├── test ├── alpha │ ├── example │ ├── dune │ ├── test_python_string_literals.ml │ ├── test_statistics.ml │ ├── test_rewrite_rule.ml │ ├── test_hole_extensions.ml │ ├── test_c_style_comments.ml │ ├── test_server.ml │ └── test_optional_holes.ml ├── omega │ ├── example │ ├── dune │ ├── test_statistics.ml │ ├── test_python_string_literals.ml │ ├── test_rewrite_rule.ml │ ├── test_hole_extensions.ml │ ├── test_c_style_comments.ml │ └── test_server.ml ├── example │ ├── diff-no-newlines │ │ ├── 1 │ │ │ ├── a │ │ │ ├── b │ │ │ └── expect.patch │ │ ├── 2 │ │ │ ├── a │ │ │ ├── b │ │ │ └── expect.patch │ │ ├── 3 │ │ │ ├── a │ │ │ ├── b │ │ │ └── expect.patch │ │ ├── 4 │ │ │ ├── a │ │ │ ├── b │ │ │ └── expect.patch │ │ └── 5 │ │ │ ├── b │ │ │ ├── a │ │ │ └── expect.patch │ ├── src │ │ ├── main.c │ │ ├── depth-0.c │ │ ├── ignore-me │ │ │ └── main.c │ │ ├── depth-1 │ │ │ ├── depth-1.c │ │ │ └── depth-2 │ │ │ │ └── depth-2.c │ │ └── honor-file-extensions │ │ │ ├── honor.pb.generic │ │ │ └── honor.pb.go │ ├── templates │ │ ├── identity │ │ │ ├── match │ │ │ └── rewrite │ │ ├── implicit-equals │ │ │ ├── rewrite │ │ │ └── match │ │ ├── parse-template-no-trailing-newline │ │ │ ├── match │ │ │ ├── rewrite │ │ │ └── match_rule │ │ ├── parse-no-match-template │ │ │ └── dune-needs-a-file-to-exist-here │ │ └── parse-template-with-trailing-newline │ │ │ ├── match │ │ │ ├── rewrite │ │ │ └── match_rule │ ├── multiple-nested-templates │ │ ├── 3 │ │ │ ├── match │ │ │ └── rewrite │ │ ├── invalid-subdir │ │ │ └── nothing-here │ │ └── contains-subdirs-1-and-2 │ │ │ ├── 1 │ │ │ ├── match │ │ │ └── rewrite │ │ │ └── 2 │ │ │ ├── match │ │ │ └── rewrite │ ├── diff-preserve-slash-r │ │ ├── a │ │ ├── b │ │ └── expect.patch │ └── zip-test │ │ ├── sample-repo │ │ ├── src │ │ │ └── main.go │ │ └── vendor │ │ │ └── main.go │ │ └── sample-repo.zip ├── dune └── common │ ├── dune │ ├── test_nested_comments.ml │ ├── test_c_separators.ml │ ├── test_pipeline.ml │ ├── test_bash.ml │ ├── test_go.ml │ ├── test_user_defined_language.ml │ ├── test_c.ml │ └── test_rewrite_parts.ml ├── lib ├── matchers │ ├── syntax_config.ml │ ├── syntax_config.mli │ ├── matcher.mli │ ├── matchers.ml │ ├── matchers.mli │ ├── dune │ ├── configuration.ml │ └── types.ml ├── rewriter │ ├── rewriter.ml │ ├── rewriter.mli │ ├── dune │ ├── rewrite.mli │ ├── rewrite_template.mli │ ├── rewrite_template.ml │ └── rewrite.ml ├── match │ ├── match.ml │ ├── types.ml │ ├── location.ml │ ├── dune │ ├── range.ml │ ├── environment.mli │ ├── match.mli │ ├── match_context.ml │ └── environment.ml ├── parsers │ ├── dune │ ├── string_literals.ml │ └── comments.ml ├── interactive │ └── dune ├── language │ ├── dune │ ├── syntax.ml │ ├── rule.mli │ ├── ast.ml │ └── parser.ml ├── statistics │ ├── dune │ ├── statistics.mli │ ├── time.ml │ └── statistics.ml ├── replacement │ ├── dune │ ├── replacement.mli │ └── replacement.ml ├── comby.ml ├── comby.mli ├── server │ ├── dune │ └── server_types.ml ├── configuration │ ├── specification.ml │ ├── dune │ ├── command_input.ml │ ├── command_configuration.mli │ └── diff_configuration.ml ├── pipeline │ └── dune └── dune ├── comby ├── benchmark ├── comby-server ├── .dockerignore ├── .gitignore ├── scripts ├── build-docker-binary-releases.sh ├── run-docker-binary.sh ├── build-base-dependencies-alpine-image.sh ├── check-and-install.sh ├── release.sh └── install.sh ├── .github ├── PULL_REQUEST_TEMPLATE │ └── pull_request_template.md └── ISSUE_TEMPLATE │ ├── need_help_with_pattern.md │ ├── bug_report.md │ └── feature_request.md ├── dune ├── .travis.ocaml.yml ├── push-coverage-report.sh ├── .travis.yml ├── docs ├── CHANGELOG.md ├── third-party-licenses │ ├── ocaml-ci-scripts │ │ └── LICENSE.md │ ├── lwt │ │ └── LICENSE.md │ ├── bisect_ppx │ │ └── LICENSE.md │ ├── ppx_deriving │ │ └── LICENSE.txt │ ├── ppx_deriving_yojson │ │ └── LICENSE.txt │ ├── ppxlib │ │ └── LICENSE.md │ ├── core │ │ └── LICENSE.md │ ├── patdiff │ │ └── LICENSE.md │ ├── hack_parallel │ │ └── LICENSE │ ├── ocaml-tls │ │ └── LICENSE.md │ ├── opium │ │ └── opium.opam │ ├── lambda-term │ │ └── LICENSE │ └── pull-and-update-release-scripts.sh ├── ROADMAP.md ├── resources │ └── match-semantics.md ├── CODE_OF_CONDUCT.md ├── FEATURE_TABLE.md └── CONTRIBUTING.md ├── Dockerfile ├── dockerfiles ├── ubuntu │ └── binary-release │ │ └── Dockerfile └── alpine │ ├── base-dependencies │ └── Dockerfile │ └── binary-release │ └── Dockerfile ├── Makefile ├── src ├── dune ├── benchmark.ml └── server.ml ├── comby.opam └── README.md /test/alpha/example: -------------------------------------------------------------------------------- 1 | ../example -------------------------------------------------------------------------------- /test/omega/example: -------------------------------------------------------------------------------- 1 | ../example -------------------------------------------------------------------------------- /lib/matchers/syntax_config.ml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/matchers/syntax_config.mli: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /comby: -------------------------------------------------------------------------------- 1 | _build/install/default/bin/comby -------------------------------------------------------------------------------- /test/example/diff-no-newlines/1/a: -------------------------------------------------------------------------------- 1 | 1 -------------------------------------------------------------------------------- /test/example/diff-no-newlines/1/b: -------------------------------------------------------------------------------- 1 | 2 -------------------------------------------------------------------------------- /test/example/diff-no-newlines/4/a: -------------------------------------------------------------------------------- 1 | 1 -------------------------------------------------------------------------------- /test/example/diff-no-newlines/5/b: -------------------------------------------------------------------------------- 1 | 2 -------------------------------------------------------------------------------- /test/example/diff-no-newlines/2/a: -------------------------------------------------------------------------------- 1 | 1 2 | 1 -------------------------------------------------------------------------------- /test/example/diff-no-newlines/2/b: -------------------------------------------------------------------------------- 1 | 2 2 | 2 -------------------------------------------------------------------------------- /test/example/diff-no-newlines/3/a: -------------------------------------------------------------------------------- 1 | 1 2 | 2 -------------------------------------------------------------------------------- /test/example/diff-no-newlines/3/b: -------------------------------------------------------------------------------- 1 | 2 2 | 2 -------------------------------------------------------------------------------- /test/example/diff-no-newlines/4/b: -------------------------------------------------------------------------------- 1 | 2 2 | -------------------------------------------------------------------------------- /test/example/diff-no-newlines/5/a: -------------------------------------------------------------------------------- 1 | 1 2 | -------------------------------------------------------------------------------- /test/example/src/main.c: -------------------------------------------------------------------------------- 1 | int main() {} 2 | -------------------------------------------------------------------------------- /benchmark: -------------------------------------------------------------------------------- 1 | _build/install/default/bin/benchmark -------------------------------------------------------------------------------- /test/example/src/depth-0.c: -------------------------------------------------------------------------------- 1 | int depth_0() {} 2 | -------------------------------------------------------------------------------- /test/example/templates/identity/match: -------------------------------------------------------------------------------- 1 | :[[1]] 2 | -------------------------------------------------------------------------------- /comby-server: -------------------------------------------------------------------------------- 1 | _build/install/default/bin/comby-server -------------------------------------------------------------------------------- /test/example/multiple-nested-templates/3/match: -------------------------------------------------------------------------------- 1 | 3 2 | -------------------------------------------------------------------------------- /test/example/src/ignore-me/main.c: -------------------------------------------------------------------------------- 1 | int main() {} 2 | -------------------------------------------------------------------------------- /test/example/templates/identity/rewrite: -------------------------------------------------------------------------------- 1 | :[[1]] 2 | -------------------------------------------------------------------------------- /test/example/diff-preserve-slash-r/a: -------------------------------------------------------------------------------- 1 | 1 2 | 2 3 | 3 4 | -------------------------------------------------------------------------------- /test/example/diff-preserve-slash-r/b: -------------------------------------------------------------------------------- 1 | 2 2 | 2 3 | 3 4 | -------------------------------------------------------------------------------- /test/example/multiple-nested-templates/3/rewrite: -------------------------------------------------------------------------------- 1 | +3 2 | -------------------------------------------------------------------------------- /test/example/src/depth-1/depth-1.c: -------------------------------------------------------------------------------- 1 | int depth_1() {} 2 | -------------------------------------------------------------------------------- /test/example/templates/implicit-equals/rewrite: -------------------------------------------------------------------------------- 1 | ident 2 | -------------------------------------------------------------------------------- /test/dune: -------------------------------------------------------------------------------- 1 | (dirs common alpha) 2 | ; (dirs common omega) 3 | -------------------------------------------------------------------------------- /test/example/src/depth-1/depth-2/depth-2.c: -------------------------------------------------------------------------------- 1 | int depth_2() {} 2 | -------------------------------------------------------------------------------- /test/example/multiple-nested-templates/invalid-subdir/nothing-here: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/example/templates/implicit-equals/match: -------------------------------------------------------------------------------- 1 | (fun :[[a]] -> :[[a]]) 2 | -------------------------------------------------------------------------------- /test/example/templates/parse-template-no-trailing-newline/match: -------------------------------------------------------------------------------- 1 | :[[1]] -------------------------------------------------------------------------------- /test/example/templates/parse-template-no-trailing-newline/rewrite: -------------------------------------------------------------------------------- 1 | :[1] -------------------------------------------------------------------------------- /test/example/multiple-nested-templates/contains-subdirs-1-and-2/1/match: -------------------------------------------------------------------------------- 1 | 1 2 | -------------------------------------------------------------------------------- /test/example/multiple-nested-templates/contains-subdirs-1-and-2/2/match: -------------------------------------------------------------------------------- 1 | 2 2 | -------------------------------------------------------------------------------- /test/example/templates/parse-no-match-template/dune-needs-a-file-to-exist-here: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/example/templates/parse-template-no-trailing-newline/match_rule: -------------------------------------------------------------------------------- 1 | where true -------------------------------------------------------------------------------- /test/example/templates/parse-template-with-trailing-newline/match: -------------------------------------------------------------------------------- 1 | :[[1]] 2 | -------------------------------------------------------------------------------- /test/example/templates/parse-template-with-trailing-newline/rewrite: -------------------------------------------------------------------------------- 1 | :[1] 2 | -------------------------------------------------------------------------------- /test/example/zip-test/sample-repo/src/main.go: -------------------------------------------------------------------------------- 1 | // src 2 | func main() {} 3 | -------------------------------------------------------------------------------- /test/example/multiple-nested-templates/contains-subdirs-1-and-2/1/rewrite: -------------------------------------------------------------------------------- 1 | +1 2 | -------------------------------------------------------------------------------- /test/example/multiple-nested-templates/contains-subdirs-1-and-2/2/rewrite: -------------------------------------------------------------------------------- 1 | +2 2 | -------------------------------------------------------------------------------- /test/example/zip-test/sample-repo/vendor/main.go: -------------------------------------------------------------------------------- 1 | // vendor 2 | func main() {} 3 | -------------------------------------------------------------------------------- /test/example/templates/parse-template-with-trailing-newline/match_rule: -------------------------------------------------------------------------------- 1 | where true 2 | -------------------------------------------------------------------------------- /test/example/src/honor-file-extensions/honor.pb.generic: -------------------------------------------------------------------------------- 1 | func main() { 2 | // foo() 3 | } 4 | -------------------------------------------------------------------------------- /lib/rewriter/rewriter.ml: -------------------------------------------------------------------------------- 1 | module Rewrite = Rewrite 2 | module Rewrite_template = Rewrite_template 3 | -------------------------------------------------------------------------------- /lib/rewriter/rewriter.mli: -------------------------------------------------------------------------------- 1 | module Rewrite = Rewrite 2 | module Rewrite_template = Rewrite_template 3 | -------------------------------------------------------------------------------- /lib/matchers/matcher.mli: -------------------------------------------------------------------------------- 1 | open Types 2 | 3 | module Make (Syntax : Syntax.S) (Info : Info.S) : Matcher.S 4 | -------------------------------------------------------------------------------- /test/example/src/honor-file-extensions/honor.pb.go: -------------------------------------------------------------------------------- 1 | func main() { 2 | // in a comment foo() 3 | foo() 4 | } 5 | -------------------------------------------------------------------------------- /test/example/zip-test/sample-repo.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stefanbuck/comby/master/test/example/zip-test/sample-repo.zip -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | ** 2 | !src 3 | !lib 4 | !docs 5 | !test 6 | !Makefile 7 | !comby.opam 8 | !dune 9 | !push-coverage-report.sh 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .merlin 2 | _build 3 | *.install 4 | /dune-project 5 | \#* 6 | .\#* 7 | scripts/0.* 8 | scripts/1.* 9 | *.swp 10 | -------------------------------------------------------------------------------- /lib/match/match.ml: -------------------------------------------------------------------------------- 1 | module Location = Location 2 | module Range = Range 3 | module Environment = Environment 4 | 5 | include Types 6 | include Match_context 7 | -------------------------------------------------------------------------------- /lib/matchers/matchers.ml: -------------------------------------------------------------------------------- 1 | module Configuration = Configuration 2 | module Syntax = Types.Syntax 3 | module type Matcher = Types.Matcher.S 4 | 5 | include Languages 6 | -------------------------------------------------------------------------------- /scripts/build-docker-binary-releases.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd .. 4 | docker build --tag comby-alpine-binary-release -f dockerfiles/alpine/binary-release/Dockerfile . 5 | -------------------------------------------------------------------------------- /scripts/run-docker-binary.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # mount /tmp/host to tmp in docker and run the binary 4 | docker run -it -v /tmp/host:/tmp comby-alpine-binary-build 5 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE/pull_request_template.md: -------------------------------------------------------------------------------- 1 | **Please make sure to add at least one test case for any change that is not a trivial bug fix, or a small code cleanup.** 2 | -------------------------------------------------------------------------------- /lib/matchers/matchers.mli: -------------------------------------------------------------------------------- 1 | module Configuration = Configuration 2 | module Syntax = Types.Syntax 3 | module type Matcher = Types.Matcher.S 4 | 5 | include module type of Languages 6 | -------------------------------------------------------------------------------- /test/example/diff-preserve-slash-r/expect.patch: -------------------------------------------------------------------------------- 1 | --- a 2019-11-20 15:16:55.000000000 -0700 2 | +++ b 2019-11-20 15:17:00.000000000 -0700 3 | @@ -1,3 +1,3 @@ 4 | -1 5 | +2 6 | 2 7 | 3 8 | -------------------------------------------------------------------------------- /test/example/diff-no-newlines/4/expect.patch: -------------------------------------------------------------------------------- 1 | --- a 2019-11-20 00:20:35.000000000 -0700 2 | +++ b 2019-11-20 00:20:35.000000000 -0700 3 | @@ -1 +1 @@ 4 | -1 5 | \ No newline at end of file 6 | +2 7 | -------------------------------------------------------------------------------- /test/example/diff-no-newlines/5/expect.patch: -------------------------------------------------------------------------------- 1 | --- a 2019-11-20 00:20:54.000000000 -0700 2 | +++ b 2019-11-20 00:20:54.000000000 -0700 3 | @@ -1 +1 @@ 4 | -1 5 | +2 6 | \ No newline at end of file 7 | -------------------------------------------------------------------------------- /dune: -------------------------------------------------------------------------------- 1 | (env 2 | (dev 3 | (flags (:standard -w A-3-4-32-34-39-40-41-42-44-45-48-49-50-57))) 4 | (release 5 | (flags (:standard -w A-3-4-32-34-39-40-41-42-44-45-48-49-50-57)) 6 | (ocamlopt_flags (-O3)))) 7 | -------------------------------------------------------------------------------- /lib/match/types.ml: -------------------------------------------------------------------------------- 1 | type location = Location.t 2 | [@@deriving yojson, eq, sexp] 3 | 4 | type range = Range.t 5 | [@@deriving yojson, eq, sexp] 6 | 7 | type environment = Environment.t 8 | [@@deriving yojson] 9 | -------------------------------------------------------------------------------- /lib/parsers/dune: -------------------------------------------------------------------------------- 1 | (library 2 | (name parsers) 3 | (public_name comby.parsers) 4 | (preprocess (pps ppx_sexp_conv bisect_ppx --conditional)) 5 | (libraries angstrom ppxlib core mparser mparser.pcre)) 6 | -------------------------------------------------------------------------------- /test/example/diff-no-newlines/3/expect.patch: -------------------------------------------------------------------------------- 1 | --- a 2019-11-20 00:15:07.000000000 -0700 2 | +++ b 2019-11-20 00:15:07.000000000 -0700 3 | @@ -1,2 +1,2 @@ 4 | -1 5 | +2 6 | 2 7 | \ No newline at end of file 8 | -------------------------------------------------------------------------------- /lib/interactive/dune: -------------------------------------------------------------------------------- 1 | (library 2 | (name interactive) 3 | (public_name comby.interactive) 4 | (preprocess (pps ppx_sexp_conv)) 5 | (libraries comby.configuration comby.match ppxlib core core.uuid lwt lwt.unix)) 6 | -------------------------------------------------------------------------------- /lib/language/dune: -------------------------------------------------------------------------------- 1 | (library 2 | (name language) 3 | (public_name comby.language) 4 | (preprocess (pps ppx_sexp_conv bisect_ppx --conditional)) 5 | (libraries comby.parsers comby.match comby.rewriter ppxlib core core.uuid)) 6 | -------------------------------------------------------------------------------- /lib/rewriter/dune: -------------------------------------------------------------------------------- 1 | (library 2 | (name rewriter) 3 | (public_name comby.rewriter) 4 | (preprocess (pps ppx_deriving_yojson bisect_ppx --conditional)) 5 | (libraries comby.matchers comby.replacement ppxlib core core.uuid)) 6 | -------------------------------------------------------------------------------- /test/example/diff-no-newlines/1/expect.patch: -------------------------------------------------------------------------------- 1 | --- a 2019-11-18 21:13:06.000000000 -0700 2 | +++ b 2019-11-18 21:13:06.000000000 -0700 3 | @@ -1 +1 @@ 4 | -1 5 | \ No newline at end of file 6 | +2 7 | \ No newline at end of file 8 | -------------------------------------------------------------------------------- /lib/statistics/dune: -------------------------------------------------------------------------------- 1 | (library 2 | (name statistics) 3 | (public_name comby.statistics) 4 | (preprocess (pps ppx_deriving_yojson bisect_ppx --conditional)) 5 | (libraries yojson ppx_deriving_yojson ppx_deriving_yojson.runtime)) 6 | -------------------------------------------------------------------------------- /lib/rewriter/rewrite.mli: -------------------------------------------------------------------------------- 1 | (** if [source] is given, substitute in-place. If not, 2 | emit result separated by newlines *) 3 | val all 4 | : ?source:string 5 | -> rewrite_template:string 6 | -> Match.t list 7 | -> Replacement.result option 8 | -------------------------------------------------------------------------------- /test/example/diff-no-newlines/2/expect.patch: -------------------------------------------------------------------------------- 1 | --- a 2019-11-20 00:12:09.000000000 -0700 2 | +++ b 2019-11-20 00:12:09.000000000 -0700 3 | @@ -1,2 +1,2 @@ 4 | -1 5 | -1 6 | \ No newline at end of file 7 | +2 8 | +2 9 | \ No newline at end of file 10 | -------------------------------------------------------------------------------- /lib/match/location.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | type t = 4 | { offset : int 5 | ; line : int 6 | ; column : int 7 | } 8 | [@@deriving yojson, eq, sexp] 9 | 10 | let default = 11 | { offset = -1 12 | ; line = -1 13 | ; column = -1 14 | } 15 | -------------------------------------------------------------------------------- /.travis.ocaml.yml: -------------------------------------------------------------------------------- 1 | language: c 2 | sudo: required 3 | install: test -e .travis-ocaml.sh || wget https://raw.githubusercontent.com/ocaml/ocaml-ci-scripts/master/.travis-ocaml.sh 4 | script: bash -ex .travis-ocaml.sh 5 | env: 6 | - OCAML_VERSION=4.07 7 | os: 8 | - linux 9 | -------------------------------------------------------------------------------- /lib/match/dune: -------------------------------------------------------------------------------- 1 | (library 2 | (name match) 3 | (public_name comby.match) 4 | (preprocess (pps ppx_deriving.eq ppx_sexp_conv ppx_deriving_yojson bisect_ppx --conditional)) 5 | (libraries comby.parsers ppxlib core yojson ppx_deriving_yojson ppx_deriving_yojson.runtime)) 6 | -------------------------------------------------------------------------------- /lib/replacement/dune: -------------------------------------------------------------------------------- 1 | (library 2 | (name replacement) 3 | (public_name comby.replacement) 4 | (preprocess (pps ppx_deriving_yojson bisect_ppx --conditional)) 5 | (libraries comby.match ppxlib core yojson ppx_deriving_yojson ppx_deriving_yojson.runtime patdiff.lib)) 6 | -------------------------------------------------------------------------------- /lib/comby.ml: -------------------------------------------------------------------------------- 1 | module Language = Language 2 | module Matchers = Matchers 3 | module Match = Match 4 | module Replacement = Replacement 5 | module Rewriter = Rewriter 6 | module Server_types = Server_types 7 | module Statistics = Statistics 8 | module Configuration = Configuration 9 | -------------------------------------------------------------------------------- /lib/comby.mli: -------------------------------------------------------------------------------- 1 | module Language = Language 2 | module Matchers = Matchers 3 | module Match = Match 4 | module Replacement = Replacement 5 | module Rewriter = Rewriter 6 | module Server_types = Server_types 7 | module Statistics = Statistics 8 | module Configuration = Configuration 9 | -------------------------------------------------------------------------------- /lib/match/range.ml: -------------------------------------------------------------------------------- 1 | type t = 2 | { match_start : Location.t [@key "start"] 3 | ; match_end : Location.t [@key "end"] 4 | } 5 | [@@deriving yojson, eq, sexp] 6 | 7 | let default = 8 | { match_start = Location.default 9 | ; match_end = Location.default 10 | } 11 | -------------------------------------------------------------------------------- /lib/server/dune: -------------------------------------------------------------------------------- 1 | (library 2 | (name server_types) 3 | (public_name comby.server_types) 4 | (preprocess (pps ppx_deriving.show ppx_deriving_yojson bisect_ppx --conditional)) 5 | (libraries comby.match comby.replacement ppxlib core yojson ppx_deriving_yojson ppx_deriving_yojson.runtime)) 6 | -------------------------------------------------------------------------------- /lib/configuration/specification.ml: -------------------------------------------------------------------------------- 1 | open Language 2 | 3 | type t = 4 | { match_template : string 5 | ; rule : Rule.t option 6 | ; rewrite_template : string option 7 | } 8 | 9 | let create ?rewrite_template ?rule ~match_template () = 10 | { match_template; rule; rewrite_template } 11 | -------------------------------------------------------------------------------- /lib/statistics/statistics.mli: -------------------------------------------------------------------------------- 1 | module Time = Time 2 | module Timer = Timer 3 | 4 | type t = 5 | { number_of_files : int 6 | ; lines_of_code : int 7 | ; number_of_matches : int 8 | ; total_time : float 9 | } 10 | [@@deriving yojson] 11 | 12 | val empty : t 13 | 14 | val merge : t -> t -> t 15 | -------------------------------------------------------------------------------- /lib/pipeline/dune: -------------------------------------------------------------------------------- 1 | (library 2 | (name pipeline) 3 | (public_name comby.pipeline) 4 | (preprocess (pps ppx_sexp_conv ppx_deriving_yojson bisect_ppx --conditional)) 5 | (libraries comby.interactive comby.statistics comby.parsers comby.match comby.language camlzip ppxlib core yojson ppx_deriving_yojson hack_parallel)) 6 | -------------------------------------------------------------------------------- /push-coverage-report.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | bisect-ppx-report \ 4 | -I _build/default/ \ 5 | --coveralls coverage.json \ 6 | --service-name travis-ci \ 7 | --service-job-id $TRAVIS_JOB_ID \ 8 | `find . -name 'bisect*.out'` 9 | curl -L -F json_file=@./coverage.json https://coveralls.io/api/v1/jobs 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: c 2 | services: 3 | - docker 4 | sudo: required 5 | before_install: 6 | - make docker-test-build 7 | script: 8 | - docker run -it -e TRAVIS_JOB_ID="$TRAVIS_JOB_ID" comby-local-test-build:latest /bin/bash -c "make && make clean && make build-with-coverage && make test && ./push-coverage-report.sh && ./benchmark" 9 | -------------------------------------------------------------------------------- /lib/language/syntax.ml: -------------------------------------------------------------------------------- 1 | let rule_prefix = "where" 2 | let start_match_pattern = "match" 3 | let start_rewrite_pattern = "rewrite" 4 | let equal = "==" 5 | let not_equal = "!=" 6 | let variable_left_delimiter = ":[" 7 | let variable_right_delimiter = "]" 8 | let true' = "true" 9 | let false' = "false" 10 | let pipe_operator = "|" 11 | let arrow = "->" 12 | -------------------------------------------------------------------------------- /scripts/build-base-dependencies-alpine-image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd .. 4 | docker build --no-cache --tag comby/base-dependencies-alpine-3.10 -f dockerfiles/alpine/base-dependencies/Dockerfile . 5 | docker push comby/base-dependencies-alpine-3.10:latest 6 | echo "Now run ./build-docker-binary-releases.sh to make sure the base dependencies image is updated." 7 | -------------------------------------------------------------------------------- /lib/configuration/dune: -------------------------------------------------------------------------------- 1 | (library 2 | (name configuration) 3 | (public_name comby.configuration) 4 | (preprocess (pps ppx_deriving.show ppx_sexp_conv ppx_sexp_message ppx_deriving_yojson bisect_ppx --conditional)) 5 | (libraries camlzip comby.statistics comby.parsers comby.match comby.language ppxlib core core.uuid mparser mparser.pcre yojson ppx_deriving_yojson hack_parallel)) 6 | -------------------------------------------------------------------------------- /lib/matchers/dune: -------------------------------------------------------------------------------- 1 | (copy_files# alpha/*.ml{,i}) 2 | ; (copy_files# omega/*.ml{,i}) 3 | 4 | (library 5 | (name matchers) 6 | (public_name comby.matchers) 7 | (preprocess (pps ppx_here ppx_sexp_conv ppx_sexp_message ppx_deriving_yojson bisect_ppx --conditional)) 8 | (libraries comby.parsers comby.match angstrom ppxlib core core.uuid mparser mparser.pcre yojson ppx_deriving_yojson)) 9 | -------------------------------------------------------------------------------- /lib/dune: -------------------------------------------------------------------------------- 1 | (library 2 | (name comby) 3 | (public_name comby) 4 | (preprocess (pps ppx_deriving.show ppx_deriving.eq ppx_sexp_conv bisect_ppx --conditional)) 5 | (libraries 6 | ppxlib 7 | core 8 | mparser 9 | mparser.pcre 10 | comby.language 11 | comby.match 12 | comby.matchers 13 | comby.pipeline 14 | comby.replacement 15 | comby.rewriter 16 | comby.server_types 17 | comby.statistics)) 18 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/need_help_with_pattern.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: I need help writing a pattern, or have questions 3 | about: Please see the description and head over to Gitter for questions about usage 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | If you're trying to rewrite something and need help or clarification please head over to Gitter: https://gitter.im/comby-tools/community and don't post this issue. Thanks! 11 | -------------------------------------------------------------------------------- /lib/language/rule.mli: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | open Matchers 4 | open Match 5 | 6 | open Ast 7 | 8 | type t = Ast.t 9 | 10 | type result = bool * environment option 11 | 12 | val sat : result -> bool 13 | 14 | val result_env : result -> environment option 15 | 16 | val create : string -> expression list Or_error.t 17 | 18 | val apply 19 | : ?matcher:(module Matcher) 20 | -> ?substitute_in_place:bool 21 | -> t 22 | -> environment 23 | -> result 24 | -------------------------------------------------------------------------------- /lib/rewriter/rewrite_template.mli: -------------------------------------------------------------------------------- 1 | open Match 2 | 3 | (** substitute returns the result and variables substituted for *) 4 | val substitute : string -> Environment.t -> (string * string list) 5 | 6 | val of_match_context : Match.t -> source:string -> (string * string) 7 | 8 | val get_offsets_for_holes : string -> string list -> (string * int) list 9 | 10 | val get_offsets_after_substitution : (string * int) list -> Environment.t -> (string * int) list 11 | -------------------------------------------------------------------------------- /lib/matchers/configuration.ml: -------------------------------------------------------------------------------- 1 | type match_kind = 2 | | Exact 3 | | Fuzzy 4 | 5 | type t = 6 | { match_kind : match_kind 7 | ; significant_whitespace : bool 8 | ; disable_substring_matching : bool 9 | } 10 | 11 | let create 12 | ?(disable_substring_matching = false) 13 | ?(match_kind = Fuzzy) 14 | ?(significant_whitespace = false) 15 | () = 16 | { match_kind 17 | ; significant_whitespace 18 | ; disable_substring_matching 19 | } 20 | -------------------------------------------------------------------------------- /lib/configuration/command_input.ml: -------------------------------------------------------------------------------- 1 | type single_input_kind = 2 | [ `String of string 3 | | `Path of string 4 | ] 5 | 6 | type t = 7 | [ `Paths of string list 8 | | `Zip of string 9 | | single_input_kind 10 | ] 11 | 12 | let show_input_kind = 13 | function 14 | | `Paths _ -> Format.sprintf "Paths..." 15 | | `Path path -> Format.sprintf "Path: %s" path 16 | | `String _ -> Format.sprintf "A long string..." 17 | | `Zip _ -> Format.sprintf "Zip..." 18 | -------------------------------------------------------------------------------- /docs/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 0.9.0 2 | 3 | - Adds interactive mode with the `-review` option. Edit can be set with `-editor`, and default accept behavior can be toggled with `-default-no` (as in [codemod](https://github.com/facebook/codemod)). 4 | - `-match-only` returns matches on single lines, prefixed by the matched files, like `grep`. Newlines are converted to `\n`. 5 | - Allow rewrite templates to contain `[hole\n]`, `[ hole]`, `:[hole.]` syntax which substitute for variable `hole`. 6 | 7 | ## 0.8.0 8 | 9 | Changelog starts 10 | -------------------------------------------------------------------------------- /lib/replacement/replacement.mli: -------------------------------------------------------------------------------- 1 | open Match 2 | 3 | type t = 4 | { range : range 5 | ; replacement_content : string 6 | ; environment : environment 7 | } 8 | [@@deriving yojson] 9 | 10 | type result = 11 | { rewritten_source : string 12 | ; in_place_substitutions : t list 13 | } 14 | [@@deriving yojson] 15 | 16 | val to_json 17 | : ?path:string 18 | -> ?replacements:t list 19 | -> ?rewritten_source:string 20 | -> diff:string 21 | -> unit 22 | -> Yojson.Safe.json 23 | 24 | val empty_result : result 25 | -------------------------------------------------------------------------------- /test/common/dune: -------------------------------------------------------------------------------- 1 | (library 2 | (name common_test_integration) 3 | (modules 4 | test_nested_comments 5 | test_c 6 | test_bash 7 | test_go 8 | test_c_separators 9 | test_pipeline 10 | test_rewrite_parts 11 | test_user_defined_language) 12 | (inline_tests) 13 | (preprocess (pps ppx_expect ppx_sexp_message ppx_deriving_yojson ppx_deriving_yojson.runtime)) 14 | (libraries 15 | comby 16 | cohttp-lwt-unix 17 | core 18 | camlzip)) 19 | 20 | (alias 21 | (name runtest) 22 | (deps (source_tree example) (source_tree example/src/.ignore-me))) 23 | -------------------------------------------------------------------------------- /lib/statistics/time.ml: -------------------------------------------------------------------------------- 1 | let start () = Unix.gettimeofday () 2 | 3 | let stop start = 4 | (Unix.gettimeofday () -. start) *. 1000.0 5 | 6 | exception Time_out 7 | 8 | let time_out ~after f args = 9 | let behavior = 10 | Sys.(signal sigalrm @@ Signal_handle (fun _ -> raise Time_out)) 11 | in 12 | let cancel_alarm () = 13 | Unix.alarm 0 |> ignore; 14 | Sys.(set_signal sigalrm behavior) 15 | in 16 | Unix.alarm after |> ignore; 17 | match f args with 18 | | result -> 19 | cancel_alarm (); 20 | result 21 | | exception exc -> 22 | cancel_alarm (); 23 | raise exc 24 | -------------------------------------------------------------------------------- /scripts/check-and-install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ui 4 | 5 | if which tput >/dev/null 2>&1; then 6 | colors=$(tput colors) 7 | fi 8 | 9 | if [ -t 1 ] && [ -n "$colors" ] && [ "$colors" -ge 8 ]; then 10 | YELLOW="$(tput setaf 3)" 11 | NORMAL="$(tput sgr0)" 12 | else 13 | YELLOW="" 14 | NORMAL="" 15 | fi 16 | 17 | SUCCESS_IN_PATH=$(command -v comby || echo notinpath) 18 | 19 | if [ $SUCCESS_IN_PATH == "notinpath" ]; then 20 | printf "${YELLOW}[-]${NORMAL} Looks like comby is not installed on your PATH. Let's try install it!\n" 21 | bash <(curl -sL get.comby.dev) 22 | fi 23 | 24 | exit 0 25 | 26 | -------------------------------------------------------------------------------- /lib/match/environment.mli: -------------------------------------------------------------------------------- 1 | type t 2 | [@@deriving yojson] 3 | 4 | val create : unit -> t 5 | 6 | val vars : t -> string list 7 | 8 | val add : ?range:Range.t -> t -> string -> string -> t 9 | 10 | val lookup : t -> string -> string option 11 | 12 | val update : t -> string -> string -> t 13 | 14 | val lookup_range : t -> string -> Range.t option 15 | 16 | val update_range : t -> string -> Range.t -> t 17 | 18 | val furthest_match : t -> int 19 | 20 | val equal : t -> t -> bool 21 | 22 | val merge : t -> t -> t 23 | 24 | val copy : t -> t 25 | 26 | val exists : t -> string -> bool 27 | 28 | val to_string : t -> string 29 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **Reproducing** 14 | - If this looks like a bug in the matcher, go to https://comby.live and paste a shared link below: 15 | 16 | 17 | - If this is not about a matcher, please describe the bug: 18 | 19 | **Expected behavior** 20 | A clear and concise description of what you expected to happen. 21 | 22 | **Additional context** 23 | Add any other context about the problem here. 24 | -------------------------------------------------------------------------------- /lib/language/ast.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | type atom = 4 | | Variable of string 5 | | String of string 6 | [@@deriving sexp] 7 | 8 | type antecedent = atom 9 | [@@deriving sexp] 10 | 11 | type expression = 12 | | True 13 | | False 14 | | Equal of atom * atom 15 | | Not_equal of atom * atom 16 | | Match of atom * (antecedent * consequent) list 17 | | RewriteTemplate of string 18 | | Rewrite of atom * (antecedent * expression) 19 | and consequent = expression list 20 | [@@deriving sexp] 21 | 22 | let (=) left right = Equal (left, right) 23 | 24 | let (<>) left right = Not_equal (left, right) 25 | 26 | type t = expression list 27 | [@@deriving sexp] 28 | -------------------------------------------------------------------------------- /test/alpha/dune: -------------------------------------------------------------------------------- 1 | (library 2 | (name alpha_test_integration) 3 | (modules 4 | test_optional_holes 5 | test_match_rule 6 | test_hole_extensions 7 | test_python_string_literals 8 | test_integration 9 | test_statistics 10 | test_cli 11 | test_c_style_comments 12 | test_string_literals 13 | test_special_matcher_cases 14 | test_generic 15 | test_rewrite_rule 16 | test_server 17 | test_substring_disabled) 18 | (inline_tests) 19 | (preprocess (pps ppx_expect ppx_sexp_message ppx_deriving_yojson ppx_deriving_yojson.runtime)) 20 | (libraries 21 | comby 22 | cohttp-lwt-unix 23 | core 24 | camlzip)) 25 | 26 | (alias 27 | (name runtest) 28 | (deps (source_tree example) (source_tree example/src/.ignore-me))) 29 | -------------------------------------------------------------------------------- /lib/language/parser.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | open MParser 4 | open MParser_PCRE.Tokens 5 | 6 | open Ast 7 | 8 | let variable_parser s = 9 | (string Syntax.variable_left_delimiter 10 | >> (many (alphanum <|> char '_') |>> String.of_char_list) 11 | << string Syntax.variable_right_delimiter) s 12 | 13 | let value_parser s = 14 | string_literal s 15 | 16 | let operator_parser s = 17 | ((string Syntax.equal) 18 | <|> (string Syntax.not_equal)) s 19 | 20 | let atom_parser s = 21 | ((variable_parser >>= fun variable -> return (Variable variable)) 22 | <|> (value_parser >>= fun value -> return (String value))) s 23 | 24 | let rewrite_template_parser s = 25 | (value_parser >>= fun value -> return (RewriteTemplate value)) s 26 | -------------------------------------------------------------------------------- /test/omega/dune: -------------------------------------------------------------------------------- 1 | (library 2 | (name omega_test_integration) 3 | (modules 4 | ; 5 | ; TODO 6 | ; 7 | test_generic 8 | test_match_rule 9 | test_hole_extensions 10 | test_python_string_literals 11 | test_integration 12 | test_statistics 13 | test_cli 14 | test_c_style_comments 15 | test_string_literals 16 | test_special_matcher_cases 17 | test_generic 18 | test_rewrite_rule 19 | test_server 20 | test_substring_disabled) 21 | (inline_tests) 22 | (preprocess (pps ppx_expect ppx_sexp_message ppx_deriving_yojson ppx_deriving_yojson.runtime)) 23 | (libraries 24 | comby 25 | cohttp-lwt-unix 26 | core 27 | camlzip)) 28 | 29 | (alias 30 | (name runtest) 31 | (deps (source_tree example) (source_tree example/src/.ignore-me))) 32 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ########################################################################################################################## 2 | ### Binary build for testing. Pulled from registry. The base-dependencies-alpine spec builds this. Used by .travis.yml ### 3 | ########################################################################################################################## 4 | FROM comby/base-dependencies-alpine-3.10:latest 5 | 6 | WORKDIR /home/comby 7 | 8 | COPY Makefile /home/comby/ 9 | COPY comby.opam /home/comby/ 10 | COPY dune /home/comby/ 11 | COPY docs /home/comby/docs 12 | COPY src /home/comby/src 13 | COPY lib /home/comby/lib 14 | COPY test /home/comby/test 15 | COPY push-coverage-report.sh /home/comby/ 16 | 17 | RUN sudo chown -R $(whoami) /home/comby 18 | -------------------------------------------------------------------------------- /docs/third-party-licenses/ocaml-ci-scripts/LICENSE.md: -------------------------------------------------------------------------------- 1 | ## ISC License 2 | 3 | Permission to use, copy, modify, and distribute this software for any 4 | purpose with or without fee is hereby granted, provided that the above 5 | copyright notice and this permission notice appear in all copies. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **One quick thing** 11 | 12 | If your feature request is about changing or updating `:[hole]` syntax, please [read some of the principles and guidelines on the syntax design](https://github.com/comby-tools/comby/blob/master/docs/CONTRIBUTING.md#comby-syntax-rationale-and-proposal-guidelines) before posting. If not, please proceed. 13 | 14 | **Is your feature request related to a problem? Please describe.** 15 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 16 | 17 | **Describe the solution you'd like** 18 | A clear and concise description of what you want to happen. 19 | 20 | **Additional context** 21 | Add any other context or screenshots about the feature request here. 22 | -------------------------------------------------------------------------------- /dockerfiles/ubuntu/binary-release/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ocaml/opam2:ubuntu-18.04-opam 2 | 3 | WORKDIR /home/comby 4 | 5 | RUN sudo apt-get install -y \ 6 | libpcre3-dev \ 7 | pkg-config \ 8 | zlib1g-dev \ 9 | m4 \ 10 | libgmp-dev 11 | 12 | RUN opam init --no-setup --disable-sandboxing --compiler=4.09.0 13 | 14 | COPY Makefile /home/comby/ 15 | COPY comby.opam /home/comby/ 16 | COPY dune /home/comby/ 17 | COPY docs /home/comby/docs 18 | COPY src /home/comby/src 19 | COPY lib /home/comby/lib 20 | COPY test /home/comby/test 21 | COPY push-coverage-report.sh /home/comby/ 22 | 23 | RUN sudo chown -R $(whoami) /home/comby 24 | 25 | RUN eval $(opam env) && opam install . --deps-only -y 26 | # Next command must be a single run command for $(opam env) to take effect. 27 | RUN eval $(opam env) && make && make test && make clean && make release 28 | 29 | # The binary is now available in /home/comby/_build/default/src/main.exe 30 | -------------------------------------------------------------------------------- /lib/statistics/statistics.ml: -------------------------------------------------------------------------------- 1 | module Time = Time 2 | module Timer = Timer 3 | 4 | type t = 5 | { number_of_files : int 6 | ; lines_of_code : int 7 | ; number_of_matches : int 8 | ; total_time : float 9 | } 10 | [@@deriving yojson] 11 | 12 | let empty = 13 | { number_of_files = 0 14 | ; lines_of_code = 0 15 | ; number_of_matches = 0 16 | ; total_time = 0.0 17 | } 18 | 19 | let merge 20 | { number_of_files 21 | ; lines_of_code 22 | ; number_of_matches 23 | ; total_time 24 | } 25 | { number_of_files = number_of_files' 26 | ; lines_of_code = lines_of_code' 27 | ; number_of_matches = number_of_matches' 28 | ; total_time = total_time' 29 | } = 30 | { number_of_files = number_of_files + number_of_files' 31 | ; lines_of_code = lines_of_code + lines_of_code' 32 | ; number_of_matches = number_of_matches + number_of_matches' 33 | ; total_time = total_time +. total_time' 34 | } 35 | -------------------------------------------------------------------------------- /dockerfiles/alpine/base-dependencies/Dockerfile: -------------------------------------------------------------------------------- 1 | ######################### 2 | ### Base Dependencies ### 3 | ######################### 4 | FROM ocaml/opam2:alpine-3.10-ocaml-4.09 5 | 6 | WORKDIR /home/comby 7 | 8 | # Install alpine system dependencies. 9 | RUN sudo apk --no-cache add \ 10 | pcre-dev \ 11 | m4 \ 12 | linux-headers \ 13 | perl \ 14 | gmp-dev \ 15 | zlib-dev 16 | 17 | # Copy the source files. 18 | COPY Makefile /home/comby/ 19 | COPY comby.opam /home/comby/ 20 | COPY dune /home/comby/ 21 | COPY docs /home/comby/docs 22 | COPY src /home/comby/src 23 | COPY lib /home/comby/lib 24 | COPY test /home/comby/test 25 | COPY push-coverage-report.sh /home/comby/ 26 | 27 | # Prepare for build. 28 | RUN sudo chown -R $(whoami) /home/comby 29 | 30 | # Build and install the OCaml dependencies. 31 | RUN eval $(opam env) && opam install . --deps-only -y 32 | # Delete the source files 33 | RUN sudo rm -rf /home/comby 34 | -------------------------------------------------------------------------------- /test/common/test_nested_comments.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | open Matchers 4 | 5 | let configuration = Configuration.create ~match_kind:Fuzzy () 6 | 7 | let print_matches matches = 8 | List.map matches ~f:(fun matched -> `String matched.Match.matched) 9 | |> (fun matches -> `List matches) 10 | |> Yojson.Safe.pretty_to_string 11 | |> print_string 12 | 13 | (* See https://stackoverflow.com/questions/6698039/nested-comments-in-c-c *) 14 | let%expect_test "nested_multiline_ocaml" = 15 | let source = {|int nest = /*/*/ 0 */**/ 1;|} in 16 | let template = {|0 * 1|} in 17 | (* 0 is not commented out *) 18 | let matches_no_nesting = C.all ~configuration ~template ~source in 19 | print_matches matches_no_nesting; 20 | [%expect_exact {|[ "0 */**/ 1" ]|}]; 21 | 22 | let matches_no_nesting = C_nested_comments.all ~configuration ~template ~source in 23 | (* 0 is commented out *) 24 | print_matches matches_no_nesting; 25 | [%expect_exact {|[]|}]; 26 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: build comby comby-server benchmark 2 | 3 | build: 4 | dune build --profile dev 5 | 6 | build-with-coverage: 7 | @BISECT_ENABLE=yes dune build --profile dev 8 | 9 | release: 10 | @dune build --profile release 11 | 12 | comby comby-server benchmark: 13 | @ln -s _build/install/default/bin/$@ ./$@ 14 | 15 | run-server: 16 | @./comby-server -p 8888 17 | 18 | run-staging-server: 19 | @./comby-server -p 8887 20 | 21 | install: 22 | @dune install 23 | 24 | doc: 25 | @dune build @doc 26 | 27 | test: 28 | @dune runtest 29 | 30 | coverage: 31 | @bisect-ppx-report -I _build/default/ -html coverage/ `find . -name 'bisect*.out'` 32 | 33 | clean: 34 | @dune clean 35 | 36 | uninstall: 37 | @dune uninstall 38 | 39 | promote: 40 | @dune promote 41 | 42 | docker-test-build: 43 | docker build -t comby-local-test-build . 44 | 45 | .PHONY: all build build-with-coverage release run-server run-staging-server install doc test coverage clean uninstall promote docker-test-build 46 | -------------------------------------------------------------------------------- /lib/replacement/replacement.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | open Match 4 | 5 | type t = 6 | { range : range 7 | ; replacement_content : string 8 | ; environment : environment 9 | } 10 | [@@deriving yojson] 11 | 12 | type result = 13 | { rewritten_source : string 14 | ; in_place_substitutions : t list 15 | } 16 | [@@deriving yojson] 17 | 18 | let empty_result = 19 | { rewritten_source = "" 20 | ; in_place_substitutions = [] 21 | } 22 | [@@deriving yojson] 23 | 24 | let to_json ?path ?replacements ?rewritten_source ~diff () = 25 | let uri = 26 | match path with 27 | | Some path -> `String path 28 | | None -> `Null 29 | in 30 | match replacements, rewritten_source with 31 | | Some replacements, Some rewritten_source -> 32 | `Assoc 33 | [ "uri", uri 34 | ; "rewritten_source", `String rewritten_source 35 | ; "in_place_substitutions", `List (List.map ~f:to_yojson replacements) 36 | ; "diff", `String diff 37 | ] 38 | | _ -> 39 | `Assoc 40 | [ "uri", uri 41 | ; "diff", `String diff 42 | ] 43 | -------------------------------------------------------------------------------- /docs/ROADMAP.md: -------------------------------------------------------------------------------- 1 | # Roadmap 2 | 3 | The purpose of this file is to give a high-level overview of planned features. The intent is to communicate the relative priority of these features. See this as a hopeful wishlist, rather than an exact timeline or promise, since it's difficult to predict how long some features may take to implement. 4 | 5 | ### Oct 2019 6 | - [x] [Interactive editing mode](https://github.com/comby-tools/comby/pull/104), a la codemod. 7 | 8 | ### In the coming 3 to 6 months (Nov 2019 - Mar 2020) 9 | - [ ] Optional holes [in progress](https://github.com/comby-tools/comby/pull/133) 10 | - [ ] A more efficient rewrite of the parser engine (in progress) 11 | - [ ] Matching support for arbitrary tagged delimiters, like HTML `

foo

`. 12 | - [ ] Indentation-sensitive support. 13 | - [ ] Official support for match rules that contain hole syntax. This needs planning to document and test the supported semantics. 14 | - [ ] Official support for rewrite rules and semantics combined with match rules. 15 | - [ ] Switching horizontal/vertical layout in comby.live. 16 | - [ ] Tight matching 17 | -------------------------------------------------------------------------------- /src/dune: -------------------------------------------------------------------------------- 1 | (executables 2 | (libraries comby core ppx_deriving_yojson ppx_deriving_yojson.runtime hack_parallel camlzip patdiff.lib) 3 | (preprocess (pps ppx_deriving_yojson ppx_let ppx_deriving.show ppx_sexp_conv)) 4 | (modules main) 5 | (names main)) 6 | 7 | (executables 8 | (libraries comby core opium ppx_deriving_yojson ppx_deriving_yojson.runtime hack_parallel) 9 | (preprocess (pps ppx_deriving_yojson ppx_let ppx_deriving.show ppx_sexp_conv)) 10 | (modules server) 11 | (names server)) 12 | 13 | (executables 14 | (libraries comby core opium ppx_deriving_yojson ppx_deriving_yojson.runtime hack_parallel) 15 | (preprocess (pps ppx_deriving_yojson ppx_let ppx_deriving.show ppx_sexp_conv)) 16 | (modules benchmark) 17 | (names benchmark)) 18 | 19 | (alias 20 | (name DEFAULT) 21 | (deps main.exe)) 22 | 23 | (alias 24 | (name DEFAULT) 25 | (deps server.exe)) 26 | 27 | (alias 28 | (name DEFAULT) 29 | (deps benchmark.exe)) 30 | 31 | (install 32 | (section bin) 33 | (files (main.exe as comby))) 34 | 35 | (install 36 | (section bin) 37 | (files (benchmark.exe as benchmark))) 38 | 39 | (install 40 | (section bin) 41 | (files (server.exe as comby-server))) 42 | -------------------------------------------------------------------------------- /docs/third-party-licenses/lwt/LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 1999-2019, the Authors of Lwt (docs/AUTHORS) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /docs/third-party-licenses/bisect_ppx/LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright © 2008-2019 Xavier Clerc, Leonid Rozenberg, Anton Bachin 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the “Software”), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 7 | of the Software, and to permit persons to whom the Software is furnished to do 8 | so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /docs/third-party-licenses/ppx_deriving/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014-2016 whitequark 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /docs/third-party-licenses/ppx_deriving_yojson/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014-2018 whitequark 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /docs/third-party-licenses/ppxlib/LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2018 Jane Street Group, LLC 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /docs/third-party-licenses/core/LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2008--2019 Jane Street Group, LLC 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /docs/third-party-licenses/patdiff/LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2005--2019 Jane Street Group, LLC 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /test/common/test_c_separators.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | open Matchers 4 | open Rewriter 5 | 6 | let configuration = Configuration.create ~match_kind:Fuzzy () 7 | 8 | let run source match_template rewrite_template = 9 | C.first ~configuration match_template source 10 | |> function 11 | | Ok result -> 12 | Rewrite.all ~source ~rewrite_template [result] 13 | |> (fun x -> Option.value_exn x) 14 | |> (fun { rewritten_source; _ } -> rewritten_source) 15 | |> print_string 16 | | Error _ -> 17 | print_string rewrite_template 18 | 19 | let%expect_test "whitespace_should_not_matter_between_separators" = 20 | let source = {|*p|} in 21 | let match_template = {|*:[1]|} in 22 | let rewrite_template = {|:[1]|} in 23 | run source match_template rewrite_template; 24 | [%expect_exact {|p|}]; 25 | 26 | let source = {|* p|} in 27 | let match_template = {|*:[1]|} in 28 | let rewrite_template = {|:[1]|} in 29 | run source match_template rewrite_template; 30 | [%expect_exact {| p|}]; 31 | 32 | let source = {|* p|} in 33 | let match_template = {|* :[1]|} in 34 | let rewrite_template = {|:[1]|} in 35 | run source match_template rewrite_template; 36 | [%expect_exact {|p|}] 37 | -------------------------------------------------------------------------------- /docs/third-party-licenses/hack_parallel/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2013-present, Facebook, Inc. 4 | Modified work Copyright (c) 2018-2019 Rijnard van Tonder 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /docs/third-party-licenses/ocaml-tls/LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, David Kaloper and Hannes Mehnert 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, this 11 | list of conditions and the following disclaimer in the documentation and/or 12 | other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 18 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 21 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /dockerfiles/alpine/binary-release/Dockerfile: -------------------------------------------------------------------------------- 1 | ####################################### 2 | ### Source build for alpine release ### 3 | ####################################### 4 | FROM comby/base-dependencies-alpine-3.10:latest AS source-build-alpine 5 | 6 | WORKDIR /home/comby 7 | 8 | COPY Makefile /home/comby/ 9 | COPY comby.opam /home/comby/ 10 | COPY dune /home/comby/ 11 | COPY docs /home/comby/docs 12 | COPY src /home/comby/src 13 | COPY lib /home/comby/lib 14 | COPY test /home/comby/test 15 | COPY push-coverage-report.sh /home/comby/ 16 | 17 | RUN sudo chown -R $(whoami) /home/comby 18 | # Next command must be a single run command for $(opam env) to take effect. 19 | RUN eval $(opam env) && make && make test && make clean && make release 20 | 21 | ################################################### 22 | ### Binary distribution on minimal alpine image ### 23 | ################################################### 24 | FROM alpine:3.10 25 | 26 | LABEL maintainer="Rijnard van Tonder " 27 | LABEL name="comby" 28 | LABEL org.opencontainers.image.source="https://github.com/comby-tools/comby" 29 | 30 | RUN apk --no-cache add pcre tini 31 | COPY --from=source-build-alpine /home/comby/_build/default/src/main.exe /usr/local/bin/comby 32 | COPY --from=source-build-alpine /home/comby/docs/third-party-licenses /usr/local/bin/comby-third-party-licenses 33 | 34 | WORKDIR / 35 | ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/comby"] 36 | -------------------------------------------------------------------------------- /docs/third-party-licenses/opium/opium.opam: -------------------------------------------------------------------------------- 1 | # This file is generated by dune, edit dune-project instead 2 | opam-version: "2.0" 3 | synopsis: "Sinatra like web toolkit based on Lwt + Cohttp" 4 | description: """ 5 | Opium is a minimalistic library for quickly binding functions to http routes. Its features include (but not limited to): 6 | 7 | Middleware system for app independent components 8 | A simple router for matching urls and parsing parameters 9 | Request/Response pretty printing for easier debugging 10 | """ 11 | maintainer: ["Rudi Grinberg "] 12 | authors: ["Rudi Grinberg"] 13 | license: "MIT" 14 | homepage: "https://github.com/rgrinberg/opium" 15 | doc: "https://rgrinberg.github.io/opium" 16 | bug-reports: "https://github.com/rgrinberg/opium/issues" 17 | depends: [ 18 | "ocaml" {>= "4.04.2"} 19 | "dune" {>= "1.11"} 20 | "base" {>= "v0.11.0"} 21 | "stdio" {>= "v0.11.0"} 22 | "opium_core" 23 | "hmap" 24 | "yojson" 25 | "cohttp-lwt-unix" 26 | "lwt" 27 | "logs" 28 | "cmdliner" 29 | "ppx_fields_conv" 30 | "ppx_sexp_conv" 31 | "re" 32 | "magic-mime" 33 | "alcotest" {with-test} 34 | ] 35 | build: [ 36 | ["dune" "subst"] {pinned} 37 | [ 38 | "dune" 39 | "build" 40 | "-p" 41 | name 42 | "-j" 43 | jobs 44 | "@install" 45 | "@runtest" {with-test} 46 | "@doc" {with-doc} 47 | ] 48 | ] 49 | dev-repo: "git+https://github.com/rgrinberg/opium.git" 50 | -------------------------------------------------------------------------------- /comby.opam: -------------------------------------------------------------------------------- 1 | opam-version: "2.0" 2 | version: "dev" 3 | maintainer: "rvantonder@gmail.com" 4 | authors: "Rijnard van Tonder" 5 | homepage: "https://github.com/comby-tools/comby" 6 | bug-reports: "https://github.com/comby-tools/comby/issues" 7 | dev-repo: "git+https://github.com/comby-tools/comby.git" 8 | license: "Apache-2.0" 9 | build: [ 10 | ["dune" "build" "-p" name "-j" jobs "@install"] 11 | ] 12 | depends: [ 13 | "ocaml" {>= "4.08.1"} 14 | "core" {>= "0.12.2"} 15 | "lwt" {>= "4.3.0"} 16 | "mparser-comby" 17 | "ppxlib" 18 | "ppx_deriving" 19 | "angstrom" 20 | "hack_parallel" 21 | "opium" 22 | "pcre" 23 | "oasis" 24 | "tls" 25 | "camlzip" 26 | "patdiff" 27 | "lwt_react" 28 | "ppx_deriving_yojson" 29 | "ppx_tools_versioned" {>= "5.2.3"} 30 | "bisect_ppx" {dev & >= "2.0.0"} 31 | ] 32 | pin-depends: [ 33 | ["bisect_ppx.git" "git+https://github.com/aantron/bisect_ppx.git"] 34 | ["mparser-comby.git" "git+https://github.com/comby-tools/mparser.git"] 35 | ["lwt.git" "git+https://github.com/rvantonder/lwt.git#4.3.0-with-captain"] 36 | ["patdiff.git" "git+https://github.com/rvantonder/patdiff.git#0.12.1-patch-compatible-diffs"] 37 | ] 38 | depexts: [ 39 | [ 40 | "pkg-config" 41 | "libpcre3-dev" 42 | "zlib1g-dev" 43 | "m4" 44 | "libgmp-dev" 45 | ] {os-distribution = "ubuntu"} 46 | [ 47 | "pkg-config" 48 | "pcre" 49 | "gmp" 50 | ] {os-distribution = "macos"} 51 | ] 52 | synopsis: "A tool for structural code search and replace that supports ~every language" 53 | description: "" 54 | -------------------------------------------------------------------------------- /test/common/test_pipeline.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | open Hack_parallel 3 | 4 | open Pipeline 5 | open Command_configuration 6 | open Matchers 7 | 8 | let matcher = (module Generic : Matchers.Matcher) 9 | 10 | let configuration = 11 | { sources = `String "source" 12 | ; specifications = [] 13 | ; exclude_directory_prefix = "." 14 | ; file_filters = None 15 | ; run_options = 16 | { sequential = true 17 | ; verbose = false 18 | ; match_timeout = 3 19 | ; number_of_workers = 4 20 | ; dump_statistics = false 21 | ; substitute_in_place = true 22 | ; disable_substring_matching = false 23 | } 24 | ; output_printer = (fun _ -> ()) 25 | ; interactive_review = None 26 | } 27 | 28 | let%expect_test "interactive_paths" = 29 | let _, count = 30 | let scheduler = Scheduler.create ~number_of_workers:0 () in 31 | Pipeline.with_scheduler scheduler ~f:( 32 | Pipeline.process_paths_for_interactive 33 | ~sequential:false 34 | ~f:(fun ~input: _ ~path:_ -> (None, 0)) []) 35 | in 36 | print_string (Format.sprintf "%d" count); 37 | [%expect_exact {|0|}] 38 | 39 | let%expect_test "launch_editor" = 40 | let configuration = 41 | { configuration 42 | with interactive_review = 43 | Some 44 | { editor = "vim" 45 | ; default_is_accept = true 46 | } 47 | } 48 | in 49 | let result = 50 | try Pipeline.run matcher configuration; "passed" 51 | with _exc -> "Not a tty" 52 | in 53 | print_string result; 54 | [%expect_exact {|Not a tty|}] 55 | -------------------------------------------------------------------------------- /docs/third-party-licenses/lambda-term/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011, Jeremie Dimino 2 | All rights reserved. 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are met: 5 | 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | * Neither the name of Jeremie Dimino nor the names of his 12 | contributors may be used to endorse or promote products derived 13 | from this software without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 16 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE FOR ANY 19 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /lib/match/match.mli: -------------------------------------------------------------------------------- 1 | module Location : sig 2 | type t = 3 | { offset : int 4 | ; line : int 5 | ; column : int 6 | } 7 | [@@deriving yojson, eq, sexp] 8 | 9 | val default : t 10 | end 11 | 12 | type location = Location.t 13 | [@@deriving yojson, eq, sexp] 14 | 15 | module Range : sig 16 | type t = 17 | { match_start : location [@key "start"] 18 | ; match_end : location [@key "end"] 19 | } 20 | [@@deriving yojson, eq, sexp] 21 | 22 | val default : t 23 | end 24 | 25 | type range = Range.t 26 | [@@deriving yojson, eq, sexp] 27 | 28 | module Environment : sig 29 | type t 30 | [@@deriving yojson, eq] 31 | 32 | val create : unit -> t 33 | 34 | val vars : t -> string list 35 | 36 | val add : ?range:range -> t -> string -> string -> t 37 | 38 | val lookup : t -> string -> string option 39 | 40 | val update : t -> string -> string -> t 41 | 42 | val lookup_range : t -> string -> range option 43 | 44 | val update_range : t -> string -> range -> t 45 | 46 | val furthest_match : t -> int 47 | 48 | val equal : t -> t -> bool 49 | 50 | val copy : t -> t 51 | 52 | val merge : t -> t -> t 53 | 54 | val to_string : t -> string 55 | 56 | val exists : t -> string -> bool 57 | end 58 | 59 | type environment = Environment.t 60 | [@@deriving yojson] 61 | 62 | type t = 63 | { range : range 64 | ; environment : environment 65 | ; matched : string 66 | } 67 | [@@deriving yojson] 68 | 69 | val create : unit -> t 70 | 71 | val pp : Format.formatter -> string option * t list -> unit 72 | 73 | val pp_json_lines : Format.formatter -> string option * t list -> unit 74 | 75 | val pp_match_count : Format.formatter -> string option * t list -> unit 76 | -------------------------------------------------------------------------------- /lib/server/server_types.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | open Match 4 | 5 | module In = struct 6 | 7 | type substitution_request = 8 | { rewrite_template : string [@key "rewrite"] 9 | ; environment : Environment.t 10 | ; id : int 11 | } 12 | [@@deriving yojson] 13 | 14 | type match_request = 15 | { source : string 16 | ; match_template : string [@key "match"] 17 | ; rule : string option [@default None] 18 | ; language : string [@default "generic"] 19 | ; id : int 20 | } 21 | [@@deriving yojson] 22 | 23 | type rewrite_request = 24 | { source : string 25 | ; match_template : string [@key "match"] 26 | ; rewrite_template : string [@key "rewrite"] 27 | ; rule : string option [@default None] 28 | ; language : string [@default "generic"] 29 | ; substitution_kind : string [@default "in_place"] 30 | ; id : int 31 | } 32 | [@@deriving yojson] 33 | end 34 | 35 | module Out = struct 36 | 37 | module Matches = struct 38 | type t = 39 | { matches : Match.t list 40 | ; source : string 41 | ; id : int 42 | } 43 | [@@deriving yojson] 44 | 45 | let to_string = 46 | Fn.compose Yojson.Safe.pretty_to_string to_yojson 47 | 48 | end 49 | 50 | module Rewrite = struct 51 | type t = 52 | { rewritten_source : string 53 | ; in_place_substitutions : Replacement.t list 54 | ; id : int 55 | } 56 | [@@deriving yojson] 57 | 58 | let to_string = 59 | Fn.compose Yojson.Safe.pretty_to_string to_yojson 60 | 61 | end 62 | 63 | module Substitution = struct 64 | 65 | type t = 66 | { result : string 67 | ; id : int 68 | } 69 | [@@deriving yojson] 70 | 71 | let to_string = 72 | Fn.compose Yojson.Safe.pretty_to_string to_yojson 73 | 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /lib/match/match_context.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | type t = 4 | { range : Range.t 5 | ; environment : Environment.t 6 | ; matched : string 7 | } 8 | [@@deriving yojson] 9 | 10 | let create () = 11 | { range = Range.default 12 | ; environment = Environment.create () 13 | ; matched = "" 14 | } 15 | 16 | let to_json source_path matches = 17 | let json_matches matches = `List (List.map ~f:to_yojson matches) in 18 | let uri = 19 | match source_path with 20 | | Some path -> `String path 21 | | None -> `Null 22 | in 23 | `Assoc 24 | [ ("uri", uri) 25 | ; ("matches", json_matches matches) 26 | ] 27 | 28 | let pp_source_path ppf source_path = 29 | match source_path with 30 | | Some path -> Format.fprintf ppf "%s:" path 31 | | None -> Format.fprintf ppf "" 32 | 33 | let pp_line_number ppf start_line = 34 | Format.fprintf ppf "%d:" start_line 35 | 36 | let pp ppf (source_path, matches) = 37 | if matches = [] then 38 | () 39 | else 40 | let matched = 41 | List.map matches ~f:(fun { matched; range; _ } -> 42 | let matched = String.substr_replace_all matched ~pattern:"\n" ~with_:"\\n" in 43 | let line = range.match_start.line in 44 | Format.asprintf "%a%a%s" pp_source_path source_path pp_line_number line matched) 45 | |> String.concat ~sep:"\n" 46 | in 47 | Format.fprintf ppf "%s@." matched 48 | 49 | let pp_match_count ppf (source_path, matches) = 50 | let l = List.length matches in 51 | if l > 1 then 52 | Format.fprintf ppf "%a%d matches\n" pp_source_path source_path (List.length matches) 53 | else if l = 1 then 54 | Format.fprintf ppf "%a%d match\n" pp_source_path source_path (List.length matches) 55 | 56 | let pp_json_lines ppf (source_path, matches) = 57 | Format.fprintf ppf "%s" @@ Yojson.Safe.to_string @@ to_json source_path matches 58 | -------------------------------------------------------------------------------- /scripts/release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | VERSION="0.x.0" 4 | 5 | if [ -z "$1" ]; then 6 | echo "Need arg: what version to release?" 7 | exit 1 8 | fi 9 | 10 | echo -n "Did you bump the number in main.ml?" 11 | read X 12 | echo -n "Did you commit release script changes?" 13 | read X 14 | 15 | VERSION=$1 16 | 17 | rm -rf $VERSION 18 | mkdir -p $VERSION $VERSION/0 $VERSION/get 19 | 20 | cd ../docs/third-party-licenses 21 | ./pull-and-update-release-scripts.sh 22 | cd ../.. 23 | 24 | comby '"0.x.0"' "\"$VERSION\"" .ml -d src -i 25 | 26 | # Build ubuntu docker binary release and copy binary 27 | docker build --tag comby-ubuntu-build . -f dockerfiles/ubuntu/binary-release/Dockerfile 28 | docker run --rm --entrypoint cat comby-ubuntu-build:latest /home/comby/_build/default/src/main.exe > scripts/$VERSION/comby-$VERSION-x86_64-linux 29 | cd scripts/$VERSION && tar czvf comby-$VERSION-x86_64-linux.tar.gz comby-$VERSION-x86_64-linux && cd ../.. 30 | 31 | # Build mac binary 32 | make clean 33 | make release 34 | make test 35 | 36 | git checkout -- . 37 | 38 | OS=$(uname -s || echo dunno) 39 | 40 | if [ "$OS" = "Darwin" ]; then 41 | cp _build/default/src/main.exe scripts/$VERSION/comby-$VERSION-x86_64-macos 42 | cd scripts/$VERSION && tar czvf comby-$VERSION-x86_64-macos.tar.gz comby-$VERSION-x86_64-macos && cd ../.. 43 | fi 44 | 45 | cp scripts/check-and-install.sh scripts/$VERSION/0/index.html 46 | cp scripts/install-with-licenses.sh scripts/$VERSION/get/index.html 47 | comby '"0.x.0"' "$VERSION" .html -i -d scripts/$VERSION 48 | 49 | # Alpine docker image 50 | cd scripts 51 | ./build-docker-binary-releases.sh 52 | docker tag comby-alpine-binary-release:latest comby/comby:$VERSION 53 | echo "test: 'docker run -it comby/comby:$VERSION -version'" 54 | echo "push: 'docker push comby/comby:$VERSION'" 55 | echo "tag latest: 'docker tag comby/comby:$VERSION comby/comby:latest" 56 | echo "push: 'docker push comby/comby:latest" 57 | -------------------------------------------------------------------------------- /test/common/test_bash.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | open Matchers 4 | open Rewriter 5 | 6 | let configuration = Configuration.create ~match_kind:Fuzzy () 7 | 8 | let run_bash source match_template rewrite_template = 9 | Bash.first ~configuration match_template source 10 | |> function 11 | | Ok result -> 12 | Rewrite.all ~source ~rewrite_template [result] 13 | |> (fun x -> Option.value_exn x) 14 | |> (fun { rewritten_source; _ } -> rewritten_source) 15 | |> print_string 16 | | Error _ -> 17 | print_string rewrite_template 18 | 19 | let run_go source match_template rewrite_template = 20 | Go.first ~configuration match_template source 21 | |> function 22 | | Ok result -> 23 | Rewrite.all ~source ~rewrite_template [result] 24 | |> (fun x -> Option.value_exn x) 25 | |> (fun { rewritten_source; _ } -> rewritten_source) 26 | |> print_string 27 | | Error _ -> 28 | print_string rewrite_template 29 | 30 | let%expect_test "custom_long_delimiters" = 31 | let source = 32 | {| 33 | case 34 | case 35 | block 1 36 | esac 37 | 38 | case 39 | block 2 40 | esac 41 | esac 42 | |} 43 | in 44 | let match_template = {|case :[1] esac|} in 45 | let rewrite_template = {|case nuked blocks esac|} in 46 | 47 | run_bash source match_template rewrite_template; 48 | [%expect_exact {| 49 | case nuked blocks esac 50 | |}] 51 | 52 | let%expect_test "custom_long_delimiters_doesn't_work_in_go" = 53 | let source = 54 | {| 55 | case 56 | case 57 | block 1 58 | esac 59 | 60 | case 61 | block 2 62 | esac 63 | esac 64 | |} 65 | in 66 | let match_template = {|case :[1] esac|} in 67 | let rewrite_template = {|case nuked blocks esac|} in 68 | 69 | run_go source match_template rewrite_template; 70 | [%expect_exact {| 71 | case nuked blocks esac 72 | 73 | case 74 | block 2 75 | esac 76 | esac 77 | |}] 78 | -------------------------------------------------------------------------------- /test/alpha/test_python_string_literals.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | open Matchers 4 | 5 | let configuration = Configuration.create ~match_kind:Fuzzy () 6 | 7 | let print_matches matches = 8 | List.map matches ~f:(fun matched -> `String matched.Match.matched) 9 | |> (fun matches -> `List matches) 10 | |> Yojson.Safe.pretty_to_string 11 | |> print_string 12 | 13 | 14 | let%expect_test "matched_contains_raw_literal_quotes" = 15 | let source = {|"""blah"""|} in 16 | let template = {|""":[[1]]"""|} in 17 | let matches = Python.all ~configuration ~template ~source in 18 | print_matches matches; 19 | [%expect_exact {|[ "\"\"\"blah\"\"\"" ]|}] 20 | 21 | let%expect_test "interpreted_string_does_not_match_raw_literal" = 22 | let source = {|"""blah""" "blah"|} in 23 | let template = {|":[[1]]"|} in 24 | let matches = Python.all ~configuration ~template ~source in 25 | print_matches matches; 26 | [%expect_exact {|[ "\"blah\"" ]|}] 27 | 28 | let%expect_test "interpreted_string_does_not_match_raw_literal_containing_quote" = 29 | let source = {|"""blah""" """bl"ah""" "blah"|} in 30 | let template = {|":[[1]]"|} in 31 | let matches = Python.all ~configuration ~template ~source in 32 | print_matches matches; 33 | [%expect_exact {|[ "\"blah\"" ]|}] 34 | 35 | let%expect_test "raw_string_matches_string_containing_quote" = 36 | let source = {|"""bl"ah"""|} in 37 | let template = {|""":[1]"""|} in 38 | let matches = Python.all ~configuration ~template ~source in 39 | print_matches matches; 40 | [%expect_exact {|[ "\"\"\"bl\"ah\"\"\"" ]|}] 41 | 42 | let%expect_test "invalid_raw_string_in_python_but_matches_because_ignores_after" = 43 | let source = {|"""""""|} in 44 | let template = {|""":[1]"""|} in 45 | let matches = Python.all ~configuration ~template ~source in 46 | print_matches matches; 47 | [%expect_exact {|[ "\"\"\"\"\"\"" ]|}] 48 | 49 | let%expect_test "raw_string_captures_escape_sequences" = 50 | let source = {|"""\""""|} in 51 | let template = {|""":[1]"""|} in 52 | let matches = Python.all ~configuration ~template ~source in 53 | print_matches matches; 54 | [%expect_exact {|[ "\"\"\"\\\"\"\"" ]|}] 55 | -------------------------------------------------------------------------------- /test/alpha/test_statistics.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | open Language 4 | open Matchers 5 | open Match 6 | 7 | let configuration = Configuration.create ~match_kind:Fuzzy () 8 | 9 | let format s = 10 | let s = s |> String.chop_prefix_exn ~prefix:"\n" in 11 | let leading_indentation = 12 | Option.value_exn (String.lfindi s ~f:(fun _ c -> c <> ' ')) in 13 | s 14 | |> String.split ~on:'\n' 15 | |> List.map ~f:(Fn.flip String.drop_prefix leading_indentation) 16 | |> String.concat ~sep:"\n" 17 | |> String.chop_suffix_exn ~suffix:"\n" 18 | 19 | let %expect_test "statistics" = 20 | let template = 21 | {| 22 | def :[fn_name](:[fn_params]) 23 | |} 24 | |> format 25 | in 26 | 27 | let source = 28 | {| 29 | def foo(bar): 30 | pass 31 | 32 | def bar(bazz): 33 | pass 34 | |} 35 | |> format 36 | in 37 | 38 | let rule = 39 | {| where true 40 | |} 41 | |> Rule.create 42 | |> Or_error.ok_exn 43 | in 44 | Go.all ~configuration ~template ~source 45 | |> List.filter ~f:(fun { environment; _ } -> 46 | Rule.(sat @@ apply rule environment)) 47 | |> fun matches -> 48 | let statistics = 49 | Statistics. 50 | { number_of_files = 1 51 | ; lines_of_code = 5 52 | ; number_of_matches = List.length matches 53 | ; total_time = 0.0 54 | } 55 | in 56 | statistics 57 | |> Statistics.to_yojson 58 | |> Yojson.Safe.pretty_to_string 59 | |> print_string; 60 | [%expect {| 61 | { 62 | "number_of_files": 1, 63 | "lines_of_code": 5, 64 | "number_of_matches": 2, 65 | "total_time": 0.0 66 | } |}]; 67 | 68 | let statistics' = 69 | Statistics.merge 70 | { number_of_files = 1 71 | ; lines_of_code = 10 72 | ; number_of_matches = 1 73 | ; total_time = 1.5 74 | } 75 | statistics 76 | in 77 | statistics' 78 | |> Statistics.to_yojson 79 | |> Yojson.Safe.pretty_to_string 80 | |> print_string; 81 | [%expect {| 82 | { 83 | "number_of_files": 2, 84 | "lines_of_code": 15, 85 | "number_of_matches": 3, 86 | "total_time": 1.5 87 | } |}] 88 | -------------------------------------------------------------------------------- /test/omega/test_statistics.ml: -------------------------------------------------------------------------------- 1 | (*open Core 2 | 3 | open Language 4 | open Matchers 5 | open Match 6 | 7 | let configuration = Configuration.create ~match_kind:Fuzzy () 8 | 9 | let format s = 10 | let s = s |> String.chop_prefix_exn ~prefix:"\n" in 11 | let leading_indentation = 12 | Option.value_exn (String.lfindi s ~f:(fun _ c -> c <> ' ')) in 13 | s 14 | |> String.split ~on:'\n' 15 | |> List.map ~f:(Fn.flip String.drop_prefix leading_indentation) 16 | |> String.concat ~sep:"\n" 17 | |> String.chop_suffix_exn ~suffix:"\n" 18 | 19 | let %expect_test "statistics" = 20 | let template = 21 | {| 22 | def :[fn_name](:[fn_params]) 23 | |} 24 | |> format 25 | in 26 | 27 | let source = 28 | {| 29 | def foo(bar): 30 | pass 31 | 32 | def bar(bazz): 33 | pass 34 | |} 35 | |> format 36 | in 37 | 38 | let rule = 39 | {| where true 40 | |} 41 | |> Rule.create 42 | |> Or_error.ok_exn 43 | in 44 | Go.all ~configuration ~template ~source 45 | |> List.filter ~f:(fun { environment; _ } -> 46 | Rule.(sat @@ apply rule environment)) 47 | |> fun matches -> 48 | let statistics = 49 | Statistics. 50 | { number_of_files = 1 51 | ; lines_of_code = 5 52 | ; number_of_matches = List.length matches 53 | ; total_time = 0.0 54 | } 55 | in 56 | statistics 57 | |> Statistics.to_yojson 58 | |> Yojson.Safe.pretty_to_string 59 | |> print_string; 60 | [%expect {| 61 | { 62 | "number_of_files": 1, 63 | "lines_of_code": 5, 64 | "number_of_matches": 2, 65 | "total_time": 0.0 66 | } |}]; 67 | 68 | let statistics' = 69 | Statistics.merge 70 | { number_of_files = 1 71 | ; lines_of_code = 10 72 | ; number_of_matches = 1 73 | ; total_time = 1.5 74 | } 75 | statistics 76 | in 77 | statistics' 78 | |> Statistics.to_yojson 79 | |> Yojson.Safe.pretty_to_string 80 | |> print_string; 81 | [%expect {| 82 | { 83 | "number_of_files": 2, 84 | "lines_of_code": 15, 85 | "number_of_matches": 3, 86 | "total_time": 1.5 87 | } |}] 88 | *) 89 | -------------------------------------------------------------------------------- /test/omega/test_python_string_literals.ml: -------------------------------------------------------------------------------- 1 | (*open Core 2 | 3 | open Matchers 4 | 5 | let configuration = Configuration.create ~match_kind:Fuzzy () 6 | 7 | let print_matches matches = 8 | List.map matches ~f:(fun matched -> `String matched.Match.matched) 9 | |> (fun matches -> `List matches) 10 | |> Yojson.Safe.pretty_to_string 11 | |> print_string 12 | 13 | 14 | let%expect_test "matched_contains_raw_literal_quotes" = 15 | let source = {|"""blah"""|} in 16 | let template = {|""":[[1]]"""|} in 17 | let matches = Python.all ~configuration ~template ~source in 18 | print_matches matches; 19 | [%expect_exact {|[ "\"\"\"blah\"\"\"" ]|}] 20 | 21 | let%expect_test "interpreted_string_does_not_match_raw_literal" = 22 | let source = {|"""blah""" "blah"|} in 23 | let template = {|":[[1]]"|} in 24 | let matches = Python.all ~configuration ~template ~source in 25 | print_matches matches; 26 | [%expect_exact {|[ "\"blah\"" ]|}] 27 | 28 | let%expect_test "interpreted_string_does_not_match_raw_literal_containing_quote" = 29 | let source = {|"""blah""" """bl"ah""" "blah"|} in 30 | let template = {|":[[1]]"|} in 31 | let matches = Python.all ~configuration ~template ~source in 32 | print_matches matches; 33 | [%expect_exact {|[ "\"blah\"" ]|}] 34 | 35 | let%expect_test "raw_string_matches_string_containing_quote" = 36 | let source = {|"""bl"ah"""|} in 37 | let template = {|""":[1]"""|} in 38 | let matches = Python.all ~configuration ~template ~source in 39 | print_matches matches; 40 | [%expect_exact {|[ "\"\"\"bl\"ah\"\"\"" ]|}] 41 | 42 | let%expect_test "invalid_raw_string_in_python_but_matches_because_ignores_after" = 43 | let source = {|"""""""|} in 44 | let template = {|""":[1]"""|} in 45 | let matches = Python.all ~configuration ~template ~source in 46 | print_matches matches; 47 | [%expect_exact {|[ "\"\"\"\"\"\"" ]|}] 48 | 49 | let%expect_test "raw_string_captures_escape_sequences" = 50 | let source = {|"""\""""|} in 51 | let template = {|""":[1]"""|} in 52 | let matches = Python.all ~configuration ~template ~source in 53 | print_matches matches; 54 | [%expect_exact {|[ "\"\"\"\\\"\"\"" ]|}] 55 | *) 56 | -------------------------------------------------------------------------------- /lib/configuration/command_configuration.mli: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | module Printer : sig 4 | type printable_result = 5 | | Matches of 6 | { source_path : string option 7 | ; matches : Match.t list 8 | } 9 | | Replacements of 10 | { source_path : string option 11 | ; replacements : Replacement.t list 12 | ; result : string 13 | ; source_content : string 14 | } 15 | 16 | type t = printable_result -> unit 17 | end 18 | 19 | type interactive_review = 20 | { editor : string 21 | ; default_is_accept : bool 22 | } 23 | 24 | type output_options = 25 | { color : bool 26 | ; json_lines : bool 27 | ; json_only_diff : bool 28 | ; overwrite_file_in_place : bool 29 | ; diff : bool 30 | ; stdout : bool 31 | ; substitute_in_place : bool 32 | ; count : bool 33 | ; interactive_review : interactive_review option 34 | } 35 | 36 | type anonymous_arguments = 37 | { match_template : string 38 | ; rewrite_template : string 39 | ; file_filters : string list option 40 | } 41 | 42 | type user_input_options = 43 | { rule : string 44 | ; stdin : bool 45 | ; specification_directories : string list option 46 | ; anonymous_arguments : anonymous_arguments option 47 | ; file_filters : string list option 48 | ; zip_file : string option 49 | ; match_only : bool 50 | ; target_directory : string 51 | ; directory_depth : int option 52 | ; exclude_directory_prefix : string 53 | } 54 | 55 | type run_options = 56 | { sequential : bool 57 | ; verbose : bool 58 | ; match_timeout : int 59 | ; number_of_workers : int 60 | ; dump_statistics : bool 61 | ; substitute_in_place : bool 62 | ; disable_substring_matching : bool 63 | } 64 | 65 | type user_input = 66 | { input_options : user_input_options 67 | ; run_options : run_options 68 | ; output_options : output_options 69 | } 70 | 71 | type t = 72 | { sources : Command_input.t 73 | ; specifications : Specification.t list 74 | ; file_filters : string list option 75 | ; exclude_directory_prefix : string 76 | ; run_options : run_options 77 | ; output_printer : Printer.t 78 | ; interactive_review : interactive_review option 79 | } 80 | 81 | val create : user_input -> t Or_error.t 82 | -------------------------------------------------------------------------------- /test/common/test_go.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | open Language 4 | open Matchers 5 | open Match 6 | open Rewriter 7 | 8 | let configuration = Configuration.create ~match_kind:Fuzzy () 9 | 10 | let run ?(rule = "where true") source match_template rewrite_template = 11 | let rule = Rule.create rule |> Or_error.ok_exn in 12 | Go.first ~configuration match_template source 13 | |> function 14 | | Ok ({environment; _ } as result) -> 15 | if Rule.(sat @@ apply rule environment) then 16 | Rewrite.all ~source ~rewrite_template [result] 17 | |> (fun x -> Option.value_exn x) 18 | |> (fun { rewritten_source; _ } -> rewritten_source) |> print_string 19 | else 20 | assert false 21 | | Error _ -> 22 | print_string rewrite_template 23 | 24 | let%expect_test "gosimple_s1000" = 25 | let source = 26 | {| 27 | select { 28 | case x := <-ch: 29 | fmt.Println(x) 30 | } 31 | |} 32 | in 33 | 34 | let match_template = 35 | {| 36 | select { 37 | case :[1] := :[2]: 38 | :[3] 39 | } 40 | |} 41 | in 42 | 43 | let rewrite_template = 44 | {| 45 | :[1] := :[2] 46 | :[3] 47 | |} 48 | in 49 | run source match_template rewrite_template; 50 | [%expect_exact {| 51 | x := <-ch 52 | fmt.Println(x) 53 | |}] 54 | 55 | let%expect_test "gosimple_s1001" = 56 | let source = 57 | {| 58 | for i, x := range src { 59 | dst[i] = x 60 | } 61 | |} 62 | in 63 | 64 | let match_template = 65 | {| 66 | for :[index_define], :[src_element_define] := range :[src_array] { 67 | :[dst_array][:[index_use]] = :[src_element_use] 68 | } 69 | |} 70 | in 71 | 72 | let rewrite_template = 73 | {| 74 | copy(:[dst_array], :[src_array]) 75 | |} 76 | in 77 | 78 | let rule = {|where :[index_define] == :[index_use], :[src_element_define] == :[src_element_use]|} in 79 | 80 | run ~rule source match_template rewrite_template; 81 | [%expect_exact {| 82 | copy(dst, src) 83 | |}] 84 | 85 | let%expect_test "gosimple_s1003" = 86 | let source = 87 | {| 88 | if strings.Index(x, y) != -1 { ignore } 89 | |} 90 | in 91 | 92 | let match_template = 93 | {| 94 | if strings.:[1](x, y) != -1 { :[_] } 95 | |} 96 | in 97 | 98 | let rewrite_template = {|:[1]|} in 99 | 100 | run source match_template rewrite_template; 101 | [%expect_exact {|Index|}] 102 | -------------------------------------------------------------------------------- /lib/matchers/types.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | module Syntax = struct 4 | 5 | type escapable_string_literals = 6 | { delimiters : string list 7 | ; escape_character: char 8 | } 9 | [@@deriving yojson] 10 | 11 | type comment_kind = 12 | | Multiline of string * string 13 | | Nested_multiline of string * string 14 | | Until_newline of string 15 | [@@deriving yojson] 16 | 17 | type t = { 18 | user_defined_delimiters : (string * string) list; 19 | escapable_string_literals : escapable_string_literals option; [@default None] 20 | raw_string_literals : (string * string) list; 21 | comments : comment_kind list; 22 | } 23 | [@@deriving yojson] 24 | 25 | module type S = sig 26 | val user_defined_delimiters : (string * string) list 27 | val escapable_string_literals : escapable_string_literals option 28 | val raw_string_literals : (string * string) list 29 | val comments : comment_kind list 30 | end 31 | 32 | end 33 | 34 | module Info = struct 35 | module type S = sig 36 | val name : string 37 | val extensions : string list 38 | end 39 | end 40 | 41 | type dimension = 42 | | Code 43 | | Escapable_string_literal 44 | | Raw_string_literal 45 | | Comment 46 | 47 | type id = string 48 | type including = char list 49 | type until = char option 50 | 51 | module Hole = struct 52 | 53 | type sort = 54 | | Everything 55 | | Alphanum 56 | | Non_space 57 | | Line 58 | | Blank 59 | 60 | type t = 61 | { sort : sort 62 | ; identifier : string 63 | ; dimension : dimension 64 | ; optional : bool 65 | } 66 | 67 | let sorts () = 68 | [ Everything 69 | ; Alphanum 70 | ; Non_space 71 | ; Line 72 | ; Blank 73 | ] 74 | end 75 | 76 | type hole = Hole.t 77 | 78 | module Omega = struct 79 | type omega_match_production = 80 | { offset : int 81 | ; identifier : string 82 | ; text : string 83 | } 84 | 85 | type production = 86 | | Unit 87 | | String of string 88 | | Hole of hole 89 | | Match of omega_match_production 90 | end 91 | 92 | type production = 93 | | Unit 94 | | String of string 95 | | Hole of hole 96 | 97 | module Matcher = struct 98 | module type S = sig 99 | include Info.S 100 | 101 | val first 102 | : ?configuration:Configuration.t 103 | -> ?shift:int 104 | -> string 105 | -> string 106 | -> Match.t Or_error.t 107 | 108 | val all 109 | : ?configuration:Configuration.t 110 | -> template:string 111 | -> source:string 112 | -> Match.t list 113 | end 114 | end 115 | -------------------------------------------------------------------------------- /docs/resources/match-semantics.md: -------------------------------------------------------------------------------- 1 | ## Weak delimiter matching 2 | 3 | Suppose we have a pattern like `(:[1])`. We could weaken delimiter matching to 4 | allow matching, for example, anything except parens, i.e., `(])` would be valid 5 | and could be matched against. In strict delimiter matching, where `[]` must be 6 | balanced, `(])` would not be valid. This could, in theory, give a bit of a 7 | performance bump since we wouldn't neet to ensure well-balancedness with 8 | respect to any other delimiters besides `()`. 9 | 10 | Weak delimiter matching only works for unique delimiters. For example, the 11 | trick does not work for a closing delimiter like `end` in Ruby where multiple 12 | opening delimiters like `class` or `def` close with `end`. In this case, we 13 | have no choice but to check well-balancedness of all delimiters with the same 14 | closing delimiter. 15 | 16 | ## Matching alphanumeric delimiters 17 | 18 | A neat thing about Comby is that holes can match at the character level (not 19 | just word boundaries). This makes it a little bit more challenging to identify 20 | alphanumeric delimiters like `for`, `end`, because a character sequence like 21 | `for` in the word `before` would, under normal circumstances, under character 22 | matching, trigger delimiter matching, and Comby will look for an `end` to the 23 | `for` in `before`. This problem doesn't come up for `()` delimiters, for 24 | example, because punctuation isn't mixed with alphanumberic sequences. The way 25 | we deal with this complexity in Comby is as follows: 26 | 27 | We detect alpanumeric delimiters by requiring surrounding content (such as 28 | whitespace or other delimiters). Before something like 'def', we expect 29 | whitespace, punctuation delimiters like ')', and nothing. After 'def', we 30 | expect similar sequences. Buf after something like 'end', we expect the same 31 | but also punctuation like `;` or `.`. 32 | 33 | There's some complexity for detecting these cases: we don't consume the 34 | trailing whitespace, because that would stop us from detecting `begin begin`, 35 | separated by a single space. So the trailing part is only checked as a look 36 | ahead, but not consumed. We do, however, need to consume the prefix in order to 37 | advance the state (otherwise, the prefix whitespace would be handled normally, 38 | and we would re-encounter the delimiter subsequently). 39 | 40 | ## Choice operator behavior and 'attempt' 41 | 42 | If `a1` succees in the parse sequence `a1 >>= a2 >>= a3 <|> b1 >>= b2 >>= b3`, 43 | the whole parser will fail, and `b1 >>= ...` will never be attempted. Putting 44 | `attempt @@ a1 >>= a2 >>= a3 <|> attempt @@ b1 >>= ...` will backtrack if `a1` 45 | succeeds but `a2` fails, and will then try `b1`. This pattern is important for 46 | disambiguating holes `:[1]` and `:[[1]]`, and alphanumeric sequences (like 47 | `begin`, `struct`) where we need to check if the sequence initiates a balanced 48 | delimiter matching or not. 49 | -------------------------------------------------------------------------------- /docs/third-party-licenses/pull-and-update-release-scripts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | LIBS="ppx_deriving_yojson core ppxlib ppx_deriving hack_parallel opium pcre-ocaml ocaml-tls camlzip bisect_ppx mparser ocaml-ci-scripts patdiff lambda-term lwt" 4 | 5 | rm ALL.txt 2> /dev/null 6 | for l in $LIBS; do rm -rf $l; done 7 | 8 | # MIT 9 | mkdir patdiff && \ 10 | wget -P patdiff https://raw.githubusercontent.com/janestreet/patdiff/master/LICENSE.md 11 | 12 | # MIT 13 | mkdir ppx_deriving_yojson && \ 14 | wget -P ppx_deriving_yojson https://raw.githubusercontent.com/ocaml-ppx/ppx_deriving_yojson/master/LICENSE.txt 15 | 16 | # MIT 17 | mkdir core && \ 18 | wget -P core https://raw.githubusercontent.com/janestreet/core/master/LICENSE.md 19 | 20 | # MIT 21 | mkdir ppxlib && \ 22 | wget -P ppxlib https://raw.githubusercontent.com/ocaml-ppx/ppxlib/master/LICENSE.md 23 | 24 | # MIT 25 | mkdir ppx_deriving && \ 26 | wget -P ppx_deriving https://raw.githubusercontent.com/ocaml-ppx/ppx_deriving/master/LICENSE.txt 27 | 28 | # MIT 29 | mkdir hack_parallel && \ 30 | wget -P hack_parallel https://raw.githubusercontent.com/rvantonder/hack-parallel/master/LICENSE 31 | 32 | # MIT 33 | mkdir opium && \ 34 | wget -P opium https://raw.githubusercontent.com/rgrinberg/opium/master/opium.opam 35 | 36 | # LGPL 37 | mkdir pcre-ocaml && \ 38 | wget -P pcre-ocaml https://raw.githubusercontent.com/mmottl/pcre-ocaml/master/LICENSE.md 39 | 40 | # BSD-2 41 | mkdir ocaml-tls && \ 42 | wget -P ocaml-tls https://raw.githubusercontent.com/mirleft/ocaml-tls/master/LICENSE.md 43 | 44 | # LGPL 45 | mkdir camlzip && \ 46 | wget -P camlzip https://raw.githubusercontent.com/xavierleroy/camlzip/master/LICENSE 47 | 48 | # MIT 49 | mkdir bisect_ppx && \ 50 | wget -P bisect_ppx https://raw.githubusercontent.com/aantron/bisect_ppx/master/LICENSE.md 51 | 52 | # LGPL 53 | mkdir mparser && \ 54 | wget -P mparser https://raw.githubusercontent.com/comby-tools/mparser/master/LICENSE.txt 55 | 56 | # ISC 57 | mkdir ocaml-ci-scripts && \ 58 | wget -P ocaml-ci-scripts https://raw.githubusercontent.com/ocaml/ocaml-ci-scripts/master/LICENSE.md 59 | 60 | # MIT 61 | mkdir lwt && \ 62 | wget -P lwt https://raw.githubusercontent.com/ocsigen/lwt/master/LICENSE.md 63 | 64 | # BSD-3 65 | mkdir lambda-term && \ 66 | wget -P lambda-term https://raw.githubusercontent.com/ocaml-community/lambda-term/master/LICENSE 67 | 68 | 69 | # ALL.txt 70 | for l in $LIBS; do 71 | F=$(ls $l | head -n 1) 72 | echo "LICENSE FOR $l:" >> ALL.txt 73 | echo "" >> ALL.txt 74 | cat $l/$F >> ALL.txt 75 | echo "" >> ALL.txt 76 | done 77 | 78 | # update release script 79 | cp ../../scripts/install.sh . 80 | printf "\n\n# comby license\n" >> install.sh 81 | echo "#" >> install.sh 82 | sed 's/^/# /' ../../LICENSE >> install.sh 83 | echo "" >> install.sh 84 | printf "# Begin third party licenses\n\n" >> install.sh 85 | sed 's/^/# /' ALL.txt >> install.sh 86 | mv install.sh ../../scripts/install-with-licenses.sh 87 | -------------------------------------------------------------------------------- /lib/parsers/string_literals.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | module Omega = struct 4 | open Angstrom 5 | 6 | let (|>>) p f = 7 | p >>= fun x -> return (f x) 8 | 9 | module Escapable = struct 10 | module type S = sig 11 | val delimiter : string 12 | val escape : char 13 | end 14 | 15 | module Make (M : S) = struct 16 | (* delimiters can be escaped and parsing continues within the string body *) 17 | let escaped_char_s = 18 | any_char 19 | 20 | let char_token_s = 21 | ((char M.escape *> escaped_char_s >>= fun c -> return (Format.sprintf {|%c%c|} M.escape c)) 22 | <|> (any_char |>> String.of_char) 23 | ) 24 | 25 | 26 | let base_string_literal = 27 | ((string M.delimiter *> (many_till char_token_s (string M.delimiter)) 28 | |>> String.concat) 29 | >>= fun result -> 30 | return (Format.sprintf {|%s%s|} M.delimiter result) 31 | (* unlike Alpha, do not suffix the ending delimiter, it was captured by the delimiter parser in many_till *) 32 | ) 33 | end 34 | end 35 | end 36 | 37 | module Alpha = struct 38 | open MParser 39 | 40 | (** Assumes the left and right delimiter are the same, and that these can be 41 | escaped. Does not parse a string body containing newlines (as usual when 42 | escaping with \n) *) 43 | module Escapable = struct 44 | module type S = sig 45 | val delimiter : string 46 | val escape : char 47 | end 48 | 49 | module Make (M : S) = struct 50 | (* delimiters can be escaped and parsing continues within the string body *) 51 | let escaped_char_s s = 52 | any_char s 53 | 54 | let char_token_s s = 55 | ((char M.escape >> escaped_char_s >>= fun c -> return (Format.sprintf {|%c%c|} M.escape c)) 56 | <|> (any_char |>> String.of_char) 57 | ) 58 | s 59 | 60 | let base_string_literal s = 61 | ((string M.delimiter >> (many_until char_token_s (string M.delimiter)) 62 | |>> String.concat) 63 | >>= fun result -> 64 | return (Format.sprintf {|%s%s%s|} M.delimiter result M.delimiter) 65 | ) 66 | s 67 | end 68 | end 69 | 70 | (** Quoted or raw strings. Allows different left and right delimiters, and 71 | disallows any sort of escaping. Does not support raw strings with identifiers 72 | yet, e.g., {blah||blah} (OCaml) or delim``delim 73 | syntax (Go) *) 74 | module Raw = struct 75 | module type S = sig 76 | val left_delimiter : string 77 | val right_delimiter : string 78 | end 79 | 80 | module Make (M : S) = struct 81 | let char_token_s s = 82 | (any_char_or_nl |>> String.of_char) s 83 | 84 | let base_string_literal s = 85 | (( 86 | string M.left_delimiter >> (many_until char_token_s (string M.right_delimiter)) 87 | |>> String.concat "raw string literal body") 88 | >>= fun result -> 89 | return (Format.sprintf {|%s%s%s|} M.left_delimiter result M.right_delimiter) 90 | ) 91 | s 92 | end 93 | end 94 | end 95 | -------------------------------------------------------------------------------- /lib/rewriter/rewrite_template.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | open Match 4 | 5 | let debug = 6 | Sys.getenv "DEBUG_COMBY" 7 | |> Option.is_some 8 | 9 | 10 | let substitute template env = 11 | let substitution_formats = 12 | [ ":[ ", "]" 13 | ; ":[", ".]" 14 | ; ":[", "\\n]" 15 | ; ":[[", "]]" 16 | ; ":[", "]" 17 | (* optional syntax *) 18 | ; ":[? ", "]" 19 | ; ":[ ?", "]" 20 | ; ":[?", ".]" 21 | ; ":[?", "\\n]" 22 | ; ":[[?", "]]" 23 | ; ":[?", "]" 24 | ] 25 | in 26 | Environment.vars env 27 | |> List.fold ~init:(template, []) ~f:(fun (acc, vars) variable -> 28 | match Environment.lookup env variable with 29 | | Some value -> 30 | List.find_map substitution_formats ~f:(fun (left,right) -> 31 | let pattern = left^variable^right in 32 | if Option.is_some (String.substr_index template ~pattern) then 33 | Some (String.substr_replace_all acc ~pattern ~with_:value, variable::vars) 34 | else 35 | None) 36 | |> Option.value ~default:(acc,vars) 37 | | None -> acc, vars) 38 | 39 | let of_match_context 40 | { range = 41 | { match_start = { offset = start_index; _ } 42 | ; match_end = { offset = end_index; _ } } 43 | ; _ 44 | } 45 | ~source = 46 | if debug then Format.printf "Start idx: %d@.End idx: %d@." start_index end_index; 47 | let before_part = 48 | if start_index = 0 then 49 | "" 50 | else 51 | String.slice source 0 start_index 52 | in 53 | let after_part = String.slice source end_index (String.length source) in 54 | let hole_id = Uuid_unix.(Fn.compose Uuid.to_string create ()) in 55 | let rewrite_template = String.concat [before_part; ":["; hole_id; "]"; after_part] in 56 | hole_id, rewrite_template 57 | 58 | (* return the offset for holes (specified by variables) in a given match template *) 59 | let get_offsets_for_holes rewrite_template variables = 60 | let sorted_variables = 61 | List.fold variables ~init:[] ~f:(fun acc variable -> 62 | match String.substr_index rewrite_template ~pattern:(":["^variable^"]") with 63 | | Some index -> 64 | (variable, index)::acc 65 | | None -> acc) 66 | |> List.sort ~compare:(fun (_, i1) (_, i2) -> i1 - i2) 67 | |> List.map ~f:fst 68 | in 69 | List.fold sorted_variables ~init:(rewrite_template, []) ~f:(fun (rewrite_template, acc) variable -> 70 | match String.substr_index rewrite_template ~pattern:(":["^variable^"]") with 71 | | Some index -> 72 | let rewrite_template = 73 | String.substr_replace_all rewrite_template ~pattern:(":["^variable^"]") ~with_:"" in 74 | rewrite_template, (variable, index)::acc 75 | | None -> rewrite_template, acc) 76 | |> snd 77 | 78 | (* pretend we substituted vars in offsets with environment. return what the offsets are after *) 79 | let get_offsets_after_substitution offsets environment = 80 | List.fold_right offsets ~init:([],0) ~f:(fun (var, offset) (acc, shift) -> 81 | match Environment.lookup environment var with 82 | | None -> failwith "Expected var" 83 | | Some s -> 84 | let offset' = offset + shift in 85 | let shift = shift + String.length s in 86 | ((var, offset')::acc), shift) 87 | |> fst 88 | -------------------------------------------------------------------------------- /test/common/test_user_defined_language.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | open Matchers 3 | open Rewriter 4 | 5 | let configuration = Configuration.create ~match_kind:Fuzzy () 6 | 7 | let run (module M : Matchers.Matcher) source match_template rewrite_template = 8 | M.first ~configuration match_template source 9 | |> function 10 | | Ok result -> 11 | Rewrite.all ~source ~rewrite_template [result] 12 | |> (fun x -> Option.value_exn x) 13 | |> (fun {rewritten_source; _} -> rewritten_source) 14 | |> print_string 15 | | Error _ -> 16 | print_string rewrite_template 17 | 18 | let%expect_test "user_defined_language" = 19 | let c = 20 | Syntax. 21 | { user_defined_delimiters = [("case", "esac")] 22 | ; escapable_string_literals = None 23 | ; raw_string_literals = [] 24 | ; comments = [Multiline ("/*", "*/"); Until_newline "//"] 25 | } 26 | in 27 | let user_lang = Matchers.create c in 28 | let source = 29 | {| 30 | case 31 | case 32 | block 1 33 | esac 34 | 35 | case 36 | block 2 37 | esac 38 | esac 39 | /* 40 | case 41 | ignore this 42 | esac 43 | */ 44 | // case 45 | // ignore this 46 | // esac 47 | |} 48 | in 49 | let match_template = {|case :[1] esac|} in 50 | let rewrite_template = {|case nuked blocks esac|} in 51 | run user_lang source match_template rewrite_template ; 52 | [%expect_exact {| 53 | case nuked blocks esac 54 | /* 55 | case 56 | ignore this 57 | esac 58 | */ 59 | // case 60 | // ignore this 61 | // esac 62 | |}] 63 | 64 | let%expect_test "user_defined_language_from_json" = 65 | let json = 66 | {|{ 67 | "user_defined_delimiters": [ 68 | ["case", "esac"] 69 | ], 70 | "escapable_string_literals": { 71 | "delimiters": ["\""], 72 | "escape_character": "\\" 73 | }, 74 | "raw_string_literals": [], 75 | "comments": [ 76 | [ "Multiline", "/*", "*/" ], 77 | [ "Until_newline", "//" ] 78 | ] 79 | } 80 | |} 81 | in 82 | let user_lang = 83 | Yojson.Safe.from_string json 84 | |> Matchers.Syntax.of_yojson 85 | |> Result.ok_or_failwith 86 | |> Matchers.create 87 | in 88 | let source = "" in 89 | let match_template = {|""|} in 90 | let rewrite_template = {|""|} in 91 | run user_lang source match_template rewrite_template ; 92 | [%expect_exact {|""|}] 93 | 94 | let%expect_test "user_defined_language_from_json_optional_escapable" = 95 | let json = 96 | {|{ 97 | "user_defined_delimiters": [ 98 | ["case", "esac"] 99 | ], 100 | "raw_string_literals": [], 101 | "comments": [ 102 | [ "Multiline", "/*", "*/" ], 103 | [ "Until_newline", "//" ] 104 | ] 105 | } 106 | |} 107 | in 108 | let user_lang = 109 | Yojson.Safe.from_string json 110 | |> Matchers.Syntax.of_yojson 111 | |> Result.ok_or_failwith 112 | |> Matchers.create 113 | in 114 | let source = "" in 115 | let match_template = {|""|} in 116 | let rewrite_template = {|""|} in 117 | run user_lang source match_template rewrite_template ; 118 | [%expect_exact {|""|}] 119 | -------------------------------------------------------------------------------- /lib/match/environment.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | module Data = struct 4 | type t = 5 | { value : string 6 | ; range : Range.t 7 | } 8 | [@@deriving yojson, eq, sexp] 9 | end 10 | 11 | open Data 12 | type data = Data.t 13 | [@@deriving yojson, eq, sexp] 14 | 15 | type t = data Core.String.Map.t 16 | 17 | let create () : t = 18 | String.Map.empty 19 | 20 | let vars (env : t) : string list = 21 | Map.keys env 22 | 23 | let add ?(range = Range.default) (env : t) (var : string) (value : string) : t = 24 | Map.add env ~key:var ~data:{ value; range } 25 | |> function 26 | | `Duplicate -> env 27 | | `Ok env -> env 28 | 29 | let lookup (env : t) (var : string) : string option = 30 | Map.find env var 31 | |> Option.map ~f:(fun { value; _ } -> value) 32 | 33 | let lookup_range (env : t) (var : string) : Range.t option = 34 | Map.find env var 35 | |> Option.map ~f:(fun { range; _ } -> range) 36 | 37 | let fold (env : t) = 38 | Map.fold env 39 | 40 | let update env var value = 41 | Map.change env var ~f:(Option.map ~f:(fun result -> { result with value })) 42 | 43 | let update_range env var range = 44 | Map.change env var ~f:(Option.map ~f:(fun result -> { result with range })) 45 | 46 | let to_string env = 47 | Map.fold env ~init:"" ~f:(fun ~key:variable ~data:{ value; _ } acc -> 48 | Format.sprintf "%s |-> %s\n%s" variable value acc) 49 | 50 | let furthest_match env = 51 | Map.fold 52 | env 53 | ~init:0 54 | ~f:(fun ~key:_ ~data:{ range = { match_start = { offset; _ }; _ }; _ } max -> 55 | Int.max offset max) 56 | 57 | let equal env1 env2 = 58 | Map.equal Data.equal env1 env2 59 | 60 | let merge env1 env2 = 61 | Map.merge_skewed env1 env2 ~combine:(fun ~key:_ v1 _ -> v1) 62 | 63 | let copy env = 64 | fold env ~init:(create ()) ~f:(fun ~key ~data:{ value; range } env' -> 65 | add ~range env' key value) 66 | 67 | let exists env key = 68 | Option.is_some (lookup env key) 69 | 70 | let to_yojson env : Yojson.Safe.json = 71 | let s = 72 | Map.fold_right env ~init:[] ~f:(fun ~key:variable ~data:{value; range} acc -> 73 | let item = 74 | `Assoc 75 | [ ("variable", `String variable) 76 | ; ("value", `String value) 77 | ; ("range", Range.to_yojson range) 78 | ] 79 | in 80 | item::acc) 81 | in 82 | `List s 83 | 84 | let of_yojson (json : Yojson.Safe.json) = 85 | let open Yojson.Safe.Util in 86 | let env = create () in 87 | match json with 88 | | `List l -> 89 | List.fold l ~init:env ~f:(fun env json -> 90 | let variable = member "variable" json |> to_string_option in 91 | let value = member "value" json |> to_string_option in 92 | let range = 93 | member "range" json 94 | |> function 95 | | `Null -> Some Range.default 96 | | json -> 97 | Range.of_yojson json 98 | |> function 99 | | Ok range -> Some range 100 | | Error _ -> None 101 | in 102 | match variable, value with 103 | | Some variable, Some value -> 104 | add env ?range variable value 105 | | _ -> 106 | env) 107 | |> Result.return 108 | | _ -> Error "Invalid JSON for environment" 109 | -------------------------------------------------------------------------------- /docs/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | This code of conduct is adapted from the [Django Code of 4 | Conduct](https://www.djangoproject.com/conduct/). 5 | 6 | Here are a few ground rules that we ask Comby project participants to adhere 7 | to. This code applies equally to contributors and those seeking help and 8 | guidance. 9 | 10 | This isn't an exhaustive list of things that you can't do. Rather, take it in 11 | the spirit in which it's intended - a guide to make it easier to enrich all of 12 | us and the technical communities in which we participate. 13 | 14 | This code of conduct applies to all spaces managed by the Comby project. This 15 | includes the issue tracker and any other forums created which the community 16 | uses for communication. 17 | 18 | If you believe someone is violating the code of conduct, we ask that you report 19 | it by emailing rvantonder@gmail.com. 20 | 21 | - Be friendly and patient. 22 | 23 | - Be welcoming. We strive to be a community that welcomes and supports people 24 | of all backgrounds and identities. This includes, but is not limited to members 25 | of any race, ethnicity, culture, national origin, colour, immigration status, 26 | social and economic class, educational level, sex, sexual orientation, gender 27 | identity and expression, age, size, family status, political belief, religion, 28 | and mental and physical ability. 29 | 30 | - Be considerate. Your work will be used by other people, and you in turn will 31 | depend on the work of others. Any decision you take will affect users and 32 | colleagues, and you should take those consequences into account when making 33 | decisions. Remember that we're a world-wide community, so you might not be 34 | communicating in someone else's primary language. 35 | 36 | - Be respectful. Not all of us will agree all the time, but disagreement is no 37 | excuse for poor behavior and poor manners. We might all experience some 38 | frustration now and then, but we cannot allow that frustration to turn into a 39 | personal attack. It's important to remember that a community where people feel 40 | uncomfortable or threatened is not a productive one. Participants should be 41 | respectful when interacting with other participants. 42 | 43 | - Be careful in the words that you choose. We are a community of professionals, 44 | and we conduct ourselves professionally. Be kind to others. Do not insult or 45 | put down other participants. Harassment and other exclusionary behavior aren't 46 | acceptable. This includes, but is not limited to: Violent threats or language 47 | directed against another person, discriminatory jokes and language, posting 48 | sexually explicit or violent material, posting (or threatening to post) other 49 | people's personally identifying information ("doxing"), personal insults, 50 | especially those using racist or sexist terms, unwelcome sexual attention, 51 | encouraging any of the above behavior, or repeated harassment of others. In 52 | general, if someone asks you to stop, then stop. 53 | 54 | - When we disagree, try to understand why. Disagreements, both social and 55 | technical, happen all the time and participating in open source projects is no 56 | exception. It is important that we resolve disagreements and differing views 57 | constructively. Remember that we're different. Different people have different 58 | perspectives on issues. Being unable to understand why someone holds a 59 | viewpoint doesn't mean that they're wrong. Don't forget that it is human to err 60 | and blaming each other doesn't get us anywhere. Instead, focus on helping to 61 | resolve issues and learning from mistakes. 62 | -------------------------------------------------------------------------------- /test/alpha/test_rewrite_rule.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | open Language 4 | open Matchers 5 | open Match 6 | open Rewriter 7 | 8 | let configuration = Configuration.create ~match_kind:Fuzzy () 9 | 10 | let format s = 11 | let s = String.chop_prefix_exn ~prefix:"\n" s in 12 | let leading_indentation = Option.value_exn (String.lfindi s ~f:(fun _ c -> c <> ' ')) in 13 | s 14 | |> String.split ~on:'\n' 15 | |> List.map ~f:(Fn.flip String.drop_prefix leading_indentation) 16 | |> String.concat ~sep:"\n" 17 | |> String.chop_suffix_exn ~suffix:"\n" 18 | 19 | let run_rule source match_template rewrite_template rule = 20 | Generic.first ~configuration match_template source 21 | |> function 22 | | Error _ -> print_string "bad" 23 | | Ok result -> 24 | match result with 25 | | ({ environment; _ } as m) -> 26 | let e = Rule.(result_env @@ apply rule environment) in 27 | match e with 28 | | None -> print_string "bad bad" 29 | | Some e -> 30 | { m with environment = e } 31 | |> List.return 32 | |> Rewrite.all ~source ~rewrite_template 33 | |> (fun x -> Option.value_exn x) 34 | |> (fun { rewritten_source; _ } -> rewritten_source) 35 | |> print_string 36 | 37 | let%expect_test "rewrite_rule" = 38 | let source = {|int|} in 39 | let match_template = {|:[1]|} in 40 | let rewrite_template = {|:[1]|} in 41 | 42 | let rule = 43 | {| 44 | where rewrite :[1] { "int" -> "expect" } 45 | |} 46 | |> Rule.create 47 | |> Or_error.ok_exn 48 | in 49 | 50 | run_rule source match_template rewrite_template rule; 51 | [%expect_exact {|expect|}] 52 | 53 | let%expect_test "sequenced_rewrite_rule" = 54 | let source = {|{ { a : { b : { c : d } } } }|} in 55 | let match_template = {|{ :[a] : :[rest] }|} in 56 | let rewrite_template = {|{ :[a] : :[rest] }|} in 57 | 58 | let rule = 59 | {| 60 | where 61 | rewrite :[a] { "a" -> "qqq" }, 62 | rewrite :[rest] { "{ b : { :[other] } }" -> "{ :[other] }" } 63 | |} 64 | |> Rule.create 65 | |> Or_error.ok_exn 66 | in 67 | 68 | run_rule source match_template rewrite_template rule; 69 | [%expect_exact {|{ { qqq : { c : d } } }|}] 70 | 71 | let%expect_test "rewrite_rule_for_list" = 72 | let source = {|[1, 2, 3, 4,]|} in 73 | let match_template = {|[:[contents]]|} in 74 | let rewrite_template = {|[:[contents]]|} in 75 | 76 | let rule = 77 | {| 78 | where rewrite :[contents] { ":[[x]]," -> ":[[x]];" } 79 | |} 80 | |> Rule.create 81 | |> Or_error.ok_exn 82 | in 83 | 84 | run_rule source match_template rewrite_template rule; 85 | [%expect_exact {|[1; 2; 3; 4;]|}] 86 | 87 | let%expect_test "rewrite_rule_for_list_strip_last" = 88 | let source = {|[1, 2, 3, 4]|} in 89 | let match_template = {|[:[contents]]|} in 90 | let rewrite_template = {|[:[contents]]|} in 91 | 92 | let rule = 93 | {| 94 | where rewrite :[contents] { ":[x], " -> ":[x]; " } 95 | |} 96 | |> Rule.create 97 | |> Or_error.ok_exn 98 | in 99 | 100 | run_rule source match_template rewrite_template rule; 101 | [%expect_exact {|[1; 2; 3; 4]|}] 102 | 103 | let%expect_test "haskell_example" = 104 | let source = {| 105 | (concat 106 | [ "blah blah blah" 107 | , "blah" 108 | ]) 109 | |} in 110 | let match_template = {|(concat [:[contents]])|} in 111 | let rewrite_template = {|(:[contents])|} in 112 | 113 | let rule = 114 | {| 115 | where rewrite :[contents] { "," -> "++" } 116 | |} 117 | |> Rule.create 118 | |> Or_error.ok_exn 119 | in 120 | 121 | run_rule source match_template rewrite_template rule; 122 | [%expect_exact {| 123 | ( "blah blah blah" 124 | ++ "blah" 125 | ) 126 | |}] 127 | -------------------------------------------------------------------------------- /test/omega/test_rewrite_rule.ml: -------------------------------------------------------------------------------- 1 | (*open Core 2 | 3 | open Language 4 | open Matchers 5 | open Match 6 | open Rewriter 7 | 8 | let configuration = Configuration.create ~match_kind:Fuzzy () 9 | 10 | let format s = 11 | let s = String.chop_prefix_exn ~prefix:"\n" s in 12 | let leading_indentation = Option.value_exn (String.lfindi s ~f:(fun _ c -> c <> ' ')) in 13 | s 14 | |> String.split ~on:'\n' 15 | |> List.map ~f:(Fn.flip String.drop_prefix leading_indentation) 16 | |> String.concat ~sep:"\n" 17 | |> String.chop_suffix_exn ~suffix:"\n" 18 | 19 | let run_rule source match_template rewrite_template rule = 20 | Generic.first ~configuration match_template source 21 | |> function 22 | | Error _ -> print_string "bad" 23 | | Ok result -> 24 | match result with 25 | | ({ environment; _ } as m) -> 26 | let e = Rule.(result_env @@ apply rule environment) in 27 | match e with 28 | | None -> print_string "bad bad" 29 | | Some e -> 30 | { m with environment = e } 31 | |> List.return 32 | |> Rewrite.all ~source ~rewrite_template 33 | |> (fun x -> Option.value_exn x) 34 | |> (fun { rewritten_source; _ } -> rewritten_source) 35 | |> print_string 36 | 37 | let%expect_test "rewrite_rule" = 38 | let source = {|int|} in 39 | let match_template = {|:[1]|} in 40 | let rewrite_template = {|:[1]|} in 41 | 42 | let rule = 43 | {| 44 | where rewrite :[1] { "int" -> "expect" } 45 | |} 46 | |> Rule.create 47 | |> Or_error.ok_exn 48 | in 49 | 50 | run_rule source match_template rewrite_template rule; 51 | [%expect_exact {|expect|}] 52 | 53 | let%expect_test "sequenced_rewrite_rule" = 54 | let source = {|{ { a : { b : { c : d } } } }|} in 55 | let match_template = {|{ :[a] : :[rest] }|} in 56 | let rewrite_template = {|{ :[a] : :[rest] }|} in 57 | 58 | let rule = 59 | {| 60 | where 61 | rewrite :[a] { "a" -> "qqq" }, 62 | rewrite :[rest] { "{ b : { :[other] } }" -> "{ :[other] }" } 63 | |} 64 | |> Rule.create 65 | |> Or_error.ok_exn 66 | in 67 | 68 | run_rule source match_template rewrite_template rule; 69 | [%expect_exact {|{ { qqq : { c : d } } }|}] 70 | 71 | let%expect_test "rewrite_rule_for_list" = 72 | let source = {|[1, 2, 3, 4,]|} in 73 | let match_template = {|[:[contents]]|} in 74 | let rewrite_template = {|[:[contents]]|} in 75 | 76 | let rule = 77 | {| 78 | where rewrite :[contents] { ":[[x]]," -> ":[[x]];" } 79 | |} 80 | |> Rule.create 81 | |> Or_error.ok_exn 82 | in 83 | 84 | run_rule source match_template rewrite_template rule; 85 | [%expect_exact {|[1; 2; 3; 4;]|}] 86 | 87 | let%expect_test "rewrite_rule_for_list_strip_last" = 88 | let source = {|[1, 2, 3, 4]|} in 89 | let match_template = {|[:[contents]]|} in 90 | let rewrite_template = {|[:[contents]]|} in 91 | 92 | let rule = 93 | {| 94 | where rewrite :[contents] { ":[x], " -> ":[x]; " } 95 | |} 96 | |> Rule.create 97 | |> Or_error.ok_exn 98 | in 99 | 100 | run_rule source match_template rewrite_template rule; 101 | [%expect_exact {|[1; 2; 3; 4]|}] 102 | 103 | let%expect_test "haskell_example" = 104 | let source = {| 105 | (concat 106 | [ "blah blah blah" 107 | , "blah" 108 | ]) 109 | |} in 110 | let match_template = {|(concat [:[contents]])|} in 111 | let rewrite_template = {|(:[contents])|} in 112 | 113 | let rule = 114 | {| 115 | where rewrite :[contents] { "," -> "++" } 116 | |} 117 | |> Rule.create 118 | |> Or_error.ok_exn 119 | in 120 | 121 | run_rule source match_template rewrite_template rule; 122 | [%expect_exact {| 123 | ( "blah blah blah" 124 | ++ "blah" 125 | ) 126 | |}] 127 | *) 128 | -------------------------------------------------------------------------------- /docs/FEATURE_TABLE.md: -------------------------------------------------------------------------------- 1 | # Feature Table 2 | 3 | This document tabulates the current feature stability and development status. 4 | 5 | ## Legend 6 | 7 | | Status | Icon | Description | 8 | |-------------------|-------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 9 | | Stable | :white_check_mark: | There are no planned changes to this feature in the forseeable future. | 10 | | Beta | :b: | The feature is available and in beta for quality control testing, after which it becomes stable | 11 | | Experimental | :alembic: | This feature is available but how it works may change in the near future. If you rely on this feature, you may need to update the project that uses it in the future. | 12 | | Under development | :building_construction: | This feature is planned or being actively developed and is not currently available. | 13 | | Requested | :bulb: | This feature has been requested but is not being actively developed. | 14 | 15 | ## Features 16 | 17 | | Name | Status | Description | 18 | |---------------------------------------------|-------------------------|-------------------------------------------------------------------------------------------------------| 19 | | Basic template syntax | :white_check_mark: | See [basic syntax](https://comby.dev/#match-syntax) | 20 | | Matching for balanced alphanumeric keywords | :white_check_mark: | | 21 | | Interactive review mode | :white_check_mark: | Use `-review` for interactive review like in [codemod](https://github.com/facebook/codemod) | 22 | | Sub-matching rule expressions | :alembic: | See [sub-matching syntax](https://comby.dev/#experimental-language-features-sub-matching) | 23 | | Nested rewrite rule expressions | :alembic: | See [rewrite expression syntax](https://comby.dev/#experimental-language-features-rewrite-expression) | 24 | | Optional holes | :alembic: | See [syntax](https://comby.dev/#match-syntax) and [usage tips](https://comby.dev/#tips-and-tricks) | 25 | | Matching for arbitrary balanced tags | :building_construction: | See [roadmap](https://github.com/comby-tools/comby/blob/master/docs/ROADMAP.md) | 26 | | Indentation-sensitive matching | :building_construction: | See [#65](https://github.com/comby-tools/comby/issues/65) and [roadmap](https://github.com/comby-tools/comby/blob/master/docs/ROADMAP.md) | 27 | | Interactive mode with git-patch | :bulb: | See [#134](https://github.com/comby-tools/comby/issues/134) | 28 | | Editor extensions | :bulb: | See [#103](https://github.com/comby-tools/comby/issues/103) | 29 | -------------------------------------------------------------------------------- /test/common/test_c.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | open Matchers 4 | open Rewriter 5 | 6 | let configuration = Configuration.create ~match_kind:Fuzzy () 7 | 8 | let run source match_template rewrite_template = 9 | C.first ~configuration match_template source 10 | |> function 11 | | Ok result -> 12 | Rewrite.all ~source ~rewrite_template [result] 13 | |> (fun x -> Option.value_exn x) 14 | |> (fun { rewritten_source; _ } -> rewritten_source) 15 | |> print_string 16 | | Error _ -> 17 | print_string rewrite_template 18 | 19 | let%expect_test "comments_1" = 20 | let source = {|match this /**/ expect end|} in 21 | let match_template = {|match this :[1] end|} in 22 | let rewrite_template = {|:[1]|} in 23 | 24 | run source match_template rewrite_template; 25 | [%expect_exact {|expect|}] 26 | 27 | let%expect_test "comments_2" = 28 | let source = {|match this /* */ expect end|} in 29 | let match_template = {|match this :[1] end|} in 30 | let rewrite_template = {|:[1]|} in 31 | 32 | run source match_template rewrite_template; 33 | [%expect_exact {|expect|}] 34 | 35 | let%expect_test "comments_3" = 36 | let source = {|match this /* blah blah */ expect /**/ end|} in 37 | let match_template = {|match this :[1] end|} in 38 | let rewrite_template = {|:[1]|} in 39 | 40 | run source match_template rewrite_template; 41 | [%expect_exact {|expect|}] 42 | 43 | let%expect_test "comments_4" = 44 | let source = {|match this expect/**/end|} in 45 | let match_template = {|match this :[1]end|} in 46 | let rewrite_template = {|:[1]|} in 47 | 48 | run source match_template rewrite_template; 49 | [%expect_exact {|expect|}] 50 | 51 | let%expect_test "comments_5" = 52 | let source = {|match this expect /**/end|} in 53 | let match_template = {|match this :[1] end|} in 54 | let rewrite_template = {|:[1]|} in 55 | 56 | run source match_template rewrite_template; 57 | [%expect_exact {|expect|}] 58 | 59 | let%expect_test "comments_6" = 60 | let source = {|/* don't match this (a) end */|} in 61 | let match_template = {|match this :[1] end|} in 62 | let rewrite_template = {|nothing matches|} in 63 | 64 | run source match_template rewrite_template; 65 | [%expect_exact {|nothing matches|}] 66 | 67 | let%expect_test "comments_7" = 68 | let source = {|/* don't match /**/ this (a) end */|} in 69 | let match_template = {|match this :[1] end|} in 70 | let rewrite_template = {|nothing matches|} in 71 | 72 | run source match_template rewrite_template; 73 | [%expect_exact {|nothing matches|}] 74 | 75 | let%expect_test "comments_8" = 76 | let source = {|(/* don't match this (a) end */)|} in 77 | let match_template = {|match this :[1] end|} in 78 | let rewrite_template = {|nothing matches|} in 79 | 80 | run source match_template rewrite_template; 81 | [%expect_exact {|nothing matches|}] 82 | 83 | let%expect_test "comments_9" = 84 | let source = {|/* don't match this (a) end */ do match this (b) end|} in 85 | let match_template = {|match this :[1] end|} in 86 | let rewrite_template = {|:[1]|} in 87 | 88 | run source match_template rewrite_template; 89 | [%expect_exact {|/* don't match this (a) end */ do (b)|}] 90 | 91 | let%expect_test "comments_10" = 92 | let source = {|/* don't match this (a) end */ do match this () end|} in 93 | let match_template = {|match this :[1] end|} in 94 | let rewrite_template = {|:[1]|} in 95 | 96 | run source match_template rewrite_template; 97 | [%expect_exact {|/* don't match this (a) end */ do ()|}] 98 | 99 | let%expect_test "comments_11" = 100 | let source = {|do match this (b) end /* don't match this (a) end */|} in 101 | let match_template = {|match this :[1] end|} in 102 | let rewrite_template = {|:[1]|} in 103 | 104 | run source match_template rewrite_template; 105 | [%expect_exact {|do (b) /* don't match this (a) end */|}] 106 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # comby 2 | 3 | [![Apache-2.0](https://img.shields.io/badge/license-Apache-blue.svg)](LICENSE) 4 | [![Build Status](https://travis-ci.com/comby-tools/comby.svg?branch=master)](https://travis-ci.com/comby-tools/comby) 5 | ![Coveralls github](https://img.shields.io/coveralls/github/comby-tools/comby) 6 | [![Downloads](https://img.shields.io/github/downloads/comby-tools/comby/total.svg?color=orange)](Downloads) 7 | [![Commit](https://img.shields.io/github/last-commit/comby-tools/comby.svg)](Commit) 8 | [![Gitter](https://img.shields.io/gitter/room/comby-tools/comby.svg?color=teal)](https://gitter.im/comby-tools/community) 9 | 10 | ![](https://user-images.githubusercontent.com/888624/64916761-0b657780-d752-11e9-96e2-cd81a2681139.gif) 11 | 12 | ### See the [usage documentation](https://comby.dev). 13 | [A short example below](https://github.com/comby-tools/comby#isnt-a-regex-approach-like-sed-good-enough) shows how comby simplifies matching and rewriting compared to regex approaches like `sed`. 14 | 15 |
16 | Comby supports interactive review mode (click here to see it in action). 17 | 18 | ![](https://user-images.githubusercontent.com/888624/69503010-b8870980-0ed2-11ea-828d-68c152ed9def.gif) 19 | 20 |
21 | 22 | **Need help writing patterns or have other problems? Post them in [Gitter](https://gitter.im/comby-tools/community).** 23 | 24 | ## Install (pre-built binaries) 25 | 26 | ### Mac OS X 27 | 28 | - `brew install comby` 29 | 30 | ### Ubuntu Linux 31 | 32 | - `bash <(curl -sL get.comby.dev)` 33 | 34 | - **Arch and other Linux**: The PCRE library is dynamically linked in the Ubuntu binary. For other distributions, like Arch, a fixup is needed: `ln -s /usr/lib/libpcre.so /usr/lib/libpcre.so.3`. Alternatively, consider [building from source](https://github.com/comby-tools/comby#build-from-source). 35 | 36 | 37 | ### Windows 38 | 39 | - [Install the Windows Subsystem for Linux](https://docs.microsoft.com/en-us/windows/wsl/install-win10) and install Ubuntu. Then run `bash <(curl -sL get.comby.dev)` 40 | 41 | 42 | ### Docker 43 | 44 | - `docker pull comby/comby` 45 | 46 |
47 | click to expand an example invocation for the docker image 48 | 49 | Running with docker on `stdin`: 50 | 51 | ```bash 52 | echo '(👋 hi)' | docker run -a stdin -a stdout -i comby/comby '(:[emoji] hi)' 'bye :[emoji]' lisp -stdin 53 | ``` 54 | 55 | 56 | 57 |
58 | 59 | 60 | 61 | ### Or [try it live](https://bit.ly/2UXkonD). 62 | 63 | ## Isn't a regex approach like sed good enough? 64 | 65 | Sometimes, yes. But often, small changes and refactorings are complicated by nested expressions, comments, or strings. Consider the following C-like snippet. Say the challenge is to rewrite the two `if` conditions to the value `1`. Can you write a regular expression that matches the contents of the two if condition expressions, and only those two? Feel free to share your pattern with [@rvtond](https://twitter.com/rvtond) on Twitter. 66 | 67 | ```c 68 | if (fgets(line, 128, file_pointer) == Null) // 1) if (...) returns 0 69 | return 0; 70 | ... 71 | if (scanf("%d) %d", &x, &y) == 2) // 2) if (scanf("%d) %d", &x, &y) == 2) returns 0 72 | return 0; 73 | ``` 74 | 75 | To match these with comby, all you need to write is `if (:[condition])`, and specify one flag that this language is C-like. The replacement is `if (1)`. See the [live example](https://bit.ly/30935ou). 76 | 77 | ## Build from source 78 | 79 | - Install [opam](https://opam.ocaml.org/doc/Install.html) 80 | 81 | - Create a new switch if you don't have OCaml installed: 82 | 83 | ``` 84 | opam init 85 | opam switch create 4.09.0 4.09.0 86 | ``` 87 | 88 | - Install OS dependencies: 89 | 90 | - **Linux:** `sudo apt-get install pkg-config libpcre3-dev` 91 | 92 | - **Mac:** `brew install pkg-config pcre` 93 | 94 | - Then install the library dependencies: 95 | 96 | ``` 97 | git clone https://github.com/comby-tools/comby 98 | cd comby && opam install . --deps-only -y 99 | ``` 100 | 101 | - Build and test 102 | 103 | ``` 104 | make 105 | make test 106 | ``` 107 | 108 | - If you want to install `comby` on your `PATH`, run 109 | 110 | ``` 111 | make install 112 | ``` 113 | -------------------------------------------------------------------------------- /docs/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Contributions to this project are welcome and should be submitted via GitHub 4 | pull requests. If you plan to embark on implementing a significant feature 5 | (anything besides trivial bug fixes, or small code cleanups), please first 6 | [create an issue](https://github.com/comby-tools/comby/issues/new/choose) to 7 | discuss your change. 8 | 9 | Here are some specific implementation guidelines, and also the rationale and 10 | background behind Comby's current feature set and behavior. Please keep these 11 | in mind if you plan to implement or suggest a change to the tool behavior, and 12 | be patient to discuss such changes. 13 | 14 | ### PRs 15 | 16 | - Essentially every PR (corresponding to a new feature) should include one or 17 | more test cases. 18 | 19 | - Your PR build may fail if there is a significant performance regression (> 20% slower). 20 | 21 | - Your PR build may fail if there is a reduction in code coverage. That's usually OK for small reductions. 22 | 23 | ### Comby syntax rationale and proposal guidelines 24 | 25 | *If you are proposing a change to do with Comby metasyntax, like `:[hole]`, 26 | please read this section. Otherwise, feel free to ignore it.* 27 | 28 | This section discusses choices in the metasyntax of Comby (like `:[hole]`, 29 | i.e., syntax that does not refer to concrete source code), and things to 30 | consider if you are thinking of proposing a change. 31 | 32 | **Template syntax.** Comby implements hole syntax like `:[hole]` that is 33 | different from familiar syntaxes found as in, for example, regular expressions. 34 | This syntax has been gradually expanded from `:[hole]` to [four other 35 | forms](https://comby.dev/#match-syntax) over the course of months. These forms 36 | roughly correspond to basic [POSIX character 37 | classes](https://en.wikibooks.org/wiki/Regular_Expressions/POSIX_Basic_Regular_Expressions). 38 | The design intends to keep the feature set of metasyntax small. This simplifies 39 | the complexity of understanding and writing declarative templates. It can be 40 | tempting to grasp for familiar syntax as in regular expressions, and it might 41 | sometimes feel like similar behavior is missing from Comby. However, it's not a 42 | light consideration to support similar regex syntax, as this can interact in 43 | undesirable ways with Comby's current features. For example, suppose Comby 44 | supported a way to match arbitrary regex within templates. Would patterns like 45 | `(` then potentially break the ability to parse balanced parentheses later? How 46 | would it be avoided? It's not so simple to blacklist the `(` from this 47 | hypothetical regex support within Comby, because the set and significance of 48 | delimiters like `(` can change depending on language. In Bash, we might have to 49 | blacklist `case` and `esac` from hypothetical regex support, which becomes 50 | awkward to blacklist in regex because these are alphanumeric words. 51 | 52 | Thus, introducing new syntax is tempting, but needs careful consideration: it 53 | is very easy to add syntax to a language, but difficult to take it away (or 54 | change the underlying behavior) once it is supported. In general, increasing 55 | the amount of syntactic forms induces a higher learning curve, higher potential 56 | cause for confusion for users, and more documentation. Lowering syntax 57 | variations lead to more obvious and comprehensible templates, where the idea 58 | strives to be minimal and useful, rather than comprehensive and 59 | over-engineered. Thus, being conservative and thoughtful about declarative 60 | syntax, and thinking through the implications, can help to avoid problems down 61 | the line. Thus, to set expectations: be patient and understanding with 62 | proposals and feedback related to changing syntax (both yours and others). 63 | 64 | **Rules.** Comby implements rules that are separate, optional inputs. One rule 65 | is associated with a match and rewrite template. Rules impose constraints on 66 | matches and can extend rewriting. It is an explicit decision to separate rules 67 | (and their constraints) from templates. This avoids conflating constraints and 68 | concrete syntax. Compare to regular expressions which mix these concerns, 69 | leading to the possiblity of very complex expressions (e.g., with match groups 70 | and `|` clauses), and the need to escape these metacharacters (or concrete 71 | characters). Separate rules in Comby achieve a more declarative way for 72 | introducing constraints, and deliver readable and escape-free match and rewrite 73 | templates. Thus, proposing syntax changes related to constraint should usually 74 | be considered in the context of rules. 75 | -------------------------------------------------------------------------------- /test/alpha/test_hole_extensions.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | open Matchers 4 | open Rewriter 5 | 6 | let configuration = Configuration.create ~match_kind:Fuzzy () 7 | 8 | let run_all ?(configuration = configuration) source match_template rewrite_template = 9 | Generic.all ~configuration ~template:match_template ~source 10 | |> function 11 | | [] -> print_string "No matches." 12 | | results -> 13 | Option.value_exn (Rewrite.all ~source ~rewrite_template results) 14 | |> (fun { rewritten_source; _ } -> rewritten_source) 15 | |> print_string 16 | 17 | let%expect_test "non_space" = 18 | let run = run_all in 19 | let source = {| foo. foo.bar.quux derp|} in 20 | let match_template = {|:[x.]|} in 21 | let rewrite_template = {|{:[x]}|} in 22 | run source match_template rewrite_template; 23 | [%expect_exact {| {foo.} {foo.bar.quux} {derp}|}] 24 | 25 | let%expect_test "only_space" = 26 | let run = run_all in 27 | let source = {| foo. foo.bar.quux derp|} in 28 | let match_template = {|:[ x]|} in 29 | let rewrite_template = {|{:[x]}|} in 30 | run source match_template rewrite_template; 31 | [%expect_exact {|{ }foo.{ }foo.bar.quux{ }derp|}] 32 | 33 | let%expect_test "up_to_newline" = 34 | let run = run_all in 35 | let source = 36 | {| 37 | foo. 38 | foo.bar.quux 39 | derp 40 | |} in 41 | let match_template = {|:[x\n]|} in 42 | let rewrite_template = {|{:[x]}|} in 43 | run source match_template rewrite_template; 44 | [%expect_exact {|{ 45 | }{foo. 46 | }{foo.bar.quux 47 | }{derp 48 | }|}] 49 | 50 | let%expect_test "match_empty_in_newline_hole" = 51 | let run = run_all in 52 | let source = 53 | {|stuff 54 | after 55 | |} in 56 | let match_template = {|stuff:[x\n]|} in 57 | let rewrite_template = {|{->:[x]<-}|} in 58 | run source match_template rewrite_template; 59 | [%expect_exact {|{-> 60 | <-}after 61 | |}] 62 | 63 | let%expect_test "leading_indentation" = 64 | let run = run_all in 65 | let source = 66 | {| 67 | foo. bar bazz 68 | foo.bar.quux 69 | derp 70 | |} in 71 | let match_template = {|:[ leading_indentation]:[rest\n]|} in 72 | let rewrite_template = {|{:[leading_indentation]}:[rest]|} in 73 | run source match_template rewrite_template; 74 | [%expect_exact {| 75 | { }foo. bar bazz 76 | { }foo.bar.quux 77 | { }derp 78 | |}] 79 | 80 | let%expect_test "non_space_partial_match" = 81 | let run = run_all in 82 | let source = {| foo. foo.bar.quux derp|} in 83 | let match_template = {|foo.:[x.]ux|} in 84 | let rewrite_template = {|{:[x]}|} in 85 | run source match_template rewrite_template; 86 | [%expect_exact {| foo. {bar.qu} derp|}] 87 | 88 | let%expect_test "non_space_does_not_match_reserved_delimiters" = 89 | let run = run_all in 90 | let source = {|fo.o(x)|} in 91 | let match_template = {|:[f.]|} in 92 | let rewrite_template = {|{:[f]}|} in 93 | run source match_template rewrite_template; 94 | [%expect_exact {|{fo.o}({x})|}] 95 | 96 | let%expect_test "alphanum_partial_match" = 97 | let run = run_all in 98 | let source = {| foo. foo.bar.quux derp|} in 99 | let match_template = {|foo.b:[x]r.quux|} in 100 | let rewrite_template = {|{:[x]}|} in 101 | run source match_template rewrite_template; 102 | [%expect_exact {| foo. {a} derp|}] 103 | 104 | let%expect_test "newline_matcher_should_not_be_sat_on_space" = 105 | let run = run_all in 106 | let source = 107 | {|a b c d 108 | e f g h|} in 109 | let match_template = {|:[line\n] |} in 110 | let rewrite_template = {|{:[line]}|} in 111 | run source match_template rewrite_template; 112 | [%expect_exact {|{a b c d 113 | }e f g h|}]; 114 | 115 | let run = run_all in 116 | let source = 117 | {|a b c d 118 | e f g h|} in 119 | let match_template = {|:[line\n]:[next]|} in 120 | let rewrite_template = {|{:[line]}|} in 121 | run source match_template rewrite_template; 122 | [%expect_exact {|{a b c d 123 | }|}]; 124 | 125 | let run = run_all in 126 | let source = 127 | {|a b c d 128 | e f g h 129 | |} in 130 | let match_template = {|:[line1\n]:[next\n]|} in 131 | let rewrite_template = {|{:[line1]|:[next]}|} in 132 | run source match_template rewrite_template; 133 | [%expect_exact {|{a b c d 134 | |e f g h 135 | }|}] 136 | 137 | let%expect_test "implicit_equals" = 138 | let run = run_all in 139 | let source = {|a b a|} in 140 | let match_template = {|:[[x]] :[[m]] :[[x]]|} in 141 | let rewrite_template = {|:[m]|} in 142 | run source match_template rewrite_template; 143 | [%expect_exact {|b|}] 144 | -------------------------------------------------------------------------------- /test/omega/test_hole_extensions.ml: -------------------------------------------------------------------------------- 1 | (*open Core 2 | 3 | open Matchers 4 | open Rewriter 5 | 6 | let configuration = Configuration.create ~match_kind:Fuzzy () 7 | 8 | let run_all ?(configuration = configuration) source match_template rewrite_template = 9 | Generic.all ~configuration ~template:match_template ~source 10 | |> function 11 | | [] -> print_string "No matches." 12 | | results -> 13 | Option.value_exn (Rewrite.all ~source ~rewrite_template results) 14 | |> (fun { rewritten_source; _ } -> rewritten_source) 15 | |> print_string 16 | 17 | let%expect_test "non_space" = 18 | let run = run_all in 19 | let source = {| foo. foo.bar.quux derp|} in 20 | let match_template = {|:[x.]|} in 21 | let rewrite_template = {|{:[x]}|} in 22 | run source match_template rewrite_template; 23 | [%expect_exact {| {foo.} {foo.bar.quux} {derp}|}] 24 | 25 | let%expect_test "only_space" = 26 | let run = run_all in 27 | let source = {| foo. foo.bar.quux derp|} in 28 | let match_template = {|:[ x]|} in 29 | let rewrite_template = {|{:[x]}|} in 30 | run source match_template rewrite_template; 31 | [%expect_exact {|{ }foo.{ }foo.bar.quux{ }derp|}] 32 | 33 | let%expect_test "up_to_newline" = 34 | let run = run_all in 35 | let source = 36 | {| 37 | foo. 38 | foo.bar.quux 39 | derp 40 | |} in 41 | let match_template = {|:[x\n]|} in 42 | let rewrite_template = {|{:[x]}|} in 43 | run source match_template rewrite_template; 44 | [%expect_exact {|{ 45 | }{foo. 46 | }{foo.bar.quux 47 | }{derp 48 | }|}] 49 | 50 | let%expect_test "match_empty_in_newline_hole" = 51 | let run = run_all in 52 | let source = 53 | {|stuff 54 | after 55 | |} in 56 | let match_template = {|stuff:[x\n]|} in 57 | let rewrite_template = {|{->:[x]<-}|} in 58 | run source match_template rewrite_template; 59 | [%expect_exact {|{-> 60 | <-}after 61 | |}] 62 | 63 | let%expect_test "leading_indentation" = 64 | let run = run_all in 65 | let source = 66 | {| 67 | foo. bar bazz 68 | foo.bar.quux 69 | derp 70 | |} in 71 | let match_template = {|:[ leading_indentation]:[rest\n]|} in 72 | let rewrite_template = {|{:[leading_indentation]}:[rest]|} in 73 | run source match_template rewrite_template; 74 | [%expect_exact {| 75 | { }foo. bar bazz 76 | { }foo.bar.quux 77 | { }derp 78 | |}] 79 | 80 | let%expect_test "non_space_partial_match" = 81 | let run = run_all in 82 | let source = {| foo. foo.bar.quux derp|} in 83 | let match_template = {|foo.:[x.]ux|} in 84 | let rewrite_template = {|{:[x]}|} in 85 | run source match_template rewrite_template; 86 | [%expect_exact {| foo. {bar.qu} derp|}] 87 | 88 | let%expect_test "non_space_does_not_match_reserved_delimiters" = 89 | let run = run_all in 90 | let source = {|fo.o(x)|} in 91 | let match_template = {|:[f.]|} in 92 | let rewrite_template = {|{:[f]}|} in 93 | run source match_template rewrite_template; 94 | [%expect_exact {|{fo.o}({x})|}] 95 | 96 | let%expect_test "alphanum_partial_match" = 97 | let run = run_all in 98 | let source = {| foo. foo.bar.quux derp|} in 99 | let match_template = {|foo.b:[x]r.quux|} in 100 | let rewrite_template = {|{:[x]}|} in 101 | run source match_template rewrite_template; 102 | [%expect_exact {| foo. {a} derp|}] 103 | 104 | let%expect_test "newline_matcher_should_not_be_sat_on_space" = 105 | let run = run_all in 106 | let source = 107 | {|a b c d 108 | e f g h|} in 109 | let match_template = {|:[line\n] |} in 110 | let rewrite_template = {|{:[line]}|} in 111 | run source match_template rewrite_template; 112 | [%expect_exact {|{a b c d 113 | }e f g h|}]; 114 | 115 | let run = run_all in 116 | let source = 117 | {|a b c d 118 | e f g h|} in 119 | let match_template = {|:[line\n]:[next]|} in 120 | let rewrite_template = {|{:[line]}|} in 121 | run source match_template rewrite_template; 122 | [%expect_exact {|{a b c d 123 | }|}]; 124 | 125 | let run = run_all in 126 | let source = 127 | {|a b c d 128 | e f g h 129 | |} in 130 | let match_template = {|:[line1\n]:[next\n]|} in 131 | let rewrite_template = {|{:[line1]|:[next]}|} in 132 | run source match_template rewrite_template; 133 | [%expect_exact {|{a b c d 134 | |e f g h 135 | }|}] 136 | 137 | let%expect_test "implicit_equals" = 138 | let run = run_all in 139 | let source = {|a b a|} in 140 | let match_template = {|:[[x]] :[[m]] :[[x]]|} in 141 | let rewrite_template = {|:[m]|} in 142 | run source match_template rewrite_template; 143 | [%expect_exact {|b|}] 144 | *) 145 | -------------------------------------------------------------------------------- /lib/configuration/diff_configuration.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | (* This is the default patdiff configuration, except whitespace is toggled to 4 | true. See patdiff/lib/configuration record for options.*) 5 | let default context = 6 | Format.sprintf 7 | {|;; -*- scheme -*- 8 | ;; patdiff Configuration file 9 | 10 | ( 11 | (context %d) 12 | 13 | (line_same 14 | ((prefix ((text " |") (style ((bg bright_black) (fg black))))))) 15 | 16 | (keep_whitespace true) 17 | 18 | (line_old 19 | ((prefix ((text "-|") (style ((bg red)(fg black))))) 20 | (style ((fg red))) 21 | (word_same (dim)))) 22 | 23 | (line_new 24 | ((prefix ((text "+|") (style ((bg green)(fg black))))) 25 | (style ((fg green))))) 26 | 27 | (line_unified 28 | ((prefix ((text "!|") (style ((bg yellow)(fg black))))))) 29 | 30 | (header_old 31 | ((prefix ((text "------ ") (style ((fg red))))) 32 | (style (bold)))) 33 | 34 | (header_new 35 | ((prefix ((text "++++++ ") (style ((fg green))))) 36 | (style (bold)))) 37 | 38 | (hunk 39 | ((prefix ((text "@|") (style ((bg bright_black) (fg black))))) 40 | (suffix ((text " ============================================================") (style ()))) 41 | (style (bold)))) 42 | )|} context 43 | 44 | let terminal ?(context = 16) () = 45 | Patdiff_lib.Configuration.Config.t_of_sexp (Sexp.of_string (default context)) 46 | |> Patdiff_lib.Configuration.parse 47 | 48 | let diff_configuration = 49 | {|;; -*- scheme -*- 50 | ;; patdiff Configuration file 51 | 52 | ( 53 | (context 1) 54 | 55 | (line_same 56 | ((prefix ((text " |") (style ((bg bright_black) (fg black))))))) 57 | 58 | (line_old 59 | ((prefix ((text "-|") (style ((bg red)(fg black))))) 60 | (style ((fg red))) 61 | (word_same (dim)))) 62 | 63 | (line_new 64 | ((prefix ((text "+|") (style ((bg green)(fg black))))) 65 | (style ((fg green))))) 66 | 67 | (line_unified 68 | ((prefix ((text "!|") (style ((bg yellow)(fg black))))))) 69 | 70 | (header_old 71 | ((prefix ((text "------ ") (style ((fg red))))) 72 | (style (bold)))) 73 | 74 | (header_new 75 | ((prefix ((text "++++++ ") (style ((fg green))))) 76 | (style (bold)))) 77 | 78 | (hunk 79 | ((prefix ((text "@|") (style ((bg bright_black) (fg black))))) 80 | (suffix ((text " =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=") (style ()))) 81 | (style (bold)))) 82 | )|} 83 | 84 | let match_diff () = 85 | Patdiff_lib.Configuration.Config.t_of_sexp (Sexp.of_string diff_configuration) 86 | |> Patdiff_lib.Configuration.parse 87 | 88 | (* Needs (unrefined true), otherwise it just prints without colors. Unrefined true 89 | will diff on a line basis. line_unified is ignored for unrefined, but 90 | will still create a prefix width of 2 in the diff if it is "!|" *) 91 | let plain_configuration = 92 | {|;; -*- scheme -*- 93 | ;; patdiff Configuration file 94 | 95 | ( 96 | (context 3) 97 | ;; unrefined: output every changed line. unrefined false will merge + and - lines if they are similar. 98 | (unrefined true) 99 | 100 | (line_same 101 | ((prefix ((text " ") (style ()))))) 102 | 103 | (line_old 104 | ((prefix ((text "-") (style ()))) 105 | (style ()) 106 | (word_same (dim)))) 107 | 108 | (line_new 109 | ((prefix ((text "+") (style ()))) 110 | (style ()))) 111 | 112 | (header_old 113 | ((prefix ((text "--- ") (style ()))) 114 | (style ()))) 115 | 116 | (header_new 117 | ((prefix ((text "+++ ") (style ()))) 118 | (style ()))) 119 | 120 | (hunk 121 | ((prefix ((text "@@ ") (style ()))) 122 | (suffix ((text " @@") (style ()))) 123 | (style ()))) 124 | ) 125 | |} 126 | 127 | let plain () = 128 | Patdiff_lib.Configuration.Config.t_of_sexp (Sexp.of_string plain_configuration) 129 | |> Patdiff_lib.Configuration.parse 130 | 131 | type kind = 132 | | Plain 133 | | Colored 134 | | Html 135 | | Default 136 | | Match_only 137 | 138 | let get_diff kind source_path source_content result = 139 | let open Patdiff_lib in 140 | let source_path = 141 | match source_path with 142 | | Some path -> path 143 | | None -> "/dev/null" 144 | in 145 | let configuration = 146 | match kind with 147 | | Plain -> plain () 148 | | Colored 149 | | Html 150 | | Default -> terminal ~context:3 () 151 | | Match_only -> match_diff () 152 | in 153 | let prev = Patdiff_core.{ name = source_path; text = source_content } in 154 | let next = Patdiff_core.{ name = source_path; text = result } in 155 | 156 | Compare_core.diff_strings ~print_global_header:true configuration ~prev ~next 157 | |> function 158 | | `Different diff -> Some diff 159 | | `Same -> None 160 | -------------------------------------------------------------------------------- /lib/parsers/comments.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | module Omega = struct 4 | open Angstrom 5 | 6 | let (|>>) p f = 7 | p >>= fun x -> return (f x) 8 | 9 | let between left _right p = 10 | left *> p (*<* right*) 11 | 12 | let to_string from until between : string = 13 | from ^ (String.of_char_list between) ^ until 14 | 15 | let anything_including_newlines ~until = 16 | (* until is not consumed in Alpha. Angstrom consumes it. It needs to not 17 | consume because we're doing that for 'between'; but now between is 18 | changed to not expect it. The point is: it does the right thing but 19 | diverges from Alpha and is a little bit... weird *) 20 | (many_till any_char (string until)) 21 | 22 | let anything_excluding_newlines () = 23 | anything_including_newlines ~until:"\n" 24 | 25 | let non_nested_comment from until = 26 | between 27 | (string from) 28 | (string until) 29 | (anything_including_newlines ~until) 30 | |>> to_string from until 31 | 32 | 33 | module Multiline = struct 34 | module type S = sig 35 | val left : string 36 | val right : string 37 | end 38 | 39 | module Make (M : S) = struct 40 | let comment = non_nested_comment M.left M.right 41 | end 42 | end 43 | 44 | (* XXX consumes the newline *) 45 | let until_newline start = 46 | (string start *> anything_excluding_newlines () 47 | |>> fun l -> start^(String.of_char_list l)) 48 | 49 | module Until_newline = struct 50 | module type S = sig 51 | val start : string 52 | end 53 | 54 | module Make (M : S) = struct 55 | let comment = until_newline M.start 56 | end 57 | end 58 | 59 | end 60 | 61 | module Alpha = struct 62 | open MParser 63 | 64 | let to_string from until between : string = 65 | from ^ (String.of_char_list between) ^ until 66 | 67 | let anything_including_newlines ~until = 68 | (many 69 | (not_followed_by (string until) "" 70 | >>= fun () -> any_char_or_nl)) 71 | 72 | let anything_excluding_newlines ~until = 73 | (many 74 | (not_followed_by (string until) "" 75 | >>= fun () -> any_char)) 76 | 77 | (** a parser for comments with delimiters [from] and [until] that do not nest *) 78 | let non_nested_comment from until s = 79 | (between 80 | (string from) 81 | (string until) 82 | (anything_including_newlines ~until) 83 | |>> to_string from until 84 | ) s 85 | 86 | let until_newline start s = 87 | (string start >> anything_excluding_newlines ~until:"\n" 88 | |>> fun l -> start^(String.of_char_list l)) s 89 | 90 | let any_newline comment_string s = 91 | (string comment_string >> anything_excluding_newlines ~until:"\n" |>> fun l -> (comment_string^String.of_char_list l)) s 92 | 93 | let is_not p s = 94 | if is_ok (p s) then 95 | Empty_failed (unknown_error s) 96 | else 97 | match read_char s with 98 | | Some c -> 99 | Consumed_ok (c, advance_state s 1, No_error) 100 | | None -> 101 | Empty_failed (unknown_error s) 102 | 103 | (** A nested comment parser *) 104 | let nested_comment from until s = 105 | let reserved = skip ((string from) <|> (string until)) in 106 | let rec grammar s = 107 | ((comment_delimiters >>= fun string -> return string) 108 | <|> 109 | (is_not reserved >>= fun c -> return (Char.to_string c))) 110 | s 111 | 112 | and comment_delimiters s = 113 | (between 114 | (string from) 115 | (string until) 116 | ((many grammar) >>= fun result -> 117 | return (String.concat result))) 118 | s 119 | in 120 | (comment_delimiters |>> fun content -> 121 | from ^ content ^ until) s 122 | 123 | (** a parser for, e.g., /* ... */ style block comments. Non-nested. *) 124 | module Multiline = struct 125 | module type S = sig 126 | val left : string 127 | val right : string 128 | end 129 | 130 | module Make (M : S) = struct 131 | let comment s = non_nested_comment M.left M.right s 132 | end 133 | end 134 | 135 | module Until_newline = struct 136 | module type S = sig 137 | val start : string 138 | end 139 | 140 | module Make (M : S) = struct 141 | let comment s = until_newline M.start s 142 | end 143 | end 144 | 145 | module Nested_multiline = struct 146 | module type S = sig 147 | val left : string 148 | val right : string 149 | end 150 | 151 | module Make (M : S) = struct 152 | let comment s = nested_comment M.left M.right s 153 | end 154 | end 155 | end 156 | -------------------------------------------------------------------------------- /scripts/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ue 4 | # set -x 5 | 6 | RELEASE_VERSION="0.x.0" 7 | RELEASE_TAG="0.x.0" 8 | RELEASE_URL="https://github.com/comby-tools/comby/releases" 9 | 10 | INSTALL_DIR=/usr/local/bin 11 | 12 | function ctrl_c() { 13 | rm -f $TMP/$RELEASE_BIN &> /dev/null 14 | printf "${RED}[-]${NORMAL} Installation cancelled. Please see https://github.com/comby-tools/comby/releases if you prefer to install manually.\n" 15 | exit 1 16 | } 17 | 18 | trap ctrl_c INT 19 | 20 | if which tput >/dev/null 2>&1; then 21 | colors=$(tput colors) 22 | fi 23 | if [ -t 1 ] && [ -n "$colors" ] && [ "$colors" -ge 8 ]; then 24 | RED="$(tput setaf 1)" 25 | GREEN="$(tput setaf 2)" 26 | YELLOW="$(tput setaf 3)" 27 | BOLD="$(tput bold)" 28 | NORMAL="$(tput sgr0)" 29 | else 30 | RED="" 31 | GREEN="" 32 | YELLOW="" 33 | BOLD="" 34 | NORMAL="" 35 | fi 36 | 37 | EXISTS=$(command -v comby || echo) 38 | 39 | if [ -n "$EXISTS" ]; then 40 | INSTALL_DIR=$(dirname $EXISTS) 41 | fi 42 | 43 | if [ ! -d "$INSTALL_DIR" ]; then 44 | printf "${YELLOW}[-]${NORMAL} $INSTALL_DIR does not exist. Please download the binary from ${RELEASE_URL} and install it manually.\n" 45 | exit 1 46 | fi 47 | 48 | TMP=${TMPDIR:-/tmp} 49 | 50 | ARCH=$(uname -m || echo dunno) 51 | case "$ARCH" in 52 | x86_64|amd64) ARCH="x86_64";; 53 | # x86|i?86) ARCH="i686";; 54 | *) ARCH="OTHER" 55 | esac 56 | 57 | OS=$(uname -s || echo dunno) 58 | 59 | if [ "$OS" = "Darwin" ]; then 60 | OS=macos 61 | fi 62 | 63 | if [ "$OS" = "Linux" ]; then 64 | OS=linux 65 | fi 66 | 67 | RELEASE_BIN="comby-${RELEASE_TAG}-${ARCH}-${OS}" 68 | RELEASE_TAR="$RELEASE_BIN.tar.gz" 69 | RELEASE_URL="https://github.com/comby-tools/comby/releases/download/${RELEASE_TAG}/${RELEASE_TAR}" 70 | 71 | 72 | if [ ! -e "$TMP/$RELEASE_TAR" ]; then 73 | printf "${GREEN}[+]${NORMAL} Downloading ${YELLOW}comby $RELEASE_VERSION${NORMAL}\n" 74 | 75 | SUCCESS=$(curl -s -L -o "$TMP/$RELEASE_TAR" "$RELEASE_URL" --write-out "%{http_code}") 76 | 77 | if [ "$SUCCESS" == "404" ]; then 78 | printf "${RED}[-]${NORMAL} No binary release available for your system.\n" 79 | rm -f $TMP/$RELEASE_TAR 80 | exit 1 81 | fi 82 | printf "${GREEN}[+]${NORMAL} Download complete.\n" 83 | fi 84 | 85 | cd "$TMP" && tar -xzf "$RELEASE_TAR" 86 | chmod 755 "$TMP/$RELEASE_BIN" 87 | 88 | echo "${GREEN}[+]${NORMAL} Installing comby to $INSTALL_DIR" 89 | if [ ! $OS == "macos" ]; then 90 | sudo cp "$TMP/$RELEASE_BIN" "$INSTALL_DIR/comby" 91 | else 92 | cp "$TMP/$RELEASE_BIN" "$INSTALL_DIR/comby" 93 | fi 94 | 95 | SUCCESS_IN_PATH=$(command -v comby || echo notinpath) 96 | 97 | if [ $SUCCESS_IN_PATH == "notinpath" ]; then 98 | printf "${BOLD}[*]${NORMAL} Comby is not in your PATH. You should add $INSTALL_DIR to your PATH.\n" 99 | rm -f $TMP/$RELEASE_TAR 100 | rm -f $TMP/$RELEASE_BIN 101 | exit 1 102 | fi 103 | 104 | 105 | CHECK=$(printf 'printf("hello world!\\\n");' | $INSTALL_DIR/comby 'printf("hello :[1]!\\n");' 'printf("hello comby!\\n");' .c -stdin || echo broken) 106 | if [ "$CHECK" == "broken" ]; then 107 | printf "${RED}[-]${NORMAL} ${YELLOW}comby${NORMAL} did not install correctly.\n" 108 | printf "${YELLOW}[-]${NORMAL} My guess is that you need to install the pcre library on your system. Try:\n" 109 | if [ $OS == "macos" ]; then 110 | printf "${YELLOW}[*]${NORMAL} ${BOLD}brew install pcre && bash <(curl -sL get.comby.dev)${NORMAL}\n" 111 | else 112 | printf "${YELLOW}[*]${NORMAL} ${BOLD}sudo apt-get install libpcre3-dev && bash <(curl -sL get.comby.dev)${NORMAL}\n" 113 | fi 114 | rm -f $TMP/$RELEASE_BIN 115 | rm -f $TMP/$RELEASE_TAR 116 | exit 1 117 | fi 118 | 119 | rm -f $TMP/$RELEASE_BIN 120 | rm -f $TMP/$RELEASE_TAR 121 | 122 | printf "${GREEN}[+]${NORMAL} ${YELLOW}comby${NORMAL} is installed!\n" 123 | printf "${GREEN}[+]${NORMAL} The licenses for this distribution are included in this script. Licenses are also available at https://github.com/comby-tools/comby/tree/master/docs/third-party-licenses\n" 124 | printf "${GREEN}[+]${NORMAL} Running example command:\n" 125 | echo "${YELLOW}------------------------------------------------------------" 126 | printf "${YELLOW}comby${NORMAL} 'printf(\"${GREEN}:[1] :[2]${NORMAL}!\")' 'printf(\"comby, ${GREEN}:[1]${NORMAL}!\")' .c -stdin << EOF\n" 127 | printf "int main(void) {\n" 128 | printf " printf(\"hello world!\");\n" 129 | printf "}\n" 130 | printf "EOF\n" 131 | echo "${YELLOW}------------------------------------------------------------" 132 | printf "${GREEN}[+]${NORMAL} Output:\n" 133 | echo "${GREEN}------------------------------------------------------------${NORMAL}" 134 | comby 'printf(":[1] :[2]!")' 'printf("comby, :[1]!")' .c -stdin << EOF 135 | int main(void) { 136 | printf("hello world!"); 137 | } 138 | EOF 139 | echo "${GREEN}------------------------------------------------------------${NORMAL}" 140 | -------------------------------------------------------------------------------- /lib/rewriter/rewrite.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | open Match 4 | open Replacement 5 | 6 | let debug = 7 | Sys.getenv "DEBUG_COMBY" 8 | |> Option.is_some 9 | 10 | 11 | let substitute_match_contexts (matches: Match.t list) source replacements = 12 | if debug then 13 | Format.printf "Matches: %d | Replacements: %d@." (List.length matches) (List.length replacements); 14 | let rewrite_template, environment = 15 | List.fold2_exn 16 | matches replacements 17 | ~init:(source, Environment.create ()) 18 | ~f:(fun (rewrite_template, accumulator_environment) 19 | ({ environment = _match_environment; _ } as match_) 20 | { replacement_content; _ } -> 21 | (* create a hole in the rewrite template based on this match context *) 22 | let hole_id, rewrite_template = Rewrite_template.of_match_context match_ ~source:rewrite_template in 23 | if debug then Format.printf "Hole: %s in %s@." hole_id rewrite_template; 24 | (* add this match context replacement to the environment *) 25 | let accumulator_environment = Environment.add accumulator_environment hole_id replacement_content in 26 | (* update match context replacements offset *) 27 | rewrite_template, accumulator_environment) 28 | in 29 | if debug then Format.printf "Env:@.%s" (Environment.to_string environment); 30 | if debug then Format.printf "Rewrite in:@.%s@." rewrite_template; 31 | let rewritten_source = Rewrite_template.substitute rewrite_template environment |> fst in 32 | let offsets = Rewrite_template.get_offsets_for_holes rewrite_template (Environment.vars environment) in 33 | if debug then 34 | Format.printf "Replacements: %d | Offsets 1: %d@." (List.length replacements) (List.length offsets); 35 | let offsets = Rewrite_template.get_offsets_after_substitution offsets environment in 36 | if debug then 37 | Format.printf "Replacements: %d | Offsets 2: %d@." (List.length replacements) (List.length offsets); 38 | let in_place_substitutions = 39 | List.map2_exn replacements offsets ~f:(fun replacement (_uid, offset) -> 40 | let match_start = { Location.default with offset } in 41 | let offset = offset + String.length replacement.replacement_content in 42 | let match_end = { Location.default with offset } in 43 | let range = Range.{ match_start; match_end } in 44 | { replacement with range }) 45 | in 46 | { rewritten_source 47 | ; in_place_substitutions 48 | } 49 | 50 | (* 51 | store range information for this match_context replacement: 52 | (a) its offset in the original source 53 | (b) its replacement context (to calculate the range) 54 | (c) an environment of values that are updated to reflect their relative offset in the rewrite template 55 | *) 56 | let substitute_in_rewrite_template rewrite_template ({ environment; _ } : Match.t) = 57 | let replacement_content, vars_substituted_for = Rewrite_template.substitute rewrite_template environment in 58 | let offsets = Rewrite_template.get_offsets_for_holes rewrite_template (Environment.vars environment) in 59 | let offsets = Rewrite_template.get_offsets_after_substitution offsets environment in 60 | let environment = 61 | List.fold offsets ~init:(Environment.create ()) ~f:(fun acc (var, relative_offset) -> 62 | if List.mem vars_substituted_for var ~equal:String.equal then 63 | let value = Option.value_exn (Environment.lookup environment var) in 64 | (* FIXME(RVT): Location does not update row/column here *) 65 | let start_location = 66 | Location.{ default with offset = relative_offset } 67 | in 68 | let end_location = 69 | let offset = relative_offset + String.length value in 70 | Location.{ default with offset } 71 | in 72 | let range = 73 | Range. 74 | { match_start = start_location 75 | ; match_end = end_location 76 | } 77 | in 78 | Environment.add ~range acc var value 79 | else 80 | acc) 81 | in 82 | { replacement_content 83 | ; environment 84 | ; range = 85 | { match_start = { Location.default with offset = 0 } 86 | ; match_end = Location.default 87 | } 88 | } 89 | 90 | let all ?source ~rewrite_template matches : result option = 91 | if matches = [] then None else 92 | match source with 93 | (* in-place substitution *) 94 | | Some source -> 95 | let matches : Match.t list = List.rev matches in 96 | matches 97 | |> List.map ~f:(substitute_in_rewrite_template rewrite_template) 98 | |> substitute_match_contexts matches source 99 | |> Option.some 100 | (* no in place substitution, emit result separated by newlines *) 101 | | None -> 102 | matches 103 | |> List.map ~f:(substitute_in_rewrite_template rewrite_template) 104 | |> List.map ~f:(fun { replacement_content; _ } -> replacement_content) 105 | |> String.concat ~sep:"\n" 106 | |> (fun rewritten_source -> { rewritten_source; in_place_substitutions = [] }) 107 | |> Option.some 108 | -------------------------------------------------------------------------------- /test/alpha/test_c_style_comments.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | open Matchers 4 | open Rewriter 5 | 6 | let configuration = Configuration.create ~match_kind:Fuzzy () 7 | 8 | let all ?(configuration = configuration) template source = 9 | C.all ~configuration ~template ~source 10 | 11 | let print_matches matches = 12 | List.map matches ~f:Match.to_yojson 13 | |> (fun matches -> `List matches) 14 | |> Yojson.Safe.pretty_to_string 15 | |> print_string 16 | 17 | let%expect_test "rewrite_comments_1" = 18 | let template = "replace this :[1] end" in 19 | let source = "/* don't replace this () end */ do replace this () end" in 20 | let rewrite_template = "X" in 21 | 22 | all template source 23 | |> (fun matches -> 24 | Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) 25 | |> (fun { rewritten_source; _ } -> rewritten_source) 26 | |> print_string; 27 | [%expect_exact "/* don't replace this () end */ do X"] 28 | 29 | let%expect_test "rewrite_comments_2" = 30 | let template = 31 | {| 32 | if (:[1]) { :[2] } 33 | |} 34 | in 35 | 36 | let source = 37 | {| 38 | /* if (fake_condition_body_must_be_non_empty) { fake_body; } */ 39 | // if (fake_condition_body_must_be_non_empty) { fake_body; } 40 | if (real_condition_body_must_be_empty) { 41 | int i; 42 | int j; 43 | } 44 | |} 45 | in 46 | 47 | let rewrite_template = 48 | {| 49 | if (:[1]) {} 50 | |} 51 | in 52 | 53 | all template source 54 | |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) 55 | |> (fun { rewritten_source; _ } -> rewritten_source) 56 | |> print_string; 57 | [%expect_exact 58 | {| 59 | /* if (fake_condition_body_must_be_non_empty) { fake_body; } */ 60 | if (real_condition_body_must_be_empty) {} 61 | |}] 62 | 63 | let%expect_test "capture_comments" = 64 | let template = {|if (:[1]) { :[2] }|} in 65 | let source = {|if (true) { /* some comment */ console.log(z); }|} in 66 | let matches = all template source in 67 | print_matches matches; 68 | [%expect_exact {|[ 69 | { 70 | "range": { 71 | "start": { "offset": 0, "line": 1, "column": 1 }, 72 | "end": { "offset": 48, "line": 1, "column": 49 } 73 | }, 74 | "environment": [ 75 | { 76 | "variable": "1", 77 | "value": "true", 78 | "range": { 79 | "start": { "offset": 4, "line": 1, "column": 5 }, 80 | "end": { "offset": 8, "line": 1, "column": 9 } 81 | } 82 | }, 83 | { 84 | "variable": "2", 85 | "value": "console.log(z);", 86 | "range": { 87 | "start": { "offset": 31, "line": 1, "column": 32 }, 88 | "end": { "offset": 46, "line": 1, "column": 47 } 89 | } 90 | } 91 | ], 92 | "matched": "if (true) { /* some comment */ console.log(z); }" 93 | } 94 | ]|}] 95 | 96 | let%expect_test "single_quote_in_comment" = 97 | let template = 98 | {| {:[1]} |} 99 | in 100 | 101 | let source = 102 | {| 103 | /*'*/ 104 | {test} 105 | |} 106 | in 107 | 108 | let rewrite_template = 109 | {| 110 | {:[1]} 111 | |} 112 | in 113 | 114 | all template source 115 | |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) 116 | |> (fun { rewritten_source; _ } -> rewritten_source) 117 | |> print_string; 118 | [%expect_exact 119 | {| 120 | {test} 121 | |}] 122 | 123 | let%expect_test "single_quote_in_comment" = 124 | let template = 125 | {| {:[1]} |} 126 | in 127 | 128 | let source = 129 | {| 130 | { 131 | a = 1; 132 | /* Events with mask == AE_NONE are not set. So let's initiaize the 133 | * vector with it. */ 134 | for (i = 0; i < setsize; i++) 135 | } 136 | |} 137 | in 138 | 139 | let rewrite_template = 140 | {| 141 | {:[1]} 142 | |} 143 | in 144 | 145 | all template source 146 | |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) 147 | |> (fun { rewritten_source; _ } -> rewritten_source) 148 | |> print_string; 149 | [%expect_exact 150 | {| 151 | { 152 | a = 1; 153 | /* Events with mask == AE_NONE are not set. So let's initiaize the 154 | * vector with it. */ 155 | for (i = 0; i < setsize; i++) 156 | } 157 | |}] 158 | 159 | let%expect_test "single_quote_in_comment" = 160 | let template = 161 | {| {:[1]} |} 162 | in 163 | 164 | let source = 165 | {| 166 | { 167 | a = 1; 168 | /* ' */ 169 | for (i = 0; i < setsize; i++) 170 | } 171 | |} 172 | in 173 | 174 | let rewrite_template = 175 | {| 176 | {:[1]} 177 | |} 178 | in 179 | 180 | all template source 181 | |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) 182 | |> (fun { rewritten_source; _ } -> rewritten_source) 183 | |> print_string; 184 | [%expect_exact 185 | {| 186 | { 187 | a = 1; 188 | /* ' */ 189 | for (i = 0; i < setsize; i++) 190 | } 191 | |}] 192 | 193 | let%expect_test "give_back_the_comment_characters_for_newline_comments_too" = 194 | let template = 195 | {| {:[1]} |} 196 | in 197 | 198 | let source = 199 | {| 200 | { 201 | // a comment 202 | } 203 | |} 204 | in 205 | 206 | let rewrite_template = 207 | {| 208 | {:[1]} 209 | |} 210 | in 211 | 212 | all template source 213 | |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) 214 | |> (fun { rewritten_source; _ } -> rewritten_source) 215 | |> print_string; 216 | [%expect_exact 217 | {| 218 | { 219 | // a comment 220 | } 221 | |}] 222 | -------------------------------------------------------------------------------- /test/omega/test_c_style_comments.ml: -------------------------------------------------------------------------------- 1 | (*open Core 2 | 3 | open Matchers 4 | open Rewriter 5 | 6 | let configuration = Configuration.create ~match_kind:Fuzzy () 7 | 8 | let all ?(configuration = configuration) template source = 9 | C.all ~configuration ~template ~source 10 | 11 | let print_matches matches = 12 | List.map matches ~f:Match.to_yojson 13 | |> (fun matches -> `List matches) 14 | |> Yojson.Safe.pretty_to_string 15 | |> print_string 16 | 17 | let%expect_test "rewrite_comments_1" = 18 | let template = "replace this :[1] end" in 19 | let source = "/* don't replace this () end */ do replace this () end" in 20 | let rewrite_template = "X" in 21 | 22 | all template source 23 | |> (fun matches -> 24 | Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) 25 | |> (fun { rewritten_source; _ } -> rewritten_source) 26 | |> print_string; 27 | [%expect_exact "/* don't replace this () end */ do X"] 28 | 29 | let%expect_test "rewrite_comments_2" = 30 | let template = 31 | {| 32 | if (:[1]) { :[2] } 33 | |} 34 | in 35 | 36 | let source = 37 | {| 38 | /* if (fake_condition_body_must_be_non_empty) { fake_body; } */ 39 | // if (fake_condition_body_must_be_non_empty) { fake_body; } 40 | if (real_condition_body_must_be_empty) { 41 | int i; 42 | int j; 43 | } 44 | |} 45 | in 46 | 47 | let rewrite_template = 48 | {| 49 | if (:[1]) {} 50 | |} 51 | in 52 | 53 | all template source 54 | |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) 55 | |> (fun { rewritten_source; _ } -> rewritten_source) 56 | |> print_string; 57 | [%expect_exact 58 | {| 59 | /* if (fake_condition_body_must_be_non_empty) { fake_body; } */ 60 | if (real_condition_body_must_be_empty) {} 61 | |}] 62 | 63 | let%expect_test "capture_comments" = 64 | let template = {|if (:[1]) { :[2] }|} in 65 | let source = {|if (true) { /* some comment */ console.log(z); }|} in 66 | let matches = all template source in 67 | print_matches matches; 68 | [%expect_exact {|[ 69 | { 70 | "range": { 71 | "start": { "offset": 0, "line": 1, "column": 1 }, 72 | "end": { "offset": 48, "line": 1, "column": 49 } 73 | }, 74 | "environment": [ 75 | { 76 | "variable": "1", 77 | "value": "true", 78 | "range": { 79 | "start": { "offset": 4, "line": 1, "column": 5 }, 80 | "end": { "offset": 8, "line": 1, "column": 9 } 81 | } 82 | }, 83 | { 84 | "variable": "2", 85 | "value": "console.log(z);", 86 | "range": { 87 | "start": { "offset": 31, "line": 1, "column": 32 }, 88 | "end": { "offset": 46, "line": 1, "column": 47 } 89 | } 90 | } 91 | ], 92 | "matched": "if (true) { /* some comment */ console.log(z); }" 93 | } 94 | ]|}] 95 | 96 | let%expect_test "single_quote_in_comment" = 97 | let template = 98 | {| {:[1]} |} 99 | in 100 | 101 | let source = 102 | {| 103 | /*'*/ 104 | {test} 105 | |} 106 | in 107 | 108 | let rewrite_template = 109 | {| 110 | {:[1]} 111 | |} 112 | in 113 | 114 | all template source 115 | |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) 116 | |> (fun { rewritten_source; _ } -> rewritten_source) 117 | |> print_string; 118 | [%expect_exact 119 | {| 120 | {test} 121 | |}] 122 | 123 | let%expect_test "single_quote_in_comment" = 124 | let template = 125 | {| {:[1]} |} 126 | in 127 | 128 | let source = 129 | {| 130 | { 131 | a = 1; 132 | /* Events with mask == AE_NONE are not set. So let's initiaize the 133 | * vector with it. */ 134 | for (i = 0; i < setsize; i++) 135 | } 136 | |} 137 | in 138 | 139 | let rewrite_template = 140 | {| 141 | {:[1]} 142 | |} 143 | in 144 | 145 | all template source 146 | |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) 147 | |> (fun { rewritten_source; _ } -> rewritten_source) 148 | |> print_string; 149 | [%expect_exact 150 | {| 151 | { 152 | a = 1; 153 | /* Events with mask == AE_NONE are not set. So let's initiaize the 154 | * vector with it. */ 155 | for (i = 0; i < setsize; i++) 156 | } 157 | |}] 158 | 159 | let%expect_test "single_quote_in_comment" = 160 | let template = 161 | {| {:[1]} |} 162 | in 163 | 164 | let source = 165 | {| 166 | { 167 | a = 1; 168 | /* ' */ 169 | for (i = 0; i < setsize; i++) 170 | } 171 | |} 172 | in 173 | 174 | let rewrite_template = 175 | {| 176 | {:[1]} 177 | |} 178 | in 179 | 180 | all template source 181 | |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) 182 | |> (fun { rewritten_source; _ } -> rewritten_source) 183 | |> print_string; 184 | [%expect_exact 185 | {| 186 | { 187 | a = 1; 188 | /* ' */ 189 | for (i = 0; i < setsize; i++) 190 | } 191 | |}] 192 | 193 | let%expect_test "give_back_the_comment_characters_for_newline_comments_too" = 194 | let template = 195 | {| {:[1]} |} 196 | in 197 | 198 | let source = 199 | {| 200 | { 201 | // a comment 202 | } 203 | |} 204 | in 205 | 206 | let rewrite_template = 207 | {| 208 | {:[1]} 209 | |} 210 | in 211 | 212 | all template source 213 | |> (fun matches -> Option.value_exn (Rewrite.all ~source ~rewrite_template matches)) 214 | |> (fun { rewritten_source; _ } -> rewritten_source) 215 | |> print_string; 216 | [%expect_exact 217 | {| 218 | { 219 | // a comment 220 | } 221 | |}] 222 | *) 223 | -------------------------------------------------------------------------------- /src/benchmark.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | module Time = Core_kernel.Time_ns.Span 4 | 5 | let debug = false 6 | 7 | let epsilon_same = 0.05 8 | let epsilon_warn = 0.2 9 | 10 | let read_with_timeout read_from_channel = 11 | let read_from_fd = Unix.descr_of_in_channel read_from_channel in 12 | let read_from_channel = 13 | Unix.select 14 | ~read:[read_from_fd] 15 | ~write:[] 16 | ~except:[] 17 | ~timeout:(`After (Time.of_int_sec 2)) 18 | () 19 | |> (fun { Unix.Select_fds.read; _ } -> List.hd_exn read) 20 | |> Unix.in_channel_of_descr 21 | in 22 | let result = In_channel.input_all read_from_channel in 23 | if debug then Format.printf "Read: %s@." result; 24 | result 25 | 26 | let read_stats command = 27 | let open Unix.Process_channels in 28 | if debug then Format.printf "Running: %s@." command; 29 | let { stderr; _ } = Unix.open_process_full ~env:[||] command in 30 | read_with_timeout stderr 31 | |> Yojson.Safe.from_string 32 | |> Statistics.of_yojson 33 | |> function 34 | | Ok statistics -> statistics 35 | | Error _ -> failwith "Error reading stats from stderr" 36 | 37 | let run ~iterations ~command = 38 | List.fold 39 | (List.init iterations ~f:ident) 40 | ~init:Statistics.empty 41 | ~f:(fun statistics _ -> Statistics.merge (read_stats command) statistics) 42 | |> fun { total_time; _ } -> total_time /. (Int.to_float iterations) 43 | 44 | let bench_diff ~iterations baseline_command new_command = 45 | let time_baseline = run ~iterations ~command:baseline_command in 46 | let time_new_command = run ~iterations ~command:new_command in 47 | if debug then Format.printf "time_baseline: %f@." time_baseline; 48 | if debug then Format.printf "time_new: %f@." time_new_command; 49 | let delta_x = 1.0 /. (time_new_command /. time_baseline) in 50 | if delta_x < 1.0 then begin 51 | let percentage_delta = 1.0 -. delta_x in 52 | Format.printf "SLOWER: %.4fx of master@." delta_x; 53 | if percentage_delta > epsilon_warn then begin 54 | Format.printf 55 | "FAIL: benchmark epsilon exceeded (%.4f > %.4f)@." 56 | percentage_delta epsilon_warn; 57 | 1 58 | end 59 | else begin 60 | Format.printf "PASS: difference negligible (%.4f)@." percentage_delta; 61 | 0 62 | end 63 | end 64 | else 65 | let percentage_delta = delta_x -. 1.0 in 66 | if percentage_delta > epsilon_same then begin 67 | Format.printf "PASS: %.4fx faster@." delta_x; 68 | 0 69 | end 70 | else begin 71 | Format.printf "PASS: difference negligible (%.4f)@." percentage_delta; 72 | 0 73 | end 74 | 75 | let with_temp_dir f = 76 | let dir = Filename.temp_dir "bench_" "_comby" in 77 | let result = f dir in 78 | match Unix.system (Format.sprintf "rm -rf %s" dir) with 79 | | Ok () -> 80 | if debug then Format.printf "Successfully removed temp dir@."; 81 | result 82 | | Error _ -> 83 | if debug then Format.printf "Failed to remove temp dir@."; 84 | result 85 | 86 | let with_zip dir f = 87 | let output_zip = dir ^/ "master.zip" in 88 | let repo_uri = "https://github.com/comby-tools/comby" in 89 | let curl_command = 90 | Format.sprintf "curl -L -o %s %s/zipball/master/ &> /dev/null" output_zip repo_uri 91 | in 92 | begin 93 | match Unix.system curl_command with 94 | | Ok () -> 95 | if debug then Format.printf "Download to %s OK@." output_zip; 96 | (* XXX for some reason Unix.system does not give enough time, and 97 | we can run into '"/tmp/bench_.tmp.efb1e5_comby/master.zip: No such file or directory"'. This is a hack. Actually test for the file and wait *) 98 | Unix.sleep 2; 99 | let result = f output_zip in 100 | Unix.remove output_zip; 101 | result 102 | | Error _ -> 103 | Unix.remove output_zip; 104 | Format.eprintf "Failed to execute curl command: %s" curl_command; 105 | 1 106 | end 107 | 108 | let with_master_comby dir f = 109 | let new_comby = dir ^/ "new_comby" in 110 | let baseline_comby = dir ^/ "comby" in 111 | match Unix.system (Format.sprintf "make release && cp $(pwd)/comby %s" new_comby) with 112 | | Ok () -> 113 | if debug then Format.printf "new_comby copy OK@."; 114 | begin 115 | match 116 | Unix.system 117 | (Format.sprintf 118 | "git clone --depth=50 --branch=master https://github.com/comby-tools/comby.git %s/comby-master && \ 119 | make -C %s/comby-master release && \ 120 | cp %s/comby-master/comby %s" dir dir dir baseline_comby) 121 | with 122 | | Ok () -> 123 | if debug then Format.printf "master comby make and copy OK@."; 124 | let result = f baseline_comby new_comby in 125 | Unix.remove baseline_comby; 126 | Unix.remove new_comby; 127 | result 128 | | Error _ -> 129 | Unix.remove baseline_comby; 130 | Unix.remove new_comby; 131 | Format.eprintf "Failed to clone master and build baseline"; 132 | 1 133 | end 134 | | Error _ -> 135 | Format.eprintf "Failed to make this release and copy to temp dir"; 136 | 1 137 | 138 | let zip_bench () = 139 | let match_template = "let" in 140 | let rewrite_template = "derp" in 141 | let go baseline_binary new_binary zip_file = 142 | let command_args = 143 | Format.sprintf "'%s' '%s' .ml,.mli -matcher .ml -sequential -stats -zip %s > /dev/null" 144 | match_template 145 | rewrite_template 146 | zip_file 147 | in 148 | let baseline_command = Format.sprintf "%s %s" baseline_binary command_args in 149 | let new_command = Format.sprintf "%s %s" new_binary command_args in 150 | bench_diff ~iterations:10 baseline_command new_command 151 | in 152 | with_temp_dir (fun dir -> 153 | with_master_comby dir (fun baseline_comby new_comby -> 154 | with_zip dir (go baseline_comby new_comby))) 155 | 156 | 157 | 158 | let fs_bench = 159 | () 160 | 161 | let match_only_bench = 162 | () 163 | 164 | let rewrite_bench = 165 | () 166 | 167 | let () = 168 | exit (zip_bench ()) 169 | -------------------------------------------------------------------------------- /test/alpha/test_server.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | open Lwt.Infix 4 | 5 | open Match 6 | open Server_types 7 | 8 | let binary_path = "../../../../comby-server" 9 | 10 | let port = "9991" 11 | 12 | let pid' = ref None 13 | 14 | let launch port = 15 | Unix.create_process ~prog:binary_path ~args:["-p"; port] 16 | |> fun { pid; _ } -> pid' := Some pid 17 | 18 | let post endpoint json = 19 | let uri = 20 | let uri endpoint = 21 | Uri.of_string ("http://127.0.0.1:" ^ port ^ "/" ^ endpoint) 22 | in 23 | match endpoint with 24 | | `Match -> uri "match" 25 | | `Rewrite -> uri "rewrite" 26 | | `Substitute -> uri "substitute" 27 | in 28 | let thread = 29 | Cohttp_lwt_unix.Client.post ~body:(`String json) uri >>= fun (_, response) -> 30 | match response with 31 | | `Stream response -> Lwt_stream.get response >>= fun result -> Lwt.return result 32 | | _ -> Lwt.return None 33 | in 34 | match Lwt_unix.run thread with 35 | | None -> "FAIL" 36 | | Some result -> result 37 | 38 | (* FIXME(RVT) use wait *) 39 | let launch () = 40 | launch port; 41 | Unix.sleep 2 42 | 43 | let kill () = 44 | match !pid' with 45 | | None -> () 46 | | Some pid -> 47 | match Signal.send Signal.kill (`Pid pid) with 48 | | `Ok -> () 49 | | `No_such_process -> () 50 | 51 | let () = launch () 52 | 53 | let%expect_test "post_request" = 54 | 55 | let source = "hello world" in 56 | let match_template = "hello :[1]" in 57 | let rule = Some {|where :[1] == "world"|} in 58 | let language = "generic" in 59 | 60 | In.{ source; match_template; rule; language; id = 0 } 61 | |> In.match_request_to_yojson 62 | |> Yojson.Safe.to_string 63 | |> post `Match 64 | |> print_string; 65 | 66 | [%expect {| 67 | { 68 | "matches": [ 69 | { 70 | "range": { 71 | "start": { "offset": 0, "line": 1, "column": 1 }, 72 | "end": { "offset": 11, "line": 1, "column": 12 } 73 | }, 74 | "environment": [ 75 | { 76 | "variable": "1", 77 | "value": "world", 78 | "range": { 79 | "start": { "offset": 6, "line": 1, "column": 7 }, 80 | "end": { "offset": 11, "line": 1, "column": 12 } 81 | } 82 | } 83 | ], 84 | "matched": "hello world" 85 | } 86 | ], 87 | "source": "hello world", 88 | "id": 0 89 | } |}]; 90 | 91 | 92 | let source = "hello world" in 93 | let match_template = "hello :[1]" in 94 | let rule = Some {|where :[1] = "world"|} in 95 | let language = "generic" in 96 | 97 | In.{ source; match_template; rule; language; id = 0 } 98 | |> In.match_request_to_yojson 99 | |> Yojson.Safe.to_string 100 | |> post `Match 101 | |> print_string; 102 | 103 | [%expect {| 104 | Error in line 1, column 7: 105 | where :[1] = "world" 106 | ^ 107 | Expecting "false", "match", "rewrite" or "true" 108 | Backtracking occurred after: 109 | Error in line 1, column 12: 110 | where :[1] = "world" 111 | ^ 112 | Expecting "!=" or "==" |}]; 113 | 114 | let substitution_kind = "in_place" in 115 | let source = "hello world" in 116 | let match_template = "hello :[1]" in 117 | let rule = Some {|where :[1] == "world"|} in 118 | let rewrite_template = ":[1], hello" in 119 | let language = "generic" in 120 | 121 | In.{ source; match_template; rewrite_template; rule; language; substitution_kind; id = 0} 122 | |> In.rewrite_request_to_yojson 123 | |> Yojson.Safe.to_string 124 | |> post `Rewrite 125 | |> print_string; 126 | 127 | [%expect {| 128 | { 129 | "rewritten_source": "world, hello", 130 | "in_place_substitutions": [ 131 | { 132 | "range": { 133 | "start": { "offset": 0, "line": -1, "column": -1 }, 134 | "end": { "offset": 12, "line": -1, "column": -1 } 135 | }, 136 | "replacement_content": "world, hello", 137 | "environment": [ 138 | { 139 | "variable": "1", 140 | "value": "world", 141 | "range": { 142 | "start": { "offset": 0, "line": -1, "column": -1 }, 143 | "end": { "offset": 5, "line": -1, "column": -1 } 144 | } 145 | } 146 | ] 147 | } 148 | ], 149 | "id": 0 150 | } |}]; 151 | 152 | let substitution_kind = "newline_separated" in 153 | let source = "hello world {} hello world" in 154 | let match_template = "hello :[[1]]" in 155 | let rule = Some {|where :[1] == "world"|} in 156 | let rewrite_template = ":[1], hello" in 157 | let language = "generic" in 158 | 159 | In.{ source; match_template; rewrite_template; rule; language; substitution_kind; id = 0} 160 | |> In.rewrite_request_to_yojson 161 | |> Yojson.Safe.to_string 162 | |> post `Rewrite 163 | |> print_string; 164 | 165 | [%expect {| 166 | { 167 | "rewritten_source": "world, hello\nworld, hello", 168 | "in_place_substitutions": [], 169 | "id": 0 170 | } |}]; 171 | 172 | (* test there must be at least one predicate in a rule *) 173 | let source = "hello world" in 174 | let match_template = "hello :[1]" in 175 | let rule = Some {|where |} in 176 | let language = "generic" in 177 | 178 | let request = In.{ source; match_template; rule; language; id = 0 } in 179 | let json = In.match_request_to_yojson request |> Yojson.Safe.to_string in 180 | let result = post `Match json in 181 | 182 | print_string result; 183 | [%expect {| 184 | Error in line 1, column 7: 185 | where 186 | ^ 187 | Expecting ":[", "false", "match", "rewrite", "true" or string literal |}] 188 | 189 | let%expect_test "post_substitute" = 190 | 191 | let rewrite_template = ":[1] hi :[2]" in 192 | let environment = Environment.create () in 193 | let environment = Environment.add environment "1" "oh" in 194 | let environment = Environment.add environment "2" "there" in 195 | 196 | In.{ rewrite_template; environment; id = 0 } 197 | |> In.substitution_request_to_yojson 198 | |> Yojson.Safe.to_string 199 | |> post `Substitute 200 | |> print_string; 201 | 202 | [%expect {| { "result": "oh hi there", "id": 0 } |}] 203 | 204 | let () = kill () 205 | -------------------------------------------------------------------------------- /test/omega/test_server.ml: -------------------------------------------------------------------------------- 1 | (*open Core 2 | 3 | open Lwt.Infix 4 | 5 | open Match 6 | open Server_types 7 | 8 | let binary_path = "../../../comby-server" 9 | 10 | let port = "9991" 11 | 12 | let pid' = ref None 13 | 14 | let launch port = 15 | Unix.create_process ~prog:binary_path ~args:["-p"; port] 16 | |> fun { pid; _ } -> pid' := Some pid 17 | 18 | let post endpoint json = 19 | let uri = 20 | let uri endpoint = 21 | Uri.of_string ("http://127.0.0.1:" ^ port ^ "/" ^ endpoint) 22 | in 23 | match endpoint with 24 | | `Match -> uri "match" 25 | | `Rewrite -> uri "rewrite" 26 | | `Substitute -> uri "substitute" 27 | in 28 | let thread = 29 | Cohttp_lwt_unix.Client.post ~body:(`String json) uri >>= fun (_, response) -> 30 | match response with 31 | | `Stream response -> Lwt_stream.get response >>= fun result -> Lwt.return result 32 | | _ -> Lwt.return None 33 | in 34 | match Lwt_unix.run thread with 35 | | None -> "FAIL" 36 | | Some result -> result 37 | 38 | (* FIXME(RVT) use wait *) 39 | let launch () = 40 | launch port; 41 | Unix.sleep 2 42 | 43 | let kill () = 44 | match !pid' with 45 | | None -> () 46 | | Some pid -> 47 | match Signal.send Signal.kill (`Pid pid) with 48 | | `Ok -> () 49 | | `No_such_process -> () 50 | 51 | let () = launch () 52 | 53 | let%expect_test "post_request" = 54 | 55 | let source = "hello world" in 56 | let match_template = "hello :[1]" in 57 | let rule = Some {|where :[1] == "world"|} in 58 | let language = "generic" in 59 | 60 | In.{ source; match_template; rule; language; id = 0 } 61 | |> In.match_request_to_yojson 62 | |> Yojson.Safe.to_string 63 | |> post `Match 64 | |> print_string; 65 | 66 | [%expect {| 67 | { 68 | "matches": [ 69 | { 70 | "range": { 71 | "start": { "offset": 0, "line": 1, "column": 1 }, 72 | "end": { "offset": 11, "line": 1, "column": 12 } 73 | }, 74 | "environment": [ 75 | { 76 | "variable": "1", 77 | "value": "world", 78 | "range": { 79 | "start": { "offset": 6, "line": 1, "column": 7 }, 80 | "end": { "offset": 11, "line": 1, "column": 12 } 81 | } 82 | } 83 | ], 84 | "matched": "hello world" 85 | } 86 | ], 87 | "source": "hello world", 88 | "id": 0 89 | } |}]; 90 | 91 | 92 | let source = "hello world" in 93 | let match_template = "hello :[1]" in 94 | let rule = Some {|where :[1] = "world"|} in 95 | let language = "generic" in 96 | 97 | In.{ source; match_template; rule; language; id = 0 } 98 | |> In.match_request_to_yojson 99 | |> Yojson.Safe.to_string 100 | |> post `Match 101 | |> print_string; 102 | 103 | [%expect {| 104 | Error in line 1, column 7: 105 | where :[1] = "world" 106 | ^ 107 | Expecting "false", "match", "rewrite" or "true" 108 | Backtracking occurred after: 109 | Error in line 1, column 12: 110 | where :[1] = "world" 111 | ^ 112 | Expecting "!=" or "==" |}]; 113 | 114 | let substitution_kind = "in_place" in 115 | let source = "hello world" in 116 | let match_template = "hello :[1]" in 117 | let rule = Some {|where :[1] == "world"|} in 118 | let rewrite_template = ":[1], hello" in 119 | let language = "generic" in 120 | 121 | In.{ source; match_template; rewrite_template; rule; language; substitution_kind; id = 0} 122 | |> In.rewrite_request_to_yojson 123 | |> Yojson.Safe.to_string 124 | |> post `Rewrite 125 | |> print_string; 126 | 127 | [%expect {| 128 | { 129 | "rewritten_source": "world, hello", 130 | "in_place_substitutions": [ 131 | { 132 | "range": { 133 | "start": { "offset": 0, "line": -1, "column": -1 }, 134 | "end": { "offset": 12, "line": -1, "column": -1 } 135 | }, 136 | "replacement_content": "world, hello", 137 | "environment": [ 138 | { 139 | "variable": "1", 140 | "value": "world", 141 | "range": { 142 | "start": { "offset": 0, "line": -1, "column": -1 }, 143 | "end": { "offset": 5, "line": -1, "column": -1 } 144 | } 145 | } 146 | ] 147 | } 148 | ], 149 | "id": 0 150 | } |}]; 151 | 152 | let substitution_kind = "newline_separated" in 153 | let source = "hello world {} hello world" in 154 | let match_template = "hello :[[1]]" in 155 | let rule = Some {|where :[1] == "world"|} in 156 | let rewrite_template = ":[1], hello" in 157 | let language = "generic" in 158 | 159 | In.{ source; match_template; rewrite_template; rule; language; substitution_kind; id = 0} 160 | |> In.rewrite_request_to_yojson 161 | |> Yojson.Safe.to_string 162 | |> post `Rewrite 163 | |> print_string; 164 | 165 | [%expect {| 166 | { 167 | "rewritten_source": "world, hello\nworld, hello", 168 | "in_place_substitutions": [], 169 | "id": 0 170 | } |}]; 171 | 172 | (* test there must be at least one predicate in a rule *) 173 | let source = "hello world" in 174 | let match_template = "hello :[1]" in 175 | let rule = Some {|where |} in 176 | let language = "generic" in 177 | 178 | let request = In.{ source; match_template; rule; language; id = 0 } in 179 | let json = In.match_request_to_yojson request |> Yojson.Safe.to_string in 180 | let result = post `Match json in 181 | 182 | print_string result; 183 | [%expect {| 184 | Error in line 1, column 7: 185 | where 186 | ^ 187 | Expecting ":[", "false", "match", "rewrite", "true" or string literal |}] 188 | 189 | let%expect_test "post_substitute" = 190 | 191 | let rewrite_template = ":[1] hi :[2]" in 192 | let environment = Environment.create () in 193 | let environment = Environment.add environment "1" "oh" in 194 | let environment = Environment.add environment "2" "there" in 195 | 196 | In.{ rewrite_template; environment; id = 0 } 197 | |> In.substitution_request_to_yojson 198 | |> Yojson.Safe.to_string 199 | |> post `Substitute 200 | |> print_string; 201 | 202 | [%expect {| { "result": "oh hi there", "id": 0 } |}] 203 | 204 | let () = kill () 205 | *) 206 | -------------------------------------------------------------------------------- /src/server.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | open Opium.Std 3 | 4 | open Comby 5 | open Language 6 | open Matchers 7 | open Rewriter 8 | open Server_types 9 | 10 | let (>>|) = Lwt.Infix.(>|=) 11 | 12 | let debug = 13 | match Sys.getenv "DEBUG" with 14 | | None -> false 15 | | Some _ -> true 16 | 17 | let timeout = 18 | match Sys.getenv "TIMEOUT" with 19 | | None -> 30 (* seconds *) 20 | | Some t -> Int.of_string t 21 | 22 | 23 | let max_request_length = 24 | match Sys.getenv "MAX_REQUEST_LENGTH" with 25 | | None -> Int.max_value 26 | | Some max -> Int.of_string max 27 | 28 | let check_too_long s = 29 | let n = String.length s in 30 | if n > max_request_length then 31 | Error 32 | (Format.sprintf 33 | "The source input is a bit big! Make it %d characters shorter, \ 34 | or click 'Run in Terminal' below to install and run comby locally :)" 35 | (n - max_request_length)) 36 | else 37 | Ok s 38 | 39 | let perform_match request = 40 | App.string_of_body_exn request 41 | >>| check_too_long 42 | >>| Result.map ~f:(Fn.compose In.match_request_of_yojson Yojson.Safe.from_string) 43 | >>| Result.join 44 | >>| function 45 | | Ok ({ source; match_template; rule; language; id } as request) -> 46 | if debug then Format.printf "Received %s@." (Yojson.Safe.pretty_to_string (In.match_request_to_yojson request)); 47 | let matcher = 48 | match Matchers.select_with_extension language with 49 | | Some matcher -> matcher 50 | | None -> (module Matchers.Generic) 51 | in 52 | let run ?rule () = 53 | let configuration = Configuration.create ~match_kind:Fuzzy () in 54 | let matches = 55 | Pipeline.with_timeout timeout (`String "") ~f:(fun () -> 56 | Pipeline.timed_run matcher ?rule ~configuration ~template:match_template ~source ()) 57 | in 58 | Out.Matches.to_string { matches; source; id } 59 | in 60 | let code, result = 61 | match Option.map rule ~f:Rule.create with 62 | | None -> 200, run () 63 | | Some Ok rule -> 200, run ~rule () 64 | | Some Error error -> 400, Error.to_string_hum error 65 | in 66 | if debug then Format.printf "Result (%d) %s@." code result; 67 | respond ~code:(`Code code) (`String result) 68 | | Error error -> 69 | if debug then Format.printf "Result (400) %s@." error; 70 | respond ~code:(`Code 400) (`String error) 71 | 72 | let perform_rewrite request = 73 | App.string_of_body_exn request 74 | >>| check_too_long 75 | >>| Result.map ~f:(Fn.compose In.rewrite_request_of_yojson Yojson.Safe.from_string) 76 | >>| Result.join 77 | >>| function 78 | | Ok ({ source; match_template; rewrite_template; rule; language; substitution_kind; id } as request) -> 79 | if debug then Format.printf "Received %s@." (Yojson.Safe.pretty_to_string (In.rewrite_request_to_yojson request)); 80 | let matcher = 81 | match Matchers.select_with_extension language with 82 | | Some matcher -> matcher 83 | | None -> (module Matchers.Generic) 84 | in 85 | let source_substitution, substitute_in_place = 86 | match substitution_kind with 87 | | "newline_separated" -> None, false 88 | | "in_place" | _ -> Some source, true 89 | in 90 | let default = 91 | Out.Rewrite.to_string 92 | { rewritten_source = "" 93 | ; in_place_substitutions = [] 94 | ; id 95 | } 96 | in 97 | let run ?rule () = 98 | let configuration = Configuration.create ~match_kind:Fuzzy () in 99 | let matches = 100 | Pipeline.with_timeout timeout (`String "") ~f:(fun () -> 101 | Pipeline.timed_run 102 | matcher 103 | ?rule 104 | ~substitute_in_place 105 | ~configuration 106 | ~template:match_template 107 | ~source 108 | ()) 109 | in 110 | Rewrite.all matches ?source:source_substitution ~rewrite_template 111 | |> Option.value_map ~default ~f:(fun Replacement.{ rewritten_source; in_place_substitutions } -> 112 | Out.Rewrite.to_string 113 | { rewritten_source 114 | ; in_place_substitutions 115 | ; id 116 | }) 117 | in 118 | let code, result = 119 | match Option.map rule ~f:Rule.create with 120 | | None -> 200, run () 121 | | Some Ok rule -> 200, run ~rule () 122 | | Some Error error -> 400, Error.to_string_hum error 123 | in 124 | if debug then Format.printf "Result (%d): %s@." code result; 125 | respond ~code:(`Code code) (`String result) 126 | | Error error -> 127 | if debug then Format.printf "Result (400): %s@." error; 128 | respond ~code:(`Code 400) (`String error) 129 | 130 | let perform_environment_substitution request = 131 | App.string_of_body_exn request 132 | >>| Yojson.Safe.from_string 133 | >>| In.substitution_request_of_yojson 134 | >>| function 135 | | Ok ({ rewrite_template; environment; id } as request) -> 136 | if debug then Format.printf "Received %s@." (Yojson.Safe.pretty_to_string (In.substitution_request_to_yojson request)); 137 | let code, result = 138 | 200, 139 | Out.Substitution.to_string 140 | { result = fst @@ Rewrite_template.substitute rewrite_template environment 141 | ; id 142 | } 143 | in 144 | if debug then Format.printf "Result (%d) %s@." code result; 145 | respond ~code:(`Code code) (`String result) 146 | | Error error -> 147 | if debug then Format.printf "Result (400) %s@." error; 148 | respond ~code:(`Code 400) (`String error) 149 | 150 | let add_cors_headers (headers: Cohttp.Header.t): Cohttp.Header.t = 151 | Cohttp.Header.add_list headers [ 152 | ("Access-Control-Allow-Origin", "*"); 153 | ("Access-Control-Allow-Methods", "GET, POST, PATCH, PUT, DELETE, OPTIONS"); 154 | ("Access-Control-Allow-Headers", "Origin, Content-Type, X-Auth-Token"); 155 | ] 156 | 157 | let allow_cors = 158 | let filter handler req = 159 | handler req 160 | >>| fun response -> 161 | response 162 | |> Response.headers 163 | |> add_cors_headers 164 | |> Field.fset Response.Fields.headers response 165 | in 166 | Rock.Middleware.create ~name:"allow cors" ~filter 167 | 168 | let () = 169 | Lwt.async_exception_hook := (function 170 | | Unix.Unix_error (error, func, arg) -> 171 | Logs.warn (fun m -> 172 | m "Client connection error %s: %s(%S)" 173 | (Unix.error_message error) func arg 174 | ) 175 | | exn -> Logs.err (fun m -> m "Unhandled exception: %a" Fmt.exn exn) 176 | ); 177 | App.empty 178 | |> App.post "/match" perform_match 179 | |> App.post "/rewrite" perform_rewrite 180 | |> App.post "/substitute" perform_environment_substitution 181 | |> App.middleware allow_cors 182 | |> App.run_command 183 | -------------------------------------------------------------------------------- /test/alpha/test_optional_holes.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | open Matchers 4 | open Rewriter 5 | 6 | let configuration = Configuration.create ~match_kind:Fuzzy () 7 | 8 | let run ?(configuration = configuration) source match_template rewrite_template = 9 | Generic.all ~configuration ~template:match_template ~source 10 | |> function 11 | | [] -> print_string "No matches." 12 | | results -> 13 | Option.value_exn (Rewrite.all ~source ~rewrite_template results) 14 | |> (fun { rewritten_source; _ } -> rewritten_source) 15 | |> print_string 16 | 17 | let%expect_test "optional_holes_basic_match" = 18 | let source = {||} in 19 | let match_template = {|:[[?x]]|} in 20 | let rewrite_template = {|/:[?x]/|} in 21 | run source match_template rewrite_template; 22 | [%expect_exact {|//|}]; 23 | 24 | let source = {||} in 25 | let match_template = {|:[[?x]]:[[?y]]|} in 26 | let rewrite_template = {|/:[?x]/:[?y]/|} in 27 | run source match_template rewrite_template; 28 | [%expect_exact {|///|}]; 29 | 30 | let source = {|a |} in 31 | let match_template = {|:[[x]] :[[?y]]|} in 32 | let rewrite_template = {|/:[x]/:[?y]/|} in 33 | run source match_template rewrite_template; 34 | [%expect_exact {|/a//|}]; 35 | 36 | let source = {|a |} in 37 | let match_template = {|:[[x]] :[[?y]]|} in 38 | let rewrite_template = {|/:[x]/:[?y]/|} in 39 | run source match_template rewrite_template; 40 | [%expect_exact {|/a//|}]; 41 | 42 | let source = {|(foo )|} in 43 | let match_template = {|(:[[?x]] :[[?y]])|} in 44 | let rewrite_template = {|/:[?x]/:[?y]/|} in 45 | run source match_template rewrite_template; 46 | [%expect_exact {|/foo//|}]; 47 | 48 | let source = {|(foo)|} in 49 | let match_template = {|(:[[?x]]:[? w])|} in 50 | let rewrite_template = {|/:[?x]/:[?w]/|} in 51 | run source match_template rewrite_template; 52 | [%expect_exact {|/foo//|}]; 53 | 54 | let source = {|()|} in 55 | let match_template = {|(:[[?x]]:[? w]:[?y]:[?z.])|} in 56 | let rewrite_template = {|/:[?x]/:[?w]/:[?y]|} in 57 | run source match_template rewrite_template; 58 | [%expect_exact {|///|}]; 59 | 60 | let source = {|()|} in 61 | let match_template = {|(:[?s\n])|} in 62 | let rewrite_template = {|/:[?s]/|} in 63 | run source match_template rewrite_template; 64 | [%expect_exact {|//|}] 65 | 66 | let%expect_test "optional_holes_match_over_coalesced_whitespace" = 67 | let source = {|a c|} in 68 | let match_template = {|:[[a]] :[[?b]] :[[c]]|} in 69 | let rewrite_template = {|/:[?a]/:[?b]/:[?c]|} in 70 | run source match_template rewrite_template; 71 | [%expect_exact {|/a//c|}]; 72 | 73 | let source = {|a c|} in 74 | let match_template = {|:[[a]] :[[?b]]:[[c]]|} in 75 | let rewrite_template = {|/:[?a]/:[?b]/:[?c]|} in 76 | run source match_template rewrite_template; 77 | [%expect_exact {|/a//c|}]; 78 | 79 | let source = {|a c|} in 80 | let match_template = {|:[[a]]:[[?b]]:[[c]]|} in 81 | let rewrite_template = {|/:[?a]/:[?b]/:[?c]|} in 82 | run source match_template rewrite_template; 83 | [%expect_exact {|No matches.|}]; 84 | 85 | let source = {|a c|} in 86 | let match_template = {|:[[a]]:[[?b]] :[[?c]]|} in 87 | let rewrite_template = {|/:[?a]/:[?b]/:[?c]|} in 88 | run source match_template rewrite_template; 89 | [%expect_exact {|/a//c|}]; 90 | 91 | let source = {|a c|} in 92 | let match_template = {|a :[?b] c|} in 93 | let rewrite_template = {|/:[?b]/|} in 94 | run source match_template rewrite_template; 95 | [%expect_exact {|//|}]; 96 | 97 | let source = {|a c|} in 98 | let match_template = {|a :[?b] c|} in 99 | let rewrite_template = {|/:[?b]/|} in 100 | run source match_template rewrite_template; 101 | [%expect_exact {|//|}]; 102 | 103 | let source = {| 104 | 105 | a 106 | 107 | c 108 | 109 | |} in 110 | let match_template = {| a :[?b] c |} in 111 | let rewrite_template = {|/:[?b]/|} in 112 | run source match_template rewrite_template; 113 | [%expect_exact {|//|}]; 114 | 115 | let source = {|func foo(bar) {}|} in 116 | let match_template = {|func :[?receiver] foo(:[args])|} in 117 | let rewrite_template = {|/:[receiver]/:[args]/|} in 118 | run source match_template rewrite_template; 119 | [%expect_exact {|//bar/ {}|}]; 120 | 121 | let source = {|func foo(bar) {}|} in 122 | let match_template = {|func :[?receiver] foo(:[args])|} in 123 | let rewrite_template = {|/:[receiver]/:[args]/|} in 124 | run source match_template rewrite_template; 125 | [%expect_exact {|//bar/ {}|}]; 126 | 127 | let source = {|func (r *receiver) foo(bar) {}|} in 128 | let match_template = {|func :[?receiver] foo(:[args])|} in 129 | let rewrite_template = {|/:[receiver]/:[args]/|} in 130 | run source match_template rewrite_template; 131 | [%expect_exact {|/(r *receiver)/bar/ {}|}]; 132 | 133 | let source = {|func foo()|} in 134 | let match_template = {|func :[?receiver] foo()|} in 135 | let rewrite_template = {|/:[receiver]/|} in 136 | run source match_template rewrite_template; 137 | [%expect_exact {|//|}]; 138 | 139 | let source = {|a l|} in 140 | let match_template = {|a :[?b]asdfasdfsadf|} in 141 | let rewrite_template = {|/:[?b]/|} in 142 | run source match_template rewrite_template; 143 | [%expect_exact {|No matches.|}]; 144 | 145 | let source = {|func foo (1, 3)|} in 146 | let match_template = {|func :[?receiver] foo (1, :[?args] 3)|} in 147 | let rewrite_template = {|/:[receiver]/:[args]/|} in 148 | run source match_template rewrite_template; 149 | [%expect_exact {|///|}]; 150 | 151 | let source = {| 152 | try { 153 | foo() 154 | } catch (Exception e) { 155 | logger.error(e) 156 | hey 157 | } 158 | |} in 159 | let match_template = {| 160 | catch (:[type] :[var]) { 161 | :[?anything] 162 | logger.:[logMethod](:[var]) 163 | :[?something] 164 | } 165 | |} in 166 | let rewrite_template = {| 167 | catch (:[type] :[var]) { 168 | :[anything] 169 | logger.:[logMethod]("", :[var]) 170 | :[something] 171 | } 172 | |} in 173 | run source match_template rewrite_template; 174 | [%expect_exact {| 175 | try { 176 | foo() 177 | } 178 | catch (Exception e) { 179 | 180 | logger.error("", e) 181 | hey 182 | } 183 | |}]; 184 | 185 | let source = {|

content

more content

|} in 186 | let match_template = {||} in 187 | let rewrite_template = {||} in 188 | run source match_template rewrite_template; 189 | [%expect_exact {|content

more content

|}] 190 | 191 | let%expect_test "optional_holes_match_over_coalesced_whitespace_in_strings" = 192 | let source = {|"a c"|} in 193 | let match_template = {|"a :[?b] c"|} in 194 | let rewrite_template = {|/:[?b]/|} in 195 | run source match_template rewrite_template; 196 | [%expect_exact {|No matches.|}]; 197 | 198 | let source = {|"a c"|} in 199 | let match_template = {|"a :[?b] c"|} in 200 | let rewrite_template = {|/:[?b]/|} in 201 | run source match_template rewrite_template; 202 | [%expect_exact {|//|}]; 203 | 204 | (* Uh, turns out whitespace is significant inside strings, so this is correct 205 | until it is decided otherwise *) 206 | let source = {|"a c"|} in 207 | let match_template = {|"a :[?b] c"|} in 208 | let rewrite_template = {|/:[?b]/|} in 209 | run source match_template rewrite_template; 210 | [%expect_exact {|/ /|}] 211 | 212 | let%expect_test "optional_holes_substitute" = 213 | let source = {|()|} in 214 | let match_template = {|(:[[?x]]:[? w]:[?y]:[?z.])|} in 215 | let rewrite_template = {|/:[x]/:[w]/:[y]/:[z]|} in 216 | run source match_template rewrite_template; 217 | [%expect_exact {|////|}] 218 | -------------------------------------------------------------------------------- /test/common/test_rewrite_parts.ml: -------------------------------------------------------------------------------- 1 | open Core 2 | 3 | open Match 4 | open Matchers 5 | open Rewriter 6 | 7 | let%expect_test "get_offsets_for_holes" = 8 | let rewrite_template = {|1234:[1]1234:[2]|} in 9 | let result = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "2"] in 10 | print_s [%message (result : (string * int) list)]; 11 | [%expect_exact {|(result ((2 8) (1 4))) 12 | |}] 13 | 14 | let%expect_test "get_offsets_for_holes_after_substitution_1" = 15 | let rewrite_template = {|1234:[1]1234:[2]|} in 16 | let offsets = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "2"] in 17 | let environment = 18 | Environment.create () 19 | |> (fun environment -> Environment.add environment "1" "333") 20 | |> (fun environment -> Environment.add environment "2" "22") 21 | in 22 | let result = Rewrite_template.get_offsets_after_substitution offsets environment in 23 | print_s [%message (result : (string * int) list)]; 24 | [%expect_exact {|(result ((2 11) (1 4))) 25 | |}] 26 | 27 | let%expect_test "get_offsets_for_holes_after_substitution_1" = 28 | let rewrite_template = {|1234:[1]1234:[3]11:[2]|} in 29 | let offsets = Rewrite_template.get_offsets_for_holes rewrite_template ["1"; "3"; "2"] in 30 | let environment = 31 | Environment.create () 32 | |> (fun environment -> Environment.add environment "1" "333") 33 | |> (fun environment -> Environment.add environment "3" "333") 34 | |> (fun environment -> Environment.add environment "2" "22") 35 | in 36 | let result = Rewrite_template.get_offsets_after_substitution offsets environment in 37 | print_s [%message (result : (string * int) list)]; 38 | [%expect_exact {|(result ((2 16) (3 11) (1 4))) 39 | |}] 40 | 41 | 42 | let configuration = Configuration.create ~match_kind:Fuzzy () 43 | 44 | let all ?(configuration = configuration) template source = 45 | C.all ~configuration ~template ~source 46 | 47 | let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy" = 48 | let source = {|123433312343331122|} in 49 | let match_template = {|1234:[1]1234:[3]11:[2]|} in 50 | let rewrite_template = {|1234:[1]1234:[3]11:[2]|} in 51 | all match_template source 52 | |> Rewrite.all ~source ~rewrite_template 53 | |> (function 54 | | Some rewrite_result -> print_string (Yojson.Safe.pretty_to_string (Replacement.result_to_yojson rewrite_result)) 55 | | None -> print_string "BROKEN EXPECT"); 56 | [%expect_exact {|{ 57 | "rewritten_source": "123433312343331122", 58 | "in_place_substitutions": [ 59 | { 60 | "range": { 61 | "start": { "offset": 0, "line": -1, "column": -1 }, 62 | "end": { "offset": 18, "line": -1, "column": -1 } 63 | }, 64 | "replacement_content": "123433312343331122", 65 | "environment": [ 66 | { 67 | "variable": "1", 68 | "value": "333", 69 | "range": { 70 | "start": { "offset": 4, "line": -1, "column": -1 }, 71 | "end": { "offset": 7, "line": -1, "column": -1 } 72 | } 73 | }, 74 | { 75 | "variable": "2", 76 | "value": "22", 77 | "range": { 78 | "start": { "offset": 16, "line": -1, "column": -1 }, 79 | "end": { "offset": 18, "line": -1, "column": -1 } 80 | } 81 | }, 82 | { 83 | "variable": "3", 84 | "value": "333", 85 | "range": { 86 | "start": { "offset": 11, "line": -1, "column": -1 }, 87 | "end": { "offset": 14, "line": -1, "column": -1 } 88 | } 89 | } 90 | ] 91 | } 92 | ] 93 | }|}] 94 | 95 | let%expect_test "comments_in_string_literals_should_not_be_treated_as_comments_by_fuzzy" = 96 | let source = {|123433312343331122;123433312343331122;|} in 97 | let match_template = {|1234:[1]1234:[3]11:[2];|} in 98 | let rewrite_template = {|1234:[1]1234:[3]11:[2];|} in 99 | all match_template source 100 | |> Rewrite.all ~source ~rewrite_template 101 | |> (function 102 | | Some rewrite_result -> print_string (Yojson.Safe.pretty_to_string (Replacement.result_to_yojson rewrite_result)) 103 | | None -> print_string "BROKEN EXPECT"); 104 | [%expect_exact {|{ 105 | "rewritten_source": "123433312343331122;123433312343331122;", 106 | "in_place_substitutions": [ 107 | { 108 | "range": { 109 | "start": { "offset": 19, "line": -1, "column": -1 }, 110 | "end": { "offset": 38, "line": -1, "column": -1 } 111 | }, 112 | "replacement_content": "123433312343331122;", 113 | "environment": [ 114 | { 115 | "variable": "1", 116 | "value": "333", 117 | "range": { 118 | "start": { "offset": 4, "line": -1, "column": -1 }, 119 | "end": { "offset": 7, "line": -1, "column": -1 } 120 | } 121 | }, 122 | { 123 | "variable": "2", 124 | "value": "22", 125 | "range": { 126 | "start": { "offset": 16, "line": -1, "column": -1 }, 127 | "end": { "offset": 18, "line": -1, "column": -1 } 128 | } 129 | }, 130 | { 131 | "variable": "3", 132 | "value": "333", 133 | "range": { 134 | "start": { "offset": 11, "line": -1, "column": -1 }, 135 | "end": { "offset": 14, "line": -1, "column": -1 } 136 | } 137 | } 138 | ] 139 | }, 140 | { 141 | "range": { 142 | "start": { "offset": 0, "line": -1, "column": -1 }, 143 | "end": { "offset": 19, "line": -1, "column": -1 } 144 | }, 145 | "replacement_content": "123433312343331122;", 146 | "environment": [ 147 | { 148 | "variable": "1", 149 | "value": "333", 150 | "range": { 151 | "start": { "offset": 4, "line": -1, "column": -1 }, 152 | "end": { "offset": 7, "line": -1, "column": -1 } 153 | } 154 | }, 155 | { 156 | "variable": "2", 157 | "value": "22", 158 | "range": { 159 | "start": { "offset": 16, "line": -1, "column": -1 }, 160 | "end": { "offset": 18, "line": -1, "column": -1 } 161 | } 162 | }, 163 | { 164 | "variable": "3", 165 | "value": "333", 166 | "range": { 167 | "start": { "offset": 11, "line": -1, "column": -1 }, 168 | "end": { "offset": 14, "line": -1, "column": -1 } 169 | } 170 | } 171 | ] 172 | } 173 | ] 174 | }|}] 175 | 176 | let%expect_test "multiple_contextual_substitutions" = 177 | let source = {|foo bar foo|} in 178 | let match_template = {|foo|} in 179 | let rewrite_template = {|xxxx|} in 180 | all match_template source 181 | |> Rewrite.all ~source ~rewrite_template 182 | |> (function 183 | | Some rewrite_result -> print_string (Yojson.Safe.pretty_to_string (Replacement.result_to_yojson rewrite_result)) 184 | | None -> print_string "BROKEN EXPECT"); 185 | [%expect_exact {|{ 186 | "rewritten_source": "xxxx bar xxxx", 187 | "in_place_substitutions": [ 188 | { 189 | "range": { 190 | "start": { "offset": 9, "line": -1, "column": -1 }, 191 | "end": { "offset": 13, "line": -1, "column": -1 } 192 | }, 193 | "replacement_content": "xxxx", 194 | "environment": [] 195 | }, 196 | { 197 | "range": { 198 | "start": { "offset": 0, "line": -1, "column": -1 }, 199 | "end": { "offset": 4, "line": -1, "column": -1 } 200 | }, 201 | "replacement_content": "xxxx", 202 | "environment": [] 203 | } 204 | ] 205 | }|}] 206 | --------------------------------------------------------------------------------