├── .gitignore ├── CHECK-TESTS.sh ├── COMPARE-jsonnet.sh ├── LICENSE ├── LICENSE.json ├── LICENSE.jsonnet ├── LICENSE.md5 ├── README.md ├── ast.h ├── bridge.c ├── desugarer.cpp ├── desugarer.h ├── formatter.cpp ├── formatter.h ├── json.h ├── json.hpp ├── jsonnet.go ├── jsonnet_main └── main.go ├── jsonnet_test.go ├── lexer.cpp ├── lexer.h ├── libjsonnet.cpp ├── libjsonnet.h ├── libjsonnet_fmt.h ├── md5.cpp ├── md5.h ├── parser.cpp ├── parser.h ├── pass.cpp ├── pass.h ├── state.h ├── static_analysis.cpp ├── static_analysis.h ├── static_error.h ├── std.jsonnet.h ├── stress_loop └── stress_loop.go ├── string_utils.cpp ├── string_utils.h ├── test1.j ├── test2.j ├── unicode.h ├── vm.cpp └── vm.h /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | # Vim 27 | *~ 28 | .*.swp 29 | .*.swo 30 | 31 | # RCS 32 | RCS 33 | *,v 34 | -------------------------------------------------------------------------------- /CHECK-TESTS.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Run examples/ from google/jsonnet/ and compare output to golden files. 4 | 5 | MAIN="/tmp/$$.jsonnet_main" 6 | OUT="/tmp/$$.out" 7 | TRUE="/tmp/$$.true.golden" 8 | go build -o "$MAIN" jsonnet_main/main.go 9 | echo true > "$TRUE" 10 | 11 | E=../../google/jsonnet/examples/ 12 | S=../../google/jsonnet/test_suite/ 13 | STATUS=0 14 | for x in "$E"/*.jsonnet "$E"/terraform/*.jsonnet "$S"/*.jsonnet 15 | do 16 | case $x in 17 | */top-level-*.jsonnet ) 18 | # These top-level things don't have a printable value. 19 | continue 20 | ;; 21 | */error.* ) 22 | echo "== $x ==" 23 | if go run jsonnet_main/main.go "$x" >$OUT 2>&1 24 | then 25 | cat -nv $OUT 26 | echo "FAILED: That should have failed." 27 | STATUS=7 28 | fi 29 | ;; 30 | */stdlib.jsonnet | */tla.simple.jsonnet ) 31 | continue 32 | ;; 33 | * ) 34 | if test -f "$x.golden" 35 | then 36 | echo "== $x ==" 37 | (cd $(dirname "$x") ; $MAIN $(basename "$x")) >$OUT 2>&1 38 | if diff -b "$x.golden" "$OUT" 39 | then 40 | : ok good 41 | else 42 | STATUS=$? 43 | echo "FAILED: $x" 44 | fi 45 | else 46 | echo "== $x ==" 47 | (cd $(dirname "$x") ; $MAIN $(basename "$x")) >$OUT 2>&1 48 | if diff -b "$TRUE" "$OUT" 49 | then 50 | : ok good 51 | else 52 | STATUS=$? 53 | echo "FAILED: $x" 54 | fi 55 | fi 56 | ;; 57 | esac 58 | done 59 | 60 | case $STATUS in 61 | 0 ) echo 'ALL OKAY' >&2 ;; 62 | * ) echo 'FAILED' >&2 ;; 63 | esac 64 | 65 | rm -f "$MAIN" "$TRUE" "$OUT" 66 | exit $STATUS 67 | -------------------------------------------------------------------------------- /COMPARE-jsonnet.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # To help me keep up-to-date with jsonnet, 4 | # this will compare our copies of jsonnet files 5 | # with the ones in a jsonnet directory. 6 | # See "Usage:" a few lines below. 7 | 8 | WRITE= 9 | case "$1" in 10 | -w ) 11 | WRITE=1 12 | shift 13 | ;; 14 | esac 15 | 16 | case "$#/$1" in 17 | 0/ ) 18 | set ../../google/jsonnet/ 19 | ;; 20 | 1/*/jsonnet/ ) 21 | : ok 22 | ;; 23 | * ) 24 | echo >&2 ' 25 | Usage: 26 | sh $0 ?-w? ?/path/to/jsonnet/? 27 | 28 | This command takes one argument, the jsonnet repository directory, 29 | ending in /jsonnet/. The default is ../../google/jsonnet/. 30 | 31 | If -w is provided, then it will attempt to update files 32 | with differences from the source. 33 | ' 34 | exit 13 35 | ;; 36 | esac 37 | 38 | J="$1" 39 | test -d "$J" 40 | 41 | for x in \ 42 | ast.h \ 43 | desugarer.cpp \ 44 | desugarer.h \ 45 | formatter.cpp \ 46 | formatter.h \ 47 | json.h \ 48 | json.hpp \ 49 | lexer.cpp \ 50 | lexer.h \ 51 | libjsonnet.cpp \ 52 | libjsonnet.h \ 53 | libjsonnet_fmt.h \ 54 | md5.cpp \ 55 | md5.h \ 56 | parser.cpp \ 57 | parser.h \ 58 | pass.cpp \ 59 | pass.h \ 60 | state.h \ 61 | static_analysis.cpp \ 62 | static_analysis.h \ 63 | static_error.h \ 64 | std.jsonnet.h \ 65 | string_utils.cpp \ 66 | string_utils.h \ 67 | unicode.h \ 68 | vm.cpp \ 69 | vm.h \ 70 | # 71 | do 72 | ok=false 73 | F= 74 | for subdir in core cpp third_party/md5 third_party/json include 75 | do 76 | test -f "$J/$subdir/$x" && F="$J/$subdir/$x" 77 | if cmp "$J/$subdir/$x" "./$x" 2>/dev/null 78 | then 79 | ok=true 80 | break 81 | fi 82 | done 83 | 84 | if $ok 85 | then 86 | echo "ok: $x" 87 | else 88 | echo "******** NOT OK: $x" 89 | test -n "$WRITE" && cp -v "$F" "./$x" 90 | fi 91 | done 92 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Strick Yak 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /LICENSE.json: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013-2018 Niels Lohmann . 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | 21 | -------------------------------------------------------------------------------- /LICENSE.jsonnet: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /LICENSE.md5: -------------------------------------------------------------------------------- 1 | MD5 2 | Converted to C++ class by Frank Thilo (thilo@unix-ag.org) 3 | for bzflag (http://www.bzflag.org) 4 | 5 | based on: 6 | 7 | md5.h and md5.c 8 | reference implementation of RFC 1321 9 | 10 | Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All 11 | rights reserved. 12 | 13 | License to copy and use this software is granted provided that it 14 | is identified as the "RSA Data Security, Inc. MD5 Message-Digest 15 | Algorithm" in all material mentioning or referencing this software 16 | or this function. 17 | 18 | License is also granted to make and use derivative works provided 19 | that such works are identified as "derived from the RSA Data 20 | Security, Inc. MD5 Message-Digest Algorithm" in all material 21 | mentioning or referencing the derived work. 22 | 23 | RSA Data Security, Inc. makes no representations concerning either 24 | the merchantability of this software or the suitability of this 25 | software for any particular purpose. It is provided "as is" 26 | without express or implied warranty of any kind. 27 | 28 | These notices must be retained in any copies of any part of this 29 | documentation and/or software. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `jsonnet_cgo` 2 | 3 | Simple golang cgo wrapper around JSonnet VM. 4 | 5 | Everything in libjsonnet.h is covered except the multi-file evaluators. 6 | 7 | See jsonnet_test.go for how to use it. 8 | 9 | ## Quick example in golang: 10 | 11 | vm := jsonnet.Make() 12 | vm.ExtVar("color", "purple") 13 | 14 | x, err := vm.EvaluateSnippet(`Test_Demo`, `"dark " + std.extVar("color")`) 15 | 16 | if err != nil { 17 | panic(err) 18 | } 19 | if x != "\"dark purple\"\n" { 20 | panic("fail: we got " + x) 21 | } 22 | 23 | vm.Destroy() 24 | 25 | ## Quick examples with the command line demo program: 26 | 27 | ``` 28 | $ ( cd jsonnet_main/ ; go build -x -a ) 29 | ... 30 | mv $WORK/b001/exe/a.out jsonnet_main 31 | ... 32 | $ echo "{ a: 1, b: 2 }" | jsonnet_main/jsonnet_main /dev/stdin 33 | { 34 | "a": 1, 35 | "b": 2 36 | } 37 | $ cat test1.j 38 | { 39 | shell: "/bin/sh", 40 | awk: "/usr/bin/awk", 41 | } 42 | $ jsonnet_main/jsonnet_main test1.j 43 | { 44 | "awk": "/usr/bin/awk", 45 | "shell": "/bin/sh" 46 | } 47 | $ cat test2.j 48 | local test1 = import "test1.j"; 49 | 50 | test1 { 51 | shell: "/bin/csh", 52 | } 53 | $ jsonnet_main/jsonnet_main test2.j 54 | { 55 | "awk": "/usr/bin/awk", 56 | "shell": "/bin/csh" 57 | } 58 | $ echo ' std.extVar("a") + "bar" ' | jsonnet_main/jsonnet_main /dev/stdin a=foo 59 | "foobar" 60 | ``` 61 | 62 | ## LICENSES 63 | 64 | Notice the various `LICENSE*` files. I cannot offer legal advice, 65 | but you might find that the Apache License is the most restrictive. 66 | 67 | Most of this code comes from https://github.com/google/jsonnet 68 | and is under the Apache License, Version 2.0, January 2004, 69 | and our files that match filenames there are under that license. 70 | 71 | Notice the `third_party/` directory in that distribution. 72 | It has `json/` and `md5/` under their own licences, and our files 73 | that match filenames there are under those licenses. 74 | 75 | Anything new added here is under an MIT license in the plain `LICENSE` file. 76 | -------------------------------------------------------------------------------- /bridge.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "_cgo_export.h" 6 | 7 | char* CallImport_cgo(void *ctx, const char *base, const char *rel, char **found_here, int *success) { 8 | struct JsonnetVm* vm = ctx; 9 | return go_call_import(vm, (char*)base, (char*)rel, found_here, success); 10 | } 11 | 12 | struct JsonnetJsonValue* CallNative_cgo(void* ctx, const struct JsonnetJsonValue* const* argv, int* success) { 13 | GoUintptr key = (GoUintptr)ctx; 14 | return go_call_native(key, (struct JsonnetJsonValue**)argv, success); 15 | } 16 | -------------------------------------------------------------------------------- /desugarer.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #ifndef JSONNET_DESUGARING_H 18 | #define JSONNET_DESUGARING_H 19 | 20 | #include 21 | #include 22 | 23 | #include "ast.h" 24 | #include "vm.h" 25 | 26 | /** Translate the AST to remove syntax sugar. 27 | * \param alloc Allocator for making new identifiers / ASTs. 28 | * \param ast The AST to change. 29 | * \param tla the top level arguments. If null then do not try to process 30 | * top-level functions. 31 | */ 32 | void jsonnet_desugar(Allocator *alloc, AST *&ast, std::map *tla); 33 | 34 | #endif 35 | -------------------------------------------------------------------------------- /formatter.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #ifndef JSONNET_FORMATTER_H 18 | #define JSONNET_FORMATTER_H 19 | 20 | #include "ast.h" 21 | 22 | struct FmtOpts { 23 | char stringStyle; 24 | char commentStyle; 25 | unsigned indent; 26 | unsigned maxBlankLines; 27 | bool padArrays; 28 | bool padObjects; 29 | bool stripComments; 30 | bool stripAllButComments; 31 | bool stripEverything; 32 | bool prettyFieldNames; 33 | bool sortImports; 34 | FmtOpts(void) 35 | : stringStyle('s'), 36 | commentStyle('s'), 37 | indent(2), 38 | maxBlankLines(2), 39 | padArrays(false), 40 | padObjects(true), 41 | stripComments(false), 42 | stripAllButComments(false), 43 | stripEverything(false), 44 | prettyFieldNames(true), 45 | sortImports(true) 46 | { 47 | } 48 | }; 49 | 50 | /** The inverse of jsonnet_parse. 51 | */ 52 | std::string jsonnet_fmt(AST *ast, Fodder &final_fodder, const FmtOpts &opts); 53 | 54 | #endif // JSONNET_PARSER_H 55 | -------------------------------------------------------------------------------- /json.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #ifndef JSONNET_JSON_H 18 | #define JSONNET_JSON_H 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | 26 | struct JsonnetJsonValue { 27 | enum Kind { 28 | ARRAY, 29 | BOOL, 30 | NULL_KIND, 31 | NUMBER, 32 | OBJECT, 33 | STRING, 34 | }; 35 | 36 | JsonnetJsonValue() = default; 37 | JsonnetJsonValue(JsonnetJsonValue&) = delete; 38 | JsonnetJsonValue(JsonnetJsonValue&&) = default; 39 | 40 | JsonnetJsonValue(Kind kind, std::string string, double number) 41 | : kind(kind), string(string), number(number) 42 | { 43 | } 44 | 45 | Kind kind; 46 | std::string string; 47 | double number; // Also used for bool (0.0 and 1.0) 48 | std::vector> elements; 49 | std::map> fields; 50 | }; 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /jsonnet.go: -------------------------------------------------------------------------------- 1 | /* 2 | jsonnet is a simple Go wrapper for the JSonnet VM. 3 | 4 | See http://jsonnet.org/ 5 | */ 6 | package jsonnet 7 | 8 | // By Henry Strickland <@yak.net:strick> 9 | // Made self-contained by Marko Mikulicic 10 | 11 | /* 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | char *CallImport_cgo(void *ctx, const char *base, const char *rel, char **found_here, int *success); 20 | struct JsonnetJsonValue *CallNative_cgo(void *ctx, const struct JsonnetJsonValue *const *argv, int *success); 21 | 22 | #cgo CXXFLAGS: -std=c++0x -O3 23 | */ 24 | import "C" 25 | 26 | import ( 27 | "errors" 28 | "fmt" 29 | "reflect" 30 | "runtime" 31 | "sync" 32 | "unsafe" 33 | ) 34 | 35 | type ImportCallback func(base, rel string) (result string, path string, err error) 36 | 37 | type NativeCallback func(args ...*JsonValue) (result *JsonValue, err error) 38 | 39 | type nativeFunc struct { 40 | vm *VM 41 | argc int 42 | callback NativeCallback 43 | } 44 | 45 | // Global registry of native functions. Cgo pointer rules don't allow 46 | // us to pass go pointers directly (may not be stable), so pass uintptr 47 | // keys into this indirect map instead. 48 | var nativeFuncsMu sync.Mutex 49 | var nativeFuncsIdx uintptr 50 | var nativeFuncs = make(map[uintptr]*nativeFunc) 51 | 52 | func registerFunc(vm *VM, arity int, callback NativeCallback) uintptr { 53 | f := nativeFunc{vm: vm, argc: arity, callback: callback} 54 | 55 | nativeFuncsMu.Lock() 56 | defer nativeFuncsMu.Unlock() 57 | 58 | nativeFuncsIdx++ 59 | for nativeFuncs[nativeFuncsIdx] != nil { 60 | nativeFuncsIdx++ 61 | } 62 | 63 | nativeFuncs[nativeFuncsIdx] = &f 64 | return nativeFuncsIdx 65 | } 66 | 67 | func getFunc(key uintptr) *nativeFunc { 68 | nativeFuncsMu.Lock() 69 | defer nativeFuncsMu.Unlock() 70 | 71 | return nativeFuncs[key] 72 | } 73 | 74 | func unregisterFuncs(vm *VM) { 75 | nativeFuncsMu.Lock() 76 | defer nativeFuncsMu.Unlock() 77 | 78 | // This is inefficient if there are many 79 | // simultaneously-existing VMs... 80 | for idx, f := range nativeFuncs { 81 | if f.vm == vm { 82 | delete(nativeFuncs, idx) 83 | } 84 | } 85 | } 86 | 87 | type VM struct { 88 | guts *C.struct_JsonnetVm 89 | importCallback ImportCallback 90 | } 91 | 92 | //export go_call_native 93 | func go_call_native(key uintptr, argv **C.struct_JsonnetJsonValue, okPtr *C.int) *C.struct_JsonnetJsonValue { 94 | f := getFunc(key) 95 | vm := f.vm 96 | 97 | goArgv := make([]*JsonValue, f.argc) 98 | for i := 0; i < f.argc; i++ { 99 | p := unsafe.Pointer(uintptr(unsafe.Pointer(argv)) + unsafe.Sizeof(*argv)*uintptr(i)) 100 | argptr := (**C.struct_JsonnetJsonValue)(p) 101 | // NB: argv will be freed by jsonnet after this 102 | // function exits, so don't want (*JsonValue).destroy 103 | // finalizer. 104 | goArgv[i] = &JsonValue{ 105 | vm: vm, 106 | guts: *argptr, 107 | } 108 | } 109 | 110 | ret, err := f.callback(goArgv...) 111 | if err != nil { 112 | *okPtr = C.int(0) 113 | ret = vm.NewString(err.Error()) 114 | } else { 115 | *okPtr = C.int(1) 116 | } 117 | 118 | return ret.take() 119 | } 120 | 121 | //export go_call_import 122 | func go_call_import(vmPtr unsafe.Pointer, base, rel *C.char, pathPtr **C.char, okPtr *C.int) *C.char { 123 | vm := (*VM)(vmPtr) 124 | result, path, err := vm.importCallback(C.GoString(base), C.GoString(rel)) 125 | if err != nil { 126 | *okPtr = C.int(0) 127 | return jsonnetString(vm, err.Error()) 128 | } 129 | *pathPtr = jsonnetString(vm, path) 130 | *okPtr = C.int(1) 131 | return jsonnetString(vm, result) 132 | } 133 | 134 | // Evaluate a file containing Jsonnet code, return a JSON string. 135 | func Version() string { 136 | return C.GoString(C.jsonnet_version()) 137 | } 138 | 139 | // Create a new Jsonnet virtual machine. 140 | func Make() *VM { 141 | vm := &VM{guts: C.jsonnet_make()} 142 | return vm 143 | } 144 | 145 | // Complement of Make(). 146 | func (vm *VM) Destroy() { 147 | unregisterFuncs(vm) 148 | C.jsonnet_destroy(vm.guts) 149 | vm.guts = nil 150 | } 151 | 152 | // jsonnet often wants char* strings that were allocated via 153 | // jsonnet_realloc. This function does that. 154 | func jsonnetString(vm *VM, s string) *C.char { 155 | clen := C.size_t(len(s)) + 1 // num bytes including trailing \0 156 | 157 | // TODO: remove additional copy 158 | cstr := C.CString(s) 159 | defer C.free(unsafe.Pointer(cstr)) 160 | 161 | ret := C.jsonnet_realloc(vm.guts, nil, clen) 162 | C.memcpy(unsafe.Pointer(ret), unsafe.Pointer(cstr), clen) 163 | 164 | return ret 165 | } 166 | 167 | // Evaluate a file containing Jsonnet code, return a JSON string. 168 | func (vm *VM) EvaluateFile(filename string) (string, error) { 169 | cfilename := C.CString(filename) 170 | defer C.free(unsafe.Pointer(cfilename)) 171 | 172 | var e C.int 173 | z := C.GoString(C.jsonnet_evaluate_file(vm.guts, cfilename, &e)) 174 | if e != 0 { 175 | return "", errors.New(z) 176 | } 177 | return z, nil 178 | } 179 | 180 | // Evaluate a string containing Jsonnet code, return a JSON string. 181 | func (vm *VM) EvaluateSnippet(filename, snippet string) (string, error) { 182 | cfilename := C.CString(filename) 183 | csnippet := C.CString(snippet) 184 | defer func() { 185 | C.free(unsafe.Pointer(csnippet)) 186 | C.free(unsafe.Pointer(cfilename)) 187 | }() 188 | 189 | var e C.int 190 | z := C.GoString(C.jsonnet_evaluate_snippet(vm.guts, cfilename, csnippet, &e)) 191 | if e != 0 { 192 | return "", errors.New(z) 193 | } 194 | return z, nil 195 | } 196 | 197 | // Format a file containing Jsonnet code, return a JSON string. 198 | func (vm *VM) FormatFile(filename string) (string, error) { 199 | cfilename := C.CString(filename) 200 | defer C.free(unsafe.Pointer(cfilename)) 201 | 202 | var e C.int 203 | z := C.GoString(C.jsonnet_fmt_file(vm.guts, cfilename, &e)) 204 | if e != 0 { 205 | return "", errors.New(z) 206 | } 207 | return z, nil 208 | } 209 | 210 | // Indentation level when reformatting (number of spaces) 211 | func (vm *VM) FormatIndent(n int) { 212 | C.jsonnet_fmt_indent(vm.guts, C.int(n)) 213 | } 214 | 215 | // Format a string containing Jsonnet code, return a JSON string. 216 | func (vm *VM) FormatSnippet(filename, snippet string) (string, error) { 217 | cfilename := C.CString(filename) 218 | csnippet := C.CString(snippet) 219 | defer func() { 220 | C.free(unsafe.Pointer(csnippet)) 221 | C.free(unsafe.Pointer(cfilename)) 222 | }() 223 | 224 | var e C.int 225 | z := C.GoString(C.jsonnet_fmt_snippet(vm.guts, cfilename, csnippet, &e)) 226 | if e != 0 { 227 | return "", errors.New(z) 228 | } 229 | return z, nil 230 | } 231 | 232 | // Override the callback used to locate imports. 233 | func (vm *VM) ImportCallback(f ImportCallback) { 234 | vm.importCallback = f 235 | C.jsonnet_import_callback( 236 | vm.guts, 237 | (*C.JsonnetImportCallback)(unsafe.Pointer(C.CallImport_cgo)), 238 | unsafe.Pointer(vm)) 239 | } 240 | 241 | // NativeCallback is a helper around NativeCallbackRaw that uses 242 | // `reflect` to convert argument and result types to/from JsonValue. 243 | // `f` is expected to be a function that takes argument types 244 | // supported by `(*JsonValue).Extract` and returns `(x, error)` where 245 | // `x` is a type supported by `NewJson`. 246 | func (vm *VM) NativeCallback(name string, params []string, f interface{}) { 247 | ty := reflect.TypeOf(f) 248 | if ty.NumIn() != len(params) { 249 | panic("Wrong number of parameters") 250 | } 251 | if ty.NumOut() != 2 { 252 | panic("Wrong number of output parameters") 253 | } 254 | 255 | wrapper := func(args ...*JsonValue) (*JsonValue, error) { 256 | in := make([]reflect.Value, len(args)) 257 | for i, arg := range args { 258 | value := reflect.ValueOf(arg.Extract()) 259 | if vty := value.Type(); !vty.ConvertibleTo(ty.In(i)) { 260 | return nil, fmt.Errorf("parameter %d (type %s) cannot be converted to type %s", i, vty, ty.In(i)) 261 | } 262 | in[i] = value.Convert(ty.In(i)) 263 | } 264 | 265 | out := reflect.ValueOf(f).Call(in) 266 | 267 | result := vm.NewJson(out[0].Interface()) 268 | var err error 269 | if out[1].IsValid() && !out[1].IsNil() { 270 | err = out[1].Interface().(error) 271 | } 272 | return result, err 273 | } 274 | 275 | vm.NativeCallbackRaw(name, params, wrapper) 276 | } 277 | 278 | func (vm *VM) NativeCallbackRaw(name string, params []string, f NativeCallback) { 279 | cname := C.CString(name) 280 | defer C.free(unsafe.Pointer(cname)) 281 | 282 | // jsonnet expects this to be NULL-terminated, so the last 283 | // element is left as nil 284 | cparams := make([]*C.char, len(params)+1) 285 | for i, param := range params { 286 | cparams[i] = C.CString(param) 287 | defer C.free(unsafe.Pointer(cparams[i])) 288 | } 289 | 290 | key := registerFunc(vm, len(params), f) 291 | C.jsonnet_native_callback( 292 | vm.guts, 293 | cname, 294 | (*C.JsonnetNativeCallback)(C.CallNative_cgo), 295 | unsafe.Pointer(key), 296 | (**C.char)(unsafe.Pointer(&cparams[0]))) 297 | } 298 | 299 | // Bind a Jsonnet external var to the given value. 300 | func (vm *VM) ExtVar(key, val string) { 301 | ckey := C.CString(key) 302 | cval := C.CString(val) 303 | defer func() { 304 | C.free(unsafe.Pointer(cval)) 305 | C.free(unsafe.Pointer(ckey)) 306 | }() 307 | C.jsonnet_ext_var(vm.guts, ckey, cval) 308 | } 309 | 310 | // Bind a Jsonnet external var to the given Jsonnet code. 311 | func (vm *VM) ExtCode(key, val string) { 312 | ckey := C.CString(key) 313 | cval := C.CString(val) 314 | defer func() { 315 | C.free(unsafe.Pointer(cval)) 316 | C.free(unsafe.Pointer(ckey)) 317 | }() 318 | C.jsonnet_ext_code(vm.guts, ckey, cval) 319 | } 320 | 321 | // Bind a Jsonnet top-level argument to the given value. 322 | func (vm *VM) TlaVar(key, val string) { 323 | ckey := C.CString(key) 324 | cval := C.CString(val) 325 | defer func() { 326 | C.free(unsafe.Pointer(cval)) 327 | C.free(unsafe.Pointer(ckey)) 328 | }() 329 | C.jsonnet_tla_var(vm.guts, ckey, cval) 330 | } 331 | 332 | // Bind a Jsonnet top-level argument to the given Jsonnet code. 333 | func (vm *VM) TlaCode(key, val string) { 334 | ckey := C.CString(key) 335 | cval := C.CString(val) 336 | defer func() { 337 | C.free(unsafe.Pointer(cval)) 338 | C.free(unsafe.Pointer(ckey)) 339 | }() 340 | C.jsonnet_tla_code(vm.guts, ckey, cval) 341 | } 342 | 343 | // Set the maximum stack depth. 344 | func (vm *VM) MaxStack(v uint) { 345 | C.jsonnet_max_stack(vm.guts, C.uint(v)) 346 | } 347 | 348 | // Set the number of lines of stack trace to display (0 for all of them). 349 | func (vm *VM) MaxTrace(v uint) { 350 | C.jsonnet_max_trace(vm.guts, C.uint(v)) 351 | } 352 | 353 | // Set the number of objects required before a garbage collection cycle is allowed. 354 | func (vm *VM) GcMinObjects(v uint) { 355 | C.jsonnet_gc_min_objects(vm.guts, C.uint(v)) 356 | } 357 | 358 | // Run the garbage collector after this amount of growth in the number of objects. 359 | func (vm *VM) GcGrowthTrigger(v float64) { 360 | C.jsonnet_gc_growth_trigger(vm.guts, C.double(v)) 361 | } 362 | 363 | // Expect a string as output and don't JSON encode it. 364 | func (vm *VM) StringOutput(v bool) { 365 | if v { 366 | C.jsonnet_string_output(vm.guts, C.int(1)) 367 | } else { 368 | C.jsonnet_string_output(vm.guts, C.int(0)) 369 | } 370 | } 371 | 372 | // Add to the default import callback's library search path. 373 | func (vm *VM) JpathAdd(path string) { 374 | cpath := C.CString(path) 375 | defer C.free(unsafe.Pointer(cpath)) 376 | C.jsonnet_jpath_add(vm.guts, cpath) 377 | } 378 | 379 | /* The following are not implemented because they are trivial to implement in Go on top of the 380 | * existing API by parsing and post-processing the JSON output by regular evaluation. 381 | * 382 | * jsonnet_evaluate_file_multi 383 | * jsonnet_evaluate_snippet_multi 384 | * jsonnet_evaluate_file_stream 385 | * jsonnet_evaluate_snippet_stream 386 | */ 387 | 388 | // JsonValue represents a jsonnet JSON object. 389 | type JsonValue struct { 390 | vm *VM 391 | guts *C.struct_JsonnetJsonValue 392 | } 393 | 394 | func (v *JsonValue) Extract() interface{} { 395 | if x, ok := v.ExtractString(); ok { 396 | return x 397 | } 398 | if x, ok := v.ExtractNumber(); ok { 399 | return x 400 | } 401 | if x, ok := v.ExtractBool(); ok { 402 | return x 403 | } 404 | if ok := v.ExtractNull(); ok { 405 | return nil 406 | } 407 | panic("Unable to extract value") 408 | } 409 | 410 | // ExtractString returns the string value and true if the value was a string 411 | func (v *JsonValue) ExtractString() (string, bool) { 412 | cstr := C.jsonnet_json_extract_string(v.vm.guts, v.guts) 413 | if cstr == nil { 414 | return "", false 415 | } 416 | return C.GoString(cstr), true 417 | } 418 | 419 | func (v *JsonValue) ExtractNumber() (float64, bool) { 420 | var ret C.double 421 | ok := C.jsonnet_json_extract_number(v.vm.guts, v.guts, &ret) 422 | return float64(ret), ok != 0 423 | } 424 | 425 | func (v *JsonValue) ExtractBool() (bool, bool) { 426 | ret := C.jsonnet_json_extract_bool(v.vm.guts, v.guts) 427 | switch ret { 428 | case 0: 429 | return false, true 430 | case 1: 431 | return true, true 432 | case 2: 433 | // Not a bool 434 | return false, false 435 | default: 436 | panic("jsonnet_json_extract_number returned unexpected value") 437 | } 438 | } 439 | 440 | // ExtractNull returns true iff the value is null 441 | func (v *JsonValue) ExtractNull() bool { 442 | ret := C.jsonnet_json_extract_null(v.vm.guts, v.guts) 443 | return ret != 0 444 | } 445 | 446 | func (vm *VM) newjson(ptr *C.struct_JsonnetJsonValue) *JsonValue { 447 | v := &JsonValue{vm: vm, guts: ptr} 448 | runtime.SetFinalizer(v, (*JsonValue).destroy) 449 | return v 450 | } 451 | 452 | func (v *JsonValue) destroy() { 453 | if v.guts == nil { 454 | return 455 | } 456 | C.jsonnet_json_destroy(v.vm.guts, v.guts) 457 | v.guts = nil 458 | runtime.SetFinalizer(v, nil) 459 | } 460 | 461 | // Take ownership of the embedded ptr, effectively consuming the JsonValue 462 | func (v *JsonValue) take() *C.struct_JsonnetJsonValue { 463 | ptr := v.guts 464 | if ptr == nil { 465 | panic("taking nil pointer from JsonValue") 466 | } 467 | v.guts = nil 468 | runtime.SetFinalizer(v, nil) 469 | return ptr 470 | } 471 | 472 | func (vm *VM) NewJson(value interface{}) *JsonValue { 473 | switch val := value.(type) { 474 | case string: 475 | return vm.NewString(val) 476 | case int: 477 | return vm.NewNumber(float64(val)) 478 | case float64: 479 | return vm.NewNumber(val) 480 | case bool: 481 | return vm.NewBool(val) 482 | case nil: 483 | return vm.NewNull() 484 | case []interface{}: 485 | a := vm.NewArray() 486 | for _, v := range val { 487 | a.ArrayAppend(vm.NewJson(v)) 488 | } 489 | return a 490 | case map[string]interface{}: 491 | o := vm.NewObject() 492 | for k, v := range val { 493 | o.ObjectAppend(k, vm.NewJson(v)) 494 | } 495 | return o 496 | default: 497 | panic(fmt.Sprintf("NewJson can't handle type: %T", value)) 498 | } 499 | } 500 | 501 | func (vm *VM) NewString(v string) *JsonValue { 502 | cstr := C.CString(v) 503 | defer C.free(unsafe.Pointer(cstr)) 504 | ptr := C.jsonnet_json_make_string(vm.guts, cstr) 505 | return vm.newjson(ptr) 506 | } 507 | 508 | func (vm *VM) NewNumber(v float64) *JsonValue { 509 | ptr := C.jsonnet_json_make_number(vm.guts, C.double(v)) 510 | return vm.newjson(ptr) 511 | } 512 | 513 | func (vm *VM) NewBool(v bool) *JsonValue { 514 | var i C.int 515 | if v { 516 | i = 1 517 | } else { 518 | i = 0 519 | } 520 | ptr := C.jsonnet_json_make_bool(vm.guts, i) 521 | return vm.newjson(ptr) 522 | } 523 | 524 | func (vm *VM) NewNull() *JsonValue { 525 | ptr := C.jsonnet_json_make_null(vm.guts) 526 | return vm.newjson(ptr) 527 | } 528 | 529 | func (vm *VM) NewArray() *JsonValue { 530 | ptr := C.jsonnet_json_make_array(vm.guts) 531 | return vm.newjson(ptr) 532 | } 533 | 534 | func (v *JsonValue) ArrayAppend(item *JsonValue) { 535 | C.jsonnet_json_array_append(v.vm.guts, v.guts, item.take()) 536 | } 537 | 538 | func (vm *VM) NewObject() *JsonValue { 539 | ptr := C.jsonnet_json_make_object(vm.guts) 540 | return vm.newjson(ptr) 541 | } 542 | 543 | func (v *JsonValue) ObjectAppend(key string, value *JsonValue) { 544 | ckey := C.CString(key) 545 | defer C.free(unsafe.Pointer(ckey)) 546 | 547 | C.jsonnet_json_object_append(v.vm.guts, v.guts, ckey, value.take()) 548 | } 549 | -------------------------------------------------------------------------------- /jsonnet_main/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Command line tool to try evaluating JSonnet. 3 | 4 | Demos: 5 | echo "{ a: 1, b: 2 }" | go run jsonnet_main/main.go /dev/stdin 6 | go run jsonnet_main/main.go test1.j 7 | go run jsonnet_main/main.go test2.j 8 | echo 'std.extVar("a") + "bar"' | go run jsonnet_main/main.go /dev/stdin a=foo 9 | */ 10 | package main 11 | 12 | import "github.com/strickyak/jsonnet_cgo" 13 | 14 | import ( 15 | "flag" 16 | "fmt" 17 | "io/ioutil" 18 | "log" 19 | "path/filepath" 20 | "strings" 21 | ) 22 | 23 | var stringOutput = flag.Bool( 24 | "string_output", false, 25 | "If set, will expect a string and output it verbatim") 26 | 27 | func importFunc(base, rel string) (result string, path string, err error) { 28 | filename := filepath.Join(base, rel) 29 | contents, err := ioutil.ReadFile(filename) 30 | if err != nil { 31 | return "", "", err 32 | } 33 | return string(contents), filename, nil 34 | } 35 | 36 | func main() { 37 | flag.Parse() 38 | vm := jsonnet.Make() 39 | vm.ImportCallback(importFunc) 40 | 41 | if stringOutput != nil { 42 | vm.StringOutput(*stringOutput) 43 | } 44 | 45 | args := flag.Args() 46 | if len(args) < 1 { 47 | log.Fatal("Usage: jsonnet_main filename key1=val1 key2=val2...") 48 | } 49 | 50 | for i := 1; i < len(args); i++ { 51 | kv := strings.SplitN(args[i], "=", 2) 52 | if len(kv) != 2 { 53 | log.Fatalf("Error in jsonnet_main: Expected arg to be 'key=value': %q", args[i]) 54 | } 55 | vm.ExtVar(kv[0], kv[1]) 56 | } 57 | 58 | z, err := vm.EvaluateFile(args[0]) 59 | if err != nil { 60 | log.Fatalf("Error in jsonnet_main: %s", err) 61 | } 62 | fmt.Print(z) 63 | 64 | vm.Destroy() 65 | } 66 | -------------------------------------------------------------------------------- /jsonnet_test.go: -------------------------------------------------------------------------------- 1 | package jsonnet 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "strings" 9 | "testing" 10 | ) 11 | 12 | // Demo for the README. 13 | func Test_Demo(t *testing.T) { 14 | vm := Make() 15 | vm.ExtVar("color", "purple") 16 | 17 | x, err := vm.EvaluateSnippet(`Test_Demo`, `"dark " + std.extVar("color")`) 18 | if err != nil { 19 | panic(err) 20 | } 21 | if x != "\"dark purple\"\n" { 22 | panic("fail: we got " + x) 23 | } 24 | vm.Destroy() 25 | } 26 | 27 | // importFunc returns a couple of hardwired responses. 28 | func importFunc(base, rel string) (result string, path string, err error) { 29 | if rel == "alien.conf" { 30 | return `{ type: "alien", origin: "Ork", name: "Mork" }`, "alien.conf", nil 31 | } 32 | if rel == "human.conf" { 33 | return `{ type: "human", origin: "Earth", name: "Mendy" }`, "human.conf", nil 34 | } 35 | return "", "", errors.New(fmt.Sprintf("Cannot import %q", rel)) 36 | } 37 | 38 | // check there is no err, and a == b. 39 | func check(t *testing.T, err error, a, b string) { 40 | if err != nil { 41 | t.Errorf("got error: %q", err.Error()) 42 | } 43 | if a != b { 44 | t.Errorf("got %q but wanted %q", a, b) 45 | } 46 | } 47 | 48 | func Test_Simple(t *testing.T) { 49 | 50 | // Each time there's a new version, this will force an update to this code. 51 | check(t, nil, Version(), `v0.10.0`) 52 | 53 | vm := Make() 54 | defer vm.Destroy() 55 | vm.TlaVar("color", "purple") 56 | vm.TlaVar("size", "XXL") 57 | vm.TlaCode("gooselevel", "1234 * 10 + 5") 58 | vm.ExtVar("color", "purple") 59 | vm.ExtVar("size", "XXL") 60 | vm.ExtCode("gooselevel", "1234 * 10 + 5") 61 | vm.ImportCallback(importFunc) 62 | 63 | x, err := vm.EvaluateSnippet(`test1`, `20 + 22`) 64 | check(t, err, x, `42`+"\n") 65 | x, err = vm.EvaluateSnippet(`test2`, `function(color, size, gooselevel) color`) 66 | check(t, err, x, `"purple"`+"\n") 67 | x, err = vm.EvaluateSnippet(`test2`, `std.extVar("color")`) 68 | check(t, err, x, `"purple"`+"\n") 69 | vm.StringOutput(true) 70 | x, err = vm.EvaluateSnippet(`test2`, `"whee"`) 71 | check(t, err, x, `whee`+"\n") 72 | vm.StringOutput(false) 73 | x, err = vm.EvaluateSnippet(`test3`, ` 74 | local a = import "alien.conf"; 75 | local b = import "human.conf"; 76 | a.name + b.name 77 | `) 78 | check(t, err, x, `"MorkMendy"`+"\n") 79 | x, err = vm.EvaluateSnippet(`test4`, ` 80 | local a = import "alien.conf"; 81 | local b = a { type: "fictitious" }; 82 | b.type + b.name 83 | `) 84 | check(t, err, x, `"fictitiousMork"`+"\n") 85 | } 86 | 87 | func Test_FileScript(t *testing.T) { 88 | vm := Make() 89 | defer vm.Destroy() 90 | x, err := vm.EvaluateFile("test2.j") 91 | check(t, err, x, `{ 92 | "awk": "/usr/bin/awk", 93 | "shell": "/bin/csh" 94 | } 95 | `) 96 | } 97 | 98 | func Test_Misc(t *testing.T) { 99 | vm := Make() 100 | defer vm.Destroy() 101 | 102 | vm.MaxStack(10) 103 | vm.MaxTrace(10) 104 | vm.GcMinObjects(10) 105 | vm.GcGrowthTrigger(2.0) 106 | 107 | x, err := vm.EvaluateSnippet("Misc", ` 108 | local a = import "test2.j"; 109 | a.awk + a.shell`) 110 | check(t, err, x, `"/usr/bin/awk/bin/csh"`+"\n") 111 | } 112 | 113 | func Test_FormatFile(t *testing.T) { 114 | f, err := ioutil.TempFile("", "jsonnet-fmt-test") 115 | if err != nil { 116 | t.Fatal(err) 117 | } 118 | filename := f.Name() 119 | defer func() { 120 | f.Close() 121 | os.Remove(filename) 122 | }() 123 | 124 | data := `{ 125 | "quoted": "keys", 126 | "notevaluated": 20 + 22, 127 | "trailing": "comma",} 128 | ` 129 | if err := ioutil.WriteFile(filename, []byte(data), 0644); err != nil { 130 | t.Fatalf("WriteFile %s: %v", filename, err) 131 | } 132 | 133 | vm := Make() 134 | defer vm.Destroy() 135 | result, err := vm.FormatFile(filename) 136 | 137 | check(t, err, result, "{\n quoted: 'keys',\n notevaluated: 20 + 22,\n trailing: 'comma',\n}\n") 138 | } 139 | 140 | func Test_FormatSnippet(t *testing.T) { 141 | data := `{ 142 | "quoted": "keys", 143 | "notevaluated": 20 + 22, 144 | "trailing": "comma",} 145 | ` 146 | 147 | vm := Make() 148 | defer vm.Destroy() 149 | result, err := vm.FormatSnippet("testfoo", data) 150 | 151 | check(t, err, result, "{\n quoted: 'keys',\n notevaluated: 20 + 22,\n trailing: 'comma',\n}\n") 152 | } 153 | 154 | func Test_FormatIndent(t *testing.T) { 155 | data := `{ 156 | "quoted": "keys", 157 | "notevaluated": 20 + 22, 158 | "trailing": "comma",} 159 | ` 160 | 161 | vm := Make() 162 | defer vm.Destroy() 163 | vm.FormatIndent(1) 164 | result, err := vm.FormatSnippet("testfoo", data) 165 | 166 | check(t, err, result, "{\n quoted: 'keys',\n notevaluated: 20 + 22,\n trailing: 'comma',\n}\n") 167 | } 168 | 169 | func TestJsonString(t *testing.T) { 170 | vm := Make() 171 | defer vm.Destroy() 172 | 173 | v := vm.NewString("foo") 174 | 175 | if v2, ok := v.ExtractString(); !ok { 176 | t.Errorf("ExtractString() returned !ok") 177 | } else if v2 != "foo" { 178 | t.Errorf("Incorrect extracted string: %s", v2) 179 | } 180 | 181 | if _, ok := v.ExtractNumber(); ok { 182 | t.Errorf("ExtractNumber() returned ok") 183 | } 184 | 185 | if _, ok := v.ExtractBool(); ok { 186 | t.Errorf("ExtractBool() returned ok") 187 | } 188 | 189 | if ok := v.ExtractNull(); ok { 190 | t.Errorf("ExtractNull() returned ok") 191 | } 192 | } 193 | 194 | func TestJsonNumber(t *testing.T) { 195 | vm := Make() 196 | defer vm.Destroy() 197 | 198 | v := vm.NewNumber(42) 199 | 200 | if _, ok := v.ExtractString(); ok { 201 | t.Errorf("ExtractString() returned ok") 202 | } 203 | 204 | if v2, ok := v.ExtractNumber(); !ok { 205 | t.Errorf("ExtractNumber() returned !ok") 206 | } else if v2 != 42 { 207 | t.Errorf("ExtractNumber() returned unexpected value: %v", v2) 208 | } 209 | 210 | if _, ok := v.ExtractBool(); ok { 211 | t.Errorf("ExtractBool() returned ok") 212 | } 213 | 214 | if ok := v.ExtractNull(); ok { 215 | t.Errorf("ExtractNull() returned ok") 216 | } 217 | } 218 | 219 | func TestJsonArray(t *testing.T) { 220 | vm := Make() 221 | defer vm.Destroy() 222 | 223 | a := vm.NewArray() 224 | a.ArrayAppend(vm.NewString("foo")) 225 | a.ArrayAppend(vm.NewNull()) 226 | a.ArrayAppend(vm.NewNumber(3.14)) 227 | 228 | // Can't actually inspect array elements with this version of 229 | // jsonnet ... 230 | } 231 | 232 | func TestJsonObject(t *testing.T) { 233 | vm := Make() 234 | defer vm.Destroy() 235 | 236 | a := vm.NewObject() 237 | a.ObjectAppend("foo", vm.NewString("foo")) 238 | a.ObjectAppend("bar", vm.NewNull()) 239 | a.ObjectAppend("baz", vm.NewNumber(3.14)) 240 | 241 | // Can't actually inspect array elements with this version of 242 | // jsonnet ... 243 | } 244 | 245 | func TestNativeRaw(t *testing.T) { 246 | vm := Make() 247 | defer vm.Destroy() 248 | 249 | vm.NativeCallbackRaw("myadd", []string{"a", "b"}, func(args ...*JsonValue) (*JsonValue, error) { 250 | if len(args) != 2 { 251 | return nil, errors.New("wrong number of args") 252 | } 253 | 254 | a, ok := args[0].ExtractNumber() 255 | if !ok { 256 | return nil, errors.New("a is not a number") 257 | } 258 | b, ok := args[1].ExtractNumber() 259 | if !ok { 260 | return nil, errors.New("b is not a number") 261 | } 262 | return vm.NewNumber(a + b), nil 263 | }) 264 | 265 | x, err := vm.EvaluateSnippet("NativeRaw", `std.native("myadd")(3, 4)`) 266 | check(t, err, x, "7\n") 267 | 268 | x, err = vm.EvaluateSnippet("NativeRaw", `std.native("myadd")(42)`) 269 | if err == nil { 270 | t.Errorf("Wrong number of args failed to produce error") 271 | } 272 | 273 | x, err = vm.EvaluateSnippet("NativeRaw", `std.native("myadd")(3, "foo")`) 274 | if err == nil { 275 | t.Errorf("Go error was not translated into jsonnet error") 276 | } else if !strings.Contains(err.Error(), "b is not a number") { 277 | t.Errorf("Wrong jsonnet error: %v", err) 278 | } 279 | } 280 | 281 | func TestNative(t *testing.T) { 282 | vm := Make() 283 | defer vm.Destroy() 284 | 285 | vm.NativeCallback("myadd", []string{"a", "b"}, func(a, b float64) (float64, error) { 286 | return a + b, nil 287 | }) 288 | vm.NativeCallback("fail", []string{}, func() (interface{}, error) { 289 | return nil, fmt.Errorf("this is an error") 290 | }) 291 | 292 | x, err := vm.EvaluateSnippet("Native", `std.native("myadd")(3, 4)`) 293 | check(t, err, x, "7\n") 294 | 295 | x, err = vm.EvaluateSnippet("Native", `std.native("myadd")(42)`) 296 | if err == nil { 297 | t.Errorf("Wrong number of args failed to produce error") 298 | } 299 | 300 | x, err = vm.EvaluateSnippet("Native", `std.native("myadd")(3, "foo")`) 301 | if err == nil { 302 | t.Errorf("Wrong arg types failed to produce an error") 303 | } 304 | 305 | x, err = vm.EvaluateSnippet("Native", `std.native("fail")()`) 306 | if err == nil { 307 | t.Errorf("Go error was not propagated") 308 | } else if !strings.Contains(err.Error(), "this is an error") { 309 | t.Errorf("Go error text was not propagated") 310 | } 311 | } 312 | -------------------------------------------------------------------------------- /lexer.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "lexer.h" 24 | #include "static_error.h" 25 | #include "unicode.h" 26 | 27 | static const std::vector EMPTY; 28 | 29 | /** Is the char whitespace (excluding \n). */ 30 | static bool is_horz_ws(char c) 31 | { 32 | return c == ' ' || c == '\t' || c == '\r'; 33 | } 34 | 35 | /** Is the char whitespace. */ 36 | static bool is_ws(char c) 37 | { 38 | return c == '\n' || is_horz_ws(c); 39 | } 40 | 41 | /** Strip whitespace from both ends of a string, but only up to margin on the left hand side. */ 42 | static std::string strip_ws(const std::string &s, unsigned margin) 43 | { 44 | if (s.size() == 0) 45 | return s; // Avoid underflow below. 46 | size_t i = 0; 47 | while (i < s.length() && is_horz_ws(s[i]) && i < margin) 48 | i++; 49 | size_t j = s.size(); 50 | while (j > i && is_horz_ws(s[j - 1])) { 51 | j--; 52 | } 53 | return std::string(&s[i], &s[j]); 54 | } 55 | 56 | /** Split a string by \n and also strip left (up to margin) & right whitespace from each line. */ 57 | static std::vector line_split(const std::string &s, unsigned margin) 58 | { 59 | std::vector ret; 60 | std::stringstream ss; 61 | for (size_t i = 0; i < s.length(); ++i) { 62 | if (s[i] == '\n') { 63 | ret.emplace_back(strip_ws(ss.str(), margin)); 64 | ss.str(""); 65 | } else { 66 | ss << s[i]; 67 | } 68 | } 69 | ret.emplace_back(strip_ws(ss.str(), margin)); 70 | return ret; 71 | } 72 | 73 | /** Consume whitespace. 74 | * 75 | * Return number of \n and number of spaces after last \n. Convert \t to spaces. 76 | */ 77 | static void lex_ws(const char *&c, unsigned &new_lines, unsigned &indent, const char *&line_start, 78 | unsigned long &line_number) 79 | { 80 | indent = 0; 81 | new_lines = 0; 82 | for (; *c != '\0' && is_ws(*c); c++) { 83 | switch (*c) { 84 | case '\r': 85 | // Ignore. 86 | break; 87 | 88 | case '\n': 89 | indent = 0; 90 | new_lines++; 91 | line_number++; 92 | line_start = c + 1; 93 | break; 94 | 95 | case ' ': indent += 1; break; 96 | 97 | // This only works for \t at the beginning of lines, but we strip it everywhere else 98 | // anyway. The only case where this will cause a problem is spaces followed by \t 99 | // at the beginning of a line. However that is rare, ill-advised, and if re-indentation 100 | // is enabled it will be fixed later. 101 | case '\t': indent += 8; break; 102 | } 103 | } 104 | } 105 | 106 | /** 107 | # Consume all text until the end of the line, return number of newlines after that and indent 108 | */ 109 | static void lex_until_newline(const char *&c, std::string &text, unsigned &blanks, unsigned &indent, 110 | const char *&line_start, unsigned long &line_number) 111 | { 112 | const char *original_c = c; 113 | const char *last_non_space = c; 114 | for (; *c != '\0' && *c != '\n'; c++) { 115 | if (!is_horz_ws(*c)) 116 | last_non_space = c; 117 | } 118 | text = std::string(original_c, last_non_space - original_c + 1); 119 | // Consume subsequent whitespace including the '\n'. 120 | unsigned new_lines; 121 | lex_ws(c, new_lines, indent, line_start, line_number); 122 | blanks = new_lines == 0 ? 0 : new_lines - 1; 123 | } 124 | 125 | static bool is_upper(char c) 126 | { 127 | return c >= 'A' && c <= 'Z'; 128 | } 129 | 130 | static bool is_lower(char c) 131 | { 132 | return c >= 'a' && c <= 'z'; 133 | } 134 | 135 | static bool is_number(char c) 136 | { 137 | return c >= '0' && c <= '9'; 138 | } 139 | 140 | static bool is_identifier_first(char c) 141 | { 142 | return is_upper(c) || is_lower(c) || c == '_'; 143 | } 144 | 145 | static bool is_identifier(char c) 146 | { 147 | return is_identifier_first(c) || is_number(c); 148 | } 149 | 150 | static bool is_symbol(char c) 151 | { 152 | switch (c) { 153 | case '!': 154 | case '$': 155 | case ':': 156 | case '~': 157 | case '+': 158 | case '-': 159 | case '&': 160 | case '|': 161 | case '^': 162 | case '=': 163 | case '<': 164 | case '>': 165 | case '*': 166 | case '/': 167 | case '%': return true; 168 | } 169 | return false; 170 | } 171 | 172 | bool allowed_at_end_of_operator(char c) { 173 | switch (c) { 174 | case '+': 175 | case '-': 176 | case '~': 177 | case '!': 178 | case '$': return false; 179 | } 180 | return true; 181 | } 182 | 183 | static const std::map keywords = { 184 | {"assert", Token::ASSERT}, 185 | {"else", Token::ELSE}, 186 | {"error", Token::ERROR}, 187 | {"false", Token::FALSE}, 188 | {"for", Token::FOR}, 189 | {"function", Token::FUNCTION}, 190 | {"if", Token::IF}, 191 | {"import", Token::IMPORT}, 192 | {"importstr", Token::IMPORTSTR}, 193 | {"in", Token::IN}, 194 | {"local", Token::LOCAL}, 195 | {"null", Token::NULL_LIT}, 196 | {"self", Token::SELF}, 197 | {"super", Token::SUPER}, 198 | {"tailstrict", Token::TAILSTRICT}, 199 | {"then", Token::THEN}, 200 | {"true", Token::TRUE}, 201 | }; 202 | 203 | Token::Kind lex_get_keyword_kind(const std::string &identifier) 204 | { 205 | auto it = keywords.find(identifier); 206 | if (it == keywords.end()) 207 | return Token::IDENTIFIER; 208 | return it->second; 209 | } 210 | 211 | std::string lex_number(const char *&c, const std::string &filename, const Location &begin) 212 | { 213 | // This function should be understood with reference to the linked image: 214 | // http://www.json.org/number.gif 215 | 216 | // Note, we deviate from the json.org documentation as follows: 217 | // There is no reason to lex negative numbers as atomic tokens, it is better to parse them 218 | // as a unary operator combined with a numeric literal. This avoids x-1 being tokenized as 219 | // instead of the intended . 220 | 221 | enum State { 222 | BEGIN, 223 | AFTER_ZERO, 224 | AFTER_ONE_TO_NINE, 225 | AFTER_DOT, 226 | AFTER_DIGIT, 227 | AFTER_E, 228 | AFTER_EXP_SIGN, 229 | AFTER_EXP_DIGIT 230 | } state; 231 | 232 | std::string r; 233 | 234 | state = BEGIN; 235 | while (true) { 236 | switch (state) { 237 | case BEGIN: 238 | switch (*c) { 239 | case '0': state = AFTER_ZERO; break; 240 | 241 | case '1': 242 | case '2': 243 | case '3': 244 | case '4': 245 | case '5': 246 | case '6': 247 | case '7': 248 | case '8': 249 | case '9': state = AFTER_ONE_TO_NINE; break; 250 | 251 | default: throw StaticError(filename, begin, "couldn't lex number"); 252 | } 253 | break; 254 | 255 | case AFTER_ZERO: 256 | switch (*c) { 257 | case '.': state = AFTER_DOT; break; 258 | 259 | case 'e': 260 | case 'E': state = AFTER_E; break; 261 | 262 | default: goto end; 263 | } 264 | break; 265 | 266 | case AFTER_ONE_TO_NINE: 267 | switch (*c) { 268 | case '.': state = AFTER_DOT; break; 269 | 270 | case 'e': 271 | case 'E': state = AFTER_E; break; 272 | 273 | case '0': 274 | case '1': 275 | case '2': 276 | case '3': 277 | case '4': 278 | case '5': 279 | case '6': 280 | case '7': 281 | case '8': 282 | case '9': state = AFTER_ONE_TO_NINE; break; 283 | 284 | default: goto end; 285 | } 286 | break; 287 | 288 | case AFTER_DOT: 289 | switch (*c) { 290 | case '0': 291 | case '1': 292 | case '2': 293 | case '3': 294 | case '4': 295 | case '5': 296 | case '6': 297 | case '7': 298 | case '8': 299 | case '9': state = AFTER_DIGIT; break; 300 | 301 | default: { 302 | std::stringstream ss; 303 | ss << "couldn't lex number, junk after decimal point: " << *c; 304 | throw StaticError(filename, begin, ss.str()); 305 | } 306 | } 307 | break; 308 | 309 | case AFTER_DIGIT: 310 | switch (*c) { 311 | case 'e': 312 | case 'E': state = AFTER_E; break; 313 | 314 | case '0': 315 | case '1': 316 | case '2': 317 | case '3': 318 | case '4': 319 | case '5': 320 | case '6': 321 | case '7': 322 | case '8': 323 | case '9': state = AFTER_DIGIT; break; 324 | 325 | default: goto end; 326 | } 327 | break; 328 | 329 | case AFTER_E: 330 | switch (*c) { 331 | case '+': 332 | case '-': state = AFTER_EXP_SIGN; break; 333 | 334 | case '0': 335 | case '1': 336 | case '2': 337 | case '3': 338 | case '4': 339 | case '5': 340 | case '6': 341 | case '7': 342 | case '8': 343 | case '9': state = AFTER_EXP_DIGIT; break; 344 | 345 | default: { 346 | std::stringstream ss; 347 | ss << "couldn't lex number, junk after 'E': " << *c; 348 | throw StaticError(filename, begin, ss.str()); 349 | } 350 | } 351 | break; 352 | 353 | case AFTER_EXP_SIGN: 354 | switch (*c) { 355 | case '0': 356 | case '1': 357 | case '2': 358 | case '3': 359 | case '4': 360 | case '5': 361 | case '6': 362 | case '7': 363 | case '8': 364 | case '9': state = AFTER_EXP_DIGIT; break; 365 | 366 | default: { 367 | std::stringstream ss; 368 | ss << "couldn't lex number, junk after exponent sign: " << *c; 369 | throw StaticError(filename, begin, ss.str()); 370 | } 371 | } 372 | break; 373 | 374 | case AFTER_EXP_DIGIT: 375 | switch (*c) { 376 | case '0': 377 | case '1': 378 | case '2': 379 | case '3': 380 | case '4': 381 | case '5': 382 | case '6': 383 | case '7': 384 | case '8': 385 | case '9': state = AFTER_EXP_DIGIT; break; 386 | 387 | default: goto end; 388 | } 389 | break; 390 | } 391 | r += *c; 392 | c++; 393 | } 394 | end: 395 | return r; 396 | } 397 | 398 | // Check that b has at least the same whitespace prefix as a and returns the amount of this 399 | // whitespace, otherwise returns 0. If a has no whitespace prefix than return 0. 400 | static int whitespace_check(const char *a, const char *b) 401 | { 402 | int i = 0; 403 | while (a[i] == ' ' || a[i] == '\t') { 404 | if (b[i] != a[i]) 405 | return 0; 406 | i++; 407 | } 408 | return i; 409 | } 410 | 411 | /* 412 | static void add_whitespace(Fodder &fodder, const char *s, size_t n) 413 | { 414 | std::string ws(s, n); 415 | if (fodder.size() == 0 || fodder.back().kind != FodderElement::WHITESPACE) { 416 | fodder.emplace_back(FodderElement::WHITESPACE, ws); 417 | } else { 418 | fodder.back().data += ws; 419 | } 420 | } 421 | */ 422 | 423 | Tokens jsonnet_lex(const std::string &filename, const char *input) 424 | { 425 | unsigned long line_number = 1; 426 | const char *line_start = input; 427 | 428 | Tokens r; 429 | 430 | const char *c = input; 431 | 432 | Fodder fodder; 433 | bool fresh_line = true; // Are we tokenizing from the beginning of a new line? 434 | 435 | while (*c != '\0') { 436 | // Used to ensure we have actually advanced the pointer by the end of the iteration. 437 | const char *original_c = c; 438 | 439 | Token::Kind kind; 440 | std::string data; 441 | std::string string_block_indent; 442 | std::string string_block_term_indent; 443 | 444 | unsigned new_lines, indent; 445 | lex_ws(c, new_lines, indent, line_start, line_number); 446 | 447 | // If it's the end of the file, discard final whitespace. 448 | if (*c == '\0') 449 | break; 450 | 451 | if (new_lines > 0) { 452 | // Otherwise store whitespace in fodder. 453 | unsigned blanks = new_lines - 1; 454 | fodder.emplace_back(FodderElement::LINE_END, blanks, indent, EMPTY); 455 | fresh_line = true; 456 | } 457 | 458 | Location begin(line_number, c - line_start + 1); 459 | 460 | switch (*c) { 461 | // The following operators should never be combined with subsequent symbols. 462 | case '{': 463 | kind = Token::BRACE_L; 464 | c++; 465 | break; 466 | 467 | case '}': 468 | kind = Token::BRACE_R; 469 | c++; 470 | break; 471 | 472 | case '[': 473 | kind = Token::BRACKET_L; 474 | c++; 475 | break; 476 | 477 | case ']': 478 | kind = Token::BRACKET_R; 479 | c++; 480 | break; 481 | 482 | case ',': 483 | kind = Token::COMMA; 484 | c++; 485 | break; 486 | 487 | case '.': 488 | kind = Token::DOT; 489 | c++; 490 | break; 491 | 492 | case '(': 493 | kind = Token::PAREN_L; 494 | c++; 495 | break; 496 | 497 | case ')': 498 | kind = Token::PAREN_R; 499 | c++; 500 | break; 501 | 502 | case ';': 503 | kind = Token::SEMICOLON; 504 | c++; 505 | break; 506 | 507 | // Numeric literals. 508 | case '0': 509 | case '1': 510 | case '2': 511 | case '3': 512 | case '4': 513 | case '5': 514 | case '6': 515 | case '7': 516 | case '8': 517 | case '9': 518 | kind = Token::NUMBER; 519 | data = lex_number(c, filename, begin); 520 | break; 521 | 522 | // UString literals. 523 | case '"': { 524 | c++; 525 | for (;; ++c) { 526 | if (*c == '\0') { 527 | throw StaticError(filename, begin, "unterminated string"); 528 | } 529 | if (*c == '"') { 530 | break; 531 | } 532 | if (*c == '\\' && *(c + 1) != '\0') { 533 | data += *c; 534 | ++c; 535 | } 536 | if (*c == '\n') { 537 | // Maintain line/column counters. 538 | line_number++; 539 | line_start = c + 1; 540 | } 541 | data += *c; 542 | } 543 | c++; // Advance beyond the ". 544 | kind = Token::STRING_DOUBLE; 545 | } break; 546 | 547 | // UString literals. 548 | case '\'': { 549 | c++; 550 | for (;; ++c) { 551 | if (*c == '\0') { 552 | throw StaticError(filename, begin, "unterminated string"); 553 | } 554 | if (*c == '\'') { 555 | break; 556 | } 557 | if (*c == '\\' && *(c + 1) != '\0') { 558 | data += *c; 559 | ++c; 560 | } 561 | if (*c == '\n') { 562 | // Maintain line/column counters. 563 | line_number++; 564 | line_start = c + 1; 565 | } 566 | data += *c; 567 | } 568 | c++; // Advance beyond the '. 569 | kind = Token::STRING_SINGLE; 570 | } break; 571 | 572 | // Verbatim string literals. 573 | // ' and " quoting is interpreted here, unlike non-verbatim strings 574 | // where it is done later by jsonnet_string_unescape. This is OK 575 | // in this case because no information is lost by resoving the 576 | // repeated quote into a single quote, so we can go back to the 577 | // original form in the formatter. 578 | case '@': { 579 | c++; 580 | if (*c != '"' && *c != '\'') { 581 | std::stringstream ss; 582 | ss << "couldn't lex verbatim string, junk after '@': " << *c; 583 | throw StaticError(filename, begin, ss.str()); 584 | } 585 | const char quot = *c; 586 | c++; // Advance beyond the opening quote. 587 | for (;; ++c) { 588 | if (*c == '\0') { 589 | throw StaticError(filename, begin, "unterminated verbatim string"); 590 | } 591 | if (*c == quot) { 592 | if (*(c + 1) == quot) { 593 | c++; 594 | } else { 595 | break; 596 | } 597 | } 598 | data += *c; 599 | } 600 | c++; // Advance beyond the closing quote. 601 | if (quot == '"') { 602 | kind = Token::VERBATIM_STRING_DOUBLE; 603 | } else { 604 | kind = Token::VERBATIM_STRING_SINGLE; 605 | } 606 | } break; 607 | 608 | // Keywords 609 | default: 610 | if (is_identifier_first(*c)) { 611 | std::string id; 612 | for (; is_identifier(*c); ++c) 613 | id += *c; 614 | kind = lex_get_keyword_kind(id); 615 | data = id; 616 | 617 | } else if (is_symbol(*c) || *c == '#') { 618 | // Single line C++ and Python style comments. 619 | if (*c == '#' || (*c == '/' && *(c + 1) == '/')) { 620 | std::vector comment(1); 621 | unsigned blanks; 622 | unsigned indent; 623 | lex_until_newline(c, comment[0], blanks, indent, line_start, line_number); 624 | auto kind = fresh_line ? FodderElement::PARAGRAPH : FodderElement::LINE_END; 625 | fodder.emplace_back(kind, blanks, indent, comment); 626 | fresh_line = true; 627 | continue; // We've not got a token, just fodder, so keep scanning. 628 | } 629 | 630 | // Multi-line C style comment. 631 | if (*c == '/' && *(c + 1) == '*') { 632 | unsigned margin = c - line_start; 633 | 634 | const char *initial_c = c; 635 | c += 2; // Avoid matching /*/: skip the /* before starting the search for 636 | // */. 637 | 638 | while (!(*c == '*' && *(c + 1) == '/')) { 639 | if (*c == '\0') { 640 | auto msg = "multi-line comment has no terminating */."; 641 | throw StaticError(filename, begin, msg); 642 | } 643 | if (*c == '\n') { 644 | // Just keep track of the line / column counters. 645 | line_number++; 646 | line_start = c + 1; 647 | } 648 | ++c; 649 | } 650 | c += 2; // Move the pointer to the char after the closing '/'. 651 | 652 | std::string comment(initial_c, 653 | c - initial_c); // Includes the "/*" and "*/". 654 | 655 | // Lex whitespace after comment 656 | unsigned new_lines_after, indent_after; 657 | lex_ws(c, new_lines_after, indent_after, line_start, line_number); 658 | std::vector lines; 659 | if (comment.find('\n') >= comment.length()) { 660 | // Comment looks like /* foo */ 661 | lines.push_back(comment); 662 | fodder.emplace_back(FodderElement::INTERSTITIAL, 0, 0, lines); 663 | if (new_lines_after > 0) { 664 | fodder.emplace_back(FodderElement::LINE_END, 665 | new_lines_after - 1, 666 | indent_after, 667 | EMPTY); 668 | fresh_line = true; 669 | } 670 | } else { 671 | lines = line_split(comment, margin); 672 | assert(lines[0][0] == '/'); 673 | // Little hack to support PARAGRAPHs with * down the LHS: 674 | // Add a space to lines that start with a '*' 675 | bool all_star = true; 676 | for (auto &l : lines) { 677 | if (l[0] != '*') 678 | all_star = false; 679 | } 680 | if (all_star) { 681 | for (auto &l : lines) { 682 | if (l[0] == '*') 683 | l = " " + l; 684 | } 685 | } 686 | if (new_lines_after == 0) { 687 | // Ensure a line end after the paragraph. 688 | new_lines_after = 1; 689 | indent_after = 0; 690 | } 691 | fodder_push_back(fodder, 692 | FodderElement(FodderElement::PARAGRAPH, 693 | new_lines_after - 1, 694 | indent_after, 695 | lines)); 696 | fresh_line = true; 697 | } 698 | continue; // We've not got a token, just fodder, so keep scanning. 699 | } 700 | 701 | // Text block 702 | if (*c == '|' && *(c + 1) == '|' && *(c + 2) == '|') { 703 | c += 3; // Skip the "|||". 704 | while (is_horz_ws(*c)) ++c; // Chomp whitespace at end of line. 705 | if (*c != '\n') { 706 | auto msg = "text block syntax requires new line after |||."; 707 | throw StaticError(filename, begin, msg); 708 | } 709 | std::stringstream block; 710 | c++; // Skip the "\n" 711 | line_number++; 712 | // Skip any blank lines at the beginning of the block. 713 | while (*c == '\n') { 714 | line_number++; 715 | ++c; 716 | block << '\n'; 717 | } 718 | line_start = c; 719 | const char *first_line = c; 720 | int ws_chars = whitespace_check(first_line, c); 721 | string_block_indent = std::string(first_line, ws_chars); 722 | if (ws_chars == 0) { 723 | auto msg = "text block's first line must start with whitespace."; 724 | throw StaticError(filename, begin, msg); 725 | } 726 | while (true) { 727 | assert(ws_chars > 0); 728 | // Read up to the \n 729 | for (c = &c[ws_chars]; *c != '\n'; ++c) { 730 | if (*c == '\0') 731 | throw StaticError(filename, begin, "unexpected EOF"); 732 | block << *c; 733 | } 734 | // Add the \n 735 | block << '\n'; 736 | ++c; 737 | line_number++; 738 | line_start = c; 739 | // Skip any blank lines 740 | while (*c == '\n') { 741 | line_number++; 742 | ++c; 743 | block << '\n'; 744 | } 745 | // Examine next line 746 | ws_chars = whitespace_check(first_line, c); 747 | if (ws_chars == 0) { 748 | // End of text block 749 | // Skip over any whitespace 750 | while (*c == ' ' || *c == '\t') { 751 | string_block_term_indent += *c; 752 | ++c; 753 | } 754 | // Expect ||| 755 | if (!(*c == '|' && *(c + 1) == '|' && *(c + 2) == '|')) { 756 | auto msg = "text block not terminated with |||"; 757 | throw StaticError(filename, begin, msg); 758 | } 759 | c += 3; // Leave after the last | 760 | data = block.str(); 761 | kind = Token::STRING_BLOCK; 762 | break; // Out of the while loop. 763 | } 764 | } 765 | 766 | break; // Out of the switch. 767 | } 768 | 769 | const char *operator_begin = c; 770 | for (; is_symbol(*c); ++c) { 771 | // Not allowed // in operators 772 | if (*c == '/' && *(c + 1) == '/') 773 | break; 774 | // Not allowed /* in operators 775 | if (*c == '/' && *(c + 1) == '*') 776 | break; 777 | // Not allowed ||| in operators 778 | if (*c == '|' && *(c + 1) == '|' && *(c + 2) == '|') 779 | break; 780 | } 781 | // Not allowed to end with a + - ~ ! unless a single char. 782 | // So, wind it back if we need to (but not too far). 783 | while (c > operator_begin + 1 && !allowed_at_end_of_operator(*(c - 1))) { 784 | c--; 785 | } 786 | data += std::string(operator_begin, c); 787 | if (data == "$") { 788 | kind = Token::DOLLAR; 789 | data = ""; 790 | } else { 791 | kind = Token::OPERATOR; 792 | } 793 | } else { 794 | std::stringstream ss; 795 | ss << "Could not lex the character "; 796 | auto uc = (unsigned char)(*c); 797 | if (*c < 32) 798 | ss << "code " << unsigned(uc); 799 | else 800 | ss << "'" << *c << "'"; 801 | throw StaticError(filename, begin, ss.str()); 802 | } 803 | } 804 | 805 | // Ensure that a bug in the above code does not cause an infinite memory consuming loop due 806 | // to pushing empty tokens. 807 | if (c == original_c) { 808 | throw StaticError(filename, begin, "internal lexing error: pointer did not advance"); 809 | } 810 | 811 | Location end(line_number, (c + 1) - line_start); 812 | r.emplace_back(kind, 813 | fodder, 814 | data, 815 | string_block_indent, 816 | string_block_term_indent, 817 | LocationRange(filename, begin, end)); 818 | fodder.clear(); 819 | fresh_line = false; 820 | } 821 | 822 | Location begin(line_number, c - line_start + 1); 823 | Location end(line_number, (c + 1) - line_start + 1); 824 | r.emplace_back(Token::END_OF_FILE, fodder, "", "", "", LocationRange(filename, begin, end)); 825 | return r; 826 | } 827 | 828 | std::string jsonnet_unlex(const Tokens &tokens) 829 | { 830 | std::stringstream ss; 831 | for (const auto &t : tokens) { 832 | for (const auto &f : t.fodder) { 833 | switch (f.kind) { 834 | case FodderElement::LINE_END: { 835 | if (f.comment.size() > 0) { 836 | ss << "LineEnd(" << f.blanks << ", " << f.indent << ", " << f.comment[0] 837 | << ")\n"; 838 | } else { 839 | ss << "LineEnd(" << f.blanks << ", " << f.indent << ")\n"; 840 | } 841 | } break; 842 | 843 | case FodderElement::INTERSTITIAL: { 844 | ss << "Interstitial(" << f.comment[0] << ")\n"; 845 | } break; 846 | 847 | case FodderElement::PARAGRAPH: { 848 | ss << "Paragraph(\n"; 849 | for (const auto &line : f.comment) { 850 | ss << " " << line << '\n'; 851 | } 852 | ss << ")\n"; 853 | } break; 854 | } 855 | } 856 | if (t.kind == Token::END_OF_FILE) { 857 | ss << "EOF\n"; 858 | break; 859 | } 860 | if (t.kind == Token::STRING_DOUBLE) { 861 | ss << "\"" << t.data << "\"\n"; 862 | } else if (t.kind == Token::STRING_SINGLE) { 863 | ss << "'" << t.data << "'\n"; 864 | } else if (t.kind == Token::STRING_BLOCK) { 865 | ss << "|||\n"; 866 | ss << t.stringBlockIndent; 867 | for (const char *cp = t.data.c_str(); *cp != '\0'; ++cp) { 868 | ss << *cp; 869 | if (*cp == '\n' && *(cp + 1) != '\n' && *(cp + 1) != '\0') { 870 | ss << t.stringBlockIndent; 871 | } 872 | } 873 | ss << t.stringBlockTermIndent << "|||\n"; 874 | } else { 875 | ss << t.data << "\n"; 876 | } 877 | } 878 | return ss.str(); 879 | } 880 | -------------------------------------------------------------------------------- /lexer.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #ifndef JSONNET_LEXER_H 18 | #define JSONNET_LEXER_H 19 | 20 | #include 21 | #include 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | #include "static_error.h" 30 | #include "unicode.h" 31 | 32 | /** Whitespace and comments. 33 | * 34 | * "Fodder" (as in cannon fodder) implies this data is expendable. 35 | */ 36 | struct FodderElement { 37 | enum Kind { 38 | /** The next token, paragraph, or interstitial should be on a new line. 39 | * 40 | * A single comment string is allowed, which flows before the new line. 41 | * 42 | * The LINE_END fodder specifies the indentation level and vertical spacing before whatever 43 | * comes next. 44 | */ 45 | LINE_END, 46 | 47 | /** A C-style comment that begins and ends on the same line. 48 | * 49 | * If it follows a token (i.e., it is the first fodder element) then it appears after the 50 | * token on the same line. If it follows another interstitial, it will also flow after it 51 | * on the same line. If it follows a new line or a paragraph, it is the first thing on the 52 | * following line, after the blank lines and indentation specified by the previous fodder. 53 | * 54 | * There is exactly one comment string. 55 | */ 56 | INTERSTITIAL, 57 | 58 | /** A comment consisting of at least one line. 59 | * 60 | * // and # style commes have exactly one line. C-style comments can have more than one 61 | * line. 62 | * 63 | * All lines of the comment are indented according to the indentation level of the previous 64 | * new line / paragraph fodder. 65 | * 66 | * The PARAGRAPH fodder specifies the indentation level and vertical spacing before whatever 67 | * comes next. 68 | */ 69 | PARAGRAPH, 70 | }; 71 | Kind kind; 72 | 73 | /** How many blank lines (vertical space) before the next fodder / token. */ 74 | unsigned blanks; 75 | 76 | /** How far the next fodder / token should be indented. */ 77 | unsigned indent; 78 | 79 | /** Whatever comments are part of this fodder. 80 | * 81 | * Constraints apply. See Kind, above. 82 | * 83 | * The strings include any delimiting characters, e.g. // # and C-style comment delimiters but 84 | * not newline characters or indentation. 85 | */ 86 | std::vector comment; 87 | 88 | FodderElement(Kind kind, unsigned blanks, unsigned indent, 89 | const std::vector &comment) 90 | : kind(kind), blanks(blanks), indent(indent), comment(comment) 91 | { 92 | assert(kind != LINE_END || comment.size() <= 1); 93 | assert(kind != INTERSTITIAL || (blanks == 0 && indent == 0 && comment.size() == 1)); 94 | assert(kind != PARAGRAPH || comment.size() >= 1); 95 | } 96 | }; 97 | 98 | static inline std::ostream &operator<<(std::ostream &o, const FodderElement &f) 99 | { 100 | switch (f.kind) { 101 | case FodderElement::LINE_END: 102 | o << "END(" << f.blanks << ", " << f.indent; 103 | if (!f.comment.empty()) { 104 | o << ", " << f.comment[0]; 105 | } 106 | o << ")"; 107 | break; 108 | case FodderElement::INTERSTITIAL: 109 | o << "INT(" << f.blanks << ", " << f.indent << ", " << f.comment[0] << ")"; 110 | break; 111 | case FodderElement::PARAGRAPH: 112 | o << "PAR(" << f.blanks << ", " << f.indent << ", " << f.comment[0] << "...)"; 113 | break; 114 | } 115 | return o; 116 | } 117 | 118 | /** A sequence of fodder elements, typically between two tokens. 119 | * 120 | * A LINE_END is not allowed to follow a PARAGRAPH or a LINE_END. This can be represented by 121 | * replacing the indent of the prior fodder and increasing the number of blank lines if necessary. 122 | * If there was a comment, it can be represented by changing the LINE_END to a paragraph containing 123 | * the same single comment string. 124 | * 125 | * There must be a LINE_END or a PARAGRAPH before a PARAGRAPH. 126 | * 127 | * TODO(sbarzowski) Make it a proper class 128 | */ 129 | typedef std::vector Fodder; 130 | 131 | static inline bool fodder_has_clean_endline(const Fodder &fodder) 132 | { 133 | return !fodder.empty() && fodder.back().kind != FodderElement::INTERSTITIAL; 134 | } 135 | 136 | /** As a.push_back(elem) but preserves constraints. 137 | * 138 | * See concat_fodder below. 139 | */ 140 | static inline void fodder_push_back(Fodder &a, const FodderElement &elem) 141 | { 142 | if (fodder_has_clean_endline(a) && elem.kind == FodderElement::LINE_END) { 143 | if (elem.comment.size() > 0) { 144 | // The line end had a comment, so create a single line paragraph for it. 145 | a.emplace_back(FodderElement::PARAGRAPH, elem.blanks, elem.indent, elem.comment); 146 | } else { 147 | // Merge it into the previous line end. 148 | a.back().indent = elem.indent; 149 | a.back().blanks += elem.blanks; 150 | } 151 | } else { 152 | if (!fodder_has_clean_endline(a) && elem.kind == FodderElement::PARAGRAPH) { 153 | a.emplace_back(FodderElement::LINE_END, 0, elem.indent, std::vector()); 154 | } 155 | a.push_back(elem); 156 | } 157 | } 158 | 159 | /** As a + b but preserves constraints. 160 | * 161 | * Namely, a LINE_END is not allowed to follow a PARAGRAPH or a LINE_END. 162 | */ 163 | static inline Fodder concat_fodder(const Fodder &a, const Fodder &b) 164 | { 165 | if (a.size() == 0) 166 | return b; 167 | if (b.size() == 0) 168 | return a; 169 | Fodder r = a; 170 | // Carefully add the first element of b. 171 | fodder_push_back(r, b[0]); 172 | // Add the rest of b. 173 | for (unsigned i = 1; i < b.size(); ++i) { 174 | r.push_back(b[i]); 175 | } 176 | return r; 177 | } 178 | 179 | /** Move b to the front of a. */ 180 | static inline void fodder_move_front(Fodder &a, Fodder &b) 181 | { 182 | a = concat_fodder(b, a); 183 | b.clear(); 184 | } 185 | 186 | static inline Fodder make_fodder(const FodderElement &elem) 187 | { 188 | Fodder fodder; 189 | fodder_push_back(fodder, elem); 190 | return fodder; 191 | } 192 | 193 | static inline void ensureCleanNewline(Fodder &fodder) 194 | { 195 | if (!fodder_has_clean_endline(fodder)) { 196 | fodder_push_back(fodder, FodderElement(FodderElement::Kind::LINE_END, 0, 0, {})); 197 | } 198 | } 199 | 200 | static inline int countNewlines(const FodderElement &elem) 201 | { 202 | switch (elem.kind) { 203 | case FodderElement::INTERSTITIAL: return 0; 204 | case FodderElement::LINE_END: return 1; 205 | case FodderElement::PARAGRAPH: return elem.comment.size() + elem.blanks; 206 | } 207 | std::cerr << "Unknown FodderElement kind" << std::endl; 208 | abort(); 209 | } 210 | 211 | static inline int countNewlines(const Fodder &fodder) 212 | { 213 | int sum = 0; 214 | for (const auto &elem : fodder) { 215 | sum += countNewlines(elem); 216 | } 217 | return sum; 218 | } 219 | 220 | static inline std::ostream &operator<<(std::ostream &o, const Fodder &fodder) 221 | { 222 | bool first = true; 223 | for (const auto &f : fodder) { 224 | o << (first ? "[" : ", "); 225 | first = false; 226 | o << f; 227 | } 228 | o << (first ? "[]" : "]"); 229 | return o; 230 | } 231 | 232 | struct Token { 233 | enum Kind { 234 | // Symbols 235 | BRACE_L, 236 | BRACE_R, 237 | BRACKET_L, 238 | BRACKET_R, 239 | COMMA, 240 | DOLLAR, 241 | DOT, 242 | PAREN_L, 243 | PAREN_R, 244 | SEMICOLON, 245 | 246 | // Arbitrary length lexemes 247 | IDENTIFIER, 248 | NUMBER, 249 | OPERATOR, 250 | STRING_DOUBLE, 251 | STRING_SINGLE, 252 | STRING_BLOCK, 253 | VERBATIM_STRING_SINGLE, 254 | VERBATIM_STRING_DOUBLE, 255 | 256 | // Keywords 257 | ASSERT, 258 | ELSE, 259 | ERROR, 260 | FALSE, 261 | FOR, 262 | FUNCTION, 263 | IF, 264 | IMPORT, 265 | IMPORTSTR, 266 | IN, 267 | LOCAL, 268 | NULL_LIT, 269 | TAILSTRICT, 270 | THEN, 271 | SELF, 272 | SUPER, 273 | TRUE, 274 | 275 | // A special token that holds line/column information about the end of the file. 276 | END_OF_FILE 277 | } kind; 278 | 279 | /** Fodder before this token. */ 280 | Fodder fodder; 281 | 282 | /** Content of the token if it wasn't a keyword. */ 283 | std::string data; 284 | 285 | /** If kind == STRING_BLOCK then stores the sequence of whitespace that indented the block. */ 286 | std::string stringBlockIndent; 287 | 288 | /** If kind == STRING_BLOCK then stores the sequence of whitespace that indented the end of 289 | * the block. 290 | * 291 | * This is always fewer whitespace characters than in stringBlockIndent. 292 | */ 293 | std::string stringBlockTermIndent; 294 | 295 | UString data32(void) const 296 | { 297 | return decode_utf8(data); 298 | } 299 | 300 | LocationRange location; 301 | 302 | Token(Kind kind, const Fodder &fodder, const std::string &data, 303 | const std::string &string_block_indent, const std::string &string_block_term_indent, 304 | const LocationRange &location) 305 | : kind(kind), 306 | fodder(fodder), 307 | data(data), 308 | stringBlockIndent(string_block_indent), 309 | stringBlockTermIndent(string_block_term_indent), 310 | location(location) 311 | { 312 | } 313 | 314 | Token(Kind kind, const std::string &data = "") : kind(kind), data(data) {} 315 | 316 | static const char *toString(Kind v) 317 | { 318 | switch (v) { 319 | case BRACE_L: return "\"{\""; 320 | case BRACE_R: return "\"}\""; 321 | case BRACKET_L: return "\"[\""; 322 | case BRACKET_R: return "\"]\""; 323 | case COMMA: return "\",\""; 324 | case DOLLAR: return "\"$\""; 325 | case DOT: return "\".\""; 326 | 327 | case PAREN_L: return "\"(\""; 328 | case PAREN_R: return "\")\""; 329 | case SEMICOLON: return "\";\""; 330 | 331 | case IDENTIFIER: return "IDENTIFIER"; 332 | case NUMBER: return "NUMBER"; 333 | case OPERATOR: return "OPERATOR"; 334 | case STRING_SINGLE: return "STRING_SINGLE"; 335 | case STRING_DOUBLE: return "STRING_DOUBLE"; 336 | case VERBATIM_STRING_SINGLE: return "VERBATIM_STRING_SINGLE"; 337 | case VERBATIM_STRING_DOUBLE: return "VERBATIM_STRING_DOUBLE"; 338 | case STRING_BLOCK: return "STRING_BLOCK"; 339 | 340 | case ASSERT: return "assert"; 341 | case ELSE: return "else"; 342 | case ERROR: return "error"; 343 | case FALSE: return "false"; 344 | case FOR: return "for"; 345 | case FUNCTION: return "function"; 346 | case IF: return "if"; 347 | case IMPORT: return "import"; 348 | case IMPORTSTR: return "importstr"; 349 | case IN: return "in"; 350 | case LOCAL: return "local"; 351 | case NULL_LIT: return "null"; 352 | case SELF: return "self"; 353 | case SUPER: return "super"; 354 | case TAILSTRICT: return "tailstrict"; 355 | case THEN: return "then"; 356 | case TRUE: return "true"; 357 | 358 | case END_OF_FILE: return "end of file"; 359 | default: 360 | std::cerr << "INTERNAL ERROR: Unknown token kind: " << v << std::endl; 361 | std::abort(); 362 | } 363 | } 364 | }; 365 | 366 | /** The result of lexing. 367 | * 368 | * Because of the EOF token, this will always contain at least one token. So element 0 can be used 369 | * to get the filename. 370 | */ 371 | typedef std::list Tokens; 372 | 373 | static inline bool operator==(const Token &a, const Token &b) 374 | { 375 | if (a.kind != b.kind) 376 | return false; 377 | if (a.data != b.data) 378 | return false; 379 | return true; 380 | } 381 | 382 | static inline std::ostream &operator<<(std::ostream &o, Token::Kind v) 383 | { 384 | o << Token::toString(v); 385 | return o; 386 | } 387 | 388 | static inline std::ostream &operator<<(std::ostream &o, const Token &v) 389 | { 390 | if (v.data == "") { 391 | o << Token::toString(v.kind); 392 | } else if (v.kind == Token::OPERATOR) { 393 | o << "\"" << v.data << "\""; 394 | } else { 395 | o << "(" << Token::toString(v.kind) << ", \"" << v.data << "\")"; 396 | } 397 | return o; 398 | } 399 | 400 | /** IF the given identifier is a keyword, return its kind, otherwise return IDENTIFIER. */ 401 | Token::Kind lex_get_keyword_kind(const std::string &identifier); 402 | 403 | Tokens jsonnet_lex(const std::string &filename, const char *input); 404 | 405 | std::string jsonnet_unlex(const Tokens &tokens); 406 | 407 | #endif // JSONNET_LEXER_H 408 | -------------------------------------------------------------------------------- /libjsonnet.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include 18 | #include 19 | #include 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | extern "C" { 28 | #include "libjsonnet.h" 29 | #include "libjsonnet_fmt.h" 30 | } 31 | 32 | #include "desugarer.h" 33 | #include "formatter.h" 34 | #include "json.h" 35 | #include "parser.h" 36 | #include "static_analysis.h" 37 | #include "vm.h" 38 | 39 | static void memory_panic(void) 40 | { 41 | fputs("FATAL ERROR: a memory allocation error occurred.\n", stderr); 42 | abort(); 43 | } 44 | 45 | static char *from_string(JsonnetVm *vm, const std::string &v) 46 | { 47 | char *r = jsonnet_realloc(vm, nullptr, v.length() + 1); 48 | std::strcpy(r, v.c_str()); 49 | return r; 50 | } 51 | 52 | static char *default_import_callback(void *ctx, const char *dir, const char *file, 53 | char **found_here_cptr, int *success); 54 | 55 | const char *jsonnet_json_extract_string(JsonnetVm *vm, const struct JsonnetJsonValue *v) 56 | { 57 | (void)vm; 58 | if (v->kind != JsonnetJsonValue::STRING) 59 | return nullptr; 60 | return v->string.c_str(); 61 | } 62 | 63 | int jsonnet_json_extract_number(struct JsonnetVm *vm, const struct JsonnetJsonValue *v, double *out) 64 | { 65 | (void)vm; 66 | if (v->kind != JsonnetJsonValue::NUMBER) 67 | return 0; 68 | *out = v->number; 69 | return 1; 70 | } 71 | 72 | int jsonnet_json_extract_bool(struct JsonnetVm *vm, const struct JsonnetJsonValue *v) 73 | { 74 | (void)vm; 75 | if (v->kind != JsonnetJsonValue::BOOL) 76 | return 2; 77 | return v->number != 0; 78 | } 79 | 80 | int jsonnet_json_extract_null(struct JsonnetVm *vm, const struct JsonnetJsonValue *v) 81 | { 82 | (void)vm; 83 | return v->kind == JsonnetJsonValue::NULL_KIND; 84 | } 85 | 86 | JsonnetJsonValue *jsonnet_json_make_string(JsonnetVm *vm, const char *v) 87 | { 88 | (void)vm; 89 | JsonnetJsonValue *r = new JsonnetJsonValue(); 90 | r->kind = JsonnetJsonValue::STRING; 91 | r->string = v; 92 | return r; 93 | } 94 | 95 | JsonnetJsonValue *jsonnet_json_make_number(struct JsonnetVm *vm, double v) 96 | { 97 | (void)vm; 98 | JsonnetJsonValue *r = new JsonnetJsonValue(); 99 | r->kind = JsonnetJsonValue::NUMBER; 100 | r->number = v; 101 | return r; 102 | } 103 | 104 | JsonnetJsonValue *jsonnet_json_make_bool(struct JsonnetVm *vm, int v) 105 | { 106 | (void)vm; 107 | JsonnetJsonValue *r = new JsonnetJsonValue(); 108 | r->kind = JsonnetJsonValue::BOOL; 109 | r->number = v != 0; 110 | return r; 111 | } 112 | 113 | JsonnetJsonValue *jsonnet_json_make_null(struct JsonnetVm *vm) 114 | { 115 | (void)vm; 116 | JsonnetJsonValue *r = new JsonnetJsonValue(); 117 | r->kind = JsonnetJsonValue::NULL_KIND; 118 | return r; 119 | } 120 | 121 | JsonnetJsonValue *jsonnet_json_make_array(JsonnetVm *vm) 122 | { 123 | (void)vm; 124 | JsonnetJsonValue *r = new JsonnetJsonValue(); 125 | r->kind = JsonnetJsonValue::ARRAY; 126 | return r; 127 | } 128 | 129 | void jsonnet_json_array_append(JsonnetVm *vm, JsonnetJsonValue *arr, JsonnetJsonValue *v) 130 | { 131 | (void)vm; 132 | assert(arr->kind == JsonnetJsonValue::ARRAY); 133 | arr->elements.emplace_back(v); 134 | } 135 | 136 | JsonnetJsonValue *jsonnet_json_make_object(JsonnetVm *vm) 137 | { 138 | (void)vm; 139 | JsonnetJsonValue *r = new JsonnetJsonValue(); 140 | r->kind = JsonnetJsonValue::OBJECT; 141 | return r; 142 | } 143 | 144 | void jsonnet_json_object_append(JsonnetVm *vm, JsonnetJsonValue *obj, const char *f, 145 | JsonnetJsonValue *v) 146 | { 147 | (void)vm; 148 | assert(obj->kind == JsonnetJsonValue::OBJECT); 149 | obj->fields[std::string(f)] = std::unique_ptr(v); 150 | } 151 | 152 | void jsonnet_json_destroy(JsonnetVm *vm, JsonnetJsonValue *v) 153 | { 154 | (void)vm; 155 | delete v; 156 | } 157 | 158 | struct JsonnetVm { 159 | double gcGrowthTrigger; 160 | unsigned maxStack; 161 | unsigned gcMinObjects; 162 | unsigned maxTrace; 163 | std::map ext; 164 | std::map tla; 165 | JsonnetImportCallback *importCallback; 166 | VmNativeCallbackMap nativeCallbacks; 167 | void *importCallbackContext; 168 | bool stringOutput; 169 | std::vector jpaths; 170 | 171 | FmtOpts fmtOpts; 172 | bool fmtDebugDesugaring; 173 | 174 | JsonnetVm(void) 175 | : gcGrowthTrigger(2.0), 176 | maxStack(500), 177 | gcMinObjects(1000), 178 | maxTrace(20), 179 | importCallback(default_import_callback), 180 | importCallbackContext(this), 181 | stringOutput(false), 182 | fmtDebugDesugaring(false) 183 | { 184 | jpaths.emplace_back("/usr/share/jsonnet-" + std::string(jsonnet_version()) + "/"); 185 | jpaths.emplace_back("/usr/local/share/jsonnet-" + std::string(jsonnet_version()) + "/"); 186 | } 187 | }; 188 | 189 | enum ImportStatus { IMPORT_STATUS_OK, IMPORT_STATUS_FILE_NOT_FOUND, IMPORT_STATUS_IO_ERROR }; 190 | 191 | static enum ImportStatus try_path(const std::string &dir, const std::string &rel, 192 | std::string &content, std::string &found_here, 193 | std::string &err_msg) 194 | { 195 | std::string abs_path; 196 | if (rel.length() == 0) { 197 | err_msg = "the empty string is not a valid filename"; 198 | return IMPORT_STATUS_IO_ERROR; 199 | } 200 | // It is possible that rel is actually absolute. 201 | if (rel[0] == '/') { 202 | abs_path = rel; 203 | } else { 204 | abs_path = dir + rel; 205 | } 206 | 207 | if (abs_path[abs_path.length() - 1] == '/') { 208 | err_msg = "attempted to import a directory"; 209 | return IMPORT_STATUS_IO_ERROR; 210 | } 211 | 212 | std::ifstream f; 213 | f.open(abs_path.c_str()); 214 | if (!f.good()) 215 | return IMPORT_STATUS_FILE_NOT_FOUND; 216 | try { 217 | content.assign(std::istreambuf_iterator(f), std::istreambuf_iterator()); 218 | } catch (const std::ios_base::failure &io_err) { 219 | err_msg = io_err.what(); 220 | return IMPORT_STATUS_IO_ERROR; 221 | } 222 | if (!f.good()) { 223 | err_msg = strerror(errno); 224 | return IMPORT_STATUS_IO_ERROR; 225 | } 226 | 227 | found_here = abs_path; 228 | 229 | return IMPORT_STATUS_OK; 230 | } 231 | 232 | static char *default_import_callback(void *ctx, const char *dir, const char *file, 233 | char **found_here_cptr, int *success) 234 | { 235 | auto *vm = static_cast(ctx); 236 | 237 | std::string input, found_here, err_msg; 238 | 239 | ImportStatus status = try_path(dir, file, input, found_here, err_msg); 240 | 241 | std::vector jpaths(vm->jpaths); 242 | 243 | // If not found, try library search path. 244 | while (status == IMPORT_STATUS_FILE_NOT_FOUND) { 245 | if (jpaths.size() == 0) { 246 | *success = 0; 247 | const char *err = "no match locally or in the Jsonnet library paths."; 248 | char *r = jsonnet_realloc(vm, nullptr, std::strlen(err) + 1); 249 | std::strcpy(r, err); 250 | return r; 251 | } 252 | status = try_path(jpaths.back(), file, input, found_here, err_msg); 253 | jpaths.pop_back(); 254 | } 255 | 256 | if (status == IMPORT_STATUS_IO_ERROR) { 257 | *success = 0; 258 | return from_string(vm, err_msg); 259 | } else { 260 | assert(status == IMPORT_STATUS_OK); 261 | *success = 1; 262 | *found_here_cptr = from_string(vm, found_here); 263 | return from_string(vm, input); 264 | } 265 | } 266 | 267 | #define TRY try { 268 | #define CATCH(func) \ 269 | } \ 270 | catch (const std::bad_alloc &) \ 271 | { \ 272 | memory_panic(); \ 273 | } \ 274 | catch (const std::exception &e) \ 275 | { \ 276 | std::cerr << "Something went wrong during " func ", please report this: " << e.what() \ 277 | << std::endl; \ 278 | abort(); \ 279 | } 280 | 281 | const char *jsonnet_version(void) 282 | { 283 | return LIB_JSONNET_VERSION; 284 | } 285 | 286 | JsonnetVm *jsonnet_make(void) 287 | { 288 | TRY 289 | return new JsonnetVm(); 290 | CATCH("jsonnet_make") 291 | return nullptr; 292 | } 293 | 294 | void jsonnet_destroy(JsonnetVm *vm) 295 | { 296 | TRY 297 | delete vm; 298 | CATCH("jsonnet_destroy") 299 | } 300 | 301 | void jsonnet_max_stack(JsonnetVm *vm, unsigned v) 302 | { 303 | vm->maxStack = v; 304 | } 305 | 306 | void jsonnet_gc_min_objects(JsonnetVm *vm, unsigned v) 307 | { 308 | vm->gcMinObjects = v; 309 | } 310 | 311 | void jsonnet_gc_growth_trigger(JsonnetVm *vm, double v) 312 | { 313 | vm->gcGrowthTrigger = v; 314 | } 315 | 316 | void jsonnet_string_output(struct JsonnetVm *vm, int v) 317 | { 318 | vm->stringOutput = bool(v); 319 | } 320 | 321 | void jsonnet_import_callback(struct JsonnetVm *vm, JsonnetImportCallback *cb, void *ctx) 322 | { 323 | vm->importCallback = cb; 324 | vm->importCallbackContext = ctx; 325 | } 326 | 327 | void jsonnet_native_callback(struct JsonnetVm *vm, const char *name, JsonnetNativeCallback *cb, 328 | void *ctx, const char *const *params) 329 | { 330 | std::vector params2; 331 | for (; *params != nullptr; params++) 332 | params2.push_back(*params); 333 | vm->nativeCallbacks[name] = VmNativeCallback{cb, ctx, params2}; 334 | } 335 | 336 | void jsonnet_ext_var(JsonnetVm *vm, const char *key, const char *val) 337 | { 338 | vm->ext[key] = VmExt(val, false); 339 | } 340 | 341 | void jsonnet_ext_code(JsonnetVm *vm, const char *key, const char *val) 342 | { 343 | vm->ext[key] = VmExt(val, true); 344 | } 345 | 346 | void jsonnet_tla_var(JsonnetVm *vm, const char *key, const char *val) 347 | { 348 | vm->tla[key] = VmExt(val, false); 349 | } 350 | 351 | void jsonnet_tla_code(JsonnetVm *vm, const char *key, const char *val) 352 | { 353 | vm->tla[key] = VmExt(val, true); 354 | } 355 | 356 | void jsonnet_fmt_debug_desugaring(JsonnetVm *vm, int v) 357 | { 358 | vm->fmtDebugDesugaring = v; 359 | } 360 | 361 | void jsonnet_fmt_indent(JsonnetVm *vm, int v) 362 | { 363 | vm->fmtOpts.indent = v; 364 | } 365 | 366 | void jsonnet_fmt_max_blank_lines(JsonnetVm *vm, int v) 367 | { 368 | vm->fmtOpts.maxBlankLines = v; 369 | } 370 | 371 | void jsonnet_fmt_string(JsonnetVm *vm, int v) 372 | { 373 | if (v != 'd' && v != 's' && v != 'l') 374 | v = 'l'; 375 | vm->fmtOpts.stringStyle = v; 376 | } 377 | 378 | void jsonnet_fmt_comment(JsonnetVm *vm, int v) 379 | { 380 | if (v != 'h' && v != 's' && v != 'l') 381 | v = 'l'; 382 | vm->fmtOpts.commentStyle = v; 383 | } 384 | 385 | void jsonnet_fmt_pad_arrays(JsonnetVm *vm, int v) 386 | { 387 | vm->fmtOpts.padArrays = v; 388 | } 389 | 390 | void jsonnet_fmt_pad_objects(JsonnetVm *vm, int v) 391 | { 392 | vm->fmtOpts.padObjects = v; 393 | } 394 | 395 | void jsonnet_fmt_pretty_field_names(JsonnetVm *vm, int v) 396 | { 397 | vm->fmtOpts.prettyFieldNames = v; 398 | } 399 | 400 | void jsonnet_fmt_sort_imports(JsonnetVm *vm, int v) 401 | { 402 | vm->fmtOpts.sortImports = v; 403 | } 404 | 405 | void jsonnet_max_trace(JsonnetVm *vm, unsigned v) 406 | { 407 | vm->maxTrace = v; 408 | } 409 | 410 | void jsonnet_jpath_add(JsonnetVm *vm, const char *path_) 411 | { 412 | if (std::strlen(path_) == 0) 413 | return; 414 | std::string path = path_; 415 | if (path[path.length() - 1] != '/') 416 | path += '/'; 417 | vm->jpaths.emplace_back(path); 418 | } 419 | 420 | static char *jsonnet_fmt_snippet_aux(JsonnetVm *vm, const char *filename, const char *snippet, 421 | int *error) 422 | { 423 | try { 424 | Allocator alloc; 425 | std::string json_str; 426 | AST *expr; 427 | std::map files; 428 | Tokens tokens = jsonnet_lex(filename, snippet); 429 | 430 | // std::cout << jsonnet_unlex(tokens); 431 | 432 | expr = jsonnet_parse(&alloc, tokens); 433 | Fodder final_fodder = tokens.front().fodder; 434 | 435 | if (vm->fmtDebugDesugaring) 436 | jsonnet_desugar(&alloc, expr, &vm->tla); 437 | 438 | json_str = jsonnet_fmt(expr, final_fodder, vm->fmtOpts); 439 | 440 | json_str += "\n"; 441 | 442 | *error = false; 443 | return from_string(vm, json_str); 444 | 445 | } catch (StaticError &e) { 446 | std::stringstream ss; 447 | ss << "STATIC ERROR: " << e << std::endl; 448 | *error = true; 449 | return from_string(vm, ss.str()); 450 | } 451 | } 452 | 453 | char *jsonnet_fmt_file(JsonnetVm *vm, const char *filename, int *error) 454 | { 455 | TRY 456 | std::ifstream f; 457 | f.open(filename); 458 | if (!f.good()) { 459 | std::stringstream ss; 460 | ss << "Opening input file: " << filename << ": " << strerror(errno); 461 | *error = true; 462 | return from_string(vm, ss.str()); 463 | } 464 | std::string input; 465 | input.assign(std::istreambuf_iterator(f), std::istreambuf_iterator()); 466 | 467 | return jsonnet_fmt_snippet_aux(vm, filename, input.c_str(), error); 468 | CATCH("jsonnet_fmt_file") 469 | return nullptr; // Never happens. 470 | } 471 | 472 | char *jsonnet_fmt_snippet(JsonnetVm *vm, const char *filename, const char *snippet, int *error) 473 | { 474 | TRY 475 | return jsonnet_fmt_snippet_aux(vm, filename, snippet, error); 476 | CATCH("jsonnet_fmt_snippet") 477 | return nullptr; // Never happens. 478 | } 479 | 480 | namespace { 481 | enum EvalKind { REGULAR, MULTI, STREAM }; 482 | } // namespace 483 | 484 | static char *jsonnet_evaluate_snippet_aux(JsonnetVm *vm, const char *filename, const char *snippet, 485 | int *error, EvalKind kind) 486 | { 487 | try { 488 | Allocator alloc; 489 | AST *expr; 490 | Tokens tokens = jsonnet_lex(filename, snippet); 491 | 492 | expr = jsonnet_parse(&alloc, tokens); 493 | 494 | jsonnet_desugar(&alloc, expr, &vm->tla); 495 | 496 | unsigned max_stack = vm->maxStack; 497 | 498 | // For the stdlib desugaring. 499 | max_stack++; 500 | 501 | // For the TLA desugaring. 502 | max_stack++; 503 | 504 | jsonnet_static_analysis(expr); 505 | switch (kind) { 506 | case REGULAR: { 507 | std::string json_str = jsonnet_vm_execute(&alloc, 508 | expr, 509 | vm->ext, 510 | max_stack, 511 | vm->gcMinObjects, 512 | vm->gcGrowthTrigger, 513 | vm->nativeCallbacks, 514 | vm->importCallback, 515 | vm->importCallbackContext, 516 | vm->stringOutput); 517 | json_str += "\n"; 518 | *error = false; 519 | return from_string(vm, json_str); 520 | } break; 521 | 522 | case MULTI: { 523 | std::map files = 524 | jsonnet_vm_execute_multi(&alloc, 525 | expr, 526 | vm->ext, 527 | max_stack, 528 | vm->gcMinObjects, 529 | vm->gcGrowthTrigger, 530 | vm->nativeCallbacks, 531 | vm->importCallback, 532 | vm->importCallbackContext, 533 | vm->stringOutput); 534 | size_t sz = 1; // final sentinel 535 | for (const auto &pair : files) { 536 | sz += pair.first.length() + 1; // include sentinel 537 | sz += pair.second.length() + 2; // Add a '\n' as well as sentinel 538 | } 539 | char *buf = (char *)::malloc(sz); 540 | if (buf == nullptr) 541 | memory_panic(); 542 | std::ptrdiff_t i = 0; 543 | for (const auto &pair : files) { 544 | memcpy(&buf[i], pair.first.c_str(), pair.first.length() + 1); 545 | i += pair.first.length() + 1; 546 | memcpy(&buf[i], pair.second.c_str(), pair.second.length()); 547 | i += pair.second.length(); 548 | buf[i] = '\n'; 549 | i++; 550 | buf[i] = '\0'; 551 | i++; 552 | } 553 | buf[i] = '\0'; // final sentinel 554 | *error = false; 555 | return buf; 556 | } break; 557 | 558 | case STREAM: { 559 | std::vector documents = 560 | jsonnet_vm_execute_stream(&alloc, 561 | expr, 562 | vm->ext, 563 | max_stack, 564 | vm->gcMinObjects, 565 | vm->gcGrowthTrigger, 566 | vm->nativeCallbacks, 567 | vm->importCallback, 568 | vm->importCallbackContext); 569 | size_t sz = 1; // final sentinel 570 | for (const auto &doc : documents) { 571 | sz += doc.length() + 2; // Add a '\n' as well as sentinel 572 | } 573 | char *buf = (char *)::malloc(sz); 574 | if (buf == nullptr) 575 | memory_panic(); 576 | std::ptrdiff_t i = 0; 577 | for (const auto &doc : documents) { 578 | memcpy(&buf[i], doc.c_str(), doc.length()); 579 | i += doc.length(); 580 | buf[i] = '\n'; 581 | i++; 582 | buf[i] = '\0'; 583 | i++; 584 | } 585 | buf[i] = '\0'; // final sentinel 586 | *error = false; 587 | return buf; 588 | } break; 589 | 590 | default: 591 | fputs("INTERNAL ERROR: bad value of 'kind', probably memory corruption.\n", stderr); 592 | abort(); 593 | } 594 | 595 | } catch (StaticError &e) { 596 | std::stringstream ss; 597 | ss << "STATIC ERROR: " << e << std::endl; 598 | *error = true; 599 | return from_string(vm, ss.str()); 600 | 601 | } catch (RuntimeError &e) { 602 | std::stringstream ss; 603 | ss << "RUNTIME ERROR: " << e.msg << std::endl; 604 | const long max_above = vm->maxTrace / 2; 605 | const long max_below = vm->maxTrace - max_above; 606 | const long sz = e.stackTrace.size(); 607 | for (long i = 0; i < sz; ++i) { 608 | const auto &f = e.stackTrace[i]; 609 | if (vm->maxTrace > 0 && i >= max_above && i < sz - max_below) { 610 | if (i == max_above) 611 | ss << "\t..." << std::endl; 612 | } else { 613 | ss << "\t" << f.location << "\t" << f.name << std::endl; 614 | } 615 | } 616 | *error = true; 617 | return from_string(vm, ss.str()); 618 | } 619 | 620 | return nullptr; // Quiet, compiler. 621 | } 622 | 623 | static char *jsonnet_evaluate_file_aux(JsonnetVm *vm, const char *filename, int *error, 624 | EvalKind kind) 625 | { 626 | std::ifstream f; 627 | f.open(filename); 628 | if (!f.good()) { 629 | std::stringstream ss; 630 | ss << "Opening input file: " << filename << ": " << strerror(errno); 631 | *error = true; 632 | return from_string(vm, ss.str()); 633 | } 634 | std::string input; 635 | input.assign(std::istreambuf_iterator(f), std::istreambuf_iterator()); 636 | 637 | return jsonnet_evaluate_snippet_aux(vm, filename, input.c_str(), error, kind); 638 | } 639 | 640 | char *jsonnet_evaluate_file(JsonnetVm *vm, const char *filename, int *error) 641 | { 642 | TRY 643 | return jsonnet_evaluate_file_aux(vm, filename, error, REGULAR); 644 | CATCH("jsonnet_evaluate_file") 645 | return nullptr; // Never happens. 646 | } 647 | 648 | char *jsonnet_evaluate_file_multi(JsonnetVm *vm, const char *filename, int *error) 649 | { 650 | TRY 651 | return jsonnet_evaluate_file_aux(vm, filename, error, MULTI); 652 | CATCH("jsonnet_evaluate_file_multi") 653 | return nullptr; // Never happens. 654 | } 655 | 656 | char *jsonnet_evaluate_file_stream(JsonnetVm *vm, const char *filename, int *error) 657 | { 658 | TRY 659 | return jsonnet_evaluate_file_aux(vm, filename, error, STREAM); 660 | CATCH("jsonnet_evaluate_file_stream") 661 | return nullptr; // Never happens. 662 | } 663 | 664 | char *jsonnet_evaluate_snippet(JsonnetVm *vm, const char *filename, const char *snippet, int *error) 665 | { 666 | TRY 667 | return jsonnet_evaluate_snippet_aux(vm, filename, snippet, error, REGULAR); 668 | CATCH("jsonnet_evaluate_snippet") 669 | return nullptr; // Never happens. 670 | } 671 | 672 | char *jsonnet_evaluate_snippet_multi(JsonnetVm *vm, const char *filename, const char *snippet, 673 | int *error) 674 | { 675 | TRY 676 | return jsonnet_evaluate_snippet_aux(vm, filename, snippet, error, MULTI); 677 | CATCH("jsonnet_evaluate_snippet_multi") 678 | return nullptr; // Never happens. 679 | } 680 | 681 | char *jsonnet_evaluate_snippet_stream(JsonnetVm *vm, const char *filename, const char *snippet, 682 | int *error) 683 | { 684 | TRY 685 | return jsonnet_evaluate_snippet_aux(vm, filename, snippet, error, STREAM); 686 | CATCH("jsonnet_evaluate_snippet_stream") 687 | return nullptr; // Never happens. 688 | } 689 | 690 | char *jsonnet_realloc(JsonnetVm *vm, char *str, size_t sz) 691 | { 692 | (void)vm; 693 | if (str == nullptr) { 694 | if (sz == 0) 695 | return nullptr; 696 | auto *r = static_cast(::malloc(sz)); 697 | if (r == nullptr) 698 | memory_panic(); 699 | return r; 700 | } else { 701 | if (sz == 0) { 702 | ::free(str); 703 | return nullptr; 704 | } else { 705 | auto *r = static_cast(::realloc(str, sz)); 706 | if (r == nullptr) 707 | memory_panic(); 708 | return r; 709 | } 710 | } 711 | } 712 | -------------------------------------------------------------------------------- /libjsonnet.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #ifndef LIB_JSONNET_H 18 | #define LIB_JSONNET_H 19 | 20 | #include 21 | 22 | /** \file This file is a library interface for evaluating Jsonnet. It is written in C++ but exposes 23 | * a C interface for easier wrapping by other languages. See \see jsonnet_lib_test.c for an example 24 | * of using the library. 25 | */ 26 | 27 | #define LIB_JSONNET_VERSION "v0.13.0" 28 | 29 | /** Return the version string of the Jsonnet interpreter. Conforms to semantic versioning 30 | * http://semver.org/ If this does not match LIB_JSONNET_VERSION then there is a mismatch between 31 | * header and compiled library. 32 | */ 33 | const char *jsonnet_version(void); 34 | 35 | /** Jsonnet virtual machine context. */ 36 | struct JsonnetVm; 37 | 38 | /** Create a new Jsonnet virtual machine. */ 39 | struct JsonnetVm *jsonnet_make(void); 40 | 41 | /** Set the maximum stack depth. */ 42 | void jsonnet_max_stack(struct JsonnetVm *vm, unsigned v); 43 | 44 | /** Set the number of objects required before a garbage collection cycle is allowed. */ 45 | void jsonnet_gc_min_objects(struct JsonnetVm *vm, unsigned v); 46 | 47 | /** Run the garbage collector after this amount of growth in the number of objects. */ 48 | void jsonnet_gc_growth_trigger(struct JsonnetVm *vm, double v); 49 | 50 | /** Expect a string as output and don't JSON encode it. */ 51 | void jsonnet_string_output(struct JsonnetVm *vm, int v); 52 | 53 | /** Callback used to load imports. 54 | * 55 | * The returned char* should be allocated with jsonnet_realloc. It will be cleaned up by 56 | * libjsonnet when no-longer needed. 57 | * 58 | * \param ctx User pointer, given in jsonnet_import_callback. 59 | * \param base The directory containing the code that did the import. 60 | * \param rel The path imported by the code. 61 | * \param found_here Set this byref param to path to the file, absolute or relative to the 62 | * process's CWD. This is necessary so that imports from the content of the imported file can 63 | * be resolved correctly. Allocate memory with jsonnet_realloc. Only use when *success = 1. 64 | * \param success Set this byref param to 1 to indicate success and 0 for failure. 65 | * \returns The content of the imported file, or an error message. 66 | */ 67 | typedef char *JsonnetImportCallback(void *ctx, const char *base, const char *rel, char **found_here, 68 | int *success); 69 | 70 | /** An opaque type which can only be utilized via the jsonnet_json_* family of functions. 71 | */ 72 | struct JsonnetJsonValue; 73 | 74 | /** If the value is a string, return it as UTF8 otherwise return NULL. 75 | */ 76 | const char *jsonnet_json_extract_string(struct JsonnetVm *vm, const struct JsonnetJsonValue *v); 77 | 78 | /** If the value is a number, return 1 and store the number in out, otherwise return 0. 79 | */ 80 | int jsonnet_json_extract_number(struct JsonnetVm *vm, const struct JsonnetJsonValue *v, 81 | double *out); 82 | 83 | /** Return 0 if the value is false, 1 if it is true, and 2 if it is not a bool. 84 | */ 85 | int jsonnet_json_extract_bool(struct JsonnetVm *vm, const struct JsonnetJsonValue *v); 86 | 87 | /** Return 1 if the value is null, else 0. 88 | */ 89 | int jsonnet_json_extract_null(struct JsonnetVm *vm, const struct JsonnetJsonValue *v); 90 | 91 | /** Convert the given UTF8 string to a JsonnetJsonValue. 92 | */ 93 | struct JsonnetJsonValue *jsonnet_json_make_string(struct JsonnetVm *vm, const char *v); 94 | 95 | /** Convert the given double to a JsonnetJsonValue. 96 | */ 97 | struct JsonnetJsonValue *jsonnet_json_make_number(struct JsonnetVm *vm, double v); 98 | 99 | /** Convert the given bool (1 or 0) to a JsonnetJsonValue. 100 | */ 101 | struct JsonnetJsonValue *jsonnet_json_make_bool(struct JsonnetVm *vm, int v); 102 | 103 | /** Make a JsonnetJsonValue representing null. 104 | */ 105 | struct JsonnetJsonValue *jsonnet_json_make_null(struct JsonnetVm *vm); 106 | 107 | /** Make a JsonnetJsonValue representing an array. 108 | * 109 | * Assign elements with jsonnet_json_array_append. 110 | */ 111 | struct JsonnetJsonValue *jsonnet_json_make_array(struct JsonnetVm *vm); 112 | 113 | /** Add v to the end of the array. 114 | */ 115 | void jsonnet_json_array_append(struct JsonnetVm *vm, struct JsonnetJsonValue *arr, 116 | struct JsonnetJsonValue *v); 117 | 118 | /** Make a JsonnetJsonValue representing an object with the given number of fields. 119 | * 120 | * Every index of the array must have a unique value assigned with jsonnet_json_array_element. 121 | */ 122 | struct JsonnetJsonValue *jsonnet_json_make_object(struct JsonnetVm *vm); 123 | 124 | /** Add the field f to the object, bound to v. 125 | * 126 | * This replaces any previous binding of the field. 127 | */ 128 | void jsonnet_json_object_append(struct JsonnetVm *vm, struct JsonnetJsonValue *obj, const char *f, 129 | struct JsonnetJsonValue *v); 130 | 131 | /** Clean up a JSON subtree. 132 | * 133 | * This is useful if you want to abort with an error mid-way through building a complex value. 134 | */ 135 | void jsonnet_json_destroy(struct JsonnetVm *vm, struct JsonnetJsonValue *v); 136 | 137 | /** Callback to provide native extensions to Jsonnet. 138 | * 139 | * The returned JsonnetJsonValue* should be allocated with jsonnet_realloc. It will be cleaned up 140 | * along with the objects rooted at argv by libjsonnet when no-longer needed. Return a string upon 141 | * failure, which will appear in Jsonnet as an error. The argv pointer is an array whose size 142 | * matches the array of parameters supplied when the native callback was originally registered. 143 | * 144 | * \param ctx User pointer, given in jsonnet_native_callback. 145 | * \param argv Array of arguments from Jsonnet code. 146 | * \param success Set this byref param to 1 to indicate success and 0 for failure. 147 | * \returns The content of the imported file, or an error message. 148 | */ 149 | typedef struct JsonnetJsonValue *JsonnetNativeCallback(void *ctx, 150 | const struct JsonnetJsonValue *const *argv, 151 | int *success); 152 | 153 | /** Allocate, resize, or free a buffer. This will abort if the memory cannot be allocated. It will 154 | * only return NULL if sz was zero. 155 | * 156 | * \param buf If NULL, allocate a new buffer. If an previously allocated buffer, resize it. 157 | * \param sz The size of the buffer to return. If zero, frees the buffer. 158 | * \returns The new buffer. 159 | */ 160 | char *jsonnet_realloc(struct JsonnetVm *vm, char *buf, size_t sz); 161 | 162 | /** Override the callback used to locate imports. 163 | */ 164 | void jsonnet_import_callback(struct JsonnetVm *vm, JsonnetImportCallback *cb, void *ctx); 165 | 166 | /** Register a native extension. 167 | * 168 | * This will appear in Jsonnet as a function type and can be accessed from std.nativeExt("foo"). 169 | * 170 | * DO NOT register native callbacks with side-effects! Jsonnet is a lazy functional language and 171 | * will call your function when you least expect it, more times than you expect, or not at all. 172 | * 173 | * \param vm The vm. 174 | * \param name The name of the function as visible to Jsonnet code, e.g. "foo". 175 | * \param cb The PURE function that implements the behavior you want. 176 | * \param ctx User pointer, stash non-global state you need here. 177 | * \param params NULL-terminated array of the names of the params. Must be valid identifiers. 178 | */ 179 | void jsonnet_native_callback(struct JsonnetVm *vm, const char *name, JsonnetNativeCallback *cb, 180 | void *ctx, const char *const *params); 181 | 182 | /** Bind a Jsonnet external var to the given string. 183 | * 184 | * Argument values are copied so memory should be managed by caller. 185 | */ 186 | void jsonnet_ext_var(struct JsonnetVm *vm, const char *key, const char *val); 187 | 188 | /** Bind a Jsonnet external var to the given code. 189 | * 190 | * Argument values are copied so memory should be managed by caller. 191 | */ 192 | void jsonnet_ext_code(struct JsonnetVm *vm, const char *key, const char *val); 193 | 194 | /** Bind a string top-level argument for a top-level parameter. 195 | * 196 | * Argument values are copied so memory should be managed by caller. 197 | */ 198 | void jsonnet_tla_var(struct JsonnetVm *vm, const char *key, const char *val); 199 | 200 | /** Bind a code top-level argument for a top-level parameter. 201 | * 202 | * Argument values are copied so memory should be managed by caller. 203 | */ 204 | void jsonnet_tla_code(struct JsonnetVm *vm, const char *key, const char *val); 205 | 206 | /** Set the number of lines of stack trace to display (0 for all of them). */ 207 | void jsonnet_max_trace(struct JsonnetVm *vm, unsigned v); 208 | 209 | /** Add to the default import callback's library search path. 210 | * 211 | * The search order is last to first, so more recently appended paths take precedence. 212 | */ 213 | void jsonnet_jpath_add(struct JsonnetVm *vm, const char *v); 214 | 215 | /** Evaluate a file containing Jsonnet code, return a JSON string. 216 | * 217 | * The returned string should be cleaned up with jsonnet_realloc. 218 | * 219 | * \param filename Path to a file containing Jsonnet code. 220 | * \param error Return by reference whether or not there was an error. 221 | * \returns Either JSON or the error message. 222 | */ 223 | char *jsonnet_evaluate_file(struct JsonnetVm *vm, const char *filename, int *error); 224 | 225 | /** Evaluate a string containing Jsonnet code, return a JSON string. 226 | * 227 | * The returned string should be cleaned up with jsonnet_realloc. 228 | * 229 | * \param filename Path to a file (used in error messages). 230 | * \param snippet Jsonnet code to execute. 231 | * \param error Return by reference whether or not there was an error. 232 | * \returns Either JSON or the error message. 233 | */ 234 | char *jsonnet_evaluate_snippet(struct JsonnetVm *vm, const char *filename, const char *snippet, 235 | int *error); 236 | 237 | /** Evaluate a file containing Jsonnet code, return a number of named JSON files. 238 | * 239 | * The returned character buffer contains an even number of strings, the filename and JSON for each 240 | * JSON file interleaved. It should be cleaned up with jsonnet_realloc. 241 | * 242 | * \param filename Path to a file containing Jsonnet code. 243 | * \param error Return by reference whether or not there was an error. 244 | * \returns Either the error, or a sequence of strings separated by \0, terminated with \0\0. 245 | */ 246 | char *jsonnet_evaluate_file_multi(struct JsonnetVm *vm, const char *filename, int *error); 247 | 248 | /** Evaluate a string containing Jsonnet code, return a number of named JSON files. 249 | * 250 | * The returned character buffer contains an even number of strings, the filename and JSON for each 251 | * JSON file interleaved. It should be cleaned up with jsonnet_realloc. 252 | * 253 | * \param filename Path to a file containing Jsonnet code. 254 | * \param snippet Jsonnet code to execute. 255 | * \param error Return by reference whether or not there was an error. 256 | * \returns Either the error, or a sequence of strings separated by \0, terminated with \0\0. 257 | */ 258 | char *jsonnet_evaluate_snippet_multi(struct JsonnetVm *vm, const char *filename, 259 | const char *snippet, int *error); 260 | 261 | /** Evaluate a file containing Jsonnet code, return a number of JSON files. 262 | * 263 | * The returned character buffer contains several strings. It should be cleaned up with 264 | * jsonnet_realloc. 265 | * 266 | * \param filename Path to a file containing Jsonnet code. 267 | * \param error Return by reference whether or not there was an error. 268 | * \returns Either the error, or a sequence of strings separated by \0, terminated with \0\0. 269 | */ 270 | char *jsonnet_evaluate_file_stream(struct JsonnetVm *vm, const char *filename, int *error); 271 | 272 | /** Evaluate a string containing Jsonnet code, return a number of JSON files. 273 | * 274 | * The returned character buffer contains several strings. It should be cleaned up with 275 | * jsonnet_realloc. 276 | * 277 | * \param filename Path to a file containing Jsonnet code. 278 | * \param snippet Jsonnet code to execute. 279 | * \param error Return by reference whether or not there was an error. 280 | * \returns Either the error, or a sequence of strings separated by \0, terminated with \0\0. 281 | */ 282 | char *jsonnet_evaluate_snippet_stream(struct JsonnetVm *vm, const char *filename, 283 | const char *snippet, int *error); 284 | 285 | /** Complement of \see jsonnet_vm_make. */ 286 | void jsonnet_destroy(struct JsonnetVm *vm); 287 | 288 | #endif // LIB_JSONNET_H 289 | -------------------------------------------------------------------------------- /libjsonnet_fmt.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #ifndef LIB_JSONNET_FMT_H 18 | #define LIB_JSONNET_FMT_H 19 | 20 | #include 21 | 22 | /** \file This file is a library interface for formatting Jsonnet code. 23 | */ 24 | 25 | /** Jsonnet virtual machine context. */ 26 | struct JsonnetVm; 27 | 28 | /** Indentation level when reformatting (number of spaeces). 29 | * 30 | * \param n Number of spaces, must be > 0. 31 | */ 32 | void jsonnet_fmt_indent(struct JsonnetVm *vm, int n); 33 | 34 | /** Indentation level when reformatting (number of spaeces). 35 | * 36 | * \param n Number of spaces, must be > 0. 37 | */ 38 | void jsonnet_fmt_max_blank_lines(struct JsonnetVm *vm, int n); 39 | 40 | /** Preferred style for string literals ("" or ''). 41 | * 42 | * \param c String style as a char ('d', 's', or 'l' (leave)). 43 | */ 44 | void jsonnet_fmt_string(struct JsonnetVm *vm, int c); 45 | 46 | /** Preferred style for line comments (# or //). 47 | * 48 | * \param c Comment style as a char ('h', 's', or 'l' (leave)). 49 | */ 50 | void jsonnet_fmt_comment(struct JsonnetVm *vm, int c); 51 | 52 | /** Whether to add an extra space on the inside of arrays. 53 | */ 54 | void jsonnet_fmt_pad_arrays(struct JsonnetVm *vm, int v); 55 | 56 | /** Whether to add an extra space on the inside of objects. 57 | */ 58 | void jsonnet_fmt_pad_objects(struct JsonnetVm *vm, int v); 59 | 60 | /** Use syntax sugar where possible with field names. 61 | */ 62 | void jsonnet_fmt_pretty_field_names(struct JsonnetVm *vm, int v); 63 | 64 | /** Sort top-level imports in alphabetical order 65 | */ 66 | void jsonnet_fmt_sort_imports(struct JsonnetVm *vm, int v); 67 | 68 | /** If set to 1, will reformat the Jsonnet input after desugaring. */ 69 | void jsonnet_fmt_debug_desugaring(struct JsonnetVm *vm, int v); 70 | 71 | /** Reformat a file containing Jsonnet code, return a Jsonnet string. 72 | * 73 | * The returned string should be cleaned up with jsonnet_realloc. 74 | * 75 | * \param filename Path to a file containing Jsonnet code. 76 | * \param error Return by reference whether or not there was an error. 77 | * \returns Either Jsonnet code or the error message. 78 | */ 79 | char *jsonnet_fmt_file(struct JsonnetVm *vm, const char *filename, int *error); 80 | 81 | /** Reformat a string containing Jsonnet code, return a Jsonnet string. 82 | * 83 | * The returned string should be cleaned up with jsonnet_realloc. 84 | * 85 | * \param filename Path to a file (used in error messages). 86 | * \param snippet Jsonnet code to execute. 87 | * \param error Return by reference whether or not there was an error. 88 | * \returns Either Jsonnet code or the error message. 89 | */ 90 | char *jsonnet_fmt_snippet(struct JsonnetVm *vm, const char *filename, const char *snippet, 91 | int *error); 92 | 93 | #endif // LIB_JSONNET_FMT_H 94 | -------------------------------------------------------------------------------- /md5.cpp: -------------------------------------------------------------------------------- 1 | /* MD5 2 | converted to C++ class by Frank Thilo (thilo@unix-ag.org) 3 | for bzflag (http://www.bzflag.org) 4 | 5 | based on: 6 | 7 | md5.h and md5.c 8 | reference implemantion of RFC 1321 9 | 10 | Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All 11 | rights reserved. 12 | 13 | License to copy and use this software is granted provided that it 14 | is identified as the "RSA Data Security, Inc. MD5 Message-Digest 15 | Algorithm" in all material mentioning or referencing this software 16 | or this function. 17 | 18 | License is also granted to make and use derivative works provided 19 | that such works are identified as "derived from the RSA Data 20 | Security, Inc. MD5 Message-Digest Algorithm" in all material 21 | mentioning or referencing the derived work. 22 | 23 | RSA Data Security, Inc. makes no representations concerning either 24 | the merchantability of this software or the suitability of this 25 | software for any particular purpose. It is provided "as is" 26 | without express or implied warranty of any kind. 27 | 28 | These notices must be retained in any copies of any part of this 29 | documentation and/or software. 30 | 31 | */ 32 | 33 | /* interface header */ 34 | #include "md5.h" 35 | 36 | /* system implementation headers */ 37 | #include 38 | #include 39 | 40 | 41 | // Constants for MD5Transform routine. 42 | #define S11 7 43 | #define S12 12 44 | #define S13 17 45 | #define S14 22 46 | #define S21 5 47 | #define S22 9 48 | #define S23 14 49 | #define S24 20 50 | #define S31 4 51 | #define S32 11 52 | #define S33 16 53 | #define S34 23 54 | #define S41 6 55 | #define S42 10 56 | #define S43 15 57 | #define S44 21 58 | 59 | /////////////////////////////////////////////// 60 | 61 | // F, G, H and I are basic MD5 functions. 62 | inline MD5::uint4 MD5::F(uint4 x, uint4 y, uint4 z) { 63 | return (x&y) | (~x&z); 64 | } 65 | 66 | inline MD5::uint4 MD5::G(uint4 x, uint4 y, uint4 z) { 67 | return (x&z) | (y&~z); 68 | } 69 | 70 | inline MD5::uint4 MD5::H(uint4 x, uint4 y, uint4 z) { 71 | return x^y^z; 72 | } 73 | 74 | inline MD5::uint4 MD5::I(uint4 x, uint4 y, uint4 z) { 75 | return y ^ (x | ~z); 76 | } 77 | 78 | // rotate_left rotates x left n bits. 79 | inline MD5::uint4 MD5::rotate_left(uint4 x, int n) { 80 | return (x << n) | (x >> (32-n)); 81 | } 82 | 83 | // FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4. 84 | // Rotation is separate from addition to prevent recomputation. 85 | inline void MD5::FF(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac) { 86 | a = rotate_left(a+ F(b,c,d) + x + ac, s) + b; 87 | } 88 | 89 | inline void MD5::GG(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac) { 90 | a = rotate_left(a + G(b,c,d) + x + ac, s) + b; 91 | } 92 | 93 | inline void MD5::HH(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac) { 94 | a = rotate_left(a + H(b,c,d) + x + ac, s) + b; 95 | } 96 | 97 | inline void MD5::II(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac) { 98 | a = rotate_left(a + I(b,c,d) + x + ac, s) + b; 99 | } 100 | 101 | ////////////////////////////////////////////// 102 | 103 | // default ctor, just initailize 104 | MD5::MD5() 105 | { 106 | init(); 107 | } 108 | 109 | ////////////////////////////////////////////// 110 | 111 | // nifty shortcut ctor, compute MD5 for string and finalize it right away 112 | MD5::MD5(const std::string &text) 113 | { 114 | init(); 115 | update(text.c_str(), text.length()); 116 | finalize(); 117 | } 118 | 119 | ////////////////////////////// 120 | 121 | void MD5::init() 122 | { 123 | finalized=false; 124 | 125 | count[0] = 0; 126 | count[1] = 0; 127 | 128 | // load magic initialization constants. 129 | state[0] = 0x67452301; 130 | state[1] = 0xefcdab89; 131 | state[2] = 0x98badcfe; 132 | state[3] = 0x10325476; 133 | } 134 | 135 | ////////////////////////////// 136 | 137 | // decodes input (unsigned char) into output (uint4). Assumes len is a multiple of 4. 138 | void MD5::decode(uint4 output[], const uint1 input[], size_type len) 139 | { 140 | for (unsigned int i = 0, j = 0; j < len; i++, j += 4) 141 | output[i] = ((uint4)input[j]) | (((uint4)input[j+1]) << 8) | 142 | (((uint4)input[j+2]) << 16) | (((uint4)input[j+3]) << 24); 143 | } 144 | 145 | ////////////////////////////// 146 | 147 | // encodes input (uint4) into output (unsigned char). Assumes len is 148 | // a multiple of 4. 149 | void MD5::encode(uint1 output[], const uint4 input[], size_type len) 150 | { 151 | for (size_type i = 0, j = 0; j < len; i++, j += 4) { 152 | output[j] = input[i] & 0xff; 153 | output[j+1] = (input[i] >> 8) & 0xff; 154 | output[j+2] = (input[i] >> 16) & 0xff; 155 | output[j+3] = (input[i] >> 24) & 0xff; 156 | } 157 | } 158 | 159 | ////////////////////////////// 160 | 161 | // apply MD5 algo on a block 162 | void MD5::transform(const uint1 block[blocksize]) 163 | { 164 | uint4 a = state[0], b = state[1], c = state[2], d = state[3], x[16]; 165 | decode (x, block, blocksize); 166 | 167 | /* Round 1 */ 168 | FF (a, b, c, d, x[ 0], S11, 0xd76aa478); /* 1 */ 169 | FF (d, a, b, c, x[ 1], S12, 0xe8c7b756); /* 2 */ 170 | FF (c, d, a, b, x[ 2], S13, 0x242070db); /* 3 */ 171 | FF (b, c, d, a, x[ 3], S14, 0xc1bdceee); /* 4 */ 172 | FF (a, b, c, d, x[ 4], S11, 0xf57c0faf); /* 5 */ 173 | FF (d, a, b, c, x[ 5], S12, 0x4787c62a); /* 6 */ 174 | FF (c, d, a, b, x[ 6], S13, 0xa8304613); /* 7 */ 175 | FF (b, c, d, a, x[ 7], S14, 0xfd469501); /* 8 */ 176 | FF (a, b, c, d, x[ 8], S11, 0x698098d8); /* 9 */ 177 | FF (d, a, b, c, x[ 9], S12, 0x8b44f7af); /* 10 */ 178 | FF (c, d, a, b, x[10], S13, 0xffff5bb1); /* 11 */ 179 | FF (b, c, d, a, x[11], S14, 0x895cd7be); /* 12 */ 180 | FF (a, b, c, d, x[12], S11, 0x6b901122); /* 13 */ 181 | FF (d, a, b, c, x[13], S12, 0xfd987193); /* 14 */ 182 | FF (c, d, a, b, x[14], S13, 0xa679438e); /* 15 */ 183 | FF (b, c, d, a, x[15], S14, 0x49b40821); /* 16 */ 184 | 185 | /* Round 2 */ 186 | GG (a, b, c, d, x[ 1], S21, 0xf61e2562); /* 17 */ 187 | GG (d, a, b, c, x[ 6], S22, 0xc040b340); /* 18 */ 188 | GG (c, d, a, b, x[11], S23, 0x265e5a51); /* 19 */ 189 | GG (b, c, d, a, x[ 0], S24, 0xe9b6c7aa); /* 20 */ 190 | GG (a, b, c, d, x[ 5], S21, 0xd62f105d); /* 21 */ 191 | GG (d, a, b, c, x[10], S22, 0x2441453); /* 22 */ 192 | GG (c, d, a, b, x[15], S23, 0xd8a1e681); /* 23 */ 193 | GG (b, c, d, a, x[ 4], S24, 0xe7d3fbc8); /* 24 */ 194 | GG (a, b, c, d, x[ 9], S21, 0x21e1cde6); /* 25 */ 195 | GG (d, a, b, c, x[14], S22, 0xc33707d6); /* 26 */ 196 | GG (c, d, a, b, x[ 3], S23, 0xf4d50d87); /* 27 */ 197 | GG (b, c, d, a, x[ 8], S24, 0x455a14ed); /* 28 */ 198 | GG (a, b, c, d, x[13], S21, 0xa9e3e905); /* 29 */ 199 | GG (d, a, b, c, x[ 2], S22, 0xfcefa3f8); /* 30 */ 200 | GG (c, d, a, b, x[ 7], S23, 0x676f02d9); /* 31 */ 201 | GG (b, c, d, a, x[12], S24, 0x8d2a4c8a); /* 32 */ 202 | 203 | /* Round 3 */ 204 | HH (a, b, c, d, x[ 5], S31, 0xfffa3942); /* 33 */ 205 | HH (d, a, b, c, x[ 8], S32, 0x8771f681); /* 34 */ 206 | HH (c, d, a, b, x[11], S33, 0x6d9d6122); /* 35 */ 207 | HH (b, c, d, a, x[14], S34, 0xfde5380c); /* 36 */ 208 | HH (a, b, c, d, x[ 1], S31, 0xa4beea44); /* 37 */ 209 | HH (d, a, b, c, x[ 4], S32, 0x4bdecfa9); /* 38 */ 210 | HH (c, d, a, b, x[ 7], S33, 0xf6bb4b60); /* 39 */ 211 | HH (b, c, d, a, x[10], S34, 0xbebfbc70); /* 40 */ 212 | HH (a, b, c, d, x[13], S31, 0x289b7ec6); /* 41 */ 213 | HH (d, a, b, c, x[ 0], S32, 0xeaa127fa); /* 42 */ 214 | HH (c, d, a, b, x[ 3], S33, 0xd4ef3085); /* 43 */ 215 | HH (b, c, d, a, x[ 6], S34, 0x4881d05); /* 44 */ 216 | HH (a, b, c, d, x[ 9], S31, 0xd9d4d039); /* 45 */ 217 | HH (d, a, b, c, x[12], S32, 0xe6db99e5); /* 46 */ 218 | HH (c, d, a, b, x[15], S33, 0x1fa27cf8); /* 47 */ 219 | HH (b, c, d, a, x[ 2], S34, 0xc4ac5665); /* 48 */ 220 | 221 | /* Round 4 */ 222 | II (a, b, c, d, x[ 0], S41, 0xf4292244); /* 49 */ 223 | II (d, a, b, c, x[ 7], S42, 0x432aff97); /* 50 */ 224 | II (c, d, a, b, x[14], S43, 0xab9423a7); /* 51 */ 225 | II (b, c, d, a, x[ 5], S44, 0xfc93a039); /* 52 */ 226 | II (a, b, c, d, x[12], S41, 0x655b59c3); /* 53 */ 227 | II (d, a, b, c, x[ 3], S42, 0x8f0ccc92); /* 54 */ 228 | II (c, d, a, b, x[10], S43, 0xffeff47d); /* 55 */ 229 | II (b, c, d, a, x[ 1], S44, 0x85845dd1); /* 56 */ 230 | II (a, b, c, d, x[ 8], S41, 0x6fa87e4f); /* 57 */ 231 | II (d, a, b, c, x[15], S42, 0xfe2ce6e0); /* 58 */ 232 | II (c, d, a, b, x[ 6], S43, 0xa3014314); /* 59 */ 233 | II (b, c, d, a, x[13], S44, 0x4e0811a1); /* 60 */ 234 | II (a, b, c, d, x[ 4], S41, 0xf7537e82); /* 61 */ 235 | II (d, a, b, c, x[11], S42, 0xbd3af235); /* 62 */ 236 | II (c, d, a, b, x[ 2], S43, 0x2ad7d2bb); /* 63 */ 237 | II (b, c, d, a, x[ 9], S44, 0xeb86d391); /* 64 */ 238 | 239 | state[0] += a; 240 | state[1] += b; 241 | state[2] += c; 242 | state[3] += d; 243 | 244 | // Zeroize sensitive information. 245 | memset(x, 0, sizeof x); 246 | } 247 | 248 | ////////////////////////////// 249 | 250 | // MD5 block update operation. Continues an MD5 message-digest 251 | // operation, processing another message block 252 | void MD5::update(const unsigned char input[], size_type length) 253 | { 254 | // compute number of bytes mod 64 255 | size_type index = count[0] / 8 % blocksize; 256 | 257 | // Update number of bits 258 | if ((count[0] += (length << 3)) < (length << 3)) 259 | count[1]++; 260 | count[1] += (length >> 29); 261 | 262 | // number of bytes we need to fill in buffer 263 | size_type firstpart = 64 - index; 264 | 265 | size_type i; 266 | 267 | // transform as many times as possible. 268 | if (length >= firstpart) 269 | { 270 | // fill buffer first, transform 271 | memcpy(&buffer[index], input, firstpart); 272 | transform(buffer); 273 | 274 | // transform chunks of blocksize (64 bytes) 275 | for (i = firstpart; i + blocksize <= length; i += blocksize) 276 | transform(&input[i]); 277 | 278 | index = 0; 279 | } 280 | else 281 | i = 0; 282 | 283 | // buffer remaining input 284 | memcpy(&buffer[index], &input[i], length-i); 285 | } 286 | 287 | ////////////////////////////// 288 | 289 | // for convenience provide a verson with signed char 290 | void MD5::update(const char input[], size_type length) 291 | { 292 | update((const unsigned char*)input, length); 293 | } 294 | 295 | ////////////////////////////// 296 | 297 | // MD5 finalization. Ends an MD5 message-digest operation, writing the 298 | // the message digest and zeroizing the context. 299 | MD5& MD5::finalize() 300 | { 301 | static unsigned char padding[64] = { 302 | 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 303 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 304 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 305 | }; 306 | 307 | if (!finalized) { 308 | // Save number of bits 309 | unsigned char bits[8]; 310 | encode(bits, count, 8); 311 | 312 | // pad out to 56 mod 64. 313 | size_type index = count[0] / 8 % 64; 314 | size_type padLen = (index < 56) ? (56 - index) : (120 - index); 315 | update(padding, padLen); 316 | 317 | // Append length (before padding) 318 | update(bits, 8); 319 | 320 | // Store state in digest 321 | encode(digest, state, 16); 322 | 323 | // Zeroize sensitive information. 324 | memset(buffer, 0, sizeof buffer); 325 | memset(count, 0, sizeof count); 326 | 327 | finalized=true; 328 | } 329 | 330 | return *this; 331 | } 332 | 333 | ////////////////////////////// 334 | 335 | // return hex representation of digest as string 336 | std::string MD5::hexdigest() const 337 | { 338 | if (!finalized) 339 | return ""; 340 | 341 | char buf[33]; 342 | for (int i=0; i<16; i++) 343 | sprintf(buf+i*2, "%02x", digest[i]); 344 | buf[32]=0; 345 | 346 | return std::string(buf); 347 | } 348 | 349 | ////////////////////////////// 350 | 351 | std::ostream& operator<<(std::ostream& out, MD5 md5) 352 | { 353 | return out << md5.hexdigest(); 354 | } 355 | 356 | ////////////////////////////// 357 | 358 | std::string md5(const std::string str) 359 | { 360 | MD5 md5 = MD5(str); 361 | 362 | return md5.hexdigest(); 363 | } 364 | 365 | -------------------------------------------------------------------------------- /md5.h: -------------------------------------------------------------------------------- 1 | /* MD5 2 | converted to C++ class by Frank Thilo (thilo@unix-ag.org) 3 | for bzflag (http://www.bzflag.org) 4 | 5 | based on: 6 | 7 | md5.h and md5.c 8 | reference implementation of RFC 1321 9 | 10 | Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All 11 | rights reserved. 12 | 13 | License to copy and use this software is granted provided that it 14 | is identified as the "RSA Data Security, Inc. MD5 Message-Digest 15 | Algorithm" in all material mentioning or referencing this software 16 | or this function. 17 | 18 | License is also granted to make and use derivative works provided 19 | that such works are identified as "derived from the RSA Data 20 | Security, Inc. MD5 Message-Digest Algorithm" in all material 21 | mentioning or referencing the derived work. 22 | 23 | RSA Data Security, Inc. makes no representations concerning either 24 | the merchantability of this software or the suitability of this 25 | software for any particular purpose. It is provided "as is" 26 | without express or implied warranty of any kind. 27 | 28 | These notices must be retained in any copies of any part of this 29 | documentation and/or software. 30 | 31 | */ 32 | 33 | #ifndef BZF_MD5_H 34 | #define BZF_MD5_H 35 | 36 | #include 37 | #include 38 | 39 | 40 | // a small class for calculating MD5 hashes of strings or byte arrays 41 | // it is not meant to be fast or secure 42 | // 43 | // usage: 1) feed it blocks of uchars with update() 44 | // 2) finalize() 45 | // 3) get hexdigest() string 46 | // or 47 | // MD5(std::string).hexdigest() 48 | // 49 | // assumes that char is 8 bit and int is 32 bit 50 | class MD5 51 | { 52 | public: 53 | typedef unsigned int size_type; // must be 32bit 54 | 55 | MD5(); 56 | MD5(const std::string& text); 57 | void update(const unsigned char *buf, size_type length); 58 | void update(const char *buf, size_type length); 59 | MD5& finalize(); 60 | std::string hexdigest() const; 61 | friend std::ostream& operator<<(std::ostream&, MD5 md5); 62 | 63 | private: 64 | void init(); 65 | typedef unsigned char uint1; // 8bit 66 | typedef unsigned int uint4; // 32bit 67 | enum {blocksize = 64}; // VC6 won't eat a const static int here 68 | 69 | void transform(const uint1 block[blocksize]); 70 | static void decode(uint4 output[], const uint1 input[], size_type len); 71 | static void encode(uint1 output[], const uint4 input[], size_type len); 72 | 73 | bool finalized; 74 | uint1 buffer[blocksize]; // bytes that didn't fit in last 64 byte chunk 75 | uint4 count[2]; // 64bit counter for number of bits (lo, hi) 76 | uint4 state[4]; // digest so far 77 | uint1 digest[16]; // the result 78 | 79 | // low level logic operations 80 | static inline uint4 F(uint4 x, uint4 y, uint4 z); 81 | static inline uint4 G(uint4 x, uint4 y, uint4 z); 82 | static inline uint4 H(uint4 x, uint4 y, uint4 z); 83 | static inline uint4 I(uint4 x, uint4 y, uint4 z); 84 | static inline uint4 rotate_left(uint4 x, int n); 85 | static inline void FF(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac); 86 | static inline void GG(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac); 87 | static inline void HH(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac); 88 | static inline void II(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac); 89 | }; 90 | 91 | std::string md5(const std::string str); 92 | 93 | #endif 94 | -------------------------------------------------------------------------------- /parser.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #ifndef JSONNET_PARSER_H 18 | #define JSONNET_PARSER_H 19 | 20 | #include 21 | 22 | #include "ast.h" 23 | #include "lexer.h" 24 | #include "unicode.h" 25 | 26 | /** Parse a given JSON++ string. 27 | * 28 | * \param alloc Used to allocate the AST nodes. The Allocator must outlive the 29 | * AST pointer returned. 30 | * \param tokens The list of tokens (all tokens are popped except EOF). 31 | * \returns The parsed abstract syntax tree. 32 | */ 33 | AST *jsonnet_parse(Allocator *alloc, Tokens &tokens); 34 | 35 | /** Outputs a number, trying to preserve precision as well as possible. 36 | */ 37 | std::string jsonnet_unparse_number(double v); 38 | 39 | /** The inverse of jsonnet_parse. 40 | */ 41 | std::string jsonnet_unparse_jsonnet(const AST *ast, const Fodder &final_fodder, unsigned indent, 42 | bool pad_arrays, bool pad_objects, char comment_style); 43 | 44 | #endif // JSONNET_PARSER_H 45 | -------------------------------------------------------------------------------- /pass.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include "pass.h" 18 | 19 | void CompilerPass::fodder(Fodder &fodder) 20 | { 21 | for (auto &f : fodder) 22 | fodderElement(f); 23 | } 24 | 25 | void CompilerPass::specs(std::vector &specs) 26 | { 27 | for (auto &spec : specs) { 28 | fodder(spec.openFodder); 29 | switch (spec.kind) { 30 | case ComprehensionSpec::FOR: 31 | fodder(spec.varFodder); 32 | fodder(spec.inFodder); 33 | expr(spec.expr); 34 | break; 35 | case ComprehensionSpec::IF: expr(spec.expr); break; 36 | } 37 | } 38 | } 39 | 40 | void CompilerPass::params(Fodder &fodder_l, ArgParams ¶ms, Fodder &fodder_r) 41 | { 42 | fodder(fodder_l); 43 | for (auto ¶m : params) { 44 | fodder(param.idFodder); 45 | if (param.expr) { 46 | fodder(param.eqFodder); 47 | expr(param.expr); 48 | } 49 | fodder(param.commaFodder); 50 | } 51 | fodder(fodder_r); 52 | } 53 | 54 | void CompilerPass::fieldParams(ObjectField &field) 55 | { 56 | if (field.methodSugar) { 57 | params(field.fodderL, field.params, field.fodderR); 58 | } 59 | } 60 | 61 | void CompilerPass::fields(ObjectFields &fields) 62 | { 63 | for (auto &field : fields) { 64 | switch (field.kind) { 65 | case ObjectField::LOCAL: { 66 | fodder(field.fodder1); 67 | fodder(field.fodder2); 68 | fieldParams(field); 69 | fodder(field.opFodder); 70 | expr(field.expr2); 71 | } break; 72 | 73 | case ObjectField::FIELD_ID: 74 | case ObjectField::FIELD_STR: 75 | case ObjectField::FIELD_EXPR: { 76 | if (field.kind == ObjectField::FIELD_ID) { 77 | fodder(field.fodder1); 78 | 79 | } else if (field.kind == ObjectField::FIELD_STR) { 80 | expr(field.expr1); 81 | 82 | } else if (field.kind == ObjectField::FIELD_EXPR) { 83 | fodder(field.fodder1); 84 | expr(field.expr1); 85 | fodder(field.fodder2); 86 | } 87 | fieldParams(field); 88 | fodder(field.opFodder); 89 | expr(field.expr2); 90 | 91 | } break; 92 | 93 | case ObjectField::ASSERT: { 94 | fodder(field.fodder1); 95 | expr(field.expr2); 96 | if (field.expr3 != nullptr) { 97 | fodder(field.opFodder); 98 | expr(field.expr3); 99 | } 100 | } break; 101 | } 102 | 103 | fodder(field.commaFodder); 104 | } 105 | } 106 | 107 | void CompilerPass::expr(AST *&ast_) 108 | { 109 | fodder(ast_->openFodder); 110 | visitExpr(ast_); 111 | } 112 | 113 | void CompilerPass::visit(Apply *ast) 114 | { 115 | expr(ast->target); 116 | params(ast->fodderL, ast->args, ast->fodderR); 117 | if (ast->tailstrict) { 118 | fodder(ast->tailstrictFodder); 119 | } 120 | } 121 | 122 | void CompilerPass::visit(ApplyBrace *ast) 123 | { 124 | expr(ast->left); 125 | expr(ast->right); 126 | } 127 | 128 | void CompilerPass::visit(Array *ast) 129 | { 130 | for (auto &element : ast->elements) { 131 | expr(element.expr); 132 | fodder(element.commaFodder); 133 | } 134 | fodder(ast->closeFodder); 135 | } 136 | 137 | void CompilerPass::visit(ArrayComprehension *ast) 138 | { 139 | expr(ast->body); 140 | fodder(ast->commaFodder); 141 | specs(ast->specs); 142 | fodder(ast->closeFodder); 143 | } 144 | 145 | void CompilerPass::visit(Assert *ast) 146 | { 147 | expr(ast->cond); 148 | if (ast->message != nullptr) { 149 | fodder(ast->colonFodder); 150 | expr(ast->message); 151 | } 152 | fodder(ast->semicolonFodder); 153 | expr(ast->rest); 154 | } 155 | 156 | void CompilerPass::visit(Binary *ast) 157 | { 158 | expr(ast->left); 159 | fodder(ast->opFodder); 160 | expr(ast->right); 161 | } 162 | 163 | void CompilerPass::visit(Conditional *ast) 164 | { 165 | expr(ast->cond); 166 | fodder(ast->thenFodder); 167 | if (ast->branchFalse != nullptr) { 168 | expr(ast->branchTrue); 169 | fodder(ast->elseFodder); 170 | expr(ast->branchFalse); 171 | } else { 172 | expr(ast->branchTrue); 173 | } 174 | } 175 | 176 | void CompilerPass::visit(Error *ast) 177 | { 178 | expr(ast->expr); 179 | } 180 | 181 | void CompilerPass::visit(Function *ast) 182 | { 183 | params(ast->parenLeftFodder, ast->params, ast->parenRightFodder); 184 | expr(ast->body); 185 | } 186 | 187 | void CompilerPass::visit(Import *ast) 188 | { 189 | visit(ast->file); 190 | } 191 | 192 | void CompilerPass::visit(Importstr *ast) 193 | { 194 | visit(ast->file); 195 | } 196 | 197 | void CompilerPass::visit(InSuper *ast) 198 | { 199 | expr(ast->element); 200 | } 201 | 202 | void CompilerPass::visit(Index *ast) 203 | { 204 | expr(ast->target); 205 | if (ast->id != nullptr) { 206 | } else { 207 | if (ast->isSlice) { 208 | if (ast->index != nullptr) 209 | expr(ast->index); 210 | if (ast->end != nullptr) 211 | expr(ast->end); 212 | if (ast->step != nullptr) 213 | expr(ast->step); 214 | } else { 215 | expr(ast->index); 216 | } 217 | } 218 | } 219 | 220 | void CompilerPass::visit(Local *ast) 221 | { 222 | assert(ast->binds.size() > 0); 223 | for (auto &bind : ast->binds) { 224 | fodder(bind.varFodder); 225 | if (bind.functionSugar) { 226 | params(bind.parenLeftFodder, bind.params, bind.parenRightFodder); 227 | } 228 | fodder(bind.opFodder); 229 | expr(bind.body); 230 | fodder(bind.closeFodder); 231 | } 232 | expr(ast->body); 233 | } 234 | 235 | void CompilerPass::visit(Object *ast) 236 | { 237 | fields(ast->fields); 238 | fodder(ast->closeFodder); 239 | } 240 | 241 | void CompilerPass::visit(DesugaredObject *ast) 242 | { 243 | for (AST *assert : ast->asserts) { 244 | expr(assert); 245 | } 246 | for (auto &field : ast->fields) { 247 | expr(field.name); 248 | expr(field.body); 249 | } 250 | } 251 | 252 | void CompilerPass::visit(ObjectComprehension *ast) 253 | { 254 | fields(ast->fields); 255 | specs(ast->specs); 256 | fodder(ast->closeFodder); 257 | } 258 | 259 | void CompilerPass::visit(ObjectComprehensionSimple *ast) 260 | { 261 | expr(ast->field); 262 | expr(ast->value); 263 | expr(ast->array); 264 | } 265 | 266 | void CompilerPass::visit(Parens *ast) 267 | { 268 | expr(ast->expr); 269 | fodder(ast->closeFodder); 270 | } 271 | 272 | void CompilerPass::visit(SuperIndex *ast) 273 | { 274 | if (ast->id != nullptr) { 275 | } else { 276 | expr(ast->index); 277 | } 278 | } 279 | 280 | void CompilerPass::visit(Unary *ast) 281 | { 282 | expr(ast->expr); 283 | } 284 | 285 | #define VISIT(var,astType,astClass) \ 286 | case astType: { \ 287 | assert(dynamic_cast(var)); \ 288 | auto *ast = static_cast(var); \ 289 | visit(ast); \ 290 | } break 291 | 292 | void CompilerPass::visitExpr(AST *&ast_) 293 | { 294 | switch(ast_->type) { 295 | VISIT(ast_, AST_APPLY, Apply); 296 | VISIT(ast_, AST_APPLY_BRACE, ApplyBrace); 297 | VISIT(ast_, AST_ARRAY, Array); 298 | VISIT(ast_, AST_ARRAY_COMPREHENSION, ArrayComprehension); 299 | // VISIT(ast_, AST_ARRAY_COMPREHENSION, ArrayComprehensionSimple); 300 | VISIT(ast_, AST_ASSERT, Assert); 301 | VISIT(ast_, AST_BINARY, Binary); 302 | VISIT(ast_, AST_BUILTIN_FUNCTION, BuiltinFunction); 303 | VISIT(ast_, AST_CONDITIONAL, Conditional); 304 | VISIT(ast_, AST_DESUGARED_OBJECT, DesugaredObject); 305 | VISIT(ast_, AST_DOLLAR, Dollar); 306 | VISIT(ast_, AST_ERROR, Error); 307 | VISIT(ast_, AST_FUNCTION, Function); 308 | VISIT(ast_, AST_IMPORT, Import); 309 | VISIT(ast_, AST_IMPORTSTR, Importstr); 310 | VISIT(ast_, AST_INDEX, Index); 311 | VISIT(ast_, AST_IN_SUPER, InSuper); 312 | VISIT(ast_, AST_LITERAL_BOOLEAN, LiteralBoolean); 313 | VISIT(ast_, AST_LITERAL_NULL, LiteralNull); 314 | VISIT(ast_, AST_LITERAL_NUMBER, LiteralNumber); 315 | VISIT(ast_, AST_LITERAL_STRING, LiteralString); 316 | VISIT(ast_, AST_LOCAL, Local); 317 | VISIT(ast_, AST_OBJECT, Object); 318 | VISIT(ast_, AST_OBJECT_COMPREHENSION, ObjectComprehension); 319 | VISIT(ast_, AST_OBJECT_COMPREHENSION_SIMPLE, ObjectComprehensionSimple); 320 | VISIT(ast_, AST_PARENS, Parens); 321 | VISIT(ast_, AST_SELF, Self); 322 | VISIT(ast_, AST_SUPER_INDEX, SuperIndex); 323 | VISIT(ast_, AST_UNARY, Unary); 324 | VISIT(ast_, AST_VAR, Var); 325 | default: 326 | std::cerr << "INTERNAL ERROR: Unknown AST: " << ast_ << std::endl; 327 | std::abort(); 328 | break; 329 | } 330 | } 331 | 332 | void CompilerPass::file(AST *&body, Fodder &final_fodder) 333 | { 334 | expr(body); 335 | fodder(final_fodder); 336 | } 337 | 338 | /** A pass that clones the AST it is given. */ 339 | class ClonePass : public CompilerPass { 340 | public: 341 | ClonePass(Allocator &alloc) : CompilerPass(alloc) {} 342 | virtual void expr(AST *&ast); 343 | }; 344 | 345 | #define CLONE(var,astType,astClass) \ 346 | case astType: { \ 347 | assert(dynamic_cast(var)); \ 348 | auto *ast = static_cast(var); \ 349 | var = alloc.clone(ast); \ 350 | } break 351 | 352 | void ClonePass::expr(AST *&ast_) 353 | { 354 | switch(ast_->type) { 355 | CLONE(ast_, AST_APPLY, Apply); 356 | CLONE(ast_, AST_APPLY_BRACE, ApplyBrace); 357 | CLONE(ast_, AST_ARRAY, Array); 358 | CLONE(ast_, AST_ARRAY_COMPREHENSION, ArrayComprehension); 359 | // CLONE(ast_, AST_ARRAY_COMPREHENSION, ArrayComprehensionSimple); 360 | CLONE(ast_, AST_ASSERT, Assert); 361 | CLONE(ast_, AST_BINARY, Binary); 362 | CLONE(ast_, AST_BUILTIN_FUNCTION, BuiltinFunction); 363 | CLONE(ast_, AST_CONDITIONAL, Conditional); 364 | CLONE(ast_, AST_DESUGARED_OBJECT, DesugaredObject); 365 | CLONE(ast_, AST_DOLLAR, Dollar); 366 | CLONE(ast_, AST_ERROR, Error); 367 | CLONE(ast_, AST_FUNCTION, Function); 368 | CLONE(ast_, AST_IMPORT, Import); 369 | CLONE(ast_, AST_IMPORTSTR, Importstr); 370 | CLONE(ast_, AST_INDEX, Index); 371 | CLONE(ast_, AST_IN_SUPER, InSuper); 372 | CLONE(ast_, AST_LITERAL_BOOLEAN, LiteralBoolean); 373 | CLONE(ast_, AST_LITERAL_NULL, LiteralNull); 374 | CLONE(ast_, AST_LITERAL_NUMBER, LiteralNumber); 375 | CLONE(ast_, AST_LITERAL_STRING, LiteralString); 376 | CLONE(ast_, AST_LOCAL, Local); 377 | CLONE(ast_, AST_OBJECT, Object); 378 | CLONE(ast_, AST_OBJECT_COMPREHENSION, ObjectComprehension); 379 | CLONE(ast_, AST_OBJECT_COMPREHENSION_SIMPLE, ObjectComprehensionSimple); 380 | CLONE(ast_, AST_PARENS, Parens); 381 | CLONE(ast_, AST_SELF, Self); 382 | CLONE(ast_, AST_SUPER_INDEX, SuperIndex); 383 | CLONE(ast_, AST_UNARY, Unary); 384 | CLONE(ast_, AST_VAR, Var); 385 | default: 386 | std::cerr << "INTERNAL ERROR: Unknown AST: " << ast_ << std::endl; 387 | std::abort(); 388 | break; 389 | } 390 | 391 | CompilerPass::expr(ast_); 392 | } 393 | 394 | AST *clone_ast(Allocator &alloc, AST *ast) 395 | { 396 | AST *r = ast; 397 | ClonePass(alloc).expr(r); 398 | return r; 399 | } 400 | -------------------------------------------------------------------------------- /pass.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #ifndef JSONNET_PASS_H 18 | #define JSONNET_PASS_H 19 | 20 | #include "ast.h" 21 | 22 | /** A generic Pass that does nothing but can be extended to easily define real passes. 23 | */ 24 | class CompilerPass { 25 | public: 26 | protected: 27 | Allocator &alloc; 28 | 29 | public: 30 | CompilerPass(Allocator &alloc) : alloc(alloc) {} 31 | 32 | virtual void fodderElement(FodderElement &) {} 33 | 34 | virtual void fodder(Fodder &fodder); 35 | 36 | virtual void specs(std::vector &specs); 37 | 38 | virtual void params(Fodder &fodder_l, ArgParams ¶ms, Fodder &fodder_r); 39 | 40 | virtual void fieldParams(ObjectField &field); 41 | 42 | virtual void fields(ObjectFields &fields); 43 | 44 | virtual void expr(AST *&ast_); 45 | 46 | virtual void visit(Apply *ast); 47 | 48 | virtual void visit(ApplyBrace *ast); 49 | 50 | virtual void visit(Array *ast); 51 | 52 | virtual void visit(ArrayComprehension *ast); 53 | 54 | virtual void visit(Assert *ast); 55 | 56 | virtual void visit(Binary *ast); 57 | 58 | virtual void visit(BuiltinFunction *) {} 59 | 60 | virtual void visit(Conditional *ast); 61 | 62 | virtual void visit(Dollar *) {} 63 | 64 | virtual void visit(Error *ast); 65 | 66 | virtual void visit(Function *ast); 67 | 68 | virtual void visit(Import *ast); 69 | 70 | virtual void visit(Importstr *ast); 71 | 72 | virtual void visit(InSuper *ast); 73 | 74 | virtual void visit(Index *ast); 75 | 76 | virtual void visit(Local *ast); 77 | 78 | virtual void visit(LiteralBoolean *) {} 79 | 80 | virtual void visit(LiteralNumber *) {} 81 | 82 | virtual void visit(LiteralString *) {} 83 | 84 | virtual void visit(LiteralNull *) {} 85 | 86 | virtual void visit(Object *ast); 87 | 88 | virtual void visit(DesugaredObject *ast); 89 | 90 | virtual void visit(ObjectComprehension *ast); 91 | 92 | virtual void visit(ObjectComprehensionSimple *ast); 93 | 94 | virtual void visit(Parens *ast); 95 | 96 | virtual void visit(Self *) {} 97 | 98 | virtual void visit(SuperIndex *ast); 99 | 100 | virtual void visit(Unary *ast); 101 | 102 | virtual void visit(Var *) {} 103 | 104 | virtual void visitExpr(AST *&ast_); 105 | 106 | virtual void file(AST *&body, Fodder &final_fodder); 107 | }; 108 | 109 | /** Return an equivalent AST that can be modified without affecting the original. 110 | * 111 | * This is a deep copy. 112 | */ 113 | AST *clone_ast(Allocator &alloc, AST *ast); 114 | 115 | #endif 116 | -------------------------------------------------------------------------------- /state.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #ifndef JSONNET_STATE_H 18 | #define JSONNET_STATE_H 19 | 20 | namespace { 21 | 22 | /** Mark & sweep: advanced by 1 each GC cycle. 23 | */ 24 | typedef unsigned char GarbageCollectionMark; 25 | 26 | /** Supertype of everything that is allocated on the heap. 27 | */ 28 | struct HeapEntity { 29 | enum Type : unsigned char { 30 | THUNK, 31 | ARRAY, 32 | CLOSURE, 33 | STRING, 34 | SIMPLE_OBJECT, 35 | COMPREHENSION_OBJECT, 36 | EXTENDED_OBJECT, 37 | }; 38 | GarbageCollectionMark mark; 39 | Type type; 40 | HeapEntity(Type type_) : type(type_) {} 41 | virtual ~HeapEntity() {} 42 | }; 43 | 44 | /** Tagged union of all values. 45 | * 46 | * Primitives (<= 8 bytes) are copied by value. Otherwise a pointer to a HeapEntity is used. 47 | */ 48 | struct Value { 49 | enum Type { 50 | NULL_TYPE = 0x0, // Unfortunately NULL is a macro in C. 51 | BOOLEAN = 0x1, 52 | NUMBER = 0x2, 53 | 54 | ARRAY = 0x10, 55 | FUNCTION = 0x11, 56 | OBJECT = 0x12, 57 | STRING = 0x13 58 | }; 59 | Type t; 60 | union { 61 | HeapEntity *h; 62 | double d; 63 | bool b; 64 | } v; 65 | bool isHeap(void) const 66 | { 67 | return t & 0x10; 68 | } 69 | }; 70 | 71 | /** Convert the type into a string, for error messages. */ 72 | std::string type_str(Value::Type t) 73 | { 74 | switch (t) { 75 | case Value::NULL_TYPE: return "null"; 76 | case Value::BOOLEAN: return "boolean"; 77 | case Value::NUMBER: return "number"; 78 | case Value::ARRAY: return "array"; 79 | case Value::FUNCTION: return "function"; 80 | case Value::OBJECT: return "object"; 81 | case Value::STRING: return "string"; 82 | default: 83 | std::cerr << "INTERNAL ERROR: Unknown type: " << t << std::endl; 84 | std::abort(); 85 | return ""; // Quiet, compiler. 86 | } 87 | } 88 | 89 | /** Convert the value's type into a string, for error messages. */ 90 | std::string type_str(const Value &v) 91 | { 92 | return type_str(v.t); 93 | } 94 | 95 | struct HeapThunk; 96 | 97 | /** Stores the values bound to variables. 98 | * 99 | * Each nested local statement, function call, and field access has its own binding frame to 100 | * give the values for the local variable, function parameters, or upValues. 101 | */ 102 | typedef std::map BindingFrame; 103 | 104 | /** Supertype of all objects. Types of Value::OBJECT will point at these. */ 105 | struct HeapObject : public HeapEntity { 106 | HeapObject(Type type) : HeapEntity(type) {} 107 | }; 108 | 109 | /** Hold an unevaluated expression. This implements lazy semantics. 110 | */ 111 | struct HeapThunk : public HeapEntity { 112 | /** Whether or not the thunk was forced. */ 113 | bool filled; 114 | 115 | /** The result when the thunk was forced, if filled == true. */ 116 | Value content; 117 | 118 | /** Used in error tracebacks. */ 119 | const Identifier *name; 120 | 121 | /** The captured environment. 122 | * 123 | * Note, this is non-const because we have to add cyclic references to it. 124 | */ 125 | BindingFrame upValues; 126 | 127 | /** The captured self variable, or nullptr if there was none. \see CallFrame. */ 128 | HeapObject *self; 129 | 130 | /** The offset from the captured self variable. \see CallFrame. */ 131 | unsigned offset; 132 | 133 | /** Evaluated to force the thunk. */ 134 | const AST *body; 135 | 136 | HeapThunk(const Identifier *name, HeapObject *self, unsigned offset, const AST *body) 137 | : HeapEntity(THUNK), filled(false), name(name), self(self), offset(offset), body(body) 138 | { 139 | } 140 | 141 | void fill(const Value &v) 142 | { 143 | content = v; 144 | filled = true; 145 | self = nullptr; 146 | upValues.clear(); 147 | } 148 | }; 149 | 150 | struct HeapArray : public HeapEntity { 151 | // It is convenient for this to not be const, so that we can add elements to it one at a 152 | // time after creation. Thus, elements are not GCed as the array is being 153 | // created. 154 | std::vector elements; 155 | HeapArray(const std::vector &elements) 156 | : HeapEntity(ARRAY), elements(elements) 157 | { 158 | } 159 | }; 160 | 161 | /** Supertype of all objects that are not super objects or extended objects. */ 162 | struct HeapLeafObject : public HeapObject { 163 | HeapLeafObject(Type type) : HeapObject(type) {} 164 | }; 165 | 166 | /** Objects created via the simple object constructor construct. */ 167 | struct HeapSimpleObject : public HeapLeafObject { 168 | /** The captured environment. */ 169 | const BindingFrame upValues; 170 | 171 | struct Field { 172 | /** Will the field appear in output? */ 173 | ObjectField::Hide hide; 174 | /** Expression that is evaluated when indexing this field. */ 175 | AST *body; 176 | }; 177 | 178 | /** The fields. 179 | * 180 | * These are evaluated in the captured environment and with self and super bound 181 | * dynamically. 182 | */ 183 | const std::map fields; 184 | 185 | /** The object's invariants. 186 | * 187 | * These are evaluated in the captured environment with self and super bound. 188 | */ 189 | ASTs asserts; 190 | 191 | HeapSimpleObject(const BindingFrame &up_values, 192 | const std::map fields, ASTs asserts) 193 | : HeapLeafObject(SIMPLE_OBJECT), upValues(up_values), fields(fields), asserts(asserts) 194 | { 195 | } 196 | }; 197 | 198 | /** Objects created by the + construct. */ 199 | struct HeapExtendedObject : public HeapObject { 200 | /** The left hand side of the construct. */ 201 | HeapObject *left; 202 | 203 | /** The right hand side of the construct. */ 204 | HeapObject *right; 205 | 206 | HeapExtendedObject(HeapObject *left, HeapObject *right) 207 | : HeapObject(EXTENDED_OBJECT), left(left), right(right) 208 | { 209 | } 210 | }; 211 | 212 | /** Objects created by the ObjectComprehensionSimple construct. */ 213 | struct HeapComprehensionObject : public HeapLeafObject { 214 | /** The captured environment. */ 215 | const BindingFrame upValues; 216 | 217 | /** The expression used to compute the field values. */ 218 | const AST *value; 219 | 220 | /** The identifier of bound variable in that construct. */ 221 | const Identifier *const id; 222 | 223 | /** Binding for id. 224 | * 225 | * For each field, holds the value that should be bound to id. This is the corresponding 226 | * array element from the original array used to define this object. This should not really 227 | * be a thunk, but it makes the implementation easier. 228 | * 229 | * It is convenient to make this non-const to allow building up the values one by one, so that 230 | * the garbage collector can see them at each intermediate point. 231 | */ 232 | std::map compValues; 233 | 234 | HeapComprehensionObject(const BindingFrame &up_values, const AST *value, const Identifier *id, 235 | const std::map &comp_values) 236 | : HeapLeafObject(COMPREHENSION_OBJECT), upValues(up_values), value(value), id(id), compValues(comp_values) 237 | { 238 | } 239 | }; 240 | 241 | /** Stores the function itself and also the captured environment. 242 | * 243 | * Either body is non-null and builtinName is "", or body is null and builtin refers to a built-in 244 | * function. In the former case, the closure represents a user function, otherwise calling it 245 | * will trigger the builtin function to execute. Params is empty when the function is a 246 | * builtin. 247 | */ 248 | struct HeapClosure : public HeapEntity { 249 | /** The captured environment. */ 250 | const BindingFrame upValues; 251 | /** The captured self variable, or nullptr if there was none. \see Frame. */ 252 | HeapObject *self; 253 | /** The offset from the captured self variable. \see Frame.*/ 254 | unsigned offset; 255 | struct Param { 256 | const Identifier *id; 257 | const AST *def; 258 | Param(const Identifier *id, const AST *def) : id(id), def(def) {} 259 | }; 260 | typedef std::vector Params; 261 | const Params params; 262 | const AST *body; 263 | std::string builtinName; 264 | HeapClosure(const BindingFrame &up_values, HeapObject *self, unsigned offset, 265 | const Params ¶ms, const AST *body, const std::string &builtin_name) 266 | : HeapEntity(CLOSURE), 267 | upValues(up_values), 268 | self(self), 269 | offset(offset), 270 | params(params), 271 | body(body), 272 | builtinName(builtin_name) 273 | { 274 | } 275 | }; 276 | 277 | /** Stores a simple string on the heap. */ 278 | struct HeapString : public HeapEntity { 279 | const UString value; 280 | HeapString(const UString &value) : HeapEntity(STRING), value(value) {} 281 | }; 282 | 283 | /** The heap does memory management, i.e. garbage collection. */ 284 | class Heap { 285 | /** How many objects must exist in the heap before we bother doing garbage collection? 286 | */ 287 | unsigned gcTuneMinObjects; 288 | 289 | /** How much must the heap have grown since the last cycle to trigger a collection? 290 | */ 291 | double gcTuneGrowthTrigger; 292 | 293 | /** Value used to mark entities at the last garbage collection cycle. */ 294 | GarbageCollectionMark lastMark; 295 | 296 | /** The heap entities (strings, arrays, objects, functions, etc). 297 | * 298 | * Not all may be reachable, all should have o->mark == this->lastMark. Entities are 299 | * removed from the heap via O(1) swap with last element, so the ordering of entities is 300 | * arbitrary and changes every garbage collection cycle. 301 | */ 302 | std::vector entities; 303 | 304 | /** The number of heap entities at the last garbage collection cycle. */ 305 | unsigned long lastNumEntities; 306 | 307 | /** The number of heap entities now. */ 308 | unsigned long numEntities; 309 | 310 | /** Add the HeapEntity inside v to vec, if the value exists on the heap. 311 | */ 312 | void addIfHeapEntity(Value v, std::vector &vec) 313 | { 314 | if (v.isHeap()) 315 | vec.push_back(v.v.h); 316 | } 317 | 318 | /** Add the HeapEntity inside v to vec, if the value exists on the heap. 319 | */ 320 | void addIfHeapEntity(HeapEntity *v, std::vector &vec) 321 | { 322 | vec.push_back(v); 323 | } 324 | 325 | public: 326 | Heap(unsigned gc_tune_min_objects, double gc_tune_growth_trigger) 327 | : gcTuneMinObjects(gc_tune_min_objects), 328 | gcTuneGrowthTrigger(gc_tune_growth_trigger), 329 | lastMark(0), 330 | lastNumEntities(0), 331 | numEntities(0) 332 | { 333 | } 334 | 335 | ~Heap(void) 336 | { 337 | // Nothing is marked, everything will be collected. 338 | sweep(); 339 | } 340 | 341 | /** Garbage collection: Mark v, and entities reachable from v. */ 342 | void markFrom(Value v) 343 | { 344 | if (v.isHeap()) 345 | markFrom(v.v.h); 346 | } 347 | 348 | /** Garbage collection: Mark heap entities reachable from the given heap entity. */ 349 | void markFrom(HeapEntity *from) 350 | { 351 | assert(from != nullptr); 352 | const GarbageCollectionMark thisMark = lastMark + 1; 353 | struct State { 354 | HeapEntity *ent; 355 | std::vector children; 356 | State(HeapEntity *ent) : ent(ent) {} 357 | }; 358 | 359 | std::vector stack; 360 | stack.emplace_back(from); 361 | 362 | while (stack.size() > 0) { 363 | size_t curr_index = stack.size() - 1; 364 | State &s = stack[curr_index]; 365 | HeapEntity *curr = s.ent; 366 | if (curr->mark != thisMark) { 367 | curr->mark = thisMark; 368 | 369 | switch(curr->type) { 370 | case HeapEntity::SIMPLE_OBJECT: { 371 | assert(dynamic_cast(curr)); 372 | auto *obj = static_cast(curr); 373 | for (auto upv : obj->upValues) 374 | addIfHeapEntity(upv.second, s.children); 375 | break; 376 | } 377 | case HeapEntity::EXTENDED_OBJECT: { 378 | assert(dynamic_cast(curr)); 379 | auto *obj = static_cast(curr); 380 | addIfHeapEntity(obj->left, s.children); 381 | addIfHeapEntity(obj->right, s.children); 382 | break; 383 | } 384 | case HeapEntity::COMPREHENSION_OBJECT: { 385 | assert(dynamic_cast(curr)); 386 | auto *obj = static_cast(curr); 387 | for (auto upv : obj->upValues) 388 | addIfHeapEntity(upv.second, s.children); 389 | for (auto upv : obj->compValues) 390 | addIfHeapEntity(upv.second, s.children); 391 | break; 392 | } 393 | case HeapEntity::ARRAY: { 394 | assert(dynamic_cast(curr)); 395 | auto *arr = static_cast(curr); 396 | for (auto el : arr->elements) 397 | addIfHeapEntity(el, s.children); 398 | break; 399 | } 400 | case HeapEntity::CLOSURE: { 401 | assert(dynamic_cast(curr)); 402 | auto *func = static_cast(curr); 403 | for (auto upv : func->upValues) 404 | addIfHeapEntity(upv.second, s.children); 405 | if (func->self) 406 | addIfHeapEntity(func->self, s.children); 407 | break; 408 | } 409 | case HeapEntity::THUNK: { 410 | assert(dynamic_cast(curr)); 411 | auto *thunk = static_cast(curr); 412 | if (thunk->filled) { 413 | if (thunk->content.isHeap()) 414 | addIfHeapEntity(thunk->content.v.h, s.children); 415 | } else { 416 | for (auto upv : thunk->upValues) 417 | addIfHeapEntity(upv.second, s.children); 418 | if (thunk->self) 419 | addIfHeapEntity(thunk->self, s.children); 420 | } 421 | break; 422 | } 423 | case HeapEntity::STRING: 424 | assert(dynamic_cast(curr)); 425 | break; 426 | default: 427 | assert(false); 428 | break; 429 | } 430 | } 431 | 432 | if (s.children.size() > 0) { 433 | HeapEntity *next = s.children[s.children.size() - 1]; 434 | s.children.pop_back(); 435 | stack.emplace_back(next); // CAUTION: s invalidated here 436 | } else { 437 | stack.pop_back(); // CAUTION: s invalidated here 438 | } 439 | } 440 | } 441 | 442 | /** Delete everything that was not marked since the last collection. */ 443 | void sweep(void) 444 | { 445 | lastMark++; 446 | // Heap shrinks during this loop. Do not cache entities.size(). 447 | for (unsigned long i = 0; i < entities.size(); ++i) { 448 | HeapEntity *x = entities[i]; 449 | if (x->mark != lastMark) { 450 | delete x; 451 | if (i != entities.size() - 1) { 452 | // Swap it with the back. 453 | entities[i] = entities[entities.size() - 1]; 454 | } 455 | entities.pop_back(); 456 | --i; 457 | } 458 | } 459 | lastNumEntities = numEntities = entities.size(); 460 | } 461 | 462 | /** Is it time to initiate a GC cycle? */ 463 | bool checkHeap(void) 464 | { 465 | return numEntities > gcTuneMinObjects && 466 | numEntities > gcTuneGrowthTrigger * lastNumEntities; 467 | } 468 | 469 | /** Allocate a heap entity. 470 | * 471 | * If the heap is large enough (\see gcTuneMinObjects) and has grown by enough since the 472 | * last collection cycle (\see gcTuneGrowthTrigger), a collection cycle is performed. 473 | */ 474 | template 475 | T *makeEntity(Args &&... args) 476 | { 477 | T *r = new T(std::forward(args)...); 478 | entities.push_back(r); 479 | r->mark = lastMark; 480 | numEntities = entities.size(); 481 | return r; 482 | } 483 | }; 484 | 485 | } // namespace 486 | 487 | #endif // JSONNET_STATE_H 488 | -------------------------------------------------------------------------------- /static_analysis.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include 18 | 19 | #include "ast.h" 20 | #include "static_analysis.h" 21 | #include "static_error.h" 22 | 23 | typedef std::set IdSet; 24 | 25 | /** Inserts all of s into r. */ 26 | static void append(IdSet &r, const IdSet &s) 27 | { 28 | r.insert(s.begin(), s.end()); 29 | } 30 | 31 | /** Statically analyse the given ast. 32 | * 33 | * \param ast_ The AST. 34 | * \param in_object Whether or not ast_ is within the lexical scope of an object AST. 35 | * \param vars The variables defined within lexical scope of ast_. 36 | * \returns The free variables in ast_. 37 | */ 38 | static IdSet static_analysis(AST *ast_, bool in_object, const IdSet &vars) 39 | { 40 | IdSet r; 41 | 42 | switch (ast_->type) { 43 | case AST_APPLY: { 44 | assert(dynamic_cast(ast_)); 45 | auto* ast = static_cast(ast_); 46 | append(r, static_analysis(ast->target, in_object, vars)); 47 | for (const auto &arg : ast->args) 48 | append(r, static_analysis(arg.expr, in_object, vars)); 49 | } break; 50 | case AST_APPLY_BRACE: { 51 | assert(dynamic_cast(ast_)); 52 | // Nothing to do. 53 | } break; 54 | case AST_ARRAY: { 55 | assert(dynamic_cast(ast_)); 56 | auto* ast = static_cast(ast_); 57 | for (auto &el : ast->elements) 58 | append(r, static_analysis(el.expr, in_object, vars)); 59 | } break; 60 | case AST_BINARY: { 61 | assert(dynamic_cast(ast_)); 62 | auto* ast = static_cast(ast_); 63 | append(r, static_analysis(ast->left, in_object, vars)); 64 | append(r, static_analysis(ast->right, in_object, vars)); 65 | } break; 66 | case AST_BUILTIN_FUNCTION: { 67 | assert(dynamic_cast(ast_)); 68 | // Nothing to do. 69 | } break; 70 | case AST_CONDITIONAL: { 71 | assert(dynamic_cast(ast_)); 72 | auto* ast = static_cast(ast_); 73 | append(r, static_analysis(ast->cond, in_object, vars)); 74 | append(r, static_analysis(ast->branchTrue, in_object, vars)); 75 | append(r, static_analysis(ast->branchFalse, in_object, vars)); 76 | } break; 77 | case AST_ERROR: { 78 | assert(dynamic_cast(ast_)); 79 | auto* ast = static_cast(ast_); 80 | append(r, static_analysis(ast->expr, in_object, vars)); 81 | } break; 82 | case AST_FUNCTION: { 83 | assert(dynamic_cast(ast_)); 84 | auto* ast = static_cast(ast_); 85 | auto new_vars = vars; 86 | IdSet params; 87 | for (const auto &p : ast->params) { 88 | if (params.find(p.id) != params.end()) { 89 | std::string msg = "Duplicate function parameter: " + encode_utf8(p.id->name); 90 | throw StaticError(ast_->location, msg); 91 | } 92 | params.insert(p.id); 93 | new_vars.insert(p.id); 94 | } 95 | 96 | auto fv = static_analysis(ast->body, in_object, new_vars); 97 | for (const auto &p : ast->params) { 98 | if (p.expr != nullptr) 99 | append(fv, static_analysis(p.expr, in_object, new_vars)); 100 | } 101 | for (const auto &p : ast->params) 102 | fv.erase(p.id); 103 | append(r, fv); 104 | } break; 105 | case AST_IMPORT: { 106 | assert(dynamic_cast(ast_)); 107 | // Nothing to do. 108 | } break; 109 | case AST_IMPORTSTR: { 110 | assert(dynamic_cast(ast_)); 111 | // Nothing to do. 112 | } break; 113 | case AST_IN_SUPER: { 114 | assert(dynamic_cast(ast_)); 115 | auto* ast = static_cast(ast_); 116 | if (!in_object) 117 | throw StaticError(ast_->location, "Can't use super outside of an object."); 118 | append(r, static_analysis(ast->element, in_object, vars)); 119 | } break; 120 | case AST_INDEX: { 121 | assert(dynamic_cast(ast_)); 122 | auto* ast = static_cast(ast_); 123 | append(r, static_analysis(ast->target, in_object, vars)); 124 | append(r, static_analysis(ast->index, in_object, vars)); 125 | } break; 126 | case AST_LOCAL: { 127 | assert(dynamic_cast(ast_)); 128 | auto* ast = static_cast(ast_); 129 | IdSet ast_vars; 130 | for (const auto &bind : ast->binds) { 131 | ast_vars.insert(bind.var); 132 | } 133 | auto new_vars = vars; 134 | append(new_vars, ast_vars); 135 | IdSet fvs; 136 | for (const auto &bind : ast->binds) { 137 | append(fvs, static_analysis(bind.body, in_object, new_vars)); 138 | } 139 | 140 | append(fvs, static_analysis(ast->body, in_object, new_vars)); 141 | 142 | for (const auto &bind : ast->binds) 143 | fvs.erase(bind.var); 144 | 145 | append(r, fvs); 146 | } break; 147 | case AST_LITERAL_BOOLEAN: { 148 | assert(dynamic_cast(ast_)); 149 | // Nothing to do. 150 | } break; 151 | case AST_LITERAL_NUMBER: { 152 | assert(dynamic_cast(ast_)); 153 | // Nothing to do. 154 | } break; 155 | case AST_LITERAL_STRING: { 156 | assert(dynamic_cast(ast_)); 157 | // Nothing to do. 158 | } break; 159 | case AST_LITERAL_NULL: { 160 | assert(dynamic_cast(ast_)); 161 | // Nothing to do. 162 | } break; 163 | case AST_DESUGARED_OBJECT: { 164 | assert(dynamic_cast(ast_)); 165 | auto* ast = static_cast(ast_); 166 | for (auto &field : ast->fields) { 167 | append(r, static_analysis(field.name, in_object, vars)); 168 | append(r, static_analysis(field.body, true, vars)); 169 | } 170 | for (AST *assert : ast->asserts) { 171 | append(r, static_analysis(assert, true, vars)); 172 | } 173 | } break; 174 | case AST_OBJECT_COMPREHENSION_SIMPLE: { 175 | assert(dynamic_cast(ast_)); 176 | auto* ast = static_cast(ast_); 177 | auto new_vars = vars; 178 | new_vars.insert(ast->id); 179 | append(r, static_analysis(ast->field, false, new_vars)); 180 | append(r, static_analysis(ast->value, true, new_vars)); 181 | r.erase(ast->id); 182 | append(r, static_analysis(ast->array, in_object, vars)); 183 | } break; 184 | case AST_SELF: { 185 | assert(dynamic_cast(ast_)); 186 | if (!in_object) 187 | throw StaticError(ast_->location, "Can't use self outside of an object."); 188 | } break; 189 | case AST_SUPER_INDEX: { 190 | assert(dynamic_cast(ast_)); 191 | auto* ast = static_cast(ast_); 192 | if (!in_object) 193 | throw StaticError(ast_->location, "Can't use super outside of an object."); 194 | append(r, static_analysis(ast->index, in_object, vars)); 195 | } break; 196 | case AST_UNARY: { 197 | assert(dynamic_cast(ast_)); 198 | auto* ast = static_cast(ast_); 199 | append(r, static_analysis(ast->expr, in_object, vars)); 200 | } break; 201 | case AST_VAR: { 202 | assert(dynamic_cast(ast_)); 203 | auto* ast = static_cast(ast_); 204 | if (vars.find(ast->id) == vars.end()) { 205 | throw StaticError(ast->location, "Unknown variable: " + encode_utf8(ast->id->name)); 206 | } 207 | r.insert(ast->id); 208 | } break; 209 | default: 210 | std::cerr << "INTERNAL ERROR: Unknown AST: " << ast_ << std::endl; 211 | std::abort(); 212 | break; 213 | } 214 | 215 | for (auto *id : r) 216 | ast_->freeVariables.push_back(id); 217 | 218 | return r; 219 | } 220 | 221 | void jsonnet_static_analysis(AST *ast) 222 | { 223 | static_analysis(ast, false, IdSet{}); 224 | } 225 | -------------------------------------------------------------------------------- /static_analysis.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #ifndef JSONNET_STATIC_ANALYSIS_H 18 | #define JSONNET_STATIC_ANALYSIS_H 19 | 20 | #include "ast.h" 21 | 22 | /** Check the ast for appropriate use of self, super, and correctly bound variables. Also 23 | * initialize the freeVariables member of function and object ASTs. 24 | */ 25 | void jsonnet_static_analysis(AST *ast); 26 | 27 | #endif 28 | -------------------------------------------------------------------------------- /static_error.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #ifndef JSONNET_STATIC_ERROR_H 18 | #define JSONNET_STATIC_ERROR_H 19 | 20 | #include 21 | #include 22 | 23 | struct Location { 24 | unsigned long line; 25 | unsigned long column; 26 | Location(void) : line(0), column(0) {} 27 | Location(unsigned long line, unsigned long column) : line(line), column(column) {} 28 | bool isSet(void) const 29 | { 30 | return line != 0; 31 | } 32 | Location successor(void) const 33 | { 34 | return Location(this->line, this->column + 1); 35 | } 36 | }; 37 | 38 | static inline std::ostream &operator<<(std::ostream &o, const Location &loc) 39 | { 40 | o << loc.line << ":" << loc.column; 41 | return o; 42 | } 43 | 44 | struct LocationRange { 45 | std::string file; 46 | // [begin, end) 47 | Location begin, end; 48 | LocationRange(void) {} 49 | /** This is useful for special locations, e.g. manifestation entry point. */ 50 | LocationRange(const std::string &msg) : file(msg) {} 51 | LocationRange(const std::string &file, const Location &begin, const Location &end) 52 | : file(file), begin(begin), end(end) 53 | { 54 | } 55 | bool isSet(void) const 56 | { 57 | return begin.isSet(); 58 | } 59 | }; 60 | 61 | static inline std::ostream &operator<<(std::ostream &o, const LocationRange &loc) 62 | { 63 | if (loc.file.length() > 0) 64 | o << loc.file; 65 | if (loc.isSet()) { 66 | if (loc.file.length() > 0) 67 | o << ":"; 68 | if (loc.begin.line == loc.end.line) { 69 | if (loc.begin.column == loc.end.column - 1) { 70 | o << loc.begin; 71 | } else { 72 | o << loc.begin << "-" << loc.end.column; 73 | } 74 | } else { 75 | o << "(" << loc.begin << ")-(" << loc.end << ")"; 76 | } 77 | } 78 | return o; 79 | } 80 | 81 | struct StaticError { 82 | LocationRange location; 83 | std::string msg; 84 | StaticError(const std::string &msg) : msg(msg) {} 85 | StaticError(const std::string &filename, const Location &location, const std::string &msg) 86 | : location(filename, location, location.successor()), msg(msg) 87 | { 88 | } 89 | StaticError(const LocationRange &location, const std::string &msg) 90 | : location(location), msg(msg) 91 | { 92 | } 93 | 94 | std::string toString() const 95 | { 96 | std::stringstream ss; 97 | if (location.isSet()) { 98 | ss << location << ":"; 99 | } 100 | ss << " " << msg; 101 | return ss.str(); 102 | } 103 | }; 104 | 105 | static inline std::ostream &operator<<(std::ostream &o, const StaticError &err) 106 | { 107 | o << err.toString(); 108 | return o; 109 | } 110 | 111 | #endif // JSONNET_ERROR_H 112 | -------------------------------------------------------------------------------- /stress_loop/stress_loop.go: -------------------------------------------------------------------------------- 1 | // A program to stress the JSonnet VM and its cgo wrappings. 2 | // Run `top` in another shell, to see how much we leak memory. 3 | package main 4 | 5 | import ( 6 | "flag" 7 | "fmt" 8 | "log" 9 | "runtime" 10 | "strings" 11 | "time" 12 | 13 | J "github.com/strickyak/jsonnet_cgo" 14 | ) 15 | 16 | type Unit struct{} 17 | 18 | var L = flag.Int("l", 1000, "Repeat the big loop this many times") 19 | var N = flag.Int("n", 100, "Repeat quick operations this many times.") 20 | var P = flag.Int("p", 100, "How many goroutines running a JSonnet vm in parallel.") 21 | var F = fmt.Sprintf 22 | var Quiet = false 23 | 24 | // Say something to the log, unless Quiet. 25 | func Say(format string, args ...interface{}) { 26 | if !Quiet { 27 | log.Printf(format, args...) 28 | } 29 | } 30 | 31 | // Repeat the action so many times. 32 | func Repeat(times int, action func()) { 33 | for i := 0; i < times; i++ { 34 | action() 35 | } 36 | } 37 | 38 | // ExercizeOneVM and then signal it is finished. 39 | // It panics only if something goes really wrong. 40 | func ExercizeOneVM(finished chan Unit) { 41 | vm := J.Make() 42 | vm.JpathAdd("/tmp") 43 | 44 | // concatStringsNative is an instance of NativeCallback. 45 | // It extracts string representations of string, number, bool, or null arguments. 46 | concatStringsNativeRaw := func(args ...*J.JsonValue) (result *J.JsonValue, err error) { 47 | z := "" 48 | for _, a := range args { 49 | z += F("%v", a.Extract()) 50 | } 51 | return vm.NewString(z), nil 52 | } 53 | concat4Strings := func(a, b, c, d string) (string, error) { return a + b + c + d, nil } 54 | 55 | Repeat(*N, func() { vm.MaxStack(999) }) 56 | Repeat(*N, func() { vm.MaxTrace(0) }) 57 | Repeat(*N, func() { vm.GcMinObjects(10) }) 58 | Repeat(*N, func() { vm.GcGrowthTrigger(2.0) }) 59 | Repeat(*N, func() { vm.ExtVar("color", "purple") }) 60 | Repeat(*N, func() { vm.TlaVar("shade", "dark") }) 61 | 62 | // Concerning Native Callbacks, 63 | // See https://gist.github.com/sparkprime/5b2ab0a1b72beceab2cf5ea524db228c 64 | // and https://github.com/google/jsonnet/issues/108 65 | Repeat(*N, func() { vm.NativeCallbackRaw("Concat2", []string{"a", "b"}, concatStringsNativeRaw) }) 66 | Repeat(*N, func() { vm.NativeCallbackRaw("Concat3", []string{"a", "b", "c"}, concatStringsNativeRaw) }) 67 | Repeat(*N, func() { vm.NativeCallback("Concat4", []string{"a", "b", "c", "d"}, concat4Strings) }) 68 | { 69 | got, err := vm.EvaluateSnippet( 70 | "SNIPPET-1", 71 | `function(shade) (std.native("Concat3")(shade, "-", std.extVar("color")))`) 72 | if err != nil { 73 | panic(err) 74 | } 75 | Say("1: %q", got) 76 | want := "\"dark-purple\"\n" 77 | if got != want { 78 | panic(F("got %q wanted %q", got, want)) 79 | } 80 | } 81 | Repeat(*N, func() { vm.TlaCode("amount", "8 * 111") }) 82 | { 83 | got, err := vm.EvaluateSnippet( 84 | "SNIPPET-2", 85 | `function(shade, amount) (amount * 10)`) 86 | if err != nil { 87 | panic(err) 88 | } 89 | Say("1b: %s", got) 90 | want := "8880\n" 91 | if got != want { 92 | panic(F("got %q wanted %q", got, want)) 93 | } 94 | } 95 | Repeat(*N, func() { vm.StringOutput(true) }) 96 | { 97 | // Provoke an error. Pass 3 instead of 4 arguments to std.native("Concat4"). 98 | _, err := vm.EvaluateSnippet( 99 | "SNIPPET-3", 100 | `function(shade, amount) ( 101 | std.native("Concat4")( 102 | std.native("Concat2")(shade, "-"), 103 | "", 104 | std.extVar("color")))`) 105 | if !strings.HasPrefix( 106 | F("%v", err), 107 | "RUNTIME ERROR: Function parameter d not bound in call.") { 108 | panic(F("SNIPPET-3: Got wrong error string prefix: %q", err)) 109 | } 110 | } 111 | { 112 | got, err := vm.EvaluateSnippet( 113 | "SNIPPET-4", 114 | `function(shade, amount) ( 115 | std.native("Concat4")( 116 | std.native("Concat2")(shade, "-"), 117 | "", 118 | "", 119 | std.extVar("color")))`) 120 | if err != nil { 121 | panic(err) 122 | } 123 | Say("2: %s", got) 124 | want := "dark-purple\n" 125 | if got != want { 126 | panic(F("got %q wanted %q", got, want)) 127 | } 128 | } 129 | Repeat(*N, func() { 130 | _, err := vm.FormatSnippet( 131 | "SNIPPET-5", 132 | `{ "hello" : "world" }`) 133 | if err != nil { 134 | panic(err) 135 | } 136 | }) 137 | Repeat(*N, func() { _ = J.Version() }) 138 | vm.Destroy() 139 | finished <- Unit{} 140 | } 141 | 142 | func main() { 143 | flag.Parse() 144 | startMain := time.Now() 145 | for loops := 1; loops <= *L; loops++ { 146 | startLoop := time.Now() 147 | 148 | finished := make(chan Unit, *P) 149 | for i := 0; i < *P; i++ { 150 | go ExercizeOneVM(finished) // Start them in parallel. 151 | } 152 | Say("Waiting...") 153 | for i := 0; i < *P; i++ { 154 | <-finished // Wait for all to finish. 155 | } 156 | 157 | duration := time.Now().Sub(startLoop) 158 | totalDuration := time.Now().Sub(startMain) 159 | log.Printf("Finished %d loops. This loop %.3f sec. Average %.3f sec per loop.", 160 | loops, duration.Seconds(), totalDuration.Seconds()/float64(loops)) 161 | 162 | runtime.GC() // Don't let garbage accumulate, unless it's leaked. 163 | 164 | // Only be verbose on the first pass. 165 | Quiet = true 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /string_utils.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #include 18 | 19 | #include "static_error.h" 20 | #include "string_utils.h" 21 | 22 | UString jsonnet_string_unparse(const UString &str, bool single) 23 | { 24 | UStringStream ss; 25 | ss << (single ? U'\'' : U'\"'); 26 | ss << jsonnet_string_escape(str, single); 27 | ss << (single ? U'\'' : U'\"'); 28 | return ss.str(); 29 | } 30 | 31 | UString jsonnet_string_escape(const UString &str, bool single) 32 | { 33 | UStringStream ss; 34 | for (std::size_t i = 0; i < str.length(); ++i) { 35 | char32_t c = str[i]; 36 | switch (c) { 37 | case U'\"': ss << (single ? U"\"" : U"\\\""); break; 38 | case U'\'': ss << (single ? U"\\\'" : U"\'"); break; 39 | case U'\\': ss << U"\\\\"; break; 40 | case U'\b': ss << U"\\b"; break; 41 | case U'\f': ss << U"\\f"; break; 42 | case U'\n': ss << U"\\n"; break; 43 | case U'\r': ss << U"\\r"; break; 44 | case U'\t': ss << U"\\t"; break; 45 | case U'\0': ss << U"\\u0000"; break; 46 | default: { 47 | if (c < 0x20 || (c >= 0x7f && c <= 0x9f)) { 48 | // Unprintable, use \u 49 | std::stringstream ss8; 50 | ss8 << "\\u" << std::hex << std::setfill('0') << std::setw(4) 51 | << (unsigned long)(c); 52 | ss << decode_utf8(ss8.str()); 53 | } else { 54 | // Printable, write verbatim 55 | ss << c; 56 | } 57 | } 58 | } 59 | } 60 | return ss.str(); 61 | } 62 | 63 | UString jsonnet_string_unescape(const LocationRange &loc, const UString &s) 64 | { 65 | UString r; 66 | const char32_t *s_ptr = s.c_str(); 67 | for (const char32_t *c = s_ptr; *c != U'\0'; ++c) { 68 | switch (*c) { 69 | case '\\': 70 | switch (*(++c)) { 71 | case '"': 72 | case '\'': r += *c; break; 73 | 74 | case '\\': r += *c; break; 75 | 76 | case '/': r += *c; break; 77 | 78 | case 'b': r += '\b'; break; 79 | 80 | case 'f': r += '\f'; break; 81 | 82 | case 'n': r += '\n'; break; 83 | 84 | case 'r': r += '\r'; break; 85 | 86 | case 't': r += '\t'; break; 87 | 88 | case 'u': { 89 | ++c; // Consume the 'u'. 90 | unsigned long codepoint = 0; 91 | // Expect 4 hex digits. 92 | for (unsigned i = 0; i < 4; ++i) { 93 | auto x = (unsigned char)(c[i]); 94 | unsigned digit; 95 | if (x == '\0') { 96 | auto msg = "Truncated unicode escape sequence in string literal."; 97 | throw StaticError(loc, msg); 98 | } else if (x >= '0' && x <= '9') { 99 | digit = x - '0'; 100 | } else if (x >= 'a' && x <= 'f') { 101 | digit = x - 'a' + 10; 102 | } else if (x >= 'A' && x <= 'F') { 103 | digit = x - 'A' + 10; 104 | } else { 105 | std::stringstream ss; 106 | ss << "Malformed unicode escape character, " 107 | << "should be hex: '" << x << "'"; 108 | throw StaticError(loc, ss.str()); 109 | } 110 | codepoint *= 16; 111 | codepoint += digit; 112 | } 113 | 114 | r += codepoint; 115 | 116 | // Leave us on the last char, ready for the ++c at 117 | // the outer for loop. 118 | c += 3; 119 | } break; 120 | 121 | case '\0': { 122 | auto msg = "Truncated escape sequence in string literal."; 123 | throw StaticError(loc, msg); 124 | } 125 | 126 | default: { 127 | std::stringstream ss; 128 | std::string utf8; 129 | encode_utf8(*c, utf8); 130 | ss << "Unknown escape sequence in string literal: '" << utf8 << "'"; 131 | throw StaticError(loc, ss.str()); 132 | } 133 | } 134 | break; 135 | 136 | default: 137 | // Just a regular letter. 138 | r += *c; 139 | } 140 | } 141 | return r; 142 | } 143 | -------------------------------------------------------------------------------- /string_utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #ifndef JSONNET_STRING_H 18 | #define JSONNET_STRING_H 19 | 20 | #include "lexer.h" 21 | 22 | /** Unparse the string. */ 23 | UString jsonnet_string_unparse(const UString &str, bool single); 24 | 25 | // Note that the following two functions do not handle the quoting of ' and " 26 | // inside verbatim strings because that quoting is reversible. Thus, that 27 | // quoting is done at lexing time and undone again at pretty-printing time. 28 | 29 | /** Escape special characters. */ 30 | UString jsonnet_string_escape(const UString &str, bool single); 31 | 32 | /** Resolve escape chracters in the string. */ 33 | UString jsonnet_string_unescape(const LocationRange &loc, const UString &s); 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /test1.j: -------------------------------------------------------------------------------- 1 | { 2 | shell: "/bin/sh", 3 | awk: "/usr/bin/awk", 4 | } 5 | -------------------------------------------------------------------------------- /test2.j: -------------------------------------------------------------------------------- 1 | local test1 = import "test1.j"; 2 | 3 | test1 { 4 | shell: "/bin/csh", 5 | } 6 | -------------------------------------------------------------------------------- /unicode.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #ifndef JSONNET_UNICODE_H 18 | #define JSONNET_UNICODE_H 19 | 20 | /** Substituted when a unicode translation format encoding error is encountered. */ 21 | #define JSONNET_CODEPOINT_ERROR 0xfffd 22 | #define JSONNET_CODEPOINT_MAX 0x110000 23 | 24 | /** Convert a unicode codepoint to UTF8. 25 | * 26 | * \param x The unicode codepoint. 27 | * \param s The UTF-8 string to append to. 28 | * \returns The number of characters appended. 29 | */ 30 | static inline int encode_utf8(char32_t x, std::string &s) 31 | { 32 | if (x >= JSONNET_CODEPOINT_MAX) 33 | x = JSONNET_CODEPOINT_ERROR; 34 | 35 | // 00ZZZzzz 00zzYYYY 00Yyyyxx 00xxxxxx 36 | long bytes = ((x & 0x1C0000) << 6) | ((x & 0x03F000) << 4) | ((x & 0x0FC0) << 2) | (x & 0x3F); 37 | 38 | if (x < 0x80) { 39 | s.push_back((char)x); 40 | return 1; 41 | } else if (x < 0x800) { // note that capital 'Y' bits must be 0 42 | bytes |= 0xC080; 43 | s.push_back((bytes >> 8) & 0xFF); 44 | s.push_back((bytes >> 0) & 0xFF); 45 | return 2; 46 | } else if (x < 0x10000) { // note that 'z' bits must be 0 47 | bytes |= 0xE08080; 48 | s.push_back((bytes >> 16) & 0xFF); 49 | s.push_back((bytes >> 8) & 0xFF); 50 | s.push_back((bytes >> 0) & 0xFF); 51 | return 3; 52 | } else if (x < 0x110000) { // note that capital 'Z' bits must be 0 53 | bytes |= 0xF0808080; 54 | s.push_back((bytes >> 24) & 0xFF); 55 | s.push_back((bytes >> 16) & 0xFF); 56 | s.push_back((bytes >> 8) & 0xFF); 57 | s.push_back((bytes >> 0) & 0xFF); 58 | return 4; 59 | } else { 60 | std::cerr << "Should never get here." << std::endl; 61 | abort(); 62 | } 63 | } 64 | 65 | /** Convert the UTF8 byte sequence in the given string to a unicode code point. 66 | * 67 | * \param str The string. 68 | * \param i The index of the string from which to start decoding and returns the index of the last 69 | * byte of the encoded codepoint. 70 | * \returns The decoded unicode codepoint. 71 | */ 72 | static inline char32_t decode_utf8(const std::string &str, size_t &i) 73 | { 74 | char c0 = str[i]; 75 | if ((c0 & 0x80) == 0) { // 0xxxxxxx 76 | return c0; 77 | } else if ((c0 & 0xE0) == 0xC0) { // 110yyyxx 10xxxxxx 78 | if (i + 1 >= str.length()) { 79 | return JSONNET_CODEPOINT_ERROR; 80 | } 81 | char c1 = str[++i]; 82 | if ((c1 & 0xC0) != 0x80) { 83 | return JSONNET_CODEPOINT_ERROR; 84 | } 85 | return ((c0 & 0x1F) << 6ul) | (c1 & 0x3F); 86 | } else if ((c0 & 0xF0) == 0xE0) { // 1110yyyy 10yyyyxx 10xxxxxx 87 | if (i + 2 >= str.length()) { 88 | return JSONNET_CODEPOINT_ERROR; 89 | } 90 | char c1 = str[++i]; 91 | if ((c1 & 0xC0) != 0x80) { 92 | return JSONNET_CODEPOINT_ERROR; 93 | } 94 | char c2 = str[++i]; 95 | if ((c2 & 0xC0) != 0x80) { 96 | return JSONNET_CODEPOINT_ERROR; 97 | } 98 | return ((c0 & 0xF) << 12ul) | ((c1 & 0x3F) << 6) | (c2 & 0x3F); 99 | } else if ((c0 & 0xF8) == 0xF0) { // 11110zzz 10zzyyyy 10yyyyxx 10xxxxxx 100 | if (i + 3 >= str.length()) { 101 | return JSONNET_CODEPOINT_ERROR; 102 | } 103 | char c1 = str[++i]; 104 | if ((c1 & 0xC0) != 0x80) { 105 | return JSONNET_CODEPOINT_ERROR; 106 | } 107 | char c2 = str[++i]; 108 | if ((c2 & 0xC0) != 0x80) { 109 | return JSONNET_CODEPOINT_ERROR; 110 | } 111 | char c3 = str[++i]; 112 | if ((c3 & 0xC0) != 0x80) { 113 | return JSONNET_CODEPOINT_ERROR; 114 | } 115 | return ((c0 & 0x7) << 24ul) | ((c1 & 0x3F) << 12ul) | ((c2 & 0x3F) << 6) | (c3 & 0x3F); 116 | } else { 117 | return JSONNET_CODEPOINT_ERROR; 118 | } 119 | } 120 | 121 | /** A string class capable of holding unicode codepoints. */ 122 | typedef std::basic_string UString; 123 | 124 | static inline void encode_utf8(const UString &s, std::string &r) 125 | { 126 | for (char32_t cp : s) 127 | encode_utf8(cp, r); 128 | } 129 | 130 | static inline std::string encode_utf8(const UString &s) 131 | { 132 | std::string r; 133 | encode_utf8(s, r); 134 | return r; 135 | } 136 | 137 | static inline UString decode_utf8(const std::string &s) 138 | { 139 | UString r; 140 | for (size_t i = 0; i < s.length(); ++i) 141 | r.push_back(decode_utf8(s, i)); 142 | return r; 143 | } 144 | 145 | /** A stringstream-like class capable of holding unicode codepoints. 146 | * The C++ standard does not support std::basic_stringstream 168 | UStringStream &operator<<(T c) 169 | { 170 | std::stringstream ss; 171 | ss << c; 172 | for (char c : ss.str()) 173 | buf.push_back(char32_t(c)); 174 | return *this; 175 | } 176 | UString str() 177 | { 178 | return buf; 179 | } 180 | }; 181 | 182 | #endif // JSONNET_UNICODE_H 183 | -------------------------------------------------------------------------------- /vm.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2015 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #ifndef JSONNET_VM_H 18 | #define JSONNET_VM_H 19 | 20 | #include 21 | 22 | #include "ast.h" 23 | 24 | /** A single line of a stack trace from a runtime error. 25 | */ 26 | struct TraceFrame { 27 | LocationRange location; 28 | std::string name; 29 | TraceFrame(const LocationRange &location, const std::string &name = "") 30 | : location(location), name(name) 31 | { 32 | } 33 | }; 34 | 35 | /** Exception that is thrown by the interpreter when it reaches an error construct, or divide by 36 | * zero, array bounds error, dynamic type error, etc. 37 | */ 38 | struct RuntimeError { 39 | std::vector stackTrace; 40 | std::string msg; 41 | RuntimeError(const std::vector stack_trace, const std::string &msg) 42 | : stackTrace(stack_trace), msg(msg) 43 | { 44 | } 45 | }; 46 | 47 | /** Holds native callback and context. */ 48 | struct VmNativeCallback { 49 | JsonnetNativeCallback *cb; 50 | void *ctx; 51 | std::vector params; 52 | }; 53 | 54 | typedef std::map VmNativeCallbackMap; 55 | 56 | /** Stores external values / code. */ 57 | struct VmExt { 58 | std::string data; 59 | bool isCode; 60 | VmExt() : isCode(false) {} 61 | VmExt(const std::string &data, bool is_code) : data(data), isCode(is_code) {} 62 | }; 63 | 64 | /** Execute the program and return the value as a JSON string. 65 | * 66 | * \param alloc The allocator used to create the ast. 67 | * \param ast The program to execute. 68 | * \param ext The external vars / code. 69 | * \param max_stack Recursion beyond this level gives an error. 70 | * \param gc_min_objects The garbage collector does not run when the heap is this small. 71 | * \param gc_growth_trigger Growth since last garbage collection cycle to trigger a new cycle. 72 | * \param import_callback A callback to handle imports 73 | * \param import_callback_ctx Context param for the import callback. 74 | * \param output_string Whether to expect a string and output it without JSON encoding 75 | * \throws RuntimeError reports runtime errors in the program. 76 | * \returns The JSON result in string form. 77 | */ 78 | std::string jsonnet_vm_execute(Allocator *alloc, const AST *ast, 79 | const std::map &ext, unsigned max_stack, 80 | double gc_min_objects, double gc_growth_trigger, 81 | const VmNativeCallbackMap &natives, 82 | JsonnetImportCallback *import_callback, void *import_callback_ctx, 83 | bool string_output); 84 | 85 | /** Execute the program and return the value as a number of named JSON files. 86 | * 87 | * This assumes the given program yields an object whose keys are filenames. 88 | * 89 | * \param alloc The allocator used to create the ast. 90 | * \param ast The program to execute. 91 | * \param ext The external vars / code. 92 | * \param tla The top-level arguments (strings or code). 93 | * \param max_stack Recursion beyond this level gives an error. 94 | * \param gc_min_objects The garbage collector does not run when the heap is this small. 95 | * \param gc_growth_trigger Growth since last garbage collection cycle to trigger a new cycle. 96 | * \param import_callback A callback to handle imports 97 | * \param import_callback_ctx Context param for the import callback. 98 | * \param output_string Whether to expect a string and output it without JSON encoding 99 | * \throws RuntimeError reports runtime errors in the program. 100 | * \returns A mapping from filename to the JSON strings for that file. 101 | */ 102 | std::map jsonnet_vm_execute_multi( 103 | Allocator *alloc, const AST *ast, const std::map &ext, unsigned max_stack, 104 | double gc_min_objects, double gc_growth_trigger, const VmNativeCallbackMap &natives, 105 | JsonnetImportCallback *import_callback, void *import_callback_ctx, bool string_output); 106 | 107 | /** Execute the program and return the value as a stream of JSON files. 108 | * 109 | * This assumes the given program yields an array whose elements are individual 110 | * JSON files. 111 | * 112 | * \param alloc The allocator used to create the ast. 113 | * \param ast The program to execute. 114 | * \param ext The external vars / code. 115 | * \param tla The top-level arguments (strings or code). 116 | * \param max_stack Recursion beyond this level gives an error. 117 | * \param gc_min_objects The garbage collector does not run when the heap is this small. 118 | * \param gc_growth_trigger Growth since last garbage collection cycle to trigger a new cycle. 119 | * \param import_callback A callback to handle imports 120 | * \param import_callback_ctx Context param for the import callback. 121 | * \param output_string Whether to expect a string and output it without JSON encoding 122 | * \throws RuntimeError reports runtime errors in the program. 123 | * \returns A mapping from filename to the JSON strings for that file. 124 | */ 125 | std::vector jsonnet_vm_execute_stream( 126 | Allocator *alloc, const AST *ast, const std::map &ext, unsigned max_stack, 127 | double gc_min_objects, double gc_growth_trigger, const VmNativeCallbackMap &natives, 128 | JsonnetImportCallback *import_callback, void *import_callback_ctx); 129 | 130 | #endif 131 | --------------------------------------------------------------------------------