├── .cargo └── config ├── .gitignore ├── .travis.yml ├── Cargo.toml ├── LICENSE-APACHE ├── LICENSE-MIT ├── Makefile ├── README.md ├── cargo_log_parser ├── Cargo.toml └── src │ └── lib │ ├── mod.rs │ └── tests │ ├── fixtures │ ├── check-with-error-parallel.log │ └── check-with-error.log │ └── mod.rs ├── ci └── script.sh ├── discovery_parser ├── Cargo.toml ├── examples │ └── filter_apis.rs ├── src │ ├── discovery.rs │ └── lib.rs └── tests │ ├── spec.json │ └── tests.rs ├── google_api_auth ├── Cargo.toml └── src │ ├── lib.rs │ └── yup_oauth2.rs ├── google_api_bytes ├── Cargo.toml └── src │ └── lib.rs ├── google_cli_generator ├── Cargo.toml ├── build.rs ├── src │ ├── all.rs │ ├── cargo.rs │ ├── cli │ │ ├── liquid_filters.rs │ │ ├── mod.rs │ │ └── model.rs │ ├── lib.rs │ └── util.rs ├── templates │ ├── 01_main.rs.liquid │ └── 02_main.rs.liquid └── tests │ ├── generate.rs │ ├── output │ └── .gitignore │ └── spec.json ├── google_cli_shared ├── Cargo.toml ├── README.md └── src │ └── lib.rs ├── google_field_selector ├── Cargo.toml ├── src │ └── lib.rs └── tests │ └── tests.rs ├── google_field_selector_derive ├── Cargo.toml └── src │ └── lib.rs ├── google_rest_api_generator ├── Cargo.toml ├── build.rs ├── examples │ └── any_api.rs ├── gen_include │ ├── README.md │ ├── error.rs │ ├── iter.rs │ ├── multipart.rs │ ├── parsed_string.rs │ ├── percent_encode_consts.rs │ └── resumable_upload.rs └── src │ ├── cargo.rs │ ├── lib.rs │ ├── markdown.rs │ ├── method_actions.rs │ ├── method_builder.rs │ ├── package_doc.rs │ ├── path_templates.rs │ ├── resource_actions.rs │ └── resource_builder.rs ├── justfile ├── mcp ├── Cargo.toml └── src │ ├── cmds │ ├── cargo_errors.rs │ ├── completions.rs │ ├── fetch_specs.rs │ ├── generate.rs │ ├── map_index.rs │ ├── mod.rs │ ├── substitute.rs │ └── util.rs │ ├── main.rs │ └── options │ ├── cargo_errors.rs │ ├── completions.rs │ ├── fetch_specs.rs │ ├── generate.rs │ ├── map_index.rs │ ├── mod.rs │ └── substitute.rs ├── shared ├── Cargo.toml ├── src │ ├── lib.rs │ └── rustfmt.rs └── tests │ ├── fixtures │ └── known-versions │ ├── shared.rs │ └── snapshots │ └── parse_version__it_works_for_all_known_inputs.snap ├── templating ├── Cargo.toml ├── src │ ├── lib.rs │ ├── liquid.rs │ ├── spec.rs │ └── util.rs └── tests │ └── spec.rs ├── tests └── mcp │ ├── fixtures │ └── substitute │ │ ├── complex.tpl │ │ ├── data-for-complex.tpl.yml │ │ ├── data.json.hbs │ │ ├── partials │ │ ├── base0.hbs │ │ ├── base1.hbs │ │ └── template.hbs │ │ └── the-answer.hbs │ ├── included-stateless-substitute.sh │ ├── journey-tests.sh │ ├── snapshots │ └── substitute │ │ ├── data-stdin-json-data-validated-fix-with-replacements-stdout │ │ ├── data-stdin-json-single-template-stdout │ │ ├── data-stdin-yaml-multi-template-stdout │ │ ├── data-stdin-yaml-multi-template-stdout-explicit-separator │ │ ├── data-stdin-yaml-multi-template-to-same-file │ │ ├── data-stdin-yaml-multi-template-to-same-file-again │ │ ├── data-stdin-yaml-multi-template-to-same-file-again-output │ │ ├── data-stdin-yaml-multi-template-to-same-file-explicit-separator │ │ ├── data-stdin-yaml-multi-template-to-same-file-explicit-separator-output │ │ ├── data-stdin-yaml-multi-template-to-same-file-output │ │ ├── data-stdin-yaml-single-relative-template-stdout │ │ ├── data-stdin-yaml-single-template-file-non-existing-directory │ │ ├── data-stdin-yaml-single-template-output-file-with-nonexisting-directory │ │ └── some │ │ │ └── sub │ │ │ └── directory │ │ │ └── output │ │ ├── data-stdin-yaml-single-template-stdout │ │ ├── fail-data-stdin-and-no-specs │ │ ├── fail-data-stdin-json-data-validated-stdout │ │ ├── fail-data-stdin-template-misses-key │ │ ├── fail-data-stdin-template-stdin │ │ ├── fail-invalid-data-format │ │ ├── fail-invalid-data-format-multi-document-yaml │ │ ├── fail-multiple-templates-from-stdin │ │ ├── fail-no-data-and-no-specs │ │ ├── fail-not-enough-replacements │ │ ├── fail-source-is-destination-single-spec │ │ ├── fail-validation-data-stdin-json-template │ │ ├── liquid │ │ └── filter-base64 │ │ ├── template-from-complex-template │ │ ├── template-stdin-hbs-output-stdout │ │ ├── template-stdin-hbs-output-stdout-to-file │ │ ├── template-stdin-hbs-output-stdout-to-file-output │ │ └── validation-success-data-stdin-json-template │ └── utilities.sh └── uri_template_parser ├── Cargo.toml └── src └── lib.rs /.cargo/config: -------------------------------------------------------------------------------- 1 | [build] 2 | target-dir = "./target" 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The folder containing all generated output 2 | /gen/ 3 | 4 | # Snapshots by insta 5 | *.snap.new 6 | 7 | # Ignore any target directory (not just the one at the project root) 8 | target/ 9 | 10 | # Ignore common editor settings 11 | .vscode/ 12 | .idea/ 13 | 14 | # Created by https://www.gitignore.io/api/rust,linux,macos,windows,vim 15 | # Edit at https://www.gitignore.io/?templates=rust,linux,macos,windows,vim 16 | 17 | ### Linux ### 18 | *~ 19 | 20 | # temporary files which can be created if a process still has a handle open of a deleted file 21 | .fuse_hidden* 22 | 23 | # KDE directory preferences 24 | .directory 25 | 26 | # Linux trash folder which might appear on any partition or disk 27 | .Trash-* 28 | 29 | # .nfs files are created when an open file is removed but is still being accessed 30 | .nfs* 31 | 32 | ### macOS ### 33 | # General 34 | .DS_Store 35 | .AppleDouble 36 | .LSOverride 37 | 38 | # Icon must end with two \r 39 | Icon 40 | 41 | # Thumbnails 42 | ._* 43 | 44 | # Files that might appear in the root of a volume 45 | .DocumentRevisions-V100 46 | .fseventsd 47 | .Spotlight-V100 48 | .TemporaryItems 49 | .Trashes 50 | .VolumeIcon.icns 51 | .com.apple.timemachine.donotpresent 52 | 53 | # Directories potentially created on remote AFP share 54 | .AppleDB 55 | .AppleDesktop 56 | Network Trash Folder 57 | Temporary Items 58 | .apdisk 59 | 60 | ### Rust ### 61 | # Generated by Cargo 62 | # will have compiled files and executables 63 | /target/ 64 | 65 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 66 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 67 | Cargo.lock 68 | 69 | # These are backup files generated by rustfmt 70 | **/*.rs.bk 71 | 72 | ### Vim ### 73 | # Swap 74 | [._]*.s[a-v][a-z] 75 | [._]*.sw[a-p] 76 | [._]s[a-rt-v][a-z] 77 | [._]ss[a-gi-z] 78 | [._]sw[a-p] 79 | 80 | # Session 81 | Session.vim 82 | Sessionx.vim 83 | 84 | # Temporary 85 | .netrwhist 86 | 87 | # Auto-generated tag files 88 | tags 89 | 90 | # Persistent undo 91 | [._]*.un~ 92 | 93 | # Coc configuration directory 94 | .vim 95 | 96 | ### Windows ### 97 | # Windows thumbnail cache files 98 | Thumbs.db 99 | Thumbs.db:encryptable 100 | ehthumbs.db 101 | ehthumbs_vista.db 102 | 103 | # Dump file 104 | *.stackdump 105 | 106 | # Folder config file 107 | [Dd]esktop.ini 108 | 109 | # Recycle Bin used on file shares 110 | $RECYCLE.BIN/ 111 | 112 | # Windows Installer files 113 | *.cab 114 | *.msi 115 | *.msix 116 | *.msm 117 | *.msp 118 | 119 | # Windows shortcuts 120 | *.lnk 121 | 122 | # End of https://www.gitignore.io/api/rust,linux,macos,windows,vim 123 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: rust 2 | script: 3 | ./ci/script.sh 4 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [profile.release] 2 | opt-level = 3 3 | # the flags below reduce binary size, but increase build time. Let's keep the latter low for now. 4 | #lto = true 5 | #codegen-units = 1 6 | debug = false 7 | rpath = false 8 | debug-assertions = false 9 | 10 | [workspace] 11 | 12 | members = [ 13 | "discovery_parser", 14 | "google_api_auth", 15 | "google_api_bytes", 16 | "google_field_selector", 17 | "google_field_selector_derive", 18 | "google_rest_api_generator", 19 | "google_cli_generator", 20 | "uri_template_parser", 21 | "cargo_log_parser", 22 | "shared", 23 | "templating", 24 | "mcp", 25 | ] 26 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Permission is hereby granted, free of charge, to any person obtaining a copy 2 | of this software and associated documentation files (the "Software"), to deal 3 | in the Software without restriction, including without limitation the rights 4 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 5 | copies of the Software, and to permit persons to whom the Software is 6 | furnished to do so, subject to the following conditions: 7 | 8 | The above copyright notice and this permission notice shall be included in 9 | all copies or substantial portions of the Software. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 12 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 13 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 14 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 15 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 16 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 17 | SOFTWARE. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | help: 2 | $(info Please use 'just' instead) 3 | $(info just --list | list all targets) 4 | $(info See how to get it here: https://github.com/casey/just#installation) 5 | -------------------------------------------------------------------------------- /cargo_log_parser/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "cargo_log_parser" 3 | version = "0.1.0" 4 | authors = ["Sebastian Thiel "] 5 | edition = "2018" 6 | 7 | [lib] 8 | path = "src/lib/mod.rs" 9 | 10 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 11 | 12 | [dependencies] 13 | nom = "5.0.1" 14 | -------------------------------------------------------------------------------- /cargo_log_parser/src/lib/mod.rs: -------------------------------------------------------------------------------- 1 | use nom::{ 2 | branch::alt, 3 | bytes::{streaming::tag, streaming::take_till, streaming::take_till1}, 4 | character::streaming::line_ending, 5 | combinator::{complete, map, map_parser, map_res, opt}, 6 | multi::fold_many0, 7 | sequence::{delimited, terminated, tuple}, 8 | IResult, 9 | }; 10 | use std::convert::TryFrom; 11 | use std::string::FromUtf8Error; 12 | 13 | #[derive(Clone, Debug, PartialEq, Eq)] 14 | pub struct CrateWithError { 15 | pub name: String, 16 | } 17 | 18 | impl TryFrom<&[u8]> for CrateWithError { 19 | type Error = FromUtf8Error; 20 | 21 | fn try_from(name: &[u8]) -> Result { 22 | Ok(CrateWithError { 23 | name: String::from_utf8(name.to_owned())?, 24 | }) 25 | } 26 | } 27 | 28 | fn quoted_name(input: &[u8]) -> IResult<&[u8], &[u8]> { 29 | let backtick = || tag(b"`"); 30 | delimited(backtick(), take_till1(|b| b == b'`'), backtick())(input) 31 | } 32 | 33 | #[derive(Clone, Debug, PartialEq, Eq)] 34 | enum Line { 35 | Other, 36 | Error(CrateWithError), 37 | } 38 | 39 | impl From for Line { 40 | fn from(c: CrateWithError) -> Self { 41 | Line::Error(c) 42 | } 43 | } 44 | 45 | impl From<&[u8]> for Line { 46 | fn from(_: &[u8]) -> Self { 47 | Line::Other 48 | } 49 | } 50 | 51 | pub fn parse_errors(input: &[u8]) -> IResult<&[u8], Vec> { 52 | fold_many0( 53 | |i: &[u8]| { 54 | if i.len() == 0 { 55 | return Err(nom::Err::Error((i, nom::error::ErrorKind::Eof))); 56 | } 57 | opt(alt(( 58 | map(complete(line_with_error), Line::from), 59 | map(line_without_ending, Line::from), 60 | )))(i) 61 | }, 62 | Vec::new(), 63 | |mut acc, c| { 64 | if let Some(Line::Error(c)) = c { 65 | acc.push(c); 66 | } 67 | acc 68 | }, 69 | )(input) 70 | } 71 | 72 | fn is_newline(b: u8) -> bool { 73 | b == b'\n' || b == b'\r' 74 | } 75 | 76 | fn line_without_ending(input: &[u8]) -> IResult<&[u8], &[u8]> { 77 | terminated(take_till(is_newline), line_ending)(input) 78 | } 79 | 80 | fn line_with_error(input: &[u8]) -> IResult<&[u8], CrateWithError> { 81 | map_parser( 82 | line_without_ending, 83 | map_res( 84 | tuple((tag(b"error:"), take_till1(|b| b == b'`'), quoted_name)), 85 | |(_, _, name)| CrateWithError::try_from(name), 86 | ), 87 | )(input) 88 | } 89 | 90 | #[cfg(test)] 91 | mod tests; 92 | -------------------------------------------------------------------------------- /cargo_log_parser/src/lib/tests/fixtures/check-with-error-parallel.log: -------------------------------------------------------------------------------- 1 | Checking google-calendar3 v0.1.0 (/Users/sthiel/dev/google-apis/generated/gen/calendar/v3/lib) 2 | Checking google-groupsmigration1 v0.1.0 (/Users/sthiel/dev/google-apis/generated/gen/groupsmigration/v1/lib) 3 | Checking google-oslogin1_beta v0.1.0 (/Users/sthiel/dev/google-apis/generated/gen/oslogin/v1beta/lib) 4 | Checking google-dns2_beta1 v0.1.0 (/Users/sthiel/dev/google-apis/generated/gen/dns/v2beta1/lib) 5 | Checking google-dataproc1 v0.1.0 (/Users/sthiel/dev/google-apis/generated/gen/dataproc/v1/lib) 6 | Checking google-videointelligence1_p3beta1 v0.1.0 (/Users/sthiel/dev/google-apis/generated/gen/videointelligence/v1p3beta1/lib) 7 | Checking google-compute1 v0.1.0 (/Users/sthiel/dev/google-apis/generated/gen/compute/v1/lib) 8 | Checking google-pagespeedonline5 v0.1.0 (/Users/sthiel/dev/google-apis/generated/gen/pagespeedonline/v5/lib) 9 | Checking google-videointelligence1_beta2 v0.1.0 (/Users/sthiel/dev/google-apis/generated/gen/videointelligence/v1beta2/lib) 10 | Checking google-videointelligence1_p2beta1 v0.1.0 (/Users/sthiel/dev/google-apis/generated/gen/videointelligence/v1p2beta1/lib) 11 | Checking google-oauth2 v0.1.0 (/Users/sthiel/dev/google-apis/generated/gen/oauth/v2/lib) 12 | Checking google-container1_beta1 v0.1.0 (/Users/sthiel/dev/google-apis/generated/gen/container/v1beta1/lib) 13 | error[E0609]: no field `request` on type `resources::archive::InsertRequestBuilder<'a, A>` 14 | --> groupsmigration/v1/lib/src/lib.rs:210:63 15 | | 16 | 210 | let request_json = ::serde_json::to_vec(&self.request)?; 17 | | ^^^^^^^ help: a field with a similar name exists: `reqwest` 18 | 19 | error: aborting due to previous error 20 | 21 | For more information about this error, try `rustc --explain E0609`. 22 | error: Could not compile `google-groupsmigration1`. 23 | warning: build failed, waiting for other jobs to finish... 24 | error[E0412]: cannot find type `GetCertForOpenIdConnectRequestBuilder` in this scope 25 | --> oauth/v2/lib/src/lib.rs:241:51 26 | | 27 | 241 | pub fn get_cert_for_open_id_connect(&self) -> GetCertForOpenIdConnectRequestBuilder { 28 | | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ not found in this scope 29 | help: possible candidate is found in another module, you can import it into scope 30 | | 31 | 1 | use crate::resources::GetCertForOpenIdConnectRequestBuilder; 32 | | 33 | 34 | error[E0422]: cannot find struct, variant or union type `GetCertForOpenIdConnectRequestBuilder` in this scope 35 | --> oauth/v2/lib/src/lib.rs:242:9 36 | | 37 | 242 | GetCertForOpenIdConnectRequestBuilder { 38 | | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ not found in this scope 39 | help: possible candidate is found in another module, you can import it into scope 40 | | 41 | 1 | use crate::resources::GetCertForOpenIdConnectRequestBuilder; 42 | | 43 | 44 | error[E0412]: cannot find type `TokeninfoRequestBuilder` in this scope 45 | --> oauth/v2/lib/src/lib.rs:255:32 46 | | 47 | 255 | pub fn tokeninfo(&self) -> TokeninfoRequestBuilder { 48 | | ^^^^^^^^^^^^^^^^^^^^^^^ not found in this scope 49 | help: possible candidate is found in another module, you can import it into scope 50 | | 51 | 1 | use crate::resources::TokeninfoRequestBuilder; 52 | | 53 | 54 | error[E0422]: cannot find struct, variant or union type `TokeninfoRequestBuilder` in this scope 55 | --> oauth/v2/lib/src/lib.rs:256:9 56 | | 57 | 256 | TokeninfoRequestBuilder { 58 | | ^^^^^^^^^^^^^^^^^^^^^^^ not found in this scope 59 | help: possible candidate is found in another module, you can import it into scope 60 | | 61 | 1 | use crate::resources::TokeninfoRequestBuilder; 62 | | 63 | 64 | error[E0412]: cannot find type `RunpagespeedCategoryItems` in module `crate::resources::pagespeedapi::params` 65 | --> pagespeedonline/v5/lib/src/lib.rs:737:68 66 | | 67 | 737 | Option>, 68 | | ^^^^^^^^^^^^^^^^^^^^^^^^^ not found in `crate::resources::pagespeedapi::params` 69 | 70 | error[E0412]: cannot find type `RunpagespeedCategoryItems` in module `crate::resources::pagespeedapi::params` 71 | --> pagespeedonline/v5/lib/src/lib.rs:754:78 72 | | 73 | 754 | value: impl Into>, 74 | | ^^^^^^^^^^^^^^^^^^^^^^^^^ not found in `crate::resources::pagespeedapi::params` 75 | 76 | error: aborting due to 4 previous errors 77 | 78 | Some errors have detailed explanations: E0412, E0422. 79 | For more information about an error, try `rustc --explain E0412`. 80 | error: Could not compile `google-oauth2`. 81 | warning: build failed, waiting for other jobs to finish... 82 | error: aborting due to 2 previous errors 83 | 84 | For more information about this error, try `rustc --explain E0412`. 85 | error: Could not compile `google-pagespeedonline5`. 86 | warning: build failed, waiting for other jobs to finish... 87 | error: build failed 88 | -------------------------------------------------------------------------------- /cargo_log_parser/src/lib/tests/mod.rs: -------------------------------------------------------------------------------- 1 | fn assert_complete_error(res: Result>) { 2 | panic!( 3 | "Not complete: {}", 4 | match res { 5 | Ok(t) => format!("{:?}", t), 6 | Err(nom::Err::Failure(_)) | Err(nom::Err::Error(_)) => return, 7 | Err(e) => format!("{:?}", e), 8 | } 9 | ) 10 | } 11 | 12 | fn assert_incomplete_error(res: Result>) { 13 | panic!( 14 | "Not incomplete: {}", 15 | match res { 16 | Ok(t) => format!("{:?}", t), 17 | Err(nom::Err::Incomplete(_)) => return, 18 | Err(e) => format!("{:?}", e), 19 | } 20 | ) 21 | } 22 | 23 | mod quoted { 24 | use super::super::quoted_name; 25 | use super::assert_incomplete_error; 26 | #[test] 27 | fn it_works_on_valid_input() { 28 | assert_eq!( 29 | quoted_name(b"`hello-there1`"), 30 | Ok((&b""[..], &b"hello-there1"[..])) 31 | ); 32 | } 33 | #[test] 34 | fn fails_on_partial_input() { 35 | assert_incomplete_error(quoted_name(b"`hello-")) 36 | } 37 | } 38 | 39 | mod line { 40 | use crate::line_without_ending; 41 | use crate::tests::assert_incomplete_error; 42 | 43 | #[test] 44 | fn it_succeeds_on_valid_input() { 45 | assert_eq!( 46 | line_without_ending(b"foo\n").unwrap(), 47 | (&b""[..], &b"foo"[..]) 48 | ); 49 | } 50 | 51 | #[test] 52 | fn it_needs_a_complete_line() { 53 | assert_incomplete_error(line_without_ending(b"foo")) 54 | } 55 | } 56 | 57 | mod parse_errors { 58 | use crate::{parse_errors, CrateWithError}; 59 | static CARGO_ERRORS: &[u8] = include_bytes!("./fixtures/check-with-error.log"); 60 | static CARGO_ERRORS_PARALLEL: &[u8] = 61 | include_bytes!("./fixtures/check-with-error-parallel.log"); 62 | 63 | #[test] 64 | fn it_succeeds_on_valid_sequential_input() { 65 | assert_eq!( 66 | parse_errors(CARGO_ERRORS).unwrap(), 67 | ( 68 | &b""[..], 69 | vec![ 70 | CrateWithError { name: "!".into() }, 71 | CrateWithError { 72 | name: "google-urlshortener1".into() 73 | } 74 | ] 75 | ) 76 | ); 77 | } 78 | #[test] 79 | fn it_succeeds_on_valid_parallel_input() { 80 | assert_eq!( 81 | parse_errors(CARGO_ERRORS_PARALLEL).unwrap(), 82 | ( 83 | &b""[..], 84 | vec![ 85 | CrateWithError { 86 | name: "google-groupsmigration1".into() 87 | }, 88 | CrateWithError { 89 | name: "google-oauth2".into() 90 | }, 91 | CrateWithError { 92 | name: "google-pagespeedonline5".into() 93 | } 94 | ] 95 | ) 96 | ); 97 | } 98 | } 99 | 100 | mod error_line { 101 | use super::super::line_with_error; 102 | use crate::tests::{assert_complete_error, assert_incomplete_error}; 103 | use crate::CrateWithError; 104 | 105 | #[test] 106 | fn it_succeeds_and_parses_the_correct_crate_name_on_valid_input() { 107 | assert_eq!( 108 | line_with_error(&b"error: Could not compile `google-groupsmigration1`.\n"[..]), 109 | Ok(( 110 | &b""[..], 111 | CrateWithError { 112 | name: "google-groupsmigration1".into() 113 | } 114 | )) 115 | ); 116 | assert_incomplete_error(line_with_error( 117 | &b"error: Could not compile `google-groupsmigration1`"[..], 118 | )); 119 | assert_incomplete_error(line_with_error(&b"error: Could not "[..])); 120 | assert_incomplete_error(line_with_error(&b"err"[..])); 121 | } 122 | 123 | #[test] 124 | fn it_fails_on_invalid_input() { 125 | assert_complete_error(line_with_error( 126 | b" Checking google-videointelligence1_p3beta1 v0.1.0 (/Users/some/lib)\n", 127 | )); 128 | 129 | assert_incomplete_error(line_with_error( 130 | b" Checking google-videointelligence1_p3beta1 v0.1.0 (/Users/s", 131 | )); 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /ci/script.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # This script takes care of testing your crate 3 | set -eux -o pipefail 4 | 5 | 6 | curl -LSfs https://japaric.github.io/trust/install.sh | \ 7 | sh -s -- --git casey/just --target x86_64-unknown-linux-musl --force 8 | 9 | just tests 10 | 11 | [ -d generated ] || git clone --depth=1 https://github.com/google-apis-rs/generated 12 | 13 | cd generated 14 | export MCP=$PWD/../target/debug/mcp 15 | ./ci/script.sh 16 | -------------------------------------------------------------------------------- /discovery_parser/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "discovery_parser" 3 | version = "0.1.0" 4 | authors = ["Glenn Griffin "] 5 | edition = "2018" 6 | 7 | [lib] 8 | doctest = false 9 | 10 | [features] 11 | # Don't allow unknown fields in serde parsing. Useful in development to ensure 12 | # no features are being missed accidentally. 13 | strict = [] 14 | 15 | [dependencies] 16 | serde = { version = "1", features = ["derive"] } 17 | 18 | [dev-dependencies] 19 | reqwest = "0.9.18" 20 | serde_json = "1" 21 | futures = "0.1" 22 | tokio = "0.1" 23 | -------------------------------------------------------------------------------- /discovery_parser/examples/filter_apis.rs: -------------------------------------------------------------------------------- 1 | use discovery_parser::DiscoveryRestDesc; 2 | use reqwest; 3 | use serde::Deserialize; 4 | 5 | #[derive(Debug, Deserialize)] 6 | #[serde(rename_all = "camelCase")] 7 | struct ApiList { 8 | items: Vec, 9 | } 10 | 11 | #[derive(Debug, Deserialize)] 12 | #[serde(rename_all = "camelCase")] 13 | struct ApiSpec { 14 | name: String, 15 | discovery_rest_url: String, 16 | } 17 | 18 | fn count_resources<'a>( 19 | resources: impl Iterator, 20 | ) -> usize { 21 | resources 22 | .map(|resource| { 23 | let sub_resources: usize = count_resources(resource.resources.values()); 24 | 1 + sub_resources 25 | }) 26 | .sum() 27 | } 28 | 29 | fn count_methods<'a>(resources: impl Iterator) -> usize { 30 | resources 31 | .map(|resource| { 32 | let sub_methods: usize = count_methods(resource.resources.values()); 33 | resource.methods.len() + sub_methods 34 | }) 35 | .sum() 36 | } 37 | 38 | fn main() -> Result<(), Box> { 39 | use discovery_parser::{AuthDesc, Oauth2Desc}; 40 | for_each_api(|rest_desc| { 41 | if let Some(AuthDesc { 42 | oauth2: Oauth2Desc { scopes }, 43 | }) = &rest_desc.auth 44 | { 45 | for scope in scopes.keys() { 46 | println!("{} -> {}", scope, const_id_for_scope(&scope)); 47 | } 48 | } 49 | })?; 50 | Ok(()) 51 | } 52 | 53 | fn const_id_for_scope(mut scope: &str) -> String { 54 | const GOOGLE_AUTH_PREFIX: &str = "https://www.googleapis.com/auth/"; 55 | scope = scope.trim_start_matches("https://www.googleapis.com/auth/"); 56 | scope = scope.trim_start_matches("https://"); 57 | scope = scope.trim_end_matches("/"); 58 | let mut scope = scope.replace(&['.', '/', '-'][..], "_"); 59 | scope.make_ascii_uppercase(); 60 | scope 61 | } 62 | 63 | fn for_each_resource(rest_desc: &DiscoveryRestDesc, mut f: F) 64 | where 65 | F: FnMut(&discovery_parser::ResourceDesc), 66 | { 67 | fn per_resource(res: &discovery_parser::ResourceDesc, f: &mut F) 68 | where 69 | F: FnMut(&discovery_parser::ResourceDesc), 70 | { 71 | for sub_resource in res.resources.values() { 72 | per_resource(sub_resource, f); 73 | } 74 | f(res) 75 | } 76 | 77 | for resource in rest_desc.resources.values() { 78 | per_resource(resource, &mut f); 79 | } 80 | } 81 | 82 | fn for_each_api(mut f: F) -> Result<(), Box> 83 | where 84 | F: FnMut(&DiscoveryRestDesc), 85 | { 86 | let client = reqwest::Client::new(); 87 | let all_apis: ApiList = client 88 | .get("https://www.googleapis.com/discovery/v1/apis") 89 | .send()? 90 | .json()?; 91 | println!("There are {} apis", all_apis.items.len()); 92 | for api in all_apis.items { 93 | match get_api(&client, &api.discovery_rest_url) { 94 | Ok(rest_desc) => f(&rest_desc), 95 | Err(err) => eprintln!("Failed to get {}: {}", &api.discovery_rest_url, err), 96 | } 97 | } 98 | Ok(()) 99 | } 100 | 101 | fn get_api( 102 | client: &reqwest::Client, 103 | url: &str, 104 | ) -> Result> { 105 | eprintln!("Fetching {}", url); 106 | Ok(client.get(url).send()?.error_for_status()?.json()?) 107 | } 108 | -------------------------------------------------------------------------------- /discovery_parser/src/discovery.rs: -------------------------------------------------------------------------------- 1 | // Example code that deserializes and serializes the model. 2 | // extern crate serde; 3 | // #[macro_use] 4 | // extern crate serde_derive; 5 | // extern crate serde_json; 6 | // 7 | // use generated_module::[object Object]; 8 | // 9 | // fn main() { 10 | // let json = r#"{"answer": 42}"#; 11 | // let model: [object Object] = serde_json::from_str(&json).unwrap(); 12 | // } 13 | 14 | #[derive(Serialize, Deserialize)] 15 | pub struct ApiIndexV1 { 16 | #[serde(rename = "kind")] 17 | pub kind: String, 18 | 19 | #[serde(rename = "discoveryVersion")] 20 | pub discovery_version: String, 21 | 22 | #[serde(rename = "items")] 23 | pub items: Vec, 24 | } 25 | 26 | #[derive(Serialize, Deserialize)] 27 | pub struct Item { 28 | #[serde(rename = "kind")] 29 | pub kind: Kind, 30 | 31 | #[serde(rename = "id")] 32 | pub id: String, 33 | 34 | #[serde(rename = "name")] 35 | pub name: String, 36 | 37 | #[serde(rename = "version")] 38 | pub version: String, 39 | 40 | #[serde(rename = "title")] 41 | pub title: String, 42 | 43 | #[serde(rename = "description")] 44 | pub description: String, 45 | 46 | #[serde(rename = "discoveryRestUrl")] 47 | pub discovery_rest_url: String, 48 | 49 | #[serde(rename = "icons")] 50 | pub icons: Icons, 51 | 52 | #[serde(rename = "documentationLink")] 53 | pub documentation_link: Option, 54 | 55 | #[serde(rename = "preferred")] 56 | pub preferred: bool, 57 | 58 | #[serde(rename = "discoveryLink")] 59 | pub discovery_link: Option, 60 | 61 | #[serde(rename = "labels")] 62 | pub labels: Option>, 63 | } 64 | 65 | #[derive(Serialize, Deserialize)] 66 | pub struct Icons { 67 | #[serde(rename = "x16")] 68 | pub x16: String, 69 | 70 | #[serde(rename = "x32")] 71 | pub x32: String, 72 | } 73 | 74 | #[derive(Serialize, Deserialize)] 75 | pub enum Kind { 76 | #[serde(rename = "discovery#directoryItem")] 77 | DiscoveryDirectoryItem, 78 | } 79 | 80 | #[derive(Serialize, Deserialize)] 81 | pub enum Label { 82 | #[serde(rename = "labs")] 83 | Labs, 84 | 85 | #[serde(rename = "limited_availability")] 86 | LimitedAvailability, 87 | } 88 | -------------------------------------------------------------------------------- /discovery_parser/src/lib.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | use std::collections::BTreeMap; 3 | 4 | /// The content of this module was generated by [`quicktype`](https://quicktype.io) 5 | pub mod generated { 6 | use serde::{Deserialize, Serialize}; 7 | include!("discovery.rs"); 8 | } 9 | 10 | #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] 11 | #[serde(rename_all = "camelCase")] 12 | pub struct DiscoveryError { 13 | pub error: ErrorMsg, 14 | } 15 | 16 | #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] 17 | #[serde(rename_all = "camelCase")] 18 | pub struct ErrorMsg { 19 | pub code: u32, 20 | pub message: String, 21 | pub status: String, 22 | } 23 | 24 | #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] 25 | #[serde(rename_all = "camelCase", untagged)] 26 | pub enum RestDescOrErr { 27 | RestDesc(DiscoveryRestDesc), 28 | Err(DiscoveryError), 29 | } 30 | 31 | #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] 32 | #[serde(rename_all = "camelCase")] 33 | #[cfg_attr(feature = "strict", serde(deny_unknown_fields))] 34 | pub struct DiscoveryRestDesc { 35 | pub kind: Option, 36 | pub etag: Option, 37 | pub discovery_version: Option, 38 | pub id: String, 39 | pub name: String, 40 | pub canonical_name: Option, 41 | pub fully_encode_reserved_expansion: Option, 42 | pub version: String, 43 | pub revision: String, 44 | pub title: String, 45 | pub description: String, 46 | pub owner_domain: String, 47 | pub owner_name: String, 48 | #[serde(default)] 49 | pub icons: BTreeMap, 50 | pub documentation_link: Option, 51 | pub protocol: String, 52 | pub base_url: String, 53 | pub base_path: String, 54 | pub root_url: String, 55 | pub service_path: String, 56 | pub batch_path: String, 57 | #[serde(rename = "version_module")] 58 | pub version_module: Option, 59 | pub package_path: Option, 60 | pub labels: Option>, 61 | pub features: Option>, 62 | #[serde(default)] 63 | pub parameters: BTreeMap, 64 | pub auth: Option, 65 | #[serde(default)] 66 | pub schemas: BTreeMap, 67 | pub resources: BTreeMap, 68 | #[serde(default)] 69 | pub methods: BTreeMap, 70 | } 71 | 72 | #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] 73 | #[serde(rename_all = "camelCase")] 74 | #[cfg_attr(feature = "strict", serde(deny_unknown_fields))] 75 | pub struct ParamDesc { 76 | pub description: Option, 77 | pub default: Option, 78 | pub location: String, 79 | #[serde(default)] 80 | pub required: bool, 81 | #[serde(rename = "type")] 82 | pub typ: String, 83 | pub format: Option, 84 | pub minimum: Option, 85 | pub maximum: Option, 86 | pub pattern: Option, 87 | #[serde(default, rename = "enum")] 88 | pub enumeration: Vec, 89 | #[serde(default)] 90 | pub enum_descriptions: Vec, 91 | #[serde(default)] 92 | pub repeated: bool, 93 | } 94 | 95 | #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] 96 | #[serde(rename_all = "camelCase")] 97 | #[cfg_attr(feature = "strict", serde(deny_unknown_fields))] 98 | pub struct AuthDesc { 99 | pub oauth2: Oauth2Desc, 100 | } 101 | 102 | #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] 103 | #[serde(rename_all = "camelCase")] 104 | #[cfg_attr(feature = "strict", serde(deny_unknown_fields))] 105 | pub struct Oauth2Desc { 106 | #[serde(default)] 107 | pub scopes: BTreeMap, 108 | } 109 | 110 | #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] 111 | #[serde(rename_all = "camelCase")] 112 | #[cfg_attr(feature = "strict", serde(deny_unknown_fields))] 113 | pub struct ScopeDesc { 114 | pub description: String, 115 | } 116 | 117 | #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] 118 | #[serde(rename_all = "camelCase")] 119 | #[cfg_attr(feature = "strict", serde(deny_unknown_fields))] 120 | pub struct SchemaDesc { 121 | pub id: String, 122 | pub description: Option, 123 | #[serde(flatten, rename = "type")] 124 | pub typ: TypeDesc, 125 | } 126 | 127 | #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] 128 | #[serde(rename_all = "camelCase")] 129 | pub struct PropertyDesc { 130 | pub description: Option, 131 | 132 | #[serde(flatten)] 133 | pub typ: RefOrType, 134 | } 135 | 136 | #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] 137 | #[serde(rename_all = "camelCase", untagged)] 138 | pub enum RefOrType { 139 | #[serde(with = "ref_target")] 140 | Ref(String), 141 | Type(T), 142 | } 143 | 144 | mod ref_target { 145 | use serde::{Deserialize, Deserializer, Serialize, Serializer}; 146 | use std::borrow::Cow; 147 | 148 | #[derive(Debug, Serialize, Deserialize)] 149 | #[serde(rename_all = "camelCase")] 150 | struct RefTarget<'a> { 151 | #[serde(rename = "$ref")] 152 | reference: Cow<'a, str>, 153 | } 154 | 155 | pub(super) fn serialize(x: &String, serializer: S) -> Result 156 | where 157 | S: Serializer, 158 | { 159 | RefTarget { 160 | reference: Cow::Borrowed(x.as_str()), 161 | } 162 | .serialize(serializer) 163 | } 164 | 165 | pub(super) fn deserialize<'de, D>(deserializer: D) -> Result 166 | where 167 | D: Deserializer<'de>, 168 | { 169 | let rt = RefTarget::deserialize(deserializer)?; 170 | Ok(rt.reference.into_owned()) 171 | } 172 | } 173 | 174 | #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] 175 | #[serde(rename_all = "camelCase")] 176 | pub struct TypeDesc { 177 | #[serde(rename = "type")] 178 | pub typ: String, 179 | pub format: Option, 180 | #[serde(default, rename = "enum")] 181 | pub enumeration: Vec, 182 | #[serde(default)] 183 | pub enum_descriptions: Vec, 184 | #[serde(default)] 185 | pub properties: BTreeMap, 186 | #[serde(default)] 187 | pub additional_properties: Option>, 188 | pub items: Option>>, 189 | } 190 | 191 | impl TypeDesc { 192 | pub fn from_param(param: ParamDesc) -> TypeDesc { 193 | let type_desc = TypeDesc { 194 | typ: param.typ, 195 | format: param.format, 196 | enumeration: param.enumeration, 197 | enum_descriptions: param.enum_descriptions, 198 | properties: BTreeMap::new(), 199 | additional_properties: None, 200 | items: None, 201 | }; 202 | 203 | if param.repeated { 204 | // Repeated params should be represented as arrays of this type. 205 | TypeDesc { 206 | typ: "array".to_owned(), 207 | format: None, 208 | enumeration: Vec::new(), 209 | enum_descriptions: Vec::new(), 210 | properties: BTreeMap::new(), 211 | additional_properties: None, 212 | items: Some(Box::new(RefOrType::Type(type_desc))), 213 | } 214 | } else { 215 | type_desc 216 | } 217 | } 218 | } 219 | 220 | #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] 221 | #[serde(rename_all = "camelCase")] 222 | pub struct ResourceDesc { 223 | #[serde(default)] 224 | pub resources: BTreeMap, 225 | #[serde(default)] 226 | pub methods: BTreeMap, 227 | } 228 | 229 | #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] 230 | #[serde(rename_all = "camelCase")] 231 | pub struct MethodDesc { 232 | pub id: String, 233 | pub path: String, 234 | pub http_method: String, 235 | pub description: Option, 236 | #[serde(default)] 237 | pub parameters: BTreeMap, 238 | #[serde(default)] 239 | pub parameter_order: Vec, 240 | pub request: Option>, 241 | pub response: Option>, 242 | #[serde(default)] 243 | pub scopes: Vec, 244 | #[serde(default)] 245 | pub supports_media_download: bool, 246 | #[serde(default)] 247 | pub use_media_download_service: bool, 248 | #[serde(default)] 249 | pub supports_subscription: bool, 250 | #[serde(default)] 251 | pub supports_media_upload: bool, 252 | pub media_upload: Option, 253 | } 254 | 255 | #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] 256 | #[serde(rename_all = "camelCase")] 257 | pub struct MediaUpload { 258 | pub accept: Vec, 259 | pub max_size: Option, 260 | pub protocols: UploadProtocols, 261 | } 262 | 263 | #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] 264 | #[serde(rename_all = "camelCase")] 265 | pub struct UploadProtocols { 266 | pub simple: Option, 267 | pub resumable: Option, 268 | } 269 | 270 | #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] 271 | #[serde(rename_all = "camelCase")] 272 | pub struct UploadProtocol { 273 | pub multipart: bool, 274 | pub path: String, 275 | } 276 | -------------------------------------------------------------------------------- /discovery_parser/tests/tests.rs: -------------------------------------------------------------------------------- 1 | use discovery_parser::DiscoveryRestDesc; 2 | use std::error::Error; 3 | 4 | const API_SPEC: &str = include_str!("./spec.json"); 5 | 6 | #[test] 7 | fn parse_one_api() -> Result<(), Box> { 8 | let desc: DiscoveryRestDesc = serde_json::from_str(API_SPEC)?; 9 | println!("{:#?}", desc); 10 | Ok(()) 11 | } 12 | -------------------------------------------------------------------------------- /google_api_auth/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "google_api_auth" 3 | version = "0.1.0" 4 | authors = ["Glenn Griffin "] 5 | edition = "2018" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [features] 10 | default = [] 11 | with-yup-oauth2 = ["yup-oauth2", "tokio"] 12 | 13 | [dependencies] 14 | yup-oauth2 = { version = "^4.1", optional = true } 15 | tokio = { version = "0.2", optional = true } 16 | hyper = "^0.13" 17 | -------------------------------------------------------------------------------- /google_api_auth/src/lib.rs: -------------------------------------------------------------------------------- 1 | /// GetAccessToken provides an oauth2 access token. It's used by google api 2 | /// client libraries to retrieve access tokens when making http requests. This 3 | /// library optionally provides a variety of implementations, but users are also 4 | /// free to implement whatever logic they want for retrieving a token. 5 | pub trait GetAccessToken: ::std::fmt::Debug + Send + Sync { 6 | fn access_token(&self) -> Result>; 7 | } 8 | 9 | impl From for Box 10 | where 11 | T: GetAccessToken + 'static, 12 | { 13 | fn from(x: T) -> Self { 14 | Box::new(x) 15 | } 16 | } 17 | 18 | #[cfg(feature = "with-yup-oauth2")] 19 | pub mod yup_oauth2; 20 | -------------------------------------------------------------------------------- /google_api_auth/src/yup_oauth2.rs: -------------------------------------------------------------------------------- 1 | use hyper::client::connect::Connect; 2 | use yup_oauth2::authenticator::Authenticator; 3 | 4 | pub fn from_authenticator(auth: Authenticator, scopes: I) -> impl crate::GetAccessToken 5 | where 6 | C: Connect + Clone + Send + Sync + 'static, 7 | I: IntoIterator, 8 | S: Into, 9 | { 10 | YupAuthenticator { 11 | auth, 12 | scopes: scopes.into_iter().map(Into::into).collect(), 13 | } 14 | } 15 | 16 | struct YupAuthenticator { 17 | auth: Authenticator, 18 | scopes: Vec, 19 | } 20 | 21 | impl ::std::fmt::Debug for YupAuthenticator { 22 | fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { 23 | write!(f, "{}", "YupAuthenticator{..}") 24 | } 25 | } 26 | 27 | impl crate::GetAccessToken for YupAuthenticator 28 | where 29 | C: Connect + Clone + Send + Sync + 'static, 30 | { 31 | fn access_token(&self) -> Result> { 32 | let fut = self.auth.token(&self.scopes); 33 | let mut runtime = ::tokio::runtime::Runtime::new().expect("unable to start tokio runtime"); 34 | Ok(runtime.block_on(fut)?.as_str().to_string()) 35 | } 36 | } 37 | 38 | #[cfg(test)] 39 | mod tests { 40 | use super::*; 41 | use crate::GetAccessToken; 42 | use yup_oauth2 as oauth2; 43 | 44 | #[tokio::test] 45 | async fn it_works() { 46 | let auth = oauth2::InstalledFlowAuthenticator::builder( 47 | oauth2::ApplicationSecret::default(), 48 | yup_oauth2::InstalledFlowReturnMethod::HTTPRedirect, 49 | ) 50 | .build() 51 | .await 52 | .expect("failed to build"); 53 | 54 | let auth = from_authenticator(auth, vec!["foo", "bar"]); 55 | 56 | fn this_should_work(_x: T) {}; 57 | this_should_work(auth); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /google_api_bytes/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "google_api_bytes" 3 | version = "0.1.0" 4 | authors = ["Glenn Griffin "] 5 | edition = "2018" 6 | 7 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 8 | 9 | [dependencies] 10 | radix64 = "0.6" 11 | serde = "1" 12 | -------------------------------------------------------------------------------- /google_api_bytes/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Bytes in google apis are represented as urlsafe base64 encoded strings. 2 | // This defines a Bytes type that is a simple wrapper around a Vec used 3 | // internally to handle byte fields in google apis. 4 | use radix64::URL_SAFE as BASE64_CFG; 5 | use serde::{Deserialize, Deserializer, Serialize, Serializer}; 6 | use std::fmt; 7 | 8 | #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] 9 | pub struct Bytes(pub Vec); 10 | 11 | impl ::std::convert::From> for Bytes { 12 | fn from(x: Vec) -> Bytes { 13 | Bytes(x) 14 | } 15 | } 16 | 17 | impl ::std::convert::From for Vec { 18 | fn from(x: Bytes) -> Vec { 19 | x.0 20 | } 21 | } 22 | 23 | impl AsRef<[u8]> for Bytes { 24 | fn as_ref(&self) -> &[u8] { 25 | self.0.as_ref() 26 | } 27 | } 28 | 29 | impl fmt::Display for Bytes { 30 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 31 | ::radix64::Display::new(BASE64_CFG, &self.0).fmt(f) 32 | } 33 | } 34 | 35 | impl Serialize for Bytes { 36 | fn serialize(&self, serializer: S) -> ::std::result::Result 37 | where 38 | S: Serializer, 39 | { 40 | let encoded = BASE64_CFG.encode(&self.0); 41 | encoded.serialize(serializer) 42 | } 43 | } 44 | 45 | impl<'de> Deserialize<'de> for Bytes { 46 | fn deserialize(deserializer: D) -> ::std::result::Result 47 | where 48 | D: Deserializer<'de>, 49 | { 50 | let encoded = String::deserialize(deserializer)?; 51 | let decoded = BASE64_CFG 52 | .decode(&encoded) 53 | .map_err(|_| ::serde::de::Error::custom("invalid base64 input"))?; 54 | Ok(Bytes(decoded)) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /google_cli_generator/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "google_cli_generator" 3 | version = "0.1.0" 4 | authors = ["Sebastian Thiel "] 5 | edition = "2018" 6 | 7 | [lib] 8 | doctest = false 9 | 10 | [dependencies] 11 | google_rest_api_generator = { path = "../google_rest_api_generator" } 12 | discovery_parser = { path = "../discovery_parser" } 13 | shared = { path = "../shared" } 14 | serde = { version = "1", default-features = false, features = ["derive"] } 15 | serde_json = "1.0.40" 16 | log = "0.4.8" 17 | liquid = "0.19" 18 | liquid-error = "0.19.0" 19 | crossbeam = "0.7.2" 20 | 21 | [dev-dependencies] 22 | toml_edit = "0.1.5" 23 | tempfile = "3.1.0" 24 | simple_logger = "1.3.0" 25 | 26 | [build-dependencies] 27 | chrono = "0.4.7" 28 | 29 | -------------------------------------------------------------------------------- /google_cli_generator/build.rs: -------------------------------------------------------------------------------- 1 | use chrono::prelude::*; 2 | use std::error::Error; 3 | 4 | fn main() -> Result<(), Box> { 5 | println!( 6 | "cargo:rustc-env=GIT_HASH={}", 7 | std::process::Command::new("git") 8 | .args(&["describe", "--always"]) 9 | .stdout(std::process::Stdio::piped()) 10 | .output() 11 | .ok() 12 | .and_then(|out| std::str::from_utf8(&out.stdout).map(str::to_string).ok()) 13 | .unwrap_or_else(|| "".to_owned()) 14 | ); 15 | println!( 16 | "cargo:rustc-env=BUILD_DATE={}", 17 | Utc::today().format("%Y-%m-%d") 18 | ); 19 | Ok(()) 20 | } 21 | -------------------------------------------------------------------------------- /google_cli_generator/src/all.rs: -------------------------------------------------------------------------------- 1 | use super::CombinedMetadata; 2 | use crate::cli; 3 | use discovery_parser::DiscoveryRestDesc; 4 | use google_rest_api_generator::generate as generate_library; 5 | use std::{error::Error, path::Path}; 6 | 7 | pub enum Build { 8 | ApiAndCliInParallelNoErrorHandling, 9 | ApiAndCli, 10 | OnlyCli, 11 | OnlyApi, 12 | } 13 | 14 | pub fn generate( 15 | discovery_desc: &DiscoveryRestDesc, 16 | base_dir: impl AsRef, 17 | mode: Build, 18 | ) -> Result<(), Box> { 19 | let constants = shared::Standard::default(); 20 | std::fs::write( 21 | base_dir.as_ref().join(constants.metadata_path), 22 | serde_json::to_string_pretty(&CombinedMetadata::default())?, 23 | )?; 24 | 25 | let lib_dir = base_dir.as_ref().join(&constants.lib_dir); 26 | let cli_dir = base_dir.as_ref().join(&constants.cli_dir); 27 | use self::Build::*; 28 | match mode { 29 | ApiAndCliInParallelNoErrorHandling => { 30 | let _ignore_errors_while_cli_gen_may_fail = crossbeam::scope(|s| { 31 | s.spawn(|_| generate_library(lib_dir, &discovery_desc).map_err(|e| e.to_string())); 32 | s.spawn(|_| cli::generate(cli_dir, &discovery_desc).map_err(|e| e.to_string())); 33 | }); 34 | } 35 | ApiAndCli => { 36 | generate_library(lib_dir, &discovery_desc)?; 37 | cli::generate(cli_dir, &discovery_desc)?; 38 | } 39 | OnlyCli => cli::generate(cli_dir, &discovery_desc)?, 40 | OnlyApi => generate_library(lib_dir, &discovery_desc)?, 41 | } 42 | 43 | Ok(()) 44 | } 45 | -------------------------------------------------------------------------------- /google_cli_generator/src/cargo.rs: -------------------------------------------------------------------------------- 1 | const CARGO_TOML: &str = r#" 2 | [package] 3 | name = "{crate_name}" 4 | version = "{crate_version}" 5 | authors = ["Sebastian Thiel "] 6 | edition = "2018" 7 | # for now, let's not even accidentally publish these 8 | publish = false 9 | 10 | [[bin]] 11 | name = "{bin_name}" 12 | path = "{bin_path}" 13 | 14 | [dependencies] 15 | yup-oauth2 = "^3.1" 16 | google_api_auth = { git = "https://github.com/google-apis-rs/generator", features = ["with-yup-oauth2"] } 17 | clap = "^2.33" 18 | serde_json = "1.0.40" 19 | dirs = "2.0" 20 | google_cli_shared = { git = "https://github.com/google-apis-rs/generator", version = "0.1.0" } 21 | default-boxed = "0.1.6" 22 | "#; 23 | 24 | pub(crate) fn cargo_toml(api: &shared::Api, standard: &shared::Standard) -> String { 25 | let mut doc = CARGO_TOML 26 | .trim() 27 | .replace("{crate_name}", &api.cli_crate_name) 28 | .replace( 29 | "{crate_version}", 30 | &api.cli_crate_version 31 | .as_ref() 32 | .expect("available crate version"), 33 | ) 34 | .replace("{bin_name}", &api.bin_name) 35 | .replace("{bin_path}", &standard.main_path); 36 | 37 | doc.push_str(&format!("\n[dependencies.{}]\n", api.lib_crate_name)); 38 | doc.push_str(&format!("path = \"../lib\"\n")); 39 | doc.push_str(&format!( 40 | "version = \"{}\"\n", 41 | api.lib_crate_version 42 | .as_ref() 43 | .expect("available crate version") 44 | )); 45 | 46 | doc 47 | } 48 | -------------------------------------------------------------------------------- /google_cli_generator/src/cli/liquid_filters.rs: -------------------------------------------------------------------------------- 1 | use liquid::compiler::Filter; 2 | use liquid::derive::*; 3 | use liquid::error::Result; 4 | use liquid::interpreter::Context; 5 | use liquid::value::Value; 6 | 7 | #[derive(Clone, ParseFilter, FilterReflection)] 8 | #[filter( 9 | name = "rust_string_literal", 10 | description = "make any string printable as a Rust string", 11 | parsed(RustStringLiteralFilter) 12 | )] 13 | pub struct RustStringLiteral; 14 | 15 | #[derive(Debug, Default, Display_filter)] 16 | #[name = "rust_string_literal"] 17 | struct RustStringLiteralFilter; 18 | 19 | impl Filter for RustStringLiteralFilter { 20 | fn evaluate(&self, input: &Value, _context: &Context) -> Result { 21 | Ok(Value::scalar(format!("{:?}", input.to_str()))) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /google_cli_generator/src/cli/mod.rs: -------------------------------------------------------------------------------- 1 | use discovery_parser::DiscoveryRestDesc; 2 | use log::info; 3 | use std::{ 4 | cmp::Ordering, convert::TryFrom, error::Error, ffi::OsStr, fs, io::Write, path::Path, 5 | time::Instant, 6 | }; 7 | 8 | use crate::cargo; 9 | use google_rest_api_generator::APIDesc; 10 | use model::Model; 11 | 12 | mod liquid_filters; 13 | mod model; 14 | 15 | pub fn generate( 16 | output_dir: impl AsRef, 17 | discovery_desc: &DiscoveryRestDesc, 18 | ) -> Result<(), Box> { 19 | const MAIN_RS: &str = r#" 20 | "#; 21 | let time = Instant::now(); 22 | info!("cli: building api desc"); 23 | let api_desc = APIDesc::from_discovery(discovery_desc); 24 | let api = shared::Api::try_from(discovery_desc)?; 25 | 26 | let constants = shared::Standard::default(); 27 | let output_dir = output_dir.as_ref(); 28 | let cargo_toml_path = output_dir.join(&constants.cargo_toml_path); 29 | let main_path = output_dir.join(&constants.main_path); 30 | 31 | fs::create_dir_all(&main_path.parent().expect("file in directory"))?; 32 | 33 | let cargo_contents = cargo::cargo_toml(&api, &constants); 34 | fs::write(&cargo_toml_path, &cargo_contents)?; 35 | 36 | info!("cli: writing main '{}'", main_path.display()); 37 | let mut rustfmt_writer = shared::RustFmtWriter::new(fs::File::create(&main_path)?)?; 38 | rustfmt_writer.write_all(MAIN_RS.as_bytes())?; 39 | 40 | let templates_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("templates"); 41 | let engine = liquid::ParserBuilder::with_liquid() 42 | .filter(liquid_filters::RustStringLiteral) 43 | .build()?; 44 | let model = into_liquid_object(Model::new(api, discovery_desc, &api_desc))?; 45 | let mut templates: Vec<_> = templates_dir 46 | .read_dir()? 47 | .collect::, _>>()? 48 | .into_iter() 49 | .filter(|e| { 50 | e.file_type().map(|e| e.is_file()).unwrap_or(false) 51 | && e.path().extension() == Some(OsStr::new("liquid")) 52 | }) 53 | .collect(); 54 | templates.sort_by(|l, r| { 55 | l.path() 56 | .file_name() 57 | .and_then(|fl| r.path().file_name().map(|fr| fl.cmp(fr))) 58 | .unwrap_or(Ordering::Equal) 59 | }); 60 | 61 | for entry in templates { 62 | let template = fs::read_to_string(entry.path())?; 63 | let template = engine.parse(&template).map_err(|err| { 64 | format!( 65 | "Failed to parse liquid template at '{}': {}", 66 | entry.path().display(), 67 | err 68 | ) 69 | })?; 70 | let rendered = template.render(&model)?; 71 | rustfmt_writer.write_all(rendered.as_bytes())?; 72 | } 73 | 74 | rustfmt_writer.close()?; 75 | info!("cli: done in {:?}", time.elapsed()); 76 | 77 | Ok(()) 78 | } 79 | 80 | fn into_liquid_object(src: impl serde::Serialize) -> Result> { 81 | let src = serde_json::to_value(src)?; 82 | let dst = serde_json::from_value(src)?; 83 | match dst { 84 | liquid::value::Value::Object(obj) => Ok(obj), 85 | _ => Err("Data model root must be an object".to_owned().into()), 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /google_cli_generator/src/cli/model.rs: -------------------------------------------------------------------------------- 1 | use crate::util::concat_with_and; 2 | use discovery_parser::DiscoveryRestDesc; 3 | use google_rest_api_generator::{APIDesc, Method as ApiMethod, Resource as ApiResource}; 4 | use serde::Serialize; 5 | use shared::Api; 6 | use std::{collections::VecDeque, iter::FromIterator}; 7 | 8 | const APP_IDENT: &str = "app"; 9 | 10 | #[derive(Serialize)] 11 | pub struct Model { 12 | /// The name of the top-level app identifier 13 | app_ident: String, 14 | /// The name of the crate for 'use ' statement 15 | lib_crate_name_for_use: String, 16 | /// The name of the CLI program 17 | program_name: String, 18 | /// The full semantic version of the CLI 19 | cli_version: String, 20 | /// A one-line summary of what the API does 21 | description: String, 22 | /// A list of resources, along with their capabilities, with methods or without 23 | resources: Vec, 24 | } 25 | 26 | impl Model { 27 | pub fn new(api: Api, desc: &DiscoveryRestDesc, api_desc: &APIDesc) -> Self { 28 | Model { 29 | app_ident: APP_IDENT.to_owned(), 30 | lib_crate_name_for_use: api.lib_crate_name.replace('-', "_"), 31 | program_name: api.bin_name, 32 | cli_version: api.cli_crate_version.expect("available cli crate version"), 33 | description: desc.description.clone(), 34 | resources: { 35 | let mut deque = VecDeque::from_iter(api_desc.resources.iter()); 36 | let mut resources = Vec::new(); 37 | while let Some(api_resource) = deque.pop_front() { 38 | resources.push(Resource::from(api_resource)); 39 | deque.extend(api_resource.resources.iter()); 40 | } 41 | resources 42 | }, 43 | } 44 | } 45 | } 46 | 47 | #[derive(Serialize)] 48 | struct Resource { 49 | parent_ident: String, 50 | ident: String, 51 | name: String, 52 | about: String, 53 | methods: Vec, 54 | } 55 | 56 | impl From<&ApiResource> for Resource { 57 | fn from(r: &ApiResource) -> Self { 58 | let name = r.ident.to_string(); 59 | let parent_count = r.parent_path.segments.len().saturating_sub(2); // skip top-level resources module 60 | let parent_ident = if parent_count == 0 { 61 | APP_IDENT.into() 62 | } else { 63 | format!( 64 | "{}{}", 65 | r.parent_path 66 | .segments 67 | .last() 68 | .expect("at least one item") 69 | .ident 70 | .to_string(), 71 | parent_count.saturating_sub(1) 72 | ) 73 | }; 74 | let about = if r.methods.is_empty() { 75 | format!( 76 | "sub-resources: {}", 77 | concat_with_and(r.resources.iter().map(|r| r.ident.to_string())) 78 | ) 79 | } else { 80 | format!( 81 | "methods: {}", 82 | concat_with_and(r.methods.iter().map(|m| m.ident.to_string())) 83 | ) 84 | }; 85 | 86 | Resource { 87 | parent_ident, 88 | ident: format!("{}{}", name, parent_count), 89 | name, 90 | about, 91 | methods: r.methods.iter().map(Method::from).collect(), 92 | } 93 | } 94 | } 95 | 96 | #[derive(Serialize)] 97 | struct Method { 98 | name: String, 99 | about: Option, 100 | } 101 | 102 | impl From<&ApiMethod> for Method { 103 | fn from(m: &ApiMethod) -> Self { 104 | Method { 105 | name: m.ident.to_string(), 106 | about: m.description.clone(), 107 | } 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /google_cli_generator/src/lib.rs: -------------------------------------------------------------------------------- 1 | use google_rest_api_generator::Metadata as ApiMetadata; 2 | use serde::{Deserialize, Serialize}; 3 | 4 | #[derive(Serialize, Deserialize, PartialEq, Eq, Debug)] 5 | pub struct Metadata { 6 | pub git_hash: String, 7 | pub ymd_date: String, 8 | } 9 | 10 | impl Default for Metadata { 11 | fn default() -> Self { 12 | Metadata { 13 | git_hash: env!("GIT_HASH").into(), 14 | ymd_date: env!("BUILD_DATE").into(), 15 | } 16 | } 17 | } 18 | 19 | #[derive(Serialize, Deserialize, PartialEq, Eq, Debug, Default)] 20 | pub struct CombinedMetadata { 21 | pub cli_generator: Metadata, 22 | pub api_generator: ApiMetadata, 23 | } 24 | 25 | pub mod all; 26 | mod cargo; 27 | pub mod cli; 28 | mod util; 29 | -------------------------------------------------------------------------------- /google_cli_generator/src/util.rs: -------------------------------------------------------------------------------- 1 | pub fn concat_with_and(items: impl Iterator>) -> String { 2 | let items = items.map(|s| s.as_ref().to_owned()).collect::>(); 3 | if items.is_empty() { 4 | return String::new(); 5 | } 6 | let mut buf = items[..items.len() - 1].join(", "); 7 | if items.len() > 1 { 8 | buf.push_str(" and "); 9 | } 10 | if items.len() > 0 { 11 | buf.push_str(items.last().expect("last element")); 12 | } 13 | buf 14 | } 15 | 16 | #[cfg(test)] 17 | mod tests { 18 | mod concat_with_and { 19 | use super::super::concat_with_and; 20 | #[test] 21 | fn empty_input_yields_empty_string() { 22 | assert_eq!(concat_with_and(Vec::<&str>::new().iter()), ""); 23 | } 24 | 25 | #[test] 26 | fn single_input_yields_item() { 27 | assert_eq!(concat_with_and(["foo"].iter()), "foo"); 28 | } 29 | 30 | #[test] 31 | fn two_input_yields_items_connected_with_and() { 32 | assert_eq!(concat_with_and(["foo", "bar"].iter()), "foo and bar"); 33 | } 34 | 35 | #[test] 36 | fn multiple_input_yields_last_two_items_connected_with_and_the_others_with_comma() { 37 | assert_eq!( 38 | concat_with_and(["foo", "bar", "baz"].iter()), 39 | "foo, bar and baz" 40 | ); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /google_cli_generator/templates/01_main.rs.liquid: -------------------------------------------------------------------------------- 1 | 2 | use clap::{Arg, AppSettings, App, SubCommand}; 3 | use default_boxed::DefaultBoxed; 4 | 5 | #[derive(DefaultBoxed)] 6 | struct Outer<'a, 'b> { 7 | inner: HeapApp<'a, 'b> 8 | } 9 | 10 | struct HeapApp<'a, 'b> { 11 | app: App<'a, 'b> 12 | } 13 | 14 | impl<'a, 'b> Default for HeapApp<'a, 'b> { 15 | fn default() -> Self { 16 | let mut {{app_ident}} = App::new("{{program_name}}") 17 | .setting(clap::AppSettings::ColoredHelp) 18 | .author("Sebastian Thiel ") 19 | .version("{{cli_version}}") 20 | .about({{ description | rust_string_literal }}) 21 | .after_help("All documentation details can be found at ") 22 | .arg(Arg::with_name("scope") 23 | .long("scope") 24 | .help("Specify the authentication method should be executed in. Each scope requires the user to grant this application permission to use it. If unset, it defaults to the shortest scope url for a particular method.") 25 | .multiple(true) 26 | .takes_value(true)) 27 | .arg(Arg::with_name("folder") 28 | .long("config-dir") 29 | .help("A directory into which we will store our persistent data. Defaults to a user-writable directory that we will create during the first invocation." ) 30 | .multiple(false) 31 | .takes_value(true)) 32 | .arg(Arg::with_name("debug") 33 | .long("debug") 34 | .help("Provide more output to aid with debugging") 35 | .multiple(false) 36 | .takes_value(false)); 37 | 38 | {%- for r in resources %} 39 | let mut {{r.ident}} = SubCommand::with_name("{{r.name}}") 40 | .setting(AppSettings::ColoredHelp) 41 | .about({{r.about | rust_string_literal }}); 42 | {%- for m in r.methods %} 43 | { 44 | let mcmd = SubCommand::with_name("{{m.name}}") 45 | {%- if m.about -%} 46 | .about({{ m.about | rust_string_literal }}); 47 | {%- endif %}; 48 | {{r.ident}} = {{r.ident}}.subcommand(mcmd); 49 | } 50 | {%- endfor %} 51 | {%- endfor -%} 52 | 53 | {%- for r in resources reversed %} 54 | {{r.parent_ident}} = {{r.parent_ident}}.subcommand({{r.ident}}); 55 | {%- endfor %} 56 | 57 | Self { 58 | {{app_ident}} 59 | } 60 | } 61 | } -------------------------------------------------------------------------------- /google_cli_generator/templates/02_main.rs.liquid: -------------------------------------------------------------------------------- 1 | use {{lib_crate_name_for_use}} as api; 2 | 3 | fn main() { 4 | // TODO: set homedir afterwards, once the address is unmovable, or use Pin for the very first time 5 | // to allow a self-referential structure :D! 6 | let _home_dir = dirs::config_dir().expect("configuration directory can be obtained").join("google-service-cli"); 7 | let outer = Outer::default_boxed(); 8 | let app = outer.inner.app; 9 | let _matches = app.get_matches(); 10 | } -------------------------------------------------------------------------------- /google_cli_generator/tests/generate.rs: -------------------------------------------------------------------------------- 1 | use discovery_parser::DiscoveryRestDesc; 2 | use google_cli_generator as lib; 3 | use serde_json; 4 | use shared; 5 | use simple_logger; 6 | use std::convert::TryFrom; 7 | use std::{ 8 | error::Error, 9 | io, 10 | path::Path, 11 | process::Stdio, 12 | process::{Command, ExitStatus}, 13 | str::FromStr, 14 | }; 15 | use tempfile::TempDir; 16 | use toml_edit; 17 | 18 | static SPEC: &str = include_str!("spec.json"); 19 | 20 | #[test] 21 | fn valid_code_is_produced_for_complex_spec() -> Result<(), Box> { 22 | simple_logger::init_with_level("INFO".parse()?)?; 23 | let spec: DiscoveryRestDesc = serde_json::from_str(SPEC)?; 24 | let temp_dir = TempDir::new_in(Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/output"))?; 25 | lib::all::generate(&spec, &temp_dir, lib::all::Build::ApiAndCli)?; 26 | 27 | let standard = shared::Standard::default(); 28 | let lib_path = temp_dir.path().join(&standard.lib_dir); 29 | let cli_path = temp_dir.path().join(&standard.cli_dir); 30 | 31 | let api = shared::Api::try_from(&spec)?; 32 | fixup_deps(&lib_path, &standard)?; 33 | fixup_deps(&cli_path, &standard)?; 34 | fixup_cli(&cli_path, &api, &standard)?; 35 | 36 | let status = cargo(&cli_path, "check")?; 37 | assert!(status.success(), "cargo check failed on library"); 38 | 39 | Ok(()) 40 | } 41 | 42 | fn cargo(current_dir: &Path, sub_command: &str) -> Result { 43 | let mut cmd = Command::new("cargo"); 44 | cmd.arg(sub_command) 45 | .stdin(Stdio::null()) 46 | .stdout(Stdio::inherit()) 47 | .stderr(Stdio::inherit()) 48 | .current_dir(current_dir); 49 | 50 | if std::env::var("TRAIN_MODE").is_ok() { 51 | cmd.arg("--offline"); 52 | } 53 | 54 | cmd.status() 55 | } 56 | 57 | fn fixup_cli( 58 | path: &Path, 59 | api: &shared::Api, 60 | standard: &shared::Standard, 61 | ) -> Result<(), Box> { 62 | let (cargo_toml_path, mut document) = toml_document(path, &standard)?; 63 | 64 | document["dependencies"][&api.lib_crate_name]["path"] = toml_edit::value( 65 | Path::new("..") 66 | .join(&standard.lib_dir) 67 | .to_str() 68 | .expect("valid utf8"), 69 | ); 70 | 71 | std::fs::write(&cargo_toml_path, &document.to_string().as_bytes())?; 72 | Ok(()) 73 | } 74 | 75 | fn fixup_deps(path: &Path, standard: &shared::Standard) -> Result<(), Box> { 76 | let (cargo_toml_path, mut document) = toml_document(path, &standard)?; 77 | 78 | document["workspace"] = toml_edit::table(); 79 | let dependencies = &mut document["dependencies"]; 80 | for name in &[ 81 | "google_field_selector", 82 | "google_api_auth", 83 | "google_api_bytes", 84 | "google_cli_shared", 85 | ] { 86 | let dep = dependencies[name].as_inline_table_mut(); 87 | if let Some(dep) = dep { 88 | dep.remove("git"); 89 | dep.get_or_insert("path", format!("../../../../../{}", name)); 90 | } 91 | } 92 | 93 | std::fs::write(&cargo_toml_path, &document.to_string().as_bytes())?; 94 | Ok(()) 95 | } 96 | 97 | fn toml_document( 98 | path: &Path, 99 | standard: &shared::Standard, 100 | ) -> Result<(std::path::PathBuf, toml_edit::Document), Box> { 101 | let cargo_toml_path = path.join(&standard.cargo_toml_path); 102 | let toml = std::fs::read_to_string(&cargo_toml_path)?; 103 | let document = toml_edit::Document::from_str(&toml)?; 104 | 105 | Ok((cargo_toml_path, document)) 106 | } 107 | -------------------------------------------------------------------------------- /google_cli_generator/tests/output/.gitignore: -------------------------------------------------------------------------------- 1 | .tmp*/ 2 | -------------------------------------------------------------------------------- /google_cli_shared/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | 3 | name = "google_cli_shared" 4 | version = "0.1.0" 5 | authors = ["Sebastian Thiel "] 6 | description = "Code shared by all Google CLI crates" 7 | edition = "2018" 8 | publish = false 9 | 10 | [dependencies] 11 | mime = "^ 0.3.13" 12 | serde_json = "^ 1.0" 13 | yup-oauth2 = "^3.1" 14 | strsim = "^0.9" 15 | 16 | [workspace] 17 | -------------------------------------------------------------------------------- /google_cli_shared/README.md: -------------------------------------------------------------------------------- 1 | A crate with shared code for use in generated CLIs. 2 | -------------------------------------------------------------------------------- /google_field_selector/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "google_field_selector" 3 | version = "0.1.0" 4 | authors = ["Glenn Griffin "] 5 | edition = "2018" 6 | 7 | [lib] 8 | doctest = false 9 | 10 | [dependencies] 11 | google_field_selector_derive = { version = "0.1.0", path = "../google_field_selector_derive" } 12 | 13 | [dev-dependencies] 14 | chrono = { version = "0.4", features = ["serde"] } 15 | serde = { version = "1", features = ["derive"] } 16 | -------------------------------------------------------------------------------- /google_field_selector/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub use google_field_selector_derive::FieldSelector; 2 | use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, LinkedList, VecDeque}; 3 | 4 | /// This is the purpose of the crate. Given a type that implements FieldSelector 5 | /// this will return a string that can be used in the `fields` parameter of 6 | /// google API's to request a partial response that contains the fields of T. 7 | /// 8 | /// See [Google 9 | /// Docs](https://developers.google.com/discovery/v1/performance#partial-response) 10 | /// for more details. 11 | pub fn to_string() -> String { 12 | fn append_fields(parents: &mut Vec<&str>, fields: &[Field], output: &mut String) { 13 | let mut iter = fields.iter(); 14 | let mut next = iter.next(); 15 | while let Some(field) = next { 16 | append_field(parents, &field, output); 17 | next = iter.next(); 18 | if next.is_some() { 19 | output.push_str(","); 20 | } 21 | } 22 | } 23 | 24 | fn append_field(parents: &mut Vec<&str>, field: &Field, output: &mut String) { 25 | let append_parents = |output: &mut String| { 26 | for &parent in parents.iter() { 27 | output.push_str(parent); 28 | output.push_str("/"); 29 | } 30 | }; 31 | 32 | match field { 33 | Field::Glob => { 34 | append_parents(output); 35 | output.push_str("*"); 36 | } 37 | Field::Named { 38 | field_name, 39 | field_type, 40 | } => match field_type { 41 | FieldType::Leaf => { 42 | append_parents(output); 43 | output.push_str(field_name); 44 | } 45 | FieldType::Container(inner_field_type) => { 46 | append_parents(output); 47 | output.push_str(field_name); 48 | match &**inner_field_type { 49 | FieldType::Leaf | FieldType::Container(_) => {} 50 | FieldType::Struct(fields) => { 51 | output.push_str("("); 52 | append_fields(&mut Vec::new(), fields, output); 53 | output.push_str(")"); 54 | } 55 | } 56 | } 57 | FieldType::Struct(fields) => { 58 | parents.push(field_name); 59 | append_fields(parents, fields, output); 60 | parents.pop(); 61 | } 62 | }, 63 | } 64 | } 65 | 66 | let mut output = String::new(); 67 | append_fields(&mut Vec::new(), &T::fields(), &mut output); 68 | output 69 | } 70 | 71 | pub enum Field { 72 | Glob, 73 | Named { 74 | field_name: &'static str, 75 | field_type: FieldType, 76 | }, 77 | } 78 | 79 | pub enum FieldType { 80 | Leaf, 81 | Struct(Vec), 82 | Container(Box), 83 | } 84 | 85 | pub trait ToFieldType { 86 | fn field_type() -> FieldType; 87 | } 88 | 89 | /// FieldSelector provides a google api compatible field selector. This trait 90 | /// will typically be generated from a procedural macro using 91 | /// #[derive(FieldSelector)] 92 | pub trait FieldSelector { 93 | fn fields() -> Vec; 94 | } 95 | 96 | macro_rules! leaf_field_type { 97 | ($t:ty) => { 98 | impl ToFieldType for $t { 99 | fn field_type() -> FieldType { 100 | FieldType::Leaf 101 | } 102 | } 103 | }; 104 | } 105 | 106 | leaf_field_type!(bool); 107 | leaf_field_type!(char); 108 | leaf_field_type!(i8); 109 | leaf_field_type!(i16); 110 | leaf_field_type!(i32); 111 | leaf_field_type!(i64); 112 | leaf_field_type!(i128); 113 | leaf_field_type!(isize); 114 | leaf_field_type!(u8); 115 | leaf_field_type!(u16); 116 | leaf_field_type!(u32); 117 | leaf_field_type!(u64); 118 | leaf_field_type!(u128); 119 | leaf_field_type!(usize); 120 | leaf_field_type!(f32); 121 | leaf_field_type!(f64); 122 | leaf_field_type!(String); 123 | 124 | // For field selection we treat Options as invisible, proxying to the inner type. 125 | impl ToFieldType for Option 126 | where 127 | T: ToFieldType, 128 | { 129 | fn field_type() -> FieldType { 130 | T::field_type() 131 | } 132 | } 133 | 134 | // implement ToFieldType for std::collections types. 135 | // Vec, VecDeque, HashSet, BTreeSet, LinkedList, all act as containers of other elements. 136 | 137 | impl ToFieldType for Vec 138 | where 139 | T: ToFieldType, 140 | { 141 | fn field_type() -> FieldType { 142 | FieldType::Container(Box::new(T::field_type())) 143 | } 144 | } 145 | 146 | impl ToFieldType for VecDeque 147 | where 148 | T: ToFieldType, 149 | { 150 | fn field_type() -> FieldType { 151 | FieldType::Container(Box::new(T::field_type())) 152 | } 153 | } 154 | 155 | impl ToFieldType for HashSet 156 | where 157 | T: ToFieldType, 158 | { 159 | fn field_type() -> FieldType { 160 | FieldType::Container(Box::new(T::field_type())) 161 | } 162 | } 163 | 164 | impl ToFieldType for BTreeSet 165 | where 166 | T: ToFieldType, 167 | { 168 | fn field_type() -> FieldType { 169 | FieldType::Container(Box::new(T::field_type())) 170 | } 171 | } 172 | 173 | impl ToFieldType for LinkedList 174 | where 175 | T: ToFieldType, 176 | { 177 | fn field_type() -> FieldType { 178 | FieldType::Container(Box::new(T::field_type())) 179 | } 180 | } 181 | 182 | // HashMap and BTreeMap are not considered containers for the purposes of 183 | // selections. The google api does not provide a mechanism to specify fields of 184 | // key/value pairs. 185 | impl ToFieldType for HashMap { 186 | fn field_type() -> FieldType { 187 | FieldType::Leaf 188 | } 189 | } 190 | 191 | impl ToFieldType for BTreeMap { 192 | fn field_type() -> FieldType { 193 | FieldType::Leaf 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /google_field_selector/tests/tests.rs: -------------------------------------------------------------------------------- 1 | #![allow(dead_code)] 2 | 3 | use google_field_selector::{to_string, FieldSelector}; 4 | use serde::Deserialize; 5 | use std::collections::HashMap; 6 | 7 | #[derive(Deserialize, FieldSelector)] 8 | #[serde(rename_all = "camelCase")] 9 | struct File { 10 | id: String, 11 | mime_type: String, 12 | sharing_user: Option, 13 | } 14 | 15 | #[derive(Deserialize, FieldSelector)] 16 | #[serde(rename_all = "camelCase")] 17 | struct UserInfo { 18 | me: bool, 19 | email_address: String, 20 | user_attrs: HashMap, 21 | } 22 | 23 | #[test] 24 | fn basic() { 25 | #[derive(Deserialize, FieldSelector)] 26 | #[serde(rename_all = "camelCase")] 27 | struct Response { 28 | next_page_token: String, 29 | files: Vec, 30 | } 31 | 32 | assert_eq!( 33 | to_string::(), 34 | "nextPageToken,files(id,mimeType,sharingUser/me,sharingUser/emailAddress,sharingUser/userAttrs)" 35 | ); 36 | } 37 | 38 | #[test] 39 | fn generic_with_flatten() { 40 | #[derive(Deserialize, FieldSelector)] 41 | #[serde(rename_all = "camelCase")] 42 | struct Response 43 | where 44 | T: FieldSelector, 45 | { 46 | next_page_token: String, 47 | #[serde(flatten)] 48 | payload: T, 49 | } 50 | 51 | #[derive(Deserialize, FieldSelector)] 52 | #[serde(rename_all = "camelCase")] 53 | struct ListFiles { 54 | files: Vec, 55 | } 56 | assert_eq!( 57 | to_string::>(), 58 | "nextPageToken,files(id,mimeType,sharingUser/me,sharingUser/emailAddress,sharingUser/userAttrs)" 59 | ); 60 | } 61 | 62 | #[test] 63 | fn external_types() { 64 | use chrono::{DateTime, Utc}; 65 | #[derive(Deserialize)] 66 | struct MyCustomVec(Vec); 67 | 68 | #[derive(Deserialize, FieldSelector)] 69 | struct File { 70 | id: String, 71 | 72 | // Specify that DateTime is a leaf node. Don't treat it as a nested 73 | // struct that we can specify subselections of. 74 | #[field_selector(leaf)] 75 | viewed_by_me_time: DateTime, 76 | } 77 | 78 | #[derive(Deserialize, FieldSelector)] 79 | #[serde(rename_all = "camelCase")] 80 | struct Response { 81 | next_page_token: String, 82 | 83 | // Specify that MyCustomVec should be treated as a container holding 84 | // elements of File. 85 | #[field_selector(container_of = "File")] 86 | files: MyCustomVec, 87 | } 88 | 89 | assert_eq!( 90 | to_string::(), 91 | "nextPageToken,files(id,viewed_by_me_time)" 92 | ); 93 | } 94 | 95 | #[test] 96 | fn glob_selector() { 97 | use google_field_selector::{Field, FieldType, ToFieldType}; 98 | #[derive(Deserialize)] 99 | struct Foo; 100 | 101 | impl FieldSelector for Foo { 102 | fn fields() -> Vec { 103 | vec![Field::Glob] 104 | } 105 | } 106 | 107 | impl ToFieldType for Foo { 108 | fn field_type() -> FieldType { 109 | FieldType::Struct(Self::fields()) 110 | } 111 | } 112 | 113 | assert_eq!(to_string::(), "*"); 114 | 115 | #[derive(Deserialize, FieldSelector)] 116 | #[serde(rename_all = "camelCase")] 117 | struct NestedFoo { 118 | attr_1: String, 119 | foo: Foo, 120 | } 121 | 122 | assert_eq!(to_string::(), "attr1,foo/*"); 123 | 124 | #[derive(Deserialize, FieldSelector)] 125 | #[serde(rename_all = "camelCase")] 126 | struct ContainedFoo { 127 | attr_1: String, 128 | foo: Vec, 129 | } 130 | 131 | assert_eq!(to_string::(), "attr1,foo(*)"); 132 | } 133 | -------------------------------------------------------------------------------- /google_field_selector_derive/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "google_field_selector_derive" 3 | version = "0.1.0" 4 | authors = ["Glenn Griffin "] 5 | edition = "2018" 6 | 7 | [lib] 8 | name = "google_field_selector_derive" 9 | doctest = false 10 | proc-macro = true 11 | 12 | [dependencies] 13 | serde_derive_internals = "0.25" 14 | syn = "1.0.3" 15 | quote = "1" 16 | proc-macro2 = "1" 17 | -------------------------------------------------------------------------------- /google_field_selector_derive/src/lib.rs: -------------------------------------------------------------------------------- 1 | extern crate proc_macro; 2 | 3 | use proc_macro::TokenStream; 4 | use proc_macro2::TokenStream as TokenStream2; 5 | use quote::quote; 6 | use serde_derive_internals as serdei; 7 | use std::error::Error; 8 | use syn::{parse_macro_input, DeriveInput}; 9 | 10 | #[proc_macro_derive(FieldSelector, attributes(field_selector))] 11 | pub fn derive_field_selector(input: TokenStream) -> TokenStream { 12 | let input = parse_macro_input!(input as DeriveInput); 13 | expand_derive_field_selector(&input).unwrap().into() 14 | } 15 | 16 | fn expand_derive_field_selector(input: &DeriveInput) -> Result> { 17 | let ctx = serdei::Ctxt::new(); 18 | let cont = serdei::ast::Container::from_ast(&ctx, input, serdei::Derive::Deserialize); 19 | if let Err(errors) = ctx.check() { 20 | return Err(errors 21 | .into_iter() 22 | .map(|e| e.to_string()) 23 | .collect::>() 24 | .join("\n") 25 | .into()); 26 | } 27 | let cont = cont.expect("no option if there are no errors"); 28 | let field_output: Vec = match cont.data { 29 | serdei::ast::Data::Struct(serdei::ast::Style::Struct, fields) => { 30 | fields.iter().map(selector_for_field).collect() 31 | } 32 | _ => return Err("Only able to derive FieldSelector for plain Struct".into()), 33 | }; 34 | 35 | let ident = cont.ident; 36 | let (impl_generics, ty_generics, where_clause) = cont.generics.split_for_impl(); 37 | let dummy_const = syn::Ident::new( 38 | &format!("_IMPL_FIELD_SELECTOR_FOR_{}", ident), 39 | proc_macro2::Span::call_site(), 40 | ); 41 | Ok(quote! { 42 | const #dummy_const: () = { 43 | extern crate google_field_selector as _google_field_selector; 44 | impl #impl_generics _google_field_selector::FieldSelector for #ident #ty_generics #where_clause { 45 | fn fields() -> Vec<_google_field_selector::Field> { 46 | let mut fields = Vec::new(); 47 | #(#field_output)* 48 | fields 49 | } 50 | } 51 | 52 | impl #impl_generics _google_field_selector::ToFieldType for #ident #ty_generics #where_clause { 53 | fn field_type() -> _google_field_selector::FieldType { 54 | _google_field_selector::FieldType::Struct(::fields()) 55 | } 56 | } 57 | }; 58 | }) 59 | } 60 | 61 | fn selector_for_field<'a>(field: &serdei::ast::Field<'a>) -> TokenStream2 { 62 | enum AttrOverride { 63 | ContainerOf(syn::ExprPath), 64 | Leaf, 65 | } 66 | let syn_field = field.original; 67 | let attr_override = syn_field.attrs.iter().find_map(|attr| { 68 | let metalist = match attr.parse_meta() { 69 | Ok(meta @ syn::Meta::List(_)) => meta, 70 | _ => return None, 71 | }; 72 | if metalist.path().get_ident().map(|n| n.to_string()) != Some("field_selector".into()) { 73 | return None; 74 | } 75 | let nestedlist = match metalist { 76 | syn::Meta::List(syn::MetaList { nested, .. }) => nested, 77 | _ => return None, 78 | }; 79 | for meta in nestedlist.iter() { 80 | match meta { 81 | syn::NestedMeta::Meta(syn::Meta::NameValue(syn::MetaNameValue { 82 | path: name, 83 | lit: syn::Lit::Str(value), 84 | .. 85 | })) if name.get_ident().map(|n| n.to_string()) == Some("container_of".into()) => { 86 | if let Ok(typ_path) = value.parse() { 87 | return Some(AttrOverride::ContainerOf(typ_path)); 88 | } 89 | } 90 | syn::NestedMeta::Meta(syn::Meta::Path(word)) 91 | if word.get_ident().map(|x| x.to_string()) == Some("leaf".into()) => 92 | { 93 | return Some(AttrOverride::Leaf); 94 | } 95 | _ => {} 96 | } 97 | } 98 | None 99 | }); 100 | 101 | let field_name = field.attrs.name().deserialize_name(); 102 | match attr_override { 103 | Some(AttrOverride::ContainerOf(type_path)) => { 104 | quote! { 105 | fields.push( 106 | _google_field_selector::Field::Named{ 107 | field_name: #field_name, 108 | field_type: _google_field_selector::FieldType::Container( 109 | Box::new(<#type_path as _google_field_selector::ToFieldType>::field_type())) 110 | } 111 | ); 112 | } 113 | } 114 | Some(AttrOverride::Leaf) => { 115 | quote! { 116 | fields.push( 117 | _google_field_selector::Field::Named{ 118 | field_name: #field_name, 119 | field_type: _google_field_selector::FieldType::Leaf 120 | } 121 | ); 122 | } 123 | } 124 | None => { 125 | let typ = field.ty; 126 | if field.attrs.flatten() { 127 | quote! { 128 | fields.extend(<#typ as _google_field_selector::FieldSelector>::fields()); 129 | } 130 | } else { 131 | quote! { 132 | fields.push( 133 | _google_field_selector::Field::Named{ 134 | field_name: #field_name, 135 | field_type: <#typ as _google_field_selector::ToFieldType>::field_type() 136 | } 137 | ); 138 | } 139 | } 140 | } 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /google_rest_api_generator/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "google_rest_api_generator" 3 | version = "0.1.0" 4 | authors = ["Glenn Griffin "] 5 | edition = "2018" 6 | 7 | [lib] 8 | doctest = false 9 | 10 | [dependencies] 11 | discovery_parser = { path = "../discovery_parser" } 12 | uri_template_parser = { path = "../uri_template_parser" } 13 | shared = { path = "../shared" } 14 | proc-macro2 = "1.0" 15 | quote = "1.0" 16 | Inflector = "0.11.4" 17 | syn = { version = "1.0", features = ["full", "extra-traits"] } 18 | serde = { version = "1", features = ["derive"] } 19 | serde_json = "1" 20 | log = "0.4.8" 21 | pulldown-cmark-to-cmark = "1.2.2" 22 | pulldown-cmark = "0.5.3" 23 | percent-encoding = "2.1.0" 24 | reqwest = "0.9.19" 25 | 26 | [dev-dependencies] 27 | http = "0.1.17" 28 | env_logger = "0.6.2" 29 | structopt = "0.2.18" 30 | tempfile = "3.1.0" 31 | simple_logger = "1.3.0" 32 | 33 | [build-dependencies] 34 | chrono = "0.4.7" 35 | 36 | -------------------------------------------------------------------------------- /google_rest_api_generator/build.rs: -------------------------------------------------------------------------------- 1 | use chrono::prelude::*; 2 | use std::error::Error; 3 | 4 | fn main() -> Result<(), Box> { 5 | println!( 6 | "cargo:rustc-env=GIT_HASH={}", 7 | std::process::Command::new("git") 8 | .args(&["describe", "--always"]) 9 | .stdout(std::process::Stdio::piped()) 10 | .output() 11 | .ok() 12 | .and_then(|out| std::str::from_utf8(&out.stdout).map(str::to_string).ok()) 13 | .unwrap_or_else(|| "".to_owned()) 14 | ); 15 | println!( 16 | "cargo:rustc-env=BUILD_DATE={}", 17 | Utc::today().format("%Y-%m-%d") 18 | ); 19 | Ok(()) 20 | } 21 | -------------------------------------------------------------------------------- /google_rest_api_generator/examples/any_api.rs: -------------------------------------------------------------------------------- 1 | use discovery_parser::DiscoveryRestDesc; 2 | use google_rest_api_generator as generator; 3 | use std::error::Error; 4 | use std::path::PathBuf; 5 | use structopt::StructOpt; 6 | 7 | #[derive(Debug, StructOpt)] 8 | #[structopt( 9 | name = "any_api", 10 | about = "Generate Rust API bindings for a Google Rest API" 11 | )] 12 | struct Opt { 13 | #[structopt(long = "name")] 14 | name: String, 15 | #[structopt(long = "version")] 16 | version: String, 17 | #[structopt(long = "output_dir", default_value = "/tmp", parse(from_os_str))] 18 | output_dir: PathBuf, 19 | } 20 | 21 | /// Alter the URL to generate output for a different API. 22 | /// Otherwise, prefer using the machinery in https://github.com/google-apis-rs/generated to 23 | /// generate any API, CLI and more 24 | fn main() -> Result<(), Box> { 25 | ::env_logger::builder() 26 | .default_format_timestamp_nanos(true) 27 | .init(); 28 | let opt = Opt::from_args(); 29 | let desc: DiscoveryRestDesc = reqwest::get(&format!( 30 | "https://www.googleapis.com/discovery/v1/apis/{}/{}/rest", 31 | &opt.name, &opt.version 32 | ))? 33 | .error_for_status()? 34 | .json()?; 35 | let project_name = format!("google_{}_{}", &desc.name, &desc.version); 36 | let project_dir = opt.output_dir.join(&project_name); 37 | println!("Writing to {:?}", &project_dir); 38 | generator::generate(&project_dir, &desc)?; 39 | Ok(()) 40 | } 41 | -------------------------------------------------------------------------------- /google_rest_api_generator/gen_include/README.md: -------------------------------------------------------------------------------- 1 | Files in this directory are included verbatim in the generated lib.rs file. Thus they compile in the context of the generated code, not in the context of the this (the generator) crate. -------------------------------------------------------------------------------- /google_rest_api_generator/gen_include/error.rs: -------------------------------------------------------------------------------- 1 | #[derive(Debug)] 2 | pub enum Error { 3 | OAuth2(Box), 4 | JSON(::serde_json::Error), 5 | Reqwest { 6 | reqwest_err: ::reqwest::Error, 7 | body: Option, 8 | }, 9 | Other(Box), 10 | } 11 | 12 | impl Error { 13 | pub fn json_error(&self) -> Option<&::serde_json::Error> { 14 | match self { 15 | Error::OAuth2(_) => None, 16 | Error::JSON(err) => Some(err), 17 | Error::Reqwest { .. } => None, 18 | Error::Other(_) => None, 19 | } 20 | } 21 | } 22 | 23 | impl ::std::fmt::Display for Error { 24 | fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { 25 | match self { 26 | Error::OAuth2(err) => write!(f, "OAuth2 Error: {}", err), 27 | Error::JSON(err) => write!(f, "JSON Error: {}", err), 28 | Error::Reqwest { reqwest_err, body } => { 29 | write!(f, "Reqwest Error: {}", reqwest_err)?; 30 | if let Some(body) = body { 31 | write!(f, ": {}", body)?; 32 | } 33 | Ok(()) 34 | } 35 | Error::Other(err) => write!(f, "Uknown Error: {}", err), 36 | } 37 | } 38 | } 39 | 40 | impl ::std::error::Error for Error {} 41 | 42 | impl From<::serde_json::Error> for Error { 43 | fn from(err: ::serde_json::Error) -> Error { 44 | Error::JSON(err) 45 | } 46 | } 47 | 48 | impl From<::reqwest::Error> for Error { 49 | fn from(reqwest_err: ::reqwest::Error) -> Error { 50 | Error::Reqwest { 51 | reqwest_err, 52 | body: None, 53 | } 54 | } 55 | } 56 | 57 | /// Check the response to see if the status code represents an error. If so 58 | /// convert it into the Reqwest variant of Error. 59 | fn error_from_response(response: ::reqwest::blocking::Response) -> Result<::reqwest::blocking::Response, Error> { 60 | match response.error_for_status_ref() { 61 | Err(reqwest_err) => { 62 | let body = response.text().ok(); 63 | Err(Error::Reqwest { reqwest_err, body }) 64 | } 65 | Ok(_) => Ok(response), 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /google_rest_api_generator/gen_include/iter.rs: -------------------------------------------------------------------------------- 1 | pub mod iter { 2 | pub trait IterableMethod { 3 | fn set_page_token(&mut self, value: String); 4 | fn execute(&mut self) -> Result 5 | where 6 | T: ::serde::de::DeserializeOwned; 7 | } 8 | 9 | pub struct PageIter{ 10 | pub method: M, 11 | pub finished: bool, 12 | pub _phantom: ::std::marker::PhantomData, 13 | } 14 | 15 | impl PageIter 16 | where 17 | M: IterableMethod, 18 | T: ::serde::de::DeserializeOwned, 19 | { 20 | pub(crate) fn new(method: M) -> Self { 21 | PageIter{ 22 | method, 23 | finished: false, 24 | _phantom: ::std::marker::PhantomData, 25 | } 26 | } 27 | } 28 | 29 | impl Iterator for PageIter 30 | where 31 | M: IterableMethod, 32 | T: ::serde::de::DeserializeOwned, 33 | { 34 | type Item = Result; 35 | 36 | fn next(&mut self) -> Option> { 37 | if self.finished { 38 | return None; 39 | } 40 | let paginated_result: ::serde_json::Map = match self.method.execute() { 41 | Ok(r) => r, 42 | Err(err) => return Some(Err(err)), 43 | }; 44 | if let Some(next_page_token) = paginated_result.get("nextPageToken").and_then(|t| t.as_str()) { 45 | self.method.set_page_token(next_page_token.to_owned()); 46 | } else { 47 | self.finished = true; 48 | } 49 | 50 | Some(match ::serde_json::from_value(::serde_json::Value::Object(paginated_result)) { 51 | Ok(resp) => Ok(resp), 52 | Err(err) => Err(err.into()) 53 | }) 54 | } 55 | } 56 | 57 | pub struct PageItemIter{ 58 | items_field: &'static str, 59 | page_iter: PageIter>, 60 | items: ::std::vec::IntoIter, 61 | } 62 | 63 | impl PageItemIter 64 | where 65 | M: IterableMethod, 66 | T: ::serde::de::DeserializeOwned, 67 | { 68 | pub(crate) fn new(method: M, items_field: &'static str) -> Self { 69 | PageItemIter{ 70 | items_field, 71 | page_iter: PageIter::new(method), 72 | items: Vec::new().into_iter(), 73 | } 74 | } 75 | } 76 | 77 | impl Iterator for PageItemIter 78 | where 79 | M: IterableMethod, 80 | T: ::serde::de::DeserializeOwned, 81 | { 82 | type Item = Result; 83 | 84 | fn next(&mut self) -> Option> { 85 | loop { 86 | if let Some(v) = self.items.next() { 87 | return Some(Ok(v)); 88 | } 89 | 90 | let next_page = self.page_iter.next(); 91 | match next_page { 92 | None => return None, 93 | Some(Err(err)) => return Some(Err(err)), 94 | Some(Ok(next_page)) => { 95 | let mut next_page: ::serde_json::Map = next_page; 96 | let items_array = match next_page.remove(self.items_field) { 97 | Some(items) => items, 98 | None => return Some(Err(crate::Error::Other(format!("no {} field found in iter response", self.items_field).into()))), 99 | }; 100 | let items_vec: Result, _> = ::serde_json::from_value(items_array); 101 | match items_vec { 102 | Ok(items) => self.items = items.into_iter(), 103 | Err(err) => return Some(Err(err.into())), 104 | } 105 | } 106 | } 107 | } 108 | } 109 | } 110 | } -------------------------------------------------------------------------------- /google_rest_api_generator/gen_include/multipart.rs: -------------------------------------------------------------------------------- 1 | #[allow(dead_code)] 2 | mod multipart { 3 | pub(crate) struct RelatedMultiPart { 4 | parts: Vec, 5 | boundary: String, 6 | } 7 | 8 | impl RelatedMultiPart { 9 | pub(crate) fn new() -> Self { 10 | RelatedMultiPart { 11 | parts: Vec::new(), 12 | boundary: ::textnonce::TextNonce::sized(68).unwrap().0, 13 | } 14 | } 15 | 16 | pub(crate) fn new_part(&mut self, part: Part) { 17 | self.parts.push(part); 18 | } 19 | 20 | pub(crate) fn boundary(&self) -> &str { 21 | &self.boundary 22 | } 23 | 24 | pub(crate) fn into_reader(self) -> RelatedMultiPartReader { 25 | let boundary_marker = boundary_marker(&self.boundary); 26 | RelatedMultiPartReader { 27 | state: RelatedMultiPartReaderState::WriteBoundary { 28 | start: 0, 29 | boundary: format!("{}\r\n", &boundary_marker), 30 | }, 31 | boundary: boundary_marker, 32 | next_body: None, 33 | parts: self.parts.into_iter(), 34 | } 35 | } 36 | } 37 | 38 | pub(crate) struct Part { 39 | content_type: ::mime::Mime, 40 | body: Box, 41 | } 42 | 43 | impl Part { 44 | pub(crate) fn new( 45 | content_type: ::mime::Mime, 46 | body: Box, 47 | ) -> Part { 48 | Part { content_type, body } 49 | } 50 | } 51 | 52 | pub(crate) struct RelatedMultiPartReader { 53 | state: RelatedMultiPartReaderState, 54 | boundary: String, 55 | next_body: Option>, 56 | parts: std::vec::IntoIter, 57 | } 58 | 59 | enum RelatedMultiPartReaderState { 60 | WriteBoundary { 61 | start: usize, boundary: String, 62 | }, 63 | WriteContentType { 64 | start: usize, 65 | content_type: Vec, 66 | }, 67 | WriteBody { 68 | body: Box, 69 | }, 70 | } 71 | 72 | impl ::std::io::Read for RelatedMultiPartReader { 73 | fn read(&mut self, buf: &mut [u8]) -> ::std::io::Result { 74 | use RelatedMultiPartReaderState::*; 75 | let mut bytes_written: usize = 0; 76 | loop { 77 | let rem_buf = &mut buf[bytes_written..]; 78 | match &mut self.state { 79 | WriteBoundary { start, boundary } => { 80 | let bytes_to_copy = std::cmp::min(boundary.len() - *start, rem_buf.len()); 81 | rem_buf[..bytes_to_copy] 82 | .copy_from_slice(&boundary.as_bytes()[*start..*start + bytes_to_copy]); 83 | *start += bytes_to_copy; 84 | bytes_written += bytes_to_copy; 85 | if *start == boundary.len() { 86 | let next_part = match self.parts.next() { 87 | None => break, 88 | Some(part) => part, 89 | }; 90 | self.next_body = Some(next_part.body); 91 | self.state = WriteContentType { 92 | start: 0, 93 | content_type: format!("Content-Type: {}\r\n\r\n", next_part.content_type) 94 | .into_bytes(), 95 | }; 96 | } else { 97 | break; 98 | } 99 | } 100 | WriteContentType { 101 | start, 102 | content_type, 103 | } => { 104 | let bytes_to_copy = std::cmp::min(content_type.len() - *start, rem_buf.len()); 105 | rem_buf[..bytes_to_copy] 106 | .copy_from_slice(&content_type[*start..*start + bytes_to_copy]); 107 | *start += bytes_to_copy; 108 | bytes_written += bytes_to_copy; 109 | if *start == content_type.len() { 110 | self.state = WriteBody { 111 | body: self.next_body.take().unwrap(), 112 | }; 113 | } else { 114 | break; 115 | } 116 | } 117 | WriteBody { body } => { 118 | let written = body.read(rem_buf)?; 119 | bytes_written += written; 120 | if written == 0 { 121 | self.state = WriteBoundary { 122 | start: 0, 123 | boundary: format!("\r\n{}\r\n", &self.boundary), 124 | }; 125 | } else { 126 | break; 127 | } 128 | } 129 | } 130 | } 131 | Ok(bytes_written) 132 | } 133 | } 134 | 135 | fn boundary_marker(boundary: &str) -> String { 136 | let mut marker = String::with_capacity(boundary.len() + 2); 137 | marker.push_str("--"); 138 | marker.push_str(boundary); 139 | marker 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /google_rest_api_generator/gen_include/parsed_string.rs: -------------------------------------------------------------------------------- 1 | // A serde helper module that can be used with the `with` attribute 2 | // to deserialize any string to a FromStr type and serialize any 3 | // Display type to a String. Google API's encode i64, u64 values as 4 | // strings. 5 | #[allow(dead_code)] 6 | mod parsed_string { 7 | pub fn serialize(value: &Option, serializer: S) -> ::std::result::Result 8 | where 9 | T: ::std::fmt::Display, 10 | S: ::serde::Serializer, 11 | { 12 | use ::serde::Serialize; 13 | value.as_ref().map(|x| x.to_string()).serialize(serializer) 14 | } 15 | 16 | pub fn deserialize<'de, T, D>(deserializer: D) -> ::std::result::Result, D::Error> 17 | where 18 | T: ::std::str::FromStr, 19 | T::Err: ::std::fmt::Display, 20 | D: ::serde::de::Deserializer<'de>, 21 | { 22 | use ::serde::Deserialize; 23 | match Option::::deserialize(deserializer)? { 24 | Some(x) => Ok(Some(x.parse().map_err(::serde::de::Error::custom)?)), 25 | None => Ok(None), 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /google_rest_api_generator/gen_include/percent_encode_consts.rs: -------------------------------------------------------------------------------- 1 | #[allow(dead_code)] 2 | const SIMPLE: &::percent_encoding::AsciiSet = &::percent_encoding::NON_ALPHANUMERIC 3 | .remove(b'-') 4 | .remove(b'.') 5 | .remove(b'_') 6 | .remove(b'~'); 7 | 8 | #[allow(dead_code)] 9 | const RESERVED: &::percent_encoding::AsciiSet = &SIMPLE 10 | .remove(b'%') 11 | .remove(b':') 12 | .remove(b'/') 13 | .remove(b'?') 14 | .remove(b'#') 15 | .remove(b'[') 16 | .remove(b']') 17 | .remove(b'@') 18 | .remove(b'!') 19 | .remove(b'$') 20 | .remove(b'&') 21 | .remove(b'\'') 22 | .remove(b'(') 23 | .remove(b')') 24 | .remove(b'*') 25 | .remove(b'+') 26 | .remove(b',') 27 | .remove(b';') 28 | .remove(b'='); 29 | -------------------------------------------------------------------------------- /google_rest_api_generator/gen_include/resumable_upload.rs: -------------------------------------------------------------------------------- 1 | pub struct ResumableUpload { 2 | reqwest: ::reqwest::blocking::Client, 3 | url: String, 4 | progress: Option, 5 | } 6 | 7 | impl ResumableUpload { 8 | pub fn new(reqwest: ::reqwest::blocking::Client, url: String) -> Self { 9 | ResumableUpload { 10 | reqwest, 11 | url, 12 | progress: None, 13 | } 14 | } 15 | 16 | pub fn url(&self) -> &str { 17 | &self.url 18 | } 19 | 20 | pub fn upload(&mut self, mut reader: R) -> Result<(), Box> 21 | where 22 | R: ::std::io::Read + ::std::io::Seek + Send + 'static, 23 | { 24 | let reader_len = { 25 | let start = reader.seek(::std::io::SeekFrom::Current(0))?; 26 | let end = reader.seek(::std::io::SeekFrom::End(0))?; 27 | reader.seek(::std::io::SeekFrom::Start(start))?; 28 | end 29 | }; 30 | let progress = match self.progress { 31 | Some(progress) => progress, 32 | None => { 33 | let req = self.reqwest.request(::reqwest::Method::PUT, &self.url); 34 | let req = req.header(::reqwest::header::CONTENT_LENGTH, 0); 35 | let req = req.header( 36 | ::reqwest::header::CONTENT_RANGE, 37 | format!("bytes */{}", reader_len), 38 | ); 39 | let resp = req.send()?.error_for_status()?; 40 | match resp.headers().get(::reqwest::header::RANGE) { 41 | Some(range_header) => { 42 | let (_, progress) = parse_range_header(range_header) 43 | .map_err(|e| format!("invalid RANGE header: {}", e))?; 44 | progress + 1 45 | } 46 | None => 0, 47 | } 48 | } 49 | }; 50 | 51 | reader.seek(::std::io::SeekFrom::Start(progress as u64))?; 52 | let content_length = reader_len - progress as u64; 53 | let content_range = format!("bytes {}-{}/{}", progress, reader_len - 1, reader_len); 54 | let req = self.reqwest.request(::reqwest::Method::PUT, &self.url); 55 | let req = req.header(::reqwest::header::CONTENT_RANGE, content_range); 56 | let req = req.body(::reqwest::blocking::Body::sized(reader, content_length)); 57 | req.send()?.error_for_status()?; 58 | Ok(()) 59 | } 60 | } 61 | 62 | fn parse_range_header( 63 | range: &::reqwest::header::HeaderValue, 64 | ) -> Result<(i64, i64), Box> { 65 | let range = range.to_str()?; 66 | if !range.starts_with("bytes ") { 67 | return Err(r#"does not begin with "bytes""#.to_owned().into()); 68 | } 69 | let range = &range[6..]; 70 | let slash_idx = range 71 | .find('/') 72 | .ok_or_else(|| r#"does not contain"#.to_owned())?; 73 | let (begin, end) = range.split_at(slash_idx); 74 | let end = &end[1..]; // remove '/' 75 | let begin: i64 = begin.parse()?; 76 | let end: i64 = end.parse()?; 77 | Ok((begin, end)) 78 | } 79 | -------------------------------------------------------------------------------- /google_rest_api_generator/src/cargo.rs: -------------------------------------------------------------------------------- 1 | const CARGO_TOML: &str = r#" 2 | [package] 3 | name = "{crate_name}" 4 | version = "{crate_version}" 5 | authors = ["Glenn Griffin String { 23 | let mut doc = CARGO_TOML 24 | .trim() 25 | .replace("{crate_name}", crate_name) 26 | .replace( 27 | "{crate_version}", 28 | &api.lib_crate_version 29 | .as_ref() 30 | .expect("available crate version"), 31 | ); 32 | 33 | if include_bytes_dep { 34 | doc.push_str("\n[dependencies.google_api_bytes]\n"); 35 | doc.push_str("git = \"https://github.com/google-apis-rs/generator\"\n"); 36 | } 37 | doc 38 | } 39 | -------------------------------------------------------------------------------- /google_rest_api_generator/src/markdown.rs: -------------------------------------------------------------------------------- 1 | use pulldown_cmark::Parser; 2 | use pulldown_cmark_to_cmark::fmt::cmark; 3 | 4 | /// Currently does the following 5 | /// * look for code blocks, and rewrite them as 'text'. Sometimes, these are code in any other language, thus far never in Rust. 6 | /// Cargo doc will complain (warn) if the code block is not valid Rust, and we don't want to get into the habit of ignoring warnings. 7 | /// On the bright side, cargo doc does exactly what we do, it interprets these blocks as text in the end. 8 | pub fn sanitize(md: &str) -> String { 9 | let mut output = String::with_capacity(2048); 10 | cmark( 11 | Parser::new_ext(&md, pulldown_cmark::Options::all()).map(|e| { 12 | use pulldown_cmark::Event::*; 13 | match e { 14 | Start(ref tag) => { 15 | use pulldown_cmark::Tag::*; 16 | match tag { 17 | CodeBlock(code) => Start(CodeBlock(format!("text{}", code).into())), 18 | _ => e, 19 | } 20 | } 21 | _ => e, 22 | } 23 | }), 24 | &mut output, 25 | None, 26 | ) 27 | .unwrap(); 28 | output 29 | } 30 | -------------------------------------------------------------------------------- /google_rest_api_generator/src/method_actions.rs: -------------------------------------------------------------------------------- 1 | use crate::{markdown, Method, Param, ParamInitMethod}; 2 | use proc_macro2::TokenStream; 3 | use quote::quote; 4 | use syn::parse_quote; 5 | 6 | pub(crate) fn generate(method: &Method, global_params: &[Param]) -> TokenStream { 7 | let method_ident = &method.ident; 8 | let method_builder_type = method.builder_name(); 9 | let mut required_args: Vec = Vec::new(); 10 | let mut method_builder_initializers: Vec = Vec::new(); 11 | if let Some(req) = method.request.as_ref() { 12 | let ty = req.type_path(); 13 | required_args.push(parse_quote! {request: #ty}); 14 | method_builder_initializers.push(parse_quote! {request}); 15 | } 16 | required_args.extend(method.params.iter().filter(|p| p.required).map(|param| { 17 | let name = ¶m.ident; 18 | let init_method: syn::FnArg = match param.init_method() { 19 | ParamInitMethod::BytesInit => parse_quote! {#name: impl Into>}, 20 | ParamInitMethod::IntoImpl(into_typ) => parse_quote! {#name: impl Into<#into_typ>}, 21 | ParamInitMethod::ByValue => { 22 | let ty = param.typ.type_path(); 23 | parse_quote! {#name: #ty} 24 | } 25 | }; 26 | init_method 27 | })); 28 | let all_params = global_params.into_iter().chain(method.params.iter()); 29 | method_builder_initializers.extend(all_params.map(|param| { 30 | let name = ¶m.ident; 31 | let field_pattern: syn::FieldValue = if param.required { 32 | match param.init_method() { 33 | ParamInitMethod::BytesInit => parse_quote! {#name: #name.into().into()}, 34 | ParamInitMethod::IntoImpl(_) => parse_quote! {#name: #name.into()}, 35 | ParamInitMethod::ByValue => parse_quote! {#name}, 36 | } 37 | } else { 38 | parse_quote! {#name: None} 39 | }; 40 | field_pattern 41 | })); 42 | let method_description = &method 43 | .description 44 | .as_ref() 45 | .map(|s| markdown::sanitize(s.as_str())) 46 | .unwrap_or_else(String::new); 47 | quote! { 48 | #[doc = #method_description] 49 | pub fn #method_ident(&self#(, #required_args)*) -> #method_builder_type { 50 | #method_builder_type{ 51 | reqwest: &self.reqwest, 52 | auth: self.auth_ref(), 53 | #(#method_builder_initializers,)* 54 | } 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /google_rest_api_generator/src/package_doc.rs: -------------------------------------------------------------------------------- 1 | use crate::{APIDesc, Resource}; 2 | use std::fmt; 3 | 4 | pub(crate) fn generate(api_desc: &APIDesc) -> String { 5 | let mut output = "# Resources and Methods\n".to_owned(); 6 | for resource in &api_desc.resources { 7 | generate_resource(resource, &mut output); 8 | } 9 | output 10 | } 11 | 12 | fn generate_resource(resource: &Resource, output: &mut String) { 13 | let mod_path = module_path(resource); 14 | let indent_amount = resource.parent_path.segments.len(); 15 | use fmt::Write; 16 | for _ in 0..indent_amount * 2 { 17 | output.push(' '); 18 | } 19 | output.push_str("* "); 20 | write!( 21 | output, 22 | "[{}]({}/struct.{}.html)\n", 23 | &resource.ident, 24 | &mod_path, 25 | resource.action_type_name() 26 | ) 27 | .unwrap(); 28 | if !resource.methods.is_empty() { 29 | for _ in 0..(indent_amount + 1) * 2 { 30 | output.push(' '); 31 | } 32 | output.push_str("* "); 33 | let mut first_method = true; 34 | for method in &resource.methods { 35 | if !first_method { 36 | output.push_str(", "); 37 | } 38 | write!( 39 | output, 40 | "[*{}*]({}/struct.{}.html)", 41 | &method.id, 42 | &mod_path, 43 | method.builder_name() 44 | ) 45 | .unwrap(); 46 | first_method = false; 47 | } 48 | output.push('\n'); 49 | } 50 | for resource in &resource.resources { 51 | generate_resource(resource, output); 52 | } 53 | } 54 | 55 | fn module_path(resource: &Resource) -> String { 56 | use std::fmt::Write; 57 | let mut output = String::new(); 58 | for seg in resource.parent_path.segments.iter().skip(1) { 59 | write!(&mut output, "{}/", &seg.ident).unwrap(); 60 | } 61 | write!(&mut output, "{}", &resource.ident).unwrap(); 62 | output 63 | } 64 | -------------------------------------------------------------------------------- /google_rest_api_generator/src/path_templates.rs: -------------------------------------------------------------------------------- 1 | // Path templates are uri templates as described in RFC 6570. However they are 2 | // only used to define paths in google api's meaning the implementation can be 3 | // simplified greatly if we only support the subset of the uri template syntax 4 | // used by google apis. The defined subset is that only the simple {var} and 5 | // reserved {+var} operator is supported, with no modifiers (prefix or explode). 6 | // We use the uri_template_parser crate to parse the template into an AST, 7 | // validate that the AST conforms to the subset supported in path templates, and 8 | // then generate code to define the path based on the parameters in use by the 9 | // method. 10 | use uri_template_parser as parser; 11 | 12 | #[derive(Debug, Clone, Eq, PartialEq, Hash, Ord, PartialOrd)] 13 | pub(crate) struct PathTemplate<'a> { 14 | nodes: Vec>, 15 | } 16 | 17 | #[derive(Debug, Clone, Eq, PartialEq, Hash, Ord, PartialOrd)] 18 | pub(crate) enum PathAstNode<'a> { 19 | Lit(&'a str), 20 | Var { 21 | var_name: &'a str, 22 | expansion_style: ExpansionStyle, 23 | }, 24 | } 25 | 26 | #[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd)] 27 | pub(crate) enum ExpansionStyle { 28 | Simple { prefix: Option }, 29 | Reserved { prefix: Option }, 30 | PathSegment, // implies explode (atleast for now). 31 | } 32 | 33 | impl<'a> PathAstNode<'a> { 34 | fn from_parser_ast_node(n: parser::AstNode<'a>) -> Result, String> { 35 | Ok(match n { 36 | parser::AstNode::Lit(lit) => PathAstNode::Lit(lit), 37 | parser::AstNode::Expr(expr) => { 38 | if expr.var_spec_list.len() != 1 { 39 | return Err(format!( 40 | "Unsupported number of variables in uri template varspec: {}", 41 | expr.var_spec_list.len() 42 | )); 43 | } 44 | let var_spec = &expr.var_spec_list[0]; 45 | let expansion_style = match (expr.operator, var_spec.modifier) { 46 | (parser::Operator::Simple, parser::Modifier::NoModifier) => { 47 | ExpansionStyle::Simple { prefix: None } 48 | } 49 | (parser::Operator::Simple, parser::Modifier::Prefix(prefix)) => { 50 | ExpansionStyle::Simple { 51 | prefix: Some(prefix), 52 | } 53 | } 54 | (parser::Operator::Reserved, parser::Modifier::NoModifier) => { 55 | ExpansionStyle::Reserved { prefix: None } 56 | } 57 | (parser::Operator::Reserved, parser::Modifier::Prefix(prefix)) => { 58 | ExpansionStyle::Reserved { 59 | prefix: Some(prefix), 60 | } 61 | } 62 | (parser::Operator::PathSegment, parser::Modifier::Explode) => { 63 | ExpansionStyle::PathSegment 64 | } 65 | (operator, modifier) => { 66 | return Err(format!( 67 | "Unsupported uri template: op: {:?} mod: {:?}", 68 | operator, modifier 69 | )) 70 | } 71 | }; 72 | PathAstNode::Var { 73 | var_name: var_spec.var_name, 74 | expansion_style, 75 | } 76 | } 77 | }) 78 | } 79 | } 80 | 81 | impl<'a> PathTemplate<'a> { 82 | pub(crate) fn new(tmpl: &str) -> Result { 83 | let nodes = parser::ast_nodes(tmpl) 84 | .ok_or_else(|| "Failed to parse uri template".to_owned())? 85 | .into_iter() 86 | .map(PathAstNode::from_parser_ast_node) 87 | .collect::, String>>()?; 88 | Ok(PathTemplate { nodes }) 89 | } 90 | 91 | pub(crate) fn nodes(&self) -> impl Iterator { 92 | self.nodes.iter() 93 | } 94 | } 95 | 96 | #[cfg(test)] 97 | mod tests { 98 | use super::*; 99 | 100 | #[test] 101 | fn test_template() { 102 | assert_eq!( 103 | PathTemplate::new("foobar"), 104 | Ok(PathTemplate { 105 | nodes: vec![PathAstNode::Lit("foobar"),] 106 | }) 107 | ); 108 | assert_eq!( 109 | PathTemplate::new("{project}/managedZones/{+managedZone}/changes"), 110 | Ok(PathTemplate { 111 | nodes: vec![ 112 | PathAstNode::Var { 113 | var_name: "project", 114 | expansion_style: ExpansionStyle::Simple { prefix: None } 115 | }, 116 | PathAstNode::Lit("/managedZones/"), 117 | PathAstNode::Var { 118 | var_name: "managedZone", 119 | expansion_style: ExpansionStyle::Reserved { prefix: None } 120 | }, 121 | PathAstNode::Lit("/changes"), 122 | ], 123 | }) 124 | ); 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /google_rest_api_generator/src/resource_actions.rs: -------------------------------------------------------------------------------- 1 | use crate::Resource; 2 | use proc_macro2::TokenStream; 3 | use quote::quote; 4 | 5 | pub(crate) fn generate(resource: &Resource) -> TokenStream { 6 | let parent_path = &resource.parent_path; 7 | let resource_ident = &resource.ident; 8 | let action_ident = resource.action_type_name(); 9 | let description = format!( 10 | "Actions that can be performed on the {} resource", 11 | &resource.ident 12 | ); 13 | quote! { 14 | #[doc= #description] 15 | pub fn #resource_ident(&self) -> #parent_path::#resource_ident::#action_ident { 16 | #parent_path::#resource_ident::#action_ident{ 17 | reqwest: &self.reqwest, 18 | auth: self.auth_ref(), 19 | } 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /google_rest_api_generator/src/resource_builder.rs: -------------------------------------------------------------------------------- 1 | use crate::{method_actions, method_builder, Param, Resource, Type}; 2 | use proc_macro2::TokenStream; 3 | use quote::quote; 4 | use std::collections::BTreeMap; 5 | 6 | pub(crate) fn generate( 7 | root_url: &str, 8 | service_path: &str, 9 | global_params: &[Param], 10 | resource: &Resource, 11 | schemas: &BTreeMap, 12 | ) -> TokenStream { 13 | let ident = &resource.ident; 14 | let param_type_defs = resource 15 | .methods 16 | .iter() 17 | .flat_map(|method| method.params.iter()) 18 | .fold(Vec::new(), |accum, param| { 19 | param.typ.fold_nested(accum, |mut accum, typ| { 20 | if let Some(type_def) = typ.type_def(schemas) { 21 | accum.push(type_def); 22 | } 23 | accum 24 | }) 25 | }); 26 | let method_builders = resource.methods.iter().map(|method| { 27 | method_builder::generate( 28 | root_url, 29 | service_path, 30 | global_params, 31 | method, 32 | &resource.action_type_name(), 33 | schemas, 34 | ) 35 | }); 36 | let nested_resource_mods = resource 37 | .resources 38 | .iter() 39 | .map(|resource| generate(root_url, service_path, global_params, resource, schemas)); 40 | 41 | let method_actions = resource 42 | .methods 43 | .iter() 44 | .map(|method| method_actions::generate(method, global_params)); 45 | let nested_resource_actions = resource 46 | .resources 47 | .iter() 48 | .map(|sub_resource| crate::resource_actions::generate(sub_resource)); 49 | let action_ident = resource.action_type_name(); 50 | quote! { 51 | pub mod #ident { 52 | pub mod params { 53 | #(#param_type_defs)* 54 | } 55 | 56 | pub struct #action_ident<'a> { 57 | pub(crate) reqwest: &'a reqwest::blocking::Client, 58 | pub(crate) auth: &'a dyn ::google_api_auth::GetAccessToken, 59 | } 60 | impl<'a> #action_ident<'a> { 61 | fn auth_ref(&self) -> &dyn ::google_api_auth::GetAccessToken { 62 | self.auth 63 | } 64 | 65 | #(#method_actions)* 66 | #(#nested_resource_actions)* 67 | } 68 | 69 | #(#method_builders)* 70 | #(#nested_resource_mods)* 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- 1 | api_index_path := "../generated/etc/api-index.v1.json" 2 | mcpd := "target/debug/mcp" 3 | 4 | # run *all* the tests 5 | tests: mcp-tests cargo-tests 6 | 7 | # run all tests for the 'master control program' 8 | mcp-tests: mcpd 9 | tests/mcp/journey-tests.sh {{mcpd}} 10 | 11 | # run all tests driven by cargo 12 | cargo-tests: 13 | cargo test --tests --examples --all-features 14 | 15 | # update everything that was generated in repository 16 | update-generated-fixtures: discovery-spec known-versions-fixture discovery-rs 17 | 18 | # build the master control program in debug mode 19 | mcpd: 20 | cargo build 21 | 22 | # fetch the spec used as fixture in our tests 23 | discovery-spec: 24 | curl https://www.googleapis.com/discovery/v1/apis/admin/directory_v1/rest -o discovery_parser/tests/spec.json 25 | curl https://www.googleapis.com/discovery/v1/apis/youtube/v3/rest -o google_rest_api_generator/tests/spec.json 26 | 27 | # Update a fixture with all API versions encountered in the Google API index 28 | known-versions-fixture: 29 | # version 1.6 known to be working 30 | jq -r '.items[].version' < {{api_index_path}} | sort | uniq > shared/tests/fixtures/known-versions 31 | 32 | # A generated file with types supported the deserializtion of the Google API index 33 | discovery-rs: 34 | # version 15.0.199 known to be working 35 | quicktype --lang rust --visibility=public {{api_index_path}} > discovery_parser/src/discovery.rs 36 | 37 | -------------------------------------------------------------------------------- /mcp/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "mcp" 3 | version = "0.1.0" 4 | authors = ["Sebastian Thiel "] 5 | edition = "2018" 6 | description = "A 'master control program' for handling generation of Google APIs" 7 | exclude = ["src/shared/snapshots/"] 8 | publish = false 9 | 10 | [[bin]] 11 | name = "mcp" 12 | doctest = false 13 | 14 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 15 | 16 | [dependencies] 17 | discovery_parser = { path = "../discovery_parser" } 18 | google_cli_generator = { path = "../google_cli_generator" } 19 | cargo_log_parser = { path = "../cargo_log_parser" } 20 | shared = { path = "../shared" } 21 | templating = { path = "../templating" } 22 | clap = "2.33.0" 23 | failure = "0.1.5" 24 | itertools = "0.8.0" 25 | atty = "0.2.13" 26 | failure-tools = "4.0.2" 27 | structopt = "0.3" 28 | serde_json = "1.0.40" 29 | log = "0.4.8" 30 | simple_logger = "1.3.0" 31 | reqwest = "0.9.19" 32 | rayon = "1.1.0" 33 | serde = "1.0.99" 34 | ci_info = "0.8.0" 35 | nom = "5.0.1" 36 | 37 | -------------------------------------------------------------------------------- /mcp/src/cmds/cargo_errors.rs: -------------------------------------------------------------------------------- 1 | use crate::options::cargo_errors::Args; 2 | use cargo_log_parser::parse_errors; 3 | use failure::{bail, format_err, Error, ResultExt}; 4 | use log::{error, info, warn}; 5 | use shared::{Api, MappedIndex}; 6 | use std::{ 7 | ffi::OsString, 8 | fs, 9 | io::{self, Read, Write}, 10 | path::Path, 11 | process::{Command, Stdio}, 12 | }; 13 | 14 | pub fn execute( 15 | Args { 16 | index_path, 17 | cargo_manifest_path, 18 | output_directory, 19 | cargo_arguments, 20 | }: Args, 21 | ) -> Result<(), Error> { 22 | let mut last_excludes_len = 0; 23 | let index: MappedIndex = serde_json::from_slice(&fs::read(index_path)?)?; 24 | let mut excludes = Vec::<&Api>::new(); 25 | let filter_parse_result = |parsed: Vec| { 26 | parsed 27 | .into_iter() 28 | .map(|c| c.name) 29 | .filter_map(|n| index.api.iter().find(|api| api.lib_crate_name == n)) 30 | }; 31 | 32 | loop { 33 | let mut args = cargo_arguments.clone(); 34 | args.push("--all".into()); 35 | args.push("--manifest-path".into()); 36 | args.push(cargo_manifest_path.clone().into()); 37 | args.extend( 38 | excludes 39 | .iter() 40 | .map(|api| format!("--exclude={}", api.lib_crate_name).into()), 41 | ); 42 | command_info("", &args); 43 | let mut cargo = Command::new("cargo") 44 | .args(&args) 45 | .stderr(Stdio::piped()) 46 | .stdout(Stdio::inherit()) 47 | .stdin(Stdio::null()) 48 | .spawn() 49 | .with_context(|_| "failed to launch cargo")?; 50 | 51 | let mut input = Vec::new(); 52 | let mut print_from = 0_usize; 53 | loop { 54 | let written_bytes = io::stderr().write(&input[print_from..])?; 55 | print_from += written_bytes; 56 | 57 | let to_read = match parse_errors(&input).map(|(i, r)| (i.len(), r)) { 58 | Ok((input_left_len, parsed)) => { 59 | let input_len = input.len(); 60 | input = input.into_iter().skip(input_len - input_left_len).collect(); 61 | print_from = 0; 62 | excludes.extend((filter_parse_result)(parsed)); 63 | 128 64 | } 65 | Err(nom::Err::Incomplete(needed)) => { 66 | match needed { 67 | nom::Needed::Unknown => 1, // read one byte 68 | nom::Needed::Size(len) => len, 69 | } 70 | } 71 | Err(nom::Err::Failure(_e)) | Err(nom::Err::Error(_e)) => { 72 | bail!("TODO: proper error conversion if parsing really fails") 73 | } 74 | }; 75 | 76 | if let Some(_) = cargo.try_wait()? { 77 | break; 78 | } 79 | 80 | if let Err(e) = cargo 81 | .stderr 82 | .as_mut() 83 | .expect("cargo_output is set") 84 | .take(to_read as u64) 85 | .read_to_end(&mut input) 86 | { 87 | error!("Failed to read cargo output: {}", e); 88 | break; 89 | } 90 | } 91 | 92 | cargo 93 | .stderr 94 | .as_mut() 95 | .expect("cargo_output is set") 96 | .read_to_end(&mut input)?; 97 | 98 | io::stderr().write(&input[print_from..])?; 99 | 100 | match parse_errors(&input) { 101 | Ok((_, parsed)) => { 102 | excludes.extend(filter_parse_result(parsed)); 103 | } 104 | Err(nom::Err::Error(e)) | Err(nom::Err::Failure(e)) => { 105 | error!("Ignoring parse error after cargo ended: {:?}", e.1); 106 | } 107 | Err(nom::Err::Incomplete(_)) => panic!( 108 | "Could not parse remaining input: {:?}", 109 | std::str::from_utf8(&input) 110 | ), 111 | }; 112 | 113 | collect_errors( 114 | &excludes[last_excludes_len..], 115 | &cargo_arguments, 116 | output_directory.as_path(), 117 | )?; 118 | 119 | let workspace_cargo_status = cargo.try_wait()?.expect("cargo ended"); 120 | 121 | if workspace_cargo_status.success() { 122 | info!("Cargo finished successfully."); 123 | if !excludes.is_empty() { 124 | info!( 125 | "Recorded errors for the following workspace members: {:?}", 126 | excludes 127 | .iter() 128 | .map(|a| &a.lib_crate_name) 129 | .collect::>() 130 | ); 131 | } 132 | return Ok(()); 133 | } else { 134 | if last_excludes_len == excludes.len() { 135 | bail!( 136 | "cargo seems to fail permanently and makes no progress: {:?}. Probably we cannot parse crate names from the error output.", 137 | workspace_cargo_status 138 | ); 139 | } 140 | last_excludes_len = excludes.len(); 141 | } 142 | } 143 | } 144 | 145 | fn command_info(prefix: &str, args: &[OsString]) { 146 | info!( 147 | "({}) Running 'cargo {}'", 148 | prefix, 149 | args.iter() 150 | .map(|o| o.to_string_lossy()) 151 | .collect::>() 152 | .join(" ") 153 | ); 154 | } 155 | 156 | fn collect_errors( 157 | apis: &[&Api], 158 | cargo_arguments: &[OsString], 159 | output_directory: &Path, 160 | ) -> Result<(), Error> { 161 | for api in apis { 162 | let mut args = cargo_arguments.to_owned(); 163 | let manifest_path = output_directory.join(&api.lib_cargo_file); 164 | args.push("--manifest-path".into()); 165 | args.push(manifest_path.into()); 166 | command_info(&api.lib_crate_name, &args); 167 | 168 | let output = Command::new("cargo") 169 | .args(&args) 170 | .stderr(Stdio::piped()) 171 | .stdout(Stdio::piped()) 172 | .stdin(Stdio::null()) 173 | .output() 174 | .with_context(|_| "failed to launch cargo and collect output")?; 175 | let output_path = output_directory.join(&api.cargo_error_file); 176 | if output.status.success() { 177 | warn!("Command succeeded unexpectedly - no file error log written."); 178 | continue; 179 | } 180 | let mut fh = fs::OpenOptions::new() 181 | .write(true) 182 | .create(true) 183 | .truncate(true) 184 | .open(&output_path) 185 | .with_context(|_| { 186 | format_err!("Could not output path at '{}'", output_path.display()) 187 | })?; 188 | fh.write_all(&output.stderr)?; 189 | fh.write_all(&output.stdout)?; 190 | fh.flush()?; 191 | info!("Wrote cargo error log to '{}'", output_path.display()); 192 | } 193 | Ok(()) 194 | } 195 | -------------------------------------------------------------------------------- /mcp/src/cmds/completions.rs: -------------------------------------------------------------------------------- 1 | use crate::options::completions::Args; 2 | use clap::{App, Shell}; 3 | use failure::{err_msg, format_err, Error, ResultExt}; 4 | use std::{io::stdout, path::Path, str::FromStr}; 5 | 6 | pub fn execute(mut app: App, Args { shell }: Args) -> Result<(), Error> { 7 | let shell = Path::new(&shell) 8 | .file_name() 9 | .and_then(|f| f.to_str()) 10 | .or_else(|| shell.to_str()) 11 | .ok_or_else(|| { 12 | format_err!( 13 | "'{}' as shell string contains invalid characters", 14 | shell.to_string_lossy() 15 | ) 16 | }) 17 | .and_then(|s| { 18 | Shell::from_str(s) 19 | .map_err(err_msg) 20 | .with_context(|_| format!("The shell '{}' is unsupported", s)) 21 | .map_err(Into::into) 22 | })?; 23 | app.gen_completions_to(crate::PROGRAM_NAME, shell, &mut stdout()); 24 | Ok(()) 25 | } 26 | -------------------------------------------------------------------------------- /mcp/src/cmds/fetch_specs.rs: -------------------------------------------------------------------------------- 1 | use super::util::{log_error_and_continue, logged_write}; 2 | use crate::options::fetch_specs::Args; 3 | use ci_info; 4 | use discovery_parser::{generated::ApiIndexV1, DiscoveryRestDesc, RestDescOrErr}; 5 | use failure::{err_msg, format_err, Error, ResultExt}; 6 | use google_cli_generator::{all, CombinedMetadata as Metadata}; 7 | use log::info; 8 | use rayon::prelude::*; 9 | use shared::{Api, MappedIndex, SkipIfErrorIsPresent}; 10 | use std::{convert::TryFrom, convert::TryInto, fs, io, path::Path, time::Instant}; 11 | 12 | fn write_artifacts( 13 | api: &Api, 14 | spec: DiscoveryRestDesc, 15 | output_dir: &Path, 16 | ) -> Result { 17 | let spec_path = output_dir.join(&api.spec_file); 18 | fs::create_dir_all( 19 | &spec_path 20 | .parent() 21 | .ok_or_else(|| format_err!("invalid spec path - needs parent"))?, 22 | ) 23 | .with_context(|_| { 24 | format_err!( 25 | "Could not create artifact output directory at '{}'", 26 | output_dir.display() 27 | ) 28 | })?; 29 | 30 | // TODO: if no additional processing is done on the data, just pass it as String to avoid 31 | // ser-de. This is not relevant for performance, but can simplify code a bit. 32 | logged_write( 33 | &spec_path, 34 | serde_json::to_string_pretty(&spec)?.as_bytes(), 35 | "spec", 36 | )?; 37 | Ok(spec) 38 | } 39 | 40 | fn fetch_spec(api: &Api) -> Result { 41 | reqwest::get(&api.rest_url) 42 | .with_context(|_| format_err!("Could not fetch spec from '{}'", api.rest_url)) 43 | .map_err(Error::from) 44 | .and_then(|mut r: reqwest::Response| { 45 | let res: RestDescOrErr = r.json().with_context(|_| { 46 | format_err!("Could not deserialize spec at '{}'", api.rest_url) 47 | })?; 48 | match res { 49 | RestDescOrErr::RestDesc(v) => Ok(v), 50 | RestDescOrErr::Err(err) => Err(format_err!("{:?}", err.error)), 51 | } 52 | }) 53 | .with_context(|_| format_err!("Error fetching spec from '{}'", api.rest_url)) 54 | .map_err(Into::into) 55 | } 56 | 57 | fn generate_code( 58 | desc: DiscoveryRestDesc, 59 | info: &ci_info::types::CiInfo, 60 | spec_directory: &Path, 61 | output_directory: &Path, 62 | ) -> Result { 63 | let api = Api::try_from(&desc)?.validated( 64 | info, 65 | spec_directory, 66 | output_directory, 67 | SkipIfErrorIsPresent::Generator, 68 | )?; 69 | let should_generate = (|| -> Result<_, Error> { 70 | let cargo_path = output_directory.join(&api.lib_cargo_file); 71 | if !cargo_path.exists() { 72 | info!( 73 | "Need to generate '{}' as it was never generated before.", 74 | api.lib_crate_name 75 | ); 76 | return Ok(true); 77 | } 78 | let metadata_path = output_directory.join(&api.metadata_file); 79 | let previous_metadata = fs::read(&metadata_path) 80 | .map_err(Error::from) 81 | .and_then(|data| serde_json::from_slice::(&data).map_err(Error::from)) 82 | .ok(); 83 | let current_metadata = Metadata::default(); 84 | if previous_metadata.as_ref() != Some(¤t_metadata) { 85 | info!("Generator changed for '{}'. Last generated content stamped with {:?}, latest version is {:?}", api.lib_crate_name, previous_metadata, current_metadata); 86 | return Ok(true); 87 | } 88 | let spec_path = spec_directory.join(&api.spec_file); 89 | let buf = match fs::read(&spec_path) { 90 | Ok(v) => Ok(v), 91 | Err(ref e) if e.kind() == io::ErrorKind::NotFound => return Ok(true), 92 | Err(e) => Err(e).with_context(|_| { 93 | format_err!("Could not read spec file at '{}'", spec_path.display()) 94 | }), 95 | }?; 96 | let local_api: DiscoveryRestDesc = serde_json::from_slice(&buf)?; 97 | Ok(local_api != desc) 98 | })()?; 99 | if !should_generate { 100 | info!("Skipping generation of '{}' as it is up to date", api.id); 101 | return Ok(desc); 102 | } 103 | all::generate( 104 | &desc, 105 | output_directory.join(&api.gen_dir), 106 | all::Build::ApiAndCli, // TODO: change this to the parallel mode once error handling works correctly 107 | ) 108 | .map_err(|e| { 109 | let error = e.to_string(); 110 | let error_path = output_directory.join(api.gen_error_file); 111 | fs::write(&error_path, &error).ok(); 112 | info!( 113 | "Api '{}' failed to generate, marked it at '{}'", 114 | api.id, 115 | error_path.display() 116 | ); 117 | err_msg(error) 118 | })?; 119 | Ok(desc) 120 | } 121 | 122 | pub fn execute( 123 | Args { 124 | index_path, 125 | spec_directory, 126 | output_directory, 127 | }: Args, 128 | ) -> Result<(), Error> { 129 | let input = fs::read_to_string(&index_path)?; 130 | let index: MappedIndex = serde_json::from_str::(&input) 131 | .map_err(Error::from) 132 | .and_then(TryInto::try_into) 133 | .or_else(|_| serde_json::from_str(&input)) 134 | .map_err(Error::from) 135 | .with_context(|_| { 136 | format_err!( 137 | "Could not read google api index at '{}'", 138 | index_path.display() 139 | ) 140 | })?; 141 | let time = Instant::now(); 142 | let info = ci_info::get(); 143 | index 144 | .api 145 | .par_iter() 146 | .map(|api| fetch_spec(api).map(|r| (api, r))) 147 | .filter_map(log_error_and_continue) 148 | .map(|(api, v)| { 149 | generate_code(v, &info, &spec_directory, &output_directory).map(|v| (api, v)) 150 | }) 151 | .filter_map(log_error_and_continue) 152 | .map(|(api, v)| write_artifacts(api, v, &spec_directory)) 153 | .filter_map(log_error_and_continue) 154 | .for_each(|api| info!("Successfully processed {}:{}", api.name, api.version)); 155 | info!( 156 | "Fetched and generated {} specs in {}s", 157 | index.api.len(), 158 | time.elapsed().as_secs() 159 | ); 160 | Ok(()) 161 | } 162 | -------------------------------------------------------------------------------- /mcp/src/cmds/generate.rs: -------------------------------------------------------------------------------- 1 | use crate::options::generate::Args; 2 | use discovery_parser::DiscoveryRestDesc; 3 | use failure::{format_err, Error, ResultExt}; 4 | use google_cli_generator::all; 5 | use std::fs; 6 | 7 | pub fn execute( 8 | Args { 9 | spec_json_path, 10 | output_directory, 11 | }: Args, 12 | ) -> Result<(), Error> { 13 | let desc: DiscoveryRestDesc = { serde_json::from_slice(&fs::read(&spec_json_path)?) } 14 | .with_context(|_| format_err!("Could read spec file at '{}'", spec_json_path.display()))?; 15 | 16 | all::generate(&desc, output_directory, all::Build::ApiAndCli) 17 | .map_err(|e| format_err!("{}", e.to_string()))?; 18 | Ok(()) 19 | } 20 | -------------------------------------------------------------------------------- /mcp/src/cmds/map_index.rs: -------------------------------------------------------------------------------- 1 | use super::util::logged_write; 2 | use crate::options::map_index::Args; 3 | use discovery_parser::generated::ApiIndexV1; 4 | use failure::{format_err, Error, ResultExt}; 5 | use shared::MappedIndex; 6 | use std::{convert::TryFrom, fs}; 7 | 8 | pub fn execute( 9 | Args { 10 | discovery_json_path, 11 | output_file, 12 | spec_directory, 13 | output_directory, 14 | }: Args, 15 | ) -> Result<(), Error> { 16 | let index: ApiIndexV1 = { serde_json::from_slice(&fs::read(&discovery_json_path)?) } 17 | .with_context(|_| { 18 | format_err!( 19 | "Could read spec file at '{}'", 20 | discovery_json_path.display() 21 | ) 22 | })?; 23 | 24 | let index: MappedIndex = 25 | MappedIndex::try_from(index)?.validated(&spec_directory, &output_directory); 26 | logged_write( 27 | output_file, 28 | serde_json::to_string_pretty(&index)?.as_bytes(), 29 | "mapped api index", 30 | ) 31 | } 32 | -------------------------------------------------------------------------------- /mcp/src/cmds/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod cargo_errors; 2 | pub mod completions; 3 | pub mod fetch_specs; 4 | pub mod generate; 5 | pub mod map_index; 6 | pub mod substitute; 7 | mod util; 8 | -------------------------------------------------------------------------------- /mcp/src/cmds/substitute.rs: -------------------------------------------------------------------------------- 1 | use crate::options::substitute::Args; 2 | use failure::Error; 3 | use itertools::Itertools; 4 | use templating::StreamOrPath; 5 | 6 | use templating::substitute; 7 | 8 | pub fn execute(args: Args) -> Result<(), Error> { 9 | let args = args.sanitized()?; 10 | let replacements: Vec<_> = args.replacements.into_iter().tuples().collect(); 11 | substitute( 12 | &args.data.unwrap_or(StreamOrPath::Stream), 13 | &args.specs, 14 | &args.separator, 15 | args.validate, 16 | &replacements, 17 | ) 18 | } 19 | -------------------------------------------------------------------------------- /mcp/src/cmds/util.rs: -------------------------------------------------------------------------------- 1 | use failure::{format_err, Error, ResultExt}; 2 | use failure_tools::print_causes; 3 | use log::{error, info}; 4 | use std::{fs, path::Path}; 5 | 6 | pub fn log_error_and_continue>(r: Result) -> Option { 7 | match r { 8 | Ok(v) => Some(v), 9 | Err(e) => { 10 | let e = e.into(); 11 | let mut buf = Vec::new(); 12 | let e_display = e.to_string(); 13 | print_causes(e, &mut buf); 14 | error!("{}", String::from_utf8(buf).unwrap_or(e_display)); 15 | None 16 | } 17 | } 18 | } 19 | 20 | pub fn logged_write, C: AsRef<[u8]>>( 21 | path: P, 22 | contents: C, 23 | kind: &str, 24 | ) -> Result<(), Error> { 25 | fs::write(path.as_ref(), contents).with_context(|_| { 26 | format_err!( 27 | "Could not write {kind} file at '{}'", 28 | path.as_ref().display(), 29 | kind = kind, 30 | ) 31 | })?; 32 | info!( 33 | "Wrote file {kind} at '{}'", 34 | path.as_ref().display(), 35 | kind = kind 36 | ); 37 | Ok(()) 38 | } 39 | -------------------------------------------------------------------------------- /mcp/src/main.rs: -------------------------------------------------------------------------------- 1 | use structopt::StructOpt; 2 | 3 | const PROGRAM_NAME: &str = "mcp"; 4 | 5 | mod cmds; 6 | /// taken from share-secrets-safely/tools 7 | mod options; 8 | 9 | use options::Args; 10 | use options::SubCommand::*; 11 | 12 | fn main() { 13 | let args = Args::from_args(); 14 | simple_logger::init_with_level(args.log_level).ok(); 15 | let res = match args.cmd { 16 | CargoErrors(args) => cmds::cargo_errors::execute(args), 17 | MapApiIndex(args) => cmds::map_index::execute(args), 18 | Completions(args) => cmds::completions::execute(Args::clap(), args), 19 | FetchApiSpecs(args) => cmds::fetch_specs::execute(args), 20 | Generate(args) => cmds::generate::execute(args), 21 | Substitute(args) => cmds::substitute::execute(args), 22 | }; 23 | failure_tools::ok_or_exit(res); 24 | } 25 | -------------------------------------------------------------------------------- /mcp/src/options/cargo_errors.rs: -------------------------------------------------------------------------------- 1 | use std::{ffi::OsString, path::PathBuf}; 2 | use structopt::StructOpt; 3 | 4 | #[derive(Debug, StructOpt)] 5 | #[structopt(setting = structopt::clap::AppSettings::ColoredHelp)] 6 | pub struct Args { 7 | /// The mapped index with information about APIs. It should correspond to the Cargo workspace file. 8 | #[structopt(parse(from_os_str))] 9 | pub index_path: PathBuf, 10 | 11 | /// The path to the cargo.toml file defining the workspace for all generated API code 12 | #[structopt(parse(from_os_str))] 13 | pub cargo_manifest_path: PathBuf, 14 | 15 | /// The directory into which we will wrote the generated APIs for dumping error information 16 | #[structopt(parse(from_os_str))] 17 | pub output_directory: PathBuf, 18 | 19 | /// All arguments to be provided to cargo 20 | #[structopt(parse(from_os_str))] 21 | #[structopt(min_values = 1)] 22 | pub cargo_arguments: Vec, 23 | } 24 | -------------------------------------------------------------------------------- /mcp/src/options/completions.rs: -------------------------------------------------------------------------------- 1 | use std::ffi::OsString; 2 | use structopt::StructOpt; 3 | 4 | #[derive(Debug, StructOpt)] 5 | #[structopt(setting = structopt::clap::AppSettings::ColoredHelp)] 6 | pub struct Args { 7 | /// The name of the shell, or the path to the shell as exposed by the $SHELL variable 8 | #[structopt(parse(from_os_str))] 9 | pub shell: OsString, 10 | } 11 | -------------------------------------------------------------------------------- /mcp/src/options/fetch_specs.rs: -------------------------------------------------------------------------------- 1 | use std::path::PathBuf; 2 | use structopt::StructOpt; 3 | 4 | #[derive(Debug, StructOpt)] 5 | #[structopt(setting = structopt::clap::AppSettings::ColoredHelp)] 6 | pub struct Args { 7 | /// Either the original Google index, or the mapped index we produced prior 8 | #[structopt(parse(from_os_str))] 9 | pub index_path: PathBuf, 10 | 11 | /// The directory into which we will write all downloaded specifications 12 | #[structopt(parse(from_os_str))] 13 | pub spec_directory: PathBuf, 14 | 15 | /// The directory into which we will write the generated APIs 16 | #[structopt(parse(from_os_str))] 17 | pub output_directory: PathBuf, 18 | } 19 | -------------------------------------------------------------------------------- /mcp/src/options/generate.rs: -------------------------------------------------------------------------------- 1 | use std::path::PathBuf; 2 | use structopt::StructOpt; 3 | 4 | #[derive(Debug, StructOpt)] 5 | #[structopt(setting = structopt::clap::AppSettings::ColoredHelp)] 6 | /// Generate libraries and command-line interface for said libraries based on Google API specifications. 7 | /// 8 | /// The output will always be formatted using rustfmt. You can set the RUSTFMT environment variable to an 9 | /// empty value to prevent formatting, which can safe some time during generation. 10 | pub struct Args { 11 | /// The Google API specification as downloaded from the discovery service 12 | #[structopt(parse(from_os_str))] 13 | pub spec_json_path: PathBuf, 14 | 15 | /// The directory into which we will write all generated data 16 | #[structopt(parse(from_os_str))] 17 | pub output_directory: PathBuf, 18 | } 19 | -------------------------------------------------------------------------------- /mcp/src/options/map_index.rs: -------------------------------------------------------------------------------- 1 | use std::path::PathBuf; 2 | use structopt::StructOpt; 3 | 4 | #[derive(Debug, StructOpt)] 5 | #[structopt(setting = structopt::clap::AppSettings::ColoredHelp)] 6 | /// Transform the Google API index into something we can use further when dealing with substitutions. 7 | pub struct Args { 8 | /// The index with all API specification URLs as provided by Google's discovery API 9 | #[structopt(parse(from_os_str))] 10 | pub discovery_json_path: PathBuf, 11 | 12 | /// The path to which to write the digest 13 | #[structopt(parse(from_os_str))] 14 | pub output_file: PathBuf, 15 | 16 | /// The directory into which the `fetch-specs` subcommand writes its files, see `Standard::spec_dir` 17 | #[structopt(parse(from_os_str))] 18 | pub spec_directory: PathBuf, 19 | 20 | /// The directory into which files will be generated into 21 | #[structopt(parse(from_os_str))] 22 | pub output_directory: PathBuf, 23 | } 24 | -------------------------------------------------------------------------------- /mcp/src/options/mod.rs: -------------------------------------------------------------------------------- 1 | pub fn _output_formats() -> &'static [&'static str] { 2 | &["json", "yaml"] 3 | } 4 | 5 | use structopt::StructOpt; 6 | 7 | #[derive(StructOpt, Debug)] 8 | #[structopt(setting = structopt::clap::AppSettings::ColoredHelp)] 9 | #[structopt(setting = structopt::clap::AppSettings::VersionlessSubcommands)] 10 | #[structopt(setting = structopt::clap::AppSettings::DeriveDisplayOrder)] 11 | pub struct Args { 12 | /// The desired log level. 13 | #[structopt(short = "l", long = "log-level", default_value = "INFO")] 14 | #[structopt(possible_values = &["INFO", "ERROR", "DEBUG", "TRACE"])] 15 | pub log_level: log::Level, 16 | #[structopt(subcommand)] 17 | pub(crate) cmd: SubCommand, 18 | } 19 | 20 | #[derive(StructOpt, Debug)] 21 | pub enum SubCommand { 22 | #[structopt(name = "fetch-api-specs")] 23 | /// Fetch all API specs, in parallel, and generate their Rust code 24 | FetchApiSpecs(fetch_specs::Args), 25 | #[structopt(name = "completions")] 26 | /// generate completions for supported shells 27 | Completions(completions::Args), 28 | #[structopt(name = "generate")] 29 | /// generate APIs and CLIs for a Google API specification 30 | Generate(generate::Args), 31 | #[structopt(name = "map-api-index")] 32 | /// Transform the API index into data we can use during substitution 33 | MapApiIndex(map_index::Args), 34 | #[structopt(name = "substitute")] 35 | #[structopt(alias = "sub")] 36 | /// Substitutes templates using structured data. 37 | Substitute(substitute::Args), 38 | #[structopt(name = "cargo-errors")] 39 | /// Run cargo on workspace files and collect errors as we go 40 | CargoErrors(cargo_errors::Args), 41 | } 42 | 43 | pub mod cargo_errors; 44 | pub mod completions; 45 | pub mod fetch_specs; 46 | pub mod generate; 47 | pub mod map_index; 48 | pub mod substitute; 49 | -------------------------------------------------------------------------------- /mcp/src/options/substitute.rs: -------------------------------------------------------------------------------- 1 | use failure::{bail, Error}; 2 | use std::ffi::OsString; 3 | use structopt::StructOpt; 4 | use templating::{Spec, StreamOrPath}; 5 | 6 | #[derive(Debug, StructOpt)] 7 | #[structopt(setting = structopt::clap::AppSettings::ColoredHelp)] 8 | #[structopt(setting = structopt::clap::AppSettings::AllowLeadingHyphen)] 9 | /// Substitutes templates using structured data. 10 | /// 11 | /// The idea is to build a tree of data that is used to substitute in various templates, using multiple inputs and outputs. 12 | /// That way, secrets (like credentials) can be extracted from the vault just once and used wherever needed without them touching disk. 13 | /// Liquid is used as template engine, and it's possible to refer to and inherit from other templates by their file-stem. 14 | /// Read more on their website at https://shopify.github.io/liquid . 15 | pub struct Args { 16 | #[structopt(parse(from_os_str))] 17 | #[structopt(set = structopt::clap::ArgSettings::RequireEquals)] 18 | #[structopt( 19 | short = "s", 20 | long = "separator", 21 | name = "separator", 22 | default_value = "\n" 23 | )] 24 | /// The string to use to separate multiple documents that are written to the same stream. 25 | /// 26 | /// This can be useful to output a multi-document YAML file from multiple input templates 27 | /// to stdout if the separator is '---'. 28 | /// The separator is also used when writing multiple templates into the same file, like in 'a:out b:out'. 29 | pub separator: OsString, 30 | 31 | #[structopt(set = structopt::clap::ArgSettings::RequireEquals)] 32 | #[structopt(use_delimiter = true)] 33 | #[structopt( 34 | long = "replace", 35 | value_delimiter = ":", 36 | value_name = "find-this:replace-with-that" 37 | )] 38 | /// A simple find & replace for values for the string data to be placed into the template. \ 39 | /// The word to find is the first specified argument, the second one is the word to replace it with, \ 40 | /// e.g. -r=foo:bar. 41 | pub replacements: Vec, 42 | 43 | #[structopt(long = "validate", short = "v")] 44 | /// If set, the instantiated template will be parsed as YAML or JSON. 45 | /// If both of them are invalid, the command will fail. 46 | pub validate: bool, 47 | 48 | #[structopt(set = structopt::clap::ArgSettings::RequireEquals)] 49 | #[structopt(short = "d", long = "data", name = "path")] 50 | #[structopt(parse(try_from_str = stream_from_str))] 51 | /// Structured data in YAML or JSON format to use when instantiating/substituting the template. 52 | /// If set, everything from standard input is interpreted as template. 53 | pub data: Option, 54 | 55 | #[structopt(set = structopt::clap::ArgSettings::RequireEquals)] 56 | #[structopt(value_name = "template-spec")] 57 | #[structopt(parse(try_from_str = spec_from_str))] 58 | /// Identifies the how to map template files to output. 59 | /// 60 | /// The syntax is ':'. 61 | /// and are a relative or absolute paths to the source templates or 62 | /// destination files respectively. 63 | /// If is unspecified, the template will be read from stdin, e.g. ':output'. Only one spec can read from stdin. 64 | /// If is unspecified, the substituted template will be output to stdout, e.g 'input.hbs:' 65 | /// or 'input.hbs'. Multiple templates are separated by the '--separator' accordingly. This is particularly useful for YAML files 66 | /// where the separator should be `$'---\\n'` 67 | pub specs: Vec, 68 | } 69 | 70 | fn stream_from_str(v: &str) -> Result { 71 | Ok(StreamOrPath::from(v)) 72 | } 73 | 74 | fn spec_from_str(v: &str) -> Result { 75 | Ok(Spec::from(v)) 76 | } 77 | 78 | impl Args { 79 | pub fn sanitized(self) -> Result { 80 | if self.replacements.len() % 2 != 0 { 81 | bail!("Please provide --replace-value arguments in pairs of two. First the value to find, second the one to replace it with"); 82 | } 83 | Ok(self) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /shared/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "shared" 3 | version = "0.1.0" 4 | authors = ["Sebastian Thiel "] 5 | edition = "2018" 6 | publish = false 7 | 8 | [lib] 9 | doctest = false 10 | 11 | [dependencies] 12 | serde = { version = "1", features = ["derive"] } 13 | discovery_parser = { path = "../discovery_parser" } 14 | failure = "0.1.5" 15 | log = "0.4.8" 16 | ci_info = "0.8.0" 17 | toolchain_find = "0.1.4" 18 | 19 | [dev-dependencies] 20 | itertools = "0.8.0" 21 | insta = "0.11" 22 | 23 | -------------------------------------------------------------------------------- /shared/src/rustfmt.rs: -------------------------------------------------------------------------------- 1 | use std::fs::File; 2 | use std::io; 3 | use std::path::PathBuf; 4 | use std::process::{Child, Command, Stdio}; 5 | 6 | pub enum RustFmtWriter { 7 | Formatted(Child), 8 | Unformatted(File), 9 | } 10 | 11 | impl RustFmtWriter { 12 | pub fn new(output_file: File) -> Result> { 13 | Ok(match rustfmt_path() { 14 | Some(path) => RustFmtWriter::Formatted( 15 | Command::new(path) 16 | .arg("--edition=2018") 17 | .stderr(Stdio::inherit()) 18 | .stdout(output_file) 19 | .stdin(Stdio::piped()) 20 | .spawn()?, 21 | ), 22 | None => RustFmtWriter::Unformatted(output_file), 23 | }) 24 | } 25 | 26 | pub fn close(self) -> Result<(), Box> { 27 | match self { 28 | RustFmtWriter::Formatted(mut cmd) => { 29 | if cmd.wait()?.success() { 30 | Ok(()) 31 | } else { 32 | Err("rustfmt exited with error".to_owned().into()) 33 | } 34 | } 35 | RustFmtWriter::Unformatted(file) => Ok(file.sync_all()?), 36 | } 37 | } 38 | } 39 | 40 | impl io::Write for RustFmtWriter { 41 | fn write(&mut self, buf: &[u8]) -> io::Result { 42 | match self { 43 | RustFmtWriter::Formatted(cmd) => cmd.stdin.as_mut().unwrap().write(buf), 44 | RustFmtWriter::Unformatted(file) => file.write(buf), 45 | } 46 | } 47 | 48 | fn flush(&mut self) -> io::Result<()> { 49 | match self { 50 | RustFmtWriter::Formatted(_) => Ok(()), 51 | RustFmtWriter::Unformatted(file) => file.flush(), 52 | } 53 | } 54 | } 55 | 56 | fn rustfmt_path() -> Option { 57 | match std::env::var_os("RUSTFMT") { 58 | Some(which) => { 59 | if which.is_empty() { 60 | None 61 | } else { 62 | Some(PathBuf::from(which)) 63 | } 64 | } 65 | None => toolchain_find::find_installed_component("rustfmt"), 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /shared/tests/fixtures/known-versions: -------------------------------------------------------------------------------- 1 | alpha 2 | beta 3 | datatransfer_v1 4 | directory_v1 5 | reports_v1 6 | v1 7 | v1.1 8 | v1.2 9 | v1.3 10 | v1.4 11 | v1alpha 12 | v1alpha1 13 | v1alpha2 14 | v1b3 15 | v1beta 16 | v1beta1 17 | v1beta1a 18 | v1beta2 19 | v1beta3 20 | v1beta4 21 | v1beta5 22 | v1configuration 23 | v1management 24 | v1p1alpha1 25 | v1p1beta1 26 | v1p2beta1 27 | v1p3beta1 28 | v2 29 | v2.1 30 | v2.4 31 | v2alpha1 32 | v2beta 33 | v2beta1 34 | v2beta2 35 | v2beta3 36 | v3 37 | v3.1 38 | v3.2 39 | v3.3 40 | v3beta1 41 | v3p1alpha1 42 | v3p1beta1 43 | v4 44 | v4.1 45 | v5 46 | -------------------------------------------------------------------------------- /shared/tests/shared.rs: -------------------------------------------------------------------------------- 1 | mod sanitized_name { 2 | use shared::sanitized_name; 3 | 4 | #[test] 5 | fn it_does_not_alter_anything_else() { 6 | assert_eq!(sanitized_name("2foo"), "2foo"); 7 | assert_eq!(sanitized_name("fo2oo"), "fo2oo"); 8 | assert_eq!(sanitized_name("foo"), "foo"); 9 | } 10 | #[test] 11 | fn it_strips_numbers_off_the_tail() { 12 | // specifically for adexchangebuyer , actually 13 | assert_eq!(sanitized_name("foo2"), "foo"); 14 | assert_eq!(sanitized_name("foo20"), "foo") 15 | } 16 | } 17 | 18 | mod lib_crate_name { 19 | use shared::lib_crate_name; 20 | 21 | #[test] 22 | fn it_produces_a_valid_crate_name() { 23 | assert_eq!( 24 | lib_crate_name("youtube", "v2.0").unwrap(), 25 | "google-youtube2d0" 26 | ) 27 | } 28 | } 29 | 30 | mod make_target { 31 | use shared::make_target; 32 | 33 | #[test] 34 | fn it_produces_a_valid_make_target() { 35 | assert_eq!(make_target("youtube", "v1.3").unwrap(), "youtube1d3") 36 | } 37 | } 38 | 39 | mod parse_version { 40 | use insta::assert_snapshot; 41 | use itertools::Itertools; 42 | use shared::parse_version; 43 | use std::io::{BufRead, BufReader}; 44 | 45 | const KNOWN_VERSIONS: &str = include_str!("./fixtures/known-versions"); 46 | 47 | #[test] 48 | fn it_works_for_all_known_inputs() { 49 | let expected = BufReader::new(KNOWN_VERSIONS.as_bytes()) 50 | .lines() 51 | .filter_map(Result::ok) 52 | .map(|api_version| { 53 | format!( 54 | "{input} {output}", 55 | input = api_version, 56 | output = parse_version(&api_version).unwrap() 57 | ) 58 | }) 59 | .join("\n"); 60 | assert_snapshot!(expected); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /shared/tests/snapshots/parse_version__it_works_for_all_known_inputs.snap: -------------------------------------------------------------------------------- 1 | --- 2 | created: "2019-08-19T08:12:56.238951Z" 3 | creator: insta@0.10.1 4 | source: mcp/src/shared.rs 5 | expression: expected 6 | --- 7 | alpha alpha 8 | beta beta 9 | datatransfer_v1 1_datatransfer 10 | directory_v1 1_directory 11 | reports_v1 1_reports 12 | v1 1 13 | v1.1 1d1 14 | v1.2 1d2 15 | v1.3 1d3 16 | v1.4 1d4 17 | v1alpha 1_alpha 18 | v1alpha1 1_alpha1 19 | v1alpha2 1_alpha2 20 | v1b3 1_b3 21 | v1beta 1_beta 22 | v1beta1 1_beta1 23 | v1beta1a 1_beta1a 24 | v1beta2 1_beta2 25 | v1beta3 1_beta3 26 | v1beta4 1_beta4 27 | v1beta5 1_beta5 28 | v1configuration 1_configuration 29 | v1management 1_management 30 | v1p1alpha1 1_p1alpha1 31 | v1p1beta1 1_p1beta1 32 | v1p2beta1 1_p2beta1 33 | v1p3beta1 1_p3beta1 34 | v2 2 35 | v2.1 2d1 36 | v2.4 2d4 37 | v2alpha1 2_alpha1 38 | v2beta 2_beta 39 | v2beta1 2_beta1 40 | v2beta2 2_beta2 41 | v2beta3 2_beta3 42 | v3 3 43 | v3.1 3d1 44 | v3.2 3d2 45 | v3.3 3d3 46 | v3beta1 3_beta1 47 | v3p1alpha1 3_p1alpha1 48 | v3p1beta1 3_p1beta1 49 | v4 4 50 | v4.1 4d1 51 | v5 5 52 | -------------------------------------------------------------------------------- /templating/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | authors = ["Sebastian Thiel "] 3 | name = "templating" 4 | version = "1.0.0" 5 | edition = "2018" 6 | publish = false 7 | 8 | description = "Provide a nice interface to a templating engine" 9 | license = "MIT" 10 | 11 | [lib] 12 | doctest = false 13 | 14 | [dependencies] 15 | failure = "0.1.1" 16 | atty = "0.2.8" 17 | serde_json = "1.0.11" 18 | serde_yaml = "0.8.8" 19 | yaml-rust = "0.4.0" 20 | liquid = "0.19" 21 | liquid-error = "0.19.0" 22 | -------------------------------------------------------------------------------- /templating/src/lib.rs: -------------------------------------------------------------------------------- 1 | #![forbid(unsafe_code)] 2 | 3 | extern crate serde_json as json; 4 | extern crate serde_yaml as yaml; 5 | mod liquid; 6 | mod spec; 7 | mod util; 8 | pub use crate::liquid::*; 9 | -------------------------------------------------------------------------------- /templating/src/liquid.rs: -------------------------------------------------------------------------------- 1 | use atty; 2 | use failure::{bail, err_msg, format_err, Error, ResultExt}; 3 | use json; 4 | use liquid; 5 | use yaml_rust; 6 | 7 | use std::{ 8 | collections::BTreeSet, 9 | ffi::OsStr, 10 | fs::File, 11 | io::{self, stdin}, 12 | os::unix::ffi::OsStrExt, 13 | }; 14 | 15 | pub use super::{ 16 | spec::*, 17 | util::{de_json_or_yaml, validate}, 18 | }; 19 | 20 | pub fn substitute( 21 | input_data: &StreamOrPath, 22 | specs: &[Spec], 23 | separator: &OsStr, 24 | try_deserialize: bool, 25 | replacements: &[(String, String)], 26 | ) -> Result<(), Error> { 27 | use self::StreamOrPath::*; 28 | let mut own_specs = Vec::new(); 29 | 30 | let (dataset, specs) = match *input_data { 31 | Stream => { 32 | if atty::is(atty::Stream::Stdin) { 33 | bail!("Stdin is a TTY. Cannot substitute a template without any data.") 34 | } else { 35 | let stdin = stdin(); 36 | let locked_stdin = stdin.lock(); 37 | (de_json_or_yaml(locked_stdin)?, specs) 38 | } 39 | } 40 | Path(ref p) => ( 41 | de_json_or_yaml(File::open(&p).context(format!( 42 | "Could not open input data file at '{}'", 43 | p.display() 44 | ))?)?, 45 | if specs.is_empty() { 46 | own_specs.push(Spec { 47 | src: Stream, 48 | dst: Stream, 49 | }); 50 | &own_specs 51 | } else { 52 | specs 53 | }, 54 | ), 55 | }; 56 | 57 | validate(input_data, specs)?; 58 | let dataset = substitute_in_data(dataset, replacements); 59 | let (engine, dataset) = ( 60 | liquid::ParserBuilder::with_liquid().build()?, 61 | into_liquid_object(dataset)?, 62 | ); 63 | 64 | let mut seen_file_outputs = BTreeSet::new(); 65 | let mut seen_writes_to_stdout = 0; 66 | let mut buf = Vec::::new(); 67 | let mut ibuf = String::new(); 68 | 69 | for spec in specs { 70 | let append = match spec.dst { 71 | Path(ref p) => { 72 | let seen = seen_file_outputs.contains(p); 73 | seen_file_outputs.insert(p); 74 | seen 75 | } 76 | Stream => { 77 | seen_writes_to_stdout += 1; 78 | false 79 | } 80 | }; 81 | 82 | let mut ostream = spec.dst.open_as_output(append)?; 83 | if seen_writes_to_stdout > 1 || append { 84 | ostream.write_all(separator.as_bytes())?; 85 | } 86 | 87 | { 88 | let mut istream = spec.src.open_as_input()?; 89 | let ostream_for_template: &mut dyn io::Write = if try_deserialize { 90 | &mut buf 91 | } else { 92 | &mut ostream 93 | }; 94 | 95 | ibuf.clear(); 96 | istream.read_to_string(&mut ibuf)?; 97 | let tpl = engine.parse(&ibuf).map_err(|err| { 98 | format_err!("{}", err).context(format!( 99 | "Failed to parse liquid template at '{}'", 100 | spec.src.name() 101 | )) 102 | })?; 103 | 104 | let rendered = tpl.render(&dataset).map_err(|err| { 105 | format_err!("{}", err).context(format!( 106 | "Failed to render template from template at '{}'", 107 | spec.src.short_name() 108 | )) 109 | })?; 110 | ostream_for_template.write_all(rendered.as_bytes())?; 111 | } 112 | 113 | if try_deserialize { 114 | { 115 | let str_buf = ::std::str::from_utf8(&buf).context(format!( 116 | "Validation of template output at '{}' failed as it was not valid UTF8", 117 | spec.dst.name() 118 | ))?; 119 | yaml_rust::YamlLoader::load_from_str(str_buf).context(format!( 120 | "Validation of template output at '{}' failed. It's neither valid YAML, nor JSON", 121 | spec.dst.name() 122 | ))?; 123 | } 124 | let mut read = io::Cursor::new(buf); 125 | io::copy(&mut read, &mut ostream) 126 | .map_err(|_| err_msg("Failed to output validated template to destination."))?; 127 | buf = read.into_inner(); 128 | buf.clear(); 129 | } 130 | } 131 | Ok(()) 132 | } 133 | 134 | fn into_liquid_object(src: json::Value) -> Result { 135 | let dst = json::from_value(src)?; 136 | match dst { 137 | liquid::value::Value::Object(obj) => Ok(obj), 138 | _ => Err(err_msg("Data model root must be an object")), 139 | } 140 | } 141 | 142 | fn substitute_in_data(mut d: json::Value, r: &[(String, String)]) -> json::Value { 143 | if r.is_empty() { 144 | return d; 145 | } 146 | 147 | { 148 | use json::Value::*; 149 | let mut stack = vec![&mut d]; 150 | while let Some(v) = stack.pop() { 151 | match *v { 152 | String(ref mut s) => { 153 | *s = r 154 | .iter() 155 | .fold(s.to_owned(), |s, &(ref f, ref t)| s.replace(f.as_str(), t)) 156 | } 157 | Array(ref mut v) => stack.extend(v.iter_mut()), 158 | Object(ref mut m) => stack.extend(m.iter_mut().map(|(_, v)| v)), 159 | _ => continue, 160 | } 161 | } 162 | } 163 | 164 | d 165 | } 166 | -------------------------------------------------------------------------------- /templating/src/spec.rs: -------------------------------------------------------------------------------- 1 | use atty; 2 | use failure::{bail, Error, ResultExt}; 3 | 4 | use std::{ 5 | ffi::OsStr, 6 | fmt, 7 | fs::create_dir_all, 8 | fs::{File, OpenOptions}, 9 | io::{self, stdin, stdout}, 10 | path::PathBuf, 11 | }; 12 | 13 | #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] 14 | pub enum StreamOrPath { 15 | Stream, 16 | Path(PathBuf), 17 | } 18 | 19 | impl StreamOrPath { 20 | pub fn is_stream(&self) -> bool { 21 | match *self { 22 | StreamOrPath::Stream => true, 23 | StreamOrPath::Path(_) => false, 24 | } 25 | } 26 | 27 | pub fn name(&self) -> &str { 28 | match *self { 29 | StreamOrPath::Stream => "stream", 30 | StreamOrPath::Path(ref p) => p.to_str().unwrap_or(""), 31 | } 32 | } 33 | 34 | pub fn short_name(&self) -> &str { 35 | match *self { 36 | StreamOrPath::Stream => "stream", 37 | StreamOrPath::Path(ref p) => p.file_stem().map_or("", |s| { 38 | s.to_str().unwrap_or("") 39 | }), 40 | } 41 | } 42 | 43 | pub fn open_as_output(&self, append: bool) -> Result, Error> { 44 | Ok(match *self { 45 | StreamOrPath::Stream => Box::new(stdout()), 46 | StreamOrPath::Path(ref p) => { 47 | if let Some(dir) = p.parent() { 48 | create_dir_all(dir).context(format!( 49 | "Could not create directory leading towards '{}'", 50 | p.display(), 51 | ))?; 52 | } 53 | Box::new( 54 | OpenOptions::new() 55 | .create(true) 56 | .truncate(!append) 57 | .write(true) 58 | .append(append) 59 | .open(p) 60 | .context(format!("Could not open '{}' for writing", p.display()))?, 61 | ) 62 | } 63 | }) 64 | } 65 | 66 | pub fn open_as_input(&self) -> Result, Error> { 67 | Ok(match *self { 68 | StreamOrPath::Stream => { 69 | if atty::is(atty::Stream::Stdin) { 70 | bail!("Cannot read from standard input while a terminal is connected") 71 | } else { 72 | Box::new(stdin()) 73 | } 74 | } 75 | StreamOrPath::Path(ref p) => Box::new( 76 | File::open(p).context(format!("Could not open '{}' for reading", p.display()))?, 77 | ), 78 | }) 79 | } 80 | } 81 | 82 | impl<'a> From<&'a OsStr> for StreamOrPath { 83 | fn from(p: &'a OsStr) -> Self { 84 | StreamOrPath::Path(PathBuf::from(p)) 85 | } 86 | } 87 | 88 | impl<'a> From<&'a str> for StreamOrPath { 89 | fn from(s: &str) -> Self { 90 | use self::StreamOrPath::*; 91 | if s.is_empty() { 92 | Stream 93 | } else { 94 | Path(PathBuf::from(s)) 95 | } 96 | } 97 | } 98 | 99 | impl fmt::Display for StreamOrPath { 100 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 101 | use self::StreamOrPath::*; 102 | match *self { 103 | Stream => Ok(()), 104 | Path(ref p) => p.display().fmt(f), 105 | } 106 | } 107 | } 108 | 109 | #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] 110 | pub struct Spec { 111 | pub src: StreamOrPath, 112 | pub dst: StreamOrPath, 113 | } 114 | 115 | impl Spec { 116 | pub fn sep() -> char { 117 | ':' 118 | } 119 | } 120 | 121 | impl fmt::Display for Spec { 122 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 123 | use self::fmt::Write; 124 | use self::StreamOrPath::*; 125 | match (&self.src, &self.dst) { 126 | (&Stream, &Stream) => f.write_char(Spec::sep()), 127 | (&Path(ref p), &Stream) => p.display().fmt(f), 128 | (&Stream, &Path(ref p)) => f.write_char(Spec::sep()).and(p.display().fmt(f)), 129 | (&Path(ref p1), &Path(ref p2)) => p1 130 | .display() 131 | .fmt(f) 132 | .and(f.write_char(Spec::sep())) 133 | .and(p2.display().fmt(f)), 134 | } 135 | } 136 | } 137 | 138 | impl<'a> From<&'a str> for Spec { 139 | fn from(src: &'a str) -> Self { 140 | use self::StreamOrPath::*; 141 | let mut it = src.splitn(2, Spec::sep()); 142 | match (it.next(), it.next()) { 143 | (None, Some(_)) | (None, None) => unreachable!(), 144 | (Some(p), None) => Spec { 145 | src: StreamOrPath::from(p), 146 | dst: Stream, 147 | }, 148 | (Some(p1), Some(p2)) => Spec { 149 | src: StreamOrPath::from(p1), 150 | dst: StreamOrPath::from(p2), 151 | }, 152 | } 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /templating/src/util.rs: -------------------------------------------------------------------------------- 1 | use failure::{bail, Error, Fail, ResultExt}; 2 | use json; 3 | use yaml; 4 | 5 | use std::{io::Cursor, io::Read}; 6 | 7 | pub use super::spec::*; 8 | 9 | pub fn validate(data: &StreamOrPath, specs: &[Spec]) -> Result<(), Error> { 10 | if specs.is_empty() { 11 | bail!("No spec provided, neither from standard input, nor from file") 12 | } 13 | let count_of_specs_needing_stdin = specs.iter().filter(|s| s.src.is_stream()).count(); 14 | if count_of_specs_needing_stdin > 1 { 15 | bail!("Cannot read more than one template spec from standard input") 16 | } 17 | if data.is_stream() && count_of_specs_needing_stdin == 1 { 18 | bail!("Data is read from standard input, as well as one template. Please choose one") 19 | } 20 | if let Some(spec_which_overwrites_input) = specs 21 | .iter() 22 | .filter_map(|s| { 23 | use self::StreamOrPath::*; 24 | match (&s.src, &s.dst) { 25 | (&Path(ref src), &Path(ref dst)) => src 26 | .canonicalize() 27 | .and_then(|src| dst.canonicalize().map(|dst| (src, dst))) 28 | .ok() 29 | .and_then(|(src, dst)| if src == dst { Some(s) } else { None }), 30 | _ => None, 31 | } 32 | }) 33 | .next() 34 | { 35 | bail!( 36 | "Refusing to overwrite input file at '{}' with output", 37 | spec_which_overwrites_input.src 38 | ) 39 | } 40 | Ok(()) 41 | } 42 | 43 | pub fn de_json_or_yaml(mut reader: R) -> Result { 44 | let mut buf = Vec::::new(); 45 | reader 46 | .read_to_end(&mut buf) 47 | .context("Could not read input stream data deserialization")?; 48 | 49 | Ok( 50 | match json::from_reader::<_, json::Value>(Cursor::new(&buf)) { 51 | Ok(v) => v, 52 | Err(json_err) => yaml::from_reader(Cursor::new(&buf)).map_err(|yaml_err| { 53 | yaml_err 54 | .context("YAML deserialization failed") 55 | .context(json_err) 56 | .context("JSON deserialization failed") 57 | .context("Could not deserialize data, tried JSON and YAML") 58 | })?, 59 | }, 60 | ) 61 | } 62 | -------------------------------------------------------------------------------- /templating/tests/spec.rs: -------------------------------------------------------------------------------- 1 | extern crate templating; 2 | 3 | use std::path::PathBuf; 4 | use templating::{Spec, StreamOrPath::*}; 5 | 6 | #[cfg(test)] 7 | mod parse { 8 | use super::*; 9 | 10 | #[test] 11 | fn empty() { 12 | let actual = Spec::from(""); 13 | assert_eq!( 14 | actual, 15 | Spec { 16 | src: Stream, 17 | dst: Stream, 18 | } 19 | ); 20 | 21 | assert_eq!(format!("{}", actual), ":") 22 | } 23 | 24 | #[test] 25 | fn colon() { 26 | let actual = Spec::from(":"); 27 | assert_eq!( 28 | actual, 29 | Spec { 30 | src: Stream, 31 | dst: Stream, 32 | } 33 | ); 34 | 35 | assert_eq!(format!("{}", actual), ":") 36 | } 37 | 38 | #[test] 39 | fn stream_path() { 40 | let actual = Spec::from(":foo"); 41 | assert_eq!( 42 | actual, 43 | Spec { 44 | src: Stream, 45 | dst: Path(PathBuf::from("foo")), 46 | } 47 | ); 48 | 49 | assert_eq!(format!("{}", actual), ":foo") 50 | } 51 | 52 | #[test] 53 | fn path_stream() { 54 | let actual = Spec::from("foo:"); 55 | assert_eq!( 56 | actual, 57 | Spec { 58 | src: Path(PathBuf::from("foo")), 59 | dst: Stream, 60 | } 61 | ); 62 | 63 | assert_eq!(format!("{}", actual), "foo") 64 | } 65 | 66 | #[test] 67 | fn path_path() { 68 | let actual = Spec::from("foo:bar"); 69 | assert_eq!( 70 | actual, 71 | Spec { 72 | src: Path(PathBuf::from("foo")), 73 | dst: Path(PathBuf::from("bar")), 74 | } 75 | ); 76 | 77 | assert_eq!(format!("{}", actual), "foo:bar") 78 | } 79 | 80 | #[test] 81 | fn absolute_path_absolute_path() { 82 | let actual = Spec::from("/foo/sub:/bar/sub"); 83 | assert_eq!( 84 | actual, 85 | Spec { 86 | src: Path(PathBuf::from("/foo/sub")), 87 | dst: Path(PathBuf::from("/bar/sub")), 88 | } 89 | ); 90 | 91 | assert_eq!(format!("{}", actual), "/foo/sub:/bar/sub") 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /tests/mcp/fixtures/substitute/complex.tpl: -------------------------------------------------------------------------------- 1 | {{ page.title }} 2 | 3 | {% if user %} 4 | Hello {{ user.name }}! 5 | {% endif %} 6 | 7 | {{ "/my/fancy/url" | append: ".html" }} 8 | 9 | 10 | {% if product.title contains 'Pack' %} 11 | This product's title contains the word Pack. 12 | {% endif %} 13 | 14 | 15 | 16 | {% assign my_string = "Hello World!" %} 17 | 18 | 19 | {% assign my_int = 25 %} 20 | {% assign my_float = 39.756 %} 21 | 22 | 23 | {% assign foo = true %} 24 | {% assign bar = false %} 25 | 26 | {% for user in site.users %} 27 | {{ user }} 28 | {% endfor %} 29 | 30 | 31 | {% assign my_variable = "tomato" %} 32 | {{ my_variable }} 33 | 34 | {%- assign my_variable = "tomato" -%} 35 | {{ my_variable }} 36 | 37 | Anything you put between {% comment %} and {% endcomment %} tags 38 | is turned into a comment. 39 | 40 | 41 | 42 | {% unless product.title == 'Awesome Shoes' %} 43 | These shoes are not awesome. 44 | {% endunless %} 45 | 46 | {% if product.title != 'Awesome Shoes' %} 47 | These shoes are not awesome. 48 | {% endif %} 49 | 50 | {% if customer.name == 'kevin' %} 51 | Hey Kevin! 52 | {% elsif customer.name == 'anonymous' %} 53 | Hey Anonymous! 54 | {% else %} 55 | Hi Stranger! 56 | {% endif %} 57 | 58 | {% assign handle = 'cake' %} 59 | {% case handle %} 60 | {% when 'cake' %} 61 | This is a cake 62 | {% when 'cookie' %} 63 | This is a cookie 64 | {% else %} 65 | This is not a cake nor a cookie 66 | {% endcase %} 67 | 68 | 69 | {% for i in (1..5) %} 70 | {% if i == 4 %} 71 | {% break %} 72 | {% else %} 73 | {{ i }} 74 | {% endif %} 75 | {% endfor %} 76 | 77 | 78 | {% for i in (1..5) %} 79 | {% if i == 4 %} 80 | {% continue %} 81 | {% else %} 82 | {{ i }} 83 | {% endif %} 84 | {% endfor %} 85 | 86 | {% for item in array limit:2 %} 87 | {{ item }} 88 | {%- endfor -%} 89 | 90 | {% for i in (3..5) %} 91 | {{ i }} 92 | {% endfor %} 93 | 94 | {% assign num = 4 %} 95 | {% for i in (1..num) %} 96 | {{ i }} 97 | {% endfor %} 98 | 99 | {% for item in array reversed %} 100 | {{ item }} 101 | {% endfor %} 102 | 103 | 104 | {% raw %} 105 | In Handlebars, {{ this }} will be HTML-escaped, but 106 | {{ that }}} will not. 107 | {% endraw %} 108 | 109 | {% capture my_variable %}I am being captured.{% endcapture %} 110 | {{ my_variable }} 111 | 112 | 113 | {% assign favorite_food = 'pizza' %} 114 | {% assign age = 35 %} 115 | 116 | {% capture about_me %} 117 | I am {{ age }} and my favorite food is {{ favorite_food }}. 118 | {% endcapture %} 119 | 120 | {{ about_me }} 121 | 122 | 123 | 124 | {{ -17 | abs }} 125 | {{ "/my/fancy/url" | append: ".html" }} 126 | {{ "title" | capitalize }} 127 | {{ 1.2 | ceil }} 128 | {{ "3.5" | ceil }} 129 | {% assign site_categories = site.pages | map: 'category' | compact %} 130 | 131 | {% for category in site_categories %} 132 | {{ category }} 133 | {% endfor %} 134 | 135 | {% assign fruits = "apples, oranges, peaches" | split: ", " %} 136 | {% assign vegetables = "carrots, turnips, potatoes" | split: ", " %} 137 | 138 | {% assign everything = fruits | concat: vegetables %} 139 | 140 | {% for item in everything %} 141 | - {{ item }} 142 | {% endfor %} 143 | 144 | {% assign furniture = "chairs, tables, shelves" | split: ", " %} 145 | 146 | {% assign everything = fruits | concat: vegetables | concat: furniture %} 147 | 148 | {% for item in everything %} 149 | - {{ item }} 150 | {% endfor %} 151 | 152 | {{ "March 25, 2018" | date: "%b %d, %y" }} 153 | {{ null | default: 2.99 }} 154 | 155 | {% assign product_price = "" %} 156 | {{ product_price | default: 2.99 }} 157 | {{ 16 | divided_by: 4 }} 158 | 159 | {% assign my_integer = 7 %} 160 | {{ 20 | divided_by: my_integer }} 161 | 162 | {% assign my_integer = 7 %} 163 | {% assign my_float = my_integer | times: 1.0 %} 164 | {{ 20 | divided_by: my_float }} 165 | {{ "Parker Moore" | downcase }} 166 | {{ "Have you read 'James & the Giant Peach'?" | escape }} 167 | {{ "1 < 2 & 3" | escape_once }} 168 | 169 | {% assign my_array = "zebra, octopus, giraffe, tiger" | split: ", " %} 170 | 171 | {{ 1.2 | floor }} 172 | {{ "3.5" | floor }} 173 | {% assign beatles = "John, Paul, George, Ringo" | split: ", " %} 174 | {{ beatles | join: " and " }} 175 | {{ " So much room for activities! " | lstrip }} 176 | {{ 4 | minus: 2 }} 177 | {{ 3 | modulo: 2 }} 178 | {% capture string_with_newlines %} 179 | Hello 180 | there 181 | {% endcapture %} 182 | {{ string_with_newlines | newline_to_br }} 183 | {{ 4 | plus: 2 }} 184 | {{ "apples, oranges, and bananas" | prepend: "Some fruit: " }} 185 | {{ "I strained to see the train through the rain" | remove: "rain" }} 186 | {{ "I strained to see the train through the rain" | remove_first: "rain" }} 187 | {{ "Take my protein pills and put my helmet on" | replace: "my", "your" }} 188 | {% assign my_string = "Take my protein pills and put my helmet on" %} 189 | 190 | {{ my_string | replace_first: "my", "your" }} 191 | {{ "Ground control to Major Tom." | split: "" | reverse | join: "" }} 192 | 193 | {% assign my_array = "apples, oranges, peaches, plums" | split: ", " %} 194 | {{ my_array | reverse | join: ", " }} 195 | 196 | {{ 1.2 | round }} 197 | 198 | {{ " So much room for activities! " | rstrip }} 199 | 200 | {{ "Ground control to Major Tom." | size }} 201 | 202 | {% assign my_array = "apples, oranges, peaches, plums" | split: ", " %} 203 | {{ my_array | size }} 204 | 205 | {{ "Liquid" | slice: 2, 5 }} 206 | {{ "Liquid" | slice: -3, 2 }} 207 | 208 | {% assign my_array = "zebra, octopus, giraffe, Sally Snake" | split: ", " %} 209 | {{ my_array | sort | join: ", " }} 210 | 211 | {% assign my_array = "zebra, octopus, giraffe, Sally Snake" | split: ", " %} 212 | {{ my_array | sort_natural | join: ", " }} 213 | 214 | {{ " So much room for activities! " | strip }} 215 | 216 | {{ "Have you read Ulysses?" | strip_html }} 217 | 218 | {% capture string_with_newlines %} 219 | Hello 220 | there 221 | {% endcapture %} 222 | 223 | {{ string_with_newlines | strip_newlines }} 224 | {{ 3 | times: 2 }} 225 | {{ "Ground control to Major Tom." | truncate: 25, ", and so on" }} 226 | {{ "Ground control to Major Tom." | truncatewords: 3 }} 227 | 228 | {% assign my_array = "ants, bugs, bees, bugs, ants" | split: ", " %} 229 | {{ my_array | uniq | join: ", " }} 230 | -------------------------------------------------------------------------------- /tests/mcp/fixtures/substitute/data-for-complex.tpl.yml: -------------------------------------------------------------------------------- 1 | page: 2 | title: Introduction 3 | 4 | user: 5 | name: Harald 6 | 7 | customer: 8 | name: anonymous 9 | 10 | product: 11 | title: BarPackFoo 12 | 13 | site: 14 | pages: 15 | - category: one 16 | - category: ~ 17 | - category: two 18 | users: 19 | - Harald 20 | - Schneider 21 | - Meier 22 | 23 | null: ~ 24 | array: 25 | - 1 26 | - 2 27 | - 3 28 | - 4 29 | 30 | 31 | -------------------------------------------------------------------------------- /tests/mcp/fixtures/substitute/data.json.hbs: -------------------------------------------------------------------------------- 1 | { 2 | "password": "{{secret}}" 3 | } -------------------------------------------------------------------------------- /tests/mcp/fixtures/substitute/partials/base0.hbs: -------------------------------------------------------------------------------- 1 | 2 | {{title}} 3 | 4 |

Derived from base0.hbs

5 | {{~> page}} 6 | 7 | 8 | -------------------------------------------------------------------------------- /tests/mcp/fixtures/substitute/partials/base1.hbs: -------------------------------------------------------------------------------- 1 | 2 | {{title}} 3 | 4 |

Derived from base1.hbs

5 | {{~> page}} 6 | 7 | 8 | -------------------------------------------------------------------------------- /tests/mcp/fixtures/substitute/partials/template.hbs: -------------------------------------------------------------------------------- 1 | {{#*inline "page"}} 2 |

Rendered in partial, parent is {{parent}}

3 | {{/inline}} 4 | {{~> (parent)~}} 5 | -------------------------------------------------------------------------------- /tests/mcp/fixtures/substitute/the-answer.hbs: -------------------------------------------------------------------------------- 1 | : What is the answer to everything? 2 | : {{the-answer}} 3 | -------------------------------------------------------------------------------- /tests/mcp/included-stateless-substitute.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | template="$fixture/substitute" 4 | snapshot="$snapshot/substitute" 5 | 6 | title "'substitute' with find & replace" 7 | (with "stdin for data" 8 | (with "input as yaml containing a character that needs escaping in JSON" 9 | INPUT='secret: $sec"cret' 10 | (with "a data-template from file" 11 | precondition "fails without replacements" && { 12 | echo "$INPUT" | \ 13 | WITH_SNAPSHOT="$snapshot/fail-data-stdin-json-data-validated-stdout" \ 14 | expect_run $WITH_FAILURE "$exe" substitute --validate "$template/data.json.hbs" 15 | } 16 | 17 | (with "replacements to escape the offending character" 18 | it "succeeds thanks to replacements" && { 19 | echo "$INPUT" | \ 20 | WITH_SNAPSHOT="$snapshot/data-stdin-json-data-validated-fix-with-replacements-stdout" \ 21 | expect_run $SUCCESSFULLY "$exe" substitute --validate --replace='":\"' --replace='sec:geheim:cret:niss' "$template/data.json.hbs" 22 | } 23 | ) 24 | ) 25 | ) 26 | ) 27 | 28 | title "'substitute' subcommand" 29 | (with "stdin for data" 30 | (with "input as json" 31 | (with "single template from a file (absolute path)" 32 | it "outputs the substituted data to stdout" && { 33 | echo '{"the-answer": 42}' | \ 34 | WITH_SNAPSHOT="$snapshot/data-stdin-json-single-template-stdout" \ 35 | expect_run $SUCCESSFULLY "$exe" substitute "$template/the-answer.hbs" 36 | } 37 | ) 38 | ) 39 | (with "input as yaml" 40 | (with "single template from a file (absolute path)" 41 | (when "outputting to stdout" 42 | it "outputs the substituted data" && { 43 | echo "the-answer: 42" | \ 44 | WITH_SNAPSHOT="$snapshot/data-stdin-yaml-single-template-stdout" \ 45 | expect_run $SUCCESSFULLY "$exe" substitute "$template/the-answer.hbs" 46 | } 47 | ) 48 | (sandbox 49 | (when "outputting to a file within a non-existing directory" 50 | it "succeeds" && { 51 | echo "the-answer: 42" | \ 52 | WITH_SNAPSHOT="$snapshot/data-stdin-yaml-single-template-file-non-existing-directory" \ 53 | expect_run $SUCCESSFULLY "$exe" substitute "$template/the-answer.hbs:some/sub/directory/output" 54 | } 55 | 56 | it "creates the subdirectory which contains the file" && { 57 | expect_snapshot "$snapshot/data-stdin-yaml-single-template-output-file-with-nonexisting-directory" . 58 | } 59 | ) 60 | ) 61 | ) 62 | (sandbox 63 | (with "single template from a file (relative path)" 64 | cp "$template/the-answer.hbs" template.hbs 65 | (when "outputting to stdout" 66 | it "outputs the substituted data to stdout" && { 67 | echo "the-answer: 42" | \ 68 | WITH_SNAPSHOT="$snapshot/data-stdin-yaml-single-relative-template-stdout" \ 69 | expect_run $SUCCESSFULLY "$exe" substitute template.hbs 70 | } 71 | ) 72 | ) 73 | ) 74 | (with "multiple templates from a file (absolute path)" 75 | (with "the default document separator" 76 | it "outputs the substituted data to stdout" && { 77 | echo "the-answer: 42" | \ 78 | WITH_SNAPSHOT="$snapshot/data-stdin-yaml-multi-template-stdout" \ 79 | expect_run $SUCCESSFULLY "$exe" substitute "$template/the-answer.hbs" "$template/the-answer.hbs" 80 | } 81 | ) 82 | (with "an explicit document separator" 83 | it "outputs the substituted data to stdout" && { 84 | echo "the-answer: 42" | \ 85 | WITH_SNAPSHOT="$snapshot/data-stdin-yaml-multi-template-stdout-explicit-separator" \ 86 | expect_run $SUCCESSFULLY "$exe" substitute --separator $'<->\n' "$template/the-answer.hbs" "$template/the-answer.hbs" 87 | } 88 | ) 89 | ) 90 | ) 91 | ) 92 | 93 | 94 | (with "stdin for data" 95 | (with "input as yaml" 96 | (with "multiple template from a file to the same output file" 97 | (sandbox 98 | (with "the default document separator" 99 | it "succeeds" && { 100 | echo "the-answer: 42" | \ 101 | WITH_SNAPSHOT="$snapshot/data-stdin-yaml-multi-template-to-same-file" \ 102 | expect_run $SUCCESSFULLY "$exe" substitute "$template/the-answer.hbs:output" "$template/the-answer.hbs:output" 103 | } 104 | 105 | it "produces the expected output, which is a single document separated by the document separator" && { 106 | expect_snapshot "$snapshot/data-stdin-yaml-multi-template-to-same-file-output" output 107 | } 108 | ) 109 | (when "writing to the same output file again" 110 | it "succeeds" && { 111 | echo "the-answer: 42" | \ 112 | WITH_SNAPSHOT="$snapshot/data-stdin-yaml-multi-template-to-same-file-again" \ 113 | expect_run $SUCCESSFULLY "$exe" substitute "$template/the-answer.hbs:output" 114 | } 115 | it "overwrites the previous output file entirely" && { 116 | expect_snapshot "$snapshot/data-stdin-yaml-multi-template-to-same-file-again-output" output 117 | } 118 | ) 119 | ) 120 | (sandbox 121 | (with "the explicit document separator" 122 | it "succeeds" && { 123 | echo "the-answer: 42" | \ 124 | WITH_SNAPSHOT="$snapshot/data-stdin-yaml-multi-template-to-same-file-explicit-separator" \ 125 | expect_run $SUCCESSFULLY "$exe" substitute -s=$'---\n' "$template/the-answer.hbs:$PWD/output" "$template/the-answer.hbs:$PWD/output" 126 | } 127 | it "produces the expected output" && { 128 | expect_snapshot "$snapshot/data-stdin-yaml-multi-template-to-same-file-explicit-separator-output" output 129 | } 130 | ) 131 | ) 132 | ) 133 | ) 134 | ) 135 | (with "stdin for templates" 136 | (with "data from file" 137 | (when "writing the output to stdout" 138 | (with "implicit syntax" 139 | it "succeeds" && { 140 | echo "hello {{to-what}}" | \ 141 | WITH_SNAPSHOT="$snapshot/template-stdin-hbs-output-stdout" \ 142 | expect_run $SUCCESSFULLY "$exe" substitute -d <(echo "to-what: world") 143 | } 144 | ) 145 | (with "explicit syntax" 146 | it "succeeds" && { 147 | echo "hello {{to-what}}" | \ 148 | WITH_SNAPSHOT="$snapshot/template-stdin-hbs-output-stdout" \ 149 | expect_run $SUCCESSFULLY "$exe" substitute -d <(echo '{"to-what": "world"}') : 150 | } 151 | ) 152 | ) 153 | (sandbox 154 | (when "writing the output to a file" 155 | (with "implicit syntax" 156 | it "succeeds" && { 157 | echo "hello {{to-what}}" | \ 158 | WITH_SNAPSHOT="$snapshot/template-stdin-hbs-output-stdout-to-file" \ 159 | expect_run $SUCCESSFULLY "$exe" substitute -d <(echo "to-what: world") :output 160 | } 161 | it "produces the expected output" && { 162 | expect_snapshot "$snapshot/template-stdin-hbs-output-stdout-to-file-output" output 163 | } 164 | ) 165 | ) 166 | ) 167 | ) 168 | ) 169 | 170 | title "'substitute' (liquid) complex example" 171 | (when "feeding a complex example" 172 | it "succeeds and produces the correct output" && { 173 | WITH_SNAPSHOT="$snapshot/template-from-complex-template" \ 174 | expect_run $SUCCESSFULLY "$exe" substitute "$template/complex.tpl" < "$template/data-for-complex.tpl.yml" 175 | } 176 | ) 177 | 178 | title "'substitute' subcommand error cases" 179 | (with "invalid data in no known format" 180 | it "fails" && { 181 | WITH_SNAPSHOT="$snapshot/fail-invalid-data-format" \ 182 | expect_run $WITH_FAILURE "$exe" substitute -d <(echo 'a: *b') "$template/the-answer.hbs" 183 | } 184 | ) 185 | (with "multi-document yaml as input" 186 | it "fails" && { 187 | WITH_SNAPSHOT="$snapshot/fail-invalid-data-format-multi-document-yaml" \ 188 | expect_run $WITH_FAILURE "$exe" substitute -d <(echo $'---\n---') "$template/the-answer.hbs" 189 | } 190 | ) 191 | 192 | (with "a spec that tries to write the output to the input template" 193 | (with "a single spec" 194 | it "fails as it refuses" && { 195 | WITH_SNAPSHOT="$snapshot/fail-source-is-destination-single-spec" \ 196 | expect_run $WITH_FAILURE "$exe" substitute -d <(echo does not matter) "$rela_root/journeys/fixtures/substitute/the-answer.hbs:$template/the-answer.hbs" 197 | } 198 | ) 199 | ) 200 | (with "multiple specs indicating to read them from stdin" 201 | it "fails as this cannot be done" && { 202 | WITH_SNAPSHOT="$snapshot/fail-multiple-templates-from-stdin" \ 203 | expect_run $WITH_FAILURE "$exe" substitute -d <(echo does not matter) :first.out :second.out 204 | } 205 | ) 206 | (with "data from stdin and template from stdin" 207 | it "fails" && { 208 | WITH_SNAPSHOT="$snapshot/fail-data-stdin-template-stdin" \ 209 | expect_run $WITH_FAILURE "$exe" substitute :output 210 | } 211 | ) 212 | (with "no data specification and no spec" 213 | it "fails" && { 214 | WITH_SNAPSHOT="$snapshot/fail-no-data-and-no-specs" \ 215 | expect_run $WITH_FAILURE "$exe" substitute 216 | } 217 | ) 218 | (with "data from stdin specification and no spec" 219 | it "fails" && { 220 | echo "foo: 42" | \ 221 | WITH_SNAPSHOT="$snapshot/fail-data-stdin-and-no-specs" \ 222 | expect_run $WITH_FAILURE "$exe" substitute 223 | } 224 | ) 225 | (with "data used in the template is missing" 226 | it "fails" && { 227 | echo 'hi: 42' | \ 228 | WITH_SNAPSHOT="$snapshot/fail-data-stdin-template-misses-key" \ 229 | expect_run $WITH_FAILURE "$exe" sub "$template/the-answer.hbs" 230 | } 231 | ) 232 | (with "not enough replacement values" 233 | it "fails" && { 234 | echo 'hi: 42' | \ 235 | WITH_SNAPSHOT="$snapshot/fail-not-enough-replacements" \ 236 | expect_run $WITH_FAILURE "$exe" sub "$template/the-answer.hbs" --replace=foo 237 | } 238 | ) 239 | (with "--verify" 240 | (with "invalid data" 241 | it "fails" && { 242 | echo 'secret: sec"ret' | \ 243 | WITH_SNAPSHOT="$snapshot/fail-validation-data-stdin-json-template" \ 244 | expect_run $WITH_FAILURE "$exe" sub --validate "$template/data.json.hbs" 245 | } 246 | ) 247 | (with "valid data" 248 | it "succeeds" && { 249 | echo 'secret: geheim' | \ 250 | WITH_SNAPSHOT="$snapshot/validation-success-data-stdin-json-template" \ 251 | expect_run $SUCCESSFULLY "$exe" sub --validate "$template/data.json.hbs" 252 | } 253 | ) 254 | ) 255 | -------------------------------------------------------------------------------- /tests/mcp/journey-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu -o pipefail 4 | exe=${1:?First argument is the executable under test} 5 | exe="$(cd "${exe%/*}" && pwd)/${exe##*/}" 6 | 7 | rela_root="${0%/*}" 8 | root="$(cd "${rela_root}" && pwd)" 9 | # shellcheck source=./tests/utilities.sh 10 | source "$root/utilities.sh" 11 | 12 | WITH_FAILURE=1 13 | SUCCESSFULLY=0 14 | 15 | fixture="$root/fixtures" 16 | snapshot="$root/snapshots" 17 | 18 | # shellcheck source=./tests/included-stateless-substitute.sh 19 | source "$root/included-stateless-substitute.sh" 20 | 21 | -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/data-stdin-json-data-validated-fix-with-replacements-stdout: -------------------------------------------------------------------------------- 1 | { 2 | "password": "$geheim\"niss" 3 | } -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/data-stdin-json-single-template-stdout: -------------------------------------------------------------------------------- 1 | : What is the answer to everything? 2 | : 42 -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/data-stdin-yaml-multi-template-stdout: -------------------------------------------------------------------------------- 1 | : What is the answer to everything? 2 | : 42 3 | 4 | : What is the answer to everything? 5 | : 42 -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/data-stdin-yaml-multi-template-stdout-explicit-separator: -------------------------------------------------------------------------------- 1 | : What is the answer to everything? 2 | : 42 3 | <-> 4 | : What is the answer to everything? 5 | : 42 -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/data-stdin-yaml-multi-template-to-same-file: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-apis-rs/generator/9ad1f063e5046f2f404ea77f0116e0fd25b4246d/tests/mcp/snapshots/substitute/data-stdin-yaml-multi-template-to-same-file -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/data-stdin-yaml-multi-template-to-same-file-again: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-apis-rs/generator/9ad1f063e5046f2f404ea77f0116e0fd25b4246d/tests/mcp/snapshots/substitute/data-stdin-yaml-multi-template-to-same-file-again -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/data-stdin-yaml-multi-template-to-same-file-again-output: -------------------------------------------------------------------------------- 1 | : What is the answer to everything? 2 | : 42 3 | -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/data-stdin-yaml-multi-template-to-same-file-explicit-separator: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-apis-rs/generator/9ad1f063e5046f2f404ea77f0116e0fd25b4246d/tests/mcp/snapshots/substitute/data-stdin-yaml-multi-template-to-same-file-explicit-separator -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/data-stdin-yaml-multi-template-to-same-file-explicit-separator-output: -------------------------------------------------------------------------------- 1 | : What is the answer to everything? 2 | : 42 3 | --- 4 | : What is the answer to everything? 5 | : 42 6 | -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/data-stdin-yaml-multi-template-to-same-file-output: -------------------------------------------------------------------------------- 1 | : What is the answer to everything? 2 | : 42 3 | 4 | : What is the answer to everything? 5 | : 42 6 | -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/data-stdin-yaml-single-relative-template-stdout: -------------------------------------------------------------------------------- 1 | : What is the answer to everything? 2 | : 42 -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/data-stdin-yaml-single-template-file-non-existing-directory: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-apis-rs/generator/9ad1f063e5046f2f404ea77f0116e0fd25b4246d/tests/mcp/snapshots/substitute/data-stdin-yaml-single-template-file-non-existing-directory -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/data-stdin-yaml-single-template-output-file-with-nonexisting-directory/some/sub/directory/output: -------------------------------------------------------------------------------- 1 | : What is the answer to everything? 2 | : 42 3 | -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/data-stdin-yaml-single-template-stdout: -------------------------------------------------------------------------------- 1 | : What is the answer to everything? 2 | : 42 -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/fail-data-stdin-and-no-specs: -------------------------------------------------------------------------------- 1 | error: No spec provided, neither from standard input, nor from file -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/fail-data-stdin-json-data-validated-stdout: -------------------------------------------------------------------------------- 1 | error: Validation of template output at 'stream' failed. It's neither valid YAML, nor JSON 2 | Caused by: 3 | 1: while parsing a flow mapping, did not find expected ',' or '}' at line 2 column 21 -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/fail-data-stdin-template-misses-key: -------------------------------------------------------------------------------- 1 | error: Failed to render template from template at 'the-answer' 2 | Caused by: 3 | 1: liquid: Unknown variable 4 | with: 5 | requested variable=the-answer 6 | available variables=hi -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/fail-data-stdin-template-stdin: -------------------------------------------------------------------------------- 1 | error: Stdin is a TTY. Cannot substitute a template without any data. -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/fail-invalid-data-format: -------------------------------------------------------------------------------- 1 | error: Could not deserialize data, tried JSON and YAML 2 | Caused by: 3 | 4: JSON deserialization failed 4 | 3: expected value at line 1 column 1 5 | 2: YAML deserialization failed 6 | 1: while parsing node, found unknown anchor at line 1 column 4 -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/fail-invalid-data-format-multi-document-yaml: -------------------------------------------------------------------------------- 1 | error: Could not deserialize data, tried JSON and YAML 2 | Caused by: 3 | 4: JSON deserialization failed 4 | 3: invalid number at line 1 column 2 5 | 2: YAML deserialization failed 6 | 1: deserializing from YAML containing more than one document is not supported -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/fail-multiple-templates-from-stdin: -------------------------------------------------------------------------------- 1 | error: Cannot read more than one template spec from standard input -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/fail-no-data-and-no-specs: -------------------------------------------------------------------------------- 1 | error: Stdin is a TTY. Cannot substitute a template without any data. -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/fail-not-enough-replacements: -------------------------------------------------------------------------------- 1 | error: Please provide --replace-value arguments in pairs of two. First the value to find, second the one to replace it with -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/fail-source-is-destination-single-spec: -------------------------------------------------------------------------------- 1 | error: Data model root must be an object -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/fail-validation-data-stdin-json-template: -------------------------------------------------------------------------------- 1 | error: Validation of template output at 'stream' failed. It's neither valid YAML, nor JSON 2 | Caused by: 3 | 1: while parsing a flow mapping, did not find expected ',' or '}' at line 2 column 20 -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/liquid/filter-base64: -------------------------------------------------------------------------------- 1 | aGVsbG8= -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/template-from-complex-template: -------------------------------------------------------------------------------- 1 | Introduction 2 | 3 | 4 | Hello Harald! 5 | 6 | 7 | /my/fancy/url.html 8 | 9 | 10 | 11 | This product's title contains the word Pack. 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | Harald 28 | 29 | Schneider 30 | 31 | Meier 32 | 33 | 34 | 35 | 36 | tomatotomato 37 | 38 | Anything you put between tags 39 | is turned into a comment. 40 | 41 | 42 | 43 | 44 | These shoes are not awesome. 45 | 46 | 47 | 48 | These shoes are not awesome. 49 | 50 | 51 | 52 | Hey Anonymous! 53 | 54 | 55 | 56 | 57 | This is a cake 58 | 59 | 60 | 61 | 62 | 63 | 1 64 | 65 | 66 | 67 | 2 68 | 69 | 70 | 71 | 3 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 1 81 | 82 | 83 | 84 | 2 85 | 86 | 87 | 88 | 3 89 | 90 | 91 | 92 | 93 | 94 | 5 95 | 96 | 97 | 98 | 99 | 1 100 | 2 101 | 3 102 | 103 | 4 104 | 105 | 5 106 | 107 | 108 | 109 | 110 | 1 111 | 112 | 2 113 | 114 | 3 115 | 116 | 4 117 | 118 | 119 | 120 | 4 121 | 122 | 3 123 | 124 | 2 125 | 126 | 1 127 | 128 | 129 | 130 | 131 | In Handlebars, {{ this }} will be HTML-escaped, but 132 | {{ that }}} will not. 133 | 134 | 135 | 136 | I am being captured. 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | I am 35 and my favorite food is pizza. 146 | 147 | 148 | 149 | 150 | 17 151 | /my/fancy/url.html 152 | Title 153 | 2 154 | 4 155 | 156 | 157 | 158 | one 159 | 160 | two 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | - apples 170 | 171 | - oranges 172 | 173 | - peaches 174 | 175 | - carrots 176 | 177 | - turnips 178 | 179 | - potatoes 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | - apples 188 | 189 | - oranges 190 | 191 | - peaches 192 | 193 | - carrots 194 | 195 | - turnips 196 | 197 | - potatoes 198 | 199 | - chairs 200 | 201 | - tables 202 | 203 | - shelves 204 | 205 | 206 | March 25, 2018 207 | 2.99 208 | 209 | 210 | 2.99 211 | 4 212 | 213 | 214 | 2 215 | 216 | 217 | 218 | 2.857142857142857 219 | parker moore 220 | Have you read 'James & the Giant Peach'? 221 | 1 < 2 & 3 222 | 223 | 224 | 225 | 1 226 | 3 227 | 228 | John and Paul and George and Ringo 229 | So much room for activities! 230 | 2 231 | 1 232 | 233 |
234 | Hello
235 | there
236 | 237 | 6 238 | Some fruit: apples, oranges, and bananas 239 | I sted to see the t through the 240 | I sted to see the train through the rain 241 | Take your protein pills and put your helmet on 242 | 243 | 244 | Take your protein pills and put my helmet on 245 | .moT rojaM ot lortnoc dnuorG 246 | 247 | 248 | plums, peaches, oranges, apples 249 | 250 | 1 251 | 252 | So much room for activities! 253 | 254 | 28 255 | 256 | 257 | 4 258 | 259 | quid 260 | ui 261 | 262 | 263 | Sally Snake, giraffe, octopus, zebra 264 | 265 | 266 | giraffe, octopus, Sally Snake, zebra 267 | 268 | So much room for activities! 269 | 270 | Have you read Ulysses? 271 | 272 | 273 | 274 | Hellothere 275 | 6 276 | Ground control, and so on 277 | Ground control to... 278 | 279 | 280 | ants, bugs, bees -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/template-stdin-hbs-output-stdout: -------------------------------------------------------------------------------- 1 | hello world -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/template-stdin-hbs-output-stdout-to-file: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-apis-rs/generator/9ad1f063e5046f2f404ea77f0116e0fd25b4246d/tests/mcp/snapshots/substitute/template-stdin-hbs-output-stdout-to-file -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/template-stdin-hbs-output-stdout-to-file-output: -------------------------------------------------------------------------------- 1 | hello world 2 | -------------------------------------------------------------------------------- /tests/mcp/snapshots/substitute/validation-success-data-stdin-json-template: -------------------------------------------------------------------------------- 1 | { 2 | "password": "geheim" 3 | } -------------------------------------------------------------------------------- /tests/mcp/utilities.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | WHITE="$(tput setaf 9 2>/dev/null || echo -n '')" 4 | YELLOW="$(tput setaf 3 2>/dev/null || echo -n '')" 5 | GREEN="$(tput setaf 2 2>/dev/null || echo -n '')" 6 | RED="$(tput setaf 1 2>/dev/null || echo -n '')" 7 | OFFSET=( ) 8 | STEP=" " 9 | 10 | function with_program () { 11 | local program="${1:?}" 12 | hash "$program" &>/dev/null || { 13 | function expect_run () { 14 | echo 1>&2 "${WHITE} - skipped (missing program)" 15 | } 16 | function expect_run_sh () { 17 | echo 1>&2 "${WHITE} - skipped (missing program)" 18 | } 19 | } 20 | } 21 | 22 | function title () { 23 | echo "$WHITE-----------------------------------------------------" 24 | echo "${GREEN}$*" 25 | echo "$WHITE-----------------------------------------------------" 26 | } 27 | 28 | function _context () { 29 | local name="${1:?}" 30 | shift 31 | echo 1>&2 "${YELLOW}${OFFSET[*]:-}[$name] $*" 32 | OFFSET+=("$STEP") 33 | } 34 | 35 | function step () { 36 | _note step "${WHITE}" "$*" 37 | } 38 | 39 | function stepn () { 40 | step "$*" $'\n' 41 | } 42 | 43 | function with () { 44 | _context with "$*" 45 | } 46 | 47 | function when () { 48 | _context when "$*" 49 | } 50 | 51 | function _note () { 52 | local name="${1:?}" 53 | local color="${2:-}" 54 | shift 2 55 | echo 1>&2 -n "${OFFSET[*]:-}${color}[$name] ${*// /}" 56 | } 57 | 58 | function it () { 59 | _note it "${GREEN}" "$*" 60 | } 61 | 62 | function precondition () { 63 | _note precondition "${WHITE}" "$*" 64 | } 65 | 66 | function shortcoming () { 67 | _note shortcoming "${RED}" "$*" 68 | } 69 | 70 | function step () { 71 | _note step "${WHITE}" "$*" 72 | } 73 | 74 | function stepn () { 75 | step "$*" $'\n' 76 | } 77 | 78 | function fail () { 79 | echo 1>&2 "${RED} $*" 80 | exit 1 81 | } 82 | 83 | function sandbox () { 84 | sandbox_tempdir="$(mktemp -t sandbox.XXXXXX -d)" 85 | # shellcheck disable=2064 86 | trap "popd >/dev/null" EXIT 87 | pushd "$sandbox_tempdir" >/dev/null \ 88 | || fail "Could not change directory into temporary directory." 89 | 90 | local custom_init="${1:-}" 91 | if [ -n "$custom_init" ]; then 92 | eval "$custom_init" 93 | fi 94 | } 95 | 96 | function expect_equals () { 97 | expect_run 0 test "${1:?}" = "${2:?}" 98 | } 99 | 100 | function expect_exists () { 101 | expect_run 0 test -e "${1:?}" 102 | } 103 | 104 | function expect_run_sh () { 105 | expect_run "${1:?}" bash -c -eu -o pipefail "${2:?}" 106 | } 107 | 108 | function expect_snapshot () { 109 | local expected=${1:?} 110 | local actual=${2:?} 111 | if ! [ -e "$expected" ]; then 112 | mkdir -p "${expected%/*}" 113 | cp -R "$actual" "$expected" 114 | fi 115 | expect_run 0 diff -r -N "$expected" "$actual" 116 | } 117 | 118 | function expect_run () { 119 | local expected_exit_code=$1 120 | shift 121 | local output= 122 | set +e 123 | if [[ -n "${SNAPSHOT_FILTER-}" ]]; then 124 | output="$("$@" 2>&1 | "$SNAPSHOT_FILTER")" 125 | else 126 | output="$("$@" 2>&1)" 127 | fi 128 | 129 | local actual_exit_code=$? 130 | if [[ "$actual_exit_code" == "$expected_exit_code" ]]; then 131 | if [[ -n "${WITH_SNAPSHOT-}" ]]; then 132 | local expected="$WITH_SNAPSHOT" 133 | if ! [ -f "$expected" ]; then 134 | mkdir -p "${expected%/*}" 135 | echo -n "$output" > "$expected" || exit 1 136 | fi 137 | if ! diff "$expected" <(echo -n "$output"); then 138 | echo 1>&2 "${RED} - FAIL" 139 | echo 1>&2 "${WHITE}\$ $*" 140 | echo 1>&2 "Output snapshot did not match snapshot at '$expected'" 141 | echo 1>&2 "$output" 142 | if [ -n "${ON_ERROR:-}" ]; then 143 | eval "$ON_ERROR" 144 | fi 145 | exit 1 146 | fi 147 | fi 148 | echo 1>&2 149 | else 150 | echo 1>&2 "${RED} - FAIL" 151 | echo 1>&2 "${WHITE}\$ $*" 152 | echo 1>&2 "${RED}Expected actual status $actual_exit_code to be $expected_exit_code" 153 | echo 1>&2 "$output" 154 | if [ -n "${ON_ERROR:-}" ]; then 155 | eval "$ON_ERROR" 156 | fi 157 | exit 1 158 | fi 159 | set -e 160 | } 161 | -------------------------------------------------------------------------------- /uri_template_parser/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "uri_template_parser" 3 | version = "0.1.0" 4 | authors = ["Glenn Griffin "] 5 | edition = "2018" 6 | 7 | [lib] 8 | doctest = false 9 | 10 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 11 | 12 | [dependencies] 13 | nom = "5.0.0" 14 | --------------------------------------------------------------------------------