├── .github └── workflows │ └── build.yml ├── .gitignore ├── .travis-ci.sh ├── .travis.yml ├── CHANGES ├── LICENSE ├── Makefile ├── README.md ├── THANKS ├── TODO ├── VERSION ├── doc └── piqi-ocaml.md ├── dune ├── dune-project ├── examples ├── addressbook │ ├── Makefile │ ├── Makefile.ocaml │ ├── Makefile.ocaml_ext │ ├── README │ ├── add_person.ml │ ├── addressbook.ocaml.piqi │ ├── addressbook.piq │ ├── addressbook.proto │ ├── addressbook.proto.piqi │ ├── io_json_xml_pb.ml │ ├── list_people.ml │ └── test ├── custom-types │ ├── Makefile │ ├── README │ ├── example.piqi │ ├── piqirun_custom.ml │ └── skvl.piqi └── piq-config │ ├── Makefile │ ├── README │ ├── config.ml │ ├── config.piq │ └── config.piqi ├── make ├── Makefile.dirs └── OCamlMakefile ├── opam ├── piqic-ocaml ├── .gitignore ├── Makefile ├── dune ├── make_version.sh ├── piqi.ocaml.piqi ├── piqi.piqi-ocaml.piqi ├── piqic_common.ml ├── piqic_ocaml.ml ├── piqic_ocaml_defaults.ml ├── piqic_ocaml_ext.ml ├── piqic_ocaml_in.ml ├── piqic_ocaml_out.ml ├── piqic_ocaml_types.ml └── piqic_piqi.ml ├── piqirun ├── .gitignore ├── META.in ├── Makefile ├── dune ├── piqirun.ml ├── piqirun_ext.ml ├── piqirun_ext.mli ├── test.ml └── test.ocaml └── tests ├── Makefile ├── addressbook ├── array ├── Makefile ├── Makefile.ocaml ├── packed.piqi ├── test-all.piq └── test.ml ├── custom-types ├── misc ├── Ad.piqi ├── Makefile ├── P.piqi ├── Protocol.piqi ├── empty-record.piqi ├── variant-1.piqi └── variant-2.piqi ├── misc1 ├── Makefile ├── example.proto ├── example.proto.piqi ├── id.proto ├── id.proto.piqi └── test.ml ├── packed ├── Makefile ├── Makefile.ocaml ├── packed-array.piqi ├── packed-nocompat.piqi ├── packed.piqi ├── test-all.piq ├── test_packed.ml └── test_packed_array.ml ├── perf ├── Makefile ├── Makefile.ocaml ├── README ├── addressbook.ocaml.piqi ├── addressbook.piq ├── addressbook.proto.piqi └── test.ml ├── piq-config ├── piqi ├── Makefile ├── Makefile.ocaml ├── README ├── piqi.ocaml.piqi ├── piqobj.piqi └── test.ml ├── piqirun ├── Makefile └── test.ml └── riak_pb ├── Makefile ├── README ├── riak.proto ├── riak_kv.ocaml.piqi ├── riak_kv.proto └── riak_search.proto /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | build: 6 | strategy: 7 | fail-fast: false 8 | matrix: 9 | os: 10 | - macos-latest 11 | - ubuntu-latest 12 | ocaml-version: 13 | - 4.13.x 14 | - 4.03.x 15 | 16 | runs-on: ${{ matrix.os }} 17 | 18 | steps: 19 | - name: Checkout code 20 | uses: actions/checkout@v2 21 | 22 | - name: Use OCaml ${{ matrix.ocaml-version }} 23 | uses: ocaml/setup-ocaml@v2 24 | with: 25 | ocaml-compiler: ${{ matrix.ocaml-version }} 26 | 27 | - run: opam pin add piqi . 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.[oa] 2 | 3 | # OCaml: 4 | *.cm[ioxa] 5 | *.cmxa 6 | 7 | # OCamlMakefile: 8 | *.top 9 | ._d 10 | ._bcdi 11 | ._ncdi 12 | 13 | # vim: 14 | *.swp 15 | /_build/ 16 | -------------------------------------------------------------------------------- /.travis-ci.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | # build, run tests, build package, install package 6 | wget https://raw.githubusercontent.com/ocaml/ocaml-ci-scripts/master/.travis-opam.sh 7 | 8 | 9 | . .travis-opam.sh 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: c 2 | script: "./.travis-ci.sh" 3 | env: 4 | # NOTE: testing only against the current development version of piqilib, 5 | # because compatibility is not guaranteed with previous releases 6 | - PACKAGE="piqi" OCAML_VERSION=4.03 PINS=piqilib 7 | - PACKAGE="piqi" OCAML_VERSION=4.04 PINS=piqilib 8 | - PACKAGE="piqi" OCAML_VERSION=4.05 PINS=piqilib 9 | - PACKAGE="piqi" OCAML_VERSION=4.06 PINS=piqilib 10 | - PACKAGE="piqi" OCAML_VERSION=4.07 PINS=piqilib 11 | - PACKAGE="piqi" OCAML_VERSION=4.08 PINS=piqilib 12 | -------------------------------------------------------------------------------- /CHANGES: -------------------------------------------------------------------------------- 1 | Piqi-ocaml 0.7.8 (Aug 05, 2019) 2 | =============================== 3 | 4 | - Add support for the dune build system and use it from OPAM by default 5 | - Switch CI setup from Travis to GH Actions 6 | 7 | 8 | Piqi-ocaml 0.7.7 (Oct 12, 2019) 9 | =============================== 10 | 11 | - Switch from Pervasives to Stdlib 12 | 13 | 14 | Piqi-ocaml 0.7.6 (Sep 13, 2018) 15 | =============================== 16 | 17 | - Fix sometimes incorrect handling of unknown fields (--gen-preserve-unknown-fields) 18 | - (new) add support for .internal flag for fields that should be defined in OCaml but 19 | skipped during serialization/deserialization 20 | 21 | 22 | Piqi-ocaml 0.7.5 (May 31, 2016) 23 | =============================== 24 | 25 | - Switch to tail-recursive List.map in the piqirun.ml runtime library 26 | - (new feature) Support for overriding ocaml name of an imported module 27 | 28 | 29 | Piqi-ocaml 0.7.4 (Mar 13, 2015) 30 | =============================== 31 | 32 | - Fix build and tests for OCaml < 4.02.0 33 | 34 | 35 | Piqi-ocaml 0.7.3 (Mar 13, 2015) 36 | =============================== 37 | 38 | - Fix compilation warnings for OCaml >= 4.02 39 | 40 | 41 | Piqi-ocaml 0.7.2 (May 17, 2014) 42 | =============================== 43 | 44 | New features: 45 | 46 | - Automatically prefix OCaml keywords and user-specified reserved names with 47 | underscores in generated OCaml code (thanks to Petter Urkedal) 48 | 49 | 50 | Piqi-ocaml 0.7.1 (April 17, 2014) 51 | ================================= 52 | 53 | Fixes: 54 | 55 | - 'piqic-ocaml --normalize-names' should be true by default 56 | 57 | Other changes: 58 | 59 | - piqic-ocaml: indent generated .ml code instead of relying on camlp4o 60 | 61 | 62 | Piqi-ocaml 0.7.0 (April 7, 2014) 63 | ================================ 64 | 65 | The main change of this release is the rewrite of the "piqic-ocaml" code 66 | generator on top of "piqi compile" interfaces. As a result, we were able to move 67 | the piqi-ocaml code from the main Piqi project repository into its own 68 | piqi-ocaml repo. 69 | 70 | Other important changes: 71 | 72 | - "piqic ocaml" becomes "piqic-ocaml" 73 | - "piqic ocaml-ext" turns into "piqic-ocaml --ext" 74 | - "piqirun" becomes a package with two subpackages: piqirun.pb -- for Protobuf 75 | serialization and piqirun.ext -- for multi-format serialization; piqirun.pb 76 | should be now used instead of piqirun package and piqirun.ext -- instead of 77 | piqi.lib 78 | - "--gen-defaults" compiler option is deprecated. "piqic-ocaml" now generates 79 | _piqi.default_X automatically 80 | - Much more cleaner and stable interfaces in the piqilib dependency (those 81 | that piqi-ocaml depends on); this allows us to release piqilib and 82 | piqi-ocaml independently 83 | - Drop support for OCaml <= 3.11 84 | - Deprecate and remove "pa_openin" camlp4 extension; "pa_labelscope" 85 | extensions is kept for now as a part of the piqilib package so that existing 86 | programs can still use it 87 | - Deprecate and remove "piqic-ocaml -o " command-line option 88 | 89 | Fixes: 90 | 91 | - Fix generation of incorrect code for custom OCaml types 92 | 93 | New features: 94 | 95 | - Preserve unknown Protobuf fields through deserialization-serialization cycle 96 | (new piqic-ocaml --gen-preserve-unknown-fields flag) 97 | - New conversion options: piq_frameless_output, piq_frameless_input, 98 | piq_relaxed_parsing 99 | - Add the ability to specify conversion options for generated 100 | _piqi_ext.print_/prerr_ functions 101 | - Add piqic-ocaml --piqi-version flag for printing piqilib version 102 | - Add piqic-ocaml --runtime option for specifying the name of the 103 | Protobuf serialization runtime module (default = Piqirun) 104 | 105 | Miscellaneous: 106 | 107 | - Rename conversion option: json_omit_null_fields -> json_omit_missing_fields 108 | - Improved piqi-ocaml documentation (available at doc/piqi-ocaml.md) 109 | - Continuous builds via Travis-CI 110 | 111 | 112 | Piqi 0.6.5 (October 27, 2013) 113 | ============================= 114 | 115 | Changelog for v0.6.5 and earlier releases is available here (the piqi-ocaml 116 | project used to be a subtree in the piqi repository): 117 | 118 | https://github.com/alavrik/piqi/blob/master/CHANGES 119 | 120 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | include make/Makefile.dirs 2 | 3 | 4 | DESTDIR ?= /usr/local 5 | 6 | 7 | DIRS = piqirun piqic-ocaml 8 | 9 | 10 | .PHONY: install uninstall clean distclean test 11 | 12 | 13 | install: 14 | $(MAKE) -C piqirun install 15 | -install -d $(DESTDIR)/bin 16 | install piqic-ocaml/piqic-ocaml $(DESTDIR)/bin 17 | 18 | 19 | uninstall: 20 | $(MAKE) -C piqirun uninstall 21 | rm -f $(DESTDIR)/bin/piqic-ocaml 22 | 23 | 24 | test: 25 | $(MAKE) -C tests 26 | 27 | 28 | distclean: 29 | $(MAKE) clean 30 | $(MAKE) -C tests clean 31 | 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![build](https://github.com/alavrik/piqi-ocaml/actions/workflows/build.yml/badge.svg)](https://github.com/alavrik/piqi-ocaml/actions/workflows/build.yml) 2 | 3 | 4 | Piqi is a multi-format data serialization system for OCaml. It provides a 5 | uniform interface for serializing OCaml data structures to JSON, XML and 6 | Protocol Buffers formats. 7 | 8 | 9 | A typical Piqi usage scenario involves the following steps: 10 | 11 | **1. Install piqi-ocaml** -- see installation instructions below. 12 | 13 | 14 | **2. Describe data structures using the Piqi data definition language or 15 | Protocol Buffers `.proto` files** 16 | 17 | The [Piqi](http://piqi.org/doc/piqi/) data definition language can describe many 18 | OCaml types, both primitive and user-defined. This includes integers, floats, 19 | booleans, strings, binaries, lists, records and polymorphic variants. 20 | 21 | `.piqi` modules can be converted to and from Protocol Buffers `.proto` files: 22 | 23 | piqi to-proto X.piqi 24 | piqi of-proto X.proto 25 | 26 | 27 | **3. Call the Piqi compiler (piqic-ocaml) to generate OCaml type definitions and 28 | serialization code** 29 | 30 | piqic-ocaml X.piqi 31 | 32 | 33 | **4. Use generated serializes/deserializers in a user's program** -- the desired 34 | serialization format can be specified at runtime. For 35 | [examples](examples/addressbook/io_json_xml_pb.ml): 36 | 37 | 38 | % deserialize a data structure from Protocol Buffers 39 | let buf = Piqirun.init_from_string bytes in 40 | let addressbook = Addressbook_piqi.parse_address_book buf in ... 41 | 42 | % serialize it as JSON 43 | let json = Addressbook_piqi_ext.gen_address_book addressbook `json in ... 44 | 45 | % serialize it as pretty-printed JSON 46 | let json_pretty = Addressbook_piqi_ext.gen_address_book addressbook `json_pretty in ... 47 | 48 | % serialize it as XML 49 | let xml = Addressbook_piqi_ext.gen_address_book addressbook `xml in ... 50 | 51 | 52 | Examples 53 | -------- 54 | 55 | See [examples/addressbook](examples/addressbook/) and other projects in the 56 | [examples](examples/) directory. 57 | 58 | 59 | Installation 60 | ------------ 61 | 62 | ### Installing using OPAM 63 | 64 | In order to install Piqi using [OPAM](https://opam.ocaml.org/), run the 65 | following command: 66 | 67 | opam install piqi 68 | 69 | This command will install the latest stable version of Piqi that includes `piqi` 70 | and `piqic-ocaml` executables and runtime libraries for OCaml. 71 | 72 | To install the latest development version of Piqi, use opam pinning. 73 | 74 | opam pin add -n --dev-repo piqilib 75 | opam pin add -n --dev-repo piqi 76 | opam install piqi 77 | 78 | 79 | ### Installing from source code 80 | ------------------------------- 81 | 82 | 1. Download and install [piqi and piqilib](http://github.com/alavrik/piqi) 83 | 84 | Follow general build and installation instructions from the INSTALL file. 85 | 86 | After that, build and install the `piqilib` OCaml library by running 87 | 88 | make ocaml 89 | make ocaml-install 90 | 91 | 92 | 2. Build and install `piqi-ocaml` 93 | 94 | ``` 95 | make 96 | make install 97 | ``` 98 | 99 | To uninstall: 100 | 101 | make uninstall 102 | 103 | 104 | Documentation 105 | ------------- 106 | 107 | Piqi OCaml documentation is available at http://piqi.org/doc/ocaml/ 108 | 109 | The master copy is located in this repository: 110 | [doc/piqi-ocaml.md](doc/piqi-ocaml.md) 111 | 112 | 113 | Bugs 114 | ---- 115 | 116 | Please report found problems using [GitHub 117 | issues](http://github.com/alavrik/piqi-ocaml/issues). 118 | 119 | 120 | Mailing list 121 | ------------ 122 | 123 | http://groups.google.com/group/piqi 124 | 125 | 126 | Contributing 127 | ------------ 128 | 129 | Your contributions are always welcome. Just open a pull request. Check [TODO 130 | list](TODO) for ideas. 131 | 132 | 133 | Some useful commands: 134 | 135 | make test 136 | 137 | 138 | License 139 | ------- 140 | 141 | [Apache License Version 2.0](LICENSE) 142 | 143 | -------------------------------------------------------------------------------- /THANKS: -------------------------------------------------------------------------------- 1 | The following people have contributed to piqi-ocaml: 2 | 3 | Koen De Keyser 4 | Petter Urkedal 5 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | This is a list of ideas of how to improve piqi-ocaml. 2 | 3 | Current issues and feature requests: https://github.com/alavrik/piqi-ocaml/issues 4 | 5 | 6 | New features 7 | ------------ 8 | 9 | - optionally, treat Piqi enums as OCaml integers and generate enum constants as 10 | constants instead of polymorphic variants 11 | 12 | 13 | Optimizations 14 | ------------- 15 | 16 | - profile and optimize piqirun.ml, e.g. varint encoding, record parsing, GC 17 | behavior, etc. 18 | 19 | 20 | Miscellaneous 21 | ------------- 22 | 23 | - add piqi-any example that demonstrates use of embedded JSON/XML (now that we 24 | have piqi_piqi.ml as a part of piqilib); see piqi-erlang/examples/piqi-any for 25 | details 26 | 27 | - more tests 28 | 29 | 30 | 31 | # ex: et sw=4 ts=4 32 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.7.8-dev 2 | -------------------------------------------------------------------------------- /doc/piqi-ocaml.md: -------------------------------------------------------------------------------- 1 | Overview 2 | -------- 3 | 4 | Piqi includes a data serialization system for OCaml. It can be used for 5 | serializing OCaml values in 4 different formats: Google Protocol Buffers, 6 | [JSON](/doc/encodings/#json), [XML](/doc/encodings/#xml) and [Piq](/doc/piq/). 7 | 8 | A typical Piqi usage scenario involves the following steps: 9 | 10 | **1. Build and install Piqi libraries for OCaml** 11 | : Piqi source code is distributed as a self-contained package with no external 12 | dependencies. Builds have been tested on all major platforms. 13 | 14 | The installation instructions are available 15 | [here](https://github.com/alavrik/piqi-ocaml). 16 | 17 | **2. Describe data structures using the Piqi data definition language** 18 | : The [Piqi](/doc/piqi/) data definition language can describe many OCaml 19 | types, both primitive and user-defined. This includes int, uint32 and 20 | uint64, floats, bools, strings, lists, arrays, records and polymorphic 21 | variants (including sub-variants). 22 | 23 | In addition to types supported by default, Piqi has a mechanism for adding 24 | support for arbitrary monomorphic OCaml types. It can be used, for example, 25 | to add support for OCaml's `nativeint` or `char`. 26 | 27 | Refer to the "Piqi to OCaml mapping" section below for details. 28 | 29 | **3. Call `piqic-ocaml`, the Piqi compiler for OCaml, to generate OCaml type 30 | definitions and serialization code** 31 | : See the next section for the detailed description. 32 | 33 | **4. Use generated serializes/deserializers/printers in a user's program** 34 | : When multi-format serialization mode is used, one can specify a desired 35 | format at runtime. 36 | 37 | **5. Link the user's program with the Piqi runtime library** 38 | : There are two Piqi runtime libraries: `piqirun.pb` and `piqirun.ext`. The 39 | first one is for Protocol Buffers serialization. The second one is used for 40 | multi-format serialization. See the next section for more details. 41 | 42 | The [Examples](#examples) section contains links to several sample OCaml 43 | projects that use Piqi for data serialization and demonstrate usage of steps 44 | 2--5. 45 | 46 | 47 | Piqi compiler and generated OCaml code 48 | -------------------------------------- 49 | 50 | `piqic-ocaml`, the Piqi compiler for OCaml, can generate two flavors of OCaml 51 | code. The first one is used only for Protocol Buffers serialization. The second 52 | flavor can be used for serializing multiple formats, including Protobuf, XML, 53 | JSON and Piq. 54 | 55 | Multi-format serialization is an extension of the basic Protocol Buffers 56 | serialization mode. It requires linking with a different runtime library. 57 | 58 | 59 | ### Protocol Buffers serialization 60 | 61 | When called without `--multi-format`, `piqic-ocaml` generates OCaml type 62 | definitions and code for Protocol Buffers serialization. 63 | 64 | `piqic-ocaml` command takes a Piqi module `/.piqi` and produces 65 | an output `.ml` file in the current working directory. By default, 66 | `` is a valid OCaml module name equivalent to `_piqi`. 67 | 68 | For each specified, imported or included module `.piqi`, the compiler tries 69 | to load and automatically include `.ocaml.piqi`. This mechanism is called 70 | *Extension Modules*. It is described in detail in the Piqi language 71 | [section](/doc/piqi#extensionmodules). 72 | 73 | Output directory can be overridden using the `-C` command-line option. 74 | 75 | Generated `.ml` file contains OCaml type definitions and functions 76 | for serializing and deserializing OCaml values. 77 | 78 | For each defined data type ``, `piqic-ocaml` produces several 79 | functions: 80 | 81 | - `parse_` -- for deserializing a value 82 | 83 | - `gen_` -- for serializing a value 84 | 85 | - `default_` -- type constructor: returns a minimally serializable 86 | value of this type 87 | 88 | Compiled `.ml` files should be linked with the `piqilib.pb` findlib package. For 89 | example: 90 | 91 | # generate "test_piqi.ml" 92 | piqic-ocaml test.piqi 93 | 94 | # compile and link it with the runtime library using findlib/ocamlfind 95 | ocamlfind ocamlc -linkpkg -package piqilib.pb test_piqi.ml 96 | 97 | 98 | ### Customized runtime library for Protocol Buffers serialization 99 | 100 | Under the hood, the `piqilib.pb` package resolves to a single compiled module 101 | named `Piqirun`. This is the module the generated `_piqi.ml` OCaml code uses. 102 | 103 | Sometimes, it may be useful to use a modified version of the `Piqirun` module. 104 | For example, a customized version could have extra optimizations, improve error 105 | handling or implement serialization code for [Custom OCaml 106 | types](#customocamltypes). 107 | 108 | To swap the default runtime for a customized one, call `piqic-ocaml` with 109 | `--runtime ` option, where `` is the name of the 110 | module to use instead of `Piqirun`. 111 | 112 | 113 | ### Multi-format serialization 114 | 115 | When `piqic-ocaml` is called with the `--multi-format` flag, it generates an 116 | additional code for serializing values in XML, JSON, Protobuf and Piq formats. 117 | 118 | The additional module generated by this compiler is named 119 | `_piqi_ext.ml`. 120 | 121 | The `parse_` and `gen_` functions from this module take an 122 | additional parameter specifying which serialization format to use: 123 | 124 | type input_format = [ `piq | `json | `xml | `pb | `wire ] 125 | 126 | type output_format = [ input_format | `json_pretty | `xml_pretty ] 127 | 128 | In addition, `piqic-ocaml --multi-format` generates some other functions: 129 | 130 | - `print_` -- for printing a value to `stdout` in Piq format 131 | 132 | Each `parse_` and `gen_` function accepts an optional 133 | `?opts` argument representing a set of serialization options that can be 134 | constructed using `Piqirun_ext.make_options`: 135 | 136 | (* Construct serialization options to be passed as an optional argument to 137 | * gen_ and parse_ functions. Available options: 138 | * 139 | * pretty_print 140 | * 141 | * Pretty-print generated JSON and XML output (default = true) 142 | * 143 | * json_omit_missing_fields 144 | * 145 | * Omit missing optional and empty repeated fields from JSON 146 | * output instead of representing them as {"field_name": null} and 147 | * {"field_name", []} JSON fields (default = true) 148 | * 149 | * use_strict_parsing 150 | * 151 | * Treat unknown and duplicate fields as errors when parsing JSON, 152 | * XML and Piq formats (default = false) 153 | * 154 | * piq_frameless_output 155 | * 156 | * Print a frame (i.e. : []) around a single output Piq object 157 | * (default=false) 158 | * 159 | * piq_frameless_input 160 | * 161 | * Expect a frame around a single input Piq object (default=false) 162 | * 163 | * piq_relaxed_parsing 164 | * 165 | * Parse Piq format using "relaxed" mode (default=false); 166 | * 167 | * For instance, when set to `true`, single-word string literals don't have 168 | * to be quoted 169 | *) 170 | val make_options: 171 | ?pretty_print:bool -> 172 | ?json_omit_missing_fields:bool -> 173 | ?use_strict_parsing:bool -> 174 | ?piq_frameless_output:bool -> 175 | ?piq_frameless_input:bool -> 176 | ?piq_relaxed_parsing:bool -> 177 | unit -> options 178 | 179 | 180 | Compiled `.ml` files should be linked with the `piqirun.ext` findlib package. 181 | For example: 182 | 183 | # generate "test_piqi.ml" and "test_piqi_ext.ml" 184 | piqic-ocaml --multi-format test.piqi 185 | 186 | # compile and link them with the runtime library using findlib/ocamlfind 187 | ocamlfind ocamlc -linkpkg -package piqirun.ext test_piqi.ml test_piqi_ext.ml 188 | 189 | 190 | ### Command-line parameters 191 | 192 | `piqic-ocaml` accepts the following command-line parameters. 193 | 194 | - `--multi-format` generate extended OCaml stubs for multi-format 195 | (JSON/XML/Piq/Pb) serialization, i.e. `_piqi_ext.ml`a file 196 | 197 | - `--ext` same as `--multi-format` 198 | 199 | - `--normalize-names true|false` -- convert "CamelCase"-style identifiers from 200 | the original type spec into "camel_case" OCaml names (names will be 201 | capitalized when appropriate). When the argument is `false`, the original 202 | identifiers will be lowercased without performing any additional 203 | transformations, e.g. "CamelCase" turns into "camelCase". The default value 204 | is `true`. 205 | 206 | - `--reserved-name` -- add a reserved name in addition to the standard OCaml 207 | keywords. Can be used several times. Such names will be prefixed with 208 | underscores in the generated OCaml code. 209 | 210 | - `--runtime ` name of the Protobuf serialization runtime module 211 | (default = Piqirun) 212 | 213 | - `-C ` -- specify output directory for the generated `.ml` files. 214 | 215 | - `-I ` -- add directory to the list of imported .piqi search paths 216 | 217 | - `-e ` -- try including extension for all loaded modules (can be 218 | used several times) 219 | 220 | - `--gen-preserve-unknown-fields` -- generate code that preserves unknown 221 | Protobuf fields when they are serialized back. When enabled, unknown 222 | (unrecognized) Protobuf fields are captured during de-serialization in a 223 | special 'piqi_unknown_pb' field and automatically written back when the 224 | record is serialized to Protobuf. 225 | 226 | - `--strict` treat unknown and duplicate fields as errors 227 | 228 | - `--no-warnings` -- don't print warnings 229 | 230 | - `--trace` -- turn on tracing (verbose output) 231 | 232 | - `--version` -- print piqi-ocaml version and exit 233 | 234 | - `--piqi-version` -- print piqi (piqilib) version and exit 235 | 236 | - `-h, --help` -- print command-line options help 237 | 238 | 239 | Piqi to OCaml mapping 240 | --------------------- 241 | 242 | The following sections describe how different Piqi constructs such as modules 243 | and types are mapped to OCaml. 244 | 245 | ### Modules 246 | 247 | The name of OCaml module is derived from Piqi module name unless overridden by 248 | `ocaml-module` top-level field. 249 | 250 | If Piqi module name is "example.com/foo/bar", then "Bar" (the last part of the 251 | Piqi module name) will be used as the OCaml module name. It is possible to 252 | override such default name assignment by specifying 253 | `.ocaml-module ""` in the Piqi module. 254 | 255 | #### Includes 256 | 257 | Piqi takes all "include" directives of a Piqi module, resolves them internally 258 | and produces a compound Piqi module which is then mapped to the resulting OCaml 259 | module. 260 | 261 | #### Imports 262 | 263 | Piqi "import" directives are mapped to OCaml modules in the following way. 264 | 265 | Imports that do not specify local module name are used directly as a part of 266 | Ocaml type names for imported types. For example, 267 | 268 | .import [ .module example.com/foo/bar ] 269 | .variant [ 270 | .name v 271 | .option [ 272 | .name o 273 | .type bar/t 274 | ] 275 | ] 276 | 277 | is mapped to: 278 | 279 | type v = [ `o of Bar.t ] 280 | 281 | Imports that do have imported module name result in generation of module alias. 282 | For example, 283 | 284 | .import [ .module example.com/foo/bar .name fum ] 285 | 286 | is mapped to: 287 | 288 | module Fum = Bar 289 | 290 | It is possible to override ocaml names of an imported module by using 291 | `.ocaml-name` and `.ocaml-module` properties in the import statement. For 292 | example, given an import statement like this: 293 | 294 | .import [ 295 | .module m 296 | .ocaml-module "Foo.Bar" 297 | .ocaml-name "Fum" 298 | ] 299 | 300 | `piqic-ocaml` will generate the following statement at the top of the generated 301 | `_piqi.ml`: 302 | 303 | module Fum = Foo.Bar 304 | 305 | 306 | ### Primitive types 307 | 308 | The table below represents correspondence between Piqi primitive types and OCaml 309 | types. 310 | 311 | (Mapping between Piqi and Protocol Buffers primitive type is documented 312 | [here](/doc/protobuf/#primitivetypes)). 313 | 314 | Piqi type(s) OCaml type Protobuf type(s) 315 | ---------------------------------------------------------- ------------ ------------------------------------------ 316 | bool bool bool 317 | string string string 318 | binary string bytes 319 | int, uint int sint32, uint32 320 | int32, uint32, int32-fixed, uint32-fixed, protobuf-int32 int32 sint32, uint32, sfixed32, fixed32, int32 321 | int64, uint64, int64-fixed, uint64-fixed, protobuf-int64 int64 sint64, uint64, sfixed64, fixed64, int64 322 | float, float64, float32 float double, float 323 | 324 | If there is a need to add serialization support for other OCaml types, such as 325 | `char`, `nativeint` or `bigint`, refer to [Custom OCaml 326 | types](#customocamltypes) section which describes a method for mapping custom 327 | OCaml types to Piqi types. 328 | 329 | ### User-defined types 330 | 331 | - Type names 332 | 333 | Each user-defined type is identified by its name. Piqi type names are 334 | converted to OCaml type name using the following rule. 335 | 336 | By default, Piqi identifiers are normalized and all hyphen characters are 337 | replaced by underscores. Normalization means converting "CamelCase" to 338 | "camel-case". 339 | 340 | If `--normalize false` command-line option is specified, then instead of 341 | full normalization, the first letter of the type name is uncapitalized. 342 | 343 | Sometimes it is necessary to override this rule and specify a custom OCaml 344 | name for a type. For example, when a Piqi type name conflicts with one of 345 | OCaml keywords. In such case, custom OCaml name can be specified using 346 | `.ocaml-name ""` field next to the original `.name ` 347 | entry. (This feature also works for field names, option names, import names 348 | and function names). 349 | 350 | For those Piqi fields or options that do not specify names, OCaml name is 351 | derived from the name of the Piqi type for that field. 352 | 353 | - Records are mapped to OCaml records. 354 | 355 | As a workaround for OCaml's flat namespace for record labels, Piqi puts each 356 | record definition in a separate OCaml module. The module's name is set to 357 | the record's name. 358 | 359 | For example, Piqi record 360 | 361 | .record [ 362 | .name r 363 | .field [ .name a .type int ] 364 | ] 365 | 366 | will be mapped to the following OCaml module: 367 | 368 | module R = 369 | struct 370 | record t = { mutable a : int } 371 | end 372 | 373 | (In fact, the real example would be more verbose, because Piqi uses 374 | recursive modules which require signature definition in addition to module 375 | implementation.) 376 | 377 | To make working with records defined in separate modules easier, you can use 378 | "local opens" introduced in OCaml 3.12. For example, records can be created 379 | as 380 | 381 | R.({ a = 10; b = ... }) 382 | 383 | instead of 384 | 385 | {R.a = 10; b = ...} 386 | 387 | Similarly, if you have a significant portion of code working with some 388 | record's fields, you can `open` the record's module before the code: 389 | 390 | let open R in 391 | ... 392 | 393 | (let open R in ... is a full equivalent of R.( ... )) 394 | 395 | This way, you can refer to record felds as `x.a` instead of `x.R.a`. Note 396 | that this doesn't work with several records simultaneously. 397 | 398 | 399 | **required** Piqi fields are mapped directly to OCaml record fields. 400 | 401 | 402 | **optional** Piqi fields of type `` are mapped to fields with type 403 | ` option`. 404 | 405 | **optional** Piqi fields with specified default values are mapped 406 | to OCaml fields the same way as required fields unless 407 | `.ocaml-optional` flag is specified in which case the field will 408 | have type ` option`. 409 | 410 | If a value of such field is not defined in serialized object, it 411 | will be set to the default value during deserialization. Also, see 412 | [Limitations](#limitations) section below. 413 | 414 | **repeated** Piqi fields of type `` are mapped to fields with type 415 | ` list`. It is possible to use OCaml `array` instead of `list` by 416 | specifying an additional `.ocaml-array` property in the field definition. 417 | 418 | It is possible for some fields to generate OCaml definitions but exclude 419 | them during serialization/deserialization. This is achieved by adding 420 | `.internal` flag under `.field [ ... ]`. 421 | 422 | - Enums and Variants are mapped directly to OCaml polymorphic variants. 423 | 424 | For example, these definitions: 425 | 426 | .enum [ 427 | .name e 428 | .option [ .name a ] 429 | .option [ .name b ] 430 | ] 431 | .varint [ 432 | .name v 433 | .option [ .type e ] 434 | .option [ .name f ] 435 | .option [ .name i .type int ] 436 | ] 437 | 438 | are mapped to: 439 | 440 | type e = [ `a | `b ] 441 | type v = [ e | `f | `i of int ] 442 | 443 | - List type is mapped OCaml list type. 444 | 445 | For example, 446 | 447 | .list [ 448 | .name l 449 | .type x 450 | ] 451 | 452 | is mapped to: 453 | 454 | type l = x list 455 | 456 | It is possible to use OCaml `array` instead of `list` by specifying an 457 | additional `.ocaml-array` property in the field definition, e.g.: 458 | 459 | .list [ 460 | .name l 461 | .type x 462 | .ocaml-array 463 | ] 464 | 465 | is mapped to: 466 | 467 | type l = x array 468 | 469 | - Aliases are mapped to OCaml type definitions. 470 | 471 | For example, 472 | 473 | .alias [ 474 | .name a 475 | .type x 476 | ] 477 | 478 | is mapped to: 479 | 480 | type a = x 481 | 482 | ### Custom OCaml types 483 | 484 | Piqi provides a way to define mappings between custom OCaml types and Piqi 485 | types. Such mechanism is useful when there is a need to automatically serialize 486 | an OCaml type using some relevant Piqi type, but there is no way to describe the 487 | desired OCaml type using Piqi. 488 | 489 | Inability to use Piqi to define an OCaml type would mean that the OCaml type is 490 | either a primitive built-in or abstract type (e.g. `char` or `bigint`), or some 491 | higher-order parametric type (e.g. `string Map.Make(String).t`). 492 | 493 | The mapping mechanism works as follows. Suppose we need to add support for 494 | serializing OCaml's `char` type as Piqi `int`. This can be done in a few steps: 495 | 496 | 1. First, define Piqi alias for such mapping: 497 | 498 | .alias [ 499 | % the new Piqi type 500 | .name char 501 | 502 | % the original Piqi type 503 | .type int 504 | 505 | % OCaml type (should be point to the namespace with the mapping 506 | % implementation -- see below) 507 | .ocaml-type "Piqirun_custom.char" 508 | 509 | % optionally, define a custom OCaml name for this type 510 | % .ocaml-name "char" 511 | ] 512 | 513 | 2. Second, implement runtime functions for mapping the custom OCaml type to the 514 | Piqi type: 515 | 516 | In module `piqirun_custom.ml`: 517 | 518 | type char = Char.t 519 | 520 | let char_of_int: int -> char = Char.chr 521 | let char_to_int: char -> int = Char.code 522 | 523 | After that, the only thing that's left is to compile and link 524 | `piqirun_custom.ml` with your OCaml program. 525 | 526 | More examples of how to map various OCaml types to Piqi types can be found 527 | [here](http://github.com/alavrik/piqi-ocaml/tree/master/examples/custom-types/). 528 | 529 | ### Piqi extensions 530 | 531 | There is no direct notion of Piqi extensions in OCaml: Piqi extensions are all 532 | resolved and applied to Piqi types before generating OCaml types from them. 533 | 534 | Examples 535 | -------- 536 | 537 | - The first example is based on the "addressbook" example from Protocol 538 | Buffers source distribution. It contains OCaml implementation of two simple 539 | programs: for adding a record to an addressbook and for listing addressbook 540 | contents. The programs implement the same functionality as programs from the 541 | Protocol Buffers 542 | [examples](https://github.com/google/protobuf/tree/v2.6.1/examples) written 543 | in C++, Java and Python. 544 | 545 | [examples/ocaml](http://github.com/alavrik/piqi-ocaml/tree/master/examples/) 546 | 547 | - Data serialization in XML, JSON and Piq formats using `piqic-ocaml 548 | --multi-format` 549 | 550 | In the same directory, there is the `io_json_xml_pb.ml` OCaml module. It 551 | reads and writes the addressbook data structure from the previous example in 552 | a variety of formats. 553 | 554 | - Piqi implementation itself makes another example 555 | 556 | Piqi is implemented in OCaml but the Piqi language and Piqi internal 557 | representation are defined in a series of Piqi specifications which are 558 | mapped to OCaml types. 559 | 560 | [Piqi self-specifications](http://github.com/alavrik/piqi/tree/master/piqi/) 561 | 562 | - More complicated example demonstrating complex types and module imports 563 | 564 | Piqi compiler for OCaml (`piqic-ocaml`) produces OCaml parsers and 565 | generators from Piqi self-specification 566 | ([piqi.piqi](/self-definition/#piqi_piqi)). After that, an OCaml 567 | program reads (and writes back) Piqi self-specification represented as a 568 | binary object. 569 | 570 | [tests/ocaml\_piqi](http://github.com/alavrik/piqi-ocaml/tree/master/tests/piqi/) 571 | 572 | - Examples of serializing custom OCaml types using Piqi 573 | 574 | [examples/ocaml-custom-types](http://github.com/alavrik/piqi-ocaml/tree/master/examples/custom-types/) 575 | 576 | - Example of using [Piq](/doc/piq) as a config file format 577 | 578 | [examples/ocaml-piq-config](http://github.com/alavrik/piqi-ocaml/tree/master/examples/piq-config/) 579 | 580 | Limitations 581 | ----------- 582 | 583 | The way how Piqi records are mapped to OCaml records introduces several 584 | limitations. 585 | 586 | - Limited support for defaults. 587 | 588 | There is no way to tell if the value of an optional field came from the 589 | original serialized object or it is the default value. 590 | 591 | There is no way to skip default values when serializing an optional field 592 | since concrete value for that field must be always present in the OCaml 593 | record. This behavior may be optimized in the future, but at the moment, it 594 | will produce somewhat excessive serialized representation for optional 595 | fields with default values. 596 | 597 | - No other dynamic properties. 598 | 599 | For example, in Protocol Buffers, there is a way to get the count of 600 | repeated fields and access them using field index. (Update: now it is 601 | possible to do that by specifying to use OCaml arrays instead of lists for 602 | repeated fields). 603 | 604 | An alternative method would use OCaml objects for representing records, 605 | providing "setters" and "getters" for object fields. Such method is used, for 606 | example, in Protocol Buffers mappings for C++, Java and Python languages. 607 | 608 | Although it is possible (and even easier) to implement this method for OCaml, 609 | the current method has several advantages: 610 | 611 | - Native syntax for record construction and field access. 612 | 613 | - Pattern matching works with records. 614 | 615 | - No runtime overhead of calling constructors, setters and getters. 616 | 617 | Other limitations: 618 | 619 | - Piqi runtime library hasn't been heavily optimized for performance yet. 620 | 621 | Supported OCaml and Protocol Buffers versions 622 | --------------------------------------------- 623 | 624 | Piqi works with OCaml \>= 3.12 and Protocol Buffers \>= 2.3.0 625 | -------------------------------------------------------------------------------- /dune: -------------------------------------------------------------------------------- 1 | (env (dev (flags (:standard -warn-error -A)))) 2 | -------------------------------------------------------------------------------- /dune-project: -------------------------------------------------------------------------------- 1 | (lang dune 2.0) 2 | (name piqi) 3 | 4 | (package (name piqi)) 5 | (package (name piqirun)) 6 | -------------------------------------------------------------------------------- /examples/addressbook/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all ocaml test ocaml_ext test_ext clean 2 | 3 | 4 | all: ocaml test ocaml_ext test_ext 5 | 6 | 7 | ocaml: 8 | GOAL=add_person $(MAKE) -f Makefile.ocaml 9 | GOAL=list_people $(MAKE) -f Makefile.ocaml 10 | 11 | 12 | test: 13 | ./test 14 | 15 | 16 | ocaml_ext: 17 | rm -f addressbook_piqi.cm? # forcing make to rebuild it 18 | $(MAKE) -f Makefile.ocaml_ext 19 | 20 | 21 | test_ext: ocaml_ext 22 | ./io_json_xml_pb addressbook.piq.pb 23 | 24 | 25 | clean: 26 | GOAL=add_person $(MAKE) -f Makefile.ocaml clean 27 | GOAL=list_people $(MAKE) -f Makefile.ocaml clean 28 | rm -f addressbook.piq.pb addressbook.pb.piq l1 l2 29 | $(MAKE) -f Makefile.ocaml_ext clean 30 | 31 | -------------------------------------------------------------------------------- /examples/addressbook/Makefile.ocaml: -------------------------------------------------------------------------------- 1 | OCAMLMAKEFILE := ../../make/OCamlMakefile 2 | 3 | 4 | SOURCES = addressbook_piqi.ml 5 | 6 | 7 | ifeq ($(GOAL),add_person) 8 | RESULT = add_person 9 | SOURCES += add_person.ml 10 | else 11 | RESULT = list_people 12 | SOURCES += list_people.ml 13 | endif 14 | 15 | 16 | PRE_TARGETS = addressbook_piqi.ml 17 | 18 | export OCAMLPATH := ../..:$(OCAMLPATH) 19 | PACKS = piqirun.pb 20 | 21 | 22 | PIQIC = ../../piqic-ocaml/piqic-ocaml 23 | #PIQIC_FLAGS = 24 | 25 | 26 | all: native-code #byte-code 27 | 28 | 29 | addressbook_piqi.ml: addressbook.proto.piqi 30 | $(PIQIC) $(PIQIC_FLAGS) $< 31 | 32 | 33 | clean:: 34 | rm -f *.tmp.ml 35 | 36 | 37 | include $(OCAMLMAKEFILE) 38 | -------------------------------------------------------------------------------- /examples/addressbook/Makefile.ocaml_ext: -------------------------------------------------------------------------------- 1 | OCAMLMAKEFILE := ../../make/OCamlMakefile 2 | 3 | 4 | RESULT = io_json_xml_pb 5 | 6 | 7 | SOURCES = \ 8 | $(PIQI_ML_FILES) \ 9 | io_json_xml_pb.ml 10 | 11 | 12 | export OCAMLPATH := ../..:$(OCAMLPATH) 13 | PACKS = piqirun.ext 14 | 15 | 16 | PIQI_FILES = addressbook.proto.piqi 17 | PIQI_ML_FILES = addressbook_piqi.ml addressbook_piqi_ext.ml 18 | 19 | 20 | PRE_TARGETS = $(PIQI_ML_FILES) 21 | 22 | 23 | PIQIC = ../../piqic-ocaml/piqic-ocaml 24 | PIQIC_FLAGS = --multi-format 25 | 26 | 27 | all: native-code #byte-code debug-code 28 | 29 | 30 | $(PIQI_ML_FILES): $(PIQI_FILES) 31 | set -e; \ 32 | for i in $^; do \ 33 | $(PIQIC) $(PIQIC_FLAGS) $$i ; \ 34 | done 35 | 36 | 37 | clean:: 38 | rm -f *.tmp.ml 39 | 40 | 41 | include $(OCAMLMAKEFILE) 42 | -------------------------------------------------------------------------------- /examples/addressbook/README: -------------------------------------------------------------------------------- 1 | This directory contains an example of an addressbook data structure definition 2 | and two OCaml programs that rely on it to manipulate addressbook data: 3 | 4 | addressbook.proto.piqi -- definition of the addressbook data structure 5 | 6 | add_person.ml -- adds a person to the addressbook 7 | 8 | list_people.ml -- lists the contents of the addressbook 9 | 10 | 11 | The "./test" shell script contains tests for the OCaml programs. 12 | 13 | 14 | This example is based on the Google Protocol Buffers examples: 15 | 16 | http://protobuf.googlecode.com/svn/trunk/examples/ 17 | (See their README.txt file for details) 18 | 19 | The data definition file "addressbook.proto.piqi" is converted from the original 20 | "../addressbook.proto" using the "piqi of-proto" command. 21 | 22 | OCaml-specific extensions to the converted ".proto" specificaion are defined in 23 | "addressbook.ocaml.piqi". 24 | 25 | The OCaml programs implement exactly the same functionality as Python, Java and 26 | C++ programs from the Protobuf examples. The data structure and the binary 27 | encoding of the addressbook data structure is fully compatible as well. As a 28 | result, the OCaml programs can read an addressbook structure created by 29 | Python/C++/Java programs and vice versa. 30 | 31 | This example also contains the "addressbook.piq" file which is a sample 32 | addressbook data structure represented as a Piq file. It can be converted to the 33 | binary Protocol Buffers format (using "piqi convert") and then can be 34 | manipulated by the OCaml programs. 35 | 36 | -------------------------------------------------------------------------------- /examples/addressbook/add_person.ml: -------------------------------------------------------------------------------- 1 | 2 | module A = Addressbook_piqi 3 | 4 | 5 | let default_phone_number = A.default_person_phone_number () 6 | 7 | 8 | let read_phone_type () = 9 | print_endline "Is this a mobile, home, or work phone? "; 10 | match read_line () with 11 | | "mobile" -> `mobile 12 | | "home" -> `home 13 | | "work" -> `work 14 | | _ -> 15 | print_endline "Unknown phone type. Using default."; 16 | default_phone_number.A.Person_phone_number.phone_type 17 | 18 | 19 | let read_phone_numbers () = 20 | let rec aux accu = 21 | match read_line () with 22 | | "" -> List.rev accu 23 | | number -> 24 | let phone_type = read_phone_type () in 25 | let res = 26 | A.Person_phone_number.({ 27 | number = number; 28 | phone_type = phone_type; 29 | }) 30 | in aux (res :: accu) 31 | in aux [] 32 | 33 | 34 | (* This function fills in a Person message based on user input. *) 35 | let prompt_for_address address_book = 36 | print_endline "Enter person ID number: "; 37 | let id = Int32.of_string (read_line ()) in 38 | 39 | print_endline "Enter name: "; 40 | let name = read_line () in 41 | 42 | print_endline "Enter email address (blank for none): "; 43 | let email = 44 | match read_line () with 45 | | "" -> None 46 | | x -> Some x 47 | in 48 | 49 | print_endline "Enter a phone number (or leave blank to finish): "; 50 | let phone_numbers = read_phone_numbers () in 51 | let person = 52 | A.Person.({ 53 | id = id; 54 | name = name; 55 | email = email; 56 | phone = phone_numbers; 57 | }) 58 | in 59 | A.Address_book.({ 60 | (* address_book with *) 61 | person = address_book.person @ [person] 62 | }) 63 | 64 | 65 | (* 66 | * Main function: Reads the entire address book from a file, 67 | * adds one person based on user input, then writes it back out to the same 68 | * file. 69 | *) 70 | let _ = 71 | if Array.length Sys.argv <> 2 72 | then 73 | ( Printf.eprintf "Usage: %s ADDRESS_BOOK_FILE\n" Sys.argv.(0); 74 | exit (-1) 75 | ); 76 | 77 | let address_book = 78 | try 79 | (* Read the existing address book. *) 80 | let ch = open_in_bin Sys.argv.(1) in 81 | let buf = Piqirun.init_from_channel ch in 82 | let res = A.parse_address_book buf in 83 | close_in ch; 84 | res 85 | with 86 | | Sys_error _ -> 87 | Printf.printf "%s: File not found. Creating a new file.\n" Sys.argv.(0); 88 | A.Address_book.({person = []}) 89 | | Piqirun.Error _ -> 90 | Printf.eprintf "Failed to parse address book.\n"; 91 | exit (-1) 92 | in 93 | 94 | (* Add an address. *) 95 | let address_book = prompt_for_address address_book in 96 | 97 | (* Write the new address book back to disk. *) 98 | let och = open_out_bin Sys.argv.(1) in 99 | (* NOTE: specifying -1 as the field code has a special meaning: it tells 100 | * generator not to generate the header (code/tag/len) -- just generate the 101 | * contents *) 102 | let data = A.gen_address_book address_book in 103 | Piqirun.to_channel och data; 104 | close_out och 105 | 106 | -------------------------------------------------------------------------------- /examples/addressbook/addressbook.ocaml.piqi: -------------------------------------------------------------------------------- 1 | % This is an extension for addressbook.proto.piqi module. 2 | % 3 | % This file will be automatically included by "piqic-ocaml" when it loads 4 | % addressbook.proto.piqi 5 | % 6 | % After converting addressbook.proto to addressbook.proto.piqi, we need to add 7 | % "ocaml-name" property for field "type" in "person-phone-name" record. This is 8 | % necessary because "type" is a keyword in OCaml. 9 | 10 | .include [ .module addressbook ] 11 | 12 | 13 | .extend [ 14 | .field person-phone-number.type 15 | 16 | .with.ocaml-name "phone_type" 17 | ] 18 | 19 | 20 | .custom-field ocaml-name 21 | -------------------------------------------------------------------------------- /examples/addressbook/addressbook.piq: -------------------------------------------------------------------------------- 1 | 2 | :addressbook/address-book [ 3 | 4 | .person [ 5 | .name "J. Random Hacker" 6 | .id 0 7 | 8 | .email "j.r.hacker@example.com" 9 | 10 | .phone [ 11 | .number "(111) 123 45 67" % phone is "home" by default 12 | ] 13 | 14 | .phone [ 15 | .number "(222) 123 45 67" 16 | .mobile % NOTE: this is a "shorthand" for .type.mobile 17 | ] 18 | 19 | .phone [ 20 | .number "(333) 123 45 67" 21 | .work 22 | ] 23 | 24 | % this is also valid (but not recormmended) 25 | .phone [ "(444) 123 45 67" .mobile ] 26 | ] 27 | 28 | 29 | % it is possible to omit optional "email" field 30 | % and repeated "phone" field 31 | .person [ 32 | .name "Joe User" 33 | .id 1 34 | 35 | % it is possible to omit optional "email" field 36 | % and repeated "phone" field 37 | ] 38 | 39 | 40 | % ... or even omit labels for the required fields (not recommended style 41 | % though) 42 | .person [ "Joe User Jr" 2 ] 43 | 44 | 45 | % or even like that 46 | .person [ 47 | "Joe User II" 48 | 3 49 | "joe.user@example.com" 50 | .phone [ "(444) 123 45 67" ] 51 | .phone [ "(555) 123 45 67" .work ] 52 | ] 53 | ] 54 | 55 | -------------------------------------------------------------------------------- /examples/addressbook/addressbook.proto: -------------------------------------------------------------------------------- 1 | // This file was taken from Google Protocol Buffers source distribution. It is 2 | // also available here: 3 | // 4 | // http://protobuf.googlecode.com/svn/trunk/examples/addressbook.proto 5 | 6 | package tutorial; 7 | 8 | option java_package = "com.example.tutorial"; 9 | option java_outer_classname = "AddressBookProtos"; 10 | 11 | message Person { 12 | required string name = 1; 13 | required int32 id = 2; // Unique ID number for this person. 14 | optional string email = 3; 15 | 16 | enum PhoneType { 17 | MOBILE = 0; 18 | HOME = 1; 19 | WORK = 2; 20 | } 21 | 22 | message PhoneNumber { 23 | required string number = 1; 24 | optional PhoneType type = 2 [default = HOME]; 25 | } 26 | 27 | repeated PhoneNumber phone = 4; 28 | } 29 | 30 | // Our address book file is just one of these. 31 | message AddressBook { 32 | repeated Person person = 1; 33 | } 34 | -------------------------------------------------------------------------------- /examples/addressbook/addressbook.proto.piqi: -------------------------------------------------------------------------------- 1 | % This file was generated from "addressbook.proto" using the following command: 2 | % 3 | % piqi of-proto --normalize addressbook.proto 4 | 5 | 6 | .protobuf-package "tutorial" 7 | 8 | .record [ 9 | .name person 10 | .field [ 11 | .name name 12 | .type string 13 | .code 1 14 | ] 15 | .field [ 16 | .name id 17 | .type protobuf-int32 18 | .code 2 19 | ] 20 | .field [ 21 | .name email 22 | .type string 23 | .optional 24 | .code 3 25 | ] 26 | .field [ 27 | .name phone 28 | .type person-phone-number 29 | .repeated 30 | .code 4 31 | ] 32 | ] 33 | 34 | .record [ 35 | .name person-phone-number 36 | .field [ 37 | .name number 38 | .type string 39 | .code 1 40 | ] 41 | .field [ 42 | .name type 43 | .type person-phone-type 44 | .optional 45 | .default.home 46 | .code 2 47 | ] 48 | ] 49 | 50 | .enum [ 51 | .name person-phone-type 52 | .option [ 53 | .name mobile 54 | .code 0 55 | .protobuf-name "person_mobile" 56 | ] 57 | .option [ 58 | .name home 59 | .code 1 60 | .protobuf-name "person_home" 61 | ] 62 | .option [ 63 | .name work 64 | .code 2 65 | .protobuf-name "person_work" 66 | ] 67 | ] 68 | 69 | .record [ 70 | .name address-book 71 | .field [ 72 | .name person 73 | .type person 74 | .repeated 75 | .code 1 76 | ] 77 | ] 78 | -------------------------------------------------------------------------------- /examples/addressbook/io_json_xml_pb.ml: -------------------------------------------------------------------------------- 1 | 2 | 3 | module Ab = Addressbook_piqi_ext 4 | 5 | 6 | (* Main function: Reads the entire address book from a file and prints all 7 | * the information inside in various formats *) 8 | let _ = 9 | if Array.length Sys.argv <> 2 10 | then 11 | ( Printf.eprintf "Usage: %s ADDRESS_BOOK_FILE\n" Sys.argv.(0); 12 | exit (-1) 13 | ); 14 | 15 | (* Read the existing address book in binary Protocol Buffers format *) 16 | let ch = open_in_bin Sys.argv.(1) in 17 | let buf = Piqirun.init_from_channel ch in 18 | let address_book = Addressbook_piqi.parse_address_book buf in 19 | close_in ch; 20 | 21 | (* Serialize addressbook to JSON format *) 22 | let json = Ab.gen_address_book address_book `json in 23 | Printf.printf "\n\nJSON: \n\n%s\n" json; 24 | (* Read back from JSON *) 25 | let address_book' = Ab.parse_address_book json `json in 26 | assert (address_book' = address_book); 27 | 28 | (* Serialize addressbook to pretty-printed JSON format *) 29 | let json = Ab.gen_address_book address_book `json_pretty in 30 | Printf.printf "\n\npretty-printed JSON: \n\n%s\n" json; 31 | (* Read back from JSON *) 32 | let address_book' = Ab.parse_address_book json `json in 33 | assert (address_book' = address_book); 34 | 35 | (* Serialize addressbook to pretty-printed JSON format and include "null" JSON 36 | * fields for missing optional and [] for missing required fields *) 37 | let opts = Piqirun_ext.make_options ~json_omit_missing_fields:false () in 38 | let json = Ab.gen_address_book address_book `json ~opts in 39 | Printf.printf "\n\nJSON with null fields: \n\n%s\n" json; 40 | (* Read back from JSON *) 41 | let address_book' = Ab.parse_address_book json `json in 42 | assert (address_book' = address_book); 43 | 44 | (* Serialize addressbook to XML format *) 45 | let xml = Ab.gen_address_book address_book `xml in 46 | Printf.printf "\n\nXML: \n\n%s\n" xml; 47 | (* Read back from XML *) 48 | let address_book' = Ab.parse_address_book xml `xml in 49 | assert (address_book' = address_book); 50 | 51 | (* Serialize addressbook to pretty-printed XML format *) 52 | let xml = Ab.gen_address_book address_book `xml_pretty in 53 | Printf.printf "\n\npretty-printed XML: \n\n%s\n" xml; 54 | (* Read back from XML *) 55 | let address_book' = Ab.parse_address_book xml `xml in 56 | assert (address_book' = address_book); 57 | 58 | 59 | (* Serialize addressbook to Piq format *) 60 | let piq = Ab.gen_address_book address_book `piq in 61 | Printf.printf "\n\nPiq: \n\n%s\n" piq; 62 | (* Read back from XML *) 63 | let address_book' = Ab.parse_address_book piq `piq in 64 | assert (address_book' = address_book); 65 | 66 | 67 | (* Print addressbook to stdout in Piq format *) 68 | Printf.printf "\n\nPrinting to stdout:\n\n"; 69 | Ab.print_address_book address_book; 70 | 71 | 72 | (* Print addressbook to stderr in Piq format *) 73 | Printf.eprintf "\n\nPrinting to stderr:\n\n"; 74 | Ab.prerr_address_book address_book; 75 | 76 | () 77 | 78 | -------------------------------------------------------------------------------- /examples/addressbook/list_people.ml: -------------------------------------------------------------------------------- 1 | 2 | open Addressbook_piqi 3 | 4 | 5 | let printf = Printf.printf 6 | 7 | 8 | (* Iterates though all people in the AddressBook and prints info about them. *) 9 | let list_people address_book = 10 | let print_email = function 11 | | Some email -> 12 | printf " E-mail address: %s\n" email 13 | | None -> () 14 | in 15 | let print_phone_number x = 16 | (match x.Person_phone_number.phone_type with 17 | | `mobile -> 18 | printf " Mobile phone #: " 19 | | `home -> 20 | printf " Home phone #: " 21 | | `work -> 22 | printf " Work phone #: " 23 | ); 24 | printf "%s\n" x.Person_phone_number.number 25 | in 26 | let list_person x = 27 | printf "Person ID: %ld\n" x.Person.id; 28 | printf " Name: %s\n" x.Person.name; 29 | print_email x.Person.email; 30 | List.iter print_phone_number x.Person.phone 31 | in 32 | List.iter list_person address_book.Address_book.person 33 | 34 | 35 | (* Main function: Reads the entire address book from a file and prints all 36 | * the information inside. *) 37 | let _ = 38 | if Array.length Sys.argv <> 2 39 | then 40 | ( Printf.eprintf "Usage: %s ADDRESS_BOOK_FILE\n" Sys.argv.(0); 41 | exit (-1) 42 | ); 43 | (* Read the existing address book. *) 44 | let ch = open_in_bin Sys.argv.(1) in 45 | let buf = Piqirun.init_from_channel ch in 46 | let address_book = parse_address_book buf in 47 | close_in ch; 48 | 49 | list_people address_book 50 | 51 | -------------------------------------------------------------------------------- /examples/addressbook/test: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | : ${PIQI:=piqi} 4 | 5 | 6 | set -ex 7 | 8 | # create a binary representation of the addressbook by converting it from piq 9 | $PIQI convert -t pb addressbook.piq 10 | 11 | # create another copy of addressbook records using ocaml program 12 | ./add_person addressbook.pb < l1 54 | ./list_people addressbook.pb > l2 55 | cmp l1 l2 56 | 57 | 58 | # check that the addressbook binary file created by "add_person" is readable by 59 | # "piqi convert" 60 | $PIQI convert --type addressbook/address-book addressbook.pb > addressbook.pb.piq 61 | 62 | rm addressbook.pb 63 | 64 | -------------------------------------------------------------------------------- /examples/custom-types/Makefile: -------------------------------------------------------------------------------- 1 | OCAMLMAKEFILE := ../../make/OCamlMakefile 2 | 3 | 4 | RESULT = a.out 5 | 6 | 7 | SOURCES = \ 8 | $(PIQI_ML_FILES1) \ 9 | piqirun_custom.ml \ 10 | $(PIQI_ML_FILES2) \ 11 | 12 | 13 | export OCAMLPATH := ../..:$(OCAMLPATH) 14 | PACKS = piqirun.pb num 15 | 16 | 17 | PIQI_FILES = example.piqi skvl.piqi 18 | PIQI_ML_FILES1 = skvl_piqi.ml 19 | PIQI_ML_FILES2 = example_piqi.ml 20 | PIQI_ML_FILES = $(PIQI_ML_FILES1) $(PIQI_ML_FILES2) 21 | 22 | 23 | PRE_TARGETS = $(PIQI_ML_FILES) 24 | 25 | 26 | PIQIC = ../../piqic-ocaml/piqic-ocaml 27 | #PIQIC_FLAGS = 28 | 29 | 30 | all: native-code #byte-code debug-code 31 | 32 | 33 | $(PIQI_ML_FILES): $(PIQI_FILES) 34 | set -e; \ 35 | for i in $^; do \ 36 | $(PIQIC) $(PIQIC_FLAGS) $$i ; \ 37 | done 38 | 39 | 40 | clean:: 41 | rm -f *.tmp.ml 42 | 43 | 44 | include $(OCAMLMAKEFILE) 45 | -------------------------------------------------------------------------------- /examples/custom-types/README: -------------------------------------------------------------------------------- 1 | This is an example of how to serialize arbitrary monomorphic OCaml types by 2 | mapping them to built-in or previously defined Piqi types. 3 | 4 | This mechanism can be used, for example, for serializing OCaml chars as 5 | integers, bigints as decimal strings, or even "string Map.Make(String).t" as 6 | list of {key, value} records. 7 | -------------------------------------------------------------------------------- /examples/custom-types/example.piqi: -------------------------------------------------------------------------------- 1 | % 2 | % defining custom OCaml types to be serialized as some Piqi types 3 | % 4 | 5 | % mapping OCaml char to Piqi int 6 | .alias [ 7 | .name char 8 | .type int 9 | .ocaml-type "Piqirun_custom.char" 10 | ] 11 | 12 | 13 | % mapping OCaml nativeint to Piqi int 14 | .alias [ 15 | .name ocaml-nativeint 16 | .type int 17 | .ocaml-type "Piqirun_custom.nativeint" 18 | ] 19 | 20 | 21 | % mapping OCaml big_int to Piqi string (by representing number as a decimal 22 | % string) 23 | .alias [ 24 | .name ocaml-bigint 25 | .type string 26 | .ocaml-type "Piqirun_custom.bigint" 27 | ] 28 | 29 | 30 | % 31 | % mapping OCaml string key-value map (string Map.Make(String) 32 | % to the list of {key, value} records 33 | % 34 | 35 | .import [ .module skvl ] 36 | 37 | .alias [ 38 | .name ocaml-string-key-value-list 39 | .type skvl/string-key-value-list 40 | .ocaml-type "Piqirun_custom.skvl" 41 | ] 42 | 43 | 44 | % a record that uses the above definitions 45 | .record [ 46 | .name r 47 | 48 | .field [ 49 | .name c 50 | .type char 51 | ] 52 | 53 | .field [ 54 | .name ni 55 | .type ocaml-nativeint 56 | ] 57 | 58 | .field [ 59 | .name bi 60 | .type ocaml-bigint 61 | ] 62 | 63 | .field [ 64 | .name kvl 65 | .type ocaml-string-key-value-list 66 | ] 67 | ] 68 | 69 | 70 | .custom-field ocaml-type 71 | -------------------------------------------------------------------------------- /examples/custom-types/piqirun_custom.ml: -------------------------------------------------------------------------------- 1 | (* 2 | * runtime support for OCaml custom types defined in example.piqi 3 | *) 4 | 5 | (* mapping OCaml char to Piqi int *) 6 | type char = Char.t 7 | 8 | let char_of_int: int -> char = Char.chr 9 | let char_to_int: char -> int = Char.code 10 | 11 | 12 | (* mapping OCaml nativeint to Piqi int *) 13 | type nativeint = Nativeint.t 14 | 15 | let nativeint_of_int: int -> nativeint = Nativeint.of_int 16 | let nativeint_to_int: nativeint -> int = Nativeint.to_int 17 | 18 | 19 | (* mapping OCaml big_int to Piqi string (by representing number as a decimal 20 | * string) 21 | *) 22 | type bigint = Big_int.big_int 23 | 24 | let bigint_of_string: string -> bigint = Big_int.big_int_of_string 25 | let bigint_to_string: bigint -> string = Big_int.string_of_big_int 26 | 27 | 28 | (* 29 | * mapping string key-value map (string Map.Make(String) 30 | * to the list of {key, value} records 31 | *) 32 | 33 | 34 | module M = Map.Make(String) 35 | type skvl = string M.t 36 | 37 | 38 | let list_of_map map = 39 | let l = M.fold (fun k v accu -> (k,v)::accu) map [] in 40 | List.rev l 41 | 42 | let map_of_list pairs = 43 | List.fold_left (fun accu (k,v) -> M.add k v accu) M.empty pairs 44 | 45 | 46 | open Skvl_piqi.String_key_value 47 | 48 | let skvl_of_string_key_value_list (x: Skvl_piqi.string_key_value_list) :skvl = 49 | (* 50 | M.empty 51 | *) 52 | let pairs = List.map (fun x -> x.key, x.value) x in 53 | map_of_list pairs 54 | 55 | 56 | let skvl_to_string_key_value_list (x :skvl) :Skvl_piqi.string_key_value_list = 57 | (* 58 | Skvl_piqi.default_string_key_value_list () 59 | *) 60 | let pairs = list_of_map x in 61 | List.map (fun (k, v) -> {key = k; value = v}) pairs 62 | 63 | -------------------------------------------------------------------------------- /examples/custom-types/skvl.piqi: -------------------------------------------------------------------------------- 1 | 2 | .record [ 3 | .name string-key-value 4 | 5 | .field [ 6 | .name key 7 | .type string 8 | ] 9 | .field [ 10 | .name value 11 | .type string 12 | ] 13 | ] 14 | 15 | 16 | .list [ 17 | .name string-key-value-list 18 | .type string-key-value 19 | ] 20 | 21 | -------------------------------------------------------------------------------- /examples/piq-config/Makefile: -------------------------------------------------------------------------------- 1 | OCAMLMAKEFILE := ../../make/OCamlMakefile 2 | 3 | 4 | RESULT = config 5 | 6 | 7 | SOURCES = \ 8 | $(PIQI_ML_FILES) \ 9 | config.ml 10 | 11 | 12 | export OCAMLPATH := ../..:$(OCAMLPATH) 13 | PACKS = piqirun.ext 14 | 15 | 16 | PIQI_FILES = config.piqi 17 | PIQI_ML_FILES = config_piqi.ml config_piqi_ext.ml 18 | 19 | 20 | PRE_TARGETS = $(PIQI_ML_FILES) 21 | 22 | 23 | PIQIC = ../../piqic-ocaml/piqic-ocaml 24 | PIQIC_FLAGS = --multi-format 25 | 26 | 27 | all: native-code #byte-code debug-code 28 | 29 | 30 | $(PIQI_ML_FILES): $(PIQI_FILES) 31 | set -e; \ 32 | for i in $^; do \ 33 | $(PIQIC) $(PIQIC_FLAGS) $$i ; \ 34 | done 35 | 36 | 37 | clean:: 38 | rm -f *.tmp.ml 39 | 40 | 41 | include $(OCAMLMAKEFILE) 42 | -------------------------------------------------------------------------------- /examples/piq-config/README: -------------------------------------------------------------------------------- 1 | This is an example of how to use configuration file in Piq format from OCaml 2 | program. 3 | 4 | To build the example, run "make". 5 | 6 | The resulting "./config" executable will read the config data structure from 7 | "config.piq" and print it on stdout. 8 | 9 | 10 | Files: 11 | config.piqi - the definition of the config file format 12 | config.piq - an example config file 13 | config.ml - an OCaml program that reads the "config.piq" file 14 | 15 | -------------------------------------------------------------------------------- /examples/piq-config/config.ml: -------------------------------------------------------------------------------- 1 | 2 | let read_file filename = 3 | let ch = open_in_bin filename in 4 | let len = in_channel_length ch in 5 | let buf = Buffer.create len in 6 | Buffer.add_channel buf ch len; 7 | close_in ch; 8 | Buffer.contents buf 9 | 10 | 11 | (* common serialization options *) 12 | let opts = Piqirun_ext.make_options () 13 | ~piq_frameless_input:true 14 | ~piq_frameless_output:true 15 | 16 | 17 | let read_config filename = 18 | let contents = read_file filename in 19 | try 20 | Config_piqi_ext.parse_config contents `piq ~opts 21 | with 22 | Piqi_common.Error ((file, col, line), error) -> 23 | failwith (Printf.sprintf "error at %s:%d:%d: %s" file col line error) 24 | 25 | 26 | (* test *) 27 | let _ = 28 | let config = read_config "config.piq" in 29 | Config_piqi_ext.print_config config ~opts 30 | 31 | -------------------------------------------------------------------------------- /examples/piq-config/config.piq: -------------------------------------------------------------------------------- 1 | 2 | % the shortest form 3 | [ "foo" .make ] 4 | 5 | % explicit field name 6 | [ .path "foo" .make ] 7 | 8 | % more explicit field names 9 | [ .path "foo" .generator.make ] 10 | 11 | % alternative notation for representing enums (variants) 12 | [ .path "foo" .generator (.make) ] 13 | 14 | % the longest form 15 | [ 16 | .path "foo" 17 | .generator.make 18 | ] 19 | 20 | % testing other enum values 21 | [ "bar" .latexmk ] 22 | [ "fum" .org ] 23 | [ "baz" .tensile ] 24 | 25 | -------------------------------------------------------------------------------- /examples/piq-config/config.piqi: -------------------------------------------------------------------------------- 1 | .module config 2 | 3 | .enum [ 4 | .name generator 5 | .option [ .name make ] 6 | .option [ .name latexmk ] 7 | .option [ .name org ] 8 | .option [ .name tensile ] 9 | ] 10 | 11 | .record [ 12 | .name entry 13 | .field [ 14 | .name path 15 | .type string 16 | ] 17 | .field [ 18 | .type generator 19 | ] 20 | ] 21 | 22 | .list [ 23 | .name config 24 | .type entry 25 | ] 26 | 27 | -------------------------------------------------------------------------------- /make/Makefile.dirs: -------------------------------------------------------------------------------- 1 | .PHONY: all dirs clean pre_target post_target 2 | 3 | all: pre_target dirs post_target 4 | 5 | 6 | dirs: $(DIRS) 7 | set -e; \ 8 | for dir in $(DIRS); do \ 9 | $(MAKE) -C $$dir $(MAKECMDGOALS); \ 10 | done 11 | 12 | 13 | clean:: dirs 14 | 15 | 16 | pre_target:: 17 | post_target:: 18 | 19 | 20 | # vim:ft=make 21 | -------------------------------------------------------------------------------- /opam: -------------------------------------------------------------------------------- 1 | opam-version: "2.0" 2 | synopsis: "Protocol Buffers, JSON and XML serialization system for OCaml" 3 | authors: "Anton Lavrik " 4 | maintainer: "Anton Lavrik " 5 | homepage: "https://github.com/alavrik/piqi-ocaml" 6 | bug-reports: "https://github.com/alavrik/piqi-ocaml/issues" 7 | depends: [ 8 | "ocaml" {>= "4.02.0"} 9 | "dune" {>= "2.0.0"} 10 | "piqilib" 11 | "stdlib-shims" 12 | "num" {with-test} 13 | ] 14 | dev-repo: "git://github.com/alavrik/piqi-ocaml" 15 | 16 | build: [ 17 | ["dune" "subst"] {dev} 18 | [ 19 | "dune" 20 | "build" 21 | "-p" 22 | "piqi,piqirun" 23 | "-j" 24 | jobs 25 | "@install" 26 | "@runtest" {with-test} 27 | "@doc" {with-doc} 28 | ] 29 | ] 30 | 31 | install: [ 32 | ["dune" "install" "-p" "piqi,piqirun"] 33 | ] 34 | -------------------------------------------------------------------------------- /piqic-ocaml/.gitignore: -------------------------------------------------------------------------------- 1 | /piqic-ocaml 2 | /piqic_ocaml_version.ml 3 | 4 | /piqi.piqi 5 | -------------------------------------------------------------------------------- /piqic-ocaml/Makefile: -------------------------------------------------------------------------------- 1 | OCAMLMAKEFILE := ../make/OCamlMakefile 2 | 3 | 4 | RESULT = piqic-ocaml 5 | 6 | 7 | SOURCES = \ 8 | piqic_piqi.ml \ 9 | piqic_common.ml \ 10 | \ 11 | piqic_ocaml_types.ml \ 12 | piqic_ocaml_out.ml \ 13 | piqic_ocaml_in.ml \ 14 | piqic_ocaml_defaults.ml \ 15 | piqic_ocaml_ext.ml \ 16 | \ 17 | piqic_ocaml_version.ml \ 18 | piqic_ocaml.ml \ 19 | 20 | 21 | PACKS = piqilib bytes stdlib-shims 22 | INCDIRS = ../piqirun 23 | LIBS = ../piqirun/piqirun 24 | 25 | 26 | PRE_TARGETS = piqic_ocaml_version.ml 27 | 28 | 29 | all: nc 30 | 31 | debug: dc top 32 | 33 | 34 | clean:: 35 | rm -f piqi.piqi 36 | 37 | 38 | # recompile piqi self-spec into piqic_piqi.ml 39 | PIQI ?= piqi 40 | PIQIC = ./$(RESULT) 41 | PIQIC_FLAGS = -e piqi-ocaml --embed-piqi #--trace 42 | 43 | piqi: 44 | $(PIQI) cc -o piqi.piqi 45 | $(PIQIC) $(PIQIC_FLAGS) piqi.piqi 46 | 47 | 48 | piqic_ocaml_version.ml: ../VERSION 49 | echo "let version = \"`head -1 $<`\"" >$@ 50 | 51 | 52 | include $(OCAMLMAKEFILE) 53 | -------------------------------------------------------------------------------- /piqic-ocaml/dune: -------------------------------------------------------------------------------- 1 | (executable 2 | (name piqic_ocaml) 3 | (public_name piqic-ocaml) 4 | (package piqi) 5 | (libraries piqilib piqirun bytes stdlib-shims)) 6 | 7 | (rule 8 | (target piqic_ocaml_version.ml) 9 | (deps 10 | (file make_version.sh) 11 | (file ../VERSION)) 12 | (action (with-stdout-to %{target} 13 | (bash ./make_version.sh)))) 14 | -------------------------------------------------------------------------------- /piqic-ocaml/make_version.sh: -------------------------------------------------------------------------------- 1 | echo "let version = \"$(head -1 ../VERSION)\"" 2 | -------------------------------------------------------------------------------- /piqic-ocaml/piqi.ocaml.piqi: -------------------------------------------------------------------------------- 1 | % Copyright 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2017 Anton Lavrik 2 | % 3 | % Licensed under the Apache License, Version 2.0 (the "License"); 4 | % you may not use this file except in compliance with the License. 5 | % You may obtain a copy of the License at 6 | % 7 | % http://www.apache.org/licenses/LICENSE-2.0 8 | % 9 | % Unless required by applicable law or agreed to in writing, software 10 | % distributed under the License is distributed on an "AS IS" BASIS, 11 | % WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | % See the License for the specific language governing permissions and 13 | % limitations under the License. 14 | 15 | 16 | .include [ .module piqi ] 17 | 18 | 19 | % 20 | % types 21 | % 22 | 23 | .extend [ 24 | .typedef* [ int uint ] 25 | .with.ocaml-type "int" 26 | ] 27 | 28 | .extend [ 29 | .typedef* [ int32 uint32 ] 30 | .with.ocaml-type "int32" 31 | ] 32 | 33 | .extend [ 34 | .typedef* [ int64 uint64 ] 35 | .with.ocaml-type "int64" 36 | ] 37 | 38 | .extend [ 39 | .typedef* [ float64 float32 ] 40 | .with.ocaml-type "float" 41 | ] 42 | 43 | 44 | % 45 | % names 46 | % 47 | 48 | .extend [ 49 | .option typedef.list 50 | 51 | .with.ocaml-name "list" 52 | ] 53 | 54 | .extend [ 55 | .typedef type 56 | .field any.type 57 | 58 | .with.ocaml-name "typename" 59 | ] 60 | 61 | .extend [ 62 | .typedef list 63 | 64 | .with.ocaml-name "piqi_list" 65 | ] 66 | 67 | .extend [ 68 | .field* [ piqi.module import.module ] 69 | 70 | .with.ocaml-name "modname" 71 | ] 72 | 73 | .extend [ 74 | .typedef function 75 | 76 | .with.ocaml-name "func" 77 | ] 78 | 79 | .extend [ 80 | .typedef piqi-list 81 | 82 | % ocaml-name "piqi-list" is already defined for record "list" 83 | .with.ocaml-name "piqi_bundle" 84 | ] 85 | 86 | 87 | .custom-field* [ ocaml-type ocaml-name ] 88 | 89 | -------------------------------------------------------------------------------- /piqic-ocaml/piqi.piqi-ocaml.piqi: -------------------------------------------------------------------------------- 1 | % Copyright 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2017 Anton Lavrik 2 | % 3 | % Licensed under the Apache License, Version 2.0 (the "License"); 4 | % you may not use this file except in compliance with the License. 5 | % You may obtain a copy of the License at 6 | % 7 | % http://www.apache.org/licenses/LICENSE-2.0 8 | % 9 | % Unless required by applicable law or agreed to in writing, software 10 | % distributed under the License is distributed on an "AS IS" BASIS, 11 | % WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | % See the License for the specific language governing permissions and 13 | % limitations under the License. 14 | 15 | % 16 | % OCaml-specific Piqi extensions 17 | % 18 | 19 | .include [ .module piqi ] 20 | 21 | 22 | % generate piqic_piqi.ml instead of piqi_piqi.ml so that we don't conflict with 23 | % piqi_piqi.ml that comes with the piqilib library 24 | .ocaml-module "Piqic_piqi" 25 | .custom-field ocaml-module 26 | 27 | 28 | .extend [ 29 | .typedef* [ record field variant option enum alias list import function ] 30 | 31 | .with.field [ 32 | .name ocaml-name 33 | .type string 34 | .optional 35 | ] 36 | ] 37 | 38 | 39 | .extend [ 40 | .typedef* [ piqi import ] 41 | 42 | .with.field [ 43 | .name ocaml-module 44 | .type string 45 | .optional 46 | ] 47 | ] 48 | 49 | 50 | .extend [ 51 | .typedef alias 52 | 53 | .with.field [ 54 | .name ocaml-type 55 | .type string 56 | .optional 57 | ] 58 | ] 59 | 60 | 61 | % flag for representing repeated fields and lists as OCaml arrays 62 | .extend [ 63 | .typedef* [ field list ] 64 | 65 | .with.field [ 66 | .name ocaml-array 67 | .optional 68 | ] 69 | ] 70 | 71 | 72 | % flag for representing optional fields with defaults as 'a option instead of 'a 73 | .extend [ 74 | .typedef field 75 | 76 | .with.field [ 77 | .name ocaml-optional 78 | .optional 79 | ] 80 | ] 81 | 82 | -------------------------------------------------------------------------------- /piqic-ocaml/piqic_ocaml.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright 2009, 2010, 2011, 2012, 2013, 2014, 2015 Anton Lavrik 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | *) 16 | 17 | 18 | (* 19 | * Piqi compiler for OCaml 20 | *) 21 | 22 | module C = Piqic_common 23 | open C 24 | open Iolist 25 | 26 | 27 | (* command-line flags *) 28 | let flag_pp = ref false 29 | let flag_gen_defaults = ref false (* deprecated -- always enabled by default *) 30 | let flag_embed_piqi = ref false 31 | let flag_multi_format = ref false 32 | let flag_runtime = ref "" 33 | let flag_version = ref false 34 | let flag_piqi_version = ref false 35 | 36 | 37 | let arg__pp = 38 | "--pp", Arg.Set flag_pp, 39 | "(DEPRECATED) pretty-print output using CamlP4 (camlp4o)" 40 | 41 | let arg__normalize_names = 42 | "--normalize-names", Arg.Bool (fun x -> C.flag_normalize_names := x), 43 | "true|false turn CamlCase-style names into \"camel_case\" (default = true)" 44 | 45 | let arg__reserved_name = 46 | "--reserved-name", Arg.String (fun x -> Hashtbl.add Piqic_common.ocaml_reserved x ()), 47 | "add a reserved name in addition to the standard OCaml keywords" 48 | 49 | let arg__gen_defaults = 50 | "--gen-defaults", Arg.Set flag_gen_defaults, 51 | "(DEPRECATED) always enabled: generate default value constructors for generated types" 52 | 53 | let arg__gen_preserve_unknown_fields = 54 | "--gen-preserve-unknown-fields", Arg.Set C.flag_gen_preserve_unknown_fields, 55 | "generate code that preserve unknown Protobuf fields when they are serialized back" 56 | 57 | let arg__embed_piqi = 58 | "--embed-piqi", Arg.Set flag_embed_piqi, 59 | "embed Piqi modules encoded in binary format in the generated code" 60 | 61 | let arg__multi_format = 62 | "--multi-format", Arg.Set flag_multi_format, 63 | "generate extended OCaml stubs for multi-format (JSON/XML/Piq/Pb) serialization" 64 | 65 | let arg__ext = 66 | "--ext", Arg.Set flag_multi_format, 67 | "same as --multi-format" 68 | 69 | let arg__runtime = 70 | "--runtime", Arg.Set_string flag_runtime, 71 | " name of the Protobuf serialization runtime module (default = Piqirun)" 72 | 73 | let arg__cc = 74 | "--cc", Arg.Set C.flag_cc, 75 | "compiler compiler mode -- used only for building piqilib" 76 | 77 | let arg__version = 78 | "--version", Arg.Set flag_version, 79 | "print piqi-ocaml version and exit" 80 | 81 | let arg__piqi_version = 82 | "--piqi-version", Arg.Set flag_piqi_version, 83 | "print piqi version and exit" 84 | 85 | 86 | let ocaml_pretty_print ifile ofile = 87 | (* NOTE: we need "-printer o", because Camlp4o uses the Camlp4AutoPrinter by 88 | * default (no -printer argument provided), which will either produce ocaml 89 | * code or a binary AST depending on whether the output is a terminal or not 90 | * (regardless of any -o option provided). started on a terminal, the results 91 | * of piqic-ocaml --pp was a binary AST file *) 92 | let cmd = Printf.sprintf "camlp4o -printer o -o %s %s" ofile ifile in 93 | let res = Sys.command cmd in 94 | if res <> 0 95 | then C.error ("command execution failed: " ^ cmd) 96 | 97 | 98 | let gen_output_file ofile code = 99 | if not !flag_pp 100 | then 101 | let ch = Piqi_command.open_output ofile in 102 | Iolist.to_channel ch code; 103 | Piqi_command.close_output () 104 | else 105 | begin 106 | (* prettyprint generated OCaml code using Camlp4 *) 107 | let tmp_file = ofile ^ ".tmp.ml" in 108 | (try 109 | let tmp_ch = open_out tmp_file in 110 | Iolist.to_channel tmp_ch code; 111 | close_out tmp_ch; 112 | with Sys_error s -> 113 | C.error ("error writing temporary file: " ^ s)); 114 | ocaml_pretty_print tmp_file ofile; 115 | Piqi_command.add_tmp_file tmp_file; 116 | end 117 | 118 | 119 | (* build a list of all import dependencies including the specified module and 120 | * encode each Piqi module in the list using Protobuf encoding *) 121 | let gen_embedded_piqi piqi_list = 122 | let piqi = List.hd (List.rev piqi_list) in 123 | let s = Piqirun.to_string (T.gen_piqi piqi) in 124 | iol [ 125 | ios "let piqi = "; ioq (String.escaped s); eol 126 | ] 127 | 128 | 129 | let gen_custom_runtime () = 130 | if !flag_runtime <> "" 131 | then iol [ios "module Piqirun = "; ios !flag_runtime; eol; eol] 132 | else iol [] 133 | 134 | 135 | let gen_piqi_ml context = 136 | let modname = C.top_modname context in 137 | let code = iol [ 138 | gen_custom_runtime (); 139 | 140 | Piqic_ocaml_types.gen_piqi context; 141 | Piqic_ocaml_in.gen_piqi context; 142 | Piqic_ocaml_out.gen_piqi context; 143 | Piqic_ocaml_defaults.gen_piqi context; 144 | 145 | (* NOTE: --multi-format serialization depends on --embded-piqi *) 146 | if !flag_embed_piqi || !flag_multi_format 147 | then gen_embedded_piqi context.modules 148 | else iol []; 149 | 150 | ios "include "; ios modname; eol; 151 | ] 152 | in 153 | let ofile = String.uncapitalize modname ^ ".ml" in 154 | gen_output_file ofile code 155 | 156 | 157 | let gen_piqi_ext_ml context = 158 | let code = iol [ 159 | gen_custom_runtime (); 160 | Piqic_ocaml_ext.gen_piqi context; 161 | ] 162 | in 163 | let modname = C.top_modname context in 164 | let ofile = String.uncapitalize modname ^ "_ext.ml" in 165 | 166 | gen_output_file ofile code 167 | 168 | 169 | let piqic context = 170 | (* chdir to the output directory *) 171 | Piqi_command.chdir_output !Piqi_command.odir; 172 | 173 | gen_piqi_ml context; 174 | 175 | if !flag_multi_format 176 | then gen_piqi_ext_ml context 177 | 178 | 179 | (* this is the same as calling "piqi compile ..." and reading its output but, 180 | * instead, we are just using the library functions *) 181 | let piqi_compile_piqi ifile = 182 | let self_spec_bin = T.piqi in 183 | (* by adding "ocaml" extension, we tell the library to automatically load 184 | * *.ocaml.piqi extension modules *) 185 | Piqi_compile.compile self_spec_bin ifile ~extensions:["ocaml"] 186 | 187 | 188 | let load_piqi_list ifile = 189 | let bin = piqi_compile_piqi ifile in 190 | (* read the compiled piqi bundle *) 191 | let buf = Piqirun.init_from_string bin in 192 | let bundle = T.parse_piqi_bundle buf in 193 | (* return the list of piqi modules: list of dependencies @ [input module] *) 194 | bundle.T.Piqi_bundle.piqi 195 | 196 | 197 | let piqic_file ifile = 198 | (* load input .piqi file and its dependencies *) 199 | let piqi_list = load_piqi_list ifile in 200 | let context = C.init piqi_list in 201 | piqic context 202 | 203 | 204 | let speclist = Piqi_compile.getopt_speclist @ 205 | [ 206 | Piqi_command.arg_C; 207 | arg__normalize_names; 208 | arg__reserved_name; 209 | arg__pp; 210 | arg__gen_defaults; 211 | arg__gen_preserve_unknown_fields; 212 | (* TODO: deprecated and remove in the next major release (together with --pp) *) 213 | Piqi_command.arg__keep_tmp_files; 214 | arg__embed_piqi; 215 | arg__multi_format; 216 | arg__ext; 217 | arg__runtime; 218 | arg__cc; 219 | arg__version; 220 | arg__piqi_version; 221 | ] 222 | 223 | 224 | let usage = "\ 225 | Usage: piqic-ocaml [options] <.piqi file> 226 | piqic-ocaml [--version | --piqi-version] 227 | 228 | Options:" 229 | 230 | 231 | let print_and_exit s = 232 | print_endline s; 233 | exit 0 234 | 235 | 236 | let run () = 237 | (* handle --version and --piqi-version; doing it separately from the rest of 238 | * args, because normally we expect one positional arg and this is 239 | * automatically handled by Piqi_command.parse_args () *) 240 | let argv1 = 241 | if Array.length Sys.argv > 1 242 | then Sys.argv.(1) 243 | else "" 244 | in 245 | if argv1 = "--version" 246 | then print_and_exit Piqic_ocaml_version.version 247 | else if argv1 = "--piqi-version" 248 | then print_and_exit Piqi_version.version; 249 | 250 | (* normal invocation *) 251 | Piqi_command.parse_args () ~usage ~speclist; 252 | 253 | if !flag_gen_defaults 254 | then C.warning "--gen-defaults flag is deprecated: always generating defaults"; 255 | 256 | if !flag_pp 257 | then C.warning "--pp flag is deprecated (generated code is now pretty-printed by default)"; 258 | 259 | (match Piqi_command.arg__keep_tmp_files with 260 | | _, Arg.Set keep_tmp_files, _ -> 261 | if !keep_tmp_files 262 | then C.warning "--keep-tmp-files flag is deprecated (useless without deprecated --pp)"; 263 | | _ -> () 264 | ); 265 | 266 | piqic_file !Piqi_command.ifile 267 | 268 | 269 | let _ = 270 | Piqi_command.run_command run 271 | 272 | -------------------------------------------------------------------------------- /piqic-ocaml/piqic_ocaml_defaults.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2017 Anton Lavrik 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | *) 16 | 17 | 18 | (* 19 | * This module generates default values for OCaml types generated by 20 | * Piqic_ocaml_types 21 | * 22 | * The generated default_* functions return minimal serializable values of Piqi 23 | * types 24 | * 25 | * CAVEAT: the logic is very primitive and it doesn't guarantee that default 26 | * variant values are finite 27 | *) 28 | 29 | module C = Piqic_common 30 | open C 31 | open Iolist 32 | 33 | 34 | let gen_type context typename = Piqic_ocaml_in.gen_default_type context typename 35 | 36 | 37 | let gen_int piqi_type wire_type = 38 | let wire_type = 39 | match wire_type with 40 | | Some x -> x 41 | | None -> 42 | C.get_default_wire_type piqi_type 43 | in 44 | let f = 45 | match wire_type with 46 | | `varint -> Piqirun.int64_to_varint 47 | | `zigzag_varint -> Piqirun.int64_to_zigzag_varint 48 | | `fixed32 -> Piqirun.int64_to_fixed32 49 | | `fixed64 -> Piqirun.int64_to_fixed64 50 | | `signed_varint -> Piqirun.int64_to_signed_varint 51 | | `signed_fixed32 -> Piqirun.int64_to_signed_fixed32 52 | | `signed_fixed64 -> Piqirun.int64_to_signed_fixed64 53 | | `block -> 54 | assert false 55 | in 56 | let buf = f (-1) 0L in 57 | Piqirun.to_string buf 58 | 59 | 60 | let gen_builtin_type context piqi_type ocaml_type wire_type = 61 | match piqi_type with 62 | | `any -> 63 | if context.is_self_spec 64 | then ios "default_any ()" 65 | else ios "Piqi_piqi.default_any ()" 66 | | `string | `binary -> 67 | ios "\"\"" 68 | | `bool -> 69 | ios "false" 70 | | `float -> 71 | ios "0.0" 72 | | `int -> 73 | match ocaml_type with 74 | | None | Some "int" -> ios "0" 75 | | Some "int32" -> ios "0l" 76 | | Some "int64" -> ios "0L" 77 | | _ -> 78 | (* XXX: this is the most generic way to handle it; accounting for 79 | * potential future extensions *) 80 | let typename = C.gen_builtin_type_name piqi_type ?ocaml_type in 81 | let wire_typename = C.gen_wire_type_name piqi_type wire_type in 82 | let default = gen_int piqi_type wire_type in 83 | let default_expr = iod " " [ 84 | ios "(Piqirun.parse_default"; ioq (String.escaped default); ios ")"; 85 | ] 86 | in 87 | iol [ 88 | ios "Piqirun."; 89 | ios typename; 90 | ios "_of_"; 91 | ios wire_typename; 92 | default_expr; 93 | ] 94 | 95 | 96 | (* copy-pasted Piqic_ocaml_out.gen_alias_type -- not sure how to avoid this *) 97 | let rec gen_alias_type ?wire_type context a = 98 | let open A in 99 | match a.typename with 100 | | None -> (* this is a built-in type, so piqi_type must be defined *) 101 | let piqi_type = some_of a.piqi_type in 102 | gen_builtin_type context piqi_type a.ocaml_type wire_type 103 | | Some typename -> 104 | let import, parent_piqi, typedef = C.resolve_typename context typename in 105 | match typedef with 106 | | `alias a when wire_type <> None -> 107 | (* need special handing in case when higher-level alias overrides 108 | * protobuf_wire_type *) 109 | let context = C.switch_context context parent_piqi in 110 | gen_alias_type context a ?wire_type 111 | | _ -> 112 | gen_type context typename 113 | 114 | 115 | let gen_field_default_cons context rname f = 116 | let open Field in 117 | let fname = C.mlname_of_field context f in 118 | let ffname = (* fully-qualified field name *) 119 | iol [ios rname; ios "."; ios fname] 120 | in 121 | let value = Piqic_ocaml_in.gen_field_default_value context f in 122 | (* field construction code *) 123 | iol [ffname; ios " = "; value; ios ";"] 124 | 125 | 126 | let gen_record context r = 127 | (* fully-qualified capitalized record name *) 128 | let rname = String.capitalize (some_of r.R.ocaml_name) in 129 | (* order fields by are by their integer codes *) 130 | let fields = List.sort (fun a b -> compare a.F.code b.F.code) r.R.field in 131 | let fconsl = (* field constructor list *) 132 | if fields <> [] 133 | then List.map (gen_field_default_cons context rname) fields 134 | else [ios rname; ios "._dummy = ();"] 135 | in 136 | let fconsl = 137 | if !C.flag_gen_preserve_unknown_fields 138 | then fconsl @ [iol [ios rname; ios ".piqi_unknown_pb = [];"]] 139 | else fconsl 140 | in (* fake_ function delcaration *) 141 | iol [ 142 | ios "default_"; ios (some_of r.R.ocaml_name); ios " () ="; 143 | ioi [ 144 | ios "{"; 145 | ioi (newlines fconsl); 146 | ios "}"; 147 | ] 148 | ] 149 | 150 | 151 | let gen_enum e = 152 | let open Enum in 153 | (* there must be at least one option *) 154 | let const = List.hd e.option in 155 | iol [ 156 | ios "default_"; ios (some_of e.ocaml_name); ios " () = "; 157 | C.gen_pvar_name (some_of const.O.ocaml_name) 158 | ] 159 | 160 | 161 | let gen_option context varname o = 162 | let open Option in 163 | let name = C.mlname_of_option context o in 164 | match o.typename with 165 | | None -> (* this is a flag, i.e. option without a type *) 166 | C.gen_pvar_name name 167 | | Some typename -> 168 | let import, parent_piqi, typedef = C.resolve_typename context typename in 169 | match o.ocaml_name, typedef with 170 | | None, `variant _ | None, `enum _ -> 171 | iol [ 172 | ios "("; gen_type context typename; ios " :> "; ios varname; ios ")" 173 | ] 174 | | _ -> 175 | iol [ 176 | C.gen_pvar_name name; 177 | ios " ("; gen_type context typename; ios ")"; 178 | ] 179 | 180 | 181 | let gen_variant context v = 182 | let open Variant in 183 | let name = some_of v.ocaml_name in 184 | let scoped_name = C.scoped_name context name in 185 | (* there must be at least one option *) 186 | let opt = gen_option context scoped_name (List.hd v.option) in 187 | iol [ 188 | ios "default_"; ios name; ios " () = "; opt; 189 | ] 190 | 191 | 192 | let gen_alias context a = 193 | let open Alias in 194 | (* TODO: handle a new ocaml_default property the same way we do in 195 | * piqic-erlang *) 196 | iol [ 197 | ios "default_"; ios (some_of a.ocaml_name); ios " () = "; 198 | C.gen_convert_value context a.ocaml_type "_of_" a.typename (gen_alias_type context a); 199 | ] 200 | 201 | 202 | let gen_list l = 203 | let open L in 204 | iol [ 205 | ios "default_"; ios (some_of l.ocaml_name); ios " () = "; 206 | if l.ocaml_array 207 | then ios "[||]" 208 | else ios "[]"; 209 | ] 210 | 211 | 212 | let gen_typedef context typedef = 213 | match typedef with 214 | | `record t -> gen_record context t 215 | | `variant t -> gen_variant context t 216 | | `enum t -> gen_enum t 217 | | `list t -> gen_list t 218 | | `alias t -> gen_alias context t 219 | 220 | 221 | let gen_typedefs context typedefs = 222 | if typedefs = [] 223 | then iol [] 224 | else 225 | let defs = List.map (gen_typedef context) typedefs in 226 | iol [ 227 | ios "let rec "; iod "and " (newlines defs); 228 | eol; eol 229 | ] 230 | 231 | 232 | let gen_piqi context = 233 | gen_typedefs context context.piqi.P.typedef 234 | 235 | -------------------------------------------------------------------------------- /piqic-ocaml/piqic_ocaml_ext.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright 2009, 2010, 2011, 2012, 2013, 2014, 2015 Anton Lavrik 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | *) 16 | 17 | 18 | (* 19 | * generation of interfaces for multi-format JSON/XML/Protobuf/Piq serialization 20 | *) 21 | 22 | module C = Piqic_common 23 | open C 24 | open Iolist 25 | 26 | 27 | let gen_init_piqi modname = 28 | (* init embedded piqi spec *) 29 | iol [ 30 | ios "let piqi = "; ios modname; ios ".piqi"; eol; 31 | eol; eol; 32 | ios "let () = Piqirun_ext.init_piqi piqi"; eol; 33 | ] 34 | 35 | 36 | let typedef_scoped_name context typedef = 37 | let piqi = context.piqi in 38 | let modname = some_of piqi.P.modname in 39 | let name = C.typedef_name typedef in 40 | modname ^ "/" ^ name 41 | 42 | 43 | let gen_init_piqi_type context typedef = 44 | let name = C.typedef_mlname typedef in 45 | let scoped_name = typedef_scoped_name context typedef in 46 | iol [ 47 | ios "let _"; ios name; ios "_piqi_type = "; 48 | ios "Piqirun_ext.find_piqi_type "; ioq scoped_name; 49 | eol; 50 | ] 51 | 52 | 53 | let gen_convert name input_format output_format data = 54 | let piqi_type = "_" ^ name ^ "_piqi_type" in 55 | iod " " [ 56 | ios "Piqirun_ext.convert"; 57 | ios piqi_type; ios input_format; ios output_format; ios data; 58 | ] 59 | 60 | 61 | let gen_parse modname typedef = 62 | let name = C.typedef_mlname typedef in 63 | iol [ 64 | ios "let parse_"; ios name; ios " ?opts x (format :Piqirun_ext.input_format) ="; 65 | ioi [ 66 | ios "let x_pb = "; gen_convert name "format" "`pb" "x"; ios " ?opts in"; eol; 67 | ios "let buf = Piqirun.init_from_string x_pb in"; eol; 68 | ios modname; ios ".parse_"; ios name; ios " buf" 69 | ]; 70 | eol; 71 | ] 72 | 73 | 74 | let gen_gen modname typedef = 75 | let name = C.typedef_mlname typedef in 76 | iol [ 77 | ios "let gen_"; ios name; ios " ?opts x (format :Piqirun_ext.output_format) ="; 78 | ioi [ 79 | ios "let buf = "; ios modname; ios ".gen_"; ios name; ios " x in"; eol; 80 | ios "let x_pb = Piqirun.to_string buf in"; eol; 81 | gen_convert name "`pb" "format" "x_pb"; ios " ?opts" 82 | ]; 83 | eol; 84 | ] 85 | 86 | 87 | let gen_print typedef = 88 | let name = C.typedef_mlname typedef in 89 | iol [ 90 | ios "let print_"; ios name; ios " ?opts x ="; eol; 91 | ios " Stdlib.print_endline (gen_"; ios name; ios " x `piq ?opts)"; 92 | eol; 93 | ios "let prerr_"; ios name; ios " ?opts x ="; eol; 94 | ios " Stdlib.prerr_endline (gen_"; ios name; ios " x `piq ?opts)"; 95 | eol; 96 | ] 97 | 98 | 99 | (* NOTE: the only purpose of this is to make sure that all the dependencies are 100 | * going to be linked in. Otherwise, Piqi modules can end up being missing and 101 | * uninitialized. This, in turn, leads to crash on multi-format serialization of 102 | * nested types from imported modules *) 103 | let gen_import context import = 104 | let open Import in 105 | let index = C.resolve_import context import in 106 | let piqi = index.i_piqi in 107 | iol [ 108 | ios "module "; ios (some_of import.ocaml_name); ios "_ext"; ios " = "; 109 | ios (some_of piqi.P.ocaml_module); ios "_ext"; 110 | eol; 111 | ] 112 | 113 | 114 | let gen_imports context l = 115 | let l = List.map (gen_import context) l in 116 | iol l 117 | 118 | 119 | let gen_piqi context = 120 | let piqi = context.piqi in 121 | let modname = some_of piqi.P.ocaml_module in 122 | let typedefs = piqi.P.typedef in 123 | 124 | (* XXX, TODO: skipping built-in typedefs for now; in theory we could generate 125 | * piqi_piqi_ext.ml and include it as a part of piqilib to make serialization 126 | * for built-in types to work *) 127 | let typedefs = List.filter (fun x -> not (C.is_builtin_typedef x)) typedefs in 128 | 129 | let type_initializers = List.map (gen_init_piqi_type context) typedefs in 130 | let parsers = List.map (gen_parse modname) typedefs in 131 | let generators = List.map (gen_gen modname) typedefs in 132 | let printers = List.map gen_print typedefs in 133 | 134 | iol [ 135 | gen_imports context piqi.P.import; 136 | gen_init_piqi modname; eol; eol; 137 | iol type_initializers; eol; eol; 138 | iol (newlines parsers); eol; 139 | iol (newlines generators); eol; 140 | iol (newlines printers); eol; 141 | ] 142 | 143 | -------------------------------------------------------------------------------- /piqic-ocaml/piqic_ocaml_in.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2017 Anton Lavrik 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | *) 16 | 17 | 18 | (* 19 | * generation of Protocol Buffers -> OCaml decoders 20 | *) 21 | 22 | module C = Piqic_common 23 | open C 24 | open Iolist 25 | 26 | 27 | let gen_type ?(is_packed=false) context typename = 28 | let import, parent_piqi, typedef = C.resolve_typename context typename in 29 | let packed_prefix = C.gen_packed_prefix is_packed in 30 | let parent_mod = C.gen_parent_mod import in 31 | iol [ 32 | parent_mod; 33 | packed_prefix; ios "parse_"; 34 | ios (C.typedef_mlname typedef) 35 | ] 36 | 37 | 38 | let gen_builtin_type context piqi_type ocaml_type wire_type is_packed = 39 | match piqi_type with 40 | | `any -> 41 | if context.is_self_spec 42 | then ios "parse_any" (* XXX: handle the case when ocaml_type <> None *) 43 | else ios "Piqi_piqi.parse_any" 44 | | _ -> 45 | let packed_prefix = C.gen_packed_prefix is_packed in 46 | let typename = C.gen_builtin_type_name piqi_type ?ocaml_type in 47 | let wire_typename = C.gen_wire_type_name piqi_type wire_type in 48 | iol [ 49 | gen_cc "(fun x -> let count = next_count() in refer count ("; 50 | ios "Piqirun."; 51 | ios typename; 52 | ios "_of_"; packed_prefix; 53 | ios wire_typename; 54 | gen_cc " x))"; 55 | ] 56 | 57 | 58 | (* copy-pasted Piqic_ocaml_out.gen_alias_type -- not sure how to avoid this *) 59 | let rec gen_alias_type ?wire_type ?(is_packed=false) context a = 60 | let open A in 61 | match a.typename with 62 | | None -> (* this is a built-in type, so piqi_type must be defined *) 63 | let piqi_type = some_of a.piqi_type in 64 | gen_builtin_type context piqi_type a.ocaml_type wire_type is_packed 65 | | Some typename -> 66 | let import, parent_piqi, typedef = C.resolve_typename context typename in 67 | match typedef with 68 | | `alias a when wire_type <> None -> 69 | (* need special handing in case when higher-level alias overrides 70 | * protobuf_wire_type *) 71 | let context = C.switch_context context parent_piqi in 72 | gen_alias_type context a ?wire_type ~is_packed 73 | | _ -> 74 | gen_type context typename ~is_packed 75 | 76 | 77 | (* TODO: parse defaults once at boot time rather than each time when we need to 78 | * parse a field *) 79 | let gen_default = function 80 | | None -> iol [] 81 | | Some piqi_any -> 82 | let pb = some_of piqi_any.Any.protobuf in 83 | iol [ios "~default:"; ioq (String.escaped pb) ] 84 | 85 | 86 | let esc x = ios "_" ^^ ios x 87 | 88 | 89 | let gen_default_type context typename = 90 | let import, parent_piqi, typedef = C.resolve_typename context typename in 91 | let parent_mod = C.gen_parent_mod import in 92 | iol [parent_mod; ios "default_"; ios (C.typedef_mlname typedef); ios " ()"] 93 | 94 | 95 | let gen_field_default_value context f = 96 | let open Field in 97 | let typename = some_of f.typename in 98 | match f.mode, f.default with 99 | | `required, _ -> 100 | gen_default_type context typename 101 | | `optional, Some piqi_any when not f.ocaml_optional -> 102 | let pb = some_of piqi_any.Any.protobuf in 103 | let default_str = String.escaped pb in 104 | iol [ 105 | gen_cc "(Piqloc.pause (); let res = "; 106 | gen_type context typename; 107 | ios " (Piqirun.parse_default "; ioq default_str; ios ")"; 108 | gen_cc " in Piqloc.resume (); res)"; 109 | ] 110 | | `optional, _ -> ios "None" 111 | | `repeated, _ -> 112 | if f.ocaml_array 113 | then ios "[||]" 114 | else ios "[]" 115 | 116 | 117 | let gen_field_cons context rname f = 118 | let open Field in 119 | let fname = C.mlname_of_field context f in 120 | let ffname = (* fully-qualified field name *) 121 | iol [ios rname; ios "."; ios fname] 122 | in 123 | (* field construction code *) 124 | iol [ ffname; ios " = "; esc fname; ios ";" ] 125 | 126 | 127 | let gen_field_parser context f = 128 | let open Field in 129 | let fname = C.mlname_of_field context f in 130 | let mode = C.gen_field_mode context f in 131 | let code = C.gen_code f.code in 132 | let typename = some_of f.typename in 133 | if not f.internal 134 | then 135 | let fcons = 136 | (* field constructor *) 137 | iod " " [ 138 | (* "parse_(required|optional|repeated)_field" function invocation *) 139 | ios "Piqirun.parse_" ^^ ios mode ^^ ios "_field"; 140 | code; 141 | gen_type context typename ~is_packed:f.protobuf_packed; 142 | 143 | (* when parsing packed repeated fields, we should also accept 144 | * fields in unpacked representation; therefore, specifying an 145 | * unpacked field parser as another parameter *) 146 | if f.protobuf_packed 147 | then gen_type context typename 148 | else iol []; 149 | 150 | ios "x"; 151 | gen_default f.default; 152 | ] 153 | in 154 | (* field parsing code *) 155 | iol [ ios "let "; esc fname; ios ", x = "; fcons; ios " in"; eol ] 156 | else 157 | let fcons = gen_field_default_value context f in 158 | iol [ ios "let "; esc fname; ios " = "; fcons; ios " in"; eol ] 159 | 160 | 161 | let gen_record context r = 162 | (* fully-qualified capitalized record name *) 163 | let rname = String.capitalize (some_of r.R.ocaml_name) in 164 | (* order fields by are by their integer codes *) 165 | let fields = List.sort (fun a b -> compare a.F.code b.F.code) r.R.field in 166 | let fconsl = (* field constructor list *) 167 | if fields <> [] 168 | then List.map (gen_field_cons context rname) fields 169 | else [ios rname; ios "._dummy = ();"] 170 | in 171 | let fconsl = 172 | if !C.flag_gen_preserve_unknown_fields 173 | then fconsl @ [iol [ios rname; ios ".piqi_unknown_pb = x;"]] 174 | else fconsl 175 | in 176 | let fparserl = (* field parsers list *) 177 | List.map (gen_field_parser context) fields 178 | in 179 | let rcons = (* record constructor *) 180 | iol [ 181 | iol fparserl; 182 | ios "Piqirun.check_unparsed_fields x;"; eol; 183 | ios "{"; 184 | ioi (newlines fconsl); 185 | ios "}"; 186 | ] 187 | in (* parse_ function delcaration *) 188 | iol [ 189 | ios "parse_"; ios (some_of r.R.ocaml_name); ios " x ="; 190 | ioi [ 191 | ios "let x = Piqirun.parse_record x in"; eol; 192 | gen_cc "let count = next_count() in refer count (\n"; 193 | rcons; 194 | gen_cc ")\n"; 195 | ] 196 | ] 197 | 198 | 199 | let gen_const c = 200 | let open Option in 201 | let name = C.gen_pvar_name (some_of c.ocaml_name) in 202 | iol [ios "| "; C.gen_code c.code; ios "l -> "; name] 203 | 204 | 205 | let gen_enum e ~is_packed = 206 | let open Enum in 207 | let consts = List.map gen_const e.option in 208 | let packed_prefix = C.gen_packed_prefix is_packed in 209 | iol [ 210 | packed_prefix; ios "parse_"; ios (some_of e.ocaml_name); ios " x ="; 211 | ioi [ 212 | gen_cc "let count = next_count() in refer count ("; 213 | ios "match Piqirun.int32_of_"; packed_prefix; ios "signed_varint x with"; 214 | ioi [ 215 | iol (newlines consts); 216 | ios "| x -> Piqirun.error_enum_const x"; 217 | ]; 218 | gen_cc ")\n"; 219 | ] 220 | ] 221 | 222 | 223 | let gen_enum e = 224 | (* generate two functions: one for parsing normal value; another one -- for 225 | * packed value *) 226 | iol [ 227 | gen_enum e ~is_packed:false; eol; 228 | ios "and "; 229 | gen_enum e ~is_packed:true 230 | ] 231 | 232 | 233 | let gen_option context varname o = 234 | let open Option in 235 | let name = C.mlname_of_option context o in 236 | let code = C.gen_code o.code in 237 | match o.typename with 238 | | None -> (* this is a flag, i.e. option without a type *) 239 | iol [ 240 | ios "| "; code; ios " when x = Piqirun.Varint 1 -> "; 241 | (* NOTE: providing special handling for boxed values, see "refer" *) 242 | gen_cc "let count = next_count() in refer count "; 243 | C.gen_pvar_name name; 244 | ] 245 | | Some typename -> 246 | let import, parent_piqi, typedef = C.resolve_typename context typename in 247 | match o.ocaml_name, typedef with 248 | | None, `variant _ | None, `enum _ -> 249 | (* handle variant and enum subtyping cases by lifting their labels 250 | * and clauses to the top level -- in fact, relying on OCaml here 251 | * by using # construct *) 252 | iol [ 253 | ios "| "; code; ios " -> "; 254 | ios "("; gen_type context typename; ios " x :> "; ios varname; ios ")" 255 | ] 256 | | _ -> 257 | iol [ 258 | ios "| "; code; ios " ->"; 259 | indent (ioi [ 260 | ios "let res = "; 261 | gen_cc "let count = curr_count() in refer count ("; 262 | gen_type context typename; ios " x"; 263 | gen_cc ")"; 264 | ios " in"; eol; 265 | C.gen_pvar_name name; ios " res"; 266 | ]) 267 | ] 268 | 269 | 270 | let gen_variant context v = 271 | let open Variant in 272 | let name = some_of v.ocaml_name in 273 | let scoped_name = C.scoped_name context name in 274 | let options = List.map (gen_option context scoped_name) v.option in 275 | iol [ 276 | ios "parse_"; ios name; ios " x ="; 277 | ioi [ 278 | ios "let code, x = Piqirun.parse_variant x in"; eol; 279 | gen_cc "let count = next_count() in refer count (\n"; 280 | ios "match code with"; 281 | ioi [ 282 | iol (newlines options); 283 | ios "| _ -> Piqirun.error_variant x code"; 284 | ]; 285 | gen_cc ")\n"; 286 | ] 287 | ] 288 | 289 | 290 | let gen_alias context a ~is_packed = 291 | let open Alias in 292 | let packed_prefix = C.gen_packed_prefix is_packed in 293 | iol [ 294 | packed_prefix; ios "parse_"; ios (some_of a.ocaml_name); ios " x = "; 295 | C.gen_convert_value context a.ocaml_type "_of_" a.typename ( 296 | iol [ 297 | gen_alias_type context a ?wire_type:a.protobuf_wire_type ~is_packed; 298 | ios " x"; 299 | ] 300 | ) 301 | ] 302 | 303 | 304 | let gen_alias context a = 305 | let open Alias in 306 | if C.can_be_protobuf_packed context (`alias a) 307 | then 308 | (* if a value can be packed, we need to generate two functions: one for 309 | * parsing regular (unpacked) representation, and another one for 310 | * parsing packed form *) 311 | iol [ 312 | gen_alias context a ~is_packed:false; eol; 313 | ios "and "; 314 | gen_alias context a ~is_packed:true 315 | ] 316 | else 317 | gen_alias context a ~is_packed:false 318 | 319 | 320 | let gen_list context l = 321 | let open L in 322 | let repr = C.gen_list_repr context l in 323 | iol [ 324 | ios "parse_"; ios (some_of l.ocaml_name); ios " x ="; eol; 325 | gen_cc " let count = next_count() in refer count (\n"; 326 | (* Piqirun.parse_(packed_)?(list|array|array32|array64) *) 327 | ios " Piqirun.parse_"; repr; 328 | ios " ("; gen_type context l.typename ~is_packed:l.protobuf_packed; ios ")"; 329 | 330 | (* when parsing packed repeated fields, we should also accept 331 | * fields in unpacked representation; therefore, specifying an 332 | * unpacked field parser as another parameter *) 333 | if l.protobuf_packed 334 | then iol [ 335 | ios " ("; gen_type context l.typename; ios ")"; 336 | ] 337 | else iol []; 338 | 339 | ios " x"; eol; 340 | gen_cc " )\n"; 341 | ] 342 | 343 | 344 | let gen_typedef context typedef = 345 | match typedef with 346 | | `record t -> gen_record context t 347 | | `variant t -> gen_variant context t 348 | | `enum t -> gen_enum t 349 | | `list t -> gen_list context t 350 | | `alias t -> gen_alias context t 351 | 352 | 353 | let gen_typedefs context typedefs = 354 | if typedefs = [] 355 | then iol [] 356 | else 357 | let defs = List.map (gen_typedef context) typedefs in 358 | iol [ 359 | gen_cc "let next_count = Piqloc.next_icount\n"; 360 | gen_cc "let curr_count () = !Piqloc.icount\n"; 361 | (* NOTE: providing special handling for boxed objects, since they are not 362 | * references and can not be uniquely identified. Moreover they can mask 363 | * integers which are used for enumerating objects *) 364 | gen_cc "let refer ref obj = 365 | if not (Obj.is_int (Obj.repr obj)) 366 | then Piqloc.addrefret ref obj 367 | else obj\n"; 368 | 369 | ios "let rec "; iod "and " (newlines (newlines defs)); 370 | eol 371 | ] 372 | 373 | 374 | let gen_piqi context = 375 | gen_typedefs context context.piqi.P.typedef 376 | 377 | -------------------------------------------------------------------------------- /piqic-ocaml/piqic_ocaml_out.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2017 Anton Lavrik 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | *) 16 | 17 | 18 | (* 19 | * generation of OCaml -> Protocol Buffers encoders 20 | *) 21 | 22 | module C = Piqic_common 23 | open C 24 | open Iolist 25 | 26 | 27 | let gen_type ?(is_packed=false) context typename = 28 | let import, parent_piqi, typedef = C.resolve_typename context typename in 29 | let packed_prefix = C.gen_packed_prefix is_packed in 30 | let parent_mod = C.gen_parent_mod import in 31 | iol [ 32 | parent_mod; 33 | packed_prefix; ios "gen__"; 34 | ios (C.typedef_mlname typedef) 35 | ] 36 | 37 | 38 | let gen_builtin_type context piqi_type ocaml_type wire_type is_packed = 39 | match piqi_type with 40 | | `any -> 41 | (* TODO, XXX: why do we need an extra closure here? *) 42 | if context.is_self_spec 43 | then 44 | (* XXX: handle the case when ocaml_type <> None *) 45 | ios "(fun code x -> gen__any code x)" 46 | else 47 | ios "(fun code x -> Piqi_piqi.gen__any code x)" 48 | | _ -> 49 | let packed_prefix = C.gen_packed_prefix is_packed in 50 | let typename = C.gen_builtin_type_name piqi_type ?ocaml_type in 51 | let wire_typename = C.gen_wire_type_name piqi_type wire_type in 52 | (* XXX: packed isn't used in cc mode, we can safely remove generation of 53 | * reference1 *) 54 | iol [ 55 | (if is_packed then gen_cc "(reference1 " else gen_cc "(reference "); 56 | ios "Piqirun."; 57 | ios typename; 58 | ios "_to_"; packed_prefix; 59 | ios wire_typename; 60 | gen_cc ")"; 61 | ] 62 | 63 | 64 | let rec gen_alias_type ?wire_type ?(is_packed=false) context a = 65 | let open A in 66 | match a.typename with 67 | | None -> (* this is a built-in type, so piqi_type must be defined *) 68 | let piqi_type = some_of a.piqi_type in 69 | gen_builtin_type context piqi_type a.ocaml_type wire_type is_packed 70 | | Some typename -> 71 | let import, parent_piqi, typedef = C.resolve_typename context typename in 72 | match typedef with 73 | | `alias a when wire_type <> None -> 74 | (* need special handing in case when higher-level alias overrides 75 | * protobuf_wire_type *) 76 | let context = C.switch_context context parent_piqi in 77 | gen_alias_type context a ?wire_type ~is_packed 78 | | _ -> 79 | gen_type context typename ~is_packed 80 | 81 | 82 | let gen_field context rname f = 83 | let open Field in 84 | let fname = C.mlname_of_field context f in 85 | let ffname = (* fully-qualified field name *) 86 | iod "." [ios "x"; ios rname; ios fname] 87 | in 88 | let code = C.gen_code f.code in 89 | let mode = C.gen_field_mode context f in 90 | let typename = some_of f.typename in 91 | (* field generation code *) 92 | let fgen = iod " " [ 93 | ios "Piqirun.gen_" ^^ ios mode ^^ ios "_field"; 94 | code; 95 | gen_type context typename ~is_packed:f.protobuf_packed; 96 | ffname 97 | ] 98 | in (fname, fgen) 99 | 100 | 101 | let gen_record context r = 102 | (* fully-qualified capitalized record name *) 103 | let rname = String.capitalize (some_of r.R.ocaml_name) in 104 | (* skip fields marked as .internal *) 105 | let fields = r.R.field in 106 | let fields = List.filter (fun x -> not x.F.internal) fields in 107 | (* order fields by are by their integer codes *) 108 | let fields = List.sort (fun a b -> compare a.F.code b.F.code) fields in 109 | let fgens = (* field generators list *) 110 | List.map (gen_field context rname) fields 111 | in 112 | (* field names *) 113 | let fnames, _ = List.split fgens in 114 | 115 | let esc x = ios "_" ^^ ios x in 116 | 117 | (* field generator code *) 118 | let fgens_code = List.map 119 | (fun (name, gen) -> iol [ios "let "; esc name; ios " = "; gen; ios " in"; eol]) 120 | fgens 121 | in 122 | let unknown_fields = 123 | if !C.flag_gen_preserve_unknown_fields 124 | then [iol [ios "(Piqirun.gen_parsed_field_list "; ios "x."; ios rname; ios ".piqi_unknown_pb)"]] 125 | else [ios "[]"] 126 | in (* gen_ function delcaration *) 127 | iol [ 128 | ios "gen__"; ios (some_of r.R.ocaml_name); ios " code x ="; 129 | ioi [ 130 | gen_cc "refer x;\n"; 131 | iol fgens_code; 132 | ios "Piqirun.gen_record code ("; 133 | iod " :: " ((List.map esc fnames) @ unknown_fields); 134 | ios ")"; 135 | ] 136 | ] 137 | 138 | 139 | let gen_const c = 140 | let open Option in 141 | iol [ 142 | ios "| "; C.gen_pvar_name (some_of c.ocaml_name); ios " -> "; 143 | C.gen_code c.code; ios "l"; (* ocaml int32 literal *) 144 | ] 145 | 146 | 147 | let gen_enum_consts l = 148 | let consts = List.map gen_const l in 149 | iol [ 150 | ios "(match x with"; 151 | ioi (newlines consts); 152 | ios ")" 153 | ] 154 | 155 | 156 | let gen_unpacked_enum e = 157 | let open Enum in 158 | iol [ 159 | ios "gen__"; ios (some_of e.ocaml_name); ios " code x ="; 160 | ioi [ 161 | gen_cc "refer x;\n"; 162 | ios "Piqirun.int32_to_signed_varint code "; gen_enum_consts e.option; 163 | ] 164 | ] 165 | 166 | 167 | let gen_packed_enum e = 168 | let open Enum in 169 | iol [ 170 | ios "packed_gen__"; ios (some_of e.ocaml_name); ios " x ="; 171 | ioi [ 172 | gen_cc "refer x;\n"; 173 | ios "Piqirun.int32_to_packed_signed_varint "; gen_enum_consts e.option; 174 | ] 175 | ] 176 | 177 | 178 | let gen_enum e = 179 | (* generate two functions: one for generating normal value; another one -- for 180 | * packed value *) 181 | iol [ 182 | gen_unpacked_enum e; eol; 183 | ios "and "; 184 | gen_packed_enum e; 185 | ] 186 | 187 | 188 | let gen_option context o = 189 | let open Option in 190 | let name = C.mlname_of_option context o in 191 | let code = C.gen_code o.code in 192 | match o.typename with 193 | | None -> (* this is a flag, i.e. option without a type *) 194 | iol [ 195 | ios "| "; C.gen_pvar_name name; ios " -> "; 196 | gen_cc "refer x; "; 197 | ios "Piqirun.gen_bool_field "; code; ios " true"; 198 | ] 199 | | Some typename -> 200 | let import, parent_piqi, typedef = C.resolve_typename context typename in 201 | match o.ocaml_name, typedef with 202 | | None, `variant _ | None, `enum _ -> 203 | (* handle variant and enum subtyping cases by lifting their labels 204 | * and clauses to the top level -- in fact, relying on OCaml here 205 | * by using # construct *) 206 | let scoped_typename = Piqic_ocaml_types.gen_typedef_type context typedef ?import in 207 | iol [ 208 | ios "| (#"; ios scoped_typename; ios " as x) -> "; 209 | gen_type context typename; ios " "; code; ios " x"; 210 | ] 211 | | _ -> 212 | iol [ 213 | ios "| "; C.gen_pvar_name name; ios " x -> "; 214 | gen_type context typename; ios " "; code; ios " x"; 215 | ] 216 | 217 | 218 | let gen_variant context v = 219 | let open Variant in 220 | let options = List.map (gen_option context) v.option in 221 | let typename = Piqic_ocaml_types.gen_typedef_type context (`variant v) in 222 | iol [ 223 | ios "gen__"; ios (some_of v.ocaml_name); ios " code (x:"; ios typename; ios ") ="; 224 | ioi [ 225 | gen_cc "refer x;\n"; 226 | ios "Piqirun.gen_record code [(match x with"; 227 | ioi (newlines options); 228 | ios ")]"; 229 | ] 230 | ] 231 | 232 | 233 | let gen_unpacked_alias context a = 234 | let open Alias in 235 | iol [ 236 | ios "gen__"; ios (some_of a.ocaml_name); ios " code x = "; 237 | gen_alias_type context a ?wire_type:a.protobuf_wire_type; 238 | ios " code"; 239 | C.gen_convert_value context a.ocaml_type "_to_" a.typename (ios " x"); 240 | ] 241 | 242 | 243 | let gen_packed_alias context a = 244 | let open Alias in 245 | iol [ 246 | ios "packed_gen__"; ios (some_of a.ocaml_name); ios " x = "; 247 | gen_alias_type context a ?wire_type:a.protobuf_wire_type ~is_packed:true; 248 | C.gen_convert_value context a.ocaml_type "_to_" a.typename (ios " x"); 249 | ] 250 | 251 | 252 | let gen_alias context a = 253 | let open Alias in 254 | if C.can_be_protobuf_packed context (`alias a) 255 | then 256 | (* if a value can be packed, we need to generate two functions: one for 257 | * generating regular (unpacked) representation, and another one for 258 | * generating packed form *) 259 | iol [ 260 | gen_unpacked_alias context a; eol; 261 | ios "and "; 262 | gen_packed_alias context a; 263 | ] 264 | else 265 | gen_unpacked_alias context a 266 | 267 | 268 | let gen_list context l = 269 | let open L in 270 | let repr = C.gen_list_repr context l in 271 | iol [ 272 | ios "gen__"; ios (some_of l.ocaml_name); ios " code x = "; 273 | gen_cc "reference "; 274 | (* Piqirun.gen_(packed_)?(list|array|array32|array64) *) 275 | ios "(Piqirun.gen_"; repr; ios " ("; 276 | gen_type context l.typename ~is_packed:l.protobuf_packed; 277 | ios ")) code x"; 278 | ] 279 | 280 | 281 | (* generate gen__ functions *) 282 | let gen_typedef_2 context typedef = 283 | match typedef with 284 | | `alias t -> gen_alias context t 285 | | `record t -> gen_record context t 286 | | `variant t -> gen_variant context t 287 | | `enum t -> gen_enum t 288 | | `list t -> gen_list context t 289 | 290 | 291 | (* generate gen_ functions *) 292 | let gen_typedef_1 x = 293 | let name = ios (C.typedef_mlname x) in 294 | iol [ 295 | ios "let gen_"; name; ios " x = "; 296 | ios "gen__"; name; ios " (-1) x"; 297 | ] 298 | 299 | 300 | let gen_typedefs context typedefs = 301 | if typedefs = [] 302 | then iol [] 303 | else 304 | let defs_2 = List.map (gen_typedef_2 context) typedefs in 305 | let defs_1 = List.map gen_typedef_1 typedefs in 306 | iol [ 307 | gen_cc "let next_count = Piqloc.next_ocount\n"; 308 | (* NOTE: providing special handling for boxed objects, since they are not 309 | * references and can not be uniquely identified. Moreover they can mask 310 | * integers which are used for enumerating objects *) 311 | gen_cc "let refer obj = 312 | let count = next_count () in 313 | if not (Obj.is_int (Obj.repr obj)) 314 | then Piqloc.addref obj count\n"; 315 | gen_cc "let reference f code x = refer x; f code x\n"; 316 | gen_cc "let reference1 f x = refer x; f x\n"; 317 | 318 | ios "let rec "; iod "and " (newlines (newlines defs_2)); 319 | eol; 320 | iol (newlines defs_1); 321 | eol; eol; 322 | ] 323 | 324 | 325 | let gen_piqi context = 326 | gen_typedefs context context.piqi.P.typedef 327 | 328 | -------------------------------------------------------------------------------- /piqic-ocaml/piqic_ocaml_types.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright 2009, 2010, 2011, 2012, 2013, 2014, 2015 Anton Lavrik 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | *) 16 | 17 | 18 | (* 19 | * generation of Ocaml type definitions 20 | *) 21 | 22 | module C = Piqic_common 23 | open C 24 | open Iolist 25 | 26 | 27 | let gen_builtin_type context piqi_type = 28 | match piqi_type with 29 | | `any -> 30 | if context.is_self_spec 31 | then C.scoped_name context "any" 32 | else "Piqi_piqi.any" 33 | | t -> 34 | C.gen_builtin_type_name t 35 | 36 | 37 | let gen_typedef_type ?import context typedef = 38 | let ocaml_name = C.typedef_mlname typedef in 39 | match import with 40 | | None -> (* local typedef *) 41 | C.scoped_name context ocaml_name 42 | | Some import -> 43 | let ocaml_modname = some_of import.Import.ocaml_name in 44 | (ocaml_modname ^ "." ^ ocaml_name) 45 | 46 | 47 | (* XXX: check type compatibility *) 48 | let rec gen_type context typename = 49 | let import, parent_piqi, typedef = C.resolve_typename context typename in 50 | let context = C.switch_context context parent_piqi in 51 | match typedef with 52 | | `alias a -> 53 | let ocaml_name = some_of a.A.ocaml_name in 54 | (* skip cyclic type abbreviations *) 55 | let ocaml_type = gen_alias_type context a in 56 | if ocaml_name = ocaml_type (* cyclic type abbreviation? *) 57 | then ocaml_type 58 | else gen_typedef_type context typedef ?import 59 | | _ -> (* record | variant | list | enum *) 60 | gen_typedef_type context typedef ?import 61 | 62 | 63 | and gen_alias_type context a = 64 | let open Alias in 65 | match a.ocaml_type, a.typename with 66 | | Some x, _ -> x 67 | | None, None -> 68 | (* this is an alias for a built-in type (piqi_type field must be defined 69 | * when neither of type and ocaml_type fields are present) *) 70 | gen_builtin_type context (some_of a.piqi_type) 71 | | None, Some typename -> 72 | gen_type context typename 73 | 74 | 75 | let ios_gen_type context typename = 76 | ios (gen_type context typename) 77 | 78 | 79 | let gen_field_type context f = 80 | let open F in 81 | match f.typename with 82 | | None -> ios "bool"; (* flags are represented as booleans *) 83 | | Some typename -> 84 | let deftype = ios_gen_type context typename in 85 | match f.mode with 86 | | `required -> deftype 87 | | `optional when f.default <> None && (not f.ocaml_optional) -> 88 | deftype (* optional + default *) 89 | | `optional -> deftype ^^ ios " option" 90 | | `repeated -> 91 | deftype ^^ 92 | if f.ocaml_array 93 | then ios " array" 94 | else ios " list" 95 | 96 | 97 | let gen_field context f = 98 | let open F in 99 | iol [ 100 | ios "mutable "; (* NOTE: defining all fields as mutable at the moment *) 101 | ios (C.mlname_of_field context f); 102 | ios ": "; 103 | gen_field_type context f; 104 | ios ";" 105 | ] 106 | 107 | 108 | (* generate record type in record module; see also gen_record' *) 109 | let gen_record_mod context r = 110 | let modname = String.capitalize (some_of r.R.ocaml_name) in 111 | let fields = r.R.field in 112 | let fdefs = (* field definition list *) 113 | if fields <> [] 114 | then List.map (gen_field context) fields 115 | else [ios "_dummy: unit;"] 116 | in 117 | let fdefs = 118 | if !C.flag_gen_preserve_unknown_fields 119 | then fdefs @ [ios "piqi_unknown_pb: (int * Piqirun.t) list;"] 120 | else fdefs 121 | in 122 | (* record def constructor *) 123 | let rcons = [ 124 | ios "type t = "; ios "{"; 125 | ioi (newlines fdefs); 126 | ios "}"; 127 | ] 128 | in iol [ 129 | ios modname; ios ":"; 130 | ioi [ 131 | ios "sig"; 132 | ioi rcons; 133 | ios "end"; ios " = "; ios modname; 134 | ] 135 | ] 136 | 137 | 138 | let gen_option context o = 139 | let open Option in 140 | match o.ocaml_name, o.typename with 141 | | ocaml_name, Some typename -> ( 142 | let import, parent_piqi, typedef = C.resolve_typename context typename in 143 | match ocaml_name, typedef with 144 | | None, `variant x -> 145 | (* NOTE: for some reason, ocaml complains about fully qualified 146 | * polymorphic variants in recursive modules, so we need to use 147 | * non-qualified names in this case *) 148 | if import = None (* local typedef? *) 149 | then ios (some_of x.V.ocaml_name) 150 | else ios_gen_type context typename 151 | | None, `enum x -> 152 | if import = None (* local typedef? *) 153 | then ios (some_of x.E.ocaml_name) 154 | else ios_gen_type context typename 155 | | _ -> 156 | (* same as C.mlname_of_option but avoid resoving the same type 157 | * again *) 158 | let mlname = 159 | match ocaml_name with 160 | | Some n -> n 161 | | None -> C.typedef_mlname typedef 162 | in 163 | let n = C.gen_pvar_name mlname in 164 | n ^^ ios " of " ^^ ios_gen_type context typename 165 | ) 166 | | Some n, None -> 167 | C.gen_pvar_name n 168 | | None, None -> 169 | assert false 170 | 171 | 172 | let gen_alias context a = 173 | let open Alias in 174 | let ocaml_name = some_of a.ocaml_name in 175 | let ocaml_type = gen_alias_type context a in 176 | if ocaml_name = ocaml_type (* cyclic type abbreviation? *) 177 | then [] (* avoid generating cyclic type abbreviations *) 178 | else [iol [ 179 | ios ocaml_name; ios " = "; ios ocaml_type; 180 | ]] 181 | 182 | 183 | let gen_list context l = 184 | let open L in 185 | iol [ 186 | ios (some_of l.ocaml_name); ios " = "; 187 | ios_gen_type context l.typename; 188 | if l.ocaml_array 189 | then ios " array" 190 | else ios " list"; 191 | ] 192 | 193 | 194 | let gen_options context options = 195 | let options_code = List.map (gen_option context) options in 196 | ioi [ 197 | ios "["; 198 | ioi (prefix "| " options_code |> newlines); 199 | ios "]" 200 | ] 201 | 202 | 203 | let gen_variant context v = 204 | let open Variant in 205 | iol [ 206 | ios (some_of v.ocaml_name); ios " ="; 207 | gen_options context v.option; 208 | ] 209 | 210 | 211 | let gen_enum context e = 212 | let open Enum in 213 | iol [ 214 | ios (some_of e.ocaml_name); ios " ="; 215 | gen_options context e.option; 216 | ] 217 | 218 | 219 | let gen_record context r = 220 | let name = some_of r.R.ocaml_name in 221 | let modname = String.capitalize name in 222 | iol [ ios name; ios " = "; ios (modname ^ ".t") ] 223 | 224 | 225 | let gen_typedef context typedef = 226 | match typedef with 227 | | `record t -> [gen_record context t] 228 | | `variant t -> [gen_variant context t] 229 | | `enum t -> [gen_enum context t] 230 | | `list t -> [gen_list context t] 231 | | `alias t -> gen_alias context t 232 | 233 | 234 | let gen_mod_typedef context typedef = 235 | match typedef with 236 | | `record r -> 237 | [gen_record_mod context r] 238 | (* XXX: generate modules for variants? *) 239 | | _ -> [] 240 | 241 | 242 | let gen_typedefs context (typedefs:T.typedef list) = 243 | let top_modname = C.top_modname context in 244 | (* generated typedefs that must be wrapped into ocaml modules *) 245 | let def_mods = U.flatmap (gen_mod_typedef context) typedefs in 246 | (* generated the rest of typedefs wrapped into into an ocaml module *) 247 | let other_defs = U.flatmap (gen_typedef context) typedefs in 248 | let top_mod = iol [ 249 | ios top_modname; 250 | ios ":"; 251 | ioi [ 252 | ios "sig"; 253 | ioi (prefix "type " other_defs |> newlines); 254 | ios "end"; ios " = "; ios top_modname; 255 | ] 256 | ] 257 | in 258 | let code = iol [ 259 | ios "module rec "; top_mod; 260 | iol (prefix "and " def_mods); 261 | ] 262 | in 263 | iol [ 264 | code; 265 | eol; 266 | ] 267 | 268 | 269 | let gen_import context import = 270 | let open Import in 271 | let index = C.resolve_import context import in 272 | let imported_modname = 273 | match import.ocaml_module with 274 | | Some modname -> (* local override *) 275 | modname 276 | | None -> (* original modname *) 277 | let piqi = index.i_piqi in 278 | some_of piqi.P.ocaml_module 279 | in 280 | iod " " [ 281 | ios "module "; ios (some_of import.ocaml_name); ios "="; 282 | ios imported_modname; 283 | eol; eol 284 | ] 285 | 286 | 287 | let gen_imports context l = 288 | let l = List.map (gen_import context) l in 289 | iol l 290 | 291 | 292 | let default_visitor _ = () 293 | 294 | 295 | (* depth-first graph traversal *) 296 | let dfs 297 | ?(pre_visit = default_visitor) 298 | ?(cycle_visit = default_visitor) 299 | ?(post_visit = default_visitor) 300 | (nodes: 'a list) 301 | (get_adjacent_vertixes: ('a -> 'a list)) = 302 | let black = ref [] in (* visited nodes, i.e. after last_visit *) 303 | let grey = ref [] in (* nodes between first_visit and last_visit *) 304 | let set_color node = function 305 | | `black -> black := node::!black 306 | | `grey -> grey := node::!grey 307 | in 308 | let get_color node = 309 | if List.memq node !black 310 | then `black 311 | else if List.memq node !grey 312 | then `grey 313 | else `white 314 | in 315 | let rec aux node = 316 | match get_color node with 317 | | `black -> () (* already processed -- nothing to do *) 318 | | `grey -> (* found a cycle -- run a handler and return *) 319 | cycle_visit node 320 | | `white -> 321 | set_color node `grey; 322 | pre_visit node; (* run a pre-visit handler *) 323 | 324 | List.iter aux (get_adjacent_vertixes node); 325 | 326 | set_color node `black; 327 | post_visit node (* run a post-visit handler *) 328 | in 329 | List.iter aux nodes 330 | 331 | 332 | (* topological sort of a graph *) 333 | let tsort 334 | ?(cycle_visit = (fun _ -> failwith "found a cycle!")) 335 | (nodes: 'a list) 336 | (get_adjacent_vertixes: ('a -> 'a list)) : 'a list = 337 | let stack = ref [] in 338 | let post_visit node = 339 | stack := node::!stack 340 | in 341 | dfs nodes get_adjacent_vertixes ~post_visit ~cycle_visit; 342 | List.rev !stack 343 | 344 | 345 | (* NOTE: for some reason, ocaml complains about fully qualified polymorphic 346 | * variants in recursive modules, so instead of relying on OCaml, we need to 347 | * preorder variants ourselves without relying on OCaml to figure out the order 348 | * automatically *) 349 | let order_variants context l = 350 | (* topologically sort local variant defintions *) 351 | let cycle_visit def = 352 | C.error ("cyclic OCaml variant definition: " ^ typedef_name def) 353 | in 354 | let get_adjacent_vertixes = function 355 | | `variant v -> 356 | (* get the list of included variants *) 357 | U.flatmap (fun o -> 358 | match o.O.typename with 359 | | Some typename when o.O.ocaml_name = None -> 360 | let import, parent_piqi, typedef = C.resolve_typename context typename in 361 | (match typedef with 362 | | ((`variant _) as typedef) 363 | | ((`enum _) as typedef) -> 364 | if import <> None (* imported? *) 365 | then [] (* omit any imported definitions *) 366 | else [typedef] 367 | | _ -> [] 368 | ) 369 | | _ -> [] 370 | ) v.V.option 371 | | _ -> [] 372 | in 373 | tsort l get_adjacent_vertixes ~cycle_visit 374 | 375 | 376 | (* make sure we define aliases for built-in ocaml types first; some aliases 377 | * (e.g. float) can override the default OCaml type names which results in 378 | * cyclic type definitions without such ordering *) 379 | let order_aliases l = 380 | let rank def = 381 | match def with 382 | | `alias x -> 383 | if C.is_builtin_alias x 384 | then 385 | (* aliases of built-in OCaml types go first *) 386 | if x.A.ocaml_type <> None then 1 else 2 387 | else 100 388 | | _ -> 389 | assert false 390 | in 391 | let compare_alias a b = 392 | rank a - rank b 393 | in 394 | List.stable_sort compare_alias l 395 | 396 | 397 | let order_typedefs context typedefs = 398 | (* we apply this specific ordering only to variants, to be more specific -- 399 | * only to those variants that include other variants by not specifying tags 400 | * for the options *) 401 | let variants, rest = 402 | List.partition (function 403 | | `variant _ | `enum _ -> true 404 | | _ -> false) 405 | typedefs 406 | in 407 | let aliases, rest = 408 | List.partition (function 409 | | `alias _ -> true 410 | | _ -> false) 411 | rest 412 | in 413 | (* return the updated list of definitions with sorted variants and aliases *) 414 | (order_aliases aliases) @ (order_variants context variants) @ rest 415 | 416 | 417 | let gen_piqi context = 418 | let piqi = context.piqi in 419 | let typedefs = order_typedefs context piqi.P.typedef in 420 | iol [ 421 | gen_imports context piqi.P.import; 422 | gen_typedefs context typedefs; 423 | eol; eol 424 | ] 425 | 426 | -------------------------------------------------------------------------------- /piqirun/.gitignore: -------------------------------------------------------------------------------- 1 | /META 2 | -------------------------------------------------------------------------------- /piqirun/META.in: -------------------------------------------------------------------------------- 1 | description = "Runtime support for stubs generated by'piqic-ocaml'" 2 | 3 | package "pb" ( 4 | description = "Piqi runtime library for Protocol Buffers serialization" 5 | archive(byte) = "piqirun.cmo" 6 | archive(native) = "piqirun.cmx" 7 | requires = "bytes stdlib-shims" 8 | ) 9 | 10 | package "ext" ( 11 | description = "Piqi runtime library for multi-format JSON/XML/Protobuf/Piq serialization" 12 | requires = "piqilib piqirun.pb" 13 | archive(byte) = "piqirun_ext.cmo" 14 | archive(native) = "piqirun_ext.cmx" 15 | ) 16 | -------------------------------------------------------------------------------- /piqirun/Makefile: -------------------------------------------------------------------------------- 1 | OCAMLMAKEFILE := ../make/OCamlMakefile 2 | 3 | 4 | RESULT = piqirun 5 | 6 | 7 | # piqirun_ext.ml depends on it 8 | PACKS = piqilib stdlib-shims 9 | 10 | 11 | SOURCES = \ 12 | piqirun.ml \ 13 | piqirun_ext.ml piqirun_ext.mli 14 | 15 | LIBINSTALL_FILES = \ 16 | piqirun.cmi piqirun.cmo piqirun.cmx piqirun.o \ 17 | piqirun_ext.mli piqirun_ext.cmi piqirun_ext.cmo piqirun_ext.cmx piqirun_ext.o \ 18 | piqirun.ml # TODO: piqirun.mli 19 | 20 | 21 | PRE_TARGETS = META 22 | 23 | 24 | all: bcl ncl 25 | 26 | 27 | debug: dcl top 28 | 29 | 30 | # NOTE: when installing, uninstall first to avoid "already installed" error 31 | install: uninstall libinstall 32 | 33 | 34 | uninstall: libuninstall 35 | 36 | 37 | META: ../VERSION META.in 38 | echo "version = \"`head -1 $<`\"" >$@ 39 | cat META.in >>$@ 40 | 41 | 42 | test: bcl 43 | ocaml piqirun.cmo test.ml 44 | 45 | 46 | # these commands are useful for debugging: 47 | #ocamlc -c -g piqirun.cmo test.ml 48 | #ocaml -init test.ocaml piqirun.cmo test.cmo 49 | # 50 | #ocamlc -g piqirun.cmo test.ml 51 | #OCAMLRUNPARAM=b ./a.out 52 | 53 | 54 | clean:: 55 | rm -f test.cm? a.out 56 | 57 | 58 | include $(OCAMLMAKEFILE) 59 | -------------------------------------------------------------------------------- /piqirun/dune: -------------------------------------------------------------------------------- 1 | (library 2 | (name piqirun) 3 | (public_name piqirun.pb) 4 | (wrapped false) 5 | (libraries piqilib stdlib-shims) 6 | (modules piqirun)) 7 | 8 | (library 9 | (name piqirun_ext) 10 | (public_name piqirun.ext) 11 | (wrapped false) 12 | (libraries piqilib stdlib-shims) 13 | (modules piqirun_ext)) 14 | -------------------------------------------------------------------------------- /piqirun/piqirun_ext.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright 2009, 2010, 2011, 2012, 2013, 2014, 2015 Anton Lavrik 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | *) 16 | 17 | (* Runtime support for JSON-XML-Protobuf-Piq serialization 18 | * 19 | * This module is used by OCaml modules generated by 20 | * "piqic-ocaml --multi-format" Piqi compiler 21 | *) 22 | 23 | 24 | type input_format = [ `piq | `json | `xml | `pb | `pib ] 25 | 26 | type output_format = [ input_format | `json_pretty | `xml_pretty ] 27 | 28 | type piqi_type = Piqi_common.T.piqtype 29 | 30 | type options = Piqi_convert.options 31 | 32 | 33 | let _ = 34 | Piqi_convert.init () 35 | 36 | 37 | let add_piqi (piqi_bin: string) = 38 | let buf = Piqi_piqirun.init_from_string piqi_bin in 39 | let piqi = Piqi.piqi_of_pb buf in 40 | Piqi_db.add_piqi piqi; 41 | () 42 | 43 | 44 | let seen = ref [] 45 | 46 | let init_piqi piqi = 47 | if not (List.memq piqi !seen) 48 | then ( 49 | seen:= piqi :: !seen; 50 | add_piqi piqi 51 | ) 52 | 53 | 54 | let find_piqi_type (typename :string) :piqi_type = 55 | Piqi_convert.find_type typename 56 | 57 | 58 | (* preallocate default convert options *) 59 | let default_options = Piqi_convert.make_options () 60 | 61 | let default_options_no_pp = 62 | { 63 | default_options with 64 | Piqi_convert.pretty_print = false 65 | } 66 | 67 | 68 | let make_options = Piqi_convert.make_options 69 | 70 | 71 | let convert 72 | ?opts 73 | (piqi_type :piqi_type) 74 | (input_format :input_format) 75 | (output_format :output_format) 76 | (data :string) :string = 77 | if output_format = (input_format :> output_format) 78 | then data 79 | else ( 80 | let output_format, default_opts = 81 | match output_format with 82 | | `json_pretty -> `json, default_options 83 | | `xml_pretty -> `xml, default_options 84 | | (#input_format as x) -> x, default_options_no_pp 85 | in 86 | let opts = 87 | match opts with 88 | | None -> default_opts 89 | | Some x -> x 90 | in 91 | Piqi_convert.convert piqi_type input_format output_format data ~opts 92 | ) 93 | 94 | -------------------------------------------------------------------------------- /piqirun/piqirun_ext.mli: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright 2009, 2010, 2011, 2012, 2013, 2014, 2015 Anton Lavrik 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | *) 16 | 17 | (* Runtime support for JSON-XML-Protobuf-Piq serialization 18 | * 19 | * This module is used by OCaml modules generated by 20 | * "piqic-ocaml --multi-format" Piqi compiler 21 | *) 22 | 23 | 24 | type input_format = [ `json | `pb | `piq | `pib | `xml ] 25 | 26 | type output_format = [ input_format | `json_pretty | `xml_pretty ] 27 | 28 | type piqi_type 29 | 30 | type options 31 | 32 | 33 | val init_piqi : string -> unit 34 | 35 | val find_piqi_type : string -> piqi_type 36 | 37 | 38 | (* Construct serialization options to be passed as an optional argument to 39 | * gen_ and parse_ functions. Available options: 40 | * 41 | * pretty_print 42 | * 43 | * Pretty-print generated JSON and XML output (default = true) 44 | * 45 | * json_omit_missing_fields 46 | * 47 | * Omit missing optional and empty repeated fields from JSON 48 | * output instead of representing them as {"field_name": null} and 49 | * {"field_name", []} JSON fields (default = true) 50 | * 51 | * use_strict_parsing 52 | * 53 | * Treat unknown and duplicate fields as errors when parsing JSON, 54 | * XML and Piq formats (default = false) 55 | * 56 | * piq_frameless_output 57 | * 58 | * Print a frame (i.e. : []) around a single output Piq object 59 | * (default=false) 60 | * 61 | * piq_frameless_input 62 | * 63 | * Expect a frame around a single input Piq object (default=false) 64 | * 65 | * piq_relaxed_parsing 66 | * 67 | * Parse Piq format using "relaxed" mode (default=false); 68 | * 69 | * For instance, when set to `true`, single-word string literals don't have 70 | * to be quoted 71 | *) 72 | val make_options: 73 | ?pretty_print:bool -> 74 | ?json_omit_missing_fields:bool -> 75 | ?json_omit_null_fields:bool -> (* deprecated: use json_omit_missing_fields instead *) 76 | ?use_strict_parsing:bool -> 77 | ?piq_frameless_output:bool -> 78 | ?piq_frameless_input:bool -> 79 | ?piq_relaxed_parsing:bool -> 80 | unit -> options 81 | 82 | val convert: 83 | ?opts:options -> 84 | piqi_type -> input_format -> output_format -> string -> string 85 | 86 | -------------------------------------------------------------------------------- /piqirun/test.ml: -------------------------------------------------------------------------------- 1 | ../tests/piqirun/test.ml -------------------------------------------------------------------------------- /piqirun/test.ocaml: -------------------------------------------------------------------------------- 1 | open Test 2 | (* 3 | #trace test_int;; 4 | #trace test_int32;; 5 | #trace test_int64;; 6 | #trace Piqirun.int64_of_varint;; 7 | #trace test_parse_varint;; 8 | #trace Piqirun.next_varint_byte;; 9 | *) 10 | #trace Piqirun.int_to_zigzag_varint;; 11 | #trace Piqirun.parse_varint;; 12 | #trace Piqirun.int_of_varint;; 13 | #trace test_zigzag_int;; 14 | #trace test_zigzag_int32;; 15 | #trace test_zigzag_int64;; 16 | #trace test_parse_zigzag_varint;; 17 | #trace Piqirun.int_of_zigzag_varint;; 18 | #trace Piqirun.zigzag_varint_of_varint;; 19 | test ();; 20 | -------------------------------------------------------------------------------- /tests/Makefile: -------------------------------------------------------------------------------- 1 | include ../make/Makefile.dirs 2 | 3 | 4 | DIRS = \ 5 | piqirun \ 6 | addressbook \ 7 | piqi \ 8 | perf \ 9 | packed \ 10 | array \ 11 | misc \ 12 | misc1 \ 13 | custom-types \ 14 | piq-config \ 15 | 16 | ifneq ($(shell which protoc), ) 17 | DIRS += riak_pb 18 | endif 19 | -------------------------------------------------------------------------------- /tests/addressbook: -------------------------------------------------------------------------------- 1 | ../examples/addressbook -------------------------------------------------------------------------------- /tests/array/Makefile: -------------------------------------------------------------------------------- 1 | PIQI ?= piqi 2 | 3 | 4 | all: 5 | $(MAKE) -f Makefile.ocaml 6 | $(PIQI) convert -t pb test-all.piq 7 | ./test 8 | cmp test-all.piq.pb test-all.piq.pb.array 9 | 10 | 11 | clean: 12 | $(MAKE) -f Makefile.ocaml clean 13 | rm -f test-all.piq.* test-all-lists.piq.* 14 | -------------------------------------------------------------------------------- /tests/array/Makefile.ocaml: -------------------------------------------------------------------------------- 1 | OCAMLMAKEFILE := ../../make/OCamlMakefile 2 | 3 | 4 | RESULT = test 5 | 6 | 7 | SOURCES = $(PIQI_ML_FILES) test.ml 8 | 9 | 10 | PIQI_FILES = packed.piqi 11 | 12 | PIQI_ML_FILES = \ 13 | packed_piqi.ml \ 14 | 15 | 16 | PRE_TARGETS = $(PIQI_ML_FILES) 17 | 18 | 19 | PIQIC = ../../piqic-ocaml/piqic-ocaml 20 | #PIQIC_FLAGS = 21 | 22 | 23 | export OCAMLPATH := ../..:$(OCAMLPATH) 24 | PACKS = piqirun.pb 25 | 26 | 27 | all: nc #top 28 | 29 | 30 | $(PIQI_ML_FILES): $(PIQI_FILES) 31 | set -e; \ 32 | for i in $^; do \ 33 | $(PIQIC) $(PIQIC_FLAGS) $$i ; \ 34 | done 35 | 36 | 37 | 38 | include $(OCAMLMAKEFILE) 39 | -------------------------------------------------------------------------------- /tests/array/packed.piqi: -------------------------------------------------------------------------------- 1 | .record [ 2 | .name r-int 3 | .field [ 4 | .type i 5 | .repeated 6 | .ocaml-array 7 | ] 8 | ] 9 | 10 | .enum [ 11 | .name e 12 | .option [ 13 | .name a 14 | .code -1 15 | 16 | ] 17 | .option [ 18 | .name b 19 | .code 2 20 | ] 21 | .option [ 22 | .name c 23 | .code 0 24 | ] 25 | .option [ 26 | .name d 27 | .code 100_000_000 28 | ] 29 | ] 30 | 31 | .record [ 32 | .name r-enum 33 | .field [ 34 | .type e 35 | .repeated 36 | .ocaml-array 37 | ] 38 | ] 39 | 40 | .record [ 41 | .name r-bool 42 | .field [ 43 | .type b 44 | .repeated 45 | .ocaml-array 46 | ] 47 | ] 48 | 49 | .record [ 50 | .name r-all 51 | .field [ 52 | .name bool 53 | .type bool 54 | .repeated 55 | .ocaml-array 56 | ] 57 | .field [ 58 | .name u32 59 | .type uint32 60 | .repeated 61 | .ocaml-array 62 | ] 63 | .field [ 64 | .name u64 65 | .type uint64 66 | .repeated 67 | .ocaml-array 68 | ] 69 | .field [ 70 | .name s32 71 | .type int32 72 | .repeated 73 | .ocaml-array 74 | ] 75 | .field [ 76 | .name s64 77 | .type int64 78 | .repeated 79 | .ocaml-array 80 | ] 81 | .field [ 82 | .name i32 83 | .type protobuf-int32 84 | .repeated 85 | .ocaml-array 86 | ] 87 | .field [ 88 | .name i64 89 | .type protobuf-int64 90 | .repeated 91 | .ocaml-array 92 | ] 93 | .field [ 94 | .name uf32 95 | .type uint32-fixed 96 | .repeated 97 | .ocaml-array 98 | ] 99 | .field [ 100 | .name uf64 101 | .type uint64-fixed 102 | .repeated 103 | .ocaml-array 104 | ] 105 | .field [ 106 | .name if32 107 | .type int32-fixed 108 | .repeated 109 | .ocaml-array 110 | ] 111 | .field [ 112 | .name if64 113 | .type int64-fixed 114 | .repeated 115 | .ocaml-array 116 | ] 117 | .field [ 118 | .name f32 119 | .type float32 120 | .repeated 121 | .ocaml-array 122 | ] 123 | .field [ 124 | .name f64 125 | .type float64 126 | .repeated 127 | .ocaml-array 128 | ] 129 | 130 | 131 | .field [ 132 | .name e 133 | .type e 134 | .repeated 135 | .ocaml-array 136 | ] 137 | .field [ 138 | .name b 139 | .type b 140 | .repeated 141 | .ocaml-array 142 | ] 143 | .field [ 144 | .name i 145 | .type i 146 | .repeated 147 | .ocaml-array 148 | ] 149 | 150 | .field [ 151 | .name lists 152 | .type r-all-lists 153 | ] 154 | ] 155 | 156 | .alias [ 157 | .name i 158 | .type int 159 | ] 160 | 161 | .alias [ 162 | .name b 163 | .type bool 164 | ] 165 | 166 | .list [ 167 | .name bool-list 168 | .type bool 169 | .ocaml-array 170 | ] 171 | 172 | .list [ 173 | .name int-list 174 | .type int 175 | .ocaml-array 176 | ] 177 | 178 | .list [ 179 | .name e-list 180 | .type e 181 | .ocaml-array 182 | ] 183 | 184 | .list [ 185 | .name i-list 186 | .type i 187 | .ocaml-array 188 | ] 189 | 190 | .list [ 191 | .name b-list 192 | .type b 193 | .ocaml-array 194 | ] 195 | 196 | 197 | .list [ 198 | .name u32-list 199 | .type uint32 200 | .ocaml-array 201 | ] 202 | .list [ 203 | .name u64-list 204 | .type uint64 205 | .ocaml-array 206 | ] 207 | .list [ 208 | .name s32-list 209 | .type int32 210 | .ocaml-array 211 | ] 212 | .list [ 213 | .name s64-list 214 | .type int64 215 | .ocaml-array 216 | ] 217 | .list [ 218 | .name i32-list 219 | .type protobuf-int32 220 | .ocaml-array 221 | ] 222 | .list [ 223 | .name i64-list 224 | .type protobuf-int64 225 | .ocaml-array 226 | ] 227 | 228 | .list [ 229 | .name uf32-list 230 | .type uint32-fixed 231 | .ocaml-array 232 | ] 233 | .list [ 234 | .name uf64-list 235 | .type uint64-fixed 236 | .ocaml-array 237 | ] 238 | .list [ 239 | .name if32-list 240 | .type int32-fixed 241 | .ocaml-array 242 | ] 243 | .list [ 244 | .name if64-list 245 | .type int64-fixed 246 | .ocaml-array 247 | ] 248 | .list [ 249 | .name f32-list 250 | .type float32 251 | .ocaml-array 252 | ] 253 | .list [ 254 | .name f64-list 255 | .type float64 256 | .ocaml-array 257 | ] 258 | 259 | 260 | .record [ 261 | .name r-all-lists 262 | .field [ 263 | .name bool 264 | .type bool-list 265 | ] 266 | .field [ 267 | .name u32 268 | .type u32-list 269 | ] 270 | .field [ 271 | .name u64 272 | .type u64-list 273 | ] 274 | .field [ 275 | .name s32 276 | .type s32-list 277 | ] 278 | .field [ 279 | .name s64 280 | .type s64-list 281 | ] 282 | .field [ 283 | .name i32 284 | .type i32-list 285 | ] 286 | .field [ 287 | .name i64 288 | .type i64-list 289 | ] 290 | .field [ 291 | .name uf32 292 | .type uf32-list 293 | ] 294 | .field [ 295 | .name uf64 296 | .type uf64-list 297 | ] 298 | .field [ 299 | .name if32 300 | .type if32-list 301 | ] 302 | .field [ 303 | .name if64 304 | .type if64-list 305 | ] 306 | .field [ 307 | .name f32 308 | .type f32-list 309 | ] 310 | .field [ 311 | .name f64 312 | .type f64-list 313 | ] 314 | 315 | 316 | .field [ 317 | .name e 318 | .type e-list 319 | ] 320 | .field [ 321 | .name b 322 | .type b-list 323 | ] 324 | .field [ 325 | .name i 326 | .type i-list 327 | ] 328 | ] 329 | 330 | 331 | .custom-field ocaml-array 332 | 333 | -------------------------------------------------------------------------------- /tests/array/test-all.piq: -------------------------------------------------------------------------------- 1 | ../packed/test-all.piq -------------------------------------------------------------------------------- /tests/array/test.ml: -------------------------------------------------------------------------------- 1 | 2 | let t () = 3 | print_endline "testing Piqi repeated fields and Piqi lists represented as OCaml arrays"; 4 | let ich = open_in_bin "test-all.piq.pb" in 5 | let buf = Piqirun.init_from_channel ich in 6 | let piqi = Packed_piqi.parse_r_all buf in 7 | 8 | let och = open_out_bin "test-all.piq.pb.array" in 9 | let data = Packed_piqi.gen_r_all piqi in 10 | Piqirun.to_channel och data; 11 | 12 | close_in ich; 13 | close_out och; 14 | () 15 | 16 | 17 | let _ = t () 18 | -------------------------------------------------------------------------------- /tests/custom-types: -------------------------------------------------------------------------------- 1 | ../examples/custom-types -------------------------------------------------------------------------------- /tests/misc/Ad.piqi: -------------------------------------------------------------------------------- 1 | .ocaml-module "Ad" 2 | .custom-field ocaml-module 3 | 4 | .alias [ 5 | .name id 6 | .type int64 7 | ] 8 | 9 | -------------------------------------------------------------------------------- /tests/misc/Makefile: -------------------------------------------------------------------------------- 1 | OCAMLMAKEFILE := ../../make/OCamlMakefile 2 | 3 | 4 | RESULT = test 5 | 6 | 7 | SOURCES = $(PIQI_ML_FILES) 8 | 9 | 10 | PIQI_FILES = \ 11 | empty-record.piqi \ 12 | \ 13 | Ad.piqi P.piqi Protocol.piqi \ 14 | \ 15 | variant-1.piqi variant-2.piqi 16 | 17 | 18 | PIQI_ML_FILES = \ 19 | empty_record_piqi.ml \ 20 | \ 21 | ad.ml p.ml protocol.ml \ 22 | \ 23 | variant_1_piqi.ml variant_2_piqi.ml 24 | 25 | 26 | PRE_TARGETS = $(PIQI_ML_FILES) 27 | 28 | 29 | PIQIC = ../../piqic-ocaml/piqic-ocaml 30 | #PIQIC_FLAGS = 31 | 32 | 33 | export OCAMLPATH := ../..:$(OCAMLPATH) 34 | PACKS = piqirun.pb 35 | 36 | 37 | all: nc #top 38 | 39 | 40 | $(PIQI_ML_FILES): $(PIQI_FILES) 41 | set -e; \ 42 | for i in $^; do \ 43 | $(PIQIC) $(PIQIC_FLAGS) $$i ; \ 44 | done 45 | 46 | 47 | 48 | include $(OCAMLMAKEFILE) 49 | -------------------------------------------------------------------------------- /tests/misc/P.piqi: -------------------------------------------------------------------------------- 1 | .ocaml-module "P" 2 | (.custom-field ocaml-module ocaml-name) 3 | 4 | .import [ 5 | .module Ad 6 | .ocaml-name "Common_ad" 7 | ] 8 | 9 | .variant [ 10 | .name reply 11 | .option [ 12 | .name page 13 | .type Ad/id 14 | ] 15 | ] 16 | 17 | .variant [ 18 | .name voidvarianttosolvepiqibug 19 | .option [ 20 | .name bad 21 | .type int 22 | ] 23 | ] 24 | 25 | -------------------------------------------------------------------------------- /tests/misc/Protocol.piqi: -------------------------------------------------------------------------------- 1 | .ocaml-module "Protocol" 2 | .custom-field ocaml-module 3 | 4 | .import [ .module Ad ] 5 | 6 | .variant [ 7 | .name op 8 | .option [ .name ping ] 9 | .option [ 10 | .name get-balance 11 | .type Ad/id 12 | ] 13 | ] 14 | 15 | -------------------------------------------------------------------------------- /tests/misc/empty-record.piqi: -------------------------------------------------------------------------------- 1 | .record [ 2 | .name r 3 | ] 4 | -------------------------------------------------------------------------------- /tests/misc/variant-1.piqi: -------------------------------------------------------------------------------- 1 | .enum [ 2 | .name e 3 | .option [ a ] 4 | ] 5 | -------------------------------------------------------------------------------- /tests/misc/variant-2.piqi: -------------------------------------------------------------------------------- 1 | 2 | .import [ variant-1 ] 3 | 4 | 5 | .variant [ 6 | .name v 7 | 8 | % testing automatic implicit reordering of variants when generating OCaml 9 | % types (type "e" is defined below, but in OCaml types it must appear above) 10 | .option [ 11 | .type v1 12 | ] 13 | 14 | % testing correct OCaml representation of imported sub-variant 15 | .option [ 16 | .type variant-1/e 17 | ] 18 | ] 19 | 20 | 21 | .variant [ 22 | .name v1 23 | .option [ 24 | .type ee 25 | ] 26 | ] 27 | 28 | 29 | .enum [ 30 | .name ee 31 | .option [ b ] 32 | ] 33 | 34 | 35 | % NOTE: without specifying ".ocaml_name l", these two mutually cyclic 36 | % definitions should trigger a piqic error (because OCaml won't compile code 37 | % containing cycles like that, even if types are finite) 38 | .variant [ 39 | .name loop1 40 | .option [ 41 | .ocaml-name "l" 42 | .type loop2 43 | ] 44 | ] 45 | 46 | .variant [ 47 | .name loop2 48 | .option [ .type loop1 ] 49 | .option [ .name foo ] 50 | ] 51 | 52 | 53 | .custom-field ocaml-name 54 | -------------------------------------------------------------------------------- /tests/misc1/Makefile: -------------------------------------------------------------------------------- 1 | OCAMLMAKEFILE := ../../make/OCamlMakefile 2 | 3 | 4 | RESULT = t 5 | SOURCES = id_piqi.ml id_piqi_ext.ml example_piqi.ml example_piqi_ext.ml test.ml 6 | 7 | 8 | PRE_TARGETS = id_piqi.ml id_piqi_ext.ml example_piqi.ml example_piqi_ext.ml 9 | 10 | 11 | export OCAMLPATH := ../..:$(OCAMLPATH) 12 | PACKS = piqirun.ext 13 | 14 | 15 | PIQI ?= piqi 16 | PIQIC = ../../piqic-ocaml/piqic-ocaml 17 | PIQIC_FLAGS = --ext 18 | 19 | 20 | all: native-code test #byte-code 21 | 22 | 23 | test: 24 | ./$(RESULT) 25 | 26 | 27 | %_piqi.ml: %.proto $(realpath $(PIQI)) $(PIQIC) 28 | ifneq ($(shell which protoc), ) 29 | $(PIQI) of-proto $< 30 | endif 31 | $(PIQIC) $(PIQIC_FLAGS) $<.piqi 32 | 33 | 34 | clean:: 35 | rm -f *.tmp.ml 36 | 37 | 38 | include $(OCAMLMAKEFILE) 39 | -------------------------------------------------------------------------------- /tests/misc1/example.proto: -------------------------------------------------------------------------------- 1 | package test; 2 | import "id.proto"; 3 | 4 | message example 5 | { 6 | required Id id = 1; 7 | } 8 | -------------------------------------------------------------------------------- /tests/misc1/example.proto.piqi: -------------------------------------------------------------------------------- 1 | .protobuf-package "test" 2 | 3 | .import [ .module id ] 4 | 5 | .record [ 6 | .name example 7 | .field [ 8 | .name id 9 | .type id/Id 10 | .code 1 11 | ] 12 | ] 13 | -------------------------------------------------------------------------------- /tests/misc1/id.proto: -------------------------------------------------------------------------------- 1 | package test; 2 | message Id 3 | { 4 | required int64 id = 1; // Unique ID 5 | } 6 | -------------------------------------------------------------------------------- /tests/misc1/id.proto.piqi: -------------------------------------------------------------------------------- 1 | .protobuf-package "test" 2 | 3 | .record [ 4 | .name Id 5 | .field [ 6 | .name id 7 | .type protobuf-int64 8 | .code 1 9 | ] 10 | ] 11 | -------------------------------------------------------------------------------- /tests/misc1/test.ml: -------------------------------------------------------------------------------- 1 | let x = Example_piqi.default_example () in 2 | Example_piqi_ext.prerr_example x 3 | -------------------------------------------------------------------------------- /tests/packed/Makefile: -------------------------------------------------------------------------------- 1 | PIQI ?= piqi 2 | 3 | 4 | all: 5 | cat packed.piqi | sed -e 's/\.protobuf-packed//' > unpacked.piqi 6 | cat test-all.piq | sed -e 's/:packed/:unpacked/' > test-all-unpacked.piq 7 | 8 | $(MAKE) -f Makefile.ocaml 9 | 10 | $(PIQI) convert -t pb test-all.piq 11 | ./test 12 | cmp test-all.piq.pb test-all.piq.pb.packed 13 | cmp test-all.piq.pb test-all.piq.pb.packed-array 14 | 15 | cp test-all.piq.pb test-all.piq.pb.orig 16 | $(PIQI) convert -t pb -o test-all.piq.pb test-all-unpacked.piq 17 | ./test 18 | cmp test-all.piq.pb.orig test-all.piq.pb.packed 19 | cmp test-all.piq.pb.orig test-all.piq.pb.packed-array 20 | 21 | 22 | clean: 23 | $(MAKE) -f Makefile.ocaml clean 24 | rm -f test-all.piq.* unpacked.piqi test-all-unpacked.piq 25 | -------------------------------------------------------------------------------- /tests/packed/Makefile.ocaml: -------------------------------------------------------------------------------- 1 | OCAMLMAKEFILE := ../../make/OCamlMakefile 2 | 3 | 4 | RESULT = test 5 | 6 | 7 | SOURCES = $(PIQI_ML_FILES) test_packed.ml test_packed_array.ml 8 | 9 | 10 | PIQI_FILES = packed-nocompat.piqi packed.piqi packed-array.piqi 11 | 12 | PIQI_ML_FILES = \ 13 | packed_nocompat_piqi.ml \ 14 | packed_piqi.ml \ 15 | packed_array_piqi.ml \ 16 | 17 | 18 | PRE_TARGETS = $(PIQI_ML_FILES) 19 | 20 | 21 | PIQIC = ../../piqic-ocaml/piqic-ocaml 22 | #PIQIC_FLAGS = 23 | 24 | 25 | export OCAMLPATH := ../..:$(OCAMLPATH) 26 | PACKS = piqirun.pb 27 | 28 | 29 | all: nc #top 30 | 31 | 32 | $(PIQI_ML_FILES): $(PIQI_FILES) 33 | set -e; \ 34 | for i in $^; do \ 35 | $(PIQIC) $(PIQIC_FLAGS) $$i ; \ 36 | done 37 | 38 | 39 | 40 | include $(OCAMLMAKEFILE) 41 | -------------------------------------------------------------------------------- /tests/packed/packed-array.piqi: -------------------------------------------------------------------------------- 1 | .record [ 2 | .name r-int 3 | .field [ 4 | .type i 5 | .repeated 6 | .protobuf-packed 7 | .ocaml-array 8 | ] 9 | ] 10 | 11 | .enum [ 12 | .name e 13 | .option [ 14 | .name a 15 | .code -1 16 | 17 | ] 18 | .option [ 19 | .name b 20 | .code 2 21 | ] 22 | .option [ 23 | .name c 24 | .code 0 25 | ] 26 | .option [ 27 | .name d 28 | .code 100_000_000 29 | ] 30 | ] 31 | 32 | .record [ 33 | .name r-enum 34 | .field [ 35 | .type e 36 | .repeated 37 | .protobuf-packed 38 | .ocaml-array 39 | ] 40 | ] 41 | 42 | .record [ 43 | .name r-bool 44 | .field [ 45 | .type b 46 | .repeated 47 | .protobuf-packed 48 | .ocaml-array 49 | ] 50 | ] 51 | 52 | .record [ 53 | .name r-all 54 | .field [ 55 | .name bool 56 | .type bool 57 | .repeated 58 | .protobuf-packed 59 | .ocaml-array 60 | ] 61 | .field [ 62 | .name u32 63 | .type uint32 64 | .repeated 65 | .protobuf-packed 66 | .ocaml-array 67 | ] 68 | .field [ 69 | .name u64 70 | .type uint64 71 | .repeated 72 | .protobuf-packed 73 | .ocaml-array 74 | ] 75 | .field [ 76 | .name s32 77 | .type int32 78 | .repeated 79 | .protobuf-packed 80 | .ocaml-array 81 | ] 82 | .field [ 83 | .name s64 84 | .type int64 85 | .repeated 86 | .protobuf-packed 87 | .ocaml-array 88 | ] 89 | .field [ 90 | .name i32 91 | .type protobuf-int32 92 | .repeated 93 | .protobuf-packed 94 | .ocaml-array 95 | ] 96 | .field [ 97 | .name i64 98 | .type protobuf-int64 99 | .repeated 100 | .protobuf-packed 101 | .ocaml-array 102 | ] 103 | .field [ 104 | .name uf32 105 | .type uint32-fixed 106 | .repeated 107 | .protobuf-packed 108 | .ocaml-array 109 | ] 110 | .field [ 111 | .name uf64 112 | .type uint64-fixed 113 | .repeated 114 | .protobuf-packed 115 | .ocaml-array 116 | ] 117 | .field [ 118 | .name if32 119 | .type int32-fixed 120 | .repeated 121 | .protobuf-packed 122 | .ocaml-array 123 | ] 124 | .field [ 125 | .name if64 126 | .type int64-fixed 127 | .repeated 128 | .protobuf-packed 129 | .ocaml-array 130 | ] 131 | 132 | .field [ 133 | .name f 134 | .type float 135 | .repeated 136 | .protobuf-packed 137 | .ocaml-array 138 | ] 139 | .field [ 140 | .name f32 141 | .type float32 142 | .repeated 143 | .protobuf-packed 144 | .ocaml-array 145 | ] 146 | .field [ 147 | .name f64 148 | .type float64 149 | .repeated 150 | .protobuf-packed 151 | .ocaml-array 152 | ] 153 | 154 | 155 | .field [ 156 | .name e 157 | .type e 158 | .repeated 159 | .protobuf-packed 160 | .ocaml-array 161 | ] 162 | .field [ 163 | .name b 164 | .type b 165 | .repeated 166 | .protobuf-packed 167 | .ocaml-array 168 | ] 169 | .field [ 170 | .name i 171 | .type i 172 | .repeated 173 | .protobuf-packed 174 | .ocaml-array 175 | ] 176 | 177 | .field [ 178 | .name lists 179 | .type r-all-lists 180 | ] 181 | ] 182 | 183 | .alias [ 184 | .name i 185 | .type int 186 | ] 187 | 188 | .alias [ 189 | .name b 190 | .type bool 191 | ] 192 | 193 | .list [ 194 | .name bool-list 195 | .type bool 196 | .protobuf-packed 197 | .ocaml-array 198 | ] 199 | 200 | .list [ 201 | .name int-list 202 | .type int 203 | .protobuf-packed 204 | .ocaml-array 205 | ] 206 | 207 | .list [ 208 | .name e-list 209 | .type e 210 | .protobuf-packed 211 | .ocaml-array 212 | ] 213 | 214 | .list [ 215 | .name i-list 216 | .type i 217 | .protobuf-packed 218 | .ocaml-array 219 | ] 220 | 221 | .list [ 222 | .name b-list 223 | .type b 224 | .protobuf-packed 225 | .ocaml-array 226 | ] 227 | 228 | 229 | .list [ 230 | .name u32-list 231 | .type uint32 232 | .protobuf-packed 233 | .ocaml-array 234 | ] 235 | .list [ 236 | .name u64-list 237 | .type uint64 238 | .protobuf-packed 239 | .ocaml-array 240 | ] 241 | .list [ 242 | .name s32-list 243 | .type int32 244 | .protobuf-packed 245 | .ocaml-array 246 | ] 247 | .list [ 248 | .name s64-list 249 | .type int64 250 | .protobuf-packed 251 | .ocaml-array 252 | ] 253 | .list [ 254 | .name i32-list 255 | .type protobuf-int32 256 | .protobuf-packed 257 | .ocaml-array 258 | ] 259 | .list [ 260 | .name i64-list 261 | .type protobuf-int64 262 | .protobuf-packed 263 | .ocaml-array 264 | ] 265 | 266 | .list [ 267 | .name uf32-list 268 | .type uint32-fixed 269 | .protobuf-packed 270 | .ocaml-array 271 | ] 272 | .list [ 273 | .name uf64-list 274 | .type uint64-fixed 275 | .protobuf-packed 276 | .ocaml-array 277 | ] 278 | .list [ 279 | .name if32-list 280 | .type int32-fixed 281 | .protobuf-packed 282 | .ocaml-array 283 | ] 284 | .list [ 285 | .name if64-list 286 | .type int64-fixed 287 | .protobuf-packed 288 | .ocaml-array 289 | ] 290 | .list [ 291 | .name f32-list 292 | .type float32 293 | .protobuf-packed 294 | .ocaml-array 295 | ] 296 | .list [ 297 | .name f64-list 298 | .type float64 299 | .protobuf-packed 300 | .ocaml-array 301 | ] 302 | 303 | 304 | .record [ 305 | .name r-all-lists 306 | .field [ 307 | .name bool 308 | .type bool-list 309 | ] 310 | .field [ 311 | .name u32 312 | .type u32-list 313 | ] 314 | .field [ 315 | .name u64 316 | .type u64-list 317 | ] 318 | .field [ 319 | .name s32 320 | .type s32-list 321 | ] 322 | .field [ 323 | .name s64 324 | .type s64-list 325 | ] 326 | .field [ 327 | .name i32 328 | .type i32-list 329 | ] 330 | .field [ 331 | .name i64 332 | .type i64-list 333 | ] 334 | .field [ 335 | .name uf32 336 | .type uf32-list 337 | ] 338 | .field [ 339 | .name uf64 340 | .type uf64-list 341 | ] 342 | .field [ 343 | .name if32 344 | .type if32-list 345 | ] 346 | .field [ 347 | .name if64 348 | .type if64-list 349 | ] 350 | .field [ 351 | .name f32 352 | .type f32-list 353 | ] 354 | .field [ 355 | .name f64 356 | .type f64-list 357 | ] 358 | 359 | 360 | .field [ 361 | .name e 362 | .type e-list 363 | ] 364 | .field [ 365 | .name b 366 | .type b-list 367 | ] 368 | .field [ 369 | .name i 370 | .type i-list 371 | ] 372 | ] 373 | 374 | 375 | .custom-field ocaml-array 376 | -------------------------------------------------------------------------------- /tests/packed/packed-nocompat.piqi: -------------------------------------------------------------------------------- 1 | % This module contains "ee" alias which is not compatible with Protocol Buffers. 2 | % 3 | % When mapped to Protobuf, another "ee" enum definition is generated with the 4 | % same enum elements, which is not allowed in Protobuf. 5 | 6 | .include [ .module packed ] 7 | 8 | .alias [ 9 | .name ee 10 | .type e 11 | ] 12 | 13 | 14 | .extend [ 15 | .typedef r-all 16 | 17 | .with.field [ 18 | .name ee 19 | .type ee 20 | .repeated 21 | .protobuf-packed 22 | ] 23 | ] 24 | 25 | 26 | .list [ 27 | .name ee-list 28 | .type ee 29 | .protobuf-packed 30 | ] 31 | 32 | -------------------------------------------------------------------------------- /tests/packed/packed.piqi: -------------------------------------------------------------------------------- 1 | .record [ 2 | .name r-int 3 | .field [ 4 | .type i 5 | .repeated 6 | .protobuf-packed 7 | ] 8 | ] 9 | 10 | .enum [ 11 | .name e 12 | .option [ 13 | .name a 14 | .code -1 15 | 16 | ] 17 | .option [ 18 | .name b 19 | .code 2 20 | ] 21 | .option [ 22 | .name c 23 | .code 0 24 | ] 25 | .option [ 26 | .name d 27 | .code 100_000_000 28 | ] 29 | ] 30 | 31 | .record [ 32 | .name r-enum 33 | .field [ 34 | .type e 35 | .repeated 36 | .protobuf-packed 37 | ] 38 | ] 39 | 40 | .record [ 41 | .name r-bool 42 | .field [ 43 | .type b 44 | .repeated 45 | .protobuf-packed 46 | ] 47 | ] 48 | 49 | .record [ 50 | .name r-all 51 | .field [ 52 | .name bool 53 | .type bool 54 | .repeated 55 | .protobuf-packed 56 | ] 57 | .field [ 58 | .name u32 59 | .type uint32 60 | .repeated 61 | .protobuf-packed 62 | ] 63 | .field [ 64 | .name u64 65 | .type uint64 66 | .repeated 67 | .protobuf-packed 68 | ] 69 | .field [ 70 | .name s32 71 | .type int32 72 | .repeated 73 | .protobuf-packed 74 | ] 75 | .field [ 76 | .name s64 77 | .type int64 78 | .repeated 79 | .protobuf-packed 80 | ] 81 | .field [ 82 | .name i32 83 | .type protobuf-int32 84 | .repeated 85 | .protobuf-packed 86 | ] 87 | .field [ 88 | .name i64 89 | .type protobuf-int64 90 | .repeated 91 | .protobuf-packed 92 | ] 93 | .field [ 94 | .name uf32 95 | .type uint32-fixed 96 | .repeated 97 | .protobuf-packed 98 | ] 99 | .field [ 100 | .name uf64 101 | .type uint64-fixed 102 | .repeated 103 | .protobuf-packed 104 | ] 105 | .field [ 106 | .name if32 107 | .type int32-fixed 108 | .repeated 109 | .protobuf-packed 110 | ] 111 | .field [ 112 | .name if64 113 | .type int64-fixed 114 | .repeated 115 | .protobuf-packed 116 | ] 117 | 118 | .field [ 119 | .name f 120 | .type float 121 | .repeated 122 | .protobuf-packed 123 | ] 124 | .field [ 125 | .name f32 126 | .type float32 127 | .repeated 128 | .protobuf-packed 129 | ] 130 | .field [ 131 | .name f64 132 | .type float64 133 | .repeated 134 | .protobuf-packed 135 | ] 136 | 137 | 138 | .field [ 139 | .name e 140 | .type e 141 | .repeated 142 | .protobuf-packed 143 | ] 144 | .field [ 145 | .name b 146 | .type b 147 | .repeated 148 | .protobuf-packed 149 | ] 150 | .field [ 151 | .name i 152 | .type i 153 | .repeated 154 | .protobuf-packed 155 | ] 156 | 157 | .field [ 158 | .name lists 159 | .type r-all-lists 160 | ] 161 | ] 162 | 163 | .alias [ 164 | .name i 165 | .type int 166 | ] 167 | 168 | .alias [ 169 | .name b 170 | .type bool 171 | ] 172 | 173 | .list [ 174 | .name bool-list 175 | .type bool 176 | .protobuf-packed 177 | ] 178 | 179 | .list [ 180 | .name int-list 181 | .type int 182 | .protobuf-packed 183 | ] 184 | 185 | .list [ 186 | .name e-list 187 | .type e 188 | .protobuf-packed 189 | ] 190 | 191 | .list [ 192 | .name i-list 193 | .type i 194 | .protobuf-packed 195 | ] 196 | 197 | .list [ 198 | .name b-list 199 | .type b 200 | .protobuf-packed 201 | ] 202 | 203 | 204 | .list [ 205 | .name u32-list 206 | .type uint32 207 | .protobuf-packed 208 | ] 209 | .list [ 210 | .name u64-list 211 | .type uint64 212 | .protobuf-packed 213 | ] 214 | .list [ 215 | .name s32-list 216 | .type int32 217 | .protobuf-packed 218 | ] 219 | .list [ 220 | .name s64-list 221 | .type int64 222 | .protobuf-packed 223 | ] 224 | .list [ 225 | .name i32-list 226 | .type protobuf-int32 227 | .protobuf-packed 228 | ] 229 | .list [ 230 | .name i64-list 231 | .type protobuf-int64 232 | .protobuf-packed 233 | ] 234 | 235 | .list [ 236 | .name uf32-list 237 | .type uint32-fixed 238 | .protobuf-packed 239 | ] 240 | .list [ 241 | .name uf64-list 242 | .type uint64-fixed 243 | .protobuf-packed 244 | ] 245 | .list [ 246 | .name if32-list 247 | .type int32-fixed 248 | .protobuf-packed 249 | ] 250 | .list [ 251 | .name if64-list 252 | .type int64-fixed 253 | .protobuf-packed 254 | ] 255 | .list [ 256 | .name f32-list 257 | .type float32 258 | .protobuf-packed 259 | ] 260 | .list [ 261 | .name f64-list 262 | .type float64 263 | .protobuf-packed 264 | ] 265 | 266 | 267 | .record [ 268 | .name r-all-lists 269 | .field [ 270 | .name bool 271 | .type bool-list 272 | ] 273 | .field [ 274 | .name u32 275 | .type u32-list 276 | ] 277 | .field [ 278 | .name u64 279 | .type u64-list 280 | ] 281 | .field [ 282 | .name s32 283 | .type s32-list 284 | ] 285 | .field [ 286 | .name s64 287 | .type s64-list 288 | ] 289 | .field [ 290 | .name i32 291 | .type i32-list 292 | ] 293 | .field [ 294 | .name i64 295 | .type i64-list 296 | ] 297 | .field [ 298 | .name uf32 299 | .type uf32-list 300 | ] 301 | .field [ 302 | .name uf64 303 | .type uf64-list 304 | ] 305 | .field [ 306 | .name if32 307 | .type if32-list 308 | ] 309 | .field [ 310 | .name if64 311 | .type if64-list 312 | ] 313 | .field [ 314 | .name f32 315 | .type f32-list 316 | ] 317 | .field [ 318 | .name f64 319 | .type f64-list 320 | ] 321 | 322 | 323 | .field [ 324 | .name e 325 | .type e-list 326 | ] 327 | .field [ 328 | .name b 329 | .type b-list 330 | ] 331 | .field [ 332 | .name i 333 | .type i-list 334 | ] 335 | ] 336 | -------------------------------------------------------------------------------- /tests/packed/test-all.piq: -------------------------------------------------------------------------------- 1 | :packed/r-all [ 2 | 3 | (.bool true false true true false false true) 4 | 5 | 6 | (.u32 0 1 100 0xffff_ffff) 7 | 8 | (.u64 0 1 100 0xffff_ffff 0xffff_ffff_ffff_ffff 18446744073709551615) 9 | 10 | 11 | (.s32 -1 0 1 100 0x7fff_ffff -0x8000_0000) 12 | 13 | (.s64 -1 0 1 100 0x7fff_ffff -0x8000_0000 14 | 0x7fff_ffff_ffff_ffff -0x8000_0000_0000_0000 15 | ) 16 | 17 | 18 | (.i32 -1 0 1 100 0x7fff_ffff -0x8000_0000) 19 | 20 | (.i64 -1 0 1 100 0x7fff_ffff -0x8000_0000 21 | 0x7fff_ffff_ffff_ffff -0x8000_0000_0000_0000 22 | ) 23 | 24 | 25 | (.uf32 0 1 100 0xffff_ffff) 26 | 27 | (.uf64 0 1 100 0xffff_ffff 0xffff_ffff_ffff_ffff 18446744073709551615) 28 | 29 | 30 | (.if32 -1 0 1 100 0x7fff_ffff -0x8000_0000) 31 | 32 | (.if64 -1 0 1 100 0x7fff_ffff -0x8000_0000 33 | 0x7fff_ffff_ffff_ffff -0x8000_0000_0000_0000 34 | ) 35 | 36 | 37 | % TODO: uncomment infinities and NaN when Piqi/Erlang support is added 38 | (.f32 -1.999999999 0.0 1.001 4.44444 123456.123456 -123456.123456) % -0.inf 0.inf 0.nan) 39 | 40 | (.f64 -1.999999999 0.0 1.001 4.44444 123456.123456 -123456.123456) % -0.inf 0.inf 0.nan) 41 | 42 | 43 | (.e .a .b .c .d .d .c .b .a .d .d .c .c .b .b .a .a) 44 | 45 | (.b true false true true false false true) 46 | 47 | (.i -100_000_000 -1 0 2 100_000_000 1 2 3 4 5 -1000 48 | -1 0 1 100 0x3fff_ffff -0x4000_0000 49 | ) 50 | 51 | .lists [ 52 | .bool [true false true true false false true] 53 | 54 | 55 | .u32 [0 1 100 0xffff_ffff] 56 | 57 | .u64 [0 1 100 0xffff_ffff 0xffff_ffff_ffff_ffff] 58 | 59 | 60 | .s32 [-1 0 1 100 0x7fff_ffff -0x8000_0000] 61 | 62 | .s64 [-1 0 1 100 0x7fff_ffff -0x8000_0000 63 | 0x7fff_ffff_ffff_ffff -0x8000_0000_0000_0000 64 | ] 65 | 66 | 67 | .i32 [-1 0 1 100 0x7fff_ffff -0x8000_0000] 68 | 69 | .i64 [-1 0 1 100 0x7fff_ffff -0x8000_0000 70 | 0x7fff_ffff_ffff_ffff -0x8000_0000_0000_0000 71 | ] 72 | 73 | 74 | .uf32 [0 1 100 0xffff_ffff] 75 | 76 | .uf64 [0 1 100 0xffff_ffff 0xffff_ffff_ffff_ffff] 77 | 78 | 79 | .if32 [-1 0 1 100 0x7fff_ffff -0x8000_0000] 80 | 81 | .if64 [-1 0 1 100 0x7fff_ffff -0x8000_0000 82 | 0x7fff_ffff_ffff_ffff -0x8000_0000_0000_0000 83 | ] 84 | 85 | 86 | % TODO: uncomment infinities and NaN when Piqi/Erlang support is added 87 | .f32 [-1.999999999 0.0 1.001 4.44444 123456.123456 -123456.123456] % -0.inf 0.inf 0.nan] 88 | 89 | .f64 [-1.999999999 0.0 1.001 4.44444 123456.123456 -123456.123456] % -0.inf 0.inf 0.nan] 90 | 91 | 92 | .e [.a .b .c .d .d .c .b .a .d .d .c .c .b .b .a .a] 93 | 94 | .b [true false true true false false true] 95 | 96 | .i [-100_000_000 -1 0 2 100_000_000 1 2 3 4 5 -1000 97 | -1 0 1 100 0x3fff_ffff -0x4000_0000 98 | ] 99 | ] 100 | ] 101 | -------------------------------------------------------------------------------- /tests/packed/test_packed.ml: -------------------------------------------------------------------------------- 1 | 2 | let t () = 3 | print_endline "testing packed repeated fields"; 4 | let ich = open_in_bin "test-all.piq.pb" in 5 | let buf = Piqirun.init_from_channel ich in 6 | let piqi = Packed_piqi.parse_r_all buf in 7 | 8 | let och = open_out_bin "test-all.piq.pb.packed" in 9 | let data = Packed_piqi.gen_r_all piqi in 10 | Piqirun.to_channel och data; 11 | 12 | close_in ich; 13 | close_out och; 14 | () 15 | 16 | 17 | let _ = t () 18 | -------------------------------------------------------------------------------- /tests/packed/test_packed_array.ml: -------------------------------------------------------------------------------- 1 | 2 | let t () = 3 | print_endline "testing packed repeated fields represented as OCaml arrays"; 4 | let ich = open_in_bin "test-all.piq.pb" in 5 | let buf = Piqirun.init_from_channel ich in 6 | let piqi = Packed_array_piqi.parse_r_all buf in 7 | 8 | let och = open_out_bin "test-all.piq.pb.packed-array" in 9 | let data = Packed_array_piqi.gen_r_all piqi in 10 | Piqirun.to_channel och data; 11 | 12 | close_in ich; 13 | close_out och; 14 | () 15 | 16 | 17 | let _ = t () 18 | -------------------------------------------------------------------------------- /tests/perf/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all ocaml test clean 2 | 3 | 4 | PIQI ?= piqi 5 | 6 | 7 | all: ocaml test 8 | 9 | 10 | ocaml: piqi-obj.piqi 11 | $(MAKE) -f Makefile.ocaml 12 | 13 | 14 | piqi-obj.piqi: 15 | $(PIQI) cc | sed -e 's/\.module .*//;s/\.code .*//' > piqi.piqi 16 | ln -sf ../../piqic-ocaml/piqi.ocaml.piqi 17 | $(PIQI) expand -e ocaml piqi.piqi > $@ 18 | 19 | 20 | test: ocaml 21 | $(PIQI) convert -t pb addressbook.piq 22 | 23 | echo ":piqi-obj/piqi [" > piqi.piq 24 | cat piqi-obj.piqi >> piqi.piq 25 | echo "]" >> piqi.piq 26 | 27 | $(PIQI) convert --no-warnings --add-defaults -t pb piqi.piq 28 | #./test 29 | 30 | 31 | clean: 32 | $(MAKE) -f Makefile.ocaml clean 33 | rm -f addressbook.piq.pb piqi-expanded.piqi piqi-obj.* piqi.ocaml.piqi piqi.piq piqi.piq.pb piqi.piqi 34 | 35 | -------------------------------------------------------------------------------- /tests/perf/Makefile.ocaml: -------------------------------------------------------------------------------- 1 | OCAMLMAKEFILE := ../../make/OCamlMakefile 2 | 3 | 4 | RESULT = test 5 | 6 | 7 | SOURCES = \ 8 | $(PIQI_ML_FILES) \ 9 | test.ml 10 | 11 | 12 | export OCAMLPATH := ../..:$(OCAMLPATH) 13 | PACKS = unix piqirun.ext 14 | 15 | 16 | PIQI_FILES = addressbook.proto.piqi piqi.piqi piqi-obj.piqi 17 | 18 | PIQI_ML_FILES = \ 19 | addressbook_piqi.ml addressbook_piqi_ext.ml \ 20 | piqi_piqi.ml piqi_piqi_ext.ml \ 21 | piqi_obj_piqi.ml piqi_obj_piqi_ext.ml 22 | 23 | 24 | PRE_TARGETS = $(PIQI_ML_FILES) 25 | 26 | 27 | PIQIC = ../../piqic-ocaml/piqic-ocaml 28 | PIQIC_FLAGS = --multi-format --no-warnings 29 | 30 | 31 | all: native-code #byte-code debug-code 32 | 33 | 34 | $(PIQI_ML_FILES): $(PIQI_FILES) 35 | set -e; \ 36 | for i in $^; do \ 37 | $(PIQIC) $(PIQIC_FLAGS) $$i ; \ 38 | done 39 | 40 | 41 | clean:: 42 | rm -f *.tmp.ml 43 | 44 | 45 | include $(OCAMLMAKEFILE) 46 | -------------------------------------------------------------------------------- /tests/perf/README: -------------------------------------------------------------------------------- 1 | Performance tests for serialization and de-serialization of OCaml data 2 | structures to/from Protocol Buffers, JSON, XML and Piq data formats. 3 | 4 | The tests are performed on two different objects. The first one is a relatively 5 | small and not very complicated "addressbook.piq". The second one is the expanded 6 | Piqi self-specification which is 10 times as large as the first one and has more 7 | complex structure. 8 | 9 | To run the tests: 10 | 11 | make 12 | 13 | ./test 14 | 15 | -------------------------------------------------------------------------------- /tests/perf/addressbook.ocaml.piqi: -------------------------------------------------------------------------------- 1 | ../addressbook/addressbook.ocaml.piqi -------------------------------------------------------------------------------- /tests/perf/addressbook.piq: -------------------------------------------------------------------------------- 1 | ../addressbook/addressbook.piq -------------------------------------------------------------------------------- /tests/perf/addressbook.proto.piqi: -------------------------------------------------------------------------------- 1 | ../addressbook/addressbook.proto.piqi -------------------------------------------------------------------------------- /tests/perf/test.ml: -------------------------------------------------------------------------------- 1 | 2 | let read_file filename = 3 | let ch = open_in_bin filename in 4 | let size = in_channel_length ch in 5 | let res = Bytes.create size in 6 | really_input ch res 0 size; 7 | close_in ch; 8 | Bytes.unsafe_to_string res 9 | 10 | 11 | let string_of_format = function 12 | | `pb -> "pb" 13 | | `json -> "json" 14 | | `json_pretty -> "json_pretty" 15 | | `xml -> "xml" 16 | | `xml_pretty -> "xml_pretty" 17 | | `piq -> "piq" 18 | | `pib -> "pib" 19 | 20 | 21 | let printf = Printf.printf 22 | 23 | 24 | let test f n = 25 | printf "count: %d\n%!" n; 26 | 27 | let t1 = Unix.gettimeofday () in 28 | 29 | for i = 1 to n 30 | do 31 | f (); 32 | (* 33 | Gc.minor (); 34 | *) 35 | Gc.minor (); 36 | done; 37 | 38 | let t2 = Unix.gettimeofday () in 39 | let seconds = t2 -. t1 in 40 | 41 | let per_second = (float_of_int n) /. seconds in 42 | let per_second = truncate per_second in 43 | 44 | printf "time: %f seconds\n%!" seconds; 45 | printf "rate: %d calls per second\n\n%!" per_second; 46 | 47 | per_second 48 | 49 | 50 | let test_convert codec format input n = 51 | let f = fun () -> codec input format in 52 | test f n 53 | 54 | 55 | let test_rw reader writer (format: Piqirun_ext.output_format) bytes n = 56 | printf "size of Protobuf binary: %d\n%!" (String.length bytes); 57 | 58 | (* read the object into OCaml term representation *) 59 | let output = reader bytes `pb in 60 | 61 | (* write the object into desired test input format *) 62 | let input = writer output format in 63 | (* 64 | printf "input: %s\n" input; 65 | *) 66 | 67 | let input_format = 68 | match format with 69 | | `json_pretty -> `json 70 | | `xml_pretty -> `xml 71 | | #Piqirun_ext.input_format as x -> x 72 | in 73 | 74 | printf "reading %s objects...\n%!" (string_of_format format); 75 | let i_rate = test_convert reader input_format input n in 76 | 77 | printf "writing %s objects...\n%!" (string_of_format format); 78 | let o_rate = test_convert writer format output n in 79 | 80 | printf "%s read/write rate: %d/%d\n\n%!" (string_of_format format) i_rate o_rate; 81 | 82 | print_newline (); 83 | 84 | () 85 | 86 | 87 | let test_rw_all reader writer bytes n = 88 | let formats = [`pb; `json; `json_pretty; `xml; `xml_pretty; `piq; `pib;] in 89 | List.iter (fun format -> test_rw reader writer format bytes n) formats 90 | 91 | 92 | let test_addressbook () = 93 | printf "*** testing OCaml serialization of medium objects ***\n\n"; 94 | 95 | let filename = "addressbook.piq.pb" in 96 | 97 | (* Read the addressbook encoded in Protobuf format *) 98 | let bytes = read_file filename in 99 | 100 | let reader = Addressbook_piqi_ext.parse_address_book in 101 | let writer = Addressbook_piqi_ext.gen_address_book in 102 | 103 | let n = 100000 in 104 | 105 | test_rw_all reader writer bytes n; 106 | (* 107 | test_rw_all reader writer bytes n; 108 | 109 | test_rw reader writer `pb bytes n; 110 | test_rw reader writer `json bytes n; 111 | test_rw reader writer `json_pretty bytes n; 112 | test_rw reader writer `xml bytes n; 113 | test_rw reader writer `xml_pretty bytes n; 114 | test_rw reader writer `pib bytes n; 115 | test_rw reader writer `piq bytes n; 116 | *) 117 | () 118 | 119 | 120 | let test_piqi () = 121 | printf "*** testing OCaml serialization of big objects ***\n\n"; 122 | 123 | let filename = "piqi.piq.pb" in 124 | 125 | (* Read the Piqi self-specification encoded in Protobuf format *) 126 | let bytes = read_file filename in 127 | 128 | let reader = Piqi_obj_piqi_ext.parse_piqi in 129 | let writer = Piqi_obj_piqi_ext.gen_piqi in 130 | 131 | let n = 20000 in 132 | 133 | test_rw_all reader writer bytes n; 134 | (* 135 | test_rw_all reader writer bytes n; 136 | 137 | test_rw reader writer `pb bytes n; 138 | test_rw reader writer `json bytes n; 139 | test_rw reader writer `json_pretty bytes n; 140 | test_rw reader writer `xml bytes n; 141 | test_rw reader writer `xml_pretty bytes n; 142 | test_rw reader writer `pib bytes n; 143 | test_rw reader writer `piq bytes n; 144 | *) 145 | () 146 | 147 | 148 | let set_gc_options () = 149 | (* Don't set custom options if the OCAMLRUNPARAM environment variable is 150 | * defined *) 151 | try ignore (Sys.getenv "OCAMLRUNPARAM") 152 | with Not_found -> 153 | let opt = Gc.get () in 154 | opt.Gc.minor_heap_size <- 4 * 1024 * 1024; (* Minor heap size: 4m *) 155 | opt.Gc.space_overhead <- 200; (* run major GC less frequently, but waste more RAM *) 156 | Gc.set opt 157 | 158 | 159 | let _ = 160 | (* 161 | Gc.compact (); 162 | Gc.print_stat stdout; 163 | print_newline (); 164 | *) 165 | set_gc_options (); 166 | 167 | test_addressbook (); 168 | test_piqi (); 169 | () 170 | 171 | -------------------------------------------------------------------------------- /tests/piq-config: -------------------------------------------------------------------------------- 1 | ../examples/piq-config -------------------------------------------------------------------------------- /tests/piqi/Makefile: -------------------------------------------------------------------------------- 1 | PIQI ?= piqi 2 | 3 | 4 | all: prep test 5 | 6 | 7 | prep: 8 | $(PIQI) cc > piqi.piqi 9 | $(PIQI) convert --add-defaults -t pb piqi.piqi 10 | 11 | 12 | test: 13 | $(MAKE) -f Makefile.ocaml 14 | ./otest 15 | cmp *.pb 16 | 17 | 18 | clean: 19 | $(MAKE) -f Makefile.ocaml clean 20 | rm -f piqi.piqi piqi.piqi.pb* t.* 21 | 22 | -------------------------------------------------------------------------------- /tests/piqi/Makefile.ocaml: -------------------------------------------------------------------------------- 1 | OCAMLMAKEFILE := ../../make/OCamlMakefile 2 | 3 | 4 | RESULT = otest 5 | 6 | 7 | SOURCES = $(PIQI_ML_SOURCES) test.ml 8 | PIQI_ML_SOURCES = piqi_piqi.ml piqobj_piqi.ml 9 | 10 | 11 | PRE_TARGETS = $(PIQI_ML_SOURCES) 12 | 13 | 14 | PIQIC = ../../piqic-ocaml/piqic-ocaml 15 | #PIQIC_FLAGS = 16 | 17 | 18 | export OCAMLPATH := ../..:$(OCAMLPATH) 19 | PACKS = piqirun.pb 20 | 21 | 22 | all: bc #top 23 | 24 | 25 | $(PIQI_ML_SOURCES): *.piqi 26 | $(PIQIC) $(PIQIC_FLAGS) piqi.piqi 27 | $(PIQIC) $(PIQIC_FLAGS) piqobj.piqi 28 | 29 | 30 | include $(OCAMLMAKEFILE) 31 | -------------------------------------------------------------------------------- /tests/piqi/README: -------------------------------------------------------------------------------- 1 | 2 | Compile complex piqi specification "piqi/piqi.piqi" to produce OCaml 3 | parsers and generators. 4 | 5 | Then build a program which reads and writes back a serialized binary object 6 | which happens to be a serialized "piqi.piqi" specification. 7 | 8 | Also testing piqi imports in OCaml by compiling and linking "piqobj.piqi" which 9 | imports "piqi.piqi". 10 | 11 | -------------------------------------------------------------------------------- /tests/piqi/piqi.ocaml.piqi: -------------------------------------------------------------------------------- 1 | ../../piqic-ocaml/piqi.ocaml.piqi -------------------------------------------------------------------------------- /tests/piqi/piqobj.piqi: -------------------------------------------------------------------------------- 1 | % Internal representation of Piq data 2 | % 3 | % Copyright 2009, 2010, 2011, 2012 Anton Lavrik 4 | 5 | 6 | .import [ .module piqi ] 7 | 8 | 9 | .variant [ 10 | .name typedef 11 | 12 | .option [ .type record ] 13 | .option [ .type variant ] 14 | .option [ .type enum ] 15 | .option [ .type list ] 16 | .option [ .type alias ] 17 | ] 18 | 19 | 20 | .variant [ 21 | .name obj 22 | 23 | .option [ .type typedef ] 24 | 25 | % built-in objects 26 | .option [ 27 | .name int 28 | .type int64 29 | ] 30 | 31 | .option [ 32 | .name uint 33 | .type uint64 34 | ] 35 | 36 | .option [ .type float ] 37 | .option [ .type bool ] 38 | .option [ .type string ] 39 | .option [ .type binary ] 40 | .option [ .type any ] 41 | ] 42 | 43 | 44 | .record [ 45 | .name record 46 | .field [ 47 | .name t 48 | .type piqi/record 49 | ] 50 | .field [ 51 | .type field 52 | .repeated 53 | ] 54 | ] 55 | 56 | 57 | .record [ 58 | .name field 59 | .field [ 60 | .name t 61 | .type piqi/field 62 | ] 63 | .field [ 64 | .type obj 65 | .optional 66 | ] 67 | ] 68 | 69 | 70 | .record [ 71 | .name variant 72 | .field [ 73 | .name t 74 | .type piqi/variant 75 | ] 76 | .field [ .type option ] 77 | ] 78 | 79 | 80 | .record [ 81 | .name option 82 | .field [ 83 | .name t 84 | .type piqi/option 85 | ] 86 | .field [ 87 | .type obj 88 | .optional 89 | ] 90 | ] 91 | 92 | 93 | .record [ 94 | .name list 95 | .field [ 96 | .name t 97 | .type piqi/list 98 | ] 99 | .field [ 100 | .type obj 101 | .repeated 102 | ] 103 | ] 104 | 105 | 106 | .record [ 107 | .name alias 108 | .field [ 109 | .name t 110 | .type piqi/alias 111 | ] 112 | .field [ .type obj ] 113 | ] 114 | 115 | 116 | .record [ 117 | .name any 118 | .field [ 119 | .name t 120 | .type piqi/alias 121 | ] 122 | .field [ .type obj .optional ] 123 | ] 124 | 125 | 126 | .alias [ 127 | .name enum 128 | .type variant 129 | ] 130 | 131 | 132 | .custom-field ocaml-name 133 | -------------------------------------------------------------------------------- /tests/piqi/test.ml: -------------------------------------------------------------------------------- 1 | 2 | let t () = 3 | let ich = open_in_bin "piqi.piqi.pb" in 4 | let buf = Piqirun.init_from_channel ich in 5 | let piqi = Piqi_piqi.parse_piqi buf in 6 | 7 | let och = open_out_bin "piqi.piqi.pb.pb" in 8 | let data = Piqi_piqi.gen_piqi piqi in 9 | Piqirun.to_channel och data; 10 | 11 | close_in ich; 12 | close_out och; 13 | () 14 | 15 | 16 | let _ = t () 17 | -------------------------------------------------------------------------------- /tests/piqirun/Makefile: -------------------------------------------------------------------------------- 1 | OCAMLMAKEFILE := ../../make/OCamlMakefile 2 | 3 | 4 | RESULT = test 5 | 6 | SOURCES = test.ml 7 | 8 | 9 | export OCAMLPATH := ../..:$(OCAMLPATH) 10 | PACKS = piqirun.pb 11 | 12 | 13 | all: bc 14 | ./test 15 | 16 | 17 | 18 | include $(OCAMLMAKEFILE) 19 | -------------------------------------------------------------------------------- /tests/piqirun/test.ml: -------------------------------------------------------------------------------- 1 | (* 2 | Copyright 2009, 2010, 2011, 2012, 2013 Anton Lavrik 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | *) 16 | 17 | open Piqirun 18 | 19 | 20 | (* superposition operator *) 21 | let ( ** ) f g x = f (g x) 22 | 23 | 24 | let assert_eq a b = assert (a = b) 25 | 26 | 27 | let test_key x = 28 | let test_parse x = 29 | (IBuf.of_string ** OBuf.to_string) x 30 | in 31 | assert (x = ((fun x -> let _, code = parse_field_header x in code) ** test_parse ** gen_key 0) x) 32 | 33 | 34 | let test_gen_parse x = 35 | (init_from_string ** to_string) x 36 | 37 | 38 | (* 39 | * Names of the test_* functions correspond to Piqi types: there are 12 40 | * different integer types 41 | *) 42 | 43 | let test_int x = 44 | assert (x = (int_of_zigzag_varint ** test_gen_parse ** int_to_zigzag_varint (-1)) x) 45 | 46 | let test_int32 x = 47 | assert (x = (int32_of_zigzag_varint ** test_gen_parse ** int32_to_zigzag_varint (-1)) x) 48 | 49 | let test_int64 x = 50 | assert (x = (int64_of_zigzag_varint ** test_gen_parse ** int64_to_zigzag_varint (-1)) x) 51 | 52 | 53 | let test_fixed_int32 x = 54 | assert (x = (int32_of_signed_fixed32 ** test_gen_parse ** int32_to_signed_fixed32 (-1)) x) 55 | 56 | let test_fixed_int64 x = 57 | assert (x = (int64_of_signed_fixed64 ** test_gen_parse ** int64_to_signed_fixed64 (-1)) x) 58 | 59 | 60 | let test_proto_int x = (* NOTE: this type is not currently defined in Piqi *) 61 | assert (x = (int_of_signed_varint ** test_gen_parse ** int_to_signed_varint (-1)) x) 62 | 63 | let test_proto_int32 x = 64 | assert (x = (int32_of_signed_varint ** test_gen_parse ** int32_to_signed_varint (-1)) x) 65 | 66 | let test_proto_int64 x = 67 | assert (x = (int64_of_signed_varint ** test_gen_parse ** int64_to_signed_varint (-1)) x) 68 | 69 | 70 | let test_uint x = 71 | assert (x = (int_of_varint ** test_gen_parse ** int_to_varint (-1)) x) 72 | 73 | let test_uint32 x = 74 | assert (x = (int32_of_varint ** test_gen_parse ** int32_to_varint (-1)) x) 75 | 76 | let test_uint64 x = 77 | assert (x = (int64_of_varint ** test_gen_parse ** int64_to_varint (-1)) x) 78 | 79 | 80 | let test_fixed_uint32 x = 81 | assert (x = (int32_of_fixed32 ** test_gen_parse ** int32_to_fixed32 (-1)) x) 82 | 83 | let test_fixed_uint64 x = 84 | assert (x = (int64_of_fixed64 ** test_gen_parse ** int64_to_fixed64 (-1)) x) 85 | 86 | 87 | (* 88 | * Testing packed 89 | *) 90 | 91 | let test_packed_gen_parse x = 92 | (IBuf.of_string ** OBuf.to_string) x 93 | 94 | 95 | (* 96 | * Names of the test_* functions correspond to Piqi types: there are 12 97 | * different integer types 98 | *) 99 | 100 | let test_packed_int x = 101 | assert (x = (int_of_packed_zigzag_varint ** test_packed_gen_parse ** int_to_packed_zigzag_varint) x) 102 | 103 | let test_packed_int32 x = 104 | assert (x = (int32_of_packed_zigzag_varint ** test_packed_gen_parse ** int32_to_packed_zigzag_varint) x) 105 | 106 | let test_packed_int64 x = 107 | assert (x = (int64_of_packed_zigzag_varint ** test_packed_gen_parse ** int64_to_packed_zigzag_varint) x) 108 | 109 | 110 | let test_packed_fixed_int32 x = 111 | assert (x = (int32_of_packed_signed_fixed32 ** test_packed_gen_parse ** int32_to_packed_signed_fixed32) x) 112 | 113 | let test_packed_fixed_int64 x = 114 | assert (x = (int64_of_packed_signed_fixed64 ** test_packed_gen_parse ** int64_to_packed_signed_fixed64) x) 115 | 116 | 117 | let test_packed_proto_int x = (* NOTE: this type is not currently defined in Piqi *) 118 | assert (x = (int_of_packed_signed_varint ** test_packed_gen_parse ** int_to_packed_signed_varint) x) 119 | 120 | let test_packed_proto_int32 x = 121 | assert (x = (int32_of_packed_signed_varint ** test_packed_gen_parse ** int32_to_packed_signed_varint) x) 122 | 123 | let test_packed_proto_int64 x = 124 | assert (x = (int64_of_packed_signed_varint ** test_packed_gen_parse ** int64_to_packed_signed_varint) x) 125 | 126 | 127 | let test_packed_uint x = 128 | assert (x = (int_of_packed_varint ** test_packed_gen_parse ** int_to_packed_varint) x) 129 | 130 | let test_packed_uint32 x = 131 | assert (x = (int32_of_packed_varint ** test_packed_gen_parse ** int32_to_packed_varint) x) 132 | 133 | let test_packed_uint64 x = 134 | assert (x = (int64_of_packed_varint ** test_packed_gen_parse ** int64_to_packed_varint) x) 135 | 136 | 137 | let test_packed_fixed_uint32 x = 138 | assert (x = (int32_of_packed_fixed32 ** test_packed_gen_parse ** int32_to_packed_fixed32) x) 139 | 140 | let test_packed_fixed_uint64 x = 141 | assert (x = (int64_of_packed_fixed64 ** test_packed_gen_parse ** int64_to_packed_fixed64) x) 142 | 143 | 144 | 145 | let int_input = 146 | [ 147 | 0; 1; 2; 3; -1; -2; -3; 148 | min_int; min_int + 1; min_int + 2; min_int + 3; 149 | max_int; max_int - 1; max_int - 2; max_int - 3; 150 | ] 151 | 152 | 153 | let uint_input = 154 | let max_uint = lnot 0 in 155 | [ 156 | 0; 1; 2; 3; 157 | max_uint; 158 | ] 159 | 160 | 161 | open Int32 162 | 163 | let int32_input = 164 | let int_intput = List.map (fun x -> of_int x) int_input in 165 | int_intput @ 166 | [ 167 | min_int; succ min_int; succ (succ min_int); succ (succ (succ min_int)); 168 | max_int; pred max_int; pred (pred max_int); pred (pred (pred max_int)); 169 | ] 170 | 171 | 172 | let uint32_input = 173 | let max_uint = lognot 0l in 174 | [ 175 | 0l; 1l; 2l; 3l; 176 | max_uint; 177 | ] 178 | 179 | 180 | open Int64 181 | 182 | let int64_input = 183 | let int_intput = List.map (fun x -> of_int x) int_input in 184 | let int32_intput = List.map (fun x -> of_int32 x) int32_input in 185 | int_intput @ 186 | int32_intput @ 187 | [ 188 | min_int; succ min_int; succ (succ min_int); succ (succ (succ min_int)); 189 | max_int; pred max_int; pred (pred max_int); pred (pred (pred max_int)); 190 | ] 191 | 192 | 193 | let uint64_input = 194 | let max_uint = lognot 0L in 195 | let uint32_intput = List.map (fun x -> int64_of_uint32 x) uint32_input in 196 | uint32_intput @ 197 | [ 198 | max_uint; 199 | ] 200 | 201 | 202 | let max_key = (1 lsl 29) - 1 203 | 204 | let key_input = [ 1; 2; 3; max_key - 1; max_key ] 205 | 206 | 207 | (* TODO: 208 | * tests for malformed/broken/unexpectedly terminated input 209 | * tests for OCaml's type overflows 210 | * tests for cross-type reading, e.g. int64 -> int32, varint -> int64, etc. 211 | * tests for bools, floats and other types 212 | * 213 | *) 214 | 215 | let test _ = 216 | List.iter test_key key_input; 217 | 218 | (* tests for integer fields *) 219 | 220 | List.iter test_int int_input; 221 | List.iter test_int32 int32_input; 222 | List.iter test_int64 int64_input; 223 | 224 | List.iter test_fixed_int32 int32_input; 225 | List.iter test_fixed_int64 int64_input; 226 | 227 | List.iter test_proto_int int_input; 228 | List.iter test_proto_int32 int32_input; 229 | List.iter test_proto_int64 int64_input; 230 | 231 | 232 | List.iter test_uint uint_input; 233 | List.iter test_uint32 uint32_input; 234 | List.iter test_uint64 uint64_input; 235 | 236 | List.iter test_fixed_uint32 uint32_input; 237 | List.iter test_fixed_uint64 uint64_input; 238 | 239 | (* tests for packed integers *) 240 | 241 | List.iter test_packed_int int_input; 242 | List.iter test_packed_int32 int32_input; 243 | List.iter test_packed_int64 int64_input; 244 | 245 | List.iter test_packed_fixed_int32 int32_input; 246 | List.iter test_packed_fixed_int64 int64_input; 247 | 248 | List.iter test_packed_proto_int int_input; 249 | List.iter test_packed_proto_int32 int32_input; 250 | List.iter test_packed_proto_int64 int64_input; 251 | 252 | 253 | List.iter test_packed_uint uint_input; 254 | List.iter test_packed_uint32 uint32_input; 255 | List.iter test_packed_uint64 uint64_input; 256 | 257 | List.iter test_packed_fixed_uint32 uint32_input; 258 | List.iter test_packed_fixed_uint64 uint64_input; 259 | 260 | () 261 | 262 | 263 | let _ = 264 | if !Sys.interactive 265 | then () 266 | else test () 267 | 268 | -------------------------------------------------------------------------------- /tests/riak_pb/Makefile: -------------------------------------------------------------------------------- 1 | OCAMLMAKEFILE := ../../make/OCamlMakefile 2 | 3 | 4 | RESULT = riak_pb 5 | 6 | 7 | SOURCES = \ 8 | $(PIQI_ML_FILES) 9 | 10 | 11 | PROTO_FILES = riak.proto riak_kv.proto riak_search.proto 12 | 13 | PIQI_FILES = $(PROTO_FILES:%=%.piqi) 14 | 15 | PIQI_ML_FILES = $(PROTO_FILES:%.proto=%_piqi.ml) 16 | 17 | 18 | export OCAMLPATH := ../..:$(OCAMLPATH) 19 | PACKS = piqirun.pb 20 | 21 | 22 | PIQI ?= piqi 23 | PIQIC = ../../piqic-ocaml/piqic-ocaml 24 | #PIQIC_FLAGS = 25 | 26 | 27 | PRE_TARGETS = $(PIQI_FILES) $(PIQI_ML_FILES) 28 | 29 | 30 | all: native-code-library #byte-code 31 | 32 | 33 | $(PIQI_ML_FILES): $(PIQI_FILES) 34 | set -e; \ 35 | for i in $^; do \ 36 | $(PIQIC) $(PIQIC_FLAGS) $$i ; \ 37 | done 38 | 39 | 40 | $(PIQI_FILES): $(PROTO_FILES) 41 | set -e; \ 42 | for i in $^; do \ 43 | $(PIQI) of-proto $$i ; \ 44 | done 45 | 46 | 47 | clean:: 48 | rm -f *.tmp.ml *.proto.piqi *.ml 49 | 50 | 51 | include $(OCAMLMAKEFILE) 52 | -------------------------------------------------------------------------------- /tests/riak_pb/README: -------------------------------------------------------------------------------- 1 | Riak Protocol Buffers definitions were downloaded from 2 | 3 | git@github.com:basho/riak_pb.git 4 | 5 | commit: ab277e0900887699aeafcd8c1d0495fc5b4e304e 6 | -------------------------------------------------------------------------------- /tests/riak_pb/riak.proto: -------------------------------------------------------------------------------- 1 | /* ------------------------------------------------------------------- 2 | ** 3 | ** riak.proto: Protocol buffers for Riak 4 | ** 5 | ** Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | ** 7 | ** This file is provided to you under the Apache License, 8 | ** Version 2.0 (the "License"); you may not use this file 9 | ** except in compliance with the License. You may obtain 10 | ** a copy of the License at 11 | ** 12 | ** http://www.apache.org/licenses/LICENSE-2.0 13 | ** 14 | ** Unless required by applicable law or agreed to in writing, 15 | ** software distributed under the License is distributed on an 16 | ** "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | ** KIND, either express or implied. See the License for the 18 | ** specific language governing permissions and limitations 19 | ** under the License. 20 | ** 21 | ** ------------------------------------------------------------------- 22 | */ 23 | 24 | /* 25 | ** Revision: 1.2 26 | */ 27 | 28 | // Error response - may be generated for any Req 29 | message RpbErrorResp { 30 | required bytes errmsg = 1; 31 | required uint32 errcode = 2; 32 | } 33 | 34 | // Get server info request - no message defined, just send RpbGetServerInfoReq message code 35 | message RpbGetServerInfoResp { 36 | optional bytes node = 1; 37 | optional bytes server_version = 2; 38 | } 39 | 40 | // Key/value pair - used for user metadata, indexes, search doc fields 41 | message RpbPair { 42 | required bytes key = 1; 43 | optional bytes value = 2; 44 | } 45 | -------------------------------------------------------------------------------- /tests/riak_pb/riak_kv.ocaml.piqi: -------------------------------------------------------------------------------- 1 | .include [ .module riak_kv ] 2 | 3 | .extend [ 4 | (.field RpbListKeysResp.done RpbMapRedResp.done) 5 | 6 | .with.ocaml-name "isdone" 7 | ] 8 | 9 | .custom-field ocaml-name 10 | -------------------------------------------------------------------------------- /tests/riak_pb/riak_kv.proto: -------------------------------------------------------------------------------- 1 | /* ------------------------------------------------------------------- 2 | ** 3 | ** riak_kv.proto: Protocol buffers for riak KV 4 | ** 5 | ** Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. 6 | ** 7 | ** This file is provided to you under the Apache License, 8 | ** Version 2.0 (the "License"); you may not use this file 9 | ** except in compliance with the License. You may obtain 10 | ** a copy of the License at 11 | ** 12 | ** http://www.apache.org/licenses/LICENSE-2.0 13 | ** 14 | ** Unless required by applicable law or agreed to in writing, 15 | ** software distributed under the License is distributed on an 16 | ** "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | ** KIND, either express or implied. See the License for the 18 | ** specific language governing permissions and limitations 19 | ** under the License. 20 | ** 21 | ** ------------------------------------------------------------------- 22 | */ 23 | 24 | /* 25 | ** Revision: 1.2 26 | */ 27 | 28 | import "riak.proto"; // for RpbPair 29 | 30 | // Get ClientId Request - no message defined, just send RpbGetClientIdReq message code 31 | message RpbGetClientIdResp { 32 | required bytes client_id = 1; // Client id in use for this connection 33 | } 34 | 35 | message RpbSetClientIdReq { 36 | required bytes client_id = 1; // Client id to use for this connection 37 | } 38 | // Set ClientId Request - no message defined, just send RpbSetClientIdReq message code 39 | 40 | 41 | // Get Request - retrieve bucket/key 42 | message RpbGetReq { 43 | required bytes bucket = 1; 44 | required bytes key = 2; 45 | optional uint32 r = 3; 46 | optional uint32 pr = 4; 47 | optional bool basic_quorum = 5; 48 | optional bool notfound_ok = 6; 49 | optional bytes if_modified = 7; // fail if the supplied vclock does not match 50 | optional bool head = 8; // return everything but the value 51 | optional bool deletedvclock = 9; // return the tombstone's vclock, if applicable 52 | } 53 | 54 | // Get Response - if the record was not found there will be no content/vclock 55 | message RpbGetResp { 56 | repeated RpbContent content = 1; 57 | optional bytes vclock = 2; // the opaque vector clock for the object 58 | optional bool unchanged = 3; 59 | } 60 | 61 | 62 | // Put request - if options.return_body is set then the updated metadata/data for 63 | // the key will be returned. 64 | message RpbPutReq { 65 | required bytes bucket = 1; 66 | optional bytes key = 2; 67 | optional bytes vclock = 3; 68 | required RpbContent content = 4; 69 | optional uint32 w = 5; 70 | optional uint32 dw = 6; 71 | optional bool return_body = 7; 72 | optional uint32 pw = 8; 73 | optional bool if_not_modified = 9; 74 | optional bool if_none_match = 10; 75 | optional bool return_head = 11; 76 | } 77 | 78 | // Put response - same as get response with optional key if one was generated 79 | message RpbPutResp { 80 | repeated RpbContent content = 1; 81 | optional bytes vclock = 2; // the opaque vector clock for the object 82 | optional bytes key = 3; // the key generated, if any 83 | } 84 | 85 | 86 | // Delete request 87 | message RpbDelReq { 88 | required bytes bucket = 1; 89 | required bytes key = 2; 90 | optional uint32 rw = 3; 91 | optional bytes vclock = 4; 92 | optional uint32 r = 5; 93 | optional uint32 w = 6; 94 | optional uint32 pr = 7; 95 | optional uint32 pw = 8; 96 | optional uint32 dw = 9; 97 | } 98 | 99 | // Delete response - not defined, will return a RpbDelResp on success or RpbErrorResp on failure 100 | 101 | // List buckets request - no message defined, just send RpbListBucketsReq 102 | 103 | // List buckets response 104 | message RpbListBucketsResp { 105 | repeated bytes buckets = 1; 106 | } 107 | 108 | 109 | // List keys in bucket request 110 | message RpbListKeysReq { 111 | required bytes bucket = 1; 112 | } 113 | 114 | // List keys in bucket response - one or more of these packets will be sent 115 | // the last one will have done set true (and may not have any keys in it) 116 | message RpbListKeysResp { 117 | repeated bytes keys = 1; 118 | optional bool done = 2; 119 | } 120 | 121 | // Get bucket properties request 122 | message RpbGetBucketReq { 123 | required bytes bucket = 1; 124 | } 125 | 126 | // Get bucket properties response 127 | message RpbGetBucketResp { 128 | required RpbBucketProps props = 1; 129 | } 130 | 131 | // Set bucket properties request 132 | message RpbSetBucketReq { 133 | required bytes bucket = 1; 134 | required RpbBucketProps props = 2; 135 | } 136 | 137 | 138 | // Set bucket properties response - no message defined, just send RpbSetBucketResp 139 | 140 | 141 | // Map/Reduce request 142 | message RpbMapRedReq { 143 | required bytes request = 1; 144 | required bytes content_type = 2; 145 | } 146 | 147 | // Map/Reduce response 148 | // one or more of these packets will be sent the last one will have done set 149 | // true (and may not have phase/data in it) 150 | message RpbMapRedResp { 151 | optional uint32 phase = 1; 152 | optional bytes response = 2; 153 | optional bool done = 3; 154 | } 155 | 156 | // Secondary Index query request 157 | message RpbIndexReq { 158 | enum IndexQueryType { 159 | eq = 0; 160 | range = 1; 161 | } 162 | 163 | required bytes bucket = 1; 164 | required bytes index = 2; 165 | required IndexQueryType qtype = 3; 166 | optional bytes key = 4; 167 | optional bytes range_min = 5; 168 | optional bytes range_max = 6; 169 | } 170 | 171 | // Secondary Index query response 172 | message RpbIndexResp { 173 | repeated bytes keys = 1; 174 | } 175 | 176 | // Content message included in get/put responses 177 | // Holds the value and associated metadata 178 | message RpbContent { 179 | required bytes value = 1; 180 | optional bytes content_type = 2; // the media type/format 181 | optional bytes charset = 3; 182 | optional bytes content_encoding = 4; 183 | optional bytes vtag = 5; 184 | repeated RpbLink links = 6; // links to other resources 185 | optional uint32 last_mod = 7; 186 | optional uint32 last_mod_usecs = 8; 187 | repeated RpbPair usermeta = 9; // user metadata stored with the object 188 | repeated RpbPair indexes = 10; // user metadata stored with the object 189 | optional bool deleted = 11; 190 | } 191 | 192 | // Link metadata 193 | message RpbLink { 194 | optional bytes bucket = 1; 195 | optional bytes key = 2; 196 | optional bytes tag = 3; 197 | } 198 | 199 | // Bucket properties 200 | message RpbBucketProps { 201 | optional uint32 n_val = 1; 202 | optional bool allow_mult = 2; 203 | } 204 | -------------------------------------------------------------------------------- /tests/riak_pb/riak_search.proto: -------------------------------------------------------------------------------- 1 | /* ------------------------------------------------------------------- 2 | ** 3 | ** riak_search.proto: Protocol buffers for Riak Search 4 | ** 5 | ** Copyright (c) 2012 Basho Technologies, Inc. All Rights Reserved. 6 | ** 7 | ** This file is provided to you under the Apache License, 8 | ** Version 2.0 (the "License"); you may not use this file 9 | ** except in compliance with the License. You may obtain 10 | ** a copy of the License at 11 | ** 12 | ** http://www.apache.org/licenses/LICENSE-2.0 13 | ** 14 | ** Unless required by applicable law or agreed to in writing, 15 | ** software distributed under the License is distributed on an 16 | ** "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | ** KIND, either express or implied. See the License for the 18 | ** specific language governing permissions and limitations 19 | ** under the License. 20 | ** 21 | ** ------------------------------------------------------------------- 22 | */ 23 | 24 | /* 25 | ** Revision: 1.2 26 | */ 27 | 28 | import "riak.proto"; 29 | 30 | message RpbSearchDoc { 31 | repeated RpbPair fields = 1; 32 | } 33 | 34 | message RpbSearchQueryReq { 35 | required bytes q = 1; // Query string 36 | required bytes index = 2; // Index 37 | optional uint32 rows = 3; // Limit rows 38 | optional uint32 start = 4; // Starting offset 39 | optional bytes sort = 5; // Sort order 40 | optional bytes filter = 6; // Inline fields filtering query 41 | optional bytes df = 7; // Default field 42 | optional bytes op = 8; // Default op 43 | repeated bytes fl = 9; // Return fields limit (for ids only, generally) 44 | optional bytes presort = 10; // Presort (key / score) 45 | } 46 | 47 | message RpbSearchQueryResp { 48 | repeated RpbSearchDoc docs = 1; // Result documents 49 | optional float max_score = 2; // Maximum score 50 | optional uint32 num_found = 3; // Number of results 51 | } 52 | --------------------------------------------------------------------------------