├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── DataRestrictions.md ├── LICENSE ├── README.md ├── data ├── dimension_pos.nt ├── earnings-periods.nt ├── earnings.nt ├── earnings_metadata.nt ├── geo-labels.nt ├── healthy-life-expectancy-periods.nt ├── healthy_life_expectancy.nt ├── healthy_life_expectancy_metadata.nt ├── labels.nt ├── measure_properties.nt └── member_labels.nt ├── doc ├── ogi-cubiql.png └── table2qb-cubiql.md ├── project.clj ├── resources ├── base-schema.edn ├── config.edn ├── find-all-dimensions.sparql ├── find-all-measures.sparql ├── is-measure-numeric.sparql ├── log4j2.xml └── measure-components-query.sparql ├── src └── cubiql │ ├── config.clj │ ├── context.clj │ ├── core.clj │ ├── data.clj │ ├── dataset_model.clj │ ├── main.clj │ ├── queries.clj │ ├── query_model.clj │ ├── resolvers.clj │ ├── schema.clj │ ├── schema │ └── mapping │ │ ├── dataset.clj │ │ └── labels.clj │ ├── schema_model.clj │ ├── server.clj │ ├── types.clj │ ├── types │ └── scalars.clj │ ├── util.clj │ └── vocabulary.clj └── test └── cubiql ├── query_model_test.clj ├── resolvers_test.clj ├── schema └── mapping │ └── labels_test.clj ├── schema_test.clj ├── types_test.clj └── util_test.clj /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | pom.xml 5 | pom.xml.asc 6 | *.jar 7 | *.class 8 | /.lein-* 9 | /.nrepl-port 10 | .hgignore 11 | .hg/ 12 | .idea/ 13 | *.iml 14 | logs 15 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: clojure 2 | lein: 2.7.1 3 | jdk: 4 | - oraclejdk8 5 | 6 | after_success: 7 | - lein uberjar 8 | 9 | deploy: 10 | provider: releases 11 | api_key: "$RELEASE_OAUTH_TOKEN" 12 | file: "target/uberjar/cubiql-$TRAVIS_TAG-standalone.jar" 13 | skip_cleanup: true 14 | on: 15 | tags: true 16 | condition: $TRAVIS_TAG =~ ^[0-9]+\.[0-9]+\.[0-9]+$ 17 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | All notable changes to this project will be documented in this file. This change log follows the conventions of [keepachangelog.com](http://keepachangelog.com/). 3 | 4 | ## [Unreleased] 5 | ### Changed 6 | - Add a new arity to `make-widget-async` to provide a different widget shape. 7 | 8 | ## [0.1.1] - 2017-08-28 9 | ### Changed 10 | - Documentation on how to make the widgets. 11 | 12 | ### Removed 13 | - `make-widget-sync` - we're all async, all the time. 14 | 15 | ### Fixed 16 | - Fixed widget maker to keep working when daylight savings switches over. 17 | 18 | ## 0.1.0 - 2017-08-28 19 | ### Added 20 | - Files from the new template. 21 | - Widget maker public API - `make-widget-sync`. 22 | 23 | [Unreleased]: https://github.com/your-name/clj-graphql/compare/0.1.1...HEAD 24 | [0.1.1]: https://github.com/your-name/clj-graphql/compare/0.1.0...0.1.1 25 | -------------------------------------------------------------------------------- /DataRestrictions.md: -------------------------------------------------------------------------------- 1 | ## CubiQL RDF data restrictions 2 | 3 | This list contains the CubiQL data restrictions. The RDF data should apply to these restrictions when running CubiQL against your own SPARQL end point. 4 | 5 | CubiQL requires data to be modeled using the [RDF Data Cube Vocabulary](https://www.w3.org/TR/vocab-data-cube/). However, there are some more assumptions/restrictions for the data to be compatible with CubiQL: 6 | 7 | - Multiple measures should be modeled using the measure dimension approach (i.e. use qb:measureType) 8 | - Always use the qb:measureType even if there is only one measure. 9 | - The codes used for each of the cube dimensions (except the geo and time dimensions) should be defined at a separate code list (e.g. skos:ConceptScheme). The code list should contain **only** the codes used. 10 | - A code list should be defined also for the qb:measureType dimension 11 | - This code list can be associated to either a qb:ComponentSpecification or a qb:DimensionProperty. 12 | - The predicate to associate this code list to the qb:ComponentSpecification or a qb:DimensionProperty can be defined at the configuration (it can be qb:codeList or any other property) 13 | - Only URIs (NOT String or xsd:date) should be used for the values of the dimension (code lists cannot be defined based on strings) 14 | - The geo dimension defined at the cofiguration should take values that have a label 15 | - The time dimension defined at the configuration should take values URIs defined by reference.data.gov.uk e.g. http://reference.data.gov.uk/id/year/2016 16 | - If geo and/or time dimensions do not match the 2 above criteria then they should **not** be defined at the configuration and they will be handled like all the other dimensions 17 | 18 | Temporal requirements that will be fixed: 19 | - Datasets, dimensions, measures and codelist members should all have a single `rdfs:label` with a language tag matching the configured `:schema-label-language` in the configuration (nil can be specified to use strings without an associated language tag). This value is used to generate elements of the GraphQL schema such as field names and enum values. Additional `rdfs:label`s with language tags can be defined, although only a single label should be defined for each element for a particular language. The requirement for string literal labels will be lifted when the GraphQL schema mapping is defined explicitly, see [#10](https://github.com/Swirrl/cubiql/issues/10) and 20 | [#40](https://github.com/Swirrl/cubiql/issues/40). 21 | 22 | 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC 2 | LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM 3 | CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 4 | 5 | 1. DEFINITIONS 6 | 7 | "Contribution" means: 8 | 9 | a) in the case of the initial Contributor, the initial code and 10 | documentation distributed under this Agreement, and 11 | 12 | b) in the case of each subsequent Contributor: 13 | 14 | i) changes to the Program, and 15 | 16 | ii) additions to the Program; 17 | 18 | where such changes and/or additions to the Program originate from and are 19 | distributed by that particular Contributor. A Contribution 'originates' from 20 | a Contributor if it was added to the Program by such Contributor itself or 21 | anyone acting on such Contributor's behalf. Contributions do not include 22 | additions to the Program which: (i) are separate modules of software 23 | distributed in conjunction with the Program under their own license 24 | agreement, and (ii) are not derivative works of the Program. 25 | 26 | "Contributor" means any person or entity that distributes the Program. 27 | 28 | "Licensed Patents" mean patent claims licensable by a Contributor which are 29 | necessarily infringed by the use or sale of its Contribution alone or when 30 | combined with the Program. 31 | 32 | "Program" means the Contributions distributed in accordance with this 33 | Agreement. 34 | 35 | "Recipient" means anyone who receives the Program under this Agreement, 36 | including all Contributors. 37 | 38 | 2. GRANT OF RIGHTS 39 | 40 | a) Subject to the terms of this Agreement, each Contributor hereby grants 41 | Recipient a non-exclusive, worldwide, royalty-free copyright license to 42 | reproduce, prepare derivative works of, publicly display, publicly perform, 43 | distribute and sublicense the Contribution of such Contributor, if any, and 44 | such derivative works, in source code and object code form. 45 | 46 | b) Subject to the terms of this Agreement, each Contributor hereby grants 47 | Recipient a non-exclusive, worldwide, royalty-free patent license under 48 | Licensed Patents to make, use, sell, offer to sell, import and otherwise 49 | transfer the Contribution of such Contributor, if any, in source code and 50 | object code form. This patent license shall apply to the combination of the 51 | Contribution and the Program if, at the time the Contribution is added by the 52 | Contributor, such addition of the Contribution causes such combination to be 53 | covered by the Licensed Patents. The patent license shall not apply to any 54 | other combinations which include the Contribution. No hardware per se is 55 | licensed hereunder. 56 | 57 | c) Recipient understands that although each Contributor grants the licenses 58 | to its Contributions set forth herein, no assurances are provided by any 59 | Contributor that the Program does not infringe the patent or other 60 | intellectual property rights of any other entity. Each Contributor disclaims 61 | any liability to Recipient for claims brought by any other entity based on 62 | infringement of intellectual property rights or otherwise. As a condition to 63 | exercising the rights and licenses granted hereunder, each Recipient hereby 64 | assumes sole responsibility to secure any other intellectual property rights 65 | needed, if any. For example, if a third party patent license is required to 66 | allow Recipient to distribute the Program, it is Recipient's responsibility 67 | to acquire that license before distributing the Program. 68 | 69 | d) Each Contributor represents that to its knowledge it has sufficient 70 | copyright rights in its Contribution, if any, to grant the copyright license 71 | set forth in this Agreement. 72 | 73 | 3. REQUIREMENTS 74 | 75 | A Contributor may choose to distribute the Program in object code form under 76 | its own license agreement, provided that: 77 | 78 | a) it complies with the terms and conditions of this Agreement; and 79 | 80 | b) its license agreement: 81 | 82 | i) effectively disclaims on behalf of all Contributors all warranties and 83 | conditions, express and implied, including warranties or conditions of title 84 | and non-infringement, and implied warranties or conditions of merchantability 85 | and fitness for a particular purpose; 86 | 87 | ii) effectively excludes on behalf of all Contributors all liability for 88 | damages, including direct, indirect, special, incidental and consequential 89 | damages, such as lost profits; 90 | 91 | iii) states that any provisions which differ from this Agreement are offered 92 | by that Contributor alone and not by any other party; and 93 | 94 | iv) states that source code for the Program is available from such 95 | Contributor, and informs licensees how to obtain it in a reasonable manner on 96 | or through a medium customarily used for software exchange. 97 | 98 | When the Program is made available in source code form: 99 | 100 | a) it must be made available under this Agreement; and 101 | 102 | b) a copy of this Agreement must be included with each copy of the Program. 103 | 104 | Contributors may not remove or alter any copyright notices contained within 105 | the Program. 106 | 107 | Each Contributor must identify itself as the originator of its Contribution, 108 | if any, in a manner that reasonably allows subsequent Recipients to identify 109 | the originator of the Contribution. 110 | 111 | 4. COMMERCIAL DISTRIBUTION 112 | 113 | Commercial distributors of software may accept certain responsibilities with 114 | respect to end users, business partners and the like. While this license is 115 | intended to facilitate the commercial use of the Program, the Contributor who 116 | includes the Program in a commercial product offering should do so in a 117 | manner which does not create potential liability for other Contributors. 118 | Therefore, if a Contributor includes the Program in a commercial product 119 | offering, such Contributor ("Commercial Contributor") hereby agrees to defend 120 | and indemnify every other Contributor ("Indemnified Contributor") against any 121 | losses, damages and costs (collectively "Losses") arising from claims, 122 | lawsuits and other legal actions brought by a third party against the 123 | Indemnified Contributor to the extent caused by the acts or omissions of such 124 | Commercial Contributor in connection with its distribution of the Program in 125 | a commercial product offering. The obligations in this section do not apply 126 | to any claims or Losses relating to any actual or alleged intellectual 127 | property infringement. In order to qualify, an Indemnified Contributor must: 128 | a) promptly notify the Commercial Contributor in writing of such claim, and 129 | b) allow the Commercial Contributor to control, and cooperate with the 130 | Commercial Contributor in, the defense and any related settlement 131 | negotiations. The Indemnified Contributor may participate in any such claim 132 | at its own expense. 133 | 134 | For example, a Contributor might include the Program in a commercial product 135 | offering, Product X. That Contributor is then a Commercial Contributor. If 136 | that Commercial Contributor then makes performance claims, or offers 137 | warranties related to Product X, those performance claims and warranties are 138 | such Commercial Contributor's responsibility alone. Under this section, the 139 | Commercial Contributor would have to defend claims against the other 140 | Contributors related to those performance claims and warranties, and if a 141 | court requires any other Contributor to pay any damages as a result, the 142 | Commercial Contributor must pay those damages. 143 | 144 | 5. NO WARRANTY 145 | 146 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON 147 | AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 148 | EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR 149 | CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A 150 | PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the 151 | appropriateness of using and distributing the Program and assumes all risks 152 | associated with its exercise of rights under this Agreement , including but 153 | not limited to the risks and costs of program errors, compliance with 154 | applicable laws, damage to or loss of data, programs or equipment, and 155 | unavailability or interruption of operations. 156 | 157 | 6. DISCLAIMER OF LIABILITY 158 | 159 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY 160 | CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, 161 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION 162 | LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 163 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 164 | ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE 165 | EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY 166 | OF SUCH DAMAGES. 167 | 168 | 7. GENERAL 169 | 170 | If any provision of this Agreement is invalid or unenforceable under 171 | applicable law, it shall not affect the validity or enforceability of the 172 | remainder of the terms of this Agreement, and without further action by the 173 | parties hereto, such provision shall be reformed to the minimum extent 174 | necessary to make such provision valid and enforceable. 175 | 176 | If Recipient institutes patent litigation against any entity (including a 177 | cross-claim or counterclaim in a lawsuit) alleging that the Program itself 178 | (excluding combinations of the Program with other software or hardware) 179 | infringes such Recipient's patent(s), then such Recipient's rights granted 180 | under Section 2(b) shall terminate as of the date such litigation is filed. 181 | 182 | All Recipient's rights under this Agreement shall terminate if it fails to 183 | comply with any of the material terms or conditions of this Agreement and 184 | does not cure such failure in a reasonable period of time after becoming 185 | aware of such noncompliance. If all Recipient's rights under this Agreement 186 | terminate, Recipient agrees to cease use and distribution of the Program as 187 | soon as reasonably practicable. However, Recipient's obligations under this 188 | Agreement and any licenses granted by Recipient relating to the Program shall 189 | continue and survive. 190 | 191 | Everyone is permitted to copy and distribute copies of this Agreement, but in 192 | order to avoid inconsistency the Agreement is copyrighted and may only be 193 | modified in the following manner. The Agreement Steward reserves the right to 194 | publish new versions (including revisions) of this Agreement from time to 195 | time. No one other than the Agreement Steward has the right to modify this 196 | Agreement. The Eclipse Foundation is the initial Agreement Steward. The 197 | Eclipse Foundation may assign the responsibility to serve as the Agreement 198 | Steward to a suitable separate entity. Each new version of the Agreement will 199 | be given a distinguishing version number. The Program (including 200 | Contributions) may always be distributed subject to the version of the 201 | Agreement under which it was received. In addition, after a new version of 202 | the Agreement is published, Contributor may elect to distribute the Program 203 | (including its Contributions) under the new version. Except as expressly 204 | stated in Sections 2(a) and 2(b) above, Recipient receives no rights or 205 | licenses to the intellectual property of any Contributor under this 206 | Agreement, whether expressly, by implication, estoppel or otherwise. All 207 | rights in the Program not expressly granted under this Agreement are 208 | reserved. 209 | 210 | This Agreement is governed by the laws of the State of New York and the 211 | intellectual property laws of the United States of America. No party to this 212 | Agreement will bring a legal action under this Agreement more than one year 213 | after the cause of action arose. Each party waives its rights to a jury trial 214 | in any resulting litigation. 215 | -------------------------------------------------------------------------------- /data/dimension_pos.nt: -------------------------------------------------------------------------------- 1 | . 2 | . 3 | . 4 | "Reference Area"@en . 5 | "The country or geographic area to which the measured statistical phenomenon relates."@en . 6 | . 7 | . 8 | . 9 | . 10 | . 11 | . 12 | "Reference Area"@en . 13 | "The country or geographic area to which the measured statistical phenomenon relates."@en . 14 | . 15 | . 16 | . 17 | . 18 | . 19 | "Reference Period"@en . 20 | "The period of time or point in time to which the measured observation is intended to refer."@en . 21 | . 22 | . 23 | . 24 | . 25 | . 26 | "Reference Period"@en . 27 | "The period of time or point in time to which the measured observation is intended to refer."@en . 28 | . 29 | . 30 | . 31 | . 32 | . 33 | "measure type"@en . 34 | "Generic measure dimension, the value of this dimension indicates which measure (from the set of measures in the DSD) is being given by the obsValue (or other primary measure)"@en . 35 | . 36 | . 37 | . 38 | . 39 | . 40 | "measure type"@en . 41 | "Generic measure dimension, the value of this dimension indicates which measure (from the set of measures in the DSD) is being given by the obsValue (or other primary measure)"@en . 42 | . 43 | . 44 | . 45 | . 46 | . 47 | "Gender" . 48 | . 49 | . 50 | . 51 | . 52 | "Gender" . 53 | . 54 | . 55 | . 56 | . 57 | "Population Group" . 58 | . 59 | . 60 | . 61 | . 62 | "Confidence Interval" . 63 | . 64 | . 65 | -------------------------------------------------------------------------------- /data/earnings_metadata.nt: -------------------------------------------------------------------------------- 1 | . 2 | . 3 | . 4 | . 5 | . 6 | "Earnings" . 7 | "Earnings"@en . 8 | "Urningz"@de . 9 | "Eauxnings"@fr . 10 | 11 | "Earnings" . 12 | "Earnings"@en . 13 | "Urningz"@de . 14 | "Eauxnings"@fr . 15 | 16 | "Median gross weekly earnings (\u00A3s) by gender and workplace/residence measure. " . 17 | "Median gross weekly urningz (\u00A3s) by gender und workplace/residence measure. "@de . 18 | "Median gross weekly eauxnings (\u00A3s) by gender et workplace/residence measure. "@fr . 19 | 20 | . 21 | . 22 | "2014-07-29T02:00:00+02:00"^^ . 23 | "2016-02-09T12:08:35Z"^^ . 24 | . 25 | . 26 | . 27 | . 28 | . 29 | . 30 | . 31 | . 32 | "The median gross weekly earnings (before deductions for Tax & National Insurance) of full-time employees on adult rates, whose pay for the survey period was not affected by absence. The self-employed are excluded from this study. This information is obtained from the Annual Survey of Hours & Earnings (ASHE). This is an Office for National Statistics (ONS) publication" . 33 | -------------------------------------------------------------------------------- /data/geo-labels.nt: -------------------------------------------------------------------------------- 1 | "North Lanarkshire" . 2 | "Shetland Islands" . 3 | "Na h-Eileanan Siar" . 4 | "Falkirk" . 5 | "Scotland" . 6 | "Inverclyde" . 7 | "East Ayrshire" . 8 | "Scottish Borders" . 9 | "Fife" . 10 | "Orkney Islands" . 11 | "Perth and Kinross" . 12 | "North Ayrshire" . 13 | "West Dunbartonshire" . 14 | "Midlothian" . 15 | "South Ayrshire" . 16 | "Stirling" . 17 | "Aberdeenshire" . 18 | "South Lanarkshire" . 19 | "East Lothian" . 20 | "Renfrewshire" . 21 | "West Lothian" . 22 | "Aberdeen City" . 23 | "Glasgow City" . 24 | "Highland" . 25 | "Dundee City" . 26 | "East Renfrewshire" . 27 | "Dumfries and Galloway" . 28 | "City of Edinburgh" . 29 | "Argyll and Bute" . 30 | "Clackmannanshire" . 31 | "East Dunbartonshire" . 32 | "Angus" . 33 | "Moray" . 34 | -------------------------------------------------------------------------------- /data/healthy_life_expectancy_metadata.nt: -------------------------------------------------------------------------------- 1 | . 2 | . 3 | . 4 | . 5 | . 6 | "Healthy Life Expectancy" . 7 | "Healthy Life Expectancy" . 8 | "Years of Healthy Life Expectancy (including confidence intervals) by gender" . 9 | . 10 | . 11 | "2014-07-29T02:00:00+02:00"^^ . 12 | "2016-02-10T10:14:02Z"^^ . 13 | . 14 | . 15 | . 16 | . 17 | . 18 | . 19 | . 20 | . 21 | "Life expectancy (LE) is an estimate of how many years a person might be expected to live, whereas healthy life expectancy (HLE) is an estimate of how many years they might live in a 'healthy' state. HLE is a key summary measure of a population's health.\r\n\r\nHLE is an estimate of how long the average person might be expected to live in a 'healthy' state. Like LE, it is most often expressed for an entire lifetime from the time of birth. HLE at birth is the number of years that a new-born baby would live in good health if they experienced the death rates and levels of general health of the local population at the time of their birth, throughout their life.\r\n\r\nHLE is calculated by combining LE and a measure of 'healthy' health: in these HLE analyses for Scotland the measure used is self-assessed general health. This is self-reported by survey or Census respondents but has been shown to reflect both mental and physical health.\r\n" . 22 | -------------------------------------------------------------------------------- /data/labels.nt: -------------------------------------------------------------------------------- 1 | "Reference Period"@en . 2 | "2004" . 3 | "2003" . 4 | "2002" . 5 | "1998" . 6 | "1999" . 7 | "2000" . 8 | "2001" . 9 | "2005" . 10 | "2006" . 11 | "2007" . 12 | "2008" . 13 | "2009" . 14 | "2010" . 15 | "2011" . 16 | "2012" . 17 | "measure type"@en . 18 | "Median" . 19 | "Gender" . 20 | "All" . 21 | "Male" . 22 | "Female" . 23 | "Population Group" . 24 | "Workplace Based" . 25 | "Residence Based" . 26 | -------------------------------------------------------------------------------- /data/measure_properties.nt: -------------------------------------------------------------------------------- 1 | . 2 | . 3 | "Median" . 4 | . 5 | . 6 | -------------------------------------------------------------------------------- /data/member_labels.nt: -------------------------------------------------------------------------------- 1 | "Count" . 2 | "Male" . 3 | "Male" . 4 | "Female" . 5 | "Female" . 6 | "West Dunbartonshire" . 7 | "West Lothian" . 8 | "Scotland" . 9 | "Scotland" . 10 | "Clackmannanshire" . 11 | "Dumfries and Galloway" . 12 | "East Ayrshire" . 13 | "East Lothian" . 14 | "East Renfrewshire" . 15 | "Na h-Eileanan Siar" . 16 | "Falkirk" . 17 | "Highland" . 18 | "Inverclyde" . 19 | "Midlothian" . 20 | "Moray" . 21 | "North Ayrshire" . 22 | "Orkney Islands" . 23 | "Scottish Borders" . 24 | "Shetland Islands" . 25 | "South Ayrshire" . 26 | "South Lanarkshire" . 27 | "Stirling" . 28 | "Aberdeen City" . 29 | "Aberdeenshire" . 30 | "Argyll and Bute" . 31 | "City of Edinburgh" . 32 | "Renfrewshire" . 33 | "Angus" . 34 | "Dundee City" . 35 | "North Lanarkshire" . 36 | "East Dunbartonshire" . 37 | "Glasgow City" . 38 | "Fife" . 39 | "Perth and Kinross" . 40 | "All" . 41 | "2003" . 42 | "2004" . 43 | "2005" . 44 | "2005" . 45 | "2006" . 46 | "2006" . 47 | "2007" . 48 | "2007" . 49 | "2008" . 50 | "2008" . 51 | "2009" . 52 | "2009" . 53 | "2010" . 54 | "2010" . 55 | "2011" . 56 | "2012" . 57 | "2001" . 58 | "2001" . 59 | "2002" . 60 | "2002" . 61 | "Median" . 62 | "1998" . 63 | "1998" . 64 | "1999" . 65 | "1999" . 66 | "2000" . 67 | "2000" . 68 | "Residence Based" . 69 | "Workplace Based" . 70 | "1993" . 71 | "1994" . 72 | "1995" . 73 | "1996" . 74 | "1980" . 75 | "1981" . 76 | "1982" . 77 | "1983" . 78 | "1984" . 79 | "1985" . 80 | "1986" . 81 | "1987" . 82 | "1988" . 83 | "1989" . 84 | "1990" . 85 | "1991" . 86 | "1992" . 87 | "All" . 88 | "Lower" . 89 | "Upper" . 90 | -------------------------------------------------------------------------------- /doc/ogi-cubiql.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Swirrl/cubiql/bd09e1b3a2e3ece3a3e52cb5d06ce95d64b09ed7/doc/ogi-cubiql.png -------------------------------------------------------------------------------- /doc/table2qb-cubiql.md: -------------------------------------------------------------------------------- 1 | # Table2qb - CubiQL End-to-end example 2 | 3 | Steps: 4 | 1. Create RDF ttl files using Table2qb 5 | 2. (Optional) Load the RDF files into a triple store 6 | 3. Run CubiQL against the generated RDF data. 7 | 8 | ## Step 1: Table2qb 9 | 10 | [cubiql](https://github.com/Swirrl/cubiql/) is a command-line tool for generating RDF data cubes from tidy CSV data. See the `table2qb` repository for 11 | installation instructions and an [example](https://github.com/Swirrl/table2qb/blob/master/examples/employment/README.md) of using it to generate data cubes. 12 | 13 | ### Requirements 14 | 15 | - Java 8 16 | - Unix, Linux, Mac, Windows environments 17 | - On Windows run `table2qb` with `java -jar table2qb.jar exec ...` 18 | 19 | ### How to use 20 | 21 | **Create the codelists using the `codelist-pipeline`** 22 | 23 | Example: 24 | 25 | `table2qb exec codelist-pipeline --codelist-csv csv/gender.csv --codelist-name "Gender" --codelist-slug "gender" --base-uri http://statistics.gov.scot/ --output-file ttl/gender.ttl` 26 | 27 | - Codelists should exist for every dimension. If codelists currently exists there is no need to create new ones. 28 | - A separate CSV is required for each codelist. 29 | - The codelist CSV files should contain all the possibles values for each codelist 30 | - An example CSV is: [gender.csv](https://github.com/Swirrl/table2qb/blob/master/examples/employment/csv/gender.csv) 31 | - At the CSV use the column `Parent Notation` to define hierarchies 32 | - The URIs created by the codelist pipeline 33 | - Concept scheme (codelist): `{base-uri}/def/concept-scheme/{codelist-slug}` 34 | - Concepts: `{base-uri}/def/concept/{codelist-slug}/{CSV_column_ Notation}` 35 | 36 | **Create the cube components using the `components-pipeline`** 37 | 38 | Example: 39 | `table2qb exec components-pipeline --input-csv csv/components.csv --base-uri http://statistics.gov.scot/ --output-file ttl/components.ttl` 40 | 41 | - The cube components are the `qb:DimensionProperty`, `qb:MeasureProperty` and `qb:AttributeProperty`. 42 | - At the CSV use one row for each of the cube components. If some of the components have already been created (e.g. for another cube) do not include them in the CSV. 43 | - At the CSV column `Component Type` the types are: `Dimension`, `Measure` and `Attribute` 44 | - At the CSV column `Codelist` use the URIs created by the `codelist-pipeline` 45 | - An example CSV is: [components.csv](https://github.com/Swirrl/table2qb/blob/master/examples/employment/csv/components.csv) 46 | - The URIs created by the `components-pipeline`: 47 | - Dimension: `{base-uri}/def/dimension/{CSV_column_ Label}` 48 | - Measure: `{base-uri}/def/measure/{CSV_column_ Label}` 49 | - Attribute: `{base-uri}/def/attribute/{CSV_column_ Label}` 50 | 51 | **Create the cube DSD and observations using the `cube-pipeline`** 52 | 53 | Example: 54 | `table2qb exec cube-pipeline --input-csv csv/input.csv --dataset-name "Employment" --dataset-slug "employment" --column-config csv/columns.csv --base-uri http://statistics.gov.scot/ --output-file ttl/cube.ttl` 55 | 56 | - Use a CSV file to define the mappings between a CSV column and the relevant cube component. 57 | - The CSV should contain one row per component i.e. `qb:DimensionProperty`, `qb:MeasureProperty` and `qb:AttributeProperty`. Including a row for the `qb:measureType`. It should also contain a row for the observation `value`. 58 | - At the CSV, the column `component_attachment` can be: `qb:dimension`, `qb:measure`, `qb:attribute`. 59 | - At the CSV, the column `property_template` should match with the URIs created for the components by the `components-pipeline`. Use the csv column `name` if required at the template (e.g. `http://example.gr/def/measure/{measure_type}`) 60 | - At the CSV, the column `value_template` should match with the URIs created for the concepts by the `codelist-pipeline`. Use the csv column `name` if required at the template (e.g. `http://example.gr/def/concept/stationid/{station_id}`) 61 | - If there are numbers at the dimension values (not at the value of the observation) use `datatype` `string`. Otherwise if `datatype` `number` is used then the URIs will have the form e.g. `http://example.gr/def/condept/year/2000.0` 62 | - At the CSV row that has the mapping for the measure (i.e. `component_attachment` `qb:measure`), leave the `value_template` empty. 63 | - At the CSV row for the value (i.e. with `name` `value`). Leave the `component_attachment` and `value_template` empty. 64 | - At each CSV row use a `value_transformation` if required. Possible values are: `slugize`, `unitize` or blank. `slugize` converts column values into URI components, e.g. `(slugize "Some column value")` is `some-column-value`. `unitize` translates literal `£` into `GBP`, e.g. (unitize "£ 100") is `gpb-100`. **Be careful**: the `slugize` converts all to non-capital letters. The URIs of the dimension values should match with the the concept URIs created through the `codelist-pipeline` 65 | 66 | **Advice:** 67 | - Use the same base URI at all pipelines. Although it is not mandatory it will easy the transformation process. 68 | - **Be careful to use URIs that match between the pipelines. E.g. the `property_template` URI at `cube-pipeline` should match with the URIs created for the components by the `components-pipeline`.** 69 | 70 | A complete example can be found at [Github](https://github.com/Swirrl/table2qb/tree/master/examples/employment). 71 | 72 | ## Step 2: (Optional) Load RDF to triple store 73 | 74 | CubiQL can be run directly against a local directory containing the generated RDF data, however this loads the data into memory so is unsuitable for large amounts of data. 75 | First loading the data into a dedicated triple store is therefore recommended. 76 | 77 | Supported triple stores: 78 | - Virtuoso 79 | - Stardog 80 | - Fuseki ? 81 | 82 | What to load at the triple store: 83 | - All the RDF ttl created by Table2qb: 84 | - RDF for each of the codelists 85 | - RDF for the cube components 86 | - RDF for the DSD and the observations 87 | - RDF for the [QB vocabulary](https://raw.githubusercontent.com/UKGovLD/publishing-statistical-data/master/specs/src/main/vocab/cube.ttl) 88 | 89 | Configuration of the triple store: 90 | - Set the max limit of the returned results e.g. 100000 (depends on the size and number of cubes). 91 | - At Virtuoso the limit is defined by the parameter `ResultSetMaxRows` 92 | 93 | ## Step 3: CubiQL 94 | 95 | Run CubiQL using the default configuration: 96 | 97 | `java -jar cubiql-standalone.jar --port 8085 --endpoint http://myendpoint.com` 98 | 99 | if running against a local directory containing the data the `--endpoint` parameter should specify the path to the directory: 100 | 101 | `java -jar cubiql-standalone.jar --port 8085 --endpoint ./ttl` 102 | 103 | The default configuration: 104 | ``` 105 | {:geo-dimension-uri nil 106 | :time-dimension-uri nil 107 | :codelist-source "component" 108 | :codelist-predicate "http://publishmydata.com/def/qb/codesUsed" 109 | :codelist-label-uri "http://www.w3.org/2000/01/rdf-schema#label" 110 | :dataset-label-uri "http://www.w3.org/2000/01/rdf-schema#label" 111 | :schema-label-language nil 112 | :max-observations-page-size 2000} 113 | ``` 114 | 115 | If default configuration does not match your data, then use another configuration file: 116 | 117 | `java -jar cubiql-standalone.jar --port 8085 --endpoint http://myendpoint.com/ --configuration myconfig.edn` 118 | 119 | Configuration parameters: 120 | 121 | - `:geo-dimension-uri` defines the geo dimension. The values of the geo dimension should have a labe. 122 | - `:time-dimension-uri` defines the time dimension. The values of the time dimensions should be defined by the `reference.data.gov.uk` e.g. `http://reference.data.gov.uk/id/year/2016` 123 | - `:codelist-source` defines the source of the codelist that contains only the values used at the cube. The source can be: (i) `"component"` or (ii) `"dimension"`. By default Table2qb uses the `"component"`. 124 | - `:codelist-predicate` defines the predicate that connects the `:codelist-source` with the codelist that contains only the values used at the cube. By default Table2qb uses: `http://publishmydata.com/def/qb/codesUsed`. 125 | - `:codelist-label-uri` defines the label property that is used at the codelists. By default Table2qb uses: `http://www.w3.org/2000/01/rdf-schema#label ` 126 | - `:dataset-label-uri` defines the label property that is used at the dataset i.e. cube, DSD, components. By default Table2qb uses: `http://www.w3.org/2000/01/rdf-schema#label` 127 | - Datasets, dimensions, measures and codelist members should all have a label with a language tag matching the `:schema-label-language`. `nil` can be specified to use strings without an associated language tag. 128 | - `:max-observations-page-size` defines the maximum page size e.g. if you need to get all the observations with one query. 129 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject swirrl/cubiql "0.7.0-SNAPSHOT" 2 | :description "Query RDF Datacubes with GraphQL" 3 | :url "https://github.com/Swirrl/cubiql" 4 | :license {:name "Eclipse Public License" 5 | :url "http://www.eclipse.org/legal/epl-v10.html"} 6 | :dependencies [[org.clojure/clojure "1.9.0"] 7 | [com.walmartlabs/lacinia "0.23.0-rc-1"] 8 | [com.walmartlabs/lacinia-pedestal "0.5.0-rc-2"] 9 | [org.clojure/data.json "0.2.6"] 10 | [grafter "0.11.5"] 11 | [org.clojure/tools.cli "0.3.5"] 12 | 13 | ;; logging 14 | [org.slf4j/slf4j-api "1.7.25"] 15 | [org.apache.logging.log4j/log4j-slf4j-impl "2.17.1"] 16 | 17 | [org.slf4j/jul-to-slf4j "1.7.25"] 18 | [org.slf4j/jcl-over-slf4j "1.7.25"] 19 | 20 | ;;configuration 21 | [aero "1.1.3"]] 22 | :main ^:skip-aot cubiql.main 23 | :target-path "target/%s" 24 | :profiles {:uberjar {:aot :all}} 25 | 26 | :release-tasks [["vcs" "assert-committed"] 27 | ["change" "version" 28 | "leiningen.release/bump-version" "release"] 29 | ["vcs" "commit"] 30 | ["vcs" "tag"] 31 | ["change" "version" "leiningen.release/bump-version"] 32 | ["vcs" "commit"] 33 | ;;["vcs" "push"] 34 | ]) 35 | -------------------------------------------------------------------------------- /resources/base-schema.edn: -------------------------------------------------------------------------------- 1 | {:objects 2 | {:qb 3 | {:fields 4 | {:datasets {:type (list :dataset) 5 | :resolve :resolve-datasets 6 | :args {:dimensions {:type :filter} 7 | :measures {:type :filter} 8 | :attributes {:type :filter} 9 | :componentValue {:type :valuefilter} 10 | :uri {:type :uri}}}}} 11 | 12 | :dataset 13 | {:implements [:dataset_meta] 14 | :fields 15 | {:uri {:type :uri :description "Dataset URI"} 16 | :title {:type String :description "Dataset title"} 17 | :description {:type (list String) :description "Dataset descriptions"} 18 | :licence {:type (list :uri) :description "URIs of the licences the dataset is published under"} 19 | :issued {:type (list :DateTime) :description "When the dataset was issued"} 20 | :modified {:type :DateTime :description "When the dataset was last modified"} 21 | :publisher {:type (list :uri) :description "URIs of the publishers of the dataset"} 22 | :schema {:type String :description "Name of the GraphQL query root field corresponding to this dataset"} 23 | :dimensions {:type (list :dim) 24 | :resolve :resolve-dataset-dimensions 25 | :description "Dimensions within the dataset"} 26 | :measures {:type (list :measure) 27 | :resolve :resolve-dataset-measures 28 | :description "Measure types within the dataset"}}} 29 | 30 | :dim 31 | {:fields 32 | {:uri {:type :uri :description "URI of the dimension"} 33 | :label {:type String :description "Label for the dimension"} 34 | :values {:type (list :dim_value) :description "Code list of values for the dimension"} 35 | :enum_name {:type String :description "Name of the corresponding enum value"}}} 36 | 37 | :measure 38 | {:fields 39 | {:uri {:type :uri :description "URI of the measure"} 40 | :label {:type String :description "Label for the measure"} 41 | :enum_name {:type String :description "Name of the corresponding enum value"}}} 42 | 43 | :unmapped_dim_value 44 | {:implements [:resource] 45 | :fields 46 | {:uri {:type :uri :description "URI of the dimension value"} 47 | :label {:type String :description "Label for the dimension value"}}} 48 | 49 | :enum_dim_value 50 | {:implements [:resource] 51 | :fields 52 | {:uri {:type :uri :description "URI of the dimension value"} 53 | :label {:type String :description "Label for the dimension value"} 54 | :enum_name {:type String :description "Name of the corresponding enum value"}}} 55 | 56 | :ref_period 57 | {:fields 58 | {:uri {:type :uri :description "URI of the reference period"} 59 | :label {:type String :description "Label for the reference period"} 60 | :start {:type :DateTime :description "Start time for the period"} 61 | :end {:type :DateTime :description "End time for the period"}}} 62 | 63 | :ref_area 64 | {:implements [:resource] 65 | :fields 66 | {:uri {:type :uri :description "URI of the reference area"} 67 | :label {:type String :description "Label for the reference area"}}}} 68 | 69 | :interfaces 70 | {:dataset_meta 71 | {:description "Fields common to generic and specific dataset schemas" 72 | :fields 73 | {:uri {:type :uri :description "Dataset URI"} 74 | :title {:type String :description "Dataset title"} 75 | :description {:type (list String) :description "Dataset description"} 76 | :schema {:type String :description "Name of the GraphQL query root field corresponding to this dataset"} 77 | :dimensions {:type (list :dim) :description "Dimensions within the dataset"} 78 | :measures {:type (list :measure) :description "Measure types within the dataset"}}} 79 | :resource 80 | {:description "Resource with a URI and optional label" 81 | :fields 82 | {:uri {:type :uri :description "URI of the resource"} 83 | :label {:type String :description "Optional label"}}}} 84 | 85 | :enums 86 | {:sort_direction 87 | {:description "Which direction to sort a dimension or measure in" 88 | :values [:ASC :DESC]}} 89 | 90 | :unions 91 | {:dim_value 92 | {:members [:enum_dim_value :unmapped_dim_value]}} 93 | 94 | :input-objects 95 | {:filter 96 | {:fields 97 | {:or {:type (list :uri) 98 | :description "List of URIs for which at least one must be contained within matching datasets."} 99 | :and {:type (list :uri) 100 | :description "List of URIs which must all be contained within matching datasets."}}} 101 | 102 | :valuefilter 103 | {:fields 104 | {:or {:type (list :componentvalue) 105 | :description "List of :componentvalues for which at least one must be contained within matching datasets."} 106 | :and {:type (list :componentvalue) 107 | :description "List of :componentvalues which must all be contained within matching datasets."}}} 108 | 109 | :componentvalue 110 | {:fields 111 | {:component {:type :uri :description "The URI of a dimension or attribute"} 112 | :values {:type (list :uri) :description "A list of dimension/attribute values"} 113 | :levels {:type (list :uri) :description "A list of URIs of the hierarchical levels of dimension/attribute value"}}} 114 | 115 | :ref_period_filter 116 | {:fields 117 | {:uri {:type :uri :description "URI of the reference period"} 118 | :starts_before {:type :DateTime :description "Latest start time for the reference period"} 119 | :starts_after {:type :DateTime :description "Earliest start time for the reference period"} 120 | :ends_before {:type :DateTime :description "Latest end time for the reference period"} 121 | :ends_after {:type :DateTime :description "Earliest end time for the reference period"}}} 122 | 123 | :page_selector 124 | {:fields 125 | {:first {:type Int :description "Number of results to retrive."} 126 | :after {:type :SparqlCursor :description "Cursor to the start of the results page"}}}} 127 | 128 | :queries 129 | {:cubiql 130 | {:type :qb 131 | :resolve :resolve-cuibiql 132 | :args {:lang_preference {:type String}}}}} 133 | -------------------------------------------------------------------------------- /resources/config.edn: -------------------------------------------------------------------------------- 1 | {:geo-dimension-uri "http://purl.org/linked-data/sdmx/2009/dimension#refArea" 2 | :time-dimension-uri "http://purl.org/linked-data/sdmx/2009/dimension#refPeriod" 3 | :codelist-source "component" 4 | :codelist-label-uri "http://www.w3.org/2000/01/rdf-schema#label" 5 | :dataset-label-uri "http://www.w3.org/2000/01/rdf-schema#label" 6 | :schema-label-language nil 7 | :max-observations-page-size 2000} -------------------------------------------------------------------------------- /resources/find-all-dimensions.sparql: -------------------------------------------------------------------------------- 1 | PREFIX rdfs: 2 | PREFIX qb: 3 | 4 | SELECT ?dim ?label ?range WHERE { 5 | ?dim a qb:DimensionProperty . 6 | OPTIONAL { ?dim rdfs:range ?range } 7 | } 8 | -------------------------------------------------------------------------------- /resources/find-all-measures.sparql: -------------------------------------------------------------------------------- 1 | PREFIX rdfs: 2 | PREFIX qb: 3 | SELECT ?measure WHERE { 4 | ?measure a qb:MeasureProperty . 5 | } 6 | -------------------------------------------------------------------------------- /resources/is-measure-numeric.sparql: -------------------------------------------------------------------------------- 1 | PREFIX qb: 2 | 3 | SELECT ?numeric WHERE { 4 | ?obs a qb:Observation . 5 | ?obs ?measure ?val . 6 | BIND(isNumeric(?val) AS ?numeric) 7 | } LIMIT 1 8 | -------------------------------------------------------------------------------- /resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %d %p %c{1.} [%t] %m%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /resources/measure-components-query.sparql: -------------------------------------------------------------------------------- 1 | PREFIX qb: 2 | 3 | SELECT * WHERE { 4 | ?ds qb:structure ?dsd . 5 | ?dsd a qb:DataStructureDefinition . 6 | ?dsd qb:component ?comp . 7 | ?comp qb:measure ?measure . 8 | } 9 | -------------------------------------------------------------------------------- /src/cubiql/config.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.config 2 | (:require [cubiql.vocabulary :refer :all] 3 | [aero.core :as aero] 4 | [clojure.java.io :as io]) 5 | (:import [java.net URI])) 6 | 7 | (defn read-config 8 | ([] (read-config (io/resource "config.edn"))) 9 | ([source] 10 | (aero/read-config source))) 11 | 12 | (defn geo-dimension [config] 13 | (if-let [config-geo (:geo-dimension-uri config)] 14 | (URI. config-geo))) 15 | 16 | (defn time-dimension [config] 17 | (if-let [config-time (:time-dimension-uri config)] 18 | (URI. config-time))) 19 | 20 | ;;accepted values: [dimension component] 21 | (defn codelist-source [{config-codelist :codelist-source :as config}] 22 | ;;Return the default "?dim" if :codelist-source is not defined at configuration 23 | (case config-codelist 24 | "dimension" "?dim" 25 | "component" "?comp" 26 | "?dim")) 27 | 28 | (defn codelist-predicate [{predicate :codelist-predicate :as config}] 29 | (if (nil? predicate) 30 | qb:codeList 31 | (URI. predicate))) 32 | 33 | (defn codelist-label [{config-cl-label :codelist-label-uri :as config}] 34 | ;;Return the default skos:prefLabel if :codelist-label-uri is not defined at configuration 35 | (if (nil? config-cl-label) 36 | skos:prefLabel 37 | (URI. config-cl-label))) 38 | 39 | (defn dataset-label [{config-dataset-label :dataset-label-uri :as config}] 40 | ;;Return the default rdfs:label if :dataset-label-uri is not defined at configuration 41 | (if (nil? config-dataset-label) 42 | rdfs:label 43 | (URI. config-dataset-label))) 44 | 45 | (defn schema-label-language [config] 46 | (:schema-label-language config)) 47 | 48 | (defn ignored-codelist-dimensions 49 | "Returns a collection of URIs for dimensions which should not have an associated codelist." 50 | [config] 51 | (remove nil? [(geo-dimension config) 52 | (time-dimension config)])) 53 | 54 | (defn max-observations-page-size [config] 55 | (:max-observations-page-size config)) -------------------------------------------------------------------------------- /src/cubiql/context.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.context 2 | "Functions for managing the graphql execution context passed into resolver functions" 3 | (:require [cubiql.util :as util] 4 | [clojure.walk :as walk] 5 | [com.walmartlabs.lacinia.executor :as executor])) 6 | 7 | (defn create 8 | "Creates a context map from a repository and collection of datasets" 9 | [repo datasets dataset-mappings config] 10 | (let [uri->dataset (util/keyed-by :uri datasets) 11 | uri->dataset-mapping (util/keyed-by :uri dataset-mappings)] 12 | {:repo repo :uri->dataset uri->dataset :uri->dataset-mapping uri->dataset-mapping :config config})) 13 | 14 | (defn get-dataset 15 | "Fetches a dataset from the context by its URI. Returns nil if the dataset was not found." 16 | [context dataset-uri] 17 | (get-in context [:uri->dataset dataset-uri])) 18 | 19 | (defn get-dataset-mapping [context dataset-uri] 20 | (get-in context [:uri->dataset-mapping dataset-uri])) 21 | 22 | (defn get-repository 23 | "Gets the SPARQL repository from the context" 24 | [context] 25 | (:repo context)) 26 | 27 | (defn get-configuration [context] 28 | (:config context)) 29 | 30 | (defn un-namespace-keys [m] 31 | (walk/postwalk (fn [x] 32 | (if (map? x) 33 | (util/map-keys (fn [k] (keyword (name k))) x) 34 | x)) m)) 35 | 36 | (defn flatten-selections [m] 37 | (walk/postwalk (fn [x] 38 | (if (and (map? x) (contains? x :selections)) 39 | (:selections x) 40 | x)) m)) 41 | 42 | (defn get-selections [context] 43 | (-> context (executor/selections-tree) (un-namespace-keys) (flatten-selections))) 44 | -------------------------------------------------------------------------------- /src/cubiql/core.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.core 2 | (:require [com.walmartlabs.lacinia.schema :as lschema] 3 | [com.walmartlabs.lacinia.util :refer [attach-resolvers]] 4 | [com.walmartlabs.lacinia :refer [execute]] 5 | [cubiql.util :refer [read-edn-resource]] 6 | [cubiql.types :refer :all :as types] 7 | [cubiql.types.scalars :as scalars] 8 | [cubiql.schema :as schema] 9 | [cubiql.resolvers :as resolvers] 10 | [cubiql.schema-model :as sm] 11 | [cubiql.schema.mapping.labels :as mapping] 12 | [cubiql.config :as config] 13 | [cubiql.dataset-model :as ds-model])) 14 | 15 | (defn can-generate-schema? 16 | "Indicates whether a GraphQL schema can be generated for the given dataset" 17 | [dataset] 18 | (not (empty? (types/dataset-dimension-measures dataset)))) 19 | 20 | (defn build-schema [dataset-mappings] 21 | (let [base-schema (read-edn-resource "base-schema.edn") 22 | base-schema (assoc base-schema :scalars scalars/custom-scalars) 23 | {:keys [qb-fields schema]} (schema/get-qb-fields-schema dataset-mappings) 24 | base-schema (update-in base-schema [:objects :qb :fields] merge qb-fields) 25 | {:keys [resolvers] :as combined-schema} (sm/merge-schemas base-schema schema) 26 | query-resolvers (merge {:resolve-observation-sparql-query resolvers/resolve-observations-sparql-query 27 | :resolve-datasets (resolvers/wrap-options resolvers/resolve-datasets) 28 | :resolve-dataset-dimensions (schema/create-global-dataset-dimensions-resolver dataset-mappings) 29 | :resolve-dataset-measures schema/global-dataset-measures-resolver 30 | :resolve-cuibiql resolvers/resolve-cubiql} 31 | resolvers)] 32 | (attach-resolvers (dissoc combined-schema :resolvers) query-resolvers))) 33 | 34 | (defn- get-schema-components [repo config] 35 | (let [datasets (ds-model/get-all-datasets repo config) 36 | datasets (filter can-generate-schema? datasets) 37 | dataset-mappings (mapping/get-dataset-mapping-models repo datasets config)] 38 | {:schema (build-schema dataset-mappings) 39 | :datasets datasets 40 | :dataset-mappings dataset-mappings})) 41 | 42 | (defn get-schema [repo] 43 | (:schema (get-schema-components repo (config/read-config)))) 44 | 45 | (defn build-schema-context [repo config] 46 | (let [result (get-schema-components repo config)] 47 | (update result :schema lschema/compile))) 48 | -------------------------------------------------------------------------------- /src/cubiql/data.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.data 2 | (:require 3 | [grafter.rdf.repository :as repo] 4 | [grafter.rdf.formats :as formats] 5 | [grafter.rdf :refer [add]] 6 | [grafter.rdf.formats :as formats] 7 | [clojure.java.io :as io]) 8 | (:import [java.io File FileFilter] 9 | [java.net URI URISyntaxException])) 10 | 11 | (defn- ^FileFilter create-file-filter [p] 12 | (reify FileFilter 13 | (accept [_this file] 14 | (p file)))) 15 | 16 | (defn- is-rdf-file? [^File file] 17 | (some? (formats/filename->rdf-format file))) 18 | 19 | (defn directory-repo 20 | "Creates a sail repository from a directory containing RDF files" 21 | [^File dir] 22 | {:pre [(and (.isDirectory dir) (.exists dir))]} 23 | (let [rdf-files (.listFiles dir (create-file-filter is-rdf-file?))] 24 | (apply repo/fixture-repo rdf-files))) 25 | 26 | (defn file->endpoint [^File f] 27 | (if (.exists f) 28 | (if (.isDirectory f) 29 | (directory-repo f) 30 | (repo/fixture-repo f)) 31 | (throw (IllegalArgumentException. (format "%s does not exist" f))))) 32 | 33 | (defmulti uri->endpoint (fn [^URI uri] (keyword (.getScheme uri)))) 34 | 35 | (defmethod uri->endpoint :http [^URI uri] 36 | (repo/sparql-repo uri)) 37 | 38 | (defmethod uri->endpoint :https [^URI uri] 39 | (repo/sparql-repo uri)) 40 | 41 | (defmethod uri->endpoint :file [^URI uri] 42 | (let [f (File. uri)] 43 | (file->endpoint f))) 44 | 45 | (defmethod uri->endpoint :default [^URI uri] 46 | (let [f (File. (str uri))] 47 | (file->endpoint f))) 48 | 49 | (defn parse-endpoint [^String endpoint-str] 50 | (try 51 | (let [uri (URI. endpoint-str)] 52 | (uri->endpoint uri)) 53 | (catch URISyntaxException ex 54 | (let [f (io/file endpoint-str)] 55 | (file->endpoint f))))) 56 | 57 | (defn get-test-repo [] 58 | (directory-repo (io/file "data"))) 59 | 60 | (defn get-scotland-repo [] 61 | (repo/sparql-repo "https://staging-drafter-sg.publishmydata.com/v1/sparql/live")) 62 | -------------------------------------------------------------------------------- /src/cubiql/dataset_model.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.dataset-model 2 | (:require [cubiql.config :as config] 3 | [cubiql.util :as util] 4 | [grafter.rdf.sparql :as sp] 5 | [clojure.string :as string] 6 | [cubiql.types :as types] 7 | [cubiql.vocabulary :refer :all])) 8 | 9 | (defn find-all-datasets-query [configuration] 10 | (str 11 | (let [dataset-label (config/dataset-label configuration)] 12 | (str 13 | "PREFIX qb: " 14 | "SELECT ?ds ?name WHERE {" 15 | " ?ds a qb:DataSet ." 16 | " ?ds <" dataset-label "> ?name ." 17 | "}")))) 18 | 19 | (defn find-all-datasets [repo configuration] 20 | (let [q (find-all-datasets-query configuration) 21 | results (util/eager-query repo q) 22 | by-uri (group-by :ds results)] 23 | (map (fn [[ds-uri bindings]] 24 | (let [names (map :name bindings)] 25 | {:ds ds-uri 26 | :name (util/find-best-language names (config/schema-label-language configuration))})) 27 | by-uri))) 28 | 29 | (defn dimension-bindings->dimensions [dimension-bindings] 30 | (map (fn [[dim-uri bindings]] 31 | (let [range (:range (first bindings))] 32 | {:uri dim-uri 33 | :range range})) 34 | (group-by :dim dimension-bindings))) 35 | 36 | (defn find-all-dimensions [repo] 37 | (let [results (sp/query "find-all-dimensions.sparql" repo)] 38 | (dimension-bindings->dimensions results))) 39 | 40 | (defn measure-bindings->measures [measure-bindings] 41 | (map #(util/rename-key % :measure :uri) measure-bindings)) 42 | 43 | (defn- is-measure-numeric? 44 | "Queries for an observation for the specified measure. Returns true if the value of the observation is numeric, false 45 | if there are no matching observations or the value is non-numeric." 46 | [repo measure-uri] 47 | (let [results (sp/query "is-measure-numeric.sparql" {:measure measure-uri} repo)] 48 | (if-let [solution (first results)] 49 | (util/xmls-boolean->boolean (:numeric solution)) 50 | false))) 51 | 52 | (defn find-numeric-measures [repo all-measures] 53 | (into #{} (keep (fn [{:keys [uri] :as measure}] 54 | (if (is-measure-numeric? repo uri) 55 | uri)) 56 | all-measures))) 57 | 58 | (defn find-all-measures [repo] 59 | (let [results (sp/query "find-all-measures.sparql" repo) 60 | measures (measure-bindings->measures results) 61 | numeric-measures (find-numeric-measures repo measures)] 62 | (map (fn [{:keys [uri] :as measure}] 63 | (assoc measure :is-numeric? (contains? numeric-measures uri))) 64 | measures))) 65 | 66 | (defn get-dimension-components-query [configuration] 67 | (str 68 | "PREFIX qb: " 69 | "SELECT * WHERE {" 70 | " ?ds qb:structure ?dsd ." 71 | " ?dsd a qb:DataStructureDefinition ." 72 | " ?dsd qb:component ?comp ." 73 | " ?comp qb:dimension ?dim ." 74 | " OPTIONAL { " (config/codelist-source configuration) " <" (config/codelist-predicate configuration) "> ?codelist }" 75 | "}")) 76 | 77 | (defn get-dimension-components [repo configuration] 78 | (let [q (get-dimension-components-query configuration) 79 | results (util/eager-query repo q)] 80 | (->> results 81 | (map (fn [bindings] (util/rename-key bindings :dim :dimension))) 82 | (util/distinct-by :comp)))) 83 | 84 | (defn get-measure-components [repo] 85 | (sp/query "measure-components-query.sparql" repo)) 86 | 87 | (defn get-codelists-query [configuration] 88 | (let [dimension-filters (map (fn [dim] (str "FILTER(?dim != <" dim ">)")) (config/ignored-codelist-dimensions configuration))] 89 | (str 90 | "PREFIX qb: " 91 | "PREFIX skos: " 92 | "SELECT DISTINCT ?codelist WHERE {" 93 | " ?ds a qb:DataSet ." 94 | " ?ds qb:structure ?dsd ." 95 | " ?dsd a qb:DataStructureDefinition ." 96 | " ?dsd qb:component ?comp ." 97 | " ?comp qb:dimension ?dim ." 98 | (string/join "\n" dimension-filters) 99 | (config/codelist-source configuration) " <" (config/codelist-predicate configuration) "> ?codelist ." 100 | "}"))) 101 | 102 | (defn get-all-codelists [repo configuration] 103 | (let [q (get-codelists-query configuration) 104 | results (util/eager-query repo q)] 105 | (into #{} (map :codelist results)))) 106 | 107 | (defn set-component-orders [components] 108 | (let [has-order? (comp some? :order) 109 | with-order (filter has-order? components) 110 | without-order (remove has-order? components) 111 | max-order (if (seq with-order) 112 | (apply max (map :order with-order)) 113 | 0)] 114 | (concat (sort-by :order with-order) 115 | (map-indexed (fn [idx comp] 116 | (assoc comp :order (+ max-order idx 1))) 117 | without-order)))) 118 | 119 | (defn is-measure-type-dimension? [{:keys [uri] :as dimension}] 120 | (= qb:measureType uri)) 121 | 122 | (defn get-dimension-type [{:keys [uri range] :as dim} codelist-uri codelists configuration] 123 | (cond 124 | (= (config/geo-dimension configuration) uri) 125 | types/ref-area-type 126 | 127 | (= (config/time-dimension configuration) uri) 128 | types/ref-period-type 129 | 130 | (= qb:measureType uri) 131 | types/measure-dimension-type 132 | 133 | (= xsd:decimal range) 134 | types/decimal-type 135 | 136 | (= xsd:string range) 137 | types/string-type 138 | 139 | (contains? codelists codelist-uri) 140 | types/enum-type 141 | 142 | :else 143 | (types/->UnmappedType range))) 144 | 145 | (defn construct-dataset [{ds-uri :ds ds-name :name :as dataset} dimension-components measure-components uri->dimension uri->measure codelists configuration] 146 | (let [ordered-dim-components (set-component-orders dimension-components) 147 | dimensions (mapv (fn [{dim-uri :dimension order :order codelist-uri :codelist :as comp}] 148 | (let [dimension (util/strict-get uri->dimension dim-uri :key-desc "Dimension") 149 | type (get-dimension-type dimension codelist-uri codelists configuration)] 150 | (types/->Dimension dim-uri order type))) 151 | ordered-dim-components) 152 | ordered-measure-components (set-component-orders measure-components) 153 | measures (mapv (fn [{measure-uri :measure order :order :as comp}] 154 | (let [{:keys [is-numeric?]} (util/strict-get uri->measure measure-uri :key-desc "Measure")] 155 | (types/->MeasureType measure-uri order is-numeric?))) 156 | ordered-measure-components)] 157 | (types/->Dataset ds-uri ds-name dimensions measures))) 158 | 159 | (defn construct-datasets [datasets dimension-components measure-components dimensions measures codelists configuration] 160 | (let [ds->dimension-components (group-by :ds dimension-components) 161 | ds->measure-components (group-by :ds measure-components) 162 | uri->dimension (util/strict-map-by :uri dimensions) 163 | uri->measure (util/strict-map-by :uri measures)] 164 | (map (fn [{uri :ds :as ds}] 165 | (let [dimension-components (get ds->dimension-components uri) 166 | measure-components (get ds->measure-components uri)] 167 | (construct-dataset ds dimension-components measure-components uri->dimension uri->measure codelists configuration))) 168 | datasets))) 169 | 170 | (defn get-all-datasets 171 | "1. Find all datasets 172 | 2. Find all components 173 | 3. Find all codelists 174 | 4. Find all dimensions 175 | 5. Find all measures 176 | 6. Find which measures are numeric" 177 | [repo configuration] 178 | (let [datasets (find-all-datasets repo configuration) 179 | dimension-components (get-dimension-components repo configuration) 180 | measure-components (get-measure-components repo) 181 | dimensions (find-all-dimensions repo) 182 | measures (find-all-measures repo) 183 | codelists (get-all-codelists repo configuration)] 184 | (construct-datasets datasets dimension-components measure-components dimensions measures codelists configuration))) 185 | -------------------------------------------------------------------------------- /src/cubiql/main.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.main 2 | (:require [cubiql.server :as server] 3 | [clojure.tools.cli :as cli] 4 | [cubiql.data :as data] 5 | [cubiql.config :as config] 6 | [clojure.java.io :as io]) 7 | (:gen-class) 8 | (:import [java.io File])) 9 | 10 | (def cli-options 11 | [["-p" "--port PORT" "Port number to start the server on" 12 | :default 8080 13 | :parse-fn #(Integer/parseInt %) 14 | :validate [#(< 0 % 65536) "Port number must be in the range (0, 65536)"]] 15 | 16 | ["-e" "--endpoint ENDPOINT" "Uri of the SPARQL query endpoint to search for datasets. Uses the test repository if not specified" 17 | :parse-fn data/parse-endpoint] 18 | 19 | ["-c" "--configuration CONFIGURATION" "File containing data cube configuration" 20 | :parse-fn io/file 21 | :validate [(fn [^File f] (.exists f)) "Configuration file not found"]]]) 22 | 23 | (defn print-usage [arg-summary] 24 | (println "Usage: cubiql OPTIONS") 25 | (println "The following options are available:") 26 | (println) 27 | (println arg-summary)) 28 | 29 | (defn print-errors-and-usage [errors arg-summary] 30 | (binding [*out* *err*] 31 | (doseq [err errors] 32 | (println err)) 33 | (println) 34 | (print-usage arg-summary))) 35 | 36 | (defn parse-arguments [args] 37 | (let [{:keys [options] :as result} (cli/parse-opts args cli-options) 38 | endpoint (:endpoint options)] 39 | (if (nil? endpoint) 40 | (update result :errors conj "Endpoint required") 41 | result))) 42 | 43 | (defn -main 44 | [& args] 45 | (let [{:keys [options summary errors]} (parse-arguments args)] 46 | (if (some? errors) 47 | (do 48 | (print-errors-and-usage errors summary) 49 | (System/exit 1)) 50 | (let [{:keys [port endpoint configuration]} options 51 | config (if (some? configuration) 52 | (config/read-config configuration) 53 | (config/read-config))] 54 | (server/start-server port endpoint config) 55 | (println "Started server on port " port))))) 56 | -------------------------------------------------------------------------------- /src/cubiql/queries.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.queries 2 | (:require [clojure.string :as string] 3 | [cubiql.types :as types] 4 | [cubiql.vocabulary :refer :all] 5 | [cubiql.query-model :as qm] 6 | [cubiql.config :as config] 7 | [cubiql.util :as util] 8 | [cubiql.types.scalars :as scalars] 9 | [cubiql.schema.mapping.dataset :as dsm])) 10 | 11 | (defn- add-observation-filter-measure-bindings [dataset-mapping model] 12 | (if (dsm/has-measure-type-dimension? dataset-mapping) 13 | (-> model 14 | (qm/add-binding [[:mp qb:measureType]] ::qm/var) 15 | (qm/add-binding [[:mv (qm/->QueryVar "mp")]] ::qm/var)) 16 | (reduce (fn [m {{:keys [uri order] :as measure} :measure :as measure-mapping}] 17 | (let [measure-key (keyword (str "mv" order))] 18 | (qm/add-binding m [[measure-key uri]] ::qm/var))) 19 | model 20 | (dsm/measures dataset-mapping)))) 21 | 22 | (defn get-observation-filter-model [dataset-mapping dim-filter] 23 | (let [m (add-observation-filter-measure-bindings dataset-mapping qm/empty-model)] 24 | (reduce (fn [m [dim value]] 25 | (types/apply-filter dim m value)) 26 | m 27 | dim-filter))) 28 | 29 | (defn apply-model-projections [filter-model dataset observation-selections config] 30 | (reduce (fn [m dm] 31 | (types/apply-projection dm m observation-selections config)) 32 | filter-model 33 | (types/dataset-dimension-measures dataset))) 34 | 35 | (defn apply-model-order-by [model order-by-dims-measures config] 36 | (reduce (fn [m [dim-measure direction]] 37 | (types/apply-order-by dim-measure m direction config)) 38 | model 39 | order-by-dims-measures)) 40 | 41 | (defn filter-model->observations-query [filter-model dataset order-by observation-selections config] 42 | (-> filter-model 43 | (apply-model-projections dataset observation-selections config) 44 | (apply-model-order-by order-by config))) 45 | 46 | (defn get-observation-query [{ds-uri :uri :as dataset} filter-model order-by observation-selections config] 47 | (let [model (filter-model->observations-query filter-model dataset order-by observation-selections config)] 48 | (qm/get-query model "obs" ds-uri))) 49 | 50 | (defn get-observation-page-query [dataset filter-model limit offset order-by-dim-measures observation-selections config] 51 | (str 52 | (get-observation-query dataset filter-model order-by-dim-measures observation-selections config) 53 | " LIMIT " limit " OFFSET " offset)) 54 | 55 | (defn get-dimensions-or [{dims-or :or} configuration] 56 | (if (empty? dims-or) 57 | "" 58 | (let [union-clauses (map (fn [dim] 59 | (str "{ ?struct qb:component ?comp ." 60 | " ?comp qb:dimension <" dim "> . }")) 61 | dims-or)] 62 | (str 63 | "{ SELECT DISTINCT ?ds WHERE {" 64 | " ?ds a qb:DataSet ." 65 | " ?ds qb:structure ?struct ." 66 | " ?struct a qb:DataStructureDefinition ." 67 | (string/join " UNION " union-clauses) 68 | "} }")))) 69 | 70 | (defn get-dimensions-filter [{dims-and :and} configuration] 71 | (if (empty? dims-and) 72 | "" 73 | (let [and-clauses (map-indexed (fn [idx uri] 74 | (let [comp-var (str "?comp" (inc idx))] 75 | (str 76 | "?struct qb:component " comp-var ". \n" 77 | comp-var " a qb:ComponentSpecification .\n" 78 | comp-var " qb:dimension <" (str uri) "> .\n"))) 79 | dims-and)] 80 | (str (string/join "\n" and-clauses))))) 81 | 82 | (defn get-measures-filter [{meas-and :and} configuration] 83 | (if (empty? meas-and) 84 | "" 85 | (let [and-clauses (map-indexed (fn [idx uri] 86 | (let [comp-var (str "?compMeas" (inc idx))] 87 | (str 88 | "?struct qb:component " comp-var ". \n" 89 | comp-var " a qb:ComponentSpecification .\n" 90 | comp-var " qb:measure <" (str uri) "> .\n"))) 91 | meas-and)] 92 | (str (string/join and-clauses))))) 93 | 94 | (defn get-measures-or [{meas-or :or} configuration] 95 | (if (empty? meas-or) 96 | "" 97 | (let [union-clauses (map (fn [meas] 98 | (str "{ ?struct qb:component ?comp .\n" 99 | " ?comp qb:measure <" meas "> . }\n")) 100 | meas-or)] 101 | (str 102 | "{ SELECT DISTINCT ?ds WHERE {\n" 103 | " ?ds a qb:DataSet .\n" 104 | " ?ds qb:structure ?struct .\n" 105 | " ?struct a qb:DataStructureDefinition .\n" 106 | (string/join " UNION " union-clauses) 107 | "} }")))) 108 | 109 | (defn get-attributes-filter [{attr-and :and} configuration] 110 | (if (empty? attr-and) 111 | "" 112 | (let [and-clauses (map-indexed (fn [idx uri] 113 | (let [comp-var (str "?compAttr" (inc idx))] 114 | (str 115 | "?struct qb:component " comp-var ". \n" 116 | comp-var " a qb:ComponentSpecification .\n" 117 | comp-var " qb:attribute <" (str uri) "> .\n"))) 118 | attr-and)] 119 | (str (string/join and-clauses))))) 120 | 121 | (defn get-attributes-or [{attr-or :or} configuration] 122 | (if (empty? attr-or) 123 | "" 124 | (let [union-clauses (map (fn [attr] 125 | (str "{ ?struct qb:component ?comp .\n" 126 | " ?comp qb:attribute <" attr "> . }\n")) 127 | attr-or)] 128 | (str 129 | "{ SELECT DISTINCT ?ds WHERE {\n" 130 | " ?ds a qb:DataSet .\n" 131 | " ?ds qb:structure ?struct .\n" 132 | " ?struct a qb:DataStructureDefinition .\n" 133 | (string/join " UNION " union-clauses) 134 | "} }")))) 135 | 136 | (defn get-data-filter [{data-and :and} configuration] 137 | (if (empty? data-and) 138 | "" 139 | (let [codelist-predicate (config/codelist-predicate configuration) 140 | and-clauses (map-indexed (fn [idx {comp :component vals :values levs :levels}] 141 | (let [incidx (str (inc idx))] 142 | (str " ?struct qb:component ?compdata" incidx " .\n" 143 | " ?compdata" incidx " qb:dimension|qb:attribute <" comp "> .\n" ;;the component can be either a dimension or attribute 144 | " ?compdata" incidx " <" codelist-predicate "> ?cl" incidx ".\n" ;the codelist should contain ONLY the values used at the dataset 145 | (if (some? vals) 146 | (let [cl-vals 147 | (map (fn[uri] (str " ?cl" incidx " skos:member <" uri ">.\n")) vals)] 148 | (str (string/join cl-vals)))) 149 | (if (some? levs) 150 | (let [cl-levs 151 | (map (fn[uri] (str " ?cl" incidx " skos:member/ <" uri ">.\n")) levs)] 152 | (str (string/join cl-levs))))))) 153 | data-and)] 154 | (str (string/join and-clauses))))) 155 | 156 | (defn get-data-or [{data-or :or} configuration] 157 | (if (empty? data-or) 158 | "" 159 | (let [codelist-predicate (config/codelist-predicate configuration) 160 | union-clauses (map (fn [{comp :component vals :values levs :levels}] 161 | (str "{ ?struct qb:component ?comp .\n" 162 | " ?comp qb:dimension|qb:attribute <" comp "> .\n" ;;the component can be either a dimension or attribute 163 | " ?comp <" codelist-predicate "> ?cl.\n" ;;the codelist should contain ONLY the values used at the dataset 164 | " ?cl skos:member ?mem.\n" 165 | (if (some? vals) 166 | (let [members 167 | (map (fn[uri] (str " ?mem=<" uri ">")) vals)] 168 | (str "FILTER(" 169 | (string/join "||" members) 170 | ")"))) 171 | (if (some? levs) 172 | (str " ?mem ?lev.\n" 173 | (let [levelsOr 174 | (map (fn[uri] (str " ?lev=<" uri ">")) levs)] 175 | (str "FILTER(" 176 | (string/join "||" levelsOr) 177 | ")")))) 178 | "}\n")) 179 | data-or)] 180 | (str 181 | "{ SELECT DISTINCT ?ds WHERE {\n" 182 | " ?ds a qb:DataSet .\n" 183 | " ?ds qb:structure ?struct .\n" 184 | " ?struct a qb:DataStructureDefinition .\n" 185 | (string/join " UNION " union-clauses) 186 | "} }")))) 187 | 188 | (defn get-datasets-query [dimensions measures attributes componentValue uri configuration] 189 | (let [dataset-label (config/dataset-label configuration) 190 | schema-lang (config/schema-label-language configuration)] 191 | (str 192 | "PREFIX rdfs: " 193 | "PREFIX qb: " 194 | "PREFIX dcterms: " 195 | "PREFIX skos: " 196 | "SELECT DISTINCT ?ds ?name WHERE {" 197 | " ?ds a qb:DataSet ." 198 | " ?ds qb:structure ?struct ." 199 | " ?struct a qb:DataStructureDefinition ." 200 | (get-dimensions-or dimensions configuration) 201 | (get-measures-or measures configuration) 202 | (get-attributes-or attributes configuration) 203 | (get-data-or componentValue configuration) 204 | " ?ds <" (str dataset-label) "> ?name ." 205 | " FILTER(LANG(?name) = \"" schema-lang "\")" 206 | (get-dimensions-filter dimensions configuration) 207 | (get-measures-filter measures configuration) 208 | (get-attributes-filter attributes configuration) 209 | (get-data-filter componentValue configuration) 210 | (if (some? uri) 211 | (str "FILTER(?ds = <" uri ">) .")) 212 | "}"))) 213 | 214 | (defn get-datasets [repo dimensions measures attributes componentValue uri configuration] 215 | (let [q (get-datasets-query dimensions measures attributes componentValue uri configuration) 216 | results (util/eager-query repo q)] 217 | (map (util/convert-binding-labels [:name]) results))) 218 | 219 | (defn- get-datasets-metadata-query [dataset-uris configuration lang] 220 | (let [label-predicate (str (config/dataset-label configuration))] 221 | (str 222 | "PREFIX qb: " 223 | "PREFIX rdfs: " 224 | "PREFIX dcterms: " 225 | "SELECT DISTINCT * WHERE {" 226 | " VALUES ?ds { " (string/join " " (map #(str "<" % ">") dataset-uris)) " }" 227 | " ?ds a qb:DataSet ." 228 | "{" 229 | " ?ds <" label-predicate "> ?title ." 230 | (when lang 231 | (str "FILTER(LANG(?title) = \"" lang "\")")) 232 | "}" 233 | "UNION {" 234 | " ?ds rdfs:comment ?description ." 235 | (when lang 236 | (str "FILTER(LANG(?description) = \"" lang "\")")) 237 | "}" 238 | "UNION { ?ds dcterms:issued ?issued . }" 239 | "UNION { ?ds dcterms:publisher ?publisher . }" 240 | "UNION { ?ds dcterms:license ?licence . }" 241 | "UNION {" 242 | " SELECT ?modified WHERE {" 243 | " ?ds dcterms:modified ?modified ." 244 | " } ORDER BY DESC(?modified) LIMIT 1" 245 | "}" 246 | "}"))) 247 | 248 | (def metadata-keys #{:title :description :issued :publisher :licence :modified}) 249 | 250 | (defn process-dataset-metadata-bindings [bindings] 251 | (let [{:keys [title description issued publisher licence modified]} (util/to-multimap bindings)] 252 | {:title (util/label->string (first title)) ;;TODO: allow multiple titles? 253 | :description (mapv util/label->string description) 254 | :issued (mapv scalars/grafter-date->datetime issued) 255 | :publisher (or publisher []) 256 | :licence (or licence []) 257 | :modified (some-> (first modified) (scalars/grafter-date->datetime))})) 258 | 259 | (defn get-dataset-metadata [repo dataset-uri configuration lang] 260 | (let [q (get-datasets-metadata-query [dataset-uri] configuration lang) 261 | bindings (util/eager-query repo q)] 262 | (process-dataset-metadata-bindings bindings))) 263 | 264 | (defn get-datasets-metadata [repo dataset-uris configuration lang] 265 | (let [q (get-datasets-metadata-query dataset-uris configuration lang) 266 | results (util/eager-query repo q)] 267 | (group-by :ds results))) 268 | 269 | (defn get-dimension-codelist-values-query [ds-uri configuration lang] 270 | (let [codelist-label (config/codelist-label configuration) 271 | codelist-predicate (config/codelist-predicate configuration)] 272 | (str 273 | "PREFIX qb: " 274 | "PREFIX skos: " 275 | "PREFIX rdfs: " 276 | "PREFIX ui: " 277 | "SELECT ?dim ?member ?label WHERE {" 278 | "<" (str ds-uri) "> qb:structure ?struct ." 279 | "?struct a qb:DataStructureDefinition ." 280 | "?struct qb:component ?comp ." 281 | "?comp qb:dimension ?dim ." 282 | (config/codelist-source configuration) " <" codelist-predicate "> ?list ." 283 | "{" 284 | " ?list skos:member ?member ." 285 | " OPTIONAL {" 286 | " ?member <" (str codelist-label) "> ?label ." 287 | (when lang 288 | (str "FILTER(LANG(?label) = \"" lang "\") .")) 289 | " }" 290 | "} UNION {" 291 | " ?member skos:inScheme ?list ." 292 | " OPTIONAL {" 293 | " ?member <" (str codelist-label) "> ?label ." 294 | (when lang 295 | (str "FILTER(LANG(?label) = \"" lang "\") .")) 296 | " }" 297 | "}" 298 | "}"))) 299 | 300 | (defn get-dimension-codelist-values [repo {:keys [uri] :as dataset} config lang] 301 | (let [dimvalues-query (get-dimension-codelist-values-query uri config lang) 302 | results (util/eager-query repo dimvalues-query)] 303 | (map (util/convert-binding-labels [:label]) results))) 304 | 305 | (defn get-all-enum-dimension-values 306 | "Gets all codelist members for all dimensions across all datasets. Each dimension is expected to have a 307 | single label without a language code. Each codelist item should have at most one label without a language 308 | code used to generate the enum name." 309 | [configuration] 310 | (let [codelist-label (config/codelist-label configuration) 311 | codelist-predicate (config/codelist-predicate configuration) 312 | ignored-dimensions (config/ignored-codelist-dimensions configuration) 313 | dimension-filters (map (fn [dim-uri] (format "FILTER(?dim != <%s>)" dim-uri)) ignored-dimensions)] 314 | (str 315 | "PREFIX qb: " 316 | "PREFIX rdfs: " 317 | "PREFIX skos: " 318 | "SELECT * WHERE {" 319 | " ?ds qb:structure ?struct ." 320 | " ?struct a qb:DataStructureDefinition ." 321 | " ?struct qb:component ?comp ." 322 | " ?comp qb:dimension ?dim ." 323 | (string/join "\n" dimension-filters) 324 | " OPTIONAL { ?dim rdfs:comment ?doc }" 325 | (config/codelist-source configuration) " <" codelist-predicate "> ?codelist ." 326 | "{" 327 | " ?codelist skos:member ?member ." 328 | " ?member <" (str codelist-label) "> ?vallabel ." 329 | "}" 330 | "UNION {" 331 | " ?member skos:inScheme ?codelist ." 332 | " ?member rdfs:label ?vallabel ." 333 | "}" 334 | 335 | "}"))) 336 | 337 | (defn get-measures-by-lang-query [ds-uri lang configuration] 338 | (str 339 | "PREFIX qb: " 340 | "PREFIX rdfs: " 341 | "SELECT ?mt ?label WHERE {" 342 | " <" ds-uri "> qb:structure ?struct ." 343 | " ?struct qb:component ?comp ." 344 | " ?comp qb:measure ?mt ." 345 | " ?mt a qb:MeasureProperty ." 346 | " OPTIONAL {" 347 | " ?mt <" (config/dataset-label configuration) "> ?label ." 348 | " FILTER(LANG(?label) = \"" lang "\")" 349 | " }" 350 | "}")) 351 | 352 | (defn get-dimensions-by-lang-query [ds-uri lang configuration] 353 | (str 354 | "PREFIX qb: " 355 | "PREFIX rdfs: " 356 | "SELECT ?dim ?label WHERE {" 357 | " <" ds-uri "> qb:structure ?struct ." 358 | " ?struct qb:component ?comp ." 359 | " ?comp qb:dimension ?dim ." 360 | " ?dim a qb:DimensionProperty ." 361 | " OPTIONAL {" 362 | " ?dim <" (config/dataset-label configuration) "> ?label ." 363 | " FILTER(LANG(?label) = \"" lang "\")" 364 | " }" 365 | "}")) 366 | 367 | (defn get-dimension-labels-query [configuration] 368 | (str 369 | "PREFIX qb: " 370 | "PREFIX rdfs: " 371 | "SELECT ?dim ?label ?doc WHERE {" 372 | " ?dim a qb:DimensionProperty ." 373 | " { ?dim <" (config/dataset-label configuration) "> ?label . }" 374 | " UNION { ?dim rdfs:comment ?doc . }" 375 | "}")) 376 | 377 | (defn get-measure-labels-query [configuration] 378 | (let [dataset-label (config/dataset-label configuration)] 379 | (str 380 | "PREFIX rdfs: " 381 | "PREFIX qb: " 382 | "SELECT ?measure ?label WHERE {" 383 | " ?measure a qb:MeasureProperty ." 384 | " ?measure <" (str dataset-label) "> ?label ." 385 | "}"))) -------------------------------------------------------------------------------- /src/cubiql/query_model.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.query-model 2 | "Represents a simplified model of a SPARQL query. Within the model, items at specified 'paths' are defined which 3 | can be constrained to a specific value or a collection of associated predicates. The path is a sequence of [key predicate] 4 | pairs where the key identifies the binding in the resulting SPARQL query and the predicate defines the predicate URI 5 | for the RDF relation." 6 | (:require [clojure.string :as string] 7 | [cubiql.vocabulary :refer :all] 8 | [cubiql.vocabulary :refer [time:DateTime]]) 9 | (:import [java.net URI] 10 | [java.util Date] 11 | [java.text SimpleDateFormat] 12 | [java.time.temporal TemporalAccessor] 13 | [java.time.format DateTimeFormatter])) 14 | 15 | (defprotocol QueryItem 16 | "Protocol representing items that have a specific representation within a SPARQL query." 17 | (query-format [this] 18 | "Returns the string representation of this value within a SPARQL query")) 19 | 20 | (defrecord QueryVar [name] 21 | QueryItem 22 | (query-format [_this] (str "?" name))) 23 | 24 | (extend-protocol QueryItem 25 | String 26 | (query-format [s] (str "\"" s "\"")) 27 | 28 | URI 29 | (query-format [uri] (str "<" uri ">")) 30 | 31 | TemporalAccessor 32 | (query-format [dt] 33 | (str "\"" (.format DateTimeFormatter/ISO_OFFSET_DATE_TIME dt) "\"^^")) 34 | 35 | Date 36 | (query-format [date] 37 | ;;TODO: fix date format to correctly include time zone 38 | ;;Stardog treats a time zone offset of 0000 different from Z so just hardcode to Z for now 39 | (let [fmt (SimpleDateFormat. "yyyy-MM-dd'T'HH:mm:ss")] 40 | (str "\"" (.format fmt date) "Z\"^^")))) 41 | 42 | (def empty-model {:bindings {} 43 | :filters {} 44 | :order-by []}) 45 | 46 | (defn- update-binding-value [old-value new-value] 47 | (cond 48 | (= ::var old-value) new-value 49 | (= ::var new-value) old-value 50 | :else (throw (Exception. "Cannot bind multiple values to the same path")))) 51 | 52 | (defn- update-binding-spec [old-spec new-spec] 53 | (-> old-spec 54 | (update ::match update-binding-value (::match new-spec)) 55 | (update ::optional? #(and %1 %2) (::optional? new-spec)))) 56 | 57 | (defn- format-path [path] 58 | (string/join " -> " (map (fn [[key uri]] (str "[" key ", " uri "]")) path))) 59 | 60 | (defn- format-key-path [key-path] 61 | (string/join " -> " (map name key-path))) 62 | 63 | (defn- update-binding [binding path-prefix path binding-spec] 64 | (let [[[key uri :as path-item] & ps] path] 65 | (if-let [[existing-uri spec] (get binding key)] 66 | (if (= uri existing-uri) 67 | (if (seq? ps) 68 | (let [updated (update-binding spec (conj path-prefix path-item) ps binding-spec)] 69 | (assoc binding key [uri updated])) 70 | ;;update existing match spec 71 | (assoc binding key [uri (update-binding-spec spec binding-spec)])) 72 | (throw (IllegalArgumentException. (str "Mismatched URIs for key path " (format-path (conj path-prefix path-item)) " - existing = " existing-uri " attempted = " uri)))) 73 | (let [inner (if (seq? ps) 74 | (update-binding {} (conj path-prefix path-item) ps binding-spec) 75 | binding-spec)] 76 | (assoc binding key [uri inner]))))) 77 | 78 | (defn add-binding 79 | "Adds a binding for the specified path to the given model. Path should be a sequence of [key predicate] pairs 80 | where predicate is the predicate URI for the corresponding triple pattern. Value can be either an object 81 | implementing the QueryItem protocol or the special value ::var indicating a variable. " 82 | [model path value & {:keys [optional?] 83 | :or {optional? false}}] 84 | (if (= 0 (count path)) 85 | model 86 | (update model :bindings (fn [b] (update-binding b [] path {::match value ::optional? optional?}))))) 87 | 88 | (defn- key-path->spec-path [key-path] 89 | (mapcat (fn [k] [k 1]) key-path)) 90 | 91 | (defn- get-spec-by-key-path [{:keys [bindings] :as qm} key-path] 92 | (get-in bindings (key-path->spec-path key-path))) 93 | 94 | (defn- key-path-valid? [{:keys [bindings] :as model} key-path] 95 | (and (seq key-path) 96 | (let [spec (get-spec-by-key-path model key-path)] 97 | (and (map? spec) (contains? spec ::match))))) 98 | 99 | (defn add-filter 100 | "Adds a filter to a path in the given query. A binding for the key path should already exist." 101 | [model key-path filter] 102 | (if (key-path-valid? model key-path) 103 | (update model :filters (fn [fm] (update-in fm key-path (fnil conj []) filter))) 104 | (throw (IllegalArgumentException. (str "No existing binding for key path " (format-key-path key-path)))))) 105 | 106 | (defn add-order-by 107 | "Adds an ORDER BY clause to this query." 108 | [model order-by] 109 | (update model :order-by conj order-by)) 110 | 111 | (defn key-path->var-name 112 | "Converts a binding key path into the corresponding SPARQL variable." 113 | [key-path] 114 | (string/join "" (map name key-path))) 115 | 116 | (defn key-path->var-key 117 | "Converts a binding key path into the corresponding SPARQL variable key." 118 | [key-path] 119 | (keyword (key-path->var-name key-path))) 120 | 121 | (defn- key-path->query-var [key-path] 122 | (->QueryVar (key-path->var-name key-path))) 123 | 124 | (defn- is-literal? [l] 125 | (and (some? l) (not= ::var l))) 126 | 127 | (defn- get-query-var-bindings [{:keys [bindings] :as model}] 128 | (->> bindings 129 | (filter (fn [[k [pred m]]] (is-literal? (::match m)))) 130 | (map (fn [[k [pred m]]] 131 | [(::match m) k])))) 132 | 133 | (defn- format-query-var-binding [[uri var-kw]] 134 | (let [query-var (->QueryVar (name var-kw))] 135 | (str "BIND(" (query-format uri) " AS " (query-format query-var) ")"))) 136 | 137 | (defn- get-query-order-by [{:keys [order-by]}] 138 | (map (fn [ordering] 139 | (let [dir (if (contains? ordering :ASC) :ASC :DESC) 140 | key-path (or (:ASC ordering) (:DESC ordering))] 141 | [dir (key-path->query-var key-path)])) 142 | order-by)) 143 | 144 | (defn- format-ordering [[dir query-var]] 145 | (if (= :ASC dir) 146 | (query-format query-var) 147 | (str "DESC(" (query-format query-var) ")"))) 148 | 149 | (defn- format-query-order-by [orderings] 150 | (if (empty? orderings) 151 | "" 152 | (str "ORDER BY " (string/join " " (map format-ordering orderings))))) 153 | 154 | (defn- get-path-bgps [key-path parent [predicate binding-spec]] 155 | (let [match (::match binding-spec) 156 | optional? (::optional? binding-spec) 157 | nested (dissoc binding-spec ::match ::optional?) 158 | path-var-name (key-path->var-name key-path) 159 | object (if (is-literal? match) match (->QueryVar path-var-name)) 160 | child-bgps (mapcat (fn [[key binding-spec]] 161 | (get-path-bgps (conj key-path key) object binding-spec)) 162 | nested)] 163 | (conj child-bgps {::s parent ::p predicate ::o object ::optional? optional?}))) 164 | 165 | (defn- get-query-triple-patterns [{:keys [bindings] :as model} obs-var-name] 166 | (mapcat (fn [[key spec-pair]] 167 | (get-path-bgps [key] (->QueryVar obs-var-name) spec-pair)) 168 | bindings)) 169 | 170 | (defn- format-query-triple-pattern [#::{:keys [s p o optional?]}] 171 | (let [tp (str (query-format s) " " (query-format p) " " (query-format o) " .")] 172 | (if optional? 173 | (str "OPTIONAL { " tp " }") 174 | tp))) 175 | 176 | (defn- get-filters [path-prefix k v] 177 | (if (map? v) 178 | (mapcat (fn [[ck cv]] 179 | (get-filters (conj path-prefix k) ck cv)) 180 | v) 181 | (let [query-var (key-path->query-var (conj path-prefix k))] 182 | (map (fn [f] [query-var f]) v)))) 183 | 184 | (defn- get-query-filters [{:keys [filters] :as model}] 185 | (mapcat (fn [[k v]] (get-filters [] k v)) filters)) 186 | 187 | (defn- format-filter [[query-var [fun value]]] 188 | (format "FILTER(%s %s %s)" (query-format query-var) fun (query-format value))) 189 | 190 | (defn get-query 191 | "Returns the observation SPARQL query for the given query model." 192 | [model obs-var-name dataset-uri] 193 | (let [var-bindings (get-query-var-bindings model) 194 | binding-clauses (string/join " " (map format-query-var-binding var-bindings)) 195 | obs-var (->QueryVar obs-var-name)] 196 | (str 197 | "PREFIX qb: " 198 | "SELECT * WHERE {" 199 | binding-clauses 200 | " " (query-format obs-var) " a qb:Observation ." 201 | " " (query-format obs-var) " qb:dataSet " (query-format dataset-uri) " ." 202 | (string/join " " (map format-query-triple-pattern (get-query-triple-patterns model obs-var-name))) 203 | (string/join " " (map format-filter (get-query-filters model))) 204 | "}" (format-query-order-by (get-query-order-by model))))) 205 | 206 | (defn get-observation-count-query 207 | "Returns the SPARQL query for finding the number of matching observations for the given query model." 208 | [model obs-var-name dataset-uri] 209 | (let [obs-var (->QueryVar obs-var-name)] 210 | (str 211 | "PREFIX qb: " 212 | "SELECT (COUNT(*) AS ?c) WHERE {" 213 | " " (query-format obs-var) " a qb:Observation ." 214 | " " (query-format obs-var) " qb:dataSet " (query-format dataset-uri) " ." 215 | (string/join " " (map format-query-triple-pattern (get-query-triple-patterns model obs-var-name))) 216 | (string/join " " (map format-filter (get-query-filters model))) 217 | "}"))) 218 | 219 | (defn get-observation-aggregation-query 220 | "Returns the SPARQL query for aggregating the specified measure for an observation query model." 221 | [model aggregation-fn dataset-uri measure-uri] 222 | (let [measure-var-name "mv" 223 | obs-var-name "obs" 224 | sparql-fn (string/upper-case (name aggregation-fn))] 225 | (str 226 | "PREFIX qb: " 227 | "SELECT (" sparql-fn "(?mv) AS ?" (name aggregation-fn) ") WHERE {" 228 | " ?obs a qb:Observation ." 229 | " ?obs qb:dataSet " (query-format dataset-uri) " ." 230 | (string/join " " (map format-query-triple-pattern (get-query-triple-patterns model obs-var-name))) 231 | (string/join " " (map format-filter (get-query-filters model))) 232 | " ?obs " (query-format measure-uri) " ?" measure-var-name " ." 233 | "}"))) 234 | 235 | (defn get-path-binding-value 236 | "Returns the value bound at the given key path." 237 | [model key-path] 238 | (::match (get-spec-by-key-path model key-path))) 239 | 240 | (defn is-path-binding-optional? 241 | "Returns whether the binding at the given key paths is OPTIONAL." 242 | [model key-path] 243 | (::optional? (get-spec-by-key-path model key-path))) 244 | 245 | (defn get-path-filters 246 | "Returns all the filters associated with the binding at the given key path." 247 | [{:keys [filters] :as qm} key-path] 248 | (get-in filters key-path)) 249 | 250 | (defn get-order-by 251 | "Returns the ORDER BY clauses for the given query model." 252 | [qm] 253 | (:order-by qm)) 254 | -------------------------------------------------------------------------------- /src/cubiql/resolvers.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.resolvers 2 | (:require [cubiql.queries :as queries] 3 | [cubiql.util :as util] 4 | [cubiql.context :as context] 5 | [cubiql.query-model :as qm] 6 | [clojure.spec.alpha :as s] 7 | [cubiql.config :as config]) 8 | (:import [cubiql.types Dimension MeasureType])) 9 | 10 | (s/def ::order-direction #{:ASC :DESC}) 11 | (s/def ::dimension #(instance? Dimension %)) 12 | (s/def ::measure #(instance? MeasureType %)) 13 | (s/def ::dimension-measure (s/or :dimension ::dimension :measure ::measure)) 14 | (s/def ::order-item (s/cat :dimmeasure ::dimension-measure :direction ::order-direction)) 15 | (s/def ::order-by (s/coll-of ::order-item)) 16 | (s/def ::dimension-filter (constantly true)) ;TODO: specify properly 17 | (s/def ::dimensions-filter (s/map-of ::dimension ::dimension-filter)) 18 | 19 | (defn get-observation-count [repo ds-uri filter-model] 20 | (let [query (qm/get-observation-count-query filter-model "obs" ds-uri) 21 | results (util/eager-query repo query)] 22 | (:c (first results)))) 23 | 24 | (defn total-count-required? 25 | "Whether the total number of matching observations needs to be queried based on the selected observations 26 | fields." 27 | [{:keys [page] :as selections}] 28 | (or (contains? selections :total_matches) 29 | (contains? page :next_page))) 30 | 31 | (defn resolve-observations [context args field] 32 | (let [{:keys [uri] :as dataset} (::dataset field) 33 | ds-mapping (context/get-dataset-mapping context uri) 34 | repo (context/get-repository context) 35 | dimension-filter (::dimensions-filter args) 36 | filter-model (queries/get-observation-filter-model ds-mapping dimension-filter) 37 | selections (context/get-selections context) 38 | total-matches (if (total-count-required? selections) 39 | (get-observation-count repo uri filter-model))] 40 | (merge 41 | (select-keys args [::dimensions-filter ::order-by]) 42 | {::dataset dataset 43 | ::filter-model filter-model 44 | :total_matches total-matches 45 | :aggregations {::dimensions-filter dimension-filter ::filter-model filter-model :ds-uri uri}}))) 46 | 47 | (defn resolve-observations-sparql-query [context _args obs-field] 48 | (let [config (context/get-configuration context) 49 | #::{:keys [dataset observation-selections order-by filter-model]} obs-field 50 | model (queries/filter-model->observations-query filter-model dataset order-by observation-selections config)] 51 | (qm/get-query model "obs" (:uri dataset)))) 52 | 53 | (def default-limit 10) 54 | (def default-max-observations-page-size 1000) 55 | 56 | (defn get-limit [args configuration] 57 | (let [max-limit (or (config/max-observations-page-size configuration) default-max-observations-page-size)] 58 | (min (max 0 (or (:first args) default-limit)) max-limit))) 59 | 60 | (defn get-offset [args] 61 | (max 0 (or (:after args) 0))) 62 | 63 | (defn calculate-next-page-offset [offset limit total-matches] 64 | (if (some? total-matches) 65 | (let [next-offset (+ offset limit)] 66 | (if (> total-matches next-offset) 67 | next-offset)))) 68 | 69 | (defn wrap-pagination-resolver [inner-resolver] 70 | (fn [context args observations-field] 71 | (let [configuration (context/get-configuration context) 72 | limit (get-limit args configuration) 73 | offset (get-offset args) 74 | total-matches (:total_matches observations-field) 75 | page {::page-offset offset ::page-size limit} 76 | result (inner-resolver context (assoc args ::page page) observations-field) 77 | page-count (count (::observation-results result)) 78 | next-page (calculate-next-page-offset offset limit total-matches)] 79 | (assoc result :next_page next-page 80 | :count page-count)))) 81 | 82 | (defn inner-resolve-observations-page [context args observations-field] 83 | (let [order-by (::order-by observations-field) 84 | dataset (::dataset observations-field) 85 | observation-selections (::observation-selections observations-field) 86 | filter-model (::filter-model observations-field) 87 | #::{:keys [page-offset page-size]} (::page args) 88 | config (context/get-configuration context) 89 | query (queries/get-observation-page-query dataset filter-model page-size page-offset order-by observation-selections config) 90 | repo (context/get-repository context) 91 | results (util/eager-query repo query)] 92 | {::observation-results results})) 93 | 94 | (def resolve-observations-page (wrap-pagination-resolver inner-resolve-observations-page)) 95 | 96 | (defn get-lang [field] 97 | (get-in field [::options ::lang])) 98 | 99 | (defn wrap-options [inner-resolver] 100 | (fn [context args field] 101 | (let [opts (::options field) 102 | result (inner-resolver context args field)] 103 | (cond 104 | (map? result) 105 | (assoc result ::options opts) 106 | 107 | (seqable? result) 108 | (map (fn [r] (assoc r ::options opts)) result) 109 | 110 | :else 111 | (throw (ex-info "Unexpected result type when associating options" {:result result})))))) 112 | 113 | (defn resolve-datasets [context {:keys [dimensions measures attributes componentValue uri] :as args} parent-field] 114 | (let [repo (context/get-repository context) 115 | config (context/get-configuration context) 116 | lang (get-lang parent-field) 117 | results (queries/get-datasets repo dimensions measures attributes componentValue uri config) 118 | ds-uris (map :ds results) 119 | ds->metadata-bindings (queries/get-datasets-metadata repo ds-uris config lang)] 120 | (mapv (fn [ds] 121 | (let [{:keys [uri] :as dataset} (util/rename-key ds :ds :uri :strict? true) 122 | metadata-bindings (get ds->metadata-bindings uri) 123 | metadata (queries/process-dataset-metadata-bindings metadata-bindings) 124 | with-metadata (merge dataset metadata) 125 | {:keys [schema] :as dataset-mapping} (context/get-dataset-mapping context uri)] 126 | (assoc with-metadata :schema (name schema)))) 127 | results))) 128 | 129 | (defn exec-observation-aggregation [repo dataset measure filter-model aggregation-fn] 130 | (let [q (qm/get-observation-aggregation-query filter-model aggregation-fn (:uri dataset) (:uri measure)) 131 | results (util/eager-query repo q)] 132 | (get (first results) aggregation-fn))) 133 | 134 | (defn resolve-observations-aggregation [aggregation-fn 135 | context 136 | {:keys [measure] :as args} 137 | {:keys [ds-uri] :as aggregation-field}] 138 | (let [repo (context/get-repository context) 139 | dataset (context/get-dataset context ds-uri) 140 | filter-model (::filter-model aggregation-field)] 141 | (exec-observation-aggregation repo dataset measure filter-model aggregation-fn))) 142 | 143 | (defn- resolve-dataset-measures [repo dataset-uri lang configuration] 144 | (let [q (queries/get-measures-by-lang-query dataset-uri lang configuration) 145 | results (util/eager-query repo q)] 146 | (mapv (fn [{:keys [mt label]}] 147 | {:uri mt 148 | :label (util/label->string label)}) 149 | results))) 150 | 151 | (defn- metadata-requested? 152 | "Whether any items of dataset metadata have been selected in the query" 153 | [selections] 154 | (boolean (some (fn [k] (contains? selections k)) queries/metadata-keys))) 155 | 156 | (defn dataset-resolver [{:keys [uri] :as dataset-mapping}] 157 | (fn [context _args field] 158 | (let [repo (context/get-repository context) 159 | config (context/get-configuration context) 160 | lang (get-lang field) 161 | selections (context/get-selections context) 162 | dataset (context/get-dataset context uri) 163 | metadata (if (metadata-requested? selections) 164 | (queries/get-dataset-metadata repo uri config lang) 165 | {}) 166 | dataset-result (merge dataset-mapping metadata)] 167 | (assoc dataset-result ::dataset dataset)))) 168 | 169 | (defn combine-dimension-results [dimension-results dimension-codelist-results] 170 | (let [dimension-uri->codelist (group-by :dim dimension-codelist-results)] 171 | (mapv (fn [{:keys [uri] :as dim}] 172 | (if-let [codelist-members (get dimension-uri->codelist uri)] 173 | (let [values (map (fn [{:keys [member label]}] {:uri member :label label}) codelist-members)] 174 | (assoc dim :values values)) 175 | dim)) 176 | dimension-results))) 177 | 178 | (defn get-dataset-dimensions [repo dataset-uri lang configuration] 179 | (let [q (queries/get-dimensions-by-lang-query dataset-uri lang configuration) 180 | results (util/eager-query repo q)] 181 | (map (fn [{:keys [dim label] :as bindings}] 182 | {:uri dim 183 | :label (util/label->string label)}) 184 | results))) 185 | 186 | (defn dataset-dimensions-resolver [context _args {dataset-uri :uri :as ds-field}] 187 | (let [lang (get-lang ds-field) 188 | repo (context/get-repository context) 189 | dataset (context/get-dataset context dataset-uri) 190 | config (context/get-configuration context) 191 | dimension-results (get-dataset-dimensions repo dataset-uri lang config) 192 | dimension-codelists (queries/get-dimension-codelist-values repo dataset config lang)] 193 | (combine-dimension-results dimension-results dimension-codelists))) 194 | 195 | (defn resolve-cubiql [_context {lang :lang_preference :as args} _field] 196 | {::options {::lang lang}}) 197 | 198 | (defn create-dataset-dimensions-resolver [dataset-mapping] 199 | (fn [context _args {dataset-uri :uri :as field}] 200 | (let [lang (get-lang field) 201 | repo (context/get-repository context) 202 | config (context/get-configuration context) 203 | dimension-results (get-dataset-dimensions repo dataset-uri lang config) 204 | dimension-codelists (queries/get-dimension-codelist-values repo dataset-mapping config lang)] 205 | (combine-dimension-results dimension-results dimension-codelists)))) 206 | 207 | (defn create-dataset-measures-resolver [dataset-mapping] 208 | (fn [context _args field] 209 | (let [lang (get-lang field) 210 | repo (context/get-repository context) 211 | configuration (context/get-configuration context)] 212 | (resolve-dataset-measures repo (:uri dataset-mapping) lang configuration)))) -------------------------------------------------------------------------------- /src/cubiql/schema.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.schema 2 | (:require [cubiql.types :as types] 3 | [cubiql.resolvers :as resolvers] 4 | [cubiql.schema-model :as sm] 5 | [cubiql.context :as context] 6 | [cubiql.schema.mapping.dataset :as dsm] 7 | [com.walmartlabs.lacinia.schema :as ls] 8 | [cubiql.util :as util] 9 | [cubiql.schema.mapping.dataset :as ds-mapping] 10 | [grafter.rdf :as rdf]) 11 | (:import [cubiql.types EnumType RefPeriodType RefAreaType DecimalType StringType UnmappedType StringMeasureType FloatMeasureType MappedEnumType GroupMapping 12 | MeasureDimensionType])) 13 | 14 | (defprotocol ArgumentTransform 15 | (transform-argument [this graphql-value] 16 | "Converts a GraphQL argument value from the incoming query into the corresponding RDF value")) 17 | 18 | (defprotocol ResultTransform 19 | (transform-result [this inner-value] 20 | "Transforms an RDF result from a query result into the corresponding GraphQL schema value")) 21 | 22 | (defprotocol ToGraphQLInputType 23 | (->input-type-name [this] 24 | "Returns the name of the type to use for the argument representation of this type")) 25 | 26 | (defprotocol ToGraphQLOutputType 27 | (->output-type-name [this] 28 | "Return the name of the type to use for the output representation of this type")) 29 | 30 | (extend-protocol ToGraphQLInputType 31 | RefAreaType 32 | (->input-type-name [_ref-area-type] :uri) 33 | 34 | RefPeriodType 35 | (->input-type-name [_ref-period-type] :ref_period_filter) 36 | 37 | EnumType 38 | (->input-type-name [_enum-type] :uri) 39 | 40 | DecimalType 41 | (->input-type-name [_decimal-type] 'Float) 42 | 43 | StringType 44 | (->input-type-name [_string-type] 'String) 45 | 46 | MeasureDimensionType 47 | (->input-type-name [_measure-dimension-type] :uri) 48 | 49 | UnmappedType 50 | (->input-type-name [_unmapped-type] 'String) 51 | 52 | MappedEnumType 53 | (->input-type-name [{:keys [enum-type-name]}] enum-type-name)) 54 | 55 | (extend-protocol ToGraphQLOutputType 56 | RefAreaType 57 | (->output-type-name [_ref-area-type] :ref_area) 58 | 59 | RefPeriodType 60 | (->output-type-name [_ref-period-type] :ref_period) 61 | 62 | EnumType 63 | (->output-type-name [_enum-type] :uri) 64 | 65 | DecimalType 66 | (->output-type-name [_decimal-type] 'Float) 67 | 68 | StringType 69 | (->output-type-name [_string-type] 'String) 70 | 71 | MeasureDimensionType 72 | (->output-type-name [_measure-dimension-type] :uri) 73 | 74 | UnmappedType 75 | (->output-type-name [_unmapped-type] 'String) 76 | 77 | MappedEnumType 78 | (->output-type-name [{:keys [enum-type-name]}] enum-type-name) 79 | 80 | StringMeasureType 81 | (->output-type-name [_string-type] 'String) 82 | 83 | FloatMeasureType 84 | (->output-type-name [_float-type] 'Float)) 85 | 86 | (defn identity-transform [_type value] value) 87 | 88 | (extend-protocol ResultTransform 89 | RefAreaType 90 | (transform-result [_ref-area-type result] result) 91 | 92 | RefPeriodType 93 | (transform-result [_ref-period-type result] result) 94 | 95 | StringType 96 | (transform-result [_string-type result] (str result)) 97 | 98 | UnmappedType 99 | (transform-result [_type result] (str result)) 100 | 101 | FloatMeasureType 102 | (transform-result [_this r] (some-> r double)) 103 | 104 | StringMeasureType 105 | (transform-result [_this r] (some-> r str)) 106 | 107 | MappedEnumType 108 | (transform-result [{:keys [items] :as _this} result] 109 | (->> items 110 | (util/find-first (fn [{:keys [value]}] 111 | (= value result))) 112 | (:name)))) 113 | 114 | (def default-result-transform-impl 115 | {:transform-result identity-transform}) 116 | 117 | (extend EnumType ResultTransform default-result-transform-impl) 118 | (extend DecimalType ResultTransform default-result-transform-impl) 119 | (extend MeasureDimensionType default-result-transform-impl) 120 | 121 | (def default-argument-transform-impl 122 | {:transform-argument identity-transform}) 123 | 124 | (extend RefAreaType ArgumentTransform default-argument-transform-impl) 125 | (extend RefPeriodType ArgumentTransform default-argument-transform-impl) 126 | (extend EnumType ArgumentTransform default-argument-transform-impl) 127 | (extend DecimalType ArgumentTransform default-argument-transform-impl) 128 | (extend StringType ArgumentTransform default-argument-transform-impl) 129 | (extend MeasureDimensionType default-argument-transform-impl) 130 | (extend UnmappedType ArgumentTransform default-argument-transform-impl) 131 | 132 | (extend-protocol ArgumentTransform 133 | MappedEnumType 134 | (transform-argument [{:keys [items] :as _this} graphql-value] 135 | (:value (types/find-item-by-name graphql-value items))) 136 | 137 | GroupMapping 138 | (transform-argument [{:keys [items] :as _this} graphql-value] 139 | (:value (types/find-item-by-name graphql-value items))) 140 | 141 | UnmappedType 142 | (transform-argument [{:keys [type-uri] :as _unmapped-type} graphql-value] 143 | ;;map values as string literals if no type associated with the dimension 144 | (if (some? type-uri) 145 | (rdf/literal graphql-value type-uri) 146 | graphql-value))) 147 | 148 | (defn create-aggregation-resolver [dataset-mapping aggregation-fn aggregation-measures-enum] 149 | (fn [context {:keys [measure] :as args} field] 150 | (let [measure-uri (transform-argument aggregation-measures-enum measure) 151 | {:keys [type] :as measure-mapping} (dsm/get-measure-by-uri dataset-mapping measure-uri) 152 | result (resolvers/resolve-observations-aggregation aggregation-fn context {:measure measure-mapping} field)] 153 | (transform-result type result)))) 154 | 155 | (defn get-order-by 156 | "Returns an ordered list of [component-uri sort-direction] given a sequence of component URIs and an associated 157 | (possibly partial) specification for the order direction of each field. If the sort direction is not specified 158 | for an ordered field it will be sorted in ascending order." 159 | [{:keys [order order_spec] :as args} dataset-mapping] 160 | (map (fn [dm-uri] 161 | (let [{:keys [uri] :as comp} (dsm/get-component-by-uri dataset-mapping dm-uri) 162 | direction (get order_spec uri :ASC)] 163 | [(dsm/component-mapping->component comp) direction])) 164 | order)) 165 | 166 | (defn map-dimension-filter [dimensions dataset-mapping] 167 | (into {} (map (fn [{:keys [uri dimension] :as dim-mapping}] 168 | [dimension (get dimensions uri)]) 169 | (dsm/dimensions dataset-mapping)))) 170 | 171 | (defn map-dataset-observation-args [{:keys [dimensions order order_spec]} dataset-mapping] 172 | (let [mapped-dimensions (into {} (map (fn [[field-name value]] 173 | (let [{:keys [uri type]} (dsm/get-dimension-by-field-name dataset-mapping field-name)] 174 | [uri (transform-argument type value)])) 175 | dimensions)) 176 | mapped-order (mapv (fn [component-enum] 177 | (:uri (dsm/get-component-by-enum-name dataset-mapping component-enum))) 178 | order) 179 | mapped-order-spec (into {} (map (fn [[field-name dir]] 180 | [(:uri (dsm/get-component-by-field-name dataset-mapping field-name)) dir]) 181 | order_spec))] 182 | {:dimensions mapped-dimensions 183 | :order mapped-order 184 | :order_spec mapped-order-spec})) 185 | 186 | (defn get-observation-selections [context] 187 | (get-in (context/get-selections context) [:page :observation])) 188 | 189 | (defn map-observation-selections [dataset-mapping selections] 190 | (into {} (keep (fn [{:keys [field-name uri] :as comp}] 191 | (when (contains? selections field-name) 192 | [uri (get selections field-name)])) 193 | (dsm/components dataset-mapping)))) 194 | 195 | (defn create-observation-resolver [dataset-mapping] 196 | (fn [context args field] 197 | (let [{:keys [dimensions] :as mapped-args} (map-dataset-observation-args args dataset-mapping) 198 | updated-args {::resolvers/dimensions-filter (map-dimension-filter dimensions dataset-mapping) 199 | ::resolvers/order-by (get-order-by mapped-args dataset-mapping)} 200 | selected-observation-fields (get-observation-selections context) 201 | result (resolvers/resolve-observations context updated-args field)] 202 | (assoc result ::resolvers/observation-selections (map-observation-selections dataset-mapping selected-observation-fields))))) 203 | 204 | (defn get-measure-type-measure-value 205 | "Gets the value for the specified measure from a map of observation query bindings for a dataset with an 206 | explicit measure dimension. The observation should have a single measure defined by the qb:measureType which 207 | is bound to the ?mp variable." 208 | [{:keys [uri] :as measure} {:keys [mp mv] :as bindings}] 209 | (if (= uri mp) 210 | mv)) 211 | 212 | (defn get-multi-measure-value 213 | "Gets the value for the specified measure from a map of observation query bindings for a dataset with multiple 214 | measure values per observation. A value for each measure should be associated with the observation." 215 | [{:keys [order] :as measure} bindings] 216 | (let [measure-key (keyword (str "mv" order))] 217 | (get bindings measure-key))) 218 | 219 | (defn- map-measure-values 220 | "Returns a sequence of [field-name value] pairs for each measure type defined for the given dataset for a single 221 | bindings row of the observations query results." 222 | [dataset-model bindings] 223 | (let [measure-value-fn (if (dsm/has-measure-type-dimension? dataset-model) 224 | get-measure-type-measure-value 225 | get-multi-measure-value)] 226 | (map (fn [{:keys [field-name type measure] :as measure-mapping}] 227 | (let [value (measure-value-fn measure bindings)] 228 | [field-name (transform-result type value)])) 229 | (dsm/measures dataset-model)))) 230 | 231 | (defn get-observation-result [dataset-model bindings] 232 | (let [dimension-results (map (fn [{:keys [field-name type dimension] :as dimension-mapping}] 233 | (let [result (types/project-result dimension bindings)] 234 | [field-name (transform-result type result)])) 235 | (dsm/dimensions dataset-model)) 236 | measure-results (map-measure-values dataset-model bindings)] 237 | (into {:uri (:obs bindings)} (concat dimension-results measure-results)))) 238 | 239 | (defn create-aggregation-field [dataset-mapping field-name aggregation-measures-enum-mapping aggregation-fn] 240 | {field-name 241 | {:type 'Float 242 | :args {:measure {:type (sm/non-null aggregation-measures-enum-mapping) :description "The measure to aggregate"}} 243 | :resolve (create-aggregation-resolver dataset-mapping aggregation-fn aggregation-measures-enum-mapping)}}) 244 | 245 | (defn get-aggregations-schema-model [dataset-mapping aggregation-measures-enum-mapping] 246 | {:type 247 | {:fields 248 | (merge 249 | (create-aggregation-field dataset-mapping :max aggregation-measures-enum-mapping :max) 250 | (create-aggregation-field dataset-mapping :min aggregation-measures-enum-mapping :min) 251 | (create-aggregation-field dataset-mapping :sum aggregation-measures-enum-mapping :sum) 252 | (create-aggregation-field dataset-mapping :average aggregation-measures-enum-mapping :avg))}}) 253 | 254 | (defn dataset-observation-dimensions-input-schema-model [dataset-mapping] 255 | (into {} (map (fn [{:keys [field-name type] :as dim}] 256 | [field-name {:type (->input-type-name type)}]) 257 | (dsm/dimensions dataset-mapping)))) 258 | 259 | (defn dataset-observation-schema-model [dataset-mapping] 260 | (let [field-types (map (fn [{:keys [field-name type] :as comp}] 261 | [field-name {:type (->output-type-name type)}]) 262 | (dsm/components dataset-mapping))] 263 | (into {:uri {:type :uri}} 264 | field-types))) 265 | 266 | (defn dataset-order-spec-schema-model [dataset-mapping] 267 | (into {} (map (fn [{:keys [field-name] :as comp}] 268 | [field-name {:type :sort_direction}])) 269 | (dsm/components dataset-mapping))) 270 | 271 | (defn create-dataset-observations-page-resolver [dataset-mapping] 272 | (fn [context args observations-field] 273 | (let [result (resolvers/resolve-observations-page context args observations-field) 274 | mapped-result (mapv (fn [obs-bindings] 275 | (get-observation-result dataset-mapping obs-bindings)) 276 | (::resolvers/observation-results result))] 277 | (assoc result :observation mapped-result)))) 278 | 279 | (defn get-observation-schema-model [dataset-mapping] 280 | (let [dimensions-measures-enum-mapping (dsm/components-enum-group dataset-mapping) 281 | obs-model {:type 282 | {:fields 283 | {:sparql 284 | {:type 'String 285 | :description "SPARQL query used to retrieve matching observations." 286 | :resolve :resolve-observation-sparql-query} 287 | :page 288 | {:type 289 | {:fields 290 | {:next_page {:type :SparqlCursor :description "Cursor to the next page of results"} 291 | :count {:type 'Int} 292 | :observation {:type [{:fields (dataset-observation-schema-model dataset-mapping)}] :description "List of observations on this page"}}} 293 | :args {:after {:type :SparqlCursor} 294 | :first {:type 'Int}} 295 | :description "Page of results to retrieve." 296 | :resolve (create-dataset-observations-page-resolver dataset-mapping)} 297 | :total_matches {:type 'Int}}} 298 | :args 299 | {:dimensions {:type {:fields (dataset-observation-dimensions-input-schema-model dataset-mapping)}} 300 | :order {:type [dimensions-measures-enum-mapping]} 301 | :order_spec {:type {:fields (dataset-order-spec-schema-model dataset-mapping)}}} 302 | :resolve (resolvers/wrap-options (create-observation-resolver dataset-mapping))} 303 | aggregation-measures-enum-mapping (dsm/aggregation-measures-enum-group dataset-mapping)] 304 | (if (nil? aggregation-measures-enum-mapping) 305 | obs-model 306 | (let [aggregation-fields (get-aggregations-schema-model dataset-mapping aggregation-measures-enum-mapping)] 307 | (assoc-in obs-model [:type :fields :aggregations] aggregation-fields))))) 308 | 309 | ;;TODO: move? replace with protocol? 310 | (defn- is-enum-type? [type] 311 | (instance? MappedEnumType type)) 312 | 313 | (defn- annotate-dimension-values [{:keys [type] :as dimension-mapping} dimension-values] 314 | (when dimension-values 315 | (if (is-enum-type? type) 316 | (mapv (fn [{:keys [uri] :as enum-value}] 317 | (let [enum-item (util/find-first #(= uri (:value %)) (:items type))] 318 | (-> enum-value 319 | (assoc :enum_name (name (:name enum-item))) 320 | (ls/tag-with-type :enum_dim_value)))) 321 | dimension-values) 322 | (mapv #(ls/tag-with-type % :unmapped_dim_value) dimension-values)))) 323 | 324 | (defn annotate-dataset-dimensions [dataset-mapping dimensions] 325 | (mapv (fn [{:keys [uri] :as dim}] 326 | (let [{:keys [enum-name] :as dim-mapping} (dsm/get-dimension-by-uri dataset-mapping uri)] 327 | (-> dim 328 | (assoc :enum_name (name enum-name)) 329 | (update :values #(annotate-dimension-values dim-mapping %))))) 330 | dimensions)) 331 | 332 | (defn create-dataset-dimensions-resolver [dataset-mapping] 333 | (fn [context args field] 334 | (let [inner-resolver (resolvers/create-dataset-dimensions-resolver dataset-mapping) 335 | results (inner-resolver context args field)] 336 | (annotate-dataset-dimensions dataset-mapping results)))) 337 | 338 | (defn create-global-dataset-dimensions-resolver [all-dataset-mappings] 339 | (let [uri->dataset-mapping (util/strict-map-by :uri all-dataset-mappings)] 340 | (fn [context args {:keys [uri] :as dataset-mapping-field}] 341 | (let [dataset-mapping (util/strict-get uri->dataset-mapping uri) 342 | results (resolvers/dataset-dimensions-resolver context args dataset-mapping-field)] 343 | (annotate-dataset-dimensions dataset-mapping results))))) 344 | 345 | (defn map-dataset-measure-results [dataset-mapping results] 346 | (mapv (fn [{:keys [uri] :as result}] 347 | (let [measure-mapping (dsm/get-measure-by-uri dataset-mapping uri)] 348 | (assoc result :enum_name (name (:enum-name measure-mapping))))) 349 | results)) 350 | 351 | (defn create-dataset-measures-resolver [dataset-mapping] 352 | (fn [context args field] 353 | (let [inner-resolver (resolvers/create-dataset-measures-resolver dataset-mapping) 354 | results (inner-resolver context args field)] 355 | (map-dataset-measure-results dataset-mapping results)))) 356 | 357 | ;;TODO: refactor with create-dataset-measures-resolver 358 | (defn global-dataset-measures-resolver [context args {:keys [uri] :as dataset-field}] 359 | (let [dataset-mapping (context/get-dataset-mapping context uri) 360 | inner-resolver (resolvers/create-dataset-measures-resolver dataset-mapping) 361 | results (inner-resolver context args dataset-mapping)] 362 | (map-dataset-measure-results dataset-mapping results))) 363 | 364 | (defn create-dataset-resolver [dataset-mapping] 365 | (resolvers/wrap-options 366 | ;;TODO: add spec for resolver result? 367 | ;;should contain keys defined in dataset schema 368 | (resolvers/dataset-resolver dataset-mapping))) 369 | 370 | (defn get-query-schema-model [{:keys [schema] :as dataset-mapping}] 371 | (let [observations-model (get-observation-schema-model dataset-mapping)] 372 | {schema 373 | {:type 374 | {:implements [:dataset_meta] 375 | :fields {:uri {:type :uri :description "Dataset URI"} 376 | :title {:type 'String :description "Dataset title"} 377 | :description {:type ['String] :description "Dataset descriptions"} 378 | :licence {:type [:uri] :description "URIs of the licences the dataset is published under"} 379 | :issued {:type [:DateTime] :description "When the dataset was issued"} 380 | :modified {:type :DateTime :description "When the dataset was last modified"} 381 | :publisher {:type [:uri] :description "URIs of the publishers of the dataset"} 382 | :schema {:type 'String :description "Name of the GraphQL query root field corresponding to this dataset"} 383 | :dimensions {:type [:dim] 384 | :resolve (create-dataset-dimensions-resolver dataset-mapping) 385 | :description "Dimensions within the dataset"} 386 | :measures {:type [:measure] 387 | :description "Measure types within the dataset" 388 | :resolve (create-dataset-measures-resolver dataset-mapping)} 389 | :observations observations-model} 390 | :description (dsm/description dataset-mapping)} 391 | :resolve (create-dataset-resolver dataset-mapping)}})) 392 | 393 | (defn dimension->enum-schema [{:keys [type] :as dim}] 394 | (when (instance? MappedEnumType type) 395 | (let [{:keys [enum-type-name doc items]} type] 396 | (if (some? doc) 397 | {enum-type-name {:values (mapv :name items) :description doc}} 398 | {enum-type-name {:values (mapv :name items)}})))) 399 | 400 | (defn dataset-enum-types-schema [dataset-mapping] 401 | (apply merge (map (fn [dim] 402 | (dimension->enum-schema dim)) 403 | (dsm/dimensions dataset-mapping)))) 404 | 405 | (defn get-qb-fields-schema [dataset-mappings] 406 | (reduce (fn [{:keys [qb-fields] :as acc} dsm] 407 | (let [m (get-query-schema-model dsm) 408 | [field-name field] (first m) 409 | field-schema (sm/visit-field [field-name] field-name field :objects) 410 | ds-enums-schema {:enums (dataset-enum-types-schema dsm)} 411 | field (::sm/field field-schema) 412 | schema (::sm/schema field-schema) 413 | schema (sm/merge-schemas schema ds-enums-schema)] 414 | {:qb-fields (assoc qb-fields field-name field) 415 | :schema (sm/merge-schemas (:schema acc) schema)})) 416 | {:qb-fields {} :schema {}} 417 | dataset-mappings)) 418 | -------------------------------------------------------------------------------- /src/cubiql/schema/mapping/dataset.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.schema.mapping.dataset 2 | (:require [cubiql.util :as util] 3 | [cubiql.vocabulary :refer [qb:measureType]])) 4 | 5 | (def uri :uri) 6 | (def schema :schema) 7 | (def dimensions :dimensions) 8 | (def measures :measures) 9 | 10 | (defn description [dataset-mapping] 11 | ;;TODO: add dataset description to mapping! 12 | "") 13 | 14 | (defn components [dataset-mapping] 15 | (concat (dimensions dataset-mapping) (measures dataset-mapping))) 16 | 17 | (defn numeric-measure-mappings [dataset-mapping] 18 | (seq (filter :is-numeric? (measures dataset-mapping)))) 19 | 20 | (defn- find-by-uri [components uri] 21 | (util/find-first (fn [comp] (= uri (:uri comp))) components)) 22 | 23 | (defn get-component-by-uri [dataset-mapping uri] 24 | (find-by-uri (components dataset-mapping) uri)) 25 | 26 | (defn get-dimension-by-uri [dataset-mapping uri] 27 | (find-by-uri (dimensions dataset-mapping) uri)) 28 | 29 | (defn get-measure-by-uri [dataset-mapping uri] 30 | (find-by-uri (measures dataset-mapping) uri)) 31 | 32 | (defn- find-by-enum-name [components enum-name] 33 | (util/find-first (fn [comp] (= enum-name (:enum-name comp))) components)) 34 | 35 | (defn get-component-by-enum-name [dataset-mapping enum-name] 36 | (find-by-enum-name (components dataset-mapping) enum-name)) 37 | 38 | (defn- find-by-field-name [components field-name] 39 | (util/find-first (fn [comp] (= field-name (:field-name comp))) components)) 40 | 41 | (defn get-component-by-field-name [dataset-mapping field-name] 42 | (find-by-field-name (components dataset-mapping) field-name)) 43 | 44 | (defn get-dimension-by-field-name [dataset-mapping field-name] 45 | (find-by-field-name (dimensions dataset-mapping) field-name)) 46 | 47 | (defn component-mapping->component [comp] 48 | (or (:dimension comp) (:measure comp))) 49 | 50 | (defn components-enum-group [dataset-mapping] 51 | (:components-enum dataset-mapping)) 52 | 53 | (defn aggregation-measures-enum-group [dataset-mapping] 54 | (:aggregation-measures-enum dataset-mapping)) 55 | 56 | (defn has-measure-type-dimension? 57 | "Whether the given dataset has an explicit qb:measureType dimension" 58 | [dataset-mapping] 59 | (let [dims (dimensions dataset-mapping) 60 | measure-dim (some (fn [dim] (= qb:measureType (:uri dim))) dims)] 61 | (some? measure-dim))) -------------------------------------------------------------------------------- /src/cubiql/schema/mapping/labels.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.schema.mapping.labels 2 | "Creates GraphQL schema mappings from the labels associated with types and values" 3 | (:require [clojure.spec.alpha :as s] 4 | [cubiql.util :as util] 5 | [clojure.string :as string] 6 | [cubiql.types :as types] 7 | [cubiql.config :as config] 8 | [cubiql.queries :as queries] 9 | [cubiql.dataset-model :as dsm])) 10 | 11 | ;;TODO: add/use spec for graphql enum values 12 | (s/def ::graphql-enum keyword?) 13 | 14 | (defn get-identifier-segments [label] 15 | (let [segments (re-seq #"[a-zA-Z0-9]+" (str label))] 16 | (if (empty? segments) 17 | (throw (IllegalArgumentException. (format "Cannot construct identifier from label '%s'" label))) 18 | (let [^Character first-char (ffirst segments)] 19 | (if (Character/isDigit first-char) 20 | (cons "a" segments) 21 | segments))))) 22 | 23 | (defn- segments->enum-value [segments] 24 | (->> segments 25 | (map string/upper-case) 26 | (string/join "_") 27 | (keyword))) 28 | 29 | (defn segments->schema-key [segments] 30 | (->> segments 31 | (map string/lower-case) 32 | (string/join "_") 33 | (keyword))) 34 | 35 | (defn label->field-name [label] 36 | (segments->schema-key (get-identifier-segments label))) 37 | 38 | (defn label->enum-name 39 | ([label] 40 | (segments->enum-value (get-identifier-segments label))) 41 | ([label n] 42 | (let [label-segments (get-identifier-segments label)] 43 | (segments->enum-value (concat label-segments [(str n)]))))) 44 | 45 | (defn create-group-mapping 46 | ([name mappings] (create-group-mapping name mappings identity)) 47 | ([name mappings val-f] 48 | ;;TODO: handle multiple mappings to the same label 49 | (let [items (mapv (fn [{:keys [label] :as mapping}] 50 | (types/->EnumMappingItem (label->enum-name label) (val-f mapping) label)) 51 | mappings)] 52 | (types/->GroupMapping name items)))) 53 | 54 | (defn components-enum-group [schema components] 55 | (let [mapping-name (keyword (str (name schema) "_dimension_measures"))] 56 | (create-group-mapping mapping-name components :uri))) 57 | 58 | (defn aggregation-measures-enum-group [schema measures] 59 | (if-let [aggregation-measures (seq (filter :is-numeric? measures))] 60 | (let [mapping-name (keyword (str (name schema) "_aggregation_measures"))] 61 | (create-group-mapping mapping-name aggregation-measures :uri)))) 62 | 63 | (defn create-enum-mapping [enum-label enum-doc code-list] 64 | (let [by-enum-name (group-by #(label->enum-name (:label %)) code-list) 65 | items (mapcat (fn [[enum-name item-results]] 66 | (if (= 1 (count item-results)) 67 | (map (fn [{:keys [member label]}] 68 | (types/->EnumMappingItem enum-name member label)) 69 | item-results) 70 | (map-indexed (fn [n {:keys [member label]}] 71 | (types/->EnumMappingItem (label->enum-name label (inc n)) member label)) 72 | item-results))) 73 | by-enum-name)] 74 | {:label enum-label :doc (or enum-doc "") :items (vec items)})) 75 | 76 | (defn get-measure-type 77 | "Returns the mapped datatype for the given measure" 78 | [m] 79 | (if (types/is-numeric-measure? m) 80 | types/float-measure-type 81 | types/string-measure-type)) 82 | 83 | (defn get-dimension-codelist [dimension-member-bindings configuration] 84 | (map (fn [[member-uri member-bindings]] 85 | (let [labels (map :vallabel member-bindings)] 86 | {:member member-uri :label (util/find-best-language labels (config/schema-label-language configuration))})) 87 | (group-by :member dimension-member-bindings))) 88 | 89 | (defn- get-dataset-enum-mappings [dataset-member-bindings dimension-labels configuration] 90 | (let [dimension-member-bindings (group-by :dim dataset-member-bindings) 91 | field-mappings (map (fn [[dim-uri dim-members]] 92 | (let [{dim-label :label enum-doc :doc} (get dimension-labels dim-uri) 93 | codelist (get-dimension-codelist dim-members configuration)] 94 | [dim-uri (create-enum-mapping dim-label enum-doc codelist)])) 95 | dimension-member-bindings)] 96 | (into {} field-mappings))) 97 | 98 | (defn get-datasets-enum-mappings [datasets codelist-member-bindings dimension-labels configuration] 99 | (let [ds-members (group-by :ds codelist-member-bindings) 100 | dataset-mappings (map (fn [dataset] 101 | (let [ds-uri (:uri dataset) 102 | ds-codelist-member-bindings (get ds-members ds-uri)] 103 | [ds-uri (get-dataset-enum-mappings ds-codelist-member-bindings dimension-labels configuration)])) 104 | datasets)] 105 | (into {} dataset-mappings))) 106 | 107 | ;;schema mappings 108 | 109 | (defn get-all-enum-mappings [repo datasets dimension-labels config] 110 | (let [enum-dimension-values-query (queries/get-all-enum-dimension-values config) 111 | results (util/eager-query repo enum-dimension-values-query) 112 | dataset-enum-values (map (util/convert-binding-labels [:vallabel]) results)] 113 | (get-datasets-enum-mappings datasets dataset-enum-values dimension-labels config))) 114 | 115 | (defn field-name->type-name [field-name ds-schema] 116 | (keyword (str (name ds-schema) "_" (name field-name) "_type"))) 117 | 118 | (defn identify-dimension-labels [dimension-bindings configuration] 119 | (util/map-values (fn [bindings] 120 | (let [{:keys [label doc]} (util/to-multimap bindings)] 121 | {:label (util/find-best-language label (config/schema-label-language configuration)) 122 | :doc (util/find-best-language doc (config/schema-label-language configuration))})) 123 | (group-by :dim dimension-bindings))) 124 | 125 | (defn identify-measure-labels [measure-bindings configuration] 126 | (util/map-values (fn [bindings] 127 | (let [labels (map :label bindings)] 128 | (util/find-best-language labels (config/schema-label-language configuration)))) 129 | (group-by :measure measure-bindings))) 130 | 131 | (defn find-dimension-labels [repo configuration] 132 | (let [q (queries/get-dimension-labels-query configuration) 133 | results (util/eager-query repo q)] 134 | (identify-dimension-labels results configuration))) 135 | 136 | (defn find-measure-labels [repo configuration] 137 | (let [q (queries/get-measure-labels-query configuration) 138 | results (util/eager-query repo q)] 139 | (identify-measure-labels results configuration))) 140 | 141 | (defn- measure->enum-item [{:keys [label enum-name uri] :as measure}] 142 | (types/->EnumMappingItem enum-name uri label)) 143 | 144 | (defn measures-enum-type [enum-name measures doc] 145 | (let [items (mapv measure->enum-item measures)] 146 | (types/->MappedEnumType enum-name types/measure-dimension-type doc items))) 147 | 148 | (defn- get-dimension-mapping [schema {:keys [uri] :as dimension} ds-enum-mappings {:keys [label doc]} measures] 149 | (let [dimension-type (:type dimension) 150 | field-name (label->field-name label) 151 | enum-name (field-name->type-name field-name schema) 152 | mapped-type (cond 153 | (contains? ds-enum-mappings uri) 154 | (let [enum-mapping (get ds-enum-mappings uri)] 155 | (types/->MappedEnumType enum-name dimension-type (:doc enum-mapping) (:items enum-mapping))) 156 | 157 | (dsm/is-measure-type-dimension? dimension) 158 | (measures-enum-type enum-name measures doc) 159 | 160 | :else 161 | dimension-type)] 162 | {:uri uri 163 | :label label 164 | :doc doc 165 | :field-name field-name 166 | :enum-name (label->enum-name label) 167 | :type mapped-type 168 | :dimension dimension})) 169 | 170 | (defn- get-measure-mapping [{:keys [uri is-numeric?] :as measure} label] 171 | {:uri uri 172 | :label label 173 | :field-name (label->field-name label) 174 | :enum-name (label->enum-name label) 175 | :type (get-measure-type measure) 176 | :is-numeric? is-numeric? 177 | :measure measure}) 178 | 179 | (defn dataset-name->schema-name [label] 180 | (segments->schema-key (cons "dataset" (get-identifier-segments label)))) 181 | 182 | (defn dataset-schema [ds] 183 | (keyword (dataset-name->schema-name (:name ds)))) 184 | 185 | (defn resolve-dimension-labels [{:keys [uri] :as dimension} dimension-uri->labels] 186 | (if (dsm/is-measure-type-dimension? dimension) 187 | (merge (get dimension-uri->labels uri) {:label "Measure type" :doc "Generic measure type dimension"}) 188 | (util/strict-get dimension-uri->labels uri))) 189 | 190 | (defn build-dataset-mapping-model [{:keys [uri] :as dataset} ds-enum-mappings dimension-labels measure-labels] 191 | (let [schema (dataset-schema dataset) 192 | measures (mapv (fn [{:keys [uri] :as measure}] 193 | (let [label (util/strict-get measure-labels uri)] 194 | (get-measure-mapping measure label))) 195 | (types/dataset-measures dataset)) 196 | dimensions (mapv (fn [dim] 197 | (let [labels (resolve-dimension-labels dim dimension-labels)] 198 | (get-dimension-mapping schema dim ds-enum-mappings labels measures))) 199 | (types/dataset-dimensions dataset))] 200 | {:uri uri 201 | :schema schema 202 | :dimensions dimensions 203 | :measures measures 204 | :components-enum (components-enum-group schema (concat dimensions measures)) 205 | :aggregation-measures-enum (aggregation-measures-enum-group schema measures)})) 206 | 207 | (defn get-dataset-mapping-models [repo datasets configuration] 208 | (let [dimension-labels (find-dimension-labels repo configuration) 209 | measure-labels (find-measure-labels repo configuration) 210 | enum-mappings (get-all-enum-mappings repo datasets dimension-labels configuration)] 211 | (mapv (fn [{:keys [uri] :as ds}] 212 | (let [ds-enum-mappings (get enum-mappings uri {})] 213 | (build-dataset-mapping-model ds ds-enum-mappings dimension-labels measure-labels))) 214 | datasets))) 215 | -------------------------------------------------------------------------------- /src/cubiql/schema_model.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.schema-model 2 | (:require [clojure.string :as string]) 3 | (:import [cubiql.types GroupMapping])) 4 | 5 | (defn is-graphql-type? [x] 6 | (or (symbol? x) 7 | (and (keyword? x) 8 | (nil? (namespace x))))) 9 | 10 | (defn is-type-ref? [x] 11 | (and (keyword? x) 12 | (some? (namespace x)))) 13 | 14 | (defrecord NonNull [type-def]) 15 | (def non-null ->NonNull) 16 | 17 | (defn is-enum-mapping? [type] 18 | (instance? GroupMapping type)) 19 | 20 | (defn merge-schemas [s1 s2] 21 | (merge-with (fn [v1 v2] 22 | (cond 23 | (and (map? v1) (map? v2)) (merge-schemas v1 v2) 24 | 25 | (nil? v1) v2 26 | 27 | (nil? v2) v1 28 | 29 | :else v2)) 30 | s1 s2)) 31 | 32 | (defn path->object-name [path] 33 | (keyword (string/join "_" (map name path)))) 34 | 35 | (declare visit-object) 36 | 37 | (defn visit-type [path type type-schema-key] 38 | (cond 39 | (instance? NonNull type) 40 | (let [type-def (:type-def type) 41 | element-type (visit-type path type-def type-schema-key)] 42 | {::name (list 'non-null (::name element-type)) 43 | ::schema (::schema element-type)}) 44 | 45 | (is-enum-mapping? type) 46 | (let [{:keys [name items]} type] 47 | {::name name 48 | ::schema {:enums {name {:values (mapv :name items)}}}}) 49 | 50 | (map? type) 51 | (let [obj-result (visit-object path type type-schema-key) 52 | type-name (path->object-name path) 53 | type-schema {type-schema-key {type-name (::object obj-result)}}] 54 | {::schema (merge-schemas (::schema obj-result) type-schema) 55 | ::name type-name}) 56 | 57 | (is-graphql-type? type) 58 | {::name type ::schema {}} 59 | 60 | (is-type-ref? type) 61 | {::name type ::schema {}} 62 | 63 | (vector? type) 64 | (let [type-def (first type) 65 | element-type (visit-type path type-def type-schema-key)] 66 | {::name (list 'list (::name element-type)) 67 | ::schema (::schema element-type)}))) 68 | 69 | (defn visit-arg [path {:keys [type] :as arg-def}] 70 | (let [type-result (visit-type path type :input-objects) 71 | out-arg (assoc arg-def :type (::name type-result))] 72 | {::schema (::schema type-result) ::arg out-arg})) 73 | 74 | (defn visit-args [path args] 75 | (reduce (fn [acc [arg-name arg-def]] 76 | (let [arg-result (visit-arg (conj path arg-name) arg-def)] 77 | (-> acc 78 | (update ::schema merge-schemas (::schema arg-result)) 79 | (assoc-in [::args arg-name] (::arg arg-result))))) 80 | {::schema {} ::args {}} 81 | args)) 82 | 83 | (defn path->resolver-name [path] 84 | (keyword (str "resolve_" (string/join "_" (map name path))))) 85 | 86 | (defn visit-resolver [path resolver] 87 | (cond 88 | (fn? resolver) 89 | (let [resolver-name (path->resolver-name path)] 90 | {::resolver resolver-name ::schema {:resolvers {resolver-name resolver}}}) 91 | 92 | (keyword? resolver) 93 | {::resolver resolver ::schema {}} 94 | 95 | :else 96 | (throw (IllegalArgumentException. "Expected fn or keyword for resolver")))) 97 | 98 | (defn visit-field [path field-name {:keys [type args resolve] :as field} type-schema-key] 99 | (let [type-result (visit-type path type type-schema-key) 100 | result {::field (assoc field :type (::name type-result)) ::schema (::schema type-result)} 101 | result (if (some? args) 102 | (let [args-result (visit-args path args)] 103 | (-> result 104 | (assoc-in [::field :args] (::args args-result)) 105 | (update ::schema merge-schemas (::schema args-result)))) 106 | result) 107 | result (if (some? resolve) 108 | (let [resolver-result (visit-resolver path resolve)] 109 | (-> result 110 | (assoc-in [::field :resolve] (::resolver resolver-result)) 111 | (update ::schema merge-schemas (::schema resolver-result)))) 112 | result)] 113 | ;;TODO: map resolver arguments? 114 | result)) 115 | 116 | (defn visit-object 117 | ([path object-def] (visit-object path object-def :objects)) 118 | ([path {:keys [fields] :as object-def} type-schema-key] 119 | (let [fields-result (reduce (fn [acc [field-name field-def]] 120 | (let [result (visit-field (conj path field-name) field-name field-def type-schema-key)] 121 | (-> acc 122 | (update ::schema merge-schemas (::schema result)) 123 | (update-in [::fields] assoc field-name (::field result))))) 124 | {::schema {} ::fields {}} 125 | fields) 126 | schema (::schema fields-result) 127 | out-fields (::fields fields-result) 128 | out-obj (assoc object-def :fields out-fields) 129 | obj-type-name (path->object-name path) 130 | obj-schema {type-schema-key {obj-type-name out-obj}}] 131 | {::schema (merge-schemas schema obj-schema) 132 | ::object out-obj}))) 133 | 134 | (defn visit-query [query-name {:keys [type resolve] :as query-def}] 135 | (let [path [query-name] 136 | object-def (visit-object path type :objects) 137 | resolver-def (visit-resolver path resolve)] 138 | (merge-schemas 139 | (merge-schemas (::schema object-def) (::schema resolver-def)) 140 | {:queries 141 | {query-name 142 | {:type query-name 143 | :resolve (::resolver resolver-def)}}}))) 144 | 145 | (defn visit-queries [queries-def] 146 | (reduce (fn [acc [query-name query-def]] 147 | (let [query-schema (visit-query query-name query-def)] 148 | (merge-schemas acc query-schema))) 149 | {} 150 | queries-def)) 151 | -------------------------------------------------------------------------------- /src/cubiql/server.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.server 2 | (:require [cubiql.core :as core] 3 | [cubiql.context :as context] 4 | [com.walmartlabs.lacinia.pedestal :as lp] 5 | [io.pedestal.http :as http])) 6 | 7 | (def cors-config {:allowed-origins (constantly true) 8 | :creds false 9 | :max-age (* 60 60 2) ;2 hours 10 | :methods "GET, POST, OPTIONS"}) 11 | 12 | (defn create-server 13 | ([port repo config] 14 | (let [{:keys [schema datasets dataset-mappings]} (core/build-schema-context repo config) 15 | context (context/create repo datasets dataset-mappings config) 16 | opts {:app-context context 17 | :port port 18 | :graphiql true} 19 | service-map (lp/service-map schema opts)] 20 | (-> service-map 21 | (assoc ::http/allowed-origins cors-config) 22 | (http/create-server))))) 23 | 24 | (defn start-server [port repo config] 25 | (http/start (create-server port repo config))) 26 | -------------------------------------------------------------------------------- /src/cubiql/types.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.types 2 | "Functions for mapping DSD elements to/from GraphQL types" 3 | (:require [cubiql.query-model :as qm] 4 | [cubiql.vocabulary :refer [time:hasBeginning time:hasEnd time:inXSDDateTime rdfs:label]] 5 | [cubiql.types.scalars :refer [grafter-date->datetime]] 6 | [cubiql.util :as util] 7 | [cubiql.config :as config])) 8 | 9 | (defprotocol SparqlFilterable 10 | (apply-filter [this model graphql-value])) 11 | 12 | (defprotocol SparqlQueryable 13 | (apply-order-by [this model direction config])) 14 | 15 | (defprotocol SparqlTypeProjection 16 | "Protocol for projecting a set of observation SPARQL bindings into a data structure 17 | for the type" 18 | (project-type-result [type root-key bindings] 19 | "Extracts any values associated with the the dimensions key from a map of SPARQL bindings and returns 20 | a data structure representing this type.")) 21 | 22 | (defrecord RefAreaType []) 23 | (defrecord RefPeriodType []) 24 | (defrecord EnumType []) 25 | (defrecord DecimalType []) 26 | (defrecord StringType []) 27 | (defrecord MeasureDimensionType []) 28 | (defrecord UnmappedType [type-uri]) 29 | 30 | (defn find-item-by-name [name items] 31 | (util/find-first #(= name (:name %)) items)) 32 | 33 | (defn find-item-by-value [value items] 34 | ) 35 | 36 | (defrecord EnumMappingItem [name value label]) 37 | 38 | (defrecord MappedEnumType [enum-type-name type doc items]) 39 | 40 | (defrecord GroupMapping [name items]) 41 | 42 | (def ref-area-type (->RefAreaType)) 43 | (def ref-period-type (->RefPeriodType)) 44 | (def decimal-type (->DecimalType)) 45 | (def string-type (->StringType)) 46 | (def enum-type (->EnumType)) 47 | (def measure-dimension-type (->MeasureDimensionType)) 48 | 49 | (defn maybe-add-period-filter [model dim-key dim-uri interval-key filter-fn dt] 50 | (if (some? dt) 51 | (let [key-path [[dim-key dim-uri] interval-key [:time time:inXSDDateTime]]] 52 | (-> model 53 | (qm/add-binding key-path ::qm/var) 54 | (qm/add-filter (map first key-path) [filter-fn dt]))) 55 | model)) 56 | 57 | (defn apply-ref-period-filter [model dim-key dim-uri {:keys [uri starts_before starts_after ends_before ends_after] :as filter}] 58 | (if (nil? filter) 59 | (qm/add-binding model [[dim-key dim-uri]] ::qm/var) 60 | (let [model (if (some? uri) (qm/add-binding model [[dim-key dim-uri]] uri) model)] 61 | (if (and (nil? starts_before) (nil? starts_after) (nil? ends_before) (nil? ends_after)) 62 | (qm/add-binding model [[dim-key dim-uri]] ::qm/var) 63 | ;; add bindings/filter for each filter 64 | (-> model 65 | (maybe-add-period-filter dim-key dim-uri [:begin time:hasBeginning] '<= starts_before) 66 | (maybe-add-period-filter dim-key dim-uri [:begin time:hasBeginning] '>= starts_after) 67 | (maybe-add-period-filter dim-key dim-uri [:end time:hasEnd] '<= ends_before) 68 | (maybe-add-period-filter dim-key dim-uri [:end time:hasEnd] '>= ends_after)))))) 69 | 70 | (defprotocol SparqlResultProjector 71 | (apply-projection [this model selections config]) 72 | (project-result [this sparql-binding])) 73 | 74 | (extend-protocol SparqlTypeProjection 75 | RefPeriodType 76 | (project-type-result [_type dim-key bindings] 77 | {:uri (get bindings dim-key) 78 | :label (get bindings (qm/key-path->var-key [dim-key :label])) 79 | :start (some-> (get bindings (qm/key-path->var-key [dim-key :begin :time])) grafter-date->datetime) 80 | :end (some-> (get bindings (qm/key-path->var-key [dim-key :end :time])) grafter-date->datetime)}) 81 | 82 | RefAreaType 83 | (project-type-result [_type dim-key bindings] 84 | {:uri (get bindings dim-key) 85 | :label (get bindings (qm/key-path->var-key [dim-key :label]))}) 86 | 87 | EnumType 88 | (project-type-result [_type dim-key bindings] 89 | (get bindings dim-key)) 90 | 91 | DecimalType 92 | (project-type-result [_type dim-key bindings] 93 | (get bindings dim-key)) 94 | 95 | StringType 96 | (project-type-result [_type dim-key bindings] 97 | (get bindings dim-key)) 98 | 99 | MeasureDimensionType 100 | (project-type-result [_type dim-key bindings] 101 | (get bindings dim-key)) 102 | 103 | UnmappedType 104 | (project-type-result [_type dim-key bindings] 105 | (some-> (get bindings dim-key) str))) 106 | 107 | (defprotocol TypeResultProjector 108 | (apply-type-projection [type dim-key uri model field-selections configuration])) 109 | 110 | (extend-protocol TypeResultProjector 111 | RefPeriodType 112 | (apply-type-projection [_type dim-key uri model field-selections configuration] 113 | (let [codelist-label (config/dataset-label configuration) 114 | model (qm/add-binding model [[dim-key uri]] ::qm/var) 115 | model (if (contains? field-selections :label) 116 | (qm/add-binding model [[dim-key uri] [:label codelist-label]] ::qm/var) 117 | model) 118 | model (if (contains? field-selections :start) 119 | (qm/add-binding model [[dim-key uri] [:begin time:hasBeginning] [:time time:inXSDDateTime]] ::qm/var) 120 | model)] 121 | (if (contains? field-selections :end) 122 | (qm/add-binding model [[dim-key uri] [:end time:hasEnd] [:time time:inXSDDateTime]] ::qm/var) 123 | model))) 124 | 125 | RefAreaType 126 | (apply-type-projection [_type dim-key uri model field-selections configuration] 127 | (let [label-selected? (contains? field-selections :label) 128 | codelist-label (config/dataset-label configuration)] 129 | (if label-selected? 130 | (qm/add-binding model [[dim-key uri] [:label codelist-label]] ::qm/var) 131 | model))) 132 | 133 | EnumType 134 | (apply-type-projection [_type _dim-key _uri model _field-selections _configuration] 135 | model) 136 | 137 | DecimalType 138 | (apply-type-projection [_type _dim-key _uri model _field-selections _configuration] 139 | model) 140 | 141 | StringType 142 | (apply-type-projection [_type _dim-key _uri model _field-selections _configuration] 143 | model) 144 | 145 | MeasureDimensionType 146 | (apply-type-projection [_type _dim-key _uri model _field-selections _configuration] 147 | model) 148 | 149 | UnmappedType 150 | (apply-type-projection [_type _dim-key _uri model _field-selections _configuration] 151 | model)) 152 | 153 | (defprotocol TypeOrderBy 154 | (apply-type-order-by [type dim-key dimension-uri model direction configuration])) 155 | 156 | (defn- default-type-order-by [_type dim-key _dimension-uri model direction _configuration] 157 | ;;NOTE: binding should have already been added 158 | (qm/add-order-by model {direction [dim-key]})) 159 | 160 | (def default-type-order-by-impl {:apply-type-order-by default-type-order-by}) 161 | 162 | (defn- ref-area-order-by [type dim-key dimension-uri model direction configuration] 163 | (let [codelist-label (config/dataset-label configuration)] 164 | (-> model 165 | (qm/add-binding [[dim-key dimension-uri] [:label codelist-label]] ::qm/var) 166 | (qm/add-order-by {direction [dim-key :label]})))) 167 | 168 | (extend RefAreaType TypeOrderBy {:apply-type-order-by ref-area-order-by}) 169 | (extend RefPeriodType TypeOrderBy default-type-order-by-impl) 170 | (extend EnumType TypeOrderBy default-type-order-by-impl) 171 | (extend DecimalType TypeOrderBy default-type-order-by-impl) 172 | (extend StringType TypeOrderBy default-type-order-by-impl) 173 | (extend MeasureDimensionType TypeOrderBy default-type-order-by-impl) 174 | (extend UnmappedType TypeOrderBy default-type-order-by-impl) 175 | 176 | (defprotocol TypeFilter 177 | (apply-type-filter [type dim-key dimension-uri model sparql-value])) 178 | 179 | (defn default-type-filter [_type dim-key dimension-uri model sparql-value] 180 | (let [value (or sparql-value ::qm/var)] 181 | (qm/add-binding model [[dim-key dimension-uri]] value))) 182 | 183 | (defn- ref-period-type-filter [_type dim-key dimension-uri model sparql-value] 184 | (apply-ref-period-filter model dim-key dimension-uri sparql-value)) 185 | 186 | (def default-type-filter-impl {:apply-type-filter default-type-filter}) 187 | 188 | (extend RefAreaType TypeFilter default-type-filter-impl) 189 | (extend RefPeriodType TypeFilter {:apply-type-filter ref-period-type-filter}) 190 | (extend EnumType TypeFilter default-type-filter-impl) 191 | (extend DecimalType TypeFilter default-type-filter-impl) 192 | (extend StringType TypeFilter default-type-filter-impl) 193 | (extend MeasureDimensionType TypeFilter default-type-filter-impl) 194 | (extend UnmappedType TypeFilter default-type-filter-impl) 195 | 196 | ;;measure types 197 | ;;TODO: combine with dimension types? 198 | (defrecord FloatMeasureType []) 199 | (defrecord StringMeasureType []) 200 | 201 | (def float-measure-type (->FloatMeasureType)) 202 | (def string-measure-type (->StringMeasureType)) 203 | 204 | (defrecord Dimension [uri order type] 205 | SparqlQueryable 206 | (apply-order-by [_this model direction configuration] 207 | (let [dim-key (keyword (str "dim" order))] 208 | (apply-type-order-by type dim-key uri model direction configuration))) 209 | 210 | SparqlFilterable 211 | (apply-filter [this model sparql-value] 212 | (let [dim-key (keyword (str "dim" order))] 213 | (apply-type-filter type dim-key uri model sparql-value))) 214 | 215 | SparqlResultProjector 216 | (apply-projection [this model observation-selections configuration] 217 | (let [dim-key (keyword (str "dim" order)) 218 | ;;TODO: move field selections into caller? 219 | field-selections (get observation-selections uri)] 220 | (apply-type-projection type dim-key uri model field-selections configuration))) 221 | 222 | (project-result [_this bindings] 223 | (let [dim-key (keyword (str "dim" order))] 224 | (project-type-result type dim-key bindings)))) 225 | 226 | (defrecord MeasureType [uri order is-numeric?] 227 | SparqlQueryable 228 | (apply-order-by [_this model direction _configuration] 229 | (qm/add-order-by model {direction [(keyword (str "mv"))]})) 230 | 231 | SparqlResultProjector 232 | (apply-projection [_this model selections config] 233 | model) 234 | 235 | (project-result [_this binding] 236 | (throw (ex-info "Not supported - measure values are bound differently depending on the existence of a qb:measureDimension within the dataset" {})))) 237 | 238 | (defrecord Dataset [uri name dimensions measures]) 239 | 240 | (defn dataset-aggregate-measures [{:keys [measures] :as ds}] 241 | (filter :is-numeric? measures)) 242 | 243 | (defn dataset-dimension-measures [{:keys [dimensions measures] :as ds}] 244 | (concat dimensions measures)) 245 | 246 | (defn dataset-dimensions [ds] 247 | (:dimensions ds)) 248 | 249 | (defn dataset-measures [ds] 250 | (:measures ds)) 251 | 252 | (defn is-numeric-measure? [m] 253 | (:is-numeric? m)) 254 | 255 | (defn get-dataset-dimension-measure-by-uri [dataset uri] 256 | (util/find-first #(= uri (:uri %)) (dataset-dimension-measures dataset))) 257 | 258 | (defn get-dataset-dimension-by-uri [dataset dimension-uri] 259 | (util/find-first (fn [dim] (= dimension-uri (:uri dim))) (dataset-dimensions dataset))) 260 | 261 | (defn get-dataset-measure-by-uri [{:keys [measures] :as dataset} uri] 262 | (util/find-first #(= uri (:uri %)) measures)) 263 | 264 | -------------------------------------------------------------------------------- /src/cubiql/types/scalars.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.types.scalars 2 | (:require [cubiql.util :as util] 3 | [com.walmartlabs.lacinia.schema :as lschema]) 4 | (:import [java.net URI] 5 | [java.util Base64 Date] 6 | [java.time.format DateTimeFormatter] 7 | [java.time ZonedDateTime ZoneOffset] 8 | [org.openrdf.model Literal] 9 | [javax.xml.datatype XMLGregorianCalendar])) 10 | 11 | (defn parse-sparql-cursor [^String base64-str] 12 | (let [bytes (.decode (Base64/getDecoder) base64-str) 13 | offset (util/bytes->long bytes)] 14 | (if (neg? offset) 15 | (throw (IllegalArgumentException. "Invalid cursor")) 16 | offset))) 17 | 18 | (defn serialise-sparql-cursor [offset] 19 | {:pre [(>= offset 0)]} 20 | (let [bytes (util/long->bytes offset) 21 | enc (Base64/getEncoder)] 22 | (.encodeToString enc bytes))) 23 | 24 | (defn parse-datetime [dt-string] 25 | (.parse DateTimeFormatter/ISO_OFFSET_DATE_TIME dt-string)) 26 | 27 | (defn serialise-datetime [dt] 28 | (.format DateTimeFormatter/ISO_OFFSET_DATE_TIME dt)) 29 | 30 | (def custom-scalars 31 | {:SparqlCursor 32 | {:parse (lschema/as-conformer parse-sparql-cursor) 33 | :serialize (lschema/as-conformer serialise-sparql-cursor)} 34 | 35 | :uri {:parse (lschema/as-conformer #(URI. %)) 36 | :serialize (lschema/as-conformer str)} 37 | 38 | :DateTime 39 | {:parse (lschema/as-conformer parse-datetime) 40 | :serialize (lschema/as-conformer serialise-datetime)}}) 41 | 42 | (defn date->datetime 43 | "Converts a java.util.Date to a java.time.ZonedDateTime." 44 | [^Date date] 45 | (ZonedDateTime/ofInstant (.toInstant date) ZoneOffset/UTC)) 46 | 47 | (defn grafter-date->datetime 48 | "Converts all known date literal representations used by Grafter into the corresponding 49 | DateTime." 50 | [dt] 51 | (cond 52 | (instance? Date dt) 53 | (date->datetime dt) 54 | 55 | (instance? Literal dt) 56 | (let [^Literal dt dt 57 | ^XMLGregorianCalendar xml-cal (.calendarValue dt) 58 | cal (.toGregorianCalendar xml-cal) 59 | date (.getTime cal)] 60 | (date->datetime date)) 61 | 62 | :else 63 | (throw (IllegalArgumentException. (str "Unexpected date representation: " dt))))) 64 | -------------------------------------------------------------------------------- /src/cubiql/util.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.util 2 | (:require [clojure.java.io :as io] 3 | [clojure.edn :as edn] 4 | [grafter.rdf.repository :as repo] 5 | [grafter.rdf.protocols :as pr]) 6 | (:import [java.io PushbackReader] 7 | [java.nio ByteBuffer] 8 | [grafter.rdf.protocols LangString] 9 | [org.openrdf.repository RepositoryConnection])) 10 | 11 | (defn read-edn 12 | "Reads EDN from the given source." 13 | [source] 14 | (with-open [pbr (PushbackReader. (io/reader source))] 15 | (edn/read pbr))) 16 | 17 | (defn read-edn-resource 18 | "Loads EDN from the named resource." 19 | [resource-name] 20 | (if-let [r (io/resource resource-name)] 21 | (read-edn r) 22 | (throw (IllegalArgumentException. "Resource not found")))) 23 | 24 | (defn rename-key 25 | "Renames the key k in the map m to new-k. If the optional keyword argument strict? is true 26 | then an exception will be thrown if k does not exist in m." 27 | [m k new-k & {:keys [strict?] :or {strict? false}}] 28 | (cond 29 | (contains? m k) 30 | (let [v (get m k)] 31 | (-> m 32 | (assoc new-k v) 33 | (dissoc k))) 34 | 35 | strict? 36 | (throw (IllegalArgumentException. (format "Source key %s not found in input map" (str k)))) 37 | 38 | :else 39 | m)) 40 | 41 | (defn keyed-by 42 | "Returns a map {key item} for each element in the source sequence items according to the key function. 43 | If multiple items map to the same key, the last matching item will be the one included in the result map." 44 | [key-fn items] 45 | (into {} (map (fn [i] [(key-fn i) i]) items))) 46 | 47 | (defn strict-map-by 48 | "Returns a map {key item} for the given sequence items and key function f. Throws an exception 49 | if any of the items in the input sequence map to the same key." 50 | [f items] 51 | (reduce (fn [acc i] 52 | (let [k (f i)] 53 | (if (contains? acc k) 54 | (throw (ex-info (str "Duplicate entries for k " k) 55 | {:existing (get acc k) 56 | :duplicate i})) 57 | (assoc acc k i)))) 58 | {} 59 | items)) 60 | 61 | (defn map-values 62 | "Maps each value in the map m according to the mapping function f." 63 | [f m] 64 | (into {} (map (fn [[k v]] [k (f v)]) m))) 65 | 66 | (defn map-keys 67 | "Maps each key in the map m according to the mapping function f. If multiple keys in m are 68 | mapped to the same value by f, no guarantees are made about which key will be chosen." 69 | [f m] 70 | (into {} (map (fn [[k v]] [(f k) v]) m))) 71 | 72 | (defn distinct-by 73 | "Returns a sequence containing distinct elements by the given key function." 74 | [f s] 75 | (let [keys (atom #{})] 76 | (filter (fn [v] 77 | (let [k (f v)] 78 | (if (contains? @keys k) 79 | false 80 | (do 81 | (swap! keys conj k) 82 | true)))) 83 | s))) 84 | 85 | (defn long->bytes [i] 86 | {:post [(= 8 (alength %))]} 87 | (let [buf (ByteBuffer/allocate 8)] 88 | (.putLong buf i) 89 | (.array buf))) 90 | 91 | (defn bytes->long [^bytes bytes] 92 | {:pre [(= 8 (alength bytes))]} 93 | (let [buf (ByteBuffer/wrap bytes)] 94 | (.getLong buf))) 95 | 96 | (defn eager-query 97 | "Executes a SPARQL query against the given repository and eagerly evaluates the results. This prevents 98 | connections being left open by lazy sequence operators." 99 | [repo sparql-string] 100 | (with-open [^RepositoryConnection conn (repo/->connection repo)] 101 | (doall (repo/query conn sparql-string)))) 102 | 103 | (defn find-first 104 | "Returns the first item in s which satisfies the predicate p. Returns nil if no items satisfy p." 105 | [p s] 106 | (first (filter p s))) 107 | 108 | (defn label->string 109 | "Converts a grafter string type into a java string." 110 | [l] 111 | (some-> l str)) 112 | 113 | (defn convert-binding-labels 114 | "Returns a function which converts each label associated with the specified keys to a string." 115 | [keys] 116 | (fn [bindings] 117 | (reduce (fn [acc k] 118 | (update acc k label->string)) 119 | bindings 120 | keys))) 121 | 122 | (defprotocol HasLang 123 | (get-language-tag [this] 124 | "Returns the language tag associated with this item, or nil if there is no language.")) 125 | 126 | (extend-protocol HasLang 127 | String 128 | (get-language-tag [_s] nil) 129 | 130 | LangString 131 | (get-language-tag [ls] (name (pr/lang ls)))) 132 | 133 | (defn- score-language-match 134 | "Scores how closely the given string matches the specified language. A higher score indicates a better match, 135 | a score of 0 indicates no match." 136 | [s lang] 137 | (let [lang-tag (get-language-tag s)] 138 | (cond 139 | (= lang lang-tag) 3 140 | (and (some? lang) (nil? lang-tag)) 2 141 | (= lang-tag "en") 1 142 | :else 0))) 143 | 144 | (defn find-best-language 145 | "Searches for the Grafter string with the closest matching language label. Strings which match the requested 146 | language tag exactly are preferred, followed by string literals without a language, then english labels. If neither 147 | a matching language, string literal or english label can be found then nil is returned." 148 | [labels lang] 149 | (when (seq labels) 150 | (let [with-scores (map (fn [l] [l (score-language-match l lang)]) labels) 151 | [label score] (apply max-key second with-scores)] 152 | (if (pos? score) 153 | (label->string label))))) 154 | 155 | (defn strict-get 156 | "Retrieves the value associated with the key k from the map m. Throws an exception if the key 157 | does not exist. The optional :key-desc keyword argument can be used to describe the keys in the 158 | map and will be included in the error message for the exception thrown if the key is not found." 159 | [m k & {:keys [key-desc] :or {key-desc "Key"}}] 160 | (let [v (get m k ::missing)] 161 | (if (= ::missing v) 162 | (throw (ex-info (format "%s %s not found" key-desc (str k)) {})) 163 | v))) 164 | 165 | (defn to-multimap 166 | "Takes a sequence of maps and returns a map of the form {:key [values]} for each key encountered in all 167 | of the input maps. Any entries where a key is mapped to a nil value is ignored." 168 | [maps] 169 | (let [non-nil-pairs (mapcat (fn [m] (filter (comp some? val) m)) maps)] 170 | (reduce (fn [acc [k v]] 171 | (update acc k (fnil conj []) v)) 172 | {} 173 | non-nil-pairs))) 174 | 175 | (defn xmls-boolean->boolean 176 | "XML Schema allows boolean values to be represented as the literals true, false 0 or 1. 177 | This function converts the clojure representation of those values into the corresponding 178 | boolean value." 179 | [xb] 180 | (if (number? xb) 181 | (= 1 xb) 182 | (boolean xb))) -------------------------------------------------------------------------------- /src/cubiql/vocabulary.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.vocabulary 2 | (:import [java.net URI])) 3 | 4 | (def sdmx:refArea (URI. "http://purl.org/linked-data/sdmx/2009/dimension#refArea")) 5 | (def sdmx:refPeriod (URI. "http://purl.org/linked-data/sdmx/2009/dimension#refPeriod")) 6 | (def time:hasBeginning (URI. "http://www.w3.org/2006/time#hasBeginning")) 7 | (def time:hasEnd (URI. "http://www.w3.org/2006/time#hasEnd")) 8 | (def time:inXSDDateTime (URI. "http://www.w3.org/2006/time#inXSDDateTime")) 9 | (def rdfs:label (URI. "http://www.w3.org/2000/01/rdf-schema#label")) 10 | (def skos:prefLabel (URI. "http://www.w3.org/2004/02/skos/core#prefLabel")) 11 | (def qb:codeList (URI. "http://purl.org/linked-data/cube#codeList")) 12 | (def qb:measureType (URI. "http://purl.org/linked-data/cube#measureType")) 13 | 14 | (def xsd:decimal (URI. "http://www.w3.org/2001/XMLSchema#decimal")) 15 | (def xsd:string (URI. "http://www.w3.org/2001/XMLSchema#string")) 16 | (def time:DateTime (URI. "http://www.w3.org/2006/time#inXSDDateTime")) -------------------------------------------------------------------------------- /test/cubiql/query_model_test.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.query-model-test 2 | (:require [clojure.test :refer :all] 3 | [cubiql.query-model :refer :all :as qm] 4 | [cubiql.vocabulary :refer [rdfs:label]]) 5 | (:import [java.net URI])) 6 | 7 | (deftest add-binding-test 8 | (testing "Default" 9 | (let [value "value" 10 | qm (add-binding empty-model [[:key (URI. "http://predicate")]] value)] 11 | (is (= value (get-path-binding-value qm [:key]))) 12 | (is (= false (is-path-binding-optional? qm [:key]))))) 13 | 14 | (testing "Optional" 15 | (let [value (URI. "http://value") 16 | qm (add-binding empty-model [[:key (URI. "http://predicate")]] value :optional? true)] 17 | (is (= value (get-path-binding-value qm [:key]))) 18 | (is (= true (is-path-binding-optional? qm [:key]))))) 19 | 20 | (testing "Required" 21 | (let [value (URI. "http://value") 22 | qm (add-binding empty-model [[:key (URI. "http://predicate")]] value :optional? false)] 23 | (is (= value (get-path-binding-value qm [:key]))) 24 | (is (= false (is-path-binding-optional? qm [:key]))))) 25 | 26 | (testing "Optional then required" 27 | (let [predicate (URI. "http://predicate") 28 | value "value" 29 | qm (-> empty-model 30 | (add-binding [[:key predicate]] ::qm/var :optional? true) 31 | (add-binding [[:key predicate]] value))] 32 | (is (= value (get-path-binding-value qm [:key]))) 33 | (is (= false (is-path-binding-optional? qm [:key]))))) 34 | 35 | (testing "Required then optional" 36 | (let [predicate (URI. "http://predicate") 37 | value "value" 38 | qm (-> empty-model 39 | (add-binding [[:key predicate]] ::qm/var :optional? false) 40 | (add-binding [[:key predicate]] value :optional true))] 41 | (is (= value (get-path-binding-value qm [:key]))) 42 | (is (= false (is-path-binding-optional? qm [:key]))))) 43 | 44 | (testing "Multiple values" 45 | (let [v1 "value1" 46 | v2 (URI. "http://value2") 47 | predicate (URI. "http://predicate") 48 | qm (add-binding empty-model [[:key predicate]] v1)] 49 | (is (thrown? Exception (add-binding qm [[:key predicate]] v2))))) 50 | 51 | (testing "Inconsistent predicates" 52 | (let [p1 (URI. "http://p1") 53 | p2 (URI. "http://p2") 54 | qm (add-binding empty-model [[:k p1] [:label rdfs:label]] ::qm/var)] 55 | (is (thrown? Exception (add-binding qm [[:k p2] [:label rdfs:label]] "value")))))) 56 | 57 | (deftest add-filter-test 58 | (testing "Existing binding" 59 | (let [f1 ['= "value"] 60 | f2 ['>= 3] 61 | predicate (URI. "http://p") 62 | qm (-> empty-model 63 | (add-binding [[:key predicate]] ::qm/var) 64 | (add-filter [:key] f1) 65 | (add-filter [:key] f2))] 66 | (is (= #{f1 f2} (set (get-path-filters qm [:key])))))) 67 | 68 | (testing "Non-existent binding" 69 | (is (thrown? Exception (add-filter empty-model [:invalid :path] ['>= 3]))))) 70 | 71 | (deftest add-order-by-test 72 | (let [qm (-> empty-model 73 | (add-binding [[:dim1 (URI. "http://dim1")] [:label rdfs:label]] "label") 74 | (add-binding [[:dim2 (URI. "http://dim2")]] (URI. "http://value2")) 75 | (add-order-by {:ASC [:dim2]}) 76 | (add-order-by {:DESC [:dim1 :label]}))] 77 | (is (= [{:ASC [:dim2]} {:DESC [:dim1 :label]}]) 78 | (get-order-by qm)))) 79 | 80 | -------------------------------------------------------------------------------- /test/cubiql/resolvers_test.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.resolvers-test 2 | (:require [clojure.test :refer :all] 3 | [cubiql.resolvers :refer :all]) 4 | (:import [java.net URI])) 5 | 6 | (deftest combine-dimension-results-test 7 | (let [dim1-uri (URI. "http://dim/gender") 8 | dim2-uri (URI. "http://dim2") 9 | dim3-uri (URI. "http://dim3") 10 | 11 | dim1-member1-uri (URI. "http://male") 12 | dim1-member2-uri (URI. "http://female") 13 | dim3-member1-uri (URI. "http://dim3mem1") 14 | 15 | codelist-members [{:dim dim1-uri :member dim1-member1-uri :label "Male"} 16 | {:dim dim3-uri :member dim3-member1-uri :label "Member"} 17 | {:dim dim1-uri :member dim1-member2-uri :label "Female"}] 18 | dimension-results [{:uri dim1-uri :label "Gender"} 19 | {:uri dim2-uri} 20 | {:uri dim3-uri :label "Dimension 3"}]] 21 | 22 | (is (= [{:uri dim1-uri :label "Gender" :values [{:uri dim1-member1-uri :label "Male"} 23 | {:uri dim1-member2-uri :label "Female"}]} 24 | {:uri dim2-uri} 25 | {:uri dim3-uri :label "Dimension 3" :values [{:uri dim3-member1-uri :label "Member"}]}] 26 | (combine-dimension-results dimension-results 27 | codelist-members))))) 28 | 29 | (deftest get-limit-test 30 | (are [expected requested configured-max] (= expected (get-limit {:first requested} {:max-observations-page-size configured-max})) 31 | ;no configured limit, no requested page size 32 | default-limit nil nil 33 | 34 | ;no requested page size, configured limit greater than default 35 | default-limit nil (* 2 default-limit) 36 | 37 | ;no requested page size, configured limit less than default 38 | (- default-limit 10) nil (- default-limit 10) 39 | 40 | ;no configured limit, requested exceeds default max 41 | default-max-observations-page-size (+ default-max-observations-page-size 10) nil 42 | 43 | ;no configured limit, requested less than default max 44 | (- default-max-observations-page-size 10) (- default-max-observations-page-size 10) nil 45 | 46 | ;less than configured limit 47 | 9000 9000 10000 48 | 49 | ;more than configured limit 50 | 10000 20000 10000 51 | 52 | ;requested negative 53 | 0 -1 nil 54 | )) 55 | 56 | (deftest total-count-required?-test 57 | (are [selections expected] (= expected (total-count-required? selections)) 58 | ;;total_matches selected 59 | {:total_matches nil} true 60 | 61 | ;;next_page requested 62 | {:page {:next_page nil}} true 63 | 64 | ;;total_matches and next_page requested 65 | {:total_matches nil :page {:next_page nil}} true 66 | 67 | ;;total matches and next page not requested 68 | {:sparql nil :page {:count nil :observation {:median nil}}} false)) 69 | 70 | (deftest calculate-next-page-offset-test 71 | (are [offset limit total-matches expected] (= expected (calculate-next-page-offset offset limit total-matches)) 72 | ;;unknown total-matches 73 | 100 10 nil nil 74 | 75 | ;;not last page 76 | 100 20 200 120 77 | 78 | ;;last page 79 | 100 20 110 nil)) -------------------------------------------------------------------------------- /test/cubiql/schema/mapping/labels_test.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.schema.mapping.labels-test 2 | (:require [clojure.test :refer :all] 3 | [cubiql.schema.mapping.labels :refer :all] 4 | [cubiql.types :as types] 5 | [grafter.rdf :as rdf]) 6 | (:import [java.net URI] 7 | [cubiql.types StringMeasureType FloatMeasureType])) 8 | 9 | (defn enum-name-value-map [enum-group] 10 | (into {} (map (juxt :name :value)) (:items enum-group))) 11 | 12 | (deftest components-enum-group-test 13 | (let [dim1-uri (URI. "http://dim1") 14 | measure1-uri (URI. "http://measure1") 15 | components [{:uri dim1-uri 16 | :label "Gender" 17 | :field-name :gender} 18 | {:uri measure1-uri 19 | :label "Count" 20 | :field-name :count}] 21 | components-enum-group (components-enum-group :dataset_test components) 22 | name->uri (into {} (map (juxt :name :value) (:items components-enum-group)))] 23 | (is (= :dataset_test_dimension_measures (:name components-enum-group))) 24 | (is (= {:GENDER dim1-uri :COUNT measure1-uri} name->uri)))) 25 | 26 | (deftest aggregation-measures-enum-group-test 27 | (testing "With numeric measures" 28 | (let [measure1-uri (URI. "http://measure1") 29 | measure2-uri (URI. "http://measure2") 30 | measure3-uri (URI. "http://measure3") 31 | m1 {:uri measure1-uri :label "Ratio" :is-numeric? true} 32 | m2 {:uri measure2-uri :label "Not numeric" :is-numeric? false} 33 | m3 {:uri measure3-uri :label "Count" :is-numeric? true} 34 | aggregation-enum (aggregation-measures-enum-group :dataset_test [m1 m2 m3])] 35 | (is (= :dataset_test_aggregation_measures (:name aggregation-enum))) 36 | (is (= {:RATIO measure1-uri :COUNT measure3-uri} (enum-name-value-map aggregation-enum))))) 37 | 38 | (testing "Without numeric measures" 39 | (let [m1 {:uri (URI. "http://measure1") :label "Measure 1" :is-numeric? false} 40 | m2 {:uri (URI. "http://measure2") :label "Measure 2" :is-numeric? false}] 41 | (is (nil? (aggregation-measures-enum-group :dataset_test [m1 m2])))))) 42 | 43 | (deftest get-dataset-observations-result-mapping-test 44 | ) 45 | 46 | (deftest get-datasets-enum-mappings-test 47 | (let [ds1-uri (URI. "http://ds1") 48 | ds2-uri (URI. "http://ds2") 49 | dim1-uri (URI. "http://dim1") 50 | dim2-uri (URI. "http://dim2") 51 | dim3-uri (URI. "http://dim3") 52 | 53 | dim1-label "Dimension 1" 54 | dim2-label "Dimension 2" 55 | dim3-label "Dimension 3" 56 | 57 | dim1-doc "Description for dimension 1" 58 | 59 | dim1-val1-uri (URI. "http://dim1val1") 60 | dim1-val2-uri (URI. "http://dim1val2") 61 | 62 | dim2-val1-uri (URI. "http://dim2val1") 63 | dim2-val2-uri (URI. "http://dim2val2") 64 | dim2-val3-uri (URI. "http://dim2val3") 65 | dim2-val4-uri (URI. "http://dim2val4") 66 | 67 | dim3-val1-uri (URI. "http://dim3val1") 68 | 69 | bindings [{:ds ds1-uri :dim dim1-uri :member dim1-val1-uri :vallabel "First"} 70 | {:ds ds1-uri :dim dim1-uri :member dim1-val2-uri :vallabel "Second"} 71 | {:ds ds2-uri :dim dim2-uri :member dim2-val1-uri :vallabel "Male"} 72 | {:ds ds2-uri :dim dim2-uri :member dim2-val2-uri :vallabel "Female"} 73 | {:ds ds2-uri :dim dim2-uri :member dim2-val3-uri :vallabel "All"} 74 | {:ds ds2-uri :dim dim2-uri :member dim2-val4-uri :vallabel "All"} 75 | {:ds ds2-uri :dim dim3-uri :member dim3-val1-uri :vallabel "Label"}] 76 | 77 | config {} 78 | datasets [(types/->Dataset ds1-uri :dataset1 [(types/->Dimension dim1-uri 1 types/enum-type)] []) 79 | (types/->Dataset ds2-uri :dataset2 [(types/->Dimension dim2-uri 1 types/enum-type) 80 | (types/->Dimension dim3-uri 2 types/enum-type)] [])] 81 | dimension-labels {dim1-uri {:label dim1-label :doc dim1-doc} 82 | dim2-uri {:label dim2-label :doc nil} 83 | dim3-uri {:label dim3-label :doc ""}} 84 | result (get-datasets-enum-mappings datasets bindings dimension-labels config)] 85 | (is (= {ds1-uri {dim1-uri {:label dim1-label :doc dim1-doc :items [(types/->EnumMappingItem :FIRST dim1-val1-uri "First") 86 | (types/->EnumMappingItem :SECOND dim1-val2-uri "Second")]}} 87 | ds2-uri {dim2-uri {:label dim2-label :doc "" :items [(types/->EnumMappingItem :MALE dim2-val1-uri "Male") 88 | (types/->EnumMappingItem :FEMALE dim2-val2-uri "Female") 89 | (types/->EnumMappingItem :ALL_1 dim2-val3-uri "All") 90 | (types/->EnumMappingItem :ALL_2 dim2-val4-uri "All")]} 91 | dim3-uri {:label dim3-label :doc "" :items [(types/->EnumMappingItem :LABEL dim3-val1-uri "Label")]}}} 92 | result)))) 93 | 94 | (deftest identify-dimension-labels-test 95 | (let [dim1-uri (URI. "http://dim1") 96 | dim2-uri (URI. "http://dim2") 97 | dim3-uri (URI. "http://dim3") 98 | 99 | bindings [{:dim dim1-uri :label (rdf/language "First dimension" :en) :doc nil} 100 | {:dim dim1-uri :label (rdf/language "Primero dimencion" :es) :doc nil} 101 | {:dim dim1-uri :label nil :doc "Dimension 1"} 102 | {:dim dim2-uri :label (rdf/language "Dimension 2" :en) :doc nil} 103 | {:dim dim2-uri :label nil :doc (rdf/language "Dimension two" :en)} 104 | {:dim dim2-uri :label nil :doc (rdf/language "Dimencion numero dos" :es)} 105 | {:dim dim3-uri :label "Dimension 3" :doc nil}] 106 | 107 | config {:schema-label-language "en"}] 108 | (is (= {dim1-uri {:label "First dimension" :doc "Dimension 1"} 109 | dim2-uri {:label "Dimension 2" :doc "Dimension two"} 110 | dim3-uri {:label "Dimension 3" :doc nil}} 111 | (identify-dimension-labels bindings config))))) 112 | 113 | (deftest identify-measure-labels-test 114 | (let [measure1-uri (URI. "http://measure1") 115 | measure2-uri (URI. "http://measure2") 116 | 117 | bindings [{:measure measure1-uri :label (rdf/language "First measure" :en)} 118 | {:measure measure1-uri :label "Second label"} 119 | {:measure measure2-uri :label "Second measure"} 120 | {:measure measure2-uri :label (rdf/language "Segundo measure" :es)}] 121 | 122 | config {:schema-label-language "en"}] 123 | (is (= {measure1-uri "First measure" 124 | measure2-uri "Second measure"} 125 | (identify-measure-labels bindings config))))) 126 | 127 | (deftest get-measure-type-test 128 | (let [measure-uri (URI. "http://measure") 129 | order 1] 130 | (testing "Numeric" 131 | (let [measure (types/->MeasureType measure-uri order true) 132 | mt (get-measure-type measure)] 133 | (is (instance? FloatMeasureType mt)))) 134 | 135 | (testing "Non-numeric" 136 | (let [measure (types/->MeasureType measure-uri 1 false) 137 | t (get-measure-type measure)] 138 | (is (instance? StringMeasureType t)))))) -------------------------------------------------------------------------------- /test/cubiql/schema_test.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.schema-test 2 | (:require [clojure.test :refer :all] 3 | [cubiql.schema :refer :all] 4 | [cubiql.types :as types] 5 | [cubiql.vocabulary :refer [qb:measureType]] 6 | [com.walmartlabs.lacinia.schema :as ls]) 7 | (:import [java.net URI])) 8 | 9 | (deftest dataset-observation-schema-model-test 10 | (let [dim1 {:field-name :dim1 :type types/string-type} 11 | dim2 {:field-name :dim2 :type types/decimal-type} 12 | measure1 {:field-name :measure1 :type (types/->FloatMeasureType)} 13 | measure2 {:field-name :measure2 :type (types/->StringMeasureType)} 14 | dsm {:uri (URI. "http://test") 15 | :schema :dataset_test 16 | :dimensions [dim1 dim2] 17 | :measures [measure1 measure2]} 18 | schema-model (dataset-observation-schema-model dsm)] 19 | (is (= {:uri {:type :uri} 20 | :dim1 {:type 'String} 21 | :dim2 {:type 'Float} 22 | :measure1 {:type 'Float} 23 | :measure2 {:type 'String}} 24 | schema-model)))) 25 | 26 | (deftest dataset-observation-dimensions-input-schema-model-test 27 | (let [dim1 {:field-name :dim1 :type types/string-type} 28 | dim2 {:field-name :dim2 :type types/decimal-type} 29 | dim3 {:field-name :dim3 :type (types/->MappedEnumType :enum-name types/enum-type nil [])} 30 | dsm {:uri (URI. "http://test") 31 | :schema :dataset_test 32 | :dimensions [dim1 dim2 dim3] 33 | :measures []} 34 | schema-model (dataset-observation-dimensions-input-schema-model dsm)] 35 | (is (= {:dim1 {:type 'String} 36 | :dim2 {:type 'Float} 37 | :dim3 {:type :enum-name}} 38 | schema-model)))) 39 | 40 | (deftest get-order-by-test 41 | (let [dim1-uri (URI. "http://dim1") 42 | dim2-uri (URI. "http://dim2") 43 | dim3-uri (URI. "http://dim3") 44 | measure1-uri (URI. "http://measure1") 45 | dim1 (types/->Dimension dim1-uri 1 (types/->DecimalType)) 46 | dim2 (types/->Dimension dim2-uri 2 (types/->StringType)) 47 | dim3 (types/->Dimension dim3-uri 3 types/enum-type) 48 | measure1 (types/->MeasureType measure1-uri 1 true) 49 | 50 | dim1-mapping {:uri dim1-uri :field-name :dim1 :dimension dim1} 51 | dim2-mapping {:uri dim2-uri :field-name :dim2 :dimension dim2} 52 | dim3-mapping {:uri dim3-uri :field-name :dim3 :dimension dim3} 53 | measure1-mapping {:uri measure1-uri :field-name :measure1 :measure measure1} 54 | 55 | dsm {:uri (URI. "http://test") 56 | :schema :dataset_test 57 | :dimensions [dim1-mapping dim2-mapping dim3-mapping] 58 | :measures [measure1-mapping]} 59 | order [dim1-uri measure1-uri dim2-uri] 60 | order-spec {measure1-uri :ASC dim2-uri :DESC}] 61 | (is (= [[dim1 :ASC] [measure1 :ASC] [dim2 :DESC]] 62 | (get-order-by {:order order :order_spec order-spec} dsm))))) 63 | 64 | (deftest map-dimension-filter-test 65 | (let [dim1-uri (URI. "http://dim1") 66 | dim2-uri (URI. "http://dim2") 67 | dim3-uri (URI. "http://dim3") 68 | dim4-uri (URI. "http://dim4") 69 | 70 | dim2-value (URI. "http://value2") 71 | 72 | dim1 (types/->Dimension dim1-uri 1 types/decimal-type) 73 | dim2 (types/->Dimension dim2-uri 2 types/enum-type) 74 | dim3 (types/->Dimension dim3-uri 3 types/string-type) 75 | dim4 (types/->Dimension dim4-uri 4 types/decimal-type) 76 | 77 | dsm {:uri (URI. "http://test") 78 | :schema :dataset_test 79 | :dimensions [{:uri dim1-uri :dimension dim1} 80 | {:uri dim2-uri :dimension dim2} 81 | {:uri dim3-uri :dimension dim3} 82 | {:uri dim4-uri :dimension dim4}] 83 | :measures []} 84 | dimension-args {dim1-uri 5 85 | dim2-uri dim2-value 86 | dim3-uri "value1"}] 87 | (is (= {dim1 5 88 | dim2 (URI. "http://value2") 89 | dim3 "value1" 90 | dim4 nil} 91 | (map-dimension-filter dimension-args dsm))))) 92 | 93 | (deftest map-dataset-observation-args-test 94 | (let [dim1-uri (URI. "http://dim1") 95 | dim2-uri (URI. "http://dim2") 96 | dim3-uri (URI. "http://dim3") 97 | measure1-uri (URI. "http://measure1") 98 | 99 | dim2-value1 (URI. "http://value1") 100 | dim2-value2 (URI. "http://value2") 101 | dim2-mapped-type (types/->MappedEnumType :enum2 types/enum-type "" [(types/->EnumMappingItem :VALUE1 dim2-value1 "Value 1") 102 | (types/->EnumMappingItem :VALUE2 dim2-value2 "Value 2")]) 103 | dsm {:uri (URI. "http://test") 104 | :schema :dataset_test 105 | :dimensions [{:uri dim1-uri :field-name :dim1 :type types/decimal-type :enum-name :DIM1} 106 | {:uri dim2-uri :field-name :dim2 :type dim2-mapped-type :enum-name :DIM2} 107 | {:uri dim3-uri :field-name :dim3 :type types/string-type :enum-name :DIM3}] 108 | :measures [{:uri measure1-uri :field-name :measure1 :type (types/->FloatMeasureType) :enum-name :MEASURE1}]} 109 | args {:dimensions {:dim1 4 :dim2 :VALUE2 :dim3 "value"} 110 | :order [:MEASURE1 :DIM1 :DIM3] 111 | :order_spec {:dim1 :ASC :dim3 :DESC :measure1 :DESC}} 112 | expected {:dimensions {dim1-uri 4 dim2-uri dim2-value2 dim3-uri "value"} 113 | :order [measure1-uri dim1-uri dim3-uri] 114 | :order_spec {dim1-uri :ASC dim3-uri :DESC measure1-uri :DESC}}] 115 | (is (= expected (map-dataset-observation-args args dsm))))) 116 | 117 | (deftest map-dataset-measure-results-test 118 | (let [measure1-uri (URI. "http://measure1") 119 | measure2-uri (URI. "http://measure2") 120 | measure1-label "First measure" 121 | measure2-label "Second measure" 122 | measure1-name :MEASURE1 123 | measure2-name :OTHER_NAME 124 | 125 | measure1 {:uri measure1-uri :enum-name measure1-name} 126 | measure2 {:uri measure2-uri :enum-name measure2-name} 127 | 128 | results [{:uri measure1-uri :label measure1-label} 129 | {:uri measure2-uri :label measure2-label}] 130 | 131 | dsm {:uri (URI. "http://test-dataset") 132 | :schema :dataset_test 133 | :dimensions [] 134 | :measures [measure1 measure2]}] 135 | (is (= [{:uri measure1-uri :label measure1-label :enum_name (name measure1-name)} 136 | {:uri measure2-uri :label measure2-label :enum_name (name measure2-name)}] 137 | (map-dataset-measure-results dsm results))))) 138 | 139 | (deftest annotate-dataset-dimensions-test 140 | (let [dim1-uri (URI. "http://dim1") 141 | dim2-uri (URI. "http://dim2") 142 | dim3-uri (URI. "http://dim3") 143 | 144 | dim1-val1 {:uri (URI. "http://dim1val1") :label "Value 1"} 145 | dim1-val2 {:uri (URI. "http://dim1val2") :label "Value 2"} 146 | dim1-result {:uri dim1-uri :values [dim1-val1 dim1-val2]} 147 | dim2-result {:uri dim2-uri :values nil} 148 | 149 | dim3-val1-uri (URI. "http://dim3val1") 150 | dim3-val2-uri (URI. "http://dim3val2") 151 | dim3-val3-uri (URI. "http://dim3val3") 152 | dim3-result {:uri dim3-uri :values [{:uri dim3-val1-uri :label "Value 1"} 153 | {:uri dim3-val2-uri :label "Value 2"} 154 | {:uri dim3-val3-uri :label "Value 3"}]} 155 | 156 | dsm {:uri (URI. "http://test-dataset") 157 | :schema :dataset_test 158 | :dimensions [{:uri dim1-uri :type types/ref-area-type :enum-name :DIM1} 159 | {:uri dim2-uri :type types/decimal-type :enum-name :DIM2} 160 | {:uri dim3-uri 161 | :type (types/->MappedEnumType :dim3 types/enum-type "" [(types/->EnumMappingItem :VALUE1 dim3-val1-uri "Value 1") 162 | (types/->EnumMappingItem :VALUE3 dim3-val3-uri "Value 3") 163 | (types/->EnumMappingItem :VALUE2 dim3-val2-uri "Value 2")]) 164 | :enum-name :DIM3}]} 165 | 166 | result (annotate-dataset-dimensions dsm [dim1-result dim2-result dim3-result])] 167 | (is (= [{:uri dim1-uri :enum_name "DIM1" :values [(ls/tag-with-type dim1-val1 :unmapped_dim_value) 168 | (ls/tag-with-type dim1-val2 :unmapped_dim_value)]} 169 | {:uri dim2-uri :enum_name "DIM2" :values nil} 170 | {:uri dim3-uri :enum_name "DIM3" :values [(ls/tag-with-type 171 | {:uri dim3-val1-uri :label "Value 1" :enum_name "VALUE1"} :enum_dim_value) 172 | (ls/tag-with-type 173 | {:uri dim3-val2-uri :label "Value 2" :enum_name "VALUE2"} :enum_dim_value) 174 | (ls/tag-with-type 175 | {:uri dim3-val3-uri :label "Value 3" :enum_name "VALUE3"} :enum_dim_value)]}] 176 | result)))) 177 | 178 | (deftest dataset-order-spec-schema-model-test 179 | (let [dsm {:uri (URI. "http://test-dataset") 180 | :schema :dataset_test 181 | :dimensions [{:field-name :gender} 182 | {:field-name :area}] 183 | :measures [{:field-name :median} 184 | {:field-name :count}]}] 185 | (is (= {:gender {:type :sort_direction} 186 | :area {:type :sort_direction} 187 | :median {:type :sort_direction} 188 | :count {:type :sort_direction}}) 189 | (dataset-order-spec-schema-model dsm)))) 190 | 191 | (deftest map-observation-selections-test 192 | (let [dim1-uri (URI. "http://dim1") 193 | dim2-uri (URI. "http://dim2") 194 | dim3-uri (URI. "http://dim3") 195 | dim4-uri (URI. "http://dim4") 196 | measure1-uri (URI. "http://measure1") 197 | measure2-uri (URI. "http://measure2") 198 | 199 | dsm {:uri (URI. "http://test-dataset") 200 | :schema :dataset_test 201 | :dimensions [{:uri dim1-uri :type types/ref-area-type :field-name :dim1} 202 | {:uri dim2-uri :type types/ref-period-type :field-name :dim2} 203 | {:uri dim3-uri :type types/enum-type :field-name :dim3} 204 | {:uri dim4-uri :type types/decimal-type :field-name :dim4}] 205 | :measures [{:uri measure1-uri :type (types/->FloatMeasureType) :field-name :measure1} 206 | {:uri measure2-uri :type (types/->StringMeasureType) :field-name :measure2}]} 207 | 208 | dim1-selections {:label nil :uri nil} 209 | dim2-selections {:uri nil :start nil} 210 | selections {:uri nil 211 | :dim1 dim1-selections 212 | :dim2 dim2-selections 213 | :dim3 nil 214 | :measure2 nil}] 215 | (is (= {dim1-uri dim1-selections 216 | dim2-uri dim2-selections 217 | dim3-uri nil 218 | measure2-uri nil} 219 | (map-observation-selections dsm selections))))) 220 | 221 | (deftest dataset-enum-types-schema-test 222 | (let [dim1 {:uri (URI. "http://dim1") 223 | :type types/string-type} 224 | dim2 {:uri (URI. "http://dim2") 225 | :type (types/->MappedEnumType :enum1 types/enum-type "description" [(types/->EnumMappingItem :VALUE1 (URI. "http://val1") "value1") 226 | (types/->EnumMappingItem :VALUE2 (URI. "http://val2") "value2")])} 227 | dim3 {:uri (URI. "http://dim3") 228 | :type types/decimal-type} 229 | dim4 {:uri (URI. "http://dim4") 230 | :type (types/->MappedEnumType :enum2 types/enum-type nil [(types/->EnumMappingItem :VALUE3 (URI. "http://val3") "value3")])} 231 | dsm {:uri (URI. "http://test") 232 | :schema :dataset_test 233 | :dimensions [dim1 dim2 dim3 dim4] 234 | :measures []} 235 | enums-schema (dataset-enum-types-schema dsm)] 236 | (is (= {:enum1 {:values [:VALUE1 :VALUE2] :description "description"} 237 | :enum2 {:values [:VALUE3]}} 238 | enums-schema)))) 239 | 240 | (deftest get-observation-result-test 241 | (testing "With qb:measureType dimension" 242 | (let [dimension1-uri (URI. "http://dimension1") 243 | measure1-uri (URI. "http://measure1") 244 | measure2-uri (URI. "http://measure2") 245 | dim1 {:uri dimension1-uri 246 | :field-name :gender 247 | :type (types/->StringType) 248 | :dimension (types/->Dimension dimension1-uri 1 (types/->StringType))} 249 | dim2 {:uri qb:measureType 250 | :field-name :measure_type 251 | :type (types/->MappedEnumType :dataset_test_measure_type_type (types/->MeasureDimensionType) "Measure type" [(types/->EnumMappingItem :MEASURE1 measure1-uri "Measure 1") 252 | (types/->EnumMappingItem :MEASURE2 measure2-uri "Measure 2")]) 253 | :dimension (types/->Dimension qb:measureType 2 (types/->MeasureDimensionType))} 254 | measure1 {:uri measure1-uri 255 | :label "Measure 1" 256 | :field-name :measure1 257 | :type (types/->StringMeasureType) 258 | :measure (types/->MeasureType measure1-uri 1 false)} 259 | measure2 {:uri measure2-uri 260 | :label "Measure 2" 261 | :field-name :measure2 262 | :type (types/->FloatMeasureType) 263 | :measure (types/->MeasureType measure2-uri 2 true)} 264 | dsm {:uri (URI. "http://test-dataset") 265 | :schema :dataset_test 266 | :dimensions [dim1 dim2] 267 | :measures [measure1 measure2]} 268 | obs-uri (URI. "http://obs") 269 | bindings {:obs obs-uri :dim1 "MALE" :dim2 measure1-uri :mp measure1-uri :mv "VALUE"}] 270 | (is (= {:uri obs-uri :gender "MALE" :measure_type :MEASURE1 :measure1 "VALUE" :measure2 nil} 271 | (get-observation-result dsm bindings))))) 272 | 273 | (testing "Without qb:measureType dimension" 274 | (let [dimension1-uri (URI. "http://dimension1") 275 | dimension2-uri (URI. "http://dimension2") 276 | measure1-uri (URI. "http://measure1") 277 | measure2-uri (URI. "http://measure2") 278 | dim1 {:uri dimension1-uri 279 | :field-name :gender 280 | :type (types/->MappedEnumType :dataset_test_gender_type types/enum-type "Gender" [(types/->EnumMappingItem :MALE (URI. "http://male") "Male") 281 | (types/->EnumMappingItem :FEMALE (URI. "http://female") "Female")]) 282 | :dimension (types/->Dimension dimension1-uri 1 types/enum-type)} 283 | dim2 {:uri dimension2-uri 284 | :field-name :area 285 | :type types/ref-area-type 286 | :dimension (types/->Dimension dimension2-uri 2 types/ref-area-type)} 287 | measure1 {:uri measure1-uri 288 | :label "Count" 289 | :field-name :count 290 | :type (types/->FloatMeasureType) 291 | :measure (types/->MeasureType measure1-uri 1 true)} 292 | measure2 {:uri measure2-uri 293 | :field-name :category 294 | :type (types/->StringMeasureType) 295 | :measure (types/->MeasureType measure2-uri 2 false)} 296 | dsm {:uri (URI. "http://test-dataset") 297 | :schema :dataset_test 298 | :dimensions [dim1 dim2] 299 | :measures [measure1 measure2]} 300 | obs-uri (URI. "http://obs") 301 | bindings {:obs obs-uri :dim1 (URI. "http://female") :dim2 (URI. "http://area") :dim2label "Area" :mv1 34 :mv2 "LOW"}] 302 | (is (= {:uri obs-uri 303 | :gender :FEMALE 304 | :area {:uri (URI. "http://area") 305 | :label "Area"} 306 | :count 34.0 307 | :category "LOW"} 308 | (get-observation-result dsm bindings)))))) 309 | 310 | (deftest get-measure-type-measure-value-test 311 | (testing "Measure matching measure type" 312 | (let [uri (URI. "http://measure") 313 | measure (types/->MeasureType uri 1 true) 314 | value 4 315 | bindings {:mp uri :mv 4}] 316 | (is (= value (get-measure-type-measure-value measure bindings))))) 317 | 318 | (testing "Measure not matching measure type" 319 | (let [uri (URI. "http://measure1") 320 | measure (types/->MeasureType uri 1 true) 321 | bindings {:mp (URI. "http://measure2") :mv 5}] 322 | (is (nil? (get-measure-type-measure-value measure bindings)))))) 323 | 324 | (deftest get-multi-measure-value-test 325 | (let [measure1-uri (URI. "http://measure1") 326 | measure2-uri (URI. "http://measure2") 327 | measure1 (types/->MeasureType measure1-uri 1 true) 328 | measure2 (types/->MeasureType measure2-uri 2 false) 329 | bindings {:mv1 4 :mv2 "value"}] 330 | (is (= 4 (get-multi-measure-value measure1 bindings))) 331 | (is (= "value" (get-multi-measure-value measure2 bindings))))) -------------------------------------------------------------------------------- /test/cubiql/types_test.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.types-test 2 | (:require [clojure.test :refer :all] 3 | [cubiql.types :refer :all] 4 | [cubiql.types.scalars :as scalars] 5 | [cubiql.types :as types] 6 | [cubiql.query-model :as qm]) 7 | (:import [java.net URI] 8 | [java.util Date])) 9 | 10 | (deftest project-result-test 11 | (testing "Dimension with ref period type" 12 | (let [dim (->Dimension (URI. "http://dim") 1 ref-period-type) 13 | period-uri (URI. "http://period") 14 | period-label "Period label" 15 | period-start (Date. 112, 0 1) 16 | period-end (Date. 113 5 13) 17 | bindings {:dim1 period-uri :dim1label period-label :dim1begintime period-start :dim1endtime period-end} 18 | expected {:uri period-uri 19 | :label period-label 20 | :start (scalars/grafter-date->datetime period-start) 21 | :end (scalars/grafter-date->datetime period-end)}] 22 | (is (= expected (project-result dim bindings))))) 23 | 24 | (testing "Dimensions with ref area type" 25 | (let [dim (->Dimension (URI. "http://dim") 1 (->RefAreaType)) 26 | area-uri (URI. "http://refarea") 27 | area-label "Area label" 28 | bindings {:dim1 area-uri :dim1label area-label}] 29 | (is (= {:uri area-uri :label area-label} (project-result dim bindings))))) 30 | 31 | (testing "Dimension with codelist type" 32 | (let [type types/enum-type 33 | dim (->Dimension (URI. "http://dim") 1 type) 34 | value (URI. "http://val1") 35 | bindings {:dim1 value}] 36 | (is (= value (project-result dim bindings)))))) 37 | 38 | (deftest apply-filter-test 39 | (testing "Ref period dimension" 40 | (let [dim-uri (URI. "http://dim") 41 | dim (->Dimension dim-uri 1 ref-period-type) 42 | starts-after (scalars/parse-datetime "2000-01-01T00:00:00Z") 43 | ends-after (scalars/parse-datetime "2000-08-01T00:00:00Z") 44 | filter {:starts_after starts-after 45 | :ends_after ends-after} 46 | model (apply-filter dim qm/empty-model filter)] 47 | (is (= ::qm/var (qm/get-path-binding-value model [:dim1 :begin :time]))) 48 | (is (= ::qm/var (qm/get-path-binding-value model [:dim1 :end :time])))))) 49 | 50 | -------------------------------------------------------------------------------- /test/cubiql/util_test.clj: -------------------------------------------------------------------------------- 1 | (ns cubiql.util-test 2 | (:require [clojure.test :refer :all] 3 | [cubiql.util :refer :all] 4 | [grafter.rdf :as rdf])) 5 | 6 | (deftest rename-key-test 7 | (testing "Key exists" 8 | (is (= {:new-key 1} (rename-key {:old-key 1} :old-key :new-key)))) 9 | 10 | (testing "Key missing strict" 11 | (is (thrown? IllegalArgumentException (rename-key {:k 1} :old-key :new-key :strict? true)))) 12 | 13 | (testing "Key missing non-strict" 14 | (let [m {:k 1}] 15 | (is (= m (rename-key m :old-key :new-key)))))) 16 | 17 | (deftest find-best-language-test 18 | (testing "Exact language match" 19 | (let [strings [(rdf/language "English" :en) 20 | (rdf/language "Deutsche" :de) 21 | (rdf/language "Francais" :fr) 22 | "No langauge"]] 23 | (is (= "English" (find-best-language strings "en"))))) 24 | 25 | (testing "English match" 26 | (let [strings [(rdf/language "English" :en) 27 | (rdf/language "Deutsche" :de) 28 | (rdf/language "Espanol" :es)]] 29 | (is (= "English" (find-best-language strings "fr"))))) 30 | 31 | (testing "String literal matches" 32 | (let [strings [(rdf/language "Francais" :fr) 33 | (rdf/language "Espanol" :es) 34 | "No language"]] 35 | (is (= "No language" (find-best-language strings "de"))))) 36 | 37 | (testing "No matches" 38 | (let [strings [(rdf/language "Espanol" :es) 39 | (rdf/language "Francais" :fr) 40 | (rdf/language "Deutsche" :de)]] 41 | (is (nil? (find-best-language strings "jp"))))) 42 | 43 | (testing "Empty collection" 44 | (is (nil? (find-best-language [] "en"))))) 45 | 46 | (deftest strict-get-test 47 | (testing "Key exists" 48 | (is (= 1 (strict-get {:a 1 :b 2} :a)))) 49 | 50 | (testing "Key does not exist" 51 | (is (thrown? Exception (strict-get {:a 1 :b 2} :missing)))) 52 | 53 | (testing "Key does not exist with description" 54 | (is (thrown-with-msg? Exception #"Widget :missing not found" (strict-get {:a 1 :b 2} :missing :key-desc "Widget"))))) 55 | 56 | (deftest strict-map-by-test 57 | (testing "Unique keys" 58 | (let [index->item (fn [i] {:key i :value (str "value" i)}) 59 | items (map index->item (range 1 11)) 60 | expected (into {} (map (fn [v] [(:key v) v]) items))] 61 | (is (= expected (strict-map-by :key items))))) 62 | 63 | (testing "Duplicate keys" 64 | (let [items [{:key 1 :value "value1"} 65 | {:key 2 :value "value2"} 66 | {:key 1 :value "value3"}]] 67 | (is (thrown? Exception (strict-map-by :key items)))))) 68 | 69 | (deftest to-multimap-test 70 | (let [maps [{:a 1 :b 2} 71 | {:a 3 :b nil :c 4} 72 | {:a 5 :d 6} 73 | {:a nil :b 7 :c 8 :d 9} 74 | {} 75 | {:c 10}]] 76 | (is (= {:a [1 3 5] 77 | :b [2 7] 78 | :c [4 8 10] 79 | :d [6 9]} 80 | (to-multimap maps))))) 81 | 82 | (deftest xmls-boolean->boolean-test 83 | (are [b expected] (= expected (xmls-boolean->boolean b)) 84 | false false 85 | true true 86 | 0 false 87 | 0N false 88 | 1 true 89 | 1N true 90 | nil false)) --------------------------------------------------------------------------------