├── _config.yml
├── stata.toc
├── dbnomics.pkg
├── LICENSE
├── README.md
├── dbnomics.sthlp
├── dbnomics.html
└── dbnomics.ado
/_config.yml:
--------------------------------------------------------------------------------
1 | theme: jekyll-theme-slate
--------------------------------------------------------------------------------
/stata.toc:
--------------------------------------------------------------------------------
1 | p dbnomics Stata client for DB.nomics, the world's economic database
2 |
--------------------------------------------------------------------------------
/dbnomics.pkg:
--------------------------------------------------------------------------------
1 | d 'DBNOMICS': Stata client for DB.nomics, the world's economic database
2 | d
3 | d dbnomics provides a suite of tools to search, browse and import
4 | d time series data from DB.nomics, the world's economic database
5 | d (https://db.nomics.world). DB.nomics is a web-based platform
6 | d that aggregates and maintains time series data from various
7 | d statistical agencies across the world. dbnomics only works
8 | d with Stata 14.0 or higher.
9 | d dbnomics provides an interface between Stata and DB.nomics' API
10 | d (https://api.db.nomics.world/apidocs). It enables creating
11 | d custom queries through Stata's options syntax (see Examples).
12 | d
13 | d KW: dbnomics
14 | d KW: data management
15 | d KW: economic data
16 | d
17 | d Requires: Stata version 14.0, libjson and moss from SSC (q.v.)
18 | d
19 | d Distribution-Date: 20200617
20 | d
21 | d Author: Simone Signore, European Investment Bank Group
22 | d Support: email signoresimone at yahoo [dot] it
23 | d
24 | f dbnomics.ado
25 | f dbnomics.sthlp
26 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2018 dreameater89
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # dbnomics
2 | Stata client for DB.nomics, the world's economic database (https://db.nomics.world)
3 |
4 | ## Description
5 |
6 | dbnomics provides a suite of tools to search, browse and import time series data from DB.nomics, the world's economic database (https://db.nomics.world). DB.nomics is a web-based platform that aggregates and maintains time series data from various statistical agencies across the world. dbnomics only works with Stata 14.0 or higher.
7 |
8 | dbnomics provides an interface between Stata and DB.nomics' API (https://api.db.nomics.world/apidocs). It enables creating custom queries through Stata's options syntax (see Examples). To achieve this, the command relies on Erik Lindsley's libjson Mata library (ssc install libjson).
9 |
10 | ## Installation
11 |
12 | `net install dbnomics, from("https://dbnomics-stata.github.io/dbnomics")`
13 |
14 | ## Syntax
15 |
16 | ```Stata
17 | . //Load list of providers
18 | . dbnomics providers [, clear insecure]
19 |
20 | . //Load content tree for a given provider
21 | . dbnomics tree , provider(PRcode) [clear insecure]
22 |
23 | . //Load dataset structure given a provider and dataset
24 | . dbnomics datastructure , provider(PRcode) dataset(DScode) [clear insecure]
25 |
26 | . // Load list of series for a given provider and dataset
27 | . dbnomics series , provider(PRcode) dataset(DScode) [dimensions_opt|sdmx(SDMX_mask)] [clear insecure limit(int) offset(int)]
28 |
29 | . // Import series for a given provider and dataset
30 | . dbnomics import , provider(PRcode) dataset(DScode) [dimensions_opt|sdmx(SDMX_mask)|seriesids(SERIES_list)] [clear insecure limit(int) offset(int)]
31 |
32 | . // Load a single series
33 | . dbnomics use series_id , provider(PRcode) dataset(DScode) [clear insecure]
34 |
35 | . // Search for data across DB.nomics' providers
36 | . dbnomics find search_str [, clear insecure limit(int) offset(int)]
37 |
38 | . // Load and display recently updated datasets
39 | . dbnomics news [, clear insecure limit(int) offset(int)]
40 | ```
41 |
42 | ## Options
43 | ### Main
44 |
45 | - **provider(**PRcode**)** sets the source with the data of interest. The list of data providers is regularly updated by the DB.nomics team. To get the list of available data providers, type **dbnomics providers [, clear]**.
46 |
47 | - **dataset(**DScode**)** sets the dataset with the time series of interest. To get a hierarchy of all datasets linked to a particular provider(*PRcode*), type **dbnomics tree, provider(PRcode) [clear]**. (**Note**: some datasets in the category tree may not be accessible).
48 |
49 | - **clear** clears data in memory before loading data from DB.nomics.
50 |
51 | - **insecure** instructs dbnomics to access the DB.nomics platform via the http protocol, instead of the more secure https.
52 |
53 | ### series
54 |
55 | - **dimensions_opt(**dim_values**)** are options that depend on the specific provider(*PRcode*) and dataset(*DScode*) and are used to identify a subset of series. A list of dimensions can be obtained using **dbnomics datastructure, provider(PRcode) dataset(DScode)**. For example, if the dimensions of dataset(*DScode*) are *freq.unit.geo*, accepted options for **dbnomics series,(...)** and **dbnomics import,(...)** are **freq(**codelist**)**, **unit(**codelist**)** and **geo(**codelist**)**. Each **dimension_opt(**dim_values**)** can contain one or more values in a space or comma-separated list, so as to select multiple dimensions at once (*e.g.*, a list of countries). **Note:** dbnomics is not able to validate user-inputted **dimension_opt(**dim_values**)**; if **dimension_opt(**dim_values**)** is incorrectly specified, dbnomics may throw an error or return the message no series found. See the Examples section.
56 |
57 | - **sdmx(**SDMX_mask**)** accepts an "SDMX mask" containing a list of dimensions separated with a dot "." character. The dimensions are specific to each provider(*PRcode*) and dataset(*DScode*) and must be provided in the order specified by the dataset(*DScode*) data structure. The data structure can be obtained using **dbnomics datastructure, provider(PRcode) dataset(DScode)**. **Note:** some providers do not support this feature. In such case dbnomics may throw an error or return the message "no series found". See the Examples section.
58 |
59 | **Note:** **dimensions_opt(**dim_values**)** and **sdmx(**SDMX_mask**)** are mutually exclusive.
60 |
61 | - **limit(**int**)** sets the maximum number of series to load. It limits the number of series identified by the **dbnomics series** or **dbnomics import** commands, resulting in faster download of information. A Warning message is issued if the total number of series identified is larger than the inputted limit(int).
62 |
63 | - **offset(**int**)** skips the first int series when loading data from dbnomics. This option can be combined with **limit(**int**)** to get a specific subset of series.
64 |
65 | ### import
66 |
67 | - **dimensions_opt(**dim_values**)** see above.
68 |
69 | - **sdmx(**SDMX_mask**)** see above.
70 |
71 | - **seriesids(**SERIES_list**)** accepts a comma-separated list of series
72 | identifiers that belong to a provider(*PRcode*) and dataset(*DScode*).
73 |
74 | **Note:** **dimensions_opt(**dim_values**)**, **sdmx(**SDMX_mask**)** and **seriesids(**SERIES_list**)** are mutually exclusive.
75 |
76 | - **limit(**int**)** sets the maximum number of series to load. It limits the number of series identified by the **dbnomics series** or **dbnomics import** commands, resulting in faster download of information. A Warning message is issued if the total number of series identified is larger than the inputted limit(int).
77 |
78 | - **offset(**int**)** skips the first int series when loading data from dbnomics. This option can be combined with **limit(**int**)** to get a specific subset of series. See the Examples section.
79 |
80 | ### use
81 |
82 | - series_id is the unique identifier of a time series belonging to provider(*PRcode*) and dataset(*DScode*).
83 |
84 | ### find/news
85 |
86 | - **limit(**int**)** sets the maximum number of results to load and display.
87 |
88 | - **offset(**int**)** skips the first int results. May be combined with **limit(**int**)** to get a specific subset of results.
89 |
90 | ## Remarks
91 |
92 | **dbnomics** has two main dependencies:
93 |
94 | 1. The Mata json library **libjson** by Erik Lindsley, needed to parse JSON strings. It can be found on SSC: `ssc install libjson`.
95 | 2. The routine **moss** by Robert Picard & Nicholas J. Cox, needed to clean unicode sequences. It can be found on SSC: `ssc install moss`.
96 |
97 | After each API call, dbnomics stores significant metadata in the form of dataset characteristics. Type *char li _dta[]* after dbnomics to obtain important info about the data, e.g., the API endpoint.
98 |
99 | ## Examples
100 |
101 | ```Stata
102 | . // Search for producer price data on DB.nomics' platform:
103 | . dbnomics find "producer price", clear
104 | (output omitted)
105 |
106 | . // Load the list of available providers with additional metadata:
107 | . dbnomics providers, clear
108 | Fetching providers list
109 | (63 providers read)
110 |
111 | . // Show recently updated datasets:
112 | . dbnomics news, clear
113 | (output omitted)
114 |
115 | . // Load the dataset tree of of the AMECO provider:
116 | . dbnomics tree, provider(AMECO) clear
117 |
118 | . // Analyse the structure of dataset PIGOT for provider AMECO:
119 | . dbnomics datastructure, provider(AMECO) dataset(PIGOT) clear
120 | Price deflator gross fixed capital formation: other investment
121 | 82 series found. Order of dimensions: (freq.unit.geo)
122 |
123 | . // List all series in AMECO/PIGOT containing deflators in national currency:
124 | . dbnomics series, provider(AMECO) dataset(PIGOT) unit(national-currency-2015-100) clear
125 | 39 of 82 series selected. Order of dimensions: (freq.unit.geo)
126 |
127 | . // Import all series in AMECO/PIGOT containing deflators in national currency:
128 | . dbnomics import, provider(AMECO) dataset(PIGOT) unit(national-currency-2015-100) clear
129 | Processing 39 series
130 | ........................................
131 | 39 series found and imported
132 |
133 | . // Import a few AMECO/PIGOT series:
134 | . dbnomics import, pr(AMECO) d(PIGOT) seriesids(ESP.3.1.0.0.PIGOT,SVN.3.1.0.0.PIGOT,LVA.3.1.99.0.PIGOT) clear
135 | Processing 3 series
136 | ...
137 | 3 series found and imported
138 |
139 | . // Import one specific series from AMECO/PIGOT:
140 | . dbnomics use ESP.3.1.0.0.PIGOT, pr(AMECO) d(PIGOT) clear
141 | Annually – (National currency: 2015 = 100) – Spain (AMECO/PIGOT/ESP.3.1.0.0.PIGOT)
142 | (62 observations read)
143 |
144 | . // Eurostat supports SMDX queries. Import all series in Eurostat/ei_bsin_q_r2 related to Belgium:
145 | . dbnomics import, provider(Eurostat) dataset(ei_bsin_q_r2) geo(BE) s_adj(NSA) clear
146 | Processing 14 series
147 | ..............
148 | 14 series found and imported
149 |
150 | . // Do the same using sdmx:
151 | . dbnomics import, provider(Eurostat) dataset(ei_bsin_q_r2) sdmx(Q..NSA.BE) clear
152 | Processing 14 series
153 | ..............
154 | 14 series found and imported
155 |
156 | . // The Eurostat/urb_ctran dataset offers 12280 series, more than permitted at once by DB.nomics:
157 | . dbnomics series, provider(Eurostat) dataset(urb_ctran) clear
158 | Warning: series set larger than dbnomics maximum provided items.
159 | Use the offset option to load series beyond the 1000th one.
160 | 12280 series selected. Order of dimensions: (FREQ.indic_ur.cities). Only #1 to #1000 retrieved
161 |
162 | . // Using limit and offset, we can instruct dbnomics to only get series #1001 to #1100:
163 | . dbnomics import, provider(Eurostat) dataset(urb_ctran) limit(100) offset(1000) clear
164 | Processing 100 series
165 | .................................................. 50
166 | .................................................. 100
167 | 12280 series found and #1001 to #1100 loaded
168 | ```
169 |
170 | ## Bugs?
171 |
172 | Write at: signoresimone at yahoo [dot] it
173 |
--------------------------------------------------------------------------------
/dbnomics.sthlp:
--------------------------------------------------------------------------------
1 | {smcl}
2 | {* *! version 1.1.2 30oct2018}{...}
3 | {viewerjumpto "Syntax" "dbnomics##syntax"}{...}
4 | {viewerjumpto "Description" "dbnomics##description"}{...}
5 | {viewerjumpto "Options" "dbnomics##options"}{...}
6 | {viewerjumpto "Remarks" "dbnomics##remarks"}{...}
7 | {viewerjumpto "Examples" "dbnomics##examples"}{...}
8 | {viewerjumpto "Stored results" "dbnomics##stored"}{...}
9 | {viewerjumpto "Author" "dbnomics##author"}{...}
10 |
11 | {title:Title}
12 |
13 | {phang}
14 | {bf:dbnomics} {hline 2} Stata client for DB.nomics, the world's economic database ({browse "https://db.nomics.world":https://db.nomics.world})
15 |
16 | {marker description}{...}
17 | {title:Description}
18 |
19 | {pstd}
20 | {cmd:dbnomics} provides a suite of tools to search, browse and import time series data from DB.nomics, the world's economic database ({browse "https://db.nomics.world":https://db.nomics.world}).
21 | DB.nomics is a web-based platform that aggregates and maintains time series data from various statistical agencies across the world.
22 | {cmd:dbnomics} only works with Stata 14.0 or higher.
23 |
24 | {pstd}
25 | {cmd:dbnomics} provides an interface between Stata and DB.nomics' API ({browse "https://api.db.nomics.world/apidocs":https://api.db.nomics.world/apidocs}). It enables creating custom queries through Stata's
26 | {help options:options} syntax (see {help dbnomics##examples:Examples}). To achieve this, the command relies on Erik Lindsley's libjson Mata library ({stata ssc install libjson:ssc install libjson}).
27 |
28 | {marker syntax}{...}
29 | {title:Syntax}
30 |
31 | {p 8 8 2} {it: Load list of providers} {p_end}
32 | {p 8 8 2} {cmdab:dbnomics} {cmdab:provider:s} [{cmd:,} {opt clear insecure}] {p_end}
33 |
34 | {p 8 8 2} {it: Load content tree for a given provider} {p_end}
35 | {p 8 8 2} {cmdab:dbnomics} {cmdab:tree} {cmd:,} {opt pr:ovider(PRcode)} [{opt clear insecure}] {p_end}
36 |
37 | {p 8 8 2} {it: Load dataset structure given a provider and dataset} {p_end}
38 | {p 8 8 2} {cmdab:dbnomics} {cmdab:data:structure} {cmd:,} {opt pr:ovider(PRcode)} {opt d:ataset(DScode)} [{opt clear} {opt insecure}] {p_end}
39 |
40 | {p 8 8 2} {it: Load list of series for a given provider and dataset} {p_end}
41 | {p 8 8 2} {cmdab:dbnomics} {cmdab:series} {cmd:,} {opt pr:ovider(PRcode)} {opt d:ataset(DScode)} [{it:dimensions_opt}|{opt sdmx(SDMX_mask)}] [{opt clear} {opt insecure} {opt limit(int)} {opt offset(int)}] {p_end}
42 |
43 | {p 8 8 2} {it: Import series for a given provider and dataset} {p_end}
44 | {p 8 8 2} {cmdab:dbnomics} {cmdab:import} {cmd:,} {opt pr:ovider(PRcode)} {opt d:ataset(DScode)} [{it:dimensions_opt}|{opt sdmx(SDMX_mask)}|{opt series:ids(SERIES_list)}] [{opt clear} {opt insecure} {opt limit(int)} {opt offset(int)}] {p_end}
45 |
46 | {p 8 8 2} {it: Load a single series} {p_end}
47 | {p 8 8 2} {cmdab:dbnomics} {cmdab:use} {it:series_id} {cmd:,} {opt pr:ovider(PRcode)} {opt d:ataset(DScode)} [{opt clear} {opt insecure}] {p_end}
48 |
49 | {p 8 8 2} {it: Search for data across DB.nomics' providers} {p_end}
50 | {p 8 8 2} {cmdab:dbnomics} {cmdab:find} {it:search_str} [{cmd:,} {opt clear} {opt insecure} {opt limit(int)} {opt offset(int)}] {p_end}
51 |
52 | {p 8 8 2} {it: Load and display recently updated datasets} {p_end}
53 | {p 8 8 2} {cmdab:dbnomics} {cmdab:news} [{cmd:,} {opt clear} {opt insecure} {opt limit(int)} {opt offset(int)}] {p_end}
54 |
55 |
56 | {synoptset 25 tabbed}{...}
57 | {synopthdr}
58 | {synoptline}
59 | {syntab:Main}
60 | {synopt:{bf:provider(}{it:PRcode}{bf:)}}declare the reference data provider ({it:e.g.} AMECO, IMF, Eurostat) {p_end}
61 | {synopt:{bf:dataset(}{it:DScode}{bf:)}}declare the reference dataset ({it:e.g.} PIGOT, BOP, ei_bsin_m) {p_end}
62 | {synopt:{bf:clear}}replace data in memory {p_end}
63 | {synopt:{bf:insecure}}connect via {it:http} instead of the default https.{p_end}
64 |
65 | {syntab:For {cmd:series} and {cmd:import} only}
66 | {synopt:{it:dimensions_opt}}provider- and dataset-specific options to filter series (see {help dbnomics##examples:Examples}){p_end}
67 | {synopt:{bf:sdmx(}{it:SDMX_mask}{bf:)}}input an SDMX mask to select specific series ({bf:Note:} not all providers support this option){p_end}
68 | {synopt:{bf:limit(}{it:int}{bf:)}}limit the number of series to load{p_end}
69 | {synopt:{bf:offset(}{it:int}{bf:)}}skip the first {it:int} result(s){p_end}
70 |
71 | {syntab:For {cmd:import} only}
72 | {synopt:{bf:seriesids(}{it:SERIES_list}{bf:)}}input a comma-separated list to load specific time series{p_end}
73 |
74 | {syntab:For {cmd:use} only}
75 | {synopt:{it:series_id}}identifier of the series (see {help dbnomics##examples:Examples}){p_end}
76 |
77 | {syntab:For {cmd:find} and {cmd:news} only}
78 | {synopt:{bf:limit(}{it:int}{bf:)}}limit the number of results to display{p_end}
79 | {synopt:{bf:offset(}{it:int}{bf:)}}skip the first {it:int} result(s){p_end}
80 | {synoptline}
81 | {p2colreset}{...}
82 |
83 | {marker options}{...}
84 | {title:Options}
85 | {dlgtab:Main}
86 |
87 | {phang}
88 | {opt provider(PRcode)} sets the source with the data of interest. The list of data providers is regularly updated by the DB.nomics team.
89 | To get the list of available data providers, type {cmdab:dbnomics} {cmdab:provider:s} [{cmd:,} {opt clear}].{p_end}
90 |
91 | {phang}
92 | {opt dataset(DScode)} sets the dataset with the time series of interest.
93 | To get a hierarchy of all datasets linked to a particular {opt provider(PRcode)}, type {cmdab:dbnomics} {cmdab:tree}{cmd:,} {cmdab:pr:ovider(}{it:PRcode}{cmd:)} [{opt clear}].
94 | ({bf:Note:} some datasets in the category tree may not be accessible). {p_end}
95 |
96 | {phang}
97 | {opt clear} clears data in memory before loading data from DB.nomics. {p_end}
98 |
99 | {phang}
100 | {opt insecure} instructs dbnomics to access the DB.nomics platform via the {it:http} protocol, instead of the more secure {it:https}. {p_end}
101 |
102 | {bf:{dlgtab:series}}
103 |
104 | {marker dimensions_opt}{...}
105 | {phang}
106 | {opt dimensions_opt(dim_values)} are options that depend on the specific {opt provider(PRcode)} and {opt dataset(DScode)} and are used to identify a subset of series.
107 | A list of dimensions can be obtained using {cmd: dbnomics datastructure,} {opt provider(PRcode)} {opt dataset(DScode)}.
108 | For example, if the dimensions of {opt dataset(DScode)} are {cmd:freq.unit.geo}, accepted options for{cmd: dbnomics series,}{opt (...)} and {cmd: dbnomics import,}{opt (...)}
109 | are {opt freq(codelist)}, {opt unit(codelist)} and {opt geo(codelist)}.
110 | Each {opt dimension_opt(dim_values)} can contain one or more values in a space or comma-separated list, so as to select multiple dimensions at once ({it:e.g.}, a list of countries).
111 | {it:Note:} {cmd:dbnomics} is not able to validate user-inputted {opt dimension_opt(dim_values)};
112 | if {opt dimension_opt(dim_values)} is incorrectly specified, {cmd:dbnomics} may throw an error or return the message {err: no series found}. See {help dbnomics##examples:Examples}. {p_end}
113 |
114 | {marker sdmx}{...}
115 | {phang}
116 | {opt sdmx(SDMX_mask)} accepts an "SDMX mask" containing a list of dimensions separated with a dot "." character.
117 | The dimensions are specific to each {opt provider(PRcode)} and {opt dataset(DScode)} and must be provided in the order specified by the {opt dataset(DScode)} data structure.
118 | The data structure can be obtained using {cmd: dbnomics datastructure,} {opt provider(PRcode)} {opt dataset(DScode)}. {it:Note:} some providers do not support this feature.
119 | In such case {cmd: dbnomics} may throw an error or return the message {err:no series found}. {p_end}
120 |
121 | {p 8 8 2}{it:Note}: {opt dimensions_opt(dim_values)} and {opt sdmx(SDMX_mask)} are mutually exclusive. {p_end}
122 |
123 | {phang}
124 | {opt limit(int)} sets a limit to the number of series to load via dbnomics. A {err:Warning} message is issued if the total number of series identified is larger than the inputted {opt limit(int)}. {p_end}
125 |
126 | {phang}
127 | {opt offset(int)} skips the first {it:int} series when loading data from dbnomics. This option can be combined with {opt limit(int)} to get a specific subset of series. {p_end}
128 |
129 | {bf:{dlgtab:import}}
130 |
131 | {phang}
132 | {opt dimensions_opt(dim_values)} {help dbnomics##dimensions_opt:see above}.{p_end}
133 |
134 | {phang}
135 | {opt sdmx(SDMX_mask)} {help dbnomics##sdmx:see above}.{p_end}
136 |
137 | {phang}
138 | {opt seriesids(SERIES_list)} accepts a comma-separated list of series identifiers that belong to a {opt provider(PRcode)} and {opt dataset(DScode)}. {p_end}
139 |
140 | {p 8 8 2}{it:Note}: {opt dimensions_opt(dim_values)}, {opt sdmx(SDMX_mask)} and {opt seriesids(SERIES_list)} are mutually exclusive. {p_end}
141 |
142 | {phang}
143 | {opt limit(int)} sets a limit to the number of series to load via dbnomics. A {err:Warning} message is issued if the total number of series identified is larger than the inputted {opt limit(int)}. {p_end}
144 |
145 | {phang}
146 | {opt offset(int)} skips the first {it:int} series when loading data from dbnomics. This option can be combined with {opt limit(int)} to get a specific subset of series. See {help dbnomics##examples:Examples}. {p_end}
147 |
148 | {bf:{dlgtab:use}}
149 |
150 | {phang}{it:series_id} is the unique identifier of a time series belonging to {opt provider(PRcode)} and {opt dataset(DScode)}.{p_end}
151 |
152 | {bf:{dlgtab:find/news}}
153 |
154 | {phang}{opt limit(int)} sets the maximum number of results to load and display. {p_end}
155 |
156 | {phang}
157 | {opt offset(int)} skips the first {it:int} results. May be combined with {opt limit(int)} to get a specific subset of results. {p_end}
158 |
159 | {phang}
160 |
161 |
162 | {marker remarks}{...}
163 | {title:Remarks}
164 | {*:{pstd} The DB.nomics data model is characterised by the following hiearchy: {it: provider -> datasets -> time series -> observations}.{p_end}}
165 | {pstd} {cmd:dbnomics} has two main dependencies: {p_end}
166 |
167 | {pstd} 1) The Mata json library {cmd:libjson} by Erik Lindsley, needed to parse JSON strings. It can be found on SSC: {stata ssc install libjson}. {p_end}
168 | {pstd} 2) The routine {cmd:moss} by Robert Picard & Nicholas J. Cox, needed to clean some unicode sequences. It can be found on SSC: {stata ssc install moss}. {p_end}
169 |
170 | {pstd} After each API call, {cmd:dbnomics} stores significant metadata in the form of {help char:dataset characteristics}.
171 | Type {cmd:{stata "char li _dta[]":char li _dta[]}} after {cmd:dbnomics} to obtain important info about the data, {it:e.g.}, the API endpoint. {p_end}
172 |
173 | {marker examples}{...}
174 | {title:Examples}
175 |
176 | {pstd}{it:Search for producer price data on the DB.nomics platform:}{p_end}
177 | {space 8}{cmd:. dbnomics find "producer price", clear}{space 4}{cmd:///}{space 2}{txt:({stata `"dbnomics find "producer price", clear "':click to run})}
178 | {space 8}{it:(output omitted)}
179 |
180 | {pstd}{it:Load the list of available providers with additional metadata:}{p_end}
181 | {space 8}{cmd:. dbnomics }{cmdab:provider:s}{cmd:, clear}{space 4}{cmd:///}{space 2}{txt:({stata "dbnomics providers, clear":click to run})}
182 |
183 | {pstd}{it:Show recently updated datasets:}{p_end}
184 | {space 8}{cmd:. dbnomics news, clear}{space 4}{cmd:///}{space 2}{txt:({stata "dbnomics news, clear ":click to run})}
185 | {space 8}{it:(output omitted)}
186 |
187 | {pstd}{it:Load the dataset tree of of the {bf:AMECO} provider:}{p_end}
188 | {space 8}{cmd:. dbnomics tree, }{cmdab:pr:ovider}{cmd:(AMECO) clear}{space 4}{cmd:///}{space 2}{txt:({stata "dbnomics tree, provider(AMECO) clear":click to run})}
189 |
190 | {pstd}{it:Analyse the structure of dataset {bf:PIGOT} for provider {bf:AMECO}:}{p_end}
191 | {space 8}{cmd:. dbnomics }{cmdab:data:structure}{cmd:, }{cmdab:pr:ovider}{cmd:(AMECO) {cmdab:d:ataset}{cmd:(PIGOT)} clear}{space 4}{cmd:///}{space 2}{txt:({stata "dbnomics datastructure, provider(AMECO) dataset(PIGOT) clear":click to run})}
192 | {space 8}{cmd:Price deflator gross fixed capital formation: other investment}
193 | {space 8}82 series found. Order of dimensions: (freq.unit.geo)
194 |
195 | {pstd}{it:List all series in {bf:AMECO/}{bf:PIGOT} containing deflators in national currency:}{p_end}
196 | {space 8}{cmd:. dbnomics }{cmdab:series}{cmd:, }{cmdab:pr:ovider}{cmd:(AMECO) {cmdab:d:ataset}{cmd:(PIGOT)} {cmd:unit(national-currency-2015-100)} clear}
197 | {space 8}{txt:({stata "dbnomics series, provider(AMECO) dataset(PIGOT) unit(national-currency-2015-100) clear":click to run})}
198 | {space 8}39 of 82 series selected. Order of dimensions: (freq.unit.geo)
199 |
200 | {pstd}{it:Import all series in {bf:AMECO/}{bf:PIGOT} containing deflators in national currency:}{p_end}
201 | {space 8}{cmd:. dbnomics }{cmdab:import}{cmd:, }{cmdab:pr:ovider}{cmd:(AMECO) {cmdab:d:ataset}{cmd:(PIGOT)} {cmd:unit(national-currency-2015-100)} clear}
202 | {space 8}{txt:({stata "dbnomics import, provider(AMECO) dataset(PIGOT) unit(national-currency-2015-100) clear":click to run})}
203 | {space 8}Processing 39 series
204 | {space 8}........................................
205 | {space 8}{bf:39 series found and imported}
206 |
207 | {pstd}{it:Import a few {bf:AMECO/}{bf:PIGOT} series:}{p_end}
208 | {space 8}{cmd:. dbnomics import, pr(AMECO) d(PIGOT) }{cmdab:series:ids}{cmd:(ESP.3.1.0.0.PIGOT,SVN.3.1.0.0.PIGOT,LVA.3.1.99.0.PIGOT) clear}
209 | {space 8}{txt:({stata "dbnomics import, pr(AMECO) d(PIGOT) seriesids(ESP.3.1.0.0.PIGOT,SVN.3.1.0.0.PIGOT,LVA.3.1.99.0.PIGOT) clear":click to run})}
210 | {space 8}Processing 3 series
211 | {space 8}...
212 | {space 8}{bf:3 series found and imported}
213 |
214 | {pstd}{it:Import one specific series from {bf:AMECO/}{bf:PIGOT}:}{p_end}
215 | {space 8}{cmd:. dbnomics use ESP.3.1.0.0.PIGOT, pr(AMECO) d(PIGOT)}{cmd: clear}
216 | {space 8}{txt:({stata "dbnomics use ESP.3.1.0.0.PIGOT, pr(AMECO) d(PIGOT) clear":click to run})}
217 | {space 8}Annually – (National currency: 2015 = 100) – Spain (AMECO/PIGOT/ESP.3.1.0.0.PIGOT)
218 | {space 8}(62 observations read)
219 |
220 | {pstd}{it:{bf:Eurostat} supports SMDX queries}. {it:Import all series in {bf:Eurostat/}{bf:ei_bsin_q_r2} related to Belgium:}{p_end}
221 | {space 8}{cmd:. dbnomics }{cmdab:import}{cmd:, }{cmdab:pr:ovider}{cmd:(Eurostat) {cmdab:d:ataset}{cmd:(ei_bsin_q_r2)} {cmd:geo(BE) s_adj(NSA)} clear}
222 | {space 8}{txt:({stata "dbnomics import, provider(Eurostat) dataset(ei_bsin_q_r2) geo(BE) s_adj(NSA) clear":click to run})}
223 | {space 8}Processing 14 series
224 | {space 8}..............
225 | {space 8}{bf:14 series found and imported}
226 |
227 | {pstd}{it:Do the same using {cmd:sdmx}:}{p_end}
228 | {space 8}{cmd:. dbnomics }{cmdab:import}{cmd:, }{cmdab:pr:ovider}{cmd:(Eurostat) {cmdab:d:ataset}{cmd:(ei_bsin_q_r2)} {cmd:sdmx(Q..NSA.BE)} clear}
229 | {space 8}{txt:({stata "dbnomics import, provider(Eurostat) dataset(ei_bsin_q_r2) sdmx(Q..NSA.BE) clear":click to run})}
230 | {space 8}Processing 14 series
231 | {space 8}..............
232 | {space 8}{bf:14 series found and imported}
233 |
234 | {pstd}{it:The {bf:Eurostat/}{bf:urb_ctran} dataset offers 12280 series, more than permitted at once by DB.nomics}:{p_end}
235 | {space 8}{cmd:. dbnomics }{cmdab:series}{cmd:, }{cmdab:pr:ovider}{cmd:(Eurostat) {cmdab:d:ataset}{cmd:(urb_ctran)} clear}
236 | {space 8}{txt:({stata "dbnomics series, pr(Eurostat) d(urb_ctran) clear":click to run})}
237 | {space 8}{err:Warning: series set larger than dbnomics maximum provided items.}
238 | {space 8}{err:Use the {cmd:offset} option to load series beyond the 1000th one.}
239 | {space 8}{txt:12280 series selected. Order of dimensions: (FREQ.indic_ur.cities). {bf:Only #1 to #1000 retrieved}}
240 |
241 | {pstd}{it:Using {cmd:limit} and {cmd:offset}, we can instruct {cmd:dbnomics} to only get series #1001 to #1100}:{p_end}
242 | {space 8}{cmd:. dbnomics }{cmdab:import}{cmd:, }{cmdab:pr:ovider}{cmd:(Eurostat) {cmdab:d:ataset}{cmd:(urb_ctran)} {cmd:limit(100)} {cmd:offset(1000)} clear}
243 | {space 8}{txt:({stata "dbnomics import, pr(Eurostat) d(urb_ctran) limit(100) offset(1000) clear":click to run})}
244 | {space 8}Processing 100 series
245 | {space 8}.................................................. 50
246 | {space 8}.................................................. 100
247 | {space 8}{bf:12280 series found and #1001 to #1100 loaded}
248 |
249 | {marker results}{...}
250 | {title:Stored results}
251 |
252 | {pstd}
253 | {cmd:dbnomics} stores the following in {cmd:r()}:
254 |
255 | {synoptset 15 tabbed}{...}
256 | {p2col 5 15 19 2: Local}{p_end}
257 | {synopt:{cmd:endpoint}}name of {cmd:dbnomics} subcommand{p_end}
258 | {synopt:{cmd:cmd#}}command to load # result shown (For {cmd:find} and {cmd:news} only){p_end}
259 | {p2colreset}{...}
260 |
261 | {marker author}{...}
262 | {title:Author}
263 |
264 | {pstd}
265 | Simone Signore{break}
266 | signoresimone at yahoo [dot] it {p_end}
267 | https://dbnomics-stata.github.io/dbnomics/
268 |
269 |
--------------------------------------------------------------------------------
/dbnomics.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
15 |
16 |
17 |
18 |
19 | Title
20 |
21 | dbnomics -- Stata client for DB.nomics, the world's economic database (https://db.nomics.world)
22 |
23 | Description
24 |
25 | dbnomics provides a suite of tools to search, browse and import time series data from DB.nomics, the
26 | world's economic database (https://db.nomics.world). DB.nomics is a web-based platform that aggregates
27 | and maintains time series data from various statistical agencies across the world. dbnomics only works
28 | with Stata 14.0 or higher.
29 |
30 | dbnomics provides an interface between Stata and DB.nomics' API (https://api.db.nomics.world/apidocs). It
31 | enables creating custom queries through Stata's options syntax (see Examples). To achieve this, the
32 | command relies on Erik Lindsley's libjson Mata library (ssc install libjson).
33 |
34 | Syntax
35 |
36 | Load list of providers
37 | dbnomics providers [, clear insecure]
38 |
39 | Load content tree for a given provider
40 | dbnomics tree , provider(PRcode) [clear insecure]
41 |
42 | Load dataset structure given a provider and dataset
43 | dbnomics datastructure , provider(PRcode) dataset(DScode) [clear insecure]
44 |
45 | Load list of series for a given provider and dataset
46 | dbnomics series , provider(PRcode) dataset(DScode) [dimensions_opt|sdmx(SDMX_mask)] [clear insecure
47 | limit(int) offset(int)]
48 |
49 | Import series for a given provider and dataset
50 | dbnomics import , provider(PRcode) dataset(DScode)
51 | [dimensions_opt|sdmx(SDMX_mask)|seriesids(SERIES_list)] [clear insecure limit(int) offset(int)]
52 |
53 | Load a single series
54 | dbnomics use series_id , provider(PRcode) dataset(DScode) [clear insecure]
55 |
56 | Search for data across DB.nomics' providers
57 | dbnomics find search_str [, clear insecure limit(int) offset(int)]
58 |
59 | Load and display recently updated datasets
60 | dbnomics news [, clear insecure limit(int) offset(int)]
61 |
62 |
63 | options Description
64 | ----------------------------------------------------------------------------------------------------------
65 | Main
66 | provider(PRcode) declare the reference data provider (e.g. AMECO, IMF, Eurostat)
67 | dataset(DScode) declare the reference dataset (e.g. PIGOT, BOP, ei_bsin_m)
68 | clear replace data in memory
69 | insecure connect via http instead of the default https.
70 |
71 | For series and import only
72 | dimensions_opt provider- and dataset-specific options to filter series (see Examples)
73 | sdmx(SDMX_mask) input an SDMX mask to select specific series (Note: not all providers support
74 | this option)
75 | limit(int) limit the number of series to load
76 | offset(int) skip the first int result(s)
77 |
78 | For import only
79 | seriesids(SERIES_list) input a comma-separated list to load specific time series
80 |
81 | For use only
82 | series_id identifier of the series (see Examples)
83 |
84 | For find and news only
85 | limit(int) limit the number of results to display
86 | offset(int) skip the first int result(s)
87 | ----------------------------------------------------------------------------------------------------------
88 |
89 | Options
90 | +------+
91 | ----+ Main +----------------------------------------------------------------------------------------------
92 |
93 | provider(PRcode) sets the source with the data of interest. The list of data providers is regularly
94 | updated by the DB.nomics team. To get the list of available data providers, type dbnomics providers
95 | [, clear].
96 |
97 | dataset(DScode) sets the dataset with the time series of interest. To get a hierarchy of all datasets
98 | linked to a particular provider(PRcode), type dbnomics tree, provider(PRcode) [clear]. (Note: some
99 | datasets in the category tree may not be accessible).
100 |
101 | clear clears data in memory before loading data from DB.nomics.
102 |
103 | insecure instructs dbnomics to access the DB.nomics platform via the http protocol, instead of the more
104 | secure https.
105 |
106 | +--------+
107 | ----+ series +--------------------------------------------------------------------------------------------
108 |
109 | dimensions_opt(dim_values) are options that depend on the specific provider(PRcode) and dataset(DScode)
110 | and are used to identify a subset of series. A list of dimensions can be obtained using dbnomics
111 | datastructure, provider(PRcode) dataset(DScode). For example, if the dimensions of dataset(DScode)
112 | are freq.unit.geo, accepted options for dbnomics series,(...) and dbnomics import,(...) are
113 | freq(codelist), unit(codelist) and geo(codelist). Each dimension_opt(dim_values) can contain one or
114 | more values in a space or comma-separated list, so as to select multiple dimensions at once (e.g., a
115 | list of countries). Note: dbnomics is not able to validate user-inputted dimension_opt(dim_values);
116 | if dimension_opt(dim_values) is incorrectly specified, dbnomics may throw an error or return the
117 | message no series found. See Examples.
118 |
119 | sdmx(SDMX_mask) accepts an "SDMX mask" containing a list of dimensions separated with a dot "." character.
120 | The dimensions are specific to each provider(PRcode) and dataset(DScode) and must be provided in the
121 | order specified by the dataset(DScode) data structure. The data structure can be obtained using
122 | dbnomics datastructure, provider(PRcode) dataset(DScode). Note: some providers do not support this
123 | feature. In such case dbnomics may throw an error or return the message no series found.
124 |
125 | Note: dimensions_opt(dim_values) and sdmx(SDMX_mask) are mutually exclusive.
126 |
127 | limit(int) sets a limit to the number of series to load via dbnomics. A Warning message is issued if the
128 | total number of series identified is larger than the inputted limit(int).
129 |
130 | offset(int) skips the first int series when loading data from dbnomics. This option can be combined with
131 | limit(int) to get a specific subset of series.
132 |
133 | +--------+
134 | ----+ import +--------------------------------------------------------------------------------------------
135 |
136 | dimensions_opt(dim_values) see above.
137 |
138 | sdmx(SDMX_mask) see above.
139 |
140 | seriesids(SERIES_list) accepts a comma-separated list of series identifiers that belong to a
141 | provider(PRcode) and dataset(DScode).
142 |
143 | Note: dimensions_opt(dim_values), sdmx(SDMX_mask) and seriesids(SERIES_list) are mutually exclusive.
144 |
145 | limit(int) sets a limit to the number of series to load via dbnomics. A Warning message is issued if the
146 | total number of series identified is larger than the inputted limit(int).
147 |
148 | offset(int) skips the first int series when loading data from dbnomics. This option can be combined with
149 | limit(int) to get a specific subset of series. See Examples.
150 |
151 | +-----+
152 | ----+ use +-----------------------------------------------------------------------------------------------
153 |
154 | series_id is the unique identifier of a time series belonging to provider(PRcode) and dataset(DScode).
155 |
156 | +-----------+
157 | ----+ find/news +-----------------------------------------------------------------------------------------
158 |
159 | limit(int) sets the maximum number of results to load and display.
160 |
161 | offset(int) skips the first int results. May be combined with limit(int) to get a specific subset of
162 | results.
163 |
164 |
165 |
166 | Remarks
167 |
168 | dbnomics has two main dependencies:
169 |
170 | 1) The Mata json library libjson by Erik Lindsley, needed to parse JSON strings. It can be found on SSC:
171 | ssc install libjson.
172 | 2) The routine moss by Robert Picard & Nicholas J. Cox, needed to clean some unicode sequences. It can be
173 | found on SSC: ssc install moss.
174 |
175 | After each API call, dbnomics stores significant metadata in the form of dataset characteristics. Type
176 | char li _dta[] after dbnomics to obtain important info about the data, e.g., the API endpoint.
177 |
178 | Examples
179 |
180 | Search for producer price data on the DB.nomics platform:
181 | . dbnomics find "producer price", clear /// (click to run)
182 | (output omitted)
183 |
184 | Load the list of available providers with additional metadata:
185 | . dbnomics providers, clear /// (click to run)
186 |
187 | Show recently updated datasets:
188 | . dbnomics news, clear /// (click to run)
189 | (output omitted)
190 |
191 | Load the dataset tree of of the AMECO provider:
192 | . dbnomics tree, provider(AMECO) clear /// (click to run)
193 |
194 | Analyse the structure of dataset PIGOT for provider AMECO:
195 | . dbnomics datastructure, provider(AMECO) dataset(PIGOT) clear /// (click to run)
196 | Price deflator gross fixed capital formation: other investment
197 | 82 series found. Order of dimensions: (freq.unit.geo)
198 |
199 | List all series in AMECO/PIGOT containing deflators in national currency:
200 | . dbnomics series, provider(AMECO) dataset(PIGOT) unit(national-currency-2015-100) clear
201 | (click to run)
202 | 39 of 82 series selected. Order of dimensions: (freq.unit.geo)
203 |
204 | Import all series in AMECO/PIGOT containing deflators in national currency:
205 | . dbnomics import, provider(AMECO) dataset(PIGOT) unit(national-currency-2015-100) clear
206 | (click to run)
207 | Processing 39 series
208 | ........................................
209 | 39 series found and imported
210 |
211 | Import a few AMECO/PIGOT series:
212 | . dbnomics import, pr(AMECO) d(PIGOT) seriesids(ESP.3.1.0.0.PIGOT,SVN.3.1.0.0.PIGOT,LVA.3.1.99.0.PIGOT)
213 | > clear
214 | (click to run)
215 | Processing 3 series
216 | ...
217 | 3 series found and imported
218 |
219 | Import one specific series from AMECO/PIGOT:
220 | . dbnomics use ESP.3.1.0.0.PIGOT, pr(AMECO) d(PIGOT) clear
221 | (click to run)
222 | Annually – (National currency: 2015 = 100) – Spain (AMECO/PIGOT/ESP.3.1.0.0.PIGOT)
223 | (62 observations read)
224 |
225 | Eurostat supports SMDX queries. Import all series in Eurostat/ei_bsin_q_r2 related to Belgium:
226 | . dbnomics import, provider(Eurostat) dataset(ei_bsin_q_r2) geo(BE) s_adj(NSA) clear
227 | (click to run)
228 | Processing 14 series
229 | ..............
230 | 14 series found and imported
231 |
232 | Do the same using sdmx:
233 | . dbnomics import, provider(Eurostat) dataset(ei_bsin_q_r2) sdmx(Q..NSA.BE) clear
234 | (click to run)
235 | Processing 14 series
236 | ..............
237 | 14 series found and imported
238 |
239 | The Eurostat/urb_ctran dataset offers 12280 series, more than permitted at once by DB.nomics:
240 | . dbnomics series, provider(Eurostat) dataset(urb_ctran) clear
241 | (click to run)
242 | Warning: series set larger than dbnomics maximum provided items.
243 | Use the offset option to load series beyond the 1000th one.
244 | 12280 series selected. Order of dimensions: (FREQ.indic_ur.cities). Only #1 to #1000 retrieved
245 |
246 | Using limit and offset, we can instruct dbnomics to only get series #1001 to #1100:
247 | . dbnomics import, provider(Eurostat) dataset(urb_ctran) limit(100) offset(1000) clear
248 | (click to run)
249 | Processing 100 series
250 | .................................................. 50
251 | .................................................. 100
252 | 12280 series found and #1001 to #1100 loaded
253 |
254 | Stored results
255 |
256 | dbnomics stores the following in r():
257 |
258 | Local
259 | endpoint name of dbnomics subcommand
260 | cmd# command to load # result shown (For find and news only)
261 |
262 | Author
263 |
264 | Simone Signore
265 | signoresimone at yahoo [dot] it
266 | https://dbnomics-stata.github.io/dbnomics/
267 |
268 |
269 |
270 |
271 |
--------------------------------------------------------------------------------
/dbnomics.ado:
--------------------------------------------------------------------------------
1 | *! Ver 1.2.0 17jun2020 Simone Signore
2 | *! Stata API client for db.nomics.world. Requires libjson and moss
3 | capture program drop dbnomics
4 |
5 | /* Main wrapper command */
6 | program dbnomics, rclass
7 |
8 | /* Version 14.0 used to be necessary because only http secure calls were possible.
9 | Now that http is available, I guess 14.0 is still safer because of unicode.
10 | Also version 13 crashes due to a weird pointer issue */
11 | version 14.0
12 |
13 | /* Changelog
14 | 20mar2018 v1.0.0 Initial release
15 | 08may2018 v1.0.1 Fixed syntax parsing bug
16 | 23may2018 v1.0.2 Updated to API ver 0.18.0
17 | 15oct2018 v1.0.3 Updated to API ver 0.21.5
18 | 19oct2018 v1.1.0 Added news API and smart listing (0.21.6)
19 | 21oct2018 v1.1.1 Improved parsing engine, added search endpoint (ver 0.21.6)
20 | 30oct2018 v1.1.2 Fixed bug in dbnomics news
21 | 17jun2020 v1.2.0 Updated to API ver 0.22.0.1, added insecure request
22 | */
23 |
24 | /*TODO:
25 | Add new metadata info to payloads [n] --- decided args are not worthy to include
26 | Evaluate whether to add new align_period and complete_missing_periods options to account for new API parameters [n] --- decided against that, best to keep the payload as small as possible
27 | Stata can't copy internet files when the server throws an HTTP error. There's no way to parse error messages then []
28 | dbnomics_list causes a strange Stata crash when trying to visualise 25+ results. Capped the option at 25, perhaps the routine should be rewritten entirely []
29 | */
30 |
31 | /* Housekeeping: taken from insheetjson */
32 | /* Check if libjson exists */
33 | mata: if (findexternal("libjson()")) {} else printf("{err:Error: The required JSON library (libjson) seems to be missing so this command will fail. Read the help file for more information.}\n");
34 |
35 | /* Check libjson version */
36 | mata: if (libjson::checkVersion((1,0,2))) {} else printf("{err: The JSON library version is not compatible with this command and so will likely fail. Please update libjson.}\n");
37 |
38 | /* Check whether moss is installed */
39 | capture which moss
40 | if _rc {
41 | di as smcl `"{err:Error: the package {cmd:moss} is required by {cmd:dbnomics}. Try {stata "ssc install moss":ssc install moss}}"'
42 | exit 111
43 | }
44 |
45 | syntax [anything(name=subcall id="subcall list")], [CLEAR INSECURE *]
46 |
47 | /* Setup API endpoint */
48 | local apipath = cond("`insecure'" != "", "http", "https") + "://api.db.nomics.world/v22"
49 |
50 | /* Declare API call hard limit */
51 | global S_dbnomics_hard_limit = 1000
52 |
53 | /* Parse subcall*/
54 | if inlist(`"`subcall'"',"provider","providers") {
55 | dbnomics_providers `apipath', `clear' `macval(options)'
56 | }
57 | else if `"`subcall'"' == "tree" {
58 | dbnomics_tree `apipath', `clear' `macval(options)'
59 | }
60 | else if strpos("datastructure", `"`subcall'"') & length(`"`subcall'"') >= 4 {
61 | dbnomics_structure `apipath', `clear' `macval(options)'
62 | }
63 | else if `"`subcall'"' == "series" {
64 | dbnomics_series `apipath', `clear' `macval(options)'
65 | }
66 | else if `"`subcall'"' == "import" {
67 | /* timer on 1 */
68 | dbnomics_import `apipath', `clear' `macval(options)'
69 | /* timer off 1
70 | timer list 1
71 | timer clear 1 */
72 | }
73 | else if `"`subcall'"' == "news" {
74 | dbnomics_news `apipath', `clear' `macval(options)' `insecure'
75 | }
76 | else if (substr(`"`subcall'"',1,4) == "use ") {
77 | tokenize `macval(subcall)'
78 | dbnomics_use `2', `clear' path(`apipath') `macval(options)' delim(",")
79 | /* di as err "Sorry, the {err:{bf:dbnomics use}} API is deprecated. Use {cmd:dbnomics import, seriesids(...)} instead."
80 | exit 198 */
81 | }
82 | else if (substr(`"`subcall'"',1,5) == "find ") {
83 | tokenize `"`subcall'"'
84 | dbnomics_query `2', `clear' `macval(options)' path(`apipath') `insecure'
85 | }
86 | else {
87 | di as err "dbnomics: unknown subcommand "`""`subcall'""'""
88 | exit 198
89 | }
90 |
91 | return add
92 | return local endpoint "`subcall'"
93 |
94 | /* Housekeeping */
95 | /* Done at the subroutine level */
96 |
97 | end
98 |
99 | /*Subroutines*/
100 | /*1) Providers table */
101 | program dbnomics_providers
102 |
103 | syntax anything(name=path), [CLEAR]
104 |
105 | /* Setup call*/
106 | local apipath = "`path'/providers"
107 |
108 | /* Parse clear option*/
109 | if ("`clear'" == "") {
110 | if `c(width)' > 0 {
111 | di as err "no; data in memory would be lost. Use the {cmd:clear} option"
112 | exit 4
113 | }
114 | }
115 | else {
116 | clear
117 | }
118 |
119 | display as txt "Fetching providers list"
120 |
121 | /* Save json locally to reduce server load over multiple calls */
122 | tempfile jdata
123 | capture copy "`apipath'" `jdata', replace
124 | if (inrange(_rc,630,696) | _rc == 601) {
125 | if (_rc == 601) di as err "server returned error. Check `apipath' for possibly more info about the issue."
126 | char _dta[endpoint] "`apipath'"
127 | char _dta[errormsg] "`=_rc'"
128 | exit _rc
129 | }
130 | else if (_rc > 0) {
131 | di as smcl "{err:failed to reach the dbnomics servers. Check your internet connection, or try using the {cmd:insecure} option.}"
132 | char _dta[endpoint] "`apipath'"
133 | char _dta[errormsg] "`=_rc'"
134 | exit _rc
135 | }
136 |
137 | /* Parse JSON */
138 | mata: providers = fetchjson("`jdata'", "");
139 | mata: provobj = fetchjson("`jdata'", "providers");
140 | mata: provnode = provobj->getNode("docs");
141 |
142 | /* Check for error in response */
143 | mata: parseresperr(providers);
144 |
145 | /* Parse metadata */
146 | mata: st_local("nomicsmeta", parsemeta(providers));
147 |
148 | /* Call mata function */
149 | mata: pushdata(json2table(provnode), jsoncolsArray(provnode, 0)');
150 |
151 | /* Reduce space (this can be automated in the future)*/
152 | qui compress
153 |
154 | /* Housekeeping */
155 | quietly {
156 |
157 | cleanutf8
158 | destring _all, replace
159 | remove_destrchar _all
160 | auto_labels _all
161 |
162 | /* Parse dates */
163 | unab varlist : _all
164 | local dbdates "converted_at indexed_at created_at"
165 | local dbdates : list dbdates & varlist
166 |
167 | local iter 0
168 |
169 | foreach dbd of local dbdates {
170 | tempvar dbd`iter'
171 | gen `dbd`iter'' = clock(`dbd',"YMD#hms#")
172 | order `dbd`iter'', after(`dbd')
173 | la var `dbd`iter'' "`: var lab `dbd''"
174 | drop `dbd'
175 | clonevar `dbd' = `dbd`iter''
176 | format %tc `dbd'
177 | order `dbd', after(`dbd`iter++'')
178 | }
179 |
180 | /* Order dataset */
181 | local ordlist "code name region"
182 | local ordlist : list ordlist & varlist
183 | capture order `ordlist', first
184 | }
185 |
186 | /* Add metadata as dataset data characteristic */
187 | char _dta[endpoint] "`apipath'"
188 | char _dta[_meta] "`nomicsmeta'"
189 |
190 | di as text `"(`=_N' `=plural(_N, "provider")' read)"'
191 |
192 | /* Housekeeping */
193 | capture mata : mata drop providers provobj provnode
194 |
195 | end
196 |
197 | /*2) Dataset trees */
198 | program dbnomics_tree
199 |
200 | syntax anything(name=path), PRovider(string) [KEYS(string) CLEAR LEVel(name)]
201 |
202 | /* Setup call*/
203 | local apipath = "`path'/providers/`provider'"
204 |
205 | /* Parse clear option*/
206 | if ("`clear'" == "") {
207 | if `c(width)' > 0 {
208 | di as err "no; data in memory would be lost. Use {cmd:clear} option"
209 | exit 4
210 | }
211 | }
212 | else {
213 | clear
214 | }
215 |
216 | /* Save json locally to reduce server load over multiple calls */
217 | tempfile jdata
218 | capture copy "`apipath'" `jdata', replace
219 | if (inrange(_rc,630,696) | _rc == 601) {
220 | if (_rc == 601) di as smcl `"{err:server returned error. Make sure {cmd:`provider'} is a valid DB.nomics provider, or check {browse "`apipath'":`apipath'} for possibly more info about the issue.}"' /*"'*/
221 | char _dta[endpoint] "`apipath'"
222 | char _dta[errormsg] "`=_rc'"
223 | exit _rc
224 | }
225 | else if (_rc > 0) {
226 | di as smcl "{err:failed to reach the dbnomics servers. Check your internet connection, or try using the {cmd:insecure} option.}"
227 | char _dta[endpoint] "`apipath'"
228 | char _dta[errormsg] "`=_rc'"
229 | exit _rc
230 | }
231 | else {
232 | display as txt "Fetching category tree for `provider'"
233 | }
234 |
235 | /* Parse JSON */
236 | mata: tree = fetchjson("`jdata'", "");
237 |
238 | /* Check for error in response */
239 | mata: parseresperr(tree);
240 |
241 | /* Parse metadata */
242 | mata: st_local("nomicsmeta", parsemeta(tree));
243 |
244 | /* Call mata function */
245 | tempname lvl
246 | mata: pushdata(parsetree(tree, treekeys("`keys'")), ("`lvl'", treekeys("`keys'")));
247 |
248 | /* Reduce space (this can be automated in the future)*/
249 | qui compress
250 |
251 | /* Parse level */
252 | tempvar lvlenc lvlgroup
253 | quietly {
254 | gen `lvlenc' = real(`lvl')
255 | if ("`level'" != "") {
256 | confirm new variable `level'
257 | egen `level' = group(`lvlenc')
258 | lab var `level' "Level"
259 | order `level'
260 | }
261 | else {
262 | confirm new variable level
263 | egen level = group(`lvlenc')
264 | lab var level "Level"
265 | order level
266 | }
267 | }
268 |
269 | /* Housekeeping */
270 | quietly {
271 | cleanutf8
272 | auto_labels _all
273 | capture {
274 | format `=subinstr("`: format name'","%","%-",1)' name
275 | replace name = ustrupper(name) if level == 1
276 | replace name = " "*(level-1) + name
277 | }
278 |
279 | }
280 |
281 | /* Add provider metadata */
282 | mata: metadata = ("website","terms_of_use","region","name");
283 | mata: providermeta = fetchkeyvals(tree->getNode("provider"), metadata);
284 | mata: for (kk=1; kk<=cols(metadata); kk++) st_lchar("_dta", metadata[kk], providermeta[kk]);
285 |
286 | /* Add metadata as dataset data characteristic */
287 | char _dta[provider] "`provider'"
288 | char _dta[endpoint] "`apipath'"
289 | char _dta[_meta] "`nomicsmeta'"
290 |
291 | /* Housekeeping */
292 | capture mata : mata drop tree metadata providermeta kk
293 |
294 | end
295 |
296 | /*3) Datastructure */
297 | program dbnomics_structure
298 |
299 | syntax anything(name=path), PRovider(string) Dataset(string) [CLEAR noSTAT]
300 |
301 | /* Setup call*/
302 | local apipath = "`path'/series/`provider'/`dataset'?facets=true&metadata=true&format=json&limit=0&offset=0"
303 |
304 | /* Parse clear option*/
305 | if ("`clear'" == "") {
306 | if `c(width)' > 0 {
307 | di as err "no; data in memory would be lost. Use {cmd:clear} option"
308 | exit 4
309 | }
310 | }
311 | else {
312 | clear
313 | }
314 |
315 | /* Save json locally to reduce server load over multiple calls */
316 | tempfile jdata
317 | capture copy "`apipath'" `jdata', replace
318 | if (inrange(_rc,630,696) | _rc == 601) {
319 | if (_rc == 601) di as smcl `"{err:server returned error. Make sure {cmd:`provider'} and {cmd:`dataset'} are valid DB.nomics provider and dataset respectively, or check {browse "`path'/series/`provider'/`dataset'":`path'/series/`provider'/`dataset'} for possibly more info about the issue.}"' /*"'*/
320 | char _dta[errormsg] "`=_rc'"
321 | char _dta[endpoint] "`apipath'"
322 | exit _rc
323 | }
324 | else if (_rc > 0) {
325 | di as smcl "{err:failed to reach the dbnomics servers. Check your internet connection, or try using the {cmd:insecure} option.}"
326 | char _dta[endpoint] "`apipath'"
327 | char _dta[errormsg] "`=_rc'"
328 | exit _rc
329 | }
330 |
331 | /* Parse JSON */
332 | mata: structure = fetchjson("`jdata'", "");
333 |
334 | /* Check for error in response */
335 | mata: parseresperr(structure);
336 |
337 | /* Parse metadata */
338 | mata: st_local("nomicsmeta", parsemeta(structure));
339 |
340 | /* Parse dataset structure */
341 | /* mata: datainfo = fetchjson("`jdata'", "dataset"); */
342 | mata: datainfo = structure->getNode("dataset");
343 | mata: datastruct = datainfo->getNode("dimensions_values_labels");
344 | /* Check whether null structure */
345 | mata: st_local("nullstruct", strofreal(datastruct==NULL));
346 |
347 | /* Proceed accordingly */
348 | if ("`nullstruct'" == "0") {
349 |
350 | /* Parse UNDATA formatting exception */
351 | if (`"`provider'"' == "UNDATA") {
352 | mata: tablestruct = dict2tablev2(datastruct, dictdim(datastruct)[.,2]);
353 | mata: pushdata(tablestruct[.,2..4], tokenizer("dimensions_values_labels", "_"));
354 | }
355 | else {
356 | mata: tablestruct = dict2table(datastruct, dictdim(datastruct)[.,2]);
357 | mata: pushdata(tablestruct[.,1..3], tokenizer("dimensions_values_labels", "_"));
358 | }
359 |
360 | /* Add additional statistics (default) */
361 | if ("`stat'" == "") {
362 |
363 | /* Select facets node */
364 | mata: statstruct = structure->getNode("series_dimensions_facets");
365 |
366 | /* Series stats matrix */
367 | local nofacets 0
368 | mata: tablesstat = dict2tablev2(statstruct, dictdim(statstruct)[.,2]);
369 |
370 | /* Capture empty node */
371 | mata: st_local("nofacets", strofreal(tablesstat == "0"));
372 |
373 | if (`nofacets' == 0) {
374 |
375 | tempfile statdata
376 | preserve
377 |
378 | drop _all
379 |
380 | /* Keep only additional data */
381 | mata: tablesstat = select(tablesstat, tablesstat[., 2] :!= "label");
382 |
383 | /* Keep relevant cols */
384 | mata: pushdata(tablesstat, tokenizer("tracker_dimensions_values_seriescount", "_"));
385 |
386 | qui save `statdata'
387 | restore
388 |
389 | /* Parse UNDATA formatting exception */
390 | if (`"`provider'"' == "UNDATA") {
391 | qui replace labels = trim(substr(values, strpos(values,",")+1, .))
392 | qui replace values = trim(substr(values, 2, strpos(values,",")))
393 | qui replace values = substr(values,1,length(values)-1)
394 | qui replace labels = substr(labels,1,length(labels)-1)
395 | }
396 |
397 | qui merge 1:1 dimensions values using `statdata', nogen norep
398 | capture drop tracker
399 |
400 | }
401 |
402 | /* Mata housekeeping */
403 | mata : mata drop statstruct tablesstat
404 | }
405 |
406 | /* Reduce space (this can be automated in the future)*/
407 | qui compress
408 |
409 | /* Housekeeping */
410 | quietly {
411 | cleanutf8
412 | format `=subinstr("`: format values'","%","%-",1)' values
413 | format `=subinstr("`: format labels'","%","%-",1)' labels
414 | /* Use destring carefully */
415 | unab nodestr: dimensions values
416 | unab allvars : _all
417 | local todestr : list allvars - nodestr
418 | destring `todestr', replace
419 | remove_destrchar _all
420 | auto_labels _all
421 | }
422 |
423 | /* Mata housekeeping */
424 | mata : mata drop tablestruct
425 | }
426 | else {
427 | di as smcl "{err:Warning. Dataset structure not found for {cmd:`dataset'}}"
428 | }
429 |
430 | /* Add provider metadata */
431 | mata: metadata = ("nb_series","code","name");
432 | mata: datafeat = fetchkeyvals(datainfo, metadata);
433 | mata: for (kk=1; kk<=cols(metadata); kk++) st_lchar("_dta", metadata[kk], datafeat[kk]);
434 |
435 | /* Finally get datastructure template */
436 | mata: structinfo = parsestructure(datainfo);
437 | mata: st_local("dtstructure", structinfo);
438 | mata: st_lchar("_dta", "dtstructure", structinfo);
439 |
440 | local seriesnum : char _dta[nb_series1]
441 | display as res "`: char _dta[name1]' `: char _dta[name2]'"
442 | display as txt "`seriesnum' series found. Order of dimensions: (`dtstructure')"
443 |
444 | /* Add metadata as dataset data characteristic */
445 | char _dta[provider] "`provider'"
446 | char _dta[dataset] "`dataset'"
447 | char _dta[endpoint] "`apipath'"
448 | char _dta[_meta] "`nomicsmeta'"
449 |
450 | /* Housekeeping */
451 | capture mata : mata drop structure datainfo datastruct metadata datafeat kk structinfo
452 |
453 | end
454 |
455 | /*4. Series */
456 | program dbnomics_series
457 |
458 | syntax anything(name=path), PRovider(string) Dataset(string) [LIMIT(numlist integer max=1 <= ${S_dbnomics_hard_limit}) OFFSET(numlist integer max=1 >= 0) SDMX(string asis) CLEAR *]
459 |
460 | /* Set limit and offset if not provided */
461 | if ("`limit'" == "") local limit = ${S_dbnomics_hard_limit}
462 | if ("`offset'" == "") local offset = 0
463 |
464 | /* smdx and dimensions mutually exclusive */
465 | if (`"`sdmx'"' != "" & "`macval(options)'" != "") {
466 | di as smcl "{err:Options {cmd:sdmx} and {cmd:dimensions} are mutually exclusive.}"
467 | exit 4
468 | }
469 |
470 | /* Parse filtering options */
471 | _optdict `macval(options)'
472 | if (`"`dimdict'"' != "") local thequery "&dimensions=`dimdict'"
473 |
474 | if (`"`sdmx'"' != "") mata: st_local("thequery", urlencode(`"`sdmx'"'))
475 |
476 | /* Parse clear option*/
477 | if ("`clear'" == "") {
478 | if `c(width)' > 0 {
479 | di as err "no; data in memory would be lost. Use {cmd:clear} option"
480 | exit 4
481 | }
482 | }
483 | else {
484 | clear
485 | }
486 |
487 | /* Capture limit override */
488 | local override 0
489 | if (`limit' != ${S_dbnomics_hard_limit}) local override 1
490 |
491 | /* Setup call*/
492 | if (`"`sdmx'"' != "") {
493 | local apipath = `"`path'/series/`provider'/`dataset'/`macval(thequery)'?facets=true&metadata=true&format=json&limit=`limit'&offset=`offset'&observations=false"'
494 | }
495 | else {
496 | local apipath = `"`path'/series/`provider'/`dataset'?facets=true&metadata=true&format=json&limit=`limit'&offset=`offset'`macval(thequery)'&observations=false"'
497 | }
498 |
499 | /* Save json locally to reduce server load over multiple calls */
500 | tempfile jdata
501 | capture copy `"`macval(apipath)'"' `jdata', replace
502 | if (inrange(_rc,630,696) | _rc == 601) {
503 | /* Determine whether it's a provider/dataset issue or sdmx or else */
504 | char _dta[errormsg] "`=_rc'"
505 | local theissue = _rc
506 | if (_rc == 601) {
507 | capture copy `"`path'/series/`provider'/`dataset'?facets=false&metadata=false&format=json&limit=0&offset=0&observations=false"' `jdata', replace
508 | if (_rc == 0) {
509 | di as smcl `"{err:server returned error. Make sure `macval(options)'`sdmx' are valid DB.nomics filters, or check {browse "`apipath'":`apipath'} for possibly more info about the issue.}"' /*"'*/
510 | }
511 | else {
512 | di as smcl `"{err:server returned error. Make sure {cmd:`provider'} and {cmd:`dataset'} are valid DB.nomics provider and dataset respectively, or check {browse "`apipath'":`apipath'} for possibly more info about the issue.}"' /*"'*/
513 | }
514 | char _dta[endpoint] "`apipath'"
515 | exit `theissue'
516 | }
517 | else {
518 | char _dta[endpoint] "`apipath'"
519 | error `theissue'
520 | exit `theissue'
521 | }
522 | }
523 | else if (_rc > 0) {
524 | di as smcl "{err:failed to reach the dbnomics servers. Check your internet connection, or try using the {cmd:insecure} option.}"
525 | char _dta[endpoint] "`apipath'"
526 | char _dta[errormsg] "`=_rc'"
527 | exit _rc
528 | }
529 |
530 | /* Parse JSON */
531 | mata: structure = fetchjson("`jdata'", "");
532 |
533 | /* Check for error in response */
534 | mata: parseresperr(structure);
535 |
536 | /* Parse metadata */
537 | mata: st_local("nomicsmeta", parsemeta(structure));
538 |
539 | /* Parse dataset structure */
540 | /* mata: datainfo = fetchjson("`jdata'", "dataset"); */
541 | mata: datainfo = structure->getNode("dataset");
542 |
543 | /* Tot. series num. */
544 | mata: numseries = fetchkeyvals(datainfo, ("nb_series"));
545 | mata: st_local("series_count", numseries[1]);
546 |
547 | /* Parse series node */
548 | /* mata: seriesinfo = fetchjson("`jdata'", "series"); */
549 | mata: seriesinfo = structure->getNode("series");
550 |
551 | /* Found series num */
552 | mata: fndseries = fetchkeyvals(seriesinfo, ("num_found"));
553 | mata: st_local("num_found", fndseries[1]);
554 |
555 | if (`limit' < min(`series_count',`num_found')) {
556 | if ((min(`series_count',`num_found') < ${S_dbnomics_hard_limit}) & (`override')) {
557 | display as smcl "{err:Warning: series set not complete. Consider removing the {cmd:limit} option.}"
558 | }
559 | else if (`override' == 0) {
560 | display as smcl "{err:Warning: series set larger than dbnomics maximum provided items.}" _n "{err:Use the {cmd:offset} option to load series beyond the ${S_dbnomics_hard_limit}th one.}"
561 | }
562 | }
563 |
564 | /* Parse series list */
565 | mata: seriesdata = seriesinfo->getNode("docs");
566 | capture mata: pushdata(json2table(seriesdata), jsoncolsArray(seriesdata, 0)');
567 |
568 | if (_rc > 0) {
569 | if (`limit' > 0) {
570 | display as smcl "{err:Warning: no series found}"
571 | }
572 | }
573 | else {
574 |
575 | /* Housekeeping */
576 | quietly {
577 | cleanutf8
578 | destring _all, replace
579 | remove_destrchar _all
580 | auto_labels _all
581 | }
582 |
583 | /* Reduce space (this can be automated in the future)*/
584 | qui compress
585 | }
586 |
587 | /* Add provider metadata */
588 | mata: metadata = ("nb_series","code","name");
589 | mata: datafeat = fetchkeyvals(datainfo, metadata);
590 | mata: for (kk=1; kk<=cols(metadata); kk++) st_lchar("_dta", metadata[kk], datafeat[kk]);
591 |
592 | /* Finally get datastructure template */
593 | mata: structinfo = parsestructure(datainfo);
594 | mata: st_local("dtstructure", structinfo);
595 | mata: st_lchar("_dta", "dtstructure", structinfo);
596 |
597 | /* Display result */
598 | if ((`"`thequery'"' != "") | (`"`sdmx'"' != "")) local series_parsed "`num_found' of "
599 |
600 | display as txt "`series_parsed'`series_count' series selected. Order of dimensions: (`dtstructure')" _c
601 | if (`limit' == 0) {
602 | display as smcl "{txt:. }{bf:None retrieved}"
603 | }
604 | else if (`limit' < min(`series_count',`num_found')) {
605 | display as smcl "{txt:. }{bf:Only #`=`offset'+1' to #`=min(`limit'+`offset',`series_parsed'`series_count')' retrieved}"
606 | }
607 | else {
608 | display as txt ""
609 | }
610 |
611 | /* Add metadata as dataset data characteristic */
612 | char _dta[provider] "`provider'"
613 | char _dta[dataset] "`dataset'"
614 | char _dta[endpoint] "`apipath'"
615 | char _dta[_meta] "`nomicsmeta'"
616 |
617 | /* Housekeeping */
618 | capture mata : mata drop datafeat datainfo fndseries kk metadata numseries seriesdata seriesinfo structinfo structure
619 |
620 | end
621 |
622 | /*5. Import one or more series */
623 | program dbnomics_import
624 |
625 | syntax anything(name=path), PRovider(string) Dataset(string) [LIMIT(numlist integer max=1 <=${S_dbnomics_hard_limit}) OFFSET(numlist integer max=1 >= 0) SDMX(string asis) SERIESids(string asis) CLEAR *]
626 |
627 | /* Set limit and offset if not provided */
628 | if ("`limit'" == "") local limit = ${S_dbnomics_hard_limit}
629 | if ("`offset'" == "") local offset = 0
630 |
631 | /* smdx and dimensions mutually exclusive */
632 | if (`"`sdmx'"' != "" & `"`macval(options)'"' != "") {
633 | di as smcl "{err:Options {cmd:sdmx} and {cmd:dimensions} are mutually exclusive.}"
634 | exit 198
635 | }
636 | /* seriesids and dimensions mutually exclusive */
637 | if (`"`seriesids'"' != "" & `"`macval(options)'"' != "") {
638 | di as smcl "{err:Options {cmd:seriesids} and {cmd:dimensions} are mutually exclusive.}"
639 | exit 198
640 | }
641 | /* seriesids and sdmx mutually exclusive */
642 | if (`"`seriesids'"' != "" & `"`sdmx'"' != "") {
643 | di as smcl "{err:Options {cmd:sdmx} and {cmd:seriesids} are mutually exclusive.}"
644 | exit 198
645 | }
646 |
647 | /* Parse filtering options */
648 | _optdict `macval(options)'
649 | if (`"`dimdict'"' != "") local thequery "&dimensions=`dimdict'"
650 |
651 | if (`"`sdmx'"' != "") mata: st_local("thequery", urlencode(`"`sdmx'"'))
652 |
653 | /* Parse list of series (must be comma separated)*/
654 | if (`"`seriesids'"' != "") {
655 | local thequery "series_ids="
656 | gettoken series oseries : seriesids, parse(",")
657 | while ("`series'" != "") {
658 | if ("`series'" != ",") local thequery `"`thequery'`provider'/`dataset'/`macval(series)',"'
659 | gettoken series oseries : oseries, parse(",")
660 | }
661 | local thequery = substr(`"`macval(thequery)'"', 1, length(`"`macval(thequery)'"') - 1)
662 | }
663 |
664 | /* Parse clear option*/
665 | if ("`clear'" == "") {
666 | if `c(width)' > 0 {
667 | di as err "no; data in memory would be lost. Use {cmd:clear} option"
668 | exit 4
669 | }
670 | }
671 | else {
672 | clear
673 | }
674 |
675 | /* Capture limit override */
676 | local override 0
677 | if (`limit' != ${S_dbnomics_hard_limit}) local override 1
678 |
679 | /* Setup call*/
680 | if (`"`seriesids'"' != "") {
681 | /* local apipath = "`path'/series?`thequery'&limit=`limit'&offset=`offset'" */
682 | local apipath = `"`path'/series?`macval(thequery)'&facets=false&metadata=false&observations=true"'
683 | }
684 | else if (`"`sdmx'"' != "") {
685 | local apipath = `"`path'/series/`provider'/`dataset'/`macval(thequery)'?facets=false&metadata=false&format=json&limit=`limit'&offset=`offset'&observations=true"'
686 | }
687 | else {
688 | local apipath = `"`path'/series/`provider'/`dataset'?facets=false&metadata=false&format=json&limit=`limit'&offset=`offset'`macval(thequery)'&observations=true"'
689 | }
690 |
691 | /* Save json locally to reduce server load over multiple calls */
692 | tempfile jdata
693 | capture copy `"`macval(apipath)'"' `jdata', replace
694 | if (inrange(_rc,630,696) | _rc == 601) {
695 | /* Determine whether it's a provider/dataset issue or sdmx or else */
696 | char _dta[errormsg] "`=_rc'"
697 | local theissue = _rc
698 | if (_rc == 601) {
699 | capture copy `"`path'/series/`provider'/`dataset'?facets=false&metadata=false&format=json&limit=0&offset=0&observations=false"' `jdata', replace
700 | if (_rc == 0) {
701 | di as smcl `"{err:server returned error. Make sure `macval(options)'`sdmx' are valid DB.nomics filters, or check {browse "`apipath'":`apipath'} for possibly more info about the issue.}"' /*"'*/
702 | }
703 | else {
704 | di as smcl `"{err:server returned error. Make sure {cmd:`provider'} and {cmd:`dataset'} are valid DB.nomics provider and dataset respectively, or check {browse "`apipath'":`apipath'} for possibly more info about the issue.}"' /*"'*/
705 | }
706 | char _dta[endpoint] "`apipath'"
707 | exit `theissue'
708 | }
709 | else {
710 | char _dta[endpoint] "`apipath'"
711 | error `theissue'
712 | exit `theissue'
713 | }
714 | }
715 | else if (_rc > 0) {
716 | di as smcl "{err:failed to reach the dbnomics servers. Check your internet connection, or try using the {cmd:insecure} option.}"
717 | char _dta[endpoint] "`apipath'"
718 | char _dta[errormsg] "`=_rc'"
719 | exit _rc
720 | }
721 |
722 | /* Parse JSON */
723 | mata: structure = fetchjson("`jdata'", "");
724 |
725 | /* Check for error in response */
726 | mata: parseresperr(structure);
727 |
728 | /* Parse metadata */
729 | mata: st_local("nomicsmeta", parsemeta(structure));
730 |
731 | /* This API does not return dataset info anymore. No need for variables below */
732 | /* Parse dataset structure. May be empty */
733 | /* mata: datainfo = fetchjson("`jdata'", "dataset"); */
734 | /* mata: datainfo = structure->getNode("dataset"); */
735 |
736 | /* Tot. series num. */
737 | /* mata: numseries = fetchkeyvals(datainfo, ("nb_series"));
738 | mata: st_local("series_count", numseries[1]); */
739 |
740 | /* Parse series node */
741 | /* mata: seriesinfo = fetchjson("`jdata'", "series"); */
742 | mata: seriesinfo = structure->getNode("series");
743 | mata: numseries = fetchkeyvals(seriesinfo, ("num_found"));
744 | mata: st_local("series_found", numseries[1]);
745 |
746 | if (`series_found' == 0) {
747 | display as smcl "{err:no series found}"
748 | local loopsize 0
749 | }
750 | else {
751 | /* Data is the array containing matching series */
752 | mata: srsdata = seriesinfo->getNode("docs");
753 |
754 | if (`limit' < `series_found') {
755 | if ((`series_found' < ${S_dbnomics_hard_limit}) & (`override')) {
756 | display as smcl "{err:Warning: series set not complete. Consider removing the {cmd:limit} option.}"
757 | }
758 | else if (`override' == 0) {
759 | display as smcl "{err:Warning: series set larger than dbnomics maximum provided items.}" _n "{err:Use the {cmd:offset} option to load series beyond the ${S_dbnomics_hard_limit}th one.}"
760 | }
761 | }
762 |
763 | tempfile theseries
764 |
765 | nobreak {
766 | quietly {
767 |
768 |
769 | local appendlist
770 | local loopsize = min(`limit'+`offset',`series_found') - `offset'
771 |
772 | nois di as smcl "{txt}Processing `loopsize' series"
773 |
774 | /* Note: this may fail for huge list of series if the c(macrolen) is hit */
775 | forval jj = 1/`loopsize' {
776 |
777 | tempfile dbseries`jj'
778 | if (`jj' > 1) local appendlist "`appendlist' "`dbseries`jj''""
779 |
780 | drop _all
781 |
782 | /* Parse series data */
783 | mata: seriesformat(srsdata, `jj');
784 | save `dbseries`jj''
785 |
786 | /* Progress report */
787 | nois _dots `jj' 0
788 |
789 | }
790 |
791 | use `dbseries1', clear
792 | if (`"`appendlist'"' != "") {
793 | append using `appendlist', gen(series_num)
794 | qui replace series_num = series_num + `offset'
795 | }
796 | else {
797 | qui gen byte series_num = 0 + `offset'
798 | }
799 |
800 | }
801 | }
802 |
803 | /* Reduce space (this can be automated in the future)*/
804 | qui compress
805 |
806 | /* Housekeeping */
807 | quietly {
808 | cleanutf8
809 | destring _all, replace
810 | remove_destrchar _all
811 | auto_labels _all
812 | foreach v of varlist _all {
813 | capture confirm new variable `=subinstr(itrim(trim(subinstr(strlower("`: var lab `v''"),"_"," ",.)))," ","_",.)'
814 | if !_rc {
815 | rename `v' `=subinstr(itrim(trim(subinstr(strlower("`: var lab `v''"),"_"," ",.)))," ","_",.)'
816 | }
817 | }
818 | }
819 | }
820 |
821 | /* Add provider metadata */
822 | /* mata: metadata = ("code","name");
823 | mata: datafeat = fetchkeyvals(datainfo, metadata);
824 | mata: for (kk=1; kk<=cols(metadata); kk++) st_lchar("_dta", metadata[kk], datafeat[kk]); */
825 |
826 | /* Setup reporting of loaded series */
827 | local series_loaded "`=min(`limit',`series_found')' "
828 | if ("`series_loaded'" == "`series_found' ") {
829 | local series_loaded
830 | }
831 | else if ((`series_found' > ${S_dbnomics_hard_limit})|(`offset'>0)) {
832 | if (`limit' > 1) {
833 | local series_loaded "#`=`offset'+1' to #`=min(`limit'+`offset', `series_found')' "
834 | }
835 | else {
836 | local series_loaded "#`=`offset'+1' "
837 | }
838 | }
839 |
840 | /* Avoid extra jump when nr of series is an exact multiple of 50 */
841 | if (mod(`loopsize',50) != 0) {
842 | display as smcl _n "{res}`series_found' series found and `series_loaded'loaded"
843 | }
844 | else {
845 | display as smcl "{res}`series_found' series found and `series_loaded'loaded"
846 | }
847 |
848 | /* Add metadata as dataset data characteristic */
849 | char _dta[provider] "`provider'"
850 | char _dta[dataset] "`dataset'"
851 | char _dta[endpoint] "`apipath'"
852 | char _dta[_meta] "`nomicsmeta'"
853 |
854 | /* Housekeeping */
855 | capture mata : mata drop datafeat kk metadata numseries srsdata seriesinfo structure
856 |
857 | end
858 |
859 |
860 | /*6. Use single series */
861 | program dbnomics_use
862 |
863 | syntax anything(name=series), PRovider(string) Dataset(string) PATH(string asis) [CLEAR DELIMiter(passthru)]
864 |
865 | /* Parse clear option*/
866 | if ("`clear'" == "") {
867 | if `c(width)' > 0 {
868 | di as err "no; data in memory would be lost. Use {cmd:clear} option"
869 | exit 4
870 | }
871 | }
872 | else {
873 | clear
874 | }
875 |
876 | local apipath = `"`path'/series/`provider'/`dataset'/`series'?format=csv&complete_missing_periods=false&align_periods=false"'
877 |
878 | /* Save csv locally to reduce server load in case of multiple calls */
879 | tempfile csvdata
880 | capture copy `"`macval(apipath)'"' `csvdata', replace
881 | if (inrange(_rc,630,696) | _rc == 601) {
882 | /* Determine whether it's a provider/dataset issue or sdmx or else */
883 | if (_rc == 601) {
884 | capture copy `"`path'/series/`provider'/`dataset'?facets=false&metadata=false&format=json&limit=0&offset=0&observations=false"' `jdata', replace
885 | if (_rc == 0) {
886 | di as smcl `"{err:server returned error. Make sure `series' is a valid DB.nomics series, or check {browse "`apipath'":`apipath'} for possibly more info about the issue.}"' /*"'*/
887 | }
888 | else {
889 | di as smcl `"{err:server returned error. Make sure {cmd:`provider'} and {cmd:`dataset'} are valid DB.nomics provider and dataset respectively, or check {browse "`apipath'":`apipath'} for possibly more info about the issue.}"' /*"'*/
890 | }
891 | }
892 | char _dta[endpoint] "`apipath'"
893 | char _dta[errormsg] "`=_rc'"
894 | exit _rc
895 | }
896 | else if (_rc > 0) {
897 | di as smcl "{err:failed to reach the dbnomics servers. Check your internet connection, or try using the {cmd:insecure} option.}"
898 | char _dta[endpoint] "`apipath'"
899 | char _dta[errormsg] "`=_rc'"
900 | exit _rc
901 | }
902 |
903 | /* Import series name: that's the header of the value column (2nd column) */
904 | quietly {
905 |
906 | tempfile theseries
907 | import delimited toss series_name using `csvdata', `delimiter' clear rowrange(1:1) varnames(nonames) encoding(utf-8)
908 | local series_name = series_name[1]
909 | gen byte dummy = 1
910 | save `theseries'
911 |
912 | /* Import full dataset */
913 | import delimited period value using `csvdata', `delimiter' clear encoding(utf-8) rowrange(2)
914 | gen code = "`series'"
915 | gen byte dummy = 1
916 | merge m:1 dummy using `theseries', assert(3) nogen norep keepusing(series_name)
917 | drop dummy
918 |
919 | }
920 |
921 | di as text `"`series_name'"'
922 | di as text `"(`=_N' `=plural(_N, "observation")' read)"'
923 |
924 | /* Housekeeping */
925 | quietly {
926 | cleanutf8
927 | destring _all, replace
928 | remove_destrchar _all
929 | auto_labels _all
930 | foreach v of varlist _all {
931 | capture confirm new variable `=subinstr(itrim(trim(subinstr(strlower("`: var lab `v''"),"_"," ",.)))," ","_",.)'
932 | if !_rc {
933 | rename `v' `=subinstr(itrim(trim(subinstr(strlower("`: var lab `v''"),"_"," ",.)))," ","_",.)'
934 | }
935 | }
936 | }
937 |
938 | /* Add metadata as dataset data characteristic */
939 | char _dta[provider] "`provider'"
940 | char _dta[dataset] "`dataset'"
941 | char _dta[series] "`series'"
942 | char _dta[series_name] "`series_name'"
943 | char _dta[endpoint] "`apipath'"
944 |
945 | end
946 |
947 | /*7. Last updates */
948 | program dbnomics_news
949 |
950 | syntax anything(name=path), [CLEAR LIMIT(numlist integer max=1 <=100) INSECURE]
951 |
952 | /* Set limit if not provided */
953 | if ("`limit'" == "") local limit = 20
954 |
955 | /* Setup call*/
956 | local apipath = "`path'/last-updates"
957 |
958 | /* Parse clear option*/
959 | if ("`clear'" == "") {
960 | if `c(width)' > 0 {
961 | di as err "no; data in memory would be lost. Use the {cmd:clear} option"
962 | exit 4
963 | }
964 | }
965 | else {
966 | clear
967 | }
968 |
969 | display as txt "Downloading recently added datasets..."
970 |
971 | /* Save json locally to reduce server load over multiple calls */
972 | tempfile jdata
973 | capture copy "`apipath'" `jdata', replace
974 | if (inrange(_rc,630,696) | _rc == 601) {
975 | if (_rc == 601) di as smcl `"{err:server returned error. Check {browse "`apipath'":`apipath'} for possibly more info about the issue.}"'
976 | char _dta[endpoint] "`apipath'"
977 | char _dta[errormsg] "`=_rc'"
978 | exit _rc
979 | }
980 | else if (_rc > 0) {
981 | di as smcl "{err:failed to reach the dbnomics servers. Check your internet connection, or try using the {cmd:insecure} option.}"
982 | char _dta[endpoint] "`apipath'"
983 | char _dta[errormsg] "`=_rc'"
984 | exit _rc
985 | }
986 |
987 | /* Parse JSON */
988 | mata: thenews = fetchjson("`jdata'", "");
989 | mata: datanode = thenews->getNode("datasets");
990 | mata: datadocs = datanode->getNode("docs");
991 |
992 | /* Check for error in response */
993 | mata: parseresperr(thenews);
994 |
995 | /* Parse metadata */
996 | mata: st_local("nomicsmeta", parsemeta(thenews));
997 |
998 | /* Call mata function */
999 | mata: pushdata(json2table(datadocs), jsoncolsArray(datadocs, 0)');
1000 |
1001 | /* Reduce space (this can be automated in the future)*/
1002 | qui compress
1003 |
1004 | /* Housekeeping */
1005 | quietly {
1006 | cleanutf8
1007 | destring _all, replace
1008 | remove_destrchar _all
1009 | auto_labels _all
1010 | }
1011 |
1012 | quietly {
1013 | /* Parse dates */
1014 | unab varlist : _all
1015 | local dbdates "converted_at indexed_at"
1016 | local dbdates : list dbdates & varlist
1017 |
1018 | local iter 0
1019 |
1020 | foreach dbd of local dbdates {
1021 | tempvar dbd`iter'
1022 | gen `dbd`iter'' = clock(`dbd',"YMD#hms#")
1023 | order `dbd`iter'', after(`dbd')
1024 | la var `dbd`iter'' "`: var lab `dbd''"
1025 | drop `dbd'
1026 | clonevar `dbd' = `dbd`iter''
1027 | format %tc `dbd'
1028 | order `dbd', after(`dbd`iter++'')
1029 | }
1030 |
1031 | /* Order dataset */
1032 | local ordlist "provider_code name provider_name code nb_series indexed_at"
1033 | local ordlist : list ordlist & varlist
1034 | capture order `ordlist', first
1035 |
1036 | }
1037 |
1038 | /* Display results */
1039 | dbnomics_list `ordlist', target(code) pr(provider_code) subcall(structure) apipath("`path'") show(`limit') `insecure'
1040 |
1041 | /* Add metadata as dataset data characteristic */
1042 | char _dta[endpoint] "`apipath'"
1043 | char _dta[_meta] "`nomicsmeta'"
1044 |
1045 | /* Housekeeping */
1046 | capture mata : mata drop thenews datadocs datanode
1047 |
1048 | end
1049 |
1050 | /*8. Search data and series */
1051 | program dbnomics_query, rclass
1052 |
1053 | syntax anything(name=query), [CLEAR PATH(string asis) LIMIT(numlist integer max=1 <=100) OFFSET(integer 0) INSECURE]
1054 |
1055 | /* Set limit if not provided */
1056 | if ("`limit'" == "") local limit = 10
1057 |
1058 | /* Parse clear option*/
1059 | if ("`clear'" == "") {
1060 | if `c(width)' > 0 {
1061 | di as err "no; data in memory would be lost. Use the {cmd:clear} option"
1062 | exit 4
1063 | }
1064 | }
1065 | else {
1066 | clear
1067 | }
1068 |
1069 | display as txt "Searching for {bf:`query'} in datasets and series..." _c
1070 |
1071 | /* Save json locally to reduce server load over multiple calls */
1072 | tempfile jdata
1073 |
1074 | /* Setup call*/
1075 | mata: st_local("queryenc", urlencode(`"`query'"'))
1076 | local apipath `"`path'/search?q=`queryenc'&limit=`limit'&offset=`offset'"'
1077 |
1078 | capture copy "`apipath'" `jdata', replace
1079 | if (inrange(_rc,630,696) | _rc == 601) {
1080 | if (_rc == 601) di as smcl `"{err:server returned error. Check {browse "`apipath'":`apipath'} for possibly more info about the issue.}"'
1081 | exit _rc
1082 | }
1083 | else if (_rc > 0) {
1084 | di as smcl "{err:failed to reach the dbnomics servers. Check your internet connection, or try using the {cmd:insecure} option.}"
1085 | char _dta[endpoint] "`apipath'"
1086 | char _dta[errormsg] "`=_rc'"
1087 | exit _rc
1088 | }
1089 |
1090 | /* Parse JSON */
1091 | mata: qmain = fetchjson("`jdata'", "");
1092 |
1093 | /* Parse number of results */
1094 | /* mata: qresult = fetchjson("`jdata'", "results"); */
1095 | mata: qresult = qmain->getNode("results");
1096 | mata: numseries = fetchkeyvals(qresult, ("num_found"));
1097 | mata: st_local("results_found", numseries[1]);
1098 |
1099 | /* Check for error in response */
1100 | mata: parseresperr(qmain);
1101 |
1102 | /* Parse metadata */
1103 | mata: st_local("nomicsmeta", parsemeta(qmain));
1104 |
1105 | /* Call mata function */
1106 | if (`results_found' > 0) {
1107 |
1108 | mata: resnode = qresult->getNode("docs");
1109 | mata: pushdata(json2table(resnode), jsoncolsArray(resnode, 0)');
1110 |
1111 | /* Reduce space (this can be automated in the future)*/
1112 | qui compress
1113 |
1114 | /* Housekeeping */
1115 | quietly {
1116 | cleanutf8
1117 | destring _all, replace
1118 | remove_destrchar _all
1119 | auto_labels _all
1120 | }
1121 |
1122 | quietly {
1123 | /* Parse dates */
1124 | unab varlist : _all
1125 | local dbdates "converted_at indexed_at"
1126 | local dbdates : list dbdates & varlist
1127 |
1128 | local iter 0
1129 |
1130 | foreach dbd of local dbdates {
1131 | tempvar dbd`iter'
1132 | gen `dbd`iter'' = clock(`dbd',"YMD#hms#")
1133 | order `dbd`iter'', after(`dbd')
1134 | la var `dbd`iter'' "`: var lab `dbd''"
1135 | drop `dbd'
1136 | clonevar `dbd' = `dbd`iter''
1137 | format %tc `dbd'
1138 | order `dbd', after(`dbd`iter++'')
1139 | }
1140 |
1141 | /* Order dataset */
1142 | local ordlist "type id provider_code name provider_name code nb_series nb_matching_series indexed_at"
1143 | local ordlist : list ordlist & varlist
1144 | capture order `ordlist', first
1145 |
1146 | /* Control data */
1147 | local ctrlist "_version_ json_data_commit_ref"
1148 | local ctrlist : list ctrlist & varlist
1149 | capture order `ctrlist', last
1150 |
1151 | /* Prepare to display results */
1152 | local displaylist "type provider_name name code nb_series nb_matching_series indexed_at"
1153 | local displaylist : list displaylist & varlist
1154 | }
1155 |
1156 | if (`limit' < `results_found') local extramsg " (first `=min(`limit',25)' shown)"
1157 | display as txt "`results_found' `=plural(`results_found', "result")' found`extramsg'." _n
1158 |
1159 | /* Display results */
1160 | tempvar subcallvar
1161 | /* In the v22 API, server only responds datasets */
1162 | gen `subcallvar' = "structure"
1163 | /* Stata crashes with more than 25 results, set maximum display length to 25 */
1164 | dbnomics_list `displaylist', target(code) pr(provider_code) subcall(`subcallvar') apipath("`path'") shownum(25) varsubcall `insecure'
1165 |
1166 | capture mata : mata drop resnode
1167 |
1168 | }
1169 | else {
1170 | display as err "no results found." _n
1171 | }
1172 |
1173 | /* Propagate rclass */
1174 | return add
1175 |
1176 | /* Add metadata as dataset data characteristic */
1177 | char _dta[endpoint] "`apipath'"
1178 | char _dta[_meta] "`nomicsmeta'"
1179 |
1180 | /* Housekeeping */
1181 | capture mata : mata drop qresult numseries qmain
1182 |
1183 | end
1184 |
1185 |
1186 |
1187 | /* 99. Utilities */
1188 |
1189 | /* List data with smart linking */
1190 | capture program drop dbnomics_list
1191 | program dbnomics_list, rclass
1192 |
1193 | syntax varlist(min=2), Target(varname) SUBCALL(string) APIPATH(string) [SHOWnum(integer 20) PRoviderlist(varname) Datasetlist(varname) VARSUBCALL INSECURE]
1194 |
1195 | /* Allocate screen space */
1196 | local cc = 1
1197 | local colspan0 = 1
1198 | /* unab varlist : _all TO REMOVE */
1199 | local maxshare = round(1.40 * (`c(linesize)'/ `:list sizeof varlist'), 1)
1200 | foreach var of local varlist {
1201 | local varlen = length("`var'") + 3
1202 | local col`cc' : format `var'
1203 | /* default colspan*/
1204 | local colspan`cc' = max(`varlen', 10) + `colspan`=`cc'-1''
1205 | if (regexm("`col`cc''","%([0-9]+).*[a-z]$")) local colspan`cc' = min(`maxshare', max(`varlen', real(regexs(1)))) + `colspan`=`cc++'-1'' + `=("`var'" == "`target'")'
1206 | }
1207 |
1208 | /* Write headers */
1209 | local tablelen = `colspan`=`cc'-1'' + 3
1210 | local cc = 0
1211 | quietly {
1212 | noi di as smcl "{c TLC}{hline `tablelen'}{c TRC}" _n "{c |}" _c
1213 | foreach var of local varlist {
1214 | noi di as smcl "{col `colspan`cc++''}`:var lab `var''" _c
1215 | }
1216 | noi di as smcl "{col `tablelen'} {c |}" _n "{c BLC}{hline `tablelen'}{c BRC}" _c
1217 | }
1218 |
1219 | quietly {
1220 | forval ii = 1/`=min(`shownum', `c(N)')' {
1221 |
1222 | local cc = 0
1223 | noi di as smcl _n " " _c
1224 |
1225 | if ("`varsubcall'" != "") {
1226 | local subcall_loop = `subcall'[`ii']
1227 | }
1228 | else {
1229 | local subcall_loop : copy local subcall
1230 | }
1231 |
1232 | foreach var of local varlist {
1233 |
1234 | capture confirm string var `var'
1235 | local istr = (_rc==0)
1236 |
1237 | /* Check if variable is the target var */
1238 | if ("`var'" == "`target'") {
1239 | /* Build command */
1240 | if ("`subcall_loop'" == "structure") {
1241 | local ocomm "dbnomics data, pr(`=`providerlist'[`ii']') d(`=`target'[`ii']') clear `insecure'"
1242 | }
1243 | else if ("`subcall_loop'" == "import") {
1244 | local ocomm "dbnomics import, pr(`=`providerlist'[`ii']') d(`=`datasetlist'[`ii']') series(`=`target'[`ii']') clear `insecure'"
1245 | }
1246 |
1247 |
1248 | if (`istr') {
1249 | local olab = abbrev("`=`var'[`ii']'",`colspan`=`cc'+1'' - `colspan`cc'')
1250 | local osmcl `"{txt:{stata "`ocomm'":`olab'}}"'
1251 | }
1252 | else {
1253 | local osmcl `"{txt:{stata "`ocomm'":`=`var'[`ii']'}}"'
1254 | }
1255 | }
1256 | else {
1257 | if (`istr') {
1258 | local olab = abbrev("`=`var'[`ii']'",`colspan`=`cc'+1'' - `colspan`cc'')
1259 | local osmcl `"`olab'"'
1260 | }
1261 | else {
1262 | local osmcl `"`=`var'[`ii']'"'
1263 | }
1264 | }
1265 |
1266 | if inlist("`: format `var''","%tc","%tC","%td") {
1267 | noi di as smcl `"{col `colspan`cc++''}"' `: format `var'' `osmcl' _c
1268 | }
1269 | else {
1270 | noi di as smcl `"{col `colspan`cc++''}`osmcl'"' _c
1271 | }
1272 |
1273 |
1274 | }
1275 |
1276 | return local cmd`ii' `"`ocomm'"'
1277 |
1278 | }
1279 |
1280 | noi di as txt _n "({it:Click on a highlighted link to load related data})"
1281 | }
1282 |
1283 | end
1284 |
1285 | /* Replace UNICODE characters */
1286 | program cleanutf8
1287 |
1288 | args noutf
1289 | tempfile runtoclean
1290 |
1291 | if ("`noutf'" != "1") {
1292 | quietly {
1293 | preserve
1294 |
1295 | tostring _all, replace force
1296 |
1297 | stack _all, into(strings) clear
1298 | drop _stack
1299 | duplicates drop
1300 |
1301 | moss strings, match("(\\u[0-9a-f][0-9a-f][0-9a-f][0-9a-f])") regex pref(uni)
1302 | keep if inrange(unicount,1,.)
1303 |
1304 | if (`c(N)' == 0) {
1305 | restore
1306 | cleanutf8 1
1307 | exit 0
1308 | }
1309 |
1310 | /* pause */
1311 |
1312 | unab matches: unimatch?*
1313 |
1314 | if (`:list sizeof matches' > 1) {
1315 | stack `matches', into(unistr) clear
1316 | drop _stack
1317 | }
1318 | else {
1319 | keep `matches'
1320 | rename `matches' unistr
1321 | }
1322 | duplicates drop
1323 | gen unichar = ustrunescape(unistr)
1324 | gen command = `"replace \`1' = subinstr(\`1',""' + unistr + `"",""' + unichar + `"",.)"'
1325 |
1326 | outfile command using `runtoclean', noquote
1327 |
1328 | restore
1329 | }
1330 | }
1331 |
1332 | foreach v of varlist _all {
1333 | capture confirm string variable `v'
1334 | if !_rc {
1335 | if ("`noutf'" != "1") run `runtoclean' `v', nostop
1336 | qui replace `v' = subinstr(`v',"\n","",.)
1337 | qui replace `v' = subinstr(`v',`"\""',`"""',.)
1338 | qui replace `v' = subinstr(`v',"\/","/",.)
1339 | qui replace `v' = "" if `v' == `""""'
1340 | }
1341 | }
1342 |
1343 | end
1344 |
1345 | /* Clean var chars */
1346 | program remove_destrchar
1347 |
1348 | syntax varlist
1349 |
1350 | foreach var of local varlist {
1351 | local thechars : char `var'[]
1352 | foreach ch of local thechars {
1353 | if inlist("`ch'","destring","destring_cmd") {
1354 | char `var'[`ch'] ""
1355 | }
1356 | }
1357 | }
1358 |
1359 | end
1360 |
1361 | /* Gen var labels */
1362 | program auto_labels
1363 |
1364 | syntax varlist
1365 |
1366 | foreach var of local varlist {
1367 | local thelab = strupper(substr("`var'",1,1)) + subinstr(substr("`var'",2,.),"_"," ",.)
1368 | lab var `var' "`thelab'"
1369 | }
1370 |
1371 | end
1372 |
1373 | /* DEPRECATED: Check SDMX compatibility. As of API v22, all providers seem to accept an SDMX mask, when the data structure allows */
1374 | * program _sdmx_check
1375 |
1376 | * args prtocheck
1377 |
1378 | * quietly {
1379 | * /* Try to get up-to-date list from the web */
1380 | * capture nobreak {
1381 | * preserve
1382 | * import delimited settings config using "https://git.nomics.world/dbnomics/dbnomics-api/raw/master/dbnomics_api/application.cfg", delim("=") clear
1383 | * keep if trim(itrim(settings)) == "SERIES_CODE_MASK_COMPATIBLE_PROVIDERS"
1384 | * replace config = subinstr(subinstr(subinstr(subinstr(config,",","",.), "}","",.), "{","",.),`"""',"",.) /*"'*/
1385 | * local checklist = config[1]
1386 | * restore
1387 | * }
1388 | * if _rc {
1389 | * local checklist "BIS ECB Eurostat FED IMF IMF-WEO INSEE OECD WTO"
1390 | * }
1391 | * local thetest : list prtocheck & checklist
1392 | * }
1393 |
1394 | * /* Return test results */
1395 | * c_local sdmx_compatible = (`"`thetest'"' != "")
1396 |
1397 | * end
1398 |
1399 | /* Compile dimensions dict based on macval(options) */
1400 | capture program drop _optdict
1401 | program _optdict
1402 |
1403 | local cmdorig : copy local 0
1404 |
1405 | if (`"`cmdorig'"' == "") {
1406 | c_local dimdict `""'
1407 | exit
1408 | }
1409 |
1410 | tokenize `"`0'"', parse(")")
1411 |
1412 | local i 1
1413 | local optlist
1414 | local optsyn
1415 |
1416 | /* Parse options */
1417 | while (`"`macval(`i')'"' != "") {
1418 |
1419 | local optfull "``=2*`i'-1''"
1420 | gettoken optcmd optval : optfull, parse("(")
1421 |
1422 | /* Here syntax-encode optcmd */
1423 | mata: st_local("optcmd_enc", syntaxencode(`"`optcmd'"'))
1424 |
1425 | if ("`optcmd_enc'" != "") local cmdorig : subinstr local cmdorig `"`optcmd'"' `"`optcmd_enc'"'
1426 | if ("`optcmd_enc'" != "") local optlist "`optlist' `optcmd_enc'"
1427 | if ("`optcmd_enc'" != "") local optsyn "`optsyn' `=strlower("`optcmd_enc'")'(string asis)"
1428 |
1429 | if ("`=strlower("`optcmd_enc'")'" != "`optcmd_enc'") local cmdorig : subinstr local cmdorig `"`optcmd_enc'("' `"`=strlower("`optcmd_enc'")'("'
1430 |
1431 | /* if ("`optval'" != "") local optvals "`optvals' `=substr(`"`optval'"',2,.)'" */
1432 |
1433 | local `i++'
1434 | }
1435 |
1436 | /* Parse options parameters */
1437 | local 0 `", `macval(cmdorig)'"'
1438 | syntax [anything], `optsyn'
1439 |
1440 | local thedict "{"
1441 |
1442 | foreach opt of local optlist {
1443 |
1444 | local theopt `""``=strlower("`opt'")''""'
1445 |
1446 | /* Here syntax-decode optcmd */
1447 | mata: st_local("optcmd", syntaxdecode(`"`opt'"'))
1448 |
1449 | local theoptdict : subinstr local theopt `"" ""' `"",""', all
1450 | local theoptdict2 : subinstr local theoptdict `""""' `"""', all
1451 |
1452 | local thedict "`thedict'"`optcmd'":[`theoptdict2'],"
1453 |
1454 | }
1455 |
1456 | /* Finalise dict */
1457 | local thedict = substr(`"`thedict'"',1,length(`"`thedict'"')-1) + "}"
1458 |
1459 | /* URL encode dict */
1460 | mata: st_local("output", urlencode(`"`thedict'"'))
1461 |
1462 | /* yield dict */
1463 | c_local dimdict `"`output'"'
1464 |
1465 | end
1466 |
1467 | /* Begin mata operations */
1468 | mata
1469 |
1470 | /* Procedure to extract series data */
1471 | void seriesformat(pointer (class libjson scalar) scalar data, real scalar cursor) {
1472 |
1473 | pointer (class libjson scalar) scalar series
1474 | pointer (class libjson scalar) scalar cell
1475 | real scalar itk
1476 | string matrix thedata
1477 | string matrix oinfo
1478 | string matrix oinfo_p
1479 | string matrix odata
1480 | string matrix output
1481 | string matrix ainfo
1482 | string matrix adata
1483 |
1484 | /* Loop through series */
1485 | series = data->getArrayValue(cursor);
1486 |
1487 | /* Parse nr. of kkeys */
1488 | selector = series->listAttributeNames(0);
1489 | /* printf("%s isonefine \n", strofreal(cols(selector))); */
1490 |
1491 | /*Initialise output*/
1492 | thedata = J(0,0,"");
1493 | scollector = J(1,0,"");
1494 |
1495 | /* Series data (period-value) */
1496 | /* printf("made it here \n") */
1497 | for (kk=1; kk<=cols(selector); kk++) {
1498 | cell = series->getAttribute(selector[kk]);
1499 | if ((cell->isArray()) && (cell->bracketArrayScalarValues() != "[]")) {
1500 | scollector = (scollector, selector[kk]);
1501 | cellarray = parsearray(cell,0)';
1502 | if (rows(thedata) == 0) {
1503 | thedata = J(rows(cellarray), 0, "");
1504 | }
1505 | thedata = (thedata, cellarray);
1506 | }
1507 | }
1508 |
1509 | /* Parse Other series info */
1510 | oinfo = dict2tablev2(series, 2);
1511 |
1512 | /* Filter out stuff that's already been parsed */
1513 | for (kk=1; kk<=cols(scollector); kk++) {
1514 | oinfo = select(oinfo, oinfo[.,1]:!=scollector[kk]);
1515 | }
1516 |
1517 | /* Ad-hoc parser for list of lists with observation attributes */
1518 | if (series->getAttribute("observations_attributes") != NULL) {
1519 |
1520 | oinfo = select(oinfo, oinfo[.,1]:!="observations_attributes");
1521 |
1522 | /* Get obs attributes data */
1523 | oadata = parseattributeslol(series->getAttribute("observations_attributes"), rows(thedata));
1524 |
1525 | /* Split header from content */
1526 | oainfo = oadata[1,.];
1527 | oadata = oadata[2..rows(oadata),.];
1528 |
1529 | }
1530 | else {
1531 | oainfo = J(1,0,"");
1532 | oadata = J(rows(thedata),0,"");
1533 | }
1534 |
1535 | /* Transpose oinfo */
1536 | oinfo_p = oinfo';
1537 |
1538 | /* Adjust other info */
1539 | odata = J(rows(thedata), 1, oinfo_p[2,.]);
1540 |
1541 | /*Combine dataset*/
1542 | output = thedata, oadata, odata;
1543 |
1544 | /* Export data */
1545 | pushdata(output, (scollector, oainfo, oinfo_p[1,.]));
1546 |
1547 | }
1548 |
1549 | /* NEW: ad-hoc function to parse observation_attributes list-of-lists */
1550 | string matrix parseattributeslol(pointer (class libjson scalar) scalar node, real scalar rowfit) {
1551 |
1552 | pointer (class libjson scalar) scalar sublist
1553 | real scalar alen
1554 | real scalar kk
1555 | string matrix collector
1556 | string matrix oadata
1557 | string matrix oheader
1558 |
1559 | /* Initialise collector given rowfit */
1560 | collector = J(rowfit+1,0,"");
1561 |
1562 | if (node->isArray() == 0) {
1563 | return(J(0,0,""))
1564 | }
1565 | else {
1566 | /* Get list size */
1567 | alen = node->arrayLength();
1568 |
1569 | /* Loop through list of lists */
1570 | for (kk=1; kk<=alen; kk++) {
1571 |
1572 | /* Get sub-list */
1573 | sublist = node->getArrayValue(kk);
1574 |
1575 | /* The format of these sublists is: "HEADER", ["content"] */
1576 | oheader = J(1,1, sublist->getArrayValue(1)->getString("",""))
1577 |
1578 | /* If the content is a string scalar, the payload provides a string, not a list */
1579 | if (sublist->getArrayValue(2)->isArray() == 1) {
1580 | oadata = parsearray(sublist->getArrayValue(2), 0);
1581 | }
1582 | else if (sublist->getArrayValue(2)->isString() == 1) {
1583 | oadata = J(1,rowfit,sublist->getArrayValue(2)->getString("",""));
1584 | }
1585 | else {
1586 | oadata = J(1,rowfit,"");
1587 | }
1588 |
1589 | /* Piece things together and update collector */
1590 | collector = (collector, (oheader, oadata)')
1591 |
1592 | }
1593 |
1594 | /* Return output */
1595 | return(collector);
1596 | }
1597 | }
1598 |
1599 | string scalar parsestructure(pointer (class libjson scalar) scalar node) {
1600 |
1601 | pointer (class libjson scalar) scalar templ
1602 |
1603 | /* Two strategies:*/
1604 | /* 1) List in dimensions_codes_order is available */
1605 | templ = node->getNode("dimensions_codes_order");
1606 | if ((templ!=NULL) && (templ->arrayLength() > 0)) {
1607 | return(parsearray(templ, 1));
1608 | } else {
1609 | /* 2) Get list of attribute names */
1610 | string scalar output
1611 | string rowvector columns
1612 | templ = node->getNode("dimensions_values_labels");
1613 | if (templ!=NULL) {
1614 | columns = templ->listAttributeNames(0);
1615 | output = columns[1];
1616 | for (kk=2; kk<=cols(columns); kk++) output = output + "." + columns[kk]
1617 | return(output);
1618 | } else {
1619 | output = "Not Available"
1620 | return(output);
1621 | }
1622 | }
1623 | }
1624 |
1625 | real rowvector dictdim(pointer (class libjson scalar) scalar node) {
1626 |
1627 | string rowvector selector
1628 | string matrix collector
1629 | real scalar NR
1630 | real scalar NC
1631 |
1632 | /* Parse nr. of kkeys */
1633 | selector = node->listAttributeNames(0);
1634 | NR = cols(selector);
1635 | /* Flatten and get longest col */
1636 | collector = node->flattenToKV();
1637 | /* Initialise max */
1638 | NC = 1
1639 | for (kk=1; kk<=rows(collector); kk++) {
1640 | NC = rowmax((NC, cols(tokenizer(collector[kk,1], ":"))));
1641 | }
1642 | return((NR, NC + 1));
1643 | }
1644 |
1645 | string rowvector tokenizer(string scalar toparse, string scalar punct) {
1646 | tok = tokens(toparse,punct);
1647 | NC = ceil(cols(tok)/2);
1648 | res = J(1,NC,"");
1649 | for (kk=1; kk<=NC; kk++) {
1650 | res[kk]=tok[kk*2-1];
1651 | }
1652 | return(res);
1653 | }
1654 |
1655 | /* Parse dict of dicts. Assumption: at most x nested level */
1656 | string matrix dict2table(pointer (class libjson scalar) scalar node, real scalar depth) {
1657 |
1658 | string matrix output
1659 | string matrix content
1660 | string matrix yield
1661 | string matrix isempty
1662 | string rowvector selector
1663 | pointer (class libjson scalar) scalar cell
1664 | real scalar kk
1665 |
1666 | /* Capture empty node */
1667 | isempty = node->flattenToKV();
1668 | if (rows(isempty) == 0) {
1669 | return("0");
1670 | }
1671 | if (node==NULL) {
1672 | return("0");
1673 | }
1674 |
1675 | /* Parse nr. of kkeys */
1676 | selector = node->listAttributeNames(0);
1677 |
1678 | /*Initialise output*/
1679 | output = J(0, depth, "");
1680 |
1681 | for (kk=1; kk<=cols(selector); kk++) {
1682 |
1683 | cell = node->getAttribute(selector[kk]);
1684 |
1685 | if (cell==NULL) {
1686 | return(0);
1687 | exit();
1688 | } else if (cell->isObject()) {
1689 | if (depth <= 2) {
1690 | content = cell->flattenToKV();
1691 | } else {
1692 | content = dict2table(cell, depth - 1);
1693 | }
1694 | if (cols(content) < cols(output)) {
1695 | yield = (J(rows(content), 1, selector[kk]), content, J(rows(content), cols(output) - cols(content) - 1, ""));
1696 | } else {
1697 | yield = content;
1698 | }
1699 | output = output \ yield;
1700 | } else if (cell->isString()) {
1701 | output = output \ (selector[kk], cell->getString("",""), J(1, cols(output) - 2, ""));
1702 | } else if (cell->isArray()) {
1703 | if (cell->bracketArrayScalarValues() == "[]") {
1704 | content = json2table(cell)
1705 | if (cols(content) < cols(output)) {
1706 | yield = (strofreal(range(1, rows(content), 1)), content, J(rows(content), cols(output) - (cols(content) + 1), ""));
1707 | } else {
1708 | yield = content;
1709 | }
1710 | output = output \ yield;
1711 | } else {
1712 | output = output \ (selector[kk], cell->bracketArrayScalarValues(), J(1, cols(output) - 2, ""));
1713 | }
1714 | }
1715 |
1716 | /* Skip cell if none of the above */
1717 | /* else { return(0); exit(); } */
1718 | }
1719 | return(output);
1720 | }
1721 |
1722 | /* Parse dict of dicts with complex structure. Assumption: at most x nested level */
1723 | string matrix dict2tablev2(pointer (class libjson scalar) scalar node, real scalar depth) {
1724 |
1725 | string matrix output
1726 | string matrix content
1727 | string matrix yield
1728 | string matrix isempty
1729 | string rowvector selector
1730 | pointer (class libjson scalar) scalar cell
1731 | real scalar kk
1732 |
1733 | /* Capture empty node */
1734 | isempty = node->flattenToKV();
1735 | if (rows(isempty) == 0) {
1736 | return("0");
1737 | }
1738 | if (node==NULL) {
1739 | return("0");
1740 | }
1741 |
1742 | /* Parse nr. of kkeys */
1743 | selector = node->listAttributeNames(0);
1744 |
1745 | /*Initialise output*/
1746 | output = J(0, depth, "");
1747 |
1748 | for (kk=1; kk<=cols(selector); kk++) {
1749 |
1750 | cell = node->getAttribute(selector[kk]);
1751 |
1752 | if (cell==NULL) {
1753 | return(0);
1754 | exit();
1755 | } else if (cell->isObject()) {
1756 | if (depth <= 2) {
1757 | content = cell->flattenToKV();
1758 | } else {
1759 | content = dict2table(cell, depth - 1);
1760 | }
1761 | if (cols(content) < cols(output)) {
1762 | yield = (J(rows(content), 1, selector[kk]), content, J(rows(content), cols(output) - cols(content) - 1, ""));
1763 | } else {
1764 | yield = content;
1765 | }
1766 | output = output \ yield;
1767 | } else if (cell->isString()) {
1768 | output = output \ (selector[kk], cell->getString("",""), J(1, cols(output) - 2, ""));
1769 | } else if (cell->isArray()) {
1770 | if (cell->bracketArrayScalarValues() == "[]") {
1771 | content = json2table(cell)
1772 | content = (J(rows(content),1,selector[kk]), content)
1773 | if (cols(content) < cols(output)) {
1774 | yield = (strofreal(range(1, rows(content), 1)), content, J(rows(content), cols(output) - (cols(content) + 1), ""));
1775 | } else {
1776 | yield = content;
1777 | }
1778 | output = output \ yield;
1779 | } else {
1780 | output = output \ (selector[kk], cell->bracketArrayScalarValues(), J(1, cols(output) - 2, ""));
1781 | }
1782 | }
1783 |
1784 | /* Skip cell if none of the above */
1785 | /* else { return(0); exit(); } */
1786 | }
1787 | return(output);
1788 | }
1789 |
1790 | string matrix parsetree(pointer (class libjson scalar) scalar node, string rowvector dictkeys) {
1791 |
1792 | pointer (class libjson scalar) scalar provnode
1793 | string matrix thetree
1794 |
1795 | /*Extract relevant node*/
1796 | provnode = node->getNode("category_tree");
1797 |
1798 | /* Build tree table */
1799 | thetree = getrecursive(provnode, dictkeys, 0);
1800 |
1801 | /*Output*/
1802 | return(thetree);
1803 |
1804 | }
1805 |
1806 | string rowvector treekeys(string scalar keylist) {
1807 |
1808 | string rowvector tok
1809 | real scalar NC
1810 | string scalar dictkeys
1811 |
1812 | if (keylist != "") {
1813 | tok = tokens(keylist,",");
1814 | NC = ceil(cols(tok)/2);
1815 | if (NC > 0) {
1816 | dictkeys=J(1,NC,"");
1817 | for (kk=1; kk<=NC; kk++) dictkeys[kk]=tok[kk*2-1];
1818 | }
1819 | else {
1820 | printf("{err: Invalid key list}\n", selector);
1821 | exit(error(198));
1822 | }
1823 | }
1824 | else {
1825 | dictkeys = ("code","name","doc_href");
1826 | }
1827 | return(dictkeys);
1828 | }
1829 |
1830 | string matrix fetchjson(string scalar url, string scalar path) {
1831 |
1832 | class libjson scalar w
1833 | pointer (class libjson scalar) scalar node
1834 |
1835 | /* Import JSON data*/
1836 | jstr = w.getrawcontents(url ,J(0,0,""));
1837 |
1838 | /* Fill any empty JSON object that would screw up the libjson parse command */
1839 | jstr = subinstr(jstr, `":{},"',`":{"null":true},"');
1840 | /*jstr = subinstr(jstr, `""dimensions":{},"',""); */
1841 |
1842 | /*Parse contents*/
1843 | node = w.parse(jstr);
1844 |
1845 | /* Parse path option */
1846 | if (path != "") {
1847 | pointer (class libjson scalar) scalar pnode
1848 | pnode = node->getNode(path);
1849 | if (pnode != NULL) {
1850 | return(pnode);
1851 | }
1852 | }
1853 | return(node);
1854 | }
1855 |
1856 | void parseresperr(pointer (class libjson scalar) scalar node) {
1857 |
1858 | pointer (class libjson scalar) scalar provnode
1859 |
1860 | /* Extract important node */
1861 | provnode = node->getNode("message");
1862 |
1863 | /* Extract error message */
1864 | if (provnode==NULL) {
1865 | /*No error key found*/
1866 | exit(0);
1867 | } else {
1868 | /* Display error description and exit 601 */
1869 | if (provnode->isString()) output = provnode->getString("","");
1870 | printf("{err: %s}\n", output);
1871 | exit(601);
1872 | }
1873 | }
1874 |
1875 | string scalar parsemeta(pointer (class libjson scalar) scalar node) {
1876 |
1877 | pointer (class libjson scalar) scalar provnode
1878 | pointer (class libjson scalar) scalar metanode
1879 | string rowvector provnode_attr
1880 | string scalar output
1881 |
1882 | /* Extract important node */
1883 | provnode = node->getNode("_meta");
1884 |
1885 | /* Extract error message */
1886 | /* Initialise output str */
1887 | output = ""
1888 |
1889 | if (provnode==NULL) {
1890 | /*No meta data found*/
1891 | return(output);
1892 | } else {
1893 | /* Get attributes*/
1894 | provnode_attr = provnode->listAttributeNames(0);
1895 |
1896 | /* Loop through attributes and fill output */
1897 | pointer (class libjson scalar) scalar cell
1898 |
1899 | for (k=1; k<=cols(provnode_attr); k++) {
1900 | /* Get attr name */
1901 | kk = provnode_attr[k];
1902 | /* Get attr content */
1903 | cell = provnode->getAttribute(kk);
1904 |
1905 | if (cell->isString()) {
1906 | if (output == "") {
1907 | output = kk + ": " + cell->getString("","");
1908 | } else {
1909 | output = output + ". " + kk + ": " + cell->getString("","");
1910 | }
1911 | }
1912 | }
1913 | return(output);
1914 | }
1915 | }
1916 |
1917 | void pushdata(string matrix ptable, string rowvector pheaders) {
1918 |
1919 | string rowvector pheadersp
1920 |
1921 | /* Ensure headers are proper stata var names*/
1922 | pheadersp = J(rows(pheaders),cols(pheaders),"")
1923 |
1924 | for (r=1; r<=cols(pheaders); r++) {
1925 | if (strlen(pheaders[1,r])>32) {
1926 | pheadersp[1,r] = strtoname(substr(pheaders[1,r],1,16)+substr(pheaders[1,r],-16,.));
1927 | }
1928 | else {
1929 | pheadersp[1,r] = strtoname(pheaders[1,r]);
1930 | }
1931 | }
1932 |
1933 | /*Add info to dataset*/
1934 | st_addobs(rows(ptable));
1935 | st_sstore(.,st_addvar("str2045", pheadersp), ptable);
1936 |
1937 | }
1938 |
1939 | string matrix json2table(pointer (class libjson scalar) scalar provnode) {
1940 |
1941 | /* Define json elements as libjson pointers */
1942 | pointer (class libjson scalar) scalar arrayval
1943 | pointer (class libjson scalar) scalar cell
1944 | pointer (string rowvector) scalar selectors
1945 |
1946 | real scalar NC
1947 | real scalar NR
1948 | string matrix res
1949 |
1950 | /* Get dimensions */
1951 | NC = strtoreal(jsoncolsArray(provnode, 1));
1952 | if (provnode->isArray()) {
1953 | NR = provnode->arrayLength();
1954 | }
1955 | else if (provnode->isObject()) {
1956 | string rowvector cols
1957 | cols = provnode->listAttributeNames(0);
1958 | NR = rows(cols');
1959 | }
1960 |
1961 | /* provnode is always an Array */
1962 | selectors = getcolsArray(provnode);
1963 |
1964 | /* Initialise output */
1965 | res = J(NR, NC, "");
1966 |
1967 | /* Shamelessly adapted from insheetjson */
1968 | /* Loop through rows and parse columns */
1969 | for (r=1; r<=NR; r++) {
1970 |
1971 | /* Get rth item from tableroot */
1972 | arrayval = provnode->getArrayValue(r);
1973 |
1974 | /* Loop through columns of rth row and parse cells */
1975 | for(c=1; c<=NC; c++) {
1976 |
1977 | /* The following is repeated from above */
1978 | /* Get cell content from rownod */
1979 | cell = arrayval->getNode(*selectors[c]);
1980 |
1981 | /* Cell is not empty: */
1982 | if (cell) {
1983 | /*Case 1: cell contains string. Getstring behaves like the dict.get() command in python */
1984 | if (cell->isString()) res[r,c] = cell->getString("","");
1985 | /* Case 2: cell contains array. Return list containing array values */
1986 | else if (cell->isArray()) res[r,c] = cell->bracketArrayScalarValues();
1987 | /* Case 3: cell is object. Return flattened json */
1988 | else if (cell->isObject()) {
1989 | if (substr(strtrim(cell->toString()), 1, 1) == "{") res[r,c] = substr(strtrim(cell->toString()),2,.)
1990 | else res[r,c] = strtrim(cell->toString())
1991 | }
1992 | }
1993 |
1994 | /* If cell is not found leave res with blank */
1995 |
1996 | }
1997 | }
1998 | return(res);
1999 | }
2000 |
2001 | string matrix getrecursive(pointer (class libjson scalar) scalar node, string rowvector dictkeys, real scalar level) {
2002 |
2003 | /* Loop through submitted key vector and fill output */
2004 | pointer (class libjson scalar) scalar cell
2005 | string matrix output
2006 | real scalar NR
2007 |
2008 | /*Initialise output*/
2009 | output = J(0,cols(dictkeys)+1,"");
2010 |
2011 | /*Case 0: node must be Array*/
2012 | if ( node->isObject() ) {
2013 | /* Capture node object */
2014 | output = output \ (strofreal(level), fetchkeyvals(node, dictkeys));
2015 | /*Try navigating to children object */
2016 | cell = node->getNode("children");
2017 | /* Build exception*/
2018 | if (cell==NULL) {
2019 | /*Reached the end of the tree*/
2020 | return(output);
2021 | }
2022 | else {
2023 | output = output \ getrecursive(cell, dictkeys, level + 1);
2024 | return(output);
2025 | }
2026 | }
2027 | else if ( node->isArray() ) {
2028 | /* Get array length*/
2029 | NR = node->arrayLength();
2030 | /*Exit if array length is zero*/
2031 | if (NR < 1) return(output);
2032 | for (r=1; r<=NR; r++) {
2033 | cell = node->getArrayValue(r);
2034 | output = output \ getrecursive(cell, dictkeys, level + 1);
2035 | }
2036 | return(output);
2037 | }
2038 | else {
2039 | return(output);
2040 | }
2041 | }
2042 |
2043 | pointer (string rowvector) getcolsArray(pointer (class libjson scalar) scalar node) {
2044 |
2045 | real scalar NR
2046 | string rowvector collector
2047 | string colvector uniquecols
2048 | pointer (string rowvector) scalar colsel
2049 |
2050 | pointer (class libjson scalar) scalar arrayval
2051 |
2052 | NR = node->arrayLength();
2053 | collector = J(1,0,"");
2054 |
2055 | for (r=1; r<=NR; r++) {
2056 |
2057 | /* Get inner array val*/
2058 | arrayval = node->getArrayValue(r);
2059 |
2060 | /* Update collector with node attributes */
2061 | collector = collector, arrayval->listAttributeNames(0);
2062 |
2063 | }
2064 |
2065 | /*Use uniqrows to list all atributes*/
2066 | uniquecols = uniqrows(collector')';
2067 |
2068 | colsel=J(1,cols(uniquecols),NULL)
2069 | for (k=1; k<=cols(uniquecols); k++) colsel[k] = & (uniquecols[k]);
2070 |
2071 | return(colsel);
2072 | }
2073 |
2074 | string rowvector parsearray(pointer (class libjson scalar) scalar node, real scalar nflag) {
2075 |
2076 | pointer (class libjson scalar) scalar cell
2077 | string rowvector collector
2078 | real scalar NR
2079 |
2080 | NR = node->arrayLength();
2081 | collector = J(1,NR,"");
2082 |
2083 | for (r=1; r<=NR; r++) {
2084 |
2085 | /* Get inner array val*/
2086 | cell = node->getArrayValue(r);
2087 |
2088 | if (cell->isString()) {
2089 |
2090 | if (nflag==. || nflag==0) {
2091 | /* Update collector with node attributes */
2092 | collector[r] = cell->getString("","");
2093 | }
2094 | else {
2095 | if (collector[1] == "") {
2096 | collector[1] = cell->getString("","");
2097 | } else {
2098 | collector[1] = collector[1] + "." + cell->getString("","");
2099 | }
2100 | }
2101 | } else {
2102 | return(0);
2103 | }
2104 | }
2105 | if (nflag==. || nflag==0) {
2106 | return(collector);
2107 | } else {
2108 | return(collector[1]);
2109 | }
2110 | }
2111 |
2112 | /* Returns either columns of JSON node or nr. of rows. Accepts an Array of dicts */
2113 | string matrix jsoncolsArray(pointer (class libjson scalar) scalar node, real scalar nflag) {
2114 |
2115 | real scalar NR
2116 | string rowvector collector
2117 | string colvector uniquecols
2118 |
2119 | pointer (class libjson scalar) scalar arrayval
2120 |
2121 | NR = node->arrayLength();
2122 | collector = J(1,0,"");
2123 |
2124 | for (r=1; r<=NR; r++) {
2125 |
2126 | /* Get inner array val*/
2127 | arrayval = node->getArrayValue(r);
2128 |
2129 | /* Update collector with node attributes */
2130 | collector = collector, arrayval->listAttributeNames(0);
2131 |
2132 | }
2133 |
2134 | /*Use uniqrows to list all atributes*/
2135 | uniquecols = uniqrows(collector');
2136 |
2137 | if (nflag==. || nflag==0) {
2138 | return(uniquecols)
2139 | }
2140 | else {
2141 | return(strofreal(rows(uniquecols)));
2142 | }
2143 | }
2144 |
2145 | string matrix fetchkeyvals(pointer (class libjson scalar) scalar node, string rowvector dictkeys) {
2146 |
2147 | /* Loop through submitted key vector and fill output */
2148 | pointer (class libjson scalar) scalar cell
2149 | string rowvector output
2150 |
2151 | /* Initialise output */
2152 | output = J(1, cols(dictkeys), "");
2153 |
2154 | if (node==NULL) {
2155 | /*No error key found*/
2156 | return(output);
2157 | }
2158 | else {
2159 | for (k=1; k<=cols(dictkeys); k++) {
2160 | /* Get attr content */
2161 | cell = node->getAttribute(dictkeys[k]);
2162 |
2163 | if (cell==NULL) {
2164 | return(output);
2165 | } else if (cell->isString()) {
2166 | output[k] = cell->getString("","");
2167 | } else {
2168 | output[k] = "";
2169 | }
2170 | }
2171 | return(output);
2172 | }
2173 | }
2174 |
2175 | /* URL encode, taken from libjson_source */
2176 | string scalar urlencode(string scalar s) {
2177 |
2178 | res = J(1,0,.);
2179 | a=ascii(s);
2180 |
2181 | for(c=1;c<=cols(a); c++) {
2182 | if ((a[c]>=44 && a[c]<=59) || (a[c]>=64 && a[c]<=122)) {
2183 | res=(res,a[c]);
2184 | } else {
2185 | h1 = floor(a[c]/16);
2186 | h2 = mod(a[c],16);
2187 | if (h1<10) {
2188 | h1=h1+48;
2189 | } else {
2190 | h1=h1+55;
2191 | }
2192 | if (h2<10) {
2193 | h2=h2+48;
2194 | } else {
2195 | h2=h2+55;
2196 | }
2197 | res=(res, 37, h1,h2);
2198 | }
2199 | }
2200 |
2201 | return(char(res));
2202 | }
2203 |
2204 | /* Convert to syntax-approved string */
2205 | string scalar syntaxencode(string scalar input) {
2206 |
2207 | real rowvector ascinput
2208 | string scalar output
2209 |
2210 | ascinput = ascii(input);
2211 | output = "";
2212 | for (kk=1; kk<=cols(ascinput); kk++) {
2213 | if ((ascinput[kk] >= 1 && ascinput[kk] <=45) || (ascinput[kk] >= 58 && ascinput[kk] <=64) || (ascinput[kk] >= 91 && ascinput[kk] <=94) || (ascinput[kk] >= 123 && ascinput[kk] <=126) || (ascinput[kk] == 47 || ascinput[kk] == 96)) {
2214 | output = output + "_" + strofreal(ascinput[kk]) + "_";
2215 | } else {
2216 | output = output + char(ascinput[kk]);
2217 | }
2218 | }
2219 |
2220 | return(output);
2221 | }
2222 |
2223 | /* Decode from syntax-approved string */
2224 | string scalar syntaxdecode(string scalar input) {
2225 |
2226 | string scalar output
2227 |
2228 | output = input;
2229 | for (kk=1; kk<=128; kk++) {
2230 | output = subinstr(output, "_" + strofreal(kk) + "_", char(kk));
2231 | }
2232 |
2233 | return(output);
2234 | }
2235 |
2236 |
2237 | end
2238 |
2239 | exit
2240 |
2241 |
--------------------------------------------------------------------------------