',
21 | 'ES index corresponding to the cache to clear'
22 | );
23 |
24 | program.parse();
25 | const options = program.opts();
26 |
27 | const main = async () => {
28 | const cursor = cache.find({ url: { $regex: `.*/${options.index}$` } })
29 | for await (const doc of cursor) {
30 | await cache.deleteOne(doc);
31 | }
32 | await cursor.close();
33 | process.exit(0);
34 | }
35 |
36 | main();
37 |
--------------------------------------------------------------------------------
/fe/src/lib/app/components/explore/coverage/utils.js:
--------------------------------------------------------------------------------
1 | import * as _ from 'lamb';
2 | import {isIterableEmpty} from '@svizzle/utils';
3 |
4 | /* trandsformations */
5 |
6 | export const initSelectedFieldSetsMap = _.pipe([
7 | _.mapWith(_.collect([
8 | _.getKey('id'),
9 | _.always(false)
10 | ])),
11 | _.fromPairs
12 | ]);
13 |
14 | export const initSelectedFieldsMap = _.pipe([
15 | _.mapWith(_.collect([
16 | _.identity,
17 | _.always(false)
18 | ])),
19 | _.fromPairs
20 | ]);
21 |
22 | export const makeGetFieldSetsFor = fieldIds => _.pipe([
23 | _.pick(fieldIds),
24 | _.values,
25 | ]);
26 |
27 | export const makeGetSelectedFields = (fieldSets, fieldIds) => _.pipe([
28 | _.pick(fieldIds),
29 | _.values,
30 | _.mapWith(_.getKey('fields')),
31 | _.unless(isIterableEmpty, _.reduceRightWith(_.intersection, fieldSets))
32 | ])
33 |
34 | export const getTruthyKeys = _.pipe([
35 | _.pairs,
36 | _.filterWith(_.getAt(1)),
37 | _.mapWith(_.getAt(0)),
38 | ]);
39 |
40 | /* data formatting */
41 |
42 | export const getPercent = (value, total) => `${Math.round(value / total * 100)}%`;
43 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/spec/geo_bounds.js:
--------------------------------------------------------------------------------
1 | import {
2 | geoPointObject,
3 | geoPointString
4 | } from '$lib/elasticsearch/types/fields.js';
5 | import {optional} from '$lib/elasticsearch/types/params.js';
6 | import {booleanD, string} from '$lib/types/index.js';
7 | import {field, missing} from '$lib/elasticsearch/aggs/ref/requestDoc.js';
8 | import response from '$lib/elasticsearch/aggs/response/geoBounds.js';
9 |
10 | export default {
11 | id: 'geo_bounds',
12 | availability: {
13 | from: '1.3'
14 | },
15 | docPath: '/search-aggregations-metrics-geobounds-aggregation.html',
16 | docs: 'Computes the bounding box containing all geo values for a field.',
17 | fieldType: geoPointObject,
18 | label: 'Geo Bounds',
19 | request: {
20 | field: string,
21 | missing: optional(geoPointString),
22 | wrap_longitude: optional(booleanD(true)),
23 | },
24 | requestDoc: {
25 | field,
26 | missing,
27 | wrap_longitude: 'Specifies whether the bounding box should be allowed to overlap the international date line.',
28 | },
29 | response,
30 | tag: 'metric',
31 | version: '7.9',
32 | };
33 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/spec/sum.js:
--------------------------------------------------------------------------------
1 | import {esNumericButBoolean} from '$lib/elasticsearch/aggs/ref/typeGroups.js';
2 | import {optional, script} from '$lib/elasticsearch/types/params.js';
3 | import {string, number} from '$lib/types/index.js';
4 | import {
5 | field,
6 | missing,
7 | script as scriptDoc,
8 | } from '$lib/elasticsearch/aggs/ref/requestDoc.js';
9 | import response from '$lib/elasticsearch/aggs/response/value.js';
10 |
11 | export default {
12 | id: 'sum',
13 | availability: {
14 | from: '1.3'
15 | },
16 | docPath: '/search-aggregations-metrics-sum-aggregation.html',
17 | docs: 'Sums up numeric values that are extracted from the aggregated documents.',
18 | fieldType: esNumericButBoolean,
19 | label: 'Sum',
20 | request: { // [0]
21 | field: string,
22 | missing: optional(number), // [1]
23 | script: optional(script),
24 | },
25 | requestDoc: {
26 | field,
27 | missing,
28 | script: scriptDoc,
29 | },
30 | response,
31 | tag: 'metric',
32 | version: '7.9',
33 | };
34 |
35 | // [0] 7.9: no params table
36 | // [1] TODO add constraint to be same type as the field type
37 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/utils/aggParams.js:
--------------------------------------------------------------------------------
1 | import * as _ from 'lamb';
2 | import {
3 | makeIsIncluded,
4 | makeMergeAppliedFnMap,
5 | valuesWith,
6 | } from '@svizzle/utils';
7 | import {
8 | nativeKey,
9 | shapeKey,
10 | defaultKey,
11 | isAppDefaultKey
12 | } from '$lib/types/index.js';
13 | import {optionalKey} from '$lib/elasticsearch/types/params.js';
14 |
15 | import {is_required} from '$lib/elasticsearch/types/params.utils.js';
16 |
17 | const getParamInfo = valuesWith(
18 | (paramType, paramId) => ({
19 | paramId,
20 | type: paramType,
21 | required: is_required(paramType),
22 | displayText: JSON.stringify(paramType, null, 2)
23 | })
24 | );
25 |
26 | export const getParamsInfo = _.pipe([
27 | getParamInfo,
28 | _.filterWith(_.not(_.pipe([
29 | _.getKey('paramId'),
30 | makeIsIncluded([nativeKey, shapeKey, optionalKey, defaultKey, isAppDefaultKey])
31 | ]))),
32 | _.sortWith([_.not(_.getKey('required'))])
33 | ]);
34 |
35 | export const mergeDocs = (partialParamsInfo, docs) =>
36 | partialParamsInfo.map(makeMergeAppliedFnMap({
37 | documentation: param => docs[param.paramId]
38 | }));
39 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/spec/stats.js:
--------------------------------------------------------------------------------
1 | import {esNumericButBoolean} from '$lib/elasticsearch/aggs/ref/typeGroups.js';
2 | import {optional, script} from '$lib/elasticsearch/types/params.js';
3 | import {string, number} from '$lib/types/index.js';
4 | import {
5 | field,
6 | missing,
7 | script as scriptDoc,
8 | } from '$lib/elasticsearch/aggs/ref/requestDoc.js';
9 | import response from '$lib/elasticsearch/aggs/response/stats.js';
10 |
11 | export default {
12 | id: 'stats',
13 | availability: {
14 | from: '1.3'
15 | },
16 | docPath: '/search-aggregations-metrics-stats-aggregation.html',
17 | docs: 'Computes stats over numeric values extracted from the aggregated documents.',
18 | fieldType: esNumericButBoolean,
19 | label: 'Stats',
20 | request: { // [0]
21 | field: string,
22 | missing: optional(number), // [1]
23 | script: optional(script),
24 | },
25 | requestDoc: {
26 | field,
27 | missing,
28 | script: scriptDoc,
29 | },
30 | response,
31 | tag: 'metric',
32 | version: '7.9',
33 | };
34 |
35 | // [0] 7.9: no params table
36 | // [1] TODO add constraint to be same type as the field type
37 |
--------------------------------------------------------------------------------
/fe/src/lib/utils/svizzle/utils/[any-array]-[array-object].js:
--------------------------------------------------------------------------------
1 | /**
2 | * @module @svizzle/utils/[any-array]-[array-object]
3 | */
4 |
5 | import * as _ from 'lamb';
6 |
7 | /**
8 | * Return a function expecting an array and returning an object
9 | * with keys and values defined by the provided function, which expects a value
10 | * and returns a pair [key, value].
11 | *
12 | * @function
13 | * @arg {function} valueToPair - (Any -> Array) Turns a value into a pair
14 | * @return {function} - (Array -> Object)
15 | *
16 | * @example
17 | > valueToPair = x => [`${x}${x}`, `${x}${x}${x}`];
18 | > arrayToObject1 = arrayToObjectWith(valueToPair)
19 | > arrayToObject1(['a', 'b', 1])
20 | {aa: 'aaa', bb: 'bbb', 11: '111'}
21 | >
22 | > valueIndexToPair = (x, i) => [`${i}${i}`, `${x}${x}${x}`];
23 | > arrayToObject2 = arrayToObjectWith(valueIndexToPair)
24 | > arrayToObject2(['a', 'b', 1])
25 | {'00': 'aaa', '11': 'bbb', '22': '111'}
26 | *
27 | * @version next
28 | */
29 | export const arrayToObjectWith = valueIndexToPair => _.pipe([
30 | _.mapWith(valueIndexToPair),
31 | _.fromPairs,
32 | ]);
33 | // FIXME unused as of now
34 |
--------------------------------------------------------------------------------
/fe/src/lib/app/machines/explore/route.config.js:
--------------------------------------------------------------------------------
1 | import {historyConfig} from './history.config.js';
2 | import {resultsConfig} from './results.config.js';
3 | import {searchConfig} from './search.config.js';
4 | import {selectingConfig} from './selecting.config.js';
5 |
6 | export const exploreConfig = {
7 | id: 'ExploreRoute',
8 | initial: 'Interactive',
9 | states: {
10 | Interactive: {
11 | type: 'parallel',
12 | states: {
13 | // order counts for rendering in the visualiser
14 | History: historyConfig,
15 | Selecting: selectingConfig,
16 | Results: resultsConfig,
17 | Search: searchConfig
18 | },
19 | on: {
20 | DATASET_UPDATED: {
21 | target: '#ExploreRoute.Interactive',
22 | actions: [
23 | 'selectDataset',
24 | 'setDataset',
25 | 'setNeededFields',
26 | 'setURL',
27 | ]
28 | },
29 | RESET_SOURCES: {
30 | target: '#ExploreRoute.Interactive',
31 | actions: ['resetSources']
32 | },
33 | SELECT_SOURCE: {
34 | target: '#ExploreRoute.Interactive',
35 | actions: ['selectSource']
36 | },
37 | },
38 | }
39 | }
40 | };
41 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/response/bucketsRange.js:
--------------------------------------------------------------------------------
1 | import {optional} from '$lib/elasticsearch/types/params.js';
2 | import {
3 | arrayOf,
4 | integer,
5 | string,
6 | number,
7 | objectOf,
8 | } from '$lib/types/index.js';
9 |
10 | export default {
11 | id: 'buckets_range',
12 | doc: {
13 | doc_count: 'The amount of documents in a bucket.',
14 | from: 'Start of the range (included).',
15 | key: 'The range key (e.g. `*-100.0`, `100.0-200.0`, `200.0-*`, or custom if ranges are named`).',
16 | to: 'End of the range (excluded).',
17 | },
18 | shape: {
19 | buckets: arrayOf(
20 | objectOf({
21 | doc_count: integer,
22 | from: optional(number),
23 | key: string,
24 | to: optional(number),
25 | })
26 | )
27 | },
28 | tag: 'multi-bucket',
29 | }
30 |
31 | /*
32 | {
33 | "buckets": [
34 | {
35 | "doc_count": 2,
36 | "key": "*-100.0",
37 | "to": 100.0
38 | },
39 | {
40 | "doc_count": 2,
41 | "from": 100.0,
42 | "key": "100.0-200.0",
43 | "to": 200.0
44 | },
45 | {
46 | "doc_count": 3,
47 | "from": 200.0,
48 | "key": "200.0-*"
49 | }
50 | ]
51 | }
52 | */
53 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2023 Nesta
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/fe/src/lib/app/components/elementary/Tab.svelte:
--------------------------------------------------------------------------------
1 |
22 |
23 | {#if isTitleSlot}
24 |
30 |
32 | {/if}
33 |
34 | {#if isContentSlot && isSelected}
35 |
16 | {#each aggs as aggsForKey}
17 | {@const path = aggsForKey.path}
18 |
19 |
20 | {getSummary(path[1])}
21 | by
22 | {getSummary(path[0])}
23 |
24 |
25 | {#each aggsForKey.result as agg}
26 |
31 | {/each}
32 |
34 | {/each}
35 |
36 |
37 |
44 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/types/fields.utils.js:
--------------------------------------------------------------------------------
1 | import * as _ from 'lamb';
2 | import {makeIsIncluded} from '@svizzle/utils';
3 |
4 | /* ES string fields */
5 |
6 | export const isWithKeywordTypeId =
7 | makeIsIncluded(['textWithKeyword', 'textWithKeywordArray']);
8 |
9 | export const keywordFieldTypes = [
10 | 'keyword',
11 | 'keywordArray',
12 | 'textWithKeyword',
13 | 'textWithKeywordArray'
14 | ];
15 |
16 | /* all */
17 |
18 | export const makeGetFieldsOfTypes = types => _.pipe([
19 | _.pairs,
20 | _.filterWith(_.pipe([
21 | _.getPath('1.type'),
22 | makeIsIncluded(types)
23 | ])),
24 | _.mapWith(_.getAt(0))
25 | ]);
26 |
27 | /* dates */
28 |
29 | /* TODO
30 | export const isValidDate
31 | - isString
32 |
33 | export const isValidDate_YYYYMMDD_dash
34 | - regex passes date_YYYYMMDD_dash.format.value
35 |
36 | export const date_YYYYMMDD_dash_time
37 | - regex passes date_YYYYMMDD_dash_time.format.value
38 | */
39 |
40 |
41 | /* Geo-point datatype */
42 |
43 | /* TODO
44 | export const isValidGeoPointObj
45 | - isShape('object')
46 | - lat passes numberWithin([-90, 90]),
47 | - lon passes numberWithin([-180, 180]),
48 | */
49 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/spec/string_stats.js:
--------------------------------------------------------------------------------
1 | import {esSearchableString} from '$lib/elasticsearch/aggs/ref/typeGroups.js';
2 | import {optional, script} from '$lib/elasticsearch/types/params.js';
3 | import {booleanD, string} from '$lib/types/index.js';
4 | import {
5 | field,
6 | missing,
7 | script as scriptDoc,
8 | } from '$lib/elasticsearch/aggs/ref/requestDoc.js';
9 | import response from '$lib/elasticsearch/aggs/response/stringStats.js';
10 |
11 | export default {
12 | id: 'string_stats',
13 | availability: {
14 | from: '7.6'
15 | },
16 | docPath: '/search-aggregations-metrics-string-stats-aggregation.html',
17 | docs: 'Computes statistics over string values extracted from the aggregated documents',
18 | fieldType: esSearchableString,
19 | label: 'String Stats',
20 | request: { // [0]
21 | field: string,
22 | missing: optional(string),
23 | script: optional(script),
24 | show_distribution: optional(booleanD(false)),
25 | },
26 | requestDoc: {
27 | field,
28 | missing,
29 | show_distribution: 'TODO',
30 | script: scriptDoc,
31 | },
32 | response,
33 | tag: 'metric',
34 | version: '7.9',
35 | };
36 |
37 | // [0] 7.9: no params table
38 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/spec/boxplot.js:
--------------------------------------------------------------------------------
1 | import {optional, script} from '$lib/elasticsearch/types/params.js';
2 | import {integerD, number, string} from '$lib/types/index.js';
3 | import {
4 | field,
5 | missing,
6 | script as scriptDoc
7 | } from '$lib/elasticsearch/aggs/ref/requestDoc.js';
8 | import response from '$lib/elasticsearch/aggs/response/boxplot.js';
9 |
10 | export default {
11 | id: 'boxplot',
12 | availability: {
13 | from: '7.7'
14 | },
15 | docPath: '/search-aggregations-metrics-boxplot-aggregation.html',
16 | docs: 'Computes boxplot of numeric values extracted from the aggregated documents. ',
17 | fieldType: number,
18 | label: 'Boxplot',
19 | request: { // [0]
20 | compression: optional(integerD(100)),
21 | field: string,
22 | missing: optional(number),
23 | script: optional(script)
24 | },
25 | requestDoc: {
26 | compression: 'Approximate algorithms must balance memory utilization with estimation accuracy. This balance can be controlled using a compression parameter.',
27 | field,
28 | missing,
29 | script: scriptDoc
30 | },
31 | response,
32 | tag: 'metric',
33 | version: '7.9',
34 | };
35 |
36 | // [0] 7.9: no params table
37 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/response/bucketsTextScore.js:
--------------------------------------------------------------------------------
1 | import {
2 | arrayOf,
3 | integer,
4 | number,
5 | objectOf,
6 | string,
7 | } from '$lib/types/index.js';
8 |
9 | export default {
10 | id: 'buckets_text_score',
11 | doc: {
12 | '{bucket}.bg_count': 'TODO.',
13 | '{bucket}.doc_count': 'The amount of documents in a bucket.',
14 | '{bucket}.key': 'The bucket term.',
15 | '{bucket}.score': 'The bucket score.',
16 | bg_count: 'TODO',
17 | doc_count: 'The amount of documents in a bucket.',
18 | },
19 | shape: {
20 | bg_count: integer,
21 | buckets: arrayOf(
22 | objectOf({
23 | bg_count: integer,
24 | doc_count: integer,
25 | key: string,
26 | score: number,
27 | })
28 | ),
29 | doc_count: integer,
30 | },
31 | tag: 'multi-bucket',
32 | }
33 |
34 | /*
35 | {
36 | "bg_count": 4814301,
37 | "buckets": [
38 | {
39 | "bg_count": 751990,
40 | "doc_count": 503375,
41 | "key": "North America",
42 | "score": 0.7828902342051492
43 | },
44 | {
45 | "bg_count": 1926,
46 | "doc_count": 1247,
47 | "key": "Europe",
48 | "score": 0.0018404593616494103
49 | }
50 | ],
51 | "doc_count": 1153445
52 | }
53 |
54 | */
55 |
--------------------------------------------------------------------------------
/fe/src/routes/+page.svelte:
--------------------------------------------------------------------------------
1 |
2 | Dapsboard - Home
3 |
4 |
5 |
6 |
7 | Nesta DAPS dashboard
8 | Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris feugiat diam et felis bibendum fringilla. Integer finibus rutrum luctus. Nullam porttitor ante tortor. Aenean iaculis lectus ut aliquam lacinia. Aliquam ornare urna libero. Quisque est libero, porttitor ut bibendum ut, iaculis in nunc. Aliquam lectus erat, pretium eu finibus nec, vulputate non lectus. Proin facilisis iaculis sapien, in malesuada lectus molestie quis. Maecenas in nisi eu eros molestie consectetur. Cras tincidunt quis lectus id maximus.
9 |
10 |
11 |
12 |
36 |
--------------------------------------------------------------------------------
/fe/src/lib/app/components/AggSelector.svelte:
--------------------------------------------------------------------------------
1 |
14 |
15 |
16 |
23 | setAggDocs?.(option.value)}
26 | on:mouseout={() => setAggDocs?.(null)}
27 | >
28 |
{option.text}
29 |
34 |
36 |
37 |
48 |
--------------------------------------------------------------------------------
/fe/src/lib/app/machines/builder/tester.config.js:
--------------------------------------------------------------------------------
1 | import { routeConfig } from './route.config.js';
2 |
3 | const tester_context = {
4 | autoExecute: false,
5 | cached: false,
6 | hideDisabledAxes: true,
7 | hideDisabledAggs: false,
8 | hideDisabledDatasets: false,
9 | hideDisabledItems: true,
10 | matching: false,
11 | selectionComplete: false,
12 | showFullResponse: false,
13 | queryReady: false
14 | };
15 |
16 | export const builderTesterConfig = {
17 | id: 'TestingBuilder',
18 | type: 'parallel',
19 | context: tester_context,
20 | states: {
21 | GuardsConfig: {
22 | initial: 'Idle',
23 | states: {
24 | Idle: {
25 | on: {
26 | SELECTION_COMPLETE_TOGGLED: {
27 | target: 'Idle',
28 | actions: ['toggleSelectionComplete']
29 | },
30 | QUERY_READY_TOGGLED: {
31 | target: 'Idle',
32 | actions: ['toggleQueryReady']
33 | },
34 | MATCHING_TOGGLED: {
35 | target: 'Idle',
36 | actions: ['toggleMatching']
37 | },
38 | CACHED_TOGGLED: {
39 | target: 'Idle',
40 | actions: ['toggleCached']
41 | }
42 | }
43 | }
44 | }
45 | },
46 | BuilderRoute: routeConfig
47 | }
48 | };
49 |
--------------------------------------------------------------------------------
/fe/src/lib/app/components/explore/viz/AggResults.svelte:
--------------------------------------------------------------------------------
1 |
22 |
23 | {#if results}
24 | {#each resultsByFieldSets as [key, aggs]}
25 | {@const depth = key.split(',').length - 1}
26 |
27 | Fields {key}
28 |
36 | {/each}
37 | {/if}
38 |
39 |
46 |
--------------------------------------------------------------------------------
/fe/svelte.config.js:
--------------------------------------------------------------------------------
1 | import path from 'node:path';
2 | import {fileURLToPath} from 'node:url';
3 |
4 | import adapterAuto from '@sveltejs/adapter-auto';
5 | import adapterNetlify from '@sveltejs/adapter-netlify';
6 | import adapterStatic from '@sveltejs/adapter-static';
7 |
8 | import {readJson} from '@svizzle/file';
9 |
10 | const __dirname = path.dirname(fileURLToPath(import.meta.url));
11 | const SOURCES_PATH = path.resolve(__dirname, './src/lib/app/data/sidebar.json')
12 | const sources = await readJson(SOURCES_PATH);
13 |
14 | const sourceIds = sources.map(({source}) => source);
15 | const exploreRoutes = sourceIds.map(id => `/explore/${id}`);
16 |
17 | // eslint-disable-next-line no-process-env
18 | const {ADAPTER} = process.env;
19 |
20 | const adapterMap = {
21 | auto: adapterAuto(),
22 | netlify: adapterNetlify({
23 | edge: false,
24 | split: false
25 | }),
26 | static: adapterStatic()
27 | }
28 |
29 | /** @type {import('@sveltejs/kit').Config} */
30 | const config = {
31 | extensions: ['.svelte', '.md'],
32 | kit: {
33 | adapter: adapterMap[ADAPTER] || adapterMap.auto,
34 | prerender: {
35 | entries: ['*', ...exploreRoutes]
36 | }
37 | }
38 | };
39 |
40 | export default config;
41 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/spec/extended_stats.js:
--------------------------------------------------------------------------------
1 | import {esNumeric} from '$lib/elasticsearch/aggs/ref/typeGroups.js';
2 | import {optional, script} from '$lib/elasticsearch/types/params.js';
3 | import {floatD, number, string} from '$lib/types/index.js';
4 | import {
5 | field,
6 | missing,
7 | script as scriptDoc,
8 | } from '$lib/elasticsearch/aggs/ref/requestDoc.js';
9 | import response from '$lib/elasticsearch/aggs/response/extendedStats.js';
10 |
11 | export default {
12 | id: 'extended_stats',
13 | availability: {
14 | from: '1.3'
15 | },
16 | docPath: '/search-aggregations-metrics-extendedstats-aggregation.html',
17 | docs: 'Calculates an approximate count of distinct values.',
18 | label: 'Extended Stats',
19 | fieldType: esNumeric,
20 | request: { // [0]
21 | field: string,
22 | missing: optional(number),
23 | script: optional(script),
24 | sigma: optional(floatD(2)),
25 | },
26 | requestDoc: {
27 | field,
28 | missing,
29 | script: scriptDoc,
30 | sigma: 'Can be any non-negative double which controls how many standard deviations +/- from the mean should be displayed.'
31 | },
32 | response,
33 | tag: 'metric',
34 | version: '7.9',
35 | };
36 |
37 | // [0] 7.9: no params table
38 |
--------------------------------------------------------------------------------
/fe/src/lib/app/components/explore/suggestions/FieldMenu.svelte:
--------------------------------------------------------------------------------
1 |
14 |
15 | {#each fieldStats as field}
16 | selectField(field.name)}
21 | >
22 | {field.name}
23 | {field.count}
24 |
25 | {/each}
26 |
27 |
47 |
--------------------------------------------------------------------------------
/fe/src/lib/app/components/elementary/Collapsible.svelte:
--------------------------------------------------------------------------------
1 |
12 |
13 |
29 |
30 |
59 |
--------------------------------------------------------------------------------
/be/src/coverage.js:
--------------------------------------------------------------------------------
1 | import * as _ from 'lamb';
2 |
3 | import { count, getMappings } from 'dap_dv_backends_utils/es/index.mjs';
4 | import { scroll } from 'dap_dv_backends_utils/es/search.mjs';
5 |
6 | const isFieldInDoc = (field, doc) => {
7 | if (field in doc) {
8 | return doc[field] !== null;
9 | }
10 | return false;
11 | }
12 |
13 | const computeSet = (fields, doc) => {
14 | const set = _.filter(fields, f => isFieldInDoc(f, doc));
15 | const setString = _.join(set, '&');
16 | return setString;
17 | }
18 |
19 | export const coverage = async (domain, index) => {
20 | const total = await count(domain, index);
21 | const mapping = await getMappings(domain, index);
22 | const fields = _.keys(
23 | mapping[index].mappings.properties
24 | || mapping[index].mappings._doc.properties
25 | );
26 | const scroller = scroll(domain, index, {size: 10000});
27 | const counts = {};
28 | let progress = 0;
29 | let page;
30 | for await (page of scroller) {
31 | _.forEach(
32 | page.hits.hits,
33 | doc => {
34 | const set = computeSet(fields, doc._source);
35 | counts[set] = set in counts ? counts[set] + 1 : 1;
36 | }
37 | );
38 | progress += page.hits.hits.length;
39 | console.log(progress / total);
40 | }
41 | return counts;
42 | }
43 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/ref/aggToResponseType.js:
--------------------------------------------------------------------------------
1 | export default {
2 | "auto_date_histogram": "buckets_date",
3 | "avg": "value",
4 | "boxplot": "boxplot",
5 | "cardinality": "value",
6 | "date_histogram": "buckets_date",
7 | "date_range": "buckets_date_range",
8 | "extended_stats": "extended_stats",
9 | "geo_bounds": "geo_bounds",
10 | "geo_centroid": "geo_centroid",
11 | "geo_distance": "buckets_range",
12 | "geohash_grid": "buckets_geohash_grid",
13 | "geotile_grid": "buckets_geotile_grid",
14 | "histogram": "buckets_number",
15 | "max": "value",
16 | "median_absolute_deviation": "value",
17 | "min": "value",
18 | "missing": "doc_count",
19 | "percentile_ranks": "numkeyToNum",
20 | "percentiles": "numkeyToNum",
21 | "range": "buckets_range",
22 | "rare_terms": "buckets_number",
23 | "rate": "value",
24 | "scripted_metric": "value",
25 | "significant_terms": "buckets_text_score",
26 | "significant_text": "buckets_text_score",
27 | "stats": "stats",
28 | "string_stats": "string_stats",
29 | "sum": "value",
30 | "t_test": "value",
31 | "terms": "buckets_terms",
32 | "top_hits": "hits",
33 | "top_metrics": "top",
34 | "value_count": "value",
35 | "variable_width_histogram": "buckets_number",
36 | "weighted_avg": "value"
37 | }
38 |
--------------------------------------------------------------------------------
/specs/indices/hpmt_epc_v0.yaml:
--------------------------------------------------------------------------------
1 | source:
2 | -
3 | provider_name: EPC
4 | provider_url: https://epc.opendatacommunities.org/
5 | dataset:
6 | api_type: ElasticSearch
7 | api_version: 7.10
8 | config_url: https://gist.githubusercontent.com/doogyb/05f58239c674feb0975b7c582c993b1a/raw/177fca03e4ec82f188294b6dc5129ce246ecdae3/epc_with_hp_install_dates_v0_config.json
9 | endpoint_url: https://es.annotations.dap-tools.uk/epc_with_hp_install_dates
10 | provider_name: DAPS (Nesta)
11 | provider_url: https://github.com/nestauk/nesta
12 | schema:
13 | postcode:
14 | type: textWithKeyword
15 | postcode_district:
16 | type: textWithKeyword
17 | tenure:
18 | type: textWithKeyword
19 | built_form:
20 | type: textWithKeyword
21 | property_type:
22 | type: textWithKeyword
23 | construction_age_band:
24 | type: textWithKeyword
25 | number_habitable_rooms:
26 | type: float
27 | mains_gas_flag:
28 | type: textWithKeyword
29 | energy_consumption_current:
30 | type: float
31 | current_energy_rating:
32 | type: textWithKeyword
33 | country:
34 | type: textWithKeyword
35 | hp_installed:
36 | type: boolean
37 | hp_install_date:
38 | type: date
39 | version: 0
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/README.md:
--------------------------------------------------------------------------------
1 | # ElasticSearch Aggregations
2 |
3 | ## Composition Rules
4 |
5 | * Metric aggregations can't have children
6 | * Most bucketing aggregations can have other bucketing or metric
7 | aggregations as children.
8 | * The `Global` aggregator can only be top level.
9 | * The `Reverse Nested` aggregator can only be defined inside a `Nested`
10 | aggregation.
11 | * The `Significant Text` aggregation can be used anywhere but:
12 | > Re-analyzing large result sets will require a lot of time and memory. It is
13 | > recommended that the `significant_text` aggregation is used as a child of
14 | > either the `sampler` or `diversified sampler` aggregation to limit the
15 | > analysis to a small selection of top-matching documents e.g. 200. This will
16 | > typically improve speed, memory use and quality of results.
17 |
18 |
19 | # ElasticSearch Filters
20 |
21 | ## `geo_bounding_box`
22 |
23 | See https://www.elastic.co/guide/en/elasticsearch/reference/7.0/search-aggregations-bucket-geotilegrid-aggregation.html#_high_precision_requests_2
24 |
25 | > When requesting detailed buckets (typically for displaying a "zoomed in" map) a filter like `geo_bounding_box` should be applied to narrow the subject area otherwise potentially millions of buckets will be created and returned.
26 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/utils/version.spec.js:
--------------------------------------------------------------------------------
1 | import assert from 'node:assert';
2 |
3 | import {makeIsAggVersionCompatible} from './version.js';
4 |
5 | describe('elasticsearch/aggs/utils/version', function () {
6 | describe('makeIsAggVersionCompatible', function () {
7 | const isCompatible = makeIsAggVersionCompatible('6.3');
8 |
9 | it('same `from`: OK', function () {
10 | assert.deepStrictEqual(isCompatible({
11 | availability: {from: '6.3'}
12 | }), true);
13 | });
14 | it('lower `from`: OK', function () {
15 | assert.deepStrictEqual(isCompatible({
16 | availability: {from: '6.1'}
17 | }), true);
18 | });
19 | it('higher `from`: fails', function () {
20 | assert.deepStrictEqual(isCompatible({
21 | availability: {from: '7.0'}
22 | }), false);
23 | });
24 | it('same `to`: OK', function () {
25 | assert.deepStrictEqual(isCompatible({
26 | availability: {from: '0.1', to: '6.3'}
27 | }), true);
28 | });
29 | it('higher `to`: OK', function () {
30 | assert.deepStrictEqual(isCompatible({
31 | availability: {from: '0.1', to: '6.9'}
32 | }), true);
33 | });
34 | it('lower `to`: fails', function () {
35 | assert.deepStrictEqual(isCompatible({
36 | availability: {from: '0.1', to: '5.4'}
37 | }), false);
38 | });
39 | });
40 | });
41 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/spec/range.js:
--------------------------------------------------------------------------------
1 | import {optional, script} from '$lib/elasticsearch/types/params.js';
2 | import {
3 | arrayOf,
4 | booleanD,
5 | number,
6 | objectOf,
7 | someOf,
8 | string,
9 | } from '$lib/types/index.js';
10 | import {field, keyed, script as scriptDoc} from '$lib/elasticsearch/aggs/ref/requestDoc.js';
11 | import response from '$lib/elasticsearch/aggs/response/bucketsRange.js';
12 |
13 | export default {
14 | id: 'range',
15 | availability: {
16 | from: '1.3'
17 | },
18 | docPath: '/search-aggregations-bucket-range-aggregation.html',
19 | docs: 'A multi-bucket value source based aggregation that enables the user to define a set of ranges - each representing a bucket.',
20 | fieldType: number,
21 | label: 'Range',
22 | needsParent: false,
23 | request: {// [0]
24 | field: string,
25 | keyed: optional(booleanD(false)),
26 | ranges: arrayOf(objectOf({
27 | key: optional(string),
28 | __extent: someOf({
29 | from: number,
30 | to: number,
31 | }),
32 | })),
33 | script: optional(script)
34 | },
35 | requestDoc: {
36 | field,
37 | keyed,
38 | ranges: 'Set of ranges, each representing a bucket.',
39 | script: scriptDoc
40 | },
41 | response,
42 | subAggs: true,
43 | tag: 'bucketing',
44 | version: '7.9',
45 | };
46 |
47 | // [0] 7.9: no params table
48 |
--------------------------------------------------------------------------------
/fe/src/lib/utils/net.js:
--------------------------------------------------------------------------------
1 | function logError (e, s) {
2 | console.log('Error: ', e, s);
3 | }
4 |
5 | /*
6 | class ESError extends Error {
7 | constructor(message, json) {
8 | super(message);
9 | this.jsonMessage = json;
10 | }
11 | }
12 | */
13 |
14 | export async function request (
15 | method,
16 | url,
17 | { headers = {}, data, fetch = window && window.fetch, type = 'json' } = {}
18 | ) {
19 | let reqOptions = { method, headers };
20 | if (data) {
21 | if (data instanceof FormData) {
22 | reqOptions.headers["content-type"] = "multipart/form-data";
23 | reqOptions.body = data;
24 | } else {
25 | reqOptions.headers["content-type"] = "application/json";
26 | reqOptions.body = JSON.stringify(data);
27 | }
28 | }
29 | let response;
30 | try {
31 | response = await fetch(url, reqOptions);
32 | if (Math.floor(response.status / 100) !== 2) {
33 | throw new Error('API request failed!');
34 | }
35 | return response[type]();
36 | } catch (error) {
37 | if (response) {
38 | // FIXME not all HTTP error responses necesarily have a JSON body
39 | error.jsonMessage = await response.clone().json();
40 | error.httpStatus = response.status;
41 | error.httpStatusText = response.statusText;
42 | }
43 |
44 | logError('request() failure', error, error.stack);
45 | throw error;
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/response/bucketsDateRange.js:
--------------------------------------------------------------------------------
1 | import {optional} from '$lib/elasticsearch/types/params.js';
2 | import {
3 | arrayOf,
4 | integer,
5 | string,
6 | number,
7 | objectOf,
8 | } from '$lib/types/index.js';
9 |
10 | export default {
11 | id: 'buckets_date_range',
12 | doc: {
13 | doc_count: 'The amount of documents in a bucket.',
14 | from_as_string: 'Start of the range (included) - as a string.',
15 | from: 'Start of the range (included).',
16 | key: 'The range key (e.g. `*-100.0`, `100.0-200.0`, `200.0-*`, or custom if ranges are named`).',
17 | to_as_string: 'End of the range (excluded) - as a string.',
18 | to: 'End of the range (excluded).',
19 | },
20 | shape: {
21 | buckets: arrayOf(
22 | objectOf({
23 | doc_count: integer,
24 | from_as_string: optional(string),
25 | from: optional(number),
26 | key: string,
27 | to_as_string: optional(string),
28 | to: optional(number),
29 | })
30 | )
31 | },
32 | tag: 'multi-bucket',
33 | }
34 |
35 | /*
36 | {
37 | "buckets": [
38 | {
39 | "doc_count": 7,
40 | "key": "*-10-2015",
41 | "to_as_string": "10-2015",
42 | "to": 1.4436576E12
43 | },
44 | {
45 | "doc_count": 0,
46 | "from_as_string": "10-2015",
47 | "from": 1.4436576E12,
48 | "key": "10-2015-*"
49 | }
50 | ]
51 | }
52 | */
53 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/response/hits.js:
--------------------------------------------------------------------------------
1 | import {
2 | arrayOf,
3 | enumsOf,
4 | float,
5 | integer,
6 | object,
7 | objectOf,
8 | unionOf,
9 | } from '$lib/types/index.js';
10 |
11 | export default {
12 | id: 'hits',
13 | doc: {
14 | 'hits.total.value': 'The amount of selected top documents in a certain bucket.', // inferred, TBD
15 | 'hits.total.relation': 'TODO.',
16 | 'hits.max_score': 'The max score among the selected top documents', // inferred, TBD
17 | 'hits.hits': 'The array of selected top documents',
18 | },
19 | docLong: {},
20 | shape: {
21 | hits: objectOf({
22 | total: objectOf({
23 | value: integer,
24 | relation: enumsOf(['eq'])
25 | }),
26 | max_score: unionOf(float, null),
27 | hits: arrayOf(object) // array of documents
28 | })
29 | },
30 | tag: 'sub-aggregator', // you don't use it to get a value
31 | }
32 |
33 | /*
34 | {
35 | "hits": {
36 | "total": {
37 | "relation": "eq",
38 | "value": 3
39 | },
40 | "max_score": null,
41 | "hits": [
42 | {
43 | "_index": "sales",
44 | "_type": "_doc",
45 | "_id": "AVnNBmauCQpcRyxw6ChK",
46 | "_source": {
47 | "date": "2015/03/01 00:00:00",
48 | "price": 200
49 | },
50 | "sort": [
51 | 1425168000000
52 | ],
53 | "_score": null
54 | }
55 | ]
56 | }
57 | }
58 |
59 | */
60 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/utils/query.js:
--------------------------------------------------------------------------------
1 | import * as _ from 'lamb';
2 |
3 | import {getDefault, hasDefault} from '$lib/types/index.js';
4 | import {stopWords} from '$lib/utils/stopwords.js';
5 |
6 | // params that startsWith('__') should be required, with no default
7 | export const makeRequestToQuery = fieldName => _.pipe([
8 | _.collect([
9 | obj => _.has(obj, 'field') ? {field: fieldName} : {},
10 | _.pipe([_.pickIf(hasDefault), _.mapValuesWith(getDefault)])
11 | ]),
12 | _.apply(_.merge)
13 | ]);
14 |
15 | /**
16 | * Get how many items have a `searchTerm` in the input `fields`
17 | */
18 | export const getCountQuery = (fields, searchTerm) => ({
19 | size: 0,
20 | aggs: {
21 | messages: {
22 | filters: {
23 | filters: _.fromPairs(fields.map(name => [name, {
24 | term: {
25 | [name]: searchTerm
26 | }
27 | }]))
28 | }
29 | }
30 | }
31 | });
32 |
33 | /**
34 | * Get significant words for a `term` in the input `field`
35 | */
36 | export const getSuggestionsQuery = (field, term) => ({
37 | size: 0,
38 | query: {
39 | term: {
40 | [field]: term
41 | }
42 | },
43 | aggs: {
44 | [field]: {
45 | significant_text: {
46 | field,
47 | // TODO include/exclude according to ES version
48 | include: stopWords,
49 | min_doc_count: 1,
50 | size: 20
51 | }
52 | }
53 | }
54 | });
55 |
--------------------------------------------------------------------------------
/fe/src/lib/app/machines/builder/tester.options.js:
--------------------------------------------------------------------------------
1 | import { assign } from 'xstate';
2 |
3 | export const builderTesterOptions = {
4 | actions: {
5 | toggleAutoExecute: assign({
6 | autoExecute: ctx => !ctx.autoExecute
7 | }),
8 | toggleCached: assign({
9 | cached: ctx => !ctx.cached
10 | }),
11 | toggleHideDisabledAxes: assign({
12 | hideDisabledAxes: ctx => !ctx.hideDisabledAxes
13 | }),
14 | toggleHideDisabledAggs: assign({
15 | hideDisabledAggs: ctx => !ctx.hideDisabledAggs
16 | }),
17 | toggleHideDisabledDatasets: assign({
18 | hideDisabledDatasets: ctx => !ctx.hideDisabledDatasets
19 | }),
20 | toggleHideDisabledItems: assign({
21 | hideDisabledItems: ctx => !ctx.hideDisabledItems
22 | }),
23 | toggleShowFullResponse: assign({
24 | showFullResponse: ctx => !ctx.showFullResponse
25 | }),
26 | toggleMatching: assign({
27 | matching: ctx => !ctx.matching
28 | }),
29 | toggleSelectionComplete: assign({
30 | selectionComplete: ctx => !ctx.selectionComplete
31 | }),
32 | toggleQueryReady: assign({
33 | queryReady: ctx => !ctx.queryReady
34 | })
35 | },
36 | guards: {
37 | isAutoExecute: ctx => ctx.autoExecute,
38 | isInCache: ctx => ctx.cached,
39 | isMatching: ctx => ctx.matching,
40 | isSelectionComplete: ctx => ctx.selectionComplete,
41 | isQueryReady: ctx => ctx.queryReady
42 | }
43 | };
44 |
--------------------------------------------------------------------------------
/fe/src/lib/app/components/elementary/TabContainer.svelte:
--------------------------------------------------------------------------------
1 |
4 |
5 |
28 |
29 |
30 |
31 |
33 |
34 |
36 |
6 |
7 |
8 | {#each sources as {source, releases}}
9 |
10 |
11 | {source}
12 |
13 |
14 | {#each releases as {id}}
15 |
16 |
19 | {id}
20 |
21 |
22 | {/each}
23 |
24 |
25 | {/each}
26 |
27 |
28 |
31 |
32 |
33 |
76 |
--------------------------------------------------------------------------------
/fe/src/lib/app/components/elementary/RadioList.svelte:
--------------------------------------------------------------------------------
1 |
27 |
28 |
33 | {#if isNil(defaultValue)}
34 |
35 | Unset
43 |
44 | {/if}
45 | {#each typeObject as item}
46 |
47 | {item}
55 |
56 | {/each}
57 |
58 |
59 |
73 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/ref/typeGroups.js:
--------------------------------------------------------------------------------
1 | import {
2 | date_YYYYMMDD_dash_time,
3 | date_YYYYMMDD_dash,
4 | date_YYYYMMDD_T_dash_time_ms,
5 | date,
6 | geoPointObject,
7 | keyword,
8 | keywordArray,
9 | text,
10 | textArray,
11 | textWithKeyword,
12 | textWithKeywordArray,
13 | } from '$lib/elasticsearch/types/fields.js';
14 | import {
15 | boolean,
16 | float,
17 | integer,
18 | unionOf,
19 | } from '$lib/types/index.js';
20 |
21 | export const esDates = unionOf(
22 | date,
23 | date_YYYYMMDD_dash,
24 | date_YYYYMMDD_dash_time,
25 | date_YYYYMMDD_T_dash_time_ms
26 | );
27 |
28 | export const esNumeric = unionOf(
29 | boolean,
30 | date,
31 | date_YYYYMMDD_dash,
32 | date_YYYYMMDD_dash_time,
33 | float,
34 | integer,
35 | );
36 |
37 | export const esNumericButBoolean = unionOf(
38 | date,
39 | date_YYYYMMDD_dash,
40 | date_YYYYMMDD_dash_time,
41 | float,
42 | integer,
43 | );
44 |
45 | export const esSearchableField = unionOf(
46 | boolean,
47 | date_YYYYMMDD_dash_time,
48 | date_YYYYMMDD_dash,
49 | date,
50 | float,
51 | geoPointObject,
52 | integer,
53 | keyword,
54 | keywordArray,
55 | textWithKeyword,
56 | textWithKeywordArray,
57 | );
58 |
59 | export const esSearchableString = unionOf(
60 | keyword,
61 | keywordArray,
62 | textWithKeyword,
63 | textWithKeywordArray,
64 | );
65 |
66 | export const esStrings = unionOf(
67 | keyword,
68 | keywordArray,
69 | text,
70 | textArray,
71 | textWithKeyword,
72 | textWithKeywordArray,
73 | );
74 |
75 | // TODO at some point we might need to use
76 | // https://www.elastic.co/guide/en/elasticsearch/reference/7.9/mapping-types.html
77 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/utils/aggQuery.js:
--------------------------------------------------------------------------------
1 | import * as _ from 'lamb';
2 |
3 | import aggsIdByFieldType from '$lib/elasticsearch/config/aggsIdByFieldType.js';
4 | import {getESType} from '$lib/utils/specs.js';
5 |
6 | const hasKeywordField = _.anyOf([
7 | _.is('textWithKeyword'),
8 | _.is('textWithKeywordArray'),
9 | ]);
10 |
11 | const createKeywordFieldQuery = fieldName => ({field: `${fieldName}.keyword`});
12 |
13 | function defaultAggregationFactory (fieldName, fieldSpec) {
14 | const fieldType = getESType(fieldSpec);
15 | if (hasKeywordField(fieldType)) {
16 | return createKeywordFieldQuery(fieldName);
17 | }
18 | return { field: fieldName };
19 | }
20 |
21 | const queryFactories = {
22 | // composite: () => ({}),
23 | // filter: () => ({}),
24 | // filters: () => ({}),
25 | // sampler: () => ({}),
26 | // eslint-disable-next-line no-unused-vars
27 | weighted_avg: () => ({})
28 | }
29 |
30 | export function buildAggregation (aggId, fieldName, fieldSpec) {
31 | if (aggId in queryFactories) {
32 | return queryFactories[aggId](fieldName, fieldSpec);
33 | }
34 |
35 | return defaultAggregationFactory(fieldName, fieldSpec);
36 | }
37 |
38 | export function constructQuery (schema) {
39 | const aggs = {};
40 | for (let fieldName in schema) {
41 | const fieldType = getESType(schema[fieldName]);
42 | const aggIds = aggsIdByFieldType[fieldType];
43 | for (let i in aggIds) {
44 | const aggId = aggIds[i];
45 | const aggKey = `${fieldName}_${aggId}`;
46 | aggs[aggKey] = {
47 | [aggId]: buildAggregation(aggId, fieldName, schema[fieldName])
48 | };
49 | }
50 | }
51 |
52 | return {
53 | size: 0,
54 | aggs
55 | };
56 | }
57 |
--------------------------------------------------------------------------------
/specs/responses/examples/terms/response.json:
--------------------------------------------------------------------------------
1 | {
2 | "primary": {
3 | "doc_count_error_upper_bound": 5949,
4 | "sum_other_doc_count": 2704253,
5 | "buckets": [
6 | {
7 | "key": "",
8 | "doc_count": 967649
9 | },
10 | {
11 | "key": "Humans",
12 | "doc_count": 115416
13 | },
14 | {
15 | "key": "Animals",
16 | "doc_count": 77388
17 | },
18 | {
19 | "key": "Goals",
20 | "doc_count": 50954
21 | },
22 | {
23 | "key": "Mice",
24 | "doc_count": 31962
25 | },
26 | {
27 | "key": "Female",
28 | "doc_count": 23720
29 | },
30 | {
31 | "key": "United States",
32 | "doc_count": 17385
33 | },
34 | {
35 | "key": "Male",
36 | "doc_count": 15240
37 | },
38 | {
39 | "key": "Rats",
40 | "doc_count": 14940
41 | },
42 | {
43 | "key": "Child",
44 | "doc_count": 14118
45 | },
46 | {
47 | "key": "DNA",
48 | "doc_count": 13309
49 | },
50 | {
51 | "key": "Brain",
52 | "doc_count": 12787
53 | },
54 | {
55 | "key": "Mutation",
56 | "doc_count": 10901
57 | },
58 | {
59 | "key": "Public Health",
60 | "doc_count": 10590
61 | },
62 | {
63 | "key": "Phenotype",
64 | "doc_count": 10449
65 | },
66 | {
67 | "key": "Neoplasms",
68 | "doc_count": 10443
69 | },
70 | {
71 | "key": "RNA, Messenger",
72 | "doc_count": 9693
73 | },
74 | {
75 | "key": "Signal Transduction",
76 | "doc_count": 8605
77 | },
78 | {
79 | "key": "Adult",
80 | "doc_count": 8596
81 | },
82 | {
83 | "key": "Models, Animal",
84 | "doc_count": 8412
85 | }
86 | ]
87 | }
88 | }
89 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/spec/percentile_ranks.js:
--------------------------------------------------------------------------------
1 | import {esNumeric} from '$lib/elasticsearch/aggs/ref/typeGroups.js';
2 | import {optional, script} from '$lib/elasticsearch/types/params.js';
3 | import {
4 | arrayOf,
5 | booleanD,
6 | integer,
7 | number,
8 | objectOf,
9 | string,
10 | } from '$lib/types/index.js';
11 | import {
12 | keyed,
13 | field,
14 | missing,
15 | script as scriptDoc,
16 | } from '$lib/elasticsearch/aggs/ref/requestDoc.js';
17 | import response from '$lib/elasticsearch/aggs/response/numToNum.js';
18 |
19 | export default {
20 | id: 'percentile_ranks',
21 | availability: {
22 | from: '1.3'
23 | },
24 | docPath: '/search-aggregations-metrics-percentile-rank-aggregation.html',
25 | docs: 'Calculates one or more percentile ranks over numeric values extracted from the aggregated documents.',
26 | fieldType: esNumeric,
27 | label: 'Percentile Ranks',
28 | request: { // [0]
29 | field: string,
30 | hdr: optional(objectOf({
31 | number_of_significant_value_digits: integer
32 | })),
33 | keyed: optional(booleanD(true)),
34 | missing: optional(number),
35 | script: optional(script),
36 | values: arrayOf(number),
37 | },
38 | requestDoc: {
39 | field,
40 | hdr: '(High Dynamic Range Histogram) is an alternative implementation that can be useful when calculating percentile ranks for latency measurements as it can be faster than the t-digest implementation with the trade-off of a larger memory footprint.',
41 | keyed,
42 | missing,
43 | script: scriptDoc,
44 | values: 'List of treshold values to use for ranking.',
45 | },
46 | response,
47 | tag: 'metric',
48 | version: '7.9',
49 | };
50 |
51 | // [0] 7.9: no params table
52 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/spec/scripted_metric.js:
--------------------------------------------------------------------------------
1 | import {optional, metricScript} from '$lib/elasticsearch/types/params.js';
2 | import {any, object} from '$lib/types/index.js';
3 | import response from '$lib/elasticsearch/aggs/response/value.js';
4 |
5 | export default {
6 | id: 'scripted_metric',
7 | availability: {
8 | from: '1.4'
9 | },
10 | docPath: '/search-aggregations-metrics-scripted-metric-aggregation.html',
11 | docs: 'A metric aggregation that executes using scripts to provide a metric output.',
12 | fieldType: any,
13 | label: 'Scripted Metric',
14 | request: { // [0]
15 | combine_script: metricScript,
16 | init_script: optional(metricScript),
17 | map_script: metricScript,
18 | params: optional(object),
19 | reduce_script: metricScript,
20 | },
21 | requestDoc: {
22 | combine_script: 'Executed once on each shard after document collection is complete. Allows the aggregation to consolidate the state returned from each shard.',
23 | init_script: 'Executed prior to any collection of documents. Allows the aggregation to set up any initial state.',
24 | map_script: 'Executed once per document collected.',
25 | params: 'An object whose contents will be passed as variables to the init_script, map_script and combine_script. This can be useful to allow the user to control the behavior of the aggregation and for storing state between the scripts.',
26 | reduce_script: 'Executed once on the coordinating node after all shards have returned their results. The script is provided with access to a variable states which is an array of the result of the combine_script on each shard.',
27 | },
28 | response,
29 | tag: 'metric',
30 | version: '7.9',
31 | };
32 |
33 | // [0] 7.9: no params table
34 |
--------------------------------------------------------------------------------
/be/src/hooks.js:
--------------------------------------------------------------------------------
1 | import {
2 | authenticate,
3 | parseBasicAuth
4 | } from 'dap_dv_backends_utils/auth/authentication.mjs';
5 | import * as _ from 'lamb';
6 |
7 | import {CROSS_ORIGIN_DOMAINS, PROTECTED_DATASETS} from './conf.js';
8 |
9 | const AUTH_ENDPOINT = 'https://authentication.dap-tools.uk/authenticate';
10 |
11 | const isCrossOriginRequestAuthorised = request => {
12 | const {origin} = request.headers;
13 | const isAllowed = _.reduce(
14 | CROSS_ORIGIN_DOMAINS,
15 | (acc, curr) => acc || curr.test(origin),
16 | false
17 | );
18 | return isAllowed;
19 | }
20 |
21 | export const authenticationHook = async req => {
22 |
23 | req.notAuthorised = false;
24 |
25 | if ('authorization' in req.headers) {
26 | const authHeader = req.headers.authorization;
27 | if (!authHeader.startsWith('Basic')) {
28 | req.notAuthorised = true;
29 | req.errorMessage = 'Basic Authorization Header required';
30 | }
31 | const { email, token } = parseBasicAuth(authHeader);
32 | const isAuthorized = await authenticate(AUTH_ENDPOINT, email, token);
33 | // if user is authenticated then they can access all datasets
34 | if (!isAuthorized) {
35 | req.notAuthorised = true;
36 | req.errorMessage = 'Token cannot be authenticated';
37 | }
38 | } else {
39 | // check that the non-signed request is coming from dapsboard
40 | if (!isCrossOriginRequestAuthorised(req)) {
41 | req.notAuthorised = true;
42 | req.errorMessage = 'CORS policy is blocking this request';
43 | }
44 | const urlParts = _.split(req.url, '/');
45 | const dataset = urlParts.at(-2).trim();
46 | if (_.isIn(PROTECTED_DATASETS, dataset)) {
47 | req.notAuthorised = true;
48 | req.errorMessage = 'This dataset is protected';
49 | }
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/spec/top_hits.js:
--------------------------------------------------------------------------------
1 | import {esSearchableField} from '$lib/elasticsearch/aggs/ref/typeGroups.js';
2 | import {optional, sortOrder} from '$lib/elasticsearch/types/params.js';
3 | import {
4 | arrayOf,
5 | integerD,
6 | objectOf,
7 | recordLike,
8 | string,
9 | unionOf
10 | } from '$lib/types/index.js';
11 | import response from '$lib/elasticsearch/aggs/response/hits.js';
12 |
13 | export default {
14 | id: 'top_hits',
15 | availability: {
16 | from: '1.3'
17 | },
18 | docPath: '/search-aggregations-metrics-top-hits-aggregation.html',
19 | docs: 'Keeps track of the most relevant document being aggregated. This aggregator is intended to be used as a sub aggregator, so that the top matching documents can be aggregated per bucket.',
20 | fieldType: esSearchableField, // TBD
21 | label: 'Top Hits',
22 | needsParent: true,
23 | request: { // [0]
24 | _source: optional(objectOf({
25 | includes: unionOf(string, arrayOf(string))
26 | })),
27 | from: optional(integerD(0, true)),
28 | size: optional(integerD(3)),
29 | sort: optional(arrayOf(recordLike({
30 | values: objectOf({
31 | order: sortOrder
32 | })
33 | }))),
34 | },
35 | requestDoc: {
36 | _source: 'If _source is requested then just the part of the source of the nested object is returned, not the entire source of the document.',
37 | from: 'The offset from the first result you want to fetch.',
38 | size: 'The maximum number of top matching hits to return per bucket. By default the top three matching hits are returned.',
39 | sort: 'How the top matching hits should be sorted. By default the hits are sorted by the score of the main query.',
40 | },
41 | response,
42 | tag: 'metric',
43 | version: '7.9',
44 | };
45 |
46 | // [0] 7.9: no params table
47 |
--------------------------------------------------------------------------------
/fe/src/lib/app/components/elementary/BooleanRadios.svelte:
--------------------------------------------------------------------------------
1 |
26 |
27 |
32 | {#if isNil(defaultValue)}
33 |
34 | Unset
42 |
43 | {/if}
44 |
45 | True
53 |
54 |
55 | False
63 |
64 |
65 |
66 |
80 |
--------------------------------------------------------------------------------
/fe/src/lib/app/utils/net.js:
--------------------------------------------------------------------------------
1 | import {get} from 'svelte/store';
2 |
3 | import {
4 | _credentials,
5 | _isAuthModalOpen
6 | } from '$lib/app/stores/auth.js';
7 | import {request} from '$lib/utils/net.js';
8 |
9 | const authBaseURL = 'https://api.dap-tools.uk/auth';
10 |
11 | const getTokenRequestEndpointURL = email =>
12 | `${authBaseURL}/request?email=${email}`;
13 | const getTokenVerifyEndpointURL = (email, token) =>
14 | `${authBaseURL}/authenticate?email=${email}&token=${token}`;
15 |
16 | export const authedRequest = async (
17 | method,
18 | url,
19 | options = {}
20 | ) => {
21 | const headers = {...options.headers};
22 |
23 | const credentials = get(_credentials);
24 | if (credentials?.token) {
25 | const {email, token} = credentials;
26 | headers.Authorization = `Basic ${btoa(`${email}:${token}`)}`;
27 | }
28 | let response;
29 | try {
30 | response = await request(
31 | method,
32 | url,
33 | {
34 | ...options,
35 | headers
36 | }
37 | );
38 | } catch (error) {
39 | if (error.httpStatus === 401 || error.httpStatus === 403) {
40 | _credentials.set(null);
41 | _isAuthModalOpen.set(true);
42 | }
43 | throw error;
44 | }
45 | return response;
46 | }
47 |
48 | export const requestNestaToken = async email => {
49 | const endpoint = getTokenRequestEndpointURL(email);
50 | const response = await fetch(endpoint);
51 |
52 | const result = {};
53 | if (response.status !== 204) {
54 | result.error = await response.text();
55 | }
56 |
57 | return result;
58 | }
59 |
60 | export const verifyNestaToken = async (email, token) => {
61 | const endpoint = getTokenVerifyEndpointURL(email, token);
62 | const response = await fetch(endpoint);
63 | const result = await response.text();
64 |
65 | return JSON.parse(result);
66 | }
67 |
--------------------------------------------------------------------------------
/fe/src/routes/+layout.svelte:
--------------------------------------------------------------------------------
1 |
39 |
40 |
54 |
56 |
57 | {#if $_isAuthModalOpen}
58 |
37 | {#each aggParamsInfo as paramInfo (`${id}-${paramInfo.paramId}`)}
38 | {#if !isParamSelector
39 | || ['__selection', value?.__selection].includes(paramInfo.paramId)
40 | }
41 | handleChange({[paramInfo.paramId]: event.detail})}
51 | help={false}
52 | />
53 | {/if}
54 | {/each}
55 |
56 |
57 |
63 |
--------------------------------------------------------------------------------
/fe/src/lib/app/components/elementary/PanelMenu.svelte:
--------------------------------------------------------------------------------
1 |
28 |
29 |
35 |
39 |
43 |
46 |
49 |
50 |
31 | {#each options as opt}
32 | {#if !(opt.disabled && hideDisabled) || opt.value === selectedOption}
33 | handleClick(opt.value, opt.disabled)}
38 | class='clickable'
39 | >
40 |
41 | {opt.text}
42 |
43 |
44 | {/if}
45 | {/each}
46 |
47 |
48 |
82 |
--------------------------------------------------------------------------------
/fe/src/lib/app/machines/explore/route.js:
--------------------------------------------------------------------------------
1 | import {createMachina} from '../utils.js';
2 |
3 | import {createHistoryStores} from './history.context.js';
4 | import {historyOptions} from './history.options.js';
5 | import {createResultsStores} from './results.context.js';
6 | import {resultsOptions} from './results.options.js';
7 | import {exploreConfig} from './route.config.js';
8 | import {createRouteStores} from './route.context.js';
9 | import {routeOptions} from './route.options.js';
10 | import {createSelectionStores} from './selecting.context.js';
11 | import {selectingOptions} from './selecting.options.js';
12 | import {createSearchStores} from './search.context.js';
13 | import {searchOptions} from './search.options.js';
14 |
15 | /* TODO
16 | // nm/svizzle/utils/array-[array-object].js: makeMergeBy
17 | // nm/app/machines/utils: mergeMachines
18 |
19 | const mergeMachines = makeMergeBy(['actions', 'guards', 'services']);
20 | mergeMachines([
21 | historyOptions,
22 | resultsOptions,
23 | routeOptions,
24 | selectingOptions
25 | ])
26 | */
27 | export const exploreOptions = {
28 | actions: {
29 | ...historyOptions.actions,
30 | ...resultsOptions.actions,
31 | ...routeOptions.actions,
32 | ...selectingOptions.actions,
33 | ...searchOptions.actions,
34 | },
35 | guards: {
36 | ...historyOptions.guards,
37 | ...resultsOptions.guards,
38 | ...routeOptions.guards,
39 | ...selectingOptions.guards,
40 | ...searchOptions.guards,
41 | },
42 | services: {
43 | ...resultsOptions.services,
44 | ...searchOptions.services,
45 | }
46 | };
47 |
48 | export const contextStores = {
49 | ...createHistoryStores(),
50 | ...createResultsStores(),
51 | ...createRouteStores(),
52 | ...createSelectionStores(),
53 | ...createSearchStores(),
54 | };
55 |
56 | export const createExploreMachine = () => createMachina(
57 | exploreConfig,
58 | exploreOptions,
59 | contextStores
60 | );
61 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/config/aggsLabels.js:
--------------------------------------------------------------------------------
1 | export const metricLabels = {
2 | // boxplot: 'BoxPlot',
3 | // scripted_metric: 'Scripted Metric',
4 | // string_stats: 'String Statistics',
5 | // string_stats: 'String Stats' // >= 7.6
6 | avg: 'Average',
7 | cardinality: 'Cardinality',
8 | extended_stats: 'Extended Statistics',
9 | geo_bounds: 'Geographic Bounds',
10 | geo_centroid: 'Geographic Centroid',
11 | max: 'Maximum',
12 | median_absolute_deviation: 'Median Abs. Dev.', // >= 6.6
13 | min: 'Minimum',
14 | percentile_ranks: 'Percentile Ranks',
15 | percentiles: 'Percentiles',
16 | stats: 'Statistics',
17 | sum: 'Sum',
18 | top_hits: 'Top Hits',
19 | value_count: 'Value Count'
20 | }
21 |
22 | export const metricMultiFieldLabels = {
23 | // matrix_stats: 'Matrix Stats',
24 | weighted_avg: 'Weighted Average',
25 | }
26 |
27 | export const topBucketLabels = {
28 | // global: 'Global'
29 | }
30 |
31 | export const bucketLabels = {
32 | // auto_date_histogram: 'Auto Date Histogram' // >= 6.5
33 | // children: 'Children',
34 | auto_date_histogram: 'Auto Date Histogram',
35 | date_histogram: 'Date Histogram',
36 | date_range: 'Date Range',
37 | geo_distance: 'Geo Distance',
38 | geohash_grid: 'Geo Hash',
39 | histogram: 'Histogram',
40 | // ip_range: 'IP Range',
41 | missing: 'Missing',
42 | nested: 'Nested',
43 | range: 'Range',
44 | rare_terms: 'Rare Terms',
45 | significant_terms: 'Significant Terms',
46 | significant_text: 'Significant Text',
47 | terms: 'Terms',
48 | }
49 |
50 | // needs to have sub-aggs to work
51 | export const bucketParentLabels = {
52 | sampler: 'Sampler',
53 | }
54 | export const bucketMultiFieldLabels = {
55 | // composite: 'Composite',
56 | // filter: 'Filter',
57 | // filters: 'Filters',
58 | }
59 |
60 | export const nestedBucketLabels = {
61 | diversified_sampler: 'Diversified Sampler',
62 | reverse_nested: 'Reverse Nested',
63 | }
64 |
--------------------------------------------------------------------------------
/fe/src/lib/app/machines/explore/route.options.js:
--------------------------------------------------------------------------------
1 | import {actions, assign} from 'xstate';
2 |
3 | import {stringifyContextStores} from '$lib/app/machines/utils.js';
4 | import {
5 | resetSources,
6 | selectSource,
7 | selectDataset,
8 | } from '$lib/app/stores/exploreStores.js';
9 | import {getDatasetOf} from '$lib/app/utils/data.js';
10 | import {getBeSearchURL} from '$lib/utils/specs.js';
11 |
12 | const {choose, log} = actions;
13 |
14 | const setDataset = (ctx, {project, source, version}) => {
15 | ctx.dataset.set({project, source, version});
16 |
17 | return ctx;
18 | }
19 |
20 | const setNeededFields = (ctx, {neededFields}) => {
21 | ctx._neededFields.set(neededFields?.split(',') || []);
22 |
23 | return ctx;
24 | }
25 |
26 | const setURL = (ctx, {project, source, version}) => {
27 | const dataset = getDatasetOf({project, source, version});
28 | const queryURL = getBeSearchURL(dataset);
29 | ctx.queryURL.set(queryURL);
30 |
31 | return ctx;
32 | }
33 |
34 | const loggedEvents = [
35 | // 'xstate.init',
36 | // 'QUERY_UPDATED',
37 | // 'SEARCH_QUERY_UPDATED'
38 | ];
39 |
40 | export const routeOptions = {
41 | actions: {
42 | conditionalLog: choose([
43 | {
44 | cond: 'logGuard',
45 | actions: [
46 | log(
47 | (context, event) => `---- context: ${stringifyContextStores(context)}\n\n---- event: ${JSON.stringify(event)}`,
48 | 'conditionalLog'
49 | )
50 | ]
51 | },
52 | ]),
53 | setDataset: assign(setDataset),
54 | setNeededFields: assign(setNeededFields),
55 | setURL: assign(setURL),
56 | resetSources: () => {
57 | resetSources()
58 | },
59 | selectDataset: (ctx, {project, source, version}) => {
60 | selectDataset({project, source, version})
61 | },
62 | selectSource: (ctx, {source}) => {
63 | selectSource(source)
64 | },
65 | },
66 | guards: {
67 | logGuard: (context, event) => loggedEvents.includes(event.type)
68 | }
69 | };
70 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/test/check_exports.spec.js:
--------------------------------------------------------------------------------
1 | import assert from 'node:assert';
2 | import path from 'node:path';
3 | import {fileURLToPath} from 'node:url';
4 |
5 | import * as _ from 'lamb';
6 | import {readDir, readFile} from '@svizzle/file';
7 | import {
8 | isIterableNotEmpty,
9 | makeEndsWith,
10 | makePrefixed,
11 | makeStartsWith,
12 | sliceStringAt,
13 | } from '@svizzle/utils';
14 |
15 | const __dirname = path.dirname(fileURLToPath(import.meta.url));
16 |
17 | const SRC_DIR_PATH = path.resolve(__dirname, '../spec/');
18 | const INDEX_PATH = path.resolve(SRC_DIR_PATH, 'index.js');
19 | const REGEX = /export \{default as .*\} from '\.\/(.*).js';\n/ug;
20 |
21 | const isAggModule = _.allOf([
22 | makeEndsWith('.js'),
23 | _.not(makeStartsWith('_')),
24 | _.not(makeStartsWith('index')),
25 | _.not(makeEndsWith('.todo.js')),
26 | _.not(makeEndsWith('.spec.js')),
27 | ]);
28 |
29 | describe('elasticsearch/aggs/spec/: exports', function () {
30 | it('All agg valid modules are exported', async function () {
31 | const leftovers = await Promise.all([
32 | readDir(SRC_DIR_PATH).then(_.pipe([
33 | _.filterWith(isAggModule),
34 | _.mapWith(sliceStringAt([0, -3])),
35 | _.sortWith([])
36 | ])),
37 |
38 | readFile(INDEX_PATH, 'utf-8')
39 | .then(srcIndex => [...srcIndex.matchAll(REGEX)].map(_.getAt(1)))
40 | .then(_.sortWith([])),
41 | ])
42 | .then(([modules, exported]) => _.pullFrom(modules, exported))
43 | .catch(err => console.error(err));
44 |
45 | const gotDupes = isIterableNotEmpty(leftovers);
46 | if (gotDupes) {
47 | const list = leftovers.map(makePrefixed('- ')).join('\n');
48 | console.log(`\n======================\n✋`)
49 | console.log(`elasticsearch/aggs/spec/index.js not exporting modules with these filenames:\n\n${list}`)
50 | console.log(`======================\n`)
51 | }
52 |
53 | assert.deepStrictEqual(gotDupes, false);
54 | });
55 | });
56 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/spec/geo_distance.js:
--------------------------------------------------------------------------------
1 | import {geoPoint} from '$lib/elasticsearch/types/fields.js';
2 | import {optional} from '$lib/elasticsearch/types/params.js';
3 | import {
4 | arrayOf,
5 | booleanD,
6 | enumsOf,
7 | number,
8 | objectOf,
9 | string,
10 | someOf,
11 | unionOf,
12 | } from '$lib/types/index.js';
13 | import {field, keyed} from '$lib/elasticsearch/aggs/ref/requestDoc.js';
14 | import response from '$lib/elasticsearch/aggs/response/bucketsRange.js';
15 |
16 | export default {
17 | id: 'geo_distance',
18 | availability: {
19 | from: '1.3'
20 | },
21 | docPath: '/search-aggregations-bucket-geodistance-aggregation.html',
22 | docs: 'A multi-bucket aggregation that works on geo_point fields and conceptually works very similar to the range aggregation. The user can define a point of origin and a set of distance range buckets.',
23 | fieldType: unionOf(geoPoint, arrayOf(geoPoint)),
24 | label: 'Geo Distance',
25 | request: { // [0]
26 | distance_type: enumsOf(['arc', 'plane'], 'arc'),
27 | field: string,
28 | keyed: optional(booleanD(false)),
29 | origin: string,
30 | ranges: arrayOf(objectOf({
31 | key: optional(string),
32 | __extent: someOf({
33 | from: number,
34 | to: number,
35 | }),
36 | })),
37 | unit: enumsOf(['m', 'mi', 'in', 'yd', 'km', 'cm', 'mm'], 'm'), // TODO make a type
38 | },
39 | requestDoc: {
40 | distance_type: 'The distance calculation type can be set using the distance_type parameter.',
41 | field,
42 | keyed,
43 | origin: 'The `origin` point can accept all formats supported by the `geo_point` type.',
44 | ranges: 'Array of objects of shape {from: string, to: string}',
45 | unit: 'By default, the distance unit is m (meters) but it can also accept: mi (miles), in (inches), yd (yards), km (kilometers), cm (centimeters), mm (millimeters).',
46 | },
47 | response,
48 | subAggs: true,
49 | tag: 'bucketing',
50 | version: '7.9',
51 | };
52 |
53 | // [0] 7.9: no params table
54 |
--------------------------------------------------------------------------------
/fe/src/lib/app/machines/utils.js:
--------------------------------------------------------------------------------
1 | import {interpret, Machine} from 'xstate';
2 | import {get, writable} from 'svelte/store';
3 | import * as _ from 'lamb';
4 | import {stringify} from '@svizzle/utils';
5 |
6 | export function bindToStore (interpreter) {
7 | const machineStore = writable(interpreter.initialState, () => {
8 | // service.start();
9 | // Unsubscribing here works for machines subcribed to only once,
10 | // but can wreak havoc if subcsriptions are changiong all the time.
11 | // TODO Perhaps implement reference counting or remove if intended
12 | // for more general use.
13 | // return () => service.stop();
14 | });
15 |
16 | interpreter.onTransition(nextState => {
17 | machineStore.set(nextState);
18 | });
19 |
20 | return {
21 | subscribe: machineStore.subscribe,
22 | send: interpreter.send,
23 | };
24 | }
25 |
26 | /**
27 | * Creates an xstate machine and places it into a svelte store.
28 | * Also creates a svelte store to be passed to the machine as context
29 | *
30 | * @param machineConfig - A valid xstate configuration object
31 | * @param machineOptions - A valid xstate options object: guards, actions, services, activities
32 | * @param contextStores - The default store(s) to be passed to xstate as context
33 | * @returns An object containing two objects. One being a store containing the xstate machine, the other containing the stores passed in as context.
34 | */
35 | export function createMachina (machineConfig, machineOptions, contextStores) {
36 | const machine = Machine({
37 | ...machineConfig,
38 | context: contextStores
39 | }, machineOptions);
40 |
41 | // const interpreter = interpret(machine, { devTools: process.env.INSPECT === 'true' });
42 | const interpreter = interpret(machine, { devTools: false });
43 |
44 | interpreter.start();
45 |
46 | return {
47 | machine: bindToStore(interpreter),
48 | contextStores
49 | };
50 | }
51 |
52 |
53 | export const stringifyContextStores = _.pipe([_.mapValuesWith(get), stringify]);
54 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/spec/percentiles.js:
--------------------------------------------------------------------------------
1 | import {esNumericButBoolean} from '$lib/elasticsearch/aggs/ref/typeGroups.js';
2 | import {optional, script} from '$lib/elasticsearch/types/params.js';
3 | import {
4 | arrayOf,
5 | booleanD,
6 | float,
7 | integer,
8 | integerD,
9 | objectOf,
10 | string,
11 | number
12 | } from '$lib/types/index.js';
13 | import {
14 | field,
15 | keyed,
16 | missing,
17 | script as scriptDoc,
18 | } from '$lib/elasticsearch/aggs/ref/requestDoc.js';
19 | import response from '$lib/elasticsearch/aggs/response/numToNum.js';
20 |
21 | export default {
22 | id: 'percentiles',
23 | availability: {
24 | from: '1.3'
25 | },
26 | docPath: '/search-aggregations-metrics-percentile-aggregation.html',
27 | docs: 'Calculates one or more percentiles over numeric values extracted from the aggregated documents.',
28 | fieldType: esNumericButBoolean,
29 | label: 'Percentiles',
30 | request: { // [0]
31 | field: string,
32 | hdr: optional(objectOf({
33 | number_of_significant_value_digits: integer
34 | })),
35 | keyed: optional(booleanD(true)),
36 | missing: optional(number), // [1]
37 | percents: optional(arrayOf(float, [1, 5, 25, 50, 75, 95, 99])),
38 | script: optional(script),
39 | tdigest: optional(objectOf({
40 | compression: integerD(100)
41 | })),
42 | },
43 | requestDoc: {
44 | field,
45 | hdr: 'hdr object indicates that HDR Histogram should be used to calculate the percentiles and specific settings for this algorithm can be specified inside the object. Format: {number_of_significant_value_digits: integer}',
46 | keyed,
47 | missing,
48 | percents: 'List of treshold values to use for ranking.',
49 | script: scriptDoc,
50 | tdigest: 'Format: {compression: integer}: The compression parameter limits the maximum number of nodes to 20 * compression.',
51 | },
52 | response,
53 | tag: 'metric',
54 | version: '7.9',
55 | };
56 |
57 | // [0] 7.9: no params table
58 | // [1] TODO add constraint to be same type as the field type
59 |
--------------------------------------------------------------------------------
/fe/src/lib/app/machines/explore/results.config.js:
--------------------------------------------------------------------------------
1 | export const resultsConfig = {
2 | id: 'Results',
3 | initial: 'Idle',
4 | on: {
5 | AGGS_HIERARCHY_UPDATED: {
6 | target: '#Results.CheckMatching'
7 | }
8 | },
9 | states: {
10 | Idle: {
11 | },
12 | CheckMatching: {
13 | entry: ['conditionalLog'],
14 | always: [
15 | {
16 | target: 'Matching',
17 | cond: 'isMatching'
18 | },
19 | { target: 'Dirty' }
20 | ]
21 | },
22 | Matching: {
23 | id: 'Matching',
24 | entry: [
25 | 'conditionalLog',
26 | 'sendResultsUpdated'
27 | ],
28 | },
29 | Dirty: {
30 | id: 'Dirty',
31 | initial: 'CheckingCache',
32 | entry: ['conditionalLog'],
33 | states: {
34 | CheckingCache: {
35 | entry: [
36 | 'conditionalLog',
37 | 'setCacheKey',
38 | ],
39 | always: [
40 | {
41 | target: '#Matching',
42 | cond: 'isInCache',
43 | actions: ['loadFromCache']
44 | },
45 | {
46 | target: 'UpdatingQueue',
47 | actions: ['initQueue']
48 | }
49 | ]
50 | },
51 | UpdatingQueue: {
52 | entry: [
53 | 'conditionalLog',
54 | 'updateQueue',
55 | ],
56 | always: [
57 | {
58 | target: 'Pending',
59 | cond: 'hasQueuedAggs',
60 | },
61 | {
62 | target: '#Matching',
63 | actions: [
64 | 'updateCache',
65 | 'updateCurrentResults',
66 | 'resetQueue',
67 | ]
68 | },
69 | ]
70 | },
71 | Pending: {
72 | entry: ['conditionalLog'],
73 | invoke: {
74 | id: 'request',
75 | src: 'apiRequest',
76 | onDone: {
77 | target: 'UpdatingQueue',
78 | actions: [
79 | 'updateQueueResults',
80 | ]
81 | },
82 | onError: {
83 | target: 'Error',
84 | }
85 | }
86 | },
87 | Error: {
88 | id: 'Error',
89 | entry: [
90 | 'conditionalLog',
91 | 'getError'
92 | ],
93 | }
94 | }
95 | }
96 | }
97 | };
98 |
--------------------------------------------------------------------------------
/fe/src/lib/app/machines/builder/formediting.config.js:
--------------------------------------------------------------------------------
1 | export const formEditingConfig = {
2 | initial: 'Idle',
3 | states: {
4 | Idle: {
5 | on: {
6 | AUTO_EXEC_TOGGLED: {
7 | target: 'Idle',
8 | actions: ['toggleAutoExecute','log']
9 | },
10 | DATASET_CHANGED: {
11 | target: 'Idle',
12 | actions: ['setDataset']
13 | },
14 | FORM_ADDED: {
15 | target: 'Idle',
16 | actions: ['spawnNestedForm']
17 | },
18 | SELECTION_COMPLETED: {
19 | target: 'Idle',
20 | actions: ['configureSubforms']
21 | },
22 | SELECTION_INCOMPLETE: {
23 | target: 'Idle',
24 | actions: ['deleteNestedForms']
25 | },
26 | FORM_SELECTED: {
27 | target: 'Idle',
28 | actions: [
29 | 'selectForm',
30 | 'sendEdited'
31 | ]
32 | },
33 | HIDE_DISABLED_FORMS_TOGGLED: {
34 | target: 'Idle',
35 | actions: ['toggleHideDisabledForms']
36 | },
37 | HIDE_DISABLED_AGGS_TOGGLED: {
38 | target: 'Idle',
39 | actions: ['toggleHideDisabledAggs']
40 | },
41 | HIDE_DISABLED_DSETS_TOGGLED: {
42 | target: 'Idle',
43 | actions: ['toggleHideDisabledDatasets']
44 | },
45 | HIDE_DISABLED_FIELDS_TOGGLED: {
46 | target: 'Idle',
47 | actions: ['toggleHideDisabledFields']
48 | },
49 | REQUEST_TAB_SELECTED: {
50 | target: 'Idle',
51 | actions: ['setTab']
52 | },
53 | RESULT_SIZE_CHANGED: {
54 | target: 'Idle',
55 | actions: ['setResultSize']
56 | },
57 | SHOW_FULL_RESPONSE_TOGGLED: {
58 | target: 'Idle',
59 | actions: ['toggleShowFullResponse']
60 | },
61 | TYPINGS_CHANGED: {
62 | target: 'Idle',
63 | actions: ['setTypings']
64 | },
65 | PARSING_START: {
66 | target: 'Idle',
67 | actions: ['setParsing']
68 | },
69 | PARSING_DONE: {
70 | target: 'Idle',
71 | actions: ['setParsingDone']
72 | },
73 | FORM_CHANGED: {
74 | target: 'Idle',
75 | actions: ['sendTreeChanged']
76 | }
77 | }
78 | }
79 | }
80 | };
81 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/spec/geohash_grid.js:
--------------------------------------------------------------------------------
1 | import {geoBounds, geoPointObject} from '$lib/elasticsearch/types/fields.js';
2 | import {optional} from '$lib/elasticsearch/types/params.js';
3 | import {
4 | integer,
5 | integerD,
6 | intWithin,
7 | string
8 | } from '$lib/types/index.js';
9 | import {field} from '$lib/elasticsearch/aggs/ref/requestDoc.js';
10 | import response from '$lib/elasticsearch/aggs/response/bucketsGeohashGrid.js';
11 |
12 | export default {
13 | id: 'geohash_grid',
14 | availability: {
15 | from: '1.3'
16 | },
17 | docPath: '/search-aggregations-bucket-geohashgrid-aggregation.html',
18 | docs: 'A multi-bucket aggregation that works on `geo_point` fields and groups points into buckets that represent cells in a grid. The resulting grid can be sparse and only contains cells that have matching data.',
19 | fieldType: geoPointObject,
20 | label: 'Geohash Grid',
21 | request: { // [1]
22 | field: string,
23 | bounds: optional(geoBounds),
24 | precision: optional(intWithin([0, 12], 5)),
25 | shard_size: optional(integer), // [2]
26 | size: optional(integerD(10000)),
27 | },
28 | requestDoc: {
29 | field,
30 | bounds: 'The bounding box to filter the points in the bucket.',
31 | precision: 'The string length of the geohashes used to define cells/buckets in the results. Defaults to 5.',
32 | shard_size: 'To allow for more accurate counting of the top cells returned in the final result the aggregation defaults to returning max(10,(size x number-of-shards)) buckets from each shard. If this heuristic is undesirable, the number considered from each shard can be over-ridden using this parameter.',
33 | size: 'The maximum number of geohash buckets to return (defaults to 10,000).',
34 | },
35 | response,
36 | subAggs: true,
37 | tag: 'bucketing',
38 | version: '7.9',
39 | };
40 |
41 | // [1] params table: https://www.elastic.co/guide/en/elasticsearch/reference/7.10/search-aggregations-bucket-geohashgrid-aggregation.html#_options_3
42 | // [2] ES default is -> max(10,(size x number-of-shards)) buckets from each shard.
43 |
--------------------------------------------------------------------------------
/fe/src/lib/app/machines/builder/history.options.js:
--------------------------------------------------------------------------------
1 | import * as _ from 'lamb';
2 | import { isObjNotEmpty, isNotNil } from '@svizzle/utils';
3 | import { get } from 'svelte/store';
4 | import { RISON } from 'rison2';
5 |
6 | const pickNonNil = _.pickIf(isNotNil);
7 |
8 | function formData (machine) {
9 | const data = {
10 | name: machine.context.id,
11 | }
12 | const selection = get(machine.context.selection);
13 |
14 | if (selection && isObjNotEmpty(selection)) {
15 | const result = pickNonNil(selection);
16 | isObjNotEmpty(result) && (data.selection = result);
17 | }
18 | const params = get(machine.context.params);
19 | params && isObjNotEmpty(params) && (data.params = params);
20 | return data;
21 | }
22 |
23 | function collectConfiguration (ctx) {
24 | const config = {
25 | size: get(ctx.resultSize),
26 | form: get(get(ctx.selectedForm).machine).context.id,
27 | forms: get(ctx.forms).map(form => formData(get(form.machine))),
28 | }
29 |
30 | const dataset = get(ctx.dataset);
31 | dataset && (config.dataset = dataset);
32 | return config;
33 | }
34 |
35 | function updateEntry (ctx, event) {
36 | if (get(ctx.isParsing)) {
37 | return;
38 | }
39 | const query = RISON.stringify(collectConfiguration(ctx));
40 | const url = `${window.location.pathname}?q=${query}`;
41 | let updateType = event.init
42 | ? 'pushState'
43 | : 'replaceState';
44 | if (url === ctx.lastGoodURL) {
45 | updateType = 'replaceState';
46 | }
47 | history[updateType](null, window.title, url);
48 | }
49 |
50 | export const historyOptions = {
51 | actions: {
52 | /**
53 | * When editing starts reserves a history slot for the session
54 | */
55 | initEntry: ctx => updateEntry(ctx, {init: true}),
56 | /**
57 | * Updates the history slot on edits
58 | */
59 | updateEntry,
60 | commitLastGoodURL: ctx => {
61 | ctx.lastGoodURL = location.pathname + location.search;
62 | }
63 | },
64 | guards: {
65 | isActiveForm: (ctx, event) => {
66 | const selectedForm = get(ctx.selectedForm);
67 | return selectedForm && selectedForm.id === event.formId;
68 | }
69 | }
70 | };
71 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/spec/weighted_avg.js:
--------------------------------------------------------------------------------
1 | import {esNumericButBoolean} from '$lib/elasticsearch/aggs/ref/typeGroups.js';
2 | import {optional, script} from '$lib/elasticsearch/types/params.js';
3 | import {
4 | object,
5 | objectOf,
6 | string,
7 | number
8 | } from '$lib/types/index.js';
9 | import {missing, script as scriptDoc} from '$lib/elasticsearch/aggs/ref/requestDoc.js';
10 | import response from '$lib/elasticsearch/aggs/response/value.js';
11 |
12 | export default {
13 | id: 'weighted_avg',
14 | availability: {
15 | from: '6.4'
16 | },
17 | docPath: '/search-aggregations-metrics-weight-avg-aggregation.html',
18 | docs: 'Computes the weighted average of numeric values that are extracted from the aggregated documents.',
19 | fieldType: esNumericButBoolean,
20 | label: 'Weighted Average',
21 | request: { // [0]
22 | format: optional(object),
23 | value_type: optional(object),
24 | value: objectOf({
25 | field: string,
26 | missing: optional(number), // [1]
27 | script: optional(script),
28 | }),
29 | weight: objectOf({
30 | field: string,
31 | missing: optional(number), // [1]
32 | script: optional(script),
33 | }),
34 | },
35 | requestDoc: {
36 | format: 'The numeric response formatter.',
37 | value_type: 'A hint about the values for pure scripts or unmapped fields.',
38 | value: 'The configuration for the field or script that provides the values',
39 | 'value.field': 'The field that values should be extracted from.',
40 | 'value.missing': missing,
41 | 'value.script': scriptDoc,
42 | weight: 'The configuration for the field or script that provides the weights',
43 | 'weight.field': 'The field that values should be extracted from.',
44 | 'weight.missing': missing,
45 | 'weight.script': scriptDoc,
46 | },
47 | response,
48 | tag: 'metric',
49 | version: '7.9',
50 | };
51 |
52 | // [0] 7.9: params table at https://www.elastic.co/guide/en/elasticsearch/reference/7.9/search-aggregations-metrics-weight-avg-aggregation.html#search-aggregations-metrics-weight-avg-aggregation
53 | // [1] TODO add constraint to be same type as the field type
54 |
--------------------------------------------------------------------------------
/fe/src/lib/app/components/svizzle/InputWidget.svelte:
--------------------------------------------------------------------------------
1 |
39 |
40 |
60 |
61 |
88 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/spec/t_test.js:
--------------------------------------------------------------------------------
1 | import {esNumeric} from '$lib/elasticsearch/aggs/ref/typeGroups.js';
2 | import {optional, script} from '$lib/elasticsearch/types/params.js';
3 | import {
4 | enumsOf,
5 | object,
6 | objectOf,
7 | string
8 | } from '$lib/types/index.js';
9 | import response from '$lib/elasticsearch/aggs/response/value.js';
10 |
11 | export default {
12 | id: 't_test',
13 | availability: {
14 | from: '7.8'
15 | },
16 | docPath: '/search-aggregations-metrics-ttest-aggregation.html',
17 | docs: 'Performs a statistical hypothesis test in which the test statistic follows a Student’s t-distribution under the null hypothesis on numeric values.',
18 | fieldType: esNumeric,
19 | label: 'T-test',
20 | request: { // [0]
21 | a: objectOf({
22 | field: string,
23 | filter: optional(object),
24 | script: optional(script),
25 | }),
26 | b: objectOf({
27 | field: string,
28 | filter: optional(object),
29 | script: optional(script),
30 | }),
31 | type: enumsOf(['paired', 'homoscedastic', 'heteroscedastic'])
32 | },
33 | requestDoc: {
34 | a: 'First test.',
35 | 'a.field': 'Field of numeric type for the first test.',
36 | 'a.filter': 'Filter for the first test.',
37 | 'a.script': 'Script for the first test.',
38 | b: 'Second test.',
39 | 'b.field': 'First field of numeric type for the second test.',
40 | 'b.filter': 'Filter for the second test.',
41 | 'c.script': 'Script for the first test.',
42 | type: 'The type of the test can be specified using the type parameter.',
43 | },
44 | response: {
45 | ...response,
46 | doc: {
47 | value: 'The probability value for the test.'
48 | },
49 | docLong: {
50 | value: 'It is the probability of obtaining results at least as extreme as the result processed by the aggregation, assuming that the null hypothesis is correct (which means there is no difference between population means). Smaller p-value means the null hypothesis is more likely to be incorrect and population means are indeed different.'
51 | }
52 | },
53 | tag: 'metric',
54 | version: '7.9',
55 | };
56 |
57 | // [0] 7.9: no params table
58 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/spec/index.js:
--------------------------------------------------------------------------------
1 | export {default as auto_date_histogram} from './auto_date_histogram.js';
2 | export {default as avg} from './avg.js';
3 | export {default as boxplot} from './boxplot.js';
4 | export {default as cardinality} from './cardinality.js';
5 | export {default as date_histogram} from './date_histogram.js';
6 | export {default as date_range} from './date_range.js';
7 | export {default as extended_stats} from './extended_stats.js';
8 | export {default as geo_bounds} from './geo_bounds.js';
9 | export {default as geo_centroid} from './geo_centroid.js';
10 | export {default as geo_distance} from './geo_distance.js';
11 | export {default as geohash_grid} from './geohash_grid.js';
12 | export {default as geotile_grid} from './geotile_grid.js';
13 | export {default as histogram} from './histogram.js';
14 | export {default as max} from './max.js';
15 | export {default as median_absolute_deviation} from './median_absolute_deviation.js';
16 | export {default as min} from './min.js';
17 | export {default as missing} from './missing.js';
18 | export {default as percentile_ranks} from './percentile_ranks.js';
19 | export {default as percentiles} from './percentiles.js';
20 | export {default as range} from './range.js';
21 | export {default as rare_terms} from './rare_terms.js';
22 | export {default as rate} from './rate.js';
23 | export {default as scripted_metric} from './scripted_metric.js';
24 | export {default as significant_terms} from './significant_terms.js';
25 | export {default as significant_text} from './significant_text.js';
26 | export {default as stats} from './stats.js';
27 | export {default as string_stats} from './string_stats.js';
28 | export {default as sum} from './sum.js';
29 | export {default as t_test} from './t_test.js';
30 | export {default as terms} from './terms.js';
31 | export {default as top_hits} from './top_hits.js';
32 | export {default as top_metrics} from './top_metrics.js';
33 | export {default as value_count} from './value_count.js';
34 | export {default as variable_width_histogram} from './variable_width_histogram.js';
35 | export {default as weighted_avg} from './weighted_avg.js';
36 |
--------------------------------------------------------------------------------
/fe/src/lib/app/machines/builder/history.test.md:
--------------------------------------------------------------------------------
1 | # URL sequences for testing history managemnt
2 |
3 | 1.
4 | - /
5 | - /builder
6 | - /builder?q=(dataset:mosaic_nih_v7,form:primary,forms:!((name:primary,params:(interval:4),selection:(aggregation:histogram,field:year_fiscal_funding,type:integer)),(name:secondary)),size:0)
7 | - BACK:
8 | - /builder
9 |
10 | 2.
11 | - /builder
12 | - /builder?q=(dataset:mosaic_nih_v5,form:primary,forms:!((name:primary,selection:(aggregation:histogram,type:integer))),size:0)
13 | - BACK:
14 | - /builder
15 | - FWD:
16 | - /builder?q=(dataset:mosaic_nih_v5,form:primary,forms:!((name:primary,selection:(aggregation:histogram,type:integer))),size:0)
17 | - Click on a field:
18 | - /builder?q=(dataset:mosaic_nih_v5,form:primary,forms:!((name:primary,selection:(aggregation:histogram,field:year_fiscal_funding,type:integer)),(name:secondary)),size:0)
19 | - BACK:
20 | - /builder
21 |
22 | 3.
23 | - /builder?q=(dataset:mosaic_nih_v7,form:primary,forms:!((name:primary,selection:(aggregation:histogram,field:year_fiscal_funding,type:integer)),(name:secondary)),size:0)
24 | - /builder?q=(dataset:mosaic_nih_v7,form:primary,forms:!((name:primary,selection:(aggregation:reverse_nested,field:year_fiscal_funding,type:integer)),(name:secondary)),size:0)
25 | - /builder?q=(dataset:mosaic_nih_v5,form:primary,forms:!((name:primary,selection:(aggregation:reverse_nested,field:year_fiscal_funding,type:integer)),(name:secondary)),size:0)
26 | - /builder?q=(dataset:mosaic_nih_v5,form:primary,forms:!((name:primary,selection:(aggregation:reverse_nested,field:cost_total_project,type:integer)),(name:secondary)),size:0)
27 |
28 | 4.
29 | - /builder
30 | - /builder?q=(dataset:mosaic_nih_v7,form:primary,forms:!((name:primary,selection:(aggregation:extended_stats,field:cost_total_project))),size:0)
31 | - Select second axis
32 | - Select first axis
33 | - Select second axis
34 | - Select first axis
35 | - Select second axis
36 | - BACK: /builder?q=(dataset:mosaic_nih_v7,form:primary,forms:!((name:primary,selection:(aggregation:extended_stats,field:cost_total_project))),size:0)
37 | - BACK: /builder
--------------------------------------------------------------------------------
/fe/src/lib/utils/specs.js:
--------------------------------------------------------------------------------
1 | /*
2 | This file is imported by `src/bin/make_data.js`
3 | - we can't import files generated by it like `data/datasets.json` in here
4 | - we can't import @svizzle/ui as its index exports `.svelte` files
5 | */
6 |
7 | import {
8 | isObject,
9 | isString,
10 | makePostfixed,
11 | applyFnMap
12 | } from '@svizzle/utils';
13 | import * as _ from 'lamb';
14 |
15 | import {useCache, selectedCacheURL} from '$lib/env.js';
16 |
17 | export const getSource = _.getKey('source');
18 | export const getSpecVersion = _.getKey('version');
19 | export const getApiVersion = _.getPath('spec.dataset.api_version');
20 | export const getSchema = _.getPath('spec.dataset.schema');
21 | export const getEsEndpointURL = _.getPath('spec.dataset.endpoint_url');
22 | export const getEsSearchURL = _.pipe([getEsEndpointURL, makePostfixed('/_search')]);
23 | export const getBeEndpointURL = useCache
24 | // FIXME import `es` from shared configuration with BE
25 | ? spec => `${selectedCacheURL}/es/${_.getPathIn(spec, 'spec.dataset.endpoint_url')}`
26 | : getEsEndpointURL;
27 | export const getBeCoverageEndpointURL = spec =>
28 | `${selectedCacheURL}/coverage/${_.getPathIn(spec, 'spec.dataset.endpoint_url')}`
29 | export const getBeSearchURL = _.pipe([getBeEndpointURL, makePostfixed('/_search')]);
30 | export const getFieldTypeId = _.getKey('type');
31 | export const getESType = _.adapter([
32 | _.casus(isObject, getFieldTypeId),
33 | _.casus(isString, _.identity),
34 | _.always('unknown')
35 | ]);
36 |
37 | export const getDocCount = _.getKey('doc_count');
38 | export const getLocation = _.getKey('location');
39 |
40 | // e.g. `arxlive_arxiv_v3`
41 | export const getDatasetIdOf =
42 | ({project, source, version}) => `${project}_${source}_v${version}`;
43 |
44 | export const groupBySource = _.groupBy(getSource);
45 |
46 | export const makeDatasetBySource = _.pipe([
47 | _.groupBy(getSource),
48 | _.mapValuesWith(_.sortWith([_.getKey('project'), getSpecVersion])),
49 | _.values,
50 | _.sortWith([getSource]),
51 | _.mapWith(applyFnMap({
52 | source: _.getPath('0.source'),
53 | releases: _.sortWith([getSpecVersion])
54 | }))
55 | ]);
56 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/spec/diversified_sampler.todo.js:
--------------------------------------------------------------------------------
1 | import {esSearchableField} from '$lib/elasticsearch/aggs/ref/typeGroups.js';
2 | import {optional} from '$lib/elasticsearch/types/params.js';
3 | import {enumsOf, integerD, string} from '$lib/types/index.js';
4 | import {missing} from '$lib/elasticsearch/aggs/ref/requestDoc.js';
5 |
6 | export default {
7 | id: 'diversified_sampler',
8 | availability: {
9 | from: '5.0'
10 | },
11 | docPath: '/search-aggregations-bucket-diversified-sampler-aggregation.html',
12 | docs: 'Like the sampler aggregation this is a filtering aggregation used to limit any sub aggregations processing to a sample of the top-scoring documents. The diversified_sampler aggregation adds the ability to limit the number of matches that share a common value such as an "author".',
13 | fieldType: esSearchableField,
14 | label: 'Diversified Sampler',
15 | request: {
16 | execution_hint: optional(
17 | enumsOf(['map', 'global_ordinals', 'bytes_hash'], 'global_ordinals')
18 | ),
19 | field: string,
20 | max_docs_per_value: optional(integerD(1)),
21 | missing: optional(string), // [1]
22 | shard_size: optional(integerD(100)),
23 | },
24 | requestDoc: {
25 | execution_hint: 'The optional execution_hint setting can influence the management of the values used for de-duplication. Each option will hold up to `shard_size` values in memory while performing de-duplication but the type of value held can be controlled as follows: * hold field values directly (`map`) * hold ordinals of the field as determined by the Lucene index (`global_ordinals`) * hold hashes of the field values - with potential for hash collisions (`bytes_hash`)',
26 | max_docs_per_value: 'The max_docs_per_value is an optional parameter and limits how many documents are permitted per choice of de-duplicating value. The default setting is "1".',
27 | missing,
28 | shard_size: 'The shard_size parameter limits how many top-scoring documents are collected in the sample processed on each shard. The default value is 100.'
29 | },
30 | subAggs: true,
31 | tag: 'bucketing',
32 | };
33 |
34 | // [1] TODO add constraint to be same type as the field type
35 |
--------------------------------------------------------------------------------
/fe/src/lib/app/components/elementary/TypedField.svelte:
--------------------------------------------------------------------------------
1 |
48 |
49 |
56 | {labelText.startsWith('_') ? '' : labelText}
57 | {#if help}
58 |
64 |
68 | {#if editor}
69 |
82 |
83 |
96 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/spec/variable_width_histogram.js:
--------------------------------------------------------------------------------
1 | import {optional} from '$lib/elasticsearch/types/params.js';
2 | import {integerD, number, string} from '$lib/types/index.js';
3 | import {field} from '$lib/elasticsearch/aggs/ref/requestDoc.js';
4 | import response from '$lib/elasticsearch/aggs/response/bucketsDocCount.js';
5 |
6 | export default {
7 | id: 'variable_width_histogram',
8 | availability: {
9 | from: '7.9'
10 | },
11 | docPath: '/search-aggregations-bucket-variablewidthhistogram-aggregation.html',
12 | docs: 'This is a multi-bucket aggregation similar to Histogram. However, the width of each bucket is not specified. Rather, a target number of buckets is provided and bucket intervals are dynamically determined based on the document distribution. Unlike other multi-bucket aggregations, the intervals will not necessarily have a uniform width.',
13 | fieldType: number,
14 | label: 'Auto Histogram',
15 | request: { // [0] 7.9: no params table
16 | buckets: optional(integerD(10)),
17 | field: string,
18 | initial_buffer: optional(integerD(5000)),
19 | shard_size: optional(integerD(500)),
20 | },
21 | requestDoc: {
22 | buckets: 'The target number of buckets. Bucket intervals are dynamically determined based on the document distribution.',
23 | field,
24 | initial_buffer: 'The number of individual documents that will be stored in memory on a shard before the initial bucketing algorithm is run. Bucket distribution is determined using this sample of initial_buffer documents. So, although a higher initial_buffer will use more memory, it will lead to more representative clusters.',
25 | shard_size: 'The number of buckets that the coordinating node will request from each shard.',
26 | },
27 | response,
28 | subAggs: true,
29 | tag: 'bucketing',
30 | lastChecked: '7.9',
31 | };
32 |
33 | /*
34 | [0] 7.9: no params table
35 | [1] TODO later (#201): needs a constraint
36 | > This aggregation cannot currently be nested under any aggregation that collects from more than a single bucket.
37 | https://www.elastic.co/guide/en/elasticsearch/reference/7.9/search-aggregations-bucket-variablewidthhistogram-aggregation.html
38 | */
39 |
--------------------------------------------------------------------------------
/specs/indices/types/type_templates.yaml:
--------------------------------------------------------------------------------
1 | # generic type
2 |
3 | :
4 | data_type: string
5 | description: string # OPTIONAL
6 | format: string # OPTIONAL - e.g. YYYYMMDD
7 | kind: string # e.g. density, currency
8 | # constraints
9 | regex: RegExp # OPTIONAL – if `data_type` = `string`, a regular expression that the value must match
10 | range: number[] # OPTIONAL – if `data_type` = `integer`|`float`, min and max acceptable values (included) – e.g. `[0, 1]`
11 | min: number # OPTIONAL – if `data_type` = `integer`|`float`, min acceptable value (included) – e.g. 0 for weight
12 | max: number # OPTIONAL – if `data_type` = `integer`|`float`, max acceptable value (included) – e.g. `700` for visible light wavelength (nm)
13 | values: array # OPTIONAL – if there are a handful of known acceptable values – `e.g. ['brown', 'blue', 'orange']`
14 |
15 | # GEOGRAPHY
16 |
17 | GeoRegion:
18 | data_type: object
19 | id_regex: RegExp # OPTIONAL – a regular expression useful to test ID values
20 | id: string # the region id e.g. UKF2
21 | kind: geographic region
22 | level: int # OPTIONAL – the region level, used for NUTS, e.g. 2
23 | levels: int[] # OPTIONAL – a list of all the possible levels – e.g for NUTS: [0, 1, 2, 3]
24 | name: string # name of the region, e.g. Cornwall, France
25 | provider: string # OPTIONAL - use `PROVIDED` in case `provider_url` is unknown because the data have already been reverse geocoded
26 | provider_url: URL # OPTIONAL - url of the boundaries used for reverse geocoding, if known
27 | region_type: string # the region type (e.g. `NUTS`, `LEP`, etc)
28 | year_enforced: int # OPTIONAL - year of spec being enforced
29 | year_spec: int # year of specification release
30 |
31 | # UNITS
32 |
33 | Unit:
34 | data_type: string
35 | description: string # OPTIONAL
36 | kind: string # e.g. density, currency
37 | label: string # short text
38 | unit_latex: string # e.g. \SI{32}{mg.m^{-3}}
39 | unit_string: string # e.g. `mg m^-3`
40 |
41 | Datestring:
42 | data_type: string
43 | format: string # e.g. YYYYMMDD
44 | kind: date
45 | regex: RegExp # OPTIONAL – a regular expression useful to test values
46 |
--------------------------------------------------------------------------------
/fe/src/lib/elasticsearch/aggs/spec/histogram.js:
--------------------------------------------------------------------------------
1 | import {
2 | booleanD,
3 | extent,
4 | integer,
5 | integerD,
6 | number,
7 | string,
8 | } from '$lib/types/index.js';
9 | import {optional, sortOptions} from '$lib/elasticsearch/types/params.js';
10 | import {
11 | field,
12 | keyed,
13 | min_doc_count,
14 | missing
15 | } from '$lib/elasticsearch/aggs/ref/requestDoc.js';
16 | import response from '$lib/elasticsearch/aggs/response/bucketsDocCount.js';
17 |
18 | export default {
19 | id: 'histogram',
20 | availability: {
21 | from: '1.3'
22 | },
23 | docPath: '/search-aggregations-bucket-histogram-aggregation.html',
24 | docs: 'A multi-bucket values source based aggregation that can be applied on numeric values extracted from the documents.',
25 | fieldType: number,
26 | label: 'Histogram',
27 | request: { // [0]
28 | extended_bounds: optional(extent),
29 | field: string,
30 | interval: integer,
31 | keyed: optional(booleanD(false)),
32 | min_doc_count: optional(integerD(1, true)),
33 | missing: optional(number), // [1]
34 | offset: optional(integerD(0)),
35 | order: optional(sortOptions),
36 | },
37 | requestDoc: {
38 | extended_bounds: 'With extended_bounds setting, you now can "force" the histogram aggregation to start building buckets on a specific min value and also keep on building buckets up to a max value (even if there are no documents anymore). Using extended_bounds only makes sense when `min_doc_count` is 0 (the empty buckets will never be returned if `min_doc_count` is greater than 0).',
39 | field,
40 | interval: 'When the aggregation executes, the selected field of every document will be evaluated and will be rounded down to its closest bucket. Must be a positive decimal.',
41 | keyed,
42 | min_doc_count,
43 | missing,
44 | offset: 'Shifts bucket boundaries. Must be a decimal greater than or equal to 0 and less than interval.',
45 | order: 'The order of the buckets can be customized by setting the order parameter. ',
46 | },
47 | response,
48 | subAggs: true,
49 | tag: 'bucketing',
50 | version: '7.9',
51 | };
52 |
53 | // [0] 7.9: no params table
54 | // [1] TODO add constraint to be same type as the field type
55 |
--------------------------------------------------------------------------------