├── .eslintignore ├── .eslintrc ├── .github └── workflows │ ├── ee_pull_requests.yml │ ├── pull_requests.yml │ └── releases.yml ├── .gitignore ├── .prettierignore ├── .prettierrc.json ├── ARCHITECTURE.md ├── DEV_FAQ.md ├── Dockerfile ├── Dockerfile.build ├── GOOD_FIRST_PROJECTS.md ├── HACKING.md ├── LICENSE.md ├── README.md ├── desktop ├── app.test.js ├── app.ts ├── bridge.ts ├── constants.ts ├── crud.ts ├── fs.ts ├── log.test.js ├── log.ts ├── migrations │ └── 1_init.sql ├── panel │ ├── columns.test.js │ ├── columns.ts │ ├── database.test.js │ ├── eval.ts │ ├── filagg.test.js │ ├── file.test.js │ ├── http.test.js │ ├── index.ts │ ├── program.test.js │ ├── shared.ts │ ├── testutil.js │ └── types.ts ├── partial.test.js ├── partial.ts ├── preload.ts ├── project.test.js ├── project.ts ├── rpc.test.js ├── rpc.ts ├── runner.test.js ├── runner.ts ├── scripts │ ├── desktop.build │ └── release.build ├── secret.test.js ├── secret.ts ├── settings.test.js ├── settings.ts ├── store.test.js └── store.ts ├── docker-compose.yml ├── e2e └── index.js ├── ee ├── .eslintignore ├── .eslintrc ├── .prettierignore ├── .prettierrc.json ├── LICENSE.md ├── README.md ├── desktop │ ├── app.test.js │ ├── app.ts │ ├── history.test.js │ ├── history.ts │ ├── migrations │ │ └── 2_history_ee.sql │ ├── preload.ts │ ├── rpc.ts │ ├── scripts │ │ └── desktop.build │ └── store.ts ├── jest.config.js ├── package.json ├── scripts │ └── require_copyright.sh ├── shared │ ├── constants.ts │ ├── rpc.ts │ └── state.ts ├── testsetup.js ├── tsconfig.json ├── type-overrides │ └── ace.d.ts ├── ui │ ├── History.test.jsx │ ├── History.tsx │ ├── index.tsx │ └── scripts │ │ └── ui.build └── yarn.lock ├── icon.icns ├── icon.ico ├── icon.png ├── integration ├── clickhouse.test.js ├── cockroachdb.test.js ├── cratedb.test.js ├── credential_database.test.js ├── docker.js ├── elasticsearch.test.js ├── influx.test.js ├── mongo.test.js ├── mysql.test.js ├── neo4j.test.js ├── oracle.test.js ├── postgres.test.js ├── prometheus.test.js ├── questdb.test.js ├── scylla.test.js └── sqlserver.test.js ├── jest.config.js ├── package.json ├── runner ├── .gitignore ├── cmd │ ├── dsq │ │ └── README.md │ ├── main.go │ └── main_test.go ├── database.go ├── database_airtable.go ├── database_athena.go ├── database_bigquery.go ├── database_cql.go ├── database_elasticsearch.go ├── database_google_sheets.go ├── database_influx.go ├── database_mongo.go ├── database_neo4j.go ├── database_prometheus.go ├── database_splunk.go ├── database_sqlite.go ├── database_test.go ├── errors.go ├── eval.go ├── filagg.go ├── file.go ├── file_test.go ├── go.mod ├── go.sum ├── graphtable.go ├── graphtable_test.go ├── http.go ├── http_test.go ├── json.go ├── literal.go ├── literal_test.go ├── odbc_unix.go ├── odbc_windows.go ├── plugins │ └── odbc │ │ ├── .gitignore │ │ ├── build.sh │ │ ├── go.mod │ │ ├── go.sum │ │ └── odbc.go ├── program.go ├── program_info.go ├── project.go ├── result.go ├── result_test.go ├── scripts │ ├── generate_program_type_info.sh │ ├── runner_test.build │ └── test_coverage.sh ├── settings.go ├── shape.go ├── shape_test.go ├── sql.go ├── sql_test.go ├── ssh.go ├── state.go ├── util.go └── vector.go ├── sampledata ├── Hudson_River_Park_Flora_-_Plantings___1997_-_2015.csv ├── README.md ├── nginx_logs.jsonl └── speedtests.db ├── screens ├── datastation-0.6.0-file-demo.gif ├── datastation-0.7.0-file-demo.gif ├── datastation-0.9.0-cockroach-pandas.gif ├── the-basics-code-panel1.png ├── the-basics-database-panel.png ├── the-basics-graph.png ├── the-basics-group-by.png └── the-basics-post-http.png ├── screenshot.png ├── scripts ├── build.py ├── ci │ ├── prepare_go.sh │ ├── prepare_linux.sh │ ├── prepare_linux_integration_test_setup_only.sh │ ├── prepare_macos.sh │ └── prepare_windows.ps1 ├── fail_on_diff.sh ├── gen_unsafe_certs.sh ├── generate_log_data.js ├── generate_test_data.js ├── original-line-author.sh ├── provision_db.sh ├── upgrade_node_deps.py └── validate_yaml.sh ├── server ├── app.test.js ├── app.ts ├── auth.test.js ├── auth.ts ├── config.test.js ├── config.ts ├── index.ts ├── log.test.js ├── log.ts ├── migrations │ └── 1_init.sql ├── release │ ├── config.yaml │ ├── datastation-exporter.timer │ ├── datastation.service │ └── install.sh ├── rpc.test.js ├── rpc.ts ├── runner.test.js ├── runner.ts └── scripts │ ├── build_image.sh │ ├── release.build │ ├── server.build │ ├── test_release.sh │ └── test_systemd_dockerfile ├── shared ├── array.test.js ├── array.ts ├── constants.ts ├── errors.ts ├── http.ts ├── languages │ ├── deno.json │ ├── deno.jsonnet │ ├── deno.ts │ ├── index.ts │ ├── javascript.json │ ├── javascript.jsonnet │ ├── javascript.ts │ ├── julia.json │ ├── julia.jsonnet │ ├── julia.ts │ ├── php.json │ ├── php.jsonnet │ ├── php.ts │ ├── python.json │ ├── python.jsonnet │ ├── python.ts │ ├── r.json │ ├── r.jsonnet │ ├── r.ts │ ├── ruby.json │ ├── ruby.jsonnet │ ├── ruby.ts │ ├── sql.ts │ └── types.ts ├── log.test.js ├── log.ts ├── object.test.js ├── object.ts ├── polyfill.ts ├── promise.ts ├── rpc.test.js ├── rpc.ts ├── settings.ts ├── sql.ts ├── state.ts ├── table.test.js ├── table.ts ├── text.test.js ├── text.ts ├── url.test.js └── url.ts ├── testdata ├── allformats │ ├── large.parquet │ ├── unknown │ ├── userdata.cjson │ ├── userdata.csv │ ├── userdata.json │ ├── userdata.jsonl │ ├── userdata.ods │ ├── userdata.parquet │ ├── userdata.tsv │ └── userdata.xlsx ├── bigquery │ └── population_result.json ├── csv │ └── missing_columns.csv ├── documents │ ├── 1.json │ ├── 2.json │ ├── 3.json │ └── 4.json ├── influx │ └── noaa-ndbc-data-sample.lp ├── logs │ ├── apache.error.log │ ├── combinedlogformat.log │ ├── commonlogformat.log │ ├── custom.log │ ├── nginx.access.log │ ├── syslogrfc3164.log │ └── syslogrfc5424.log ├── mongo │ └── documents.json ├── prometheus │ └── prometheus.yml └── regr │ ├── 217.xlsx │ └── multiple-sheets.xlsx ├── testsetup.js ├── tsconfig.json ├── type-overrides ├── ace.d.ts └── nodemailer.d.ts ├── ui ├── Connector.tsx ├── ConnectorList.tsx ├── DatabaseConnector.tsx ├── DownloadPanel.test.jsx ├── DownloadPanel.tsx ├── Editor.tsx ├── Footer.tsx ├── Header.tsx ├── Help.test.jsx ├── Help.tsx ├── MakeSelectProject.tsx ├── Navigation.tsx ├── NotFound.tsx ├── PageList.test.jsx ├── PageList.tsx ├── Panel.tsx ├── PanelList.tsx ├── ProjectStore.ts ├── Server.tsx ├── ServerList.tsx ├── Settings.tsx ├── Sidebar.tsx ├── Updates.tsx ├── app.tsx ├── asyncRPC.ts ├── components │ ├── Alert.tsx │ ├── Button.tsx │ ├── CodeEditor.tsx │ ├── Confirm.tsx │ ├── ContentTypePicker.tsx │ ├── Datetime.test.jsx │ ├── Datetime.tsx │ ├── Dropdown.test.jsx │ ├── Dropdown.tsx │ ├── ErrorBoundary.tsx │ ├── FieldPicker.test.jsx │ ├── FieldPicker.tsx │ ├── FileInput.tsx │ ├── FormGroup.tsx │ ├── Highlight.tsx │ ├── Input.tsx │ ├── Link.tsx │ ├── Loading.tsx │ ├── PanelSourcePicker.tsx │ ├── Radio.tsx │ ├── Select.tsx │ ├── ServerPicker.tsx │ ├── TimeSeriesRange.tsx │ ├── Toggle.tsx │ └── Tooltip.tsx ├── connectors │ ├── AirtableDetails.tsx │ ├── ApiKey.test.jsx │ ├── ApiKey.tsx │ ├── AthenaDetails.tsx │ ├── Auth.test.jsx │ ├── Auth.tsx │ ├── BigQueryDetails.tsx │ ├── CassandraDetails.tsx │ ├── Database.test.jsx │ ├── Database.tsx │ ├── ElasticsearchDetails.tsx │ ├── FluxDetails.tsx │ ├── GenericDetails.tsx │ ├── GoogleSheetsDetails.tsx │ ├── Host.test.jsx │ ├── Host.tsx │ ├── Neo4jDetails.tsx │ ├── ODBCDetails.tsx │ ├── Password.test.jsx │ ├── Password.tsx │ ├── SQLiteDetails.test.jsx │ ├── SQLiteDetails.tsx │ ├── SnowflakeDetails.tsx │ ├── Username.test.jsx │ ├── Username.tsx │ └── index.ts ├── index.html ├── index.tsx ├── panels │ ├── DatabasePanel.test.jsx │ ├── DatabasePanel.tsx │ ├── FilePanel.test.jsx │ ├── FilePanel.tsx │ ├── FilterAggregatePanel.test.jsx │ ├── FilterAggregatePanel.tsx │ ├── GraphPanel.test.jsx │ ├── GraphPanel.tsx │ ├── HTTPPanel.test.jsx │ ├── HTTPPanel.tsx │ ├── LiteralPanel.test.jsx │ ├── LiteralPanel.tsx │ ├── ProgramPanel.test.jsx │ ├── ProgramPanel.tsx │ ├── TablePanel.test.jsx │ ├── TablePanel.tsx │ ├── index.test.jsx │ ├── index.ts │ └── types.ts ├── samples.ts ├── scripts │ ├── languages.build │ ├── ui.build │ └── watch_and_serve.sh ├── shortcuts.ts ├── state.test.jsx ├── state.ts ├── style.css ├── urlState.tsx └── yarn.lock └── yarn.lock /.eslintignore: -------------------------------------------------------------------------------- 1 | ui/scripts/* -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "root": true, 3 | "parser": "@typescript-eslint/parser", 4 | "plugins": ["@typescript-eslint", "jest"], 5 | "extends": [ 6 | "eslint:recommended", 7 | "plugin:react/recommended", 8 | "plugin:@typescript-eslint/eslint-recommended", 9 | "plugin:@typescript-eslint/recommended", 10 | "plugin:react-hooks/recommended" 11 | ], 12 | "rules": { 13 | "react/jsx-no-target-blank": "off", 14 | "react/no-children-prop": "off", 15 | "react/no-unescaped-entities": "off", 16 | "@typescript-eslint/no-explicit-any": "off", 17 | "@typescript-eslint/no-unused-vars": [ 18 | "error", 19 | { 20 | "argsIgnorePattern": "^_", 21 | "varsIgnorePattern": "^_", 22 | "caughtErrorsIgnorePattern": "^_" 23 | } 24 | ] 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | node_modules 3 | ui/build 4 | build 5 | datastation 6 | yarn-error.log 7 | releases 8 | coverage 9 | certs 10 | .dir-locals.el 11 | TAGS 12 | *.cov 13 | all.out 14 | runner/report 15 | runner/runner 16 | runner/runner.exe 17 | dsq 18 | data 19 | .idea 20 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | build 3 | releases 4 | server/release/config.json -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true 3 | } 4 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:17-bullseye 2 | WORKDIR /datastation 3 | EXPOSE 8080 4 | COPY package.json /datastation 5 | COPY build /datastation/build 6 | COPY node_modules /datastation/node_modules 7 | CMD ["node", "/datastation/build/server.js"] -------------------------------------------------------------------------------- /Dockerfile.build: -------------------------------------------------------------------------------- 1 | FROM node:17-bullseye 2 | 3 | WORKDIR /datastation 4 | 5 | # Install Golang 6 | RUN curl -L https://go.dev/dl/go1.19.linux-amd64.tar.gz -o /tmp/go.tar.gz && tar -C /usr/local -xzf /tmp/go.tar.gz 7 | RUN ln -s /usr/local/go/bin/go /usr/bin/go 8 | -------------------------------------------------------------------------------- /GOOD_FIRST_PROJECTS.md: -------------------------------------------------------------------------------- 1 | # Good First Projects 2 | 3 | If any of these sound interesting, join #dev on 4 | [Discord](https://discord.multiprocess.io) and say hi! 5 | 6 | The first thing you'll be asked to do is go through one or two of the 7 | [tutorials on DataStation](https://datastation.multiprocess.io/docs/), 8 | and [try out dsq](https://github.com/multiprocessio/dsq). 9 | 10 | You'll need to have this little bit of experience using DataStation 11 | and dsq for these tasks to make sense. 12 | 13 | ## Easy 14 | 15 | * Add a new supported file type 16 | * Example: Messagepack, BSON, CBOR, UBJSON, XML, Yaml, Avro, HDF5? 17 | * See https://github.com/multiprocessio/datastation/pull/215 for how this can be done in one PR 18 | * Test out INT96 support in Parquet, add conversion to timestamp if necessary 19 | * Build dsq, fakegen for more/every os/arch 20 | * Add parquet, avro writers to fakegen 21 | * Preparation for optimized internal representation of data 22 | * Do read/write benchmarks among MessagePack/BSON/Protobuf/Avro 23 | * Make sure there’s a library for every language 24 | * Figure out how to embed the library inside DataStation 25 | * Migrate all calls reading results directly to an API layer for getting panel results (in both Go and JavaScript) 26 | * Fix in dsq too 27 | * More databases 28 | * IBM DB2, Apache Presto/Trino, Meilisearch, Apache Hive, Apache Druid, Apache Pinot, Quickwit, Couchbase, fix MongoDB, Redis, Splunk, DataDog, SumoLogic, Loggly, [New Relic](https://docs.newrelic.com/docs/apis/nerdgraph/examples/nerdgraph-nrql-tutorial), VictoriaMetrics, Impala 29 | * See https://github.com/multiprocessio/datastation/pull/219 for adding a database in one PR 30 | 31 | ## Medium 32 | 33 | * Add caching to dsq 34 | * Support zip, .tar.gz, .gz, .tar, bz2 files 35 | * HTTP Range support for faster downloads 36 | * Add support for FTP 37 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright 2021-2022 Multiprocess Labs LLC 2 | 3 | * All content that resides under the "ee/" directory of this repository, if that directory exists, is licensed under the license defined in "ee/LICENSE.md". 4 | * Content outside of the above mentioned directories or restrictions above is available under the "Apache 2.0" license as defined below. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | -------------------------------------------------------------------------------- /desktop/app.test.js: -------------------------------------------------------------------------------- 1 | require('./app'); 2 | 3 | test('stub', () => {}); 4 | -------------------------------------------------------------------------------- /desktop/app.ts: -------------------------------------------------------------------------------- 1 | import { app, ipcMain } from 'electron'; 2 | import path from 'path'; 3 | import { APP_NAME, DEBUG, VERSION } from '../shared/constants'; 4 | import log from '../shared/log'; 5 | import { IS_DESKTOP_RUNNER } from './constants'; 6 | import { configureLogger } from './log'; 7 | import { openWindow } from './project'; 8 | import { registerRPCHandlers } from './rpc'; 9 | import { initialize } from './runner'; 10 | import { Store } from './store'; 11 | 12 | const binaryExtension: Record = { 13 | darwin: '', 14 | linux: '', 15 | win32: '.exe', 16 | }; 17 | 18 | function main() { 19 | configureLogger(); 20 | log.info(APP_NAME, VERSION, DEBUG ? 'DEBUG' : ''); 21 | 22 | ['uncaughtException', 'unhandledRejection'].map((sig) => 23 | process.on(sig, log.error) 24 | ); 25 | 26 | // Just for basic unit tests 27 | if (app) { 28 | app.whenReady().then(async () => { 29 | const store = new Store(); 30 | const { handlers, project } = initialize({ 31 | subprocess: { 32 | node: path.join(__dirname, 'desktop_runner.js'), 33 | go: path.join( 34 | __dirname, 35 | 'go_desktop_runner' + binaryExtension[process.platform] 36 | ), 37 | }, 38 | additionalHandlers: store.getHandlers(), 39 | }); 40 | 41 | await openWindow(project); 42 | 43 | registerRPCHandlers(ipcMain, handlers); 44 | }); 45 | 46 | app.on('window-all-closed', function () { 47 | if (process.platform !== 'darwin') app.quit(); 48 | }); 49 | } 50 | } 51 | 52 | if (!IS_DESKTOP_RUNNER) { 53 | main(); 54 | } 55 | -------------------------------------------------------------------------------- /desktop/bridge.ts: -------------------------------------------------------------------------------- 1 | import { contextBridge, ipcRenderer, IpcRendererEvent } from 'electron'; 2 | import { RPC_ASYNC_REQUEST, RPC_ASYNC_RESPONSE } from '../shared/constants'; 3 | import log from '../shared/log'; 4 | import { Endpoint, IPCRendererResponse, WindowAsyncRPC } from '../shared/rpc'; 5 | 6 | let messageNumber = -1; 7 | 8 | export function bridgeAsyncRPC() { 9 | const asyncRPC: WindowAsyncRPC = async function < 10 | Request, 11 | Response = void, 12 | EndpointT extends string = Endpoint 13 | >(resource: EndpointT, projectId: string, body: Request): Promise { 14 | const payload = { 15 | // Assign a new message number 16 | messageNumber: ++messageNumber, 17 | resource, 18 | body, 19 | projectId, 20 | }; 21 | ipcRenderer.send(RPC_ASYNC_REQUEST, payload); 22 | 23 | const result = await new Promise>( 24 | (resolve, reject) => { 25 | try { 26 | ipcRenderer.once( 27 | `${RPC_ASYNC_RESPONSE}:${payload.messageNumber}`, 28 | (e: IpcRendererEvent, response: IPCRendererResponse) => 29 | resolve(response) 30 | ); 31 | } catch (e) { 32 | reject(e); 33 | } 34 | } 35 | ); 36 | 37 | if (result.kind === 'error') { 38 | try { 39 | throw result.error; 40 | } catch (e) { 41 | // The result.error object isn't a real Error at this point with 42 | // prototype after going through serialization. So throw it to get 43 | // a real Error instance that has full info for logs. 44 | log.error(e); 45 | throw e; 46 | } 47 | } 48 | 49 | return result.body; 50 | }; 51 | 52 | contextBridge.exposeInMainWorld('asyncRPC', asyncRPC); 53 | } 54 | -------------------------------------------------------------------------------- /desktop/constants.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs'; 2 | import os from 'os'; 3 | import path from 'path'; 4 | 5 | // process.env.HOME is not set on Windows 6 | export const HOME = os.homedir(); 7 | 8 | // This should be a const but this can actually change, only inside the runner code. 9 | export const DISK_ROOT = { value: path.join(HOME, 'DataStationProjects') }; 10 | 11 | export const PROJECT_EXTENSION = 'dsproj'; 12 | 13 | export const DSPROJ_FLAG = '--dsproj'; 14 | export const PANEL_FLAG = '--evalPanel'; 15 | export const PANEL_META_FLAG = '--metaFile'; 16 | export const SETTINGS_FILE_FLAG = '--settingsFile'; 17 | export const FS_BASE_FLAG = '--fsbase'; 18 | 19 | export const KEY_SIZE = 4096; 20 | 21 | export const LOG_FILE = path.join(DISK_ROOT.value, 'log'); 22 | 23 | // Finds where package.json is in this repo, the repo root. 24 | export const CODE_ROOT = (() => { 25 | for (let mpath of module.paths) { 26 | if (fs.existsSync(mpath)) { 27 | return path.dirname(mpath); 28 | } 29 | } 30 | 31 | return ''; 32 | })(); 33 | 34 | export const IS_DESKTOP_RUNNER = (process.argv[1] || '').includes( 35 | 'desktop_runner.js' 36 | ); 37 | -------------------------------------------------------------------------------- /desktop/fs.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs'; 2 | import path from 'path'; 3 | import { DISK_ROOT } from './constants'; 4 | export function ensureFile(f: string) { 5 | let root = path.isAbsolute(f) ? path.dirname(f) : DISK_ROOT.value; 6 | fs.mkdirSync(root, { recursive: true }); 7 | return path.isAbsolute(f) ? f : path.join(DISK_ROOT.value, f); 8 | } 9 | -------------------------------------------------------------------------------- /desktop/log.test.js: -------------------------------------------------------------------------------- 1 | require('./log'); 2 | 3 | test('stub', () => {}); 4 | -------------------------------------------------------------------------------- /desktop/log.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs'; 2 | import { EOL } from 'os'; 3 | import { logger } from '../shared/log'; 4 | import { LOG_FILE } from './constants'; 5 | import { ensureFile } from './fs'; 6 | 7 | export function configureLogger() { 8 | ensureFile(LOG_FILE); 9 | const logFd = fs.openSync(LOG_FILE, 'a'); 10 | 11 | function makeLogger(log: (m: string) => void) { 12 | return (m: string) => { 13 | try { 14 | log(m); 15 | fs.appendFileSync(logFd, m + EOL); 16 | } catch (e) { 17 | console.error(e); 18 | open(); 19 | } 20 | }; 21 | } 22 | 23 | logger.INFO = makeLogger(console.log.bind(console)); 24 | logger.ERROR = makeLogger(console.error.bind(console)); 25 | } 26 | -------------------------------------------------------------------------------- /desktop/panel/columns.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs'; 2 | import { PanelBody } from '../../shared/rpc'; 3 | import { Dispatch, RPCHandler } from '../rpc'; 4 | import { getProjectResultsFile } from '../store'; 5 | import { getProjectAndPanel } from './shared'; 6 | import { EvalHandlerResponse } from './types'; 7 | 8 | // TODO: this needs to be ported to go 9 | export const fetchResultsHandler: RPCHandler = { 10 | resource: 'fetchResults', 11 | handler: async function ( 12 | projectId: string, 13 | body: PanelBody, 14 | dispatch: Dispatch 15 | ): Promise { 16 | const { panel } = await getProjectAndPanel( 17 | dispatch, 18 | projectId, 19 | body.panelId 20 | ); 21 | 22 | // Maybe the only appropriate place to call this in this package? 23 | const projectResultsFile = getProjectResultsFile(projectId); 24 | // TODO: this is a 4GB file limit! 25 | const f = fs.readFileSync(projectResultsFile + panel.id); 26 | 27 | // Everything gets stored as JSON on disk. Even literals and files get rewritten as JSON. 28 | return { value: JSON.parse(f.toString()) }; 29 | }, 30 | }; 31 | -------------------------------------------------------------------------------- /desktop/panel/database.test.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const fs = require('fs'); 3 | 4 | const { CODE_ROOT } = require('../constants'); 5 | const { getProjectResultsFile } = require('../store'); 6 | const { ensureSigningKey } = require('../secret'); 7 | const { 8 | LiteralPanelInfo, 9 | Encrypt, 10 | DatabasePanelInfo, 11 | DatabaseConnectorInfo, 12 | } = require('../../shared/state'); 13 | const { basicDatabaseTest, withSavedPanels, RUNNERS } = require('./testutil'); 14 | 15 | const DATABASES = [ 16 | { 17 | type: 'sqlite', 18 | query: `SELECT 1 AS "1", 2.2 AS "2", true AS "true", 'string' AS "string", DATE('2021-01-01') AS "date"`, 19 | }, 20 | { 21 | type: 'sqlite', 22 | query: 23 | 'SELECT name, CAST(age AS INT) - 10 AS age, "location.city" AS city FROM DM_getPanel(0)', 24 | }, 25 | ]; 26 | 27 | ensureSigningKey(); 28 | 29 | const vendorOverride = {}; 30 | 31 | for (const subprocess of RUNNERS) { 32 | // Most databases now only work with the Go runner. 33 | if (!subprocess?.go) { 34 | continue; 35 | } 36 | 37 | for (const t of DATABASES) { 38 | describe( 39 | t.type + 40 | ' running via ' + 41 | (subprocess ? subprocess.node || subprocess.go : 'process') + 42 | ': ' + 43 | t.query, 44 | () => { 45 | test(`runs ${t.type} query`, async () => { 46 | if (process.platform !== 'linux') { 47 | return; 48 | } 49 | 50 | await basicDatabaseTest(t, vendorOverride, subprocess); 51 | // sqlserver takes a while 52 | }, 30_000); 53 | } 54 | ); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /desktop/panel/index.ts: -------------------------------------------------------------------------------- 1 | import { fetchResultsHandler } from './columns'; 2 | import { killProcessHandler, makeEvalHandler } from './eval'; 3 | 4 | export const panelHandlers = (subprocess?: { node: string; go?: string }) => [ 5 | makeEvalHandler(subprocess), 6 | fetchResultsHandler, 7 | killProcessHandler, 8 | ]; 9 | -------------------------------------------------------------------------------- /desktop/panel/shared.ts: -------------------------------------------------------------------------------- 1 | import { PanelInfo, ProjectState } from '../../shared/state'; 2 | import { Dispatch } from '../rpc'; 3 | 4 | export async function getProjectAndPanel( 5 | dispatch: Dispatch, 6 | projectId: string, 7 | panelId: string 8 | ) { 9 | if (!panelId) { 10 | throw new Error('No panel specified when trying to look up panel.'); 11 | } 12 | const project = 13 | ((await dispatch({ 14 | resource: 'getProject', 15 | projectId, 16 | body: { projectId }, 17 | })) as ProjectState) || new ProjectState(); 18 | let panelPage = 0; 19 | let panel: PanelInfo; 20 | for (; !panel && panelPage < (project.pages || []).length; panelPage++) { 21 | for (const p of project.pages[panelPage].panels || []) { 22 | if (p.id === panelId) { 23 | panel = p; 24 | break; 25 | } 26 | } 27 | } 28 | if (!panel) { 29 | throw new Error(`Unable to find panel: "${panelId.replace('"', '\\"')}".'`); 30 | } 31 | panelPage--; // Handle last loop ++ overshot 32 | return { panel, panelPage, project }; 33 | } 34 | -------------------------------------------------------------------------------- /desktop/panel/types.ts: -------------------------------------------------------------------------------- 1 | export type EvalHandlerResponse = { 2 | returnValue?: boolean; 3 | skipWrite?: boolean; 4 | stdout?: string; 5 | contentType?: string; 6 | value: any; 7 | size?: any; 8 | arrayCount?: any; 9 | }; 10 | -------------------------------------------------------------------------------- /desktop/partial.test.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const { parsePartialJSONFile } = require('./partial'); 3 | import { file as makeTmpFile } from 'tmp-promise'; 4 | 5 | describe('parsePartialJSONFile', function parsePartialJSONFileTest() { 6 | test('correctly fills out partial file', async function fillsOutPartial() { 7 | const f = await makeTmpFile(); 8 | try { 9 | const whole = '[{"foo": "bar"}, {"big": "bad"}]'; 10 | fs.writeFileSync(f.path, whole); 11 | const { size, preview } = parsePartialJSONFile(f.path, 3); 12 | expect(size).toBe(whole.length); 13 | expect(preview).toStrictEqual(`[\n { "foo": "bar" }\n]`); 14 | } finally { 15 | f.cleanup(); 16 | } 17 | }); 18 | 19 | test('handles open-close in string', async function handlesOpenCloseInString() { 20 | for (const c of ['{', '}', '[', ']']) { 21 | const f = await makeTmpFile(); 22 | try { 23 | const whole = `[{"foo": "${c}bar"}, {"big": "bad"}]`; 24 | fs.writeFileSync(f.path, whole); 25 | const { preview, size } = parsePartialJSONFile(f.path, 3); 26 | expect(size).toBe(whole.length); 27 | expect(preview).toStrictEqual(`[\n { "foo": "${c}bar" }\n]`); 28 | } finally { 29 | f.cleanup(); 30 | } 31 | } 32 | }); 33 | 34 | test('handles escaped quotes in string', async function handlesEscapedQuotesInString() { 35 | const f = await makeTmpFile(); 36 | try { 37 | const whole = `[{"foo":"bar\\" { "},{"big":"bad"}]`; 38 | expect(JSON.stringify(JSON.parse(whole))).toEqual(whole); 39 | fs.writeFileSync(f.path, whole); 40 | const { preview, size } = parsePartialJSONFile(f.path, 3); 41 | expect(size).toBe(whole.length); 42 | expect(preview).toStrictEqual(`[\n { "foo": "bar\\" { " }\n]`); 43 | } finally { 44 | f.cleanup(); 45 | } 46 | }); 47 | }); 48 | -------------------------------------------------------------------------------- /desktop/preload.ts: -------------------------------------------------------------------------------- 1 | import { bridgeAsyncRPC } from './bridge'; 2 | 3 | bridgeAsyncRPC(); 4 | -------------------------------------------------------------------------------- /desktop/project.test.js: -------------------------------------------------------------------------------- 1 | require('./project'); 2 | 3 | test('stub', () => {}); 4 | -------------------------------------------------------------------------------- /desktop/rpc.test.js: -------------------------------------------------------------------------------- 1 | const { 2 | RPC_ASYNC_REQUEST, 3 | RPC_ASYNC_RESPONSE, 4 | } = require('../shared/constants'); 5 | const { registerRPCHandlers } = require('./rpc'); 6 | 7 | test('registers handlers', async () => { 8 | let handle; 9 | const ipcMain = { 10 | on(msg, h) { 11 | expect(msg).toBe(RPC_ASYNC_REQUEST); 12 | handle = h; 13 | }, 14 | }; 15 | 16 | let finished1 = false; 17 | const handlers = [ 18 | { 19 | resource: 'test', 20 | handler: (projectId, body, dispatch, external) => { 21 | // TODO: test internal dispatch 22 | expect(projectId).toBe('test id'); 23 | expect(body).toStrictEqual({ something: 1 }); 24 | expect(external).toStrictEqual(true); 25 | expect(typeof dispatch).toBe('function'); 26 | finished1 = true; 27 | return { aresponse: true }; 28 | }, 29 | }, 30 | ]; 31 | 32 | let finished2 = false; 33 | registerRPCHandlers(ipcMain, handlers); 34 | const event = { 35 | sender: { 36 | send(channel, body) { 37 | expect(channel).toBe(`${RPC_ASYNC_RESPONSE}:88`); 38 | expect(body.kind).toBe('response'); 39 | expect(body.body).toStrictEqual({ aresponse: true }); 40 | finished2 = true; 41 | }, 42 | }, 43 | }; 44 | await handle(event, { 45 | resource: 'test', 46 | messageNumber: 88, 47 | projectId: 'test id', 48 | body: { something: 1 }, 49 | }); 50 | 51 | if (!finished1 || !finished2) { 52 | throw new Error('Test did not finish.'); 53 | } 54 | }); 55 | -------------------------------------------------------------------------------- /desktop/runner.test.js: -------------------------------------------------------------------------------- 1 | require('./runner'); 2 | 3 | test('stub', () => {}); 4 | -------------------------------------------------------------------------------- /desktop/scripts/desktop.build: -------------------------------------------------------------------------------- 1 | setenv UI_CONFIG_OVERRIDES "window.DS_CONFIG_MODE = 'desktop';" 2 | yarn build-ui 3 | 4 | # Flags from various package management guidelines: https://wiki.archlinux.org/title/Go_package_guidelines 5 | setenv_default VERSION "development" 6 | cd runner && go build -trimpath -buildmode=pie -mod=readonly -modcacherw -ldflags="-s -w -X main.VERSION={VERSION}" -o ../build/go_desktop_runner{required_ext} cmd/main.go 7 | 8 | yarn esbuild desktop/preload.ts --external:electron --sourcemap --bundle --outfile=build/preload.js 9 | yarn esbuild desktop/runner.ts --bundle --platform=node --sourcemap --external:better-sqlite3 --external:react-native-fs --external:asar --external:react-native-fetch-blob "--external:@elastic/elasticsearch" "--external:wasm-brotli" --external:prometheus-query --external:snowflake-sdk --external:ssh2 --external:ssh2-promise --external:ssh2-sftp-client --external:cpu-features --external:electron --target=node10.4 --outfile=build/desktop_runner.js 10 | yarn esbuild desktop/app.ts --bundle --platform=node --sourcemap --external:better-sqlite3 --external:react-native-fs --external:react-native-fetch-blob "--external:@elastic/elasticsearch" "--external:wasm-brotli" --external:prometheus-query --external:snowflake-sdk --external:ssh2 --external:ssh2-promise --external:asar --external:ssh2-sftp-client --external:cpu-features --external:electron --target=node10.4 --outfile=build/desktop.js 11 | 12 | rm -rf build/migrations 13 | cp -r desktop/migrations build/migrations 14 | -------------------------------------------------------------------------------- /desktop/scripts/release.build: -------------------------------------------------------------------------------- 1 | rm -rf build 2 | yarn 3 | setenv UI_ESBUILD_ARGS "--minify" 4 | setenv VERSION {arg0} 5 | # Remove v prefix from version. v12 -> 12 6 | stripleft VERSION "v" 7 | yarn build-desktop 8 | 9 | prepend "window.DS_CONFIG_MODE='desktop';" build/ui.js 10 | prepend "window.DS_CONFIG_VERSION='{VERSION}';" build/ui.js 11 | prepend "window.DS_CONFIG_VERSION='{VERSION}';" build/desktop_runner.js 12 | prepend "global.DS_CONFIG_VERSION='{VERSION}';" build/desktop.js 13 | cp icon.png build/icon.png 14 | cp icon.ico build/icon.ico 15 | cp icon.icns build/icon.icns 16 | 17 | # Clean stuff up 18 | rm -rf coverage 19 | yarn remove xlsx papaparse express express-session nodemailer openid-client cookie-parser chart.js ace-builds js-yaml passport react react-ace react-dom react-syntax-highlighter date-fns jsdom 20 | rm -rf certs 21 | rm -rf ee 22 | rm -rf *.md 23 | rm -rf *.yml 24 | rm -rf server 25 | rm -rf scripts 26 | rm -rf screens 27 | rm -rf testdata 28 | rm -rf type-overrides 29 | rm -rf *.png 30 | rm -rf TAGS 31 | rm -rf e2e 32 | rm -rf Dockerfile 33 | rm -rf dsq 34 | rm -rf *.js 35 | rm -rf tsconfig.json 36 | rm -rf yarn.lock 37 | rm -rf runner 38 | rm -rf desktop 39 | rm -rf ui 40 | rm -rf shared 41 | rm -rf releases 42 | 43 | yarn electron-rebuild 44 | 45 | # Build and package 46 | yarn electron-packager --asar --overwrite --icon=build/icon.png --out=releases --build-version={VERSION} --app-version={VERSION} . "DataStation Desktop CE" 47 | 48 | # This reference to arg0 is correct because the upload script uses the 49 | # real tag name, not our stripped version here. 50 | zip -9 -r releases/datastation-{os}-{arch}-{arg0}.zip "releases/DataStation Desktop CE-{os}-{arch}" -------------------------------------------------------------------------------- /desktop/secret.test.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const { file: makeTmpFile } = require('tmp-promise'); 3 | const { encrypt, decrypt, ensureSigningKey } = require('./secret'); 4 | 5 | async function withTmpSigningKey(cb) { 6 | const tmp = await makeTmpFile({ prefix: 'secret-project-' }); 7 | try { 8 | await ensureSigningKey(tmp.path); 9 | expect(fs.readFileSync(tmp.path).length).toBe(44); 10 | await cb(tmp); 11 | } finally { 12 | await tmp.cleanup(); 13 | } 14 | } 15 | 16 | test('encrypt, decrypt same key', () => 17 | withTmpSigningKey(async (tmp) => { 18 | const original = 'my great string'; 19 | const encrypted = await encrypt(original, tmp.path); 20 | const tmp2 = await makeTmpFile({ prefix: 'stored-secret-' }); 21 | try { 22 | fs.writeFileSync(tmp2.path, JSON.stringify(encrypted)); 23 | const eFromDisk = JSON.parse(fs.readFileSync(tmp2.path).toString()); 24 | const decrypted = await decrypt(eFromDisk, tmp.path); 25 | expect(decrypted).toBe(original); 26 | } finally { 27 | tmp2.cleanup(); 28 | } 29 | })); 30 | 31 | test('encrypt, decrypt different key', () => 32 | withTmpSigningKey(async (tmp) => { 33 | const original = 'my great string'; 34 | const encrypted = await encrypt(original, tmp.path); 35 | 36 | return withTmpSigningKey(async (newKeyTmp) => { 37 | let error; 38 | try { 39 | await decrypt(encrypted, newKeyTmp.path); 40 | } catch (e) { 41 | error = e; 42 | } 43 | 44 | expect(error.message).toBe('Could not decrypt message'); 45 | }); 46 | })); 47 | -------------------------------------------------------------------------------- /desktop/settings.test.js: -------------------------------------------------------------------------------- 1 | const { file: makeTmpFile } = require('tmp-promise'); 2 | const { DesktopSettings, loadSettings } = require('./settings'); 3 | 4 | test('loadSettings with and without existing settings', async () => { 5 | const tmp = await makeTmpFile({ prefix: 'settings-project-' }); 6 | 7 | const testSettings = new DesktopSettings(); 8 | testSettings.file = tmp.path; 9 | 10 | try { 11 | const loaded = loadSettings(tmp.path); 12 | testSettings.id = loaded.id; 13 | expect(loaded).toStrictEqual(testSettings); 14 | 15 | loaded.lastProject = 'my new project'; 16 | // Save 17 | await loaded.getUpdateHandler().handler(null, loaded); 18 | // And recheck from disk that save happened 19 | const reloaded = loadSettings(tmp.path); 20 | expect(reloaded.lastProject).toBe('my new project'); 21 | 22 | const settings = await loaded.getGetHandler().handler(); 23 | expect(settings).toStrictEqual(reloaded); 24 | } finally { 25 | await tmp.cleanup(); 26 | } 27 | }); 28 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | services: 3 | server: 4 | build: . 5 | ports: 6 | - "8080:8080" 7 | environment: 8 | DATASTATION_SERVER_ADDRESS: "0.0.0.0" 9 | volumes: 10 | - /etc/datastation:/etc/datastation 11 | deploy: 12 | restart_policy: 13 | condition: on-failure 14 | delay: 5s 15 | max_attempts: 3 16 | window: 120s 17 | -------------------------------------------------------------------------------- /ee/.eslintignore: -------------------------------------------------------------------------------- 1 | ui/scripts/* 2 | ui/state.test.js -------------------------------------------------------------------------------- /ee/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "root": true, 3 | "parser": "@typescript-eslint/parser", 4 | "plugins": [ 5 | "@typescript-eslint", 6 | "jest" 7 | ], 8 | "extends": [ 9 | "eslint:recommended", 10 | "plugin:react/recommended", 11 | "plugin:@typescript-eslint/eslint-recommended", 12 | "plugin:@typescript-eslint/recommended", 13 | "plugin:react-hooks/recommended" 14 | ], 15 | "rules": { 16 | "react/jsx-no-target-blank": 'off', 17 | "react/no-children-prop": 'off', 18 | "@typescript-eslint/no-explicit-any": 'off', 19 | '@typescript-eslint/no-unused-vars': [ 20 | 'error', 21 | { 22 | argsIgnorePattern: '^_', 23 | varsIgnorePattern: '^_', 24 | caughtErrorsIgnorePattern: '^_', 25 | }, 26 | ] 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /ee/.prettierignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | build 3 | releases 4 | server/release/config.json -------------------------------------------------------------------------------- /ee/.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true 3 | } 4 | -------------------------------------------------------------------------------- /ee/README.md: -------------------------------------------------------------------------------- 1 | # DataStation Enterprise Edition 2 | 3 | This code is source-available. You will need to purchase a subcription 4 | to acquire a license for use. 5 | 6 | Copyright 2022 Multiprocess Labs LLC 7 | -------------------------------------------------------------------------------- /ee/desktop/app.test.js: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Multiprocess Labs LLC 2 | 3 | require('./app'); 4 | 5 | test('', () => {}); 6 | -------------------------------------------------------------------------------- /ee/desktop/migrations/2_history_ee.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2022 Multiprocess Labs LLC 2 | 3 | CREATE TABLE ds_user( 4 | id TEXT PRIMARY KEY, 5 | name TEXT NOT NULL 6 | ) STRICT; 7 | 8 | CREATE TABLE ds_history( 9 | id TEXT PRIMARY KEY, 10 | tbl TEXT NOT NULL, 11 | pk TEXT NOT NULL, -- Primary key **value** of the row being edited in the table being edited 12 | dt INTEGER NOT NULL, -- UNIX timestamp 13 | error TEXT NOT NULL, 14 | old_value TEXT NOT NULL, 15 | new_value TEXT NOT NULL, 16 | user_id TEXT NOT NULL, 17 | action TEXT NOT NULL, 18 | FOREIGN KEY (user_id) REFERENCES ds_user(id) 19 | ) STRICT; 20 | 21 | CREATE INDEX ds_history_foreign_pk ON ds_history(pk); 22 | CREATE INDEX ds_history_dt_idx ON ds_history(dt); 23 | -------------------------------------------------------------------------------- /ee/desktop/preload.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Multiprocess Labs LLC 2 | 3 | import { bridgeAsyncRPC } from '../../desktop/bridge'; 4 | 5 | bridgeAsyncRPC(); 6 | -------------------------------------------------------------------------------- /ee/desktop/rpc.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Multiprocess Labs LLC 2 | 3 | import * as rpc_ce from '../../desktop/rpc'; 4 | import { Endpoint, GetHistoryRequest, GetHistoryResponse } from '../shared/rpc'; 5 | 6 | export type RPCPayload = rpc_ce.GenericRPCPayload; 7 | export type Dispatch = rpc_ce.GenericDispatch; 8 | 9 | export type RPCHandler = rpc_ce.RPCHandler; 10 | 11 | export type GetHistoryHandler = rpc_ce.RPCHandler< 12 | GetHistoryRequest, 13 | GetHistoryResponse, 14 | Endpoint 15 | >; 16 | -------------------------------------------------------------------------------- /ee/desktop/scripts/desktop.build: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Multiprocess Labs LLC 2 | 3 | rm -rf build 4 | rm -rf ../build 5 | setenv UI_TITLE "DataStation Enterprise Edition" 6 | setenv UI_CONFIG_OVERRIDES "window.DS_CONFIG_MODE = 'desktop';" 7 | yarn build-ui 8 | 9 | # Flags from various package management guidelines: https://wiki.archlinux.org/title/Go_package_guidelines 10 | cd ../runner && go build -trimpath -buildmode=pie -mod=readonly -modcacherw -ldflags="-s -w" -o ../ee/build/go_desktop_runner{required_ext} cmd/main.go 11 | 12 | yarn esbuild desktop/preload.ts --external:electron --sourcemap --bundle --outfile=./build/preload.js 13 | cd .. && yarn esbuild desktop/runner.ts --bundle --platform=node --sourcemap --external:better-sqlite3 --external:electron --target=node10.4 --outfile=ee/build/desktop_runner.js 14 | yarn esbuild desktop/app.ts --bundle --platform=node --sourcemap --external:better-sqlite3 --external:electron --target=node10.4 --outfile=build/desktop.js 15 | 16 | rm -rf build/migrations 17 | cp -r ../desktop/migrations build/ 18 | cp -r desktop/migrations/* build/migrations/ -------------------------------------------------------------------------------- /ee/jest.config.js: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Multiprocess Labs LLC 2 | 3 | module.exports = { 4 | // Only on linux do all tests run 5 | coverageThreshold: 6 | process.platform === 'linux' 7 | ? { 8 | global: { 9 | statements: 73, 10 | branches: 67, 11 | functions: 74, 12 | lines: 74, 13 | }, 14 | } 15 | : undefined, 16 | preset: 'ts-jest/presets/js-with-ts', 17 | transformIgnorePatterns: [ 18 | 'node_modules/(?!react-syntax-highlighter|refractor|node-fetch|fetch-blob)', 19 | ], 20 | setupFiles: ['../shared/polyfill.ts', './testsetup.js'], 21 | testEnvironment: 'node', 22 | testEnvironmentOptions: { 23 | url: 'http://localhost/', 24 | }, 25 | collectCoverageFrom: [ 26 | 'ui/**/*.ts', 27 | 'ui/**/*.tsx', 28 | 'shared/**/*.ts', 29 | 'desktop/**/*.ts', 30 | 'server/**/*.ts', 31 | 'server/**/*.tsx', 32 | ], 33 | modulePathIgnorePatterns: ['/releases/', '/build/'], 34 | }; 35 | -------------------------------------------------------------------------------- /ee/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "datastation-ee", 3 | "productName": "DataStation Enterprise Edition", 4 | "version": "0.0.1", 5 | "scripts": { 6 | "format": "yarn prettier --write \"**/*.ts\"", 7 | "build-desktop": "python3 ../scripts/build.py ./desktop/scripts/desktop.build", 8 | "build-ui": "python3 ../scripts/build.py ./ui/scripts/ui.build", 9 | "start-desktop": "yarn build-desktop && yarn electron --trace-warning --unhandled-rejection=warn build/desktop.js", 10 | "test": "yarn test-local --coverage", 11 | "test-local": "cross-env NODE_OPTIONS=--unhandled-rejections=warn jest --passWithNoTests " 12 | }, 13 | "devDependencies": { 14 | "@types/better-sqlite3": "^7.5.0", 15 | "@types/cookie-parser": "^1.4.3", 16 | "@types/express": "^4.17.13", 17 | "@types/express-session": "^1.17.5", 18 | "@types/js-yaml": "^4.0.5", 19 | "@types/jsesc": "^3.0.1", 20 | "@types/json-stringify-safe": "^5.0.0", 21 | "@types/lodash.debounce": "^4.0.7", 22 | "@types/nanoid": "^3.0.0", 23 | "@types/papaparse": "^5.3.2", 24 | "@types/passport": "^1.0.9", 25 | "@types/react": "17", 26 | "@types/react-dom": "17", 27 | "@types/react-syntax-highlighter": "^15.5.3", 28 | "@typescript-eslint/eslint-plugin": "^5.30.7", 29 | "@typescript-eslint/parser": "^5.30.7", 30 | "@wojtekmaj/enzyme-adapter-react-17": "^0.6.7", 31 | "cross-env": "^7.0.3", 32 | "electron": "^19.0.8", 33 | "enzyme": "^3.11.0", 34 | "esbuild": "^0.14.49", 35 | "eslint": "^8.20.0", 36 | "eslint-plugin-jest": "^26.6.0", 37 | "eslint-plugin-react": "^7.30.1", 38 | "eslint-plugin-react-hooks": "^4.6.0", 39 | "jest": "^28.1.3", 40 | "prettier": "^2.7.1", 41 | "prettier-plugin-organize-imports": "^3.0.0", 42 | "typescript": "^4.7.4" 43 | }, 44 | "dependencies": { 45 | "core-js": "^3.23.5", 46 | "react-dom": "17" 47 | }, 48 | "resolutions": { 49 | "cheerio": "1.0.0-rc.3" 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /ee/scripts/require_copyright.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | code="0" 6 | for f in $(git ls-files); do 7 | if [[ "$f" == *".json" ]] || [[ "$f" == *".lock" ]] || [[ "$f" == *"ignore" ]] || [[ "$f" == ".eslintrc" ]]; then 8 | continue 9 | fi 10 | 11 | if ! [[ "$(cat $f)" == *"Copyright 2022 Multiprocess Labs LLC"* ]]; then 12 | echo "$f" 13 | code="1" 14 | fi 15 | done 16 | 17 | exit "$code" 18 | -------------------------------------------------------------------------------- /ee/shared/constants.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Multiprocess Labs LLC 2 | 3 | export const APP_NAME = 'DataStation Enterprise Edition'; 4 | -------------------------------------------------------------------------------- /ee/shared/rpc.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Multiprocess Labs LLC 2 | 3 | import * as rpc_ce from '../../shared/rpc'; 4 | import { History } from './state'; 5 | 6 | export type Endpoint = rpc_ce.Endpoint | 'getHistory'; 7 | 8 | export type GetHistoryRequest = { lastId?: string }; 9 | export type GetHistoryResponse = { history: Array }; 10 | -------------------------------------------------------------------------------- /ee/shared/state.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Multiprocess Labs LLC 2 | 3 | import { mergeDeep, newId } from '../../shared/object'; 4 | 5 | export class History { 6 | id: string; 7 | table: string; 8 | pk: string; 9 | dt: Date; 10 | oldValue: any; 11 | newValue: any; 12 | action: 'update' | 'insert' | 'delete'; 13 | error: string; 14 | userId: string; 15 | 16 | constructor(defaults: Partial = {}) { 17 | this.id = defaults.id || newId(); 18 | this.table = defaults.table || ''; 19 | this.pk = defaults.pk || ''; 20 | this.dt = defaults.dt || new Date(); 21 | this.oldValue = defaults.oldValue || null; 22 | this.newValue = defaults.newValue || null; 23 | this.error = defaults.error || ''; 24 | this.userId = defaults.userId || ''; 25 | this.action = defaults.action || 'update'; 26 | } 27 | 28 | static fromJSON(raw: any): History { 29 | raw = raw || {}; 30 | 31 | const his = mergeDeep(new History(), raw); 32 | if (typeof raw.dt === 'number') { 33 | raw.dt *= 1000; 34 | } 35 | his.dt = 36 | (['string', 'number'].includes(typeof raw.dt) 37 | ? new Date(raw.dt) 38 | : raw.dt) || his.dt; 39 | return his; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /ee/testsetup.js: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Multiprocess Labs LLC 2 | 3 | const { act } = require('react-dom/test-utils'); 4 | const { wait } = require('../shared/promise'); 5 | require('../shared/polyfill'); 6 | 7 | // https://enzymejs.github.io/enzyme/docs/guides/jsdom.html 8 | const { JSDOM } = require('jsdom'); 9 | const { configure } = require('enzyme'); 10 | const Adapter = require('@wojtekmaj/enzyme-adapter-react-17'); 11 | 12 | configure({ adapter: new Adapter() }); 13 | 14 | const jsdom = new JSDOM('', { 15 | url: 'http://localhost/?projectId=test', 16 | }); 17 | const { window } = jsdom; 18 | 19 | class ResizeObserver { 20 | observe() {} 21 | unobserve() {} 22 | disconnect() {} 23 | } 24 | 25 | window.ResizeObserver = ResizeObserver; 26 | 27 | function copyProps(src, target) { 28 | Object.defineProperties(target, { 29 | ...Object.getOwnPropertyDescriptors(src), 30 | ...Object.getOwnPropertyDescriptors(target), 31 | }); 32 | } 33 | 34 | global.window = window; 35 | global.document = window.document; 36 | global.navigator = { 37 | userAgent: 'node.js', 38 | }; 39 | global.requestAnimationFrame = function (callback) { 40 | return setTimeout(callback, 0); 41 | }; 42 | global.cancelAnimationFrame = function (id) { 43 | clearTimeout(id); 44 | }; 45 | copyProps(window, global); 46 | 47 | window.fetch = () => { 48 | return Promise.resolve({ 49 | text() { 50 | return Promise.resolve(null); 51 | }, 52 | json() { 53 | return Promise.resolve(null); 54 | }, 55 | }); 56 | }; 57 | 58 | global.componentLoad = async function (component) { 59 | await wait(1000); 60 | await act(async () => { 61 | await wait(0); 62 | component.update(); 63 | }); 64 | }; 65 | 66 | global.throwOnErrorBoundary = function (component) { 67 | component.find('ErrorBoundary').forEach((e) => { 68 | if (e.find({ type: 'fatal' }).length) { 69 | // Weird ways to find the actual error message 70 | throw new Error(e.find('Highlight').props().children); 71 | } 72 | }); 73 | }; 74 | -------------------------------------------------------------------------------- /ee/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "noImplicitAny": true, 4 | "noUnusedLocals": true, 5 | "noEmit": true, 6 | "jsx": "react", 7 | "allowJs": true, 8 | "target": "esnext", 9 | "module": "commonjs", 10 | "skipLibCheck": true, 11 | "esModuleInterop": true, 12 | "typeRoots": [ 13 | "../node_modules/@types", 14 | "../type-overrides", 15 | "./node_modules/@types", 16 | "./type-overrides" 17 | ], 18 | "resolveJsonModule": true 19 | }, 20 | "exclude": [ 21 | "node_modules", 22 | "*.js", 23 | "build", 24 | "../node_modules", 25 | "../build", 26 | "../*.js" 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /ee/type-overrides/ace.d.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Multiprocess Labs LLC 2 | 3 | declare module 'ace-builds/src-min-noconflict/ace'; 4 | declare module 'ace-builds/src-min-noconflict/ext-language_tools'; 5 | -------------------------------------------------------------------------------- /ee/ui/index.tsx: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Multiprocess Labs LLC 2 | 3 | import { IconHistory } from '@tabler/icons'; 4 | import React from 'react'; 5 | import ReactDOM from 'react-dom'; 6 | import { App, defaultRoutes } from '../../ui/app'; 7 | import { History } from './History'; 8 | 9 | // SOURCE: https://stackoverflow.com/a/7995898/1507139 10 | const isMobile = navigator.userAgent.match( 11 | /(iPad)|(iPhone)|(iPod)|(android)|(webOS)/i 12 | ); 13 | 14 | function index() { 15 | const root = document.getElementById('root'); 16 | if (document.location.pathname.startsWith('/dashboard')) { 17 | //ReactDOM.render(, root); 18 | return; 19 | } 20 | 21 | if (!isMobile) { 22 | const routes = [...defaultRoutes()]; 23 | // Place last before settings 24 | routes.splice(routes.length - 1, 0, { 25 | endpoint: 'history', 26 | view: History, 27 | title: 'History', 28 | icon: IconHistory, 29 | }); 30 | ReactDOM.render(, root); 31 | return; 32 | } 33 | 34 | root.innerHTML = 'Please use a desktop web browser to view this app.'; 35 | } 36 | 37 | index(); 38 | -------------------------------------------------------------------------------- /ee/ui/scripts/ui.build: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Multiprocess Labs LLC 2 | 3 | mkdir build 4 | # This is overwritten in production builds 5 | setenv_default UI_ESBUILD_ARGS "" 6 | yarn esbuild --target=es2020,chrome90,firefox90,safari13,edge90 ui/index.tsx --loader:.ts=tsx --loader:.js=jsx "--external:fs" --bundle --sourcemap {UI_ESBUILD_ARGS} --outfile=build/ui.js 7 | 8 | setenv_default UI_CONFIG_OVERRIDES "" 9 | prepend {UI_CONFIG_OVERRIDES} build/ui.js 10 | 11 | setenv_default UI_TITLE "DataStation Enterprise Edition" 12 | setenv_default UI_CSP "" 13 | setenv_default UI_ROOT "" 14 | render ../ui/index.html build/index.html 15 | node-sass ../ui/style.css ./build/style.css -------------------------------------------------------------------------------- /icon.icns: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/multiprocessio/datastation/4e72a6e0af28fcda767b0b29bb5dc9b697d05cb3/icon.icns -------------------------------------------------------------------------------- /icon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/multiprocessio/datastation/4e72a6e0af28fcda767b0b29bb5dc9b697d05cb3/icon.ico -------------------------------------------------------------------------------- /icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/multiprocessio/datastation/4e72a6e0af28fcda767b0b29bb5dc9b697d05cb3/icon.png -------------------------------------------------------------------------------- /integration/clickhouse.test.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const cp = require('child_process'); 3 | 4 | const { basicDatabaseTest } = require('../desktop/panel/testutil'); 5 | const { withDocker } = require('./docker'); 6 | 7 | const BASIC_TESTS = [ 8 | { 9 | type: 'clickhouse', 10 | query: `SELECT 1 AS "1", 2.2 AS "2", true AS "true", 'string' AS "string", parseDateTimeBestEffortOrNull('2021-01-01') AS "date"`, 11 | }, 12 | ]; 13 | 14 | describe('basic clickhouse tests', () => { 15 | for (const t of BASIC_TESTS) { 16 | test( 17 | t.query, 18 | async () => { 19 | await withDocker( 20 | { 21 | image: 'docker.io/yandex/clickhouse-server:22', 22 | port: 9000, 23 | env: { 24 | CLICKHOUSE_DB: 'test', 25 | CLICKHOUSE_USER: 'test', 26 | CLICKHOUSE_PASSWORD: 'test', 27 | }, 28 | cmds: [ 29 | `clickhouse-client -d test -u test --password test -q 'SELECT 1'`, 30 | ], 31 | }, 32 | () => 33 | basicDatabaseTest(t, { 34 | clickhouse: { 35 | database: 'test', 36 | username: 'test', 37 | password: 'test', 38 | address: 'localhost', 39 | }, 40 | }) 41 | ); 42 | }, 43 | 360_000 44 | ); 45 | } 46 | }); 47 | -------------------------------------------------------------------------------- /integration/cockroachdb.test.js: -------------------------------------------------------------------------------- 1 | const cp = require('child_process'); 2 | 3 | const { basicDatabaseTest } = require('../desktop/panel/testutil'); 4 | const { withDocker } = require('./docker'); 5 | 6 | const BASIC_TESTS = [ 7 | { 8 | type: 'cockroach', 9 | query: `SELECT 1 AS "1", 2.2 AS "2", true AS "true", 'string' AS "string", CAST('2021-01-01' AS DATE) AS "date"`, 10 | }, 11 | ]; 12 | 13 | describe('basic cockroachdb tests', () => { 14 | for (const t of BASIC_TESTS) { 15 | test( 16 | t.query, 17 | async () => { 18 | await withDocker( 19 | { 20 | image: 'docker.io/cockroachdb/cockroach:v22.1.6', 21 | port: 26257, 22 | args: ['--entrypoint', 'tail'], 23 | program: ['-f', '/dev/null'], 24 | cmds: [ 25 | `mkdir -p certs cockroach-safe`, 26 | `cockroach cert create-ca --certs-dir=certs --ca-key=cockroach-safe/ca.key`, 27 | `cockroach cert create-node localhost $(hostname) --certs-dir=certs --ca-key=cockroach-safe/ca.key`, 28 | `cockroach cert create-client root --certs-dir=certs --ca-key=cockroach-safe/ca.key`, 29 | `cockroach start-single-node --certs-dir=certs --accept-sql-without-tls --background`, 30 | `cockroach sql --certs-dir=certs --host=localhost:26257 --execute "CREATE DATABASE test; CREATE USER test WITH PASSWORD 'test'; GRANT ALL ON DATABASE test TO test;"`, 31 | ], 32 | }, 33 | () => basicDatabaseTest(t) 34 | ); 35 | }, 36 | 360_000 37 | ); 38 | } 39 | }); 40 | -------------------------------------------------------------------------------- /integration/cratedb.test.js: -------------------------------------------------------------------------------- 1 | const { basicDatabaseTest } = require('../desktop/panel/testutil'); 2 | const { withDocker } = require('./docker'); 3 | 4 | const BASIC_TESTS = [ 5 | { 6 | type: 'crate', 7 | query: `SELECT 1 AS "1", 2.2 AS "2", true AS "true", 'string' AS "string", CAST('2021-01-01' AS DATE) AS "date"`, 8 | }, 9 | ]; 10 | 11 | const vendorOverride = { 12 | crate: { 13 | address: 'localhost:5439?sslmode=disable', 14 | database: 'doc', 15 | }, 16 | }; 17 | 18 | describe('basic cratedb tests', () => { 19 | for (const t of BASIC_TESTS) { 20 | test( 21 | t.query, 22 | async () => { 23 | await withDocker( 24 | { 25 | image: 'docker.io/library/crate:5.0.0', 26 | port: '5439:5432', 27 | program: ['crate', '-Cdiscovery.type=single-node'], 28 | cmds: [ 29 | `crash -c "CREATE USER test WITH (password = 'test');"`, 30 | `crash -c "GRANT ALL PRIVILEGES ON SCHEMA doc TO test;"`, 31 | ], 32 | }, 33 | () => basicDatabaseTest(t, vendorOverride) 34 | ); 35 | }, 36 | 360_000 37 | ); 38 | } 39 | }); 40 | -------------------------------------------------------------------------------- /integration/mysql.test.js: -------------------------------------------------------------------------------- 1 | const cp = require('child_process'); 2 | 3 | const { basicDatabaseTest } = require('../desktop/panel/testutil'); 4 | const { withDocker } = require('./docker'); 5 | 6 | const BASIC_TESTS = [ 7 | { 8 | type: 'mysql', 9 | query: 10 | 'SELECT 1 AS `1`, 2.2 AS `2`, true AS `true`, "string" AS `string`, CAST("2021-01-01" AS DATE) AS `date`', 11 | }, 12 | { 13 | type: 'mysql', 14 | query: 15 | 'SELECT name, CAST(age AS SIGNED) - 10 AS age, `location.city` AS city FROM DM_getPanel(0)', 16 | }, 17 | ]; 18 | 19 | describe('basic mysql tests', () => { 20 | for (const t of BASIC_TESTS) { 21 | test( 22 | t.query, 23 | async () => { 24 | await withDocker( 25 | { 26 | image: 'docker.io/library/mysql:8.0.30', 27 | port: '3306', 28 | env: { 29 | MYSQL_ROOT_PASSWORD: 'root', 30 | }, 31 | cmds: [ 32 | `mysql -h localhost -uroot -proot --execute="CREATE USER 'test'@'%' IDENTIFIED BY 'test';"`, 33 | `mysql -h localhost -uroot -proot --execute="CREATE DATABASE test;"`, 34 | `mysql -h localhost -uroot -proot --execute="GRANT ALL PRIVILEGES ON test.* TO 'test'@'%';"`, 35 | ], 36 | }, 37 | () => basicDatabaseTest(t) 38 | ); 39 | }, 40 | 360_000 41 | ); 42 | } 43 | }); 44 | -------------------------------------------------------------------------------- /integration/postgres.test.js: -------------------------------------------------------------------------------- 1 | const { basicDatabaseTest } = require('../desktop/panel/testutil'); 2 | const { withDocker } = require('./docker'); 3 | 4 | const BASIC_TESTS = [ 5 | { 6 | type: 'postgres', 7 | query: `SELECT 1 AS "1", 2.2 AS "2", true AS "true", 'string' AS "string", CAST('2021-01-01' AS DATE) AS "date"`, 8 | }, 9 | { 10 | type: 'postgres', 11 | query: 12 | 'SELECT name, CAST(age AS INT) - 10 AS age, "location.city" AS city FROM DM_getPanel(0)', 13 | }, 14 | ]; 15 | 16 | const vendorOverride = { 17 | postgres: { 18 | address: 'localhost?sslmode=disable', 19 | }, 20 | }; 21 | 22 | describe('basic postgres tests', () => { 23 | for (const t of BASIC_TESTS) { 24 | test( 25 | t.query, 26 | async () => { 27 | await withDocker( 28 | { 29 | image: 'docker.io/library/postgres:14', 30 | port: '5432', 31 | env: { 32 | POSTGRES_USER: 'test', 33 | POSTGRES_DB: 'test', 34 | POSTGRES_PASSWORD: 'test', 35 | }, 36 | }, 37 | () => basicDatabaseTest(t, vendorOverride) 38 | ); 39 | }, 40 | 360_000 41 | ); 42 | } 43 | }); 44 | -------------------------------------------------------------------------------- /integration/questdb.test.js: -------------------------------------------------------------------------------- 1 | const { basicDatabaseTest } = require('../desktop/panel/testutil'); 2 | const { withDocker } = require('./docker'); 3 | 4 | const BASIC_TESTS = [ 5 | { 6 | type: 'quest', 7 | query: `SELECT 1 AS "1", 2.2 AS "2", true AS "true", 'string' AS "string", CAST('2021-01-01' AS TIMESTAMP) AS "date"`, 8 | }, 9 | ]; 10 | 11 | const vendorOverride = { 12 | quest: { 13 | address: '?sslmode=disable', 14 | database: 'qdb', 15 | username: 'admin', 16 | password: 'quest', 17 | }, 18 | }; 19 | 20 | describe('basic questdb tests', () => { 21 | for (const t of BASIC_TESTS) { 22 | test( 23 | t.query, 24 | async () => { 25 | await withDocker( 26 | { 27 | image: 'docker.io/questdb/questdb:6.5', 28 | port: '8812', 29 | }, 30 | () => basicDatabaseTest(t, vendorOverride) 31 | ); 32 | }, 33 | 360_000 34 | ); 35 | } 36 | }); 37 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | // Only on linux do all tests run 3 | coverageThreshold: 4 | process.platform === 'linux' 5 | ? process.argv.includes('integration/credential_database.test.js') 6 | ? { 7 | global: { 8 | statements: 15, 9 | branches: 13, 10 | lines: 15, 11 | functions: 9, 12 | }, 13 | } 14 | : { 15 | global: { 16 | statements: 54, 17 | branches: 43, 18 | functions: 37, 19 | lines: 55, 20 | }, 21 | } 22 | : undefined, 23 | preset: 'ts-jest/presets/js-with-ts', 24 | transformIgnorePatterns: [ 25 | 'node_modules/(?!react-syntax-highlighter|refractor|node-fetch|fetch-blob)', 26 | ], 27 | setupFiles: ['./shared/polyfill.ts', './testsetup.js'], 28 | testEnvironment: 'node', 29 | testEnvironmentOptions: { 30 | url: 'http://localhost/', 31 | }, 32 | collectCoverageFrom: [ 33 | 'ui/**/*.ts', 34 | 'ui/**/*.tsx', 35 | 'shared/**/*.ts', 36 | 'desktop/**/*.ts', 37 | 'server/**/*.ts', 38 | 'server/**/*.tsx', 39 | ], 40 | modulePathIgnorePatterns: ['/releases/', '/build/', 'ee/'], 41 | }; 42 | -------------------------------------------------------------------------------- /runner/.gitignore: -------------------------------------------------------------------------------- 1 | runner 2 | main 3 | *build 4 | -------------------------------------------------------------------------------- /runner/cmd/dsq/README.md: -------------------------------------------------------------------------------- 1 | # dsq: Run SQL queries against JSON, CSV, Excel, Parquet, and more 2 | 3 | This has been [moved to its own repo](https://github.com/multiprocessio/dsq). -------------------------------------------------------------------------------- /runner/cmd/main_test.go: -------------------------------------------------------------------------------- 1 | //go:build testrunmain 2 | // +build testrunmain 3 | 4 | package main 5 | 6 | import "testing" 7 | 8 | func TestRunMain(t *testing.T) { 9 | main() 10 | } 11 | -------------------------------------------------------------------------------- /runner/database_airtable.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "net/url" 8 | "strings" 9 | ) 10 | 11 | type airtableResponse struct { 12 | Offset string `json:"offset"` 13 | Records []struct { 14 | Fields map[string]any `json:"fields"` 15 | } `json:"records"` 16 | } 17 | 18 | func (ec EvalContext) evalAirtable(panel *PanelInfo, dbInfo DatabaseConnectorInfoDatabase, w *ResultWriter) error { 19 | token, err := ec.decrypt(&dbInfo.ApiKey) 20 | if err != nil { 21 | return edse(err) 22 | } 23 | 24 | app := strings.TrimSpace(panel.Database.Extra["airtable_app"]) 25 | table := strings.TrimSpace(panel.Database.Table) 26 | 27 | v := url.Values{} 28 | 29 | view := strings.TrimSpace(panel.Database.Extra["airtable_view"]) 30 | if view != "" { 31 | v.Add("view", view) 32 | } 33 | 34 | q := strings.TrimSpace(panel.Content) 35 | if q != "" { 36 | v.Add("filterByFormula", q) 37 | } 38 | 39 | baseUrl := fmt.Sprintf("https://api.airtable.com/v0/%s/%s?%s", app, table, v.Encode()) 40 | 41 | var r airtableResponse 42 | offset := "" 43 | for { 44 | offsetParam := "" 45 | if offset != "" { 46 | offsetParam = "&offset=" + url.QueryEscape(offset) 47 | } 48 | rsp, err := makeHTTPRequest(httpRequest{ 49 | url: baseUrl + offsetParam, 50 | headers: []HttpConnectorInfoHeader{ 51 | { 52 | Name: "Authorization", 53 | Value: "Bearer " + token, 54 | }, 55 | }, 56 | }) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | defer rsp.Body.Close() 62 | 63 | if rsp.StatusCode >= 400 { 64 | b, _ := io.ReadAll(rsp.Body) 65 | return makeErrUser(string(b)) 66 | } 67 | 68 | err = json.NewDecoder(rsp.Body).Decode(&r) 69 | if err != nil { 70 | return err 71 | } 72 | 73 | for _, record := range r.Records { 74 | err = w.WriteRow(record.Fields) 75 | if err != nil { 76 | return edse(err) 77 | } 78 | } 79 | 80 | if offset == r.Offset || r.Offset == "" { 81 | break 82 | } 83 | 84 | offset = r.Offset 85 | } 86 | 87 | return nil 88 | } 89 | -------------------------------------------------------------------------------- /runner/database_bigquery.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "context" 5 | 6 | "cloud.google.com/go/bigquery" 7 | "google.golang.org/api/iterator" 8 | "google.golang.org/api/option" 9 | ) 10 | 11 | func (ec EvalContext) evalBigQuery(panel *PanelInfo, dbInfo DatabaseConnectorInfoDatabase, w *ResultWriter) error { 12 | ctx := context.Background() 13 | 14 | token, err := ec.decrypt(&dbInfo.ApiKey) 15 | if err != nil { 16 | return err 17 | } 18 | 19 | client, err := bigquery.NewClient(ctx, dbInfo.Database, option.WithCredentialsJSON([]byte(token))) 20 | if err != nil { 21 | return err 22 | } 23 | 24 | q := client.Query(panel.Content) 25 | it, err := q.Read(ctx) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | var fields []string 31 | var values []bigquery.Value 32 | 33 | for { 34 | err := it.Next(&values) 35 | if err == iterator.Done { 36 | return nil 37 | } 38 | if err != nil { 39 | return err 40 | } 41 | 42 | if len(fields) == 0 { 43 | // it.Schema is only populated after the first call to it.Next() 44 | for _, field := range it.Schema { 45 | fields = append(fields, field.Name) 46 | } 47 | w.SetFields(fields) 48 | } 49 | 50 | // bigquery seems to return more values than fields 51 | values = values[:len(fields)] 52 | 53 | err = w.WriteBigQueryRecord(values, false) 54 | if err != nil { 55 | return err 56 | } 57 | 58 | // Zeroes out "values" while preserving its capacity 59 | values = values[:0] 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /runner/database_cql.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "github.com/gocql/gocql" 5 | ) 6 | 7 | func (ec EvalContext) evalCQL(panel *PanelInfo, dbInfo DatabaseConnectorInfoDatabase, server *ServerInfo, w *ResultWriter) error { 8 | _, host, port, _, err := getHTTPHostPort(dbInfo.Address) 9 | if err != nil { 10 | return err 11 | } 12 | 13 | password, err := ec.decrypt(&dbInfo.Password) 14 | if err != nil { 15 | return err 16 | } 17 | 18 | return ec.withRemoteConnection(server, host, port, func(proxyHost, proxyPort string) error { 19 | cluster := gocql.NewCluster(proxyHost + ":" + proxyPort) 20 | cluster.Keyspace = dbInfo.Database 21 | cluster.Consistency = gocql.Quorum 22 | if password != "" { 23 | cluster.Authenticator = gocql.PasswordAuthenticator{ 24 | Username: dbInfo.Username, 25 | Password: password, 26 | } 27 | } 28 | 29 | sess, err := cluster.CreateSession() 30 | if err != nil { 31 | return err 32 | } 33 | defer sess.Close() 34 | 35 | iter := sess.Query(panel.Content).Iter() 36 | for { 37 | // TODO: Can we reuse this map? 38 | row := map[string]any{} 39 | if !iter.MapScan(row) { 40 | break 41 | } 42 | err := w.WriteRow(row) 43 | if err != nil { 44 | return err 45 | } 46 | } 47 | 48 | return iter.Close() 49 | }) 50 | } 51 | -------------------------------------------------------------------------------- /runner/database_mongo.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os/exec" 7 | ) 8 | 9 | func (ec EvalContext) evalMongo(panel *PanelInfo, dbInfo DatabaseConnectorInfoDatabase, server *ServerInfo, w *ResultWriter) error { 10 | _, conn, err := ec.getConnectionString(dbInfo) 11 | if err != nil { 12 | return err 13 | } 14 | 15 | authDB, ok := dbInfo.Extra["authenticationDatabase"] 16 | if !ok { 17 | authDB = "admin" 18 | } 19 | 20 | // EJSON.stringify required to make it possible to process output 21 | eval := fmt.Sprintf("'EJSON.stringify(%s)'", panel.Content) 22 | 23 | args := []string{conn, "--quiet", "--authenticationDatabase", authDB, "--eval", eval} 24 | stdout, err := exec.Command("mongosh", args...).Output() 25 | if err != nil { 26 | log.Println(err) 27 | if exitErr, ok := err.(*exec.ExitError); ok { 28 | log.Println(exitErr, ok) 29 | return makeErrUser(string(exitErr.Stderr)) 30 | } 31 | return err 32 | } 33 | 34 | var m any 35 | err = jsonUnmarshal(stdout, &m) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | rows, ok := m.([]any) 41 | if !ok { 42 | jw := w.w.(*JSONResultItemWriter) 43 | jw.raw = true 44 | o := jw.bfd 45 | enc := jsonNewEncoder(o) 46 | 47 | return enc.Encode(&m) 48 | } 49 | 50 | for _, row := range rows { 51 | if err := w.WriteRow(row); err != nil { 52 | return err 53 | } 54 | } 55 | 56 | return nil 57 | } 58 | -------------------------------------------------------------------------------- /runner/database_neo4j.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "github.com/neo4j/neo4j-go-driver/v4/neo4j" 5 | ) 6 | 7 | func (ec EvalContext) evalNeo4j(panel *PanelInfo, dbInfo DatabaseConnectorInfoDatabase, server *ServerInfo, w *ResultWriter) error { 8 | _, conn, err := ec.getConnectionString(dbInfo) 9 | if err != nil { 10 | return err 11 | } 12 | 13 | password, err := ec.decrypt(&dbInfo.Password) 14 | if err != nil { 15 | return err 16 | } 17 | 18 | driver, err := neo4j.NewDriver(conn, neo4j.BasicAuth(dbInfo.Username, password, "")) 19 | if err != nil { 20 | return err 21 | } 22 | defer driver.Close() 23 | 24 | sess := driver.NewSession(neo4j.SessionConfig{}) 25 | 26 | result, err := sess.Run(panel.Content, nil) 27 | if err != nil { 28 | return err 29 | } 30 | 31 | records, err := result.Collect() 32 | if err != nil { 33 | return nil 34 | } 35 | 36 | row := map[string]any{} 37 | for _, record := range records { 38 | for _, key := range record.Keys { 39 | value, ok := record.Get(key) 40 | if ok { 41 | row[key] = value 42 | } else { 43 | row[key] = nil 44 | } 45 | } 46 | 47 | if err := w.WriteRow(row); err != nil { 48 | return err 49 | } 50 | } 51 | 52 | return result.Err() 53 | } 54 | -------------------------------------------------------------------------------- /runner/database_splunk.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | func evalSplunk(panel *PanelInfo, dbInfo DatabaseConnectorInfoDatabase, server *ServerInfo, w *ResultWriter) error { 4 | panic("Not implemented") 5 | } 6 | -------------------------------------------------------------------------------- /runner/database_sqlite.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "github.com/multiprocessio/go-sqlite3-stdlib" 5 | ) 6 | 7 | func init() { 8 | stdlib.Register("sqlite3_extended") 9 | } 10 | -------------------------------------------------------------------------------- /runner/http_test.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func Test_getHTTPHostPort(t *testing.T) { 10 | tests := []struct { 11 | url string 12 | expTls bool 13 | expHost string 14 | expPort string 15 | expErr error 16 | }{ 17 | { 18 | "localhost", 19 | false, 20 | "localhost", 21 | "80", 22 | nil, 23 | }, 24 | { 25 | ":80", 26 | false, 27 | "localhost", 28 | "80", 29 | nil, 30 | }, 31 | { 32 | "http://localhost", 33 | false, 34 | "localhost", 35 | "80", 36 | nil, 37 | }, 38 | { 39 | "https://localhost", 40 | true, 41 | "localhost", 42 | "443", 43 | nil, 44 | }, 45 | { 46 | "https://foo.com", 47 | true, 48 | "foo.com", 49 | "443", 50 | nil, 51 | }, 52 | { 53 | "https://foo.com:444", 54 | true, 55 | "foo.com", 56 | "444", 57 | nil, 58 | }, 59 | { 60 | "foo.com:444", 61 | false, 62 | "foo.com", 63 | "444", 64 | nil, 65 | }, 66 | { 67 | "/xyz", 68 | false, 69 | "localhost", 70 | "80", 71 | nil, 72 | }, 73 | { 74 | ":443/xyz", 75 | true, 76 | "localhost", 77 | "443", 78 | nil, 79 | }, 80 | { 81 | ":90/xyz", 82 | false, 83 | "localhost", 84 | "90", 85 | nil, 86 | }, 87 | } 88 | 89 | for _, ts := range tests { 90 | tls, host, port, _, err := getHTTPHostPort(ts.url) 91 | assert.Equal(t, ts.expTls, tls, ts.url) 92 | assert.Equal(t, ts.expHost, host, ts.url) 93 | assert.Equal(t, ts.expPort, port, ts.url) 94 | assert.Equal(t, ts.expErr, err, ts.url) 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /runner/literal.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import "bytes" 4 | 5 | func (ec EvalContext) evalLiteralPanel(project *ProjectState, pageIndex int, panel *PanelInfo) error { 6 | cti := panel.Literal.ContentTypeInfo 7 | 8 | rw, err := ec.GetResultWriter(project.Id, panel.Id) 9 | if err != nil { 10 | return err 11 | } 12 | defer rw.Close() 13 | 14 | buf := bytes.NewReader([]byte(panel.Content)) 15 | br := newBufferedReader(buf) 16 | 17 | return TransformReader(br, "", cti, rw) 18 | } 19 | -------------------------------------------------------------------------------- /runner/literal_test.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func Test_transformCSV_literal(t *testing.T) { 11 | tests := []struct { 12 | in string 13 | out []map[string]any 14 | }{ 15 | { 16 | `name,age 17 | ted,10 18 | elsa,12`, 19 | []map[string]any{ 20 | {"name": "ted", "age": "10"}, 21 | {"name": "elsa", "age": "12"}, 22 | }, 23 | }, 24 | } 25 | 26 | ec, cleanup := makeTestEvalContext() 27 | defer cleanup() 28 | 29 | projectTmp, err := os.CreateTemp("", "dsq-project") 30 | assert.Nil(t, err) 31 | defer os.Remove(projectTmp.Name()) 32 | 33 | project := &ProjectState{ 34 | Id: projectTmp.Name(), 35 | Pages: []ProjectPage{ 36 | { 37 | Panels: []PanelInfo{{ 38 | Id: newId(), 39 | Type: LiteralPanel, 40 | LiteralPanelInfo: &LiteralPanelInfo{ 41 | Literal: LiteralPanelInfoLiteral{ 42 | ContentTypeInfo: ContentTypeInfo{ 43 | Type: "text/csv", 44 | }, 45 | }, 46 | }, 47 | }}, 48 | }, 49 | }, 50 | } 51 | 52 | for _, test := range tests { 53 | project.Pages[0].Panels[0].Content = test.in 54 | 55 | err = ec.evalLiteralPanel(project, 0, &project.Pages[0].Panels[0]) 56 | assert.Nil(t, err) 57 | 58 | var m []map[string]any 59 | out := ec.GetPanelResultsFile(project.Id, project.Pages[0].Panels[0].Id) 60 | outTmpBs, err := os.ReadFile(out) 61 | assert.Nil(t, err) 62 | err = jsonUnmarshal(outTmpBs, &m) 63 | assert.Nil(t, err) 64 | 65 | assert.Equal(t, test.out, m) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /runner/odbc_unix.go: -------------------------------------------------------------------------------- 1 | //go:build darwin || linux || freebsd 2 | 3 | package runner 4 | 5 | import ( 6 | "plugin" 7 | 8 | "github.com/jmoiron/sqlx" 9 | ) 10 | 11 | // On UNIX systems, openODBCDriver loads a Go plugin to prevent the runtime dependency on unixODBC. 12 | func openODBCDriver(conn string) (*sqlx.DB, error) { 13 | pl, err := plugin.Open("./runner/plugins/odbc/odbc.plugin") 14 | if err != nil { 15 | return nil, err 16 | } 17 | 18 | openDriver, err := pl.Lookup("OpenODBCDriver") 19 | if err != nil { 20 | return nil, err 21 | } 22 | 23 | return openDriver.(func(string) (*sqlx.DB, error))(conn) 24 | } 25 | -------------------------------------------------------------------------------- /runner/odbc_windows.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | _ "github.com/alexbrainman/odbc" 5 | "github.com/jmoiron/sqlx" 6 | ) 7 | 8 | func openODBCDriver(conn string) (*sqlx.DB, error) { 9 | return sqlx.Open("odbc", conn) 10 | } 11 | -------------------------------------------------------------------------------- /runner/plugins/odbc/.gitignore: -------------------------------------------------------------------------------- 1 | *.plugin -------------------------------------------------------------------------------- /runner/plugins/odbc/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | # Download unixODBC source code and compile 6 | curl -LO http://www.unixodbc.org/unixODBC-2.3.11.tar.gz 7 | tar -xvf unixODBC-2.3.11.tar.gz 8 | cd unixODBC-2.3.11 9 | ./configure 10 | make 11 | sudo make install 12 | cd .. && sudo rm -rf unixODBC-2.3.11 13 | 14 | race="" 15 | if [[ "$1" == "-race" ]]; then 16 | race="-race" 17 | fi 18 | 19 | # Compile Go plugin 20 | cd runner/plugins/odbc && go build -trimpath -buildmode=plugin -mod=readonly -modcacherw -ldflags="-s -w" $race -o ./odbc.plugin ./odbc.go && cd ../../.. 21 | -------------------------------------------------------------------------------- /runner/plugins/odbc/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/multiprocessio/datastation/plugins/odbc 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/alexbrainman/odbc v0.0.0-20211220213544-9c9a2e61c5e2 7 | github.com/jmoiron/sqlx v1.3.5 8 | ) 9 | 10 | require golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 // indirect 11 | -------------------------------------------------------------------------------- /runner/plugins/odbc/go.sum: -------------------------------------------------------------------------------- 1 | github.com/alexbrainman/odbc v0.0.0-20211220213544-9c9a2e61c5e2 h1:090cWAt7zsbdvRegKCBVwcCTghjxhUh1PK2KNSq82vw= 2 | github.com/alexbrainman/odbc v0.0.0-20211220213544-9c9a2e61c5e2/go.mod h1:c5eyz5amZqTKvY3ipqerFO/74a/8CYmXOahSr40c+Ww= 3 | github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= 4 | github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= 5 | github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= 6 | github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 7 | github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= 8 | github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= 9 | github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= 10 | github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 11 | github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= 12 | github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= 13 | golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo= 14 | golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 15 | -------------------------------------------------------------------------------- /runner/plugins/odbc/odbc.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | _ "github.com/alexbrainman/odbc" 5 | "github.com/jmoiron/sqlx" 6 | ) 7 | 8 | func OpenODBCDriver(conn string) (*sqlx.DB, error) { return sqlx.Open("odbc", conn) } 9 | -------------------------------------------------------------------------------- /runner/scripts/generate_program_type_info.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | echo "package runner 6 | 7 | // GENERATED BY ./runner/scripts/generate_program_type_info.sh. DO NOT MODIFY. 8 | 9 | var packedProgramTypeInfo = map[SupportedLanguages]string{" > runner/program_info.go 10 | for file in $(ls ./shared/languages/*.json); do 11 | echo "$(cat $file | jq '.id'): \`$(cat $file | jq -c '.')\`," >> runner/program_info.go 12 | done 13 | 14 | echo "}" >> runner/program_info.go 15 | 16 | gofmt -w -s . 17 | -------------------------------------------------------------------------------- /runner/scripts/runner_test.build: -------------------------------------------------------------------------------- 1 | yarn esbuild desktop/runner.ts --bundle --platform=node --sourcemap --external:sqlite3 --external:asar --external:react-native-fs --external:react-native-fetch-blob --external:oracledb "--external:@elastic/elasticsearch" "--external:wasm-brotli" --external:prometheus-query --external:snowflake-sdk --external:ssh2 --external:ssh2-promise --external:ssh2-sftp-client --external:cpu-features --external:electron --external:pg-native --target=node10.4 --outfile=build/desktop_runner.js 2 | 3 | cd runner/cmd && go test -trimpath -mod=readonly -modcacherw -ldflags="-s -w" -race -o ../../build/go_desktop_runner_test{required_ext} -coverpkg="." -c -tags testrunmain -------------------------------------------------------------------------------- /runner/scripts/test_coverage.sh: -------------------------------------------------------------------------------- 1 | set -eu 2 | 3 | gocovmerge coverage/go*.cov > runner/all.out 4 | cd runner 5 | gocov convert all.out | gocov report | tee report 6 | perc="$(tail report -n1 | cut -d ' ' -f 3 | cut -d '.' -f 1)" 7 | 8 | # Bump up as needed 9 | MINIMUM_COVERAGE="80" 10 | 11 | if (( "$perc" < "$MINIMUM_COVERAGE" )); then 12 | echo "Code coverage ($perc) below minimum ($MINIMUM_COVERAGE)" 13 | exit 1 14 | fi 15 | 16 | echo "Code coverage is ok!" 17 | -------------------------------------------------------------------------------- /runner/settings.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "path" 5 | ) 6 | 7 | type LanguageSettings struct { 8 | Path string `json:"path"` 9 | } 10 | 11 | type Theme string 12 | 13 | const ( 14 | LightTheme Theme = "light" 15 | DarkTheme Theme = "dark" 16 | ) 17 | 18 | type Settings struct { 19 | Id string `json:"id"` 20 | LastProject *string `json:"lastProject"` 21 | Languages map[SupportedLanguages]LanguageSettings `json:"languages"` 22 | File string `json:"file"` 23 | StdoutMaxSize int `json:"stdoutMaxSize"` 24 | Theme Theme `json:"theme"` 25 | CaCerts []struct { 26 | File string `json:"file"` 27 | } `json:"caCerts"` 28 | } 29 | 30 | var SettingsFileDefaultLocation = path.Join(CONFIG_FS_BASE, ".settings") 31 | 32 | var DefaultSettings = &Settings{ 33 | Id: newId(), 34 | File: SettingsFileDefaultLocation, 35 | StdoutMaxSize: 5000, 36 | Theme: "light", 37 | } 38 | 39 | func LoadSettings(file string) (*Settings, error) { 40 | var settings Settings 41 | err := readJSONFileInto(file, &settings) 42 | return &settings, err 43 | } 44 | -------------------------------------------------------------------------------- /runner/util.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | nanoid "github.com/matoous/go-nanoid/v2" 5 | ) 6 | 7 | func newId() string { 8 | id, err := nanoid.Generate("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", 12) 9 | if err != nil { 10 | // This really can't be possible and would be a huge problem if it happened. 11 | panic(err) 12 | } 13 | 14 | return id 15 | } 16 | -------------------------------------------------------------------------------- /runner/vector.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import "math" 4 | 5 | type Vector[T any] struct { 6 | data []T 7 | index int 8 | } 9 | 10 | func (v *Vector[T]) Insert(i int, item T) { 11 | if len(v.data) == 0 { 12 | v.data = make([]T, 8) 13 | } 14 | 15 | if i >= len(v.data) { 16 | rest := make([]T, int(math.Floor(float64(len(v.data))*.75))) 17 | v.data = append(v.data, rest...) 18 | } 19 | 20 | v.data[i] = item 21 | } 22 | 23 | func (v *Vector[T]) Reset() { 24 | v.index = 0 25 | } 26 | 27 | func (v *Vector[T]) Append(item T) { 28 | v.Insert(v.index, item) 29 | v.index++ 30 | } 31 | 32 | func (v *Vector[T]) List() []T { 33 | return v.data[:v.index] 34 | } 35 | 36 | func (v *Vector[T]) Index() int { 37 | return v.index 38 | } 39 | 40 | func (v *Vector[T]) Pop() *T { 41 | if v.index == 0 { 42 | return nil 43 | } 44 | 45 | v.index-- 46 | return &v.data[v.index] 47 | } 48 | 49 | func (v *Vector[T]) At(i int) *T { 50 | for i < 0 && v.index != 0 { 51 | i += v.index 52 | } 53 | 54 | if i < 0 { 55 | return nil 56 | } 57 | 58 | if i >= v.index { 59 | return nil 60 | } 61 | 62 | return &v.data[i] 63 | } 64 | -------------------------------------------------------------------------------- /sampledata/README.md: -------------------------------------------------------------------------------- 1 | # Sample Data 2 | 3 | ## Hudson River Park Flora - Plantings: 1997 - 2015 4 | 5 | File: [Hudson_River_Park_Flora_-_Plantings___1997_-_2015.csv](./Hudson_River_Park_Flora_-_Plantings___1997_-_2015.csv) 6 | 7 | Source: https://catalog.data.gov/dataset/hudson-river-park-flora-plantings-beginning-1997. 8 | 9 | No license. 10 | 11 | ## nginx JSON logs 12 | 13 | File: [nginx_json_logs](./nginx_json_logs) 14 | 15 | Source: https://github.com/elastic/examples/tree/master/Common%20Data%20Formats/nginx_json_logs 16 | 17 | License: [Apache 2.0](https://github.com/elastic/examples/blob/master/LICENSE) 18 | 19 | ## Washington Broadband Speed Tests 20 | 21 | File: [speedtests.db](./speedtests.db) 22 | 23 | Source: https://catalog.data.gov/dataset/broadband-speed-tests 24 | 25 | No license. 26 | -------------------------------------------------------------------------------- /sampledata/speedtests.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/multiprocessio/datastation/4e72a6e0af28fcda767b0b29bb5dc9b697d05cb3/sampledata/speedtests.db -------------------------------------------------------------------------------- /screens/datastation-0.6.0-file-demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/multiprocessio/datastation/4e72a6e0af28fcda767b0b29bb5dc9b697d05cb3/screens/datastation-0.6.0-file-demo.gif -------------------------------------------------------------------------------- /screens/datastation-0.7.0-file-demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/multiprocessio/datastation/4e72a6e0af28fcda767b0b29bb5dc9b697d05cb3/screens/datastation-0.7.0-file-demo.gif -------------------------------------------------------------------------------- /screens/datastation-0.9.0-cockroach-pandas.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/multiprocessio/datastation/4e72a6e0af28fcda767b0b29bb5dc9b697d05cb3/screens/datastation-0.9.0-cockroach-pandas.gif -------------------------------------------------------------------------------- /screens/the-basics-code-panel1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/multiprocessio/datastation/4e72a6e0af28fcda767b0b29bb5dc9b697d05cb3/screens/the-basics-code-panel1.png -------------------------------------------------------------------------------- /screens/the-basics-database-panel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/multiprocessio/datastation/4e72a6e0af28fcda767b0b29bb5dc9b697d05cb3/screens/the-basics-database-panel.png -------------------------------------------------------------------------------- /screens/the-basics-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/multiprocessio/datastation/4e72a6e0af28fcda767b0b29bb5dc9b697d05cb3/screens/the-basics-graph.png -------------------------------------------------------------------------------- /screens/the-basics-group-by.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/multiprocessio/datastation/4e72a6e0af28fcda767b0b29bb5dc9b697d05cb3/screens/the-basics-group-by.png -------------------------------------------------------------------------------- /screens/the-basics-post-http.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/multiprocessio/datastation/4e72a6e0af28fcda767b0b29bb5dc9b697d05cb3/screens/the-basics-post-http.png -------------------------------------------------------------------------------- /screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/multiprocessio/datastation/4e72a6e0af28fcda767b0b29bb5dc9b697d05cb3/screenshot.png -------------------------------------------------------------------------------- /scripts/ci/prepare_go.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | # Set up Go 6 | sudo curl -LO https://go.dev/dl/go1.19.linux-amd64.tar.gz 7 | sudo rm -rf /usr/local/go 8 | sudo tar -C /usr/local -xzf go1.19.linux-amd64.tar.gz 9 | sudo ln -s /usr/local/go/bin/go /usr/local/bin/go 10 | sudo ln -s /usr/local/go/bin/gofmt /usr/local/bin/gofmt 11 | -------------------------------------------------------------------------------- /scripts/ci/prepare_linux.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # IMPORTANT: This file should only include production 5 | # dependencies. Dependencies for integration tests should be in the 6 | # ./prepare_linux_integration_test_setup_only.sh file in this 7 | # directory. 8 | # 9 | 10 | set -ex 11 | 12 | function retry { 13 | ok="false" 14 | for i in $(seq $1); do 15 | if bash -c "$2" ; then 16 | ok="true" 17 | break 18 | fi 19 | 20 | echo "Retrying... $2" 21 | sleep 5s 22 | done 23 | 24 | if [[ "$ok" == "false" ]]; then 25 | echo "Failed after retries... $2" 26 | exit 1 27 | fi 28 | } 29 | 30 | # Set up Node.js, jq 31 | curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash - 32 | sudo apt-get update -y 33 | sudo apt-get install -y nodejs cmake 34 | 35 | # Set up Go 36 | ./scripts/ci/prepare_go.sh 37 | # 38 | # IMPORTANT: Only run in Github CI. 39 | # 40 | if [[ "$1" == "--integration-tests" ]]; then 41 | ./scripts/ci/prepare_linux_integration_test_setup_only.sh 42 | fi 43 | 44 | # Set up project 45 | sudo npm install --global yarn 46 | retry 5 yarn 47 | 48 | race="" 49 | if [[ "$1" == "--integration-tests" ]] || [[ "$1" == "--race" ]]; then 50 | race="-race" 51 | fi 52 | 53 | # Install ODBC 54 | ./runner/plugins/odbc/build.sh $race 55 | -------------------------------------------------------------------------------- /scripts/ci/prepare_macos.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eux 4 | 5 | /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" 6 | brew install cmake jq r julia node@16 npm go-jsonnet 7 | brew link --overwrite node@16 8 | 9 | # Install go 10 | sudo curl -LO https://go.dev/dl/go1.19.darwin-amd64.tar.gz 11 | sudo rm -rf /usr/local/go 12 | sudo tar -C /usr/local -xzf go1.19.darwin-amd64.tar.gz 13 | sudo mv /usr/local/go/bin/go /usr/local/bin/go 14 | sudo mv /usr/local/go/bin/gofmt /usr/local/bin/gofmt 15 | 16 | # Install Go helpers 17 | # Failing: https://github.com/google/go-jsonnet/issues/596 18 | # go install github.com/google/go-jsonnet/cmd/jsonnet@latest 19 | go install github.com/multiprocessio/httpmirror@latest 20 | cp ~/go/bin/httpmirror /usr/local/bin/httpmirror 21 | 22 | # Install JavaScript deps 23 | npm install --global yarn 24 | yarn 25 | 26 | # Install ODBC Driver 27 | brew tap microsoft/mssql-release https://github.com/Microsoft/homebrew-mssql-release 28 | brew update 29 | HOMEBREW_ACCEPT_EULA=Y brew install msodbcsql18 mssql-tools18 30 | 31 | # Install docker using colima (not present because of licenses) 32 | brew install colima 33 | brew install docker 34 | colima start 35 | 36 | # Run SQL server 37 | docker run -d -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=1StrongPwd!!" --name sqlserver --hostname sqlserver -p 1433:1433 mcr.microsoft.com/mssql/server:2019-latest 38 | 39 | # Install ODBC 40 | ./runner/plugins/odbc/build.sh 41 | -------------------------------------------------------------------------------- /scripts/fail_on_diff.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eu 4 | 5 | if [[ "$(git diff)" != "" ]]; then 6 | printf "\033[0;31mFAILURE: Unexpected diff (did you run 'yarn format'?)\n\n\033[0m" 7 | git diff --color=never 8 | exit 1 9 | fi 10 | -------------------------------------------------------------------------------- /scripts/gen_unsafe_certs.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | mkdir -p certs 4 | openssl req -newkey rsa:2048 -nodes -keyout certs/key.pem -x509 -days 3650 -out certs/cert.pem 5 | -------------------------------------------------------------------------------- /scripts/generate_log_data.js: -------------------------------------------------------------------------------- 1 | const faker = require('faker'); 2 | 3 | const data = []; 4 | for (let i = 0; i < 1000; i++) { 5 | data.push({ 6 | url: faker.image.imageUrl(), 7 | time: faker.date.past(), 8 | status: faker.random.number({ min: 200, max: 500 }), 9 | }); 10 | } 11 | 12 | console.log(JSON.stringify(data)); 13 | -------------------------------------------------------------------------------- /scripts/original-line-author.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script helps you find the original author of a line that hasn't 4 | # changed but has moved around. 5 | # 6 | # Usage: 7 | # ./scripts/original-line-author.sh runner/file.go "err := r.SkipRows(offset)" 8 | 9 | for commit in $(git --no-pager log --reverse --pretty=format:"%h"); do 10 | if [[ -z $(git ls-tree -r "$commit" --name-only | grep "$1") ]]; then 11 | continue 12 | fi 13 | 14 | f="$(git show $commit:"$1" | grep "$2")" 15 | if ! [[ -z "$f" ]] ; then 16 | git blame "$commit" -- "$1" | grep "$2" 17 | exit 0 18 | fi 19 | done 20 | 21 | exit 1 22 | -------------------------------------------------------------------------------- /scripts/provision_db.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eu 4 | 5 | # Usage: ./scripts/provision_db.sh $new_password 6 | sudo su postgres bash -c "psql -c $'CREATE USER datastation WITH PASSWORD \'$1\';'" 7 | sudo su postgres bash -c "psql -c 'CREATE DATABASE datastation WITH OWNER datastation;'" 8 | -------------------------------------------------------------------------------- /scripts/upgrade_node_deps.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import subprocess 5 | 6 | pinned = { 7 | # Can't upgrade to React 18 until we drop Enzyme 8 | 'react': '17', 9 | 'react-dom': '17', 10 | '@types/react': '17', 11 | '@types/react-dom': '17', 12 | # node-fetch 3 breaks because it requires everything to be jsmodules 13 | 'node-fetch': '2', 14 | # nanoid 4 breaks because it requires esm 15 | 'nanoid': '3', 16 | } 17 | 18 | def upgrade_section(items, extra_flag): 19 | packages = [] 20 | to_remove = [] 21 | for package, version in items: 22 | to_remove.append(package) 23 | 24 | if pinned.get(package): 25 | package += '@'+pinned[package] 26 | if version.startswith('npm:'): 27 | package += '@' + version 28 | 29 | packages.append(package) 30 | 31 | subprocess.run(['yarn', 'remove'] + to_remove, check=True) 32 | add = ['yarn', 'add'] 33 | if extra_flag: 34 | add.append(extra_flag) 35 | subprocess.run(add + packages, check=True) 36 | 37 | with open('package.json') as p: 38 | j = json.load(p) 39 | upgrade_section(j['dependencies'].items(), '') 40 | upgrade_section(j['devDependencies'].items(), '--dev') 41 | -------------------------------------------------------------------------------- /scripts/validate_yaml.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eu 4 | 5 | failed=0 6 | 7 | for yamlfile in $(git ls-files | grep '.yml\|.yaml'); do 8 | if ! $(ruby -ryaml -e "YAML.load(STDIN.read)" < $yamlfile); then 9 | printf "\n\nIn $yamlfile\n\n" 10 | failed=1 11 | fi 12 | done 13 | 14 | exit "$failed" 15 | -------------------------------------------------------------------------------- /server/app.test.js: -------------------------------------------------------------------------------- 1 | const { Config } = require('./config'); 2 | const { App, init } = require('./app'); 3 | 4 | const basicConfig = new Config(); 5 | basicConfig.auth.sessionSecret = 'a secret test secret'; 6 | // Not great but for now the tests need a working openid server, set up with garbage configuration. 7 | basicConfig.auth.openId = { 8 | realm: 'https://accounts.google.com', 9 | clientId: '12', 10 | clientSecret: '89', 11 | }; 12 | basicConfig.server.tlsKey = ''; 13 | basicConfig.server.tlsCert = ''; 14 | 15 | describe('app.serve', function () { 16 | const server = { listen: jest.fn() }; 17 | let app = new App(basicConfig, () => {}); 18 | app.http.createServer = jest.fn(() => server); 19 | app.https.createServer = jest.fn(() => server); 20 | 21 | beforeAll(async function () { 22 | await app.serve(); 23 | }); 24 | 25 | test('it creates server', () => { 26 | expect(app.http.createServer.mock.calls.length).toBe(1); 27 | expect(app.https.createServer.mock.calls.length).toBe(0); 28 | }); 29 | 30 | test('it listens', () => { 31 | expect(server.listen.mock.calls.length).toBe(1); 32 | }); 33 | }); 34 | -------------------------------------------------------------------------------- /server/auth.test.js: -------------------------------------------------------------------------------- 1 | require('./auth'); 2 | 3 | test('stub', () => {}); 4 | -------------------------------------------------------------------------------- /server/config.test.js: -------------------------------------------------------------------------------- 1 | const { Config, mergeFromEnv } = require('./config'); 2 | 3 | test('mergeFromEnv', function testMergeFromEnv() { 4 | const c = new Config(); 5 | expect(c.database.address).toBe('localhost:5432'); 6 | 7 | mergeFromEnv(c, { 8 | DATASTATION_DATABASE_ADDRESS: 'pg.domain.com', 9 | }); 10 | 11 | expect(c.database.address).toBe('pg.domain.com'); 12 | 13 | mergeFromEnv(c, { 14 | DATASTATION_SERVER_ADDRESS: '0.0.0.0:8080', 15 | }); 16 | 17 | expect(c.database.address).toBe('pg.domain.com'); 18 | expect(c.server.address).toBe('0.0.0.0:8080'); 19 | }); 20 | -------------------------------------------------------------------------------- /server/index.ts: -------------------------------------------------------------------------------- 1 | import { APP_NAME, DEBUG, VERSION } from '../shared/constants'; 2 | import { App, init } from './app'; 3 | import { readConfig } from './config'; 4 | import log from './log'; 5 | 6 | process.on('unhandledRejection', (e) => { 7 | log.error(e); 8 | }); 9 | process.on('uncaughtException', (e) => { 10 | log.error(e); 11 | }); 12 | log.info(APP_NAME, VERSION, DEBUG ? 'DEBUG' : ''); 13 | 14 | async function main() { 15 | const app = App.make(readConfig()); 16 | const { handlers } = await init(app); 17 | await app.serve(handlers); 18 | } 19 | 20 | main(); 21 | -------------------------------------------------------------------------------- /server/log.test.js: -------------------------------------------------------------------------------- 1 | require('./log'); 2 | 3 | test('stub', () => {}); 4 | -------------------------------------------------------------------------------- /server/log.ts: -------------------------------------------------------------------------------- 1 | import log from '../shared/log'; 2 | 3 | export default { 4 | ...log, 5 | fatal: (...args: any[]) => { 6 | log.error(...args); 7 | process.exit(1); 8 | }, 9 | }; 10 | -------------------------------------------------------------------------------- /server/migrations/1_init.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | CREATE TABLE projects ( 4 | project_name TEXT PRIMARY KEY, 5 | project_value JSONB NOT NULL, 6 | project_created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() 7 | ); 8 | 9 | CREATE TABLE migrations ( 10 | migration_name TEXT PRIMARY KEY, 11 | migration_created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() 12 | ); 13 | 14 | COMMIT; 15 | -------------------------------------------------------------------------------- /server/release/config.yaml: -------------------------------------------------------------------------------- 1 | auth: 2 | sessionSecret: "some good long random string" # Any strong random string for signing sessions 3 | openId: 4 | realm: https://accounts.google.com # Or some other realm 5 | clientId: "my id" 6 | clientSecret: "my secret" 7 | 8 | server: 9 | port: 443 10 | address: localhost 11 | publicUrl: https://datastation.mydomain.com # The address users will enter into the browser to use the app 12 | tlsKey: /home/server/certs/datastation.key.pem # Can be left blank and set at the reverse-proxy level if desired 13 | tlsCert: /home/server/certs/datastation.cert.pem 14 | 15 | database: 16 | address: localhost # Address of your PostgreSQL instance 17 | username: datastation # Should be a dedicated PostgreSQL user for DataStation 18 | password: "some good password" 19 | database: datastation # Should be a dedicated database within PostgreSQL for DataStation -------------------------------------------------------------------------------- /server/release/datastation-exporter.timer: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Run DataStation exporter daily 3 | 4 | [Timer] 5 | OnCalendar=daily 6 | Persistent=true 7 | ExecStart=node /usr/share/datastation/exporter.js 8 | 9 | [Install] 10 | WantedBy=timers.target -------------------------------------------------------------------------------- /server/release/datastation.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=DataStation 3 | After=network.target 4 | StartLimitIntervalSec=0 5 | 6 | [Service] 7 | Type=simple 8 | Restart=always 9 | RestartSec=1 10 | User=datastation 11 | ExecStartPre= 12 | WorkingDirectory=/usr/share/datastation 13 | ExecStart=node /usr/share/datastation/build/server.js 14 | ExecStartPost= 15 | ExecStop= 16 | ExecReload= 17 | 18 | [Install] 19 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /server/release/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | ROOT_DIR="$(readlink -f $(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)/..)" 6 | 7 | sudo mkdir -p /etc/datastation /usr/share /etc/systemd/system /home/datastation 8 | sudo cp $ROOT_DIR/release/config.yaml /etc/datastation/ 9 | sudo cp $ROOT_DIR/release/datastation.service /etc/systemd/system/ 10 | # sudo cp $ROOT_DIR/release/datastation-exporter.timer /etc/systemd/system/ 11 | sudo mv $ROOT_DIR /usr/share/datastation 12 | 13 | sudo id -u datastation >/dev/null 2>&1 || sudo useradd -r -s /bin/false datastation 14 | sudo chown -R datastation:datastation /etc/datastation /usr/share/datastation /home/datastation 15 | 16 | sudo systemctl enable datastation 17 | -------------------------------------------------------------------------------- /server/rpc.test.js: -------------------------------------------------------------------------------- 1 | const { handleRPC } = require('./rpc'); 2 | 3 | test('dispatching works', async () => { 4 | let done = false; 5 | 6 | const req = { 7 | query: { 8 | resource: 'test', 9 | projectId: 'my project', 10 | body: { abody: 'a thing' }, 11 | }, 12 | body: {}, 13 | }; 14 | 15 | const rsp = { 16 | json(msg) { 17 | expect(msg).toStrictEqual({ aresponse: true }); 18 | done = true; 19 | }, 20 | }; 21 | 22 | const handlers = [ 23 | { 24 | resource: 'test', 25 | handler: (projectId, body, dispatch, external) => { 26 | // TODO: test internal dispatch 27 | 28 | expect(projectId).toBe('my project'); 29 | expect(body).toStrictEqual({ abody: 'a thing' }); 30 | expect(typeof dispatch).toStrictEqual('function'); 31 | expect(external).toStrictEqual(true); 32 | return { aresponse: true }; 33 | }, 34 | }, 35 | ]; 36 | 37 | await handleRPC(req, rsp, handlers); 38 | 39 | expect(done).toBe(true); 40 | }); 41 | -------------------------------------------------------------------------------- /server/rpc.ts: -------------------------------------------------------------------------------- 1 | import express from 'express'; 2 | import { DispatchPayload, RPCHandler } from '../desktop/rpc'; 3 | import log from '../shared/log'; 4 | 5 | export function makeDispatch( 6 | handlers: RPCHandler[] 7 | ) { 8 | return async function dispatch( 9 | payload: DispatchPayload, 10 | external = false 11 | ) { 12 | const handler = handlers.filter((h) => h.resource === payload.resource)[0]; 13 | if (!handler) { 14 | throw new Error(`No RPC handler for resource: ${payload.resource}`); 15 | } 16 | 17 | return await handler.handler( 18 | payload.projectId, 19 | payload.body, 20 | dispatch, 21 | external 22 | ); 23 | }; 24 | } 25 | 26 | export async function handleRPC( 27 | req: express.Request, 28 | rsp: express.Response, 29 | rpcHandlers: RPCHandler[] 30 | ) { 31 | const payload = { 32 | ...req.query, 33 | ...req.body, 34 | }; 35 | 36 | const dispatch = makeDispatch(rpcHandlers); 37 | 38 | try { 39 | const rpcResponse = await dispatch(payload, true); 40 | rsp.json(rpcResponse || { message: 'ok' }); 41 | } catch (e) { 42 | log.error(e); 43 | rsp.status(400).json({ 44 | ...e, 45 | // Needs to get passed explicitly or name comes out as Error after rpc 46 | message: e.message, 47 | name: e.name, 48 | }); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /server/runner.test.js: -------------------------------------------------------------------------------- 1 | require('./runner'); 2 | 3 | test('stub', () => {}); 4 | -------------------------------------------------------------------------------- /server/runner.ts: -------------------------------------------------------------------------------- 1 | import { main as baseMain } from '../desktop/runner'; 2 | import { APP_NAME, DEBUG, VERSION } from '../shared/constants'; 3 | import log from '../shared/log'; 4 | import { App, init } from './app'; 5 | import { readConfig } from './config'; 6 | 7 | export async function main(app: App) { 8 | const { handlers } = await init(app, false); 9 | log.info(APP_NAME + ' Panel Runner', VERSION, DEBUG ? 'DEBUG' : ''); 10 | await baseMain(handlers); 11 | } 12 | 13 | if ((process.argv[1] || '').includes('server_runner.js')) { 14 | main(App.make(readConfig())); 15 | } 16 | -------------------------------------------------------------------------------- /server/scripts/build_image.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eu 4 | 5 | image=$(docker build . --quiet -f Dockerfile.build -t datastation-builder) 6 | docker run -v $(pwd):/datastation "$image" yarn build-server 7 | -------------------------------------------------------------------------------- /server/scripts/release.build: -------------------------------------------------------------------------------- 1 | rm -rf build 2 | yarn 3 | setenv UI_ESBUILD_ARGS "--minify" 4 | 5 | yarn build-server 6 | prepend "window.DS_CONFIG_MODE='server';" build/ui.js 7 | prepend "window.DS_CONFIG_VERSION='{arg0}';" build/ui.js 8 | prepend "window.DS_CONFIG_VERSION='{arg0}';" build/server_runner.js 9 | prepend "global.DS_CONFIG_VERSION='{arg0}';" build/server.js 10 | 11 | # Need to have a directory with a build directory and a node_modules directory 12 | rm -rf datastation 13 | mkdir datastation 14 | mkdir datastation/node_modules 15 | mv build datastation/ 16 | 17 | # Bring in sampledata 18 | cp -r sampledata datastation/sampledata 19 | 20 | # Bring in node_modules 21 | prepend "{{}}" datastation/package.json 22 | cd datastation && yarn add asar electron better-sqlite3 23 | rm datastation/yarn.lock 24 | rm datastation/package.json 25 | # Only need stubs not full chrome install 26 | rm -rf datastation/node_modules/electron/dist 27 | 28 | cp node_modules/better-sqlite3/build/Release/better_sqlite3.node datastation/build/ 29 | # Copy in install script and default configs 30 | cp -r server/release datastation/release 31 | 32 | mkdir releases 33 | tar -zcvf releases/datastation-server-{arch}-{arg0}.tar.gz datastation 34 | 35 | rm -rf datastation -------------------------------------------------------------------------------- /server/scripts/server.build: -------------------------------------------------------------------------------- 1 | setenv UI_CONFIG_OVERRIDES "window.DS_CONFIG_MODE = 'server'; window.DS_CONFIG_UI_TITLE = 'DataStation Server CE';" 2 | 3 | setenv UI_TITLE "DataStation Server CE" 4 | setenv UI_ROOT "/" 5 | yarn build-ui 6 | 7 | cd runner && go build -trimpath -buildmode=pie -mod=readonly -modcacherw -ldflags="-s -w" -o ../build/go_server_runner{required_ext} ./cmd/main.go 8 | 9 | yarn esbuild server/runner.ts --sourcemap --platform=node --bundle --target=node10.4 --external:sqlite3 --external:asar --external:react-native-fs --external:react-native-fetch-blob --external:cpu-features --external:electron --outfile=build/server_runner.js 10 | yarn esbuild server/index.ts --sourcemap --platform=node --bundle --external:sqlite3 --external:react-native-fs --external:asar --external:react-native-fetch-blob --external:cpu-features --external:electron --outfile=build/server.js 11 | 12 | prepend "global.DS_UI_TITLE='DataStation Server CE';" build/server.js 13 | 14 | rm -rf build/migrations 15 | cp -r desktop/migrations build/ 16 | 17 | setenv_default SERVER_CONFIG_OVERRIDES "" 18 | prepend {SERVER_CONFIG_OVERRIDES} build/server.js 19 | -------------------------------------------------------------------------------- /server/scripts/test_release.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | function docker() { 6 | podman "$@" 7 | } 8 | 9 | cp ./releases/$1 ./releases/datastation-docker-release-test.tar.gz 10 | docker build . -q -f ./server/scripts/test_systemd_dockerfile -t datastation-server-test 11 | cid=$(docker run -d datastation-server-test:latest) 12 | 13 | debug() { 14 | rv=$? 15 | if ! [[ "$rv" == "0" ]]; then 16 | docker ps -a 17 | docker logs $cid 18 | docker rm -f $cid || echo "Container already exited." 19 | fi 20 | exit $rv 21 | } 22 | trap "debug" exit 23 | 24 | function c_run() { 25 | docker exec $cid bash -c "$1" 26 | } 27 | 28 | # TODO: test out systemd settings eventually 29 | # # Copy in zip file 30 | # docker cp ./releases/$1 $cid:/ 31 | # 32 | # c_run "tar xvzf /$1" 33 | # c_run "/datastation/release/install.sh" 34 | # c_run "truncate --size 0 /etc/datastation/config.yaml" 35 | # c_run "systemctl restart datastation" 36 | 37 | # Wait for server to start 38 | sleep 10 39 | 40 | result="$(c_run 'curl localhost:8080')" 41 | 42 | expected='DataStation Server CE 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 |
51 |
Loading...
52 |
53 | 54 | ' 55 | 56 | if ! diff -u -wB <(echo "$expected") <(echo "$result"); then 57 | echo "Unexpected response body:" 58 | echo "$result" 59 | exit 1 60 | fi 61 | 62 | 63 | echo "Looks ok!" 64 | exit 0 65 | -------------------------------------------------------------------------------- /server/scripts/test_systemd_dockerfile: -------------------------------------------------------------------------------- 1 | FROM fedora 2 | 3 | RUN dnf install -y systemd tar 4 | RUN dnf module reset nodejs && dnf module install -y nodejs:16 5 | 6 | COPY /releases/datastation-docker-release-test.tar.gz /datastation-server.tar.gz 7 | RUN tar xvzf /datastation-server.tar.gz 8 | RUN mkdir -p /etc/datastation && touch /etc/datastation/config.yaml 9 | 10 | CMD ["node", "/datastation/build/server.js"] -------------------------------------------------------------------------------- /shared/array.test.js: -------------------------------------------------------------------------------- 1 | const { chunk } = require('./array'); 2 | 3 | test('chunks an array', () => { 4 | expect(chunk([1], 0)).toStrictEqual([[1]]); 5 | expect(chunk([], 1)).toStrictEqual([]); 6 | expect(chunk([1, 2, 3, 4], 2)).toStrictEqual([ 7 | [1, 2], 8 | [3, 4], 9 | ]); 10 | }); 11 | -------------------------------------------------------------------------------- /shared/array.ts: -------------------------------------------------------------------------------- 1 | export function chunk(a: Array, n: number): Array> { 2 | if (n === 0) { 3 | return [a]; 4 | } 5 | 6 | const chunks: Array> = []; 7 | 8 | let i = 0; 9 | while (i < a.length) { 10 | chunks.push(a.slice(i, (i += n))); 11 | } 12 | 13 | return chunks; 14 | } 15 | -------------------------------------------------------------------------------- /shared/constants.ts: -------------------------------------------------------------------------------- 1 | export const SITE_ROOT = 'https://datastation.multiprocess.io'; 2 | export const CHAT_LINK = 'https://discord.gg/f2wQBc4bXX'; 3 | 4 | export const RPC_ASYNC_REQUEST = 'rpcAsyncRequest'; 5 | export const RPC_ASYNC_RESPONSE = 'rpcAsyncResponse'; 6 | 7 | function getConfig(v: string, _default: T) { 8 | const key = 'DS_CONFIG_' + v; 9 | if (key in globalThis) { 10 | return (globalThis as any)[key] as T; 11 | } 12 | 13 | return _default; 14 | } 15 | 16 | export const DEBUG = getConfig('DEBUG', true); 17 | export const VERSION = getConfig('VERSION', 'development'); 18 | export const MODE = getConfig('MODE', 'browser'); 19 | export const APP_NAME = getConfig('UI_TITLE', 'DataStation Desktop CE'); 20 | 21 | export const MODE_FEATURES = { 22 | appHeader: MODE !== 'desktop', 23 | connectors: MODE !== 'browser', 24 | sql: MODE !== 'browser', 25 | shareProject: MODE === 'browser', 26 | corsOnly: MODE === 'browser', 27 | noBodyYOverflow: MODE === 'desktop', 28 | storeResults: MODE !== 'browser', 29 | useDefaultProject: MODE === 'browser', 30 | extraLanguages: MODE !== 'browser', 31 | killProcess: MODE !== 'browser', 32 | dashboard: MODE === 'server', 33 | scheduledExports: MODE === 'server', 34 | }; 35 | 36 | // There is no /docs/development/ so replace it with /docs/latest/ 37 | const DOCS_VERSION = VERSION === 'development' ? 'latest' : VERSION; 38 | export const DOCS_ROOT = SITE_ROOT + '/docs/' + DOCS_VERSION; 39 | 40 | export const IN_TESTS = globalThis.process 41 | ? process.env.JEST_WORKER_ID !== undefined 42 | : false; 43 | -------------------------------------------------------------------------------- /shared/http.ts: -------------------------------------------------------------------------------- 1 | import { ContentTypeInfoPlusParsers, parseArrayBuffer } from './text'; 2 | 3 | type FetchFunction = ( 4 | url: string, 5 | args: { 6 | headers: { [key: string]: string }; 7 | method: string; 8 | body?: string; 9 | } 10 | ) => any; 11 | 12 | export async function request( 13 | fetchFunction: FetchFunction, 14 | method: string, 15 | url: string, 16 | contentTypeInfo: ContentTypeInfoPlusParsers, 17 | headers: Array<{ name: string; value: string }> = [], 18 | content = '', 19 | require200 = false 20 | ) { 21 | if (!(url.startsWith('https://') || url.startsWith('http://'))) { 22 | url = 'http://' + url; 23 | } 24 | const headersDict: { [v: string]: string } = {}; 25 | headers.forEach((h: { value: string; name: string }) => { 26 | if (h.name.length && h.value.length) { 27 | headersDict[h.name] = h.value; 28 | } 29 | }); 30 | const res = await fetchFunction(url, { 31 | headers: headersDict, 32 | method, 33 | body: method !== 'GET' && method !== 'HEAD' ? content : undefined, 34 | }); 35 | 36 | const body = await res.arrayBuffer(); 37 | if (!contentTypeInfo.type) { 38 | let type = res.headers.get('content-type'); 39 | if ( 40 | type.startsWith('text/plain') || 41 | type.startsWith('application/octet-stream') 42 | ) { 43 | type = ''; 44 | } 45 | contentTypeInfo.type = type; 46 | } 47 | 48 | const data = await parseArrayBuffer(contentTypeInfo, url, body); 49 | if (require200 && res.status !== 200) { 50 | throw data.value; 51 | } 52 | 53 | return data; 54 | } 55 | -------------------------------------------------------------------------------- /shared/languages/deno.json: -------------------------------------------------------------------------------- 1 | { 2 | "commandArgs": ["run", "--allow-all"], 3 | "defaultPath": "deno", 4 | "id": "deno", 5 | "name": "Deno", 6 | "preamble": "\nfunction DM_getPanelFile(i) {\n return '$$RESULTS_FILE$$'+$$JSON_ID_MAP$$[i];\n}\n\nfunction DM_getPanel(i) {\n return JSON.parse(Deno.readTextFileSync('$$RESULTS_FILE$$'+$$JSON_ID_MAP$$[i]));\n}\n\nfunction DM_setPanel(v) {\n Deno.writeTextFileSync('$$PANEL_RESULTS_FILE$$', JSON.stringify(v));\n}" 7 | } 8 | -------------------------------------------------------------------------------- /shared/languages/deno.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | id: "deno", 3 | defaultPath: "deno", 4 | commandArgs: ["run", "--allow-all"], 5 | name: "Deno", 6 | preamble: " 7 | function DM_getPanelFile(i) { 8 | return '$$RESULTS_FILE$$'+$$JSON_ID_MAP$$[i]; 9 | } 10 | 11 | function DM_getPanel(i) { 12 | return JSON.parse(Deno.readTextFileSync('$$RESULTS_FILE$$'+$$JSON_ID_MAP$$[i])); 13 | } 14 | 15 | function DM_setPanel(v) { 16 | Deno.writeTextFileSync('$$PANEL_RESULTS_FILE$$', JSON.stringify(v)); 17 | }", 18 | } 19 | -------------------------------------------------------------------------------- /shared/languages/deno.ts: -------------------------------------------------------------------------------- 1 | import deno from './deno.json'; 2 | import { genericPreamble, JAVASCRIPT } from './javascript'; 3 | 4 | function preamble( 5 | resultsFile: string, 6 | panelId: string, 7 | idMap: Record 8 | ) { 9 | return genericPreamble(deno.preamble, resultsFile, panelId, idMap); 10 | } 11 | 12 | export const DENO = { 13 | ...JAVASCRIPT, 14 | name: deno.name, 15 | defaultPath: deno.defaultPath, 16 | preamble, 17 | }; 18 | -------------------------------------------------------------------------------- /shared/languages/index.ts: -------------------------------------------------------------------------------- 1 | import { DENO } from './deno'; 2 | import { JAVASCRIPT } from './javascript'; 3 | import { JULIA } from './julia'; 4 | import { PHP } from './php'; 5 | import { PYTHON } from './python'; 6 | import { R } from './r'; 7 | import { RUBY } from './ruby'; 8 | import { SQL } from './sql'; 9 | import { LanguageInfo } from './types'; 10 | 11 | export const LANGUAGES: Record = { 12 | javascript: JAVASCRIPT, 13 | deno: DENO, 14 | python: PYTHON, 15 | ruby: RUBY, 16 | julia: JULIA, 17 | r: R, 18 | sql: SQL, 19 | php: PHP, 20 | }; 21 | 22 | export type SupportedLanguages = keyof typeof LANGUAGES; 23 | -------------------------------------------------------------------------------- /shared/languages/javascript.json: -------------------------------------------------------------------------------- 1 | { 2 | "defaultPath": "node", 3 | "id": "javascript", 4 | "name": "JavaScript", 5 | "preamble": "\nfunction DM_getPanelFile(i) {\n return '$$RESULTS_FILE$$'+$$JSON_ID_MAP$$[i];\n}\n\nfunction DM_getPanel(i) {\n const fs = require('fs');\n return JSON.parse(fs.readFileSync('$$RESULTS_FILE$$'+$$JSON_ID_MAP$$[i]));\n}\n\nfunction DM_setPanel(v) {\n const fs = require('fs');\n const fd = fs.openSync('$$PANEL_RESULTS_FILE$$', 'w');\n if (Array.isArray(v)) {\n fs.writeSync(fd, '[');\n for (let i = 0; i < v.length; i++) {\n const row = v[i];\n let rowJSON = JSON.stringify(row);\n if (i < v.length - 1) {\n rowJSON += ',';\n }\n fs.writeSync(fd, rowJSON);\n }\n fs.writeSync(fd, ']');\n } else {\n fs.writeSync(fd, JSON.stringify(v));\n }\n}" 6 | } 7 | -------------------------------------------------------------------------------- /shared/languages/javascript.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | id: "javascript", 3 | defaultPath: "node", 4 | name: "JavaScript", 5 | preamble: " 6 | function DM_getPanelFile(i) { 7 | return '$$RESULTS_FILE$$'+$$JSON_ID_MAP$$[i]; 8 | } 9 | 10 | function DM_getPanel(i) { 11 | const fs = require('fs'); 12 | return JSON.parse(fs.readFileSync('$$RESULTS_FILE$$'+$$JSON_ID_MAP$$[i])); 13 | } 14 | 15 | function DM_setPanel(v) { 16 | const fs = require('fs'); 17 | const fd = fs.openSync('$$PANEL_RESULTS_FILE$$', 'w'); 18 | if (Array.isArray(v)) { 19 | fs.writeSync(fd, '['); 20 | for (let i = 0; i < v.length; i++) { 21 | const row = v[i]; 22 | let rowJSON = JSON.stringify(row); 23 | if (i < v.length - 1) { 24 | rowJSON += ','; 25 | } 26 | fs.writeSync(fd, rowJSON); 27 | } 28 | fs.writeSync(fd, ']'); 29 | } else { 30 | fs.writeSync(fd, JSON.stringify(v)); 31 | } 32 | }", 33 | } 34 | -------------------------------------------------------------------------------- /shared/languages/julia.json: -------------------------------------------------------------------------------- 1 | { 2 | "defaultPath": "julia", 3 | "id": "julia", 4 | "name": "Julia", 5 | "preamble": "\ntry\n import JSON\ncatch e\n import Pkg\n Pkg.add(\"JSON\")\n import JSON\nend\n\nfunction DM_getPanel(i)\n panelId = JSON.parse(\"$$JSON_ID_MAP_QUOTE_ESCAPED$$\")[string(i)]\n JSON.parsefile(string(\"$$RESULTS_FILE$$\", panelId))\nend\n\nfunction DM_setPanel(v)\n open(\"$$PANEL_RESULTS_FILE$$\", \"w\") do f\n JSON.print(f, v)\n end\nend\n\nfunction DM_getPanelFile(i)\n string(\"$$RESULTS_FILE$$\", JSON.parse(\"$$JSON_ID_MAP_QUOTE_ESCAPED$$\")[string(i)])\nend" 6 | } 7 | -------------------------------------------------------------------------------- /shared/languages/julia.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | id: "julia", 3 | name: "Julia", 4 | defaultPath: "julia", 5 | preamble: ' 6 | try 7 | import JSON 8 | catch e 9 | import Pkg 10 | Pkg.add("JSON") 11 | import JSON 12 | end 13 | 14 | function DM_getPanel(i) 15 | panelId = JSON.parse("$$JSON_ID_MAP_QUOTE_ESCAPED$$")[string(i)] 16 | JSON.parsefile(string("$$RESULTS_FILE$$", panelId)) 17 | end 18 | 19 | function DM_setPanel(v) 20 | open("$$PANEL_RESULTS_FILE$$", "w") do f 21 | JSON.print(f, v) 22 | end 23 | end 24 | 25 | function DM_getPanelFile(i) 26 | string("$$RESULTS_FILE$$", JSON.parse("$$JSON_ID_MAP_QUOTE_ESCAPED$$")[string(i)]) 27 | end', 28 | } 29 | -------------------------------------------------------------------------------- /shared/languages/julia.ts: -------------------------------------------------------------------------------- 1 | import { genericPreamble } from './javascript'; 2 | import julia from './julia.json'; 3 | import { EOL } from './types'; 4 | 5 | function preamble( 6 | resultsFile: string, 7 | panelId: string, 8 | idMap: Record 9 | ) { 10 | return genericPreamble(julia.preamble, resultsFile, panelId, idMap); 11 | } 12 | 13 | function defaultContent(panelIndex: number) { 14 | if (panelIndex === 0) { 15 | return 'result = []\n# Your logic here\nDM_setPanel(result)'; 16 | } 17 | 18 | return 'transform = DM_getPanel(0)\n# Your logic here\nDM_setPanel(transform)'; 19 | } 20 | 21 | function exceptionRewriter(msg: string, programPath: string) { 22 | const matcher = RegExp(`${programPath}:([1-9]*)`.replaceAll('/', '\\/'), 'g'); 23 | 24 | return msg.replace(matcher, function (_: string, line: string) { 25 | return `${programPath}:${+line - preamble('', '', {}).split(EOL).length}`; 26 | }); 27 | } 28 | 29 | export const JULIA = { 30 | name: julia.name, 31 | defaultPath: julia.defaultPath, 32 | defaultContent, 33 | preamble, 34 | exceptionRewriter, 35 | }; 36 | -------------------------------------------------------------------------------- /shared/languages/php.json: -------------------------------------------------------------------------------- 1 | { 2 | "commandArgs": ["-d", "display_errors=on"], 3 | "defaultPath": "php", 4 | "id": "php", 5 | "name": "PHP", 6 | "preamble": "\n 21 | ) { 22 | return genericPreamble(php.preamble, resultsFile, panelId, idMap); 23 | } 24 | 25 | function exceptionRewriter(msg: string, _: string) { 26 | return msg; 27 | } 28 | 29 | export const PHP = { 30 | name: php.name, 31 | defaultPath: php.defaultPath, 32 | defaultContent, 33 | preamble, 34 | exceptionRewriter, 35 | }; 36 | -------------------------------------------------------------------------------- /shared/languages/python.json: -------------------------------------------------------------------------------- 1 | { 2 | "defaultPath": "python3", 3 | "id": "python", 4 | "name": "Python", 5 | "preamble": "\ndef DM_getPanelFile(i):\n return r'$$RESULTS_FILE$$'+$$JSON_ID_MAP$$[str(i)]\n\ndef DM_getPanel(i):\n import json\n with open(r'$$RESULTS_FILE$$'+$$JSON_ID_MAP$$[str(i)]) as f:\n return json.load(f)\n\ndef DM_setPanel(v):\n import json\n with open(r'$$PANEL_RESULTS_FILE$$', 'w') as f:\n json.dump(v, f)" 6 | } 7 | -------------------------------------------------------------------------------- /shared/languages/python.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | id: "python", 3 | name: "Python", 4 | defaultPath: "python3", 5 | preamble: " 6 | def DM_getPanelFile(i): 7 | return r'$$RESULTS_FILE$$'+$$JSON_ID_MAP$$[str(i)] 8 | 9 | def DM_getPanel(i): 10 | import json 11 | with open(r'$$RESULTS_FILE$$'+$$JSON_ID_MAP$$[str(i)]) as f: 12 | return json.load(f) 13 | 14 | def DM_setPanel(v): 15 | import json 16 | with open(r'$$PANEL_RESULTS_FILE$$', 'w') as f: 17 | json.dump(v, f)", 18 | } 19 | -------------------------------------------------------------------------------- /shared/languages/r.json: -------------------------------------------------------------------------------- 1 | { 2 | "defaultPath": "Rscript", 3 | "id": "r", 4 | "name": "R", 5 | "preamble": "\ntryCatch(library(\"rjson\"), error=function(cond) {\n install.packages(\"rjson\", repos=\"https://cloud.r-project.org\")\n}, finally=library(\"rjson\"))\n\nDM_getPanel <- function(i) {\n panelId = fromJSON(\"$$JSON_ID_MAP_QUOTE_ESCAPED$$\")[[toString(i)]]\n fromJSON(file=paste(\"$$RESULTS_FILE$$\", panelId, sep=\"\"))\n}\n\nDM_setPanel <- function(v) {\n write(toJSON(v), \"$$PANEL_RESULTS_FILE$$\")\n}\n\nDM_getPanelFile <- function(i) {\n paste(\"$$RESULTS_FILE$$\", fromJSON(\"$$JSON_ID_MAP_QUOTE_ESCAPED$$\")[[toString(i)]], sep=\"\")\n}\n" 6 | } 7 | -------------------------------------------------------------------------------- /shared/languages/r.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | id: "r", 3 | name: "R", 4 | defaultPath: "Rscript", 5 | preamble: ' 6 | tryCatch(library("rjson"), error=function(cond) { 7 | install.packages("rjson", repos="https://cloud.r-project.org") 8 | }, finally=library("rjson")) 9 | 10 | DM_getPanel <- function(i) { 11 | panelId = fromJSON("$$JSON_ID_MAP_QUOTE_ESCAPED$$")[[toString(i)]] 12 | fromJSON(file=paste("$$RESULTS_FILE$$", panelId, sep="")) 13 | } 14 | 15 | DM_setPanel <- function(v) { 16 | write(toJSON(v), "$$PANEL_RESULTS_FILE$$") 17 | } 18 | 19 | DM_getPanelFile <- function(i) { 20 | paste("$$RESULTS_FILE$$", fromJSON("$$JSON_ID_MAP_QUOTE_ESCAPED$$")[[toString(i)]], sep="") 21 | } 22 | ', 23 | } 24 | -------------------------------------------------------------------------------- /shared/languages/r.ts: -------------------------------------------------------------------------------- 1 | import { genericPreamble } from './javascript'; 2 | import r from './r.json'; 3 | 4 | function defaultContent(panelIndex: number) { 5 | if (panelIndex === 0) { 6 | return 'result = c()\n# Your logic here\nDM_setPanel(result)'; 7 | } 8 | 9 | return `transform = DM_getPanel(${ 10 | panelIndex - 1 11 | })\n# Your logic here\nDM_setPanel(transform)`; 12 | } 13 | 14 | function preamble( 15 | resultsFile: string, 16 | panelId: string, 17 | idMap: Record 18 | ) { 19 | return genericPreamble(r.preamble, resultsFile, panelId, idMap); 20 | } 21 | 22 | export const R = { 23 | name: r.name, 24 | defaultPath: r.defaultPath, 25 | defaultContent, 26 | preamble, 27 | exceptionRewriter(msg: string, _: string) { 28 | return msg; 29 | }, 30 | }; 31 | -------------------------------------------------------------------------------- /shared/languages/ruby.json: -------------------------------------------------------------------------------- 1 | { 2 | "defaultPath": "ruby", 3 | "id": "ruby", 4 | "name": "Ruby", 5 | "preamble": "\ndef DM_getPanel(i)\n require 'json'\n JSON.parse(File.read('$$RESULTS_FILE$$' + JSON.parse('$$JSON_ID_MAP$$')[i.to_s]))\nend\n\ndef DM_setPanel(v)\n require 'json'\n File.write('$$PANEL_RESULTS_FILE$$', v.to_json)\nend\n\ndef DM_getPanelFile(i)\n require 'json'\n '$$RESULTS_FILE$$' + JSON.parse('$$JSON_ID_MAP$$')[i.to_s]\nend\n" 6 | } 7 | -------------------------------------------------------------------------------- /shared/languages/ruby.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | id: "ruby", 3 | name: "Ruby", 4 | defaultPath: "ruby", 5 | preamble: " 6 | def DM_getPanel(i) 7 | require 'json' 8 | JSON.parse(File.read('$$RESULTS_FILE$$' + JSON.parse('$$JSON_ID_MAP$$')[i.to_s])) 9 | end 10 | 11 | def DM_setPanel(v) 12 | require 'json' 13 | File.write('$$PANEL_RESULTS_FILE$$', v.to_json) 14 | end 15 | 16 | def DM_getPanelFile(i) 17 | require 'json' 18 | '$$RESULTS_FILE$$' + JSON.parse('$$JSON_ID_MAP$$')[i.to_s] 19 | end 20 | ", 21 | } 22 | -------------------------------------------------------------------------------- /shared/languages/ruby.ts: -------------------------------------------------------------------------------- 1 | import { genericPreamble } from './javascript'; 2 | import ruby from './ruby.json'; 3 | 4 | function defaultContent(panelIndex: number) { 5 | if (panelIndex === 0) { 6 | return 'result = []\n# Your logic here\nDM_setPanel(result)'; 7 | } 8 | 9 | return `transform = DM_getPanel(${ 10 | panelIndex - 1 11 | })\n# Your logic here\nDM_setPanel(transform)`; 12 | } 13 | 14 | function preamble( 15 | resultsFile: string, 16 | panelId: string, 17 | idMap: Record 18 | ) { 19 | return genericPreamble(ruby.preamble, resultsFile, panelId, idMap); 20 | } 21 | 22 | function exceptionRewriter(msg: string, _: string) { 23 | return msg; 24 | } 25 | 26 | export const RUBY = { 27 | name: ruby.name, 28 | defaultPath: ruby.defaultPath, 29 | defaultContent, 30 | preamble, 31 | exceptionRewriter, 32 | }; 33 | -------------------------------------------------------------------------------- /shared/languages/types.ts: -------------------------------------------------------------------------------- 1 | import { Shape } from 'shape'; 2 | import { PanelResult } from '../state'; 3 | 4 | export const EOL = /\r?\n/; 5 | 6 | export interface LanguageInfo { 7 | name: string; 8 | defaultContent: (panelIndex: number) => string; 9 | preamble: ( 10 | resultsFile: string, 11 | panelId: string, 12 | idMap: Record 13 | ) => string; 14 | defaultPath: string; 15 | exceptionRewriter: (msg: string, programPath: string) => string; 16 | inMemoryInit?: () => Promise; 17 | inMemoryEval?: ( 18 | prog: string, 19 | results: Record 20 | ) => Promise<{ stdout: string; preview: string; value: any }>; 21 | nodeEval?: ( 22 | prog: string, 23 | results: { 24 | idMap: Record; 25 | idShapeMap: Record; 26 | resultsFile: string; 27 | } 28 | ) => { stdout: string; preview: string; value: any }; 29 | } 30 | -------------------------------------------------------------------------------- /shared/log.test.js: -------------------------------------------------------------------------------- 1 | const log = require('./log'); 2 | 3 | test('it logs', () => { 4 | log.logger.INFO = jest.fn(); 5 | log.logger.ERROR = jest.fn(); 6 | 7 | log.default.info('a', 'b', 'c'); 8 | expect(log.logger.INFO.mock.calls[0][0]).toMatch( 9 | /\[INFO\] [0-9:.ZT-]+ a b c/ 10 | ); 11 | 12 | const e = new Error('my message'); 13 | log.default.error('a', e, 'c'); 14 | const words = log.logger.ERROR.mock.calls[0][0].split(' '); 15 | expect(words.filter((f) => f.length)).toStrictEqual([ 16 | '[ERROR]', 17 | words[1], // Already tested it looks like a date, 18 | 'a', 19 | 'my', 20 | 'message', 21 | 'c', 22 | '\n', 23 | ...e.stack 24 | .split('\n') 25 | .slice(1) 26 | .join('\n') 27 | .split(' ') 28 | .filter((f) => f.length), 29 | ]); 30 | }); 31 | -------------------------------------------------------------------------------- /shared/log.ts: -------------------------------------------------------------------------------- 1 | import { preview } from 'preview'; 2 | export const logger = { 3 | INFO: console.log, 4 | ERROR: console.trace, 5 | }; 6 | 7 | function safePreview(o: any): string { 8 | try { 9 | return preview(o); 10 | } catch (e) { 11 | return String(o); 12 | } 13 | } 14 | 15 | function log(level: keyof typeof logger, ...args: any[]) { 16 | const stringArgs: Array = []; 17 | let stackRecorded = ''; 18 | for (const arg of args) { 19 | if (arg instanceof Error && !stackRecorded) { 20 | stringArgs.push(arg.message); 21 | // Drop first line, which is arg.message. 22 | stackRecorded = (arg.stack.split('\n').slice(1) || []).join('\n'); 23 | continue; 24 | } 25 | 26 | stringArgs.push(safePreview(arg)); 27 | } 28 | if (stackRecorded) { 29 | stringArgs.push('\n' + safePreview(stackRecorded)); 30 | } 31 | 32 | const f = logger[level]; 33 | const now = new Date(); 34 | f(`[${level}] ${now.toISOString()} ${stringArgs.join(' ')}`); 35 | } 36 | 37 | function info(...args: any[]) { 38 | log('INFO', ...args); 39 | } 40 | 41 | function error(...args: any[]) { 42 | log('ERROR', ...args); 43 | } 44 | 45 | export default { 46 | info, 47 | error, 48 | }; 49 | -------------------------------------------------------------------------------- /shared/object.test.js: -------------------------------------------------------------------------------- 1 | const { 2 | deepEquals, 3 | mergeDeep, 4 | getPath, 5 | setPath, 6 | validate, 7 | } = require('./object'); 8 | 9 | test('getPath gets the path in the object', () => { 10 | const obj = { a: { 1: { b: '12' } } }; 11 | expect(getPath(obj, '')).toStrictEqual(obj); 12 | expect(getPath(obj, 'a.1.b')).toBe('12'); 13 | expect(getPath(obj, 'a.12.c')).toBe(undefined); 14 | expect(getPath(obj, 'a.1.c')).toBe(undefined); 15 | }); 16 | 17 | test('validate rejects missing fields', () => { 18 | const mock = jest.fn(); 19 | const obj = { a: { 1: { b: '12' } } }; 20 | validate(obj, ['a.1.b'], mock); 21 | expect(mock.mock.calls.length).toBe(0); 22 | 23 | mock.mockReset(); 24 | 25 | validate(obj, ['a.1.b', 'a.2'], mock); 26 | expect(mock.mock.calls.length).toBe(1); 27 | expect(mock.mock.calls[0][0]).toBe('a.2'); 28 | 29 | mock.mockReset(); 30 | 31 | validate(obj, ['a?.1.b', 'a.2?.b', 'a?.3'], mock); 32 | expect(mock.mock.calls.length).toBe(1); 33 | expect(mock.mock.calls[0][0]).toBe('a.3'); 34 | }); 35 | 36 | test('mergeDeep', () => { 37 | expect( 38 | mergeDeep( 39 | { a: 1, c: { b: 3 }, e: { f: 9 } }, 40 | { d: 12, c: { g: 22 }, e: { f: 10 } } 41 | ) 42 | ).toStrictEqual({ 43 | a: 1, 44 | d: 12, 45 | c: { b: 3, g: 22 }, 46 | e: { f: 10 }, 47 | }); 48 | }); 49 | 50 | test('deepEquals', () => { 51 | expect(deepEquals({}, {})).toBe(true); 52 | expect(deepEquals(1, {})).toBe(false); 53 | expect(deepEquals({ a: 2, b: { c: 3 } }, { a: 2, b: { c: 3 } })).toBe(true); 54 | expect(deepEquals({ a: 2, b: { c: 3 } }, { a: 2, b: { c: 4 } })).toBe(false); 55 | }); 56 | 57 | test('setPath', () => { 58 | const obj = { 59 | ns: [{ a: 1 }, { b: 2 }], 60 | }; 61 | setPath(obj, 'ns.1.b', 3); 62 | expect(obj).toStrictEqual({ 63 | ns: [{ a: 1 }, { b: 3 }], 64 | }); 65 | }); 66 | -------------------------------------------------------------------------------- /shared/polyfill.ts: -------------------------------------------------------------------------------- 1 | if (!(BigInt.prototype as any).toJSON) { 2 | (BigInt.prototype as any).toJSON = function () { 3 | return this.toString(); 4 | }; 5 | } 6 | 7 | if (!String.prototype.replaceAll) { 8 | String.prototype.replaceAll = function (match, replace) { 9 | if (typeof replace === 'function') { 10 | throw new Error('Node.js does not support replaceAll with functions'); 11 | } 12 | return this.split(match).join(replace); 13 | }; 14 | } 15 | -------------------------------------------------------------------------------- /shared/promise.ts: -------------------------------------------------------------------------------- 1 | export function wait(t: number) { 2 | return new Promise((r) => setTimeout(r, t)); 3 | } 4 | -------------------------------------------------------------------------------- /shared/rpc.test.js: -------------------------------------------------------------------------------- 1 | require('./rpc'); 2 | 3 | test('stub', () => {}); 4 | -------------------------------------------------------------------------------- /shared/settings.ts: -------------------------------------------------------------------------------- 1 | import { LANGUAGES, SupportedLanguages } from './languages'; 2 | import { newId } from './object'; 3 | 4 | class LanguageSettings { 5 | path: string; 6 | 7 | constructor(path?: string) { 8 | this.path = path || ''; 9 | } 10 | } 11 | 12 | export class Settings { 13 | id: string; 14 | lastProject?: string; 15 | languages: Record; 16 | file: string; 17 | stdoutMaxSize: number; 18 | autocompleteDisabled: boolean; 19 | theme: 'light' | 'dark'; 20 | caCerts: Array<{ file: string; id: string }>; 21 | surveyAug2022: boolean; 22 | 23 | constructor( 24 | file: string, 25 | id?: string, 26 | lastProject?: string, 27 | languages?: Record, 28 | stdoutMaxSize?: number 29 | ) { 30 | this.id = id || newId(); 31 | this.autocompleteDisabled = false; 32 | this.lastProject = lastProject || ''; 33 | this.languages = 34 | languages || 35 | Object.keys(LANGUAGES).reduce( 36 | (agg, lang) => ({ 37 | ...agg, 38 | [lang]: new LanguageSettings(), 39 | }), 40 | {} 41 | ); 42 | this.stdoutMaxSize = stdoutMaxSize || 5000; 43 | this.file = file; 44 | this.caCerts = []; 45 | 46 | this.surveyAug2022 = true; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /shared/table.test.js: -------------------------------------------------------------------------------- 1 | const { columnsFromObject } = require('./table'); 2 | 3 | test('invalid table', () => { 4 | let error; 5 | try { 6 | columnsFromObject(null, [], 1); 7 | } catch (e) { 8 | error = e; 9 | } 10 | 11 | expect(error.message).toBe( 12 | 'This panel requires an array of objects as input. Make sure panel [1] returns an array of objects.' 13 | ); 14 | }); 15 | 16 | test('* columns, skip missing rows', () => { 17 | const testData = [{ a: 1, b: 2 }, null, { a: 10, b: 187 }]; 18 | 19 | const result = columnsFromObject(testData, [], 1); 20 | expect(result).toStrictEqual(testData); 21 | }); 22 | -------------------------------------------------------------------------------- /shared/table.ts: -------------------------------------------------------------------------------- 1 | import { NotAnArrayOfObjectsError } from './errors'; 2 | import { getPath } from './object'; 3 | 4 | export function columnsFromObject( 5 | value: any, 6 | columns: Array, 7 | panelSource: number | string, 8 | page: number, 9 | pageSize: number 10 | ) { 11 | if (!value || !Array.isArray(value)) { 12 | throw new NotAnArrayOfObjectsError(panelSource); 13 | } 14 | 15 | if (isNaN(page)) { 16 | page = 0; 17 | } 18 | 19 | if (isNaN(pageSize)) { 20 | pageSize = 15; 21 | } 22 | 23 | return (value || []) 24 | .slice(page * pageSize, (page + 1) * pageSize) 25 | .map((row: any) => { 26 | // If none specified, select all 27 | if (!columns.length) { 28 | return row; 29 | } 30 | 31 | if (!row) { 32 | return null; 33 | } 34 | 35 | const cells: Record = {}; 36 | (columns || []).forEach((name) => { 37 | cells[name] = getPath(row, name); 38 | }); 39 | return cells; 40 | }); 41 | } 42 | -------------------------------------------------------------------------------- /shared/url.test.js: -------------------------------------------------------------------------------- 1 | const { fullHttpURL, queryParameters } = require('./url'); 2 | 3 | describe('fullHttpUrl', () => { 4 | test('basics', () => { 5 | expect(fullHttpURL('')).toBe('http://localhost'); 6 | expect(fullHttpURL('https://foo.com', null, 9090)).toBe('https://foo.com'); 7 | expect(fullHttpURL('foo.com', null, 9090)).toBe('http://foo.com:9090'); 8 | expect(fullHttpURL('https://foo.com:8080', null, 9090)).toBe( 9 | 'https://foo.com:8080' 10 | ); 11 | 12 | expect(fullHttpURL('foo.com:9090', null, 9090)).toBe('http://foo.com:9090'); 13 | 14 | expect(fullHttpURL('https://foo.com', 8080, 9090)).toBe( 15 | 'https://foo.com:8080' 16 | ); 17 | expect(fullHttpURL('localhost')).toBe('http://localhost'); 18 | expect(fullHttpURL('localhost', 443)).toBe('https://localhost'); 19 | expect(fullHttpURL('localhost/foobar', 443)).toBe( 20 | 'https://localhost/foobar' 21 | ); 22 | expect(fullHttpURL('localhost/fluber')).toBe('http://localhost/fluber'); 23 | expect(fullHttpURL('localhost/fluber', 20)).toBe( 24 | 'http://localhost:20/fluber' 25 | ); 26 | }); 27 | }); 28 | 29 | describe('queryParameters', () => { 30 | test('basics', () => { 31 | expect(queryParameters({ a: 1, b: 2, c: undefined })).toBe('a=1&b=2'); 32 | }); 33 | }); 34 | -------------------------------------------------------------------------------- /shared/url.ts: -------------------------------------------------------------------------------- 1 | export function fullHttpURL( 2 | address: string, 3 | port: number | string, 4 | defaultPort?: number 5 | ) { 6 | address = address || 'localhost'; 7 | let guessedPort = port || defaultPort || '80'; 8 | 9 | let [domain, ...path] = address.split('/'); 10 | let protocol = ''; 11 | 12 | if (address.startsWith('http://') || address.startsWith('https://')) { 13 | protocol = 14 | domain + 15 | address.slice(address.indexOf(':') + 1, address.indexOf('//') + 2); 16 | [domain, ...path] = address.slice(protocol.length).split('/'); 17 | domain = protocol + domain; 18 | // When protocol is specified, don't let defaultPort override the given port. 19 | // But users can still explicitly override the port, just not the defaultPort. 20 | if (!port) { 21 | guessedPort = ''; 22 | } 23 | } else { 24 | if (String(guessedPort) === '443') { 25 | // Return full address with path 26 | return 'https://' + address; 27 | } 28 | 29 | // Only append to _domain_ so we can add port optionally later 30 | protocol = 'http://'; 31 | domain = protocol + domain; 32 | if (String(guessedPort) === '80') { 33 | return [domain, ...path].join('/'); 34 | } 35 | } 36 | 37 | if (!guessedPort || domain.slice(protocol.length).includes(':')) { 38 | return [domain, ...path].join('/'); 39 | } 40 | 41 | return [domain + ':' + String(guessedPort), ...path].join('/'); 42 | } 43 | 44 | export function queryParameters(d: Record) { 45 | let q = ''; 46 | for (const [key, value] of Object.entries(d)) { 47 | if (value === undefined) { 48 | continue; 49 | } 50 | 51 | if (q) { 52 | q += '&'; 53 | } 54 | 55 | q += `${key}=${encodeURIComponent(value)}`; 56 | } 57 | 58 | return q; 59 | } 60 | -------------------------------------------------------------------------------- /testdata/allformats/large.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/multiprocessio/datastation/4e72a6e0af28fcda767b0b29bb5dc9b697d05cb3/testdata/allformats/large.parquet -------------------------------------------------------------------------------- /testdata/allformats/unknown: -------------------------------------------------------------------------------- 1 | hey this is unknown -------------------------------------------------------------------------------- /testdata/allformats/userdata.ods: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/multiprocessio/datastation/4e72a6e0af28fcda767b0b29bb5dc9b697d05cb3/testdata/allformats/userdata.ods -------------------------------------------------------------------------------- /testdata/allformats/userdata.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/multiprocessio/datastation/4e72a6e0af28fcda767b0b29bb5dc9b697d05cb3/testdata/allformats/userdata.parquet -------------------------------------------------------------------------------- /testdata/allformats/userdata.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/multiprocessio/datastation/4e72a6e0af28fcda767b0b29bb5dc9b697d05cb3/testdata/allformats/userdata.xlsx -------------------------------------------------------------------------------- /testdata/bigquery/population_result.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "gender": null, 4 | "geo_id": "8600000US60629", 5 | "maximum_age": null, 6 | "minimum_age": null, 7 | "population": 113916, 8 | "zipcode": "60629" 9 | }, 10 | { 11 | "gender": null, 12 | "geo_id": "8600000US79936", 13 | "maximum_age": null, 14 | "minimum_age": null, 15 | "population": 111086, 16 | "zipcode": "79936" 17 | }, 18 | { 19 | "gender": null, 20 | "geo_id": "8600000US11368", 21 | "maximum_age": null, 22 | "minimum_age": null, 23 | "population": 109931, 24 | "zipcode": "11368" 25 | }, 26 | { 27 | "gender": null, 28 | "geo_id": "8600000US00926", 29 | "maximum_age": null, 30 | "minimum_age": null, 31 | "population": 108862, 32 | "zipcode": "926" 33 | }, 34 | { 35 | "gender": null, 36 | "geo_id": "8600000US90650", 37 | "maximum_age": null, 38 | "minimum_age": null, 39 | "population": 105549, 40 | "zipcode": "90650" 41 | }, 42 | { 43 | "gender": null, 44 | "geo_id": "8600000US90011", 45 | "maximum_age": null, 46 | "minimum_age": null, 47 | "population": 103892, 48 | "zipcode": "90011" 49 | }, 50 | { 51 | "gender": null, 52 | "geo_id": "8600000US91331", 53 | "maximum_age": null, 54 | "minimum_age": null, 55 | "population": 103689, 56 | "zipcode": "91331" 57 | }, 58 | { 59 | "gender": null, 60 | "geo_id": "8600000US11226", 61 | "maximum_age": null, 62 | "minimum_age": null, 63 | "population": 101572, 64 | "zipcode": "11226" 65 | }, 66 | { 67 | "gender": null, 68 | "geo_id": "8600000US90201", 69 | "maximum_age": null, 70 | "minimum_age": null, 71 | "population": 101279, 72 | "zipcode": "90201" 73 | }, 74 | { 75 | "gender": null, 76 | "geo_id": "8600000US11373", 77 | "maximum_age": null, 78 | "minimum_age": null, 79 | "population": 100820, 80 | "zipcode": "11373" 81 | } 82 | ] 83 | -------------------------------------------------------------------------------- /testdata/csv/missing_columns.csv: -------------------------------------------------------------------------------- 1 | a,b 2 | 3,4 3 | 1,2,3 4 | 5,6 -------------------------------------------------------------------------------- /testdata/documents/2.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Android in Action, Second Edition", 3 | "isbn": "1935182722", 4 | "pageCount": 592, 5 | "publishedDate": { 6 | "$date": "2011-01-14T00:00:00.000-0800" 7 | }, 8 | "thumbnailUrl": "https://s3.amazonaws.com/AKIAJC5RLADLUMVRPFDQ.book-thumb-images/ableson2.jpg", 9 | "shortDescription": "Android in Action, Second Edition is a comprehensive tutorial for Android developers. Taking you far beyond \"Hello Android,\" this fast-paced book puts you in the driver's seat as you learn important architectural concepts and implementation strategies. You'll master the SDK, build WebKit apps using HTML 5, and even learn to extend or replace Android's built-in features by building useful and intriguing examples. ", 10 | "longDescription": "When it comes to mobile apps, Android can do almost anything and with this book, so can you! Android runs on mobile devices ranging from smart phones to tablets to countless special-purpose gadgets. It's the broadest mobile platform available. Android in Action, Second Edition is a comprehensive tutorial for Android developers. Taking you far beyond \"Hello Android,\" this fast-paced book puts you in the driver's seat as you learn important architectural concepts and implementation strategies. You'll master the SDK, build WebKit apps using HTML 5, and even learn to extend or replace Android's built-in features by building useful and intriguing examples. ", 11 | "status": "PUBLISH", 12 | "authors": ["W. Frank Ableson", "Robi Sen"], 13 | "categories": ["Java"] 14 | } 15 | -------------------------------------------------------------------------------- /testdata/documents/3.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Specification by Example", 3 | "isbn": "1617290084", 4 | "pageCount": 0, 5 | "publishedDate": { 6 | "$date": "2011-06-03T00:00:00.000-0700" 7 | }, 8 | "thumbnailUrl": "https://s3.amazonaws.com/AKIAJC5RLADLUMVRPFDQ.book-thumb-images/adzic.jpg", 9 | "status": "PUBLISH", 10 | "authors": ["Gojko Adzic"], 11 | "categories": ["Software Engineering"] 12 | } 13 | -------------------------------------------------------------------------------- /testdata/documents/4.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Flex 3 in Action", 3 | "isbn": "1933988746", 4 | "pageCount": 576, 5 | "publishedDate": { 6 | "$date": "2009-02-02T00:00:00.000-0800" 7 | }, 8 | "thumbnailUrl": "https://s3.amazonaws.com/AKIAJC5RLADLUMVRPFDQ.book-thumb-images/ahmed.jpg", 9 | "longDescription": "New web applications require engaging user-friendly interfaces and the cooler, the better. With Flex 3, web developers at any skill level can create high-quality, effective, and interactive Rich Internet Applications (RIAs) quickly and easily. Flex removes the complexity barrier from RIA development by offering sophisticated tools and a straightforward programming language so you can focus on what you want to do instead of how to do it. And now that the major components of Flex are free and open-source, the cost barrier is gone, as well! Flex 3 in Action is an easy-to-follow, hands-on Flex tutorial. Chock-full of examples, this book goes beyond feature coverage and helps you put Flex to work in real day-to-day tasks. You'll quickly master the Flex API and learn to apply the techniques that make your Flex applications stand out from the crowd. Interesting themes, styles, and skins It's in there. Working with databases You got it. Interactive forms and validation You bet. Charting techniques to help you visualize data Bam! The expert authors of Flex 3 in Action have one goal to help you get down to business with Flex 3. Fast. Many Flex books are overwhelming to new users focusing on the complexities of the language and the super-specialized subjects in the Flex eco-system; Flex 3 in Action filters out the noise and dives into the core topics you need every day. Using numerous easy-to-understand examples, Flex 3 in Action gives you a strong foundation that you can build on as the complexity of your projects increases.", 10 | "status": "PUBLISH", 11 | "authors": ["Tariq Ahmed with Jon Hirschi", "Faisal Abid"], 12 | "categories": ["Internet"] 13 | } 14 | -------------------------------------------------------------------------------- /testdata/logs/apache.error.log: -------------------------------------------------------------------------------- 1 | [Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 72.15.99.187] File does not exist: /usr/local/apache2/htdocs/favicon.ico 2 | [Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708] [client 72.15.99.187] File does not exist: /usr/local/apache2/htdocs/favicon.ico -------------------------------------------------------------------------------- /testdata/logs/combinedlogformat.log: -------------------------------------------------------------------------------- 1 | 127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)" 2 | 216.67.1.91 - leon [01/Jul/2002:12:11:52 +0000] "GET /index.html HTTP/1.1" 200 431"http://www.loganalyzer.net/" "Mozilla/4.05 [en] (WinNT; I)" "USERID=CustomerA;IMPID=01234" 3 | 203.93.245.97 - oracleuser [28/Sep/2000:23:59:07 -0700] "GET /files/search/search.jsp?s=driver&a=10 HTTP/1.0" 200 2374 "http://datawarehouse.us.oracle.com/datamining/contents.htm" "Mozilla/4.7 [en] (WinNT; I) -------------------------------------------------------------------------------- /testdata/logs/commonlogformat.log: -------------------------------------------------------------------------------- 1 | 127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 2 | 127.0.0.1 - mybot [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.jpg HTTP/1.0" 202 2326 -------------------------------------------------------------------------------- /testdata/logs/custom.log: -------------------------------------------------------------------------------- 1 | GET /path2 www.google.com 8.8.8.10 2 | POST /mypath old.misc.org 99.10.10.1 3 | DELETE /path?x=y nine.org 10.0.0.1 -------------------------------------------------------------------------------- /testdata/logs/nginx.access.log: -------------------------------------------------------------------------------- 1 | 63.249.65.4 - - [06/Dev/2021:04:10:38 +0600] "GET /news/53f8d72920ba2744fe873ebc.html HTTP/1.1" 404 177 "-" "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 2 | 63.202.65.30 - - [06/Dev/2021:04:11:24 +0600] "GET /path3?ke=y HTTP/1.1" 200 4223 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" 3 | 63.200.65.2 - - [06/Dev/2021:04:12:14 +0600] "GET /path2 HTTP/1.1" 200 4356 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)" -------------------------------------------------------------------------------- /testdata/logs/syslogrfc3164.log: -------------------------------------------------------------------------------- 1 | <34>Oct 11 22:14:15 mymachine su: 'su root' failed for lonvick on /dev/pts/8 2 | <0>1990 Oct 22 10:52:01 TZ-6 scapegoat.dmz.example.org 10.1.2.3 sched[0]: That's All Folks! 3 | -------------------------------------------------------------------------------- /testdata/logs/syslogrfc5424.log: -------------------------------------------------------------------------------- 1 | <34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8 2 | <165>1 2003-08-24T05:14:15.000003-07:00 192.0.2.1 myproc 8710 - - %% It's time to make the do-nuts. 3 | <165>1 2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"] BOMAn application event log entry.. 4 | <165>1 2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"][examplePriority@32473 class="high"] -------------------------------------------------------------------------------- /testdata/prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s # By default, scrape targets every 15 seconds. 3 | 4 | # Attach these labels to any time series or alerts when communicating with 5 | # external systems (federation, remote storage, Alertmanager). 6 | external_labels: 7 | monitor: 'codelab-monitor' 8 | 9 | # A scrape configuration containing exactly one endpoint to scrape: 10 | # Here it's Prometheus itself. 11 | scrape_configs: 12 | # The job name is added as a label `job=` to any timeseries scraped from this config. 13 | - job_name: 'prometheus' 14 | 15 | # Override the global default and scrape targets from this job every 5 seconds. 16 | scrape_interval: 1s 17 | 18 | static_configs: 19 | - targets: ['localhost:9090'] 20 | -------------------------------------------------------------------------------- /testdata/regr/217.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/multiprocessio/datastation/4e72a6e0af28fcda767b0b29bb5dc9b697d05cb3/testdata/regr/217.xlsx -------------------------------------------------------------------------------- /testdata/regr/multiple-sheets.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/multiprocessio/datastation/4e72a6e0af28fcda767b0b29bb5dc9b697d05cb3/testdata/regr/multiple-sheets.xlsx -------------------------------------------------------------------------------- /testsetup.js: -------------------------------------------------------------------------------- 1 | const { act } = require('react-dom/test-utils'); 2 | require('jest-canvas-mock'); 3 | const { wait } = require('./shared/promise'); 4 | require('./shared/polyfill'); 5 | 6 | // https://enzymejs.github.io/enzyme/docs/guides/jsdom.html 7 | const { JSDOM } = require('jsdom'); 8 | const { configure } = require('enzyme'); 9 | const Adapter = require('@wojtekmaj/enzyme-adapter-react-17'); 10 | 11 | configure({ adapter: new Adapter() }); 12 | 13 | const jsdom = new JSDOM('', { 14 | url: 'http://localhost/?projectId=test', 15 | }); 16 | const { window } = jsdom; 17 | 18 | class ResizeObserver { 19 | observe() {} 20 | unobserve() {} 21 | disconnect() {} 22 | } 23 | 24 | window.ResizeObserver = ResizeObserver; 25 | 26 | function copyProps(src, target) { 27 | Object.defineProperties(target, { 28 | ...Object.getOwnPropertyDescriptors(src), 29 | ...Object.getOwnPropertyDescriptors(target), 30 | }); 31 | } 32 | 33 | global.window = window; 34 | global.document = window.document; 35 | global.navigator = { 36 | userAgent: 'node.js', 37 | }; 38 | global.requestAnimationFrame = function (callback) { 39 | return setTimeout(callback, 0); 40 | }; 41 | global.cancelAnimationFrame = function (id) { 42 | clearTimeout(id); 43 | }; 44 | copyProps(window, global); 45 | 46 | window.fetch = () => { 47 | return Promise.resolve({ 48 | text() { 49 | return Promise.resolve(null); 50 | }, 51 | json() { 52 | return Promise.resolve(null); 53 | }, 54 | }); 55 | }; 56 | 57 | global.componentLoad = async function (component) { 58 | await wait(1000); 59 | await act(async () => { 60 | await wait(0); 61 | component.update(); 62 | }); 63 | }; 64 | 65 | global.throwOnErrorBoundary = function (component) { 66 | component.find('ErrorBoundary').forEach((e) => { 67 | if (e.find({ type: 'fatal' }).length) { 68 | // Weird ways to find the actual error message 69 | throw new Error(e.find('Highlight').props().children); 70 | } 71 | }); 72 | }; 73 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "noImplicitAny": true, 4 | "noUnusedLocals": true, 5 | "noEmit": true, 6 | "jsx": "react", 7 | "allowJs": true, 8 | "target": "esnext", 9 | "module": "commonjs", 10 | "skipLibCheck": true, 11 | "esModuleInterop": true, 12 | "typeRoots": ["./node_modules/@types", "./type-overrides"], 13 | "resolveJsonModule": true 14 | }, 15 | "exclude": ["node_modules", "*.js", "build"] 16 | } 17 | -------------------------------------------------------------------------------- /type-overrides/ace.d.ts: -------------------------------------------------------------------------------- 1 | declare module 'ace-builds/src-min-noconflict/ace'; 2 | declare module 'ace-builds/src-min-noconflict/ext-language_tools'; 3 | -------------------------------------------------------------------------------- /type-overrides/nodemailer.d.ts: -------------------------------------------------------------------------------- 1 | declare module 'nodemailer' { 2 | export interface SendMailOptions { 3 | from: string; 4 | to: string; 5 | subject: string; 6 | text?: string; 7 | html?: string; 8 | } 9 | 10 | export class SMTPTransport { 11 | sendMail(SendMailOptions): Promise; 12 | } 13 | 14 | export interface TransportOptions { 15 | host: string; 16 | port: number; 17 | auth?: { 18 | user: string; 19 | pass: string; 20 | }; 21 | } 22 | 23 | export function createTransport(TransportOptions): SMTPTransport; 24 | } 25 | -------------------------------------------------------------------------------- /ui/DatabaseConnector.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { DatabaseConnectorInfo } from '../shared/state'; 3 | import { VENDORS } from './connectors'; 4 | import { ProjectContext } from './state'; 5 | 6 | export function DatabaseConnector({ 7 | connector, 8 | updateConnector, 9 | }: { 10 | connector: DatabaseConnectorInfo; 11 | updateConnector: (dc: DatabaseConnectorInfo) => void; 12 | }) { 13 | const { servers } = React.useContext(ProjectContext).state; 14 | const { details: Details } = VENDORS[connector.database.type]; 15 | return ( 16 |
21 | ); 22 | } 23 | -------------------------------------------------------------------------------- /ui/Footer.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { APP_NAME, VERSION } from '../shared/constants'; 3 | 4 | export function Footer() { 5 | return ( 6 |
7 |

8 | 12 | Join the mailing list 13 | {' '} 14 | to stay up-to-date on releases, new tutorials, and everything else! 15 |

16 |
17 | {APP_NAME} {VERSION} 18 |
19 |
20 | ); 21 | } 22 | -------------------------------------------------------------------------------- /ui/Header.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { APP_NAME, MODE } from '../shared/constants'; 3 | import '../shared/polyfill'; 4 | import { DEFAULT_PROJECT } from '../shared/state'; 5 | import { Link } from './components/Link'; 6 | import { LocalStorageStore } from './ProjectStore'; 7 | import { UrlStateContext } from './urlState'; 8 | 9 | export function loadDefaultProject() { 10 | const store = new LocalStorageStore(); 11 | store.update(DEFAULT_PROJECT.projectName, DEFAULT_PROJECT); 12 | window.location.href = '/?projectId=' + DEFAULT_PROJECT.projectName; 13 | } 14 | 15 | export function Header() { 16 | const { 17 | state: { projectId }, 18 | } = React.useContext(UrlStateContext); 19 | 20 | return ( 21 |
22 |
23 | 24 | {APP_NAME} 25 | 26 |
27 | {MODE === 'browser' ? ( 28 | 33 | 41 | 42 | ) : ( 43 | {projectId} 44 | )} 45 |
46 |
47 |
48 | ); 49 | } 50 | -------------------------------------------------------------------------------- /ui/Help.test.jsx: -------------------------------------------------------------------------------- 1 | const React = require('react'); 2 | const enzyme = require('enzyme'); 3 | const { Help } = require('./Help'); 4 | 5 | test('help page', () => { 6 | const component = enzyme.mount(); 7 | }); 8 | -------------------------------------------------------------------------------- /ui/Navigation.tsx: -------------------------------------------------------------------------------- 1 | import { TablerIcon } from '@tabler/icons'; 2 | import React from 'react'; 3 | import { Button } from './components/Button'; 4 | import { DefaultView, UrlStateContext } from './urlState'; 5 | 6 | export function Navigation({ 7 | pages, 8 | }: { 9 | pages: Array<{ title: string; icon: TablerIcon; endpoint: View }>; 10 | }) { 11 | const { 12 | state: { projectId, view }, 13 | setState: setUrlState, 14 | } = React.useContext(UrlStateContext); 15 | 16 | return ( 17 |
18 | {pages.map((page) => ( 19 |
26 | 38 |
39 | ))} 40 |
41 | ); 42 | } 43 | -------------------------------------------------------------------------------- /ui/NotFound.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | export function NotFound() { 4 | return

Page not found!

; 5 | } 6 | -------------------------------------------------------------------------------- /ui/ServerList.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { ProjectState, ServerInfo } from '../shared/state'; 3 | import { Button } from './components/Button'; 4 | import { Server } from './Server'; 5 | 6 | export function ServerList({ 7 | state, 8 | updateServer, 9 | deleteServer, 10 | }: { 11 | state: ProjectState; 12 | updateServer: (dc: ServerInfo, position: number, insert: boolean) => void; 13 | deleteServer: (id: string) => void; 14 | }) { 15 | return ( 16 |
17 |

SSH Connections

18 | {state.servers?.map((dc: ServerInfo, i: number) => ( 19 | updateServer(dc, i, false)} 23 | deleteServer={() => deleteServer(dc.id)} 24 | /> 25 | ))} 26 |
27 | 40 |
41 |
42 | ); 43 | } 44 | -------------------------------------------------------------------------------- /ui/Sidebar.tsx: -------------------------------------------------------------------------------- 1 | import { IconChevronLeft, IconChevronRight } from '@tabler/icons'; 2 | import * as React from 'react'; 3 | import { Button } from './components/Button'; 4 | import { UrlStateContext } from './urlState'; 5 | 6 | export function Sidebar({ children }: { children: React.ReactNode }) { 7 | const { 8 | state: { sidebar: expanded }, 9 | setState: setUrlState, 10 | } = React.useContext(UrlStateContext); 11 | 12 | function setExpanded(v: boolean) { 13 | setUrlState({ sidebar: v }); 14 | } 15 | 16 | return ( 17 |
18 |
19 | 28 |
29 | {expanded && children} 30 |
31 | ); 32 | } 33 | -------------------------------------------------------------------------------- /ui/Updates.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { SITE_ROOT, VERSION } from '../shared/constants'; 3 | import { request } from '../shared/http'; 4 | import log from '../shared/log'; 5 | import { newId } from '../shared/object'; 6 | import { ContentTypeInfo } from '../shared/state'; 7 | import { SettingsContext } from './Settings'; 8 | 9 | const s = newId(); 10 | 11 | export function Updates() { 12 | const { state: settings } = React.useContext(SettingsContext); 13 | const [updates, setUpdates] = React.useState(null); 14 | React.useEffect( 15 | function getUpdates() { 16 | async function run() { 17 | try { 18 | const updates = await request( 19 | window.fetch, 20 | 'GET', 21 | `${SITE_ROOT}/api/updates?version=${VERSION}&i=${settings.id}&s=${s}`, 22 | new ContentTypeInfo(), 23 | [], 24 | '', 25 | true 26 | ); 27 | setUpdates(updates); 28 | } catch (e) { 29 | log.error(e); 30 | } 31 | } 32 | 33 | run(); 34 | }, 35 | [settings.id] 36 | ); 37 | 38 | if (!updates) { 39 | return null; 40 | } 41 | 42 | return ( 43 |
44 |
Updates
45 |
    46 | {updates.updates.map(function renderUpdate(u: string) { 47 | return
  • {u}
    • ; 48 | })} 49 |
50 |
51 | ); 52 | } 53 | -------------------------------------------------------------------------------- /ui/asyncRPC.ts: -------------------------------------------------------------------------------- 1 | import { MODE } from '../shared/constants'; 2 | import { 3 | Endpoint, 4 | PanelBody, 5 | PanelEndpoint, 6 | WindowAsyncRPC, 7 | } from '../shared/rpc'; 8 | import { PanelResult } from '../shared/state'; 9 | import { DefaultView, getUrlState } from './urlState'; 10 | 11 | export async function asyncRPC< 12 | Request = void, 13 | Response = void, 14 | EndpointT extends string = Endpoint, 15 | ViewT extends DefaultView = DefaultView 16 | >(resource: EndpointT, body: Request): Promise { 17 | const { projectId } = getUrlState(); 18 | 19 | if (MODE === 'desktop') { 20 | // this method is exposed by ../desktop/preload.ts in Electron environments 21 | const arpc = (window as any).asyncRPC as WindowAsyncRPC; 22 | return arpc(resource, projectId, body); 23 | } 24 | 25 | const rsp = await window.fetch( 26 | `/a/rpc?resource=${resource}&projectId=${projectId}`, 27 | { 28 | method: 'post', 29 | headers: { 30 | 'content-type': 'application/json', 31 | }, 32 | body: JSON.stringify({ 33 | body, 34 | }), 35 | } 36 | ); 37 | 38 | if (rsp.status === 401) { 39 | window.location.href = '/a/auth?projectId=' + projectId; 40 | return null; 41 | } 42 | 43 | if (rsp.status !== 200) { 44 | throw await rsp.json(); 45 | } 46 | 47 | return await rsp.json(); 48 | } 49 | 50 | export function panelRPC( 51 | endpoint: PanelEndpoint, 52 | panelId: string 53 | ): Promise { 54 | return asyncRPC(endpoint, { panelId }); 55 | } 56 | -------------------------------------------------------------------------------- /ui/components/Alert.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | IconAlertTriangle, 3 | IconBug, 4 | IconHeartbeat, 5 | IconInfoCircle, 6 | } from '@tabler/icons'; 7 | import * as React from 'react'; 8 | 9 | type AlertTypes = 'fatal' | 'error' | 'warning' | 'info'; 10 | 11 | export function Alert({ 12 | type, 13 | children, 14 | }: { 15 | type: AlertTypes; 16 | children: React.ReactNode; 17 | }) { 18 | const icon = ( 19 | { 20 | info: , 21 | warning: , 22 | error: , 23 | fatal: , 24 | } as Record 25 | )[type]; 26 | return ( 27 |
28 | {icon} 29 |
{children}
30 |
31 | ); 32 | } 33 | -------------------------------------------------------------------------------- /ui/components/Button.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | 3 | interface ButtonProps 4 | extends Omit< 5 | Omit, 'type'>, 6 | 'style' 7 | > { 8 | type?: 'primary' | 'outline'; 9 | icon?: boolean; 10 | options?: React.ReactNode; 11 | } 12 | 13 | export function Button({ className, type, icon, ...props }: ButtonProps) { 14 | let buttonClass = `button ${className ? ' ' + className : ''}`; 15 | if (type) { 16 | buttonClass += ` button--${type}`; 17 | } 18 | 19 | if (icon) { 20 | buttonClass += ' button--icon'; 21 | } 22 | 23 | return 40 | 50 | 51 | 52 | 53 | 54 | ); 55 | } 56 | -------------------------------------------------------------------------------- /ui/components/Datetime.test.jsx: -------------------------------------------------------------------------------- 1 | const React = require('react'); 2 | const enzyme = require('enzyme'); 3 | const { Datetime } = require('./Datetime'); 4 | 5 | test('datetime', () => { 6 | const props = { 7 | value: new Date(), 8 | onChange: () => {}, 9 | label: 'My datetime', 10 | }; 11 | const component = enzyme.mount(); 12 | }); 13 | -------------------------------------------------------------------------------- /ui/components/Datetime.tsx: -------------------------------------------------------------------------------- 1 | import format from 'date-fns/format'; 2 | import React from 'react'; 3 | import { Input, InputProps } from './Input'; 4 | 5 | export interface Datetime extends Omit { 6 | value: Date; 7 | onChange: (d: Date) => void; 8 | } 9 | 10 | export function Datetime({ 11 | value, 12 | label, 13 | className, 14 | onChange, 15 | ...props 16 | }: Datetime) { 17 | const date = format(new Date(value), 'yyyy-MM-dd'); 18 | const time = format(new Date(value), 'HH:mm'); 19 | 20 | function onDateChange(newDate: string) { 21 | const copy = new Date(value); 22 | const nd = new Date(newDate); 23 | copy.setDate(nd.getDate()); 24 | copy.setMonth(nd.getMonth()); 25 | copy.setFullYear(nd.getFullYear()); 26 | onChange(copy); 27 | } 28 | 29 | function onTimeChange(newTime: string) { 30 | const copy = new Date(value); 31 | const [h, m] = newTime.split(':'); 32 | copy.setHours(+h || 0); 33 | copy.setMinutes(+m || 0); 34 | onChange(copy); 35 | } 36 | 37 | return ( 38 |
39 | 46 | 47 |
48 | ); 49 | } 50 | -------------------------------------------------------------------------------- /ui/components/Dropdown.test.jsx: -------------------------------------------------------------------------------- 1 | const React = require('react'); 2 | const enzyme = require('enzyme'); 3 | const { Dropdown } = require('./Dropdown'); 4 | 5 | test('dropdown', () => { 6 | const props = { 7 | groups: [], 8 | title: 'My dropdown', 9 | trigger: () => {}, 10 | }; 11 | const component = enzyme.mount(); 12 | }); 13 | -------------------------------------------------------------------------------- /ui/components/Dropdown.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | export function Dropdown({ 4 | groups, 5 | title, 6 | trigger, 7 | className, 8 | }: { 9 | groups: { 10 | name: string; 11 | id: string; 12 | items: { render: (close: () => void) => React.ReactElement; id: string }[]; 13 | }[]; 14 | title?: string; 15 | trigger: (open: () => void) => React.ReactElement; 16 | className?: string; 17 | }) { 18 | const [open, toggleDropdown] = React.useState(false); 19 | 20 | return ( 21 |
{ 25 | // Ignore onBlur for child elements 26 | if (!e.currentTarget.contains(e.relatedTarget)) { 27 | toggleDropdown(false); 28 | } 29 | }} 30 | > 31 |
32 | {trigger(() => toggleDropdown(() => true))} 33 |
34 |
35 | {title &&
} 36 | {groups.map((group) => ( 37 |
38 | {group.name && ( 39 |
{group.name}
40 | )} 41 | {group.items.map((item) => ( 42 |
43 |
44 | {item.render(() => toggleDropdown(false))} 45 |
46 |
47 | ))} 48 |
49 | ))} 50 |
51 |
52 | ); 53 | } 54 | 55 | Dropdown.makeItem = ( 56 | id: string, 57 | render: (close: () => void) => React.ReactElement 58 | ) => ({ render, id }); 59 | -------------------------------------------------------------------------------- /ui/components/FieldPicker.test.jsx: -------------------------------------------------------------------------------- 1 | const { shape } = require('shape'); 2 | const { allFields, orderedObjectFields } = require('./FieldPicker'); 3 | 4 | const sample1 = shape([ 5 | { 6 | a: '1', 7 | x: 1, 8 | z: '1', 9 | c: { t: 100, n: { b: 'Kevin' } }, 10 | // Non-scalar anything 11 | d: {}, 12 | }, 13 | ]); 14 | 15 | test('orderedObjectFields preferring string', () => { 16 | expect(orderedObjectFields(sample1)).toStrictEqual([ 17 | { 18 | name: 'String', 19 | elements: [ 20 | ['a', { kind: 'scalar', name: 'string' }], 21 | ['c.n.b', { kind: 'scalar', name: 'string' }], 22 | ['z', { kind: 'scalar', name: 'string' }], 23 | ], 24 | }, 25 | { 26 | name: 'Number', 27 | elements: [ 28 | ['c.t', { kind: 'scalar', name: 'number' }], 29 | ['x', { kind: 'scalar', name: 'number' }], 30 | ], 31 | }, 32 | ]); 33 | }); 34 | 35 | test('orderedObjectFields preferring number', () => { 36 | expect(orderedObjectFields(sample1, 'number')).toStrictEqual([ 37 | { 38 | name: 'Number', 39 | elements: [ 40 | ['c.t', { kind: 'scalar', name: 'number' }], 41 | ['x', { kind: 'scalar', name: 'number' }], 42 | ], 43 | }, 44 | { 45 | name: 'String', 46 | elements: [ 47 | ['a', { kind: 'scalar', name: 'string' }], 48 | ['c.n.b', { kind: 'scalar', name: 'string' }], 49 | ['z', { kind: 'scalar', name: 'string' }], 50 | ], 51 | }, 52 | ]); 53 | }); 54 | 55 | test('all fields', () => { 56 | const sample = shape([ 57 | { a: 1, b: '2', c: { x: 9100 }, d: 'maybe' }, 58 | { a: 10, b: '99', c: { x: 80 }, e: 90 }, 59 | ]); 60 | 61 | expect(allFields(sample)).toStrictEqual([ 62 | ['a', { kind: 'scalar', name: 'number' }], 63 | ['b', { kind: 'scalar', name: 'string' }], 64 | ['c.x', { kind: 'scalar', name: 'number' }], 65 | ['d', { kind: 'scalar', name: 'string' }], 66 | ['e', { kind: 'scalar', name: 'number' }], 67 | ]); 68 | }); 69 | -------------------------------------------------------------------------------- /ui/components/FormGroup.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | export function FormGroup({ 4 | label, 5 | children, 6 | major, 7 | optional, 8 | }: { 9 | major?: boolean; 10 | label?: string; 11 | optional?: string; 12 | children: React.ReactNode; 13 | }) { 14 | const body = ( 15 |
16 | {label && } 17 |
{children}
18 |
19 | ); 20 | 21 | if (!optional) { 22 | return body; 23 | } 24 | 25 | return ( 26 |
27 | {optional} 28 | {body} 29 |
30 | ); 31 | } 32 | -------------------------------------------------------------------------------- /ui/components/Highlight.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { PrismLight as SyntaxHighlighter } from 'react-syntax-highlighter'; 3 | import vsStyle from 'react-syntax-highlighter/dist/esm/styles/prism/vs'; 4 | import vsdpStyle from 'react-syntax-highlighter/dist/esm/styles/prism/vsc-dark-plus'; 5 | import javascript from 'refractor/lang/javascript.js'; 6 | import json from 'refractor/lang/json.js'; 7 | 8 | window.addEventListener('load', function () { 9 | SyntaxHighlighter.registerLanguage('javascript', javascript); 10 | SyntaxHighlighter.registerLanguage('json', json); 11 | 12 | for (const style of [vsdpStyle, vsStyle]) { 13 | style['code[class*="language-"]'].fontFamily = ''; 14 | style['code[class*="language-"]'].fontSize = ''; 15 | style['code[class*="language-"]'].background = ''; 16 | style['pre[class*="language-"]'].fontFamily = ''; 17 | style['pre[class*="language-"]'].fontSize = ''; 18 | style['pre[class*="language-"]'].background = ''; 19 | style['pre[class*="language-"]'].overflow = ''; 20 | } 21 | }); 22 | 23 | export function Highlight({ 24 | language, 25 | children, 26 | theme = 'dark', 27 | }: { 28 | language: string; 29 | children: string; 30 | theme?: 'dark' | 'light'; 31 | }) { 32 | const style = { 33 | dark: vsdpStyle, 34 | light: vsStyle, 35 | }[theme]; 36 | return ( 37 | 38 | ); 39 | } 40 | -------------------------------------------------------------------------------- /ui/components/Link.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { UrlState, UrlStateContext } from '../urlState'; 3 | 4 | interface LinkProps { 5 | args: Partial; 6 | className?: string; 7 | children?: React.ReactNode; 8 | } 9 | 10 | export function Link({ args, className, children }: LinkProps) { 11 | const url = 12 | '?' + 13 | Object.entries(args) 14 | .map(([k, v]) => `${k}=${v}`) 15 | .join('&'); 16 | const { setState: setUrlState } = React.useContext(UrlStateContext); 17 | function navigate(e: React.SyntheticEvent) { 18 | e.preventDefault(); 19 | setUrlState(args); 20 | } 21 | 22 | return ( 23 | 24 | {children} 25 | 26 | ); 27 | } 28 | -------------------------------------------------------------------------------- /ui/components/Loading.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | export function Loading() { 4 | return ( 5 |
6 | Loading... 7 | 8 |
9 | ); 10 | } 11 | -------------------------------------------------------------------------------- /ui/components/PanelSourcePicker.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { PanelInfo } from '../../shared/state'; 3 | import { Select } from './Select'; 4 | 5 | export function PanelSourcePicker({ 6 | value, 7 | onChange, 8 | panels, 9 | currentPanel, 10 | }: { 11 | value: string; 12 | onChange: (n: string) => void; 13 | panels: Array; 14 | currentPanel: string; 15 | }) { 16 | const reversed = panels.slice().reverse(); 17 | return ( 18 | 34 | ); 35 | } 36 | -------------------------------------------------------------------------------- /ui/components/Radio.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Input, InputProps } from './Input'; 3 | 4 | interface RadioProps extends InputProps { 5 | options: Array<{ label: string; value: string }>; 6 | vertical?: boolean; 7 | } 8 | 9 | export function Radio({ 10 | options, 11 | value, 12 | label, 13 | vertical, 14 | ...props 15 | }: RadioProps) { 16 | React.useEffect(() => { 17 | if (!options.length) { 18 | return; 19 | } 20 | 21 | if (!options.map((o) => String(o.value)).includes(String(value))) { 22 | props.onChange(String(options[0].value)); 23 | } 24 | }); 25 | 26 | const radioClass = 'radio' + (vertical ? ' radio--vertical' : ''); 27 | 28 | const radio = ( 29 | 35 | {options.map((o) => { 36 | const checked = String(o.value) === String(value); 37 | return ( 38 | 49 | ); 50 | })} 51 | 52 | ); 53 | 54 | if (label) { 55 | return ( 56 | 60 | ); 61 | } 62 | 63 | return radio; 64 | } 65 | -------------------------------------------------------------------------------- /ui/components/ServerPicker.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { ServerInfo } from '../../shared/state'; 3 | import { FormGroup } from './FormGroup'; 4 | import { Select } from './Select'; 5 | 6 | export function ServerPicker({ 7 | serverId, 8 | servers, 9 | onChange, 10 | }: { 11 | serverId?: string; 12 | servers: Array; 13 | onChange: (v: string) => void; 14 | }) { 15 | const [internalValue, setInternalValue] = React.useState( 16 | serverId === '' || serverId === null ? 'null' : serverId 17 | ); 18 | React.useEffect(() => { 19 | setInternalValue(serverId === '' || serverId === null ? 'null' : serverId); 20 | }, [serverId]); 21 | 22 | if (!servers.length) { 23 | return null; 24 | } 25 | 26 | return ( 27 | 28 |
29 | 41 |
42 | 43 | ); 44 | } 45 | -------------------------------------------------------------------------------- /ui/components/Toggle.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | export function Toggle({ 4 | label, 5 | className, 6 | value, 7 | rhsLabel, 8 | onChange, 9 | }: { 10 | label: string; 11 | rhsLabel: React.ReactNode; 12 | value: boolean; 13 | onChange: () => void; 14 | className?: string; 15 | }) { 16 | return ( 17 | 25 | ); 26 | } 27 | -------------------------------------------------------------------------------- /ui/components/Tooltip.tsx: -------------------------------------------------------------------------------- 1 | import { IconHelp } from '@tabler/icons'; 2 | import React from 'react'; 3 | 4 | export function Tooltip({ children }: { children: React.ReactNode }) { 5 | const ref = React.useRef(null); 6 | const [positioned, setPositioned] = React.useState(false); 7 | React.useEffect(() => { 8 | if (ref.current && !positioned) { 9 | const b = ref.current.querySelector('.tooltip-body'); 10 | b.style.top += ref.current.offsetTop + ref.current.offsetHeight; 11 | b.style.left += ref.current.offsetLeft; 12 | setPositioned(true); 13 | } 14 | }, [positioned]); 15 | 16 | return ( 17 | { 19 | ref.current = r; 20 | }} 21 | className="tooltip" 22 | > 23 | 24 | 25 | 26 | {children} 27 | 28 | ); 29 | } 30 | -------------------------------------------------------------------------------- /ui/connectors/AirtableDetails.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { DatabaseConnectorInfo, ServerInfo } from '../../shared/state'; 3 | import { FormGroup } from '../components/FormGroup'; 4 | import { ApiKey } from './ApiKey'; 5 | 6 | export function AirtableDetails(props: { 7 | connector: DatabaseConnectorInfo; 8 | updateConnector: (c: DatabaseConnectorInfo) => void; 9 | servers: Array; 10 | }) { 11 | return ( 12 | 13 | 14 | 15 | ); 16 | } 17 | -------------------------------------------------------------------------------- /ui/connectors/ApiKey.test.jsx: -------------------------------------------------------------------------------- 1 | const React = require('react'); 2 | const enzyme = require('enzyme'); 3 | const { DatabaseConnectorInfo } = require('../../shared/state'); 4 | const { ApiKey } = require('./ApiKey'); 5 | 6 | test('ApiKey shows input and changes', async () => { 7 | const connector = new DatabaseConnectorInfo(); 8 | 9 | const changeTo = 'my-great-password'; 10 | let changed = ''; 11 | const updateConnector = jest.fn((conn) => { 12 | changed = conn.database.apiKey_encrypt.value; 13 | }); 14 | const component = enzyme.mount( 15 | 16 | ); 17 | 18 | expect(changed).toBe(''); 19 | await component 20 | .find('input') 21 | .simulate('change', { target: { value: changeTo } }); 22 | await component.find('input').simulate('blur'); 23 | expect(changed).toBe(changeTo); 24 | }); 25 | -------------------------------------------------------------------------------- /ui/connectors/ApiKey.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { DatabaseConnectorInfo, Encrypt } from '../../shared/state'; 3 | import { Input } from '../components/Input'; 4 | 5 | export function ApiKey({ 6 | connector, 7 | updateConnector, 8 | label = 'Api Key', 9 | }: { 10 | connector: DatabaseConnectorInfo; 11 | updateConnector: (c: DatabaseConnectorInfo) => void; 12 | label?: string; 13 | }) { 14 | // Don't try to show initial apiKey 15 | const [apiKey, setApiKey] = React.useState(''); 16 | function syncApiKey(p: string) { 17 | setApiKey(p); 18 | // Sync typed apiKey to state on change 19 | connector.database.apiKey_encrypt = new Encrypt(p); 20 | updateConnector(connector); 21 | } 22 | 23 | return ( 24 |
25 | syncApiKey(value)} 30 | /> 31 |
32 | ); 33 | } 34 | -------------------------------------------------------------------------------- /ui/connectors/Auth.test.jsx: -------------------------------------------------------------------------------- 1 | const React = require('react'); 2 | const enzyme = require('enzyme'); 3 | const { DatabaseConnectorInfo } = require('../../shared/state'); 4 | const { Auth } = require('./Auth'); 5 | 6 | test('Auth mounts', async () => { 7 | const connector = new DatabaseConnectorInfo(); 8 | const component = enzyme.mount( 9 | 10 | ); 11 | }); 12 | -------------------------------------------------------------------------------- /ui/connectors/Auth.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { DatabaseConnectorInfo } from '../../shared/state'; 3 | import { FormGroup } from '../components/FormGroup'; 4 | import { Select } from '../components/Select'; 5 | import { ApiKey } from './ApiKey'; 6 | import { Password } from './Password'; 7 | import { Username } from './Username'; 8 | 9 | export function Auth(props: { 10 | connector: DatabaseConnectorInfo; 11 | updateConnector: (c: DatabaseConnectorInfo) => void; 12 | apiKeyLabel?: string; 13 | }) { 14 | const { connector, apiKeyLabel } = props; 15 | 16 | const [authMethod, setAuthMethod] = React.useState( 17 | connector.database.apiKey_encrypt.value || 18 | connector.database.apiKey_encrypt.encrypted 19 | ? 'apikey' 20 | : connector.database.password_encrypt.value || 21 | connector.database.password_encrypt.encrypted 22 | ? 'basic' 23 | : 'bearer' 24 | ); 25 | 26 | return ( 27 | 28 |
29 | 37 |
38 | {authMethod === 'apikey' && } 39 | {authMethod === 'basic' && ( 40 | 41 | 42 | 43 | 44 | )} 45 | 46 | ); 47 | } 48 | -------------------------------------------------------------------------------- /ui/connectors/BigQueryDetails.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { DatabaseConnectorInfo } from '../../shared/state'; 3 | import { FormGroup } from '../components/FormGroup'; 4 | import { ApiKey } from './ApiKey'; 5 | import { Database } from './Database'; 6 | 7 | export function BigQueryDetails({ 8 | connector, 9 | updateConnector, 10 | }: { 11 | connector: DatabaseConnectorInfo; 12 | updateConnector: (c: DatabaseConnectorInfo) => void; 13 | }) { 14 | return ( 15 | 16 | 17 | 22 | 27 | 28 | 29 | ); 30 | } 31 | -------------------------------------------------------------------------------- /ui/connectors/CassandraDetails.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { DatabaseConnectorInfo, ServerInfo } from '../../shared/state'; 3 | import { FormGroup } from '../components/FormGroup'; 4 | import { ServerPicker } from '../components/ServerPicker'; 5 | import { Database } from './Database'; 6 | import { Host } from './Host'; 7 | import { Password } from './Password'; 8 | import { Username } from './Username'; 9 | 10 | export function CassandraDetails(props: { 11 | connector: DatabaseConnectorInfo; 12 | updateConnector: (c: DatabaseConnectorInfo) => void; 13 | servers: Array; 14 | }) { 15 | const { servers, connector, updateConnector } = props; 16 | 17 | return ( 18 | 19 | 20 | 21 | 26 | 27 | 28 | 29 | { 33 | connector.serverId = serverId; 34 | updateConnector(connector); 35 | }} 36 | /> 37 | 38 | ); 39 | } 40 | -------------------------------------------------------------------------------- /ui/connectors/Database.test.jsx: -------------------------------------------------------------------------------- 1 | const React = require('react'); 2 | const enzyme = require('enzyme'); 3 | const { DatabaseConnectorInfo } = require('../../shared/state'); 4 | const { wait } = require('../../shared/promise'); 5 | const { Database } = require('./Database'); 6 | 7 | test('Database shows input and changes', async () => { 8 | const connector = new DatabaseConnectorInfo(); 9 | 10 | const changeTo = 'localhost:9090'; 11 | let changed = ''; 12 | const updateConnector = jest.fn((conn) => { 13 | changed = conn.database.database; 14 | }); 15 | const component = enzyme.mount( 16 | 17 | ); 18 | 19 | expect(changed).toBe(''); 20 | await component 21 | .find('input') 22 | .simulate('change', { target: { value: changeTo } }); 23 | await component.find('input').simulate('blur'); 24 | expect(changed).toBe(changeTo); 25 | }); 26 | -------------------------------------------------------------------------------- /ui/connectors/Database.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { DatabaseConnectorInfo } from '../../shared/state'; 3 | import { Input } from '../components/Input'; 4 | 5 | export function Database({ 6 | connector, 7 | updateConnector, 8 | placeholder, 9 | label = 'Database', 10 | }: { 11 | connector: DatabaseConnectorInfo; 12 | updateConnector: (c: DatabaseConnectorInfo) => void; 13 | placeholder?: string; 14 | label?: string; 15 | }) { 16 | return ( 17 |
18 | { 22 | connector.database.database = value; 23 | updateConnector(connector); 24 | }} 25 | placeholder={placeholder} 26 | /> 27 |
28 | ); 29 | } 30 | -------------------------------------------------------------------------------- /ui/connectors/ElasticsearchDetails.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { DatabaseConnectorInfo, ServerInfo } from '../../shared/state'; 3 | import { FormGroup } from '../components/FormGroup'; 4 | import { ServerPicker } from '../components/ServerPicker'; 5 | import { Auth } from './Auth'; 6 | import { Host } from './Host'; 7 | 8 | export function ElasticsearchDetails(props: { 9 | connector: DatabaseConnectorInfo; 10 | updateConnector: (c: DatabaseConnectorInfo) => void; 11 | servers: Array; 12 | }) { 13 | const { connector, updateConnector, servers } = props; 14 | 15 | return ( 16 | 17 | 18 | 19 | 20 | 21 | { 25 | connector.serverId = serverId; 26 | updateConnector(connector); 27 | }} 28 | /> 29 | 30 | ); 31 | } 32 | -------------------------------------------------------------------------------- /ui/connectors/FluxDetails.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { DatabaseConnectorInfo, ServerInfo } from '../../shared/state'; 3 | import { FormGroup } from '../components/FormGroup'; 4 | import { ServerPicker } from '../components/ServerPicker'; 5 | import { ApiKey } from './ApiKey'; 6 | import { Database } from './Database'; 7 | import { Host } from './Host'; 8 | 9 | export function FluxDetails(props: { 10 | connector: DatabaseConnectorInfo; 11 | updateConnector: (c: DatabaseConnectorInfo) => void; 12 | servers: Array; 13 | }) { 14 | const { connector, updateConnector, servers } = props; 15 | 16 | return ( 17 | 18 | 19 | 20 | 21 | 22 | 23 | { 27 | connector.serverId = serverId; 28 | updateConnector(connector); 29 | }} 30 | /> 31 | 32 | ); 33 | } 34 | -------------------------------------------------------------------------------- /ui/connectors/GenericDetails.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { DatabaseConnectorInfo, ServerInfo } from '../../shared/state'; 3 | import { FormGroup } from '../components/FormGroup'; 4 | import { ServerPicker } from '../components/ServerPicker'; 5 | import { Database } from './Database'; 6 | import { Host } from './Host'; 7 | import { Password } from './Password'; 8 | import { Username } from './Username'; 9 | 10 | interface GenericDetailsProps { 11 | connector: DatabaseConnectorInfo; 12 | updateConnector: (c: DatabaseConnectorInfo) => void; 13 | servers: Array; 14 | skipDatabase?: boolean; 15 | } 16 | 17 | export function GenericDetails(props: GenericDetailsProps) { 18 | const { servers, connector, updateConnector, skipDatabase } = props; 19 | 20 | return ( 21 | 22 | 23 | 24 | {skipDatabase ? null : ( 25 | 26 | )} 27 | 28 | 29 | 30 | { 34 | connector.serverId = serverId; 35 | updateConnector(connector); 36 | }} 37 | /> 38 | 39 | ); 40 | } 41 | 42 | export const GenericNoDatabaseDetails = (props: GenericDetailsProps) => ( 43 | 49 | ); 50 | -------------------------------------------------------------------------------- /ui/connectors/GoogleSheetsDetails.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { DatabaseConnectorInfo } from '../../shared/state'; 3 | import { ApiKey } from './ApiKey'; 4 | 5 | export function GoogleSheetsDetails({ 6 | connector, 7 | updateConnector, 8 | }: { 9 | connector: DatabaseConnectorInfo; 10 | updateConnector: (c: DatabaseConnectorInfo) => void; 11 | }) { 12 | return ( 13 | 14 | 19 | 20 | ); 21 | } 22 | -------------------------------------------------------------------------------- /ui/connectors/Host.test.jsx: -------------------------------------------------------------------------------- 1 | const React = require('react'); 2 | const enzyme = require('enzyme'); 3 | const { DatabaseConnectorInfo } = require('../../shared/state'); 4 | const { wait } = require('../../shared/promise'); 5 | const { Host } = require('./Host'); 6 | 7 | test('Host shows input and changes', async () => { 8 | const connector = new DatabaseConnectorInfo(); 9 | 10 | const changeTo = 'localhost:9090'; 11 | let changed = ''; 12 | const updateConnector = jest.fn((conn) => { 13 | changed = conn.database.address; 14 | }); 15 | const component = enzyme.mount( 16 | 17 | ); 18 | 19 | expect(changed).toBe(''); 20 | await component 21 | .find('input') 22 | .simulate('change', { target: { value: changeTo } }); 23 | await component.find('input').simulate('blur'); 24 | expect(changed).toBe(changeTo); 25 | }); 26 | -------------------------------------------------------------------------------- /ui/connectors/Host.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { DatabaseConnectorInfo } from '../../shared/state'; 3 | import { Input, InputProps } from '../components/Input'; 4 | 5 | export function Host({ 6 | connector, 7 | updateConnector, 8 | ...props 9 | }: Partial & { 10 | connector: DatabaseConnectorInfo; 11 | updateConnector: (c: DatabaseConnectorInfo) => void; 12 | }) { 13 | return ( 14 |
15 | { 19 | connector.database.address = value; 20 | updateConnector(connector); 21 | }} 22 | {...props} 23 | /> 24 |
25 | ); 26 | } 27 | -------------------------------------------------------------------------------- /ui/connectors/Neo4jDetails.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { DatabaseConnectorInfo, ServerInfo } from '../../shared/state'; 3 | import { FormGroup } from '../components/FormGroup'; 4 | import { ServerPicker } from '../components/ServerPicker'; 5 | import { Host } from './Host'; 6 | import { Password } from './Password'; 7 | import { Username } from './Username'; 8 | 9 | export function Neo4jDetails(props: { 10 | connector: DatabaseConnectorInfo; 11 | updateConnector: (c: DatabaseConnectorInfo) => void; 12 | servers: Array; 13 | }) { 14 | const { servers, connector, updateConnector } = props; 15 | 16 | return ( 17 | 18 | 19 | 20 | 21 | 22 | 23 | { 27 | connector.serverId = serverId; 28 | updateConnector(connector); 29 | }} 30 | /> 31 | 32 | ); 33 | } 34 | -------------------------------------------------------------------------------- /ui/connectors/Password.test.jsx: -------------------------------------------------------------------------------- 1 | const React = require('react'); 2 | const enzyme = require('enzyme'); 3 | const { DatabaseConnectorInfo } = require('../../shared/state'); 4 | const { Password } = require('./Password'); 5 | 6 | test('Password shows input and changes', async () => { 7 | const connector = new DatabaseConnectorInfo(); 8 | 9 | const changeTo = 'my-great-password'; 10 | let changed = ''; 11 | const updateConnector = jest.fn((conn) => { 12 | changed = conn.database.password_encrypt.value; 13 | }); 14 | const component = enzyme.mount( 15 | 16 | ); 17 | 18 | expect(changed).toBe(''); 19 | await component 20 | .find('input') 21 | .simulate('change', { target: { value: changeTo } }); 22 | await component.find('input').simulate('blur'); 23 | expect(changed).toBe(changeTo); 24 | }); 25 | -------------------------------------------------------------------------------- /ui/connectors/Password.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { DatabaseConnectorInfo } from '../../shared/state'; 3 | import { Input } from '../components/Input'; 4 | 5 | export function Password({ 6 | connector, 7 | updateConnector, 8 | label, 9 | }: { 10 | connector: DatabaseConnectorInfo; 11 | updateConnector: (c: DatabaseConnectorInfo) => void; 12 | label?: string; 13 | }) { 14 | // Don't try to show initial password 15 | const [password, setPassword] = React.useState(''); 16 | function syncPassword(p: string) { 17 | setPassword(p); 18 | // Sync typed password to state on change 19 | connector.database.password_encrypt.value = p; 20 | connector.database.password_encrypt.encrypted = false; 21 | updateConnector(connector); 22 | } 23 | 24 | return ( 25 |
26 | syncPassword(value)} 31 | /> 32 |
33 | ); 34 | } 35 | -------------------------------------------------------------------------------- /ui/connectors/SQLiteDetails.test.jsx: -------------------------------------------------------------------------------- 1 | require('./SQLiteDetails'); 2 | 3 | test('stub', () => {}); 4 | -------------------------------------------------------------------------------- /ui/connectors/SQLiteDetails.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { DatabaseConnectorInfo, ServerInfo } from '../../shared/state'; 3 | import { FileInput } from '../components/FileInput'; 4 | import { FormGroup } from '../components/FormGroup'; 5 | import { ServerPicker } from '../components/ServerPicker'; 6 | 7 | export function SQLiteDetails({ 8 | connector, 9 | updateConnector, 10 | servers, 11 | }: { 12 | connector: DatabaseConnectorInfo; 13 | updateConnector: (c: DatabaseConnectorInfo) => void; 14 | servers: Array; 15 | }) { 16 | return ( 17 | 18 | 19 | { 25 | connector.database.database = fileName; 26 | updateConnector(connector); 27 | }} 28 | /> 29 | 30 | { 34 | connector.serverId = serverId; 35 | updateConnector(connector); 36 | }} 37 | /> 38 | 39 | ); 40 | } 41 | -------------------------------------------------------------------------------- /ui/connectors/Username.test.jsx: -------------------------------------------------------------------------------- 1 | const React = require('react'); 2 | const enzyme = require('enzyme'); 3 | const { DatabaseConnectorInfo } = require('../../shared/state'); 4 | const { wait } = require('../../shared/promise'); 5 | const { Username } = require('./Username'); 6 | 7 | test('Username shows input and changes', async () => { 8 | const connector = new DatabaseConnectorInfo(); 9 | 10 | const changeTo = 'admin'; 11 | let changed = ''; 12 | const updateConnector = jest.fn((conn) => { 13 | changed = conn.database.username; 14 | }); 15 | const component = enzyme.mount( 16 | 17 | ); 18 | 19 | expect(changed).toBe(''); 20 | await component 21 | .find('input') 22 | .simulate('change', { target: { value: changeTo } }); 23 | await component.find('input').simulate('blur'); 24 | expect(changed).toBe(changeTo); 25 | }); 26 | -------------------------------------------------------------------------------- /ui/connectors/Username.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import { DatabaseConnectorInfo } from '../../shared/state'; 3 | import { Input } from '../components/Input'; 4 | 5 | export function Username({ 6 | connector, 7 | updateConnector, 8 | label, 9 | }: { 10 | connector: DatabaseConnectorInfo; 11 | updateConnector: (c: DatabaseConnectorInfo) => void; 12 | label?: string; 13 | }) { 14 | return ( 15 |
16 | { 20 | connector.database.username = value; 21 | updateConnector(connector); 22 | }} 23 | /> 24 |
25 | ); 26 | } 27 | -------------------------------------------------------------------------------- /ui/index.html: -------------------------------------------------------------------------------- 1 | {UI_TITLE} 2 | 3 | 4 | 5 | {UI_CSP} 6 | 7 | 8 | 9 |
10 |
Loading...
11 |
12 | 13 | 14 | -------------------------------------------------------------------------------- /ui/index.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | import * as ReactDOM from 'react-dom'; 3 | import { App, defaultRoutes } from './app'; 4 | 5 | // SOURCE: https://stackoverflow.com/a/7995898/1507139 6 | const isMobile = navigator.userAgent.match( 7 | /(iPad)|(iPhone)|(iPod)|(android)|(webOS)/i 8 | ); 9 | 10 | function index() { 11 | const root = document.getElementById('root'); 12 | if (document.location.pathname.startsWith('/dashboard')) { 13 | //ReactDOM.render(, root); 14 | return; 15 | } 16 | 17 | if (!isMobile) { 18 | ReactDOM.render(, root); 19 | return; 20 | } 21 | 22 | root.innerHTML = 'Please use a desktop web browser to view this app.'; 23 | } 24 | 25 | index(); 26 | -------------------------------------------------------------------------------- /ui/panels/FilePanel.test.jsx: -------------------------------------------------------------------------------- 1 | const React = require('react'); 2 | const { act } = require('react-dom/test-utils'); 3 | const enzyme = require('enzyme'); 4 | const { 5 | ProjectState, 6 | PanelResult, 7 | ProjectPage, 8 | FilePanelInfo, 9 | } = require('../../shared/state'); 10 | const { FilePanelDetails } = require('./FilePanel'); 11 | 12 | const project = new ProjectState(); 13 | project.pages = [new ProjectPage()]; 14 | project.pages[0].panels = [new FilePanelInfo()]; 15 | 16 | test('shows file panel details', async () => { 17 | const component = enzyme.mount( 18 | {}} 22 | /> 23 | ); 24 | await componentLoad(component); 25 | }); 26 | -------------------------------------------------------------------------------- /ui/panels/FilterAggregatePanel.test.jsx: -------------------------------------------------------------------------------- 1 | const React = require('react'); 2 | const { act } = require('react-dom/test-utils'); 3 | const { wait } = require('../../shared/promise'); 4 | const enzyme = require('enzyme'); 5 | const { 6 | ProjectState, 7 | PanelResult, 8 | ProjectPage, 9 | LiteralPanelInfo, 10 | FilterAggregatePanelInfo, 11 | } = require('../../shared/state'); 12 | const { FilterAggregatePanelDetails } = require('./FilterAggregatePanel'); 13 | 14 | const project = new ProjectState(); 15 | project.pages = [new ProjectPage()]; 16 | const lp = new LiteralPanelInfo(); 17 | const fp = new FilterAggregatePanelInfo(null, { 18 | panelSource: lp.info, 19 | aggregateType: 'sum', 20 | }); 21 | project.pages[0].panels = [lp, fp]; 22 | 23 | test('shows filagg panel details', async () => { 24 | const component = enzyme.mount( 25 | { 29 | Object.assign(project.pages[0].panels[1], p); 30 | }} 31 | /> 32 | ); 33 | 34 | await componentLoad(component); 35 | 36 | expect(fp.filagg.sortAsc).toBe(false); 37 | const sortDirection = () => 38 | component.find('[data-test-id="sort-direction"] select'); 39 | await sortDirection().simulate('change', { target: { value: 'asc' } }); 40 | expect(fp.filagg.sortAsc).toBe(true); 41 | component.setProps(); 42 | expect(sortDirection().props().value).toBe('asc'); 43 | 44 | const sortField = () => component.find('[data-test-id="sort-field"] input'); 45 | await sortField().simulate('change', { target: { value: 'flubberty' } }); 46 | await sortField().simulate('blur'); 47 | await wait(); 48 | expect(fp.filagg.sortOn).toBe('flubberty'); 49 | }); 50 | -------------------------------------------------------------------------------- /ui/panels/GraphPanel.test.jsx: -------------------------------------------------------------------------------- 1 | const React = require('react'); 2 | const { act } = require('react-dom/test-utils'); 3 | const enzyme = require('enzyme'); 4 | const { 5 | ProjectState, 6 | PanelResult, 7 | ProjectPage, 8 | LiteralPanelInfo, 9 | GraphPanelInfo, 10 | } = require('../../shared/state'); 11 | const { GraphPanel, GraphPanelDetails } = require('./GraphPanel'); 12 | 13 | const project = new ProjectState(); 14 | project.pages = [new ProjectPage()]; 15 | const lp = new LiteralPanelInfo(); 16 | const gp = new GraphPanelInfo(null, { 17 | panelSource: lp.info, 18 | x: 'name', 19 | ys: [{ field: 'age', label: 'Age' }], 20 | }); 21 | project.pages[0].panels = [lp, gp]; 22 | 23 | test('shows graph panel details', async () => { 24 | const component = enzyme.mount( 25 | {}} 29 | /> 30 | ); 31 | await componentLoad(component); 32 | }); 33 | 34 | test('shows filled graph panel', async () => { 35 | gp.resultMeta = new PanelResult({ 36 | value: [ 37 | { name: 'Nora', age: 33 }, 38 | { name: 'Kay', age: 20 }, 39 | ], 40 | }); 41 | 42 | const component = enzyme.mount( 43 | {}} 47 | /> 48 | ); 49 | await componentLoad(component); 50 | }); 51 | -------------------------------------------------------------------------------- /ui/panels/HTTPPanel.test.jsx: -------------------------------------------------------------------------------- 1 | const React = require('react'); 2 | const { act } = require('react-dom/test-utils'); 3 | const enzyme = require('enzyme'); 4 | const { 5 | ProjectState, 6 | PanelResult, 7 | ProjectPage, 8 | HTTPPanelInfo, 9 | } = require('../../shared/state'); 10 | const { HTTPPanelDetails, HTTPPanelBody } = require('./HTTPPanel'); 11 | 12 | const project = new ProjectState(); 13 | project.pages = [new ProjectPage()]; 14 | project.pages[0].panels = [new HTTPPanelInfo()]; 15 | 16 | test('shows file panel details', async () => { 17 | const component = enzyme.mount( 18 | {}} 22 | /> 23 | ); 24 | await componentLoad(component); 25 | }); 26 | 27 | test('shows http panel body', async () => { 28 | const panel = new HTTPPanelInfo(); 29 | 30 | const component = enzyme.mount( 31 | 37 | ); 38 | await componentLoad(component); 39 | }); 40 | -------------------------------------------------------------------------------- /ui/panels/LiteralPanel.test.jsx: -------------------------------------------------------------------------------- 1 | const React = require('react'); 2 | const { act } = require('react-dom/test-utils'); 3 | const enzyme = require('enzyme'); 4 | const { LiteralPanelInfo } = require('../../shared/state'); 5 | const { 6 | LiteralPanel, 7 | LiteralPanelDetails, 8 | LiteralPanelBody, 9 | } = require('./LiteralPanel'); 10 | 11 | test('shows literal panel details', async () => { 12 | const panel = new LiteralPanelInfo(); 13 | const component = enzyme.mount( 14 | {}} 18 | /> 19 | ); 20 | await componentLoad(component); 21 | }); 22 | 23 | test('shows literal panel body', async () => { 24 | const panel = new LiteralPanelInfo(); 25 | 26 | const component = enzyme.mount( 27 | 33 | ); 34 | await componentLoad(component); 35 | }); 36 | -------------------------------------------------------------------------------- /ui/panels/ProgramPanel.test.jsx: -------------------------------------------------------------------------------- 1 | const React = require('react'); 2 | const { act } = require('react-dom/test-utils'); 3 | const enzyme = require('enzyme'); 4 | const { 5 | ProjectState, 6 | PanelResult, 7 | ProjectPage, 8 | ProgramPanelInfo, 9 | } = require('../../shared/state'); 10 | const { 11 | ProgramPanelDetails, 12 | ProgramInfo, 13 | ProgramPanelBody, 14 | } = require('./ProgramPanel'); 15 | 16 | const project = new ProjectState(); 17 | project.pages = [new ProjectPage()]; 18 | project.pages[0].panels = [new ProgramPanelInfo()]; 19 | 20 | test('shows program panel details', async () => { 21 | const component = enzyme.mount( 22 | {}} 26 | /> 27 | ); 28 | await componentLoad(component); 29 | }); 30 | 31 | test('shows generic program panel info', async () => { 32 | const panel = new ProgramPanelInfo(); 33 | const component = enzyme.mount(); 34 | await componentLoad(component); 35 | expect(component.text().includes('DM_setPanel(')).toBe(true); 36 | }); 37 | 38 | test('shows sql-specific program panel info', async () => { 39 | const panel = new ProgramPanelInfo(null, { 40 | type: 'sql', 41 | }); 42 | const component = enzyme.mount(); 43 | await componentLoad(component); 44 | expect(component.text().includes('SQL')).toBe(true); 45 | }); 46 | 47 | test('shows program panel body', async () => { 48 | const panel = new ProgramPanelInfo(); 49 | 50 | const component = enzyme.mount( 51 | 57 | ); 58 | await componentLoad(component); 59 | }); 60 | -------------------------------------------------------------------------------- /ui/panels/TablePanel.test.jsx: -------------------------------------------------------------------------------- 1 | const React = require('react'); 2 | const { act } = require('react-dom/test-utils'); 3 | const enzyme = require('enzyme'); 4 | const { 5 | ProjectState, 6 | PanelResult, 7 | ProjectPage, 8 | LiteralPanelInfo, 9 | TablePanelInfo, 10 | } = require('../../shared/state'); 11 | const { TablePanel, TablePanelDetails } = require('./TablePanel'); 12 | 13 | const project = new ProjectState(); 14 | project.pages = [new ProjectPage()]; 15 | const lp = new LiteralPanelInfo(); 16 | const tp = new TablePanelInfo(null, { 17 | panelSource: lp.info, 18 | columns: [ 19 | { field: 'name', label: 'Name' }, 20 | { field: 'age', label: 'Age' }, 21 | ], 22 | }); 23 | project.pages[0].panels = [lp, tp]; 24 | 25 | test('shows table panel details', async () => { 26 | const component = enzyme.mount( 27 | {}} 31 | /> 32 | ); 33 | await componentLoad(component); 34 | }); 35 | 36 | test('shows filled table panel', async () => { 37 | tp.resultMeta = new PanelResult({ 38 | value: [ 39 | { name: 'Nora', age: 33 }, 40 | { name: 'Kay', age: 20 }, 41 | ], 42 | }); 43 | 44 | const component = enzyme.mount( 45 | {}} 49 | /> 50 | ); 51 | await componentLoad(component); 52 | }); 53 | -------------------------------------------------------------------------------- /ui/panels/index.test.jsx: -------------------------------------------------------------------------------- 1 | const { PANEL_UI_DETAILS } = require('./index'); 2 | 3 | for (const [panelType, info] of Object.entries(PANEL_UI_DETAILS)) { 4 | test(panelType + ' factory', () => { 5 | const p = info.factory(12, 'my great panel'); 6 | expect(p.pageId).toBe(12); 7 | expect(p.name).toBe('my great panel'); 8 | }); 9 | } 10 | -------------------------------------------------------------------------------- /ui/panels/index.ts: -------------------------------------------------------------------------------- 1 | import { MODE } from '../../shared/constants'; 2 | import { PanelInfoType } from '../../shared/state'; 3 | import { databasePanel } from './DatabasePanel'; 4 | import { filePanel } from './FilePanel'; 5 | import { filaggPanel } from './FilterAggregatePanel'; 6 | import { graphPanel } from './GraphPanel'; 7 | import { httpPanel } from './HTTPPanel'; 8 | import { literalPanel } from './LiteralPanel'; 9 | import { programPanel } from './ProgramPanel'; 10 | import { tablePanel } from './TablePanel'; 11 | import { PanelUIDetails } from './types'; 12 | 13 | export const PANEL_UI_DETAILS: { 14 | [Property in PanelInfoType]: PanelUIDetails; 15 | } = { 16 | table: tablePanel, 17 | http: httpPanel, 18 | graph: graphPanel, 19 | program: programPanel, 20 | literal: literalPanel, 21 | database: databasePanel, 22 | file: filePanel, 23 | filagg: filaggPanel, 24 | }; 25 | 26 | export const PANEL_GROUPS: Array<{ 27 | label: string; 28 | panels: Array; 29 | }> = [ 30 | { 31 | label: 'Import from', 32 | panels: (() => { 33 | const panels: Array = ['http', 'file', 'literal']; 34 | 35 | // Weird way to make sure TypeScript type checks these strings. 36 | if (MODE !== 'browser') { 37 | panels.unshift('database'); 38 | } 39 | 40 | return panels; 41 | })(), 42 | }, 43 | { 44 | label: 'Operate with', 45 | panels: ['program', 'filagg'], 46 | }, 47 | { 48 | label: 'Visualize with', 49 | panels: ['graph', 'table'], 50 | }, 51 | ]; 52 | -------------------------------------------------------------------------------- /ui/panels/types.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ConnectorInfo, 3 | PanelInfo, 4 | PanelInfoType, 5 | PanelResult, 6 | ServerInfo, 7 | } from '../../shared/state'; 8 | 9 | export interface PanelDetailsProps { 10 | panel: T; 11 | panels: Array; 12 | updatePanel: (d: T) => void; 13 | panelIndex: number; 14 | } 15 | 16 | export interface PanelBodyProps { 17 | panel: T; 18 | panels: Array; 19 | updatePanel: (d: T) => void; 20 | keyboardShortcuts: (e: React.KeyboardEvent) => void; 21 | } 22 | 23 | export interface PanelUIDetails { 24 | icon: string; 25 | eval( 26 | panel: T, 27 | panels: Array, 28 | idMap: Record, 29 | connectors: Array, 30 | servers: Array 31 | ): Promise; 32 | id: PanelInfoType; 33 | label: string; 34 | details: React.FC>; 35 | body: React.FC> | null; 36 | hideBody?: (p: T, cs: Array) => boolean; 37 | previewable: boolean; 38 | hasStdout: boolean; 39 | info: ({ panel }: { panel: T }) => React.ReactElement | null; 40 | factory: (pageId: string, name: string) => T; 41 | dashboard?: boolean; 42 | } 43 | -------------------------------------------------------------------------------- /ui/scripts/languages.build: -------------------------------------------------------------------------------- 1 | jsonnet ./shared/languages/ruby.jsonnet -o ./shared/languages/ruby.json 2 | jsonnet ./shared/languages/r.jsonnet -o ./shared/languages/r.json 3 | jsonnet ./shared/languages/javascript.jsonnet -o ./shared/languages/javascript.json 4 | jsonnet ./shared/languages/python.jsonnet -o ./shared/languages/python.json 5 | jsonnet ./shared/languages/julia.jsonnet -o ./shared/languages/julia.json 6 | jsonnet ./shared/languages/php.jsonnet -o ./shared/languages/php.json 7 | jsonnet ./shared/languages/deno.jsonnet -o ./shared/languages/deno.json 8 | -------------------------------------------------------------------------------- /ui/scripts/ui.build: -------------------------------------------------------------------------------- 1 | mkdir build 2 | # This is overwritten in production builds 3 | setenv_default UI_ESBUILD_ARGS "" 4 | yarn esbuild --target=es2020,chrome90,firefox90,safari13,edge90 ui/index.tsx --loader:.ts=tsx --loader:.js=jsx "--external:fs" --bundle --sourcemap {UI_ESBUILD_ARGS} --outfile=build/ui.js 5 | 6 | setenv_default UI_CONFIG_OVERRIDES "" 7 | prepend {UI_CONFIG_OVERRIDES} build/ui.js 8 | 9 | setenv_default UI_TITLE "DataStation Desktop CE" 10 | setenv_default UI_CSP "" 11 | setenv_default UI_ROOT "" 12 | render ui/index.html build/index.html 13 | node-sass ./ui/style.css ./build/style.css -------------------------------------------------------------------------------- /ui/scripts/watch_and_serve.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eux 4 | 5 | # Kill all xargs/fswatch children on exit 6 | trap 'killall xargs' SIGINT SIGTERM EXIT 7 | 8 | # Build once up front 9 | yarn build-ui 10 | 11 | # Now watch for changes in the background and rebuild 12 | function watch() { 13 | for dir in $(find "$1" -type d); do 14 | fswatch -x --event Created --event Removed --event Renamed --event Updated "$dir" | grep --line-buffered -E "\.(tsx|css|ts|js|jsx)" | xargs -n1 bash -c 'yarn esbuild ui/index.tsx --loader:.ts=tsx --loader:.js=jsx "--external:fs" --bundle --sourcemap --outfile=build/ui.js && node-sass ./ui/style.css ./build/style.css' & 15 | done 16 | } 17 | 18 | watch ./ui 19 | watch ./shared 20 | 21 | # Serve the pages 22 | python3 -m http.server --directory build 8080 23 | -------------------------------------------------------------------------------- /ui/shortcuts.ts: -------------------------------------------------------------------------------- 1 | import { useHotkeys } from 'react-hotkeys-hook'; 2 | import { UrlState } from './urlState'; 3 | 4 | export function useShortcuts( 5 | urlState: UrlState, 6 | setUrlState: (a0: Partial) => void 7 | ) { 8 | useHotkeys( 9 | 'ctrl+tab', 10 | () => { 11 | if (urlState.view === 'editor') { 12 | setUrlState({ page: urlState.page + 1 }); 13 | } 14 | }, 15 | null, 16 | [urlState.page, setUrlState] 17 | ); 18 | 19 | useHotkeys( 20 | 'ctrl+shift+tab', 21 | () => { 22 | if (urlState.view === 'editor') { 23 | setUrlState({ page: urlState.page - 1 }); 24 | } 25 | }, 26 | null, 27 | [urlState.page, setUrlState] 28 | ); 29 | } 30 | -------------------------------------------------------------------------------- /ui/state.test.jsx: -------------------------------------------------------------------------------- 1 | const { makeUpdater } = require('./state'); 2 | 3 | test('test makeUpdater', async () => { 4 | const panels = [{ id: 9 }, { id: 2 }]; 5 | const updateStore = jest.fn(); 6 | const rereadStore = jest.fn(); 7 | const projectId = '12'; 8 | const update = makeUpdater(projectId, panels, updateStore, rereadStore); 9 | 10 | // Insert new at end 11 | await update({ id: 3 }, -1, true); 12 | expect(panels.map((p) => p.id)).toStrictEqual([9, 2, 3]); 13 | expect([...updateStore.mock.calls[0]]).toStrictEqual([ 14 | projectId, 15 | { id: 3 }, 16 | -1, 17 | true, 18 | [9, 2, 3], 19 | ]); 20 | 21 | // Modify existing 22 | await update({ id: 3 }, 1, false); 23 | expect(panels.map((p) => p.id)).toStrictEqual([9, 3, 2]); 24 | expect([...updateStore.mock.calls[1]]).toStrictEqual([ 25 | projectId, 26 | { id: 3 }, 27 | 1, 28 | false, 29 | [9, 3, 2], 30 | ]); 31 | 32 | // Insert new not at end 33 | await update({ id: 4 }, 2, true); 34 | expect(panels.map((p) => p.id)).toStrictEqual([9, 3, 2, 4]); 35 | expect([...updateStore.mock.calls[2]]).toStrictEqual([ 36 | projectId, 37 | { id: 4 }, 38 | 2, 39 | true, 40 | [9, 3, 4, 2], 41 | ]); 42 | }); 43 | --------------------------------------------------------------------------------