├── .github └── workflows │ ├── all-tests.yml │ ├── publish.yml │ └── release.yml ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── assets ├── beeswarm.png ├── big-file.png ├── browser-console.png ├── chart.png ├── map.png ├── nodejs-console-with-chart.png └── spatial.png ├── deno.json ├── deno.lock ├── examples ├── index.html ├── main.ts └── mainCache.ts ├── src ├── class │ ├── Simple.ts │ ├── SimpleDB.ts │ └── SimpleTable.ts ├── helpers │ ├── accumulateQuery.ts │ ├── arraysToData.ts │ ├── checkVssIndexes.ts │ ├── cleanCache.ts │ ├── cleanPath.ts │ ├── cleanSQL.ts │ ├── convertForJS.ts │ ├── extractTypes.ts │ ├── findGeoColumn.ts │ ├── getCombinations.ts │ ├── getExtension.ts │ ├── getFirstNonNullOrUndefinedValues.ts │ ├── getIdenticalColumns.ts │ ├── getName.ts │ ├── getProjection.ts │ ├── getProjectionParquet.ts │ ├── keepNumericalColumns.ts │ ├── logData.ts │ ├── mergeOptions.ts │ ├── parseDuckDBType.ts │ ├── parseTypes.ts │ ├── parseValue.ts │ ├── queryDB.ts │ ├── runQuery.ts │ ├── setDbProps.ts │ ├── shouldFlipBeforeExport.ts │ ├── stringToArray.ts │ ├── stringifyDates.ts │ ├── stringifyDatesInvert.ts │ ├── tryAI.ts │ ├── tryEmbedding.ts │ ├── unifyColumns.ts │ ├── writeDataAsArrays.ts │ └── writeProjectionsAndIndexes.ts ├── incrementVersion.ts ├── index.ts └── methods │ ├── aggregateGeoQuery.ts │ ├── aiEmbeddings.ts │ ├── aiQuery.ts │ ├── aiRowByRow.ts │ ├── aiVectorSimilarity.ts │ ├── binsQuery.ts │ ├── cache.ts │ ├── capitalizeQuery.ts │ ├── cloneColumn.ts │ ├── cloneQuery.ts │ ├── concatenateQuery.ts │ ├── convertQuery.ts │ ├── correlations.ts │ ├── correlationsQuery.ts │ ├── crossJoinQuery.ts │ ├── distanceQuery.ts │ ├── getBottom.ts │ ├── getColumns.ts │ ├── getDescription.ts │ ├── getFirstRow.ts │ ├── getGeoData.ts │ ├── getLastRow.ts │ ├── getMax.ts │ ├── getMean.ts │ ├── getMedian.ts │ ├── getMin.ts │ ├── getNbRows.ts │ ├── getQuantile.ts │ ├── getSkew.ts │ ├── getStdDev.ts │ ├── getSum.ts │ ├── getTableNames.ts │ ├── getTop.ts │ ├── getType.ts │ ├── getTypes.ts │ ├── getUniques.ts │ ├── getValues.ts │ ├── getVar.ts │ ├── insertRowsQuery.ts │ ├── join.ts │ ├── joinGeo.ts │ ├── joinGeoQuery.ts │ ├── joinQuery.ts │ ├── keepQuery.ts │ ├── linearRegressionQuery.ts │ ├── linearRegressions.ts │ ├── loadArray.ts │ ├── loadDataQuery.ts │ ├── logHistogram.ts │ ├── lowerQuery.ts │ ├── normalizeQuery.ts │ ├── outliersIQRQuery.ts │ ├── proportionsHorizontalQuery.ts │ ├── proportionsVerticalQuery.ts │ ├── quantilesQuery.ts │ ├── ranksQuery.ts │ ├── removeColumnsQuery.ts │ ├── removeDuplicatesQuery.ts │ ├── removeMissing.ts │ ├── removeMissingQuery.ts │ ├── removeQuery.ts │ ├── renameColumnQuery.ts │ ├── replaceNullsQuery.ts │ ├── replaceQuery.ts │ ├── rollingQuery.ts │ ├── roundQuery.ts │ ├── selectRowsQuery.ts │ ├── sortQuery.ts │ ├── summarize.ts │ ├── summarizeQuery.ts │ ├── trimQuery.ts │ ├── upperQuery.ts │ ├── writeDataQuery.ts │ ├── writeGeoDataQuery.ts │ └── zScoreQuery.ts └── test ├── data ├── directory │ ├── data1.csv │ ├── data2.csv │ ├── data3.csv │ └── data4ExtraColumn.csv ├── files │ ├── activefires.csv │ ├── alberta-expenses.csv │ ├── cities.csv │ ├── dailyTemperatures.csv │ ├── data.csv │ ├── data.csv.gz │ ├── data.json │ ├── data.parquet │ ├── data.tsv │ ├── data.txt │ ├── dataArrays.json │ ├── dataCompressed.parquet │ ├── dataCorrelations.json │ ├── dataCsvCompressed.txt │ ├── dataDates.csv │ ├── dataDuplicates.csv │ ├── dataExtraLines.csv │ ├── dataJustNumbers.csv │ ├── dataJustNumbers.json │ ├── dataManyDecimals.csv │ ├── dataNoHeaders.csv │ ├── dataProportions.json │ ├── dataRank.csv │ ├── dataSort.csv │ ├── dataSummarize.json │ ├── dataTidy.json │ ├── dataTrim.json │ ├── dataUntidy.json │ ├── dataUntidyWithNulls.json │ ├── dataWithMissingValues.json │ ├── employees.csv │ ├── employees.json │ ├── populations-one-sheet.xlsx │ └── populations-two-sheets.xlsx └── joins │ ├── categories.csv │ ├── dishes.csv │ ├── normals.csv │ └── projections.csv ├── geodata ├── files │ ├── CanadianProvincesAndTerritories.json │ ├── CanadianProvincesAndTerritories.shp.zip │ ├── bigCircle.json │ ├── bigCircleWithHole.json │ ├── canada-not-4326.shp.zip │ ├── canada.json │ ├── circleOverlapPolygonsGroups.json │ ├── closedLines.geojson │ ├── coordinates.csv │ ├── coordinates.geojson │ ├── data-compressed.geoparquet │ ├── data-multiple-columns.geoparquet │ ├── data.geoparquet │ ├── earthquake.geojson │ ├── economicRegions-simplified.json │ ├── firesCanada2023.csv │ ├── invalid.geojson │ ├── line.json │ ├── point.json │ ├── pointsInside.json │ ├── polygon.json │ ├── polygonInside.json │ ├── polygons.geojson │ ├── polygonsGroups.json │ ├── polygonsWithinPolygons.json │ ├── smallCircle.json │ └── triangle.json └── tests-results │ ├── CanadianProvincesAndTerritories-simplified-interior.json │ ├── CanadianProvincesAndTerritories-simplified.json │ └── bigCircleWithHoleFilled.json └── unit ├── class ├── SimpleDB.test.ts └── SimpleTable.test.ts ├── helpers ├── getExtension.test.ts ├── getName.test.ts └── getProjection.test.ts └── methods ├── accumulate.test.ts ├── addColumn.test.ts ├── addRowNumber.test.ts ├── aggregateGeo.test.ts ├── aiEmbeddings.test.ts ├── aiQuery.test.ts ├── aiRowByRow.test.ts ├── aiVectorSimilarity.test.ts ├── area.test.ts ├── bins.test.ts ├── buffer.test.ts ├── cache.test.ts ├── capitalize.test.ts ├── centroid.test.ts ├── cleanColumnNames.test.ts ├── cloneColumn.test.ts ├── cloneColumnWithOffset.test.ts ├── cloneTable.test.ts ├── concatenate.test.ts ├── convert.test.ts ├── correlations.test.ts ├── crossJoin.test.ts ├── customQuery.test.ts ├── distance.test.ts ├── fill.test.ts ├── fillHoles.test.ts ├── filter.test.ts ├── fixGeo.test.ts ├── flipCoordinates.test.ts ├── getBottom.test.ts ├── getBoundingBox.test.ts ├── getColumns.test.ts ├── getData.test.ts ├── getDescription.test.ts ├── getExtent.test.ts ├── getFirstRow.test.ts ├── getGeoData.test.ts ├── getLastRow.test.ts ├── getMax.test.ts ├── getMean.test.ts ├── getMedian.test.ts ├── getMin.test.ts ├── getNbColumns.test.ts ├── getNbRows.test.ts ├── getNbValues.test.ts ├── getQuantile.test.ts ├── getRow.test.ts ├── getSchema.test.ts ├── getSkew.test.ts ├── getStdDev.test.ts ├── getSum.test.ts ├── getTop.test.ts ├── getTypes.test.ts ├── getUniques.test.ts ├── getValues.test.ts ├── getVar.test.ts ├── hasColumn.test.ts ├── insertRows.test.ts ├── insertTables.test.ts ├── inside.test.ts ├── intersect.test.ts ├── intersection.test.ts ├── isClosedGeo.test.ts ├── isValidGeo.test.ts ├── join.test.ts ├── joinGeo.test.ts ├── keep.test.ts ├── latLon.test.ts ├── left.test.ts ├── length.test.ts ├── linearRegressions.test.ts ├── linesToPolygons.test.ts ├── loadArray.test.ts ├── loadData.test.ts ├── loadDataFromDirectory.test.ts ├── loadGeoData.test.ts ├── logBarChart.test.ts ├── logBottom.test.ts ├── logColumns.test.ts ├── logDescription.test.ts ├── logDotChart.test.ts ├── logExtent.test.ts ├── logHistogram.test.ts ├── logLineChart.test.ts ├── logNbRows.test.ts ├── logProjections.test.ts ├── logTable.test.ts ├── logUniques.test.ts ├── longer.test.ts ├── lower.test.ts ├── nbVertices.test.ts ├── normalize.test.ts ├── outliersIQR.test.ts ├── perimeter.test.ts ├── points.test.ts ├── proportionsHorizontal.test.ts ├── proportionsVertical.test.ts ├── quantiles.test.ts ├── ranks.test.ts ├── reducePrecision.test.ts ├── remove.test.ts ├── removeColumns.test.ts ├── removeDuplicates.test.ts ├── removeIntersection.test.ts ├── removeMissing.test.ts ├── removeRows.test.ts ├── renameColumns.test.ts ├── renameTable.test.ts ├── replace.test.ts ├── replaceNulls.test.ts ├── reproject.test.ts ├── right.test.ts ├── rolling.test.ts ├── round.test.ts ├── sample.test.ts ├── selectColumns.test.ts ├── selectRows.test.ts ├── setTypes.test.ts ├── simplify.test.ts ├── skip.test.ts ├── sort.test.ts ├── splitExtract.test.ts ├── summarize.test.ts ├── toSheet.test.ts ├── trim.test.ts ├── typeGeo.test.ts ├── union.test.ts ├── unnestGeo.test.ts ├── updateColumn.test.ts ├── updateWithJS.test.ts ├── upper.test.ts ├── wider.test.ts ├── writeChart.test.ts ├── writeData.test.ts ├── writeGeoData.test.ts ├── writeMap.test.ts └── zScore.test.ts /.github/workflows/all-tests.yml: -------------------------------------------------------------------------------- 1 | name: SDA TESTS 2 | 3 | on: 4 | pull_request: 5 | branches: [main] 6 | 7 | permissions: 8 | contents: read 9 | 10 | jobs: 11 | all-tests: 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - uses: actions/checkout@v4 16 | - uses: denoland/setup-deno@v2 17 | with: 18 | deno-version: v2.x 19 | - run: deno task all-tests 20 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: SDA PUBLISH 2 | on: 3 | push: 4 | tags: 5 | - "v*" 6 | 7 | jobs: 8 | publish: 9 | runs-on: ubuntu-latest 10 | permissions: 11 | contents: read 12 | id-token: write # The OIDC ID token is used for authentication with JSR. 13 | steps: 14 | - uses: actions/checkout@v3 15 | - uses: denoland/setup-deno@v2 16 | with: 17 | deno-version: v2.x 18 | - run: deno install --allow-scripts=npm:playwright-chromium && deno publish 19 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: SDA RELEASE 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*" 7 | 8 | permissions: 9 | contents: write 10 | 11 | jobs: 12 | release: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Create GitHub release 16 | env: 17 | GH_TOKEN: ${{ github.token }} 18 | tag: ${{ github.ref_name }} 19 | run: | 20 | gh release create "$tag" \ 21 | --repo="$GITHUB_REPOSITORY" \ 22 | --title="v${tag#v}" \ 23 | --generate-notes 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist/ 3 | test/output/ 4 | .DS_Store 5 | .tmp 6 | .sda-cache 7 | testRoot.csv 8 | cov_profile 9 | .env 10 | .journalism-cache -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Nael Shiab 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /assets/beeswarm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nshiab/simple-data-analysis/bdb9dff12e724c817d2bcbc5f992c60c59379860/assets/beeswarm.png -------------------------------------------------------------------------------- /assets/big-file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nshiab/simple-data-analysis/bdb9dff12e724c817d2bcbc5f992c60c59379860/assets/big-file.png -------------------------------------------------------------------------------- /assets/browser-console.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nshiab/simple-data-analysis/bdb9dff12e724c817d2bcbc5f992c60c59379860/assets/browser-console.png -------------------------------------------------------------------------------- /assets/chart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nshiab/simple-data-analysis/bdb9dff12e724c817d2bcbc5f992c60c59379860/assets/chart.png -------------------------------------------------------------------------------- /assets/map.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nshiab/simple-data-analysis/bdb9dff12e724c817d2bcbc5f992c60c59379860/assets/map.png -------------------------------------------------------------------------------- /assets/nodejs-console-with-chart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nshiab/simple-data-analysis/bdb9dff12e724c817d2bcbc5f992c60c59379860/assets/nodejs-console-with-chart.png -------------------------------------------------------------------------------- /assets/spatial.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nshiab/simple-data-analysis/bdb9dff12e724c817d2bcbc5f992c60c59379860/assets/spatial.png -------------------------------------------------------------------------------- /deno.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@nshiab/simple-data-analysis", 3 | "version": "5.6.12", 4 | "exports": { 5 | ".": "./src/index.ts" 6 | }, 7 | "tasks": { 8 | "all-tests": "deno install --allow-scripts=npm:playwright-chromium && deno fmt --check && deno lint && deno check src/index.ts && deno publish --allow-dirty --dry-run && deno test -A --fail-fast", 9 | "test-coverage": "deno test -A --fail-fast --coverage=cov_profile && deno coverage cov_profile", 10 | "patch-no-tests": "deno run -A src/incrementVersion.ts patch", 11 | "patch": "deno task all-tests && deno run -A src/incrementVersion.ts patch", 12 | "minor": "deno task all-tests && deno run -A src/incrementVersion.ts minor", 13 | "major": "deno task all-tests && deno run -A src/incrementVersion.ts major" 14 | }, 15 | "publish": { 16 | "exclude": [ 17 | "test", 18 | ".github" 19 | ] 20 | }, 21 | "nodeModulesDir": "auto", 22 | "imports": { 23 | "@duckdb/node-api": "npm:@duckdb/node-api@1.2.2-alpha.19", 24 | "@nshiab/journalism": "jsr:@nshiab/journalism@1.28.6", 25 | "@observablehq/plot": "npm:@observablehq/plot@0.6.17", 26 | "@std/assert": "jsr:@std/assert@1.0.13" 27 | }, 28 | "fmt": { 29 | "exclude": [ 30 | ".sda-cache", 31 | "test/output" 32 | ] 33 | }, 34 | "compilerOptions": { 35 | "lib": [ 36 | "dom", 37 | "deno.ns" 38 | ] 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /examples/main.ts: -------------------------------------------------------------------------------- 1 | import { SimpleDB } from "@nshiab/simple-data-analysis"; 2 | 3 | // We start a SimpleDB instance. 4 | const sdb = new SimpleDB(); 5 | 6 | // We create a new table. 7 | const provinces = sdb.newTable("provinces"); 8 | // We fetch the provinces' boundaries. It's a geoJSON. 9 | await provinces.loadGeoData( 10 | "https://raw.githubusercontent.com/nshiab/simple-data-analysis/main/test/geodata/files/CanadianProvincesAndTerritories.json", 11 | ); 12 | 13 | // Uncomment this line if you want to see the table. 14 | // await provinces.logTable() 15 | 16 | // We create a new table. 17 | const fires = sdb.newTable("fires"); 18 | // We fetch the wildfires data. It's a CSV. 19 | await fires.loadData( 20 | "https://raw.githubusercontent.com/nshiab/simple-data-analysis/main/test/geodata/files/firesCanada2023.csv", 21 | ); 22 | // We create point geometries from the lat and lon columns 23 | // and we store the points in the new column geom. 24 | await fires.points("lat", "lon", "geom"); 25 | 26 | // We match fires with provinces 27 | // and we output the results into a new table. 28 | // By default, joinGeo will automatically look 29 | // for columns storing geometries in the tables, 30 | // do a left join, and put the results 31 | // in the left table. 32 | const firesInsideProvinces = await fires.joinGeo(provinces, "inside", { 33 | outputTable: "firesInsideProvinces", 34 | }); 35 | 36 | // We remove fires that could not be matched 37 | await firesInsideProvinces.removeMissing(); 38 | 39 | // We summarize to count the number of fires 40 | // and sum up the area burnt in each province. 41 | await firesInsideProvinces.summarize({ 42 | values: "hectares", 43 | categories: "nameEnglish", 44 | summaries: ["count", "sum"], 45 | decimals: 0, 46 | }); 47 | 48 | // We rename columns. 49 | await firesInsideProvinces.renameColumns({ 50 | count: "nbFires", 51 | sum: "burntArea", 52 | }); 53 | // We want the province with 54 | // the greatest burnt area first. 55 | await firesInsideProvinces.sort({ burntArea: "desc" }); 56 | 57 | // We log the results. By default, the method 58 | // logs the first 10 rows, but we can specify 59 | // the number of rows to log. 60 | await firesInsideProvinces.logTable(12); 61 | 62 | // We can also log a bar chart of the burnt area. 63 | await firesInsideProvinces.logBarChart("nameEnglish", "burntArea"); 64 | 65 | // We close everything. 66 | await sdb.done(); 67 | -------------------------------------------------------------------------------- /src/class/Simple.ts: -------------------------------------------------------------------------------- 1 | import type { DuckDBConnection, DuckDBInstance } from "@duckdb/node-api"; 2 | 3 | export default class Simple { 4 | /** A flag indicating whether debugging information should be logged. Defaults to false. @category Properties */ 5 | debug: boolean; 6 | /** The number of rows to log. Defaults to 10. @category Properties */ 7 | nbRowsToLog: number; 8 | /** A flag indicating whether types should be logged along tables. Defaults to false. @category Properties */ 9 | types: boolean; 10 | /** The number of characters to log for text cells. By default, the whole text is logged. @category Properties */ 11 | nbCharactersToLog: number | undefined; 12 | /** A DuckDB database. @category Properties */ 13 | db!: DuckDBInstance; 14 | /** A connection to a DuckDB database. @category Properties */ 15 | connection!: DuckDBConnection; 16 | /** A flag to know if the name of the table has been attributed by default. @category Properties */ 17 | defaultTableName: boolean; 18 | /** 19 | * For internal use only. If you want to run a SQL query, use the customQuery method. @category Properties 20 | */ 21 | runQuery!: ( 22 | query: string, 23 | connection: DuckDBConnection, 24 | returnDataFromQuery: boolean, 25 | options: { 26 | debug: boolean; 27 | method: string | null; 28 | parameters: { [key: string]: unknown } | null; 29 | types?: { [key: string]: string }; 30 | }, 31 | ) => Promise< 32 | | { 33 | [key: string]: number | string | Date | boolean | null; 34 | }[] 35 | | null 36 | >; 37 | 38 | constructor( 39 | options: { 40 | debug?: boolean; 41 | nbRowsToLog?: number; 42 | nbCharactersToLog?: number; 43 | types?: boolean; 44 | } = {}, 45 | ) { 46 | this.nbRowsToLog = options.nbRowsToLog ?? 10; 47 | this.nbCharactersToLog = options.nbCharactersToLog; 48 | this.types = options.types ?? false; 49 | this.debug = options.debug ?? false; 50 | this.defaultTableName = false; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/helpers/accumulateQuery.ts: -------------------------------------------------------------------------------- 1 | import stringToArray from "./stringToArray.ts"; 2 | 3 | export default function accumulateQuery( 4 | table: string, 5 | column: string, 6 | newColumn: string, 7 | options: { 8 | categories?: string | string[]; 9 | } = {}, 10 | ) { 11 | const categories = options.categories 12 | ? stringToArray(options.categories) 13 | : []; 14 | const partition = categories.length > 0 15 | ? `PARTITION BY ${categories.map((d) => `"${d}"`).join(", ")} ` 16 | : ""; 17 | 18 | const query = 19 | `CREATE OR REPLACE TABLE "${table}" AS SELECT *, ROW_NUMBER() OVER() AS idForAccumulate FROM "${table}"; 20 | CREATE OR REPLACE TABLE "${table}" AS SELECT *, SUM("${column}") OVER (${partition}ORDER BY idForAccumulate) AS "${newColumn}" 21 | FROM "${table}" 22 | ORDER BY idForAccumulate; 23 | ALTER TABLE "${table}" DROP "idForAccumulate";`; 24 | 25 | return query; 26 | } 27 | -------------------------------------------------------------------------------- /src/helpers/arraysToData.ts: -------------------------------------------------------------------------------- 1 | import type { Table } from "npm:apache-arrow@17"; 2 | 3 | export default function tableToArrayOfObjects(table: Table) { 4 | return table.toArray().map((d) => Object.fromEntries(d)); 5 | } 6 | -------------------------------------------------------------------------------- /src/helpers/checkVssIndexes.ts: -------------------------------------------------------------------------------- 1 | import { readFileSync } from "node:fs"; 2 | import { existsSync } from "node:fs"; 3 | 4 | export default function checkVssIndexes(allIndexesFile: string): boolean { 5 | let vssIndex = false; 6 | 7 | if (existsSync(allIndexesFile)) { 8 | const indexes = JSON.parse(readFileSync(allIndexesFile, "utf-8")); 9 | for (const table of Object.keys(indexes)) { 10 | for (const index of indexes[table]) { 11 | if (index.startsWith("vss_")) { 12 | vssIndex = true; 13 | } 14 | } 15 | } 16 | } 17 | 18 | return vssIndex; 19 | } 20 | -------------------------------------------------------------------------------- /src/helpers/cleanCache.ts: -------------------------------------------------------------------------------- 1 | import { readFileSync, unlinkSync, writeFileSync } from "node:fs"; 2 | import type SimpleDB from "../class/SimpleDB.ts"; 3 | 4 | export default function cleanCache(sdb: SimpleDB) { 5 | if (sdb.cacheSourcesUsed.length > 0) { 6 | const cacheSources = JSON.parse( 7 | readFileSync(".sda-cache/sources.json", "utf-8"), 8 | ); 9 | for (const cacheId of Object.keys(cacheSources)) { 10 | if (!sdb.cacheSourcesUsed.includes(cacheId)) { 11 | if (cacheSources[cacheId].file !== null) { 12 | sdb.debug && 13 | console.log( 14 | `Removing unused file from cache: ${cacheSources[cacheId].file}`, 15 | ); 16 | unlinkSync(cacheSources[cacheId].file); 17 | } 18 | delete cacheSources[cacheId]; 19 | } 20 | } 21 | writeFileSync(".sda-cache/sources.json", JSON.stringify(cacheSources)); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/helpers/cleanPath.ts: -------------------------------------------------------------------------------- 1 | export default function cleanPath(file: string) { 2 | return file.replaceAll("'", "''"); 3 | } 4 | -------------------------------------------------------------------------------- /src/helpers/cleanSQL.ts: -------------------------------------------------------------------------------- 1 | export default function cleanSQL(query: string) { 2 | // First pass 3 | let cleaned = query 4 | .replace(/ && /g, " AND ") 5 | .replace(/ \|\| /g, " OR ") 6 | .replace(/ === /g, " = ") 7 | .replace(/ == /g, " = ") 8 | .replace(/ !== /g, " != "); 9 | 10 | if ( 11 | cleaned.includes("ALTER TABLE") && cleaned.includes("UPDATE") && 12 | cleaned.includes("SET") 13 | ) { 14 | // We pass 15 | } else { 16 | // We do a second pass 17 | cleaned = cleaned.replace(/ null(?=\s|$)/g, " NULL") // space after or end of string 18 | .replace(/ != NULL/g, " NOT NULL") 19 | .replace(/ = NULL/g, " IS NULL"); 20 | } 21 | 22 | return cleaned.trim(); 23 | } 24 | -------------------------------------------------------------------------------- /src/helpers/convertForJS.ts: -------------------------------------------------------------------------------- 1 | export default function convertForJS(rows: { 2 | [key: string]: string | number | boolean | Date | null; 3 | }[], types: { 4 | [key: string]: string; 5 | }) { 6 | if (rows[0] !== undefined) { 7 | const firstObjectKeys = Object.keys(rows[0]); 8 | for (const key of Object.keys(types)) { 9 | if (!firstObjectKeys.includes(key)) { 10 | continue; 11 | } 12 | if (types[key] === "DATE") { 13 | for (const row of rows) { 14 | row[key] = row[key] === null 15 | ? null 16 | : new Date(`${row[key]}T00:00:00.000Z`); 17 | } 18 | } else if (types[key] === "TIMESTAMP") { 19 | for (const row of rows) { 20 | row[key] = row[key] === null 21 | ? null 22 | : new Date((row[key] as string).replace(" ", "T") + "Z"); 23 | } 24 | } else if (types[key] === "BIGINT" || types[key] === "HUGEINT") { 25 | for (const row of rows) { 26 | row[key] = row[key] === null ? null : Number(row[key]); 27 | } 28 | } else if (types[key] === "GEOMETRY") { 29 | for (const row of rows) { 30 | row[key] = row[key] === null ? null : ""; 31 | } 32 | } else if (types[key].includes("FLOAT[")) { 33 | for (const row of rows) { 34 | row[key] = row[key] === null ? null : `<${types[key]}>`; 35 | } 36 | } 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/helpers/extractTypes.ts: -------------------------------------------------------------------------------- 1 | export default function extractTypes( 2 | types: { 3 | [key: string]: string | number | boolean | Date | null; 4 | }[] | null, 5 | ) { 6 | const typesObj: { [key: string]: string } = {}; 7 | 8 | if (types) { 9 | for (const t of types as { [key: string]: string }[]) { 10 | if (t.column_name) { 11 | typesObj[t.column_name] = t.column_type; 12 | } 13 | } 14 | } 15 | 16 | return typesObj; 17 | } 18 | -------------------------------------------------------------------------------- /src/helpers/findGeoColumn.ts: -------------------------------------------------------------------------------- 1 | import type SimpleTable from "../class/SimpleTable.ts"; 2 | 3 | export default async function findGeoColumn(SimpleTable: SimpleTable) { 4 | let column; 5 | 6 | const types = await SimpleTable.getTypes(); 7 | const geometries = Object.values(types).filter( 8 | (d) => d.toLowerCase() === "geometry", 9 | ); 10 | if (geometries.length === 0) { 11 | throw new Error("No column storing geometries"); 12 | } else if (geometries.length > 1) { 13 | throw new Error( 14 | "More than one column storing geometries. If the method allows to specify one, do it. Otherwise, use the selectColumns methods beforehand.", 15 | ); 16 | } else { 17 | column = Object.keys(types).find( 18 | (d) => types[d].toLowerCase() === "geometry", 19 | ); 20 | } 21 | if (typeof column !== "string") { 22 | throw new Error("No column"); 23 | } 24 | return column; 25 | } 26 | -------------------------------------------------------------------------------- /src/helpers/getCombinations.ts: -------------------------------------------------------------------------------- 1 | // from https://github.com/simple-statistics/simple-statistics/blob/main/src/combinations.js 2 | 3 | // @ts-ignore: Needs to be reworked. 4 | function getCombinations(x, k) { 5 | let i; 6 | let subI; 7 | const combinationList = []; 8 | let subsetCombinations; 9 | let next; 10 | 11 | for (i = 0; i < x.length; i++) { 12 | if (k === 1) { 13 | combinationList.push([x[i]]); 14 | } else { 15 | subsetCombinations = getCombinations( 16 | x.slice(i + 1, x.length), 17 | k - 1, 18 | ); 19 | for (subI = 0; subI < subsetCombinations.length; subI++) { 20 | next = subsetCombinations[subI]; 21 | next.unshift(x[i]); 22 | combinationList.push(next); 23 | } 24 | } 25 | } 26 | return combinationList; 27 | } 28 | 29 | export default getCombinations; 30 | -------------------------------------------------------------------------------- /src/helpers/getExtension.ts: -------------------------------------------------------------------------------- 1 | export default function getExtension(path: string) { 2 | const extensionSplit = path 3 | .replace(".gz", "") 4 | .replace(".zstd", "") 5 | .split("."); 6 | const extension = extensionSplit[extensionSplit.length - 1] 7 | .toLocaleLowerCase(); 8 | 9 | return extension; 10 | } 11 | -------------------------------------------------------------------------------- /src/helpers/getFirstNonNullOrUndefinedValues.ts: -------------------------------------------------------------------------------- 1 | export default function getFirstNonNullOrUndefinedValues( 2 | arrayOfObjects: { [key: string]: unknown }[], 3 | ) { 4 | const columns = Object.keys(arrayOfObjects[0]); 5 | const values = []; 6 | 7 | for (const col of columns) { 8 | let value = undefined; 9 | for (const d of arrayOfObjects) { 10 | if (d[col] !== null && d[col] !== undefined) { 11 | value = d[col]; 12 | break; 13 | } 14 | } 15 | if (value !== undefined) { 16 | values.push(value); 17 | } else { 18 | values.push(""); 19 | } 20 | } 21 | return values; 22 | } 23 | -------------------------------------------------------------------------------- /src/helpers/getIdenticalColumns.ts: -------------------------------------------------------------------------------- 1 | export default function getIdenticalColumns( 2 | table1Columns: string[], 3 | table2Columns: string[], 4 | ) { 5 | return table1Columns.filter((column) => table2Columns.includes(column)); 6 | } 7 | -------------------------------------------------------------------------------- /src/helpers/getName.ts: -------------------------------------------------------------------------------- 1 | export default function getName(file: string) { 2 | if (file === ":memory:") { 3 | return "memory"; 4 | } else { 5 | const nameSplit = file.split("/"); 6 | const name = nameSplit[nameSplit.length - 1]; 7 | if (!name.includes(".")) { 8 | return name; // Return the name directly if there's no extension 9 | } 10 | const nameWithoutExtension = name.split(".").slice(0, -1).join("."); 11 | return nameWithoutExtension; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/helpers/getProjection.ts: -------------------------------------------------------------------------------- 1 | import mergeOptions from "../helpers/mergeOptions.ts"; 2 | import queryDB from "../helpers/queryDB.ts"; 3 | import type SimpleDB from "../class/SimpleDB.ts"; 4 | import cleanPath from "./cleanPath.ts"; 5 | 6 | export default async function getProjection( 7 | simpleDB: SimpleDB, 8 | file: string, 9 | ) { 10 | // Load spatial may not be necessary if we change how this works 11 | const queryResult = await queryDB( 12 | simpleDB, 13 | `INSTALL spatial; 14 | LOAD spatial; 15 | SELECT layers[1].geometry_fields[1].crs.name as name, CONCAT(layers[1].geometry_fields[1].crs.auth_name, ':', layers[1].geometry_fields[1].crs.auth_code) as code, layers[1].geometry_fields[1].crs.projjson as unit, layers[1].geometry_fields[1].crs.proj4 as proj4 FROM st_read_meta('${ 16 | cleanPath(file) 17 | }')`, 18 | mergeOptions(simpleDB, { 19 | table: null, 20 | method: "getProjection()", 21 | parameters: { file }, 22 | returnDataFrom: "query", 23 | }), 24 | ); 25 | 26 | if (!queryResult) { 27 | throw new Error("No queryResults"); 28 | } 29 | 30 | const result = queryResult[0]; 31 | result.unit = JSON.parse( 32 | result.unit as string, 33 | ).coordinate_system.axis[0].unit; 34 | 35 | return result as { 36 | name: string; 37 | code: string; 38 | unit: string; 39 | proj4: string; 40 | }; 41 | } 42 | -------------------------------------------------------------------------------- /src/helpers/getProjectionParquet.ts: -------------------------------------------------------------------------------- 1 | import type SimpleTable from "../class/SimpleTable.ts"; 2 | import cleanPath from "./cleanPath.ts"; 3 | import mergeOptions from "./mergeOptions.ts"; 4 | import queryDB from "./queryDB.ts"; 5 | 6 | export default async function getProjectionParquet( 7 | SimpleTable: SimpleTable, 8 | file: string, 9 | ) { 10 | const queryResult = await queryDB( 11 | SimpleTable, 12 | `SELECT * FROM parquet_kv_metadata('${cleanPath(file)}');`, 13 | mergeOptions(SimpleTable, { 14 | table: SimpleTable.name, 15 | method: "getProjectionParquet()", 16 | parameters: { file }, 17 | returnDataFrom: "query", 18 | }), 19 | ); 20 | 21 | if (!queryResult) { 22 | throw new Error( 23 | `Could not get metadata from parquet file: ${file}`, 24 | ); 25 | } 26 | 27 | const projection = queryResult.find((d) => { 28 | return d.key?.toString() === "projections"; 29 | }); 30 | 31 | if (!projection || projection.value === null) { 32 | console.warn( 33 | `\nCould not get projection from parquet file: ${file}\n`, 34 | ); 35 | return {}; 36 | } else { 37 | return JSON.parse((projection.value as string).replaceAll("\\x22", '"')); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/helpers/keepNumericalColumns.ts: -------------------------------------------------------------------------------- 1 | export default function keepNumericalColumns(types: { [key: string]: string }) { 2 | const columns: string[] = []; 3 | for (const col of Object.keys(types)) { 4 | if ( 5 | ["FLOAT", "DOUBLE", "DECIMAL"].includes(types[col]) || 6 | types[col].includes("INT") 7 | ) { 8 | columns.push(col); 9 | } 10 | } 11 | if (columns.length === 0) { 12 | throw new Error("No numerical columns"); 13 | } 14 | return columns; 15 | } 16 | -------------------------------------------------------------------------------- /src/helpers/logData.ts: -------------------------------------------------------------------------------- 1 | export default function logData( 2 | types: { [key: string]: string } | null, 3 | data: 4 | | { 5 | [key: string]: string | number | boolean | Date | null; 6 | }[] 7 | | null, 8 | nbCharactersToLog?: number, 9 | ) { 10 | if (data === null) { 11 | console.log("Data is null"); 12 | } else { 13 | if (data.length === 0) { 14 | console.log(data); 15 | } else { 16 | const dataToBeLogged: { 17 | [key: string]: string | number | boolean | Date | null; 18 | }[] = []; 19 | const keys = Object.keys(data[0]); 20 | for (let i = 0; i < data.length; i++) { 21 | const newItem: { 22 | [key: string]: string | number | boolean | Date | null; 23 | } = {}; 24 | for (const key of keys) { 25 | if ( 26 | typeof nbCharactersToLog === "number" && 27 | typeof data[i][key] === "string" && 28 | (data[i][key] as string).length > nbCharactersToLog 29 | ) { 30 | newItem[key] = (data[i][key] as string).slice( 31 | 0, 32 | nbCharactersToLog, 33 | ) + "..."; // tested above 34 | } else { 35 | newItem[key] = data[i][key]; 36 | } 37 | } 38 | dataToBeLogged.push(newItem); 39 | } 40 | if (types !== null) { 41 | const columns = Object.keys(types); 42 | if (columns.length > 0) { 43 | for (const col of columns) { 44 | types[col] = types[col] + "/" + 45 | (data[0][col] === null ? null : typeof data[0][col]); 46 | } 47 | console.table([types]); 48 | } 49 | } 50 | console.table(dataToBeLogged); 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/helpers/mergeOptions.ts: -------------------------------------------------------------------------------- 1 | import type Simple from "../class/Simple.ts"; 2 | 3 | export default function mergeOptions( 4 | simple: Simple, 5 | options: { 6 | table: string | null; 7 | method: string | null; 8 | parameters: { [key: string]: unknown } | null; 9 | nbRowsToLog?: number; 10 | returnDataFrom?: "query" | "none"; 11 | debug?: boolean; 12 | }, 13 | ): { 14 | table: string | null; 15 | method: string | null; 16 | parameters: { [key: string]: unknown } | null; 17 | nbRowsToLog: number; 18 | nbCharactersToLog: number | undefined; 19 | returnDataFrom: "query" | "none"; 20 | debug: boolean; 21 | } { 22 | return { 23 | table: options.table, 24 | method: options.method, 25 | parameters: options.parameters, 26 | nbRowsToLog: options.nbRowsToLog ?? simple.nbRowsToLog, 27 | nbCharactersToLog: simple.nbCharactersToLog, 28 | returnDataFrom: options.returnDataFrom ?? "none", 29 | debug: options.debug ?? simple.debug, 30 | }; 31 | } 32 | -------------------------------------------------------------------------------- /src/helpers/parseDuckDBType.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ARRAY, 3 | BIGINT, 4 | BOOLEAN, 5 | DATE, 6 | DOUBLE, 7 | FLOAT, 8 | INTEGER, 9 | TIME, 10 | TIMESTAMP, 11 | TIMESTAMPTZ, 12 | VARCHAR, 13 | } from "@duckdb/node-api"; 14 | 15 | export default function parseDuckDBType(type: string) { 16 | if (type === "INTEGER") { 17 | return INTEGER; 18 | } else if (type === "BIGINT") { 19 | return BIGINT; 20 | } else if (type === "DOUBLE") { 21 | return DOUBLE; 22 | } else if (type === "VARCHAR") { 23 | return VARCHAR; 24 | } else if (type === "TIMESTAMP") { 25 | return TIMESTAMP; 26 | } else if (type === "TIMESTAMP WITH TIME ZONE") { 27 | return TIMESTAMPTZ; 28 | } else if (type === "DATE") { 29 | return DATE; 30 | } else if (type === "TIME") { 31 | return TIME; 32 | } else if (type === "BOOLEAN") { 33 | return BOOLEAN; 34 | } else if (type.includes("FLOAT[")) { 35 | // For embeddings 36 | const size = type.replace("FLOAT[", "").replace("]", ""); 37 | return ARRAY(FLOAT, parseInt(size)); 38 | } else { 39 | throw new Error(`Type ${type} not supported.`); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/helpers/parseTypes.ts: -------------------------------------------------------------------------------- 1 | export default function parseType( 2 | type: 3 | | "integer" 4 | | "float" 5 | | "number" 6 | | "string" 7 | | "date" 8 | | "time" 9 | | "datetime" 10 | | "datetimeTz" 11 | | "bigint" 12 | | "double" 13 | | "varchar" 14 | | "timestamp" 15 | | "timestamp with time zone" 16 | | "boolean" 17 | | "geometry", 18 | ) { 19 | const typeLowerCase = type.toLowerCase(); 20 | if (typeLowerCase === "integer") { 21 | return "INTEGER"; 22 | } else if (typeLowerCase === "float" || typeLowerCase === "number") { 23 | return "DOUBLE"; 24 | } else if (typeLowerCase === "string") { 25 | return "VARCHAR"; 26 | } else if (typeLowerCase === "datetime") { 27 | return "TIMESTAMP"; 28 | } else if (typeLowerCase === "datetimetz") { 29 | return "TIMESTAMP WITH TIME ZONE"; 30 | } else if ( 31 | [ 32 | "date", 33 | "time", 34 | "bigint", 35 | "hugeint", 36 | "double", 37 | "varchar", 38 | "timestamp", 39 | "timestamp with time zone", 40 | "boolean", 41 | "geometry", 42 | ].includes(typeLowerCase) 43 | ) { 44 | return type.toUpperCase(); 45 | } else { 46 | throw new Error(`Unknown type ${type}`); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/helpers/parseValue.ts: -------------------------------------------------------------------------------- 1 | export default function parseValue(value: unknown) { 2 | if (Number.isNaN(value) || value === undefined || value === null) { 3 | return "NULL"; 4 | } else if (value instanceof Date) { 5 | return `'${value.toISOString()}'`; 6 | } else if (typeof value === "string") { 7 | return `'${value.replaceAll("'", "''")}'`; 8 | } else if (typeof value === "boolean") { 9 | return value; 10 | } else if (typeof value === "number") { 11 | return value; 12 | } else { 13 | throw new Error(`Unkown type ${typeof value} of ${value}`); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/helpers/runQuery.ts: -------------------------------------------------------------------------------- 1 | import type { DuckDBConnection } from "@duckdb/node-api"; 2 | import convertForJS from "./convertForJS.ts"; 3 | 4 | export default async function runQuery( 5 | query: string, 6 | connection: DuckDBConnection, 7 | returnDataFromQuery: boolean, 8 | options: { 9 | debug: boolean; 10 | method: string | null; 11 | parameters: { [key: string]: unknown } | null; 12 | types?: { [key: string]: string }; 13 | }, 14 | ): Promise< 15 | | { 16 | [key: string]: number | string | Date | boolean | null; 17 | }[] 18 | | null 19 | > { 20 | try { 21 | if (returnDataFromQuery) { 22 | const reader = await connection.runAndReadAll( 23 | query, 24 | ); 25 | const rows = reader.getRowObjectsJson() as { 26 | [key: string]: string | number | boolean | Date | null; 27 | }[]; 28 | 29 | if (options.types) { 30 | convertForJS(rows, options.types); 31 | } 32 | 33 | return rows; 34 | } else { 35 | await connection.run(query); 36 | return null; 37 | } 38 | } catch (error) { 39 | console.warn(error); 40 | if (options.debug === false) { 41 | console.log("SDA: method causing error =>", options.method); 42 | console.log("parameters:", options.parameters); 43 | console.log("query:", query); 44 | } 45 | throw error; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/helpers/setDbProps.ts: -------------------------------------------------------------------------------- 1 | import { existsSync, readFileSync } from "node:fs"; 2 | import type SimpleDB from "../class/SimpleDB.ts"; 3 | 4 | export default async function setDbProps( 5 | simpleDB: SimpleDB, 6 | file: string, 7 | extension: string, 8 | allIndexesFile: string, 9 | ) { 10 | for (const table of await simpleDB.getTableNames()) { 11 | const t = simpleDB.newTable(table); 12 | simpleDB.tables.push(t); 13 | } 14 | const allProjectionsFile = `${ 15 | file.replace(`.${extension}`, "") 16 | }_projections.json`; 17 | if (existsSync(allProjectionsFile)) { 18 | const projections = JSON.parse( 19 | readFileSync(allProjectionsFile, "utf-8"), 20 | ); 21 | for (const table of simpleDB.tables) { 22 | if (projections[table.name]) { 23 | table.projections = projections[table.name]; 24 | } 25 | } 26 | await simpleDB.customQuery(`INSTALL spatial; LOAD spatial;`); 27 | } 28 | 29 | if (existsSync(allIndexesFile)) { 30 | const indexes = JSON.parse(readFileSync(allIndexesFile, "utf-8")); 31 | for (const table of simpleDB.tables) { 32 | if (indexes[table.name]) { 33 | table.indexes = indexes[table.name]; 34 | } 35 | } 36 | } 37 | 38 | simpleDB.tableIncrement = Math.round(Math.random() * 1000000); 39 | } 40 | -------------------------------------------------------------------------------- /src/helpers/shouldFlipBeforeExport.ts: -------------------------------------------------------------------------------- 1 | export default function shouldFlipBeforeExport(projection: string) { 2 | return projection.includes("proj=latlong"); 3 | } 4 | -------------------------------------------------------------------------------- /src/helpers/stringToArray.ts: -------------------------------------------------------------------------------- 1 | export default function stringToArray(argument: string | string[]) { 2 | if (Array.isArray(argument)) { 3 | return argument; 4 | } else if (typeof argument === "string") { 5 | return [argument]; 6 | } else { 7 | throw new Error(`argument should be a string or an array of strings.`); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/helpers/stringifyDates.ts: -------------------------------------------------------------------------------- 1 | import type SimpleTable from "../class/SimpleTable.ts"; 2 | 3 | export default async function stringifyDates( 4 | simpleTable: SimpleTable, 5 | types: { [key: string]: string }, 6 | ) { 7 | const typesKeys = Object.keys(types); 8 | const typesValues = Object.values(types); 9 | const toConvert: { [key: string]: "string" } = {}; 10 | for (let i = 0; i < typesKeys.length; i++) { 11 | if (typesValues[i] === "TIMESTAMP" || typesValues[i] === "DATE") { 12 | toConvert[typesKeys[i]] = "string"; 13 | } 14 | } 15 | if (Object.keys(toConvert).length > 0) { 16 | await simpleTable.convert(toConvert, { datetimeFormat: "%xT%T.%g%z" }); 17 | await simpleTable.replace(Object.keys(toConvert), { "+00": "Z" }); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/helpers/stringifyDatesInvert.ts: -------------------------------------------------------------------------------- 1 | import type SimpleTable from "../class/SimpleTable.ts"; 2 | 3 | export default async function stringifyDatesInvert( 4 | simpleTable: SimpleTable, 5 | types: { [key: string]: string }, 6 | ) { 7 | const typesKeys = Object.keys(types); 8 | const typesValues = Object.values(types); 9 | const toConvertBack: { [key: string]: "timestamp" | "date" } = {}; 10 | for (let i = 0; i < typesKeys.length; i++) { 11 | if (typesValues[i] === "TIMESTAMP" || typesValues[i] === "DATE") { 12 | toConvertBack[typesKeys[i]] = typesValues[i].toLowerCase() as 13 | | "timestamp" 14 | | "date"; 15 | } 16 | } 17 | if (Object.keys(toConvertBack).length > 0) { 18 | await simpleTable.replace(Object.keys(toConvertBack), { "Z": "+00" }); 19 | await simpleTable.convert(toConvertBack); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/helpers/tryEmbedding.ts: -------------------------------------------------------------------------------- 1 | import { getEmbedding } from "@nshiab/journalism"; 2 | 3 | export default async function tryEmbedding( 4 | i: number, 5 | rows: { 6 | [key: string]: string | number | boolean | Date | null; 7 | }[], 8 | text: string, 9 | newColumn: string, 10 | options: { 11 | cache?: boolean; 12 | model?: string; 13 | apiKey?: string; 14 | vertex?: boolean; 15 | project?: string; 16 | location?: string; 17 | ollama?: boolean; 18 | verbose?: boolean; 19 | } = {}, 20 | ) { 21 | // Should be improved... 22 | return rows[i][newColumn] = await getEmbedding( 23 | text, 24 | options, 25 | ) as unknown as number; 26 | } 27 | -------------------------------------------------------------------------------- /src/helpers/unifyColumns.ts: -------------------------------------------------------------------------------- 1 | import type SimpleTable from "../class/SimpleTable.ts"; 2 | 3 | export default async function unifyColumns(allTables: SimpleTable[]) { 4 | const columnsAdded: { 5 | [key: string]: string[]; 6 | } = {}; 7 | const allTypes: { [key: string]: string } = {}; 8 | const allProjections: { [key: string]: string } = {}; 9 | for (const table of allTables) { 10 | const types = await table.getTypes(); 11 | for (const key in types) { 12 | if (!allTypes[key]) { 13 | allTypes[key] = types[key]; 14 | allProjections[key] = table.projections[key]; 15 | } else { 16 | if (allTypes[key] !== types[key]) { 17 | throw new Error( 18 | `The column ${key} has different types in the tables.`, 19 | ); 20 | } else if (allProjections[key] !== table.projections[key]) { 21 | throw new Error( 22 | `The column ${key} has different projections in the tables.`, 23 | ); 24 | } 25 | } 26 | } 27 | } 28 | for (const column in allTypes) { 29 | for (const table of allTables) { 30 | if (!(await table.hasColumn(column))) { 31 | await table.addColumn( 32 | column, 33 | // Could be improved 34 | allTypes[column].toLowerCase() as 35 | | "string" 36 | | "number" 37 | | "bigint" 38 | | "boolean" 39 | | "integer" 40 | | "float" 41 | | "date" 42 | | "time" 43 | | "datetime" 44 | | "datetimeTz" 45 | | "double" 46 | | "varchar" 47 | | "timestamp" 48 | | "timestamp with time zone" 49 | | "geometry", 50 | `null`, 51 | { 52 | projection: allTypes[column] === "GEOMETRY" 53 | ? allProjections[column] 54 | : undefined, 55 | }, 56 | ); 57 | if (!columnsAdded[table.name]) { 58 | columnsAdded[table.name] = []; 59 | } 60 | columnsAdded[table.name].push(column); 61 | } 62 | } 63 | } 64 | 65 | return columnsAdded; 66 | } 67 | -------------------------------------------------------------------------------- /src/helpers/writeDataAsArrays.ts: -------------------------------------------------------------------------------- 1 | import { dataToArrays } from "jsr:@nshiab/journalism@1"; 2 | import type SimpleTable from "../class/SimpleTable.ts"; 3 | import getExtension from "./getExtension.ts"; 4 | import { writeFileSync } from "node:fs"; 5 | 6 | export default async function writeDataAsArrays( 7 | simpleTable: SimpleTable, 8 | file: string, 9 | ) { 10 | const fileExtension = getExtension(file); 11 | if (fileExtension === "json") { 12 | const data = await simpleTable.getData(); 13 | writeFileSync(file, JSON.stringify(dataToArrays(data))); 14 | } else { 15 | throw new Error("The option dataAsArrays works only with json files."); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/helpers/writeProjectionsAndIndexes.ts: -------------------------------------------------------------------------------- 1 | import { existsSync, rmSync, writeFileSync } from "node:fs"; 2 | import type { SimpleDB } from "../index.ts"; 3 | 4 | export default function writeProjectionsAndIndexes( 5 | simpleDB: SimpleDB, 6 | extension: string, 7 | file: string, 8 | ) { 9 | const allProjections: { [key: string]: { [key: string]: string } } = {}; 10 | for (const table of simpleDB.tables) { 11 | if (Object.keys(table.projections).length > 0) { 12 | allProjections[table.name] = table.projections; 13 | } 14 | } 15 | const allProjectionsFile = `${ 16 | file.replace(`.${extension}`, "") 17 | }_projections.json`; 18 | if (existsSync(allProjectionsFile)) { 19 | rmSync(allProjectionsFile); 20 | } 21 | if (Object.keys(allProjections).length > 0) { 22 | writeFileSync(allProjectionsFile, JSON.stringify(allProjections)); 23 | } 24 | 25 | const allIndexes: { [key: string]: string[] } = {}; 26 | for (const table of simpleDB.tables) { 27 | if (table.indexes.length > 0) { 28 | allIndexes[table.name] = table.indexes; 29 | } 30 | } 31 | const allIndexesFile = `${file.replace(`.${extension}`, "")}_indexes.json`; 32 | if (existsSync(allIndexesFile)) { 33 | rmSync(allIndexesFile); 34 | } 35 | if (Object.keys(allIndexes).length > 0) { 36 | writeFileSync(allIndexesFile, JSON.stringify(allIndexes)); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/incrementVersion.ts: -------------------------------------------------------------------------------- 1 | import { readFileSync, writeFileSync } from "node:fs"; 2 | import process from "node:process"; 3 | import { execSync } from "node:child_process"; 4 | 5 | // Check if we are on the main branch 6 | let branch = execSync("git rev-parse --abbrev-ref HEAD", { 7 | encoding: "utf-8", 8 | }).trim(); 9 | // Remove 'heads/' prefix if present 10 | if (branch.startsWith("heads/")) { 11 | branch = branch.slice(6); 12 | } 13 | if (branch !== "main") { 14 | throw new Error( 15 | `You can only increment the version on the main branch. Current branch is ${branch}`, 16 | ); 17 | } 18 | 19 | const args = process.argv.slice(2); 20 | const [incrementType] = args; 21 | const filePath = "deno.json"; 22 | const data = JSON.parse(readFileSync(filePath, "utf-8")); 23 | console.log(`\nCurrent version is ${data.version}`); 24 | const versionParts = data.version.split(".").map((d: string) => parseInt(d)); 25 | 26 | if (incrementType === "major") { 27 | versionParts[0] += 1; 28 | versionParts[1] = 0; 29 | versionParts[2] = 0; 30 | } else if (incrementType === "minor") { 31 | versionParts[1] += 1; 32 | versionParts[2] = 0; 33 | } else if (incrementType === "patch") { 34 | versionParts[2] += 1; 35 | } else { 36 | throw new Error("Invalid increment type"); 37 | } 38 | 39 | data.version = versionParts.join("."); 40 | writeFileSync(filePath, JSON.stringify(data, null, 2)); 41 | execSync("deno fmt"); 42 | 43 | execSync("git add -A"); 44 | execSync(`git commit -m "v${data.version}"`); 45 | execSync("git push"); 46 | 47 | console.log(`Version incremented to ${data.version}`); 48 | 49 | // Tag the current version 50 | const tagName = `v${data.version}`; 51 | execSync(`git tag ${tagName}`); 52 | execSync(`git push origin tag ${tagName}`); 53 | 54 | console.log(`Tagged with ${tagName}`); 55 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export { default as SimpleDB } from "./class/SimpleDB.ts"; 2 | export { default as SimpleTable } from "./class/SimpleTable.ts"; 3 | -------------------------------------------------------------------------------- /src/methods/aggregateGeoQuery.ts: -------------------------------------------------------------------------------- 1 | import stringToArray from "../helpers/stringToArray.ts"; 2 | 3 | export default function aggregateGeoQuery( 4 | table: string, 5 | column: string, 6 | method: "union" | "intersection", 7 | options: { 8 | categories?: string | string[]; 9 | outputTable?: string | boolean; 10 | } = {}, 11 | ) { 12 | const categoriesOptions = options.categories ?? []; 13 | const categories = stringToArray(categoriesOptions); 14 | 15 | let query = `CREATE OR REPLACE TABLE "${options.outputTable ?? table}" AS 16 | SELECT${ 17 | categories.length > 0 ? ` ${categories.map((d) => `${d}`).join(", ")},` : "" 18 | }`; 19 | 20 | if (method === "union") { 21 | query += ` ST_Union_Agg(${column}) AS ${column}`; 22 | } else if (method === "intersection") { 23 | query += ` ST_Intersection_Agg(${column}) AS ${column}`; 24 | } else { 25 | throw new Error(`Unkown method ${method}`); 26 | } 27 | 28 | query += `\nFROM "${table}"`; 29 | 30 | if (categories.length > 0) { 31 | query += `\nGROUP BY ${categories.map((d) => `${d}`).join(", ")}`; 32 | query += `\nORDER BY ${categories.map((d) => `${d} ASC`).join(", ")}`; 33 | } 34 | 35 | return query; 36 | } 37 | -------------------------------------------------------------------------------- /src/methods/aiQuery.ts: -------------------------------------------------------------------------------- 1 | import { askAI } from "@nshiab/journalism"; 2 | import type { SimpleTable } from "../index.ts"; 3 | 4 | export default async function aiQuery( 5 | simpleTable: SimpleTable, 6 | prompt: string, 7 | options: { 8 | cache?: boolean; 9 | model?: string; 10 | apiKey?: string; 11 | vertex?: boolean; 12 | project?: string; 13 | location?: string; 14 | verbose?: boolean; 15 | } = {}, 16 | ) { 17 | const p = 18 | `I have a SQL table named "${simpleTable.name}". The data is already in it with these columns:\n${ 19 | JSON.stringify(await simpleTable.getTypes()) 20 | }\nI want you to give me a SQL query to do this:\n- ${prompt}\nThe query must replace the existing "${simpleTable.name}" table. This means the the query must start with 'CREATE OR REPLACE TABLE "${simpleTable.name}"...'. Return just the query, nothing else.`; 21 | 22 | if (options.verbose) { 23 | console.log("\naiQuery()"); 24 | } 25 | 26 | // Types could be improved 27 | let query = await askAI(p, options) as unknown as string; 28 | query = query.replace("```sql", "").replace("```", "").trim(); 29 | 30 | await simpleTable.sdb.customQuery(query); 31 | } 32 | -------------------------------------------------------------------------------- /src/methods/binsQuery.ts: -------------------------------------------------------------------------------- 1 | import type SimpleTable from "../class/SimpleTable.ts"; 2 | 3 | export default async function binsQuery( 4 | SimpleTable: SimpleTable, 5 | values: string, 6 | interval: number, 7 | newColumn: string, 8 | options: { 9 | startValue?: number; 10 | } = {}, 11 | ) { 12 | const minValue = await SimpleTable.getMin(values); 13 | if (typeof minValue !== "number") { 14 | throw new Error(`minValue of ${values} is not a number`); 15 | } 16 | 17 | let startValue = 0; 18 | if (typeof options.startValue === "number") { 19 | if (startValue > minValue) { 20 | throw new Error( 21 | `startValue ${options.startValue} can't be greater than minValue ${minValue}`, 22 | ); 23 | } 24 | startValue = options.startValue; 25 | } else { 26 | startValue = minValue; 27 | } 28 | 29 | const maxValue = await SimpleTable.getMax(values); 30 | if (typeof maxValue !== "number") { 31 | throw new Error(`maxValue of ${values} is not a number`); 32 | } 33 | const endValue = maxValue; 34 | 35 | let increment = 1; 36 | let decimals = 0; 37 | const intervalAsString = interval.toString(); 38 | const decimalIndex = intervalAsString.indexOf("."); 39 | if (decimalIndex > 0) { 40 | decimals = intervalAsString.substring(decimalIndex + 1).length; 41 | increment = 1.0 / (10.0 * decimals); 42 | } 43 | 44 | const intervals: string[] = []; 45 | 46 | for (let i = startValue; i <= endValue; i += interval) { 47 | const start = i; 48 | const end = (i + interval - increment).toFixed(decimals); 49 | intervals.push( 50 | `WHEN ${values} >= ${start} AND ${values} <= ${end} THEN '[${start}-${end}]'`, 51 | ); 52 | } 53 | 54 | const query = `ALTER TABLE ${SimpleTable.name} ADD ${newColumn} VARCHAR; 55 | UPDATE ${SimpleTable.name} SET ${newColumn} = CASE 56 | ${intervals.join("\n")} 57 | END`; 58 | 59 | return query; 60 | } 61 | -------------------------------------------------------------------------------- /src/methods/capitalizeQuery.ts: -------------------------------------------------------------------------------- 1 | export default function capitalizeQuery(table: string, columns: string[]) { 2 | let query = ""; 3 | 4 | for (const column of columns) { 5 | query += 6 | `\nUPDATE "${table}" SET "${column}" = CONCAT(UPPER(LEFT("${column}", 1)), LOWER(RIGHT("${column}", LENGTH("${column}")-1)));`; 7 | } 8 | 9 | return query; 10 | } 11 | -------------------------------------------------------------------------------- /src/methods/cloneColumn.ts: -------------------------------------------------------------------------------- 1 | export default function cloneColumnQuery( 2 | table: string, 3 | originalColumn: string, 4 | newColumn: string, 5 | types: { [key: string]: string }, 6 | ) { 7 | let query = ""; 8 | 9 | const originalColumnType = types[originalColumn]; 10 | 11 | if (originalColumnType) { 12 | query += 13 | `ALTER TABLE "${table}" ADD COLUMN "${newColumn}" ${originalColumnType}; 14 | UPDATE "${table}" SET "${newColumn}" = "${originalColumn}"`; 15 | } else { 16 | throw new Error(`Can't find type of ${originalColumn}`); 17 | } 18 | 19 | return query; 20 | } 21 | -------------------------------------------------------------------------------- /src/methods/cloneQuery.ts: -------------------------------------------------------------------------------- 1 | export default function cloneQuery( 2 | table: string, 3 | newTable: string, 4 | options: { 5 | conditions?: string; 6 | } = {}, 7 | ) { 8 | return `CREATE OR REPLACE TABLE "${newTable}" AS SELECT * FROM "${table}"${ 9 | options.conditions ? ` WHERE ${options.conditions}` : "" 10 | }`; 11 | } 12 | -------------------------------------------------------------------------------- /src/methods/concatenateQuery.ts: -------------------------------------------------------------------------------- 1 | export default function concatenateQuery( 2 | table: string, 3 | columns: string[], 4 | newColumn: string, 5 | options: { separator?: string }, 6 | ) { 7 | let query = `ALTER TABLE "${table}" ADD "${newColumn}" VARCHAR; 8 | UPDATE "${table}" SET "${newColumn}" = `; 9 | if (typeof options.separator === "string") { 10 | query += `CONCAT_WS('${options.separator}', ${ 11 | columns 12 | .map((d) => `"${d}"`) 13 | .join(", ") 14 | })`; 15 | } else { 16 | query += `CONCAT(${columns.map((d) => `"${d}"`).join(", ")})`; 17 | } 18 | 19 | return query; 20 | } 21 | -------------------------------------------------------------------------------- /src/methods/correlations.ts: -------------------------------------------------------------------------------- 1 | import getCombinations from "../helpers/getCombinations.ts"; 2 | import keepNumericalColumns from "../helpers/keepNumericalColumns.ts"; 3 | import mergeOptions from "../helpers/mergeOptions.ts"; 4 | import queryDB from "../helpers/queryDB.ts"; 5 | import correlationsQuery from "./correlationsQuery.ts"; 6 | import type SimpleTable from "../class/SimpleTable.ts"; 7 | 8 | export default async function correlations( 9 | SimpleTable: SimpleTable, 10 | options: { 11 | x?: string; 12 | y?: string; 13 | categories?: string | string[]; 14 | decimals?: number; 15 | outputTable?: string | boolean; 16 | } = {}, 17 | ) { 18 | const outputTable = typeof options.outputTable === "string" 19 | ? options.outputTable 20 | : SimpleTable.name; 21 | 22 | let combinations: [string, string][] = []; 23 | if (!options.x && !options.y) { 24 | const types = await SimpleTable.getTypes(); 25 | const columns = keepNumericalColumns(types); 26 | combinations = getCombinations(columns, 2); 27 | } else if (options.x && !options.y) { 28 | const types = await SimpleTable.getTypes(); 29 | const columns = keepNumericalColumns(types); 30 | combinations = []; 31 | for (const col of columns) { 32 | if (col !== options.x) { 33 | combinations.push([options.x, col]); 34 | } 35 | } 36 | } else if (options.x && options.y) { 37 | combinations = [[options.x, options.y]]; 38 | } else { 39 | throw new Error("No combinations of x and y"); 40 | } 41 | 42 | await queryDB( 43 | SimpleTable, 44 | correlationsQuery( 45 | SimpleTable.name, 46 | outputTable, 47 | combinations, 48 | options, 49 | ), 50 | mergeOptions(SimpleTable, { 51 | table: outputTable, 52 | method: "correlations()", 53 | parameters: { 54 | options, 55 | "combinations (computed)": combinations, 56 | }, 57 | }), 58 | ); 59 | } 60 | -------------------------------------------------------------------------------- /src/methods/correlationsQuery.ts: -------------------------------------------------------------------------------- 1 | import stringToArray from "../helpers/stringToArray.ts"; 2 | 3 | export default function correlationsQuery( 4 | table: string, 5 | outputTable: string, 6 | combinations: [string, string][], 7 | options: { 8 | categories?: string | string[]; 9 | decimals?: number; 10 | }, 11 | ) { 12 | const categories = options.categories 13 | ? stringToArray(options.categories) 14 | : []; 15 | 16 | const groupBy = categories.length === 0 17 | ? "" 18 | : ` GROUP BY ${categories.map((d) => `${d}`).join(",")}`; 19 | 20 | let query = `CREATE OR REPLACE TABLE ${outputTable} AS`; 21 | 22 | let firstValue = true; 23 | for (const comb of combinations) { 24 | if (firstValue) { 25 | firstValue = false; 26 | } else { 27 | query += "\nUNION"; 28 | } 29 | const tempQuery = typeof options.decimals === "number" 30 | ? `ROUND(corr(${comb[0]}, ${comb[1]}), ${options.decimals})` 31 | : `corr(${comb[0]}, ${comb[1]})`; 32 | query += `\nSELECT ${ 33 | categories.length > 0 34 | ? `${categories.map((d) => `${d}`).join(",")}, ` 35 | : "" 36 | }'${comb[0]}' AS x, '${ 37 | comb[1] 38 | }' AS y, ${tempQuery} as corr FROM "${table}"${groupBy}`; 39 | } 40 | 41 | return query; 42 | } 43 | -------------------------------------------------------------------------------- /src/methods/crossJoinQuery.ts: -------------------------------------------------------------------------------- 1 | export default function crossJoinQuery( 2 | table: string, 3 | rightTable: string, 4 | options: { 5 | outputTable?: string | boolean; 6 | } = {}, 7 | ) { 8 | return `CREATE OR REPLACE TABLE ${ 9 | options.outputTable ?? table 10 | } AS SELECT "${table}".*, ${rightTable}.* FROM "${table}" CROSS JOIN "${rightTable}";`; 11 | } 12 | -------------------------------------------------------------------------------- /src/methods/distanceQuery.ts: -------------------------------------------------------------------------------- 1 | export default function distanceQuery( 2 | table: string, 3 | column1: string, 4 | column2: string, 5 | newColumn: string, 6 | options: { 7 | unit?: "m" | "km"; 8 | method?: "srs" | "spheroid" | "haversine"; 9 | decimals?: number; 10 | } = {}, 11 | ) { 12 | options.method = options.method ?? "srs"; 13 | 14 | if (options.method === "srs" && typeof options.unit === "string") { 15 | throw new Error( 16 | "Using the SRS unit. You can't specify options.unit unless you set options.method to 'spheroid' or 'haversine'.", 17 | ); 18 | } else if (["spheroid", "haversine"].includes(options.method)) { 19 | options.unit = options.unit ?? "m"; 20 | if (!["m", "km"].includes(options.unit)) { 21 | throw new Error( 22 | `Unknown unit ${options.unit}. Choose between 'm' and 'km'.`, 23 | ); 24 | } 25 | } 26 | 27 | let query = 28 | `ALTER TABLE "${table}" ADD ${newColumn} DOUBLE; UPDATE "${table}" SET ${newColumn} = `; 29 | 30 | if (options.method === "srs") { 31 | if (typeof options.decimals === "number") { 32 | query += 33 | `ROUND(ST_Distance(${column1}, ${column2}), ${options.decimals})`; 34 | } else { 35 | query += `ST_Distance(${column1}, ${column2})`; 36 | } 37 | } else if (options.method === "haversine") { 38 | if (typeof options.decimals === "number") { 39 | query += `ROUND(ST_Distance_Sphere(${column1}, ${column2}) ${ 40 | options.unit === "km" ? "/ 1000" : "" 41 | }, ${options.decimals});`; 42 | } else { 43 | query += `ST_Distance_Sphere(${column1}, ${column2}) ${ 44 | options.unit === "km" ? "/ 1000" : "" 45 | };`; 46 | } 47 | } else if (options.method === "spheroid") { 48 | if (typeof options.decimals === "number") { 49 | query += `ROUND(ST_Distance_Spheroid(${column1}, ${column2}) ${ 50 | options.unit === "km" ? "/ 1000" : "" 51 | }, ${options.decimals});`; 52 | } else { 53 | query += `ST_Distance_Spheroid(${column1}, ${column2}) ${ 54 | options.unit === "km" ? "/ 1000" : "" 55 | };`; 56 | } 57 | } else { 58 | throw new Error( 59 | `Uknown method ${options.method}. Choose between 'srs', 'haversine' and 'spheroid'.`, 60 | ); 61 | } 62 | 63 | return query; 64 | } 65 | -------------------------------------------------------------------------------- /src/methods/getBottom.ts: -------------------------------------------------------------------------------- 1 | import mergeOptions from "../helpers/mergeOptions.ts"; 2 | import queryDB from "../helpers/queryDB.ts"; 3 | import type SimpleTable from "../class/SimpleTable.ts"; 4 | 5 | export default async function getBottom( 6 | simpleTable: SimpleTable, 7 | count: number, 8 | options: { 9 | originalOrder?: boolean; 10 | conditions?: string; 11 | } = {}, 12 | ) { 13 | const queryResult = await queryDB( 14 | simpleTable, 15 | `WITH numberedRowsForGetBottom AS ( 16 | SELECT *, row_number() OVER () as rowNumberForGetBottom FROM ${simpleTable.name}${ 17 | options.conditions ? ` WHERE ${options.conditions}` : "" 18 | } 19 | ) 20 | SELECT * FROM numberedRowsForGetBottom ORDER BY rowNumberForGetBottom DESC LIMIT ${count};`, 21 | mergeOptions(simpleTable, { 22 | table: simpleTable.name, 23 | returnDataFrom: "query", 24 | method: "getBottom()", 25 | parameters: { count, options }, 26 | }), 27 | ); 28 | 29 | if (!queryResult) { 30 | throw new Error("No queryResult"); 31 | } 32 | 33 | const rowsRaw = queryResult.map((d) => { 34 | delete d.rowNumberForGetBottom; 35 | return d; 36 | }); 37 | const rows = options.originalOrder ? rowsRaw.reverse() : rowsRaw; 38 | 39 | return rows; 40 | } 41 | -------------------------------------------------------------------------------- /src/methods/getColumns.ts: -------------------------------------------------------------------------------- 1 | import mergeOptions from "../helpers/mergeOptions.ts"; 2 | import queryDB from "../helpers/queryDB.ts"; 3 | import type SimpleTable from "../class/SimpleTable.ts"; 4 | 5 | export default async function getColumns(simpleTable: SimpleTable) { 6 | const queryResult = await queryDB( 7 | simpleTable, 8 | `DESCRIBE ${simpleTable.name}`, 9 | mergeOptions(simpleTable, { 10 | table: simpleTable.name, 11 | returnDataFrom: "query", 12 | method: "getColumns()", 13 | parameters: {}, 14 | }), 15 | ); 16 | 17 | if (!queryResult) { 18 | throw new Error("No result"); 19 | } 20 | 21 | const columns = queryResult.map((d) => d.column_name) as string[]; 22 | 23 | return columns; 24 | } 25 | -------------------------------------------------------------------------------- /src/methods/getDescription.ts: -------------------------------------------------------------------------------- 1 | import type SimpleTable from "../class/SimpleTable.ts"; 2 | 3 | export default async function getDescription(simpleTable: SimpleTable) { 4 | const types = await simpleTable.getTypes(); 5 | const columns = await simpleTable.getColumns(); 6 | const summaryForGetDescription = await simpleTable.summarize({ 7 | values: columns, 8 | summaries: ["count", "countUnique", "countNull"], 9 | toMs: true, 10 | outputTable: "summaryForGetDescription", 11 | }); 12 | const summaryData = await summaryForGetDescription.getData(); 13 | 14 | await summaryForGetDescription.removeTable(); 15 | 16 | const description = summaryData.map((d) => ({ 17 | column: d["value"] as string, 18 | type: types[d["value"] as string], 19 | count: d["count"] as number, 20 | unique: d["countUnique"] as number, 21 | null: d["countNull"] as number, 22 | })); 23 | 24 | return description; 25 | } 26 | -------------------------------------------------------------------------------- /src/methods/getFirstRow.ts: -------------------------------------------------------------------------------- 1 | import mergeOptions from "../helpers/mergeOptions.ts"; 2 | import queryDB from "../helpers/queryDB.ts"; 3 | import type SimpleTable from "../class/SimpleTable.ts"; 4 | 5 | export default async function getFirstRow( 6 | simpleTable: SimpleTable, 7 | options: { 8 | conditions?: string; 9 | } = {}, 10 | ) { 11 | const queryResult = await queryDB( 12 | simpleTable, 13 | `SELECT * FROM ${simpleTable.name}${ 14 | options.conditions ? ` WHERE ${options.conditions}` : "" 15 | } LIMIT 1`, 16 | mergeOptions(simpleTable, { 17 | table: simpleTable.name, 18 | returnDataFrom: "query", 19 | method: "getFirstRow()", 20 | parameters: { options }, 21 | }), 22 | ); 23 | if (!queryResult) { 24 | throw new Error("No queryResult"); 25 | } 26 | 27 | const result = queryResult[0]; 28 | 29 | return result; 30 | } 31 | -------------------------------------------------------------------------------- /src/methods/getGeoData.ts: -------------------------------------------------------------------------------- 1 | import { rewind } from "@nshiab/journalism/web"; 2 | import type SimpleTable from "../class/SimpleTable.ts"; 3 | import mergeOptions from "../helpers/mergeOptions.ts"; 4 | import queryDB from "../helpers/queryDB.ts"; 5 | import shouldFlipBeforeExport from "../helpers/shouldFlipBeforeExport.ts"; 6 | // @deno-types="npm:@types/d3-geo@3" 7 | import type { GeoPermissibleObjects } from "npm:d3-geo@3"; 8 | 9 | export default async function getGeoData( 10 | SimpleTable: SimpleTable, 11 | column: string, 12 | options: { rewind?: boolean } = {}, 13 | ) { 14 | let query = ""; 15 | if (shouldFlipBeforeExport(SimpleTable.projections[column])) { 16 | query = 17 | `SELECT * EXCLUDE ${column}, ST_AsGeoJSON(ST_FlipCoordinates(${column})) as geoJsonFragment from ${SimpleTable.name};`; 18 | } else { 19 | query = 20 | `SELECT * EXCLUDE ${column}, ST_AsGeoJSON(${column}) as geoJsonFragment from ${SimpleTable.name};`; 21 | } 22 | const queryResult = await queryDB( 23 | SimpleTable, 24 | query, 25 | mergeOptions(SimpleTable, { 26 | table: null, 27 | method: "getGeoData()", 28 | parameters: { column }, 29 | returnDataFrom: "query", 30 | }), 31 | ); 32 | 33 | if (!queryResult) { 34 | throw new Error("No queryResults"); 35 | } 36 | 37 | const features = queryResult.map((d) => { 38 | const { geoJsonFragment, ...properties } = d; 39 | const geometry = JSON.parse(geoJsonFragment as string) as { 40 | "type": string; 41 | "coordinates": unknown[]; 42 | }; 43 | 44 | const feature = { 45 | type: "Feature", 46 | geometry, 47 | properties, 48 | }; 49 | 50 | return feature; 51 | }); 52 | 53 | const geoJSON = { 54 | type: "FeatureCollection", 55 | features, 56 | }; 57 | 58 | return options.rewind 59 | ? rewind(geoJSON as GeoPermissibleObjects) as { 60 | type: string; 61 | features: { 62 | type: string; 63 | geometry: { 64 | type: string; 65 | coordinates: unknown[]; 66 | }; 67 | properties: { 68 | [key: string]: string | number | boolean | Date | null; 69 | }; 70 | }[]; 71 | } 72 | : geoJSON; 73 | } 74 | -------------------------------------------------------------------------------- /src/methods/getLastRow.ts: -------------------------------------------------------------------------------- 1 | import mergeOptions from "../helpers/mergeOptions.ts"; 2 | import queryDB from "../helpers/queryDB.ts"; 3 | import type SimpleTable from "../class/SimpleTable.ts"; 4 | 5 | export default async function getLastRow( 6 | simpleTable: SimpleTable, 7 | options: { 8 | conditions?: string; 9 | } = {}, 10 | ) { 11 | const queryResult = await queryDB( 12 | simpleTable, 13 | `WITH numberedRowsForGetLastRow AS ( 14 | SELECT *, row_number() OVER () as rowNumberForGetLastRow FROM ${simpleTable.name}${ 15 | options.conditions ? ` WHERE ${options.conditions}` : "" 16 | } 17 | ) 18 | SELECT * FROM numberedRowsForGetLastRow ORDER BY rowNumberForGetLastRow DESC LIMIT 1;`, 19 | mergeOptions(simpleTable, { 20 | table: simpleTable.name, 21 | returnDataFrom: "query", 22 | method: "getLastRow()", 23 | parameters: { options }, 24 | }), 25 | ); 26 | if (!queryResult) { 27 | throw new Error("No queryResult"); 28 | } 29 | const result = queryResult[0]; 30 | delete result.rowNumberForGetLastRow; 31 | 32 | return result; 33 | } 34 | -------------------------------------------------------------------------------- /src/methods/getMax.ts: -------------------------------------------------------------------------------- 1 | import mergeOptions from "../helpers/mergeOptions.ts"; 2 | import queryDB from "../helpers/queryDB.ts"; 3 | import type SimpleTable from "../class/SimpleTable.ts"; 4 | 5 | export default async function getMax( 6 | simpleTable: SimpleTable, 7 | column: string, 8 | ) { 9 | const queryResult = await queryDB( 10 | simpleTable, 11 | `SELECT MAX("${column}") AS "${column}" FROM ${simpleTable.name}`, 12 | { 13 | ...mergeOptions(simpleTable, { 14 | table: simpleTable.name, 15 | returnDataFrom: "query", 16 | method: "getMax()", 17 | parameters: { column }, 18 | }), 19 | types: await simpleTable.getTypes(), 20 | }, 21 | ); 22 | 23 | if (!queryResult) { 24 | throw new Error("No queryResults"); 25 | } 26 | 27 | const result = queryResult[0][column]; 28 | 29 | return result; 30 | } 31 | -------------------------------------------------------------------------------- /src/methods/getMean.ts: -------------------------------------------------------------------------------- 1 | import mergeOptions from "../helpers/mergeOptions.ts"; 2 | import queryDB from "../helpers/queryDB.ts"; 3 | import type SimpleTable from "../class/SimpleTable.ts"; 4 | 5 | export default async function getMean( 6 | simpleTable: SimpleTable, 7 | column: string, 8 | options: { 9 | decimals?: number; 10 | } = {}, 11 | ) { 12 | const queryResult = await queryDB( 13 | simpleTable, 14 | typeof options.decimals === "number" 15 | ? `SELECT ROUND(AVG("${column}"), ${options.decimals}) AS "${column}" FROM ${simpleTable.name}` 16 | : `SELECT AVG("${column}") AS "${column}" FROM ${simpleTable.name}`, 17 | mergeOptions(simpleTable, { 18 | table: simpleTable.name, 19 | returnDataFrom: "query", 20 | method: "getMean()", 21 | parameters: { column, options }, 22 | }), 23 | ); 24 | 25 | if (!queryResult) { 26 | throw new Error("No queryResults"); 27 | } 28 | 29 | const result = queryResult[0][column]; 30 | 31 | return result as number; 32 | } 33 | -------------------------------------------------------------------------------- /src/methods/getMedian.ts: -------------------------------------------------------------------------------- 1 | import mergeOptions from "../helpers/mergeOptions.ts"; 2 | import queryDB from "../helpers/queryDB.ts"; 3 | import type SimpleTable from "../class/SimpleTable.ts"; 4 | 5 | export default async function getMedian( 6 | SimpleTable: SimpleTable, 7 | column: string, 8 | options: { 9 | decimals?: number; 10 | } = {}, 11 | ) { 12 | const queryResult = await queryDB( 13 | SimpleTable, 14 | typeof options.decimals === "number" 15 | ? `SELECT ROUND(MEDIAN("${column}"), ${options.decimals}) AS "${column}" FROM ${SimpleTable.name}` 16 | : `SELECT MEDIAN("${column}") AS "${column}" FROM ${SimpleTable.name}`, 17 | mergeOptions(SimpleTable, { 18 | table: SimpleTable.name, 19 | returnDataFrom: "query", 20 | method: "getMedian()", 21 | parameters: { column, options }, 22 | }), 23 | ); 24 | 25 | if (!queryResult) { 26 | throw new Error("No queryResults"); 27 | } 28 | const result = queryResult[0][column]; 29 | 30 | return result as number; 31 | } 32 | -------------------------------------------------------------------------------- /src/methods/getMin.ts: -------------------------------------------------------------------------------- 1 | import mergeOptions from "../helpers/mergeOptions.ts"; 2 | import queryDB from "../helpers/queryDB.ts"; 3 | import type SimpleTable from "../class/SimpleTable.ts"; 4 | 5 | export default async function getMin( 6 | simpleTable: SimpleTable, 7 | column: string, 8 | ) { 9 | const queryResult = await queryDB( 10 | simpleTable, 11 | `SELECT MIN("${column}") AS "${column}" FROM ${simpleTable.name}`, 12 | { 13 | ...mergeOptions(simpleTable, { 14 | table: simpleTable.name, 15 | returnDataFrom: "query", 16 | method: "getMin()", 17 | parameters: { column }, 18 | }), 19 | types: await simpleTable.getTypes(), 20 | }, 21 | ); 22 | 23 | if (!queryResult) { 24 | throw new Error("No queryResults"); 25 | } 26 | const result = queryResult[0][column]; 27 | 28 | return result; 29 | } 30 | -------------------------------------------------------------------------------- /src/methods/getNbRows.ts: -------------------------------------------------------------------------------- 1 | import mergeOptions from "../helpers/mergeOptions.ts"; 2 | import queryDB from "../helpers/queryDB.ts"; 3 | import type SimpleTable from "../class/SimpleTable.ts"; 4 | 5 | export default async function getNbRows(SimpleTable: SimpleTable) { 6 | const queryResult = await queryDB( 7 | SimpleTable, 8 | `SELECT CAST(COUNT(*) AS INTEGER) FROM ${SimpleTable.name}`, 9 | mergeOptions(SimpleTable, { 10 | table: SimpleTable.name, 11 | returnDataFrom: "query", 12 | method: "getLength()", 13 | parameters: {}, 14 | }), 15 | ); 16 | 17 | if (!queryResult) { 18 | throw new Error("No result"); 19 | } 20 | const length = queryResult[0]["CAST(count_star() AS INTEGER)"] as number; 21 | 22 | return length; 23 | } 24 | -------------------------------------------------------------------------------- /src/methods/getQuantile.ts: -------------------------------------------------------------------------------- 1 | import mergeOptions from "../helpers/mergeOptions.ts"; 2 | import queryDB from "../helpers/queryDB.ts"; 3 | import type SimpleTable from "../class/SimpleTable.ts"; 4 | 5 | export default async function getQuantile( 6 | SimpleTable: SimpleTable, 7 | column: string, 8 | quantile: number, 9 | options: { 10 | decimals?: number; 11 | } = {}, 12 | ) { 13 | const queryResult = await queryDB( 14 | SimpleTable, 15 | typeof options.decimals === "number" 16 | ? `SELECT ROUND(QUANTILE_CONT("${column}", ${quantile}), ${options.decimals}) AS "${column}" FROM ${SimpleTable.name}` 17 | : `SELECT QUANTILE_CONT("${column}", ${quantile}) AS "${column}" FROM ${SimpleTable.name}`, 18 | mergeOptions(SimpleTable, { 19 | table: SimpleTable.name, 20 | returnDataFrom: "query", 21 | method: "getQuantile()", 22 | parameters: { column, quantile, options }, 23 | }), 24 | ); 25 | 26 | if (!queryResult) { 27 | throw new Error("No queryResults"); 28 | } 29 | 30 | const result = queryResult[0][column]; 31 | return result as number; 32 | } 33 | -------------------------------------------------------------------------------- /src/methods/getSkew.ts: -------------------------------------------------------------------------------- 1 | import mergeOptions from "../helpers/mergeOptions.ts"; 2 | import queryDB from "../helpers/queryDB.ts"; 3 | import type SimpleTable from "../class/SimpleTable.ts"; 4 | 5 | export default async function getSkew( 6 | SimpleTable: SimpleTable, 7 | column: string, 8 | options: { 9 | decimals?: number; 10 | } = {}, 11 | ) { 12 | const queryResult = await queryDB( 13 | SimpleTable, 14 | typeof options.decimals === "number" 15 | ? `SELECT ROUND(SKEWNESS("${column}"), ${options.decimals}) AS "${column}" FROM ${SimpleTable.name}` 16 | : `SELECT SKEWNESS("${column}") AS "${column}" FROM ${SimpleTable.name}`, 17 | mergeOptions(SimpleTable, { 18 | table: SimpleTable.name, 19 | returnDataFrom: "query", 20 | method: "getSkew()", 21 | parameters: { column, options }, 22 | }), 23 | ); 24 | 25 | if (!queryResult) { 26 | throw new Error("No queryResults"); 27 | } 28 | 29 | const result = queryResult[0][column]; 30 | return result as number; 31 | } 32 | -------------------------------------------------------------------------------- /src/methods/getStdDev.ts: -------------------------------------------------------------------------------- 1 | import mergeOptions from "../helpers/mergeOptions.ts"; 2 | import queryDB from "../helpers/queryDB.ts"; 3 | import type SimpleTable from "../class/SimpleTable.ts"; 4 | 5 | export default async function getStdDev( 6 | SimpleTable: SimpleTable, 7 | column: string, 8 | options: { 9 | decimals?: number; 10 | } = {}, 11 | ) { 12 | const queryResult = await queryDB( 13 | SimpleTable, 14 | typeof options.decimals === "number" 15 | ? `SELECT ROUND(STDDEV("${column}"), ${options.decimals}) AS "${column}" FROM ${SimpleTable.name}` 16 | : `SELECT STDDEV("${column}") AS "${column}" FROM ${SimpleTable.name}`, 17 | mergeOptions(SimpleTable, { 18 | table: SimpleTable.name, 19 | returnDataFrom: "query", 20 | method: "getStdDev()", 21 | parameters: { column, options }, 22 | }), 23 | ); 24 | 25 | if (!queryResult) { 26 | throw new Error("No queryResults"); 27 | } 28 | 29 | const result = queryResult[0][column]; 30 | return result as number; 31 | } 32 | -------------------------------------------------------------------------------- /src/methods/getSum.ts: -------------------------------------------------------------------------------- 1 | import mergeOptions from "../helpers/mergeOptions.ts"; 2 | import queryDB from "../helpers/queryDB.ts"; 3 | import type SimpleTable from "../class/SimpleTable.ts"; 4 | 5 | export default async function getSum( 6 | SimpleTable: SimpleTable, 7 | column: string, 8 | ) { 9 | const queryResult = await queryDB( 10 | SimpleTable, 11 | `SELECT SUM("${column}") AS "${column}" FROM ${SimpleTable.name}`, 12 | mergeOptions(SimpleTable, { 13 | table: SimpleTable.name, 14 | returnDataFrom: "query", 15 | method: "getSum()", 16 | parameters: { column }, 17 | }), 18 | ); 19 | 20 | if (!queryResult) { 21 | throw new Error("No queryResults"); 22 | } 23 | 24 | const result = queryResult[0][column]; 25 | 26 | return result as number; 27 | } 28 | -------------------------------------------------------------------------------- /src/methods/getTableNames.ts: -------------------------------------------------------------------------------- 1 | import mergeOptions from "../helpers/mergeOptions.ts"; 2 | import queryDB from "../helpers/queryDB.ts"; 3 | import type SimpleDB from "../class/SimpleDB.ts"; 4 | 5 | export default async function getTableNames(simpleDB: SimpleDB) { 6 | const queryResult = await queryDB( 7 | simpleDB, 8 | `SHOW TABLES`, 9 | mergeOptions(simpleDB, { 10 | returnDataFrom: "query", 11 | table: null, 12 | method: "getTables", 13 | parameters: {}, 14 | }), 15 | ); 16 | 17 | if (!queryResult) { 18 | throw new Error("No result"); 19 | } 20 | 21 | const tables = queryResult.map((d) => d.name) as string[]; 22 | 23 | return tables; 24 | } 25 | -------------------------------------------------------------------------------- /src/methods/getTop.ts: -------------------------------------------------------------------------------- 1 | import mergeOptions from "../helpers/mergeOptions.ts"; 2 | import queryDB from "../helpers/queryDB.ts"; 3 | import type SimpleTable from "../class/SimpleTable.ts"; 4 | 5 | export default async function getTop( 6 | simpleTable: SimpleTable, 7 | count: number, 8 | options: { 9 | conditions?: string; 10 | } = {}, 11 | ) { 12 | const rows = await queryDB( 13 | simpleTable, 14 | `SELECT * FROM ${simpleTable.name}${ 15 | options.conditions ? ` WHERE ${options.conditions}` : "" 16 | } LIMIT ${count}`, 17 | mergeOptions(simpleTable, { 18 | table: simpleTable.name, 19 | returnDataFrom: "query", 20 | method: "getTop()", 21 | parameters: { count, options }, 22 | }), 23 | ); 24 | 25 | if (!rows) { 26 | throw new Error("no rows"); 27 | } 28 | 29 | return rows; 30 | } 31 | -------------------------------------------------------------------------------- /src/methods/getType.ts: -------------------------------------------------------------------------------- 1 | export default function getType(value: unknown) { 2 | if (value instanceof Date) { 3 | return "TIMESTAMP"; 4 | } else if (typeof value === "bigint" || Number.isInteger(value)) { 5 | return "BIGINT"; 6 | } else if (typeof value === "number") { 7 | return "DOUBLE"; 8 | } else if (typeof value === "string") { 9 | return "VARCHAR"; 10 | } else if (typeof value === "boolean") { 11 | return "BOOLEAN"; 12 | } else { 13 | throw new Error( 14 | `Unkown type ${typeof value} for ${value}. Using first item in array to set the column types.`, 15 | ); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/methods/getTypes.ts: -------------------------------------------------------------------------------- 1 | import mergeOptions from "../helpers/mergeOptions.ts"; 2 | import queryDB from "../helpers/queryDB.ts"; 3 | import type SimpleTable from "../class/SimpleTable.ts"; 4 | import extractTypes from "../helpers/extractTypes.ts"; 5 | 6 | export default async function getTypes(simpleTable: SimpleTable) { 7 | const types = await queryDB( 8 | simpleTable, 9 | `DESCRIBE "${simpleTable.name}"`, 10 | mergeOptions(simpleTable, { 11 | table: simpleTable.name, 12 | returnDataFrom: "query", 13 | method: "getTypes()", 14 | parameters: {}, 15 | }), 16 | ); 17 | 18 | const typesObj = extractTypes(types); 19 | 20 | return typesObj; 21 | } 22 | -------------------------------------------------------------------------------- /src/methods/getUniques.ts: -------------------------------------------------------------------------------- 1 | import mergeOptions from "../helpers/mergeOptions.ts"; 2 | import queryDB from "../helpers/queryDB.ts"; 3 | import type SimpleTable from "../class/SimpleTable.ts"; 4 | 5 | export default async function getUniques( 6 | simpleTable: SimpleTable, 7 | column: string, 8 | ) { 9 | const queryResult = await queryDB( 10 | simpleTable, 11 | `SELECT DISTINCT "${column}" FROM ${simpleTable.name} ORDER BY "${column}" ASC`, 12 | mergeOptions(simpleTable, { 13 | table: simpleTable.name, 14 | returnDataFrom: "query", 15 | method: "getUniques()", 16 | parameters: { column }, 17 | }), 18 | ); 19 | 20 | if (!queryResult) { 21 | throw new Error("No result."); 22 | } 23 | 24 | const uniques = queryResult.map((d) => d[column]); 25 | 26 | return uniques; 27 | } 28 | -------------------------------------------------------------------------------- /src/methods/getValues.ts: -------------------------------------------------------------------------------- 1 | import mergeOptions from "../helpers/mergeOptions.ts"; 2 | import queryDB from "../helpers/queryDB.ts"; 3 | import type SimpleTable from "../class/SimpleTable.ts"; 4 | 5 | export default async function getValues( 6 | simpleTable: SimpleTable, 7 | column: string, 8 | ) { 9 | const queryResult = await queryDB( 10 | simpleTable, 11 | `SELECT "${column}" FROM ${simpleTable.name}`, 12 | mergeOptions(simpleTable, { 13 | table: simpleTable.name, 14 | returnDataFrom: "query", 15 | method: "getValues()", 16 | parameters: { column }, 17 | }), 18 | ); 19 | if (!queryResult) { 20 | throw new Error("No result"); 21 | } 22 | 23 | const values = queryResult.map((d) => d[column]); 24 | 25 | return values; 26 | } 27 | -------------------------------------------------------------------------------- /src/methods/getVar.ts: -------------------------------------------------------------------------------- 1 | import mergeOptions from "../helpers/mergeOptions.ts"; 2 | import queryDB from "../helpers/queryDB.ts"; 3 | import type SimpleTable from "../class/SimpleTable.ts"; 4 | 5 | export default async function getVar( 6 | SimpleTable: SimpleTable, 7 | column: string, 8 | options: { 9 | decimals?: number; 10 | } = {}, 11 | ) { 12 | const queryResult = await queryDB( 13 | SimpleTable, 14 | typeof options.decimals === "number" 15 | ? `SELECT ROUND(VARIANCE("${column}"), ${options.decimals}) AS "${column}" FROM ${SimpleTable.name}` 16 | : `SELECT VARIANCE("${column}") AS "${column}" FROM ${SimpleTable.name}`, 17 | mergeOptions(SimpleTable, { 18 | table: SimpleTable.name, 19 | returnDataFrom: "query", 20 | method: "getVar()", 21 | parameters: { column, options }, 22 | }), 23 | ); 24 | 25 | if (!queryResult) { 26 | throw new Error("No queryResults"); 27 | } 28 | 29 | const result = queryResult[0][column]; 30 | 31 | return result as number; 32 | } 33 | -------------------------------------------------------------------------------- /src/methods/insertRowsQuery.ts: -------------------------------------------------------------------------------- 1 | import parseValue from "../helpers/parseValue.ts"; 2 | 3 | export default function insertRowsQuery( 4 | table: string, 5 | rows: { [key: string]: unknown }[], 6 | ) { 7 | const columns = Object.keys(rows[0]); 8 | 9 | let query = `INSERT INTO "${table}" (${columns.map((d) => `${d}`).join(", ")}) 10 | VALUES`; 11 | 12 | for (const row of rows) { 13 | const values = Object.values(row); 14 | query += `\n(${values.map((d) => parseValue(d)).join(", ")}),`; 15 | } 16 | 17 | return query.slice(0, query.length - 1); 18 | } 19 | -------------------------------------------------------------------------------- /src/methods/joinGeoQuery.ts: -------------------------------------------------------------------------------- 1 | export default function joinGeoQuery( 2 | leftTable: string, 3 | leftTableColumn: string, 4 | method: "intersect" | "inside" | "within", 5 | rightTable: string, 6 | rightTableColumn: string, 7 | join: "inner" | "left" | "right" | "full", 8 | outputTable: string, 9 | distance: number | undefined, 10 | distanceMethod: "srs" | "haversine" | "spheroid" | undefined, 11 | ) { 12 | let query = `CREATE OR REPLACE TABLE ${outputTable} AS SELECT *`; 13 | if (join === "inner") { 14 | query += ` FROM ${leftTable} JOIN ${rightTable}`; 15 | } else if (join === "left") { 16 | query += ` FROM ${leftTable} LEFT JOIN ${rightTable}`; 17 | } else if (join === "right") { 18 | query += ` FROM ${leftTable} RIGHT JOIN ${rightTable}`; 19 | } else if (join === "full") { 20 | query += ` FROM ${leftTable} FULL JOIN ${rightTable}`; 21 | } else { 22 | throw new Error(`Unknown ${join} join.`); 23 | } 24 | 25 | if (method === "intersect") { 26 | query += 27 | ` ON ST_Intersects(${leftTable}.${leftTableColumn}, ${rightTable}.${rightTableColumn});`; 28 | } else if (method === "inside") { 29 | // Order is important 30 | query += 31 | ` ON ST_Covers(${rightTable}.${rightTableColumn}, ${leftTable}.${leftTableColumn});`; 32 | } else if (method === "within") { 33 | if (typeof distance === "number") { 34 | if (distanceMethod === undefined || distanceMethod === "srs") { 35 | query += 36 | ` ON ST_DWithin(${leftTable}."${leftTableColumn}", ${rightTable}."${rightTableColumn}", ${distance})`; 37 | } else if (distanceMethod === "haversine") { 38 | // Maybe ST_DWithin_Sphere will be available soon? 39 | query += 40 | ` ON ST_Distance_Sphere(${leftTable}."${leftTableColumn}", ${rightTable}."${rightTableColumn}") < ${distance}`; 41 | } else if (distanceMethod === "spheroid") { 42 | // Should be using ST_DWithin_Spheroid but doesn't work? 43 | query += 44 | ` ON ST_Distance_Spheroid(${leftTable}."${leftTableColumn}", ${rightTable}."${rightTableColumn}") < ${distance}`; 45 | } else { 46 | throw new Error(`Unknown ${distanceMethod}`); 47 | } 48 | } else { 49 | throw new Error("options.distance must be a number"); 50 | } 51 | } else { 52 | throw new Error(`Unknown ${method} method`); 53 | } 54 | 55 | return query; 56 | } 57 | -------------------------------------------------------------------------------- /src/methods/joinQuery.ts: -------------------------------------------------------------------------------- 1 | export default function joinQuery( 2 | leftTable: string, 3 | rightTable: string, 4 | commonColumn: string[], 5 | join: "inner" | "left" | "right" | "full", 6 | outputTable: string, 7 | ) { 8 | let query = `CREATE OR REPLACE TABLE ${outputTable} AS SELECT *`; 9 | 10 | if (join === "inner") { 11 | query += ` FROM ${leftTable} JOIN ${rightTable}`; 12 | } else if (join === "left") { 13 | query += ` FROM ${leftTable} LEFT JOIN ${rightTable}`; 14 | } else if (join === "right") { 15 | query += ` FROM ${leftTable} RIGHT JOIN ${rightTable}`; 16 | } else if (join === "full") { 17 | query += ` FROM ${leftTable} FULL JOIN ${rightTable}`; 18 | } else { 19 | throw new Error(`Unknown ${join} join.`); 20 | } 21 | 22 | query += ` ON (${ 23 | commonColumn.map((d) => `${leftTable}."${d}" = ${rightTable}."${d}"`).join( 24 | " AND ", 25 | ) 26 | });\n`; 27 | 28 | return query; 29 | } 30 | -------------------------------------------------------------------------------- /src/methods/keepQuery.ts: -------------------------------------------------------------------------------- 1 | import parseValue from "../helpers/parseValue.ts"; 2 | 3 | export default function keepQuery( 4 | table: string, 5 | columnsAndValues: { 6 | [key: string]: 7 | | (number | string | Date | boolean | null)[] 8 | | (number | string | Date | boolean | null); 9 | }, 10 | ) { 11 | let query = 12 | `CREATE OR REPLACE TABLE "${table}" AS SELECT * FROM "${table}" WHERE\n`; 13 | const columns = Object.keys(columnsAndValues); 14 | 15 | const conditions = []; 16 | for (const column of columns) { 17 | let values = columnsAndValues[column]; 18 | if (!Array.isArray(values)) { 19 | values = [values]; 20 | } 21 | 22 | conditions.push( 23 | `"${column}" IN (${values.map((d) => parseValue(d)).join(", ")})`, 24 | ); 25 | } 26 | 27 | query += conditions.join("\nAND "); 28 | 29 | return query; 30 | } 31 | -------------------------------------------------------------------------------- /src/methods/linearRegressionQuery.ts: -------------------------------------------------------------------------------- 1 | import stringToArray from "../helpers/stringToArray.ts"; 2 | 3 | export default function linearRegressionQuery( 4 | table: string, 5 | outputTable: string, 6 | permutations: [string, string][], 7 | options: { 8 | categories?: string | string[]; 9 | decimals?: number; 10 | }, 11 | ) { 12 | let query = `CREATE OR REPLACE TABLE ${outputTable} AS`; 13 | 14 | const categories = options.categories 15 | ? stringToArray(options.categories) 16 | : []; 17 | 18 | const groupBy = categories.length === 0 19 | ? "" 20 | : ` GROUP BY ${categories.map((d) => `${d}`).join(",")}`; 21 | 22 | let firstValue = true; 23 | for (const perm of permutations) { 24 | if (firstValue) { 25 | firstValue = false; 26 | } else { 27 | query += "\nUNION"; 28 | } 29 | 30 | let tempSlop; 31 | let tempIntercept; 32 | let tempR2; 33 | if (typeof options.decimals === "number") { 34 | tempSlop = `ROUND(REGR_SLOPE(${perm[1]}, ${ 35 | perm[0] 36 | }), ${options.decimals})`; 37 | tempIntercept = `ROUND(REGR_INTERCEPT(${perm[1]}, ${ 38 | perm[0] 39 | }), ${options.decimals})`; 40 | tempR2 = `ROUND(REGR_R2(${perm[1]}, ${perm[0]}), ${options.decimals})`; 41 | } else { 42 | tempSlop = `REGR_SLOPE(${perm[1]}, ${perm[0]})`; 43 | tempIntercept = `REGR_INTERCEPT(${perm[1]}, ${perm[0]})`; 44 | tempR2 = `REGR_R2(${perm[1]}, ${perm[0]})`; 45 | } 46 | 47 | query += `\nSELECT ${ 48 | categories.length > 0 49 | ? `${categories.map((d) => `${d}`).join(",")}, ` 50 | : "" 51 | }'${perm[0]}' AS x, '${ 52 | perm[1] 53 | }' AS y, ${tempSlop} AS slope, ${tempIntercept} AS yIntercept, ${tempR2} as r2 54 | FROM "${table}"${groupBy}`; 55 | } 56 | 57 | return query; 58 | } 59 | -------------------------------------------------------------------------------- /src/methods/linearRegressions.ts: -------------------------------------------------------------------------------- 1 | import getCombinations from "../helpers/getCombinations.ts"; 2 | import keepNumericalColumns from "../helpers/keepNumericalColumns.ts"; 3 | import mergeOptions from "../helpers/mergeOptions.ts"; 4 | import queryDB from "../helpers/queryDB.ts"; 5 | import linearRegressionQuery from "./linearRegressionQuery.ts"; 6 | import type SimpleTable from "../class/SimpleTable.ts"; 7 | 8 | export default async function linearRegressions( 9 | SimpleTable: SimpleTable, 10 | options: { 11 | x?: string; 12 | y?: string; 13 | categories?: string | string[]; 14 | decimals?: number; 15 | outputTable?: string | boolean; 16 | } = {}, 17 | ) { 18 | const outputTable = typeof options.outputTable === "string" 19 | ? options.outputTable 20 | : SimpleTable.name; 21 | 22 | const permutations: [string, string][] = []; 23 | if (!options.x && !options.y) { 24 | const types = await SimpleTable.getTypes(); 25 | const columns = keepNumericalColumns(types); 26 | const combinations = getCombinations(columns, 2); 27 | for (const c of combinations) { 28 | permutations.push(c); 29 | permutations.push([c[1], c[0]]); 30 | } 31 | } else if (options.x && !options.y) { 32 | const types = await SimpleTable.getTypes(); 33 | const columns = keepNumericalColumns(types); 34 | for (const col of columns) { 35 | if (col !== options.x) { 36 | permutations.push([options.x, col]); 37 | } 38 | } 39 | } else if (options.x && options.y) { 40 | permutations.push([options.x, options.y]); 41 | } else { 42 | throw new Error("No combinations of x and y"); 43 | } 44 | 45 | await queryDB( 46 | SimpleTable, 47 | linearRegressionQuery( 48 | SimpleTable.name, 49 | outputTable, 50 | permutations, 51 | options, 52 | ), 53 | mergeOptions(SimpleTable, { 54 | table: outputTable, 55 | method: "linearRegressions()", 56 | parameters: { 57 | options, 58 | "permutations (computed)": permutations, 59 | }, 60 | }), 61 | ); 62 | } 63 | -------------------------------------------------------------------------------- /src/methods/logHistogram.ts: -------------------------------------------------------------------------------- 1 | import { logBarChart } from "jsr:@nshiab/journalism@1"; 2 | import type SimpleTable from "../class/SimpleTable.ts"; 3 | 4 | export default async function logHistogram( 5 | simpleTable: SimpleTable, 6 | values: string, 7 | options: { 8 | bins?: number; 9 | formatLabels?: (a: number, b: number) => string; 10 | compact?: boolean; 11 | width?: number; 12 | } = {}, 13 | ) { 14 | const bins = options.bins ?? 10; 15 | const formatLabels = options.formatLabels ?? ((a, b) => `[${a} | ${b})`); 16 | 17 | const data = await simpleTable.sdb.customQuery( 18 | ` 19 | WITH params AS ( 20 | SELECT 21 | ${bins} AS N_BINS, 22 | min("${values}") AS min_distance, 23 | max("${values}") AS max_distance, 24 | (max("${values}") - min("${values}")) / ${bins} AS bin_size 25 | FROM ${simpleTable.name} 26 | ), 27 | histogram AS ( 28 | SELECT 29 | floor(("${values}" - min_distance) / bin_size) AS bin_number, 30 | min_distance + floor(("${values}" - min_distance) / bin_size) * bin_size AS bin_start, 31 | min_distance + (floor(("${values}" - min_distance) / bin_size) + 1) * bin_size AS bin_end, 32 | CAST(count(*) AS INTEGER) AS frequency 33 | FROM ${simpleTable.name}, params 34 | WHERE "${values}" >= min_distance AND "${values}" < max_distance 35 | GROUP BY 1, 2, 3 36 | ) 37 | SELECT 38 | bin_start, 39 | bin_end, 40 | frequency 41 | FROM histogram 42 | ORDER BY bin_start;`, 43 | { returnDataFrom: "query" }, 44 | ); 45 | 46 | logBarChart( 47 | (data as { [key: string]: unknown }[]).map((d) => ({ 48 | binRange: formatLabels( 49 | parseFloat((d.bin_start as number).toFixed(10)), 50 | parseFloat((d.bin_end as number).toFixed(10)), 51 | ), 52 | frequency: d.frequency, 53 | })), 54 | "binRange", 55 | "frequency", 56 | { 57 | title: `Distribution of "${values}"`, 58 | totalLabel: "Number of data points", 59 | compact: options.compact, 60 | width: options.width, 61 | }, 62 | ); 63 | } 64 | -------------------------------------------------------------------------------- /src/methods/lowerQuery.ts: -------------------------------------------------------------------------------- 1 | export default function lowerQuery(table: string, columns: string[]) { 2 | let query = ""; 3 | 4 | for (const column of columns) { 5 | query += `\nUPDATE "${table}" SET "${column}" = LOWER("${column}");`; 6 | } 7 | 8 | return query; 9 | } 10 | -------------------------------------------------------------------------------- /src/methods/normalizeQuery.ts: -------------------------------------------------------------------------------- 1 | import stringToArray from "../helpers/stringToArray.ts"; 2 | 3 | export default function normalizeQuery( 4 | table: string, 5 | column: string, 6 | newColumn: string, 7 | options: { 8 | categories?: string | string[]; 9 | decimals?: number; 10 | } = {}, 11 | ) { 12 | const categories = options.categories 13 | ? stringToArray(options.categories) 14 | : []; 15 | const partition = categories.length > 0 16 | ? `PARTITION BY ${categories.map((d) => `${d}`).join(", ")}` 17 | : ""; 18 | 19 | const tempQuery = `(${column} - MIN(${column}) OVER(${partition})) 20 | / 21 | (MAX(${column}) OVER(${partition}) - MIN(${column}) OVER(${partition}))`; 22 | 23 | const query = ` 24 | CREATE OR REPLACE TABLE "${table}" AS 25 | SELECT *, ( 26 | ${ 27 | typeof options.decimals === "number" 28 | ? `ROUND(${tempQuery}, ${options.decimals})` 29 | : tempQuery 30 | } 31 | ) AS ${newColumn}, 32 | FROM "${table}" 33 | `; 34 | 35 | return query; 36 | } 37 | -------------------------------------------------------------------------------- /src/methods/outliersIQRQuery.ts: -------------------------------------------------------------------------------- 1 | import stringToArray from "../helpers/stringToArray.ts"; 2 | 3 | export default function outliersIQRQuery( 4 | table: string, 5 | column: string, 6 | newColumn: string, 7 | parity: "even" | "odd", 8 | options: { 9 | categories?: string | string[]; 10 | } = {}, 11 | ) { 12 | const categories = options.categories 13 | ? stringToArray(options.categories).map((d) => `${d}`) 14 | : []; 15 | 16 | const quantileFunc = parity === "even" ? "quantile_disc" : "quantile_cont"; 17 | 18 | const where = categories.length > 0 19 | ? `WHERE ${ 20 | categories 21 | .map((d) => `"${table}".${d} = iqr.${d}`) 22 | .join(" AND ") 23 | }` 24 | : ""; 25 | 26 | const query = `ALTER TABLE "${table}" 27 | ADD COLUMN ${newColumn} BOOLEAN; 28 | WITH iqr AS ( 29 | SELECT${categories.length > 0 ? `\n${categories},` : ""} 30 | ${quantileFunc}(${column}, 0.25) as q1, 31 | ${quantileFunc}(${column}, 0.75) as q3, 32 | (q3-q1)*1.5 as range, 33 | q1-range as lowThreshold, 34 | q3+range as highThreshold 35 | FROM "${table}" 36 | ${categories.length > 0 ? `GROUP BY ${categories}` : ""} 37 | ) 38 | UPDATE "${table}" 39 | SET ${newColumn} = CASE 40 | WHEN ${column} > (SELECT highThreshold FROM iqr ${where}) OR ${column} < (SELECT lowThreshold FROM iqr ${where}) THEN TRUE 41 | ELSE FALSE 42 | END; 43 | `; 44 | 45 | return query; 46 | } 47 | -------------------------------------------------------------------------------- /src/methods/proportionsHorizontalQuery.ts: -------------------------------------------------------------------------------- 1 | export default function proportionsHorizontalQuery( 2 | table: string, 3 | columns: string[], 4 | options: { 5 | suffix?: string; 6 | decimals?: number; 7 | } = {}, 8 | ) { 9 | let query = `CREATE OR REPLACE TABLE "${table}" AS SELECT *,`; 10 | 11 | for (const col of columns) { 12 | const tempQuery = `${col} / (${columns.map((d) => `${d}`).join(" + ")})`; 13 | if (typeof options.decimals === "number") { 14 | query += ` ROUND(${tempQuery}, ${options.decimals})`; 15 | } else { 16 | query += ` ${tempQuery}`; 17 | } 18 | query += ` AS ${col}${options.suffix ?? "Perc"},`; 19 | } 20 | 21 | query += `FROM "${table}"`; 22 | 23 | return query; 24 | } 25 | -------------------------------------------------------------------------------- /src/methods/proportionsVerticalQuery.ts: -------------------------------------------------------------------------------- 1 | import stringToArray from "../helpers/stringToArray.ts"; 2 | 3 | export default function proportionsVerticalQuery( 4 | table: string, 5 | column: string, 6 | newColumn: string, 7 | options: { 8 | categories?: string | string[]; 9 | decimals?: number; 10 | } = {}, 11 | ) { 12 | const categories = options.categories 13 | ? stringToArray(options.categories) 14 | : []; 15 | 16 | const partition = categories.length === 0 17 | ? "" 18 | : `PARTITION BY ${categories.map((d) => `${d}`).join(",")}`; 19 | 20 | let query = ""; 21 | if (typeof options.decimals === "number") { 22 | query = 23 | `CREATE OR REPLACE TABLE "${table}" AS SELECT *, ROUND(${column} / sum(${column}) OVER(${partition}), ${options.decimals}) AS ${newColumn} FROM "${table}"`; 24 | } else { 25 | query = 26 | `CREATE OR REPLACE TABLE "${table}" AS SELECT *, ${column} / sum(${column}) OVER(${partition}) AS ${newColumn} FROM "${table}"`; 27 | } 28 | 29 | return query; 30 | } 31 | -------------------------------------------------------------------------------- /src/methods/quantilesQuery.ts: -------------------------------------------------------------------------------- 1 | import stringToArray from "../helpers/stringToArray.ts"; 2 | 3 | export default function quantilesQuery( 4 | table: string, 5 | values: string, 6 | nbQuantiles: number, 7 | newColumn: string, 8 | options: { 9 | categories?: string | string[]; 10 | } = {}, 11 | ) { 12 | const categories = options.categories 13 | ? stringToArray(options.categories) 14 | : []; 15 | 16 | const partition = categories.length === 0 17 | ? "" 18 | : `PARTITION BY ${categories.map((d) => `${d}`).join(",")} `; 19 | 20 | const query = 21 | `CREATE OR REPLACE TABLE "${table}" AS SELECT *, ntile(${nbQuantiles}) OVER (${partition}ORDER BY ${values}) AS ${newColumn}, 22 | FROM "${table}"`; 23 | 24 | return query; 25 | } 26 | -------------------------------------------------------------------------------- /src/methods/ranksQuery.ts: -------------------------------------------------------------------------------- 1 | import stringToArray from "../helpers/stringToArray.ts"; 2 | 3 | export default function rankQuery( 4 | table: string, 5 | values: string, 6 | newColumn: string, 7 | options: { 8 | order?: "asc" | "desc"; 9 | categories?: string | string[]; 10 | noGaps?: boolean; 11 | } = {}, 12 | ) { 13 | const categories = options.categories 14 | ? stringToArray(options.categories) 15 | : []; 16 | 17 | const partition = categories.length === 0 18 | ? "" 19 | : `PARTITION BY ${categories.map((d) => `${d}`).join(",")} `; 20 | 21 | const query = `CREATE OR REPLACE TABLE "${table}" AS SELECT *, ${ 22 | options.noGaps ? "dense_rank()" : "rank()" 23 | } OVER (${partition}ORDER BY ${values} ${ 24 | typeof options.order === "string" ? options.order.toUpperCase() : "" 25 | }) AS ${newColumn}, 26 | FROM "${table}"`; 27 | 28 | return query; 29 | } 30 | -------------------------------------------------------------------------------- /src/methods/removeColumnsQuery.ts: -------------------------------------------------------------------------------- 1 | export default function removeColumnsQuery(table: string, columns: string[]) { 2 | let query = `ALTER TABLE "${table}"`; 3 | for (const column of columns) { 4 | query += `\nDROP COLUMN ${column}`; 5 | } 6 | return query; 7 | } 8 | -------------------------------------------------------------------------------- /src/methods/removeDuplicatesQuery.ts: -------------------------------------------------------------------------------- 1 | import stringToArray from "../helpers/stringToArray.ts"; 2 | 3 | export default function removeDuplicatesQuery( 4 | table: string, 5 | options: { 6 | on?: string | string[]; 7 | } = {}, 8 | ) { 9 | const columnsOn = options.on ? stringToArray(options.on) : null; 10 | let distinct; 11 | if (columnsOn) { 12 | distinct = `DISTINCT ON(${columnsOn.join(",")}) *`; 13 | } else { 14 | distinct = "DISTINCT *"; 15 | } 16 | 17 | return `CREATE OR REPLACE TABLE "${table}" AS SELECT ${distinct} FROM "${table}";`; 18 | } 19 | -------------------------------------------------------------------------------- /src/methods/removeMissing.ts: -------------------------------------------------------------------------------- 1 | import mergeOptions from "../helpers/mergeOptions.ts"; 2 | import queryDB from "../helpers/queryDB.ts"; 3 | import stringToArray from "../helpers/stringToArray.ts"; 4 | import removeMissingQuery from "./removeMissingQuery.ts"; 5 | import type SimpleTable from "../class/SimpleTable.ts"; 6 | 7 | export default async function removeMissing( 8 | simpleTable: SimpleTable, 9 | options: { 10 | columns?: string | string[]; 11 | missingValues?: (string | number)[]; 12 | invert?: boolean; 13 | } = {}, 14 | ) { 15 | options.missingValues = options.missingValues ?? [ 16 | "undefined", 17 | "NaN", 18 | "null", 19 | "NULL", 20 | "", 21 | ]; 22 | 23 | const types = await simpleTable.getTypes(); 24 | const allColumns = Object.keys(types); 25 | 26 | options.columns = stringToArray(options.columns ?? []); 27 | 28 | await queryDB( 29 | simpleTable, 30 | removeMissingQuery( 31 | simpleTable.name, 32 | allColumns, 33 | types, 34 | options.columns.length === 0 ? allColumns : options.columns, 35 | options, 36 | ), 37 | mergeOptions(simpleTable, { 38 | table: simpleTable.name, 39 | method: "removeMissing()", 40 | parameters: { options }, 41 | }), 42 | ); 43 | } 44 | -------------------------------------------------------------------------------- /src/methods/removeMissingQuery.ts: -------------------------------------------------------------------------------- 1 | export default function removeMissingQuery( 2 | table: string, 3 | allColumns: string[], 4 | types: { 5 | [key: string]: string; 6 | }, 7 | columns: string[], 8 | options: { 9 | missingValues?: (string | number)[]; 10 | invert?: boolean; 11 | } = {}, 12 | ) { 13 | let query = `CREATE OR REPLACE TABLE "${table}" AS SELECT ${ 14 | allColumns 15 | .map((d) => `"${d}"`) 16 | .join(", ") 17 | } FROM "${table}" 18 | WHERE`; 19 | 20 | if (options.invert) { 21 | for (let i = 0; i < columns.length; i++) { 22 | query += `\n"${columns[i]}" IS NULL OR`; 23 | if (options.missingValues) { 24 | for (const otherMissingValue of options.missingValues) { 25 | if ( 26 | typeof otherMissingValue === "string" && 27 | types[columns[i]] === "VARCHAR" 28 | ) { 29 | query += `\n"${columns[i]}" = '${otherMissingValue}' OR`; 30 | } else if ( 31 | typeof otherMissingValue === "number" && 32 | ["BIGINT", "DOUBLE"].includes(types[columns[i]]) 33 | ) { 34 | query += `\n"${columns[i]}" = ${otherMissingValue} OR`; 35 | } 36 | } 37 | } 38 | } 39 | } else { 40 | for (let i = 0; i < columns.length; i++) { 41 | query += `\n"${columns[i]}" IS NOT NULL AND`; 42 | if (options.missingValues) { 43 | for (const otherMissingValue of options.missingValues) { 44 | if ( 45 | typeof otherMissingValue === "string" && 46 | types[columns[i]] === "VARCHAR" 47 | ) { 48 | query += `\n"${columns[i]}" != '${otherMissingValue}' AND`; 49 | } else if ( 50 | typeof otherMissingValue === "number" && 51 | ["BIGINT", "DOUBLE"].includes(types[columns[i]]) 52 | ) { 53 | query += `\n"${columns[i]}" != ${otherMissingValue} AND`; 54 | } 55 | } 56 | } 57 | } 58 | } 59 | 60 | return options.invert 61 | ? query.slice(0, query.length - 3) 62 | : query.slice(0, query.length - 4); 63 | } 64 | -------------------------------------------------------------------------------- /src/methods/removeQuery.ts: -------------------------------------------------------------------------------- 1 | import parseValue from "../helpers/parseValue.ts"; 2 | 3 | export default function removeQuery( 4 | table: string, 5 | columnsAndValues: { 6 | [key: string]: 7 | | (number | string | Date | boolean | null)[] 8 | | (number | string | Date | boolean | null); 9 | }, 10 | ) { 11 | let query = 12 | `CREATE OR REPLACE TABLE "${table}" AS SELECT * FROM "${table}" WHERE\n`; 13 | const columns = Object.keys(columnsAndValues); 14 | 15 | const conditions = []; 16 | for (const column of columns) { 17 | let values = columnsAndValues[column]; 18 | if (!Array.isArray(values)) { 19 | values = [values]; 20 | } 21 | 22 | conditions.push( 23 | `"${column}" NOT IN (${values.map((d) => parseValue(d)).join(", ")})`, 24 | ); 25 | } 26 | 27 | query += conditions.join("\nAND "); 28 | 29 | return query; 30 | } 31 | -------------------------------------------------------------------------------- /src/methods/renameColumnQuery.ts: -------------------------------------------------------------------------------- 1 | export default function renameColumnQuery( 2 | table: string, 3 | oldColumns: string[], 4 | newColumns: string[], 5 | ) { 6 | let query = ""; 7 | for (let i = 0; i < oldColumns.length; i++) { 8 | query += `ALTER TABLE "${table}" RENAME COLUMN "${oldColumns[i]}" TO "${ 9 | newColumns[i] 10 | }";\n`; 11 | } 12 | return query; 13 | } 14 | -------------------------------------------------------------------------------- /src/methods/replaceNullsQuery.ts: -------------------------------------------------------------------------------- 1 | import parseValue from "../helpers/parseValue.ts"; 2 | 3 | export default function replaceNullsQuery( 4 | table: string, 5 | columns: string[], 6 | value: number | string | Date | boolean, 7 | ) { 8 | let query = ""; 9 | const valueParsed = parseValue(value); 10 | for (const column of columns) { 11 | query += 12 | `UPDATE "${table}" SET "${column}" = ${valueParsed} WHERE "${column}" IS NULL;`; 13 | } 14 | 15 | return query; 16 | } 17 | -------------------------------------------------------------------------------- /src/methods/replaceQuery.ts: -------------------------------------------------------------------------------- 1 | export default function replaceQuery( 2 | table: string, 3 | columns: string[], 4 | oldTexts: string[], 5 | newTexts: string[], 6 | options: { entireString?: boolean; regex?: boolean } = {}, 7 | ) { 8 | let query = ""; 9 | 10 | oldTexts = oldTexts.map((d) => d.replace(/'/g, "''")); 11 | newTexts = newTexts.map((d) => d.replace(/'/g, "''")); 12 | 13 | for (const column of columns) { 14 | for (let i = 0; i < oldTexts.length; i++) { 15 | if (options.entireString) { 16 | query += `UPDATE "${table}" SET "${column}" = 17 | CASE 18 | WHEN "${column}" = '${oldTexts[i]}' THEN '${newTexts[i]}' 19 | ELSE "${column}" 20 | END;\n`; 21 | } else if (options.regex) { 22 | query += 23 | `UPDATE "${table}" SET "${column}" = REGEXP_REPLACE("${column}", '${ 24 | oldTexts[i] 25 | }', '${newTexts[i]}', 'g');\n`; 26 | } else { 27 | query += `UPDATE "${table}" SET "${column}" = REPLACE("${column}", '${ 28 | oldTexts[i] 29 | }', '${newTexts[i]}');\n`; 30 | } 31 | } 32 | } 33 | 34 | return query; 35 | } 36 | -------------------------------------------------------------------------------- /src/methods/rollingQuery.ts: -------------------------------------------------------------------------------- 1 | import stringToArray from "../helpers/stringToArray.ts"; 2 | 3 | export default function rollingQuery( 4 | table: string, 5 | column: string, 6 | newColumn: string, 7 | summary: "count" | "min" | "max" | "mean" | "median" | "sum", 8 | preceding: number, 9 | following: number, 10 | options: { 11 | categories?: string | string[]; 12 | decimals?: number; 13 | } = {}, 14 | ) { 15 | const aggregates: { [key: string]: string } = { 16 | count: "COUNT", 17 | min: "MIN", 18 | max: "MAX", 19 | mean: "AVG", 20 | median: "MEDIAN", 21 | sum: "SUM", 22 | }; 23 | 24 | const categories = options.categories 25 | ? stringToArray(options.categories) 26 | : []; 27 | const partition = categories.length > 0 28 | ? `PARTITION BY ${categories.map((d) => `"${d}"`).join(", ")}` 29 | : ""; 30 | 31 | const tempQuery = `${aggregates[summary]}(${column}) OVER (${partition} 32 | ROWS BETWEEN ${preceding} PRECEDING AND ${following} FOLLOWING)`; 33 | 34 | const query = `CREATE OR REPLACE TABLE "${table}" AS SELECT *, 35 | ${ 36 | typeof options.decimals === "number" 37 | ? `ROUND(${tempQuery}, ${options.decimals})` 38 | : tempQuery 39 | } AS "${newColumn}", 40 | COUNT("${column}") OVER (${partition} 41 | ROWS BETWEEN ${preceding} PRECEDING AND ${following} FOLLOWING) as tempCountForRolling 42 | FROM "${table}"; 43 | UPDATE "${table}" SET "${newColumn}" = CASE 44 | WHEN tempCountForRolling != ${preceding + following + 1} THEN NULL 45 | ELSE ${newColumn} 46 | END; 47 | ALTER TABLE "${table}" DROP COLUMN "tempCountForRolling"; 48 | `; 49 | 50 | return query; 51 | } 52 | -------------------------------------------------------------------------------- /src/methods/roundQuery.ts: -------------------------------------------------------------------------------- 1 | export default function roundQuery( 2 | table: string, 3 | columns: string[], 4 | options: { method?: "round" | "ceiling" | "floor"; decimals?: number }, 5 | ) { 6 | let query = `UPDATE "${table}" SET`; 7 | const method = options.method?.toUpperCase() ?? "ROUND"; 8 | const decimals = options.decimals ?? 0; 9 | 10 | if (method === "ROUND") { 11 | for (const column of columns) { 12 | query += `\n${column} = ${method}(${column}, ${decimals}),`; 13 | } 14 | } else { 15 | for (const column of columns) { 16 | query += `\n${column} = ${method}(${column}),`; 17 | } 18 | } 19 | 20 | return query.slice(0, query.length - 1); 21 | } 22 | -------------------------------------------------------------------------------- /src/methods/selectRowsQuery.ts: -------------------------------------------------------------------------------- 1 | export default function selectRowsQuery( 2 | table: string, 3 | count: number | string, 4 | options: { offset?: number; outputTable?: string | boolean } = {}, 5 | ) { 6 | return `CREATE OR REPLACE TABLE ${ 7 | options.outputTable ?? table 8 | } AS SELECT * FROM "${table}" LIMIT ${count}${ 9 | typeof options.offset === "number" ? ` OFFSET ${options.offset}` : "" 10 | };`; 11 | } 12 | -------------------------------------------------------------------------------- /src/methods/sortQuery.ts: -------------------------------------------------------------------------------- 1 | export default function sortQuery( 2 | table: string, 3 | order: { [key: string]: "asc" | "desc" } | null, 4 | options: { 5 | lang?: { [key: string]: string }; 6 | } = {}, 7 | ) { 8 | let query = `CREATE OR REPLACE TABLE "${table}" AS SELECT * FROM "${table}" 9 | ORDER BY`; 10 | 11 | if (order === null) { 12 | query += " ALL; "; 13 | } else { 14 | for (const column of Object.keys(order)) { 15 | if (options.lang && options.lang[column]) { 16 | query += `\n"${column}" COLLATE ${options.lang[column]} ${ 17 | order[ 18 | column 19 | ].toUpperCase() 20 | },`; 21 | } else { 22 | query += `\n"${column}" ${order[column].toUpperCase()},`; 23 | } 24 | } 25 | } 26 | 27 | return query.slice(0, query.length - 1); 28 | } 29 | -------------------------------------------------------------------------------- /src/methods/trimQuery.ts: -------------------------------------------------------------------------------- 1 | export default function trimQuery( 2 | table: string, 3 | columns: string[], 4 | options: { character?: string; method?: "leftTrim" | "rightTrim" | "trim" }, 5 | ) { 6 | let query = ``; 7 | 8 | const method = options.method ?? "trim"; 9 | 10 | const specialCharacter = typeof options.character === "string" 11 | ? `, '${options.character}'` 12 | : ""; 13 | 14 | if (method === "trim") { 15 | for (const column of columns) { 16 | query += 17 | `\nUPDATE "${table}" SET "${column}" = TRIM("${column}"${specialCharacter});`; 18 | } 19 | } else if (method === "leftTrim") { 20 | for (const column of columns) { 21 | query += 22 | `\nUPDATE "${table}" SET "${column}" = LTRIM("${column}"${specialCharacter});`; 23 | } 24 | } else if (method === "rightTrim") { 25 | for (const column of columns) { 26 | query += 27 | `\nUPDATE "${table}" SET "${column}" = RTRIM("${column}"${specialCharacter});`; 28 | } 29 | } else { 30 | throw new Error(`Unknown method ${options.method}`); 31 | } 32 | 33 | return query; 34 | } 35 | -------------------------------------------------------------------------------- /src/methods/upperQuery.ts: -------------------------------------------------------------------------------- 1 | export default function upperQuery(table: string, columns: string[]) { 2 | let query = ""; 3 | 4 | for (const column of columns) { 5 | query += `\nUPDATE "${table}" SET "${column}" = UPPER("${column}");`; 6 | } 7 | 8 | return query; 9 | } 10 | -------------------------------------------------------------------------------- /src/methods/writeDataQuery.ts: -------------------------------------------------------------------------------- 1 | import { existsSync, rmSync } from "node:fs"; 2 | import cleanPath from "../helpers/cleanPath.ts"; 3 | 4 | export default function writeDataQuery( 5 | table: string, 6 | file: string, 7 | fileExtension: string, 8 | options: { compression?: boolean }, 9 | ) { 10 | const cleanedFile = cleanPath(file); 11 | if (fileExtension === "csv") { 12 | if (options.compression) { 13 | return `COPY "${table}" TO '${ 14 | cleanedFile + ".gz" 15 | }' (DELIMITER ',', HEADER TRUE, COMPRESSION GZIP);`; 16 | } else { 17 | return `COPY "${table}" TO '${cleanedFile}' (DELIMITER ',', HEADER TRUE);`; 18 | } 19 | } else if (fileExtension === "json") { 20 | if (options.compression) { 21 | return `COPY "${table}" TO '${ 22 | cleanedFile + ".gz" 23 | }' (FORMAT JSON, ARRAY TRUE, COMPRESSION GZIP);`; 24 | } else { 25 | return `COPY "${table}" TO '${cleanedFile}' (FORMAT JSON, ARRAY TRUE);`; 26 | } 27 | } else if (fileExtension === "parquet") { 28 | if (options.compression) { 29 | return `COPY "${table}" TO '${cleanedFile}' (FORMAT PARQUET, COMPRESSION ZSTD);`; 30 | } else { 31 | return `COPY "${table}" TO '${cleanedFile}' (FORMAT PARQUET);`; 32 | } 33 | } else if (fileExtension === "db") { 34 | if (existsSync(file)) { 35 | rmSync(file); 36 | } 37 | return `ATTACH '${cleanedFile}' AS my_database; 38 | COPY FROM DATABASE memory TO my_database; 39 | CREATE OR REPLACE TABLE my_database."${table}" AS SELECT * FROM "${table}"; 40 | DETACH my_database;`; 41 | } else if (fileExtension === "sqlite") { 42 | if (existsSync(file)) { 43 | rmSync(file); 44 | } 45 | return `INSTALL sqlite; LOAD sqlite; 46 | ATTACH '${cleanedFile}' AS my_sqlite_db (TYPE SQLITE); 47 | CREATE TABLE my_sqlite_db."${table}" AS SELECT * FROM "${table}"; 48 | DETACH my_sqlite_db;`; 49 | } else { 50 | throw new Error(`Unknown extension ${fileExtension}`); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/methods/writeGeoDataQuery.ts: -------------------------------------------------------------------------------- 1 | import cleanPath from "../helpers/cleanPath.ts"; 2 | 3 | export default function writeGeoDataQuery( 4 | table: string, 5 | file: string, 6 | fileExtension: string, 7 | options: { precision?: number } = {}, 8 | ) { 9 | if (fileExtension === "geojson" || fileExtension === "json") { 10 | const layerOptions = []; 11 | if (typeof options.precision === "number") { 12 | layerOptions.push(`COORDINATE_PRECISION=${options.precision}`); 13 | } 14 | layerOptions.push(`RFC7946=YES`); 15 | 16 | return `COPY "${table}" to '${ 17 | cleanPath(file) 18 | }' WITH (FORMAT GDAL, DRIVER 'GeoJSON'${ 19 | layerOptions.length > 0 20 | ? `, LAYER_CREATION_OPTIONS ('WRITE_NAME=NO', ${ 21 | layerOptions.map((d) => `'${d}'`).join(", ") 22 | })` 23 | : "" 24 | })`; 25 | } else { 26 | throw new Error(`Unknown extension ${fileExtension}`); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/methods/zScoreQuery.ts: -------------------------------------------------------------------------------- 1 | import stringToArray from "../helpers/stringToArray.ts"; 2 | 3 | export default function zScoreQuery( 4 | table: string, 5 | column: string, 6 | newColumn: string, 7 | options: { 8 | categories?: string | string[]; 9 | decimals?: number; 10 | } = {}, 11 | ) { 12 | const categories = options.categories 13 | ? stringToArray(options.categories) 14 | : []; 15 | const partition = categories.length > 0 16 | ? `PARTITION BY ${categories.map((d) => `${d}`).join(", ")}` 17 | : ""; 18 | 19 | const tempQuery = `(${column}-AVG(${column}) OVER(${partition})) 20 | / 21 | STDDEV_POP(${column}) OVER(${partition})`; 22 | const query = ` 23 | CREATE OR REPLACE TABLE "${table}" AS 24 | SELECT *, ( 25 | ${ 26 | typeof options.decimals === "number" 27 | ? `ROUND(${tempQuery}, ${options.decimals})` 28 | : tempQuery 29 | } 30 | 31 | ) AS ${newColumn}, 32 | FROM "${table}" 33 | `; 34 | 35 | return query; 36 | } 37 | -------------------------------------------------------------------------------- /test/data/directory/data1.csv: -------------------------------------------------------------------------------- 1 | key1,key2 2 | 1,un 3 | 2,deux 4 | 3,trois 5 | 4,quatre 6 | -------------------------------------------------------------------------------- /test/data/directory/data2.csv: -------------------------------------------------------------------------------- 1 | key1,key2 2 | 5,cinq 3 | 6,six 4 | 7,sept 5 | 8,huit 6 | -------------------------------------------------------------------------------- /test/data/directory/data3.csv: -------------------------------------------------------------------------------- 1 | key1,key2 2 | 9,neuf 3 | 10,dix 4 | 11,onze -------------------------------------------------------------------------------- /test/data/directory/data4ExtraColumn.csv: -------------------------------------------------------------------------------- 1 | key1,key2,key3 2 | 9,neuf,nine 3 | 10,dix,ten 4 | 11,onze,eleven -------------------------------------------------------------------------------- /test/data/files/alberta-expenses.csv: -------------------------------------------------------------------------------- 1 | Ministry,Position,Name,Category,Type,DateIncurred,Amount,Description,Receipt 2 | Justice,"ED, Courts Information Managem","Clarke,Gareth",Travel,Air - Out Of Province,2025-01-23T00:00:00.000Z,476.37,Attend CIO Conference, 3 | Justice,"ED, Office of the Chief Justic","Owens,Timothy",Travel,Vehicle Rental,2025-01-31T00:00:00.000Z,437.72,Chief & Council, 4 | Service Alberta and Red Tape Reduction,"ED, Red Tape Reduction","Bajcer,Paul",Daily Allowances,Meal Allowance,2025-01-28T00:00:00.000Z,44.0,Support Minister at RTR announcement, 5 | Tourism and Sport,"DM, Tourism and Sport","Goldstein,David",Daily Allowances,Meal Allowance,2025-01-21T00:00:00.000Z,57.0,DM and Dept mtgs - Edmonton - Jan 21-23, 6 | Tourism and Sport,"DM, Tourism and Sport","Goldstein,David",Daily Allowances,Meal Allowance,2025-01-22T00:00:00.000Z,57.0,DM and Dept mtgs - Edmonton - Jan 21-23, 7 | Tourism and Sport,"DM, Tourism and Sport","Goldstein,David",Daily Allowances,Meal Allowance,2025-01-23T00:00:00.000Z,13.0,DM and Dept mtgs - Edmonton - Jan 21-23, 8 | Tourism and Sport,"DM, Tourism and Sport","Goldstein,David",Travel,Per Diem - In Country,2025-01-22T00:00:00.000Z,7.35,DM and Dept mtgs - Edmonton - Jan 21-23, 9 | Tourism and Sport,"DM, Tourism and Sport","Goldstein,David",Travel,Per Diem - In Country,2025-01-23T00:00:00.000Z,7.35,DM and Dept mtgs - Edmonton - Jan 21-23, 10 | Tourism and Sport,"DM, Tourism and Sport","Goldstein,David",Travel,Taxi,2025-01-21T00:00:00.000Z,62.0,DM and Dept mtgs - Edmonton - Jan 21-23, -------------------------------------------------------------------------------- /test/data/files/cities.csv: -------------------------------------------------------------------------------- 1 | id,city 2 | 1108380,VANCOUVER 3 | 6158355,TORONTO 4 | 7024745,MONTREAL 5 | -------------------------------------------------------------------------------- /test/data/files/data.csv: -------------------------------------------------------------------------------- 1 | key1,key2 2 | 1,2 3 | 3,coucou 4 | 8,10 5 | brioche,croissant 6 | -------------------------------------------------------------------------------- /test/data/files/data.csv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nshiab/simple-data-analysis/bdb9dff12e724c817d2bcbc5f992c60c59379860/test/data/files/data.csv.gz -------------------------------------------------------------------------------- /test/data/files/data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "key1": 1, 4 | "key2": "un" 5 | }, 6 | { 7 | "key1": 2, 8 | "key2": "deux" 9 | }, 10 | { 11 | "key1": 3, 12 | "key2": "trois" 13 | }, 14 | { 15 | "key1": 4, 16 | "key2": "quatre" 17 | } 18 | ] 19 | -------------------------------------------------------------------------------- /test/data/files/data.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nshiab/simple-data-analysis/bdb9dff12e724c817d2bcbc5f992c60c59379860/test/data/files/data.parquet -------------------------------------------------------------------------------- /test/data/files/data.tsv: -------------------------------------------------------------------------------- 1 | key1 key2 2 | 1 2 3 | 3 coucou 4 | 8 10 5 | brioche croissant -------------------------------------------------------------------------------- /test/data/files/data.txt: -------------------------------------------------------------------------------- 1 | key1,key2 2 | 1,2 3 | 3,coucou 4 | 8,10 5 | brioche,croissant 6 | -------------------------------------------------------------------------------- /test/data/files/dataArrays.json: -------------------------------------------------------------------------------- 1 | { 2 | "key1": [ 3 | 1, 4 | 3, 5 | 8, 6 | "brioche" 7 | ], 8 | "key2": [ 9 | 2, 10 | "coucou", 11 | 10, 12 | "croissant" 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /test/data/files/dataCompressed.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nshiab/simple-data-analysis/bdb9dff12e724c817d2bcbc5f992c60c59379860/test/data/files/dataCompressed.parquet -------------------------------------------------------------------------------- /test/data/files/dataCorrelations.json: -------------------------------------------------------------------------------- 1 | [ 2 | { "key1": "Rubarbe", "key2": 1, "key3": 10.5, "key4": 5 }, 3 | { "key1": "Fraise", "key2": 11, "key3": 2.345, "key4": 10 }, 4 | { "key1": "Rubarbe", "key2": 2, "key3": 4.5657, "key4": 3 }, 5 | { "key1": "Fraise", "key2": 22, "key3": 12.3434, "key4": 1 } 6 | ] 7 | -------------------------------------------------------------------------------- /test/data/files/dataCsvCompressed.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nshiab/simple-data-analysis/bdb9dff12e724c817d2bcbc5f992c60c59379860/test/data/files/dataCsvCompressed.txt -------------------------------------------------------------------------------- /test/data/files/dataDates.csv: -------------------------------------------------------------------------------- 1 | date,datetime,datetimeWithMs,time,timeMs,weirdDatetime 2 | 2010-01-01,2010-01-01 14:01:12,2010-01-01 14:12:12.014,14:12:12,14:12:12.014,2010/01/01_14h_01min_12sec 3 | 2010-01-02,2010-01-02 01:12:54,2010-01-02 01:12:54.955,01:12:54,01:12:54.955,2010/01/02_01h_12min_54sec 4 | 2010-01-03,2010-01-03 02:25:01,2010-01-03 02:25:01.111,02:25:01,02:25:01.111,2010/01/03_02h_25min_54sec 5 | 2010-01-04,2010-01-04 23:25:15,2010-01-04 12:01:15.123,12:01:15,12:01:15.123,2010/01/04_23h_25min_15sec -------------------------------------------------------------------------------- /test/data/files/dataDuplicates.csv: -------------------------------------------------------------------------------- 1 | key1,key2 2 | 1,2 3 | 3,coucou 4 | 8,10 5 | 8,10 6 | brioche,croissant 7 | -------------------------------------------------------------------------------- /test/data/files/dataExtraLines.csv: -------------------------------------------------------------------------------- 1 | Annoying line, sometimes used to describe csv data. 2 | Sometimes there are two. 3 | key1,key2 4 | 1,2 5 | 3,coucou 6 | 8,10 7 | brioche,croissant -------------------------------------------------------------------------------- /test/data/files/dataJustNumbers.csv: -------------------------------------------------------------------------------- 1 | key1,key2 2 | 1.3,2 3 | 3,15 4 | 8.5,10 5 | 1,154 -------------------------------------------------------------------------------- /test/data/files/dataJustNumbers.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "key1": 1, 4 | "key2": "hi" 5 | }, 6 | { 7 | "key1": 3, 8 | "key2": "coucou" 9 | }, 10 | { 11 | "key1": 8, 12 | "key2": "how are you" 13 | }, 14 | { 15 | "key1": 21.5, 16 | "key2": "comment ça va" 17 | } 18 | ] 19 | -------------------------------------------------------------------------------- /test/data/files/dataManyDecimals.csv: -------------------------------------------------------------------------------- 1 | key1,key2 2 | 1.04355,2.945 3 | 3.243943535,4.9898 4 | 8.1,34.5 5 | 10,100 6 | -------------------------------------------------------------------------------- /test/data/files/dataNoHeaders.csv: -------------------------------------------------------------------------------- 1 | 1,2 2 | 3,coucou 3 | 8,10 4 | brioche,croissant 5 | -------------------------------------------------------------------------------- /test/data/files/dataProportions.json: -------------------------------------------------------------------------------- 1 | [ 2 | { "key1": 1, "key2": 2, "key3": 3 }, 3 | { "key1": 4, "key2": 5, "key3": 6 }, 4 | { "key1": 7, "key2": 8, "key3": 9 } 5 | ] 6 | -------------------------------------------------------------------------------- /test/data/files/dataRank.csv: -------------------------------------------------------------------------------- 1 | Name, Subject, Mark 2 | Lily,Maths,65 3 | Lily,Science,80 4 | Lily,English,70 5 | Isabella,Maths,50 6 | Isabella,Science,70 7 | Isabella,English,90 8 | Olivia,Maths,55 9 | Olivia,Science,60 10 | Olivia,English,89 -------------------------------------------------------------------------------- /test/data/files/dataSort.csv: -------------------------------------------------------------------------------- 1 | key1,key2,key3 2 | 5,À l'ouest,A 3 | 4,Extérieur,B 4 | 1,Roi,A 5 | 2,Alambic,B 6 | 56.7,Éléphant,A 7 | 900,Zéphir,A -------------------------------------------------------------------------------- /test/data/files/dataSummarize.json: -------------------------------------------------------------------------------- 1 | [ 2 | { "key1": "Rubarbe", "key2": 1, "key3": 10.5 }, 3 | { "key1": "Fraise", "key2": 11, "key3": 2.345 }, 4 | { "key1": "Rubarbe", "key2": 2, "key3": 4.5657 }, 5 | { "key1": "Fraise", "key2": 22, "key3": 12.3434 }, 6 | { "key1": "Banane", "key2": null, "key3": null }, 7 | { "key1": "Banane", "key2": null, "key3": null } 8 | ] 9 | -------------------------------------------------------------------------------- /test/data/files/dataTidy.json: -------------------------------------------------------------------------------- 1 | [ 2 | { "Department": "accounting", "year": "2015", "employees": 10 }, 3 | { "Department": "accounting", "year": "2016", "employees": 9 }, 4 | { "Department": "accounting", "year": "2017", "employees": 15 }, 5 | { "Department": "accounting", "year": "2018", "employees": 11 }, 6 | { "Department": "accounting", "year": "2019", "employees": 25 }, 7 | { "Department": "accounting", "year": "2020", "employees": 32 }, 8 | { "Department": "R&D", "year": "2015", "employees": 1 }, 9 | { "Department": "R&D", "year": "2016", "employees": 2 }, 10 | { "Department": "R&D", "year": "2017", "employees": 5 }, 11 | { "Department": "R&D", "year": "2018", "employees": 2 }, 12 | { "Department": "R&D", "year": "2019", "employees": 2 }, 13 | { "Department": "R&D", "year": "2020", "employees": 3 }, 14 | { "Department": "sales", "year": "2015", "employees": 2 }, 15 | { "Department": "sales", "year": "2016", "employees": 7 }, 16 | { "Department": "sales", "year": "2017", "employees": 15 }, 17 | { "Department": "sales", "year": "2018", "employees": 32 }, 18 | { "Department": "sales", "year": "2019", "employees": 45 }, 19 | { "Department": "sales", "year": "2020", "employees": 27 } 20 | ] 21 | -------------------------------------------------------------------------------- /test/data/files/dataTrim.json: -------------------------------------------------------------------------------- 1 | [ 2 | { "key1": "  a  ", "key2": " !@a!@" }, 3 | { "key1": "  b  ", "key2": " !@b!@" }, 4 | { "key1": "  c  ", "key2": " !@c!@" }, 5 | { "key1": "  d  ", "key2": " !@d!@" } 6 | ] 7 | -------------------------------------------------------------------------------- /test/data/files/dataUntidy.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "Department": "accounting", 4 | "2015": 10, 5 | "2016": 9, 6 | "2017": 15, 7 | "2018": 11, 8 | "2019": 25, 9 | "2020": 32 10 | }, 11 | { 12 | "Department": "R&D", 13 | "2015": 1, 14 | "2016": 2, 15 | "2017": 5, 16 | "2018": 2, 17 | "2019": 2, 18 | "2020": 3 19 | }, 20 | { 21 | "Department": "sales", 22 | "2015": 2, 23 | "2016": 7, 24 | "2017": 15, 25 | "2018": 32, 26 | "2019": 45, 27 | "2020": 27 28 | } 29 | ] 30 | -------------------------------------------------------------------------------- /test/data/files/dataUntidyWithNulls.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "Department": "accounting", 4 | "2015": null, 5 | "2016": 9, 6 | "2017": 15, 7 | "2018": 11, 8 | "2019": 25, 9 | "2020": 32 10 | }, 11 | { 12 | "Department": "R&D", 13 | "2015": 1, 14 | "2016": 2, 15 | "2017": null, 16 | "2018": 2, 17 | "2019": 2, 18 | "2020": 3 19 | }, 20 | { 21 | "Department": "sales", 22 | "2015": 2, 23 | "2016": 7, 24 | "2017": 15, 25 | "2018": 32, 26 | "2019": 45, 27 | "2020": null 28 | } 29 | ] 30 | -------------------------------------------------------------------------------- /test/data/files/dataWithMissingValues.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "key1": 1, 4 | "key2": null, 5 | "key3": 0.5 6 | }, 7 | { 8 | "key1": null, 9 | "key2": "deux", 10 | "key3": 12 11 | }, 12 | { 13 | "key1": 3, 14 | "key2": "trois", 15 | "key3": null 16 | }, 17 | { 18 | "key1": 4, 19 | "key2": "quatre", 20 | "key3": 11545.12 21 | } 22 | ] 23 | -------------------------------------------------------------------------------- /test/data/files/populations-one-sheet.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nshiab/simple-data-analysis/bdb9dff12e724c817d2bcbc5f992c60c59379860/test/data/files/populations-one-sheet.xlsx -------------------------------------------------------------------------------- /test/data/files/populations-two-sheets.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nshiab/simple-data-analysis/bdb9dff12e724c817d2bcbc5f992c60c59379860/test/data/files/populations-two-sheets.xlsx -------------------------------------------------------------------------------- /test/data/joins/categories.csv: -------------------------------------------------------------------------------- 1 | category,dishId 2 | Dessert,1 3 | Main,2 4 | Dessert,3 5 | Main,6 6 | Main,7 7 | Dessert,8 -------------------------------------------------------------------------------- /test/data/joins/dishes.csv: -------------------------------------------------------------------------------- 1 | dishId,name,country 2 | 1,Crème brûlée,France 3 | 2,Pizza,Italy 4 | 3,Churros,Mexico 5 | 4,Couscous,Morrocco 6 | 5,Mochi,Japan -------------------------------------------------------------------------------- /test/data/joins/normals.csv: -------------------------------------------------------------------------------- 1 | city,season,normals 2 | montreal,winter,-10.3 3 | montreal,spring,8.4 4 | montreal,summer,25.6 5 | montreal,fall,12.7 6 | toronto,winter,2.3 7 | toronto,spring,12.3 8 | toronto,summer,27.6 9 | toronto,fall,15.8 -------------------------------------------------------------------------------- /test/data/joins/projections.csv: -------------------------------------------------------------------------------- 1 | city,season,projections 2 | montreal,winter,-9.0 3 | montreal,spring,11.4 4 | montreal,summer,28.9 5 | montreal,fall,14.0 6 | toronto,winter,3.3 7 | toronto,spring,13.6 8 | toronto,summer,29.2 9 | toronto,fall,16.5 -------------------------------------------------------------------------------- /test/geodata/files/CanadianProvincesAndTerritories.shp.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nshiab/simple-data-analysis/bdb9dff12e724c817d2bcbc5f992c60c59379860/test/geodata/files/CanadianProvincesAndTerritories.shp.zip -------------------------------------------------------------------------------- /test/geodata/files/canada-not-4326.shp.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nshiab/simple-data-analysis/bdb9dff12e724c817d2bcbc5f992c60c59379860/test/geodata/files/canada-not-4326.shp.zip -------------------------------------------------------------------------------- /test/geodata/files/coordinates.csv: -------------------------------------------------------------------------------- 1 | name,lat,lon 2 | montreal,43.77,-79.29 3 | toronto,45.35,-73.86 4 | vancouver,49.07,-122.96 -------------------------------------------------------------------------------- /test/geodata/files/coordinates.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "properties": { 7 | "name": "toronto" 8 | }, 9 | "geometry": { 10 | "type": "Point", 11 | "coordinates": [-79.29, 43.77] 12 | }, 13 | "id": 0 14 | }, 15 | { 16 | "type": "Feature", 17 | "properties": { 18 | "name": "montreal" 19 | }, 20 | "geometry": { 21 | "type": "Point", 22 | "coordinates": [-73.86, 45.35] 23 | }, 24 | "id": 1 25 | }, 26 | { 27 | "type": "Feature", 28 | "properties": { 29 | "name": "vancouver" 30 | }, 31 | "geometry": { 32 | "type": "Point", 33 | "coordinates": [-122.96, 49.07] 34 | }, 35 | "id": 2 36 | } 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /test/geodata/files/data-compressed.geoparquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nshiab/simple-data-analysis/bdb9dff12e724c817d2bcbc5f992c60c59379860/test/geodata/files/data-compressed.geoparquet -------------------------------------------------------------------------------- /test/geodata/files/data-multiple-columns.geoparquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nshiab/simple-data-analysis/bdb9dff12e724c817d2bcbc5f992c60c59379860/test/geodata/files/data-multiple-columns.geoparquet -------------------------------------------------------------------------------- /test/geodata/files/data.geoparquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nshiab/simple-data-analysis/bdb9dff12e724c817d2bcbc5f992c60c59379860/test/geodata/files/data.geoparquet -------------------------------------------------------------------------------- /test/geodata/files/invalid.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "name": "polygon_inner_and_exterior_ring_cross", 4 | "features": [ 5 | { 6 | "type": "Feature", 7 | "properties": {}, 8 | "geometry": { 9 | "type": "Polygon", 10 | "coordinates": [ 11 | [ 12 | [13.382288, 52.515426], 13 | [13.382096, 52.514797], 14 | [13.383424, 52.51464], 15 | [13.383529, 52.515496], 16 | [13.382288, 52.515426] 17 | ], 18 | [ 19 | [13.382603, 52.514954], 20 | [13.38262, 52.516255], 21 | [13.383144, 52.515321], 22 | [13.383127, 52.514867], 23 | [13.382603, 52.514954] 24 | ] 25 | ] 26 | } 27 | } 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /test/geodata/files/line.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "properties": {}, 7 | "geometry": { 8 | "coordinates": [ 9 | [-75.68445904198826, 45.427277146491775], 10 | [-75.81934205836949, 46.05155068033537] 11 | ], 12 | "type": "LineString" 13 | } 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /test/geodata/files/point.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "properties": {}, 7 | "geometry": { 8 | "coordinates": [-73.62315106245389, 45.51412791316409], 9 | "type": "Point" 10 | } 11 | } 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /test/geodata/files/pointsInside.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "properties": { 7 | "name": "pointA" 8 | }, 9 | "geometry": { 10 | "coordinates": [-76.34553248992202, 48.241182892559266], 11 | "type": "Point" 12 | } 13 | }, 14 | { 15 | "type": "Feature", 16 | "properties": { "name": "pointB" }, 17 | "geometry": { 18 | "coordinates": [-73.18043031919933, 50.15023361660323], 19 | "type": "Point" 20 | } 21 | }, 22 | { 23 | "type": "Feature", 24 | "properties": { "name": "pointC" }, 25 | "geometry": { 26 | "coordinates": [-72.78960434234926, 48.47150751404138], 27 | "type": "Point" 28 | } 29 | }, 30 | { 31 | "type": "Feature", 32 | "properties": { "name": "pointD" }, 33 | "geometry": { 34 | "coordinates": [-72.2926406368759, 47.43075362784262], 35 | "type": "Point" 36 | } 37 | } 38 | ] 39 | } 40 | -------------------------------------------------------------------------------- /test/geodata/files/polygon.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "properties": {}, 7 | "geometry": { 8 | "coordinates": [ 9 | [ 10 | [-75.5481964022451, 45.54486812042583], 11 | [-76.0069865498526, 45.551704170943964], 12 | [-76.03627102735922, 45.34626098975511], 13 | [-75.83616043106275, 45.219198013848], 14 | [-75.36760879095283, 45.291349760674194], 15 | [-75.5481964022451, 45.54486812042583] 16 | ] 17 | ], 18 | "type": "Polygon" 19 | } 20 | } 21 | ] 22 | } 23 | -------------------------------------------------------------------------------- /test/geodata/files/polygonInside.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "properties": { "name": "container" }, 7 | "geometry": { 8 | "coordinates": [ 9 | [ 10 | [-73.86384311805543, 49.44467601022404], 11 | [-73.86384311805543, 46.84089921028934], 12 | [-71.26679234199449, 46.84089921028934], 13 | [-71.26679234199449, 49.44467601022404], 14 | [-73.86384311805543, 49.44467601022404] 15 | ] 16 | ], 17 | "type": "Polygon" 18 | }, 19 | "id": 0 20 | } 21 | ] 22 | } 23 | -------------------------------------------------------------------------------- /test/geodata/files/polygons.geojson: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "properties": { "name": "polygonA" }, 7 | "geometry": { 8 | "type": "Polygon", 9 | "coordinates": [ 10 | [ 11 | [-80.5925744, 50.3447571], 12 | [-81.4683036, 44.963885], 13 | [-75.0907732, 46.9689849], 14 | [-75.5601513, 50.1474736], 15 | [-80.5925744, 50.3447571] 16 | ] 17 | ] 18 | } 19 | }, 20 | { 21 | "type": "Feature", 22 | "properties": { "name": "polygonB" }, 23 | "geometry": { 24 | "type": "Polygon", 25 | "coordinates": [ 26 | [ 27 | [-121.9581024, 62.0110577], 28 | [-122.3017867, 56.0464801], 29 | [-112.2459009, 51.5685044], 30 | [-104.838484, 51.4335657], 31 | [-96.8420125, 53.4420801], 32 | [-98.0491012, 62.4259071], 33 | [-121.9581024, 62.0110577] 34 | ] 35 | ] 36 | } 37 | } 38 | ] 39 | } 40 | -------------------------------------------------------------------------------- /test/geodata/files/triangle.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "FeatureCollection", 3 | "features": [ 4 | { 5 | "type": "Feature", 6 | "properties": {}, 7 | "geometry": { 8 | "coordinates": [ 9 | [ 10 | [-94.52453351415484, 59.87527423109486], 11 | [-95.45894566685975, 56.32036701051533], 12 | [-81.04389945932482, 55.37237003334806], 13 | [-94.52453351415484, 59.87527423109486] 14 | ] 15 | ], 16 | "type": "Polygon" 17 | } 18 | } 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /test/unit/class/SimpleTable.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | import SimpleTable from "../../../src/class/SimpleTable.ts"; 4 | 5 | Deno.test("should create a new SimpleTable", async () => { 6 | const sdb = new SimpleDB(); 7 | const table = sdb.newTable("data"); 8 | assertEquals(table instanceof SimpleTable, true); 9 | await sdb.done(); 10 | }); 11 | -------------------------------------------------------------------------------- /test/unit/helpers/getExtension.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import getExtension from "../../../src/helpers/getExtension.ts"; 3 | 4 | Deno.test("should return the extension when csv", function () { 5 | const extension = getExtension("coucou/key2/pat.a.te.csv"); 6 | assertEquals(extension, "csv"); 7 | }); 8 | Deno.test("should return the extension when csv is compressed", function () { 9 | const extension = getExtension("coucou/key2/pat.a.te.csv.gz"); 10 | assertEquals(extension, "csv"); 11 | }); 12 | Deno.test("should return the extension when json", function () { 13 | const extension = getExtension("coucou/key2/pat.a.te.json"); 14 | assertEquals(extension, "json"); 15 | }); 16 | Deno.test("should return the extension when json is compressed", function () { 17 | const extension = getExtension("coucou/key2/pat.a.te.json.gz"); 18 | assertEquals(extension, "json"); 19 | }); 20 | Deno.test("should return the extension when parquet", function () { 21 | const extension = getExtension("coucou/key2/pat.a.te.parquet"); 22 | assertEquals(extension, "parquet"); 23 | }); 24 | -------------------------------------------------------------------------------- /test/unit/helpers/getName.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import getName from "../../../src/helpers/getName.ts"; 3 | 4 | Deno.test("should return the file name without extension for a simple file path", function () { 5 | const result = getName("example.txt"); 6 | assertEquals(result, "example"); 7 | }); 8 | 9 | Deno.test("should return the file name without extension for a nested file path", function () { 10 | const result = getName("path/to/example.txt"); 11 | assertEquals(result, "example"); 12 | }); 13 | 14 | Deno.test("should return the file name without extension for a file with multiple dots", function () { 15 | const result = getName("path/to/example.test.js"); 16 | assertEquals(result, "example.test"); 17 | }); 18 | 19 | Deno.test("should return an empty string for a file with no name but only extension", function () { 20 | const result = getName("path/to/.hiddenfile"); 21 | assertEquals(result, ""); 22 | }); 23 | 24 | Deno.test("should handle paths with no extension", function () { 25 | const result = getName("path/to/example"); 26 | assertEquals(result, "example"); 27 | }); 28 | -------------------------------------------------------------------------------- /test/unit/helpers/getProjection.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | import getProjection from "../../../src/helpers/getProjection.ts"; 4 | 5 | Deno.test("should retrieve the projection of a json file", async () => { 6 | const sdb = new SimpleDB(); 7 | const proj = await getProjection( 8 | sdb, 9 | "test/geodata/files/CanadianProvincesAndTerritories.json", 10 | ); 11 | assertEquals(proj, { 12 | name: "WGS 84", 13 | code: "EPSG:4326", 14 | unit: "degree", 15 | proj4: "+proj=longlat +datum=WGS84 +no_defs", 16 | }); 17 | await sdb.done(); 18 | }); 19 | 20 | Deno.test("should retrieve the projection of a zipped shapefile", async () => { 21 | const sdb = new SimpleDB(); 22 | const proj = await getProjection( 23 | sdb, 24 | "test/geodata/files/canada-not-4326.shp.zip", 25 | ); 26 | assertEquals(proj, { 27 | name: "NAD83 / Statistics Canada Lambert", 28 | code: "EPSG:3347", 29 | unit: "metre", 30 | proj4: 31 | "+proj=lcc +lat_0=63.390675 +lon_0=-91.8666666666667 +lat_1=49 +lat_2=77 +x_0=6200000 +y_0=3000000 +datum=NAD83 +units=m +no_defs", 32 | }); 33 | await sdb.done(); 34 | }); 35 | -------------------------------------------------------------------------------- /test/unit/methods/addRowNumber.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return a column with the row number", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadArray([ 8 | { first: "Nael", last: "Shiab" }, 9 | { first: "Graeme", last: "Bruce" }, 10 | ]); 11 | await table.addRowNumber("rowNumber"); 12 | 13 | const data = await table.getData(); 14 | 15 | assertEquals(data, [ 16 | { first: "Nael", last: "Shiab", rowNumber: 1 }, 17 | { first: "Graeme", last: "Bruce", rowNumber: 2 }, 18 | ]); 19 | await sdb.done(); 20 | }); 21 | -------------------------------------------------------------------------------- /test/unit/methods/capitalize.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should capitalize strings in one column", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadArray([{ firstName: "NAEL", lastName: "SHIAB" }]); 8 | 9 | await table.capitalize("firstName"); 10 | 11 | const data = await table.getData(); 12 | 13 | assertEquals(data, [{ firstName: "Nael", lastName: "SHIAB" }]); 14 | await sdb.done(); 15 | }); 16 | 17 | Deno.test("should capitalize strings in two columns", async () => { 18 | const sdb = new SimpleDB(); 19 | const table = sdb.newTable(); 20 | await table.loadArray([{ firstName: "NAEL", lastName: "SHIAB" }]); 21 | 22 | await table.capitalize(["firstName", "lastName"]); 23 | 24 | const data = await table.getData(); 25 | 26 | assertEquals(data, [{ firstName: "Nael", lastName: "Shiab" }]); 27 | await sdb.done(); 28 | }); 29 | 30 | Deno.test("should capitalize strings in two columns with column names containing spaces", async () => { 31 | const sdb = new SimpleDB(); 32 | const table = sdb.newTable(); 33 | await table.loadArray([{ "first Name": "NAEL", "last Name": "SHIAB" }]); 34 | 35 | await table.capitalize(["first Name", "last Name"]); 36 | 37 | const data = await table.getData(); 38 | 39 | assertEquals(data, [ 40 | { "first Name": "Nael", "last Name": "Shiab" }, 41 | ]); 42 | await sdb.done(); 43 | }); 44 | -------------------------------------------------------------------------------- /test/unit/methods/cleanColumnNames.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should clean column names", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadData("test/data/files/employees.csv"); 8 | await table.cleanColumnNames(); 9 | const columns = await table.getColumns(); 10 | assertEquals(columns, [ 11 | "name", 12 | "hireDate", 13 | "job", 14 | "salary", 15 | "departmentOrUnit", 16 | "endOfYearBonus", 17 | ]); 18 | await sdb.done(); 19 | }); 20 | -------------------------------------------------------------------------------- /test/unit/methods/cloneColumn.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should clone a column", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadArray([{ firstName: "nael", lastName: "shiab" }]); 8 | 9 | await table.cloneColumn("firstName", "firstNameCloned"); 10 | 11 | const data = await table.getData(); 12 | 13 | assertEquals(data, [ 14 | { firstName: "nael", lastName: "shiab", firstNameCloned: "nael" }, 15 | ]); 16 | 17 | await sdb.done(); 18 | }); 19 | Deno.test("should clone a column with spaces in its name", async () => { 20 | const sdb = new SimpleDB(); 21 | const table = sdb.newTable("data"); 22 | await table.loadArray([{ "first name": "nael", "last name": "shiab" }]); 23 | 24 | await table.cloneColumn("first name", "first name cloned"); 25 | 26 | const data = await table.getData(); 27 | 28 | assertEquals(data, [ 29 | { "first name": "nael", "last name": "shiab", "first name cloned": "nael" }, 30 | ]); 31 | 32 | await sdb.done(); 33 | }); 34 | Deno.test("should clone a column with geometries and keep the projection", async () => { 35 | const sdb = new SimpleDB(); 36 | const table = sdb.newTable("data"); 37 | await table.loadGeoData( 38 | "test/geodata/files/CanadianProvincesAndTerritories.json", 39 | ); 40 | 41 | await table.cloneColumn("geom", "geomClone"); 42 | 43 | assertEquals( 44 | table.projections["geom"], 45 | table.projections["geomClone"], 46 | ); 47 | 48 | await sdb.done(); 49 | }); 50 | -------------------------------------------------------------------------------- /test/unit/methods/fill.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should fill empty cells for one column", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = await sdb 7 | .newTable() 8 | .loadArray([ 9 | { first: "Nael" }, 10 | { first: null }, 11 | { first: null }, 12 | { first: "Graeme" }, 13 | { first: null }, 14 | { first: null }, 15 | { first: null }, 16 | { first: null }, 17 | { first: "Andrew" }, 18 | ]); 19 | await table.fill("first"); 20 | const data = await table.getData(); 21 | assertEquals(data, [ 22 | { first: "Nael" }, 23 | { first: "Nael" }, 24 | { first: "Nael" }, 25 | { first: "Graeme" }, 26 | { first: "Graeme" }, 27 | { first: "Graeme" }, 28 | { first: "Graeme" }, 29 | { first: "Graeme" }, 30 | { first: "Andrew" }, 31 | ]); 32 | await sdb.done(); 33 | }); 34 | 35 | Deno.test("should fill empty cells for multiple columns", async () => { 36 | const sdb = new SimpleDB(); 37 | const table = await sdb.newTable().loadArray([ 38 | { first: "Nael", job: "Senior producer" }, 39 | { first: null, job: null }, 40 | { first: null, job: "Senior producer" }, 41 | { first: "Graeme", job: "Producer" }, 42 | { first: null, job: null }, 43 | { first: null, job: "Super producer" }, 44 | { first: null, job: null }, 45 | { first: null, job: null }, 46 | { first: "Andrew", job: "Senior dev" }, 47 | ]); 48 | await table.fill(["first", "job"]); 49 | const data = await table.getData(); 50 | assertEquals(data, [ 51 | { first: "Nael", job: "Senior producer" }, 52 | { first: "Nael", job: "Senior producer" }, 53 | { first: "Nael", job: "Senior producer" }, 54 | { first: "Graeme", job: "Producer" }, 55 | { first: "Graeme", job: "Producer" }, 56 | { first: "Graeme", job: "Super producer" }, 57 | { first: "Graeme", job: "Super producer" }, 58 | { first: "Graeme", job: "Super producer" }, 59 | { first: "Andrew", job: "Senior dev" }, 60 | ]); 61 | await sdb.done(); 62 | }); 63 | -------------------------------------------------------------------------------- /test/unit/methods/fillHoles.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | import { readFileSync } from "node:fs"; 4 | 5 | Deno.test("should remove the small circle from the big circle", async () => { 6 | const sdb = new SimpleDB(); 7 | 8 | const table = sdb.newTable(); 9 | await table.loadGeoData("test/geodata/files/bigCircleWithHole.json"); 10 | 11 | await table.fillHoles(); 12 | 13 | await table.writeGeoData("test/output/bigCircleWithHoleFilled.json"); 14 | 15 | assertEquals( 16 | JSON.parse( 17 | readFileSync("test/output/bigCircleWithHoleFilled.json", { 18 | encoding: "utf-8", 19 | }), 20 | ), 21 | JSON.parse( 22 | readFileSync("test/geodata/tests-results/bigCircleWithHoleFilled.json", { 23 | encoding: "utf-8", 24 | }), 25 | ), 26 | ); 27 | await sdb.done(); 28 | }); 29 | -------------------------------------------------------------------------------- /test/unit/methods/flipCoordinates.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should flip the coordinates", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("geoData"); 7 | await table.loadGeoData("test/geodata/files/point.json"); 8 | await table.flipCoordinates(); 9 | const data = await sdb.customQuery( 10 | `SELECT ST_AsText(geom) as geomText FROM geoData;`, 11 | { returnDataFrom: "query" }, 12 | ); 13 | 14 | assertEquals(data, [ 15 | { geomText: "POINT (-73.62315106245389 45.51412791316409)" }, 16 | ]); 17 | await sdb.done(); 18 | }); 19 | 20 | Deno.test("should flip the coordinates from a specific column", async () => { 21 | const sdb = new SimpleDB(); 22 | const table = sdb.newTable("geoData"); 23 | await table.loadGeoData("test/geodata/files/point.json"); 24 | await table.flipCoordinates("geom"); 25 | const data = await sdb.customQuery( 26 | `SELECT ST_AsText(geom) as geomText FROM geoData;`, 27 | { returnDataFrom: "query" }, 28 | ); 29 | 30 | assertEquals(data, [ 31 | { geomText: "POINT (-73.62315106245389 45.51412791316409)" }, 32 | ]); 33 | await sdb.done(); 34 | }); 35 | -------------------------------------------------------------------------------- /test/unit/methods/getBoundingBox.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the bounding box in [minX, minY, maxX, maxY]", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadGeoData( 8 | "test/geodata/files/CanadianProvincesAndTerritories.json", 9 | ); 10 | const bbox = await table.getBoundingBox(); 11 | assertEquals(bbox, [-141.014, 41.981, -52.636, 83.111]); 12 | await sdb.done(); 13 | }); 14 | 15 | Deno.test("should return the bounding box in [minX, minY, maxX, maxY] from a specific column", async () => { 16 | const sdb = new SimpleDB(); 17 | const table = sdb.newTable(); 18 | await table.loadGeoData( 19 | "test/geodata/files/CanadianProvincesAndTerritories.json", 20 | ); 21 | const bbox = await table.getBoundingBox("geom"); 22 | assertEquals(bbox, [-141.014, 41.981, -52.636, 83.111]); 23 | await sdb.done(); 24 | }); 25 | -------------------------------------------------------------------------------- /test/unit/methods/getColumns.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the columns of a table", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData("test/data/files/data.csv"); 8 | 9 | const columns = await table.getColumns(); 10 | 11 | assertEquals(columns, ["key1", "key2"]); 12 | await sdb.done(); 13 | }); 14 | -------------------------------------------------------------------------------- /test/unit/methods/getDescription.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the count of null values, non null values, and distinct values in each column of a table", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData("test/data/files/employees.json"); 8 | 9 | const description = await table.getDescription(); 10 | 11 | assertEquals(description, [ 12 | { 13 | column: "Department or unit", 14 | type: "JSON", 15 | count: 51, 16 | unique: 10, 17 | null: 5, 18 | }, 19 | { 20 | column: "End-of_year-BONUS?", 21 | type: "VARCHAR", 22 | count: 51, 23 | unique: 46, 24 | null: 4, 25 | }, 26 | { 27 | column: "Hire date", 28 | type: "VARCHAR", 29 | count: 51, 30 | unique: 42, 31 | null: 5, 32 | }, 33 | { column: "Job", type: "VARCHAR", count: 51, unique: 9, null: 5 }, 34 | { column: "Name", type: "VARCHAR", count: 51, unique: 46, null: 4 }, 35 | { column: "Salary", type: "JSON", count: 51, unique: 33, null: 3 }, 36 | ]); 37 | await sdb.done(); 38 | }); 39 | -------------------------------------------------------------------------------- /test/unit/methods/getExtent.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the extent in [min, max] order", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData(["test/data/files/data.json"]); 8 | assertEquals(await table.getExtent("key1"), [1, 4]); 9 | await sdb.done(); 10 | }); 11 | -------------------------------------------------------------------------------- /test/unit/methods/getFirstRow.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the first row", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData("test/data/files/data.json"); 8 | const data = await table.getFirstRow(); 9 | assertEquals(data, { key1: 1, key2: "un" }); 10 | await sdb.done(); 11 | }); 12 | 13 | Deno.test("should return the first row found based on a condition", async () => { 14 | const sdb = new SimpleDB(); 15 | const table = sdb.newTable("data"); 16 | await table.loadData("test/data/files/data.json"); 17 | const data = await table.getFirstRow({ 18 | conditions: `key2 = 'trois'`, 19 | }); 20 | assertEquals(data, { 21 | key1: 3, 22 | key2: "trois", 23 | }); 24 | await sdb.done(); 25 | }); 26 | -------------------------------------------------------------------------------- /test/unit/methods/getLastRow.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the last row", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData("test/data/files/data.json"); 8 | const data = await table.getLastRow(); 9 | assertEquals(data, { key1: 4, key2: "quatre" }); 10 | await sdb.done(); 11 | }); 12 | 13 | Deno.test("should return the last row found based on a condition", async () => { 14 | const sdb = new SimpleDB(); 15 | const table = sdb.newTable("data"); 16 | await table.loadData("test/data/files/data.json"); 17 | const data = await table.getLastRow({ 18 | conditions: `key2 = 'trois'`, 19 | }); 20 | assertEquals(data, { key1: 3, key2: "trois" }); 21 | await sdb.done(); 22 | }); 23 | -------------------------------------------------------------------------------- /test/unit/methods/getMax.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the max value", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData(["test/data/files/data.json"]); 8 | assertEquals(await table.getMax("key1"), 4); 9 | await sdb.done(); 10 | }); 11 | Deno.test("should return the max value even when there are spaces in the column name", async () => { 12 | const sdb = new SimpleDB(); 13 | const table = sdb.newTable("data"); 14 | await table.loadData(["test/data/files/data.json"]); 15 | await table.renameColumns({ key1: "key 1" }); 16 | assertEquals(await table.getMax("key 1"), 4); 17 | await sdb.done(); 18 | }); 19 | Deno.test("should return the max value with Dates", async () => { 20 | const sdb = new SimpleDB(); 21 | const table = sdb.newTable("data"); 22 | await table.loadArray([ 23 | { key1: new Date("2020-01-01") }, 24 | { key1: new Date("2021-01-01") }, 25 | { key1: new Date("2022-01-01") }, 26 | { key1: new Date("2023-01-01") }, 27 | ]); 28 | assertEquals(await table.getMax("key1"), new Date("2023-01-01")); 29 | await sdb.done(); 30 | }); 31 | -------------------------------------------------------------------------------- /test/unit/methods/getMean.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the mean value", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData("test/data/files/data.json"); 8 | assertEquals(await table.getMean("key1"), 2.5); 9 | await sdb.done(); 10 | }); 11 | 12 | Deno.test("should return the mean value rounded", async () => { 13 | const sdb = new SimpleDB(); 14 | const table = sdb.newTable("data"); 15 | await table.loadData("test/data/files/data.json"); 16 | assertEquals(await table.getMean("key1", { decimals: 0 }), 3); 17 | await sdb.done(); 18 | }); 19 | Deno.test("should return the mean value even when there are spaces in the column name", async () => { 20 | const sdb = new SimpleDB(); 21 | const table = sdb.newTable("data"); 22 | await table.loadData("test/data/files/data.json"); 23 | await table.renameColumns({ key1: "key 1" }); 24 | assertEquals(await table.getMean("key 1"), 2.5); 25 | await sdb.done(); 26 | }); 27 | 28 | Deno.test("should return the mean value rounded even when there are spaces in the column name", async () => { 29 | const sdb = new SimpleDB(); 30 | const table = sdb.newTable("data"); 31 | await table.loadData("test/data/files/data.json"); 32 | await table.renameColumns({ key1: "key 1" }); 33 | assertEquals(await table.getMean("key 1", { decimals: 0 }), 3); 34 | await sdb.done(); 35 | }); 36 | -------------------------------------------------------------------------------- /test/unit/methods/getMedian.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the median value", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData("test/data/files/data.json"); 8 | assertEquals(await table.getMedian("key1"), 2.5); 9 | await sdb.done(); 10 | }); 11 | 12 | Deno.test("should return the median value rounded", async () => { 13 | const sdb = new SimpleDB(); 14 | const table = sdb.newTable("data"); 15 | await table.loadData("test/data/files/data.json"); 16 | assertEquals( 17 | await table.getMedian("key1", { decimals: 0 }), 18 | 3, 19 | ); 20 | await sdb.done(); 21 | }); 22 | Deno.test("should return the median value even when there are spaces in the column name", async () => { 23 | const sdb = new SimpleDB(); 24 | const table = sdb.newTable("data"); 25 | await table.loadData("test/data/files/data.json"); 26 | await table.renameColumns({ key1: "key 1" }); 27 | assertEquals(await table.getMedian("key 1"), 2.5); 28 | await sdb.done(); 29 | }); 30 | Deno.test("should return the median value rounded even when there are spaces in the column name", async () => { 31 | const sdb = new SimpleDB(); 32 | const table = sdb.newTable("data"); 33 | await table.loadData("test/data/files/data.json"); 34 | await table.renameColumns({ key1: "key 1" }); 35 | assertEquals( 36 | await table.getMedian("key 1", { decimals: 0 }), 37 | 3, 38 | ); 39 | await sdb.done(); 40 | }); 41 | -------------------------------------------------------------------------------- /test/unit/methods/getMin.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the min value", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData("test/data/files/data.json"); 8 | assertEquals(await table.getMin("key1"), 1); 9 | await sdb.done(); 10 | }); 11 | Deno.test("should return the min value even when there are spaces in the column name", async () => { 12 | const sdb = new SimpleDB(); 13 | const table = sdb.newTable("data"); 14 | await table.loadData("test/data/files/data.json"); 15 | await table.renameColumns({ key1: "key 1" }); 16 | assertEquals(await table.getMin("key 1"), 1); 17 | await sdb.done(); 18 | }); 19 | Deno.test("should return the min value with Dates", async () => { 20 | const sdb = new SimpleDB(); 21 | const table = sdb.newTable("data"); 22 | await table.loadArray([ 23 | { key1: new Date("2020-01-01") }, 24 | { key1: new Date("2021-01-01") }, 25 | { key1: new Date("2022-01-01") }, 26 | { key1: new Date("2023-01-01") }, 27 | ]); 28 | assertEquals(await table.getMin("key1"), new Date("2020-01-01")); 29 | await sdb.done(); 30 | }); 31 | -------------------------------------------------------------------------------- /test/unit/methods/getNbColumns.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the number of columns", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData(["test/data/files/employees.json"]); 8 | assertEquals(await table.getNbColumns(), 6); 9 | await sdb.done(); 10 | }); 11 | -------------------------------------------------------------------------------- /test/unit/methods/getNbRows.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the number of a rows in a table", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadArray([ 8 | { name: "Evangeline", age: 21 }, 9 | { name: "Amelia", age: 29 }, 10 | { name: "Marie", age: 30 }, 11 | { name: "Kiara", age: 31 }, 12 | { name: "Isobel", age: 31 }, 13 | { name: "Genevieve", age: 32 }, 14 | { name: "Jane", age: 32 }, 15 | { name: "Chloe", age: 33 }, 16 | { name: "Philip", age: 33 }, 17 | { name: "Morgan", age: 33 }, 18 | { name: "Jeremy", age: 34 }, 19 | { name: "Claudia", age: 35 }, 20 | { name: "Sonny", age: 57 }, 21 | { name: "Frazer", age: 64 }, 22 | { name: "Sarah", age: 64 }, 23 | { name: "Frankie", age: 65 }, 24 | ]); 25 | const length = await table.getNbRows(); 26 | 27 | assertEquals(length, 16); 28 | await sdb.done(); 29 | }); 30 | 31 | Deno.test("should return the number of a rows in a table with nul values", async () => { 32 | const sdb = new SimpleDB(); 33 | const table = sdb.newTable("data"); 34 | await table.loadArray([ 35 | { name: "Evangeline", age: 21 }, 36 | { name: "Amelia", age: 29 }, 37 | { name: "Marie", age: 30 }, 38 | { name: null, age: null }, 39 | { name: "Isobel", age: 31 }, 40 | { name: "Genevieve", age: 32 }, 41 | { name: "Jane", age: 32 }, 42 | { name: "Chloe", age: 33 }, 43 | { name: "Philip", age: 33 }, 44 | { name: "Morgan", age: 33 }, 45 | { name: "Jeremy", age: 34 }, 46 | { name: "Claudia", age: 35 }, 47 | { name: "Sonny", age: 57 }, 48 | { name: "Frazer", age: 64 }, 49 | { name: "Sarah", age: 64 }, 50 | { name: "Frankie", age: 65 }, 51 | ]); 52 | const length = await table.getNbRows(); 53 | 54 | assertEquals(length, 16); 55 | await sdb.done(); 56 | }); 57 | -------------------------------------------------------------------------------- /test/unit/methods/getNbValues.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the number of data points", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData(["test/data/files/data.json"]); 8 | assertEquals(await table.getNbValues(), 8); 9 | await sdb.done(); 10 | }); 11 | -------------------------------------------------------------------------------- /test/unit/methods/getQuantile.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return a quantile", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData("test/data/files/data.json"); 8 | assertEquals(await table.getQuantile("key1", 0.25), 1.75); 9 | await sdb.done(); 10 | }); 11 | 12 | Deno.test("should return a quantile rounded", async () => { 13 | const sdb = new SimpleDB(); 14 | const table = sdb.newTable("data"); 15 | await table.loadData("test/data/files/data.json"); 16 | assertEquals( 17 | await table.getQuantile("key1", 0.25, { 18 | decimals: 1, 19 | }), 20 | 1.8, 21 | ); 22 | await sdb.done(); 23 | }); 24 | Deno.test("should return a quantile even when there are spaces in the column name", async () => { 25 | const sdb = new SimpleDB(); 26 | const table = sdb.newTable("data"); 27 | await table.loadData("test/data/files/data.json"); 28 | await table.renameColumns({ key1: "key 1" }); 29 | assertEquals(await table.getQuantile("key 1", 0.25), 1.75); 30 | await sdb.done(); 31 | }); 32 | Deno.test("should return a quantile rounded even when there are spaces in the column name", async () => { 33 | const sdb = new SimpleDB(); 34 | const table = sdb.newTable("data"); 35 | await table.loadData("test/data/files/data.json"); 36 | await table.renameColumns({ key1: "key 1" }); 37 | assertEquals( 38 | await table.getQuantile("key 1", 0.25, { 39 | decimals: 1, 40 | }), 41 | 1.8, 42 | ); 43 | await sdb.done(); 44 | }); 45 | Deno.test("should return the median with a quantile of 0.5", async () => { 46 | const sdb = new SimpleDB(); 47 | const table = sdb.newTable("data"); 48 | await table.loadData("test/data/files/data.json"); 49 | assertEquals(await table.getQuantile("key1", 0.5), 2.5); 50 | await sdb.done(); 51 | }); 52 | -------------------------------------------------------------------------------- /test/unit/methods/getRow.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return a specific row", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData("test/data/files/employees.csv"); 8 | const data = await table.getRow(`Name === 'Grant, Douglas'`); 9 | 10 | assertEquals(data, { 11 | Name: "Grant, Douglas", 12 | "Hire date": "13-JAN-08", 13 | Job: "Clerk", 14 | Salary: "NaN", 15 | "Department or unit": "50", 16 | "End-of_year-BONUS?": "23,39%", 17 | }); 18 | await sdb.done(); 19 | }); 20 | -------------------------------------------------------------------------------- /test/unit/methods/getSchema.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the schema of a table", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData(["test/data/files/data.json"]); 8 | 9 | const schema = await table.getSchema(); 10 | assertEquals(schema, [ 11 | { 12 | column_name: "key1", 13 | column_type: "BIGINT", 14 | null: "YES", 15 | key: null, 16 | default: null, 17 | extra: null, 18 | }, 19 | { 20 | column_name: "key2", 21 | column_type: "VARCHAR", 22 | null: "YES", 23 | key: null, 24 | default: null, 25 | extra: null, 26 | }, 27 | ]); 28 | 29 | await sdb.done(); 30 | }); 31 | -------------------------------------------------------------------------------- /test/unit/methods/getSkew.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the skew", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData(["test/data/files/dataJustNumbers.csv"]); 8 | assertEquals(await table.getSkew("key1"), 1.6460497551716866); 9 | await sdb.done(); 10 | }); 11 | 12 | Deno.test("should return the skew rounded", async () => { 13 | const sdb = new SimpleDB(); 14 | const table = sdb.newTable("data"); 15 | await table.loadData(["test/data/files/dataJustNumbers.csv"]); 16 | assertEquals( 17 | await table.getSkew("key1", { decimals: 2 }), 18 | 1.65, 19 | ); 20 | await sdb.done(); 21 | }); 22 | Deno.test("should return the skew even when there are spaces in the column name", async () => { 23 | const sdb = new SimpleDB(); 24 | const table = sdb.newTable("data"); 25 | await table.loadData(["test/data/files/dataJustNumbers.csv"]); 26 | await table.renameColumns({ key1: "key 1" }); 27 | assertEquals(await table.getSkew("key 1"), 1.6460497551716866); 28 | await sdb.done(); 29 | }); 30 | Deno.test("should return the skew rounded even when there are spaces in the column name", async () => { 31 | const sdb = new SimpleDB(); 32 | const table = sdb.newTable("data"); 33 | await table.loadData(["test/data/files/dataJustNumbers.csv"]); 34 | await table.renameColumns({ key1: "key 1" }); 35 | assertEquals( 36 | await table.getSkew("key 1", { decimals: 2 }), 37 | 1.65, 38 | ); 39 | await sdb.done(); 40 | }); 41 | -------------------------------------------------------------------------------- /test/unit/methods/getStdDev.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the standard deviation", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData(["test/data/files/data.json"]); 8 | assertEquals( 9 | await table.getStdDev("key1"), 10 | 1.2909944487358056, 11 | ); 12 | await sdb.done(); 13 | }); 14 | Deno.test("should return the standard deviation rounded", async () => { 15 | const sdb = new SimpleDB(); 16 | const table = sdb.newTable("data"); 17 | await table.loadData(["test/data/files/data.json"]); 18 | assertEquals( 19 | await table.getStdDev("key1", { decimals: 3 }), 20 | 1.291, 21 | ); 22 | await sdb.done(); 23 | }); 24 | Deno.test("should return the standard deviation even when there are spaces in the column name", async () => { 25 | const sdb = new SimpleDB(); 26 | const table = sdb.newTable("data"); 27 | await table.loadData(["test/data/files/data.json"]); 28 | await table.renameColumns({ key1: "key 1" }); 29 | assertEquals( 30 | await table.getStdDev("key 1"), 31 | 1.2909944487358056, 32 | ); 33 | await sdb.done(); 34 | }); 35 | Deno.test("should return the standard deviation rounded even when there are spaces in the column name", async () => { 36 | const sdb = new SimpleDB(); 37 | const table = sdb.newTable("data"); 38 | await table.loadData(["test/data/files/data.json"]); 39 | await table.renameColumns({ key1: "key 1" }); 40 | assertEquals( 41 | await table.getStdDev("key 1", { decimals: 3 }), 42 | 1.291, 43 | ); 44 | await sdb.done(); 45 | }); 46 | -------------------------------------------------------------------------------- /test/unit/methods/getSum.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the sum", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData(["test/data/files/data.json"]); 8 | assertEquals(await table.getSum("key1"), 10); 9 | await sdb.done(); 10 | }); 11 | Deno.test("should return the sum even when there are spaces in the column name", async () => { 12 | const sdb = new SimpleDB(); 13 | const table = sdb.newTable("data"); 14 | await table.loadData(["test/data/files/data.json"]); 15 | await table.renameColumns({ key1: "key 1" }); 16 | assertEquals(await table.getSum("key 1"), 10); 17 | await sdb.done(); 18 | }); 19 | -------------------------------------------------------------------------------- /test/unit/methods/getTop.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the top 3", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData(["test/data/files/employees.csv"]); 8 | const data = await table.getTop(3); 9 | assertEquals(data, [ 10 | { 11 | Name: "OConnell, Donald", 12 | "Hire date": "21-JUN-07", 13 | Job: "Clerk", 14 | Salary: "2600", 15 | "Department or unit": "50", 16 | "End-of_year-BONUS?": "1,94%", 17 | }, 18 | { 19 | Name: "OConnell, Donald", 20 | "Hire date": "21-JUN-07", 21 | Job: "Clerk", 22 | Salary: "2600", 23 | "Department or unit": "50", 24 | "End-of_year-BONUS?": "1,94%", 25 | }, 26 | { 27 | Name: "Grant, Douglas", 28 | "Hire date": "13-JAN-08", 29 | Job: "Clerk", 30 | Salary: "NaN", 31 | "Department or unit": "50", 32 | "End-of_year-BONUS?": "23,39%", 33 | }, 34 | ]); 35 | await sdb.done(); 36 | }); 37 | 38 | Deno.test("should return the top 3 with a condition", async () => { 39 | const sdb = new SimpleDB(); 40 | const table = sdb.newTable("data"); 41 | await table.loadData(["test/data/files/employees.csv"]); 42 | const data = await table.getTop(3, { 43 | conditions: `Job = 'Programmer'`, 44 | }); 45 | assertEquals(data, [ 46 | { 47 | Name: "Hunold, Alexander", 48 | "Hire date": "03-JAN-06", 49 | Job: "Programmer", 50 | Salary: "9000", 51 | "Department or unit": "60", 52 | "End-of_year-BONUS?": "23,01%", 53 | }, 54 | { 55 | Name: "Ernst, Bruce", 56 | "Hire date": "21-MAY-07", 57 | Job: "Programmer", 58 | Salary: "6000", 59 | "Department or unit": "60", 60 | "End-of_year-BONUS?": "25,91%", 61 | }, 62 | { 63 | Name: "Austin, David", 64 | "Hire date": "NaN", 65 | Job: "Programmer", 66 | Salary: "4800", 67 | "Department or unit": "null", 68 | "End-of_year-BONUS?": "6,89%", 69 | }, 70 | ]); 71 | await sdb.done(); 72 | }); 73 | -------------------------------------------------------------------------------- /test/unit/methods/getTypes.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the types of a table", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData(["test/data/files/data.csv"]); 8 | 9 | const types = await table.getTypes(); 10 | 11 | assertEquals(types, { key1: "VARCHAR", key2: "VARCHAR" }); 12 | await sdb.done(); 13 | }); 14 | -------------------------------------------------------------------------------- /test/unit/methods/getUniques.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the unique values of a column", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData(["test/data/files/dataDuplicates.csv"]); 8 | 9 | const uniques = await table.getUniques("key1"); 10 | 11 | assertEquals(uniques, ["1", "3", "8", "brioche"]); 12 | await sdb.done(); 13 | }); 14 | -------------------------------------------------------------------------------- /test/unit/methods/getValues.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the values of a column", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData(["test/data/files/data.csv"]); 8 | 9 | const values = await table.getValues("key1"); 10 | 11 | assertEquals(values, ["1", "3", "8", "brioche"]); 12 | await sdb.done(); 13 | }); 14 | Deno.test("should return the values of a column even the name has a space in it", async () => { 15 | const sdb = new SimpleDB(); 16 | const table = sdb.newTable("data"); 17 | await table.loadArray([ 18 | { "key 1": "1", "key2": "2" }, 19 | { "key 1": "3", "key2": "4" }, 20 | ]); 21 | 22 | const values = await table.getValues("key 1"); 23 | 24 | assertEquals(values, ["1", "3"]); 25 | await sdb.done(); 26 | }); 27 | -------------------------------------------------------------------------------- /test/unit/methods/getVar.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the variance", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData(["test/data/files/data.json"]); 8 | assertEquals(await table.getVar("key1"), 1.6666666666666667); 9 | await sdb.done(); 10 | }); 11 | Deno.test("should return the variance rounded", async () => { 12 | const sdb = new SimpleDB(); 13 | const table = sdb.newTable("data"); 14 | await table.loadData(["test/data/files/data.json"]); 15 | assertEquals( 16 | await table.getVar("key1", { decimals: 6 }), 17 | 1.666667, 18 | ); 19 | await sdb.done(); 20 | }); 21 | Deno.test("should return the variance even when there are spaces in the column name", async () => { 22 | const sdb = new SimpleDB(); 23 | const table = sdb.newTable("data"); 24 | await table.loadData(["test/data/files/data.json"]); 25 | await table.renameColumns({ key1: "key 1" }); 26 | assertEquals(await table.getVar("key 1"), 1.6666666666666667); 27 | await sdb.done(); 28 | }); 29 | Deno.test("should return the variance rounded even when there are spaces in the column name", async () => { 30 | const sdb = new SimpleDB(); 31 | const table = sdb.newTable("data"); 32 | await table.loadData(["test/data/files/data.json"]); 33 | await table.renameColumns({ key1: "key 1" }); 34 | assertEquals( 35 | await table.getVar("key 1", { decimals: 6 }), 36 | 1.666667, 37 | ); 38 | await sdb.done(); 39 | }); 40 | -------------------------------------------------------------------------------- /test/unit/methods/hasColumn.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return true when the column is in the data", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.loadData("test/data/files/data.csv"); 8 | assertEquals(await table.hasColumn("key1"), true); 9 | await sdb.done(); 10 | }); 11 | 12 | Deno.test("should return false when the column is not in the data", async () => { 13 | const sdb = new SimpleDB(); 14 | const table = sdb.newTable("data"); 15 | await table.loadData("test/data/files/data.csv"); 16 | assertEquals(await table.hasColumn("keyX"), false); 17 | await sdb.done(); 18 | }); 19 | -------------------------------------------------------------------------------- /test/unit/methods/insertRows.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("add rows in a table", async () => { 5 | const sdb = new SimpleDB(); 6 | 7 | const table = sdb.newTable("data"); 8 | await table.loadData("test/data/files/data.json"); 9 | 10 | await table.insertRows([ 11 | { key1: 5, key2: "cinq" }, 12 | { key1: 6, key2: "six" }, 13 | ]); 14 | 15 | const data = await table.getData(); 16 | 17 | assertEquals(data, [ 18 | { key1: 1, key2: "un" }, 19 | { key1: 2, key2: "deux" }, 20 | { key1: 3, key2: "trois" }, 21 | { key1: 4, key2: "quatre" }, 22 | { key1: 5, key2: "cinq" }, 23 | { key1: 6, key2: "six" }, 24 | ]); 25 | 26 | await sdb.done(); 27 | }); 28 | -------------------------------------------------------------------------------- /test/unit/methods/inside.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should check if geometries are inside other geometries", async () => { 5 | const sdb = new SimpleDB(); 6 | 7 | const points = sdb.newTable("points"); 8 | await points.loadGeoData("test/geodata/files/pointsInside.json"); 9 | await points.renameColumns({ 10 | name: "points", 11 | geom: "geomPoints", 12 | }); 13 | 14 | const polygon = sdb.newTable("polygon"); 15 | await polygon.loadGeoData("test/geodata/files/polygonInside.json"); 16 | await polygon.renameColumns({ 17 | name: "polygon", 18 | geom: "geomPolygon", 19 | }); 20 | 21 | await points.crossJoin(polygon); 22 | await points.inside("geomPoints", "geomPolygon", "isInside"); 23 | await points.selectColumns(["points", "polygon", "isInside"]); 24 | await points.sort({ points: "asc" }); 25 | 26 | const data = await points.getData(); 27 | 28 | assertEquals(data, [ 29 | { points: "pointA", polygon: "container", isInside: false }, 30 | { points: "pointB", polygon: "container", isInside: false }, 31 | { points: "pointC", polygon: "container", isInside: true }, 32 | { points: "pointD", polygon: "container", isInside: true }, 33 | ]); 34 | 35 | await sdb.done(); 36 | }); 37 | -------------------------------------------------------------------------------- /test/unit/methods/latLon.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should extract the lat and lon of points", async () => { 5 | const sdb = new SimpleDB(); 6 | 7 | const table = sdb.newTable(); 8 | await table.loadGeoData("test/geodata/files/pointsInside.json"); 9 | await table.latLon("geom", "lat", "lon"); 10 | await table.removeColumns("geom"); 11 | 12 | const data = await table.getData(); 13 | 14 | assertEquals(data, [ 15 | { 16 | name: "pointA", 17 | lat: 48.241182892559266, 18 | lon: -76.34553248992202, 19 | }, 20 | { name: "pointB", lat: 50.15023361660323, lon: -73.18043031919933 }, 21 | { name: "pointC", lat: 48.47150751404138, lon: -72.78960434234926 }, 22 | { name: "pointD", lat: 47.43075362784262, lon: -72.2926406368759 }, 23 | ]); 24 | 25 | await sdb.done(); 26 | }); 27 | -------------------------------------------------------------------------------- /test/unit/methods/left.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the first two strings", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadArray([ 8 | { firstName: "Nael", lastName: "Shiab" }, 9 | { firstName: "Graeme", lastName: "Bruce" }, 10 | ]); 11 | 12 | await table.left("firstName", 2); 13 | 14 | const data = await table.getData(); 15 | 16 | assertEquals(data, [ 17 | { firstName: "Na", lastName: "Shiab" }, 18 | { firstName: "Gr", lastName: "Bruce" }, 19 | ]); 20 | 21 | await sdb.done(); 22 | }); 23 | -------------------------------------------------------------------------------- /test/unit/methods/length.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should calculate the length of geometries in meters", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadGeoData("test/geodata/files/line.json"); 8 | await table.length("length"); 9 | await table.round("length"); 10 | await table.selectColumns("length"); 11 | const data = await table.getData(); 12 | 13 | assertEquals(data, [{ length: 70175 }]); 14 | await sdb.done(); 15 | }); 16 | 17 | Deno.test("should calculate the length of geometries from a specific column in meters", async () => { 18 | const sdb = new SimpleDB(); 19 | const table = sdb.newTable(); 20 | await table.loadGeoData("test/geodata/files/line.json"); 21 | await table.length("length", { column: "geom" }); 22 | await table.round("length"); 23 | await table.selectColumns("length"); 24 | const data = await table.getData(); 25 | 26 | assertEquals(data, [{ length: 70175 }]); 27 | await sdb.done(); 28 | }); 29 | 30 | Deno.test("should calculate the length of geometries in meters from a file loaded with option toWGS84", async () => { 31 | const sdb = new SimpleDB(); 32 | const table = sdb.newTable(); 33 | await table.loadGeoData("test/geodata/files/line.json"); 34 | await table.length("length"); 35 | await table.round("length"); 36 | await table.selectColumns("length"); 37 | const data = await table.getData(); 38 | 39 | assertEquals(data, [{ length: 70175 }]); 40 | await sdb.done(); 41 | }); 42 | 43 | Deno.test("should calculate the length of geometries in kilometers", async () => { 44 | const sdb = new SimpleDB(); 45 | const table = sdb.newTable(); 46 | await table.loadGeoData("test/geodata/files/line.json"); 47 | await table.length("length", { unit: "km" }); 48 | await table.round("length"); 49 | await table.selectColumns("length"); 50 | const data = await table.getData(); 51 | 52 | assertEquals(data, [{ length: 70 }]); 53 | await sdb.done(); 54 | }); 55 | -------------------------------------------------------------------------------- /test/unit/methods/logBarChart.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should log a bar chart", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | 8 | const data = [ 9 | { category: "A", value: 10 }, 10 | { category: "B", value: 20 }, 11 | ]; 12 | await table.loadArray(data); 13 | await table.logBarChart("category", "value"); 14 | 15 | // How to test? 16 | assertEquals(true, true); 17 | await sdb.done(); 18 | }); 19 | 20 | Deno.test("should log a bar chart with options", async () => { 21 | const sdb = new SimpleDB(); 22 | const table = sdb.newTable(); 23 | 24 | const data = [ 25 | { category: "A", value: 10 }, 26 | { category: "B", value: 20 }, 27 | ]; 28 | await table.loadArray(data); 29 | await table.logBarChart("category", "value", { 30 | formatLabels: (label: unknown) => (label as string).toUpperCase(), 31 | formatValues: (value: unknown) => "$" + (value as number), 32 | }); 33 | 34 | // How to test? 35 | assertEquals(true, true); 36 | await sdb.done(); 37 | }); 38 | -------------------------------------------------------------------------------- /test/unit/methods/logBottom.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should log the last rows", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadData("test/data/files/employees.csv"); 8 | await table.logBottom(5); 9 | 10 | // How to test? 11 | assertEquals(true, true); 12 | await sdb.done(); 13 | }); 14 | -------------------------------------------------------------------------------- /test/unit/methods/logColumns.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should log columns", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadData("test/data/files/employees.csv"); 8 | await table.logColumns(); 9 | 10 | // How to test? 11 | assertEquals(true, true); 12 | await sdb.done(); 13 | }); 14 | Deno.test("should log columns with types", async () => { 15 | const sdb = new SimpleDB(); 16 | const table = sdb.newTable(); 17 | await table.loadData("test/data/files/employees.csv"); 18 | await table.logColumns({ types: true }); 19 | 20 | // How to test? 21 | assertEquals(true, true); 22 | await sdb.done(); 23 | }); 24 | -------------------------------------------------------------------------------- /test/unit/methods/logDescription.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should log a description of the table", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadData("test/data/files/employees.csv"); 8 | 9 | await table.logDescription(); 10 | 11 | // How to test? 12 | assertEquals(true, true); 13 | await sdb.done(); 14 | }); 15 | 16 | Deno.test("should not throw an error when there is no table", async () => { 17 | const sdb = new SimpleDB(); 18 | const table = sdb.newTable(); 19 | await table.logDescription(); 20 | 21 | // How to test? 22 | assertEquals(true, true); 23 | await sdb.done(); 24 | }); 25 | 26 | Deno.test("should log a description of the table containing dates", async () => { 27 | const sdb = new SimpleDB(); 28 | const temperatures = sdb.newTable("temperatures"); 29 | await temperatures.loadData( 30 | "test/data/files/dailyTemperatures.csv", 31 | ); 32 | await temperatures.logDescription(); 33 | 34 | // How to test? 35 | assertEquals(true, true); 36 | await sdb.done(); 37 | }); 38 | -------------------------------------------------------------------------------- /test/unit/methods/logDotChart.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should log a dot chart", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | 8 | const data = [ 9 | { date: new Date("2023-01-01"), value: 10 }, 10 | { date: new Date("2023-02-01"), value: 20 }, 11 | { date: new Date("2023-03-01"), value: 30 }, 12 | { date: new Date("2023-04-01"), value: 40 }, 13 | ]; 14 | await table.loadArray(data); 15 | await table.convert({ date: "string" }, { datetimeFormat: "%x" }); 16 | await table.logDotChart("date", "value"); 17 | 18 | // How to test? 19 | assertEquals(true, true); 20 | await sdb.done(); 21 | }); 22 | 23 | Deno.test("should log a dot chart with small multiples", async () => { 24 | const sdb = new SimpleDB(); 25 | const table = sdb.newTable(); 26 | 27 | const data = [ 28 | { date: new Date("2023-01-01"), value: 10, category: "A" }, 29 | { date: new Date("2023-02-01"), value: 20, category: "A" }, 30 | { date: new Date("2023-03-01"), value: 30, category: "A" }, 31 | { date: new Date("2023-04-01"), value: 40, category: "A" }, 32 | { date: new Date("2023-01-01"), value: 15, category: "B" }, 33 | { date: new Date("2023-02-01"), value: 25, category: "B" }, 34 | { date: new Date("2023-03-01"), value: 35, category: "B" }, 35 | { date: new Date("2023-04-01"), value: 45, category: "B" }, 36 | ]; 37 | await table.loadArray(data); 38 | await table.convert({ date: "string" }, { datetimeFormat: "%x" }); 39 | await table.logDotChart("date", "value", { 40 | smallMultiples: "category", 41 | }); 42 | 43 | // How to test? 44 | assertEquals(true, true); 45 | await sdb.done(); 46 | }); 47 | -------------------------------------------------------------------------------- /test/unit/methods/logExtent.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should log the extent", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadData("test/data/files/employees.csv"); 8 | await table.convert({ "Salary": "number" }, { try: true }); 9 | await table.logExtent("Salary"); 10 | 11 | // How to test? 12 | assertEquals(true, true); 13 | await sdb.done(); 14 | }); 15 | -------------------------------------------------------------------------------- /test/unit/methods/logHistogram.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | import { formatNumber } from "jsr:@nshiab/journalism@1"; 4 | 5 | Deno.test("should log a histogram", async () => { 6 | const sdb = new SimpleDB(); 7 | const table = sdb.newTable(); 8 | 9 | await table.loadData("test/data/files/dailyTemperatures.csv"); 10 | 11 | await table.logHistogram("t"); 12 | // How to test? 13 | assertEquals(true, true); 14 | 15 | await sdb.done(); 16 | }); 17 | 18 | Deno.test("should log a histogram with options", async () => { 19 | const sdb = new SimpleDB(); 20 | const table = sdb.newTable(); 21 | 22 | await table.loadData("test/data/files/dailyTemperatures.csv"); 23 | 24 | await table.logHistogram("t", { 25 | width: 10, 26 | bins: 25, 27 | compact: true, 28 | formatLabels(a, b) { 29 | return `${formatNumber(a, { decimals: 1 })} to ${ 30 | formatNumber(b, { decimals: 1 }) 31 | }°C`; 32 | }, 33 | }); 34 | // How to test? 35 | assertEquals(true, true); 36 | 37 | await sdb.done(); 38 | }); 39 | -------------------------------------------------------------------------------- /test/unit/methods/logLineChart.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should log a line chart", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | 8 | const data = [ 9 | { date: new Date("2023-01-01"), value: 10 }, 10 | { date: new Date("2023-02-01"), value: 20 }, 11 | { date: new Date("2023-03-01"), value: 30 }, 12 | { date: new Date("2023-04-01"), value: 40 }, 13 | ]; 14 | await table.loadArray(data); 15 | await table.convert({ date: "string" }, { datetimeFormat: "%x" }); 16 | await table.logLineChart("date", "value"); 17 | 18 | // How to test? 19 | assertEquals(true, true); 20 | await sdb.done(); 21 | }); 22 | 23 | Deno.test("should log a line chart with small multiples", async () => { 24 | const sdb = new SimpleDB(); 25 | const table = sdb.newTable(); 26 | 27 | const data = [ 28 | { date: new Date("2023-01-01"), value: 10, category: "A" }, 29 | { date: new Date("2023-02-01"), value: 20, category: "A" }, 30 | { date: new Date("2023-03-01"), value: 30, category: "A" }, 31 | { date: new Date("2023-04-01"), value: 40, category: "A" }, 32 | { date: new Date("2023-01-01"), value: 15, category: "B" }, 33 | { date: new Date("2023-02-01"), value: 25, category: "B" }, 34 | { date: new Date("2023-03-01"), value: 35, category: "B" }, 35 | { date: new Date("2023-04-01"), value: 45, category: "B" }, 36 | ]; 37 | await table.loadArray(data); 38 | await table.convert({ date: "string" }, { datetimeFormat: "%x" }); 39 | await table.logLineChart("date", "value", { 40 | smallMultiples: "category", 41 | }); 42 | 43 | // How to test? 44 | assertEquals(true, true); 45 | await sdb.done(); 46 | }); 47 | -------------------------------------------------------------------------------- /test/unit/methods/logNbRows.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should log the number of rows", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadData("test/data/files/employees.csv"); 8 | await table.logNbRows(); 9 | 10 | // How to test? 11 | assertEquals(true, true); 12 | await sdb.done(); 13 | }); 14 | -------------------------------------------------------------------------------- /test/unit/methods/logProjections.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should log the projections of the table, even if there is none", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadData("test/data/files/employees.csv"); 8 | 9 | await table.logProjections(); 10 | 11 | // How to test? 12 | assertEquals(true, true); 13 | await sdb.done(); 14 | }); 15 | Deno.test("should log the projections of the table (Lambert conformal conic)", async () => { 16 | const sdb = new SimpleDB(); 17 | const table = sdb.newTable(); 18 | await table.loadGeoData("test/geodata/files/canada-not-4326.shp.zip"); 19 | 20 | await table.logProjections(); 21 | 22 | // How to test? 23 | assertEquals(true, true); 24 | await sdb.done(); 25 | }); 26 | Deno.test("should log the projections of the table (Lambert conformal conic converted to WGS84)", async () => { 27 | const sdb = new SimpleDB(); 28 | const table = sdb.newTable(); 29 | await table.loadGeoData("test/geodata/files/canada-not-4326.shp.zip", { 30 | toWGS84: true, 31 | }); 32 | 33 | await table.logProjections(); 34 | 35 | // How to test? 36 | assertEquals(true, true); 37 | await sdb.done(); 38 | }); 39 | Deno.test("should log the projections of the table (geojson WGS84)", async () => { 40 | const sdb = new SimpleDB(); 41 | const table = sdb.newTable(); 42 | await table.loadGeoData( 43 | "test/geodata/files/CanadianProvincesAndTerritories.json", 44 | ); 45 | 46 | await table.logProjections(); 47 | 48 | // How to test? 49 | assertEquals(true, true); 50 | await sdb.done(); 51 | }); 52 | -------------------------------------------------------------------------------- /test/unit/methods/logUniques.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should log unique values in a column", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadData("test/data/files/employees.csv"); 8 | await table.logUniques("Name"); 9 | 10 | // How to test? 11 | assertEquals(true, true); 12 | await sdb.done(); 13 | }); 14 | 15 | Deno.test("should log stringified unique values in a column", async () => { 16 | const sdb = new SimpleDB(); 17 | const table = sdb.newTable(); 18 | await table.loadData("test/data/files/employees.csv"); 19 | await table.logUniques("Name", { stringify: true }); 20 | 21 | // How to test? 22 | assertEquals(true, true); 23 | await sdb.done(); 24 | }); 25 | -------------------------------------------------------------------------------- /test/unit/methods/lower.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should lowercase strings in one column", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadArray([{ firstName: "NAEL", lastName: "SHIAB" }]); 8 | 9 | await table.lower("firstName"); 10 | 11 | const data = await table.getData(); 12 | 13 | assertEquals(data, [{ firstName: "nael", lastName: "SHIAB" }]); 14 | await sdb.done(); 15 | }); 16 | 17 | Deno.test("should lowercase strings in two columns", async () => { 18 | const sdb = new SimpleDB(); 19 | const table = sdb.newTable(); 20 | await table.loadArray([{ firstName: "NAEL", lastName: "SHIAB" }]); 21 | 22 | await table.lower(["firstName", "lastName"]); 23 | 24 | const data = await table.getData(); 25 | 26 | assertEquals(data, [{ firstName: "nael", lastName: "shiab" }]); 27 | await sdb.done(); 28 | }); 29 | 30 | Deno.test("should lowercase strings in two columns with column names containing spaces", async () => { 31 | const sdb = new SimpleDB(); 32 | const table = sdb.newTable(); 33 | await table.loadArray([{ "first Name": "NAEL", "last Name": "SHIAB" }]); 34 | 35 | await table.lower(["first Name", "last Name"]); 36 | 37 | const data = await table.getData(); 38 | 39 | assertEquals(data, [ 40 | { "first Name": "nael", "last Name": "shiab" }, 41 | ]); 42 | await sdb.done(); 43 | }); 44 | -------------------------------------------------------------------------------- /test/unit/methods/nbVertices.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should count the number of vertices and add the result in a new column", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("geodata"); 7 | await table.loadGeoData("test/geodata/files/triangle.json"); 8 | await table.nbVertices("nbVertices"); 9 | await table.selectColumns(["nbVertices"]); 10 | 11 | const data = await table.getData(); 12 | 13 | assertEquals(data, [{ nbVertices: 4 }]); 14 | await sdb.done(); 15 | }); 16 | 17 | Deno.test("should count the number of vertices when checking a specific column", async () => { 18 | const sdb = new SimpleDB(); 19 | const table = sdb.newTable("geodata"); 20 | await table.loadGeoData("test/geodata/files/triangle.json"); 21 | await table.nbVertices("nbVertices", { column: "geom" }); 22 | await table.selectColumns(["nbVertices"]); 23 | const data = await table.getData(); 24 | 25 | assertEquals(data, [{ nbVertices: 4 }]); 26 | await sdb.done(); 27 | }); 28 | -------------------------------------------------------------------------------- /test/unit/methods/points.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should create points", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadData("test/geodata/files/coordinates.csv"); 8 | await table.convert({ lat: "double", lon: "double" }); 9 | await table.points("lat", "lon", "geom"); 10 | 11 | const data = await table.getGeoData("geom"); 12 | 13 | assertEquals(data, { 14 | type: "FeatureCollection", 15 | features: [ 16 | { 17 | type: "Feature", 18 | geometry: { type: "Point", coordinates: [-79.29, 43.77] }, 19 | properties: { name: "montreal", lat: 43.77, lon: -79.29 }, 20 | }, 21 | { 22 | type: "Feature", 23 | geometry: { type: "Point", coordinates: [-73.86, 45.35] }, 24 | properties: { name: "toronto", lat: 45.35, lon: -73.86 }, 25 | }, 26 | { 27 | type: "Feature", 28 | geometry: { type: "Point", coordinates: [-122.96, 49.07] }, 29 | properties: { name: "vancouver", lat: 49.07, lon: -122.96 }, 30 | }, 31 | ], 32 | }); 33 | 34 | await sdb.done(); 35 | }); 36 | 37 | Deno.test("should create points and add a projection", async () => { 38 | const sdb = new SimpleDB(); 39 | const table = sdb.newTable(); 40 | await table.loadData("test/geodata/files/coordinates.csv"); 41 | await table.convert({ lat: "double", lon: "double" }); 42 | await table.points("lat", "lon", "geom"); 43 | 44 | assertEquals(table.projections, { 45 | geom: "+proj=latlong +datum=WGS84 +no_defs", 46 | }); 47 | 48 | await sdb.done(); 49 | }); 50 | -------------------------------------------------------------------------------- /test/unit/methods/quantiles.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should add a column with the quantiles", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadData("test/data/files/dataRank.csv"); 8 | await table.quantiles("Mark", 4, "quantiles"); 9 | const data = await table.getData(); 10 | 11 | assertEquals(data, [ 12 | { Name: "Isabella", Subject: "Maths", Mark: 50, quantiles: 1 }, 13 | { Name: "Olivia", Subject: "Maths", Mark: 55, quantiles: 1 }, 14 | { Name: "Olivia", Subject: "Science", Mark: 60, quantiles: 1 }, 15 | { Name: "Lily", Subject: "Maths", Mark: 65, quantiles: 2 }, 16 | { Name: "Lily", Subject: "English", Mark: 70, quantiles: 2 }, 17 | { Name: "Isabella", Subject: "Science", Mark: 70, quantiles: 3 }, 18 | { Name: "Lily", Subject: "Science", Mark: 80, quantiles: 3 }, 19 | { Name: "Olivia", Subject: "English", Mark: 89, quantiles: 4 }, 20 | { Name: "Isabella", Subject: "English", Mark: 90, quantiles: 4 }, 21 | ]); 22 | 23 | await sdb.done(); 24 | }); 25 | 26 | Deno.test("should add a column with the quantiles after grouping", async () => { 27 | const sdb = new SimpleDB(); 28 | const table = sdb.newTable(); 29 | await table.loadData("test/data/files/dataRank.csv"); 30 | await table.quantiles("Mark", 2, "quantiles", { 31 | categories: "Subject", 32 | }); 33 | 34 | await table.sort({ 35 | Subject: "asc", 36 | Mark: "asc", 37 | }); 38 | 39 | const data = await table.getData(); 40 | 41 | assertEquals(data, [ 42 | { Name: "Lily", Subject: "English", Mark: 70, quantiles: 1 }, 43 | { Name: "Olivia", Subject: "English", Mark: 89, quantiles: 1 }, 44 | { Name: "Isabella", Subject: "English", Mark: 90, quantiles: 2 }, 45 | { Name: "Isabella", Subject: "Maths", Mark: 50, quantiles: 1 }, 46 | { Name: "Olivia", Subject: "Maths", Mark: 55, quantiles: 1 }, 47 | { Name: "Lily", Subject: "Maths", Mark: 65, quantiles: 2 }, 48 | { Name: "Olivia", Subject: "Science", Mark: 60, quantiles: 1 }, 49 | { Name: "Isabella", Subject: "Science", Mark: 70, quantiles: 1 }, 50 | { Name: "Lily", Subject: "Science", Mark: 80, quantiles: 2 }, 51 | ]); 52 | 53 | await sdb.done(); 54 | }); 55 | -------------------------------------------------------------------------------- /test/unit/methods/reducePrecision.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should round the coordinates to 3 decimals", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("geoData"); 7 | await table.loadGeoData("test/geodata/files/point.json"); 8 | await table.reducePrecision(3); 9 | const data = await sdb.customQuery( 10 | `SELECT ST_AsText(geom) as geomText FROM geoData;`, 11 | { returnDataFrom: "query" }, 12 | ); 13 | assertEquals(data, [{ geomText: "POINT (45.514 -73.623)" }]); 14 | await sdb.done(); 15 | }); 16 | 17 | Deno.test("should round the coordinates to 3 decimals from a specific column", async () => { 18 | const sdb = new SimpleDB(); 19 | const table = sdb.newTable("geoData"); 20 | await table.loadGeoData("test/geodata/files/point.json"); 21 | await table.reducePrecision(3, { column: "geom" }); 22 | const data = await sdb.customQuery( 23 | `SELECT ST_AsText(geom) as geomText FROM geoData;`, 24 | { returnDataFrom: "query" }, 25 | ); 26 | assertEquals(data, [{ geomText: "POINT (45.514 -73.623)" }]); 27 | await sdb.done(); 28 | }); 29 | -------------------------------------------------------------------------------- /test/unit/methods/renameColumns.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should change the name of one column", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadData(["test/data/files/data.json"]); 8 | 9 | await table.renameColumns({ 10 | key1: "A", 11 | }); 12 | const data = await table.getData(); 13 | 14 | assertEquals(data, [ 15 | { A: 1, key2: "un" }, 16 | { A: 2, key2: "deux" }, 17 | { A: 3, key2: "trois" }, 18 | { A: 4, key2: "quatre" }, 19 | ]); 20 | 21 | await sdb.done(); 22 | }); 23 | 24 | Deno.test("should change the name of multiple columns", async () => { 25 | const sdb = new SimpleDB(); 26 | const table = sdb.newTable(); 27 | await table.loadData(["test/data/files/data.json"]); 28 | 29 | await table.renameColumns({ 30 | key1: "A", 31 | key2: "B", 32 | }); 33 | const data = await table.getData(); 34 | 35 | assertEquals(data, [ 36 | { A: 1, B: "un" }, 37 | { A: 2, B: "deux" }, 38 | { A: 3, B: "trois" }, 39 | { A: 4, B: "quatre" }, 40 | ]); 41 | 42 | await sdb.done(); 43 | }); 44 | 45 | Deno.test("should change the name of a column with $ in its name", async () => { 46 | const sdb = new SimpleDB(); 47 | const table = sdb.newTable(); 48 | await table.loadArray([{ "$ value": 10 }, { "$ value": 20 }]); 49 | 50 | await table.renameColumns({ 51 | "$ value": "value", 52 | }); 53 | const data = await table.getData(); 54 | 55 | assertEquals(data, [ 56 | { value: 10 }, 57 | { value: 20 }, 58 | ]); 59 | 60 | await sdb.done(); 61 | }); 62 | -------------------------------------------------------------------------------- /test/unit/methods/renameTable.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should rename a table", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadData(["test/data/files/cities.csv"]); 8 | await table.renameTable("canadianCities"); 9 | 10 | const tables = await sdb.getTableNames(); 11 | 12 | assertEquals(tables, ["canadianCities"]); 13 | await sdb.done(); 14 | }); 15 | -------------------------------------------------------------------------------- /test/unit/methods/replaceNulls.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should replace null values in one column", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadArray([ 8 | { keyA: 1 }, 9 | { keyA: null }, 10 | { keyA: 3 }, 11 | { keyA: null }, 12 | ]); 13 | await table.replaceNulls("keyA", 0); 14 | 15 | const data = await table.getData(); 16 | 17 | assertEquals(data, [ 18 | { keyA: 1 }, 19 | { keyA: 0 }, 20 | { keyA: 3 }, 21 | { keyA: 0 }, 22 | ]); 23 | 24 | await sdb.done(); 25 | }); 26 | 27 | Deno.test("should replace null values in multiple columns", async () => { 28 | const sdb = new SimpleDB(); 29 | const table = sdb.newTable(); 30 | await table.loadArray([ 31 | { keyA: 1, keyB: 1 }, 32 | { keyA: null, keyB: 2 }, 33 | { keyA: 3, keyB: null }, 34 | { keyA: null, keyB: 4 }, 35 | ]); 36 | await table.replaceNulls(["keyA", "keyB"], 0); 37 | 38 | const data = await table.getData(); 39 | 40 | assertEquals(data, [ 41 | { keyA: 1, keyB: 1 }, 42 | { keyA: 0, keyB: 2 }, 43 | { keyA: 3, keyB: 0 }, 44 | { keyA: 0, keyB: 4 }, 45 | ]); 46 | 47 | await sdb.done(); 48 | }); 49 | -------------------------------------------------------------------------------- /test/unit/methods/right.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should return the last two strings", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadArray([ 8 | { firstName: "Nael", lastName: "Shiab" }, 9 | { firstName: "Graeme", lastName: "Bruce" }, 10 | ]); 11 | 12 | await table.right("firstName", 2); 13 | 14 | const data = await table.getData(); 15 | 16 | assertEquals(data, [ 17 | { firstName: "el", lastName: "Shiab" }, 18 | { firstName: "me", lastName: "Bruce" }, 19 | ]); 20 | 21 | await sdb.done(); 22 | }); 23 | -------------------------------------------------------------------------------- /test/unit/methods/setTypes.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should create a new SimpleTable with types", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable("data"); 7 | await table.setTypes({ name: "string", age: "number" }); 8 | const types = await table.getTypes(); 9 | assertEquals(types, { name: "VARCHAR", age: "DOUBLE" }); 10 | await sdb.done(); 11 | }); 12 | 13 | Deno.test("should create a new SimpleTable with geometry in types", async () => { 14 | const sdb = new SimpleDB(); 15 | const table = sdb.newTable("data"); 16 | await table.setTypes({ 17 | name: "string", 18 | age: "number", 19 | city: "geometry", 20 | }); 21 | const types = await table.getTypes(); 22 | assertEquals( 23 | { name: "VARCHAR", age: "DOUBLE", city: "GEOMETRY" }, 24 | types, 25 | ); 26 | await sdb.done(); 27 | }); 28 | -------------------------------------------------------------------------------- /test/unit/methods/skip.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should skip rows", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = await sdb 7 | .newTable() 8 | .loadArray([ 9 | { first: "Nael" }, 10 | { first: "Graeme" }, 11 | { first: "Andrew" }, 12 | ]); 13 | await table.skip(1); 14 | const data = await table.getData(); 15 | assertEquals(data, [{ first: "Graeme" }, { first: "Andrew" }]); 16 | await sdb.done(); 17 | }); 18 | -------------------------------------------------------------------------------- /test/unit/methods/splitExtract.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should extract a substring based on a separator and substring", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadArray([ 8 | { name: "Shiab, Nael" }, 9 | { name: "Bruce, Graeme" }, 10 | ]); 11 | 12 | await table.splitExtract("name", ",", 0, "lastName"); 13 | 14 | const data = await table.getData(); 15 | 16 | assertEquals(data, [{ name: "Shiab, Nael", lastName: "Shiab" }, { 17 | name: "Bruce, Graeme", 18 | lastName: "Bruce", 19 | }]); 20 | await sdb.done(); 21 | }); 22 | Deno.test("should extract a substring based on a separator and substring, and overwrite the original column", async () => { 23 | const sdb = new SimpleDB(); 24 | const table = sdb.newTable(); 25 | await table.loadArray([ 26 | { name: "Shiab, Nael" }, 27 | { name: "Bruce, Graeme" }, 28 | ]); 29 | 30 | await table.splitExtract("name", ",", 0, "name"); 31 | 32 | const data = await table.getData(); 33 | 34 | assertEquals(data, [{ name: "Shiab" }, { name: "Bruce" }]); 35 | await sdb.done(); 36 | }); 37 | -------------------------------------------------------------------------------- /test/unit/methods/toSheet.test.ts: -------------------------------------------------------------------------------- 1 | import "jsr:@std/dotenv/load"; 2 | import { assertEquals } from "jsr:@std/assert"; 3 | import { SimpleDB } from "../../../src/index.ts"; 4 | 5 | const email = Deno.env.get("GOOGLE_SERVICE_ACCOUNT_EMAIL"); 6 | const key = Deno.env.get("GOOGLE_PRIVATE_KEY"); 7 | 8 | if ( 9 | typeof email === "string" && 10 | email !== "" && 11 | typeof key === "string" && 12 | key !== "" 13 | ) { 14 | Deno.test("should write the data to a google sheet", async () => { 15 | const sdb = new SimpleDB(); 16 | const table = sdb.newTable(); 17 | await table.loadArray([ 18 | { first: "Nael", last: "Shiab" }, 19 | { first: "Andrew", last: "Ryan" }, 20 | ]); 21 | await table.toSheet( 22 | "https://docs.google.com/spreadsheets/d/1Ar19cP8oGYEzacfrkLWnSH7ZqImILMUrosBwnZ43EQM/edit#gid=0", 23 | { 24 | prepend: "Hi!", 25 | lastUpdate: true, 26 | timeZone: "Canada/Eastern", 27 | }, 28 | ); 29 | 30 | // Just for now. 31 | assertEquals( 32 | true, 33 | true, 34 | ); 35 | }); 36 | } else { 37 | console.log( 38 | "No GOOGLE_SERVICE_ACCOUNT_EMAIL or GOOGLE_PRIVATE_KEY in process.env", 39 | ); 40 | } 41 | -------------------------------------------------------------------------------- /test/unit/methods/updateColumn.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should update a column", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadData(["test/data/files/cities.csv"]); 8 | await table.updateColumn("city", `left("city", 3)`); 9 | 10 | const data = await table.getData(); 11 | 12 | assertEquals(data, [ 13 | { id: 1108380, city: "VAN" }, 14 | { id: 6158355, city: "TOR" }, 15 | { id: 7024745, city: "MON" }, 16 | ]); 17 | 18 | await sdb.done(); 19 | }); 20 | 21 | Deno.test("should update a column with a space in its name", async () => { 22 | const sdb = new SimpleDB(); 23 | const table = sdb.newTable(); 24 | await table.loadData("test/data/files/employees.csv"); 25 | await table.updateColumn( 26 | "Department or unit", 27 | `left("Department or unit", 1)`, 28 | ); 29 | await table.selectRows(3); 30 | 31 | const data = await table.getData(); 32 | 33 | assertEquals(data, [ 34 | { 35 | Name: "OConnell, Donald", 36 | "Hire date": "21-JUN-07", 37 | Job: "Clerk", 38 | Salary: "2600", 39 | "Department or unit": "5", 40 | "End-of_year-BONUS?": "1,94%", 41 | }, 42 | { 43 | Name: "OConnell, Donald", 44 | "Hire date": "21-JUN-07", 45 | Job: "Clerk", 46 | Salary: "2600", 47 | "Department or unit": "5", 48 | "End-of_year-BONUS?": "1,94%", 49 | }, 50 | { 51 | Name: "Grant, Douglas", 52 | "Hire date": "13-JAN-08", 53 | Job: "Clerk", 54 | Salary: "NaN", 55 | "Department or unit": "5", 56 | "End-of_year-BONUS?": "23,39%", 57 | }, 58 | ]); 59 | 60 | await sdb.done(); 61 | }); 62 | -------------------------------------------------------------------------------- /test/unit/methods/upper.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals } from "jsr:@std/assert"; 2 | import SimpleDB from "../../../src/class/SimpleDB.ts"; 3 | 4 | Deno.test("should uppercase strings in one column", async () => { 5 | const sdb = new SimpleDB(); 6 | const table = sdb.newTable(); 7 | await table.loadArray([{ firstName: "nael", lastName: "shiab" }]); 8 | 9 | await table.upper("firstName"); 10 | 11 | const data = await table.getData(); 12 | 13 | assertEquals(data, [{ firstName: "NAEL", lastName: "shiab" }]); 14 | await sdb.done(); 15 | }); 16 | 17 | Deno.test("should uppercase strings in two columns", async () => { 18 | const sdb = new SimpleDB(); 19 | const table = sdb.newTable(); 20 | await table.loadArray([{ firstName: "nael", lastName: "shiab" }]); 21 | 22 | await table.upper(["firstName", "lastName"]); 23 | 24 | const data = await table.getData(); 25 | 26 | assertEquals(data, [{ firstName: "NAEL", lastName: "SHIAB" }]); 27 | await sdb.done(); 28 | }); 29 | 30 | Deno.test("should uppercase strings in two columns with column names containing spaces", async () => { 31 | const sdb = new SimpleDB(); 32 | const table = sdb.newTable(); 33 | await table.loadArray([{ "first Name": "nael", "last Name": "shiab" }]); 34 | 35 | await table.upper(["first Name", "last Name"]); 36 | 37 | const data = await table.getData(); 38 | 39 | assertEquals(data, [ 40 | { "first Name": "NAEL", "last Name": "SHIAB" }, 41 | ]); 42 | await sdb.done(); 43 | }); 44 | --------------------------------------------------------------------------------