├── .devcontainer ├── Dockerfile └── devcontainer.json ├── .dir-locals.el ├── .gitattributes ├── .github └── workflows │ ├── build-and-deploy.yml │ ├── prs.yml │ └── prs_doc_preview.yml ├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── LICENSE ├── README-source.md ├── README.md ├── clay.edn ├── data ├── anscombe.csv ├── billboard.csv.gz ├── construction.csv ├── contacts.csv ├── family.csv ├── fish_encounters.csv ├── iris.csv ├── production.csv ├── relig_income.csv ├── stockstidyr.csv ├── us_rent_income.csv ├── warpbreaks.csv ├── who.csv.gz └── world_bank_pop.csv.gz ├── deps.edn ├── dev ├── conversion.clj ├── gendocs.clj └── readme_generation.clj ├── docs ├── .clay.html ├── .nojekyll ├── better_tables.html ├── column_api.html ├── column_api.qmd ├── column_api_files │ ├── bootstrap0.css │ ├── bootstrap2.css │ ├── html-default1.js │ ├── html-default2.js │ ├── html-default3.js │ ├── libs │ │ ├── bootstrap │ │ │ ├── bootstrap-icons.css │ │ │ ├── bootstrap-icons.woff │ │ │ ├── bootstrap.min.css │ │ │ └── bootstrap.min.js │ │ ├── clipboard │ │ │ └── clipboard.min.js │ │ └── quarto-html │ │ │ ├── anchor.min.js │ │ │ ├── popper.min.js │ │ │ ├── quarto-syntax-highlighting.css │ │ │ ├── quarto.js │ │ │ ├── tippy.css │ │ │ └── tippy.umd.min.js │ ├── md-default0.js │ └── md-default1.js ├── column_exploration.clj ├── column_exploration.html ├── index.html ├── index_files │ ├── bootstrap0.css │ ├── bootstrap2.css │ ├── html-default0.js │ ├── html-default1.js │ ├── html-default2.js │ ├── html-default3.js │ ├── katex2.js │ ├── libs │ │ ├── bootstrap │ │ │ ├── bootstrap-881eef0dd19e1eadb2abebd600cbfa02.min.css │ │ │ ├── bootstrap-icons.css │ │ │ ├── bootstrap-icons.woff │ │ │ ├── bootstrap.min.css │ │ │ └── bootstrap.min.js │ │ ├── clipboard │ │ │ └── clipboard.min.js │ │ └── quarto-html │ │ │ ├── anchor.min.js │ │ │ ├── popper.min.js │ │ │ ├── quarto-syntax-highlighting-e26fc0b31b203507ebcf2fc6137e19e2.css │ │ │ ├── quarto-syntax-highlighting.css │ │ │ ├── quarto.js │ │ │ ├── tabsets │ │ │ └── tabsets.js │ │ │ ├── tippy.css │ │ │ └── tippy.umd.min.js │ ├── md-default0.js │ └── md-default1.js ├── notebooks │ └── custom.scss └── old │ ├── README.Rmd │ ├── index.Rmd │ ├── index.html │ ├── index.md │ └── index.pdf ├── notebooks ├── custom.scss └── index.clj ├── playground.clj ├── pr-preview └── pr-100 │ ├── .clay_files │ ├── bootstrap0.css │ ├── html-default1.js │ ├── html-default2.js │ └── html-default3.js │ ├── better_tables.html │ ├── column_api.html │ ├── column_api_files │ ├── bootstrap0.css │ ├── bootstrap2.css │ ├── html-default1.js │ ├── html-default2.js │ ├── html-default3.js │ ├── libs │ │ ├── bootstrap │ │ │ ├── bootstrap-icons.css │ │ │ ├── bootstrap-icons.woff │ │ │ ├── bootstrap.min.css │ │ │ └── bootstrap.min.js │ │ ├── clipboard │ │ │ └── clipboard.min.js │ │ └── quarto-html │ │ │ ├── anchor.min.js │ │ │ ├── popper.min.js │ │ │ ├── quarto-syntax-highlighting.css │ │ │ ├── quarto.js │ │ │ ├── tippy.css │ │ │ └── tippy.umd.min.js │ ├── md-default0.js │ └── md-default1.js │ ├── column_exploration.clj │ ├── column_exploration.html │ ├── index.html │ ├── index_files │ ├── bootstrap0.css │ ├── bootstrap2.css │ ├── html-default0.js │ ├── html-default1.js │ ├── html-default2.js │ ├── html-default3.js │ ├── libs │ │ ├── bootstrap │ │ │ ├── bootstrap-icons.css │ │ │ ├── bootstrap-icons.woff │ │ │ ├── bootstrap.min.css │ │ │ └── bootstrap.min.js │ │ ├── clipboard │ │ │ └── clipboard.min.js │ │ └── quarto-html │ │ │ ├── anchor.min.js │ │ │ ├── popper.min.js │ │ │ ├── quarto-syntax-highlighting.css │ │ │ ├── quarto.js │ │ │ ├── tippy.css │ │ │ └── tippy.umd.min.js │ ├── md-default0.js │ └── md-default1.js │ ├── notebooks │ └── custom.scss │ └── old │ ├── README.Rmd │ ├── index.Rmd │ ├── index.html │ ├── index.md │ └── index.pdf ├── project.clj ├── src └── tablecloth │ ├── api.clj │ ├── api │ ├── aggregate.clj │ ├── api_template.clj │ ├── columns.clj │ ├── dataset.clj │ ├── fold_unroll.clj │ ├── group_by.clj │ ├── join_concat_ds.clj │ ├── join_separate.clj │ ├── lift_operators.clj │ ├── missing.clj │ ├── operators.clj │ ├── order_by.clj │ ├── reshape.clj │ ├── rows.clj │ ├── split.clj │ ├── unique_by.clj │ └── utils.clj │ ├── column │ ├── api.clj │ └── api │ │ ├── api_template.clj │ │ ├── column.clj │ │ ├── lift_operators.clj │ │ ├── missing.clj │ │ ├── operators.clj │ │ └── utils.clj │ ├── pipeline.clj │ └── utils │ └── codegen.clj └── test └── tablecloth ├── api ├── aggregate_test.clj ├── columns_test.clj ├── dataset_test.clj ├── fold_unroll_test.clj ├── group_by_test.clj ├── join_concat_ds_test.clj ├── join_separate_test.clj ├── missing_test.clj ├── operators_test.clj ├── reshape_test.clj └── utils_test.clj ├── column └── api │ ├── column_test.clj │ ├── missing_test.clj │ └── operators_test.clj └── common_test.clj /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM clojure:temurin-21-tools-deps-1.11.3.1456-jammy 2 | 3 | ARG USERNAME=vscode 4 | ARG USER_UID=1000 5 | ARG USER_GID=$USER_UID 6 | 7 | # Create the user 8 | RUN groupadd --gid $USER_GID $USERNAME \ 9 | && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \ 10 | # 11 | # [Optional] Add sudo support. Omit if you don't need to install software after connecting. 12 | && apt-get update \ 13 | && apt-get install -y sudo \ 14 | && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \ 15 | && chmod 0440 /etc/sudoers.d/$USERNAME 16 | 17 | # ******************************************************** 18 | # * Anything else you want to do like clean up goes here * 19 | # ******************************************************** 20 | 21 | # [Optional] Set the default user. Omit if you want to keep the default as root. 22 | USER $USERNAME 23 | SHELL ["/bin/bash", "-ec"] 24 | ENTRYPOINT ["bash"] 25 | 26 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "build": { 3 | "dockerfile": "Dockerfile" 4 | }, 5 | 6 | "features": { 7 | "ghcr.io/devcontainers-contrib/features/apt-get-packages:1": { 8 | "packages": "r-base-dev,rlwrap" 9 | }, 10 | "ghcr.io/rocker-org/devcontainer-features/quarto-cli:1": {}, 11 | "ghcr.io/rocker-org/devcontainer-features/r-apt:0": {}, 12 | "ghcr.io/rocker-org/devcontainer-features/r-packages:1": { 13 | "packages": "Rserve,data.table,rmarkdown,knitr", 14 | "additionalRepositories": "rforge= 'https://rforge.net'", 15 | "installSystemRequirements": true 16 | }, 17 | "ghcr.io/wxw-matt/devcontainer-features/command_runner:latest": { 18 | "command1": "bash < <(curl -s https://raw.githubusercontent.com/clojure-lsp/clojure-lsp/master/install)", 19 | "command2": "bash < <(curl -s https://raw.githubusercontent.com/babashka/babashka/master/install)", 20 | "command3": "bash -c 'wget https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein -O /usr/local/bin/lein && chmod +x /usr/local/bin/lein'" 21 | } 22 | }, 23 | "overrideFeatureInstallOrder": [ 24 | "ghcr.io/rocker-org/devcontainer-features/r-apt", 25 | "ghcr.io/devcontainers-contrib/features/apt-get-packages", 26 | "ghcr.io/rocker-org/devcontainer-features/r-packages" 27 | ], 28 | 29 | "customizations": { 30 | "vscode": { 31 | "extensions": [ 32 | "betterthantomorrow.calva" 33 | ] 34 | } 35 | } 36 | 37 | } 38 | 39 | 40 | -------------------------------------------------------------------------------- /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ((nil 2 | . 3 | ((cider-clojure-cli-aliases 4 | . 5 | "dev:test")))) 6 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | docs/** linguist-documentation 2 | pr-preview/** linguist-documentation 3 | -------------------------------------------------------------------------------- /.github/workflows/build-and-deploy.yml: -------------------------------------------------------------------------------- 1 | name: Build and Deploy 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | workflow_dispatch: 8 | 9 | # This is just a guard against two of these actions 10 | # running at the same time. Not likely an issue for 11 | # us right now, but it could happen. 12 | concurrency: build-and-deploy-${{ github.ref }} 13 | 14 | jobs: 15 | deploy-docs: 16 | runs-on: ubuntu-latest 17 | 18 | steps: 19 | - name: Checkout 20 | uses: actions/checkout@v4 21 | 22 | - name: Deploy documentation 23 | uses: JamesIves/github-pages-deploy-action@v4 24 | with: 25 | folder: docs 26 | branch: gh-pages # default for this action, but including to be explicit 27 | 28 | -------------------------------------------------------------------------------- /.github/workflows/prs.yml: -------------------------------------------------------------------------------- 1 | name: "PR Checks" 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - master 7 | 8 | jobs: 9 | run-tests: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - name: Checkout 14 | uses: actions/checkout@v2 15 | 16 | - name: Install java 17 | uses: actions/setup-java@v2 18 | with: 19 | distribution: 'temurin' 20 | java-version: '17' 21 | 22 | - name: Install clojure tools 23 | uses: DeLaGuardo/setup-clojure@3.2 24 | with: 25 | cli: latest 26 | lein: 2.9.6 # setting b/c latest 2.9.7 broke workflow 27 | 28 | - name: Install dependencies 29 | run: lein deps 30 | 31 | - name: Leinigen version 32 | run: lein -v 33 | 34 | - name: Test 35 | run: lein midje 36 | -------------------------------------------------------------------------------- /.github/workflows/prs_doc_preview.yml: -------------------------------------------------------------------------------- 1 | name: PR Docs Preview 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - 'docs/**/*' 7 | types: 8 | - opened 9 | - reopened 10 | - synchronize 11 | - closed 12 | 13 | concurrency: prs-doc-preview-${{ github.ref }} 14 | 15 | jobs: 16 | deploy-docs-preview: 17 | runs-on: ubuntu-latest 18 | 19 | steps: 20 | - name: Checkout 21 | uses: actions/checkout@v4 22 | 23 | - name: Deploy preview 24 | uses: rossjrw/pr-preview-action@v1 25 | with: 26 | source-dir: ./docs/ 27 | preview-branch: gh-pages 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | pom.xml 2 | pom.xml.asc 3 | *.jar 4 | *.class 5 | /lib/ 6 | /classes/ 7 | /target/ 8 | /checkouts/ 9 | .lein-deps-sum 10 | .lein-repl-history 11 | .lein-plugins/ 12 | .lein-failures 13 | .nrepl-port 14 | .cpcache/ 15 | *.csv* 16 | *.tsv* 17 | *.nippy* 18 | *.log 19 | .R* 20 | *.txt* 21 | .lsp/ 22 | .clj-kondo/ 23 | *.qmd 24 | .quarto 25 | .clay* 26 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: clojure 2 | before_install: 3 | - sudo apt-get -y install rlwrap 4 | - curl -O https://download.clojure.org/install/linux-install-1.10.1.536.sh 5 | - chmod +x linux-install-1.10.1.536.sh 6 | - sudo ./linux-install-1.10.1.536.sh 7 | addons: 8 | apt: 9 | update: true 10 | lein: 2.9.3 11 | script: lein do clean, check, test 12 | jdk: 13 | - openjdk8 14 | - openjdk11 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Scicloj 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /clay.edn: -------------------------------------------------------------------------------- 1 | {:remote-repo {:git-url "https://github.com/scicloj/tablecloth" 2 | :branch "master"} 3 | :quarto {:format {:html {:toc true 4 | :toc-depth 4 5 | :theme [:cosmo "notebooks/custom.scss"]}} 6 | :fontsize "0.8em" 7 | :highlight-style :solarized 8 | :code-block-background true 9 | :include-in-header {:text ""} 10 | :title "Tablecloth documentation"}} 11 | -------------------------------------------------------------------------------- /data/anscombe.csv: -------------------------------------------------------------------------------- 1 | x1,x2,x3,x4,y1,y2,y3,y4 2 | 10,10,10,8,8.04,9.14,7.46,6.58 3 | 8,8,8,8,6.95,8.14,6.77,5.76 4 | 13,13,13,8,7.58,8.74,12.74,7.71 5 | 9,9,9,8,8.81,8.77,7.11,8.84 6 | 11,11,11,8,8.33,9.26,7.81,8.47 7 | 14,14,14,8,9.96,8.1,8.84,7.04 8 | 6,6,6,8,7.24,6.13,6.08,5.25 9 | 4,4,4,19,4.26,3.1,5.39,12.5 10 | 12,12,12,8,10.84,9.13,8.15,5.56 11 | 7,7,7,8,4.82,7.26,6.42,7.91 12 | 5,5,5,8,5.68,4.74,5.73,6.89 13 | -------------------------------------------------------------------------------- /data/billboard.csv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scicloj/tablecloth/9ea514419e2d0777cf635a93cbd4c383b75b5d16/data/billboard.csv.gz -------------------------------------------------------------------------------- /data/construction.csv: -------------------------------------------------------------------------------- 1 | Year,Month,1 unit,2 to 4 units,5 units or more,Northeast,Midwest,South,West 2 | 2018,January,859,NA,348,114,169,596,339 3 | 2018,February,882,NA,400,138,160,655,336 4 | 2018,March,862,NA,356,150,154,595,330 5 | 2018,April,797,NA,447,144,196,613,304 6 | 2018,May,875,NA,364,90,169,673,319 7 | 2018,June,867,NA,342,76,170,610,360 8 | 2018,July,829,NA,360,108,183,594,310 9 | 2018,August,939,NA,286,90,205,649,286 10 | 2018,September,835,NA,304,117,175,560,296 11 | -------------------------------------------------------------------------------- /data/contacts.csv: -------------------------------------------------------------------------------- 1 | field,value,person_id 2 | name,Jiena McLellan,1 3 | company,Toyota,1 4 | name,John Smith,2 5 | company,google,2 6 | email,john@google.com,2 7 | name,Huxley Ratcliffe,3 8 | -------------------------------------------------------------------------------- /data/family.csv: -------------------------------------------------------------------------------- 1 | family,dob_child1,dob_child2,gender_child1,gender_child2 2 | 1,1998-11-26,2000-01-29,1,2 3 | 2,1996-06-22,NA,2,NA 4 | 3,2002-07-11,2004-04-05,2,2 5 | 4,2004-10-10,2009-08-27,1,1 6 | 5,2000-12-05,2005-02-28,2,1 7 | -------------------------------------------------------------------------------- /data/fish_encounters.csv: -------------------------------------------------------------------------------- 1 | fish,station,seen 2 | 4842,Release,1 3 | 4842,I80_1,1 4 | 4842,Lisbon,1 5 | 4842,Rstr,1 6 | 4842,Base_TD,1 7 | 4842,BCE,1 8 | 4842,BCW,1 9 | 4842,BCE2,1 10 | 4842,BCW2,1 11 | 4842,MAE,1 12 | 4842,MAW,1 13 | 4843,Release,1 14 | 4843,I80_1,1 15 | 4843,Lisbon,1 16 | 4843,Rstr,1 17 | 4843,Base_TD,1 18 | 4843,BCE,1 19 | 4843,BCW,1 20 | 4843,BCE2,1 21 | 4843,BCW2,1 22 | 4843,MAE,1 23 | 4843,MAW,1 24 | 4844,Release,1 25 | 4844,I80_1,1 26 | 4844,Lisbon,1 27 | 4844,Rstr,1 28 | 4844,Base_TD,1 29 | 4844,BCE,1 30 | 4844,BCW,1 31 | 4844,BCE2,1 32 | 4844,BCW2,1 33 | 4844,MAE,1 34 | 4844,MAW,1 35 | 4845,Release,1 36 | 4845,I80_1,1 37 | 4845,Lisbon,1 38 | 4845,Rstr,1 39 | 4845,Base_TD,1 40 | 4847,Release,1 41 | 4847,I80_1,1 42 | 4847,Lisbon,1 43 | 4848,Release,1 44 | 4848,I80_1,1 45 | 4848,Lisbon,1 46 | 4848,Rstr,1 47 | 4849,Release,1 48 | 4849,I80_1,1 49 | 4850,Release,1 50 | 4850,I80_1,1 51 | 4850,Rstr,1 52 | 4850,Base_TD,1 53 | 4850,BCE,1 54 | 4850,BCW,1 55 | 4851,Release,1 56 | 4851,I80_1,1 57 | 4854,Release,1 58 | 4854,I80_1,1 59 | 4855,Release,1 60 | 4855,I80_1,1 61 | 4855,Lisbon,1 62 | 4855,Rstr,1 63 | 4855,Base_TD,1 64 | 4857,Release,1 65 | 4857,I80_1,1 66 | 4857,Lisbon,1 67 | 4857,Rstr,1 68 | 4857,Base_TD,1 69 | 4857,BCE,1 70 | 4857,BCW,1 71 | 4857,BCE2,1 72 | 4857,BCW2,1 73 | 4858,Release,1 74 | 4858,I80_1,1 75 | 4858,Lisbon,1 76 | 4858,Rstr,1 77 | 4858,Base_TD,1 78 | 4858,BCE,1 79 | 4858,BCW,1 80 | 4858,BCE2,1 81 | 4858,BCW2,1 82 | 4858,MAE,1 83 | 4858,MAW,1 84 | 4859,Release,1 85 | 4859,I80_1,1 86 | 4859,Lisbon,1 87 | 4859,Rstr,1 88 | 4859,Base_TD,1 89 | 4861,Release,1 90 | 4861,I80_1,1 91 | 4861,Lisbon,1 92 | 4861,Rstr,1 93 | 4861,Base_TD,1 94 | 4861,BCE,1 95 | 4861,BCW,1 96 | 4861,BCE2,1 97 | 4861,BCW2,1 98 | 4861,MAE,1 99 | 4861,MAW,1 100 | 4862,Release,1 101 | 4862,I80_1,1 102 | 4862,Lisbon,1 103 | 4862,Rstr,1 104 | 4862,Base_TD,1 105 | 4862,BCE,1 106 | 4862,BCW,1 107 | 4862,BCE2,1 108 | 4862,BCW2,1 109 | 4863,Release,1 110 | 4863,I80_1,1 111 | 4864,Release,1 112 | 4864,I80_1,1 113 | 4865,Release,1 114 | 4865,I80_1,1 115 | 4865,Lisbon,1 116 | -------------------------------------------------------------------------------- /data/iris.csv: -------------------------------------------------------------------------------- 1 | Sepal.Length,Sepal.Width,Petal.Length,Petal.Width,Species 2 | 5.1,3.5,1.4,0.2,setosa 3 | 4.9,3,1.4,0.2,setosa 4 | 4.7,3.2,1.3,0.2,setosa 5 | 4.6,3.1,1.5,0.2,setosa 6 | 5,3.6,1.4,0.2,setosa 7 | 5.4,3.9,1.7,0.4,setosa 8 | 4.6,3.4,1.4,0.3,setosa 9 | 5,3.4,1.5,0.2,setosa 10 | 4.4,2.9,1.4,0.2,setosa 11 | 4.9,3.1,1.5,0.1,setosa 12 | 5.4,3.7,1.5,0.2,setosa 13 | 4.8,3.4,1.6,0.2,setosa 14 | 4.8,3,1.4,0.1,setosa 15 | 4.3,3,1.1,0.1,setosa 16 | 5.8,4,1.2,0.2,setosa 17 | 5.7,4.4,1.5,0.4,setosa 18 | 5.4,3.9,1.3,0.4,setosa 19 | 5.1,3.5,1.4,0.3,setosa 20 | 5.7,3.8,1.7,0.3,setosa 21 | 5.1,3.8,1.5,0.3,setosa 22 | 5.4,3.4,1.7,0.2,setosa 23 | 5.1,3.7,1.5,0.4,setosa 24 | 4.6,3.6,1,0.2,setosa 25 | 5.1,3.3,1.7,0.5,setosa 26 | 4.8,3.4,1.9,0.2,setosa 27 | 5,3,1.6,0.2,setosa 28 | 5,3.4,1.6,0.4,setosa 29 | 5.2,3.5,1.5,0.2,setosa 30 | 5.2,3.4,1.4,0.2,setosa 31 | 4.7,3.2,1.6,0.2,setosa 32 | 4.8,3.1,1.6,0.2,setosa 33 | 5.4,3.4,1.5,0.4,setosa 34 | 5.2,4.1,1.5,0.1,setosa 35 | 5.5,4.2,1.4,0.2,setosa 36 | 4.9,3.1,1.5,0.2,setosa 37 | 5,3.2,1.2,0.2,setosa 38 | 5.5,3.5,1.3,0.2,setosa 39 | 4.9,3.6,1.4,0.1,setosa 40 | 4.4,3,1.3,0.2,setosa 41 | 5.1,3.4,1.5,0.2,setosa 42 | 5,3.5,1.3,0.3,setosa 43 | 4.5,2.3,1.3,0.3,setosa 44 | 4.4,3.2,1.3,0.2,setosa 45 | 5,3.5,1.6,0.6,setosa 46 | 5.1,3.8,1.9,0.4,setosa 47 | 4.8,3,1.4,0.3,setosa 48 | 5.1,3.8,1.6,0.2,setosa 49 | 4.6,3.2,1.4,0.2,setosa 50 | 5.3,3.7,1.5,0.2,setosa 51 | 5,3.3,1.4,0.2,setosa 52 | 7,3.2,4.7,1.4,versicolor 53 | 6.4,3.2,4.5,1.5,versicolor 54 | 6.9,3.1,4.9,1.5,versicolor 55 | 5.5,2.3,4,1.3,versicolor 56 | 6.5,2.8,4.6,1.5,versicolor 57 | 5.7,2.8,4.5,1.3,versicolor 58 | 6.3,3.3,4.7,1.6,versicolor 59 | 4.9,2.4,3.3,1,versicolor 60 | 6.6,2.9,4.6,1.3,versicolor 61 | 5.2,2.7,3.9,1.4,versicolor 62 | 5,2,3.5,1,versicolor 63 | 5.9,3,4.2,1.5,versicolor 64 | 6,2.2,4,1,versicolor 65 | 6.1,2.9,4.7,1.4,versicolor 66 | 5.6,2.9,3.6,1.3,versicolor 67 | 6.7,3.1,4.4,1.4,versicolor 68 | 5.6,3,4.5,1.5,versicolor 69 | 5.8,2.7,4.1,1,versicolor 70 | 6.2,2.2,4.5,1.5,versicolor 71 | 5.6,2.5,3.9,1.1,versicolor 72 | 5.9,3.2,4.8,1.8,versicolor 73 | 6.1,2.8,4,1.3,versicolor 74 | 6.3,2.5,4.9,1.5,versicolor 75 | 6.1,2.8,4.7,1.2,versicolor 76 | 6.4,2.9,4.3,1.3,versicolor 77 | 6.6,3,4.4,1.4,versicolor 78 | 6.8,2.8,4.8,1.4,versicolor 79 | 6.7,3,5,1.7,versicolor 80 | 6,2.9,4.5,1.5,versicolor 81 | 5.7,2.6,3.5,1,versicolor 82 | 5.5,2.4,3.8,1.1,versicolor 83 | 5.5,2.4,3.7,1,versicolor 84 | 5.8,2.7,3.9,1.2,versicolor 85 | 6,2.7,5.1,1.6,versicolor 86 | 5.4,3,4.5,1.5,versicolor 87 | 6,3.4,4.5,1.6,versicolor 88 | 6.7,3.1,4.7,1.5,versicolor 89 | 6.3,2.3,4.4,1.3,versicolor 90 | 5.6,3,4.1,1.3,versicolor 91 | 5.5,2.5,4,1.3,versicolor 92 | 5.5,2.6,4.4,1.2,versicolor 93 | 6.1,3,4.6,1.4,versicolor 94 | 5.8,2.6,4,1.2,versicolor 95 | 5,2.3,3.3,1,versicolor 96 | 5.6,2.7,4.2,1.3,versicolor 97 | 5.7,3,4.2,1.2,versicolor 98 | 5.7,2.9,4.2,1.3,versicolor 99 | 6.2,2.9,4.3,1.3,versicolor 100 | 5.1,2.5,3,1.1,versicolor 101 | 5.7,2.8,4.1,1.3,versicolor 102 | 6.3,3.3,6,2.5,virginica 103 | 5.8,2.7,5.1,1.9,virginica 104 | 7.1,3,5.9,2.1,virginica 105 | 6.3,2.9,5.6,1.8,virginica 106 | 6.5,3,5.8,2.2,virginica 107 | 7.6,3,6.6,2.1,virginica 108 | 4.9,2.5,4.5,1.7,virginica 109 | 7.3,2.9,6.3,1.8,virginica 110 | 6.7,2.5,5.8,1.8,virginica 111 | 7.2,3.6,6.1,2.5,virginica 112 | 6.5,3.2,5.1,2,virginica 113 | 6.4,2.7,5.3,1.9,virginica 114 | 6.8,3,5.5,2.1,virginica 115 | 5.7,2.5,5,2,virginica 116 | 5.8,2.8,5.1,2.4,virginica 117 | 6.4,3.2,5.3,2.3,virginica 118 | 6.5,3,5.5,1.8,virginica 119 | 7.7,3.8,6.7,2.2,virginica 120 | 7.7,2.6,6.9,2.3,virginica 121 | 6,2.2,5,1.5,virginica 122 | 6.9,3.2,5.7,2.3,virginica 123 | 5.6,2.8,4.9,2,virginica 124 | 7.7,2.8,6.7,2,virginica 125 | 6.3,2.7,4.9,1.8,virginica 126 | 6.7,3.3,5.7,2.1,virginica 127 | 7.2,3.2,6,1.8,virginica 128 | 6.2,2.8,4.8,1.8,virginica 129 | 6.1,3,4.9,1.8,virginica 130 | 6.4,2.8,5.6,2.1,virginica 131 | 7.2,3,5.8,1.6,virginica 132 | 7.4,2.8,6.1,1.9,virginica 133 | 7.9,3.8,6.4,2,virginica 134 | 6.4,2.8,5.6,2.2,virginica 135 | 6.3,2.8,5.1,1.5,virginica 136 | 6.1,2.6,5.6,1.4,virginica 137 | 7.7,3,6.1,2.3,virginica 138 | 6.3,3.4,5.6,2.4,virginica 139 | 6.4,3.1,5.5,1.8,virginica 140 | 6,3,4.8,1.8,virginica 141 | 6.9,3.1,5.4,2.1,virginica 142 | 6.7,3.1,5.6,2.4,virginica 143 | 6.9,3.1,5.1,2.3,virginica 144 | 5.8,2.7,5.1,1.9,virginica 145 | 6.8,3.2,5.9,2.3,virginica 146 | 6.7,3.3,5.7,2.5,virginica 147 | 6.7,3,5.2,2.3,virginica 148 | 6.3,2.5,5,1.9,virginica 149 | 6.5,3,5.2,2,virginica 150 | 6.2,3.4,5.4,2.3,virginica 151 | 5.9,3,5.1,1.8,virginica 152 | -------------------------------------------------------------------------------- /data/production.csv: -------------------------------------------------------------------------------- 1 | product,country,year,production 2 | A,AI,2000,1.6372715774122846 3 | A,AI,2001,0.15870783986040046 4 | A,AI,2002,-1.567977450807575 5 | A,AI,2003,-0.44455509350133854 6 | A,AI,2004,-0.07133701045735727 7 | A,AI,2005,1.6118308960119823 8 | A,AI,2006,-0.7043468225867656 9 | A,AI,2007,-1.5355054163278994 10 | A,AI,2008,0.8390715457273077 11 | A,AI,2009,-0.3742411001291045 12 | A,AI,2010,-0.7115892568951634 13 | A,AI,2011,1.1280563436678663 14 | A,AI,2012,1.4571824668730426 15 | A,AI,2013,-1.5593410121937417 16 | A,AI,2014,-0.11695838002114005 17 | B,AI,2000,-0.026176611940051428 18 | B,AI,2001,-0.6886357642826394 19 | B,AI,2002,0.0624874105489952 20 | B,AI,2003,-0.723396863031015 21 | B,AI,2004,0.47248951964453134 22 | B,AI,2005,-0.9417386106260295 23 | B,AI,2006,-0.3478210750446564 24 | B,AI,2007,0.5242528380610967 25 | B,AI,2008,1.8323093743778909 26 | B,AI,2009,0.10706490503775681 27 | B,AI,2010,-0.329036635089033 28 | B,AI,2011,-1.7831912123478162 29 | B,AI,2012,0.6112579805886741 30 | B,AI,2013,-0.7852609221892832 31 | B,AI,2014,0.9784363513590328 32 | B,EI,2000,1.404708477530438 33 | B,EI,2001,-0.5961836879930892 34 | B,EI,2002,-0.26568578637154766 35 | B,EI,2003,0.6525780779508413 36 | B,EI,2004,0.6256499904976253 37 | B,EI,2005,-1.3453029907000869 38 | B,EI,2006,-0.9718497486278521 39 | B,EI,2007,-1.697158206855015 40 | B,EI,2008,0.045561282626706424 41 | B,EI,2009,1.1931504259618515 42 | B,EI,2010,-1.6055750319322417 43 | B,EI,2011,-0.7723549690075867 44 | B,EI,2012,-2.502627383821918 45 | B,EI,2013,-1.6275376866923248 46 | B,EI,2014,0.0332964450957834 47 | -------------------------------------------------------------------------------- /data/relig_income.csv: -------------------------------------------------------------------------------- 1 | religion,<$10k,$10-20k,$20-30k,$30-40k,$40-50k,$50-75k,$75-100k,$100-150k,>150k,Don't know/refused 2 | Agnostic,27,34,60,81,76,137,122,109,84,96 3 | Atheist,12,27,37,52,35,70,73,59,74,76 4 | Buddhist,27,21,30,34,33,58,62,39,53,54 5 | Catholic,418,617,732,670,638,1116,949,792,633,1489 6 | Don’t know/refused,15,14,15,11,10,35,21,17,18,116 7 | Evangelical Prot,575,869,1064,982,881,1486,949,723,414,1529 8 | Hindu,1,9,7,9,11,34,47,48,54,37 9 | Historically Black Prot,228,244,236,238,197,223,131,81,78,339 10 | Jehovah's Witness,20,27,24,24,21,30,15,11,6,37 11 | Jewish,19,19,25,25,30,95,69,87,151,162 12 | Mainline Prot,289,495,619,655,651,1107,939,753,634,1328 13 | Mormon,29,40,48,51,56,112,85,49,42,69 14 | Muslim,6,7,9,10,9,23,16,8,6,22 15 | Orthodox,13,17,23,32,32,47,38,42,46,73 16 | Other Christian,9,7,11,13,13,14,18,14,12,18 17 | Other Faiths,20,33,40,46,49,63,46,40,41,71 18 | Other World Religions,5,2,3,4,2,7,3,4,4,8 19 | Unaffiliated,217,299,374,365,341,528,407,321,258,597 20 | -------------------------------------------------------------------------------- /data/stockstidyr.csv: -------------------------------------------------------------------------------- 1 | time,X,Y,Z 2 | 2009-01-01,1.3098980569694743,-1.8904019256298135,-1.7794688023227476 3 | 2009-01-02,-0.299938042403242,-1.8247309017308997,2.398925133113737 4 | 2009-01-03,0.536475012262886,-1.0360685974515387,-3.98697977071294 5 | 2009-01-04,-1.8839080177420178,-0.5217838968579932,-2.8306548990407485 6 | 2009-01-05,-0.9605236142913735,-2.2168334921282327,1.4371517117119361 7 | 2009-01-06,-1.1852896576065972,-2.8935092410204866,3.3978414042157543 8 | 2009-01-07,-0.8520705598024025,-2.167948176956802,-1.2010825844501143 9 | 2009-01-08,0.2523417204877737,-0.3285411651231233,-1.5316047270579687 10 | 2009-01-09,0.40257136394542437,1.964078984798918,-6.808788298292381 11 | 2009-01-10,-0.6438350002601568,2.6861838168176297,-2.559093207111715 12 | -------------------------------------------------------------------------------- /data/us_rent_income.csv: -------------------------------------------------------------------------------- 1 | GEOID,NAME,variable,estimate,moe 2 | 01,Alabama,income,24476,136 3 | 01,Alabama,rent,747,3 4 | 02,Alaska,income,32940,508 5 | 02,Alaska,rent,1200,13 6 | 04,Arizona,income,27517,148 7 | 04,Arizona,rent,972,4 8 | 05,Arkansas,income,23789,165 9 | 05,Arkansas,rent,709,5 10 | 06,California,income,29454,109 11 | 06,California,rent,1358,3 12 | 08,Colorado,income,32401,109 13 | 08,Colorado,rent,1125,5 14 | 09,Connecticut,income,35326,195 15 | 09,Connecticut,rent,1123,5 16 | 10,Delaware,income,31560,247 17 | 10,Delaware,rent,1076,10 18 | 11,District of Columbia,income,43198,681 19 | 11,District of Columbia,rent,1424,17 20 | 12,Florida,income,25952,70 21 | 12,Florida,rent,1077,3 22 | 13,Georgia,income,27024,106 23 | 13,Georgia,rent,927,3 24 | 15,Hawaii,income,32453,218 25 | 15,Hawaii,rent,1507,18 26 | 16,Idaho,income,25298,208 27 | 16,Idaho,rent,792,7 28 | 17,Illinois,income,30684,83 29 | 17,Illinois,rent,952,3 30 | 18,Indiana,income,27247,117 31 | 18,Indiana,rent,782,3 32 | 19,Iowa,income,30002,143 33 | 19,Iowa,rent,740,4 34 | 20,Kansas,income,29126,208 35 | 20,Kansas,rent,801,5 36 | 21,Kentucky,income,24702,159 37 | 21,Kentucky,rent,713,4 38 | 22,Louisiana,income,25086,155 39 | 22,Louisiana,rent,825,4 40 | 23,Maine,income,26841,187 41 | 23,Maine,rent,808,7 42 | 24,Maryland,income,37147,152 43 | 24,Maryland,rent,1311,5 44 | 25,Massachusetts,income,34498,199 45 | 25,Massachusetts,rent,1173,5 46 | 26,Michigan,income,26987,82 47 | 26,Michigan,rent,824,3 48 | 27,Minnesota,income,32734,189 49 | 27,Minnesota,rent,906,4 50 | 28,Mississippi,income,22766,194 51 | 28,Mississippi,rent,740,5 52 | 29,Missouri,income,26999,113 53 | 29,Missouri,rent,784,4 54 | 30,Montana,income,26249,206 55 | 30,Montana,rent,751,9 56 | 31,Nebraska,income,30020,146 57 | 31,Nebraska,rent,773,4 58 | 32,Nevada,income,29019,213 59 | 32,Nevada,rent,1017,6 60 | 33,New Hampshire,income,33172,387 61 | 33,New Hampshire,rent,1052,9 62 | 34,New Jersey,income,35075,148 63 | 34,New Jersey,rent,1249,4 64 | 35,New Mexico,income,24457,214 65 | 35,New Mexico,rent,809,6 66 | 36,New York,income,31057,69 67 | 36,New York,rent,1194,3 68 | 37,North Carolina,income,26482,111 69 | 37,North Carolina,rent,844,3 70 | 38,North Dakota,income,32336,245 71 | 38,North Dakota,rent,775,9 72 | 39,Ohio,income,27435,94 73 | 39,Ohio,rent,764,2 74 | 40,Oklahoma,income,26207,101 75 | 40,Oklahoma,rent,766,3 76 | 41,Oregon,income,27389,146 77 | 41,Oregon,rent,988,4 78 | 42,Pennsylvania,income,28923,119 79 | 42,Pennsylvania,rent,885,3 80 | 44,Rhode Island,income,30210,259 81 | 44,Rhode Island,rent,957,6 82 | 45,South Carolina,income,25454,123 83 | 45,South Carolina,rent,836,4 84 | 46,South Dakota,income,28821,276 85 | 46,South Dakota,rent,696,7 86 | 47,Tennessee,income,25453,102 87 | 47,Tennessee,rent,808,4 88 | 48,Texas,income,28063,110 89 | 48,Texas,rent,952,2 90 | 49,Utah,income,27928,239 91 | 49,Utah,rent,948,6 92 | 50,Vermont,income,29351,361 93 | 50,Vermont,rent,945,11 94 | 51,Virginia,income,32545,202 95 | 51,Virginia,rent,1166,5 96 | 53,Washington,income,32318,113 97 | 53,Washington,rent,1120,4 98 | 54,West Virginia,income,23707,203 99 | 54,West Virginia,rent,681,6 100 | 55,Wisconsin,income,29868,135 101 | 55,Wisconsin,rent,813,3 102 | 56,Wyoming,income,30854,342 103 | 56,Wyoming,rent,828,11 104 | 72,Puerto Rico,income,NA,NA 105 | 72,Puerto Rico,rent,464,6 106 | -------------------------------------------------------------------------------- /data/warpbreaks.csv: -------------------------------------------------------------------------------- 1 | breaks,wool,tension 2 | 26,A,L 3 | 30,A,L 4 | 54,A,L 5 | 25,A,L 6 | 70,A,L 7 | 52,A,L 8 | 51,A,L 9 | 26,A,L 10 | 67,A,L 11 | 18,A,M 12 | 21,A,M 13 | 29,A,M 14 | 17,A,M 15 | 12,A,M 16 | 18,A,M 17 | 35,A,M 18 | 30,A,M 19 | 36,A,M 20 | 36,A,H 21 | 21,A,H 22 | 24,A,H 23 | 18,A,H 24 | 10,A,H 25 | 43,A,H 26 | 28,A,H 27 | 15,A,H 28 | 26,A,H 29 | 27,B,L 30 | 14,B,L 31 | 29,B,L 32 | 19,B,L 33 | 29,B,L 34 | 31,B,L 35 | 41,B,L 36 | 20,B,L 37 | 44,B,L 38 | 42,B,M 39 | 26,B,M 40 | 19,B,M 41 | 16,B,M 42 | 39,B,M 43 | 28,B,M 44 | 21,B,M 45 | 39,B,M 46 | 29,B,M 47 | 20,B,H 48 | 21,B,H 49 | 24,B,H 50 | 17,B,H 51 | 13,B,H 52 | 15,B,H 53 | 15,B,H 54 | 16,B,H 55 | 28,B,H 56 | -------------------------------------------------------------------------------- /data/who.csv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scicloj/tablecloth/9ea514419e2d0777cf635a93cbd4c383b75b5d16/data/who.csv.gz -------------------------------------------------------------------------------- /data/world_bank_pop.csv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scicloj/tablecloth/9ea514419e2d0777cf635a93cbd4c383b75b5d16/data/world_bank_pop.csv.gz -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:extra-paths ["data"] 2 | :deps {org.clojure/clojure {:mvn/version "1.12.0"} 3 | techascent/tech.ml.dataset {:mvn/version "7.059"}} 4 | :aliases {:dev {:extra-deps {org.scicloj/clay {:mvn/version "2-beta42"} 5 | org.scicloj/note-to-test {:mvn/version "1-alpha7"}}} 6 | :test {:extra-deps {org.scicloj/clay {:mvn/version "2-beta42"} 7 | org.scicloj/note-to-test {:mvn/version "1-alpha7"} 8 | midje/midje {:mvn/version "1.10.10"}} 9 | :extra-paths ["test"]}}} 10 | -------------------------------------------------------------------------------- /dev/conversion.clj: -------------------------------------------------------------------------------- 1 | (ns conversion 2 | (:require [clojure.string :as string])) 3 | 4 | ;; This namespace has been used to convert the old `docs/index.Rmd` documentation to the new `notebooks/index.clj` documentation. Some additional manual editing was needed to make things work afterwards. 5 | 6 | (set! *print-length* 1000) 7 | 8 | (def example-doc 9 | (slurp "docs/index.Rmd" 10 | #_"https://raw.githubusercontent.com/scicloj/tablecloth/master/docs/index.Rmd")) 11 | 12 | (defn chunk-end? [line] 13 | (= line "```")) 14 | 15 | (defn clojure-chunk-beginning? [line] 16 | (re-matches #"```\{clojure.*\}" line)) 17 | 18 | (defn clojure-chunk-options [line] 19 | (-> line 20 | (string/replace #"^```\{clojure" "") 21 | (string/replace #"\}$" ""))) 22 | 23 | (defn markdown->markdown-with-extracted-clojure-chunks [markdown] 24 | (loop [lines (string/split markdown #"\n") 25 | current-clojure-chunk-code nil 26 | current-clojure-chunk-options nil 27 | result []] 28 | (if-let [current-line (first lines)] 29 | (if current-clojure-chunk-code 30 | (if (chunk-end? current-line) 31 | (recur (rest lines) 32 | nil 33 | nil 34 | (conj result {:clojure-chunk? true 35 | :code current-clojure-chunk-code 36 | :options current-clojure-chunk-options})) 37 | (recur (rest lines) 38 | (conj current-clojure-chunk-code current-line) 39 | current-clojure-chunk-options 40 | result)) 41 | (if (clojure-chunk-beginning? current-line) 42 | (recur (rest lines) 43 | [] 44 | (clojure-chunk-options current-line) 45 | result) 46 | (recur (rest lines) 47 | nil 48 | nil 49 | (conj result current-line)))) 50 | result))) 51 | 52 | (defn clojure-chunk->markdown [{:keys [clojure-chunk? code options]}] 53 | (when clojure-chunk? 54 | (str "```{clojure" 55 | options 56 | "}\n" 57 | (string/join "\n" code) 58 | "\n```"))) 59 | 60 | (defn markdown-with-extracted-clojure-chunks->markdown [mwecc] 61 | (->> mwecc 62 | (map #(or (clojure-chunk->markdown %) 63 | %)) 64 | (string/join "\n") 65 | (format "%s\n"))) 66 | 67 | (comment 68 | (->> example-doc 69 | (spit "/tmp/0.Rmd")) 70 | 71 | (->> example-doc 72 | markdown->markdown-with-extracted-clojure-chunks 73 | markdown-with-extracted-clojure-chunks->markdown 74 | (spit "/tmp/a.Rmd")) 75 | 76 | (->> example-doc 77 | markdown->markdown-with-extracted-clojure-chunks 78 | markdown-with-extracted-clojure-chunks->markdown 79 | (= example-doc)) 80 | 81 | (->> example-doc 82 | markdown->markdown-with-extracted-clojure-chunks 83 | (partition-by string?))) 84 | 85 | 86 | (defn inner-pr-str [s] 87 | (let [s1 (pr-str s) 88 | n (count s1)] 89 | (subs s1 1 (dec n)))) 90 | 91 | (defn markdown-with-extracted-clojure-chunks->clojure [mwecc] 92 | (->> mwecc 93 | (partition-by string?) 94 | (mapcat (fn [part] 95 | (cond (-> part first string?) [(->> part 96 | (map inner-pr-str) 97 | (string/join "\n") 98 | ((fn [s] 99 | (if (seq s) 100 | (format "(md \"%s\")" s) 101 | ""))))] 102 | (-> part first :clojure-chunk?) [(->> part 103 | (mapcat 104 | (fn [{:keys [code options]}] 105 | code 106 | #_(cons (format "^{:chunk-options \"%s\"}[]\n" 107 | options) 108 | code))) 109 | (string/join "\n"))]))) 110 | (string/join "\n\n\n") 111 | (format " 112 | (ns index 113 | (:require [scicloj.kindly.v3.kind :as kind] 114 | [scicloj.kindly-default.v1.api :refer [md]] 115 | [tablecloth.api :as tc] 116 | [scicloj.note-to-test.v1.api :as note-to-test])) 117 | 118 | %s"))) 119 | 120 | (->> example-doc 121 | markdown->markdown-with-extracted-clojure-chunks 122 | markdown-with-extracted-clojure-chunks->clojure 123 | (spit "notebooks/index.clj")) 124 | -------------------------------------------------------------------------------- /dev/gendocs.clj: -------------------------------------------------------------------------------- 1 | (ns gendocs 2 | (:require [scicloj.clay.v2.api :as clay])) 3 | 4 | (clay/make! {:format [:quarto :html] 5 | :source-path "notebooks/index.clj"}) 6 | -------------------------------------------------------------------------------- /dev/readme_generation.clj: -------------------------------------------------------------------------------- 1 | (ns readme-generation 2 | (:require [clojure.string :as str] 3 | [clojure.pprint :as pp])) 4 | 5 | (set! *print-length* 1000) 6 | 7 | (defn chunk-end? [line] 8 | (= line "```")) 9 | 10 | (defn clojure-chunk-beginning? [line] 11 | (re-matches #"```\{clojure.*\}" line)) 12 | 13 | (defn clojure-chunk-options [line] 14 | (-> line 15 | (str/replace #"^```\{clojure" "") 16 | (str/replace #"\}$" ""))) 17 | 18 | (defn markdown->markdown-with-extracted-clojure-chunks [markdown] 19 | (loop [lines (str/split markdown #"\n") 20 | current-clojure-chunk-code nil 21 | current-clojure-chunk-options nil 22 | result []] 23 | (if-let [current-line (first lines)] 24 | (if current-clojure-chunk-code 25 | (if (chunk-end? current-line) 26 | (recur (rest lines) 27 | nil 28 | nil 29 | (conj result {:clojure-chunk? true 30 | :code current-clojure-chunk-code 31 | :options current-clojure-chunk-options})) 32 | (recur (rest lines) 33 | (conj current-clojure-chunk-code current-line) 34 | current-clojure-chunk-options 35 | result)) 36 | (if (clojure-chunk-beginning? current-line) 37 | (recur (rest lines) 38 | [] 39 | (clojure-chunk-options current-line) 40 | result) 41 | (recur (rest lines) 42 | nil 43 | nil 44 | (conj result current-line)))) 45 | result))) 46 | 47 | (defn markdown-with-extracted-clojure-chunks->markdown-evaluated [mwecc] 48 | (->> mwecc 49 | (map (fn [element] 50 | (if (string? element) 51 | element 52 | (let [{:keys [code options]} element 53 | options-map (-> options 54 | str/trim 55 | (str/split #"=") 56 | (->> (map read-string) 57 | (apply hash-map)))] 58 | (str (format "```{clojure}\n%s\n```\n" 59 | (->> code 60 | (str/join "\n"))) 61 | (if (-> options-map 62 | (get 'eval) 63 | (= 'FALSE)) 64 | "" 65 | (let [result (->> code 66 | (str/join "\n") 67 | load-string)] 68 | (case (-> options-map 69 | (get 'results)) 70 | "hide" "" 71 | "asis" (-> result 72 | pp/pprint 73 | with-out-str))))))))) 74 | (str/join "\n"))) 75 | 76 | 77 | (defn generate! [] 78 | (->> "README-source.md" 79 | slurp 80 | markdown->markdown-with-extracted-clojure-chunks 81 | markdown-with-extracted-clojure-chunks->markdown-evaluated 82 | (spit "README.md"))) 83 | 84 | (comment 85 | (generate!)) 86 | -------------------------------------------------------------------------------- /docs/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scicloj/tablecloth/9ea514419e2d0777cf635a93cbd4c383b75b5d16/docs/.nojekyll -------------------------------------------------------------------------------- /docs/better_tables.html: -------------------------------------------------------------------------------- 1 | 10 | -------------------------------------------------------------------------------- /docs/column_api_files/libs/bootstrap/bootstrap-icons.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scicloj/tablecloth/9ea514419e2d0777cf635a93cbd4c383b75b5d16/docs/column_api_files/libs/bootstrap/bootstrap-icons.woff -------------------------------------------------------------------------------- /docs/column_api_files/libs/quarto-html/anchor.min.js: -------------------------------------------------------------------------------- 1 | // @license magnet:?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&dn=expat.txt Expat 2 | // 3 | // AnchorJS - v4.3.1 - 2021-04-17 4 | // https://www.bryanbraun.com/anchorjs/ 5 | // Copyright (c) 2021 Bryan Braun; Licensed MIT 6 | // 7 | // @license magnet:?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&dn=expat.txt Expat 8 | !function(A,e){"use strict";"function"==typeof define&&define.amd?define([],e):"object"==typeof module&&module.exports?module.exports=e():(A.AnchorJS=e(),A.anchors=new A.AnchorJS)}(this,function(){"use strict";return function(A){function d(A){A.icon=Object.prototype.hasOwnProperty.call(A,"icon")?A.icon:"",A.visible=Object.prototype.hasOwnProperty.call(A,"visible")?A.visible:"hover",A.placement=Object.prototype.hasOwnProperty.call(A,"placement")?A.placement:"right",A.ariaLabel=Object.prototype.hasOwnProperty.call(A,"ariaLabel")?A.ariaLabel:"Anchor",A.class=Object.prototype.hasOwnProperty.call(A,"class")?A.class:"",A.base=Object.prototype.hasOwnProperty.call(A,"base")?A.base:"",A.truncate=Object.prototype.hasOwnProperty.call(A,"truncate")?Math.floor(A.truncate):64,A.titleText=Object.prototype.hasOwnProperty.call(A,"titleText")?A.titleText:""}function w(A){var e;if("string"==typeof A||A instanceof String)e=[].slice.call(document.querySelectorAll(A));else{if(!(Array.isArray(A)||A instanceof NodeList))throw new TypeError("The selector provided to AnchorJS was invalid.");e=[].slice.call(A)}return e}this.options=A||{},this.elements=[],d(this.options),this.isTouchDevice=function(){return Boolean("ontouchstart"in window||window.TouchEvent||window.DocumentTouch&&document instanceof DocumentTouch)},this.add=function(A){var e,t,o,i,n,s,a,c,r,l,h,u,p=[];if(d(this.options),"touch"===(l=this.options.visible)&&(l=this.isTouchDevice()?"always":"hover"),0===(e=w(A=A||"h2, h3, h4, h5, h6")).length)return this;for(null===document.head.querySelector("style.anchorjs")&&((u=document.createElement("style")).className="anchorjs",u.appendChild(document.createTextNode("")),void 0===(A=document.head.querySelector('[rel="stylesheet"],style'))?document.head.appendChild(u):document.head.insertBefore(u,A),u.sheet.insertRule(".anchorjs-link{opacity:0;text-decoration:none;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}",u.sheet.cssRules.length),u.sheet.insertRule(":hover>.anchorjs-link,.anchorjs-link:focus{opacity:1}",u.sheet.cssRules.length),u.sheet.insertRule("[data-anchorjs-icon]::after{content:attr(data-anchorjs-icon)}",u.sheet.cssRules.length),u.sheet.insertRule('@font-face{font-family:anchorjs-icons;src:url(data:n/a;base64,AAEAAAALAIAAAwAwT1MvMg8yG2cAAAE4AAAAYGNtYXDp3gC3AAABpAAAAExnYXNwAAAAEAAAA9wAAAAIZ2x5ZlQCcfwAAAH4AAABCGhlYWQHFvHyAAAAvAAAADZoaGVhBnACFwAAAPQAAAAkaG10eASAADEAAAGYAAAADGxvY2EACACEAAAB8AAAAAhtYXhwAAYAVwAAARgAAAAgbmFtZQGOH9cAAAMAAAAAunBvc3QAAwAAAAADvAAAACAAAQAAAAEAAHzE2p9fDzz1AAkEAAAAAADRecUWAAAAANQA6R8AAAAAAoACwAAAAAgAAgAAAAAAAAABAAADwP/AAAACgAAA/9MCrQABAAAAAAAAAAAAAAAAAAAAAwABAAAAAwBVAAIAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAMCQAGQAAUAAAKZAswAAACPApkCzAAAAesAMwEJAAAAAAAAAAAAAAAAAAAAARAAAAAAAAAAAAAAAAAAAAAAQAAg//0DwP/AAEADwABAAAAAAQAAAAAAAAAAAAAAIAAAAAAAAAIAAAACgAAxAAAAAwAAAAMAAAAcAAEAAwAAABwAAwABAAAAHAAEADAAAAAIAAgAAgAAACDpy//9//8AAAAg6cv//f///+EWNwADAAEAAAAAAAAAAAAAAAAACACEAAEAAAAAAAAAAAAAAAAxAAACAAQARAKAAsAAKwBUAAABIiYnJjQ3NzY2MzIWFxYUBwcGIicmNDc3NjQnJiYjIgYHBwYUFxYUBwYGIwciJicmNDc3NjIXFhQHBwYUFxYWMzI2Nzc2NCcmNDc2MhcWFAcHBgYjARQGDAUtLXoWOR8fORYtLTgKGwoKCjgaGg0gEhIgDXoaGgkJBQwHdR85Fi0tOAobCgoKOBoaDSASEiANehoaCQkKGwotLXoWOR8BMwUFLYEuehYXFxYugC44CQkKGwo4GkoaDQ0NDXoaShoKGwoFBe8XFi6ALjgJCQobCjgaShoNDQ0NehpKGgobCgoKLYEuehYXAAAADACWAAEAAAAAAAEACAAAAAEAAAAAAAIAAwAIAAEAAAAAAAMACAAAAAEAAAAAAAQACAAAAAEAAAAAAAUAAQALAAEAAAAAAAYACAAAAAMAAQQJAAEAEAAMAAMAAQQJAAIABgAcAAMAAQQJAAMAEAAMAAMAAQQJAAQAEAAMAAMAAQQJAAUAAgAiAAMAAQQJAAYAEAAMYW5jaG9yanM0MDBAAGEAbgBjAGgAbwByAGoAcwA0ADAAMABAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAH//wAP) format("truetype")}',u.sheet.cssRules.length)),u=document.querySelectorAll("[id]"),t=[].map.call(u,function(A){return A.id}),i=0;i\]./()*\\\n\t\b\v\u00A0]/g,"-").replace(/-{2,}/g,"-").substring(0,this.options.truncate).replace(/^-+|-+$/gm,"").toLowerCase()},this.hasAnchorJSLink=function(A){var e=A.firstChild&&-1<(" "+A.firstChild.className+" ").indexOf(" anchorjs-link "),A=A.lastChild&&-1<(" "+A.lastChild.className+" ").indexOf(" anchorjs-link ");return e||A||!1}}}); 9 | // @license-end -------------------------------------------------------------------------------- /docs/column_api_files/libs/quarto-html/quarto-syntax-highlighting.css: -------------------------------------------------------------------------------- 1 | /* quarto syntax highlight colors */ 2 | :root { 3 | --quarto-hl-kw-color: #859900; 4 | --quarto-hl-fu-color: #268bd2; 5 | --quarto-hl-va-color: #268bd2; 6 | --quarto-hl-cf-color: #859900; 7 | --quarto-hl-op-color: #859900; 8 | --quarto-hl-bu-color: #cb4b16; 9 | --quarto-hl-ex-color: #268bd2; 10 | --quarto-hl-pp-color: #cb4b16; 11 | --quarto-hl-at-color: #268bd2; 12 | --quarto-hl-ch-color: #2aa198; 13 | --quarto-hl-sc-color: #dc322f; 14 | --quarto-hl-st-color: #2aa198; 15 | --quarto-hl-vs-color: #2aa198; 16 | --quarto-hl-ss-color: #dc322f; 17 | --quarto-hl-im-color: #2aa198; 18 | --quarto-hl-dt-color: #b58900; 19 | --quarto-hl-dv-color: #2aa198; 20 | --quarto-hl-bn-color: #2aa198; 21 | --quarto-hl-fl-color: #2aa198; 22 | --quarto-hl-cn-color: #2aa198; 23 | --quarto-hl-co-color: #93a1a1; 24 | --quarto-hl-do-color: #dc322f; 25 | --quarto-hl-an-color: #268bd2; 26 | --quarto-hl-cv-color: #2aa198; 27 | --quarto-hl-re-color: #268bd2; 28 | --quarto-hl-in-color: #b58900; 29 | --quarto-hl-wa-color: #cb4b16; 30 | --quarto-hl-al-color: #d33682; 31 | --quarto-hl-er-color: #dc322f; 32 | } 33 | 34 | /* other quarto variables */ 35 | :root { 36 | --quarto-font-monospace: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; 37 | } 38 | 39 | pre > code.sourceCode > span { 40 | color: #657b83; 41 | font-style: inherit; 42 | } 43 | 44 | code span { 45 | color: #657b83; 46 | font-style: inherit; 47 | } 48 | 49 | code.sourceCode > span { 50 | color: #657b83; 51 | font-style: inherit; 52 | } 53 | 54 | div.sourceCode, 55 | div.sourceCode pre.sourceCode { 56 | color: #657b83; 57 | font-style: inherit; 58 | } 59 | 60 | code span.kw { 61 | color: #859900; 62 | font-weight: bold; 63 | } 64 | 65 | code span.fu { 66 | color: #268bd2; 67 | } 68 | 69 | code span.va { 70 | color: #268bd2; 71 | } 72 | 73 | code span.cf { 74 | color: #859900; 75 | font-weight: bold; 76 | } 77 | 78 | code span.op { 79 | color: #859900; 80 | } 81 | 82 | code span.bu { 83 | color: #cb4b16; 84 | } 85 | 86 | code span.ex { 87 | color: #268bd2; 88 | font-weight: bold; 89 | } 90 | 91 | code span.pp { 92 | color: #cb4b16; 93 | } 94 | 95 | code span.at { 96 | color: #268bd2; 97 | } 98 | 99 | code span.ch { 100 | color: #2aa198; 101 | } 102 | 103 | code span.sc { 104 | color: #dc322f; 105 | } 106 | 107 | code span.st { 108 | color: #2aa198; 109 | } 110 | 111 | code span.vs { 112 | color: #2aa198; 113 | } 114 | 115 | code span.ss { 116 | color: #dc322f; 117 | } 118 | 119 | code span.im { 120 | color: #2aa198; 121 | } 122 | 123 | code span.dt { 124 | color: #b58900; 125 | font-weight: bold; 126 | } 127 | 128 | code span.dv { 129 | color: #2aa198; 130 | } 131 | 132 | code span.bn { 133 | color: #2aa198; 134 | } 135 | 136 | code span.fl { 137 | color: #2aa198; 138 | } 139 | 140 | code span.cn { 141 | color: #2aa198; 142 | font-weight: bold; 143 | } 144 | 145 | code span.co { 146 | color: #93a1a1; 147 | font-style: italic; 148 | } 149 | 150 | code span.do { 151 | color: #dc322f; 152 | } 153 | 154 | code span.an { 155 | color: #268bd2; 156 | } 157 | 158 | code span.cv { 159 | color: #2aa198; 160 | } 161 | 162 | code span.re { 163 | color: #268bd2; 164 | background-color: #eee8d5; 165 | } 166 | 167 | code span.in { 168 | color: #b58900; 169 | } 170 | 171 | code span.wa { 172 | color: #cb4b16; 173 | } 174 | 175 | code span.al { 176 | color: #d33682; 177 | font-weight: bold; 178 | } 179 | 180 | code span.er { 181 | color: #dc322f; 182 | text-decoration: underline; 183 | } 184 | 185 | .prevent-inlining { 186 | content: ".tippy-arrow{bottom:0}.tippy-box[data-placement^=top]>.tippy-arrow:before{bottom:-7px;left:0;border-width:8px 8px 0;border-top-color:initial;transform-origin:center top}.tippy-box[data-placement^=bottom]>.tippy-arrow{top:0}.tippy-box[data-placement^=bottom]>.tippy-arrow:before{top:-7px;left:0;border-width:0 8px 8px;border-bottom-color:initial;transform-origin:center bottom}.tippy-box[data-placement^=left]>.tippy-arrow{right:0}.tippy-box[data-placement^=left]>.tippy-arrow:before{border-width:8px 0 8px 8px;border-left-color:initial;right:-7px;transform-origin:center left}.tippy-box[data-placement^=right]>.tippy-arrow{left:0}.tippy-box[data-placement^=right]>.tippy-arrow:before{left:-7px;border-width:8px 8px 8px 0;border-right-color:initial;transform-origin:center right}.tippy-box[data-inertia][data-state=visible]{transition-timing-function:cubic-bezier(.54,1.5,.38,1.11)}.tippy-arrow{width:16px;height:16px;color:#333}.tippy-arrow:before{content:"";position:absolute;border-color:transparent;border-style:solid}.tippy-content{position:relative;padding:5px 9px;z-index:1} -------------------------------------------------------------------------------- /docs/column_exploration.clj: -------------------------------------------------------------------------------- 1 | ;; # Tablecloth Column Exploration 2 | 3 | ^{:kind/hidden true} 4 | (ns intro 5 | (:require [tablecloth.api :as tc] 6 | [scicloj.clay.v2.api :as clay] 7 | [scicloj.kindly.v3.api :as kindly] 8 | [scicloj.kindly.v3.kind :as kind])) 9 | 10 | ^{:kind/hidden true} 11 | (clay/start!) 12 | 13 | ^{:kind/hidden true} 14 | (comment 15 | (clay/show-doc! "docs/column_exploration.clj" {:hide-doc? true}) 16 | (clay/write-html! "docs/column_exploration.html") 17 | ,) 18 | 19 | ;; ## What is this exploration? 20 | ;; 21 | ;; We want to add a `column` entity to tablecloth that parallels `dataset`. It will make 22 | ;; the column a first-class entity within tablecloth. 23 | 24 | ;; ## Usage 25 | 26 | (require '[tablecloth.column.api :refer [column] :as col]) 27 | 28 | ;; ### Column creation 29 | 30 | ;; We can create an empty column like this: 31 | 32 | (column) 33 | 34 | ;; We can check if it it's a column. 35 | 36 | (col/column? (column)) 37 | 38 | ;; We can create a columns with data in a number of ways 39 | 40 | (column [1 2 3 4]) 41 | 42 | (column (range 10)) 43 | 44 | ;; When you do this the types of the resulting array is determined 45 | ;; automatically from the items provided. 46 | 47 | (let [int-column (column (range 10))] 48 | (col/typeof int-column)) 49 | 50 | (let [string-column (column ["foo" "bar"])] 51 | (col/typeof string-column)) 52 | 53 | 54 | ;; ### Basic Operations 55 | 56 | ;; Operations are right now in their own namespace 57 | (require '[tablecloth.column.api.operators :as ops]) 58 | 59 | ;; With that imported we can perform a large number of operations: 60 | 61 | (def a (column [20 30 40 50])) 62 | (def b (column (range 4))) 63 | 64 | (ops/- a b) 65 | 66 | (ops/pow a 2) 67 | 68 | (ops/* 10 (ops/sin a)) 69 | 70 | (ops/< a 35) 71 | 72 | ;; All these operations take a column as their first argument and 73 | ;; return a column, so they can be chained easily. 74 | 75 | (-> a 76 | (ops/* b) 77 | (ops/< 70)) 78 | 79 | ;; ### Subsetting and accesssing 80 | 81 | ;; You can access an element in a column in exactly the same ways you 82 | ;; would in Clojure. 83 | 84 | (def myclm (column (range 5))) 85 | 86 | myclm 87 | 88 | (myclm 2) 89 | 90 | (nth myclm 2) 91 | 92 | (get myclm 2) 93 | 94 | ;; #### Selecting multiple elements 95 | 96 | ;; There are two ways to select multiple elements from a column: 97 | ;; * If you need to select a continuous subset, you can use `slice`; 98 | ;; * if you may need to select diverse elements, use `select`. 99 | ;; 100 | 101 | ;; **Slice** 102 | 103 | ;; The `slice` method allows you to use indexes to specify a portion 104 | ;; of the column to extract. 105 | 106 | (def myclm 107 | (column (repeatedly 10 #(rand-int 10)))) 108 | 109 | myclm 110 | 111 | (col/slice myclm 3 5) 112 | 113 | 114 | ;; It also supports negative indexing, making it possible to slice 115 | ;; from the end of the column: 116 | 117 | (col/slice myclm -7 -5) 118 | 119 | ;; It's also possible to slice from one direction to the beginning or 120 | ;; end: 121 | 122 | (col/slice myclm 7 :end) 123 | 124 | (col/slice myclm -3 :end) 125 | 126 | (col/slice myclm :start 7) 127 | 128 | (col/slice myclm :start -3) 129 | 130 | ;; **Select** 131 | ;; 132 | ;; The `select` fn works by taking a list of index positions: 133 | 134 | (col/select myclm [1 3 5 8]) 135 | 136 | ;; We can combine this type of selection with the operations just 137 | ;; demonstrated to select certain values. 138 | 139 | 140 | myclm 141 | 142 | ;; Let's see which positions are greter than 5. 143 | (ops/> myclm 5) 144 | 145 | 146 | ;; We can use a column of boolean values like the one above with the `select` function as well. `select` will choose all the positions that are true. It's like supplying select a list of the index positions that hold true values. 147 | (col/select myclm (ops/> myclm 5)) 148 | 149 | 150 | ;; ### Iterating over a column 151 | 152 | ;; Many operations that you might want to perform on a column are 153 | ;; available in the `tablecloth.column.api.operators` namespace. 154 | ;; However, when there is a need to do something custom, you can also 155 | ;; interate over the column. 156 | 157 | (defn calc-percent [x] 158 | (/ x 100.0)) 159 | 160 | (col/column-map myclm calc-percent) 161 | 162 | ;; It's also possible to iterate over multiple columns by supplying a 163 | ;; vector of columns: 164 | 165 | (-> [(column [5 6 7 8 9]) 166 | (column [1 2 3 4 5])] 167 | (col/column-map (partial *))) 168 | 169 | 170 | (comment 171 | (-> (column [1 nil 2 3 nil 0]) 172 | (ops/* 10)) 173 | 174 | (-> (column [1 nil 2 3 nil 0]) 175 | (ops/max [10 10 10 10 10 10])) 176 | 177 | 178 | (tech.v3.dataset.column/missing)) 179 | 180 | ;; ### Sorting a column 181 | 182 | ;; You can use `sort-column` to sort a colum 183 | 184 | (def myclm (column (repeatedly 10 #(rand-int 100)))) 185 | 186 | myclm 187 | 188 | (col/sort-column myclm) 189 | 190 | ;; As you can see, sort-columns sorts in ascending order by default, 191 | ;; but you can also specify a different order using ordering keywords 192 | ;; `:asc` and `:desc`: 193 | 194 | 195 | (col/sort-column myclm :desc) 196 | 197 | ;; Finally, sort can also accept a `comparator-fn`: 198 | 199 | (let [c (column ["1" "100" "4" "-10"])] 200 | (col/sort-column c (fn [a b] 201 | (let [a (parse-long a) 202 | b (parse-long b)] 203 | (< a b))))) 204 | 205 | 206 | ;; ### Missing values 207 | 208 | ;; The column has built-in support for basic awareness and handling of 209 | ;; missing values. Columns will be scanned for missing values when 210 | ;; created. 211 | 212 | (def myclm (column [10 nil -4 20 1000 nil -233])) 213 | 214 | ;; You can identify the set of index positions of missing values: 215 | 216 | (col/missing myclm) 217 | 218 | (col/count-missing myclm) 219 | 220 | ;; You can remove missing values: 221 | 222 | (col/drop-missing myclm) 223 | 224 | ;; Or you can replace them: 225 | 226 | (col/replace-missing myclm) 227 | 228 | ;; There are a range of built-in strategies: 229 | 230 | (col/replace-missing myclm :midpoint) 231 | 232 | 233 | ;; And you can provide your own value using a specific value or fn: 234 | 235 | (col/replace-missing myclm :value 555) 236 | 237 | (col/replace-missing myclm :value (fn [col-without-missing] 238 | (ops/mean col-without-missing))) 239 | -------------------------------------------------------------------------------- /docs/index_files/libs/bootstrap/bootstrap-icons.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scicloj/tablecloth/9ea514419e2d0777cf635a93cbd4c383b75b5d16/docs/index_files/libs/bootstrap/bootstrap-icons.woff -------------------------------------------------------------------------------- /docs/index_files/libs/quarto-html/anchor.min.js: -------------------------------------------------------------------------------- 1 | // @license magnet:?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&dn=expat.txt Expat 2 | // 3 | // AnchorJS - v5.0.0 - 2023-01-18 4 | // https://www.bryanbraun.com/anchorjs/ 5 | // Copyright (c) 2023 Bryan Braun; Licensed MIT 6 | // 7 | // @license magnet:?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&dn=expat.txt Expat 8 | !function(A,e){"use strict";"function"==typeof define&&define.amd?define([],e):"object"==typeof module&&module.exports?module.exports=e():(A.AnchorJS=e(),A.anchors=new A.AnchorJS)}(globalThis,function(){"use strict";return function(A){function u(A){A.icon=Object.prototype.hasOwnProperty.call(A,"icon")?A.icon:"",A.visible=Object.prototype.hasOwnProperty.call(A,"visible")?A.visible:"hover",A.placement=Object.prototype.hasOwnProperty.call(A,"placement")?A.placement:"right",A.ariaLabel=Object.prototype.hasOwnProperty.call(A,"ariaLabel")?A.ariaLabel:"Anchor",A.class=Object.prototype.hasOwnProperty.call(A,"class")?A.class:"",A.base=Object.prototype.hasOwnProperty.call(A,"base")?A.base:"",A.truncate=Object.prototype.hasOwnProperty.call(A,"truncate")?Math.floor(A.truncate):64,A.titleText=Object.prototype.hasOwnProperty.call(A,"titleText")?A.titleText:""}function d(A){var e;if("string"==typeof A||A instanceof String)e=[].slice.call(document.querySelectorAll(A));else{if(!(Array.isArray(A)||A instanceof NodeList))throw new TypeError("The selector provided to AnchorJS was invalid.");e=[].slice.call(A)}return e}this.options=A||{},this.elements=[],u(this.options),this.add=function(A){var e,t,o,i,n,s,a,r,l,c,h,p=[];if(u(this.options),0!==(e=d(A=A||"h2, h3, h4, h5, h6")).length){for(null===document.head.querySelector("style.anchorjs")&&((A=document.createElement("style")).className="anchorjs",A.appendChild(document.createTextNode("")),void 0===(h=document.head.querySelector('[rel="stylesheet"],style'))?document.head.appendChild(A):document.head.insertBefore(A,h),A.sheet.insertRule(".anchorjs-link{opacity:0;text-decoration:none;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}",A.sheet.cssRules.length),A.sheet.insertRule(":hover>.anchorjs-link,.anchorjs-link:focus{opacity:1}",A.sheet.cssRules.length),A.sheet.insertRule("[data-anchorjs-icon]::after{content:attr(data-anchorjs-icon)}",A.sheet.cssRules.length),A.sheet.insertRule('@font-face{font-family:anchorjs-icons;src:url(data:n/a;base64,AAEAAAALAIAAAwAwT1MvMg8yG2cAAAE4AAAAYGNtYXDp3gC3AAABpAAAAExnYXNwAAAAEAAAA9wAAAAIZ2x5ZlQCcfwAAAH4AAABCGhlYWQHFvHyAAAAvAAAADZoaGVhBnACFwAAAPQAAAAkaG10eASAADEAAAGYAAAADGxvY2EACACEAAAB8AAAAAhtYXhwAAYAVwAAARgAAAAgbmFtZQGOH9cAAAMAAAAAunBvc3QAAwAAAAADvAAAACAAAQAAAAEAAHzE2p9fDzz1AAkEAAAAAADRecUWAAAAANQA6R8AAAAAAoACwAAAAAgAAgAAAAAAAAABAAADwP/AAAACgAAA/9MCrQABAAAAAAAAAAAAAAAAAAAAAwABAAAAAwBVAAIAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAMCQAGQAAUAAAKZAswAAACPApkCzAAAAesAMwEJAAAAAAAAAAAAAAAAAAAAARAAAAAAAAAAAAAAAAAAAAAAQAAg//0DwP/AAEADwABAAAAAAQAAAAAAAAAAAAAAIAAAAAAAAAIAAAACgAAxAAAAAwAAAAMAAAAcAAEAAwAAABwAAwABAAAAHAAEADAAAAAIAAgAAgAAACDpy//9//8AAAAg6cv//f///+EWNwADAAEAAAAAAAAAAAAAAAAACACEAAEAAAAAAAAAAAAAAAAxAAACAAQARAKAAsAAKwBUAAABIiYnJjQ3NzY2MzIWFxYUBwcGIicmNDc3NjQnJiYjIgYHBwYUFxYUBwYGIwciJicmNDc3NjIXFhQHBwYUFxYWMzI2Nzc2NCcmNDc2MhcWFAcHBgYjARQGDAUtLXoWOR8fORYtLTgKGwoKCjgaGg0gEhIgDXoaGgkJBQwHdR85Fi0tOAobCgoKOBoaDSASEiANehoaCQkKGwotLXoWOR8BMwUFLYEuehYXFxYugC44CQkKGwo4GkoaDQ0NDXoaShoKGwoFBe8XFi6ALjgJCQobCjgaShoNDQ0NehpKGgobCgoKLYEuehYXAAAADACWAAEAAAAAAAEACAAAAAEAAAAAAAIAAwAIAAEAAAAAAAMACAAAAAEAAAAAAAQACAAAAAEAAAAAAAUAAQALAAEAAAAAAAYACAAAAAMAAQQJAAEAEAAMAAMAAQQJAAIABgAcAAMAAQQJAAMAEAAMAAMAAQQJAAQAEAAMAAMAAQQJAAUAAgAiAAMAAQQJAAYAEAAMYW5jaG9yanM0MDBAAGEAbgBjAGgAbwByAGoAcwA0ADAAMABAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAH//wAP) format("truetype")}',A.sheet.cssRules.length)),h=document.querySelectorAll("[id]"),t=[].map.call(h,function(A){return A.id}),i=0;i\]./()*\\\n\t\b\v\u00A0]/g,"-").replace(/-{2,}/g,"-").substring(0,this.options.truncate).replace(/^-+|-+$/gm,"").toLowerCase()},this.hasAnchorJSLink=function(A){var e=A.firstChild&&-1<(" "+A.firstChild.className+" ").indexOf(" anchorjs-link "),A=A.lastChild&&-1<(" "+A.lastChild.className+" ").indexOf(" anchorjs-link ");return e||A||!1}}}); 9 | // @license-end -------------------------------------------------------------------------------- /docs/index_files/libs/quarto-html/quarto-syntax-highlighting-e26fc0b31b203507ebcf2fc6137e19e2.css: -------------------------------------------------------------------------------- 1 | /* quarto syntax highlight colors */ 2 | :root { 3 | --quarto-hl-kw-color: #859900; 4 | --quarto-hl-fu-color: #268bd2; 5 | --quarto-hl-va-color: #268bd2; 6 | --quarto-hl-cf-color: #859900; 7 | --quarto-hl-op-color: #859900; 8 | --quarto-hl-bu-color: #cb4b16; 9 | --quarto-hl-ex-color: #268bd2; 10 | --quarto-hl-pp-color: #cb4b16; 11 | --quarto-hl-at-color: #268bd2; 12 | --quarto-hl-ch-color: #2aa198; 13 | --quarto-hl-sc-color: #dc322f; 14 | --quarto-hl-st-color: #2aa198; 15 | --quarto-hl-vs-color: #2aa198; 16 | --quarto-hl-ss-color: #dc322f; 17 | --quarto-hl-im-color: #2aa198; 18 | --quarto-hl-dt-color: #b58900; 19 | --quarto-hl-dv-color: #2aa198; 20 | --quarto-hl-bn-color: #2aa198; 21 | --quarto-hl-fl-color: #2aa198; 22 | --quarto-hl-cn-color: #2aa198; 23 | --quarto-hl-co-color: #93a1a1; 24 | --quarto-hl-do-color: #dc322f; 25 | --quarto-hl-an-color: #268bd2; 26 | --quarto-hl-cv-color: #2aa198; 27 | --quarto-hl-re-color: #268bd2; 28 | --quarto-hl-in-color: #b58900; 29 | --quarto-hl-wa-color: #cb4b16; 30 | --quarto-hl-al-color: #d33682; 31 | --quarto-hl-er-color: #dc322f; 32 | } 33 | 34 | /* other quarto variables */ 35 | :root { 36 | --quarto-font-monospace: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; 37 | } 38 | 39 | pre > code.sourceCode > span { 40 | color: #657b83; 41 | font-style: inherit; 42 | } 43 | 44 | code span { 45 | color: #657b83; 46 | font-style: inherit; 47 | } 48 | 49 | code.sourceCode > span { 50 | color: #657b83; 51 | font-style: inherit; 52 | } 53 | 54 | div.sourceCode, 55 | div.sourceCode pre.sourceCode { 56 | color: #657b83; 57 | font-style: inherit; 58 | } 59 | 60 | code span.kw { 61 | color: #859900; 62 | font-weight: bold; 63 | } 64 | 65 | code span.fu { 66 | color: #268bd2; 67 | } 68 | 69 | code span.va { 70 | color: #268bd2; 71 | } 72 | 73 | code span.cf { 74 | color: #859900; 75 | font-weight: bold; 76 | } 77 | 78 | code span.op { 79 | color: #859900; 80 | } 81 | 82 | code span.bu { 83 | color: #cb4b16; 84 | } 85 | 86 | code span.ex { 87 | color: #268bd2; 88 | font-weight: bold; 89 | } 90 | 91 | code span.pp { 92 | color: #cb4b16; 93 | } 94 | 95 | code span.at { 96 | color: #268bd2; 97 | } 98 | 99 | code span.ch { 100 | color: #2aa198; 101 | } 102 | 103 | code span.sc { 104 | color: #dc322f; 105 | } 106 | 107 | code span.st { 108 | color: #2aa198; 109 | } 110 | 111 | code span.vs { 112 | color: #2aa198; 113 | } 114 | 115 | code span.ss { 116 | color: #dc322f; 117 | } 118 | 119 | code span.im { 120 | color: #2aa198; 121 | } 122 | 123 | code span.dt { 124 | color: #b58900; 125 | font-weight: bold; 126 | } 127 | 128 | code span.dv { 129 | color: #2aa198; 130 | } 131 | 132 | code span.bn { 133 | color: #2aa198; 134 | } 135 | 136 | code span.fl { 137 | color: #2aa198; 138 | } 139 | 140 | code span.cn { 141 | color: #2aa198; 142 | font-weight: bold; 143 | } 144 | 145 | code span.co { 146 | color: #93a1a1; 147 | font-style: italic; 148 | } 149 | 150 | code span.do { 151 | color: #dc322f; 152 | } 153 | 154 | code span.an { 155 | color: #268bd2; 156 | } 157 | 158 | code span.cv { 159 | color: #2aa198; 160 | } 161 | 162 | code span.re { 163 | color: #268bd2; 164 | background-color: #eee8d5; 165 | } 166 | 167 | code span.in { 168 | color: #b58900; 169 | } 170 | 171 | code span.wa { 172 | color: #cb4b16; 173 | } 174 | 175 | code span.al { 176 | color: #d33682; 177 | font-weight: bold; 178 | } 179 | 180 | code span.er { 181 | color: #dc322f; 182 | text-decoration: underline; 183 | } 184 | 185 | .prevent-inlining { 186 | content: " code.sourceCode > span { 40 | color: #657b83; 41 | font-style: inherit; 42 | } 43 | 44 | code span { 45 | color: #657b83; 46 | font-style: inherit; 47 | } 48 | 49 | code.sourceCode > span { 50 | color: #657b83; 51 | font-style: inherit; 52 | } 53 | 54 | div.sourceCode, 55 | div.sourceCode pre.sourceCode { 56 | color: #657b83; 57 | font-style: inherit; 58 | } 59 | 60 | code span.kw { 61 | color: #859900; 62 | font-weight: bold; 63 | } 64 | 65 | code span.fu { 66 | color: #268bd2; 67 | } 68 | 69 | code span.va { 70 | color: #268bd2; 71 | } 72 | 73 | code span.cf { 74 | color: #859900; 75 | font-weight: bold; 76 | } 77 | 78 | code span.op { 79 | color: #859900; 80 | } 81 | 82 | code span.bu { 83 | color: #cb4b16; 84 | } 85 | 86 | code span.ex { 87 | color: #268bd2; 88 | font-weight: bold; 89 | } 90 | 91 | code span.pp { 92 | color: #cb4b16; 93 | } 94 | 95 | code span.at { 96 | color: #268bd2; 97 | } 98 | 99 | code span.ch { 100 | color: #2aa198; 101 | } 102 | 103 | code span.sc { 104 | color: #dc322f; 105 | } 106 | 107 | code span.st { 108 | color: #2aa198; 109 | } 110 | 111 | code span.vs { 112 | color: #2aa198; 113 | } 114 | 115 | code span.ss { 116 | color: #dc322f; 117 | } 118 | 119 | code span.im { 120 | color: #2aa198; 121 | } 122 | 123 | code span.dt { 124 | color: #b58900; 125 | font-weight: bold; 126 | } 127 | 128 | code span.dv { 129 | color: #2aa198; 130 | } 131 | 132 | code span.bn { 133 | color: #2aa198; 134 | } 135 | 136 | code span.fl { 137 | color: #2aa198; 138 | } 139 | 140 | code span.cn { 141 | color: #2aa198; 142 | font-weight: bold; 143 | } 144 | 145 | code span.co { 146 | color: #93a1a1; 147 | font-style: italic; 148 | } 149 | 150 | code span.do { 151 | color: #dc322f; 152 | } 153 | 154 | code span.an { 155 | color: #268bd2; 156 | } 157 | 158 | code span.cv { 159 | color: #2aa198; 160 | } 161 | 162 | code span.re { 163 | color: #268bd2; 164 | background-color: #eee8d5; 165 | } 166 | 167 | code span.in { 168 | color: #b58900; 169 | } 170 | 171 | code span.wa { 172 | color: #cb4b16; 173 | } 174 | 175 | code span.al { 176 | color: #d33682; 177 | font-weight: bold; 178 | } 179 | 180 | code span.er { 181 | color: #dc322f; 182 | text-decoration: underline; 183 | } 184 | 185 | .prevent-inlining { 186 | content: " { 5 | function getTabSettings() { 6 | const data = localStorage.getItem("quarto-persistent-tabsets-data"); 7 | if (!data) { 8 | localStorage.setItem("quarto-persistent-tabsets-data", "{}"); 9 | return {}; 10 | } 11 | if (data) { 12 | return JSON.parse(data); 13 | } 14 | } 15 | 16 | function setTabSettings(data) { 17 | localStorage.setItem( 18 | "quarto-persistent-tabsets-data", 19 | JSON.stringify(data) 20 | ); 21 | } 22 | 23 | function setTabState(groupName, groupValue) { 24 | const data = getTabSettings(); 25 | data[groupName] = groupValue; 26 | setTabSettings(data); 27 | } 28 | 29 | function toggleTab(tab, active) { 30 | const tabPanelId = tab.getAttribute("aria-controls"); 31 | const tabPanel = document.getElementById(tabPanelId); 32 | if (active) { 33 | tab.classList.add("active"); 34 | tabPanel.classList.add("active"); 35 | } else { 36 | tab.classList.remove("active"); 37 | tabPanel.classList.remove("active"); 38 | } 39 | } 40 | 41 | function toggleAll(selectedGroup, selectorsToSync) { 42 | for (const [thisGroup, tabs] of Object.entries(selectorsToSync)) { 43 | const active = selectedGroup === thisGroup; 44 | for (const tab of tabs) { 45 | toggleTab(tab, active); 46 | } 47 | } 48 | } 49 | 50 | function findSelectorsToSyncByLanguage() { 51 | const result = {}; 52 | const tabs = Array.from( 53 | document.querySelectorAll(`div[data-group] a[id^='tabset-']`) 54 | ); 55 | for (const item of tabs) { 56 | const div = item.parentElement.parentElement.parentElement; 57 | const group = div.getAttribute("data-group"); 58 | if (!result[group]) { 59 | result[group] = {}; 60 | } 61 | const selectorsToSync = result[group]; 62 | const value = item.innerHTML; 63 | if (!selectorsToSync[value]) { 64 | selectorsToSync[value] = []; 65 | } 66 | selectorsToSync[value].push(item); 67 | } 68 | return result; 69 | } 70 | 71 | function setupSelectorSync() { 72 | const selectorsToSync = findSelectorsToSyncByLanguage(); 73 | Object.entries(selectorsToSync).forEach(([group, tabSetsByValue]) => { 74 | Object.entries(tabSetsByValue).forEach(([value, items]) => { 75 | items.forEach((item) => { 76 | item.addEventListener("click", (_event) => { 77 | setTabState(group, value); 78 | toggleAll(value, selectorsToSync[group]); 79 | }); 80 | }); 81 | }); 82 | }); 83 | return selectorsToSync; 84 | } 85 | 86 | const selectorsToSync = setupSelectorSync(); 87 | for (const [group, selectedName] of Object.entries(getTabSettings())) { 88 | const selectors = selectorsToSync[group]; 89 | // it's possible that stale state gives us empty selections, so we explicitly check here. 90 | if (selectors) { 91 | toggleAll(selectedName, selectors); 92 | } 93 | } 94 | }); 95 | } 96 | -------------------------------------------------------------------------------- /docs/index_files/libs/quarto-html/tippy.css: -------------------------------------------------------------------------------- 1 | .tippy-box[data-animation=fade][data-state=hidden]{opacity:0}[data-tippy-root]{max-width:calc(100vw - 10px)}.tippy-box{position:relative;background-color:#333;color:#fff;border-radius:4px;font-size:14px;line-height:1.4;white-space:normal;outline:0;transition-property:transform,visibility,opacity}.tippy-box[data-placement^=top]>.tippy-arrow{bottom:0}.tippy-box[data-placement^=top]>.tippy-arrow:before{bottom:-7px;left:0;border-width:8px 8px 0;border-top-color:initial;transform-origin:center top}.tippy-box[data-placement^=bottom]>.tippy-arrow{top:0}.tippy-box[data-placement^=bottom]>.tippy-arrow:before{top:-7px;left:0;border-width:0 8px 8px;border-bottom-color:initial;transform-origin:center bottom}.tippy-box[data-placement^=left]>.tippy-arrow{right:0}.tippy-box[data-placement^=left]>.tippy-arrow:before{border-width:8px 0 8px 8px;border-left-color:initial;right:-7px;transform-origin:center left}.tippy-box[data-placement^=right]>.tippy-arrow{left:0}.tippy-box[data-placement^=right]>.tippy-arrow:before{left:-7px;border-width:8px 8px 8px 0;border-right-color:initial;transform-origin:center right}.tippy-box[data-inertia][data-state=visible]{transition-timing-function:cubic-bezier(.54,1.5,.38,1.11)}.tippy-arrow{width:16px;height:16px;color:#333}.tippy-arrow:before{content:"";position:absolute;border-color:transparent;border-style:solid}.tippy-content{position:relative;padding:5px 9px;z-index:1} -------------------------------------------------------------------------------- /docs/notebooks/custom.scss: -------------------------------------------------------------------------------- 1 | @import url(https://cdn.jsdelivr.net/npm/firacode@6.2.0/distr/fira_code.css); 2 | 3 | /*-- scss:rules --*/ 4 | .table {width:auto} 5 | 6 | code {font-family: 'Fira Code Medium', monospace;} 7 | 8 | .clay-dataset .table { 9 | @extend .table-striped; 10 | @extend .table-hover; 11 | @extend .table-responsive; 12 | } 13 | -------------------------------------------------------------------------------- /docs/old/index.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scicloj/tablecloth/9ea514419e2d0777cf635a93cbd4c383b75b5d16/docs/old/index.pdf -------------------------------------------------------------------------------- /notebooks/custom.scss: -------------------------------------------------------------------------------- 1 | @import url(https://cdn.jsdelivr.net/npm/firacode@6.2.0/distr/fira_code.css); 2 | 3 | /*-- scss:rules --*/ 4 | .table {width:auto} 5 | 6 | code {font-family: 'Fira Code Medium', monospace;} 7 | 8 | .clay-dataset .table { 9 | @extend .table-striped; 10 | @extend .table-hover; 11 | @extend .table-responsive; 12 | } 13 | -------------------------------------------------------------------------------- /pr-preview/pr-100/better_tables.html: -------------------------------------------------------------------------------- 1 | 10 | -------------------------------------------------------------------------------- /pr-preview/pr-100/column_api_files/libs/bootstrap/bootstrap-icons.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scicloj/tablecloth/9ea514419e2d0777cf635a93cbd4c383b75b5d16/pr-preview/pr-100/column_api_files/libs/bootstrap/bootstrap-icons.woff -------------------------------------------------------------------------------- /pr-preview/pr-100/column_api_files/libs/quarto-html/anchor.min.js: -------------------------------------------------------------------------------- 1 | // @license magnet:?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&dn=expat.txt Expat 2 | // 3 | // AnchorJS - v4.3.1 - 2021-04-17 4 | // https://www.bryanbraun.com/anchorjs/ 5 | // Copyright (c) 2021 Bryan Braun; Licensed MIT 6 | // 7 | // @license magnet:?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&dn=expat.txt Expat 8 | !function(A,e){"use strict";"function"==typeof define&&define.amd?define([],e):"object"==typeof module&&module.exports?module.exports=e():(A.AnchorJS=e(),A.anchors=new A.AnchorJS)}(this,function(){"use strict";return function(A){function d(A){A.icon=Object.prototype.hasOwnProperty.call(A,"icon")?A.icon:"",A.visible=Object.prototype.hasOwnProperty.call(A,"visible")?A.visible:"hover",A.placement=Object.prototype.hasOwnProperty.call(A,"placement")?A.placement:"right",A.ariaLabel=Object.prototype.hasOwnProperty.call(A,"ariaLabel")?A.ariaLabel:"Anchor",A.class=Object.prototype.hasOwnProperty.call(A,"class")?A.class:"",A.base=Object.prototype.hasOwnProperty.call(A,"base")?A.base:"",A.truncate=Object.prototype.hasOwnProperty.call(A,"truncate")?Math.floor(A.truncate):64,A.titleText=Object.prototype.hasOwnProperty.call(A,"titleText")?A.titleText:""}function w(A){var e;if("string"==typeof A||A instanceof String)e=[].slice.call(document.querySelectorAll(A));else{if(!(Array.isArray(A)||A instanceof NodeList))throw new TypeError("The selector provided to AnchorJS was invalid.");e=[].slice.call(A)}return e}this.options=A||{},this.elements=[],d(this.options),this.isTouchDevice=function(){return Boolean("ontouchstart"in window||window.TouchEvent||window.DocumentTouch&&document instanceof DocumentTouch)},this.add=function(A){var e,t,o,i,n,s,a,c,r,l,h,u,p=[];if(d(this.options),"touch"===(l=this.options.visible)&&(l=this.isTouchDevice()?"always":"hover"),0===(e=w(A=A||"h2, h3, h4, h5, h6")).length)return this;for(null===document.head.querySelector("style.anchorjs")&&((u=document.createElement("style")).className="anchorjs",u.appendChild(document.createTextNode("")),void 0===(A=document.head.querySelector('[rel="stylesheet"],style'))?document.head.appendChild(u):document.head.insertBefore(u,A),u.sheet.insertRule(".anchorjs-link{opacity:0;text-decoration:none;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}",u.sheet.cssRules.length),u.sheet.insertRule(":hover>.anchorjs-link,.anchorjs-link:focus{opacity:1}",u.sheet.cssRules.length),u.sheet.insertRule("[data-anchorjs-icon]::after{content:attr(data-anchorjs-icon)}",u.sheet.cssRules.length),u.sheet.insertRule('@font-face{font-family:anchorjs-icons;src:url(data:n/a;base64,AAEAAAALAIAAAwAwT1MvMg8yG2cAAAE4AAAAYGNtYXDp3gC3AAABpAAAAExnYXNwAAAAEAAAA9wAAAAIZ2x5ZlQCcfwAAAH4AAABCGhlYWQHFvHyAAAAvAAAADZoaGVhBnACFwAAAPQAAAAkaG10eASAADEAAAGYAAAADGxvY2EACACEAAAB8AAAAAhtYXhwAAYAVwAAARgAAAAgbmFtZQGOH9cAAAMAAAAAunBvc3QAAwAAAAADvAAAACAAAQAAAAEAAHzE2p9fDzz1AAkEAAAAAADRecUWAAAAANQA6R8AAAAAAoACwAAAAAgAAgAAAAAAAAABAAADwP/AAAACgAAA/9MCrQABAAAAAAAAAAAAAAAAAAAAAwABAAAAAwBVAAIAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAMCQAGQAAUAAAKZAswAAACPApkCzAAAAesAMwEJAAAAAAAAAAAAAAAAAAAAARAAAAAAAAAAAAAAAAAAAAAAQAAg//0DwP/AAEADwABAAAAAAQAAAAAAAAAAAAAAIAAAAAAAAAIAAAACgAAxAAAAAwAAAAMAAAAcAAEAAwAAABwAAwABAAAAHAAEADAAAAAIAAgAAgAAACDpy//9//8AAAAg6cv//f///+EWNwADAAEAAAAAAAAAAAAAAAAACACEAAEAAAAAAAAAAAAAAAAxAAACAAQARAKAAsAAKwBUAAABIiYnJjQ3NzY2MzIWFxYUBwcGIicmNDc3NjQnJiYjIgYHBwYUFxYUBwYGIwciJicmNDc3NjIXFhQHBwYUFxYWMzI2Nzc2NCcmNDc2MhcWFAcHBgYjARQGDAUtLXoWOR8fORYtLTgKGwoKCjgaGg0gEhIgDXoaGgkJBQwHdR85Fi0tOAobCgoKOBoaDSASEiANehoaCQkKGwotLXoWOR8BMwUFLYEuehYXFxYugC44CQkKGwo4GkoaDQ0NDXoaShoKGwoFBe8XFi6ALjgJCQobCjgaShoNDQ0NehpKGgobCgoKLYEuehYXAAAADACWAAEAAAAAAAEACAAAAAEAAAAAAAIAAwAIAAEAAAAAAAMACAAAAAEAAAAAAAQACAAAAAEAAAAAAAUAAQALAAEAAAAAAAYACAAAAAMAAQQJAAEAEAAMAAMAAQQJAAIABgAcAAMAAQQJAAMAEAAMAAMAAQQJAAQAEAAMAAMAAQQJAAUAAgAiAAMAAQQJAAYAEAAMYW5jaG9yanM0MDBAAGEAbgBjAGgAbwByAGoAcwA0ADAAMABAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAH//wAP) format("truetype")}',u.sheet.cssRules.length)),u=document.querySelectorAll("[id]"),t=[].map.call(u,function(A){return A.id}),i=0;i\]./()*\\\n\t\b\v\u00A0]/g,"-").replace(/-{2,}/g,"-").substring(0,this.options.truncate).replace(/^-+|-+$/gm,"").toLowerCase()},this.hasAnchorJSLink=function(A){var e=A.firstChild&&-1<(" "+A.firstChild.className+" ").indexOf(" anchorjs-link "),A=A.lastChild&&-1<(" "+A.lastChild.className+" ").indexOf(" anchorjs-link ");return e||A||!1}}}); 9 | // @license-end -------------------------------------------------------------------------------- /pr-preview/pr-100/column_api_files/libs/quarto-html/quarto-syntax-highlighting.css: -------------------------------------------------------------------------------- 1 | /* quarto syntax highlight colors */ 2 | :root { 3 | --quarto-hl-kw-color: #859900; 4 | --quarto-hl-fu-color: #268bd2; 5 | --quarto-hl-va-color: #268bd2; 6 | --quarto-hl-cf-color: #859900; 7 | --quarto-hl-op-color: #859900; 8 | --quarto-hl-bu-color: #cb4b16; 9 | --quarto-hl-ex-color: #268bd2; 10 | --quarto-hl-pp-color: #cb4b16; 11 | --quarto-hl-at-color: #268bd2; 12 | --quarto-hl-ch-color: #2aa198; 13 | --quarto-hl-sc-color: #dc322f; 14 | --quarto-hl-st-color: #2aa198; 15 | --quarto-hl-vs-color: #2aa198; 16 | --quarto-hl-ss-color: #dc322f; 17 | --quarto-hl-im-color: #2aa198; 18 | --quarto-hl-dt-color: #b58900; 19 | --quarto-hl-dv-color: #2aa198; 20 | --quarto-hl-bn-color: #2aa198; 21 | --quarto-hl-fl-color: #2aa198; 22 | --quarto-hl-cn-color: #2aa198; 23 | --quarto-hl-co-color: #93a1a1; 24 | --quarto-hl-do-color: #dc322f; 25 | --quarto-hl-an-color: #268bd2; 26 | --quarto-hl-cv-color: #2aa198; 27 | --quarto-hl-re-color: #268bd2; 28 | --quarto-hl-in-color: #b58900; 29 | --quarto-hl-wa-color: #cb4b16; 30 | --quarto-hl-al-color: #d33682; 31 | --quarto-hl-er-color: #dc322f; 32 | } 33 | 34 | /* other quarto variables */ 35 | :root { 36 | --quarto-font-monospace: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; 37 | } 38 | 39 | pre > code.sourceCode > span { 40 | color: #657b83; 41 | font-style: inherit; 42 | } 43 | 44 | code span { 45 | color: #657b83; 46 | font-style: inherit; 47 | } 48 | 49 | code.sourceCode > span { 50 | color: #657b83; 51 | font-style: inherit; 52 | } 53 | 54 | div.sourceCode, 55 | div.sourceCode pre.sourceCode { 56 | color: #657b83; 57 | font-style: inherit; 58 | } 59 | 60 | code span.kw { 61 | color: #859900; 62 | font-weight: bold; 63 | } 64 | 65 | code span.fu { 66 | color: #268bd2; 67 | } 68 | 69 | code span.va { 70 | color: #268bd2; 71 | } 72 | 73 | code span.cf { 74 | color: #859900; 75 | font-weight: bold; 76 | } 77 | 78 | code span.op { 79 | color: #859900; 80 | } 81 | 82 | code span.bu { 83 | color: #cb4b16; 84 | } 85 | 86 | code span.ex { 87 | color: #268bd2; 88 | font-weight: bold; 89 | } 90 | 91 | code span.pp { 92 | color: #cb4b16; 93 | } 94 | 95 | code span.at { 96 | color: #268bd2; 97 | } 98 | 99 | code span.ch { 100 | color: #2aa198; 101 | } 102 | 103 | code span.sc { 104 | color: #dc322f; 105 | } 106 | 107 | code span.st { 108 | color: #2aa198; 109 | } 110 | 111 | code span.vs { 112 | color: #2aa198; 113 | } 114 | 115 | code span.ss { 116 | color: #dc322f; 117 | } 118 | 119 | code span.im { 120 | color: #2aa198; 121 | } 122 | 123 | code span.dt { 124 | color: #b58900; 125 | font-weight: bold; 126 | } 127 | 128 | code span.dv { 129 | color: #2aa198; 130 | } 131 | 132 | code span.bn { 133 | color: #2aa198; 134 | } 135 | 136 | code span.fl { 137 | color: #2aa198; 138 | } 139 | 140 | code span.cn { 141 | color: #2aa198; 142 | font-weight: bold; 143 | } 144 | 145 | code span.co { 146 | color: #93a1a1; 147 | font-style: italic; 148 | } 149 | 150 | code span.do { 151 | color: #dc322f; 152 | } 153 | 154 | code span.an { 155 | color: #268bd2; 156 | } 157 | 158 | code span.cv { 159 | color: #2aa198; 160 | } 161 | 162 | code span.re { 163 | color: #268bd2; 164 | background-color: #eee8d5; 165 | } 166 | 167 | code span.in { 168 | color: #b58900; 169 | } 170 | 171 | code span.wa { 172 | color: #cb4b16; 173 | } 174 | 175 | code span.al { 176 | color: #d33682; 177 | font-weight: bold; 178 | } 179 | 180 | code span.er { 181 | color: #dc322f; 182 | text-decoration: underline; 183 | } 184 | 185 | .prevent-inlining { 186 | content: ".tippy-arrow{bottom:0}.tippy-box[data-placement^=top]>.tippy-arrow:before{bottom:-7px;left:0;border-width:8px 8px 0;border-top-color:initial;transform-origin:center top}.tippy-box[data-placement^=bottom]>.tippy-arrow{top:0}.tippy-box[data-placement^=bottom]>.tippy-arrow:before{top:-7px;left:0;border-width:0 8px 8px;border-bottom-color:initial;transform-origin:center bottom}.tippy-box[data-placement^=left]>.tippy-arrow{right:0}.tippy-box[data-placement^=left]>.tippy-arrow:before{border-width:8px 0 8px 8px;border-left-color:initial;right:-7px;transform-origin:center left}.tippy-box[data-placement^=right]>.tippy-arrow{left:0}.tippy-box[data-placement^=right]>.tippy-arrow:before{left:-7px;border-width:8px 8px 8px 0;border-right-color:initial;transform-origin:center right}.tippy-box[data-inertia][data-state=visible]{transition-timing-function:cubic-bezier(.54,1.5,.38,1.11)}.tippy-arrow{width:16px;height:16px;color:#333}.tippy-arrow:before{content:"";position:absolute;border-color:transparent;border-style:solid}.tippy-content{position:relative;padding:5px 9px;z-index:1} -------------------------------------------------------------------------------- /pr-preview/pr-100/column_exploration.clj: -------------------------------------------------------------------------------- 1 | ;; # Tablecloth Column Exploration 2 | 3 | ^{:kind/hidden true} 4 | (ns intro 5 | (:require [tablecloth.api :as tc] 6 | [scicloj.clay.v2.api :as clay] 7 | [scicloj.kindly.v3.api :as kindly] 8 | [scicloj.kindly.v3.kind :as kind])) 9 | 10 | ^{:kind/hidden true} 11 | (clay/start!) 12 | 13 | ^{:kind/hidden true} 14 | (comment 15 | (clay/show-doc! "docs/column_exploration.clj" {:hide-doc? true}) 16 | (clay/write-html! "docs/column_exploration.html") 17 | ,) 18 | 19 | ;; ## What is this exploration? 20 | ;; 21 | ;; We want to add a `column` entity to tablecloth that parallels `dataset`. It will make 22 | ;; the column a first-class entity within tablecloth. 23 | 24 | ;; ## Usage 25 | 26 | (require '[tablecloth.column.api :refer [column] :as col]) 27 | 28 | ;; ### Column creation 29 | 30 | ;; We can create an empty column like this: 31 | 32 | (column) 33 | 34 | ;; We can check if it it's a column. 35 | 36 | (col/column? (column)) 37 | 38 | ;; We can create a columns with data in a number of ways 39 | 40 | (column [1 2 3 4]) 41 | 42 | (column (range 10)) 43 | 44 | ;; When you do this the types of the resulting array is determined 45 | ;; automatically from the items provided. 46 | 47 | (let [int-column (column (range 10))] 48 | (col/typeof int-column)) 49 | 50 | (let [string-column (column ["foo" "bar"])] 51 | (col/typeof string-column)) 52 | 53 | 54 | ;; ### Basic Operations 55 | 56 | ;; Operations are right now in their own namespace 57 | (require '[tablecloth.column.api.operators :as ops]) 58 | 59 | ;; With that imported we can perform a large number of operations: 60 | 61 | (def a (column [20 30 40 50])) 62 | (def b (column (range 4))) 63 | 64 | (ops/- a b) 65 | 66 | (ops/pow a 2) 67 | 68 | (ops/* 10 (ops/sin a)) 69 | 70 | (ops/< a 35) 71 | 72 | ;; All these operations take a column as their first argument and 73 | ;; return a column, so they can be chained easily. 74 | 75 | (-> a 76 | (ops/* b) 77 | (ops/< 70)) 78 | 79 | ;; ### Subsetting and accesssing 80 | 81 | ;; You can access an element in a column in exactly the same ways you 82 | ;; would in Clojure. 83 | 84 | (def myclm (column (range 5))) 85 | 86 | myclm 87 | 88 | (myclm 2) 89 | 90 | (nth myclm 2) 91 | 92 | (get myclm 2) 93 | 94 | ;; #### Selecting multiple elements 95 | 96 | ;; There are two ways to select multiple elements from a column: 97 | ;; * If you need to select a continuous subset, you can use `slice`; 98 | ;; * if you may need to select diverse elements, use `select`. 99 | ;; 100 | 101 | ;; **Slice** 102 | 103 | ;; The `slice` method allows you to use indexes to specify a portion 104 | ;; of the column to extract. 105 | 106 | (def myclm 107 | (column (repeatedly 10 #(rand-int 10)))) 108 | 109 | myclm 110 | 111 | (col/slice myclm 3 5) 112 | 113 | 114 | ;; It also supports negative indexing, making it possible to slice 115 | ;; from the end of the column: 116 | 117 | (col/slice myclm -7 -5) 118 | 119 | ;; It's also possible to slice from one direction to the beginning or 120 | ;; end: 121 | 122 | (col/slice myclm 7 :end) 123 | 124 | (col/slice myclm -3 :end) 125 | 126 | (col/slice myclm :start 7) 127 | 128 | (col/slice myclm :start -3) 129 | 130 | ;; **Select** 131 | ;; 132 | ;; The `select` fn works by taking a list of index positions: 133 | 134 | (col/select myclm [1 3 5 8]) 135 | 136 | ;; We can combine this type of selection with the operations just 137 | ;; demonstrated to select certain values. 138 | 139 | 140 | myclm 141 | 142 | ;; Let's see which positions are greter than 5. 143 | (ops/> myclm 5) 144 | 145 | 146 | ;; We can use a column of boolean values like the one above with the `select` function as well. `select` will choose all the positions that are true. It's like supplying select a list of the index positions that hold true values. 147 | (col/select myclm (ops/> myclm 5)) 148 | 149 | 150 | ;; ### Iterating over a column 151 | 152 | ;; Many operations that you might want to perform on a column are 153 | ;; available in the `tablecloth.column.api.operators` namespace. 154 | ;; However, when there is a need to do something custom, you can also 155 | ;; interate over the column. 156 | 157 | (defn calc-percent [x] 158 | (/ x 100.0)) 159 | 160 | (col/column-map myclm calc-percent) 161 | 162 | ;; It's also possible to iterate over multiple columns by supplying a 163 | ;; vector of columns: 164 | 165 | (-> [(column [5 6 7 8 9]) 166 | (column [1 2 3 4 5])] 167 | (col/column-map (partial *))) 168 | 169 | 170 | (comment 171 | (-> (column [1 nil 2 3 nil 0]) 172 | (ops/* 10)) 173 | 174 | (-> (column [1 nil 2 3 nil 0]) 175 | (ops/max [10 10 10 10 10 10])) 176 | 177 | 178 | (tech.v3.dataset.column/missing)) 179 | 180 | ;; ### Sorting a column 181 | 182 | ;; You can use `sort-column` to sort a colum 183 | 184 | (def myclm (column (repeatedly 10 #(rand-int 100)))) 185 | 186 | myclm 187 | 188 | (col/sort-column myclm) 189 | 190 | ;; As you can see, sort-columns sorts in ascending order by default, 191 | ;; but you can also specify a different order using ordering keywords 192 | ;; `:asc` and `:desc`: 193 | 194 | 195 | (col/sort-column myclm :desc) 196 | 197 | ;; Finally, sort can also accept a `comparator-fn`: 198 | 199 | (let [c (column ["1" "100" "4" "-10"])] 200 | (col/sort-column c (fn [a b] 201 | (let [a (parse-long a) 202 | b (parse-long b)] 203 | (< a b))))) 204 | 205 | 206 | ;; ### Missing values 207 | 208 | ;; The column has built-in support for basic awareness and handling of 209 | ;; missing values. Columns will be scanned for missing values when 210 | ;; created. 211 | 212 | (def myclm (column [10 nil -4 20 1000 nil -233])) 213 | 214 | ;; You can identify the set of index positions of missing values: 215 | 216 | (col/missing myclm) 217 | 218 | (col/count-missing myclm) 219 | 220 | ;; You can remove missing values: 221 | 222 | (col/drop-missing myclm) 223 | 224 | ;; Or you can replace them: 225 | 226 | (col/replace-missing myclm) 227 | 228 | ;; There are a range of built-in strategies: 229 | 230 | (col/replace-missing myclm :midpoint) 231 | 232 | 233 | ;; And you can provide your own value using a specific value or fn: 234 | 235 | (col/replace-missing myclm :value 555) 236 | 237 | (col/replace-missing myclm :value (fn [col-without-missing] 238 | (ops/mean col-without-missing))) 239 | -------------------------------------------------------------------------------- /pr-preview/pr-100/index_files/libs/bootstrap/bootstrap-icons.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scicloj/tablecloth/9ea514419e2d0777cf635a93cbd4c383b75b5d16/pr-preview/pr-100/index_files/libs/bootstrap/bootstrap-icons.woff -------------------------------------------------------------------------------- /pr-preview/pr-100/index_files/libs/quarto-html/anchor.min.js: -------------------------------------------------------------------------------- 1 | // @license magnet:?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&dn=expat.txt Expat 2 | // 3 | // AnchorJS - v5.0.0 - 2023-01-18 4 | // https://www.bryanbraun.com/anchorjs/ 5 | // Copyright (c) 2023 Bryan Braun; Licensed MIT 6 | // 7 | // @license magnet:?xt=urn:btih:d3d9a9a6595521f9666a5e94cc830dab83b65699&dn=expat.txt Expat 8 | !function(A,e){"use strict";"function"==typeof define&&define.amd?define([],e):"object"==typeof module&&module.exports?module.exports=e():(A.AnchorJS=e(),A.anchors=new A.AnchorJS)}(globalThis,function(){"use strict";return function(A){function u(A){A.icon=Object.prototype.hasOwnProperty.call(A,"icon")?A.icon:"",A.visible=Object.prototype.hasOwnProperty.call(A,"visible")?A.visible:"hover",A.placement=Object.prototype.hasOwnProperty.call(A,"placement")?A.placement:"right",A.ariaLabel=Object.prototype.hasOwnProperty.call(A,"ariaLabel")?A.ariaLabel:"Anchor",A.class=Object.prototype.hasOwnProperty.call(A,"class")?A.class:"",A.base=Object.prototype.hasOwnProperty.call(A,"base")?A.base:"",A.truncate=Object.prototype.hasOwnProperty.call(A,"truncate")?Math.floor(A.truncate):64,A.titleText=Object.prototype.hasOwnProperty.call(A,"titleText")?A.titleText:""}function d(A){var e;if("string"==typeof A||A instanceof String)e=[].slice.call(document.querySelectorAll(A));else{if(!(Array.isArray(A)||A instanceof NodeList))throw new TypeError("The selector provided to AnchorJS was invalid.");e=[].slice.call(A)}return e}this.options=A||{},this.elements=[],u(this.options),this.add=function(A){var e,t,o,i,n,s,a,r,l,c,h,p=[];if(u(this.options),0!==(e=d(A=A||"h2, h3, h4, h5, h6")).length){for(null===document.head.querySelector("style.anchorjs")&&((A=document.createElement("style")).className="anchorjs",A.appendChild(document.createTextNode("")),void 0===(h=document.head.querySelector('[rel="stylesheet"],style'))?document.head.appendChild(A):document.head.insertBefore(A,h),A.sheet.insertRule(".anchorjs-link{opacity:0;text-decoration:none;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}",A.sheet.cssRules.length),A.sheet.insertRule(":hover>.anchorjs-link,.anchorjs-link:focus{opacity:1}",A.sheet.cssRules.length),A.sheet.insertRule("[data-anchorjs-icon]::after{content:attr(data-anchorjs-icon)}",A.sheet.cssRules.length),A.sheet.insertRule('@font-face{font-family:anchorjs-icons;src:url(data:n/a;base64,AAEAAAALAIAAAwAwT1MvMg8yG2cAAAE4AAAAYGNtYXDp3gC3AAABpAAAAExnYXNwAAAAEAAAA9wAAAAIZ2x5ZlQCcfwAAAH4AAABCGhlYWQHFvHyAAAAvAAAADZoaGVhBnACFwAAAPQAAAAkaG10eASAADEAAAGYAAAADGxvY2EACACEAAAB8AAAAAhtYXhwAAYAVwAAARgAAAAgbmFtZQGOH9cAAAMAAAAAunBvc3QAAwAAAAADvAAAACAAAQAAAAEAAHzE2p9fDzz1AAkEAAAAAADRecUWAAAAANQA6R8AAAAAAoACwAAAAAgAAgAAAAAAAAABAAADwP/AAAACgAAA/9MCrQABAAAAAAAAAAAAAAAAAAAAAwABAAAAAwBVAAIAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAMCQAGQAAUAAAKZAswAAACPApkCzAAAAesAMwEJAAAAAAAAAAAAAAAAAAAAARAAAAAAAAAAAAAAAAAAAAAAQAAg//0DwP/AAEADwABAAAAAAQAAAAAAAAAAAAAAIAAAAAAAAAIAAAACgAAxAAAAAwAAAAMAAAAcAAEAAwAAABwAAwABAAAAHAAEADAAAAAIAAgAAgAAACDpy//9//8AAAAg6cv//f///+EWNwADAAEAAAAAAAAAAAAAAAAACACEAAEAAAAAAAAAAAAAAAAxAAACAAQARAKAAsAAKwBUAAABIiYnJjQ3NzY2MzIWFxYUBwcGIicmNDc3NjQnJiYjIgYHBwYUFxYUBwYGIwciJicmNDc3NjIXFhQHBwYUFxYWMzI2Nzc2NCcmNDc2MhcWFAcHBgYjARQGDAUtLXoWOR8fORYtLTgKGwoKCjgaGg0gEhIgDXoaGgkJBQwHdR85Fi0tOAobCgoKOBoaDSASEiANehoaCQkKGwotLXoWOR8BMwUFLYEuehYXFxYugC44CQkKGwo4GkoaDQ0NDXoaShoKGwoFBe8XFi6ALjgJCQobCjgaShoNDQ0NehpKGgobCgoKLYEuehYXAAAADACWAAEAAAAAAAEACAAAAAEAAAAAAAIAAwAIAAEAAAAAAAMACAAAAAEAAAAAAAQACAAAAAEAAAAAAAUAAQALAAEAAAAAAAYACAAAAAMAAQQJAAEAEAAMAAMAAQQJAAIABgAcAAMAAQQJAAMAEAAMAAMAAQQJAAQAEAAMAAMAAQQJAAUAAgAiAAMAAQQJAAYAEAAMYW5jaG9yanM0MDBAAGEAbgBjAGgAbwByAGoAcwA0ADAAMABAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAH//wAP) format("truetype")}',A.sheet.cssRules.length)),h=document.querySelectorAll("[id]"),t=[].map.call(h,function(A){return A.id}),i=0;i\]./()*\\\n\t\b\v\u00A0]/g,"-").replace(/-{2,}/g,"-").substring(0,this.options.truncate).replace(/^-+|-+$/gm,"").toLowerCase()},this.hasAnchorJSLink=function(A){var e=A.firstChild&&-1<(" "+A.firstChild.className+" ").indexOf(" anchorjs-link "),A=A.lastChild&&-1<(" "+A.lastChild.className+" ").indexOf(" anchorjs-link ");return e||A||!1}}}); 9 | // @license-end -------------------------------------------------------------------------------- /pr-preview/pr-100/index_files/libs/quarto-html/quarto-syntax-highlighting.css: -------------------------------------------------------------------------------- 1 | /* quarto syntax highlight colors */ 2 | :root { 3 | --quarto-hl-kw-color: #859900; 4 | --quarto-hl-fu-color: #268bd2; 5 | --quarto-hl-va-color: #268bd2; 6 | --quarto-hl-cf-color: #859900; 7 | --quarto-hl-op-color: #859900; 8 | --quarto-hl-bu-color: #cb4b16; 9 | --quarto-hl-ex-color: #268bd2; 10 | --quarto-hl-pp-color: #cb4b16; 11 | --quarto-hl-at-color: #268bd2; 12 | --quarto-hl-ch-color: #2aa198; 13 | --quarto-hl-sc-color: #dc322f; 14 | --quarto-hl-st-color: #2aa198; 15 | --quarto-hl-vs-color: #2aa198; 16 | --quarto-hl-ss-color: #dc322f; 17 | --quarto-hl-im-color: #2aa198; 18 | --quarto-hl-dt-color: #b58900; 19 | --quarto-hl-dv-color: #2aa198; 20 | --quarto-hl-bn-color: #2aa198; 21 | --quarto-hl-fl-color: #2aa198; 22 | --quarto-hl-cn-color: #2aa198; 23 | --quarto-hl-co-color: #93a1a1; 24 | --quarto-hl-do-color: #dc322f; 25 | --quarto-hl-an-color: #268bd2; 26 | --quarto-hl-cv-color: #2aa198; 27 | --quarto-hl-re-color: #268bd2; 28 | --quarto-hl-in-color: #b58900; 29 | --quarto-hl-wa-color: #cb4b16; 30 | --quarto-hl-al-color: #d33682; 31 | --quarto-hl-er-color: #dc322f; 32 | } 33 | 34 | /* other quarto variables */ 35 | :root { 36 | --quarto-font-monospace: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; 37 | } 38 | 39 | pre > code.sourceCode > span { 40 | color: #657b83; 41 | font-style: inherit; 42 | } 43 | 44 | code span { 45 | color: #657b83; 46 | font-style: inherit; 47 | } 48 | 49 | code.sourceCode > span { 50 | color: #657b83; 51 | font-style: inherit; 52 | } 53 | 54 | div.sourceCode, 55 | div.sourceCode pre.sourceCode { 56 | color: #657b83; 57 | font-style: inherit; 58 | } 59 | 60 | code span.kw { 61 | color: #859900; 62 | font-weight: bold; 63 | } 64 | 65 | code span.fu { 66 | color: #268bd2; 67 | } 68 | 69 | code span.va { 70 | color: #268bd2; 71 | } 72 | 73 | code span.cf { 74 | color: #859900; 75 | font-weight: bold; 76 | } 77 | 78 | code span.op { 79 | color: #859900; 80 | } 81 | 82 | code span.bu { 83 | color: #cb4b16; 84 | } 85 | 86 | code span.ex { 87 | color: #268bd2; 88 | font-weight: bold; 89 | } 90 | 91 | code span.pp { 92 | color: #cb4b16; 93 | } 94 | 95 | code span.at { 96 | color: #268bd2; 97 | } 98 | 99 | code span.ch { 100 | color: #2aa198; 101 | } 102 | 103 | code span.sc { 104 | color: #dc322f; 105 | } 106 | 107 | code span.st { 108 | color: #2aa198; 109 | } 110 | 111 | code span.vs { 112 | color: #2aa198; 113 | } 114 | 115 | code span.ss { 116 | color: #dc322f; 117 | } 118 | 119 | code span.im { 120 | color: #2aa198; 121 | } 122 | 123 | code span.dt { 124 | color: #b58900; 125 | font-weight: bold; 126 | } 127 | 128 | code span.dv { 129 | color: #2aa198; 130 | } 131 | 132 | code span.bn { 133 | color: #2aa198; 134 | } 135 | 136 | code span.fl { 137 | color: #2aa198; 138 | } 139 | 140 | code span.cn { 141 | color: #2aa198; 142 | font-weight: bold; 143 | } 144 | 145 | code span.co { 146 | color: #93a1a1; 147 | font-style: italic; 148 | } 149 | 150 | code span.do { 151 | color: #dc322f; 152 | } 153 | 154 | code span.an { 155 | color: #268bd2; 156 | } 157 | 158 | code span.cv { 159 | color: #2aa198; 160 | } 161 | 162 | code span.re { 163 | color: #268bd2; 164 | background-color: #eee8d5; 165 | } 166 | 167 | code span.in { 168 | color: #b58900; 169 | } 170 | 171 | code span.wa { 172 | color: #cb4b16; 173 | } 174 | 175 | code span.al { 176 | color: #d33682; 177 | font-weight: bold; 178 | } 179 | 180 | code span.er { 181 | color: #dc322f; 182 | text-decoration: underline; 183 | } 184 | 185 | .prevent-inlining { 186 | content: ".tippy-arrow{bottom:0}.tippy-box[data-placement^=top]>.tippy-arrow:before{bottom:-7px;left:0;border-width:8px 8px 0;border-top-color:initial;transform-origin:center top}.tippy-box[data-placement^=bottom]>.tippy-arrow{top:0}.tippy-box[data-placement^=bottom]>.tippy-arrow:before{top:-7px;left:0;border-width:0 8px 8px;border-bottom-color:initial;transform-origin:center bottom}.tippy-box[data-placement^=left]>.tippy-arrow{right:0}.tippy-box[data-placement^=left]>.tippy-arrow:before{border-width:8px 0 8px 8px;border-left-color:initial;right:-7px;transform-origin:center left}.tippy-box[data-placement^=right]>.tippy-arrow{left:0}.tippy-box[data-placement^=right]>.tippy-arrow:before{left:-7px;border-width:8px 8px 8px 0;border-right-color:initial;transform-origin:center right}.tippy-box[data-inertia][data-state=visible]{transition-timing-function:cubic-bezier(.54,1.5,.38,1.11)}.tippy-arrow{width:16px;height:16px;color:#333}.tippy-arrow:before{content:"";position:absolute;border-color:transparent;border-style:solid}.tippy-content{position:relative;padding:5px 9px;z-index:1} -------------------------------------------------------------------------------- /pr-preview/pr-100/notebooks/custom.scss: -------------------------------------------------------------------------------- 1 | @import url(https://cdn.jsdelivr.net/npm/firacode@6.2.0/distr/fira_code.css); 2 | 3 | /*-- scss:rules --*/ 4 | .table {width:auto} 5 | 6 | code {font-family: 'Fira Code Medium', monospace;} 7 | 8 | .clay-dataset .table { 9 | @extend .table-striped; 10 | @extend .table-hover; 11 | @extend .table-responsive; 12 | } 13 | -------------------------------------------------------------------------------- /pr-preview/pr-100/old/index.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scicloj/tablecloth/9ea514419e2d0777cf635a93cbd4c383b75b5d16/pr-preview/pr-100/old/index.pdf -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject scicloj/tablecloth "7.059" 2 | :description "Dataset manipulation library built on the top of tech.ml.dataset." 3 | :url "https://github.com/scicloj/tablecloth" 4 | :license {:name "The MIT Licence" 5 | :url "https://opensource.org/licenses/MIT"} 6 | :plugins [[lein-tools-deps "0.4.5"]] 7 | :middleware [lein-tools-deps.plugin/resolve-dependencies-with-deps-edn] 8 | :lein-tools-deps/config {:config-files [:install :user :project]} 9 | :profiles {:dev {:cloverage {:runner :midje} 10 | :dependencies [[midje "1.10.10"] 11 | [org.scicloj/clay "2-beta42"]] 12 | :plugins [[lein-midje "3.2.1"] 13 | [lein-cloverage "1.2.4"]]}}) 14 | -------------------------------------------------------------------------------- /src/tablecloth/api/aggregate.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.api.aggregate 2 | (:refer-clojure :exclude [group-by concat]) 3 | (:require [tech.v3.dataset :refer [row-count concat]] 4 | 5 | [tablecloth.api.utils :refer [iterable-sequence? ->str column-names grouped? process-group-data]] 6 | [tablecloth.api.columns :refer [add-column drop-columns rename-columns]] 7 | [tablecloth.api.group-by :refer [process->ungroup ungroup group-by]] 8 | [tablecloth.api.dataset :refer [dataset rows]] 9 | [tablecloth.api.reshape :refer [pivot->wider]] 10 | [tablecloth.api.missing :refer [replace-missing]])) 11 | 12 | (defn- add-agg-result-from-seq 13 | [ds k agg-res] 14 | (reduce (fn [d [id v]] 15 | (add-column d (keyword (str (->str k) "-" (->str id))) [v])) ds agg-res)) 16 | 17 | (defn- add-agg-result 18 | [ds k agg-res] 19 | (cond 20 | (map? agg-res) (add-agg-result-from-seq ds k agg-res) ;(reduce conj tot-res agg-res) 21 | (iterable-sequence? agg-res) (add-agg-result-from-seq ds k (map-indexed vector agg-res)) 22 | :else (add-column ds k [agg-res]))) 23 | 24 | (defn- aggregate-map->dataset 25 | [ds aggregator options] 26 | (reduce (fn [d [k f]] 27 | (add-agg-result d k (f ds))) (dataset nil options) aggregator)) 28 | 29 | (defn- aggregate-map 30 | [ds aggregator options] 31 | (-> (aggregate-map->dataset ds aggregator options) 32 | (rows :as-maps) 33 | (first))) 34 | 35 | (defn aggregate 36 | "Aggregate dataset by providing: 37 | 38 | - aggregation function 39 | - map with column names and functions 40 | - sequence of aggregation functions 41 | 42 | Aggregation functions can return: 43 | - single value 44 | - seq of values 45 | - map of values with column names" 46 | ([ds aggregator] (aggregate ds aggregator nil)) 47 | ([ds aggregator {:keys [default-column-name-prefix ungroup? parallel?] 48 | :or {default-column-name-prefix "summary" ungroup? true} 49 | :as options}] 50 | (let [aggregator (cond 51 | (fn? aggregator) {default-column-name-prefix aggregator} 52 | (iterable-sequence? aggregator) (->> aggregator 53 | (interleave (map #(->> % 54 | (str default-column-name-prefix "-") 55 | keyword) (range))) 56 | (apply array-map)) 57 | :else aggregator)] 58 | (if (grouped? ds) 59 | (cond 60 | (true? ungroup?) (process->ungroup ds #(seq (aggregate-map % aggregator options)) (assoc options :add-group-as-column true)) 61 | ungroup? (ungroup (process-group-data ds #(aggregate-map->dataset % aggregator options) parallel?) 62 | (assoc options :add-group-as-column true)) 63 | :else (process-group-data ds #(aggregate-map->dataset % aggregator options) parallel?)) 64 | (aggregate-map->dataset ds aggregator options))))) 65 | 66 | (defn aggregate-columns 67 | "Aggregates each column separately" 68 | ([ds columns-aggregators] (aggregate-columns ds (column-names ds) columns-aggregators)) 69 | ([ds columns-selector column-aggregators] (aggregate-columns ds columns-selector column-aggregators nil)) 70 | ([ds columns-selector column-aggregators options] 71 | (let [aggregators (if (iterable-sequence? column-aggregators) 72 | (cycle column-aggregators) 73 | (repeat column-aggregators)) 74 | colnames (column-names ds columns-selector)] 75 | (aggregate ds (apply array-map (mapcat (fn [aggr col-name] 76 | [col-name #(aggr (% col-name))]) 77 | aggregators colnames)) 78 | options)))) 79 | 80 | ;; 81 | 82 | (defn- default-marginal 83 | [marginal-fn?] 84 | (when marginal-fn? 85 | (if (fn? marginal-fn?) marginal-fn? (partial reduce +)))) 86 | 87 | (defn- crosstab-impl 88 | [ds row-names row-vals-fn col-names col-vals-fn 89 | {:keys [replace-missing? pivot? aggregator missing-value marginal-rows marginal-cols] 90 | :or {replace-missing? true pivot? true aggregator row-count missing-value 0}}] 91 | (as-> (-> ds 92 | (group-by (fn [data-row] 93 | {:rows (row-vals-fn (map data-row row-names)) 94 | :cols (col-vals-fn (map data-row col-names))})) 95 | (aggregate aggregator)) ds 96 | (if pivot? 97 | (as-> (rename-columns ds {:rows "rows/cols"}) ds 98 | (pivot->wider ds :cols "summary" {:drop-missing? false}) 99 | (if replace-missing? 100 | (replace-missing ds :all :value missing-value) 101 | ds) 102 | (if-let [marginal-fn (default-marginal marginal-rows)] 103 | (add-column ds :summary (mapv marginal-fn (rows (drop-columns ds ["rows/cols"])))) 104 | ds) 105 | (if-let [marginal-fn (default-marginal marginal-cols)] 106 | (concat ds (aggregate ds (apply array-map (mapcat (fn [col-name] 107 | [col-name (if (= col-name "rows/cols") 108 | (constantly :summary) 109 | #(marginal-fn (% col-name)))]) 110 | (column-names ds))))) 111 | ds)) 112 | ds))) 113 | 114 | (defn crosstab 115 | "Cross tabulation of two sets of columns. 116 | 117 | Creates grouped dataset by [row-selector, col-selector] pairs and calls aggregation on each group. 118 | 119 | Options: 120 | 121 | * pivot? - create pivot table or just flat structure (default: true) 122 | * replace-missing? - replace missing values? (default: true) 123 | * missing-value - a missing value (default: 0) 124 | * aggregator - aggregating function (default: row-count) 125 | * marginal-rows, marginal-cols - adds row and/or cols, it's a sum if true. Can be a custom fn." 126 | ([ds row-selector col-selector] (crosstab ds row-selector col-selector nil)) 127 | ([ds row-selector col-selector options] 128 | (let [row-names (column-names ds row-selector) 129 | col-names (column-names ds col-selector) 130 | row-vals-fn (if (= 1 (count row-names)) first vec) 131 | col-vals-fn (if (= 1 (count col-names)) first vec)] 132 | (if (grouped? ds) 133 | (let [res (process-group-data ds #(crosstab-impl % row-names row-vals-fn col-names col-vals-fn options) 134 | (:parallel? options))] 135 | (if (:ungroup? options) (ungroup res) res)) 136 | (crosstab-impl ds row-names row-vals-fn col-names col-vals-fn options))))) 137 | 138 | #_(def ds (dataset {:a [:foo :foo :bar :bar :foo :foo] 139 | :b [:one :one :two :one :two :one] 140 | :c [:dull :dull :shiny :dull :dull :shiny]})) 141 | 142 | #_(crosstab ds :a [:b :c] {:marginal-rows true :marginal-cols true :pivot? true}) 143 | ;; => _unnamed [3 6]: 144 | ;; | rows/cols | [:one :dull] | [:two :shiny] | [:two :dull] | [:one :shiny] | :summary | 145 | ;; |-----------|-------------:|--------------:|-------------:|--------------:|---------:| 146 | ;; | :foo | 2 | 0 | 1 | 1 | 4 | 147 | ;; | :bar | 1 | 1 | 0 | 0 | 2 | 148 | ;; | :summary | 3 | 1 | 1 | 1 | 6 | 149 | 150 | -------------------------------------------------------------------------------- /src/tablecloth/api/api_template.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.api.api-template 2 | "Tablecloth API" 3 | (:refer-clojure :exclude [group-by drop concat rand-nth first last shuffle * + / - < <= > >= 4 | abs bit-and bit-and-not bit-clear bit-flip bit-not bit-or bit-set 5 | bit-shift-left bit-shift-right bit-xor and even? identity infinite? 6 | max min neg? not odd? or pos? quot rem unsigned-bit-shift-right 7 | zero?]) 8 | (:require [tech.v3.datatype.export-symbols :as exporter])) 9 | 10 | (exporter/export-symbols tech.v3.datatype 11 | clone) 12 | 13 | (exporter/export-symbols tech.v3.dataset 14 | column-count 15 | row-count 16 | set-dataset-name 17 | dataset-name 18 | column 19 | has-column? 20 | write!) 21 | 22 | (def ^{:deprecated "Use `write!` instead."} write-csv! write!) 23 | 24 | (exporter/export-symbols tech.v3.dataset.print 25 | dataset->str) 26 | 27 | (exporter/export-symbols tablecloth.api.utils 28 | column-names 29 | write-nippy! 30 | read-nippy 31 | grouped? 32 | unmark-group 33 | mark-as-group 34 | as-regular-dataset 35 | process-group-data) 36 | 37 | (exporter/export-symbols tablecloth.api.dataset 38 | dataset? 39 | empty-ds? 40 | dataset 41 | shape 42 | info 43 | columns 44 | rows 45 | get-entry 46 | print-dataset 47 | concat 48 | concat-copying) 49 | 50 | (exporter/export-symbols tablecloth.api.group-by 51 | group-by 52 | ungroup 53 | groups->seq 54 | groups->map) 55 | 56 | (exporter/export-symbols tablecloth.api.columns 57 | select-columns 58 | drop-columns 59 | rename-columns 60 | add-column 61 | add-columns 62 | add-or-replace-column 63 | add-or-replace-columns 64 | map-columns 65 | update-columns 66 | reorder-columns 67 | convert-types 68 | ->array) 69 | 70 | (exporter/export-symbols tablecloth.api.rows 71 | select-rows 72 | drop-rows 73 | map-rows 74 | head 75 | tail 76 | shuffle 77 | random 78 | rand-nth 79 | first 80 | last 81 | by-rank) 82 | 83 | (exporter/export-symbols tablecloth.api.aggregate 84 | aggregate 85 | aggregate-columns 86 | crosstab) 87 | 88 | (exporter/export-symbols tablecloth.api.order-by 89 | order-by) 90 | 91 | (exporter/export-symbols tablecloth.api.unique-by 92 | unique-by) 93 | 94 | (exporter/export-symbols tablecloth.api.missing 95 | select-missing 96 | drop-missing 97 | replace-missing 98 | fill-range-replace) 99 | 100 | (exporter/export-symbols tablecloth.api.join-separate 101 | join-columns 102 | separate-column 103 | array-column->columns 104 | columns->array-column 105 | map-column->columns 106 | ) 107 | 108 | (exporter/export-symbols tablecloth.api.fold-unroll 109 | fold-by 110 | unroll) 111 | 112 | (exporter/export-symbols tablecloth.api.reshape 113 | pivot->longer 114 | pivot->wider) 115 | 116 | (exporter/export-symbols tablecloth.api.join-concat-ds 117 | left-join 118 | right-join 119 | inner-join 120 | asof-join 121 | full-join 122 | semi-join 123 | anti-join 124 | cross-join 125 | expand 126 | complete 127 | intersect 128 | difference 129 | union 130 | bind 131 | append) 132 | 133 | (exporter/export-symbols tablecloth.api.split 134 | split 135 | split->seq) 136 | 137 | (exporter/export-symbols tablecloth.api.operators 138 | * + - / < <= > >= abs acos and asin atan atan2 bit-and bit-and-not bit-clear bit-flip bit-not bit-or bit-set bit-shift-left bit-shift-right bit-xor cbrt ceil cos cosh cummax cummin cumprod cumsum distance distance-squared dot-product eq even? exp expm1 finite? floor get-significand hypot identity ieee-remainder infinite? kurtosis log log10 log1p logistic magnitude magnitude-squared mathematical-integer? max mean mean-fast median min nan? neg? next-down next-up normalize not not-eq odd? or percentiles pos? pow quartile-1 quartile-3 quot reduce-* reduce-+ reduce-max reduce-min rem rint round shift signum sin sinh skew sq sqrt sum sum-fast tan tanh to-degrees to-radians ulp unsigned-bit-shift-right variance zero?) 139 | 140 | ;; 141 | 142 | (defn- select-or-drop 143 | "Select columns and rows" 144 | [fc fs ds columns-selector rows-selector] 145 | (let [ds (if (clojure.core/and columns-selector 146 | (not= :all columns-selector)) 147 | (fc ds columns-selector) 148 | ds)] 149 | (if (clojure.core/and rows-selector 150 | (not= :all rows-selector)) 151 | (fs ds rows-selector) 152 | ds))) 153 | 154 | (defn- select-or-drop-docstring 155 | [op] 156 | (str op " columns and rows.")) 157 | 158 | (def ^{:doc (select-or-drop-docstring "Select") 159 | :arglists '([ds columns-selector rows-selector])} 160 | select (partial select-or-drop select-columns select-rows)) 161 | (def ^{:doc (select-or-drop-docstring "Drop") 162 | :arglists '([ds columns-selector rows-selector])} 163 | drop (partial select-or-drop drop-columns drop-rows)) 164 | 165 | ;; tibble macro 166 | 167 | (defmacro let-dataset 168 | ([bindings] `(let-dataset ~bindings nil)) 169 | ([bindings options] 170 | (let [cols (take-nth 2 bindings) 171 | col-defs (into (array-map) (map vector (map keyword cols) cols))] 172 | `(let [~@bindings] 173 | (dataset ~col-defs ~options))))) 174 | 175 | ;; ungroup/group wrapper 176 | (defmacro without-grouping-> 177 | [ds & r] 178 | `(-> ~ds 179 | (unmark-group) 180 | ~@r 181 | (mark-as-group))) 182 | 183 | (comment 184 | (exporter/write-api! 'tablecloth.api.api-template 185 | 'tablecloth.api 186 | "src/tablecloth/api.clj" 187 | '[group-by drop concat rand-nth first last shuffle * + - / < <= >= > 188 | abs and bit-and bit-and-not bit-clear bit-flip bit-not bit-or bit-set 189 | bit-shift-left bit-shift-right bit-xor even? identity infinite? max min 190 | neg? not odd? or pos? quot rem unsigned-bit-shift-right zero?]) 191 | ) 192 | -------------------------------------------------------------------------------- /src/tablecloth/api/dataset.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.api.dataset 2 | (:refer-clojure :exclude [concat]) 3 | (:require 4 | [clojure.tools.logging :as logging] 5 | [tablecloth.api.utils :refer [grouped? iterable-sequence? map-inst? 6 | mark-as-group]] 7 | [tech.v3.dataset :as ds] 8 | [tech.v3.dataset.column :as col] 9 | [tech.v3.dataset.print :as p] 10 | [tech.v3.dataset.protocols :as prot] 11 | [tech.v3.dataset.tensor :as ds-tensor] 12 | [tech.v3.tensor :as tensor]) 13 | (:import 14 | [java.io FileNotFoundException])) 15 | 16 | ;;;;;;;;;;;;;;;;;;;;; 17 | ;; DATASET CREATION 18 | ;;;;;;;;;;;;;;;;;;;;; 19 | 20 | (defn dataset? 21 | "Is `ds` a `dataset` type?" 22 | [ds] 23 | (prot/is-dataset? ds)) 24 | 25 | (defn empty-ds? 26 | [ds] 27 | (zero? (ds/row-count ds))) 28 | 29 | #_(defn- fix-map-dataset 30 | "If map contains value which is not a sequence, convert it to a sequence." 31 | [map-ds] 32 | (let [c (if-let [first-seq (->> map-ds 33 | (vals) 34 | (filter iterable-sequence?) 35 | (first))] 36 | (count first-seq) 37 | 1)] 38 | (apply array-map (interleave (keys map-ds) 39 | (map #(if (iterable-sequence? %) % (repeat c %)) (vals map-ds)))))) 40 | 41 | (defn- from-tensor 42 | [data column-names layout dataset-name] 43 | (let [t (tensor/->tensor data) 44 | t (-> (if (= layout :as-columns) (tensor/transpose t [1 0]) t) 45 | (ds-tensor/tensor->dataset dataset-name))] 46 | (if column-names 47 | (ds/rename-columns t (zipmap (range (ds/column-count t)) column-names)) 48 | t))) 49 | 50 | (defn- array-of-arrays? [in] (and in (= "[[" (subs (.getName (class in)) 0 2)))) 51 | 52 | (defn dataset 53 | "Create a `dataset`. 54 | 55 | Dataset can be created from: 56 | 57 | * map of values and/or sequences 58 | * sequence of maps 59 | * sequence of columns 60 | * file or url 61 | * array of arrays 62 | * single value 63 | 64 | Single value is set only when it's not possible to find a path for given data. If tech.ml.dataset throws an exception, it's won;t be printed. To print a stack trace, set `stack-trace?` option to `true`. 65 | 66 | ds/->dataset documentation: 67 | 68 | " 69 | ([] (ds/new-dataset nil)) 70 | ([data] 71 | (dataset data nil)) 72 | ([data {:keys [single-value-column-name column-names layout dataset-name stack-trace? error-column?] 73 | :or {single-value-column-name :$value layout :as-rows stack-trace? false error-column? true} 74 | :as options}] 75 | (when (and (iterable-sequence? data) 76 | (pos? (count data)) ;; shouldn't be empty 77 | (every? iterable-sequence? data) 78 | (every? #(= 2 (count %)) data)) 79 | (logging/warn "Dataset creation behaviour changed for 2d 2-element arrays in v7.029. See https://github.com/scicloj/tablecloth/issues/142 for details.")) 80 | (cond 81 | (prot/is-dataset? data) data 82 | (map-inst? data) (ds/->dataset data options) 83 | (and (iterable-sequence? data) 84 | (every? col/is-column? data)) (ds/new-dataset options data) 85 | (or (array-of-arrays? data) 86 | (and (iterable-sequence? data) 87 | (not-every? map? data))) (from-tensor data column-names layout dataset-name) 88 | (array-of-arrays? data) (ds/new-dataset options (map ds/new-column (or column-names (range)) data)) 89 | ;; empty data but column-names exist 90 | (and (not data) 91 | column-names) (ds/new-dataset options (map (fn [n] (ds/new-column n [])) column-names)) 92 | :else (try (ds/->dataset data options) 93 | (catch FileNotFoundException fnfe (throw fnfe)) 94 | ;; create a singleton 95 | (catch Exception e 96 | (do 97 | (when stack-trace? (.printStackTrace e)) 98 | (let [row {single-value-column-name data}] 99 | (ds/->dataset (if error-column? 100 | (assoc row :$error (.getMessage e)) 101 | row) options)))))))) 102 | 103 | ;; https://github.com/scicloj/tablecloth/issues/112 104 | (alter-meta! #'dataset update :doc str (:doc (meta #'ds/->dataset))) 105 | 106 | (defn shape 107 | "Returns shape of the dataset [rows, cols]" 108 | [ds] 109 | [(ds/row-count ds) 110 | (ds/column-count ds)]) 111 | 112 | (defn- columns-info 113 | [ds] 114 | (dataset (->> (ds/columns ds) 115 | (map meta)) 116 | {:dataset-name (str (ds/dataset-name ds) " :column info")})) 117 | 118 | (defn info 119 | "Returns a statistcial information about the columns of a dataset. 120 | `result-type ` can be :descriptive or :columns" 121 | ([ds] (info ds :descriptive)) 122 | ([ds result-type] 123 | (condp = result-type 124 | :descriptive (ds/descriptive-stats ds) 125 | :columns (columns-info ds) 126 | (let [grouped? (boolean (:grouped? (meta ds))) 127 | nm (ds/dataset-name ds) 128 | inf {:name nm 129 | :grouped? grouped?}] 130 | (dataset (if grouped? 131 | (assoc inf :groups (ds/row-count inf)) 132 | (assoc inf 133 | :rows (ds/row-count ds) 134 | :columns (ds/column-count ds))) 135 | {:dataset-name (str nm " :basic info")}))))) 136 | 137 | 138 | 139 | (defn columns 140 | "Returns columns of dataset. Result type can be any of: 141 | * `:as-map` 142 | * `:as-double-arrays` 143 | * `:as-seqs` 144 | " 145 | ([ds] (columns ds :as-seqs)) 146 | ([ds result-type] 147 | (let [cols (ds/columns ds)] 148 | (case result-type 149 | :as-map (zipmap (ds/column-names ds) cols) 150 | :as-double-arrays (into-array (map double-array (ds/columns ds))) 151 | :as-seqs cols 152 | cols)))) 153 | 154 | (defn rows 155 | "Returns rows of dataset. Result type can be any of: 156 | * `:as-maps` - maps 157 | * `:as-double-arrays` - double arrays 158 | * `:as-seqs` - reader (sequence, default) 159 | * `:as-vecs` - vectors 160 | 161 | If you want to elide nils in maps set `:nil-missing?` option to false (default: `true`). 162 | Another option - `:copying?` - when true row values are copied on read (default: `false`)." 163 | ([ds] (rows ds :as-seqs)) 164 | ([ds result-type] (rows ds result-type nil)) 165 | ([ds result-type {:keys [nil-missing?] 166 | :or {nil-missing? true} 167 | :as options}] 168 | (let [options (assoc options :nil-missing? nil-missing?)] 169 | (case result-type 170 | :as-maps (ds/mapseq-reader ds options) 171 | :as-double-arrays (into-array (map double-array (ds/value-reader ds))) 172 | :as-seqs (ds/value-reader ds options) 173 | :as-vecs (ds/rowvecs ds options) 174 | (ds/value-reader ds options))))) 175 | 176 | (defn print-dataset 177 | "Prints dataset into console. For options see 178 | tech.v3.dataset.print/dataset-data->str" 179 | ([ds] (println (p/dataset->str ds))) 180 | ([ds options] (println (p/dataset->str ds options)))) 181 | 182 | ;; 183 | 184 | (defn get-entry 185 | "Returns a single value from given column and row" 186 | [ds column row] 187 | (get-in ds [column row])) 188 | 189 | ;; 190 | 191 | (defn- do-concat 192 | [concat-fs ds & datasets] 193 | (let [res (apply concat-fs ds datasets)] 194 | (if (and (grouped? ds) 195 | (every? grouped? datasets)) 196 | (-> res 197 | (ds/add-or-update-column :group-id (range (ds/row-count res))) 198 | (mark-as-group)) 199 | res))) 200 | 201 | (defn concat 202 | "Joins rows from other datasets" 203 | [dataset & datasets] 204 | (apply do-concat ds/concat dataset datasets)) 205 | 206 | (defn concat-copying 207 | "Joins rows from other datasets via a copy of data" 208 | [dataset & datasets] (apply do-concat ds/concat-copying dataset datasets)) 209 | -------------------------------------------------------------------------------- /src/tablecloth/api/fold_unroll.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.api.fold-unroll 2 | (:require [tech.v3.dataset :as ds] 3 | 4 | [tablecloth.api.unique-by :refer [unique-by]] 5 | [tablecloth.api.columns :refer [select-columns reorder-columns]] 6 | [tablecloth.api.utils :refer [column-names grouped? process-group-data]])) 7 | 8 | (defn fold-by 9 | "Group-by and pack columns into vector - the output data set has a row for each unique combination 10 | of the provided columns while each remaining column has its valu(es) collected into a vector, similar 11 | to how clojure.core/group-by works. 12 | See https://scicloj.github.io/tablecloth/index.html#Fold-by" 13 | ([ds columns-selector] (fold-by ds columns-selector vec)) 14 | ([ds columns-selector folding-function] 15 | (unique-by ds columns-selector {:strategy folding-function 16 | :add-group-as-column true}))) 17 | 18 | (defn- process-unroll 19 | [ds colnames-set colnames options] 20 | (let [unrolled-dss (map (fn [colname] 21 | (let [opts (assoc options 22 | :datatype (get-in options [:datatypes colname] :object))] 23 | [colname (-> ds 24 | (select-columns (complement (partial contains? (disj colnames-set colname)))) 25 | (ds/unroll-column colname opts))])) colnames)] 26 | (-> (fn [[_ curr] [n uds]] 27 | [_ (ds/add-column curr (uds n))]) 28 | (reduce unrolled-dss) 29 | (second) 30 | (reorder-columns (complement (partial contains? colnames-set)))))) 31 | 32 | (defn unroll 33 | "Unfolds sequences stored inside a column(s), turning it into multiple columns. Opposite of [[fold-by]]. 34 | Add each of the provided columns to the set that defines the \"uniqe key\" of each row. 35 | Thus there will be a new row for each value inside the target column(s)' value sequence. 36 | If you want instead to split the content of the columns into a set of new _columns_, look at [[separate-column]]. 37 | See https://scicloj.github.io/tablecloth/index.html#Unroll" 38 | ([ds columns-selector] (unroll ds columns-selector nil)) 39 | ([ds columns-selector options] 40 | (let [colnames (column-names ds columns-selector) 41 | colnames-set (set colnames)] 42 | (if (grouped? ds) 43 | (process-group-data ds #(process-unroll % colnames-set colnames options) (:parallel? options)) 44 | (process-unroll ds colnames-set colnames options))))) 45 | -------------------------------------------------------------------------------- /src/tablecloth/api/lift_operators.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.api.lift-operators 2 | (:require [tablecloth.api.dataset :refer [columns]] 3 | [tablecloth.api.columns :refer [select-columns add-or-replace-column]] 4 | [tablecloth.api.utils :refer [column-names]] 5 | [tablecloth.api.aggregate :refer [aggregate]] 6 | [tablecloth.utils.codegen :refer [do-lift]])) 7 | 8 | (defn get-meta [fn-sym] 9 | (-> fn-sym resolve meta)) 10 | 11 | (defn get-arglists [fn-sym] 12 | (-> fn-sym get-meta :arglists)) 13 | 14 | (defn get-docstring [fn-sym] 15 | (-> fn-sym get-meta :docstring)) 16 | 17 | (defn max-cols-allowed [arglists] 18 | (let [col-symbol-set #{'x 'y 'z} 19 | longest-arglist (reduce 20 | #(if (> (count %1) (count %2)) %1 %2) arglists)] 21 | (if (= longest-arglist '[x y & args]) 22 | Double/POSITIVE_INFINITY 23 | (count 24 | (clojure.set/intersection 25 | col-symbol-set 26 | (set longest-arglist)))))) 27 | 28 | (defn convert-arglists [arglists target-column?] 29 | (let [convert-arglist 30 | (fn [arglist] 31 | (apply conj 32 | (if target-column? 33 | '[ds target-col columns-selector] 34 | '[ds columns-selector]) 35 | (apply vector (clojure.set/difference (set arglist) #{'x 'y 'z '& 'args}))))] 36 | (->> arglists (map convert-arglist) set (apply list)))) 37 | 38 | (defn build-docstring [fn-sym] 39 | (let [original-docstring (get-docstring fn-sym) 40 | max-allowed (max-cols-allowed (get-arglists fn-sym))] 41 | 42 | (format 43 | "Applies the operation %s to the columns selected by 44 | `columns-selector` and returns a new ds with the the result in 45 | `target-col`. %s 46 | 47 | `columns-selector can be: 48 | - name 49 | - sequence of names 50 | - map of names with new names (rename) 51 | - function which filter names (via column metadata)" 52 | fn-sym 53 | (when (< max-allowed Double/POSITIVE_INFINITY) 54 | (format 55 | "This operation takes a maximum of %s columns, so 56 | `columns-selector` can yield no more than that many columns." 57 | max-allowed))))) 58 | 59 | (defn lift-op 60 | "Takes a function symbol `fn-sym` and generates a function that 61 | applies that function to one or more columns of a dataset, placing 62 | the result in the target column. 63 | 64 | Resulting signature: 65 | (lift-op [fn-sym]) => (fn [ds columns-selector target-col] ...)" 66 | [fn-sym {:keys [return-ds? make-aggregator?] 67 | :or {return-ds? false 68 | make-aggregator? false}}] 69 | (let [defn (symbol "defn") 70 | let (symbol "let") 71 | arglists (get-arglists fn-sym) 72 | max-cols (max-cols-allowed arglists) 73 | lifted-arglists (convert-arglists arglists return-ds?) 74 | new-fn-sym (symbol (name fn-sym)) 75 | new-docstring (build-docstring fn-sym)] 76 | `(~defn ~new-fn-sym 77 | ~new-docstring 78 | ~@(for [args lifted-arglists] 79 | (if make-aggregator? 80 | ;; build an aggregator fn 81 | `(~args 82 | (~let [aggregator# 83 | (fn [ds#] 84 | (~let [ds-with-selected-cols# 85 | (select-columns ds# ~'columns-selector) 86 | cols-count# 87 | (-> ds-with-selected-cols# 88 | column-names 89 | count) 90 | selected-cols# (columns ds-with-selected-cols#)] 91 | (if (>= ~max-cols cols-count#) 92 | (apply ~fn-sym (apply vector selected-cols#)) 93 | (throw (Exception. (str "Exceeded maximum number of columns allowed for operation."))))))] 94 | (aggregate ~'ds aggregator#))) 95 | ;; build either a fn that returns a dataset or the result of the operation 96 | `(~args 97 | (~let [selected-cols# (apply vector (columns 98 | (select-columns ~'ds ~'columns-selector))) 99 | args-to-pass# (concat selected-cols# [~@(drop 3 args)])] 100 | (if (>= ~max-cols (count selected-cols#)) 101 | (->> args-to-pass# 102 | (apply ~fn-sym) 103 | ~(if return-ds? `(add-or-replace-column ~'ds ~'target-col) `(identity))) 104 | (throw (Exception. (str "Exceeded maximum number of columns allowed for operation."))))))))))) 105 | 106 | (def serialized-lift-fn-lookup 107 | {'[distance 108 | distance-squared 109 | dot-product 110 | kurtosis 111 | magnitude 112 | magnitude-squared 113 | mean 114 | mean-fast 115 | median 116 | quartile-1 117 | quartile-3 118 | #_quartiles ;; this returns a vector of quartiles not sure it fits here 119 | reduce-* 120 | reduce-+ 121 | reduce-max 122 | reduce-min 123 | skew 124 | sum 125 | sum-fast 126 | variance] 127 | {:lift-fn lift-op :optional-args {:make-aggregator? true}} 128 | '[* 129 | + 130 | - 131 | / 132 | < 133 | <= 134 | > 135 | >= 136 | abs 137 | acos 138 | and 139 | asin 140 | atan 141 | atan2 142 | bit-and 143 | bit-and-not 144 | bit-clear 145 | bit-flip 146 | bit-not 147 | bit-or 148 | bit-set 149 | bit-shift-left 150 | bit-shift-right 151 | bit-xor 152 | cbrt 153 | ceil 154 | cos 155 | cosh 156 | cummax 157 | cummin 158 | cumprod 159 | cumsum 160 | eq 161 | ;; equals ;; leaving this one out. not clear its signature is right. 162 | even? 163 | exp 164 | expm1 165 | ;; fill-range ;; this one has an odd return value, not a column 166 | finite? 167 | floor 168 | get-significand 169 | hypot 170 | identity 171 | ieee-remainder 172 | infinite? 173 | log 174 | log10 175 | log1p 176 | logistic 177 | mathematical-integer? 178 | max 179 | min 180 | nan? 181 | neg? 182 | next-down 183 | next-up 184 | normalize 185 | not 186 | not-eq 187 | odd? 188 | or 189 | percentiles 190 | pos? 191 | pow 192 | quot 193 | rem 194 | rint 195 | round 196 | shift 197 | signum 198 | sin 199 | sinh 200 | sq 201 | sqrt 202 | tan 203 | tanh 204 | to-degrees 205 | to-radians 206 | ulp 207 | unsigned-bit-shift-right 208 | zero?] {:lift-fn lift-op :optional-args {:return-ds? true}}}) 209 | 210 | (comment 211 | (do-lift {:target-ns 'tablecloth.api.operators 212 | :source-ns 'tablecloth.column.api.operators 213 | :lift-fn-lookup serialized-lift-fn-lookup 214 | :deps ['tablecloth.api.lift_operators] 215 | :exclusions 216 | '[* + - / < <= > >= abs and bit-and bit-and-not bit-clear bit-flip 217 | bit-not bit-or bit-set bit-shift-left bit-shift-right bit-test bit-xor 218 | even? identity infinite? max min neg? not odd? odd? or pos? quot rem 219 | unsigned-bit-shift-right zero?]}) 220 | ,) 221 | -------------------------------------------------------------------------------- /src/tablecloth/api/missing.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.api.missing 2 | (:require [tech.v3.dataset :as ds] 3 | [tech.v3.dataset.math :as dm] 4 | 5 | [tablecloth.api.utils :refer [column-names grouped? process-group-data]] 6 | [tablecloth.api.columns :refer [select-columns]])) 7 | 8 | (defn- select-or-drop-missing 9 | "Select rows with missing values" 10 | ([f ds] (select-or-drop-missing f ds nil)) 11 | ([f ds columns-selector] 12 | (if (grouped? ds) 13 | (process-group-data ds #(select-or-drop-missing f % columns-selector)) 14 | 15 | (let [ds- (if columns-selector 16 | (select-columns ds columns-selector) 17 | ds)] 18 | (f ds (ds/missing ds-)))))) 19 | 20 | (defn- select-or-drop-missing-docstring 21 | [op] 22 | (str op " rows with missing values 23 | 24 | `columns-selector` selects columns to look at missing values")) 25 | 26 | (def ^{:doc (select-or-drop-missing-docstring "Select") 27 | :arglists '([ds] [ds columns-selector])} 28 | select-missing (partial select-or-drop-missing ds/select-rows)) 29 | 30 | (def ^{:doc (select-or-drop-missing-docstring "Drop") 31 | :arglists '([ds] [ds columns-selector])} 32 | drop-missing (partial select-or-drop-missing ds/drop-rows)) 33 | 34 | (defn replace-missing 35 | "Replaces missing values. Accepts 36 | 37 | * dataset 38 | * column selector, default: :all 39 | * strategy, default: :nearest 40 | * value (optional) 41 | * single value 42 | * sequence of values (cycled) 43 | * function, applied on column(s) with stripped missings 44 | 45 | Strategies are: 46 | 47 | `:value` - replace with given value 48 | `:up` - copy values up 49 | `:down` - copy values down 50 | `:updown` - copy values up and then down for missing values at the end 51 | `:downup` - copy values down and then up for missing values at the beginning 52 | `:mid` or `:nearest` - copy values around known values 53 | `:midpoint` - use average value from previous and next non-missing 54 | `:lerp` - trying to lineary approximate values, works for numbers and datetime, otherwise applies :nearest. For numbers always results in float datatype. 55 | " 56 | ([ds] (replace-missing ds :mid)) 57 | ([ds strategy] (replace-missing ds :all strategy)) 58 | ([ds columns-selector strategy] (replace-missing ds columns-selector strategy nil)) 59 | ([ds columns-selector strategy value] 60 | 61 | (if (grouped? ds) 62 | 63 | (process-group-data ds #(replace-missing % columns-selector strategy value)) 64 | 65 | (let [cols (column-names ds columns-selector)] 66 | (ds/replace-missing ds cols strategy value))))) 67 | 68 | (defn fill-range-replace 69 | "Fill missing up with lacking values. Accepts 70 | * dataset 71 | * column name 72 | * expected step (max-span, milliseconds in case of datetime column) 73 | * (optional) missing-strategy - how to replace missing, default :down (set to nil if none) 74 | * (optional) missing-value - optional value for replace missing 75 | " 76 | ([ds colname max-span] (fill-range-replace ds colname max-span :down)) 77 | ([ds colname max-span missing-strategy] (fill-range-replace ds colname max-span missing-strategy nil)) 78 | ([ds colname max-span missing-strategy missing-value] 79 | (if (grouped? ds) 80 | (process-group-data ds #(fill-range-replace % colname max-span missing-strategy missing-value)) 81 | (dm/fill-range-replace ds colname max-span missing-strategy missing-value)))) 82 | 83 | -------------------------------------------------------------------------------- /src/tablecloth/api/order_by.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.api.order-by 2 | (:require [tech.v3.dataset :as ds] 3 | 4 | [tablecloth.api.utils :refer [iterable-sequence? grouped? process-group-data]])) 5 | 6 | (set! *unchecked-math* :warn-on-boxed) 7 | 8 | (defn- comparator->fn 9 | [c] 10 | (cond 11 | (fn? c) c 12 | (= :desc c) #(compare %2 %1) 13 | :else compare)) 14 | 15 | (defn asc-desc-comparator 16 | [orders] 17 | (if-not (iterable-sequence? orders) 18 | (comparator->fn orders) 19 | (cond 20 | (every? #(= % :asc) orders) nil 21 | (every? #(= % :desc) orders) #(compare %2 %1) 22 | :else (let [comparators (mapv comparator->fn orders)] 23 | (fn ^long [v1 v2] 24 | (loop [v1 v1 25 | v2 v2 26 | cmptrs comparators] 27 | (if-let [cmptr (first cmptrs)] 28 | (let [^long c (cmptr (first v1) (first v2))] 29 | (if-not (zero? c) 30 | c 31 | (recur (rest v1) (rest v2) (rest cmptrs)))) 32 | 0))))))) 33 | 34 | (defn- sort-fn 35 | [columns-or-fn comp-fn] 36 | (cond 37 | (iterable-sequence? columns-or-fn) (if comp-fn 38 | (fn [ds] 39 | (ds/sort-by ds 40 | (apply juxt (map #(if (fn? %) 41 | % 42 | (fn [ds] (get ds %))) columns-or-fn)) 43 | comp-fn)) 44 | (fn [ds] 45 | (ds/sort-by ds 46 | (apply juxt (map #(if (fn? %) 47 | % 48 | (fn [ds] (get ds %))) columns-or-fn))))) 49 | (fn? columns-or-fn) (if comp-fn 50 | (fn [ds] (ds/sort-by ds columns-or-fn comp-fn)) 51 | (fn [ds] (ds/sort-by ds columns-or-fn))) 52 | :else (if comp-fn 53 | (fn [ds] (ds/sort-by-column ds columns-or-fn comp-fn)) 54 | (fn [ds] (ds/sort-by-column ds columns-or-fn))))) 55 | 56 | (defn order-by 57 | "Order dataset by: 58 | - column name 59 | - columns (as sequence of names) 60 | - key-fn 61 | - sequence of columns / key-fn 62 | Additionally you can ask the order by: 63 | - :asc 64 | - :desc 65 | - custom comparator function" 66 | ([ds columns-or-fn] (order-by ds columns-or-fn nil)) 67 | ([ds columns-or-fn comparators] (order-by ds columns-or-fn comparators nil)) 68 | ([ds columns-or-fn comparators {:keys [parallel?] :as options}] 69 | (let [comparators (or comparators (if (iterable-sequence? columns-or-fn) 70 | (repeat (count columns-or-fn) :asc) 71 | [:asc])) 72 | sorting-fn (->> comparators 73 | asc-desc-comparator 74 | (sort-fn columns-or-fn))] 75 | 76 | (if (grouped? ds) 77 | (process-group-data ds sorting-fn parallel?) 78 | (sorting-fn ds))))) 79 | -------------------------------------------------------------------------------- /src/tablecloth/api/unique_by.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.api.unique-by 2 | (:require [tech.v3.dataset :as ds] 3 | [tech.v3.datatype.protocols :as dtype-proto] 4 | [tech.v3.dataset.column :as col] 5 | [tech.v3.datatype :as dtype] 6 | [tech.v3.datatype.bitmap :as bitmap] 7 | 8 | [tablecloth.api.utils :refer [iterable-sequence? column-names grouped? process-group-data]] 9 | [tablecloth.api.dataset :refer [dataset empty-ds?]] 10 | [tablecloth.api.columns :refer [select-columns]] 11 | [tablecloth.api.group-by :refer [ungroup]])) 12 | 13 | (defn- strategy-first [_ idxs] (clojure.core/first idxs)) 14 | (defn- strategy-last [_ idxs] (clojure.core/last idxs)) 15 | (defn- strategy-random [_ idxs] (clojure.core/rand-nth idxs)) 16 | 17 | (def ^:private strategies 18 | {:first strategy-first 19 | :last strategy-last 20 | :random strategy-random}) 21 | 22 | (defn- remove-missing-from-column 23 | "The same as remove rows" 24 | [col] 25 | (let [cnt (dtype/ecount col) 26 | m (col/missing col)] 27 | (if (and (pos? cnt) 28 | (seq m)) 29 | (col/select col (-> cnt 30 | (range) 31 | (bitmap/->bitmap) 32 | (dtype-proto/set-and-not m))) 33 | col))) 34 | 35 | (defn strategy-fold 36 | ([ds columns-selector] (strategy-fold ds columns-selector nil)) 37 | ([ds columns-selector fold-fn] (strategy-fold ds columns-selector fold-fn nil)) 38 | ([ds columns-selector fold-fn ungroup-options] 39 | (let [[group-by-selector target-names] (if (fn? columns-selector) 40 | [columns-selector (ds/column-names ds)] 41 | (let [group-by-names (column-names ds columns-selector)] 42 | [group-by-names (->> group-by-names 43 | (set) 44 | (partial contains?) 45 | (complement) 46 | (column-names ds))])) 47 | fold-fn (or fold-fn vec)] 48 | (-> (tablecloth.api.group-by/group-by ds group-by-selector) 49 | (process-group-data (fn [ds] 50 | (as-> ds ds 51 | (select-columns ds target-names) 52 | (dataset [(zipmap target-names 53 | (map (comp fold-fn remove-missing-from-column) (ds/columns ds)))])))) 54 | (ungroup ungroup-options))))) 55 | 56 | (defn- unique-by-fn 57 | [strategy columns-selector selected-keys options] 58 | (if (fn? strategy) 59 | 60 | (fn [ds] (strategy-fold ds columns-selector strategy options)) 61 | 62 | (let [local-options {:keep-fn (get strategies strategy strategy-first)}] 63 | (cond 64 | (iterable-sequence? columns-selector) (let [local-options (assoc local-options :column-name-seq columns-selector)] 65 | (fn [ds] 66 | (if (= (count columns-selector) 1) 67 | (ds/unique-by-column ds local-options (clojure.core/first columns-selector)) 68 | (ds/unique-by ds local-options #(select-keys % columns-selector))))) 69 | (fn? columns-selector) (let [local-options (if selected-keys 70 | (assoc local-options :column-name-seq selected-keys) 71 | local-options)] 72 | (fn [ds] (ds/unique-by ds local-options columns-selector))) 73 | :else (fn [ds] (ds/unique-by-column ds local-options columns-selector )))))) 74 | 75 | (defn- maybe-skip-unique 76 | [ufn ds] 77 | (if (= 1 (ds/row-count ds)) ds (ufn ds))) 78 | 79 | (defn- maybe-empty 80 | [ufn ds] 81 | (if (empty-ds? ds) ds (ufn ds))) 82 | 83 | (defn unique-by 84 | "Remove rows which contains the same data 85 | `column-selector` Select columns for uniqueness 86 | `strategy` There are 4 strategies defined to handle duplicates 87 | 88 | `:first` - select first row (default) 89 | `:last` - select last row 90 | `:random` - select random row 91 | any function - apply function to a columns which are subject of uniqueness" 92 | 93 | 94 | ([ds] (unique-by ds (ds/column-names ds))) 95 | ([ds columns-selector] (unique-by ds columns-selector nil)) 96 | ([ds columns-selector {:keys [strategy select-keys parallel?] 97 | :or {strategy :first} 98 | :as options}] 99 | (let [selected-keys (column-names ds select-keys) 100 | ufn (unique-by-fn strategy columns-selector selected-keys options) 101 | ufn (partial maybe-empty (if (fn? strategy) ufn (partial maybe-skip-unique ufn)))] 102 | 103 | (if (grouped? ds) 104 | (process-group-data ds ufn parallel?) 105 | (ufn ds))))) 106 | 107 | -------------------------------------------------------------------------------- /src/tablecloth/column/api/api_template.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.column.api.api-template 2 | "Tablecloth Column API" 3 | (:refer-clojure :exclude [group-by drop concat rand-nth first last shuffle * + / - < <= > >= 4 | abs bit-and bit-and-not bit-clear bit-flip bit-not bit-or bit-set 5 | bit-shift-left bit-shift-right bit-xor and even? identity infinite? 6 | max min neg? not odd? or pos? quot rem unsigned-bit-shift-right 7 | zero?]) 8 | (:require [tech.v3.datatype.export-symbols :as exporter])) 9 | 10 | (exporter/export-symbols tablecloth.column.api.column 11 | column 12 | column? 13 | column-map 14 | typeof 15 | typeof? 16 | slice 17 | zeros 18 | ones 19 | sort-column) 20 | 21 | (exporter/export-symbols tablecloth.column.api.missing 22 | count-missing 23 | drop-missing 24 | replace-missing) 25 | 26 | (exporter/export-symbols tech.v3.dataset.column 27 | is-missing? 28 | missing 29 | select) 30 | 31 | (exporter/export-symbols tablecloth.column.api.operators 32 | * + - / < <= > >= abs acos and asin atan atan2 bit-and bit-and-not bit-clear bit-flip bit-not bit-or bit-set bit-shift-left bit-shift-right bit-xor cbrt ceil cos cosh cummax cummin cumprod cumsum descriptive-statistics distance distance-squared dot-product eq equals even? exp expm1 fill-range finite? floor get-significand hypot identity ieee-remainder infinite? kendalls-correlation kurtosis log log10 log1p logistic magnitude magnitude-squared mathematical-integer? max mean mean-fast median min nan? neg? next-down next-up normalize not not-eq odd? or pearsons-correlation percentiles pos? pow quartile-1 quartile-3 quartiles quot reduce-* reduce-+ reduce-max reduce-min rem rint round shift signum sin sinh skew spearmans-correlation sq sqrt standard-deviation sum sum-fast tan tanh to-degrees to-radians ulp unsigned-bit-shift-right variance zero?) 33 | 34 | (comment 35 | ;; Use this to generate the column api 36 | (exporter/write-api! 'tablecloth.column.api.api-template 37 | 'tablecloth.column.api 38 | "src/tablecloth/column/api.clj" 39 | '[* + - / < <= >= > 40 | abs and bit-and bit-and-not bit-clear bit-flip bit-not bit-or bit-set 41 | bit-shift-left bit-shift-right bit-xor even? identity infinite? max min 42 | neg? not odd? or pos? quot rem unsigned-bit-shift-right zero?]) 43 | ,) 44 | -------------------------------------------------------------------------------- /src/tablecloth/column/api/column.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.column.api.column 2 | (:require [tech.v3.dataset.column :as col] 3 | [tech.v3.datatype :as dtype] 4 | [tech.v3.datatype.functional :as fun] 5 | [tech.v3.datatype.argops :refer [argsort]] 6 | [tablecloth.api.utils :refer [->general-types concrete-type? type?]])) 7 | 8 | (defn column 9 | "Create a `column` from a vector or sequence. " 10 | ([] 11 | (col/new-column nil [])) 12 | ([data] 13 | (column data {:name nil})) 14 | ([data {:keys [name] 15 | :as options}] 16 | (col/new-column name data))) 17 | 18 | ;; Alias for tech.v3.dasetset.column.is-column? 19 | (defn column? 20 | "Return true or false `item` is a column." 21 | [item] 22 | (col/is-column? item)) 23 | 24 | (defn typeof 25 | "Returns the concrete type of the elements within the column `col`." 26 | [col] 27 | (dtype/elemwise-datatype col)) 28 | 29 | (defn typeof? 30 | "True|false the column's elements are of the provided type `datatype`. Can check 31 | both concrete types (e.g. :int32) or general types (:numerical, :textual, etc)." 32 | [col datatype] 33 | (let [concrete-type-of-els (dtype/elemwise-datatype col)] 34 | (if (concrete-type? datatype) 35 | (= datatype concrete-type-of-els) 36 | (not (nil? (type? datatype concrete-type-of-els)))))) 37 | 38 | (defn zeros 39 | "Create a new column filled wth `n-zeros`." 40 | [n-zeros] 41 | (column (dtype/const-reader 0 n-zeros))) 42 | 43 | (defn ones 44 | "Creates a new column filled with `n-ones`" 45 | [n-ones] 46 | (column (dtype/const-reader 1 n-ones))) 47 | 48 | (defn slice 49 | "Returns a subset of the column defined by the inclusive `from` and 50 | `to` indexes. If `to` is not provided, slices to the end of the 51 | column. If `from` is not provided (i.e. is `nil`), slices from the 52 | beginning of the column. If either `from` or `to` is a negative 53 | number, it is treated as an index from the end of the column. The 54 | `:start` and `:end` keywords can be used to represent the start and 55 | end of the column, respectively. 56 | The `step` parameter determines the increment between each index in the 57 | slice. It defaults to 1 if not provided. 58 | 59 | Examples: 60 | (def c (column [1 2 3 4 5])) 61 | (slice c 1 3) ;=> [2 3 4] 62 | (slice c 2) ;=> [3 4 5] 63 | (slice c -3 -1) ;=> [3 4 5] 64 | (slice c :start 2) ;=> [1 2 3] 65 | (slice c 2 :end) ;=> [3 4 5] 66 | (slice c -2 :end) ;=> [4 5] 67 | (slice c 0 :end 2) ;=> [1 3 5]" 68 | ([col from] 69 | (slice col from :end)) 70 | ([col from to] 71 | (slice col from to 1)) 72 | ([col from to step] 73 | (let [len (count col) 74 | from (or (when-not (or (= from :start) (nil? from)) from) 0) 75 | to (or (when-not (or (= to :end) (nil? to)) to) (dec len))] 76 | (col/select col (range (if (neg? from) (+ len from) from) 77 | (inc (if (neg? to) (+ len to) to)) 78 | step))))) 79 | 80 | ;;handle missing values 81 | (defn sort-column 82 | "Returns a sorted version of the column `col`. You can supply the ordering 83 | keywords `:asc` or `:desc` or a comparator function to `order-or-comparator`. 84 | If no comparator function is provided, the column will be sorted in 85 | ascending order." 86 | ([col] 87 | (sort-column col :asc)) 88 | ([col order-or-comparator] 89 | (if (not (or (= :asc order-or-comparator) 90 | (= :desc order-or-comparator) 91 | (fn? order-or-comparator))) 92 | (throw (IllegalArgumentException. 93 | "`order-or-comparator` must be `:asc`, `:desc`, or a function."))) 94 | (let [comparator-fn (cond 95 | (fn? order-or-comparator) order-or-comparator 96 | (= :asc order-or-comparator) compare 97 | (= :desc order-or-comparator) #(compare %2 %1)) 98 | sorted-indices (argsort comparator-fn col)] 99 | (col/select col sorted-indices)))) 100 | 101 | (defn column-map 102 | "Applies a map function `map-fn` to one or more columns. If `col` is 103 | a vector of columns, `map-fn` must have an arity equal to the number 104 | of columns. The datatype of the resulting column will be inferred, 105 | unless specified in the `options` map. Missing values can be handled 106 | by providing a `:missing-fn` in the options map. 107 | 108 | options: 109 | - :datatype - The desired datatype of the resulting column. The datatype 110 | is inferred if not provided 111 | - :missing-fn - A function that takes a sequence of columns, and returns a 112 | set of missing index positions." 113 | ([col map-fn] 114 | (column-map col map-fn {})) 115 | ([col map-fn options] 116 | (if (vector? col) 117 | (apply col/column-map map-fn options col) 118 | (col/column-map map-fn options col)))) 119 | -------------------------------------------------------------------------------- /src/tablecloth/column/api/lift_operators.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.column.api.lift-operators 2 | (:require [tablecloth.utils.codegen :refer [do-lift]] 3 | [tablecloth.column.api.column :refer [column]] 4 | [tech.v3.datatype.argtypes :refer [arg-type]])) 5 | 6 | (defn get-meta [fn-sym] 7 | (-> fn-sym resolve meta)) 8 | 9 | (defn get-arglists [fn-sym] 10 | (-> fn-sym get-meta :arglists)) 11 | 12 | (defn get-docstring [fn-sym] 13 | (-> fn-sym get-meta :doc)) 14 | 15 | (defn return-scalar-or-column [item] 16 | (let [item-type (arg-type item)] 17 | (if (= item-type :reader) 18 | (column item) 19 | item))) 20 | 21 | (defn lift-op 22 | ([fn-sym] 23 | (lift-op fn-sym nil)) 24 | ([fn-sym {:keys [new-args]}] 25 | (let [defn (symbol "defn") 26 | let (symbol "let") 27 | docstring (get-docstring fn-sym) 28 | original-args (get-arglists fn-sym) 29 | sort-by-arg-count (fn [argslist] 30 | (sort #(< (count %1) (count %2)) argslist))] 31 | (if new-args 32 | `(~defn ~(symbol (name fn-sym)) 33 | ~(or docstring "") 34 | ~@(for [[new-arg new-arg-lookup original-arg] 35 | (map vector (sort-by-arg-count (keys new-args)) 36 | (sort-by-arg-count (vals new-args)) 37 | (sort-by-arg-count original-args)) 38 | :let [filtered-original-arg (filter (partial not= '&) original-arg)]] 39 | (list 40 | (if new-arg new-arg original-arg) 41 | `(~let [original-result# (~fn-sym 42 | ~@(for [oldarg filtered-original-arg] 43 | (if (nil? (get new-arg-lookup oldarg)) 44 | oldarg 45 | (get new-arg-lookup oldarg))))] 46 | (return-scalar-or-column original-result#))))) 47 | `(~defn ~(symbol (name fn-sym)) 48 | ~(or docstring "") 49 | ~@(for [arg original-args 50 | :let [[explicit-args rest-arg-expr] (split-with (partial not= '&) arg)]] 51 | (list 52 | arg 53 | `(~let [original-result# ~(if (empty? rest-arg-expr) 54 | `(~fn-sym ~@explicit-args) 55 | `(apply ~fn-sym ~@explicit-args ~(second rest-arg-expr)))] 56 | (return-scalar-or-column original-result#))))))))) 57 | 58 | (def serialized-lift-fn-lookup 59 | {['< 60 | '<= 61 | '> 62 | '>= 63 | '* 64 | '+ 65 | '- 66 | '/ 67 | 'abs 68 | 'acos 69 | 'and 70 | 'asin 71 | 'atan 72 | 'atan2 73 | 'bit-and 74 | 'bit-and-not 75 | 'bit-clear 76 | 'bit-flip 77 | 'bit-not 78 | 'bit-or 79 | 'bit-set 80 | 'bit-shift-left 81 | 'bit-shift-right 82 | #_bit-test ;; can't get this to work yet. 83 | 'bit-xor 84 | 'cbrt 85 | 'ceil 86 | 'cos 87 | 'cosh 88 | 'cumprod 89 | 'cumsum 90 | 'cummax 91 | 'cummin 92 | 'descriptive-statistics 93 | 'distance 94 | 'distance-squared 95 | 'dot-product 96 | 'even? 97 | 'equals 98 | 'exp 99 | 'expm1 100 | 'eq 101 | 'fill-range 102 | 'finite? 103 | 'floor 104 | 'get-significand 105 | 'hypot 106 | 'identity 107 | 'ieee-remainder 108 | 'infinite? 109 | 'kendalls-correlation 110 | 'kurtosis 111 | 'log 112 | 'log10 113 | 'log1p 114 | 'logistic 115 | 'mathematical-integer? 116 | 'magnitude 117 | 'magnitude-squared 118 | 'max 119 | 'mean 120 | 'mean-fast 121 | 'median 122 | 'min 123 | 'nan? 124 | 'neg? 125 | 'next-down 126 | 'next-up 127 | 'normalize 128 | 'not 129 | 'not-eq 130 | 'odd? 131 | 'or 132 | 'pearsons-correlation 133 | 'percentiles 134 | 'pos? 135 | 'pow 136 | 'quartiles 137 | 'quartile-1 138 | 'quartile-3 139 | 'quot 140 | 'reduce-min 141 | 'reduce-max 142 | 'reduce-* 143 | 'reduce-+ 144 | 'rem 145 | 'rint 146 | 'round 147 | 'skew 148 | 'shift 149 | 'signum 150 | 'sin 151 | 'sinh 152 | 'spearmans-correlation 153 | 'sq 154 | 'sqrt 155 | 'standard-deviation 156 | 'sum 157 | 'sum-fast 158 | 'tan 159 | 'tanh 160 | 'to-degrees 161 | 'to-radians 162 | 'ulp 163 | 'unsigned-bit-shift-right 164 | 'variance 165 | 'zero?] {:lift-fn lift-op}}) 166 | 167 | 168 | (comment 169 | (do-lift {:target-ns 'tablecloth.column.api.operators 170 | :source-ns 'tech.v3.datatype.functional 171 | :lift-fn-lookup serialized-lift-fn-lookup 172 | :deps ['tablecloth.column.api.lift_operators] 173 | :exclusions 174 | '[* + - / < <= > >= abs and bit-and bit-and-not bit-clear bit-flip 175 | bit-not bit-or bit-set bit-shift-left bit-shift-right bit-test bit-xor 176 | even? identity infinite? max min neg? not odd? odd? or pos? quot rem 177 | unsigned-bit-shift-right zero?]}) 178 | ,) 179 | -------------------------------------------------------------------------------- /src/tablecloth/column/api/missing.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.column.api.missing 2 | (:require [tech.v3.dataset :as ds] 3 | [tech.v3.dataset.column :as col] 4 | [tech.v3.datatype :as dtype] 5 | [tablecloth.api :as tc])) 6 | 7 | (defn- into-ds [col] 8 | (-> {:col col} 9 | tc/dataset 10 | (tc/update-columns :col #(col/set-missing % (col/missing col))))) 11 | 12 | (defn- out-of-ds [ds] 13 | (:col ds)) 14 | 15 | (defn count-missing 16 | "Returns the number of missing values in column `col`. " 17 | [col] 18 | (-> col col/missing dtype/ecount)) 19 | 20 | (defn drop-missing 21 | "Remove missing values from column `col`." 22 | [col] 23 | (-> col 24 | (into-ds) 25 | (ds/drop-missing) 26 | (out-of-ds))) 27 | 28 | (defn replace-missing 29 | "Replace missing values in column `col` with give `strategy`. 30 | 31 | Strategies may be: 32 | 33 | - `:down` - Take the previous value, or use provided value. 34 | - `:up` - Take the next value, or use provided value. 35 | - `:downup` - Take the previous value, otherwise take the next value. 36 | - `:updown` - Take the next value, otherwise take the previous value. 37 | - `:nearest` - Use the nearest of next or previous values. (Strategy `:mid` is an alias for `:nearest`). 38 | - `:midpoint` - Use the midpoint of averaged values between previous and next (non-missing) values. 39 | - `:abb` - Impute missing value with approximate Bayesian bootstrap. 40 | See [r's ABB](https://search.r-project.org/CRAN/refmans/LaplacesDemon/html/ABB.html). 41 | - `:lerp` - Linearly interpolate values between previous and next nonmissing rows. 42 | - `:value` - Provide a value explicitly. Value may be a function in which 43 | case it will be called on the column with missing values elided 44 | and the return will be used to as the filler." 45 | ([col] 46 | (replace-missing col :nearest)) 47 | ([col strategy] 48 | (replace-missing col strategy nil)) 49 | ([col strategy value] 50 | (-> col 51 | (into-ds) 52 | (ds/replace-missing [:col] strategy value) 53 | (out-of-ds)))) 54 | -------------------------------------------------------------------------------- /src/tablecloth/column/api/utils.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.column.api.utils 2 | (:import [java.io Writer]) 3 | (:require [tech.v3.datatype.export-symbols :as exporter] 4 | [tech.v3.datatype.argtypes :refer [arg-type]] 5 | [tablecloth.column.api :refer [column]] 6 | [tech.v3.datatype.functional :as fun] 7 | [clojure.java.io :as io])) 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /src/tablecloth/pipeline.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.pipeline 2 | "Linear pipeline operations." 3 | (:refer-clojure :exclude [group-by drop concat rand-nth first last shuffle]) 4 | (:require [tablecloth.api :as api])) 5 | 6 | (defmacro build-pipelined-function 7 | [f m] 8 | (let [args (map (comp vec rest) (:arglists m))] 9 | `(defn ~(symbol (name f)) {:doc ~(:doc m) :orig (symbol (var ~f))} 10 | ~@(for [arg args 11 | :let [narg (mapv #(if (map? %) 'options %) arg) 12 | [a & r] (split-with (partial not= '&) narg)]] 13 | (list narg `(fn [ds#] 14 | (let [ctx# (if (api/dataset? ds#) 15 | {:metamorph/data ds#} ds#)] 16 | (assoc ctx# :metamorph/data (apply ~f (ctx# :metamorph/data) ~@a ~(rest r)))))))))) 17 | 18 | (def ^:private excludes '#{dataset write-csv! let-dataset without-grouping->}) 19 | 20 | (defmacro process-all-api-symbols 21 | [] 22 | (let [ps (ns-publics 'tablecloth.api)] 23 | `(do ~@(for [[f v] ps 24 | :when (not (excludes f)) 25 | :let [m (meta v) 26 | f (symbol "tablecloth.api" (name f))]] 27 | `(build-pipelined-function ~f ~m))))) 28 | 29 | (process-all-api-symbols) 30 | 31 | -------------------------------------------------------------------------------- /src/tablecloth/utils/codegen.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.utils.codegen 2 | (:import [java.io Writer]) 3 | (:require [tech.v3.datatype.export-symbols :as exporter] 4 | [clojure.java.io :as io])) 5 | 6 | (defn- writeln! 7 | ^Writer [^Writer writer strdata & strdatas] 8 | (.append writer (str strdata)) 9 | (doseq [data strdatas] 10 | (when data 11 | (.append writer (str data)))) 12 | (.append writer "\n") 13 | writer) 14 | 15 | (defn- write-empty-ln! ^Writer [^Writer writer] 16 | (writeln! writer "") 17 | writer) 18 | 19 | (defn- write-pp ^Writer [^Writer writer item] 20 | (clojure.pprint/pprint item writer) 21 | writer) 22 | 23 | (defn deserialize-lift-fn-lookup [serialized-lift-fn-lookup] 24 | (reduce (fn [m [symlist liftfn]] 25 | (loop [syms symlist 26 | result m] 27 | (if (empty? syms) 28 | result 29 | (recur (rest syms) (assoc result (first syms) liftfn))))) 30 | {} 31 | serialized-lift-fn-lookup)) 32 | 33 | (defn get-lifted [lift-fn-lookup source-ns] 34 | (let [fun-mappings (ns-publics source-ns)] 35 | (map (fn [[fnsym {:keys [lift-fn optional-args]}]] 36 | (lift-fn (symbol (name source-ns) (name fnsym)) optional-args)) 37 | (deserialize-lift-fn-lookup lift-fn-lookup)))) 38 | 39 | (defn namespace-to-path [ns-str] 40 | (-> ns-str 41 | (name) 42 | (clojure.string/replace "." "/") 43 | (clojure.string/replace "-" "_") 44 | (->> (str "./src/")) 45 | (str ".clj"))) 46 | 47 | (defn build-ns-header 48 | "Generates a namespace header with the specified target-ns and 49 | source-ns, along with optional additional dependencies and 50 | exclusions. If exclusions are provided, they will be used to exclude 51 | the specified symbol(s) from the :refer-clojure directive." 52 | ([target-ns source-ns] 53 | (build-ns-header target-ns source-ns nil nil)) 54 | ([target-ns source-ns additional-deps] 55 | (build-ns-header target-ns source-ns additional-deps nil)) 56 | ([target-ns source-ns additional-deps exclusions] 57 | (let [ns (symbol "ns")] 58 | `(~ns ~target-ns 59 | (:require [~source-ns] 60 | ~@(for [dep additional-deps] 61 | [dep])) 62 | ~@(when exclusions `((:refer-clojure :exclude ~exclusions))))))) 63 | 64 | (defn do-lift 65 | "Writes the lifted functions to target namespace. 66 | 67 | Example: 68 | {:target-ns 'tablecloth.api.operators 69 | :source-ns 'tablecloth.column.api.operators 70 | :lift-fn-lookup {['+ '- '*] lift-fn} 71 | :deps ['tablecloth.api.lift_operators] 72 | :exclusions 73 | '[* + -]}" 74 | [{:keys [target-ns source-ns lift-fn-lookup deps exclusions]}] 75 | (with-open [writer (io/writer (namespace-to-path target-ns) :append false)] 76 | (write-pp writer (build-ns-header target-ns source-ns deps exclusions)) 77 | (write-empty-ln! writer) 78 | (doseq [f (get-lifted lift-fn-lookup source-ns)] 79 | (-> writer 80 | (write-pp f) 81 | (write-empty-ln!))))) 82 | -------------------------------------------------------------------------------- /test/tablecloth/api/aggregate_test.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.api.aggregate-test 2 | (:require [tablecloth.api :as api] 3 | [tech.v3.datatype.functional :refer [sum]] 4 | [midje.sweet :refer [fact =>]])) 5 | 6 | 7 | (fact "aggregate keeps an order of columns" 8 | (-> (array-map 9 | :a [1 1] 10 | :b 2 11 | :c 3 12 | :d 4 13 | :e 5 14 | :f 6 15 | :g 7 16 | :h 8 17 | :i 9) 18 | (api/dataset) 19 | (api/aggregate-columns [:a :b :c :d :e :f :g :h :i] sum) 20 | (api/column-names)) => '(:a :b :c :d :e :f :g :h :i)) 21 | 22 | (fact "aggregate default prefix changed to custom" 23 | (-> (api/dataset {:l [:x :x :y :y :y] 24 | :a [1 2 3 4 5] 25 | :b [5 5 5 5 5]}) 26 | (api/group-by :l) 27 | (api/aggregate (fn [ds] {:sum-of-b (reduce + (ds :b))}) 28 | {:default-column-name-prefix "xxx"}) 29 | (api/column-names) 30 | (set)) => #{:xxx-sum-of-b :$group-name}) 31 | -------------------------------------------------------------------------------- /test/tablecloth/api/columns_test.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.api.columns-test 2 | (:require [tablecloth.api :as api] 3 | [tech.v3.datatype :as dtype] 4 | [midje.sweet :refer [tabular fact =>]])) 5 | 6 | 7 | ;; https://github.com/scicloj/tablecloth/issues/9 8 | (def dss (api/dataset {:idx [1 1 1 2 2 2 3 3 3] 9 | :a ["a" "b" "c" "a" "b" "c" "a" "b" "c"] 10 | "z" 1 11 | :b [1 2 3 2 3 4 3 2 1] 12 | :c [3 1 2 4 2 1 3 2 4]})) 13 | 14 | (fact "reorder-columns" 15 | (tabular (fact (-> dss 16 | (api/reorder-columns ?order) 17 | (api/column-names)) 18 | => 19 | ?expected) 20 | ?order ?expected 21 | [:ids :b :a :c] [:b :a :c :idx "z"] 22 | [:idx :b :a "z" :c] [:idx :b :a "z" :c] 23 | [:idx :b :a :C] [:idx :b :a "z" :c] 24 | [:c :A :b :e :z :idx] [:c :b :idx :a "z"] 25 | string? ["z" :idx :a :b :c] 26 | #".*[az]$" [:a "z" :idx :b :c]) 27 | (fact [:b :a :c :idx "z"] 28 | => 29 | (-> dss 30 | (api/reorder-columns :b :a [:c :ids]) 31 | (api/column-names)))) 32 | 33 | (fact "add-or-replace" 34 | (tabular (fact (-> {:x [1 2]} 35 | (api/dataset) 36 | (api/add-or-replace-column :y ?v) 37 | :y 38 | (dtype/get-datatype)) 39 | => 40 | ?expected) 41 | ?expected ?v 42 | :int64 1 43 | :float64 1.0 44 | :string "abc")) 45 | 46 | (fact "add" 47 | (tabular (fact (-> {:x [1 2]} 48 | (api/dataset) 49 | (api/add-column :y ?v) 50 | :y 51 | (dtype/get-datatype)) 52 | => 53 | ?expected) 54 | ?expected ?v 55 | :int64 1 56 | :float64 1.0 57 | :string "abc")) 58 | -------------------------------------------------------------------------------- /test/tablecloth/api/dataset_test.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.api.dataset-test 2 | (:require [tablecloth.api :as api] 3 | [tablecloth.column.api :as tcc] 4 | [tablecloth.common-test :refer [DS]] 5 | [clojure.java.io :as io] 6 | [midje.sweet :refer [tabular fact => throws]]) 7 | (:import [java.io FileNotFoundException])) 8 | 9 | (fact "dataset?" 10 | (fact (api/dataset? (api/dataset)) => true) 11 | (fact (api/dataset? DS) => true) 12 | (fact (api/dataset? {}) => false)) 13 | 14 | (fact "empty-ds" 15 | (fact (api/empty-ds? (api/dataset)) => true) 16 | (fact (api/empty-ds? (api/dataset {:a []})) => true) 17 | (fact (api/empty-ds? DS) => false)) 18 | 19 | (fact "dataset-creation" 20 | (fact (and (zero? (api/row-count (api/dataset))) 21 | (zero? (api/column-count (api/dataset)))) => true) 22 | (fact (:$value (api/dataset 999)) 23 | => [999]) 24 | (fact (get (api/dataset 999 {:single-value-column-name "my-single-value"}) "my-single-value") 25 | => [999]) 26 | (fact ((juxt #(get % 0) api/dataset-name) 27 | (api/dataset 999 {:single-value-column-name "" 28 | :dataset-name "Single value"})) 29 | => [[999] "Single value"]) 30 | (tabular (fact (-> (api/dataset ?input) 31 | (api/columns :as-map)) 32 | => ?res) 33 | ?res ?input 34 | {0 [:A :B :C] 1 [33 5 :a]} [[:A 33] [:B 5] [:C :a]] 35 | {0 [:A :B :C] 1 [[1 2 3 4 5 6] 36 | "X" 37 | :a]} [[:A [1 2 3 4 5 6]] [:B "X"] [:C :a]] 38 | {:A [33]} {:A 33} 39 | {:A [1 2 3]} {:A [1 2 3]} 40 | {:A [3 4 5] :B (repeat 3 "X")} {:A [3 4 5] :B "X"} 41 | {:a [1 99] :b [3 2]} [{:a 1 :b 3} {:b 2 :a 99}] 42 | {:a [1 2] :b [[1 2 3] [3 4]]} [{:a 1 :b [1 2 3]} {:a 2 :b [3 4]}] 43 | {:a [nil 3 11] :b [1 4 nil]} [{:a nil :b 1} {:a 3 :b 4} {:a 11}] 44 | {0 [1 3 5] 1 [2 4 6]} [[1 2] [3 4] [5 6]] 45 | {0 [1 4 7] 46 | 1 [2 5 8] 47 | 2 [3 6 9]} [[1 2 3] [4 5 6] [7 8 9]] 48 | {0 ["a" "b" "c"] 1 [1 2 3]} [["a" 1] ["b" 2] ["c" 3]]) 49 | 50 | (fact (-> (api/dataset "data/family.csv") 51 | (api/shape)) 52 | => [5 5]) 53 | (fact (-> (api/dataset "https://vega.github.io/vega-lite/examples/data/seattle-weather.csv") 54 | (api/shape)) 55 | => [1461 6]) 56 | (fact (api/dataset "not-existing.csv") 57 | => (throws FileNotFoundException))) 58 | 59 | (fact "saving" 60 | (fact (do (api/write! DS "DS.tsv.gz") 61 | (.exists (io/file "DS.tsv.gz"))) 62 | => true) 63 | (fact (-> (api/dataset "DS.tsv.gz") 64 | (api/dataset?)) 65 | => true) 66 | (fact (do (api/write! DS "DS.csv.gz") 67 | (.exists (io/file "DS.csv.gz"))) 68 | => true) 69 | (fact (-> (api/dataset "DS.csv.gz") 70 | (api/dataset?)) 71 | => true) 72 | (fact (do (api/write-nippy! DS "DS.nippy.gz") 73 | (.exists (io/file "DS.nippy.gz"))) 74 | => true) 75 | (fact (-> (api/read-nippy "DS.nippy.gz") 76 | (api/dataset?)) 77 | => true)) 78 | 79 | (fact "dataset-shape" 80 | (fact (api/shape DS) 81 | => [9 4]) 82 | (fact (api/row-count DS) 83 | => 9) 84 | (fact (api/column-count DS) 85 | => 4)) 86 | 87 | (fact "dataset-name" 88 | (fact (api/dataset-name DS) 89 | => "DS") 90 | (fact (-> [[1 2 3] [4 5 6]] 91 | (api/dataset {:dataset-name "Test"}) 92 | (api/dataset-name)) => "Test")) 93 | 94 | (fact "dataset-info" 95 | (fact 96 | (-> (api/info DS) 97 | (select-keys [:col-name :datatype])) 98 | => {:col-name [:V1 :V2 :V3 :V4] 99 | :datatype [:int64 :int64 :float64 :string]}) 100 | (fact 101 | (-> (api/info DS) 102 | (api/column-names)) 103 | => '(:col-name :datatype :n-valid :n-missing :min :mean :mode :max :standard-deviation :skew :first :last)) 104 | (fact 105 | (-> (api/info DS :basic) 106 | (api/rows :as-maps)) 107 | => [{:name "DS", :columns 4, :rows 9, :grouped? false}]) 108 | (fact 109 | (-> (api/info DS :columns) 110 | (api/rows :as-maps)) 111 | => [{:name :V1, :n-elems 9, :categorical? nil, :datatype :int64} 112 | {:name :V2, :n-elems 9, :categorical? nil, :datatype :int64} 113 | {:name :V3, :n-elems 9, :categorical? nil, :datatype :float64} 114 | {:name :V4, :n-elems 9, :categorical? true, :datatype :string}]) 115 | (fact 116 | (-> (api/info DS :columns) 117 | (api/rows :as-maps {:nil-missing? false})) 118 | => [{:name :V1, :n-elems 9, :datatype :int64} 119 | {:name :V2, :n-elems 9, :datatype :int64} 120 | {:name :V3, :n-elems 9, :datatype :float64} 121 | {:name :V4, :n-elems 9, :categorical? true, :datatype :string}])) 122 | 123 | (fact "as-double-arrays" 124 | (tabular (fact (-> (api/dataset {:a [1 2 3] 125 | :b [5 6 7]}) 126 | (?f :as-double-arrays) 127 | (->> (map seq))) 128 | = ?v) 129 | ?f ?v 130 | api/rows '((1.0 5.0) (2.0 6.0) (3.0 7.0)) 131 | api/columns '((1.0 2.0 3.0) (5.0 6.0 7.0)))) 132 | 133 | (fact "let-dataset" 134 | (fact (api/let-dataset [x (range 4) y 10 z (tcc/+ x y)]) 135 | => (api/dataset {:x [0 1 2 3] 136 | :y [10 10 10 10] 137 | :z [10 11 12 13]}))) 138 | -------------------------------------------------------------------------------- /test/tablecloth/api/fold_unroll_test.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.api.fold-unroll-test 2 | (:require [tablecloth.api :as api] 3 | [midje.sweet :refer [tabular fact =>]])) 4 | 5 | (fact "one-row-ds" 6 | (-> (api/dataset {:a [1] :b [2]}) 7 | (api/fold-by :a set) 8 | (api/rows)) 9 | => 10 | [[1 #{2}]]) 11 | 12 | (fact "empty-ds" 13 | (-> (api/dataset {:a [1]}) 14 | (api/drop-rows 0) 15 | (api/fold-by :a) 16 | (api/empty-ds?)) 17 | => 18 | true) 19 | -------------------------------------------------------------------------------- /test/tablecloth/api/group_by_test.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.api.group-by-test 2 | (:require [tablecloth.api.group-by :as sut] 3 | [clojure.test :as t])) 4 | 5 | -------------------------------------------------------------------------------- /test/tablecloth/api/join_concat_ds_test.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.api.join-concat-ds-test 2 | (:require [midje.sweet :as midje :refer [fact tabular =>]] 3 | [tablecloth.api :as api])) 4 | 5 | (def ds1 (api/dataset {:a [1 2 1 2 3 4 nil nil 4] 6 | :b (range 101 110) 7 | :c (map str "abs tract")})) 8 | (def ds2 (api/dataset {:a [nil 1 2 5 4 3 2 1 nil] 9 | :b (range 110 101 -1) 10 | :c (map str "datatable") 11 | :d (symbol "X") 12 | :e [3 4 5 6 7 nil 8 1 1]})) 13 | 14 | (fact "anti-join" 15 | (-> (api/anti-join ds1 ds2 :a) 16 | (api/rows :as-maps)) => [] 17 | (-> (api/anti-join ds2 ds1 :a) 18 | (api/rows :as-maps)) => [{:a 5, :b 107, :c "a", :d 'X, :e 6}] 19 | (-> (api/anti-join ds1 ds2 :b) 20 | (api/rows :as-maps)) => [{:b 101, :a 1, :c "a"}] 21 | (-> (api/anti-join ds2 ds1 :b) 22 | (api/rows :as-maps)) => [{:b 110, :a nil, :c "d", :d 'X, :e 3}] 23 | (-> (api/anti-join ds1 ds2 :c) 24 | (api/rows :as-maps)) => [{:c "s", :a 1, :b 103} 25 | {:c " ", :a 2, :b 104} 26 | {:c "r", :a 4, :b 106} 27 | {:c "c", :a nil, :b 108}] 28 | (-> (api/anti-join ds2 ds1 :c) 29 | (api/rows :as-maps)) => [{:c "d", :a nil, :b 110, :d 'X, :e 3} 30 | {:c "l", :a 1, :b 103, :d 'X, :e 1} 31 | {:c "e", :a nil, :b 102, :d 'X, :e 1}] 32 | (-> (api/anti-join ds1 ds2 {:left :a :right :e}) 33 | (api/rows :as-maps)) => [{:a 2, :b 102, :c "b"} {:a 2, :b 104, :c " "}] 34 | (-> (api/anti-join ds2 ds1 {:left :e :right :a}) 35 | (api/rows :as-maps)) => [{:e 5, :a 2, :b 108, :c "t", :d 'X} 36 | {:e 6, :a 5, :b 107, :c "a", :d 'X} 37 | {:e 7, :a 4, :b 106, :c "t", :d 'X} 38 | {:e 8, :a 2, :b 104, :c "b", :d 'X}] 39 | (-> (api/anti-join ds1 ds2 [:a :b]) 40 | (api/rows :as-maps)) => [{:a 1, :b 101, :c "a"} 41 | {:a 2, :b 102, :c "b"} 42 | {:a nil, :b 107, :c "a"} 43 | {:a nil, :b 108, :c "c"} 44 | {:a 4, :b 109, :c "t"}] 45 | (-> (api/anti-join ds2 ds1 [:a :b]) 46 | (api/rows :as-maps)) => [{:a nil, :b 110, :c "d", :d 'X, :e 3} 47 | {:a 1, :b 109, :c "a", :d 'X, :e 4} 48 | {:a 2, :b 108, :c "t", :d 'X, :e 5} 49 | {:a 5, :b 107, :c "a", :d 'X, :e 6} 50 | {:a nil, :b 102, :c "e", :d 'X, :e 1}]) 51 | 52 | (fact "semi-join" 53 | (-> (api/semi-join ds1 ds2 :a) 54 | (api/row-count)) => 9 55 | (-> (api/semi-join ds2 ds1 :a) 56 | (api/row-count)) => 8 57 | (-> (api/semi-join ds1 ds2 :b) 58 | (api/row-count)) => 8 59 | (-> (api/semi-join ds2 ds1 :b) 60 | (api/row-count)) => 8 61 | (-> (api/semi-join ds1 ds2 :c) 62 | (api/row-count)) => 5 63 | (-> (api/semi-join ds2 ds1 :c) 64 | (api/row-count)) => 6 65 | (-> (api/semi-join ds1 ds2 {:left :a :right :e}) 66 | (api/row-count)) => 7 67 | (-> (api/semi-join ds2 ds1 {:left :e :right :a}) 68 | (api/row-count)) => 5 69 | (-> (api/semi-join ds1 ds2 [:a :b]) 70 | (api/row-count)) => 4 71 | (-> (api/semi-join ds2 ds1 [:a :b]) 72 | (api/row-count)) => 4) 73 | 74 | (fact "eraderna int-string join" 75 | (-> (api/left-join (-> (api/dataset [{:i "foo" :y 2022}])) 76 | (-> (api/dataset [{:i "foo" :y 2022 :s "2022"} 77 | {:i "foo" :y 2023 :s "2023"}])) 78 | [:i :y]) 79 | (api/rows :as-maps)) => [{:i "foo", :y 2022, :right.i "foo", :right.y 2022, :s "2022"}] 80 | (-> (api/left-join (-> (api/dataset [{:i "foo" :y 2022}]) 81 | (api/convert-types {:y :int16})) 82 | (-> (api/dataset [{:i "foo" :y 2022 :s "2022"} 83 | {:i "foo" :y 2023 :s "2023"}])) 84 | [:i :y]) 85 | (api/rows :as-maps)) => [{:i "foo", :y 2022, :right.i "foo", :right.y 2022, :s "2022"}]) 86 | 87 | (fact "left join on shorts packed into the vector" 88 | (-> (api/left-join (-> (api/dataset [{:iy ["foo" (short 2022)]}])) 89 | (-> (api/dataset [{:iy ["foo" (long 2022)] :s "2022"} 90 | {:iy ["foo" (long 2023)] :s "2023"}])) 91 | :iy) 92 | (api/rows :as-maps)) => [{:iy ["foo", 2022], :right.iy ["foo", 2022], :s "2022"}]) 93 | 94 | (fact "multiple same rows joins or nils by eraderna" 95 | (-> (api/semi-join (api/dataset [{:k nil :v "\"nil\""} 96 | {:k "bar" :v "\"bar\""} 97 | {:k "baz" :v "\"baz\""}]) 98 | (api/dataset [{:k "baz"}]) 99 | [:k]) 100 | (api/rows :as-maps)) => [{:k "baz" :v "\"baz\""}] 101 | (-> (api/semi-join (api/dataset [{:k nil :v "\"nil\""} 102 | {:k "bar" :v "\"bar\""} 103 | {:k "baz" :v "\"baz\""}]) 104 | (api/dataset [{:k "baz"} 105 | {:k "baz"}]) 106 | [:k]) 107 | (api/rows :as-maps)) => [{:k "baz", :v "\"baz\""}] 108 | (-> (api/semi-join (api/dataset [{:k "bar" :v "\"bar\""} 109 | {:k "baz" :v "\"baz\""} 110 | {:k "baz" :v "\"baz\""}]) 111 | (api/dataset [{:k "baz"}]) 112 | [:k]) 113 | (api/rows :as-maps)) => [{:k "baz", :v "\"baz\""} {:k "baz", :v "\"baz\""}]) 114 | -------------------------------------------------------------------------------- /test/tablecloth/api/join_separate_test.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.api.join-separate-test 2 | (:require [tablecloth.api :as api] 3 | [midje.sweet :refer [tabular fact =>]])) 4 | 5 | 6 | 7 | (fact "array-column->columns works" 8 | (-> (api/dataset {:x [(double-array [1 2 3]) 9 | (double-array [4 5 6])] 10 | :y [:a :b]}) 11 | (api/array-column->columns :x) 12 | (api/rows :as-maps)) 13 | => [{:y :a 0 1.0 1 2.0 2 3.0} {:y :b 0 4.0 1 5.0 2 6.0}] 14 | 15 | (-> {"a" [[4 4] [3 2] [2 0]]} 16 | api/dataset 17 | (api/array-column->columns "a" { :prefix "c"}) 18 | (api/rows :as-maps) 19 | 20 | ) 21 | => [{"c-0" 4, "c-1" 4} {"c-0" 3, "c-1" 2} {"c-0" 2, "c-1" 0}] 22 | ) 23 | 24 | (fact "array-column->columns works can prefix columns with key-word" 25 | (-> (api/dataset {:x [(double-array [1 2 3]) 26 | (double-array [4 5 6])] 27 | :y [:a :b]}) 28 | (api/array-column->columns :x {:prefix :col}) 29 | (api/rows :as-maps)) 30 | => [{:y :a 31 | :col-0 1.0 32 | :col-1 2.0 33 | :col-2 3.0} 34 | {:y :b 35 | :col-0 4.0 36 | :col-1 5.0 37 | :col-2 6.0}]) 38 | 39 | (fact "array-column->columns works can prefix columns with string" 40 | (-> (api/dataset {:x [(double-array [1 2 3]) 41 | (double-array [4 5 6])] 42 | :y [:a :b]}) 43 | (api/array-column->columns :x {:prefix "col"}) 44 | (api/rows :as-maps)) 45 | => [{:y :a 46 | "col-0" 1.0 47 | "col-1" 2.0 48 | "col-2" 3.0} 49 | {:y :b 50 | "col-0" 4.0 51 | "col-1" 5.0 52 | "col-2" 6.0}]) 53 | 54 | (fact "columns->array columns works" 55 | (let [ds (api/dataset {0 [0.0 1 2] 56 | 1 [3.0 4 5] 57 | :x [:a :b :c]}) 58 | ds-with-array-column 59 | (-> ds 60 | (api/columns->array-column [0 1] :y))] 61 | 62 | (:x ds-with-array-column) => [:a :b :c] 63 | (->> ds-with-array-column :y (mapv vec)) 64 | => [[0.0 3.0] [1.0 4.0] [2.0 5.0]])) 65 | 66 | (fact "false-is-not-missing" 67 | (-> (api/dataset [{:a "foo" :b true} 68 | {:a "bar" :b false}]) 69 | (api/join-columns :join-columns-string [:a :b]) 70 | 71 | (api/rows) 72 | (flatten)) 73 | => ["foo-true" "bar-false"]) 74 | 75 | (fact "map-column->columns work" 76 | (-> 77 | (api/dataset {:m [{:a 1 :b 2} {:a 3 :b 4}] 78 | "n" [{:a 10 :b 20} {:a 30 :b 40}]}) 79 | (api/map-column->columns :m) 80 | (api/rows :as-maps)) 81 | => [{"n" {:a 10, :b 20}, :m-a 1, :m-b 2} {"n" {:a 30, :b 40}, :m-a 3, :m-b 4}] 82 | 83 | 84 | 85 | (-> 86 | (api/dataset {:m [{:a 1 :b 2} {:a 3 :b 4}] 87 | "n" [{:a 10 :b 20} {:a 30 :b 40}]}) 88 | (api/map-column->columns "n") 89 | (api/rows :as-maps)) 90 | => [{:m {:a 1, :b 2}, "n-a" 10, "n-b" 20} {:m {:a 3, :b 4}, "n-a" 30, "n-b" 40}] 91 | 92 | (-> 93 | (api/dataset {:m [{:a 1 :b 2 :d 4} {:a 3 :c "hello"}]}) 94 | (api/map-column->columns :m) 95 | (api/rows :as-maps)) 96 | => [{:m-a 1, :m-b 2, :m-d 4, :m-c nil} 97 | {:m-a 3, :m-b nil, :m-d nil, :m-c "hello"}]) 98 | -------------------------------------------------------------------------------- /test/tablecloth/api/missing_test.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.api.missing-test 2 | (:require [tablecloth.common-test :refer [approx]] 3 | [tablecloth.api :as api] 4 | [tech.v3.datatype.functional :as dfn] 5 | [midje.sweet :refer [tabular fact =>]])) 6 | 7 | (fact "empty-missing" 8 | (let [empty-col (api/dataset {:a [nil nil]})] 9 | (-> empty-col api/select-missing api/row-count) 10 | => 11 | 2 12 | (-> empty-col api/drop-missing api/row-count) 13 | => 14 | 0 15 | (-> empty-col (api/replace-missing :a :value 0) api/select-missing api/row-count) 16 | => 17 | 0 18 | (-> empty-col (api/replace-missing :a :value dfn/mean) api/select-missing api/row-count) 19 | => 20 | 2 21 | (-> empty-col api/replace-missing api/select-missing api/row-count) 22 | => 23 | 2)) 24 | 25 | (def ds (api/dataset {:a [nil nil nil 1.0 2 nil nil nil nil nil 4 nil 11 nil nil] 26 | :b [2 2 2 nil nil nil nil nil nil 13 nil 3 4 5 5]})) 27 | 28 | (fact "selection-and-dropping" 29 | (tabular (fact (-> ds ?s api/rows count) => ?cnt) 30 | ?cnt ?s 31 | 14 api/select-missing 32 | 1 api/drop-missing 33 | 11 (api/select-missing :a) 34 | 4 (api/drop-missing :a) 35 | 7 (api/select-missing :b) 36 | 8 (api/drop-missing :b))) 37 | 38 | (fact "strategy-mid" 39 | (fact (-> ds api/replace-missing api/select-missing api/row-count) 40 | => 41 | 0) 42 | (fact (-> ds (api/replace-missing :a :mid) api/select-missing api/row-count) 43 | => 44 | 7) 45 | (tabular (fact (-> ds ?cmd (api/column ?col) seq) 46 | => ?xs) 47 | ?xs ?cmd ?col 48 | [1.0 1.0 1.0 1.0 2.0 2.0 2.0 2.0 4.0 4.0 4.0 4.0 11.0 11.0 11.0] 49 | (api/replace-missing :a :mid) 50 | :a 51 | [2 2 2 2 2 2 13 13 13 13 13 3 4 5 5] 52 | (api/replace-missing :b :mid) 53 | :b)) 54 | 55 | (fact "strategy-down" 56 | (tabular (fact (-> ds ?cmd (api/column ?col) seq) 57 | => ?xs) 58 | ?xs ?cmd ?col 59 | [nil nil nil 1.0 2.0 2.0 2.0 2.0 2.0 2.0 4.0 4.0 11.0 11.0 11.0] 60 | (api/replace-missing :a :down) 61 | :a 62 | [2 2 2 2 2 2 2 2 2 13 13 3 4 5 5] 63 | (api/replace-missing :b :down) 64 | :b 65 | [999.0 999.0 999.0 1.0 2.0 2.0 2.0 2.0 2.0 2.0 4.0 4.0 11.0 11.0 11.0] 66 | (api/replace-missing :a :down 999.0) 67 | :a 68 | [4.5 4.5 4.5 1.0 2.0 2.0 2.0 2.0 2.0 2.0 4.0 4.0 11.0 11.0 11.0] 69 | (api/replace-missing :a :down dfn/mean) 70 | :a)) 71 | 72 | (fact "strategy-up" 73 | (tabular (fact (-> ds ?cmd (api/column ?col) seq) 74 | => ?xs) 75 | ?xs ?cmd ?col 76 | [1.0 1.0 1.0 1.0 2.0 4.0 4.0 4.0 4.0 4.0 4.0 11.0 11.0 nil nil] 77 | (api/replace-missing :a :up) 78 | :a 79 | [2 2 2 13 13 13 13 13 13 13 3 3 4 5 5] 80 | (api/replace-missing :b :up) 81 | :b 82 | [1.0 1.0 1.0 1.0 2.0 4.0 4.0 4.0 4.0 4.0 4.0 11.0 11.0 999.0 999.0] 83 | (api/replace-missing :a :up 999.0) 84 | :a 85 | [1.0 1.0 1.0 1.0 2.0 4.0 4.0 4.0 4.0 4.0 4.0 11.0 11.0 4.5 4.5] 86 | (api/replace-missing :a :up dfn/mean) 87 | :a)) 88 | 89 | (fact "strategy-lerp" 90 | (tabular (fact (-> ds ?cmd (api/column ?col) (->> (map approx))) 91 | => ?xs) 92 | ?xs ?cmd ?col 93 | (map approx 94 | [1.0 1.0 1.0 1.0 2 95 | (/ 7 3.0) (/ 8 3.0) 3.0 (/ 10 3.0) (/ 11 3.0) 96 | 4 7.5 11.0 11.0 11.0]) 97 | (api/replace-missing :a :lerp) 98 | :a 99 | (map approx 100 | [2.0 2.0 2.0 101 | 3.5714 5.1429 6.7143 8.2857 9.8571 11.4286 102 | 13.0 8.0 3.0 4.0 5.0 5.0]) 103 | (api/replace-missing :b :lerp) 104 | :b)) 105 | 106 | (fact "strategy-lerp-time" 107 | (let [dtds (api/dataset {:dt [(java.time.LocalDateTime/of 2020 1 1 1 1 1) 108 | nil nil nil 109 | (java.time.LocalDateTime/of 2020 10 1 1 1 1)]})] 110 | [(java.time.LocalDateTime/of 2020 1 1 1 1 1) 111 | (java.time.LocalDateTime/of 2020 3 9 13 1 1) 112 | (java.time.LocalDateTime/of 2020 5 17 1 1 1) 113 | (java.time.LocalDateTime/of 2020 7 24 13 1 1) 114 | (java.time.LocalDateTime/of 2020 10 1 1 1 1)] 115 | => 116 | (seq ((api/replace-missing dtds :lerp) :dt)))) 117 | 118 | (fact "strategy-value" 119 | (tabular (fact (-> ds ?cmd (api/column ?col) seq) 120 | => ?xs) 121 | ?xs ?cmd ?col 122 | [0.0 0.0 0.0 1.0 2.0 0.0 0.0 0.0 0.0 0.0 4.0 0.0 11.0 0.0 0.0] 123 | (api/replace-missing :a :value 0.0) 124 | :a 125 | [4.5 4.5 4.5 1.0 2.0 4.5 4.5 4.5 4.5 4.5 4.0 4.5 11.0 4.5 4.5] 126 | (api/replace-missing :a :value dfn/mean) 127 | :a 128 | [-10.0 -20.0 -10.0 1.0 2.0 129 | -20.0 -10.0 -20.0 -10.0 -20.0 4.0 -10.0 11.0 -20.0 -10.0] 130 | (api/replace-missing :a :value [-10.0 -20.0]) 131 | :a 132 | [nil 100.0 nil 1.0 2.0 133 | nil nil nil nil 134 | -100.0 4.0 nil 11.0 nil nil] 135 | (api/replace-missing :a :value {1 100.0 9 -100.0}) 136 | :a)) 137 | 138 | ;; otfrom case: https://clojurians.zulipchat.com/#narrow/stream/151924-data-science/topic/Simple.20Tablecloth.20beginnings/near/249217023 139 | 140 | (fact "replace missing values in grouped dataset" 141 | (fact (-> (api/dataset [{:calendar-year 2022 :age 49 :location "Barry Buddon" :hours-cycled 1.0} 142 | {:calendar-year 2024 :age 49 :location "Barry Buddon" :hours-cycled 2.4} 143 | {:calendar-year 2022 :age 49 :location "Tentsmuir Woods" :hours-cycled 3.2} 144 | {:calendar-year 2024 :age 49 :location "Tentsmuir Woods" :hours-cycled 1.4}]) 145 | (api/group-by [:age :location]) 146 | (api/fill-range-replace :calendar-year 1 nil nil) 147 | (api/replace-missing :location :down) 148 | (api/replace-missing :age :down) 149 | (api/replace-missing :hours-cycled :value 0.0) 150 | (api/ungroup) 151 | (api/rows :as-maps)) => [{:calendar-year 2022.0, :age 49, :location "Barry Buddon", :hours-cycled 1.0} {:calendar-year 2023.0, :age 49, :location "Barry Buddon", :hours-cycled 0.0} {:calendar-year 2024.0, :age 49, :location "Barry Buddon", :hours-cycled 2.4} {:calendar-year 2022.0, :age 49, :location "Tentsmuir Woods", :hours-cycled 3.2} {:calendar-year 2023.0, :age 49, :location "Tentsmuir Woods", :hours-cycled 0.0} {:calendar-year 2024.0, :age 49, :location "Tentsmuir Woods", :hours-cycled 1.4}])) 152 | 153 | -------------------------------------------------------------------------------- /test/tablecloth/api/operators_test.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.api.operators-test 2 | (:refer-clojure :exclude [* + - / < <= > >= abs and bit-and bit-and-not bit-clear bit-flip 3 | bit-not bit-or bit-set bit-shift-left bit-shift-right bit-test bit-xor 4 | even? identity infinite? max min neg? not odd? odd? or pos? quot rem 5 | unsigned-bit-shift-right zero?]) 6 | (:require [tablecloth.api :refer [dataset]] 7 | [tech.v3.datatype :refer [elemwise-datatype]] 8 | [midje.sweet :refer [fact facts =>]]) 9 | (:use [tablecloth.api.operators])) 10 | 11 | 12 | (facts 13 | "about ops that return the op result" 14 | 15 | (facts 16 | "about ops that take a maximum of one column and return a scalar" 17 | (let [ds (dataset {:label [:a :b :a :b] 18 | :value [1 2 3 4]})] 19 | (let [ops [kurtosis 20 | magnitude 21 | magnitude-squared 22 | mean 23 | mean-fast 24 | median 25 | quartile-1 26 | quartile-3 27 | reduce-* 28 | reduce-+ 29 | reduce-max 30 | reduce-min 31 | skew 32 | sum 33 | sum-fast 34 | variance]] 35 | (doseq [op ops] 36 | (fact "ops can aggregate a regular dataset" 37 | (let [result (op ds [:value])] 38 | result => tablecloth.api/dataset? 39 | (contains? result "summary") => true)) 40 | (fact "ops can aggregate a grouped ds" 41 | (let [result (-> ds 42 | (tablecloth.api/group-by :label) 43 | (op [:value]))] 44 | result => tablecloth.api/dataset? 45 | (contains? result "summary") => true)))))) 46 | 47 | (facts 48 | "about ops that take a maximum of two columns and return a scalar" 49 | (let [ds (dataset {:label [:a :b :a :b] 50 | :values1 [1 2 3 4] 51 | :values2 [5 6 7 8]})] 52 | (let [ops [distance 53 | distance-squared 54 | dot-product]] 55 | (doseq [op ops] 56 | (fact "ops can aggregate a regular dataset" 57 | (let [result (op ds [:values1 :values2])] 58 | result => tablecloth.api/dataset? 59 | (contains? result "summary"))) 60 | (fact "ops can aggregate a grouped dataset" 61 | (let [result (-> ds 62 | (tablecloth.api/group-by :label) 63 | (op [:values1 :values2]))] 64 | result => tablecloth.api/dataset? 65 | (contains? result "summary")))))))) 66 | 67 | (facts 68 | "about ops that return a dataset with a new column" 69 | 70 | (facts 71 | "about ops that take a maximum of one column" 72 | (let [ds (dataset {:a [1 2 3]})] 73 | (let [result (shift ds :b [:a] 1)] 74 | (:b result) => [1 1 2]) 75 | 76 | (let [ops [cummax 77 | cummin 78 | cumprod 79 | cumsum 80 | abs 81 | acos 82 | asin 83 | atan 84 | bit-not 85 | cbrt 86 | ceil 87 | cos 88 | cosh 89 | even? 90 | exp 91 | expm1 92 | finite? 93 | floor 94 | get-significand 95 | identity 96 | infinite? 97 | log 98 | log10 99 | log1p 100 | logistic 101 | mathematical-integer? 102 | nan? 103 | neg? 104 | next-down 105 | next-up 106 | normalize 107 | not 108 | odd? 109 | pos? 110 | rint 111 | round 112 | signum 113 | sin 114 | sinh 115 | sq 116 | sqrt 117 | tan 118 | tanh 119 | to-degrees 120 | to-radians 121 | ulp 122 | zero?]] 123 | (doseq [op ops] 124 | (let [result (op ds :b [:a])] 125 | (contains? result :b) => true))))) 126 | 127 | (facts 128 | "about ops that take a maximum of two columns" 129 | (let [ops [and eq not-eq or] 130 | ds (dataset {:a [1 2 3] 131 | :b [4 5 6]})] 132 | (doseq [op ops] 133 | (let [result (op ds :c [:a :b :c])] 134 | (contains? result :c) => true)))) 135 | 136 | (facts 137 | "about ops that take a maximum of three columns" 138 | (let [ops [< > <= >=] 139 | ds (dataset {:a [1 2 3] 140 | :b [4 5 6] 141 | :c [7 8 9]})] 142 | (doseq [op ops] 143 | (let [result (op ds :d [:a :b :c])] 144 | (contains? result :d) => true)))) 145 | 146 | (facts 147 | "about ops that can take an unlimited number of columns" 148 | (let [ops [/ 149 | - 150 | + 151 | * 152 | atan2 153 | ;; equals // this function doesn't seem to handle more than two cols 154 | hypot 155 | bit-and 156 | bit-and-not 157 | bit-clear 158 | bit-flip 159 | bit-or 160 | bit-set 161 | bit-shift-right 162 | bit-shift-left 163 | bit-xor 164 | ieee-remainder 165 | max 166 | min 167 | pow 168 | quot 169 | rem 170 | unsigned-bit-shift-right] 171 | ds (dataset {:a [1 2 3] 172 | :b [4 5 6] 173 | :c [7 8 9] 174 | :d [10 11 12]})] 175 | (doseq [op ops] 176 | (let [result (op ds :e [:a :b :c :d])] 177 | (contains? result :e) => true))))) 178 | -------------------------------------------------------------------------------- /test/tablecloth/api/utils_test.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.api.utils-test 2 | (:require [tablecloth.api :as api] 3 | [tablecloth.api.utils :as sut] 4 | [tech.v3.dataset :as ds] 5 | [tech.v3.datatype :as dtype] 6 | [clojure.string :as str] 7 | [midje.sweet :refer [tabular fact =>]])) 8 | 9 | (def ds (api/dataset "data/who.csv.gz")) 10 | (def dsg (api/group-by ds "country")) 11 | 12 | (def ds-sn (api/dataset {[1 2 3] [1 2 3] 13 | false ["false" "false" "false"] 14 | true ["true" "true" "true"] 15 | {:a 1 :b 2} [9 9 9]})) 16 | 17 | (fact "iterable-sequence" 18 | (tabular (fact (sut/iterable-sequence? ?x) 19 | => true) 20 | [?x] 21 | [] '() #{} 22 | (ds "country") 23 | (java.util.ArrayList.) 24 | (dtype/const-reader 3 5)) 25 | (tabular (fact (sut/iterable-sequence? ?x) 26 | => false) 27 | [?x] 28 | nil {})) 29 | 30 | (fact "->str" 31 | (tabular (fact (sut/->str ?y) 32 | => ?x) 33 | [?x ?y] 34 | "a" "a" 35 | "b" :a/b 36 | "c" 'a/c 37 | "b" :b 38 | "c" 'c)) 39 | 40 | (fact "column-names-regular" 41 | (fact (api/column-names ds) 42 | => (ds/column-names ds)) 43 | (fact (api/column-names ds :all) 44 | => (ds/column-names ds)) 45 | (fact (nil? (api/column-names ds nil)) 46 | => true) 47 | (tabular (fact (api/column-names ds ?y) 48 | => ?x) 49 | [?x ?y] 50 | ["country" "iso2" "iso3"] :type/string 51 | ["country" "iso2" "iso3"] :!type/numerical 52 | ["year"] "year" 53 | ["year"] ["year"] 54 | ["iso2" "year"] #{"year" "iso2"} 55 | ["new_sp_m014" "country"] ["new_sp_m014" "country"] 56 | ["iso2" "iso3"] #(str/starts-with? % "iso") 57 | ["country"] {"country" nil} ;; only keys are used 58 | ["iso2" "iso3"] #"^i.*") 59 | (fact (every? #(str/starts-with? % "new") (api/column-names ds (complement #{"country" "iso2" "iso3" "year"}))) 60 | => true) 61 | (tabular (fact (api/column-names ds ?y ?f) 62 | => ?x) 63 | [?x ?y ?f] 64 | ["iso2" "iso3"] #{"iso2" "iso3"} :name 65 | ["country" "iso2" "iso3"] #{:string} :datatype 66 | ["new_sp_m014"] (fn [{:keys [name datatype]}] 67 | (and (= datatype :int16) 68 | (str/starts-with? name "new_sp_m"))) :all) 69 | (tabular (fact (api/column-names ds-sn ?y) 70 | => ?x) 71 | [?x ?y] 72 | [false] false? 73 | [true] true? 74 | [false] #{false} 75 | [true] #{true} 76 | [[1 2 3]] vector? 77 | [{:a 1 :b 2}] map? 78 | [[1 2 3] {:a 1 :b 2}] seqable? 79 | [false true] boolean? 80 | [[1 2 3] {:a 1 :b 2}] :type/numerical 81 | [false true] :!type/numerical)) 82 | 83 | (fact "column-names-grouped" 84 | (fact (api/column-names dsg) 85 | => (ds/column-names ds)) 86 | (fact (api/column-names dsg :all) 87 | => (ds/column-names ds)) 88 | (fact (nil? (api/column-names dsg nil)) 89 | => true) 90 | (tabular (fact (api/column-names dsg ?y) 91 | => ?x) 92 | [?x ?y] 93 | ["country" "iso2" "iso3"] :type/string 94 | ["year"] "year" 95 | ["year"] ["year"] 96 | ["iso2" "year"] #{"year" "iso2"} 97 | ["new_sp_m014" "country"] ["new_sp_m014" "country"] 98 | ["iso2" "iso3"] #(str/starts-with? % "iso") 99 | ["country"] {"country" nil} ;; only keys are used 100 | ["iso2" "iso3"] #"^i.*") 101 | (fact (every? #(str/starts-with? % "new") (api/column-names dsg (complement #{"country" "iso2" "iso3" "year"}))) 102 | => true) 103 | (tabular (fact (api/column-names dsg ?y ?f) 104 | => ?x) 105 | [?x ?y ?f] 106 | ["iso2" "iso3"] #{"iso2" "iso3"} :name 107 | ["country" "iso2" "iso3"] #{:string} :datatype 108 | ["new_sp_m014"] (fn [{:keys [name datatype]}] 109 | (and (= datatype :int16) 110 | (str/starts-with? name "new_sp_m"))) :all)) 111 | 112 | (fact "rank-tests" 113 | (let [x2 [3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5]] 114 | (tabular (fact (sut/rank ?y) 115 | => ?x) 116 | [?x ?y] 117 | '(1.0 0.0 2.0 3.0 4.0) [3 1 4 15 92] 118 | '(1.0 0.0 2.0 3.0 4.0) '(1.0 0.0 2.0 3.0 4.0) 119 | '(3.5 0.5 5.0 0.5 7.0 10.0 2.0 9.0 7.0 3.5 7.0) x2 120 | '(3.5 0.5 5.0 0.5 7.0 10.0 2.0 9.0 7.0 3.5 7.0) '(3.5 0.5 5.0 0.5 7.0 10.0 2.0 9.0 7.0 3.5 7.0) 121 | ;; as in data.table, `frankv(?x, na.last=FALSE)` 122 | '(5.0 2.5 5.0 0.5 2.5 0.5 5.0) [4 1 4 nil 1 nil 4]) 123 | (tabular (fact (sut/rank ?y ?ties) 124 | => ?x) 125 | [?x ?y ?ties] 126 | [3 0 5 1 6 10 2 9 7 4 8] x2 :first 127 | [4 1 5 0 8 10 2 9 7 3 6] x2 :last 128 | '(4 1 5 1 8 10 2 9 8 4 8) x2 :max 129 | '(3 0 5 0 6 10 2 9 6 3 6) x2 :min 130 | '(2 0 3 0 4 6 1 5 4 2 4) x2 :dense) 131 | (fact (sut/rank x2 :average true) 132 | => '(6.5 9.5 5.0 9.5 3.0 0.0 8.0 1.0 3.0 6.5 3.0)) 133 | (fact (sut/rank x2 :dense true) 134 | => '(4 6 3 6 2 0 5 1 2 4 2)))) 135 | 136 | 137 | (fact "->general-types describes the set of general types for a concrete datatype" 138 | (sut/->general-types :int32) => #{:integer :numerical} 139 | (sut/->general-types :float32) => #{:float :numerical} 140 | (sut/->general-types :string) => #{:textual}) 141 | -------------------------------------------------------------------------------- /test/tablecloth/column/api/column_test.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.column.api.column-test 2 | (:require [tablecloth.column.api.column :refer [column zeros ones typeof? 3 | typeof slice sort-column]] 4 | [tech.v3.dataset.column :refer [is-column?]] 5 | [midje.sweet :refer [fact facts =>]])) 6 | 7 | (fact "`column` returns a column" 8 | (is-column? (column)) => true) 9 | 10 | (fact "`column` provides a few ways to generate a column`" 11 | (column) => [] 12 | (column [1 2 3]) => [1 2 3] 13 | (column (list 1 2 3)) =>[1 2 3] 14 | (column (range 3)) => [0 1 2]) 15 | 16 | (fact "`column` identifies the type of the elements automatically" 17 | (-> [1 2 3] 18 | (column) 19 | (tech.v3.datatype/elemwise-datatype)) => :int64 20 | (-> [true false true] 21 | (column) 22 | (tech.v3.datatype/elemwise-datatype)) => :boolean 23 | (-> [1 true false] 24 | (column) 25 | (tech.v3.datatype/elemwise-datatype)) => :object) 26 | 27 | (fact "`typeof` returns the concrete type of the elements" 28 | (typeof (column [1 2 3])) => :int64 29 | (typeof (column ["a" "b" "c"])) => :string 30 | (typeof (column [true false])) => :boolean) 31 | 32 | (fact "`typeof?` can check the concerete type of column elements" 33 | (typeof? (column [1 2 3]) :int64) => true 34 | (typeof? (column [1 2 3]) :int32) => false 35 | (typeof? (column ["a" "b" "c"]) :string) => true) 36 | 37 | (fact "`typeof?` can check the general type of column elements" 38 | (typeof? (column [1 2 3]) :integer) => true 39 | (typeof? (column [1 2 3]) :textual) => false 40 | (typeof? (column [1.0 2.0 3.0]) :numerical) => true 41 | (typeof? (column [1.0 2.0 3.0]) :logical) => false 42 | (typeof? (column ["a" "b" "c"]) :textual) => true 43 | (typeof? (column ["a" "b" "c"]) :numerical) => false 44 | (typeof? (column [true false true]) :logical) => true) 45 | 46 | (fact "`zeros` returns a column filled with zeros" 47 | (zeros 3) => [0 0 0]) 48 | 49 | (fact "`ones` returns a column filled with ones" 50 | (ones 3) => [1 1 1]) 51 | 52 | (facts "about `slice`" 53 | (let [c (column [1 2 3 4 5])] 54 | (fact "it return a subset of a column inclusively" 55 | (slice c 0 0) => [1] 56 | (slice c 0 4) => [1 2 3 4 5]) 57 | (fact "it supports negative indexing inclusively" 58 | (slice c 0 -1) 59 | (slice c -1 -1) => [5] 60 | (slice c -3 -1) => [3 4 5]) 61 | (fact "it supports 0 within negative indexing" 62 | (slice c 0 -2) => [1 2 3 4]) 63 | (fact "it supports stepped slicing" 64 | (slice c 0 4 2) => [1 3 5]) 65 | (fact "it supports using nil to indicate slice from start or end" 66 | (slice c 2) => [3 4 5] 67 | (slice c -2) => [4 5] 68 | (slice c nil 2) => [1 2 3] 69 | (slice c nil -2) => [1 2 3 4]) 70 | (fact "it supports special keywords for selecting from start or end" 71 | (slice c :start 2) => [1 2 3] 72 | (slice c 1 :end) => [2 3 4 5] 73 | (slice c -4 :end) => [2 3 4 5] 74 | (slice c :start -3) => [1 2 3]))) 75 | 76 | (facts "about `sort-column`" 77 | (let [c-ints (column [3 100 9 0 -10 43]) 78 | c-strings (column ["a" "z" "baz" "fo" "bar" "foo"])] 79 | (fact "it returns a column" 80 | (sort-column c-ints) => is-column?) 81 | (fact "it sorts in ascending order by default" 82 | (sort-column c-ints) => [-10 0 3 9 43 100] 83 | (sort-column c-strings) => ["a" "bar" "baz" "fo" "foo" "z"]) 84 | (fact "it accepts a comparator-fn" 85 | (sort-column c-strings 86 | #(> (count %1) (count %2))) => ["baz" "bar" "foo" "fo" "z" "a"]) 87 | (fact "it moves missing values to the end" 88 | (sort-column (column [nil 100 nil 3 -10])) => [-10 3 100 nil nil]))) 89 | 90 | -------------------------------------------------------------------------------- /test/tablecloth/column/api/missing_test.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.column.api.missing-test 2 | (:require [tablecloth.column.api.missing :refer [count-missing replace-missing drop-missing]] 3 | [tablecloth.column.api.column :refer [column]] 4 | [midje.sweet :refer [facts =>]])) 5 | 6 | (facts "about `count-missing`" 7 | (count-missing (column [1 2 3])) => 0) 8 | 9 | (facts "about `replace-missing`" 10 | (replace-missing (column [1 nil 3])) => [1 1 3]) 11 | 12 | (facts "about `drop-missing`" 13 | (drop-missing (column [1 nil 3])) => [1 3]) 14 | -------------------------------------------------------------------------------- /test/tablecloth/column/api/operators_test.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.column.api.operators-test 2 | (:refer-clojure :exclude [* + - / < <= > >= abs and bit-and bit-and-not bit-clear bit-flip 3 | bit-not bit-or bit-set bit-shift-left bit-shift-right bit-test bit-xor 4 | even? identity infinite? max min neg? not odd? odd? or pos? quot rem 5 | unsigned-bit-shift-right zero?]) 6 | (:require [midje.sweet :refer [fact facts =>]] 7 | [clojure.test :refer [deftest is]] 8 | [tablecloth.column.api :refer [column column? typeof]]) 9 | (:use [tablecloth.column.api.operators])) 10 | 11 | (defn sample-column [n] 12 | (column (repeatedly n #(rand-int 100)))) 13 | 14 | (defn scalar? [item] 15 | (= (tech.v3.datatype.argtypes/arg-type item) 16 | :scalar)) 17 | 18 | (facts 19 | "about 'shift" 20 | (let [a (column [1 2 3 4 5])] 21 | (shift a 2) => [1 1 1 2 3])) 22 | 23 | (facts 24 | "about 'descriptive-statistics" 25 | (let [a (sample-column 5)] 26 | ;; sanity check that we got the hash with desired data 27 | (descriptive-statistics a) => #(contains? % :standard-deviation))) 28 | 29 | (facts 30 | "about 'quartiles" 31 | (let [a (sample-column 100)] 32 | (quartiles a) => column? 33 | ;; sanity check quartiles should return a coumn of 5 values 34 | (count (quartiles a)) => 5)) 35 | 36 | (facts 37 | "about 'fill-range" 38 | (let [result (fill-range [1 5] 1)] 39 | (contains? result :result) => true 40 | (contains? result :missing) => true)) 41 | 42 | (facts 43 | "about ops that take a single column and return a column" 44 | (let [ops [abs 45 | acos 46 | asin 47 | atan 48 | bit-not 49 | cbrt 50 | ceil 51 | cos 52 | cosh 53 | cumprod 54 | cumsum 55 | cummax 56 | cummin 57 | exp 58 | expm1 59 | floor 60 | get-significand 61 | identity 62 | log 63 | log10 64 | log1p 65 | logistic 66 | next-down 67 | next-up 68 | normalize 69 | rint 70 | signum 71 | sin 72 | sinh 73 | sq 74 | sqrt 75 | tan 76 | tanh 77 | to-degrees 78 | to-radians 79 | ulp 80 | ] 81 | a (sample-column 5)] 82 | (doseq [op ops] 83 | (op a) => column?))) 84 | 85 | (facts 86 | "about ops that take a single column and return a scalar" 87 | (let [ops [magnitude 88 | reduce-max 89 | reduce-min 90 | reduce-* 91 | reduce-+ 92 | mean-fast 93 | sum-fast 94 | magnitude-squared] 95 | a (sample-column 5)] 96 | (doseq [op ops] 97 | (op a) => scalar?))) 98 | 99 | (facts 100 | "about ops that take one or more columns or scalars and return either a scalar or a column" 101 | (let [ops [/ - +] 102 | a (sample-column 5) 103 | b (sample-column 5) 104 | c (sample-column 5) 105 | d (sample-column 5)] 106 | (doseq [op ops] 107 | (op a) => column? 108 | (op a b) => column? 109 | (op a b c) => column? 110 | (op a b c d) => column? 111 | (op 1) => scalar? 112 | (op 1 2) => scalar? 113 | (op 1 2 3) => scalar?))) 114 | 115 | (facts 116 | "about comparison ops that take two or more columns and return a boolean" 117 | (let [ops [> >= < <=] 118 | a (sample-column 5) 119 | b (sample-column 5) 120 | c (sample-column 5)] 121 | (doseq [op ops] 122 | (op a b) => column? 123 | (op a b c) => column? 124 | (op 1 2) => boolean?))) 125 | 126 | (facts 127 | "about comparison ops that take two columns and return a boolean" 128 | (let [ops [equals] 129 | a (sample-column 5) 130 | b (sample-column 5)] 131 | (doseq [op ops] 132 | (op a b) => boolean?))) 133 | 134 | (facts 135 | "about comparison ops that take two columns or scalars and return a boolean or column of booleans" 136 | (let [ops [or and eq not-eq] 137 | a (sample-column 5) 138 | b (sample-column 5)] 139 | (doseq [op ops] 140 | (op a b) => column? 141 | (typeof (op a b)) => :boolean 142 | (op 1 2) => boolean?))) 143 | 144 | (facts 145 | "about ops that take a single column or scalar and return a scalar" 146 | (let [ops [kurtosis 147 | sum 148 | mean 149 | skew 150 | variance 151 | standard-deviation 152 | quartile-3 153 | quartile-1 154 | median] 155 | a (sample-column 5)] 156 | (doseq [op ops] 157 | (op a) => scalar?))) 158 | 159 | (facts 160 | "about ops that take two or more scalars or columns and return a column or scalar" 161 | (let [ops [* 162 | bit-and 163 | bit-and-not 164 | bit-clear 165 | bit-flip 166 | bit-or 167 | bit-set 168 | bit-shift-right 169 | bit-shift-left 170 | ;; bit-test 171 | bit-xor 172 | hypot 173 | ieee-remainder 174 | max 175 | min 176 | pow 177 | quot 178 | rem 179 | unsigned-bit-shift-right 180 | ] 181 | a (sample-column 5) 182 | b (sample-column 5) 183 | c (sample-column 5)] 184 | (doseq [op ops] 185 | (op a b) => column? 186 | (op a b c) => column? 187 | (op 5 5) 188 | (op 5 5 5)))) 189 | 190 | (facts 191 | "about ops that take left-hand / right-hand columns and return a scalar" 192 | (let [ops [distance 193 | dot-product 194 | distance-squared 195 | kendalls-correlation 196 | pearsons-correlation 197 | spearmans-correlation] 198 | a (sample-column 5) 199 | b (sample-column 5)] 200 | (doseq [op ops] 201 | (op a b) => scalar?))) 202 | 203 | (facts 204 | "about ops that take a single column or scalar and return boolean or column of booleans" 205 | (let [ops [finite? 206 | pos? 207 | neg? 208 | mathematical-integer? 209 | nan? 210 | even? 211 | zero? 212 | not 213 | infinite? 214 | odd?] 215 | a (sample-column 5)] 216 | (doseq [op ops] 217 | (op a) => column? 218 | (typeof (op a)) => :boolean 219 | (op 1) => boolean?))) 220 | -------------------------------------------------------------------------------- /test/tablecloth/common_test.clj: -------------------------------------------------------------------------------- 1 | (ns tablecloth.common-test 2 | (:require [tablecloth.api :as api])) 3 | 4 | (def DS (api/dataset {:V1 (take 9 (cycle [1 2])) 5 | :V2 (range 1 10) 6 | :V3 (take 9 (cycle [0.5 1.0 1.5])) 7 | :V4 (take 9 (cycle ["A" "B" "C"]))} 8 | {:dataset-name "DS"})) 9 | 10 | (defn approx 11 | ^double [^double v] 12 | (-> (Double/toString v) 13 | (BigDecimal.) 14 | (.setScale 4 BigDecimal/ROUND_HALF_UP) 15 | (.doubleValue))) 16 | 17 | --------------------------------------------------------------------------------