├── .devcontainer ├── Dockerfile └── devcontainer.json ├── .github └── workflows │ ├── ci.yml │ └── release.yml ├── .gitignore ├── .gitpod.yml ├── .vscode └── settings.json ├── LICENSE ├── README.md ├── deno.json ├── deps.ts ├── import_map.json ├── lib ├── document.ts ├── element.ts ├── helpers.ts ├── node.ts ├── query.ts ├── schema.ts ├── selector.ts ├── server.ts ├── shared.ts ├── state.ts └── types.d.ts ├── mod.ts ├── serve.ts └── tests ├── deps.ts ├── schema.test.ts └── utils.test.ts /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG VARIANT=bullseye 2 | ARG PLATFORM=linux/amd64 3 | FROM --platform=${PLATFORM} mcr.microsoft.com/vscode/devcontainers/base:0-${VARIANT} 4 | 5 | RUN export DENO_INSTALL=/deno \ 6 | && export DENO_DIR=${DENO_INSTALL}/.cache/deno \ 7 | && export PATH=${DENO_INSTALL}/bin:${PATH} \ 8 | && mkdir -p ${DENO_INSTALL} \ 9 | && curl -fsSL https://deno.land/x/install/install.sh | sh \ 10 | && chown -R vscode ${DENO_INSTALL} ; 11 | 12 | # [Optional] Uncomment this section to install additional OS packages. 13 | RUN apt-get update \ 14 | && export DEBIAN_FRONTEND=noninteractive \ 15 | && apt-get -y install \ 16 | --no-install-recommends \ 17 | build-essential \ 18 | ca-certificates \ 19 | gcc ; 20 | 21 | 22 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Deno", 3 | "build": { 4 | "dockerfile": "Dockerfile", 5 | "args": { 6 | "VARIANT": "bullseye" 7 | } 8 | }, 9 | "customizations": { 10 | "codespaces": { 11 | "repositories": { 12 | "deno911/*": { 13 | "permissions": "write-all" 14 | }, 15 | "nberlette/*": { 16 | "permissions": "write-all" 17 | } 18 | } 19 | }, 20 | "vscode": { 21 | "settings": { 22 | "deno.enable": true, 23 | "deno.lint": true, 24 | "deno.unstable": true, 25 | "deno.codeLens.test": true, 26 | "deno.codeLens.implementations": true, 27 | "deno.codeLens.references": true, 28 | "deno.codeLens.referencesAllFunctions": true, 29 | "editor.defaultFormatter": "denoland.vscode-deno", 30 | "terminal.integrated.env.linux": {}, 31 | "git.enableCommitSigning": true 32 | }, 33 | "extensions": [ 34 | "denoland.vscode-deno", 35 | "GitHub.copilot-nightly", 36 | "GitHub.copilot-labs", 37 | "editorconfig.editorconfig", 38 | "antfu.iconify", 39 | "ClearTaxEngineering.jsontographqlschema", 40 | "GraphQL.vscode-graphql", 41 | "laurencebahiirwa.deno-std-lib-snippets", 42 | "redhat.vscode-yaml", 43 | "cschleiden.vscode-github-actions", 44 | "salbert.comment-ts", 45 | "vsls-contrib.gistfs" 46 | ] 47 | } 48 | }, 49 | "remoteUser": "vscode", 50 | "forwardPorts": [3000, 8000, 8080], 51 | "postCreateCommand": "brew bundle install --global 2>/dev/null", 52 | "features": { 53 | "git": "latest", 54 | "github-cli": "latest", 55 | "sshd": "latest", 56 | "homebrew": "latest", 57 | "python": "latest", 58 | "ruby": "3.1" 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: "Deno CI" 2 | on: 3 | workflow_dispatch: 4 | repository_dispatch: 5 | push: 6 | branches: 7 | - "main" 8 | - "master" 9 | - "feat/**" 10 | - "feature/**" 11 | - "release/**" 12 | tags: 13 | - "v*" 14 | pull_request: 15 | branches: 16 | - "main" 17 | - "master" 18 | jobs: 19 | format: 20 | strategy: 21 | fail-fast: true 22 | matrix: 23 | os: [ubuntu-latest] 24 | deno-version: [1.x] 25 | name: "Format + Lint - 🦕 ${{matrix.deno-version}} on 💽 ${{matrix.os}}" 26 | runs-on: ${{matrix.os}} 27 | steps: 28 | - 29 | name: "🔧 setup: checkout" 30 | uses: actions/checkout@v3 31 | - 32 | name: "🔧 setup: 🦕 ${{matrix.deno-version}}" 33 | uses: denoland/setup-deno@main 34 | with: 35 | deno-version: ${{matrix.deno-version}} 36 | - 37 | name: "🎨 run deno fmt" 38 | run: | 39 | deno fmt --unstable \ 40 | --no-clear-screen \ 41 | --options-line-width 100 \ 42 | --options-prose-wrap preserve \ 43 | --ignore=.devcontainer,.github,.vscode ; 44 | - 45 | name: "🚨 run deno lint" 46 | run: | 47 | deno lint --unstable \ 48 | --no-clear-screen \ 49 | --ignore=.devcontainer,.github,.vscode \ 50 | --rules-exclude=no-explicit-any,no-empty-interface,no-cond-assign 51 | test: 52 | strategy: 53 | fail-fast: true 54 | matrix: 55 | os: [windows-latest, ubuntu-latest, macos-latest] 56 | deno-version: [1.16.0, 1.22.0, canary] 57 | name: "Test - 🦕 ${{matrix.deno-version}} on 💽 ${{matrix.os}}" 58 | runs-on: ${{matrix.os}} 59 | steps: 60 | - 61 | name: "🔧 setup: checkout" 62 | uses: actions/checkout@v3 63 | - 64 | name: "🔧 setup: 🦕 ${{matrix.deno-version}}" 65 | uses: denoland/setup-deno@main 66 | with: 67 | deno-version: ${{matrix.deno-version}} 68 | - 69 | name: "🧪 test: stable + checks" 70 | run: deno test -A --check --no-check=remote --jobs 4 71 | - 72 | name: "🧪 test: stable, no-checks" 73 | run: deno test -A --no-check --jobs 4 74 | - 75 | name: "🧪 test: unstable + checks" 76 | run: deno test -A --check --no-check=remote --unstable --jobs 4 77 | - 78 | name: "🧪 test: unstable, no-checks" 79 | run: deno test -A --unstable --no-check --jobs 4 80 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | tags: 5 | - 'v*' 6 | jobs: 7 | release: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | with: 12 | fetch-depth: 0 13 | - uses: actions/setup-node@v2 14 | with: 15 | node-version: '16' 16 | registry-url: https://registry.npmjs.org/ 17 | - run: npx conventional-github-releaser -p angular 18 | env: 19 | CONVENTIONAL_GITHUB_RELEASER_TOKEN: ${{secrets.GITHUB_TOKEN}} 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | Thumbs.db 3 | *.log 4 | .*.log 5 | .env* 6 | !.env.vault 7 | *.*workspace 8 | package.json 9 | test.ts 10 | -------------------------------------------------------------------------------- /.gitpod.yml: -------------------------------------------------------------------------------- 1 | image: nberlette/gitpod-enhanced:latest 2 | 3 | tasks: 4 | - # WARNING: remove this if you share your workspaces! 5 | before: | 6 | cd "$GITPOD_REPO_ROOT" || exit $?; 7 | [ -n "$DOTENV_VAULT" ] && [ ! -e .env.vault ] && 8 | echo "DOTENV_VAULT=$DOTENV_VAULT" > .env.vault; 9 | [ -n "$DOTENV_ME" ] && [ ! -e .env.me ] && 10 | echo "DOTENV_ME=$DOTENV_ME" > .env.me; 11 | if [ -e .env.me ] && [ -e .env.vault ] && [ ! -e .env ]; then 12 | which dotenv-vault &>/dev/null && 13 | dotenv-vault pull || npx -y dotenv-vault@latest pull; 14 | # expose the .env variables to current environment 15 | [ -e .env ] && { set -a; source .env; set +a; } 16 | fi 17 | # make sure we have deno installed 18 | init: | 19 | export DENO_INSTALL_ROOT="$HOME/.deno/bin" 20 | export PATH="$DENO_INSTALL_ROOT:$PATH" 21 | which deno &>/dev/null || brew install deno --quiet --overwrite 22 | gp sync-done listo 23 | - # proceed once we are ready 24 | init: gp sync-await listo 25 | command: deno task dev 2>&1 || deno task 26 | 27 | ports: 28 | - name: "Develop" 29 | port: 8000 30 | visibility: private 31 | onOpen: open-preview 32 | - name: "Preview" 33 | port: 8080 34 | visibility: public 35 | onOpen: notify 36 | 37 | github: 38 | prebuilds: 39 | master: true 40 | branches: true 41 | pullRequests: true 42 | pullRequestsFromForks: true 43 | addLabel: true 44 | addBadge: true 45 | addCheck: true 46 | 47 | vscode: 48 | extensions: 49 | - github.copilot-nightly 50 | - GitHub.copilot-labs 51 | - denoland.vscode-deno 52 | - vsls-contrib.gistfs 53 | - github.vscode-codeql 54 | - cschleiden.vscode-github-actions 55 | - editorconfig.editorconfig 56 | - jock.svg 57 | - antfu.iconify 58 | - antfu.unocss 59 | - redhat.vscode-yaml 60 | - jacano.vscode-pnpm 61 | - christian-kohler.path-intellisense 62 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "deno.enable": true, 3 | "deno.lint": true, 4 | "deno.unstable": true, 5 | "deno.codeLens.test": true, 6 | "deno.enablePaths": ["./"] 7 | } 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | 3 | Copyright (c) 2022 nyancodeid (https://github.com/nyancodeid) 4 | Copyright (c) 2022 Nicholas Berlette (https://github.com/nberlette) 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | # [🦕 DQL](https://deno.land/x/dql) 4 | 5 | ### _**Web Scraping with Deno  –  DOM + GraphQL**_ 6 | 7 |
8 | 9 | --- 10 | 11 | **`DQL`** is a web scraping module for Deno and Deno Deploy that integrates the power of [**GraphQL Queries**](https://graphql.org/learn/queries) with the DOM tree of a remote webpage or HTML document fragment. This is a fork of [**DenoQL**](https://deno.land/x/denoql) with some heavy refactoring and some additional features: 12 | 13 | - [x] Compatibility with the [**Deno Deploy**](https://deno.com/deploy) architecture 14 | - [x] Ability to pass variables alongside all queries 15 | - [x] New state-management class with additional methods 16 | - [x] Modular project structure (as opposed to a mostly single-file design) 17 | - [x] Improved types and schema structure 18 | 19 | > **Note**: _This is a work-in-progress and there is still a lot to be done._ 20 | 21 | ### 🛝  [**`GraphQL Playground`**](https://dql.deno.dev) 22 | 23 | ### 📝  [**`HackerNews Scraper`**](https://dash.deno.com/playground/dql-hn) 24 | 25 | ### 🚛  [**`Junkyard Scraper`**](https://dash.deno.com/playground/dirty-sparrow-69) 26 | 27 | --- 28 | 29 | ## `useQuery` 30 | 31 | The primary function exported by the module is the workhorse named `useQuery`: 32 | 33 | ```ts 34 | import { useQuery } from "https://deno.land/x/dql/mod.ts"; 35 | 36 | const data = await useQuery(`query { ... }`); 37 | ``` 38 | 39 | ### `QueryOptions` 40 | 41 | You can also provide a `QueryOptions` object as the second argument of `useQuery`, to further control the behavior of your query requests. All properties are optional. 42 | 43 | ```ts 44 | const data = await useQuery(`query { ... }`, { 45 | concurrency: 8, // passed directly to PQueue initializer 46 | fetch_options: { // passed directly to Fetch API requests 47 | headers: { 48 | "Authorization": "Bearer ghp_a5025a80a24defd0a7d06b4fc215bb5635a167c6", 49 | }, 50 | }, 51 | variables: {}, // variables defined in your queries 52 | operationName: "", // when using multiple queries 53 | }); 54 | ``` 55 | 56 | ## `createServer` 57 | 58 | With [**Deno Deploy**](https://dash.deno.com/new), you can deploy **`DQL`** with a GraphQL Playground in **only 2 lines of code**: 59 | 60 | ```ts 61 | import { createServer } from "https://deno.land/x/dql/mod.ts"; 62 | 63 | createServer(80, { endpoint: "https://dql.deno.dev" }); 64 | ``` 65 | 66 | `🛝` [Try the **GraphQL Playground** at **`dql.deno.dev`**](https://dql.deno.dev)\ 67 | `🦕` [View the source code in the **`Deno Playground`**](https://dash.deno.com/playground/dql) 68 | 69 | ## Command Line Usage (CLI) 70 | 71 | ```bash 72 | deno run -A --unstable https://deno.land/x/dql/serve.ts 73 | ``` 74 | 75 | #### Custom port (default is `8080`) 76 | 77 | ```bash 78 | deno run -A https://deno.land/x/dql/serve.ts --port 3000 79 | ``` 80 | 81 | > **Warning**: you need to have the [**Deno CLI**](https://deno.land) installed first. 82 | 83 | --- 84 | 85 | ## 💻 Examples 86 | 87 | ### `🚛` Junkyard Scraper · [**`Deno Playground 🦕`**](https://dash.deno.com/playground/dirty-sparrow-69) 88 | 89 | ```ts 90 | import { useQuery } from "https://deno.land/x/dql/mod.ts"; 91 | import { serve } from "https://deno.land/std@0.147.0/http/server.ts"; 92 | 93 | serve(async (res: Request) => 94 | await useQuery( 95 | ` 96 | query Junkyard ( 97 | $url: String 98 | $itemSelector: String = "table > tbody > tr" 99 | ) { 100 | vehicles: page(url: $url) { 101 | totalCount: count(selector: $itemSelector) 102 | nodes: queryAll(selector: $itemSelector) { 103 | id: index 104 | vin: text(selector: "td:nth-child(7)", trim: true) 105 | sku: text(selector: "td:nth-child(6)", trim: true) 106 | year: text(selector: "td:nth-child(1)", trim: true) 107 | model: text(selector: "td:nth-child(2) > .notranslate", trim: true) 108 | aisle: text(selector: "td:nth-child(3)", trim: true) 109 | store: text(selector: "td:nth-child(4)", trim: true) 110 | color: text(selector: "td:nth-child(5)", trim: true) 111 | date: attr(selector: "td:nth-child(8)", name: "data-value") 112 | image: src(selector: "td > a > img") 113 | } 114 | } 115 | }`, 116 | { 117 | variables: { 118 | "url": "http://nvpap.deno.dev/action=getVehicles&makes=BMW", 119 | }, 120 | }, 121 | ) 122 | .then((data) => JSON.stringify(data, null, 2)) 123 | .then((json) => 124 | new Response(json, { 125 | headers: { "content-type": "application/json;charset=utf-8" }, 126 | }) 127 | ) 128 | ); 129 | ``` 130 | 131 | ### 📝 HackerNews Scraper · [**`Deno Playground 🦕`**](https://dash.deno.com/playground/dql-hn) 132 | 133 | ```ts 134 | import { useQuery } from "https://deno.land/x/dql/mod.ts"; 135 | import { serve } from "https://deno.land/std@0.147.0/http/server.ts"; 136 | 137 | serve(async (res: Request) => 138 | await useQuery(` 139 | query HackerNews ( 140 | $url: String = "http://news.ycombinator.com" 141 | $rowSelector: String = "tr.athing" 142 | ) { 143 | page(url: $url) { 144 | title 145 | totalCount: count(selector: $rowSelector) 146 | nodes: queryAll(selector: $rowSelector) { 147 | rank: text(selector: "td span.rank", trim: true) 148 | title: text(selector: "td.title a", trim: true) 149 | site: text(selector: "span.sitestr", trim: true) 150 | url: href(selector: "td.title a") 151 | attrs: next { 152 | score: text(selector: "span.score", trim: true) 153 | user: text(selector: "a.hnuser", trim: true) 154 | date: attr(selector: "span.age", name: "title") 155 | } 156 | } 157 | } 158 | }`) 159 | .then((data) => JSON.stringify(data, null, 2)) 160 | .then((json) => 161 | new Response(json, { 162 | headers: { "content-type": "application/json;charset=utf-8" }, 163 | }) 164 | ) 165 | ); 166 | ``` 167 | 168 | ## License 169 | 170 | MIT © [**Nicholas Berlette**](https://github.com/nberlette), based on [DenoQL](https://deno.land/x/denoql). 171 | -------------------------------------------------------------------------------- /deno.json: -------------------------------------------------------------------------------- 1 | { 2 | "importMap": "./import_map.json", 3 | "compilerOptions": { 4 | "strict": true, 5 | "experimentalDecorators": true, 6 | "lib": [ 7 | "deno.window", 8 | "deno.ns", 9 | "dom", 10 | "dom.iterable", 11 | "dom.asynciterable" 12 | ], 13 | "types": [ 14 | "https://deno.land/x/graphql_deno@v15.0.0/mod.ts", 15 | "https://deno.land/x/deno_dom@v0.1.31-alpha/deno-dom-wasm.ts", 16 | "https://deno.land/x/p_queue@1.0.1/mod.ts", 17 | "./lib/types.d.ts" 18 | ] 19 | }, 20 | "fmt": { 21 | "files": { 22 | "exclude": [ 23 | ".devcontainer", 24 | ".git*", 25 | ".vscode", 26 | "*.md", 27 | "LICENSE" 28 | ] 29 | }, 30 | "options": { 31 | "proseWrap": "preserve" 32 | } 33 | }, 34 | "lint": { 35 | "files": { 36 | "exclude": [ 37 | ".devcontainer", 38 | ".git*", 39 | ".vscode", 40 | "*.md", 41 | "LICENSE" 42 | ] 43 | }, 44 | "rules": { 45 | "exclude": [ 46 | "no-explicit-any" 47 | ] 48 | } 49 | }, 50 | "tasks": { 51 | "dev": "deno run -A --unstable --watch=.,./tests,./lib serve.ts", 52 | "serve": "deno run --allow-net --unstable serve.ts", 53 | "test": "deno test -A --jobs 4", 54 | "test:nocheck": "deno test -A --no-check --jobs 4", 55 | "test:unstable": "deno test -A --unstable --jobs 4", 56 | "test:unstable:nocheck": "deno test -A --unstable --no-check --jobs 4" 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /deps.ts: -------------------------------------------------------------------------------- 1 | export { 2 | graphql, 3 | GraphQLBoolean, 4 | GraphQLDeprecatedDirective, 5 | GraphQLEnumType, 6 | GraphQLError, 7 | GraphQLInputObjectType, 8 | GraphQLInt, 9 | GraphQLInterfaceType, 10 | GraphQLList, 11 | GraphQLNonNull, 12 | GraphQLObjectType, 13 | GraphQLScalarType, 14 | GraphQLSchema, 15 | GraphQLString, 16 | graphqlSync, 17 | GraphQLUnionType, 18 | } from "https://deno.land/x/graphql_deno@v15.0.0/mod.ts"; 19 | 20 | export type { 21 | GraphQLAbstractType, 22 | GraphQLCompositeType, 23 | GraphQLEnumTypeConfig, 24 | GraphQLEnumValue, 25 | GraphQLEnumValueConfig, 26 | GraphQLEnumValueConfigMap, 27 | GraphQLField, 28 | GraphQLFieldConfig, 29 | GraphQLFieldConfigMap, 30 | GraphQLFieldMap, 31 | GraphQLFormattedError, 32 | GraphQLInputObjectTypeConfig, 33 | GraphQLInterfaceTypeConfig, 34 | GraphQLNamedType, 35 | GraphQLNullableType, 36 | GraphQLObjectTypeConfig, 37 | GraphQLSchemaConfig, 38 | GraphQLType, 39 | GraphQLTypeResolver, 40 | GraphQLUnionTypeConfig, 41 | Thunk, 42 | } from "https://deno.land/x/graphql_deno@v15.0.0/mod.ts"; 43 | 44 | export { default as PQueue } from "https://deno.land/x/p_queue@1.0.1/mod.ts"; 45 | 46 | export { 47 | denoDomDisableQuerySelectorCodeGeneration as disableCodeGen, 48 | DOMParser, 49 | } from "https://deno.land/x/deno_dom@v0.1.31-alpha/deno-dom-wasm.ts"; 50 | 51 | export type { Element } from "https://deno.land/x/deno_dom@v0.1.31-alpha/deno-dom-wasm.ts"; 52 | 53 | export { 54 | serve, 55 | type ServeInit, 56 | Server, 57 | type ServerInit, 58 | } from "https://deno.land/std@0.145.0/http/server.ts"; 59 | -------------------------------------------------------------------------------- /import_map.json: -------------------------------------------------------------------------------- 1 | { 2 | "imports": { 3 | "~/": "./", 4 | "@/": "./", 5 | "/": "./", 6 | "~~/": "./", 7 | "@@/": "./", 8 | "./": "./", 9 | "x/": "https://deno.land/x/", 10 | "std/": "https://deno.land/std@0.145.0/", 11 | "lib/": "./lib/", 12 | "esm/": "https://esm.sh/", 13 | "graphql": "https://deno.land/x/graphql_deno@v15.0.0/mod.ts", 14 | "graphql/": "https://deno.land/x/graphql_deno@v15.0.0/", 15 | "pqueue": "https://deno.land/x/p_queue@1.0.1/mod.ts", 16 | "deno-dom": "https://deno.land/x/deno_dom@v0.1.31-alpha/deno-dom-wasm.ts", 17 | "sift": "https://deno.land/x/sift@0.5.0/mod.tsx" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /lib/document.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Element, 3 | GraphQLObjectType, 4 | type GraphQLObjectTypeConfig, 5 | GraphQLString, 6 | } from "../deps.ts"; 7 | import { shared } from "./shared.ts"; 8 | import { TNode } from "./node.ts"; 9 | import { getAttributeOfElement } from "./helpers.ts"; 10 | 11 | export const TDocument = new GraphQLObjectType({ 12 | name: "Document", 13 | description: "Document, the root of the DOM tree.", 14 | interfaces: [TNode], 15 | fields: () => ({ 16 | ...shared, 17 | title: { 18 | type: GraphQLString, 19 | description: "Retrieves the document title.", 20 | resolve(element: Element) { 21 | return element?.ownerDocument?.title; 22 | }, 23 | }, 24 | meta: { 25 | type: GraphQLString, 26 | description: 27 | "Retrieves metadata from a named meta tag (if it exists), contained in the document head.", 28 | args: { 29 | name: { 30 | type: GraphQLString, 31 | description: 32 | "The name or property value of the meta tag. (e.g. 'og:image')", 33 | }, 34 | }, 35 | resolve(element: Element, { name }) { 36 | let meta = element?.querySelector(`meta[name='${name}']`); 37 | 38 | if (!meta) { 39 | meta = element?.querySelector(`meta[property='${name}']`); 40 | } 41 | try { 42 | return getAttributeOfElement(meta!, "content"); 43 | } catch { 44 | return null; 45 | } 46 | }, 47 | }, 48 | }), 49 | } as GraphQLObjectTypeConfig); 50 | -------------------------------------------------------------------------------- /lib/element.ts: -------------------------------------------------------------------------------- 1 | import { 2 | DOMParser, 3 | Element, 4 | GraphQLObjectType, 5 | type GraphQLObjectTypeConfig, 6 | GraphQLString, 7 | PQueue, 8 | } from "../deps.ts"; 9 | import { getAttributeOfElement, resolveURL } from "./helpers.ts"; 10 | import { TNode } from "./node.ts"; 11 | import { TDocument } from "./document.ts"; 12 | import { selector } from "./selector.ts"; 13 | import { shared } from "./shared.ts"; 14 | 15 | export const TElement = new GraphQLObjectType({ 16 | name: "Element", 17 | description: "A DOM element.", 18 | interfaces: [TNode], 19 | fields: () => ({ 20 | ...shared, 21 | visit: { 22 | type: TDocument, 23 | description: 24 | "If the element is a link, visit the page linked to in the href attribute.", 25 | async resolve(element: Element, _, context) { 26 | const href = element.getAttribute("href"); 27 | const base_url: string = context.state.get("base"); 28 | 29 | if (href == null) { 30 | return null; 31 | } 32 | 33 | let url = href; 34 | 35 | if (base_url) { 36 | url = resolveURL(base_url, href); 37 | context.state.set("url", url); 38 | } 39 | 40 | const options: RequestInit = context.state.get("fetch_options"); 41 | const html = await (context.state.get("queue") as PQueue).add(() => 42 | fetch(url, options).then((res) => res.text()) 43 | ); 44 | const dom = new DOMParser().parseFromString(html, "text/html"); 45 | 46 | return dom?.documentElement; 47 | }, 48 | }, 49 | visit_custom: { 50 | type: TDocument, 51 | description: 52 | "If the element is a link, visit the page linked to in the href attribute.", 53 | args: { 54 | selector, 55 | attr: { 56 | type: GraphQLString, 57 | description: "attribute name", 58 | }, 59 | }, 60 | async resolve(element: Element, { selector, attr }: TParams, context) { 61 | const base_url: string = context.state.get("base"); 62 | 63 | element = selector ? element.querySelector(selector)! : element; 64 | 65 | if (element == null) return null; 66 | if (attr == null) { 67 | attr = "href"; 68 | } 69 | 70 | const href = getAttributeOfElement(element, attr); 71 | 72 | if (href == null) return null; 73 | let url = href; 74 | 75 | if (base_url) { 76 | url = resolveURL(base_url, href); 77 | context.state.set("url", url); 78 | } 79 | 80 | const html = await (context.state.get("queue") as PQueue).add(() => 81 | fetch(url).then((res) => res.text()) 82 | ); 83 | const dom = new DOMParser().parseFromString(html, "text/html"); 84 | context.state.set("document", dom?.documentElement); 85 | return dom?.documentElement; 86 | }, 87 | }, 88 | }), 89 | } as GraphQLObjectTypeConfig); 90 | -------------------------------------------------------------------------------- /lib/helpers.ts: -------------------------------------------------------------------------------- 1 | import { Element } from "../deps.ts"; 2 | 3 | export function getAttributeOfElement( 4 | element: Element, 5 | name?: string, 6 | trim = false, 7 | ): string | null { 8 | if (element == null || name == null) { 9 | return null; 10 | } 11 | const attribute = element.getAttribute(name); 12 | if (attribute == null) { 13 | return null; 14 | } 15 | return trim ? attribute.trim() : attribute; 16 | } 17 | 18 | export function getAttributesOfElement(element: Element, trim = false) { 19 | if (element == null) { 20 | return null; 21 | } 22 | const names = element.getAttributeNames(); 23 | if (names == null || names.length === 0) { 24 | return {}; 25 | } 26 | return Object.fromEntries(names.map( 27 | (attr) => [attr, getAttributeOfElement(element, attr, trim)], 28 | )); 29 | } 30 | 31 | export function resolveURL(base: string, url: string) { 32 | const uri = new URL(url, base); 33 | return uri.toString(); 34 | } 35 | -------------------------------------------------------------------------------- /lib/node.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Element, 3 | GraphQLInterfaceType, 4 | type GraphQLInterfaceTypeConfig, 5 | } from "../deps.ts"; 6 | import { shared } from "./shared.ts"; 7 | 8 | export const TNode = new GraphQLInterfaceType({ 9 | name: "Node", 10 | description: "DOM Node implemented as either an Element or a Document.", 11 | fields: () => ({ ...shared }), 12 | } as GraphQLInterfaceTypeConfig); 13 | -------------------------------------------------------------------------------- /lib/query.ts: -------------------------------------------------------------------------------- 1 | import { graphql, PQueue } from "../deps.ts"; 2 | import { State } from "./state.ts"; 3 | import { schema } from "./schema.ts"; 4 | 5 | const defaults: QueryOptions = { 6 | concurrency: 8, 7 | fetch_options: { 8 | // ... 9 | }, 10 | variables: {}, 11 | operationName: undefined, 12 | }; 13 | 14 | export const useQuery = (query: string, options?: QueryOptions) => { 15 | const { 16 | concurrency, 17 | fetch_options, 18 | variables = {}, 19 | operationName, 20 | } = { ...defaults, ...options }; 21 | 22 | const state = new State(); 23 | const queue = new PQueue({ concurrency }); 24 | 25 | state.set("queue", queue); 26 | state.set("fetch_options", fetch_options); 27 | return graphql(schema, query, {}, { state }, variables, operationName); 28 | }; 29 | -------------------------------------------------------------------------------- /lib/schema.ts: -------------------------------------------------------------------------------- 1 | import { 2 | DOMParser, 3 | GraphQLObjectType, 4 | type GraphQLObjectTypeConfig, 5 | GraphQLSchema, 6 | GraphQLString, 7 | } from "../deps.ts"; 8 | import { TDocument } from "./document.ts"; 9 | 10 | export const schema = new GraphQLSchema({ 11 | query: new GraphQLObjectType({ 12 | name: "Query", 13 | fields: () => ({ 14 | page: { 15 | type: TDocument, 16 | args: { 17 | url: { 18 | type: GraphQLString, 19 | description: "A URL to fetch the HTML source from.", 20 | }, 21 | source: { 22 | type: GraphQLString, 23 | description: 24 | "A string containing HTML to be used as the source document.", 25 | }, 26 | }, 27 | async resolve(_, { url, source }: PageParams, context) { 28 | if (!url && !source) { 29 | throw new Error( 30 | "You need to provide either a URL or a HTML source string.", 31 | ); 32 | } 33 | 34 | if (url) { 35 | const options: RequestInit = context.state.get("fetch_options"); 36 | source = await (await fetch(url, options)).text(); 37 | context.state.set(["base", "url"], url); 38 | } else { 39 | context.state.set("base", ""); 40 | } 41 | const dom = new DOMParser().parseFromString(source!, "text/html")!; 42 | context.state.set("document", dom.documentElement); 43 | return dom.documentElement; 44 | }, 45 | }, 46 | }), 47 | } as GraphQLObjectTypeConfig, TContext>), 48 | }); 49 | -------------------------------------------------------------------------------- /lib/selector.ts: -------------------------------------------------------------------------------- 1 | import { GraphQLString } from "../deps.ts"; 2 | 3 | export const selector = { 4 | type: GraphQLString, 5 | description: "A valid CSS selector to query elements from the document DOM.", 6 | }; 7 | -------------------------------------------------------------------------------- /lib/server.ts: -------------------------------------------------------------------------------- 1 | import { serve } from "../deps.ts"; 2 | import { useQuery } from "./query.ts"; 3 | 4 | const graphqlPlayground = (endpoint = "http://localhost:8080") => 5 | `GraphQL Playground + DQL
Loading GraphQL Playground
6 | `; 7 | 8 | export function createServer(port: string | number = 8080, option?: IOptional) { 9 | const endpoint = (option?.endpoint) 10 | ? option.endpoint 11 | : `http://localhost:${port}`; 12 | 13 | const htmlResponseInit = { 14 | headers: { "content-type": "text/html;charset=utf-8" }, 15 | }; 16 | 17 | const jsonResponseInit = { 18 | headers: { "content-type": "application/json; charset=utf-8" }, 19 | }; 20 | 21 | async function handler(req: Request): Promise { 22 | switch (req.method) { 23 | case "GET": { 24 | return new Response(graphqlPlayground(endpoint), htmlResponseInit); 25 | } 26 | case "POST": { 27 | const { 28 | query, 29 | variables = {}, 30 | operationName = undefined, 31 | } = await req.json(); 32 | 33 | const response = await useQuery(query, { 34 | variables, 35 | operationName, 36 | }); 37 | 38 | return new Response( 39 | JSON.stringify(response, null, 2), 40 | jsonResponseInit, 41 | ); 42 | } 43 | default: 44 | return new Response("Invalid Method", { 45 | ...htmlResponseInit, 46 | status: 405, 47 | }); 48 | } 49 | } 50 | 51 | console.log(`🦕 DQL + GraphQL Server running on ${endpoint}`); 52 | serve(handler, { port: +port }); 53 | } 54 | -------------------------------------------------------------------------------- /lib/shared.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Element, 3 | GraphQLBoolean, 4 | type GraphQLFieldConfigMap, 5 | GraphQLInt, 6 | GraphQLList, 7 | GraphQLNonNull, 8 | GraphQLString, 9 | } from "../deps.ts"; 10 | import { selector } from "./selector.ts"; 11 | import { getAttributeOfElement } from "./helpers.ts"; 12 | import { TElement } from "./element.ts"; 13 | 14 | export const shared: GraphQLFieldConfigMap = { 15 | index: { 16 | type: GraphQLInt, 17 | description: "The node index number of the element (starting from 0).", 18 | args: { parent: selector }, 19 | resolve(element: Element, { parent }: IndexParams, context: TContext) { 20 | if (parent) { 21 | const document = context.state.get("document"); 22 | const nodes = Array.from(document.querySelectorAll(parent) ?? []); 23 | let index = -1; 24 | 25 | for (const node of nodes) { 26 | let elementParent = element.parentNode; 27 | while ( 28 | elementParent && node.compareDocumentPosition(elementParent) != 0 29 | ) { 30 | if (!elementParent) break; 31 | elementParent = elementParent.parentNode!; 32 | } 33 | if (!elementParent) continue; 34 | if (index != -1) return index; 35 | index = nodes.indexOf(elementParent); 36 | } 37 | return index; 38 | } 39 | 40 | const nodes = Array.from(element.parentElement?.childNodes ?? []); 41 | return nodes.indexOf(element) ?? -1; 42 | }, 43 | }, 44 | content: { 45 | type: GraphQLString, 46 | description: "The innerHTML content of the selected DOM node", 47 | args: { selector }, 48 | resolve(element, { selector }: ElementParams) { 49 | element = selector ? element.querySelector(selector)! : element; 50 | return element && element.innerHTML; 51 | }, 52 | }, 53 | html: { 54 | type: GraphQLString, 55 | description: "The outerHTML content of the selected DOM node", 56 | args: { selector }, 57 | resolve(element: Element, { selector }: ElementParams) { 58 | element = selector ? element.querySelector(selector)! : element; 59 | 60 | return element && element.outerHTML; 61 | }, 62 | }, 63 | text: { 64 | type: GraphQLString, 65 | description: "The text content of the selected DOM node", 66 | args: { 67 | selector, 68 | trim: { 69 | type: GraphQLBoolean, 70 | description: 71 | "Trim any leading and trailing whitespace from the value (optional, default: false)", 72 | defaultValue: false, 73 | }, 74 | }, 75 | resolve(element: Element, { selector, trim }: TextParams) { 76 | element = selector ? element.querySelector(selector)! : element; 77 | const result = element && element.textContent; 78 | return (trim) ? (result ?? "").trim() : result; 79 | }, 80 | }, 81 | table: { 82 | type: new GraphQLList(new GraphQLList(GraphQLString)), 83 | description: 84 | "Returns a two-dimensional array representing an HTML table element's contents. The first level is a list of rows (``), and each row is an array of cell (``) contents.", 85 | args: { 86 | selector, 87 | trim: { 88 | type: GraphQLBoolean, 89 | description: 90 | "Trim any leading and trailing whitespace from the values (optional, default: false)", 91 | defaultValue: false, 92 | }, 93 | }, 94 | resolve(element: Element, { selector, trim }: TextParams) { 95 | element = selector ? element.querySelector(selector)! : element; 96 | 97 | const result = element && Array.from( 98 | element.querySelectorAll("tr"), 99 | (row) => 100 | Array.from( 101 | (row as Element).querySelectorAll("td"), 102 | (td) => (trim ? td.textContent.trim() : td.textContent), 103 | ), 104 | ); 105 | 106 | return result.filter(Boolean).filter((row) => row.length > 0); 107 | }, 108 | }, 109 | tag: { 110 | type: GraphQLString, 111 | description: "The HTML tag name of the selected DOM node", 112 | args: { selector }, 113 | resolve(element: Element, { selector }: ElementParams) { 114 | element = selector ? element.querySelector(selector)! : element; 115 | return element?.tagName ?? null; 116 | }, 117 | }, 118 | attr: { 119 | type: GraphQLString, 120 | description: 121 | "The value of a given attribute from the selected node (`href`, `src`, etc.), if it exists.", 122 | args: { 123 | selector, 124 | name: { 125 | type: new GraphQLNonNull(GraphQLString), 126 | description: "The name of the attribute", 127 | }, 128 | trim: { 129 | type: GraphQLBoolean, 130 | description: 131 | "Trim any leading and trailing whitespace from the value (optional, default: false)", 132 | defaultValue: false, 133 | }, 134 | }, 135 | resolve(element: Element, { selector, name, trim }: TParams) { 136 | element = selector ? element.querySelector(selector)! : element; 137 | return getAttributeOfElement(element, name as string, trim); 138 | }, 139 | }, 140 | href: { 141 | type: GraphQLString, 142 | description: "Shorthand for `attr(name: 'href')`", 143 | args: { 144 | selector, 145 | trim: { 146 | type: GraphQLBoolean, 147 | description: 148 | "Trim any leading and trailing whitespace from the value (optional, default: false)", 149 | defaultValue: false, 150 | }, 151 | }, 152 | resolve(element: Element, { selector, trim }: TextParams) { 153 | element = selector ? element.querySelector(selector)! : element; 154 | return getAttributeOfElement(element, "href", trim); 155 | }, 156 | }, 157 | src: { 158 | type: GraphQLString, 159 | description: "Shorthand for `attr(name: 'src')`", 160 | args: { 161 | selector, 162 | trim: { 163 | type: GraphQLBoolean, 164 | description: 165 | "Trim any leading and trailing whitespace from the value (optional, default: false)", 166 | defaultValue: false, 167 | }, 168 | }, 169 | resolve(element: Element, { selector, trim }: TextParams) { 170 | element = selector ? element.querySelector(selector)! : element; 171 | if (element == null) return null; 172 | 173 | return getAttributeOfElement(element, "src", trim); 174 | }, 175 | }, 176 | class: { 177 | type: GraphQLString, 178 | description: 179 | "The class attribute of the selected node, if any exists. Formatted as a space-separated list of CSS class names.", 180 | args: { 181 | selector, 182 | trim: { 183 | type: GraphQLBoolean, 184 | description: 185 | "Trim any leading and trailing whitespace from the value (optional, default: false)", 186 | defaultValue: false, 187 | }, 188 | }, 189 | resolve(element: Element, { selector, trim }: TextParams) { 190 | element = selector ? element.querySelector(selector)! : element; 191 | if (element == null) return null; 192 | 193 | return getAttributeOfElement(element, "class", trim); 194 | }, 195 | }, 196 | classList: { 197 | type: new GraphQLList(GraphQLString), 198 | description: "An array of CSS classes extracted from the selected node.", 199 | args: { 200 | selector, 201 | }, 202 | resolve(element: Element, { selector }: ElementParams) { 203 | element = selector ? element.querySelector(selector)! : element; 204 | if (element == null) return null; 205 | return [...(element?.classList.values() ?? [])]; 206 | }, 207 | }, 208 | has: { 209 | type: GraphQLBoolean, 210 | description: 211 | "Returns true if an element with the given selector exists, otherwise false.", 212 | args: { selector }, 213 | resolve(element: Element, { selector }: ElementParams) { 214 | return !!element.querySelector(selector!); 215 | }, 216 | }, 217 | count: { 218 | type: GraphQLInt, 219 | description: 220 | "Returns the number of DOM nodes that match the given selector, or 0 if no nodes match.", 221 | args: { selector }, 222 | resolve(element: Element, { selector }: ElementParams) { 223 | if (!selector) return 0; 224 | 225 | return Array.from(element.querySelectorAll(selector)).length ?? 0; 226 | }, 227 | }, 228 | query: { 229 | type: TElement, 230 | description: 231 | "Equivalent to `Element.querySelector`. The selectors of any nested queries will be scoped to the resulting element.", 232 | args: { selector }, 233 | resolve(element: Element, { selector }: ElementParams) { 234 | return element.querySelector(selector!); 235 | }, 236 | }, 237 | queryAll: { 238 | type: new GraphQLList(TElement), 239 | description: 240 | "Equivalent to `Element.querySelectorAll`. The selectors of any nested queries will be scoped to the resulting elements.", 241 | args: { selector }, 242 | resolve(element: Element, { selector }: ElementParams) { 243 | return Array.from(element.querySelectorAll(selector!)); 244 | }, 245 | }, 246 | children: { 247 | type: new GraphQLList(TElement), 248 | description: "Children elements (not nodes) of the selected node.", 249 | resolve(element: Element) { 250 | return Array.from(element.children); 251 | }, 252 | }, 253 | childNodes: { 254 | type: new GraphQLList(TElement), 255 | description: 256 | "Child nodes (not elements) of a selected node, including any text nodes.", 257 | resolve(element: Element) { 258 | return Array.from(element.childNodes); 259 | }, 260 | }, 261 | parent: { 262 | type: TElement, 263 | description: "Parent Element of the selected node.", 264 | resolve(element: Element) { 265 | return element.parentElement; 266 | }, 267 | }, 268 | siblings: { 269 | type: new GraphQLList(TElement), 270 | description: 271 | "All elements at the same level in the tree as the current element, as well as the element itself. Equivalent to `Element.parentElement.children`.", 272 | resolve(element: Element) { 273 | const parent = element.parentElement; 274 | if (parent == null) return [element]; 275 | 276 | return Array.from(parent.children); 277 | }, 278 | }, 279 | next: { 280 | type: TElement, 281 | description: 282 | "Current element's next sibling, including any text nodes. Equivalent to `Node.nextSibling`.", 283 | resolve(element: Element) { 284 | return element.nextSibling; 285 | }, 286 | }, 287 | nextAll: { 288 | type: new GraphQLList(TElement), 289 | description: "All of the current element's next siblings", 290 | resolve(element: Element) { 291 | const siblings = []; 292 | for ( 293 | let next = element.nextSibling; 294 | next != null; 295 | next = next.nextSibling 296 | ) { 297 | siblings.push(next); 298 | } 299 | return siblings; 300 | }, 301 | }, 302 | previous: { 303 | type: TElement, 304 | description: 305 | "Current Element's previous sibling, including any text nodes. Equivalent to `Node.previousSibling`.", 306 | resolve(element: Element) { 307 | return element.previousSibling; 308 | }, 309 | }, 310 | previousAll: { 311 | type: new GraphQLList(TElement), 312 | description: "All of the current element's previous siblings", 313 | resolve(element: Element) { 314 | const siblings = []; 315 | for ( 316 | let previous = element.previousSibling; 317 | previous != null; 318 | previous = previous.previousSibling 319 | ) { 320 | siblings.push(previous); 321 | } 322 | siblings.reverse(); 323 | return siblings; 324 | }, 325 | }, 326 | }; 327 | -------------------------------------------------------------------------------- /lib/state.ts: -------------------------------------------------------------------------------- 1 | export type StateInit = 2 | | Iterable<[string, T]> 3 | | Map 4 | | [string, T][] 5 | | Record; 6 | 7 | export class State { 8 | private static $ = new Map(); 9 | private readonly $: Map; 10 | private $deleted: Set; 11 | 12 | /** 13 | * Creates a new State instance, with optional initial value. 14 | * 15 | * @param initial (optional) Initial state 16 | * @returns a new State instance with optional initial value. 17 | */ 18 | constructor(initial: StateInit = State.$) { 19 | if (initial && State.isObject(initial)) { 20 | initial = Object.entries(initial); 21 | } 22 | initial ??= []; 23 | this.$ = new Map(initial as Iterable<[string, T]>); 24 | this.$deleted = new Set(); 25 | return this; 26 | } 27 | 28 | /** 29 | * Converts state to an object that can be understood by `JSON.stringify`. 30 | * 31 | * @returns an object representing the state. 32 | * @example const state = new State([['a', 1], ['b', 2]]); 33 | * state.toJSON(); // { a: 1, b: 2 } 34 | * JSON.stringify(state); // { "a": 1, "b": 2 } 35 | */ 36 | toJSON(): Record { 37 | return Object.fromEntries(this.$.entries()); 38 | } 39 | 40 | /** 41 | * When calling `Object.prototype.toString` on a State instance, it returns 42 | * `[object State]` rather than the generic default `[object Object]`. 43 | * 44 | * @returns the string literal "State" 45 | */ 46 | public get [Symbol.toStringTag](): "State" { 47 | return "State"; 48 | } 49 | 50 | /** 51 | * Test if a given value is a plain object, and not a function, array, etc. 52 | * 53 | * @param obj The object to test. 54 | * @returns `true` if `obj` is an object literal, `false` otherwise. 55 | */ 56 | public static isObject>( 57 | obj: unknown, 58 | ): obj is O { 59 | if (Array.isArray(obj) || typeof obj === "function") return false; 60 | 61 | return (Object.prototype.toString.call(obj) === "[object Object]"); 62 | } 63 | 64 | /** 65 | * Add or update a value in a State instance by its associated key. 66 | * 67 | * @param key The key (`string`) to add or update 68 | * @param value The value to set for the given key 69 | * @returns `State` instance, for optional chaining 70 | * 71 | * @example state.set('a', 1); // { a: 1 } 72 | * @example state.set(['a', 'b'], 1); // { a: 1, b: 1 } 73 | * @example state.set({ b: 2, c: 3 }).set('a', 1); // { a: 1, b: 2, c: 3 } 74 | */ 75 | public set(key: string, value: V): State; 76 | 77 | /** 78 | * Add or update multiple keys to a specified value. 79 | * 80 | * @param key List of keys (`string[]`) to add or update 81 | * @param value The value to set for the given keys 82 | * @returns `State` instance, for optional chaining 83 | * @example state.set(['a', 'b'], 1); 84 | * // { a: 1, b: 1 } 85 | * @example state.set({ a: 1, b: 2 }); 86 | * // { a: 1, b: 2 } 87 | */ 88 | public set(key: string[], value: V): State; 89 | 90 | /** 91 | * Add or update the values for multiple keys using an object of type `Record`. 92 | * @param key An object literal with the format `{ key: value }` 93 | * @returns `State` instance, for optional chaining 94 | * @example state.set({ b: 2, c: 3 }).set('a', 1); // { a: 1, b: 2, c: 3 } 95 | */ 96 | public set(key: Record): State; 97 | public set(key: any, value?: V): State { 98 | if (State.isObject>(key)) { 99 | for (const k in key) { 100 | this.$.set(k, key[k]); 101 | } 102 | } else if (value != null) { 103 | if (typeof key === "string") { 104 | this.$.set(key, value); 105 | } else { 106 | [key].flat().forEach((k) => this.$.set(k, value)); 107 | } 108 | } 109 | return this; 110 | } 111 | 112 | /** 113 | * Returns the value associated with a key in a State instance. 114 | * @param key the key to lookup 115 | * @returns the value associated with the key (or `undefined`). 116 | */ 117 | public get(key: string, defaultValue?: V): V; 118 | 119 | /** 120 | * Returns the values in a State instance for a list of keys. 121 | * @param key array of keys to lookup 122 | * @returns An array of the values for the given keys (or `undefined`). 123 | */ 124 | public get(key: string[], defaultValue?: V): V[]; 125 | public get(key: string | string[], defaultValue?: V): V | V[] { 126 | defaultValue ??= undefined; 127 | 128 | if (Array.isArray(key)) { 129 | return key.map((k) => (this.$.get(k) ?? defaultValue)) as V[]; 130 | } 131 | return (this.$.get(key) ?? defaultValue) as V; 132 | } 133 | 134 | /** 135 | * Returns `true` if the State instance contains a value for the given key. 136 | * @param key - the key to lookup 137 | * @returns `true` if the key exists in State, `false` otherwise. 138 | * @example const state = new State([['a', 1]]); 139 | * state.has('a'); // true 140 | * state.has('c'); // false 141 | */ 142 | public has(key: string): boolean; 143 | 144 | /** 145 | * Returns an object of `{ key: true | false }` format. Each property 146 | * corresponds to a value from the provided list of keys, with the value 147 | * being `true` if it exists in the State instance, or `false` if it does not. 148 | * @param key - the keys to lookup 149 | * @returns `Record` 150 | * 151 | * @example const state = new State([['a', 1], ['b', 2]]); 152 | * state.has(['a', 'b', 'c']) // { a: true, b: true, c: false } 153 | */ 154 | public has(key: string[]): Record; 155 | public has(key: string | string[]): boolean | Record { 156 | if (Array.isArray(key)) { 157 | return key.reduce((keys, k) => ({ 158 | ...(keys ?? {}), 159 | [k]: this.has(k), 160 | }), {} as Record); 161 | } 162 | return this.$.has(key); 163 | } 164 | 165 | /** 166 | * Delete a value from a State instance by its associated key. 167 | * 168 | * @param key - the key to delete 169 | * @returns the `State` instance, for optional chaining 170 | * 171 | * @example const state = new State([['a', 1], ['b', 2], ['c', 3]]); 172 | * state.delete('a'); // { b: 2, c: 3 } 173 | * state.delete(['b', 'c']); // {} 174 | */ 175 | public delete(key: string): State; 176 | 177 | /** 178 | * Delete multiple values from a State instance. 179 | * @param key - the keys to delete 180 | * @returns the `State` instance, for optional chaining 181 | * 182 | * @example const state = new State([['a', 1], ['b', 2], ['c', 3]]); 183 | * state.delete(['a', 'b', 'c']); // {} 184 | */ 185 | public delete(key: string[]): State; 186 | public delete(key: string | string[]): State { 187 | if (Array.isArray(key)) { 188 | key.forEach((k) => this.delete(k)); 189 | } else { 190 | if (this.has(key)) { 191 | this.$deleted.add(this.get(key)); 192 | this.$.delete(key); 193 | } 194 | } 195 | return this; 196 | } 197 | 198 | /** 199 | * Clears all values from the State instance. 200 | * @returns the `State` instance, for optional chaining. 201 | * @example const state = new State([['a', 1], ['b', 2], ['c', 3]]); 202 | * state.clear(); // {} 203 | */ 204 | public clear(): State { 205 | this.$deleted = new Set([...this.$deleted, ...this.$.values()]); 206 | this.$.clear(); 207 | return this; 208 | } 209 | 210 | /** 211 | * Similar to `Array.prototype.map`, this accepts a callback function and 212 | * applies it to each value, returning an array of the collated results. 213 | * @param callbackfn - called for each value, 214 | * with any returned value then added to the final array. 215 | * @param [thisArg] - optional value to use as `this` when calling 216 | * `callbackfn`, defaults to the State instance. 217 | * @returns an array of the returned `callbackfn` values. 218 | * @example const state = new State([['a', 1], ['b', 2]]); 219 | * state.map((value, key) => value + key); // ['a1', 'b2'] 220 | */ 221 | public map( 222 | callbackfn: (value: T, key: string, map: State) => V, 223 | thisArg?: any, 224 | ): V[] { 225 | return [...this.entries].map(([key, value]) => 226 | callbackfn.apply(thisArg, [value, key, this]) 227 | ); 228 | } 229 | 230 | /** 231 | * Executes a given function for each value in the State instance. 232 | * @param callbackfn 233 | * @param [thisArg] 234 | * @returns the state instance for optional chaining. 235 | */ 236 | public forEach( 237 | callbackfn: (value: T, key: string, map: State) => void, 238 | thisArg?: any, 239 | ): State { 240 | this.$.forEach( 241 | (v, k) => callbackfn.apply(thisArg, [v, k, this]), 242 | thisArg ?? this.$, 243 | ); 244 | return this; 245 | } 246 | 247 | /** 248 | * @returns An `IterableIterator` of the keys of a State instance. 249 | */ 250 | public get keys(): IterableIterator { 251 | return this.$.keys(); 252 | } 253 | 254 | /** 255 | * @returns An `IterableIterator` of the values of a State instance. 256 | */ 257 | public get values(): IterableIterator { 258 | return this.$.values(); 259 | } 260 | 261 | /** 262 | * @returns An `IterableIterator` of the entries (key-value pairs) of a State instance. 263 | */ 264 | public get entries(): IterableIterator<[string, T]> { 265 | return this.$.entries(); 266 | } 267 | 268 | /** 269 | * @returns An `IterableIterator` of the deleted values of a State instance. 270 | */ 271 | public get deleted(): IterableIterator { 272 | return this.$deleted.values(); 273 | } 274 | 275 | /** 276 | * @returns The number of values currently in the State instance. 277 | */ 278 | public get length(): number { 279 | return +this.$.size; 280 | } 281 | } 282 | 283 | export default State; 284 | -------------------------------------------------------------------------------- /lib/types.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | /// 3 | /// 4 | /// 5 | /// 6 | 7 | type StateInit = 8 | | Iterable<[string, T]> 9 | | Map 10 | | [string, T][] 11 | | Record; 12 | 13 | interface ElementParams { 14 | selector?: string; 15 | } 16 | 17 | interface PageParams { 18 | url?: string; 19 | source?: string; 20 | } 21 | 22 | interface TextParams extends ElementParams { 23 | trim?: boolean; 24 | } 25 | 26 | interface AttrParams { 27 | name?: string; 28 | } 29 | 30 | interface IndexParams { 31 | parent?: string; 32 | } 33 | 34 | interface AllParams 35 | extends PageParams, IndexParams, ElementParams, TextParams, AttrParams { 36 | attr?: string; 37 | } 38 | 39 | type TParams = Partial; 40 | 41 | interface TContext { 42 | state: import("./state.ts").State; 43 | } 44 | 45 | type Variables = { [key: string]: any }; 46 | 47 | interface QueryOptions { 48 | concurrency?: number; 49 | fetch_options?: RequestInit; 50 | variables?: Variables; 51 | operationName?: string; 52 | } 53 | 54 | interface IOptional { 55 | endpoint?: string; 56 | } 57 | -------------------------------------------------------------------------------- /mod.ts: -------------------------------------------------------------------------------- 1 | import { State } from "./lib/state.ts"; 2 | 3 | export { createServer } from "./lib/server.ts"; 4 | export { useQuery } from "./lib/query.ts"; 5 | 6 | export { State }; 7 | 8 | export declare type StateInit = 9 | | Iterable<[string, T]> 10 | | Map 11 | | [string, T][] 12 | | Record; 13 | 14 | export declare interface TContext { 15 | state: State; 16 | } 17 | 18 | export declare interface ElementParams { 19 | selector?: string; 20 | } 21 | 22 | export declare interface PageParams { 23 | url?: string; 24 | source?: string; 25 | } 26 | 27 | export declare interface TextParams extends ElementParams { 28 | trim?: boolean; 29 | } 30 | 31 | export declare interface AttrParams { 32 | name?: string; 33 | trim?: boolean; 34 | } 35 | 36 | export declare interface IndexParams { 37 | parent?: string; 38 | } 39 | 40 | export declare interface AllParams 41 | extends PageParams, IndexParams, ElementParams, TextParams, AttrParams { 42 | attr?: string; 43 | } 44 | 45 | export declare type TParams = Partial; 46 | 47 | export declare type Variables = { [key: string]: any }; 48 | 49 | export declare interface QueryOptions { 50 | concurrency?: number; 51 | fetch_options?: RequestInit; 52 | variables?: Variables; 53 | operationName?: string; 54 | } 55 | 56 | export declare interface IOptional { 57 | endpoint?: string; 58 | } 59 | -------------------------------------------------------------------------------- /serve.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env deno run --allow-net --unstable 2 | import { createServer } from "./mod.ts"; 3 | import { parse } from "https://deno.land/std@0.145.0/flags/mod.ts"; 4 | 5 | const { port = 8080 } = parse(Deno.args) ?? {}; 6 | createServer(port); 7 | -------------------------------------------------------------------------------- /tests/deps.ts: -------------------------------------------------------------------------------- 1 | export { 2 | assertEquals, 3 | assertNotEquals, 4 | } from "https://deno.land/std@0.145.0/testing/asserts.ts"; 5 | export { resolveURL } from "../lib/helpers.ts"; 6 | export { State } from "../lib/state.ts"; 7 | export { useQuery } from "../mod.ts"; 8 | -------------------------------------------------------------------------------- /tests/schema.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals, assertNotEquals, useQuery } from "./deps.ts"; 2 | 3 | const DEFAULT_TEST_OPTIONS = { 4 | sanitizeOps: false, 5 | sanitizeResources: false, 6 | }; 7 | 8 | Deno.test({ 9 | name: "#1: no args throws errors", 10 | fn: async () => { 11 | const query = `{ page { title } }`; 12 | 13 | const response = await useQuery(query); 14 | 15 | assertEquals( 16 | response && response.errors && response.errors[0].message, 17 | "You need to provide either a URL or a HTML source string.", 18 | ); 19 | }, 20 | }); 21 | 22 | Deno.test({ 23 | name: "#2: title", 24 | fn: async () => { 25 | const html = 26 | `some title`; 27 | const query = `{ page(source: "${html}") { title } }`; 28 | const response = await useQuery(query); 29 | 30 | assertEquals("error" in response, false); 31 | assertEquals(response.data && response.data.page.title, "some title"); 32 | }, 33 | ...DEFAULT_TEST_OPTIONS, 34 | }); 35 | 36 | Deno.test({ 37 | name: "#3: fetch from URL", 38 | fn: async () => { 39 | const query = `{ 40 | page(url: "https://nyancodeid.github.io/tests/test-3-via-url.html") { 41 | title 42 | } 43 | }`; 44 | const response = await useQuery(query); 45 | 46 | assertEquals("error" in response, false); 47 | // assertEquals(response.data && response.data.page.title, "some title"); 48 | }, 49 | ...DEFAULT_TEST_OPTIONS, 50 | }); 51 | 52 | Deno.test({ 53 | name: "#4: content", 54 | fn: async () => { 55 | const html = 56 | `some titlesome body`; 57 | const query = `{ page(source: "${html}") { content } }`; 58 | 59 | const response = await useQuery(query); 60 | 61 | assertEquals("error" in response, false); 62 | assertEquals( 63 | response.data && response.data.page.content, 64 | "some titlesome body", 65 | ); 66 | }, 67 | ...DEFAULT_TEST_OPTIONS, 68 | }); 69 | 70 | Deno.test({ 71 | name: "#5: content with selector", 72 | fn: async () => { 73 | const html = 74 | `some title
bad
`; 75 | const query = `{ 76 | page(source: "${html}") { 77 | content(selector: ".selectme") 78 | } 79 | }`; 80 | 81 | const response = await useQuery(query); 82 | 83 | assertEquals("error" in response, false); 84 | assertEquals( 85 | response.data && response.data.page.content, 86 | "bad", 87 | ); 88 | }, 89 | ...DEFAULT_TEST_OPTIONS, 90 | }); 91 | 92 | Deno.test({ 93 | name: "#6: HTML content", 94 | fn: async () => { 95 | const html = 96 | `some titlesome body`; 97 | const query = `{ page(source: "${html}") { html } }`; 98 | 99 | const response = await useQuery(query); 100 | 101 | assertEquals("error" in response, false); 102 | assertEquals( 103 | response.data && response.data.page.html, 104 | html, 105 | ); 106 | }, 107 | ...DEFAULT_TEST_OPTIONS, 108 | }); 109 | 110 | Deno.test({ 111 | name: "#7: HTML content with selector", 112 | fn: async () => { 113 | const html = 114 | `some title
bad
`; 115 | const query = `{ 116 | page(source: "${html}") { 117 | html(selector: ".selectme") 118 | } 119 | }`; 120 | 121 | const response = await useQuery(query); 122 | 123 | assertEquals("error" in response, false); 124 | assertEquals( 125 | response.data && response.data.page.html, 126 | '
bad
', 127 | ); 128 | }, 129 | ...DEFAULT_TEST_OPTIONS, 130 | }); 131 | 132 | Deno.test({ 133 | name: "#8: text", 134 | fn: async () => { 135 | const html = 136 | `some title
bad
`; 137 | const query = `{ 138 | page(source: "${html}") { 139 | text 140 | } 141 | }`; 142 | 143 | const response = await useQuery(query); 144 | 145 | assertEquals("error" in response, false); 146 | assertEquals( 147 | response.data && response.data.page.text, 148 | "some titlebad", 149 | ); 150 | }, 151 | ...DEFAULT_TEST_OPTIONS, 152 | }); 153 | 154 | Deno.test({ 155 | name: "#9: text with selector", 156 | fn: async () => { 157 | const html = 158 | `some title
bad
`; 159 | const query = `{ 160 | page(source: "${html}") { 161 | text(selector: ".selectme") 162 | } 163 | }`; 164 | 165 | const response = await useQuery(query); 166 | 167 | assertEquals("error" in response, false); 168 | assertEquals( 169 | response.data && response.data.page.text, 170 | "bad", 171 | ); 172 | }, 173 | ...DEFAULT_TEST_OPTIONS, 174 | }); 175 | 176 | Deno.test({ 177 | name: "#10: tag", 178 | fn: async () => { 179 | const html = 180 | `some title
bad
`; 181 | const query = `{ 182 | page(source: "${html}") { 183 | tag 184 | } 185 | }`; 186 | 187 | const response = await useQuery(query); 188 | 189 | assertEquals("error" in response, false); 190 | assertEquals( 191 | response.data && response.data.page.tag, 192 | "HTML", 193 | ); 194 | }, 195 | ...DEFAULT_TEST_OPTIONS, 196 | }); 197 | 198 | Deno.test({ 199 | name: "#11: tag with selector", 200 | fn: async () => { 201 | const html = 202 | `some title
bad
`; 203 | const query = `{ 204 | page(source: "${html}") { 205 | tag(selector: ".selectme") 206 | } 207 | }`; 208 | 209 | const response = await useQuery(query); 210 | 211 | assertEquals("error" in response, false); 212 | assertEquals( 213 | response.data && response.data.page.tag, 214 | "DIV", 215 | ); 216 | }, 217 | ...DEFAULT_TEST_OPTIONS, 218 | }); 219 | 220 | Deno.test({ 221 | name: "#12: attr", 222 | fn: async () => { 223 | const html = 224 | `some title
bad
`; 225 | const query = `{ 226 | page(source: "${html}") { 227 | attr(name: "style") 228 | } 229 | }`; 230 | 231 | const response = await useQuery(query); 232 | 233 | assertEquals("error" in response, false); 234 | assertEquals( 235 | response.data && response.data.page.attr, 236 | "background: red;", 237 | ); 238 | }, 239 | ...DEFAULT_TEST_OPTIONS, 240 | }); 241 | 242 | Deno.test({ 243 | name: "#13: non existing attribute", 244 | fn: async () => { 245 | const html = 246 | `some title
bad
`; 247 | const query = `{ 248 | page(source: "${html}") { 249 | attr(name: "asdf") 250 | } 251 | }`; 252 | 253 | const response = await useQuery(query); 254 | 255 | assertEquals("error" in response, false); 256 | assertEquals( 257 | response.data && response.data.page.attr, 258 | null, 259 | ); 260 | }, 261 | ...DEFAULT_TEST_OPTIONS, 262 | }); 263 | 264 | Deno.test({ 265 | name: "#14: attribute with selector", 266 | fn: async () => { 267 | const html = 268 | `some title
bad
`; 269 | const query = `{ 270 | page(source: "${html}") { 271 | attr(selector: ".selectme", name: "class") 272 | } 273 | }`; 274 | 275 | const response = await useQuery(query); 276 | 277 | assertEquals("error" in response, false); 278 | assertEquals( 279 | response.data && response.data.page.attr, 280 | "selectme", 281 | ); 282 | }, 283 | ...DEFAULT_TEST_OPTIONS, 284 | }); 285 | 286 | Deno.test({ 287 | name: "#15: has", 288 | fn: async () => { 289 | const html = 290 | `some title
one
two
`; 291 | const query = `{ 292 | page(source: "${html}") { 293 | firstDiv: query(selector: "div") { 294 | isStrong: has(selector: "strong") 295 | } 296 | } 297 | }`; 298 | 299 | const response = await useQuery(query); 300 | 301 | assertEquals("error" in response, false); 302 | assertEquals( 303 | response.data && response.data.page.firstDiv.isStrong, 304 | true, 305 | ); 306 | }, 307 | ...DEFAULT_TEST_OPTIONS, 308 | }); 309 | 310 | Deno.test({ 311 | name: "#16: has not", 312 | fn: async () => { 313 | const html = 314 | `some title
one
two
`; 315 | const query = `{ 316 | page(source: "${html}") { 317 | firstDiv: query(selector: "div") { 318 | isWeak: has(selector: "weak") 319 | } 320 | } 321 | }`; 322 | 323 | const response = await useQuery(query); 324 | 325 | assertEquals("error" in response, false); 326 | assertEquals( 327 | response.data && !response.data.page.firstDiv.isWeak, 328 | true, 329 | ); 330 | }, 331 | ...DEFAULT_TEST_OPTIONS, 332 | }); 333 | 334 | Deno.test({ 335 | name: "#17: query", 336 | fn: async () => { 337 | const html = 338 | `some title
one
two
`; 339 | const query = `{ 340 | page(source: "${html}") { 341 | firstDiv: query(selector: "div") { 342 | text 343 | } 344 | } 345 | }`; 346 | 347 | const response = await useQuery(query); 348 | 349 | assertEquals("error" in response, false); 350 | assertEquals( 351 | response.data && response.data.page.firstDiv, 352 | { text: "one" }, 353 | ); 354 | }, 355 | ...DEFAULT_TEST_OPTIONS, 356 | }); 357 | 358 | Deno.test({ 359 | name: "#18: queryAll", 360 | fn: async () => { 361 | const html = 362 | `some title
one
two
`; 363 | const query = `{ 364 | page(source: "${html}") { 365 | divs: queryAll(selector: "div") { 366 | text 367 | } 368 | } 369 | }`; 370 | 371 | const response = await useQuery(query); 372 | 373 | assertEquals("error" in response, false); 374 | assertEquals( 375 | response.data && response.data.page.divs, 376 | [ 377 | { text: "one" }, 378 | { text: "two" }, 379 | ], 380 | ); 381 | }, 382 | ...DEFAULT_TEST_OPTIONS, 383 | }); 384 | 385 | Deno.test({ 386 | name: "#19: children", 387 | fn: async () => { 388 | const html = 389 | `some title
onetwo
twothree
`; 390 | const query = `{ 391 | page(source: "${html}") { 392 | kids: queryAll(selector: "div") { 393 | children { 394 | text 395 | } 396 | } 397 | } 398 | }`; 399 | 400 | const response = await useQuery(query); 401 | 402 | assertEquals("error" in response, false); 403 | assertEquals( 404 | response.data && response.data.page.kids, 405 | [ 406 | { 407 | children: [{ text: "one" }, { text: "two" }], 408 | }, 409 | { 410 | children: [{ text: "two" }, { text: "three" }], 411 | }, 412 | ], 413 | ); 414 | }, 415 | ...DEFAULT_TEST_OPTIONS, 416 | }); 417 | 418 | Deno.test({ 419 | name: "#20: childNodes", 420 | fn: async () => { 421 | const html = 422 | `some title
onetwo
twoamazingthree
`; 423 | const query = `{ 424 | page(source: "${html}") { 425 | kids: queryAll(selector: "div") { 426 | childNodes { 427 | text 428 | } 429 | } 430 | } 431 | }`; 432 | 433 | const response = await useQuery(query); 434 | 435 | assertEquals("error" in response, false); 436 | assertEquals( 437 | response.data && response.data.page.kids, 438 | [ 439 | { 440 | childNodes: [{ text: "one" }, { text: "two" }], 441 | }, 442 | { 443 | childNodes: [{ text: "two" }, { text: "amazing" }, { text: "three" }], 444 | }, 445 | ], 446 | ); 447 | }, 448 | ...DEFAULT_TEST_OPTIONS, 449 | }); 450 | 451 | Deno.test({ 452 | name: "#21: parent", 453 | fn: async () => { 454 | const html = 455 | `some title
bad
`; 456 | const query = `{ 457 | page(source: "${html}") { 458 | query(selector: "strong") { 459 | parent { 460 | attr(name: "class") 461 | } 462 | } 463 | } 464 | }`; 465 | 466 | const response = await useQuery(query); 467 | 468 | assertEquals("error" in response, false); 469 | assertEquals( 470 | response.data && response.data.page.query.parent.attr, 471 | "selectme", 472 | ); 473 | }, 474 | ...DEFAULT_TEST_OPTIONS, 475 | }); 476 | 477 | Deno.test({ 478 | name: "#22: siblings", 479 | fn: async () => { 480 | const html = 481 | `some title
bad

boom

bap
`; 482 | const query = `{ 483 | page(source: "${html}") { 484 | query(selector: "strong") { 485 | siblings { 486 | text 487 | } 488 | } 489 | } 490 | }`; 491 | 492 | const response = await useQuery(query); 493 | 494 | assertEquals("error" in response, false); 495 | assertEquals( 496 | response.data && response.data.page.query.siblings, 497 | [ 498 | { text: "bad" }, 499 | { text: "boom" }, 500 | { text: "bap" }, 501 | ], 502 | ); 503 | }, 504 | ...DEFAULT_TEST_OPTIONS, 505 | }); 506 | 507 | Deno.test({ 508 | name: "#23: siblings of root is only html", 509 | fn: async () => { 510 | const html = 511 | `nothing to see here`; 512 | const query = `{ 513 | page(source: "${html}") { 514 | siblings { 515 | tag 516 | } 517 | } 518 | }`; 519 | 520 | const response = await useQuery(query); 521 | 522 | assertEquals("error" in response, false); 523 | assertEquals( 524 | response.data && response.data.page.siblings, 525 | [{ tag: "HTML" }], 526 | ); 527 | }, 528 | ...DEFAULT_TEST_OPTIONS, 529 | }); 530 | 531 | Deno.test({ 532 | name: "#24: next", 533 | fn: async () => { 534 | const html = 535 | `some title
bad

boom

bap
`; 536 | const query = `{ 537 | page(source: "${html}") { 538 | query(selector: "strong") { 539 | next { 540 | text 541 | } 542 | } 543 | } 544 | }`; 545 | 546 | const response = await useQuery(query); 547 | 548 | assertEquals("error" in response, false); 549 | assertEquals( 550 | response.data && response.data.page.query.next.text, 551 | "boom", 552 | ); 553 | }, 554 | ...DEFAULT_TEST_OPTIONS, 555 | }); 556 | 557 | Deno.test({ 558 | name: "#25: next - bare text", 559 | fn: async () => { 560 | const html = 561 | `some title
badbare textbap
`; 562 | const query = `{ 563 | page(source: "${html}") { 564 | query(selector: "strong") { 565 | next { 566 | tag 567 | text 568 | } 569 | } 570 | } 571 | }`; 572 | 573 | const response = await useQuery(query); 574 | 575 | assertEquals("error" in response, false); 576 | assertEquals( 577 | response.data && response.data.page.query.next.tag, 578 | null, 579 | ); 580 | assertEquals( 581 | response.data && response.data.page.query.next.text, 582 | "bare text", 583 | ); 584 | }, 585 | ...DEFAULT_TEST_OPTIONS, 586 | }); 587 | 588 | Deno.test({ 589 | name: "#26: nextAll", 590 | fn: async () => { 591 | const html = 592 | `some title
badbare textbap
`; 593 | const query = `{ 594 | page(source: "${html}") { 595 | query(selector: "strong") { 596 | nextAll { 597 | tag 598 | text 599 | } 600 | } 601 | } 602 | }`; 603 | 604 | const response = await useQuery(query); 605 | 606 | assertEquals("error" in response, false); 607 | assertEquals( 608 | response.data && response.data.page.query.nextAll, 609 | [ 610 | { tag: null, text: "bare text" }, 611 | { tag: "SPAN", text: "bap" }, 612 | ], 613 | ); 614 | }, 615 | ...DEFAULT_TEST_OPTIONS, 616 | }); 617 | 618 | Deno.test({ 619 | name: "#27: previous", 620 | fn: async () => { 621 | const html = 622 | `some title
bad

boom

bap
`; 623 | const query = `{ 624 | page(source: "${html}") { 625 | query(selector: "span") { 626 | previous { 627 | text 628 | } 629 | } 630 | } 631 | }`; 632 | 633 | const response = await useQuery(query); 634 | 635 | assertEquals("error" in response, false); 636 | assertEquals( 637 | response.data && response.data.page.query.previous.text, 638 | "boom", 639 | ); 640 | }, 641 | ...DEFAULT_TEST_OPTIONS, 642 | }); 643 | 644 | Deno.test({ 645 | name: "#28: previousAll", 646 | fn: async () => { 647 | const html = 648 | `some title
badbare textbap
`; 649 | const query = `{ 650 | page(source: "${html}") { 651 | query(selector: "span") { 652 | previousAll { 653 | tag 654 | text 655 | } 656 | } 657 | } 658 | }`; 659 | 660 | const response = await useQuery(query); 661 | 662 | assertEquals("error" in response, false); 663 | assertEquals( 664 | response.data && response.data.page.query.previousAll, 665 | [ 666 | { tag: "STRONG", text: "bad" }, 667 | { tag: null, text: "bare text" }, 668 | ], 669 | ); 670 | }, 671 | ...DEFAULT_TEST_OPTIONS, 672 | }); 673 | 674 | Deno.test({ 675 | name: "#29: previousAll", 676 | fn: async () => { 677 | const html = 678 | `some title
badbare textbap
`; 679 | const query = `{ 680 | page(source: "${html}") { 681 | query(selector: "span") { 682 | previousAll { 683 | tag 684 | text 685 | } 686 | } 687 | } 688 | }`; 689 | 690 | const response = await useQuery(query); 691 | 692 | assertEquals("error" in response, false); 693 | assertEquals( 694 | response.data && response.data.page.query.previousAll, 695 | [ 696 | { tag: "STRONG", text: "bad" }, 697 | { tag: null, text: "bare text" }, 698 | ], 699 | ); 700 | }, 701 | ...DEFAULT_TEST_OPTIONS, 702 | }); 703 | 704 | Deno.test({ 705 | name: "#30: not existing selector", 706 | fn: async () => { 707 | const html = 708 | `some title
bad
`; 709 | const query = `{ 710 | page(source: "${html}") { 711 | content(selector: ".selectmenot") 712 | } 713 | }`; 714 | 715 | const response = await useQuery(query); 716 | 717 | assertEquals("error" in response, false); 718 | assertEquals( 719 | response.data && response.data.page.content, 720 | null, 721 | ); 722 | }, 723 | ...DEFAULT_TEST_OPTIONS, 724 | }); 725 | 726 | // Deno.test({ 727 | // name: "#30: visit", 728 | // fn: async () => { 729 | // const query = `{ 730 | // page(url: "https://nyancodeid.github.io/tests/test-3-via-url.html") { 731 | // link: query(selector: "a") { 732 | // visit { 733 | // text(selector: "strong") 734 | // } 735 | // } 736 | // } 737 | // }`; 738 | 739 | // const response = await useQuery(query); 740 | 741 | // assertEquals("error" in response, false); 742 | // assertEquals( 743 | // response.data && response.data.page.link.visit.text, 744 | // "we managed to visit the link!", 745 | // ); 746 | // }, 747 | // ...DEFAULT_TEST_OPTIONS, 748 | // }); 749 | 750 | Deno.test({ 751 | name: "#31: count", 752 | fn: async () => { 753 | const html = 754 | `some title
  • Item 1
  • Item 2
`; 755 | const query = `{ 756 | page(source: "${html}") { 757 | query(selector: "ul") { 758 | count(selector: "li") 759 | } 760 | } 761 | }`; 762 | 763 | const response = await useQuery(query); 764 | 765 | assertEquals("error" in response, false); 766 | assertEquals( 767 | response.data && response.data.page.query.count, 768 | 2, 769 | ); 770 | }, 771 | ...DEFAULT_TEST_OPTIONS, 772 | }); 773 | 774 | Deno.test({ 775 | name: "#32: href", 776 | fn: async () => { 777 | const html = 778 | `some titleItem 1`; 779 | const query = `{ 780 | page(source: "${html}") { 781 | query(selector: "a") { 782 | href 783 | } 784 | link: href(selector: "a") 785 | } 786 | }`; 787 | 788 | const response = await useQuery(query); 789 | 790 | assertEquals("error" in response, false); 791 | assertEquals( 792 | response.data && response.data.page.query.href, 793 | "https://nyan.web.id", 794 | ); 795 | assertEquals( 796 | response.data && response.data.page.link, 797 | "https://nyan.web.id", 798 | ); 799 | }, 800 | ...DEFAULT_TEST_OPTIONS, 801 | }); 802 | 803 | Deno.test({ 804 | name: "#33: src", 805 | fn: async () => { 806 | const html = 807 | `some title`; 808 | const query = `{ 809 | page(source: "${html}") { 810 | query(selector: "img") { 811 | src 812 | } 813 | image: src(selector: "img") 814 | } 815 | }`; 816 | 817 | const response = await useQuery(query); 818 | 819 | assertEquals("error" in response, false); 820 | assertEquals( 821 | response.data && response.data.page.query.src, 822 | "https://nyan.web.id/screenshot.png", 823 | ); 824 | assertEquals( 825 | response.data && response.data.page.image, 826 | "https://nyan.web.id/screenshot.png", 827 | ); 828 | }, 829 | ...DEFAULT_TEST_OPTIONS, 830 | }); 831 | 832 | Deno.test({ 833 | name: "#34: class", 834 | fn: async () => { 835 | const html = 836 | `some title
`; 837 | const query = `{ 838 | page(source: "${html}") { 839 | query(selector: "div") { 840 | class 841 | } 842 | box: class(selector: "div") 843 | } 844 | }`; 845 | 846 | const response = await useQuery(query); 847 | 848 | assertEquals("error" in response, false); 849 | assertEquals( 850 | response.data && response.data.page.query.class, 851 | "mx-2 my-4 bg-gray-100", 852 | ); 853 | assertEquals( 854 | response.data && response.data.page.box, 855 | "mx-2 my-4 bg-gray-100", 856 | ); 857 | }, 858 | ...DEFAULT_TEST_OPTIONS, 859 | }); 860 | 861 | Deno.test({ 862 | name: "#34: classList", 863 | fn: async () => { 864 | const html = 865 | `some title
`; 866 | const query = `{ 867 | page(source: "${html}") { 868 | query(selector: "div") { 869 | classList 870 | } 871 | classes: classList(selector: "div") 872 | } 873 | }`; 874 | 875 | const response = await useQuery(query); 876 | 877 | assertEquals("error" in response, false); 878 | assertEquals( 879 | response.data && response.data.page.query.classList, 880 | ["mx-2", "my-4", "bg-gray-100"], 881 | ); 882 | assertEquals( 883 | response.data && response.data.page.classes, 884 | ["mx-2", "my-4", "bg-gray-100"], 885 | ); 886 | }, 887 | ...DEFAULT_TEST_OPTIONS, 888 | }); 889 | 890 | Deno.test({ 891 | name: "#35: meta", 892 | fn: async () => { 893 | const html = 894 | `some title`; 895 | const query = `{ 896 | page(source: "${html}") { 897 | description: meta(name: "description") 898 | og_description: meta(name: "og:description") 899 | } 900 | }`; 901 | 902 | const response = await useQuery(query); 903 | 904 | assertEquals("error" in response, false); 905 | assertEquals(response.data?.page.description, "some description"); 906 | assertEquals(response.data?.page.og_description, "some description"); 907 | }, 908 | ...DEFAULT_TEST_OPTIONS, 909 | }); 910 | 911 | Deno.test({ 912 | name: "#36: meta not found", 913 | fn: async () => { 914 | const html = 915 | `some title`; 916 | const query = `{ 917 | page(source: "${html}") { 918 | keywords: meta(name: "keywords") 919 | } 920 | }`; 921 | 922 | const response = await useQuery(query); 923 | 924 | assertEquals("error" in response, false); 925 | assertEquals(response.data?.page.keywords, null); 926 | }, 927 | ...DEFAULT_TEST_OPTIONS, 928 | }); 929 | 930 | // Deno.test({ 931 | // name: "#37: visit_custom", 932 | // fn: async () => { 933 | // const query = `{ 934 | // page(url: "https://nyancodeid.github.io/tests/test-custom-visit.html") { 935 | // link: query(selector: "div.link") { 936 | // visit_custom(attr: "data-link") { 937 | // text(selector: "strong") 938 | // } 939 | // } 940 | // } 941 | // }`; 942 | 943 | // const response = await useQuery(query); 944 | 945 | // assertEquals("error" in response, false); 946 | // assertEquals( 947 | // response.data && response.data.page.link.visit_custom.text, 948 | // "we managed to visit the link!", 949 | // ); 950 | // }, 951 | // ...DEFAULT_TEST_OPTIONS, 952 | // }); 953 | 954 | Deno.test({ 955 | name: "#38: table", 956 | fn: async () => { 957 | const query = `{ 958 | page(url: "https://nyancodeid.github.io/tests/kurs.html") { 959 | table: query(selector: "#scrolling-table") { 960 | values: table(attr: "table.m-table-kurs") 961 | } 962 | } 963 | }`; 964 | 965 | const response = await useQuery(query); 966 | 967 | assertEquals("error" in response, false); 968 | assertNotEquals( 969 | response.data && response.data.bca_kurs.table.length, 970 | 0, 971 | ); 972 | }, 973 | ...DEFAULT_TEST_OPTIONS, 974 | }); 975 | 976 | Deno.test({ 977 | name: "#39: index", 978 | fn: async () => { 979 | const query = `{ 980 | page(source: "some title
  • one
  • two
  • three
  • four
") { 981 | items: queryAll(selector: ".items li") { 982 | index 983 | text 984 | } 985 | } 986 | }`; 987 | 988 | const response = await useQuery(query); 989 | 990 | assertEquals("error" in response, false); 991 | assertNotEquals( 992 | response.data && response.data.page.items.length, 993 | 0, 994 | ); 995 | assertEquals(response.data?.page.items.length, 4); 996 | assertEquals(response.data?.page.items[0].index, 0); 997 | assertEquals(response.data?.page.items[2].text, "three"); 998 | }, 999 | ...DEFAULT_TEST_OPTIONS, 1000 | }); 1001 | 1002 | Deno.test({ 1003 | name: "#40: index with selector", 1004 | fn: async () => { 1005 | const query = `{ 1006 | page(source: "some title
  • one
  • two
  • three
  • four
") { 1007 | items: queryAll(selector: ".items li i") { 1008 | index(parent: ".items li") 1009 | text 1010 | } 1011 | } 1012 | }`; 1013 | 1014 | const response = await useQuery(query); 1015 | 1016 | assertEquals("error" in response, false); 1017 | assertNotEquals( 1018 | response.data && response.data.page.items.length, 1019 | 0, 1020 | ); 1021 | assertEquals(response.data?.page.items.length, 4); 1022 | assertEquals(response.data?.page.items[0].index, 0); 1023 | assertEquals(response.data?.page.items[2].text, "three"); 1024 | }, 1025 | ...DEFAULT_TEST_OPTIONS, 1026 | }); 1027 | -------------------------------------------------------------------------------- /tests/utils.test.ts: -------------------------------------------------------------------------------- 1 | import { assertEquals, resolveURL, State } from "./deps.ts"; 2 | 3 | Deno.test("#1: resolveURL", () => { 4 | const base = "https://berlette.com"; 5 | 6 | const url = resolveURL(base, "favicon.svg"); 7 | 8 | assertEquals(url, "https://berlette.com/favicon.svg"); 9 | }); 10 | 11 | Deno.test("#2: resolveURL with domain", () => { 12 | const base = "https://berlette.com"; 13 | 14 | const url = resolveURL(base, "https://berlette.com/favicon.svg"); 15 | 16 | assertEquals(url, "https://berlette.com/favicon.svg"); 17 | }); 18 | 19 | Deno.test("#3: resolveURL with different domain", () => { 20 | const base = "https://berlette.com"; 21 | 22 | const url = resolveURL(base, "https://cdn.berlette.com/brand/logo.svg"); 23 | 24 | assertEquals(url, "https://cdn.berlette.com/brand/logo.svg"); 25 | }); 26 | 27 | Deno.test("#4: state - booleans", () => { 28 | const state = new State(); 29 | 30 | state.set("test-state", true); 31 | state.set("test-state", false); 32 | state.set("test-state", true); 33 | 34 | assertEquals(state.get("test-state"), true); 35 | }); 36 | 37 | Deno.test("#5: state - strings", () => { 38 | const state = new State(); 39 | 40 | state.set("test-state", "test"); 41 | state.set("test-state-2", "test2"); 42 | }); 43 | 44 | Deno.test("#6: state - length", () => { 45 | const state = new State([ 46 | ["test-state", "test-value"], 47 | ["test-state-2", "test-value-2"], 48 | ]); 49 | assertEquals(state.length, 2); 50 | // Some methods and accessors are chainable: 51 | assertEquals(state.delete("test-state").length, 1); 52 | }); 53 | 54 | Deno.test("#7: state - set/delete multiple values", () => { 55 | const state = new State(); 56 | state.set(["test-1", "test-2", "test-3"], "test-value"); 57 | assertEquals(state.length, 3); 58 | assertEquals(state.delete(["test-1", "test-3"]).length, 1); 59 | }); 60 | 61 | Deno.test("#8: state - set values with an object", () => { 62 | const state = new State(); 63 | state.set({ "test-2": "test-value-2", "test-3": "test-value-3" }); 64 | assertEquals(state.get("test-2"), "test-value-2"); 65 | assertEquals(state.get("test-3"), "test-value-3"); 66 | }); 67 | 68 | Deno.test("#9: state - get multiple values", () => { 69 | const state = new State({ 70 | "test-1": "test-value-1", 71 | "test-2": "test-value-2", 72 | "test-3": "test-value-3", 73 | }); 74 | assertEquals(state.get(["test-1", "test-2", "test-3"]), [ 75 | "test-value-1", 76 | "test-value-2", 77 | "test-value-3", 78 | ]); 79 | assertEquals(state.length, 3); 80 | }); 81 | 82 | Deno.test("#10: state - get multiple values with a default", () => { 83 | const state = new State({ 84 | "test-1": "test-value-1", 85 | "test-2": "test-value-2", 86 | }); 87 | assertEquals(state.get(["test-1", "test-2", "test-3"], "test-default"), [ 88 | "test-value-1", 89 | "test-value-2", 90 | "test-default", 91 | ]); 92 | }); 93 | --------------------------------------------------------------------------------