├── packages ├── parser │ ├── .gitignore │ ├── tsconfig.json │ ├── src │ │ ├── grammar │ │ │ ├── lex │ │ │ │ ├── shared.ts │ │ │ │ ├── slice.ts │ │ │ │ ├── request.ts │ │ │ │ └── templates.ts │ │ │ ├── getlang.ne │ │ │ ├── lexer.ts │ │ │ └── parse.ts │ │ ├── index.ts │ │ ├── passes │ │ │ ├── inference.ts │ │ │ ├── desugar │ │ │ │ ├── dropdrill.ts │ │ │ │ ├── urlinputs.ts │ │ │ │ ├── context.ts │ │ │ │ ├── links.ts │ │ │ │ ├── reqparse.ts │ │ │ │ └── slicedeps.ts │ │ │ ├── analyze.ts │ │ │ ├── desugar.ts │ │ │ ├── inference │ │ │ │ ├── calls.ts │ │ │ │ └── typeinfo.ts │ │ │ └── lineage.ts │ │ ├── parse.ts │ │ ├── utils.ts │ │ ├── print.ts │ │ └── grammar.ts │ ├── package.json │ └── CHANGELOG.md ├── ast │ ├── tsconfig.json │ ├── src │ │ ├── index.ts │ │ ├── typeinfo.ts │ │ └── ast.ts │ ├── package.json │ └── CHANGELOG.md ├── get │ ├── tsconfig.json │ ├── src │ │ ├── test.d.ts │ │ ├── index.ts │ │ ├── value.ts │ │ ├── calls.ts │ │ ├── registry.ts │ │ └── execute.ts │ ├── README.md │ ├── package.json │ └── CHANGELOG.md ├── lib │ ├── tsconfig.json │ ├── src │ │ ├── slice.ts │ │ ├── values │ │ │ ├── html │ │ │ │ ├── types.d.ts │ │ │ │ └── patch-dom.ts │ │ │ ├── headers.ts │ │ │ ├── json.ts │ │ │ ├── cookies.ts │ │ │ ├── js.ts │ │ │ └── html.ts │ │ ├── index.ts │ │ ├── core │ │ │ ├── hooks.ts │ │ │ └── errors.ts │ │ └── net │ │ │ └── http.ts │ ├── package.json │ └── CHANGELOG.md └── walker │ ├── tsconfig.json │ ├── package.json │ ├── src │ ├── wait.ts │ ├── visitor.ts │ ├── path.ts │ ├── scope.ts │ └── index.ts │ └── CHANGELOG.md ├── .gitignore ├── test ├── tsconfig.json ├── test.d.ts ├── package.json ├── expect.ts ├── helpers.ts ├── objects.spec.ts ├── modifiers.spec.ts ├── hooks.spec.ts ├── slice.spec.ts ├── modules.spec.ts └── request.spec.ts ├── assets ├── hero.png └── hero.dark.png ├── .changeset ├── proud-onions-notice.md ├── config.json └── README.md ├── .github └── workflows │ ├── pull-request.yaml │ ├── checks.yaml │ ├── discord.yaml │ └── publish.yaml ├── knip.json ├── README.md ├── scripts └── publish.sh ├── tsconfig.base.json ├── package.json ├── biome.json └── LICENSE /packages/parser/.gitignore: -------------------------------------------------------------------------------- 1 | grammar.html 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | *.log 3 | dist 4 | -------------------------------------------------------------------------------- /test/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.base.json" 3 | } 4 | -------------------------------------------------------------------------------- /assets/hero.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getlang-dev/get/HEAD/assets/hero.png -------------------------------------------------------------------------------- /packages/ast/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.base.json" 3 | } 4 | -------------------------------------------------------------------------------- /packages/get/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.base.json" 3 | } 4 | -------------------------------------------------------------------------------- /packages/lib/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.base.json" 3 | } 4 | -------------------------------------------------------------------------------- /packages/parser/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.base.json" 3 | } 4 | -------------------------------------------------------------------------------- /packages/walker/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.base.json" 3 | } 4 | -------------------------------------------------------------------------------- /assets/hero.dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/getlang-dev/get/HEAD/assets/hero.dark.png -------------------------------------------------------------------------------- /.changeset/proud-onions-notice.md: -------------------------------------------------------------------------------- 1 | --- 2 | "@getlang/get": patch 3 | --- 4 | 5 | paint inputs 6 | -------------------------------------------------------------------------------- /.github/workflows/pull-request.yaml: -------------------------------------------------------------------------------- 1 | on: pull_request 2 | 3 | jobs: 4 | checks: 5 | uses: ./.github/workflows/checks.yaml 6 | -------------------------------------------------------------------------------- /packages/ast/src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './ast.js' 2 | export type { TypeInfo } from './typeinfo.js' 3 | export { Type } from './typeinfo.js' 4 | -------------------------------------------------------------------------------- /packages/lib/src/slice.ts: -------------------------------------------------------------------------------- 1 | const AsyncFunction: any = (async () => {}).constructor 2 | 3 | export function runSlice(slice: string, context: unknown = {}) { 4 | return new AsyncFunction('$', slice)(context) 5 | } 6 | -------------------------------------------------------------------------------- /test/test.d.ts: -------------------------------------------------------------------------------- 1 | declare module 'bun:test' { 2 | interface AsymmetricMatchers { 3 | toHaveServed(url: string, opts: RequestInit): void 4 | } 5 | interface Matchers { 6 | toHaveServed(url: string, opts: RequestInit): R 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /packages/get/src/test.d.ts: -------------------------------------------------------------------------------- 1 | interface MyCustomMatchers { 2 | headers(headers: globalThis.Headers): any 3 | } 4 | declare module 'bun:test' { 5 | interface Matchers extends MyCustomMatchers {} 6 | interface AsymmetricMatchers extends MyCustomMatchers {} 7 | } 8 | -------------------------------------------------------------------------------- /knip.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://unpkg.com/knip@5/schema.json", 3 | "vitest": {}, 4 | "ignoreDependencies": [ 5 | "@changesets/cli" 6 | ], 7 | "ignoreBinaries": [ 8 | "open" 9 | ], 10 | "exclude": [ 11 | "enumMembers" 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | GETlang hero banner 4 | 5 | 6 | Visit [getlang.dev](https://getlang.dev) to view examples, take a guided tour, or try your first GET query! 7 | -------------------------------------------------------------------------------- /packages/lib/src/values/html/types.d.ts: -------------------------------------------------------------------------------- 1 | declare module '@getlang/xpath' { 2 | import type { AnyHtmlNode } from 'domhandler' 3 | 4 | function select(selector: string, node: AnyHtmlNode): Array 5 | 6 | class XPathParser { 7 | parse(selector: string): unknown 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /packages/parser/src/grammar/lex/shared.ts: -------------------------------------------------------------------------------- 1 | const id = /[a-zA-Z_]\w*/ 2 | 3 | export const patterns = { 4 | ws: /[ \t\r\f\v]+/, 5 | identifier: id, 6 | identifierExpr: new RegExp(`\\$(?:${id.source})?`), 7 | link: new RegExp(`\\@${id.source}\\)`), 8 | call: new RegExp(`\\@${id.source}`), 9 | } 10 | -------------------------------------------------------------------------------- /packages/parser/src/index.ts: -------------------------------------------------------------------------------- 1 | export { default as lexer } from './grammar/lexer.js' 2 | export { parse } from './parse.js' 3 | export { analyze } from './passes/analyze.js' 4 | export { desugar } from './passes/desugar.js' 5 | export { inference } from './passes/inference.js' 6 | export { print } from './print.js' 7 | -------------------------------------------------------------------------------- /packages/get/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | GETlang hero banner 4 | 5 | 6 | Visit [getlang.dev](https://getlang.dev) to view examples, take a guided tour, or try your first GET query! 7 | -------------------------------------------------------------------------------- /scripts/publish.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | for dir in packages/*; do 6 | tsc -p $dir 7 | sed -i.bak 's/workspace://g' "$dir/package.json" 8 | rm "$dir/package.json.bak" "$dir/tsconfig.json" 9 | done 10 | 11 | changeset publish 12 | 13 | rm -rf dist packages/*/dist 14 | git restore packages 15 | -------------------------------------------------------------------------------- /test/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test", 3 | "private": true, 4 | "type": "module", 5 | "dependencies": { 6 | "@getlang/get": "workspace:*", 7 | "@getlang/lib": "workspace:^0.2.1", 8 | "@getlang/parser": "workspace:*", 9 | "dedent": "^1.7.0" 10 | }, 11 | "devDependencies": { 12 | "jest-diff": "^30.1.2" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /.github/workflows/checks.yaml: -------------------------------------------------------------------------------- 1 | on: workflow_call 2 | 3 | jobs: 4 | checks: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@v4 8 | - uses: oven-sh/setup-bun@v2 9 | - run: bun install --frozen-lockfile 10 | - run: bun --filter @getlang/parser compile 11 | - run: bun lint 12 | - run: bun test 13 | -------------------------------------------------------------------------------- /.changeset/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://unpkg.com/@changesets/config@3.0.3/schema.json", 3 | "changelog": ["@changesets/changelog-github", { "repo": "getlang-dev/get" }], 4 | "commit": false, 5 | "fixed": [], 6 | "linked": [], 7 | "access": "public", 8 | "baseBranch": "main", 9 | "updateInternalDependencies": "minor", 10 | "ignore": [] 11 | } 12 | -------------------------------------------------------------------------------- /.github/workflows/discord.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | release: 3 | types: [published] 4 | 5 | jobs: 6 | discord-notify-release: 7 | if: startsWith(github.event.release.tag_name, '@getlang/get@') 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v4 11 | - uses: SethCohen/github-releases-to-discord@v1 12 | with: 13 | webhook_url: ${{ secrets.RELEASE_DISCORD_WEBHOOK_URL }} 14 | reduce_headings: true 15 | -------------------------------------------------------------------------------- /packages/parser/src/passes/inference.ts: -------------------------------------------------------------------------------- 1 | import type { Program, TypeInfo } from '@getlang/ast' 2 | import { resolveTypes } from './inference/typeinfo.js' 3 | 4 | type InferenceOptions = { 5 | returnTypes: { [module: string]: TypeInfo } 6 | contextType?: TypeInfo 7 | } 8 | 9 | export function inference(ast: Program, options: InferenceOptions) { 10 | const { program, returnType } = resolveTypes(ast, options) 11 | return { program, returnType } 12 | } 13 | -------------------------------------------------------------------------------- /packages/parser/src/passes/desugar/dropdrill.ts: -------------------------------------------------------------------------------- 1 | import type { Program } from '@getlang/ast' 2 | import { ScopeTracker, transform } from '@getlang/walker' 3 | 4 | export function dropDrills(ast: Program) { 5 | const scope = new ScopeTracker() 6 | 7 | return transform(ast, { 8 | scope, 9 | 10 | DrillExpr(node) { 11 | const [first, ...rest] = node.body 12 | if (rest.length === 0) { 13 | return first 14 | } 15 | }, 16 | }) 17 | } 18 | -------------------------------------------------------------------------------- /packages/lib/src/index.ts: -------------------------------------------------------------------------------- 1 | export { invariant, NullSelection } from './core/errors.js' 2 | export * from './core/hooks.js' 3 | export * as http from './net/http.js' 4 | export * as slice from './slice.js' 5 | export * as cookies from './values/cookies.js' 6 | export * as headers from './values/headers.js' 7 | export * as html from './values/html.js' 8 | export * as js from './values/js.js' 9 | export * as json from './values/json.js' 10 | 11 | export type Inputs = Record 12 | export type MaybePromise = T | Promise 13 | -------------------------------------------------------------------------------- /.changeset/README.md: -------------------------------------------------------------------------------- 1 | # Changesets 2 | 3 | Hello and welcome! This folder has been automatically generated by `@changesets/cli`, a build tool that works 4 | with multi-package repos, or single-package repos to help you version and publish your code. You can 5 | find the full documentation for it [in our repository](https://github.com/changesets/changesets) 6 | 7 | We have a quick list of common questions to get you started engaging with this project in 8 | [our documentation](https://github.com/changesets/changesets/blob/main/docs/common-questions.md) 9 | -------------------------------------------------------------------------------- /packages/lib/src/values/headers.ts: -------------------------------------------------------------------------------- 1 | import { NullSelection } from '../core/errors.js' 2 | 3 | export const select = (headers: Headers, selector: string, expand: boolean) => { 4 | if (expand && selector === 'set-cookie') { 5 | return headers.getSetCookie() 6 | } 7 | const value = headers.get(selector) 8 | if (expand) { 9 | return value ? value.split(',').map(x => x.trimStart()) : [] 10 | } 11 | return value === null ? new NullSelection(selector) : value 12 | } 13 | 14 | export const toValue = (headers: Headers) => { 15 | return Object.fromEntries(headers) 16 | } 17 | -------------------------------------------------------------------------------- /packages/ast/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@getlang/ast", 3 | "version": "0.1.0", 4 | "license": "Apache-2.0", 5 | "type": "module", 6 | "exports": { 7 | ".": { 8 | "bun": "./src/index.ts", 9 | "default": "./dist/index.js" 10 | } 11 | }, 12 | "bugs": { 13 | "url": "https://github.com/getlang-dev/get/issues" 14 | }, 15 | "repository": { 16 | "type": "git", 17 | "url": "git+https://github.com/getlang-dev/get.git", 18 | "directory": "packages/ast" 19 | }, 20 | "homepage": "https://getlang.dev", 21 | "dependencies": { 22 | "moo": "^0.5.2" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /packages/lib/src/values/json.ts: -------------------------------------------------------------------------------- 1 | import { get, toPath } from 'lodash-es' 2 | import { NullSelection } from '../core/errors.js' 3 | 4 | export const parse = (json: string) => JSON.parse(json) 5 | 6 | // only an `undefined` result is considered a null selection 7 | // if result itself is null, the key is present. This is a 8 | // valid scenario that should not raise a NullSelectionError 9 | export const select = (value: any, selector: string, expand: boolean) => { 10 | const path = toPath(selector) 11 | const fallback = expand ? [] : new NullSelection(selector) 12 | return get(value, path, fallback) 13 | } 14 | -------------------------------------------------------------------------------- /tsconfig.base.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "customConditions": ["bun"], 4 | "esModuleInterop": true, 5 | "skipLibCheck": true, 6 | "target": "es2022", 7 | "allowJs": true, 8 | "resolveJsonModule": true, 9 | "moduleDetection": "force", 10 | "isolatedModules": true, 11 | "verbatimModuleSyntax": true, 12 | "strict": true, 13 | "noUncheckedIndexedAccess": true, 14 | "noImplicitOverride": true, 15 | "module": "NodeNext", 16 | "outDir": "${configDir}/dist", 17 | "sourceMap": true, 18 | "declaration": true, 19 | "lib": ["esnext", "dom", "dom.iterable"] 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /packages/walker/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@getlang/walker", 3 | "version": "0.1.0", 4 | "license": "Apache-2.0", 5 | "type": "module", 6 | "exports": { 7 | ".": { 8 | "bun": "./src/index.ts", 9 | "default": "./dist/index.js" 10 | } 11 | }, 12 | "bugs": { 13 | "url": "https://github.com/getlang-dev/get/issues" 14 | }, 15 | "repository": { 16 | "type": "git", 17 | "url": "git+https://github.com/getlang-dev/get.git", 18 | "directory": "packages/walker" 19 | }, 20 | "homepage": "https://getlang.dev", 21 | "dependencies": { 22 | "@getlang/ast": "workspace:^0.1.0" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /packages/get/src/index.ts: -------------------------------------------------------------------------------- 1 | import type { Hooks, Inputs } from '@getlang/lib' 2 | import { buildHooks } from '@getlang/lib' 3 | import { execute as exec } from './execute.js' 4 | 5 | export function execute( 6 | source: string, 7 | inputs: Inputs = {}, 8 | hooks: Hooks = {}, 9 | ) { 10 | const system = buildHooks(hooks) 11 | return exec('Default', inputs, { 12 | ...system, 13 | import() { 14 | this.import = system.import 15 | return source 16 | }, 17 | }) 18 | } 19 | 20 | export async function executeModule( 21 | module: string, 22 | inputs: Inputs = {}, 23 | hooks: Hooks = {}, 24 | ) { 25 | return exec(module, inputs, buildHooks(hooks)) 26 | } 27 | -------------------------------------------------------------------------------- /packages/ast/src/typeinfo.ts: -------------------------------------------------------------------------------- 1 | export enum Type { 2 | Value = 'value', 3 | Html = 'html', 4 | Js = 'js', 5 | Headers = 'headers', 6 | Cookies = 'cookies', 7 | Context = 'context', 8 | List = 'list', 9 | Struct = 'struct', 10 | Never = 'never', 11 | Maybe = 'maybe', 12 | } 13 | 14 | type List = { 15 | type: Type.List 16 | of: TypeInfo 17 | } 18 | 19 | type Struct = { 20 | type: Type.Struct 21 | schema: Record 22 | } 23 | 24 | type Maybe = { 25 | type: Type.Maybe 26 | option: TypeInfo 27 | } 28 | 29 | type ScalarType = { 30 | type: Exclude 31 | } 32 | 33 | export type TypeInfo = ScalarType | List | Struct | Maybe 34 | -------------------------------------------------------------------------------- /packages/lib/src/values/cookies.ts: -------------------------------------------------------------------------------- 1 | import { mapValues } from 'lodash-es' 2 | import * as scp from 'set-cookie-parser' 3 | import { invariant, NullSelection, QuerySyntaxError } from '../core/errors.js' 4 | 5 | export const parse = (source: string) => { 6 | const cookie = scp.splitCookiesString(source) 7 | return scp.parse(cookie, { map: true }) 8 | } 9 | 10 | export const select = ( 11 | cookies: scp.CookieMap, 12 | selector: string, 13 | expand: boolean, 14 | ) => { 15 | invariant(!expand, new QuerySyntaxError('Cannot expand cookies selector')) 16 | return cookies[selector]?.value ?? new NullSelection(selector) 17 | } 18 | 19 | export const toValue = (cookies: scp.CookieMap) => { 20 | return mapValues(cookies, c => c.value) 21 | } 22 | -------------------------------------------------------------------------------- /packages/walker/src/wait.ts: -------------------------------------------------------------------------------- 1 | type MaybePromise = T | Promise 2 | 3 | export function wait( 4 | value: MaybePromise, 5 | then: (value: V) => X, 6 | ): MaybePromise { 7 | return value instanceof Promise ? value.then(then) : then(value) 8 | } 9 | 10 | export function waitMap( 11 | values: V[], 12 | mapper: (value: V) => MaybePromise, 13 | ): MaybePromise { 14 | return values.reduce( 15 | (acc, value) => 16 | acc instanceof Promise 17 | ? acc.then(async acc => { 18 | acc.push(await mapper(value)) 19 | return acc 20 | }) 21 | : wait(mapper(value), value => { 22 | acc.push(value) 23 | return acc 24 | }), 25 | [] as MaybePromise, 26 | ) 27 | } 28 | -------------------------------------------------------------------------------- /packages/get/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@getlang/get", 3 | "version": "0.3.3", 4 | "license": "Apache-2.0", 5 | "type": "module", 6 | "exports": { 7 | "bun": "./src/index.ts", 8 | "default": "./dist/index.js" 9 | }, 10 | "bugs": { 11 | "url": "https://github.com/getlang-dev/get/issues" 12 | }, 13 | "repository": { 14 | "type": "git", 15 | "url": "git+https://github.com/getlang-dev/get.git", 16 | "directory": "packages/get" 17 | }, 18 | "homepage": "https://getlang.dev", 19 | "dependencies": { 20 | "@getlang/ast": "workspace:^0.1.0", 21 | "@getlang/lib": "workspace:^0.2.1", 22 | "@getlang/parser": "workspace:^0.4.3", 23 | "@getlang/walker": "workspace:^0.1.0", 24 | "lodash-es": "^4.17.21" 25 | }, 26 | "devDependencies": { 27 | "@types/lodash-es": "^4.17.12" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /.github/workflows/publish.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - main 5 | 6 | jobs: 7 | checks: 8 | uses: ./.github/workflows/checks.yaml 9 | 10 | publish: 11 | runs-on: ubuntu-latest 12 | needs: checks 13 | permissions: 14 | id-token: write 15 | contents: write 16 | pull-requests: write 17 | issues: read 18 | steps: 19 | - uses: actions/checkout@v4 20 | - uses: oven-sh/setup-bun@v2 21 | - run: bun install --frozen-lockfile 22 | - run: bun --filter @getlang/parser compile 23 | - uses: changesets/action@v1 24 | with: 25 | title: 'chore: version packages' 26 | commit: 'chore: version packages' 27 | publish: bun pub 28 | env: 29 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 30 | NPM_TOKEN: ${{ secrets.NPM_TOKEN }} 31 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "get", 3 | "license": "Apache-2.0", 4 | "private": true, 5 | "packageManager": "bun@1.2.22", 6 | "scripts": { 7 | "fmt": "biome check --write", 8 | "lint": "bun lint:check && bun lint:types && bun lint:unused && bun lint:repo", 9 | "lint:check": "biome check", 10 | "lint:types": "cd test && tsc --noEmit", 11 | "lint:unused": "knip", 12 | "lint:repo": "sherif", 13 | "deps": "bunx taze major -r", 14 | "pub": "bash scripts/publish.sh" 15 | }, 16 | "workspaces": [ 17 | "packages/*", 18 | "test" 19 | ], 20 | "devDependencies": { 21 | "@biomejs/biome": "^2.2.4", 22 | "@changesets/changelog-github": "^0.5.1", 23 | "@changesets/cli": "^2.29.7", 24 | "@types/bun": "^1.2.22", 25 | "knip": "^5.63.1", 26 | "sherif": "^1.6.1", 27 | "typescript": "^5.9.2" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /packages/parser/src/parse.ts: -------------------------------------------------------------------------------- 1 | import type { Program } from '@getlang/ast' 2 | import { invariant } from '@getlang/lib' 3 | import { QuerySyntaxError } from '@getlang/lib/errors' 4 | import nearley from 'nearley' 5 | import lexer from './grammar/lexer.js' 6 | import grammar from './grammar.js' 7 | 8 | export function parse(source: string): Program { 9 | const parser = new nearley.Parser(nearley.Grammar.fromCompiled(grammar)) 10 | try { 11 | parser.feed(source) 12 | } catch (e: unknown) { 13 | if (typeof e === 'object' && e && 'token' in e) { 14 | throw new QuerySyntaxError( 15 | lexer.formatError(e.token, 'SyntaxError: Invalid token'), 16 | ) 17 | } 18 | throw e 19 | } 20 | 21 | const [ast, ...rest] = parser.results 22 | invariant(ast, new QuerySyntaxError('Unexpected end of input')) 23 | invariant(!rest.length, new QuerySyntaxError('Unexpected parsing error')) 24 | return ast 25 | } 26 | -------------------------------------------------------------------------------- /test/expect.ts: -------------------------------------------------------------------------------- 1 | import { expect } from 'bun:test' 2 | import { diff } from 'jest-diff' 3 | 4 | expect.extend({ 5 | async toHaveServed(received: unknown, url: string, opts: RequestInit) { 6 | const calls: [unknown, any][] = (received as any)?.mock?.calls 7 | const { method, headers = {}, body } = opts 8 | const expObj = { url, method, headers, body } 9 | 10 | let receivedObj: any 11 | 12 | for (const [url, { method, headers, body }] of calls) { 13 | const recObj = { url, method, headers, body } 14 | receivedObj ??= recObj 15 | const pass = this.equals(recObj, expObj) 16 | if (pass) { 17 | return { pass, message: () => 'Pass' } 18 | } 19 | } 20 | 21 | return { 22 | pass: false, 23 | message: () => { 24 | const diffString = diff(expObj, receivedObj) 25 | return ['Difference:', diffString].join('\n\n') 26 | }, 27 | } 28 | }, 29 | }) 30 | -------------------------------------------------------------------------------- /packages/parser/src/passes/analyze.ts: -------------------------------------------------------------------------------- 1 | import type { Program } from '@getlang/ast' 2 | import { ScopeTracker, transform } from '@getlang/walker' 3 | 4 | export function analyze(ast: Program) { 5 | const scope = new ScopeTracker() 6 | const inputs = new Set() 7 | const calls = new Set() 8 | const modifiers = new Map() 9 | const imports = new Set() 10 | let hasUnboundSelector = false 11 | 12 | transform(ast, { 13 | scope, 14 | InputExpr(node) { 15 | inputs.add(node.id.value) 16 | }, 17 | ModuleExpr(node) { 18 | imports.add(node.module.value) 19 | node.call && calls.add(node.module.value) 20 | }, 21 | SelectorExpr() { 22 | hasUnboundSelector ||= !scope.context 23 | }, 24 | ModifierExpr(node) { 25 | const mod = node.modifier.value 26 | if (!modifiers.get(mod)) { 27 | modifiers.set(mod, !scope.context) 28 | } 29 | }, 30 | }) 31 | 32 | return { 33 | inputs, 34 | imports, 35 | calls, 36 | modifiers, 37 | hasUnboundSelector, 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /packages/parser/src/grammar/lex/slice.ts: -------------------------------------------------------------------------------- 1 | import { invariant } from '@getlang/lib' 2 | import { QuerySyntaxError } from '@getlang/lib/errors' 3 | import { until } from './templates.js' 4 | 5 | const getSliceValue = (text: string) => { 6 | const src = text.slice(1, -1).replace(/\\`/g, '`') 7 | let lines = src.split('\n') 8 | const firstIdx = lines.findIndex(x => x.trim().length) 9 | invariant(firstIdx !== -1, new QuerySyntaxError('Slice must contain source')) 10 | lines = lines.slice(firstIdx) 11 | const indent = lines[0]?.match(/^\s*/)?.[0].length || 0 12 | if (indent) { 13 | lines = lines.map(x => x.replace(new RegExp(`^\\s{0,${indent}}`), '')) 14 | } 15 | return lines.join('\n').trim() 16 | } 17 | 18 | export const slice_block = { 19 | defaultType: 'slice', 20 | match: until(/\|/, { 21 | prefix: /\|/, 22 | inclusive: true, 23 | }), 24 | lineBreaks: true, 25 | value: getSliceValue, 26 | pop: 1, 27 | } 28 | 29 | export const slice = { 30 | match: until(/`/, { 31 | prefix: /`/, 32 | inclusive: true, 33 | }), 34 | lineBreaks: true, 35 | value: getSliceValue, 36 | pop: 1, 37 | } 38 | -------------------------------------------------------------------------------- /packages/ast/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # @getlang/ast 2 | 3 | ## 0.1.0 4 | 5 | ### Minor Changes 6 | 7 | - [`78660db`](https://github.com/getlang-dev/get/commit/78660db7089cebb8715c5389bac81559f2f4f60b) Thanks [@mattfysh](https://github.com/mattfysh)! - reorganize package layout 8 | 9 | ### Patch Changes 10 | 11 | - [#46](https://github.com/getlang-dev/get/pull/46) [`fe8b25b`](https://github.com/getlang-dev/get/commit/fe8b25ba179d0a770b5e00275c70e9c7fc0405cf) Thanks [@mattfysh](https://github.com/mattfysh)! - add custom modifiers 12 | 13 | - [#44](https://github.com/getlang-dev/get/pull/44) [`356a09a`](https://github.com/getlang-dev/get/commit/356a09a81cddcba4c788a103451c8c9517ec596d) Thanks [@mattfysh](https://github.com/mattfysh)! - drill-based walker 14 | 15 | - [#47](https://github.com/getlang-dev/get/pull/47) [`a2304c8`](https://github.com/getlang-dev/get/commit/a2304c824175bb75be82f08a61a7a04880fb137f) Thanks [@mattfysh](https://github.com/mattfysh)! - flexible urls with implied params 16 | 17 | - [#48](https://github.com/getlang-dev/get/pull/48) [`3e8b1c8`](https://github.com/getlang-dev/get/commit/3e8b1c8a591b5de149c6cc7556900a250273c9a5) Thanks [@mattfysh](https://github.com/mattfysh)! - primitive literals 18 | -------------------------------------------------------------------------------- /packages/lib/src/values/js.ts: -------------------------------------------------------------------------------- 1 | import type { AnyNode } from 'acorn' 2 | import { parse as acorn } from 'acorn' 3 | import esquery from 'esquery' 4 | import { 5 | ConversionError, 6 | invariant, 7 | NullSelection, 8 | SelectorSyntaxError, 9 | SliceSyntaxError, 10 | } from '../core/errors.js' 11 | 12 | export const parse = (js: string): AnyNode => { 13 | try { 14 | return acorn(js, { 15 | ecmaVersion: 'latest', 16 | allowAwaitOutsideFunction: true, 17 | }) 18 | } catch (e) { 19 | throw new SliceSyntaxError('Could not parse slice', { cause: e }) 20 | } 21 | } 22 | 23 | export const select = (node: AnyNode, selector: string, expand: boolean) => { 24 | try { 25 | const matches = esquery(node as any, selector) 26 | if (expand) { 27 | return matches 28 | } 29 | return matches.length ? matches[0] : new NullSelection(selector) 30 | } catch (e: any) { 31 | invariant( 32 | e.name !== 'SyntaxError', 33 | new SelectorSyntaxError('AST', selector, { cause: e }), 34 | ) 35 | throw e 36 | } 37 | } 38 | 39 | export const toValue = (node: AnyNode) => { 40 | invariant(node.type === 'Literal', new ConversionError(node.type)) 41 | return node.value 42 | } 43 | -------------------------------------------------------------------------------- /packages/parser/src/passes/desugar.ts: -------------------------------------------------------------------------------- 1 | import type { Program } from '@getlang/ast' 2 | import { resolveContext } from './desugar/context.js' 3 | import { dropDrills } from './desugar/dropdrill.js' 4 | import { settleLinks } from './desugar/links.js' 5 | import { RequestParsers } from './desugar/reqparse.js' 6 | import { insertSliceDeps } from './desugar/slicedeps.js' 7 | import { addUrlInputs } from './desugar/urlinputs.js' 8 | import { registerCalls } from './inference/calls.js' 9 | 10 | export type DesugarPass = ( 11 | ast: Program, 12 | tools: { 13 | parsers: RequestParsers 14 | contextual: string[] 15 | }, 16 | ) => Program 17 | 18 | const visitors = [ 19 | addUrlInputs, 20 | resolveContext, 21 | settleLinks, 22 | insertSliceDeps, 23 | dropDrills, 24 | ] 25 | 26 | export function desugar(ast: Program, contextual: string[] = []): Program { 27 | const parsers = new RequestParsers() 28 | const program = visitors.reduce((ast, pass) => { 29 | parsers.reset() 30 | return pass(ast, { parsers, contextual }) 31 | }, ast) 32 | // inference pass `registerCalls` is included in the desugar phase 33 | // it produces the list of called modules required for type inference 34 | return registerCalls(program, contextual) 35 | } 36 | -------------------------------------------------------------------------------- /packages/parser/src/passes/inference/calls.ts: -------------------------------------------------------------------------------- 1 | import type { Expr, Program } from '@getlang/ast' 2 | import { isToken } from '@getlang/ast' 3 | import { transform } from '@getlang/walker' 4 | import { LineageTracker } from '../lineage.js' 5 | 6 | export function registerCalls( 7 | ast: Program, 8 | contextual: string[] = [], 9 | ): Program { 10 | const scope = new LineageTracker() 11 | 12 | function registerCall(node: Expr) { 13 | const lineage = scope.traceLineageRoot(node) || node 14 | if (lineage?.kind === 'ModuleExpr') { 15 | lineage.call = true 16 | } 17 | } 18 | 19 | return transform(ast, { 20 | scope, 21 | 22 | TemplateExpr(node) { 23 | for (const el of node.elements) { 24 | if (!isToken(el)) { 25 | registerCall(el) 26 | } 27 | } 28 | return node 29 | }, 30 | 31 | SelectorExpr() { 32 | if (scope.context) { 33 | registerCall(scope.context) 34 | } 35 | }, 36 | 37 | ModifierExpr() { 38 | if (scope.context) { 39 | registerCall(scope.context) 40 | } 41 | }, 42 | 43 | ModuleExpr(node) { 44 | if (contextual.includes(node.module.value)) { 45 | return { ...node, call: true } 46 | } 47 | }, 48 | }) 49 | } 50 | -------------------------------------------------------------------------------- /packages/parser/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@getlang/parser", 3 | "version": "0.4.3", 4 | "license": "Apache-2.0", 5 | "type": "module", 6 | "exports": { 7 | ".": { 8 | "bun": "./src/index.ts", 9 | "default": "./dist/index.js" 10 | } 11 | }, 12 | "bugs": { 13 | "url": "https://github.com/getlang-dev/get/issues" 14 | }, 15 | "repository": { 16 | "type": "git", 17 | "url": "git+https://github.com/getlang-dev/get.git", 18 | "directory": "packages/parser" 19 | }, 20 | "homepage": "https://getlang.dev", 21 | "scripts": { 22 | "compile": "nearleyc src/grammar/getlang.ne -o src/grammar.ts", 23 | "railroad": "nearley-railroad src/grammar/getlang.ne -o grammar.html && open grammar.html" 24 | }, 25 | "dependencies": { 26 | "@getlang/ast": "workspace:^0.1.0", 27 | "@getlang/lib": "workspace:^0.2.1", 28 | "@getlang/walker": "workspace:^0.1.0", 29 | "@types/moo": "^0.5.10", 30 | "@types/nearley": "^2.11.5", 31 | "acorn": "^8.15.0", 32 | "estree-toolkit": "^1.7.13", 33 | "globals": "^16.4.0", 34 | "lodash-es": "^4.17.21", 35 | "moo": "^0.5.2", 36 | "nearley": "^2.20.1", 37 | "prettier": "^3.6.2" 38 | }, 39 | "devDependencies": { 40 | "@types/lodash-es": "^4.17.12" 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /packages/walker/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # @getlang/walker 2 | 3 | ## 0.1.0 4 | 5 | ### Minor Changes 6 | 7 | - [`78660db`](https://github.com/getlang-dev/get/commit/78660db7089cebb8715c5389bac81559f2f4f60b) Thanks [@mattfysh](https://github.com/mattfysh)! - reorganize package layout 8 | 9 | ### Patch Changes 10 | 11 | - [#46](https://github.com/getlang-dev/get/pull/46) [`fe8b25b`](https://github.com/getlang-dev/get/commit/fe8b25ba179d0a770b5e00275c70e9c7fc0405cf) Thanks [@mattfysh](https://github.com/mattfysh)! - add custom modifiers 12 | 13 | - [#44](https://github.com/getlang-dev/get/pull/44) [`356a09a`](https://github.com/getlang-dev/get/commit/356a09a81cddcba4c788a103451c8c9517ec596d) Thanks [@mattfysh](https://github.com/mattfysh)! - drill-based walker 14 | 15 | - Updated dependencies [[`fe8b25b`](https://github.com/getlang-dev/get/commit/fe8b25ba179d0a770b5e00275c70e9c7fc0405cf), [`356a09a`](https://github.com/getlang-dev/get/commit/356a09a81cddcba4c788a103451c8c9517ec596d), [`a2304c8`](https://github.com/getlang-dev/get/commit/a2304c824175bb75be82f08a61a7a04880fb137f), [`78660db`](https://github.com/getlang-dev/get/commit/78660db7089cebb8715c5389bac81559f2f4f60b), [`3e8b1c8`](https://github.com/getlang-dev/get/commit/3e8b1c8a591b5de149c6cc7556900a250273c9a5)]: 16 | - @getlang/ast@0.1.0 17 | -------------------------------------------------------------------------------- /packages/parser/src/grammar/lex/request.ts: -------------------------------------------------------------------------------- 1 | import { templateUntil } from './templates.js' 2 | 3 | const requestBlockNames = ['query', 'cookies', 'json', 'form'] 4 | 5 | const request = { 6 | request_term: { 7 | defaultType: 'nl', 8 | match: /\n\s*$/, 9 | lineBreaks: true, 10 | pop: 1, 11 | }, 12 | request_block_body_end: { 13 | match: /\n[^\S\r\n]*\[\/body\]/, 14 | lineBreaks: true, 15 | }, 16 | nl: { 17 | match: /\n/, 18 | lineBreaks: true, 19 | }, 20 | request_block_name: { 21 | match: new RegExp(`^\\s*\\[(?:${requestBlockNames.join('|')})\\]`), 22 | value: (text: string) => text.trim().slice(1, -1), 23 | }, 24 | request_block_body: { 25 | match: /^\s*\[body\]\n/, 26 | lineBreaks: true, 27 | push: 'requestBody', 28 | }, 29 | start_of_line_incl_ws: { 30 | defaultType: 'ws', 31 | match: /^\s*(?=.)/, 32 | push: 'requestKey', 33 | }, 34 | colon: ':', 35 | ws: { 36 | match: ' ', 37 | push: 'requestValue', 38 | }, 39 | } 40 | 41 | export const requestStates = { 42 | request, 43 | requestUrl: templateUntil(/\n/, { interpParams: true, next: 'request' }), 44 | requestKey: templateUntil(/:/), 45 | requestValue: templateUntil(/\n/), 46 | requestBody: templateUntil(/\n[^\S\r\n]*\[\/body\]/), 47 | } 48 | -------------------------------------------------------------------------------- /packages/walker/src/visitor.ts: -------------------------------------------------------------------------------- 1 | import type { Expr, Node, Stmt } from '@getlang/ast' 2 | import type { Path } from './index.js' 3 | 4 | // -------- Utilities 5 | 6 | type Transform = V extends readonly (infer T)[] 7 | ? Transform[] 8 | : V extends (...args: any) => any 9 | ? V 10 | : V extends object 11 | ? V extends Stmt 12 | ? S 13 | : V extends Expr 14 | ? E 15 | : { [K in keyof V]: Transform } 16 | : V 17 | 18 | type TNode = { 19 | [K in keyof N]: Transform 20 | } 21 | 22 | type NodeResult = N extends Stmt ? S : N extends Expr ? E : never 23 | 24 | type Visit = 25 | | ((node: X, path: Path) => R) 26 | | ((node: X, path: Path) => void) 27 | 28 | export type NodeVisitor = { 29 | enter: Visit 30 | exit: Visit 31 | } 32 | 33 | export type NodeConfig = 34 | | Visit 35 | | Partial> 36 | 37 | export type TransformVisitor = Partial<{ 38 | [N in Node as N['kind']]: NodeConfig 39 | }> 40 | 41 | export type ReduceVisitor = Partial<{ 42 | [N in Node as N['kind']]: NodeConfig, NodeResult> 43 | }> 44 | -------------------------------------------------------------------------------- /biome.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "./node_modules/@biomejs/biome/configuration_schema.json", 3 | "files": { 4 | "includes": [ 5 | "packages/**/*", 6 | "test/**/*", 7 | "!packages/parser/src/grammar.ts" 8 | ] 9 | }, 10 | "assist": { 11 | "actions": { 12 | "source": { 13 | "organizeImports": "on" 14 | } 15 | } 16 | }, 17 | "formatter": { 18 | "enabled": true, 19 | "indentStyle": "space", 20 | "indentWidth": 2 21 | }, 22 | "linter": { 23 | "enabled": true, 24 | "rules": { 25 | "recommended": true, 26 | "correctness": { 27 | "noUnusedVariables": "error", 28 | "noUnusedImports": "error" 29 | }, 30 | "suspicious": { 31 | "noExplicitAny": "off" 32 | }, 33 | "style": { 34 | "noUselessElse": "off", 35 | "noNonNullAssertion": "off", 36 | "useBlockStatements": "error", 37 | "useImportType": { 38 | "level": "error", 39 | "options": { 40 | "style": "separatedType" 41 | } 42 | } 43 | } 44 | } 45 | }, 46 | "javascript": { 47 | "formatter": { 48 | "semicolons": "asNeeded", 49 | "quoteStyle": "single", 50 | "arrowParentheses": "asNeeded" 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /packages/lib/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@getlang/lib", 3 | "version": "0.2.1", 4 | "license": "Apache-2.0", 5 | "type": "module", 6 | "exports": { 7 | ".": { 8 | "bun": "./src/index.ts", 9 | "default": "./dist/index.js" 10 | }, 11 | "./errors": { 12 | "bun": "./src/core/errors.ts", 13 | "default": "./dist/core/errors.js" 14 | } 15 | }, 16 | "bugs": { 17 | "url": "https://github.com/getlang-dev/get/issues" 18 | }, 19 | "repository": { 20 | "type": "git", 21 | "url": "git+https://github.com/getlang-dev/get.git", 22 | "directory": "packages/lib" 23 | }, 24 | "homepage": "https://getlang.dev", 25 | "dependencies": { 26 | "@getlang/ast": "workspace:^0.1.0", 27 | "@getlang/xpath": "0.0.35-0", 28 | "@types/esquery": "^1.5.4", 29 | "@types/lodash-es": "^4.17.12", 30 | "@types/set-cookie-parser": "^2.4.10", 31 | "acorn": "^8.15.0", 32 | "css-select": "^6.0.0", 33 | "css-what": "^7.0.0", 34 | "dom-serializer": "^2.0.0", 35 | "domelementtype": "*", 36 | "domhandler": "^5.0.3", 37 | "domutils": "^3.2.2", 38 | "esquery": "^1.6.0", 39 | "lodash-es": "^4.17.21", 40 | "parse5": "^8.0.0", 41 | "parse5-htmlparser2-tree-adapter": "^8.0.0", 42 | "set-cookie-parser": "^2.7.1" 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /packages/parser/src/utils.ts: -------------------------------------------------------------------------------- 1 | import type { Expr, RequestExpr } from '@getlang/ast' 2 | import { isToken, t } from '@getlang/ast' 3 | 4 | export const render = (template: Expr) => { 5 | if (template.kind !== 'TemplateExpr') { 6 | return null 7 | } 8 | const els = template.elements 9 | return els?.every(isToken) ? els.map(el => el.value).join('') : null 10 | } 11 | 12 | export function getContentField(req: RequestExpr) { 13 | const accept = req.headers.entries.find( 14 | e => render(e.key)?.toLowerCase() === 'accept', 15 | ) 16 | switch (accept && render(accept.value)?.toLowerCase()) { 17 | case 'application/json': 18 | return 'json' 19 | case 'application/javascript': 20 | return 'js' 21 | default: 22 | return 'html' 23 | } 24 | } 25 | 26 | function token(text: string, value = text) { 27 | return { 28 | text, 29 | value, 30 | lineBreaks: 0, 31 | offset: 0, 32 | line: 0, 33 | col: 0, 34 | } 35 | } 36 | 37 | function ident(id: string) { 38 | return t.identifierExpr(token(id)) 39 | } 40 | 41 | function template(contents: string) { 42 | return t.templateExpr([token(contents)]) 43 | } 44 | 45 | function select(selector: string) { 46 | return t.selectorExpr(template(selector), false) 47 | } 48 | 49 | export const tx = { token, ident, template, select } 50 | -------------------------------------------------------------------------------- /packages/parser/src/passes/desugar/urlinputs.ts: -------------------------------------------------------------------------------- 1 | import type { TemplateExpr } from '@getlang/ast' 2 | import { isToken, t } from '@getlang/ast' 3 | import { ScopeTracker, transform } from '@getlang/walker' 4 | import { tx } from '../../utils.js' 5 | import type { DesugarPass } from '../desugar.js' 6 | 7 | export const addUrlInputs: DesugarPass = ast => { 8 | const scope = new ScopeTracker() 9 | const implied = new Set() 10 | 11 | return transform(ast, { 12 | scope, 13 | 14 | RequestExpr: { 15 | enter(node) { 16 | function walkUrl(t: TemplateExpr) { 17 | for (const el of t.elements) { 18 | if (isToken(el)) { 19 | // continue 20 | } else if (el.kind === 'TemplateExpr') { 21 | walkUrl(el) 22 | } else if (el.kind === 'IdentifierExpr') { 23 | const id = el.id.value 24 | if (el.isUrlComponent && !scope.vars[id]) { 25 | implied.add(el.id.value) 26 | } 27 | } 28 | } 29 | } 30 | 31 | walkUrl(node.url) 32 | }, 33 | }, 34 | 35 | Program(node) { 36 | if (implied.size) { 37 | let decl = node.body.find(s => s.kind === 'DeclInputsStmt') 38 | if (!decl) { 39 | decl = t.declInputsStmt([]) 40 | node.body.unshift(decl) 41 | } 42 | for (const i of implied) { 43 | decl.inputs.push(t.InputExpr(tx.token(i), false)) 44 | } 45 | } 46 | }, 47 | }) 48 | } 49 | -------------------------------------------------------------------------------- /packages/lib/src/values/html/patch-dom.ts: -------------------------------------------------------------------------------- 1 | import ds from 'dom-serializer' 2 | // @sideEffects 3 | import type { ElementType } from 'domelementtype' 4 | import type { AnyNode } from 'domhandler' 5 | import { Element, Node } from 'domhandler' 6 | 7 | declare module 'domhandler' { 8 | class Attribute extends Node { 9 | type: ElementType.Text 10 | get nodeType(): 2 11 | value: string 12 | } 13 | 14 | type AnyHtmlNode = AnyNode | Attribute 15 | } 16 | 17 | function main() { 18 | Element.prototype.toString = function () { 19 | return ds(this) 20 | } 21 | 22 | Object.defineProperty(Node.prototype, 'nodeName', { 23 | get: function () { 24 | return this.name 25 | }, 26 | }) 27 | 28 | Object.defineProperty(Node.prototype, 'localName', { 29 | get: function () { 30 | return this.name 31 | }, 32 | }) 33 | 34 | const origAttributes = Object.getOwnPropertyDescriptor( 35 | Element.prototype, 36 | 'attributes', 37 | )?.get 38 | 39 | if (origAttributes) { 40 | Object.defineProperty(Element.prototype, 'attributes', { 41 | get: function (...args) { 42 | const attrs = origAttributes.call(this, ...args) 43 | attrs.item = (idx: number) => { 44 | const el = attrs[idx] 45 | return { ...el, nodeType: 2, localName: el.name } 46 | } 47 | return attrs 48 | }, 49 | }) 50 | } else { 51 | console.warn( 52 | '[WARN] Unable to patch DOM: Element.attributes property descriptor not found', 53 | ) 54 | } 55 | } 56 | 57 | main() 58 | -------------------------------------------------------------------------------- /packages/parser/src/passes/lineage.ts: -------------------------------------------------------------------------------- 1 | import type { Expr, Node } from '@getlang/ast' 2 | import { invariant } from '@getlang/lib' 3 | import { ValueReferenceError } from '@getlang/lib/errors' 4 | import type { Path } from '@getlang/walker' 5 | import { ScopeTracker } from '@getlang/walker' 6 | 7 | export class LineageTracker extends ScopeTracker { 8 | private lineage = new Map() 9 | 10 | getLineage(expr: Expr) { 11 | return this.lineage.get(expr) 12 | } 13 | 14 | traceLineageRoot(expr: Expr) { 15 | let parent = this.lineage.get(expr) 16 | while (parent && this.lineage.has(parent)) { 17 | parent = this.lineage.get(parent) 18 | } 19 | return parent 20 | } 21 | 22 | override exit(node: Node, path: Path) { 23 | const derive = (base: Expr) => this.lineage.set(node as Expr, base) 24 | 25 | switch (node.kind) { 26 | case 'IdentifierExpr': 27 | case 'DrillIdentifierExpr': { 28 | const id = node.id.value 29 | const value = this.lookup(id) 30 | invariant(value, new ValueReferenceError(id)) 31 | derive(value) 32 | break 33 | } 34 | 35 | case 'DrillExpr': 36 | derive(node.body.at(-1)!) 37 | break 38 | 39 | case 'ModifierExpr': 40 | case 'SelectorExpr': 41 | derive(this.context!) 42 | break 43 | 44 | case 'SubqueryExpr': 45 | if (this.extracted) { 46 | derive(this.extracted) 47 | } 48 | break 49 | } 50 | 51 | super.exit(node, path) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /packages/get/src/value.ts: -------------------------------------------------------------------------------- 1 | import type { TypeInfo } from '@getlang/ast' 2 | import { Type } from '@getlang/ast' 3 | import { cookies, headers, html, js, NullSelection } from '@getlang/lib' 4 | import { mapValues } from 'lodash-es' 5 | import { 6 | NullSelectionError, 7 | ValueTypeError, 8 | } from '../../lib/src/core/errors.js' 9 | 10 | export type RuntimeValue = { 11 | data: any 12 | typeInfo: TypeInfo 13 | } 14 | 15 | export function materialize({ data, typeInfo }: RuntimeValue): any { 16 | switch (typeInfo.type) { 17 | case Type.Html: 18 | return html.toValue(data) 19 | case Type.Js: 20 | return js.toValue(data) 21 | case Type.Headers: 22 | return headers.toValue(data) 23 | case Type.Cookies: 24 | return cookies.toValue(data) 25 | case Type.List: 26 | return data.map((item: any) => 27 | materialize({ data: item, typeInfo: typeInfo.of }), 28 | ) 29 | case Type.Struct: 30 | return mapValues(data, (v, k) => 31 | materialize({ data: v, typeInfo: typeInfo.schema[k]! }), 32 | ) 33 | case Type.Maybe: 34 | return materialize({ data, typeInfo: typeInfo.option }) 35 | case Type.Value: 36 | return data 37 | default: 38 | throw new ValueTypeError('Unsupported conversion type') 39 | } 40 | } 41 | 42 | export function assert(value: RuntimeValue) { 43 | const optional = value.typeInfo.type === Type.Maybe 44 | if (!optional && value.data instanceof NullSelection) { 45 | throw new NullSelectionError(value.data.selector) 46 | } 47 | return value 48 | } 49 | -------------------------------------------------------------------------------- /packages/walker/src/path.ts: -------------------------------------------------------------------------------- 1 | import type { Node } from '@getlang/ast' 2 | 3 | type Staging = { 4 | before: Node[] 5 | } 6 | 7 | type Mutation = Map 8 | 9 | export class Path { 10 | private staging: Staging = { before: [] } 11 | protected mutations: Mutation = new Map() 12 | public replacement?: { value: unknown } 13 | 14 | constructor( 15 | public node: N, 16 | public parent?: Path, 17 | ) {} 18 | 19 | add(node: N) { 20 | return new Path(node, this) 21 | } 22 | 23 | insertBefore(node: Node) { 24 | this.staging.before.push(node) 25 | } 26 | 27 | replace(value?: any) { 28 | this.replacement = { value } 29 | } 30 | 31 | private mutate(node: Node) { 32 | if (this.mutations.size === 0) { 33 | return node 34 | } 35 | 36 | const entries = [] 37 | for (const [key, value] of Object.entries(node)) { 38 | let val = value 39 | if (Array.isArray(value)) { 40 | val = value.flatMap(el => { 41 | const mut = this.mutations.get(el) 42 | const { before = [] } = mut ?? {} 43 | return [...before, el] 44 | }) 45 | } 46 | entries.push([key, val]) 47 | } 48 | 49 | return Object.fromEntries(entries) 50 | } 51 | 52 | apply(node: Node) { 53 | const applied = this.mutate(node) 54 | if (this.staging.before.length) { 55 | if (!this.parent) { 56 | throw new Error('Unable to apply path mutations') 57 | } 58 | this.parent.mutations.set(applied, this.staging) 59 | } 60 | return applied 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /packages/parser/src/passes/desugar/context.ts: -------------------------------------------------------------------------------- 1 | import { invariant } from '@getlang/lib' 2 | import { QuerySyntaxError } from '@getlang/lib/errors' 3 | import { ScopeTracker, transform } from '@getlang/walker' 4 | import type { DesugarPass } from '../desugar.js' 5 | 6 | export const resolveContext: DesugarPass = (ast, { parsers, contextual }) => { 7 | const scope = new ScopeTracker() 8 | 9 | const program = transform(ast, { 10 | scope, 11 | 12 | Program(node) { 13 | const body = parsers.insert(node.body) 14 | return { ...node, body } 15 | }, 16 | 17 | SubqueryExpr(node) { 18 | const body = parsers.insert(node.body) 19 | return { ...node, body } 20 | }, 21 | 22 | RequestExpr(node) { 23 | invariant(node.headers.kind === 'RequestBlockExpr', '') 24 | node.headers 25 | parsers.visit(node) 26 | }, 27 | 28 | SelectorExpr(_node, path) { 29 | const ctx = scope.context 30 | if (ctx?.kind === 'RequestExpr') { 31 | path.insertBefore(parsers.lookup(ctx)) 32 | } 33 | }, 34 | 35 | ModifierExpr(node) { 36 | const ctx = scope.context 37 | const mod = node.modifier.value 38 | if (contextual.includes(mod) && ctx?.kind === 'RequestExpr') { 39 | // replace modifier with shared parser 40 | return parsers.lookup(ctx, mod) 41 | } 42 | }, 43 | 44 | ModuleExpr(node, path) { 45 | const ctx = scope.context 46 | const module = node.module.value 47 | if (contextual.includes(module) && ctx?.kind === 'RequestExpr') { 48 | path.insertBefore(parsers.lookup(ctx)) 49 | } 50 | }, 51 | }) 52 | 53 | invariant( 54 | program.kind === 'Program', 55 | new QuerySyntaxError('Context inference exception'), 56 | ) 57 | 58 | return program 59 | } 60 | -------------------------------------------------------------------------------- /test/helpers.ts: -------------------------------------------------------------------------------- 1 | import { expect } from 'bun:test' 2 | import { executeModule } from '@getlang/get' 3 | import type { Hooks, Inputs, MaybePromise } from '@getlang/lib' 4 | import { invariant } from '@getlang/lib' 5 | import { ImportError } from '@getlang/lib/errors' 6 | import { desugar, parse, print } from '@getlang/parser' 7 | import dedent from 'dedent' 8 | import './expect.js' 9 | 10 | type ExecuteOptions = Partial<{ 11 | fetch: Fetch 12 | willThrow: boolean 13 | hooks: Hooks 14 | }> 15 | 16 | export type Fetch = (url: string, opts: RequestInit) => MaybePromise 17 | 18 | export const SELSYN = true 19 | 20 | function testIdempotency(source: string) { 21 | const print1 = print(desugar(parse(source))) 22 | const print2 = print(desugar(parse(print1))) 23 | expect(print1).toEqual(print2) 24 | } 25 | 26 | export async function execute( 27 | program: string | Record, 28 | inputs?: Inputs, 29 | options: ExecuteOptions = {}, 30 | ): Promise { 31 | const { fetch, willThrow } = options 32 | const normalized = typeof program === 'string' ? { Home: program } : program 33 | const modules: Record = {} 34 | for (const [name, source] of Object.entries(normalized)) { 35 | modules[name] = dedent(source) 36 | if (!willThrow) { 37 | testIdempotency(source) 38 | } 39 | } 40 | 41 | const hooks: Hooks = { 42 | import(module) { 43 | const src = modules[module] 44 | invariant(src, new ImportError(`Failed to import module: ${module}`)) 45 | return src 46 | }, 47 | async request(url, opts) { 48 | invariant(fetch, `Fetch required: ${url}`) 49 | const res = await fetch(url, opts) 50 | return { 51 | status: res.status, 52 | headers: res.headers, 53 | body: await res.text(), 54 | } 55 | }, 56 | ...options.hooks, 57 | } 58 | 59 | return executeModule('Home', inputs, hooks) 60 | } 61 | -------------------------------------------------------------------------------- /packages/lib/src/core/hooks.ts: -------------------------------------------------------------------------------- 1 | import type { TypeInfo } from '@getlang/ast' 2 | import type { Inputs, MaybePromise } from '../index.js' 3 | import { requestHook } from '../net/http.js' 4 | import { runSlice } from '../slice.js' 5 | import { ImportError, invariant } from './errors.js' 6 | 7 | export type ImportHook = (module: string) => MaybePromise 8 | 9 | export type CallHook = (module: string, inputs: Inputs) => MaybePromise 10 | 11 | export type RequestHook = ( 12 | url: string, 13 | opts: RequestInit, 14 | ) => MaybePromise 15 | 16 | export type SliceHook = ( 17 | slice: string, 18 | context?: unknown, 19 | ) => MaybePromise 20 | 21 | export type ExtractHook = ( 22 | module: string, 23 | inputs: Inputs, 24 | value: any, 25 | ) => MaybePromise 26 | 27 | export type Modifier = (context: any, options: Record) => any 28 | export type ModifierHook = (modifier: string) => MaybePromise< 29 | | { 30 | modifier: Modifier 31 | useContext?: boolean 32 | materialize?: boolean 33 | returnType?: TypeInfo 34 | } 35 | | undefined 36 | > 37 | 38 | export type Hooks = Partial<{ 39 | import: ImportHook 40 | request: RequestHook 41 | slice: SliceHook 42 | call: CallHook 43 | extract: ExtractHook 44 | modifier: ModifierHook 45 | }> 46 | 47 | type RequestInit = { 48 | method?: string 49 | headers?: Headers 50 | body?: string 51 | } 52 | 53 | export type Response = { 54 | status: number 55 | headers: Headers 56 | body?: string 57 | } 58 | 59 | export function buildHooks(hooks: Hooks): Required { 60 | return { 61 | import: (module: string) => { 62 | const err = 'Imports are not supported by the current runtime' 63 | invariant(hooks.import, new ImportError(err)) 64 | return hooks.import(module) 65 | }, 66 | modifier: modifier => hooks.modifier?.(modifier), 67 | call: hooks.call ?? (() => {}), 68 | request: hooks.request ?? requestHook, 69 | slice: hooks.slice ?? runSlice, 70 | extract: hooks.extract ?? (() => {}), 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /test/objects.spec.ts: -------------------------------------------------------------------------------- 1 | import { describe, expect, test } from 'bun:test' 2 | import { execute } from './helpers.js' 3 | 4 | describe('objects', () => { 5 | test('inline', async () => { 6 | const result = await execute('extract { x: `"object"` }') 7 | expect(result).toEqual({ x: 'object' }) 8 | }) 9 | 10 | test('variable', async () => { 11 | const result = await execute(` 12 | set x = { test: |true| } 13 | extract $x 14 | `) 15 | expect(result).toEqual({ test: true }) 16 | }) 17 | 18 | test('variable ref', async () => { 19 | const result = await execute(` 20 | set x = |"varref"| 21 | extract { x: $x } 22 | `) 23 | expect(result).toEqual({ x: 'varref' }) 24 | }) 25 | 26 | test('variable ref shorthand', async () => { 27 | const result = await execute(` 28 | set x = |"varref"| 29 | extract { $x } 30 | `) 31 | expect(result).toEqual({ x: 'varref' }) 32 | }) 33 | 34 | test('variable combination', async () => { 35 | const result = await execute(` 36 | set foo = |"foo"| 37 | set bar = |"bar"| 38 | extract { 39 | $foo, 40 | bar: $bar 41 | baz: |"baz"| 42 | xyz: $foo 43 | } 44 | `) 45 | 46 | expect(result).toEqual({ 47 | foo: 'foo', 48 | bar: 'bar', 49 | baz: 'baz', 50 | xyz: 'foo', 51 | }) 52 | }) 53 | 54 | test('nested inline', async () => { 55 | const result = await execute(` 56 | extract { 57 | pos: |"outer"|, 58 | value: { 59 | pos: |"inner"| 60 | } 61 | } 62 | `) 63 | expect(result).toEqual({ 64 | pos: 'outer', 65 | value: { 66 | pos: 'inner', 67 | }, 68 | }) 69 | }) 70 | 71 | test('nested variable ref shorthand', async () => { 72 | const result = await execute(` 73 | set value = { pos: |"inner"| } 74 | extract { 75 | pos: |"outer"|, 76 | $value 77 | } 78 | `) 79 | expect(result).toEqual({ 80 | pos: 'outer', 81 | value: { 82 | pos: 'inner', 83 | }, 84 | }) 85 | }) 86 | }) 87 | -------------------------------------------------------------------------------- /packages/lib/src/net/http.ts: -------------------------------------------------------------------------------- 1 | import { RequestError } from '../core/errors.js' 2 | import type { RequestHook } from '../core/hooks.js' 3 | 4 | type StringMap = Record 5 | 6 | type Blocks = { 7 | query?: StringMap 8 | cookies?: StringMap 9 | json?: StringMap 10 | form?: StringMap 11 | } 12 | 13 | // RFC 3986 compliance 14 | // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent#description 15 | const fixedEncodeURIComponent = (str: string) => { 16 | return encodeURIComponent(str).replace( 17 | /[!'()*]/g, 18 | c => `%${c.charCodeAt(0).toString(16).toUpperCase()}`, 19 | ) 20 | } 21 | 22 | export const requestHook: RequestHook = async (url, opts) => { 23 | const res = await fetch(url, opts) 24 | return { 25 | status: res.status, 26 | headers: res.headers, 27 | body: await res.text(), 28 | } 29 | } 30 | 31 | function constructUrl(start: string, query: StringMap = {}) { 32 | let url: URL 33 | let stripProtocol = false 34 | 35 | try { 36 | url = new URL(start) 37 | } catch (_) { 38 | url = new URL(`http://${start}`) 39 | stripProtocol = true 40 | } 41 | 42 | for (const entry of Object.entries(query)) { 43 | url.searchParams.append(...entry) 44 | } 45 | 46 | const str = url.toString() 47 | return stripProtocol ? str.slice(7) : str 48 | } 49 | 50 | export const request = async ( 51 | method: string, 52 | url: string, 53 | _headers: StringMap, 54 | blocks: Blocks, 55 | bodyRaw: string, 56 | hook: RequestHook, 57 | ) => { 58 | const urlString = constructUrl(url, blocks.query) 59 | 60 | // construct headers 61 | const headers = new Headers(_headers) 62 | if (blocks.cookies) { 63 | const pairs = Object.entries(blocks.cookies).map(entry => 64 | entry.map(fixedEncodeURIComponent).join('='), 65 | ) 66 | const cookieHeader = pairs.join('; ') 67 | headers.set('cookie', cookieHeader) 68 | } 69 | 70 | // construct body 71 | let body: string | undefined 72 | if (bodyRaw) { 73 | body = bodyRaw 74 | } else if (blocks.json) { 75 | body = JSON.stringify(blocks.json) 76 | } 77 | 78 | // make request 79 | try { 80 | const res = await hook(urlString, { method, headers, body }) 81 | return { url: urlString, ...res } 82 | } catch (e) { 83 | throw new RequestError(urlString, { cause: e }) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /packages/parser/src/passes/desugar/links.ts: -------------------------------------------------------------------------------- 1 | import { t } from '@getlang/ast' 2 | import { invariant } from '@getlang/lib' 3 | import { QuerySyntaxError } from '@getlang/lib/errors' 4 | import { transform } from '@getlang/walker' 5 | import { render, tx } from '../../utils.js' 6 | import type { DesugarPass } from '../desugar.js' 7 | import { LineageTracker } from '../lineage.js' 8 | 9 | export const settleLinks: DesugarPass = (ast, { parsers }) => { 10 | const scope = new LineageTracker() 11 | 12 | const ret = transform(ast, { 13 | scope, 14 | 15 | ModifierExpr(node) { 16 | invariant( 17 | node.args.kind === 'ObjectLiteralExpr', 18 | new QuerySyntaxError('Modifier options must be an object'), 19 | ) 20 | 21 | const ctx = scope.context 22 | if (node.modifier.value === 'link' && ctx) { 23 | const lineage = scope.traceLineageRoot(ctx) 24 | const hasBase = node.args.entries.some(e => render(e.key) === 'base') 25 | if (lineage?.kind === 'RequestExpr' && !hasBase) { 26 | node.args.entries.push( 27 | t.objectEntryExpr( 28 | tx.template('base'), 29 | parsers.lookup(lineage, 'link'), 30 | ), 31 | ) 32 | } 33 | } 34 | }, 35 | 36 | ModuleExpr(node) { 37 | const linkArg = node.args.entries.find(e => render(e.key) === '@link') 38 | if (!linkArg) { 39 | return 40 | } 41 | const { value } = linkArg 42 | invariant(value.kind === 'DrillExpr', 'Module links [1]') 43 | const base = scope.getLineage(value) 44 | invariant(base, 'Module links [2]') 45 | if (base.kind === 'ModifierExpr') { 46 | return 47 | } 48 | const root = scope.traceLineageRoot(value) 49 | invariant(root?.kind === 'RequestExpr', 'Module links [3]') 50 | const mod = t.modifierExpr( 51 | tx.token('link'), 52 | t.objectLiteralExpr([ 53 | t.objectEntryExpr(tx.template('base'), parsers.lookup(root, 'link')), 54 | ]), 55 | ) 56 | value.body.push(mod) 57 | }, 58 | 59 | RequestExpr(node) { 60 | parsers.visit(node) 61 | }, 62 | 63 | Program(node) { 64 | const body = parsers.insert(node.body) 65 | return { ...node, body } 66 | }, 67 | 68 | SubqueryExpr(node) { 69 | const body = parsers.insert(node.body) 70 | return { ...node, body } 71 | }, 72 | }) 73 | 74 | return ret 75 | } 76 | -------------------------------------------------------------------------------- /packages/walker/src/scope.ts: -------------------------------------------------------------------------------- 1 | import type { Node } from '@getlang/ast' 2 | import type { Path } from './index.js' 3 | 4 | class Scope { 5 | extracted?: T 6 | 7 | constructor( 8 | public vars: { [name: string]: T }, 9 | public context: T | undefined, 10 | ) {} 11 | 12 | lookup(id: string) { 13 | const value = id ? this.vars[id] : this.context 14 | return value ?? null 15 | } 16 | } 17 | 18 | export class ScopeTracker { 19 | private scopeStack: Scope[] = [] 20 | 21 | push(context: T | undefined = this.head?.context) { 22 | const vars = Object.create(this.head?.vars ?? null) 23 | this.scopeStack.push(new Scope(vars, context)) 24 | } 25 | 26 | pop() { 27 | this.scopeStack.pop() 28 | } 29 | 30 | private get head() { 31 | return this.scopeStack.at(-1) 32 | } 33 | 34 | private get ensure() { 35 | if (!this.head) { 36 | throw new Error('Invalid scope stack') 37 | } 38 | return this.head 39 | } 40 | 41 | set context(value: T) { 42 | this.ensure.context = value 43 | } 44 | 45 | get context(): T | undefined { 46 | return this.ensure.context 47 | } 48 | 49 | get vars() { 50 | return this.ensure.vars 51 | } 52 | 53 | get extracted(): T | undefined { 54 | return this.ensure.extracted 55 | } 56 | 57 | set extracted(data: T) { 58 | this.ensure.extracted = data 59 | } 60 | 61 | lookup(id: string) { 62 | return this.ensure.lookup(id) 63 | } 64 | 65 | enter(node: Node) { 66 | switch (node.kind) { 67 | case 'Program': 68 | case 'SubqueryExpr': 69 | case 'DrillExpr': 70 | this.push() 71 | break 72 | } 73 | } 74 | 75 | exit(xnode: any, path: Path) { 76 | switch (path.node.kind) { 77 | case 'Program': 78 | case 'SubqueryExpr': 79 | case 'DrillExpr': 80 | this.pop() 81 | break 82 | 83 | case 'RequestStmt': 84 | this.context = xnode.request 85 | break 86 | 87 | case 'InputExpr': 88 | this.vars[path.node.id.value] = xnode 89 | break 90 | 91 | case 'AssignmentStmt': 92 | this.vars[path.node.name.value] = xnode.value 93 | break 94 | 95 | case 'ExtractStmt': 96 | this.extracted = xnode.value 97 | break 98 | } 99 | 100 | if (path.parent?.node.kind === 'DrillExpr') { 101 | this.context = xnode 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /packages/lib/src/values/html.ts: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | import xpath from '@getlang/xpath' 4 | import { selectAll, selectOne } from 'css-select' 5 | import { parse as parseCss } from 'css-what' 6 | import type { AnyHtmlNode } from 'domhandler' 7 | import { textContent } from 'domutils' 8 | import { parse as parse5 } from 'parse5' 9 | import { adapter } from 'parse5-htmlparser2-tree-adapter' 10 | import { 11 | invariant, 12 | NullSelection, 13 | NullSelectionError, 14 | SelectorSyntaxError, 15 | } from '../core/errors.js' 16 | import './html/patch-dom.js' 17 | 18 | export { Element } from 'domhandler' 19 | 20 | export const parse = (html: string): AnyHtmlNode => { 21 | return parse5(html, { treeAdapter: adapter }) 22 | } 23 | 24 | const selectXpath = (el: AnyHtmlNode, selector: string, expand: boolean) => { 25 | try { 26 | const parseXpath = new xpath.XPathParser() 27 | parseXpath.parse(selector) 28 | } catch (e) { 29 | throw new SelectorSyntaxError('XPath', selector, { cause: e }) 30 | } 31 | 32 | let root = el 33 | if (el.nodeType === 9) { 34 | // Document -> HtmlElement 35 | const html = el.childNodes.find(x => x.nodeType === 1 && x.name === 'html') 36 | invariant(html, new NullSelectionError(selector)) 37 | root = html 38 | } 39 | 40 | const value = xpath.select(selector, root) 41 | if (expand) { 42 | return value 43 | } 44 | return value.length ? value[0] : new NullSelection(selector) 45 | } 46 | 47 | const selectCss = (el: AnyHtmlNode, selector: string, expand: boolean) => { 48 | try { 49 | parseCss(selector) 50 | } catch (e) { 51 | throw new SelectorSyntaxError('CSS', selector, { cause: e }) 52 | } 53 | const value = expand ? selectAll(selector, el) : selectOne(selector, el) 54 | if (expand) { 55 | return value ?? [] 56 | } 57 | return value === null ? new NullSelection(selector) : value 58 | } 59 | 60 | export const select = (el: AnyHtmlNode, selector: string, expand: boolean) => { 61 | return /^(\/|\.\/)/.test(selector) 62 | ? selectXpath(el, selector, expand) 63 | : selectCss(el, selector, expand) 64 | } 65 | 66 | export const toValue = (el: AnyHtmlNode) => { 67 | let str = '' 68 | if (el.nodeType === 2) { 69 | str = el.value 70 | } else if (textContent(el)) { 71 | str = textContent(el).replaceAll(/\s+/g, ' ') 72 | } 73 | return str.trim() 74 | } 75 | 76 | export const findLink = (el: AnyHtmlNode) => { 77 | const tag = el.type === 'tag' && el.name 78 | switch (tag) { 79 | case 'a': 80 | return selectXpath(el, '@href', false) 81 | case 'img': 82 | return selectXpath(el, '@src', false) 83 | default: 84 | return el 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /packages/parser/src/grammar/getlang.ne: -------------------------------------------------------------------------------- 1 | @{% 2 | import lexer from './grammar/lexer.js' 3 | import * as p from './grammar/parse.js' 4 | %} 5 | 6 | @preprocessor typescript 7 | @lexer lexer 8 | 9 | # structure 10 | program -> _ (inputs line_sep):? statements _ {% p.program %} 11 | statements -> statement (line_sep statement):* {% p.statements %} 12 | statement -> (request | assignment | extract) {% p.idd %} 13 | 14 | # keywords 15 | inputs -> "inputs" __ "{" _ input_decl (_ "," _ input_decl):* _ "}" {% p.declInputs %} 16 | assignment -> "set" __ %identifier "?":? _ "=" _ expression {% p.assignment %} 17 | extract -> "extract" __ expression {% p.extract %} 18 | 19 | # inputs 20 | input_decl -> %identifier "?":? (_ "=" _ input_default):? {% p.inputDecl %} 21 | input_default -> slice {% id %} 22 | 23 | # request 24 | request -> %request_verb template (line_sep request_block):? request_blocks {% p.request %} 25 | request_blocks -> (line_sep request_block_named):* (line_sep request_block_body):? {% p.requestBlocks %} 26 | request_block_named -> %request_block_name line_sep request_block {% p.requestBlockNamed %} 27 | request_block -> request_entry (line_sep request_entry):* {% p.requestBlock %} 28 | request_entry -> template ":" (__ template):? {% p.requestEntry %} 29 | request_block_body -> %request_block_body template %request_block_body_end {% p.requestBlockBody %} 30 | 31 | # expression 32 | expression -> drill {% id %} 33 | expression -> (drill _ %drill_arrow _):? %link _ drill {% p.link %} 34 | 35 | # drill 36 | drill -> (%drill_arrow _):? bit (_ %drill_arrow _ bit):* {% p.drill %} 37 | 38 | # drill bit 39 | bit -> (literal | slice | call | object | subquery) {% p.idd %} 40 | bit -> template {% p.selector %} 41 | bit -> %identifier_expr {% p.idbit %} 42 | 43 | # subqueries 44 | subquery -> "(" _ statements _ ")" {% p.subquery %} 45 | 46 | # calls 47 | call -> %call ("(" object ")"):? {% p.call %} 48 | 49 | # object literals 50 | object -> "{" _ (object_entry (_ ","):? _):* "}" {% p.object %} 51 | object_entry -> "@":? %identifier "?":? ":" _ expression {% p.objectEntry %} 52 | object_entry -> %identifier "?":? {% p.objectEntryShorthandSelect %} 53 | object_entry -> %identifier_expr "?":? {% p.objectEntryShorthandIdent %} 54 | 55 | # templates 56 | template -> (%str | %interpvar | interp_expr | interp_tmpl):+ {% p.template %} 57 | interp_expr -> "${" _ %identifier _ "}" {% p.interpExpr %} 58 | interp_tmpl -> "$[" _ template _ "]" {% p.interpTmpl %} 59 | 60 | # literals 61 | literal -> (%bool | %num) {% p.literal %} 62 | literal -> "'" template "'" {% p.string %} 63 | literal -> "\"" template "\"" {% p.string %} 64 | slice -> %slice {% p.slice %} 65 | 66 | # whitespace 67 | line_sep -> (%ws | %comment):* %nl _ {% p.ws %} 68 | __ -> ws:+ {% p.ws %} 69 | _ -> ws:* {% p.ws %} 70 | ws -> (%ws | %comment | %nl) {% p.ws %} 71 | -------------------------------------------------------------------------------- /packages/walker/src/index.ts: -------------------------------------------------------------------------------- 1 | import type { Node } from '@getlang/ast' 2 | import { Path } from './path.js' 3 | import type { ScopeTracker } from './scope.js' 4 | import type { 5 | NodeConfig, 6 | NodeVisitor, 7 | ReduceVisitor, 8 | TransformVisitor, 9 | } from './visitor.js' 10 | import { wait, waitMap } from './wait.js' 11 | 12 | export { ScopeTracker } from './scope.js' 13 | export type { TransformVisitor, ReduceVisitor, Path } 14 | 15 | export type WalkOptions = Visitor & { 16 | scope?: ScopeTracker 17 | } 18 | 19 | function normalize( 20 | visitor?: NodeConfig, 21 | ): NodeVisitor { 22 | if (!visitor) { 23 | return { enter: () => {}, exit: () => {} } 24 | } else if (typeof visitor === 'function') { 25 | return { enter: () => {}, exit: visitor } 26 | } else { 27 | return { 28 | enter: visitor.enter || (() => {}), 29 | exit: visitor.exit || (() => {}), 30 | } 31 | } 32 | } 33 | 34 | const isNode = (test: unknown): test is Node => 35 | typeof test === 'object' && 36 | test !== null && 37 | 'kind' in test && 38 | typeof test.kind === 'string' 39 | 40 | function walk( 41 | node: N, 42 | visitor: TransformVisitor, 43 | scope?: ScopeTracker, 44 | parent?: Path, 45 | ) { 46 | const config = visitor[node.kind] as NodeConfig 47 | const { enter, exit } = normalize(config) 48 | 49 | scope?.enter(node) 50 | 51 | const path = parent?.add(node) || new Path(node) 52 | return wait(enter(node, path), () => { 53 | let xnode: any 54 | if (path.replacement) { 55 | xnode = path.replacement.value 56 | } else { 57 | const entries = waitMap(Object.entries(node), e => { 58 | const [key, value] = e 59 | let val = value 60 | if (Array.isArray(value)) { 61 | val = waitMap(value, el => 62 | isNode(el) ? walk(el, visitor, scope, path) : el, 63 | ) 64 | } else if (isNode(value)) { 65 | val = walk(value, visitor, scope, path) 66 | } 67 | return wait(val, x => [key, x]) 68 | }) 69 | 70 | const tnode = wait(entries, Object.fromEntries) 71 | 72 | xnode = wait(tnode, t => { 73 | return wait(exit(t, path), x => x || t) 74 | }) 75 | } 76 | 77 | return wait(xnode, xnode => { 78 | const applied = path.apply(xnode) 79 | scope?.exit(applied, path) 80 | return applied 81 | }) 82 | }) 83 | } 84 | 85 | export function transform(node: Node, options: WalkOptions) { 86 | return walk(node, options, options.scope) 87 | } 88 | 89 | export function reduce( 90 | node: Node, 91 | options: WalkOptions>, 92 | ) { 93 | return walk(node, options as TransformVisitor, options.scope) 94 | } 95 | -------------------------------------------------------------------------------- /packages/parser/src/passes/desugar/reqparse.ts: -------------------------------------------------------------------------------- 1 | import type { RequestExpr, Stmt } from '@getlang/ast' 2 | import { t } from '@getlang/ast' 3 | import { invariant } from '@getlang/lib' 4 | import { QuerySyntaxError } from '@getlang/lib/errors' 5 | import { getContentField, tx } from '../../utils.js' 6 | 7 | type Parsers = Record 8 | 9 | export class RequestParsers { 10 | private requests: RequestExpr[] = [] 11 | private parsers: Parsers[] = [] 12 | 13 | private require(req: RequestExpr) { 14 | const idx = this.requests.indexOf(req) 15 | invariant(idx !== -1, new QuerySyntaxError('Unmapped request')) 16 | return idx 17 | } 18 | 19 | private id(idx: number, field: string) { 20 | return `__${field}_${idx}` 21 | } 22 | 23 | visit(req: RequestExpr) { 24 | let idx = this.requests.indexOf(req) 25 | if (idx === -1) { 26 | idx = this.requests.length 27 | this.requests.push(req) 28 | } 29 | } 30 | 31 | lookup(req: RequestExpr, field: string = getContentField(req)) { 32 | const idx = this.require(req) 33 | this.parsers[idx] ??= {} 34 | const parsers = this.parsers[idx] 35 | parsers[field] ??= { written: false } 36 | return tx.ident(this.id(idx, field)) 37 | } 38 | 39 | private writeParser(idx: number, field: string): Stmt { 40 | const req = t.drillIdentifierExpr(tx.token(''), false) 41 | const id = tx.token(this.id(idx, field)) 42 | const modbit = t.modifierExpr(tx.token(field)) 43 | 44 | switch (field) { 45 | case 'link': { 46 | const expr = t.drillExpr([req, tx.select('url')]) 47 | return t.assignmentStmt(id, expr, false) 48 | } 49 | 50 | case 'headers': { 51 | const expr = t.drillExpr([req, tx.select('headers')]) 52 | return t.assignmentStmt(id, expr, false) 53 | } 54 | 55 | case 'cookies': { 56 | const expr = t.drillExpr([ 57 | req, 58 | tx.select('headers'), 59 | tx.select('set-cookie'), 60 | modbit, 61 | ]) 62 | return t.assignmentStmt(id, expr, true) 63 | } 64 | 65 | default: { 66 | const expr = t.drillExpr([req, tx.select('body'), modbit]) 67 | return t.assignmentStmt(id, expr, false) 68 | } 69 | } 70 | } 71 | 72 | insert(stmts: Stmt[]) { 73 | return stmts.flatMap(stmt => { 74 | if (stmt.kind !== 'RequestStmt') { 75 | return stmt 76 | } 77 | const idx = this.require(stmt.request) 78 | const parsers = this.parsers[idx] 79 | if (!parsers) { 80 | return stmt 81 | } 82 | const unwritten = Object.entries(parsers).filter(e => !e[1].written) 83 | const parserStmts = unwritten.map(([field, parser]) => { 84 | parser.written = true 85 | return this.writeParser(idx, field) 86 | }) 87 | return [stmt, ...parserStmts] 88 | }) 89 | } 90 | 91 | reset() { 92 | this.requests = [] 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /packages/parser/src/grammar/lex/templates.ts: -------------------------------------------------------------------------------- 1 | import { patterns } from './shared.js' 2 | 3 | type UntilOptions = { 4 | prefix?: RegExp 5 | inclusive?: boolean 6 | } 7 | 8 | // creates a new regex that consumes characters until the 9 | // `term` regex has been reached. the regex is multiline 10 | export const until = (term: RegExp, opts: UntilOptions = {}) => { 11 | const prefix = opts.prefix ? opts.prefix.source : '' 12 | const finalGroup = opts.inclusive ? '?:' : '?=' 13 | return new RegExp( 14 | `${prefix}[^]*?[^\\\\](${finalGroup}${term.source}|(?![^]))`, 15 | ) 16 | } 17 | 18 | type TemplateUntilOptions = { 19 | interpTemplate?: boolean 20 | interpParams?: boolean 21 | next?: string 22 | } 23 | 24 | export const templateUntil = ( 25 | term: RegExp, 26 | opts: TemplateUntilOptions = {}, 27 | ) => { 28 | const { interpTemplate = true, interpParams = false, next } = opts 29 | const interpSymbols = ['$'] 30 | if (interpParams) { 31 | interpSymbols.push(':') 32 | } 33 | 34 | return { 35 | term: { 36 | defaultType: 'str', 37 | match: new RegExp(`(?=${term.source})`), 38 | lineBreaks: true, 39 | ...(next ? { next } : { pop: 1 }), 40 | }, 41 | interpexpr: { 42 | match: '${', 43 | push: 'interpExpr', 44 | }, 45 | ...(interpTemplate 46 | ? { 47 | interptmpl: { 48 | match: '$[', 49 | push: interpParams ? 'interpTmplParams' : 'interpTmpl', 50 | }, 51 | } 52 | : {}), 53 | interpvar: { 54 | match: new RegExp( 55 | `[${interpSymbols.join('')}]${patterns.identifier.source}`, 56 | ), 57 | value: (text: string) => text.slice(1), 58 | }, 59 | str: { 60 | match: until(new RegExp(`[${interpSymbols.join('')}]|${term.source}`)), 61 | value: (text: string) => text.replace(/\\(.)/g, '$1').replace(/\s/g, ' '), 62 | lineBreaks: true, 63 | }, 64 | } 65 | } 66 | 67 | // limited support for now, eventually to support expressions such as: 68 | // ${a + b} 69 | const interpExpr = { 70 | ws: patterns.ws, 71 | identifier: patterns.identifier, 72 | rbrace: { 73 | match: '}', 74 | pop: 1, 75 | }, 76 | } 77 | 78 | const interpTmpl = { 79 | rbrack: { 80 | match: ']', 81 | pop: 1, 82 | }, 83 | ...templateUntil(/]/), 84 | } 85 | 86 | const interpTmplParams = { 87 | rbrack: { 88 | match: ']', 89 | pop: 1, 90 | }, 91 | ...templateUntil(/]/, { interpParams: true }), 92 | } 93 | 94 | const stringS = { 95 | squot: { 96 | match: `'`, 97 | pop: 1, 98 | }, 99 | ...templateUntil(/'/), 100 | } 101 | 102 | const stringD = { 103 | dquot: { 104 | match: '"', 105 | pop: 1, 106 | }, 107 | ...templateUntil(/"/), 108 | } 109 | 110 | export const templateStates = { 111 | template: templateUntil(/\n|->|=>/, { interpTemplate: false }), 112 | interpExpr, 113 | interpTmpl, 114 | interpTmplParams, 115 | stringS, 116 | stringD, 117 | } 118 | -------------------------------------------------------------------------------- /packages/parser/src/grammar/lexer.ts: -------------------------------------------------------------------------------- 1 | import moo from 'moo' 2 | import { requestStates } from './lex/request.js' 3 | import { patterns } from './lex/shared.js' 4 | import { slice, slice_block } from './lex/slice.js' 5 | import { templateStates } from './lex/templates.js' 6 | 7 | const verbs = ['GET', 'PUT', 'POST', 'PATCH', 'DELETE'] 8 | const keywords = ['inputs', 'set'] 9 | 10 | const keywordsObj = Object.fromEntries(keywords.map(k => [`kw_${k}`, k])) 11 | 12 | const main = { 13 | ws: patterns.ws, 14 | nl: { 15 | match: /\n/, 16 | lineBreaks: true, 17 | }, 18 | comment: /--.*/, 19 | kw_extract: { 20 | match: /extract(?=\s)/, 21 | push: 'expr', 22 | }, 23 | drill_arrow: { 24 | match: ['->', '=>'], 25 | push: 'drillExpr', 26 | }, 27 | colon: { 28 | match: ':', 29 | push: 'expr', 30 | }, 31 | assignment: { 32 | match: '=', 33 | push: 'expr', 34 | }, 35 | request_verb: { 36 | match: new RegExp(`(?:${verbs.join('|')}) `), 37 | push: 'requestUrl', 38 | value: (text: string) => text.trim(), 39 | }, 40 | identifier: { 41 | match: patterns.identifier, 42 | type: moo.keywords(keywordsObj), 43 | }, 44 | identifier_expr: { 45 | match: patterns.identifierExpr, 46 | value: (text: string) => text.slice(1), 47 | }, 48 | symbols: /[{}(),?@]/, 49 | } 50 | 51 | const exprBase = { 52 | ws: patterns.ws, 53 | nl: { 54 | match: /\n/, 55 | lineBreaks: true, 56 | }, 57 | link: { 58 | match: patterns.link, 59 | value: (text: string) => text.slice(1, -1), 60 | }, 61 | symbols: { 62 | match: /[{(]/, 63 | pop: 1, 64 | }, 65 | template_interp: { 66 | defaultType: 'ws', 67 | match: /(?=\${)/, 68 | next: 'template', 69 | }, 70 | identifier_expr: { 71 | match: patterns.identifierExpr, 72 | value: (text: string) => text.slice(1), 73 | pop: 1, 74 | }, 75 | slice_block, 76 | slice, 77 | call: { 78 | match: patterns.call, 79 | value: (text: string) => text.slice(1), 80 | pop: 1, 81 | }, 82 | squot: { 83 | match: `'`, 84 | next: 'stringS', 85 | }, 86 | dquot: { 87 | match: `"`, 88 | next: 'stringD', 89 | }, 90 | } 91 | 92 | const expr = { 93 | ...exprBase, 94 | drill_arrow: { 95 | match: ['->', '=>'], 96 | next: 'drillExpr', 97 | }, 98 | num: { 99 | match: /\d+(?:\.\d+)?/, 100 | pop: 1, 101 | }, 102 | bool: { 103 | match: ['true', 'false'], 104 | pop: 1, 105 | }, 106 | template: { 107 | defaultType: 'ws', 108 | match: /(?=.)/, 109 | next: 'template', 110 | }, 111 | } 112 | 113 | const drillExpr = { 114 | ...exprBase, 115 | drill_arrow: ['->', '=>'], 116 | template: { 117 | defaultType: 'ws', 118 | match: /(?=.)/, 119 | next: 'template', 120 | }, 121 | } 122 | 123 | const lexer: any = moo.states({ 124 | $all: { err: moo.error }, 125 | main, 126 | expr, 127 | drillExpr, 128 | ...templateStates, 129 | ...requestStates, 130 | }) 131 | 132 | export default lexer 133 | -------------------------------------------------------------------------------- /test/modifiers.spec.ts: -------------------------------------------------------------------------------- 1 | import { describe, expect, test } from 'bun:test' 2 | import type { Modifier, ModifierHook } from '@getlang/lib' 3 | import { invariant } from '@getlang/lib' 4 | import { ValueTypeError } from '@getlang/lib/errors' 5 | import type { Fetch } from './helpers.js' 6 | import { execute as exec } from './helpers.js' 7 | 8 | function execute( 9 | source: string | Record, 10 | name: string, 11 | fn: Modifier, 12 | useContext?: boolean, 13 | fetch?: Fetch, 14 | ) { 15 | const modifier: ModifierHook = mod => { 16 | expect(mod).toEqual(name) 17 | return { modifier: fn, useContext } 18 | } 19 | return exec(source, {}, { fetch, hooks: { modifier } }) 20 | } 21 | 22 | describe('modifiers', () => { 23 | test('hook', async () => { 24 | const result = await execute('extract @rnd', 'rnd', () => 300) 25 | expect(result).toEqual(300) 26 | }) 27 | 28 | test('with context', async () => { 29 | const result = await execute( 30 | 'extract 1 -> @add_one', 31 | 'add_one', 32 | (ctx: number) => { 33 | expect(ctx).toEqual(1) 34 | return ctx + 1 35 | }, 36 | true, 37 | ) 38 | expect(result).toEqual(2) 39 | }) 40 | 41 | test('without context', async () => { 42 | const result = await execute( 43 | ` 44 | GET http://example.com 45 | 46 | set url = $ -> url 47 | set value = @nocontext 48 | 49 | extract { $url, $value } 50 | `, 51 | 'nocontext', 52 | ctx => { 53 | expect(ctx).toEqual({}) 54 | return 123 55 | }, 56 | false, 57 | () => 58 | new Response('

test

', { 59 | headers: { 60 | 'content-type': 'text/html', 61 | }, 62 | }), 63 | ) 64 | expect(result).toEqual({ url: 'http://example.com/', value: 123 }) 65 | }) 66 | 67 | test('with args', async () => { 68 | const result = await execute( 69 | 'extract @product({ a: 7, b: 6 })', 70 | 'product', 71 | (_ctx, { a, b }) => { 72 | invariant( 73 | typeof a === 'number' && typeof b === 'number', 74 | new ValueTypeError('@product expects two numbers'), 75 | ) 76 | expect(a).toEqual(7) 77 | expect(b).toEqual(6) 78 | return a * b 79 | }, 80 | ) 81 | expect(result).toEqual(42) 82 | }) 83 | 84 | test('in macros', async () => { 85 | const result = await execute( 86 | { 87 | MyMacro: `extract @add_ten`, 88 | Home: `extract 4 -> @MyMacro`, 89 | }, 90 | 'add_ten', 91 | function mymod(ctx: number) { 92 | expect(ctx).toEqual(4) 93 | return ctx + 10 94 | }, 95 | true, 96 | ) 97 | expect(result).toEqual(14) 98 | }) 99 | 100 | test('in macros, arrow function', async () => { 101 | const result = await execute( 102 | { 103 | MyMacro: `extract @add_ten`, 104 | Home: `extract 4 -> @MyMacro`, 105 | }, 106 | 'add_ten', 107 | (ctx: number) => { 108 | expect(ctx).toEqual(4) 109 | return ctx + 10 110 | }, 111 | true, 112 | ) 113 | expect(result).toEqual(14) 114 | }) 115 | }) 116 | -------------------------------------------------------------------------------- /packages/get/src/calls.ts: -------------------------------------------------------------------------------- 1 | import type { TypeInfo } from '@getlang/ast' 2 | import { Type } from '@getlang/ast' 3 | import type { Hooks, Inputs } from '@getlang/lib' 4 | import { cookies, html, invariant, js, json } from '@getlang/lib' 5 | import { ImportError, ValueReferenceError } from '@getlang/lib/errors' 6 | import { partition } from 'lodash-es' 7 | import type { Entry, Registry } from './registry.js' 8 | import type { RuntimeValue } from './value.js' 9 | import { materialize } from './value.js' 10 | 11 | export async function callModifier( 12 | registry: Registry, 13 | mod: string, 14 | args: Record, 15 | context?: RuntimeValue, 16 | ) { 17 | const entry = await registry.importMod(mod) 18 | if (entry) { 19 | let ctx: any = {} 20 | if (entry.useContext && context) { 21 | ctx = entry.materialize ? materialize(context) : context.data 22 | } 23 | return entry.mod(ctx, args) 24 | } 25 | 26 | invariant(context, 'Modifier requires context') 27 | 28 | if (mod === 'link') { 29 | invariant(typeof args.base === 'string', '@link requires base url') 30 | const data = html.findLink(context.data) 31 | const link = materialize({ data, typeInfo: context.typeInfo }) 32 | return new URL(link, args.base).toString() 33 | } 34 | 35 | const doc = materialize(context) 36 | 37 | switch (mod) { 38 | case 'html': 39 | return html.parse(doc) 40 | case 'js': 41 | return js.parse(doc) 42 | case 'json': 43 | return json.parse(doc) 44 | case 'cookies': 45 | return cookies.parse(doc) 46 | default: 47 | throw new ValueReferenceError(`Unsupported modifier: ${mod}`) 48 | } 49 | } 50 | 51 | export type Execute = (entry: Entry, inputs: Inputs) => Promise 52 | 53 | export async function callModule( 54 | registry: Registry, 55 | execute: Execute, 56 | hooks: Required, 57 | module: string, 58 | args: RuntimeValue, 59 | contextType?: TypeInfo, 60 | ) { 61 | let entry: Entry 62 | try { 63 | entry = await registry.import(module, [], contextType) 64 | } catch (e) { 65 | const err = `Failed to import module: ${module}` 66 | throw new ImportError(err, { cause: e }) 67 | } 68 | const [inputArgs, attrArgs] = partition(Object.entries(args.data), e => 69 | entry.inputs.has(e[0]), 70 | ) 71 | const inputs = materialize({ 72 | data: Object.fromEntries(inputArgs), 73 | typeInfo: args.typeInfo, 74 | }) 75 | 76 | let extracted = await hooks.call(module, inputs) 77 | if (typeof extracted === 'undefined') { 78 | extracted = await execute(entry, inputs) 79 | } 80 | await hooks.extract(module, inputs, extracted.data) 81 | 82 | function dropWarning(reason: string) { 83 | if (attrArgs.length) { 84 | const dropped = attrArgs.map(e => e[0]).join(', ') 85 | const err = [ 86 | `Module '${module}' ${reason}`, 87 | `dropping view attributes: ${dropped}`, 88 | ].join(', ') 89 | console.warn(err) 90 | } 91 | } 92 | 93 | if (entry.returnType.type !== Type.Value) { 94 | dropWarning('returned unmaterialized value') 95 | return extracted 96 | } 97 | 98 | if (typeof extracted !== 'object') { 99 | dropWarning('returned a primitive') 100 | return extracted 101 | } 102 | 103 | const raster = materialize({ 104 | data: Object.fromEntries(attrArgs), 105 | typeInfo: args.typeInfo, 106 | }) 107 | 108 | return { ...raster, ...extracted } 109 | } 110 | -------------------------------------------------------------------------------- /packages/parser/src/passes/desugar/slicedeps.ts: -------------------------------------------------------------------------------- 1 | import type { Expr } from '@getlang/ast' 2 | import { t } from '@getlang/ast' 3 | import { invariant } from '@getlang/lib' 4 | import { SliceSyntaxError } from '@getlang/lib/errors' 5 | import { ScopeTracker, transform } from '@getlang/walker' 6 | import { parse as acorn } from 'acorn' 7 | import { traverse } from 'estree-toolkit' 8 | import globals from 'globals' 9 | import { render, tx } from '../../utils.js' 10 | import type { DesugarPass } from '../desugar.js' 11 | 12 | const browserGlobals = [ 13 | ...Object.keys(globals.browser), 14 | ...Object.keys(globals.builtin), 15 | ] 16 | 17 | function parse(source: string) { 18 | try { 19 | return acorn(source, { 20 | ecmaVersion: 'latest', 21 | allowReturnOutsideFunction: true, 22 | allowAwaitOutsideFunction: true, 23 | }) 24 | } catch (e) { 25 | throw new SliceSyntaxError('Could not parse slice', { cause: e }) 26 | } 27 | } 28 | 29 | const validAutoInserts = ['ExpressionStatement', 'BlockStatement'] 30 | 31 | const analyzeSlice = (slice: string) => { 32 | let source = slice 33 | 34 | const ast = parse(slice) 35 | if (ast.body.at(-1)?.type === 'EmptyStatement') { 36 | return null 37 | } 38 | 39 | const init = ast.body[0] 40 | invariant(init, new SliceSyntaxError('Empty slice body')) 41 | if (ast.body.length === 1 && init.type !== 'ReturnStatement') { 42 | // auto-insert the return statement 43 | invariant( 44 | validAutoInserts.includes(init.type), 45 | new SliceSyntaxError(`Invalid slice body: ${init.type}`), 46 | ) 47 | source = `return ${source}` 48 | } 49 | 50 | let ids: string[] = [] 51 | traverse(ast, { 52 | $: { scope: true }, 53 | Program(path) { 54 | ids = Object.keys(path.scope?.globalBindings ?? {}) 55 | }, 56 | }) 57 | ids = ids.filter(id => !browserGlobals.includes(id)) 58 | 59 | const usesVars = ids.some(d => d !== '$') 60 | const deps = new Set(ids) 61 | if (usesVars) { 62 | const names = [...deps].join(', ') 63 | source = `var { ${names} } = $\n${source}` 64 | } 65 | 66 | // add postmark to prevent slice from being re-processed 67 | source = `${source};;` 68 | return { source, deps, usesVars } 69 | } 70 | 71 | export const insertSliceDeps: DesugarPass = ast => { 72 | const scope = new ScopeTracker() 73 | return transform(ast, { 74 | scope, 75 | 76 | SliceExpr(node, path) { 77 | const stat = analyzeSlice(node.slice.value) 78 | if (!stat) { 79 | return 80 | } 81 | 82 | const { source, deps, usesVars } = stat 83 | const xnode = { ...node, slice: tx.token(source) } 84 | 85 | let context = scope.context 86 | 87 | if (usesVars) { 88 | if (context?.kind !== 'ObjectLiteralExpr') { 89 | context = t.objectLiteralExpr([]) 90 | } 91 | const keys = new Set(context.entries.map(e => render(e.key))) 92 | const missing = deps.difference(keys) 93 | for (const dep of missing) { 94 | const id = tx.token(dep, dep === '$' ? '' : dep) 95 | context.entries.push( 96 | t.objectEntryExpr(tx.template(dep), t.identifierExpr(id), false), 97 | ) 98 | } 99 | } 100 | 101 | if (context && context !== scope.context) { 102 | invariant( 103 | path.parent?.node.kind === 'DrillExpr', 104 | 'Slice dependencies require drill expression', 105 | ) 106 | path.insertBefore(context) 107 | } 108 | 109 | return xnode 110 | }, 111 | }) 112 | } 113 | -------------------------------------------------------------------------------- /packages/lib/src/core/errors.ts: -------------------------------------------------------------------------------- 1 | export abstract class RuntimeError extends Error { 2 | toJSON() { 3 | return { name: this.name, message: this.message } 4 | } 5 | } 6 | 7 | export class FatalError extends RuntimeError { 8 | public override name = 'FatalError' 9 | 10 | constructor(options?: ErrorOptions) { 11 | super('A fatal runtime error occurred', options) 12 | } 13 | } 14 | 15 | export class QuerySyntaxError extends RuntimeError { 16 | public override name = 'SyntaxError' 17 | } 18 | 19 | export class ValueTypeError extends RuntimeError { 20 | public override name = 'TypeError' 21 | } 22 | 23 | export class ValueReferenceError extends RuntimeError { 24 | public override name = 'ReferenceError' 25 | 26 | constructor(varName: string, options?: ErrorOptions) { 27 | super(`Unable to locate variable: ${varName}`, options) 28 | } 29 | } 30 | 31 | export class SliceError extends RuntimeError { 32 | public override name = 'SliceError' 33 | 34 | constructor(options?: ErrorOptions) { 35 | super('An exception was thrown by the client-side slice', options) 36 | } 37 | } 38 | 39 | export class SliceSyntaxError extends RuntimeError { 40 | public override name = 'SliceSyntaxError' 41 | } 42 | 43 | export class ConversionError extends RuntimeError { 44 | public override name = 'ConversionError' 45 | 46 | constructor(type: string, options?: ErrorOptions) { 47 | super(`Attempted to convert unsupported type to value: ${type}`, options) 48 | } 49 | } 50 | 51 | export class SelectorSyntaxError extends RuntimeError { 52 | public override name = 'SelectorSyntaxError' 53 | 54 | constructor(type: string, selector: string, options?: ErrorOptions) { 55 | super(`Could not parse ${type} selector '${selector}'`, options) 56 | } 57 | } 58 | 59 | export class NullSelectionError extends RuntimeError { 60 | public override name = 'NullSelectionError' 61 | 62 | constructor(selector: string, options?: ErrorOptions) { 63 | super(`The selector '${selector}' did not produce a result`, options) 64 | } 65 | } 66 | 67 | export class NullInputError extends RuntimeError { 68 | public override name = 'NullInputError' 69 | 70 | constructor(inputName: string, options?: ErrorOptions) { 71 | super(`Required input '${inputName}' not provided`, options) 72 | } 73 | } 74 | 75 | export class UnknownInputsError extends RuntimeError { 76 | public override name = 'UnknownInputsError' 77 | 78 | constructor(inputNames: string[] = [], options?: ErrorOptions) { 79 | const noun = inputNames.length > 1 ? 'inputs' : 'input' 80 | super(`Unknown ${noun} provided: ${inputNames.join(', ')}`, options) 81 | } 82 | } 83 | 84 | export class RequestError extends RuntimeError { 85 | public override name = 'RequestError' 86 | 87 | constructor(url: string, options?: ErrorOptions) { 88 | super(`Request to url failed: ${url}`, options) 89 | } 90 | } 91 | 92 | export class ImportError extends RuntimeError { 93 | public override name = 'ImportError' 94 | } 95 | 96 | export class RecursiveCallError extends RuntimeError { 97 | public override name = 'RecursiveCallError' 98 | constructor(chain: string[], options?: ErrorOptions) { 99 | super(`Recursive call error: ${chain.join(' -> ')}`, options) 100 | } 101 | } 102 | 103 | export function invariant( 104 | condition: unknown, 105 | err: string | RuntimeError, 106 | ): asserts condition { 107 | if (!condition) { 108 | throw typeof err === 'string' ? new FatalError({ cause: err }) : err 109 | } 110 | } 111 | 112 | export class NullSelection { 113 | constructor(public selector: string) {} 114 | } 115 | -------------------------------------------------------------------------------- /test/hooks.spec.ts: -------------------------------------------------------------------------------- 1 | import { describe, expect, mock, test } from 'bun:test' 2 | import type { 3 | CallHook, 4 | ExtractHook, 5 | Hooks, 6 | ImportHook, 7 | RequestHook, 8 | SliceHook, 9 | } from '@getlang/lib' 10 | import { invariant } from '@getlang/lib' 11 | import { execute } from './helpers.js' 12 | 13 | describe('hook', () => { 14 | test('on request', async () => { 15 | const src = ` 16 | GET http://get.com 17 | Accept: text/html 18 | 19 | extract -> h1 20 | ` 21 | 22 | const request = mock(async () => ({ 23 | status: 200, 24 | headers: new Headers({ 'content-type': 'text/html' }), 25 | body: '

test

', 26 | })) 27 | 28 | const result = await execute(src, {}, { hooks: { request } }) 29 | 30 | expect(request).toHaveServed('http://get.com/', { 31 | method: 'GET', 32 | headers: new Headers({ 33 | Accept: 'text/html', 34 | }), 35 | }) 36 | expect(result).toEqual('test') 37 | }) 38 | 39 | test('on slice', async () => { 40 | const slice = mock(() => 3) 41 | const result = await execute('extract `1 + 2`', {}, { hooks: { slice } }) 42 | expect(slice).toHaveBeenCalledWith('return 1 + 2;;', {}) 43 | expect(result).toEqual(3) 44 | }) 45 | 46 | test('module lifecycle', async () => { 47 | const modules: Record = { 48 | Top: ` 49 | inputs { inputA } 50 | extract { value: |"top::" + inputA| } 51 | `, 52 | Mid: ` 53 | set inputA = |"bar"| 54 | extract { 55 | value: { 56 | topValue: @Top({ $inputA }) -> value 57 | midValue: |"mid"| 58 | } 59 | } 60 | `, 61 | Home: ` 62 | set inputA = |"foo"| 63 | 64 | extract { 65 | topValue: @Top({ $inputA }) -> value 66 | midValue: @Mid -> value 67 | botValue: |"bot"| 68 | } 69 | `, 70 | } 71 | 72 | const hooks: Hooks = { 73 | import: mock(async (module: string) => { 74 | const src = modules[module] 75 | invariant(src, `Unexpected import: ${module}`) 76 | return src 77 | }), 78 | call: mock(() => {}), 79 | extract: mock(() => {}), 80 | } 81 | const result = await execute(modules, {}, { hooks }) 82 | 83 | expect(result).toEqual({ 84 | topValue: 'top::foo', 85 | midValue: { 86 | topValue: 'top::bar', 87 | midValue: 'mid', 88 | }, 89 | botValue: 'bot', 90 | }) 91 | 92 | expect(hooks.import).toHaveBeenCalledTimes(3) 93 | expect(hooks.import).toHaveBeenNthCalledWith(1, 'Home') 94 | expect(hooks.import).toHaveBeenNthCalledWith(2, 'Top') 95 | expect(hooks.import).toHaveBeenNthCalledWith(3, 'Mid') 96 | 97 | expect(hooks.call).toHaveBeenCalledTimes(3) 98 | expect(hooks.call).toHaveBeenNthCalledWith(1, 'Top', { inputA: 'foo' }) 99 | expect(hooks.call).toHaveBeenNthCalledWith(2, 'Mid', {}) 100 | expect(hooks.call).toHaveBeenNthCalledWith(3, 'Top', { inputA: 'bar' }) 101 | 102 | expect(hooks.extract).toHaveBeenCalledTimes(3) 103 | expect(hooks.extract).toHaveBeenNthCalledWith( 104 | 1, 105 | 'Top', 106 | { inputA: 'foo' }, 107 | { value: 'top::foo' }, 108 | ) 109 | expect(hooks.extract).toHaveBeenNthCalledWith( 110 | 2, 111 | 'Top', 112 | { inputA: 'bar' }, 113 | { value: 'top::bar' }, 114 | ) 115 | expect(hooks.extract).toHaveBeenNthCalledWith( 116 | 3, 117 | 'Mid', 118 | {}, 119 | { 120 | value: { 121 | topValue: 'top::bar', 122 | midValue: 'mid', 123 | }, 124 | }, 125 | ) 126 | }) 127 | }) 128 | -------------------------------------------------------------------------------- /test/slice.spec.ts: -------------------------------------------------------------------------------- 1 | import { describe, expect, it, test } from 'bun:test' 2 | import { SliceError } from '@getlang/lib/errors' 3 | import { execute } from './helpers.js' 4 | 5 | describe('slice', () => { 6 | it('evaluates javascript with implicit return', async () => { 7 | const result = await execute('extract `1 + 2`') 8 | expect(result).toEqual(3) 9 | }) 10 | 11 | it('has access to script variables', async () => { 12 | const result = await execute( 13 | ` 14 | inputs { id, foo = |[1]| } 15 | 16 | set bar = |['x']| 17 | set baz = |{ id, foo, bar }| 18 | 19 | extract { $baz } 20 | `, 21 | { id: '123' }, 22 | ) 23 | expect(result).toEqual({ 24 | baz: { 25 | id: '123', 26 | foo: [1], 27 | bar: ['x'], 28 | }, 29 | }) 30 | }) 31 | 32 | it('can reference and convert variables from outer scope', async () => { 33 | // note: the slice is statically analyzed to find which script variables 34 | // are used, and converts only those to values. Any unused and incompatible 35 | // variables should not cause ConversionError's (ie the BinaryExpression) 36 | const result = await execute( 37 | ` 38 | set js = |'5 + 1'| 39 | set x = $js -> @js -> BinaryExpression 40 | set y = $js -> @js -> Literal 41 | set out = ( 42 | extract { 43 | greet: |'hi there ' + y| 44 | } 45 | ) 46 | extract $out 47 | `, 48 | ) 49 | 50 | expect(result).toEqual({ 51 | greet: 'hi there 5', 52 | }) 53 | }) 54 | 55 | it('can reference and convert context', async () => { 56 | const result = await execute( 57 | ` 58 | set html = |'

title

para 1

para 2

'| 59 | 60 | extract $html -> @html => h1, p -> |$| 61 | `, 62 | ) 63 | expect(result).toEqual(['title', 'para 1', 'para 2']) 64 | }) 65 | 66 | it('can reference both context and outer variables', async () => { 67 | const result = await execute(` 68 | set obj = |{key:"x"}| 69 | set html = |"
keyval
"| -> @html 70 | extract $html -> span -> |obj[$]| 71 | `) 72 | expect(result).toEqual('x') 73 | }) 74 | 75 | it('supports escaped backticks', async () => { 76 | const result = await execute('extract `\\`escaped\\``') 77 | expect(result).toEqual('escaped') 78 | }) 79 | 80 | it('blocks allow non-escaped backticks', async () => { 81 | const result = await execute('extract |return `escaped`|') 82 | expect(result).toEqual('escaped') 83 | }) 84 | 85 | it('converts context to value prior to run', async () => { 86 | const result = await execute(` 87 | set html = |'
  • one
  • two
'| 88 | extract $html -> @html -> //li -> |$| 89 | `) 90 | expect(result).toEqual('one') 91 | }) 92 | 93 | it('operates on list item context', async () => { 94 | const result = await execute(` 95 | set html = |'
  • one
  • two
'| 96 | extract $html -> @html => li -> |$| 97 | `) 98 | expect(result).toEqual(['one', 'two']) 99 | }) 100 | 101 | it('supports analysis on nested slices', async () => { 102 | const result = await execute(` 103 | set x = |0| 104 | 105 | extract $x -> ( 106 | set y = |1| 107 | extract $y 108 | ) 109 | `) 110 | expect(result).toEqual(1) 111 | }) 112 | 113 | describe('errors', () => { 114 | test.skip('parsing', () => { 115 | const result = execute(` 116 | extract |{ a: "b" | 117 | `) 118 | 119 | // expect(result).rejects.toThrow() 120 | return expect(result).rejects.toBeInstanceOf(SliceError) 121 | }) 122 | 123 | test('running', () => { 124 | const result = execute(` 125 | extract |({}).no.no.yes| 126 | `) 127 | return expect(result).rejects.toThrow( 128 | /^An exception was thrown by the client-side slice/, 129 | ) 130 | }) 131 | }) 132 | }) 133 | -------------------------------------------------------------------------------- /packages/lib/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # @getlang/lib 2 | 3 | ## 0.2.1 4 | 5 | ### Patch Changes 6 | 7 | - [#51](https://github.com/getlang-dev/get/pull/51) [`e9df7de`](https://github.com/getlang-dev/get/commit/e9df7dec6fc3f5424a680146d881938cea4a249a) Thanks [@mattfysh](https://github.com/mattfysh)! - modifier flags: `useContext` and `materialize` 8 | 9 | ## 0.2.0 10 | 11 | ### Minor Changes 12 | 13 | - [#43](https://github.com/getlang-dev/get/pull/43) [`0f1b076`](https://github.com/getlang-dev/get/commit/0f1b076e347b6ea8d36dd6e29af4c70a22f0fd41) Thanks [@mattfysh](https://github.com/mattfysh)! - slice dependencies analysis 14 | 15 | - [`78660db`](https://github.com/getlang-dev/get/commit/78660db7089cebb8715c5389bac81559f2f4f60b) Thanks [@mattfysh](https://github.com/mattfysh)! - reorganize package layout 16 | 17 | - [#42](https://github.com/getlang-dev/get/pull/42) [`b7385e8`](https://github.com/getlang-dev/get/commit/b7385e89ffcbe02e93fde9789d464b094ae4c726) Thanks [@mattfysh](https://github.com/mattfysh)! - module call context 18 | 19 | ### Patch Changes 20 | 21 | - [#40](https://github.com/getlang-dev/get/pull/40) [`70cb95b`](https://github.com/getlang-dev/get/commit/70cb95be361513925eb9881ee2988787a98dcf1b) Thanks [@mattfysh](https://github.com/mattfysh)! - upgrade deps 22 | 23 | - [#47](https://github.com/getlang-dev/get/pull/47) [`a2304c8`](https://github.com/getlang-dev/get/commit/a2304c824175bb75be82f08a61a7a04880fb137f) Thanks [@mattfysh](https://github.com/mattfysh)! - flexible urls with implied params 24 | 25 | - Updated dependencies [[`fe8b25b`](https://github.com/getlang-dev/get/commit/fe8b25ba179d0a770b5e00275c70e9c7fc0405cf), [`356a09a`](https://github.com/getlang-dev/get/commit/356a09a81cddcba4c788a103451c8c9517ec596d), [`a2304c8`](https://github.com/getlang-dev/get/commit/a2304c824175bb75be82f08a61a7a04880fb137f), [`78660db`](https://github.com/getlang-dev/get/commit/78660db7089cebb8715c5389bac81559f2f4f60b), [`3e8b1c8`](https://github.com/getlang-dev/get/commit/3e8b1c8a591b5de149c6cc7556900a250273c9a5)]: 26 | - @getlang/ast@0.1.0 27 | 28 | ## 0.1.5 29 | 30 | ### Patch Changes 31 | 32 | - [#35](https://github.com/getlang-dev/get/pull/35) [`0c806e0`](https://github.com/getlang-dev/get/commit/0c806e092fc1d87696fb820ac7246b5d3afb63ca) Thanks [@mattfysh](https://github.com/mattfysh)! - bugbash 33 | 34 | ## 0.1.4 35 | 36 | ### Patch Changes 37 | 38 | - [`aca9d92`](https://github.com/getlang-dev/get/commit/aca9d929e4877fc614b66740c9415f5bcd083c24) Thanks [@mattfysh](https://github.com/mattfysh)! - fix publish 39 | 40 | - Updated dependencies [[`aca9d92`](https://github.com/getlang-dev/get/commit/aca9d929e4877fc614b66740c9415f5bcd083c24)]: 41 | - @getlang/utils@0.1.5 42 | 43 | ## 0.1.3 44 | 45 | ### Patch Changes 46 | 47 | - [`3660548`](https://github.com/getlang-dev/get/commit/3660548d79dd24e13a74bf0d6f24c1fec512062d) Thanks [@mattfysh](https://github.com/mattfysh)! - prevent tsconfig publish 48 | 49 | - Updated dependencies [[`3660548`](https://github.com/getlang-dev/get/commit/3660548d79dd24e13a74bf0d6f24c1fec512062d)]: 50 | - @getlang/utils@0.1.4 51 | 52 | ## 0.1.2 53 | 54 | ### Patch Changes 55 | 56 | - [#23](https://github.com/getlang-dev/get/pull/23) [`28d0ca8`](https://github.com/getlang-dev/get/commit/28d0ca8dcf840cfc70f002d06a48cace834edcf9) Thanks [@mattfysh](https://github.com/mattfysh)! - workspace layout updates 57 | 58 | - Updated dependencies [[`28d0ca8`](https://github.com/getlang-dev/get/commit/28d0ca8dcf840cfc70f002d06a48cace834edcf9)]: 59 | - @getlang/utils@0.1.2 60 | 61 | ## 0.1.1 62 | 63 | ### Patch Changes 64 | 65 | - [#14](https://github.com/getlang-dev/get/pull/14) [`58e9988`](https://github.com/getlang-dev/get/commit/58e99887e39956ee1e3eaf669cb92fbfa188a022) Thanks [@mattfysh](https://github.com/mattfysh)! - add contextual identifier to slices reference converted or raw context 66 | 67 | - Updated dependencies [[`58e9988`](https://github.com/getlang-dev/get/commit/58e99887e39956ee1e3eaf669cb92fbfa188a022)]: 68 | - @getlang/utils@0.1.1 69 | 70 | ## 0.1.0 71 | 72 | ### Minor Changes 73 | 74 | - [`57ff46d`](https://github.com/getlang-dev/get/commit/57ff46d904484e3277ee7a8481cdf4cee4c3deb2) Thanks [@mattfysh](https://github.com/mattfysh)! - Release v0.1.0 75 | 76 | ### Patch Changes 77 | 78 | - Updated dependencies [[`57ff46d`](https://github.com/getlang-dev/get/commit/57ff46d904484e3277ee7a8481cdf4cee4c3deb2)]: 79 | - @getlang/utils@0.1.0 80 | -------------------------------------------------------------------------------- /packages/get/src/registry.ts: -------------------------------------------------------------------------------- 1 | import type { Program, TypeInfo } from '@getlang/ast' 2 | import { Type } from '@getlang/ast' 3 | import type { Hooks, Modifier } from '@getlang/lib' 4 | import { RecursiveCallError, ValueTypeError } from '@getlang/lib/errors' 5 | import { analyze, desugar, inference, parse } from '@getlang/parser' 6 | 7 | type ModEntry = { 8 | mod: Modifier 9 | useContext: boolean 10 | materialize: boolean 11 | returnType: TypeInfo 12 | } 13 | 14 | type Info = { 15 | ast: Program 16 | imports: string[] 17 | contextMods: string[] 18 | isMacro: boolean 19 | } 20 | 21 | export type Entry = { 22 | program: Program 23 | inputs: Set 24 | returnType: TypeInfo 25 | } 26 | 27 | function repr(ti: TypeInfo): string { 28 | switch (ti.type) { 29 | case Type.Maybe: 30 | return `maybe<${repr(ti.option)}>` 31 | case Type.List: 32 | return `${repr(ti.of)}[]` 33 | case Type.Struct: { 34 | const fields = Object.entries(ti.schema) 35 | .map(e => `${e[0]}: ${repr(e[1])};`) 36 | .join(' ') 37 | return `{ ${fields} }` 38 | } 39 | case Type.Context: 40 | case Type.Never: 41 | throw new ValueTypeError('Unsupported key type') 42 | default: 43 | return ti.type 44 | } 45 | } 46 | 47 | function buildImportKey(module: string, typeInfo?: TypeInfo) { 48 | let key = module 49 | if (typeInfo) { 50 | key += `<${repr(typeInfo)}>` 51 | } 52 | return key 53 | } 54 | 55 | export class Registry { 56 | private info: Record> = {} 57 | private entries: Record> = {} 58 | private modifiers: Record> = {} 59 | 60 | constructor(private hooks: Required) {} 61 | 62 | importMod(mod: string) { 63 | this.modifiers[mod] ??= Promise.resolve().then(async () => { 64 | const compiled = await this.hooks.modifier(mod) 65 | if (!compiled) { 66 | return null 67 | } 68 | 69 | const { 70 | modifier: fn, 71 | useContext = false, 72 | materialize = true, 73 | returnType = { type: Type.Value }, 74 | } = compiled 75 | 76 | return { 77 | mod: fn, 78 | useContext, 79 | materialize, 80 | returnType, 81 | } 82 | }) 83 | return this.modifiers[mod] 84 | } 85 | 86 | private getInfo(module: string) { 87 | this.info[module] ??= Promise.resolve().then(async () => { 88 | const source = await this.hooks.import(module) 89 | const ast = parse(source) 90 | const info = analyze(ast) 91 | const imports = [...info.imports] 92 | const contextMods: string[] = [] 93 | for (const mod of info.modifiers.keys()) { 94 | const entry = await this.importMod(mod) 95 | if (entry?.useContext ?? true) { 96 | contextMods.push(mod) 97 | } 98 | } 99 | const isMacro = 100 | info.hasUnboundSelector || 101 | contextMods.some(mod => info.modifiers.get(mod)) 102 | return { ast, imports, contextMods, isMacro } 103 | }) 104 | return this.info[module] 105 | } 106 | 107 | import(module: string, prev: string[] = [], contextType?: TypeInfo) { 108 | const stack = [...prev, module] 109 | if (prev.includes(module)) { 110 | throw new RecursiveCallError(stack) 111 | } 112 | const key = buildImportKey(module, contextType) 113 | this.entries[key] ??= Promise.resolve().then(async () => { 114 | const { ast, imports, contextMods } = await this.getInfo(module) 115 | const contextual = [...contextMods] 116 | for (const i of imports) { 117 | const depInfo = await this.getInfo(i) 118 | if (depInfo.isMacro) { 119 | contextual.push(i) 120 | } 121 | } 122 | const simplified = desugar(ast, contextual) 123 | const { inputs, calls, modifiers } = analyze(simplified) 124 | 125 | const returnTypes: Record = {} 126 | for (const call of calls) { 127 | const { returnType } = await this.import(call, stack) 128 | returnTypes[call] = returnType 129 | } 130 | for (const mod of modifiers.keys()) { 131 | const entry = await this.importMod(mod) 132 | if (entry) { 133 | returnTypes[mod] = entry.returnType 134 | } 135 | } 136 | 137 | const { program, returnType } = inference(simplified, { 138 | returnTypes, 139 | contextType, 140 | }) 141 | 142 | return { program, inputs, returnType } 143 | }) 144 | return this.entries[key] 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /packages/parser/src/grammar/parse.ts: -------------------------------------------------------------------------------- 1 | import type { Expr } from '@getlang/ast' 2 | import { isToken, t } from '@getlang/ast' 3 | import { invariant } from '@getlang/lib' 4 | import { QuerySyntaxError } from '@getlang/lib/errors' 5 | import { tx } from '../utils.js' 6 | 7 | type PP = nearley.Postprocessor 8 | 9 | export const program: PP = ([, maybeInputs, body]) => { 10 | const inputs = maybeInputs?.[0] 11 | const stmts = inputs ? [inputs] : [] 12 | stmts.push(...body) 13 | return t.program(stmts) 14 | } 15 | 16 | export const statements: PP = ([stmt, stmts]) => [ 17 | stmt, 18 | ...stmts.map((d: any) => d[1]), 19 | ] 20 | 21 | export const declInputs: PP = ([, , , , first, maybeRest]) => { 22 | const rest = maybeRest || [] 23 | const restIds = rest.map((d: any) => d[3]) 24 | const ids = [first, ...restIds] 25 | return t.declInputsStmt(ids) 26 | } 27 | 28 | export const inputDecl: PP = ([id, optional, maybeDefault]) => { 29 | const defaultValue = maybeDefault?.[3] 30 | return t.InputExpr(id, Boolean(optional || defaultValue), defaultValue) 31 | } 32 | 33 | export const request: PP = ([method, url, headerBlock, { blocks, body }]) => { 34 | const headers = t.requestBlockExpr(tx.token(''), headerBlock?.[1] ?? []) 35 | const req = t.requestExpr(method, url, headers, blocks, body) 36 | return t.requestStmt(req) 37 | } 38 | 39 | export const requestBlocks: PP = ([namedBlocks, maybeBody]) => { 40 | const blocks = namedBlocks.map((d: any) => d[1]) 41 | 42 | const body = maybeBody?.[1] 43 | if (body) { 44 | for (const el of body.elements) { 45 | if (isToken(el)) { 46 | // a token - restore original text 47 | el.value = el.text 48 | } 49 | } 50 | } 51 | 52 | return { blocks, body } 53 | } 54 | 55 | export const requestBlockNamed: PP = ([name, , entries]) => 56 | t.requestBlockExpr(name, entries) 57 | 58 | export const requestBlockBody: PP = ([, body]) => body 59 | 60 | export const requestBlock: PP = ([entry, entries]) => [ 61 | entry, 62 | ...entries.map((d: any) => d[1]), 63 | ] 64 | 65 | export const requestEntry: PP = ([key, , maybeValue]) => { 66 | let value = maybeValue?.[1] 67 | if (typeof value === 'undefined') { 68 | value = { 69 | ...key.value, // key is already a literal expr 70 | value: '', 71 | text: '', 72 | } 73 | } 74 | return t.requestEntryExpr(key, value) 75 | } 76 | 77 | export const assignment: PP = ([, , name, optional, , , , expr]) => 78 | t.assignmentStmt(name, expr, !!optional) 79 | 80 | export const extract: PP = ([, , exports]) => t.extractStmt(exports) 81 | 82 | export const subquery: PP = ([, , stmts]) => t.subqueryExpr(stmts) 83 | 84 | export const call: PP = ([callee, maybeInputs]) => { 85 | const inputs = maybeInputs?.[1] 86 | return /^[a-z]/.test(callee.value) 87 | ? t.modifierExpr(callee, inputs) 88 | : t.moduleExpr(callee, inputs) 89 | } 90 | 91 | export const link: PP = ([context, callee, _, link]) => { 92 | const bit = t.moduleExpr( 93 | callee, 94 | t.objectLiteralExpr([t.objectEntryExpr(tx.template('@link'), link, true)]), 95 | ) 96 | const [drill, , arrow] = context || [] 97 | const body = drill?.body || [] 98 | return t.drillExpr([...body, drillBase(bit, arrow)]) 99 | } 100 | 101 | export const object: PP = d => { 102 | const entries = d[2].map((dd: any) => dd[0]) 103 | return t.objectLiteralExpr(entries) 104 | } 105 | 106 | export const objectEntry: PP = ([callkey, identifier, optional, , , value]) => { 107 | const key = { 108 | ...identifier, 109 | value: `${callkey ? '@' : ''}${identifier.value || '$'}`, 110 | } 111 | return t.objectEntryExpr(t.templateExpr([key]), value, Boolean(optional)) 112 | } 113 | 114 | export const objectEntryShorthandSelect: PP = ([identifier, optional]) => { 115 | const value = t.templateExpr([identifier]) 116 | const selector = t.drillExpr([t.selectorExpr(value, false)]) 117 | return objectEntry([null, identifier, optional, null, null, selector]) 118 | } 119 | 120 | export const objectEntryShorthandIdent: PP = ([identifier, optional]) => { 121 | const value = t.identifierExpr(identifier) 122 | return objectEntry([null, identifier, optional, null, null, value]) 123 | } 124 | 125 | function drillBase(bit: Expr, arrow?: string): Expr { 126 | const expand = arrow === '=>' 127 | if (bit.kind === 'SelectorExpr' || bit.kind === 'DrillIdentifierExpr') { 128 | bit.expand = expand 129 | } else if (expand) { 130 | throw new QuerySyntaxError('Misplaced wide arrow drill') 131 | } 132 | return bit 133 | } 134 | 135 | export const drill: PP = ([arrow, bit, bits]) => { 136 | const expr = drillBase(bit, arrow?.[0].value) 137 | const exprs = bits.map(([, arrow, , bit]: any) => { 138 | return drillBase(bit, arrow.value) 139 | }) 140 | return t.drillExpr([expr, ...exprs]) 141 | } 142 | 143 | export const selector: PP = ([template]) => t.selectorExpr(template, false) 144 | export const idbit: PP = ([id]) => t.drillIdentifierExpr(id, false) 145 | 146 | export const template: PP = d => { 147 | const elements = d[0].reduce((els: any, dd: any) => { 148 | const el = dd[0] 149 | if (el.kind) { 150 | invariant( 151 | el.kind === 'TemplateExpr', 152 | `Unexpected template element: ${el.kind}`, 153 | ) 154 | els.push(el) 155 | } else if (el.type === 'interpvar' || el.type === 'identifier') { 156 | els.push(t.identifierExpr(el)) 157 | } else if (el.type === 'str') { 158 | if (el.value) { 159 | const prev = els.at(-1) 160 | if (prev?.type === 'str') { 161 | els.pop() 162 | els.push({ ...prev, value: prev.value + el.value }) 163 | } else { 164 | els.push(el) 165 | } 166 | } 167 | } else { 168 | throw new QuerySyntaxError(`Unknown template element: ${el.type}`) 169 | } 170 | 171 | return els 172 | }, []) 173 | 174 | const first = elements.at(0) 175 | if (first.type === 'str') { 176 | elements[0] = { ...first, value: first.value.trimLeft() } 177 | } 178 | 179 | const lastIdx = elements.length - 1 180 | const last = elements[lastIdx] 181 | if (last.type === 'str') { 182 | elements[lastIdx] = { ...last, value: last.value.trimRight() } 183 | } 184 | 185 | return t.templateExpr(elements) 186 | } 187 | 188 | export const literal: PP = ([[token]]) => t.literalExpr(token) 189 | export const string: PP = ([, template]) => template 190 | 191 | export const interpExpr: PP = ([, , token]) => token 192 | export const interpTmpl: PP = ([, , template]) => template 193 | 194 | export const slice: PP = d => t.sliceExpr(d[0]) 195 | 196 | export const ws: PP = () => null 197 | 198 | export const idd: PP = d => d[0][0] 199 | -------------------------------------------------------------------------------- /packages/parser/src/print.ts: -------------------------------------------------------------------------------- 1 | import type { Node } from '@getlang/ast' 2 | import { isToken } from '@getlang/ast' 3 | import type { ReduceVisitor } from '@getlang/walker' 4 | import { reduce } from '@getlang/walker' 5 | import { builders, printer } from 'prettier/doc' 6 | import { render } from './utils.js' 7 | 8 | type Doc = builders.Doc 9 | 10 | // NOTE: avoid using template interpolation with prettier.Doc 11 | // as the Doc may be a Doc array or Doc command 12 | 13 | const { group, indent, join, line, hardline, softline, ifBreak } = builders 14 | 15 | const printVisitor: ReduceVisitor = { 16 | Program(node) { 17 | return join(hardline, node.body) 18 | }, 19 | 20 | DeclInputsStmt(node) { 21 | return group([ 22 | 'inputs {', 23 | indent([line, join([',', line], node.inputs)]), 24 | line, 25 | '}', 26 | ]) 27 | }, 28 | 29 | RequestExpr(node) { 30 | const parts: Doc[] = [node.method.value, ' ', node.url] 31 | for (const block of node.blocks) { 32 | parts.push(block) 33 | } 34 | if (node.body) { 35 | parts.push(hardline, '[body]', hardline, node.body, hardline, '[/body]') 36 | } 37 | parts.push(hardline) // terminal 38 | return group(parts) 39 | }, 40 | 41 | RequestBlockExpr(node) { 42 | const parts: Doc[] = [] 43 | const name = node.name.value 44 | if (name) { 45 | parts.unshift(hardline, '[', name, ']') 46 | } 47 | for (const entry of node.entries) { 48 | parts.push(hardline, entry) 49 | } 50 | return parts 51 | }, 52 | 53 | RequestEntryExpr(node) { 54 | return [node.key, ': ', node.value] 55 | }, 56 | 57 | InputExpr(node) { 58 | const parts: Doc[] = [node.id.text] 59 | if (node.optional) { 60 | parts.push('?') 61 | } 62 | if (node.defaultValue) { 63 | parts.push(' = ', node.defaultValue) 64 | } 65 | return group(parts) 66 | }, 67 | 68 | LiteralExpr(node) { 69 | return String(node.value) 70 | }, 71 | 72 | RequestStmt(node) { 73 | return node.request 74 | }, 75 | 76 | AssignmentStmt(node) { 77 | return group([ 78 | 'set ', 79 | node.name.value, 80 | node.optional ? '?' : '', 81 | ' = ', 82 | node.value, 83 | ]) 84 | }, 85 | 86 | ExtractStmt(node) { 87 | return group(['extract ', node.value]) 88 | }, 89 | 90 | DrillExpr(node, { node: orig }) { 91 | return node.body.map((expr, i) => { 92 | const og = orig.body[i]! 93 | let expand = false 94 | if (og.kind === 'SelectorExpr' || og.kind === 'DrillIdentifierExpr') { 95 | expand = og.expand 96 | } 97 | if (i === 0) { 98 | return expand ? ['=> ', expr] : expr 99 | } 100 | const arrow = expand ? '=> ' : '-> ' 101 | return indent([line, arrow, expr]) 102 | }) 103 | }, 104 | 105 | ObjectEntryExpr(node, { node: orig }) { 106 | if (orig.value.kind === 'IdentifierExpr') { 107 | const key = render(orig.key) 108 | const value = orig.value.id.value 109 | if (key === value || (key === '$' && value === '')) { 110 | return node.value 111 | } 112 | } 113 | 114 | const keyGroup: Doc[] = [node.key] 115 | if (node.optional) { 116 | keyGroup.push('?') 117 | } 118 | 119 | // seperator 120 | keyGroup.push(': ') 121 | 122 | // value 123 | const value = node.value 124 | let shValue: Doc = node.value 125 | if ( 126 | Array.isArray(shValue) && 127 | shValue.length === 1 && 128 | typeof shValue[0] === 'string' 129 | ) { 130 | shValue = shValue[0] 131 | } 132 | if (typeof shValue === 'string' && node.key === shValue) { 133 | return [value, node.optional ? '?' : ''] 134 | } 135 | return group([keyGroup, value]) 136 | }, 137 | 138 | ObjectLiteralExpr(node, { node: orig }) { 139 | const shouldBreak = orig.entries.some(e => { 140 | switch (e.value.kind) { 141 | case 'SelectorExpr': 142 | return true 143 | case 'DrillExpr': 144 | return e.value.body.at(-1)!.kind === 'SelectorExpr' 145 | default: 146 | return false 147 | } 148 | }) 149 | const sep = ifBreak(line, [',', line]) 150 | return group(['{', indent([line, join(sep, node.entries)]), line, '}'], { 151 | shouldBreak, 152 | }) 153 | }, 154 | 155 | TemplateExpr(node, { node: orig }) { 156 | return node.elements.map((el, i) => { 157 | const og = orig.elements[i]! 158 | if (isToken(og)) { 159 | return og.value 160 | } 161 | 162 | if (typeof el !== 'string' && !Array.isArray(el)) { 163 | throw new Error(`Unsupported template node: ${el.type} command`) 164 | } else if (og.kind === 'TemplateExpr') { 165 | return ['$[', el, ']'] 166 | } else if (og.kind !== 'IdentifierExpr') { 167 | throw new Error(`Unexpected template node: ${og?.kind}`) 168 | } 169 | 170 | let id: Doc = [og.id.value] 171 | const nextEl = node.elements[i + 1] 172 | if (isToken(nextEl) && /^\w/.test(nextEl.value)) { 173 | // use ${id} syntax to delineate against next element in template 174 | id = ['{', id, '}'] 175 | } 176 | return [og.isUrlComponent ? ':' : '$', id] 177 | }) 178 | }, 179 | 180 | IdentifierExpr(node) { 181 | return ['$', node.id.value] 182 | }, 183 | 184 | DrillIdentifierExpr(node) { 185 | return ['$', node.id.value] 186 | }, 187 | 188 | SelectorExpr(node) { 189 | return node.selector 190 | }, 191 | 192 | ModifierExpr(node, { node: orig }) { 193 | const call: Doc[] = ['@', node.modifier.value] 194 | if (orig.args.entries.length) { 195 | call.push('(', node.args, ')') 196 | } 197 | return call 198 | }, 199 | 200 | ModuleExpr(node, { node: orig }) { 201 | const call: Doc[] = ['@', node.module.value] 202 | if (orig.args.entries.length) { 203 | call.push('(', node.args, ')') 204 | } 205 | return call 206 | }, 207 | 208 | SliceExpr(node) { 209 | const { value } = node.slice 210 | const quot = value.includes('`') ? '|' : '`' 211 | const lines = value.split('\n') 212 | return group([ 213 | quot, 214 | indent([softline, join(hardline, lines)]), 215 | softline, 216 | quot, 217 | ]) 218 | }, 219 | 220 | SubqueryExpr(node) { 221 | return ['(', indent(node.body.flatMap(x => [hardline, x])), hardline, ')'] 222 | }, 223 | } 224 | 225 | export function print(ast: Node) { 226 | if (!(ast.kind === 'Program')) { 227 | throw new Error(`Non-program AST node provided: ${ast}`) 228 | } 229 | const doc = reduce(ast, printVisitor) 230 | // propagateBreaks(doc) 231 | return printer.printDocToString(doc, { 232 | printWidth: 70, 233 | tabWidth: 2, 234 | useTabs: false, 235 | }).formatted 236 | } 237 | -------------------------------------------------------------------------------- /packages/parser/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # @getlang/parser 2 | 3 | ## 0.4.3 4 | 5 | ### Patch Changes 6 | 7 | - [#55](https://github.com/getlang-dev/get/pull/55) [`194676e`](https://github.com/getlang-dev/get/commit/194676e34bd4311af898565e045aa88a4eb5292a) Thanks [@mattfysh](https://github.com/mattfysh)! - fix modifier context (ii) 8 | 9 | ## 0.4.2 10 | 11 | ### Patch Changes 12 | 13 | - [#53](https://github.com/getlang-dev/get/pull/53) [`ecd87ab`](https://github.com/getlang-dev/get/commit/ecd87ab290fbccc7d61bc82950f82d8ed8b974d5) Thanks [@mattfysh](https://github.com/mattfysh)! - fix modifier context 14 | 15 | ## 0.4.1 16 | 17 | ### Patch Changes 18 | 19 | - [#51](https://github.com/getlang-dev/get/pull/51) [`e9df7de`](https://github.com/getlang-dev/get/commit/e9df7dec6fc3f5424a680146d881938cea4a249a) Thanks [@mattfysh](https://github.com/mattfysh)! - modifier flags: `useContext` and `materialize` 20 | 21 | - Updated dependencies [[`e9df7de`](https://github.com/getlang-dev/get/commit/e9df7dec6fc3f5424a680146d881938cea4a249a)]: 22 | - @getlang/lib@0.2.1 23 | 24 | ## 0.4.0 25 | 26 | ### Minor Changes 27 | 28 | - [#43](https://github.com/getlang-dev/get/pull/43) [`0f1b076`](https://github.com/getlang-dev/get/commit/0f1b076e347b6ea8d36dd6e29af4c70a22f0fd41) Thanks [@mattfysh](https://github.com/mattfysh)! - slice dependencies analysis 29 | 30 | - [`78660db`](https://github.com/getlang-dev/get/commit/78660db7089cebb8715c5389bac81559f2f4f60b) Thanks [@mattfysh](https://github.com/mattfysh)! - reorganize package layout 31 | 32 | - [#42](https://github.com/getlang-dev/get/pull/42) [`b7385e8`](https://github.com/getlang-dev/get/commit/b7385e89ffcbe02e93fde9789d464b094ae4c726) Thanks [@mattfysh](https://github.com/mattfysh)! - module call context 33 | 34 | ### Patch Changes 35 | 36 | - [#46](https://github.com/getlang-dev/get/pull/46) [`fe8b25b`](https://github.com/getlang-dev/get/commit/fe8b25ba179d0a770b5e00275c70e9c7fc0405cf) Thanks [@mattfysh](https://github.com/mattfysh)! - add custom modifiers 37 | 38 | - [#44](https://github.com/getlang-dev/get/pull/44) [`356a09a`](https://github.com/getlang-dev/get/commit/356a09a81cddcba4c788a103451c8c9517ec596d) Thanks [@mattfysh](https://github.com/mattfysh)! - drill-based walker 39 | 40 | - [#40](https://github.com/getlang-dev/get/pull/40) [`70cb95b`](https://github.com/getlang-dev/get/commit/70cb95be361513925eb9881ee2988787a98dcf1b) Thanks [@mattfysh](https://github.com/mattfysh)! - upgrade deps 41 | 42 | - [#47](https://github.com/getlang-dev/get/pull/47) [`a2304c8`](https://github.com/getlang-dev/get/commit/a2304c824175bb75be82f08a61a7a04880fb137f) Thanks [@mattfysh](https://github.com/mattfysh)! - flexible urls with implied params 43 | 44 | - [#48](https://github.com/getlang-dev/get/pull/48) [`3e8b1c8`](https://github.com/getlang-dev/get/commit/3e8b1c8a591b5de149c6cc7556900a250273c9a5) Thanks [@mattfysh](https://github.com/mattfysh)! - primitive literals 45 | 46 | - Updated dependencies [[`fe8b25b`](https://github.com/getlang-dev/get/commit/fe8b25ba179d0a770b5e00275c70e9c7fc0405cf), [`0f1b076`](https://github.com/getlang-dev/get/commit/0f1b076e347b6ea8d36dd6e29af4c70a22f0fd41), [`356a09a`](https://github.com/getlang-dev/get/commit/356a09a81cddcba4c788a103451c8c9517ec596d), [`70cb95b`](https://github.com/getlang-dev/get/commit/70cb95be361513925eb9881ee2988787a98dcf1b), [`a2304c8`](https://github.com/getlang-dev/get/commit/a2304c824175bb75be82f08a61a7a04880fb137f), [`78660db`](https://github.com/getlang-dev/get/commit/78660db7089cebb8715c5389bac81559f2f4f60b), [`3e8b1c8`](https://github.com/getlang-dev/get/commit/3e8b1c8a591b5de149c6cc7556900a250273c9a5), [`b7385e8`](https://github.com/getlang-dev/get/commit/b7385e89ffcbe02e93fde9789d464b094ae4c726)]: 47 | - @getlang/walker@0.1.0 48 | - @getlang/ast@0.1.0 49 | - @getlang/lib@0.2.0 50 | 51 | ## 0.3.4 52 | 53 | ### Patch Changes 54 | 55 | - [#35](https://github.com/getlang-dev/get/pull/35) [`0c806e0`](https://github.com/getlang-dev/get/commit/0c806e092fc1d87696fb820ac7246b5d3afb63ca) Thanks [@mattfysh](https://github.com/mattfysh)! - bugbash 56 | 57 | ## 0.3.3 58 | 59 | ### Patch Changes 60 | 61 | - [#31](https://github.com/getlang-dev/get/pull/31) [`1a395f9`](https://github.com/getlang-dev/get/commit/1a395f9df71d3507bc5b3841eddc9336db3a69ee) Thanks [@mattfysh](https://github.com/mattfysh)! - request parsers 62 | 63 | - Updated dependencies [[`1a395f9`](https://github.com/getlang-dev/get/commit/1a395f9df71d3507bc5b3841eddc9336db3a69ee)]: 64 | - @getlang/utils@0.1.6 65 | 66 | ## 0.3.2 67 | 68 | ### Patch Changes 69 | 70 | - [`aca9d92`](https://github.com/getlang-dev/get/commit/aca9d929e4877fc614b66740c9415f5bcd083c24) Thanks [@mattfysh](https://github.com/mattfysh)! - fix publish 71 | 72 | - Updated dependencies [[`aca9d92`](https://github.com/getlang-dev/get/commit/aca9d929e4877fc614b66740c9415f5bcd083c24)]: 73 | - @getlang/utils@0.1.5 74 | 75 | ## 0.3.1 76 | 77 | ### Patch Changes 78 | 79 | - [`3660548`](https://github.com/getlang-dev/get/commit/3660548d79dd24e13a74bf0d6f24c1fec512062d) Thanks [@mattfysh](https://github.com/mattfysh)! - prevent tsconfig publish 80 | 81 | - Updated dependencies [[`3660548`](https://github.com/getlang-dev/get/commit/3660548d79dd24e13a74bf0d6f24c1fec512062d)]: 82 | - @getlang/utils@0.1.4 83 | 84 | ## 0.3.0 85 | 86 | ### Minor Changes 87 | 88 | - [#28](https://github.com/getlang-dev/get/pull/28) [`f1bdc2c`](https://github.com/getlang-dev/get/commit/f1bdc2c8433a942f84503606790e8bcf4fb37477) Thanks [@mattfysh](https://github.com/mattfysh)! - page links 89 | 90 | ### Patch Changes 91 | 92 | - [#26](https://github.com/getlang-dev/get/pull/26) [`58bb154`](https://github.com/getlang-dev/get/commit/58bb1540f1fa50093b3a72ffb2446e7b1c6aacb0) Thanks [@mattfysh](https://github.com/mattfysh)! - interpolated template optionals 93 | 94 | - Updated dependencies [[`f1bdc2c`](https://github.com/getlang-dev/get/commit/f1bdc2c8433a942f84503606790e8bcf4fb37477)]: 95 | - @getlang/utils@0.1.3 96 | 97 | ## 0.2.2 98 | 99 | ### Patch Changes 100 | 101 | - [#23](https://github.com/getlang-dev/get/pull/23) [`28d0ca8`](https://github.com/getlang-dev/get/commit/28d0ca8dcf840cfc70f002d06a48cace834edcf9) Thanks [@mattfysh](https://github.com/mattfysh)! - workspace layout updates 102 | 103 | - Updated dependencies [[`28d0ca8`](https://github.com/getlang-dev/get/commit/28d0ca8dcf840cfc70f002d06a48cace834edcf9)]: 104 | - @getlang/utils@0.1.2 105 | 106 | ## 0.2.1 107 | 108 | ### Patch Changes 109 | 110 | - [#16](https://github.com/getlang-dev/get/pull/16) [`1c17b8d`](https://github.com/getlang-dev/get/commit/1c17b8dc6a2f417fb7948d35531b8eb970f345cc) Thanks [@mattfysh](https://github.com/mattfysh)! - bugfix: subquery context switching 111 | 112 | ## 0.2.0 113 | 114 | ### Minor Changes 115 | 116 | - [#12](https://github.com/getlang-dev/get/pull/12) [`4b81eda`](https://github.com/getlang-dev/get/commit/4b81eda1e71727f59fe7a0d26abde186ed78c876) Thanks [@mattfysh](https://github.com/mattfysh)! - rename function expressions to subqueries 117 | 118 | ### Patch Changes 119 | 120 | - [#14](https://github.com/getlang-dev/get/pull/14) [`58e9988`](https://github.com/getlang-dev/get/commit/58e99887e39956ee1e3eaf669cb92fbfa188a022) Thanks [@mattfysh](https://github.com/mattfysh)! - add contextual identifier to slices reference converted or raw context 121 | 122 | - Updated dependencies [[`58e9988`](https://github.com/getlang-dev/get/commit/58e99887e39956ee1e3eaf669cb92fbfa188a022)]: 123 | - @getlang/utils@0.1.1 124 | 125 | ## 0.1.0 126 | 127 | ### Minor Changes 128 | 129 | - [`57ff46d`](https://github.com/getlang-dev/get/commit/57ff46d904484e3277ee7a8481cdf4cee4c3deb2) Thanks [@mattfysh](https://github.com/mattfysh)! - Release v0.1.0 130 | 131 | ### Patch Changes 132 | 133 | - Updated dependencies [[`57ff46d`](https://github.com/getlang-dev/get/commit/57ff46d904484e3277ee7a8481cdf4cee4c3deb2)]: 134 | - @getlang/utils@0.1.0 135 | -------------------------------------------------------------------------------- /packages/ast/src/ast.ts: -------------------------------------------------------------------------------- 1 | import type { Token as MooToken } from 'moo' 2 | import type { TypeInfo } from './typeinfo.js' 3 | import { Type } from './typeinfo.js' 4 | 5 | export { Type } 6 | export type { TypeInfo } 7 | 8 | export type Token = Omit 9 | export function isToken(value: unknown): value is Token { 10 | return !!value && typeof value === 'object' && 'offset' in value 11 | } 12 | 13 | export type Program = { 14 | kind: 'Program' 15 | body: Stmt[] 16 | } 17 | 18 | type ExtractStmt = { 19 | kind: 'ExtractStmt' 20 | value: Expr 21 | } 22 | 23 | type AssignmentStmt = { 24 | kind: 'AssignmentStmt' 25 | name: Token 26 | value: Expr 27 | optional: boolean 28 | } 29 | 30 | export type DeclInputsStmt = { 31 | kind: 'DeclInputsStmt' 32 | inputs: InputExpr[] 33 | } 34 | 35 | export type InputExpr = { 36 | kind: 'InputExpr' 37 | id: Token 38 | optional: boolean 39 | defaultValue?: SliceExpr 40 | typeInfo: TypeInfo 41 | } 42 | 43 | type RequestStmt = { 44 | kind: 'RequestStmt' 45 | request: RequestExpr 46 | } 47 | 48 | export type RequestExpr = { 49 | kind: 'RequestExpr' 50 | method: Token 51 | url: TemplateExpr 52 | headers: RequestBlockExpr 53 | blocks: RequestBlockExpr[] 54 | body: Expr 55 | typeInfo: TypeInfo 56 | } 57 | 58 | type RequestBlockExpr = { 59 | kind: 'RequestBlockExpr' 60 | name: Token 61 | entries: RequestEntryExpr[] 62 | typeInfo: TypeInfo 63 | } 64 | 65 | type RequestEntryExpr = { 66 | kind: 'RequestEntryExpr' 67 | key: Expr 68 | value: Expr 69 | typeInfo: TypeInfo 70 | } 71 | 72 | export type TemplateExpr = { 73 | kind: 'TemplateExpr' 74 | elements: (Expr | Token)[] 75 | typeInfo: TypeInfo 76 | } 77 | 78 | type IdentifierExpr = { 79 | kind: 'IdentifierExpr' 80 | id: Token 81 | isUrlComponent: boolean 82 | typeInfo: TypeInfo 83 | } 84 | 85 | type DrillIdentifierExpr = { 86 | kind: 'DrillIdentifierExpr' 87 | id: Token 88 | expand: boolean 89 | typeInfo: TypeInfo 90 | } 91 | 92 | type SelectorExpr = { 93 | kind: 'SelectorExpr' 94 | selector: Expr 95 | expand: boolean 96 | typeInfo: TypeInfo 97 | } 98 | 99 | export type ModifierExpr = { 100 | kind: 'ModifierExpr' 101 | modifier: Token 102 | args: ObjectLiteralExpr 103 | typeInfo: TypeInfo 104 | } 105 | 106 | export type ModuleExpr = { 107 | kind: 'ModuleExpr' 108 | module: Token 109 | call: boolean 110 | args: ObjectLiteralExpr 111 | typeInfo: TypeInfo 112 | } 113 | 114 | type SubqueryExpr = { 115 | kind: 'SubqueryExpr' 116 | body: Stmt[] 117 | typeInfo: TypeInfo 118 | } 119 | 120 | type DrillExpr = { 121 | kind: 'DrillExpr' 122 | body: Expr[] 123 | typeInfo: TypeInfo 124 | } 125 | 126 | type ObjectEntryExpr = { 127 | kind: 'ObjectEntryExpr' 128 | key: Expr 129 | value: Expr 130 | optional: boolean 131 | typeInfo: TypeInfo 132 | } 133 | 134 | type ObjectLiteralExpr = { 135 | kind: 'ObjectLiteralExpr' 136 | entries: ObjectEntryExpr[] 137 | typeInfo: TypeInfo 138 | } 139 | 140 | type SliceExpr = { 141 | kind: 'SliceExpr' 142 | slice: Token 143 | typeInfo: TypeInfo 144 | } 145 | 146 | type LiteralExpr = { 147 | kind: 'LiteralExpr' 148 | value: boolean | number 149 | raw: Token 150 | typeInfo: TypeInfo 151 | } 152 | 153 | export type Stmt = 154 | | Program 155 | | ExtractStmt 156 | | AssignmentStmt 157 | | DeclInputsStmt 158 | | RequestStmt 159 | 160 | export type Expr = 161 | | InputExpr 162 | | RequestExpr 163 | | RequestBlockExpr 164 | | RequestEntryExpr 165 | | TemplateExpr 166 | | IdentifierExpr 167 | | DrillIdentifierExpr 168 | | SelectorExpr 169 | | ModifierExpr 170 | | ModuleExpr 171 | | SubqueryExpr 172 | | ObjectEntryExpr 173 | | ObjectLiteralExpr 174 | | SliceExpr 175 | | DrillExpr 176 | | LiteralExpr 177 | 178 | export type Node = Stmt | Expr 179 | 180 | const program = (body: Stmt[]): Program => ({ 181 | kind: 'Program', 182 | body, 183 | }) 184 | 185 | const assignmentStmt = ( 186 | name: Token, 187 | value: Expr, 188 | optional: boolean, 189 | ): AssignmentStmt => ({ 190 | kind: 'AssignmentStmt', 191 | name, 192 | optional, 193 | value, 194 | }) 195 | 196 | const declInputsStmt = (inputs: InputExpr[]): DeclInputsStmt => ({ 197 | kind: 'DeclInputsStmt', 198 | inputs, 199 | }) 200 | 201 | const InputExpr = ( 202 | id: Token, 203 | optional: boolean, 204 | defaultValue?: SliceExpr, 205 | ): InputExpr => ({ 206 | kind: 'InputExpr', 207 | typeInfo: { type: Type.Value }, 208 | id, 209 | optional, 210 | defaultValue, 211 | }) 212 | 213 | const extractStmt = (value: Expr): ExtractStmt => ({ 214 | kind: 'ExtractStmt', 215 | value, 216 | }) 217 | 218 | const requestStmt = (request: RequestExpr): RequestStmt => ({ 219 | kind: 'RequestStmt', 220 | request, 221 | }) 222 | 223 | const requestExpr = ( 224 | method: Token, 225 | url: TemplateExpr, 226 | headers: RequestBlockExpr, 227 | blocks: RequestBlockExpr[], 228 | body: Expr, 229 | ): RequestExpr => ({ 230 | kind: 'RequestExpr', 231 | typeInfo: { type: Type.Value }, 232 | method, 233 | url, 234 | headers, 235 | blocks, 236 | body, 237 | }) 238 | 239 | const requestBlockExpr = ( 240 | name: Token, 241 | entries: RequestEntryExpr[], 242 | ): RequestBlockExpr => ({ 243 | kind: 'RequestBlockExpr', 244 | typeInfo: { type: Type.Value }, 245 | name, 246 | entries, 247 | }) 248 | 249 | const requestEntryExpr = (key: Expr, value: Expr): RequestEntryExpr => ({ 250 | kind: 'RequestEntryExpr', 251 | typeInfo: { type: Type.Value }, 252 | key, 253 | value, 254 | }) 255 | 256 | const subqueryExpr = (body: Stmt[]): SubqueryExpr => ({ 257 | kind: 'SubqueryExpr', 258 | typeInfo: { type: Type.Value }, 259 | body, 260 | }) 261 | 262 | const drillExpr = (body: Expr[]): DrillExpr => ({ 263 | kind: 'DrillExpr', 264 | typeInfo: { type: Type.Value }, 265 | body, 266 | }) 267 | 268 | const identifierExpr = (id: Token): IdentifierExpr => ({ 269 | kind: 'IdentifierExpr', 270 | typeInfo: { type: Type.Value }, 271 | id, 272 | isUrlComponent: id.text.startsWith(':'), 273 | }) 274 | 275 | const drillIdentifierExpr = ( 276 | id: Token, 277 | expand: boolean, 278 | ): DrillIdentifierExpr => ({ 279 | kind: 'DrillIdentifierExpr', 280 | typeInfo: { type: Type.Value }, 281 | id, 282 | expand, 283 | }) 284 | 285 | const selectorExpr = (selector: Expr, expand: boolean): SelectorExpr => ({ 286 | kind: 'SelectorExpr', 287 | typeInfo: { type: Type.Value }, 288 | expand, 289 | selector, 290 | }) 291 | 292 | const modifierExpr = ( 293 | modifier: Token, 294 | inputs: ObjectLiteralExpr = objectLiteralExpr([]), 295 | ): ModifierExpr => ({ 296 | kind: 'ModifierExpr', 297 | typeInfo: { type: Type.Value }, 298 | modifier, 299 | args: inputs, 300 | }) 301 | 302 | const moduleExpr = ( 303 | module: Token, 304 | inputs: ObjectLiteralExpr = objectLiteralExpr([]), 305 | ): ModuleExpr => ({ 306 | kind: 'ModuleExpr', 307 | typeInfo: { type: Type.Value }, 308 | module, 309 | call: false, 310 | args: inputs, 311 | }) 312 | 313 | const objectEntryExpr = ( 314 | key: Expr, 315 | value: Expr, 316 | optional = false, 317 | ): ObjectEntryExpr => ({ 318 | kind: 'ObjectEntryExpr', 319 | typeInfo: { type: Type.Value }, 320 | key, 321 | optional, 322 | value, 323 | }) 324 | 325 | const objectLiteralExpr = (entries: ObjectEntryExpr[]): ObjectLiteralExpr => ({ 326 | kind: 'ObjectLiteralExpr', 327 | typeInfo: { type: Type.Value }, 328 | entries, 329 | }) 330 | 331 | const sliceExpr = (slice: Token): SliceExpr => ({ 332 | kind: 'SliceExpr', 333 | typeInfo: { type: Type.Value }, 334 | slice, 335 | }) 336 | 337 | const literalExpr = (raw: Token): LiteralExpr => ({ 338 | kind: 'LiteralExpr', 339 | typeInfo: { type: Type.Value }, 340 | raw, 341 | value: JSON.parse(raw.value), 342 | }) 343 | 344 | const templateExpr = (elements: (Expr | Token)[]): TemplateExpr => ({ 345 | kind: 'TemplateExpr', 346 | typeInfo: { type: Type.Value }, 347 | elements, 348 | }) 349 | 350 | export const t = { 351 | program, 352 | assignmentStmt, 353 | declInputsStmt, 354 | InputExpr, 355 | extractStmt, 356 | requestStmt, 357 | requestExpr, 358 | requestBlockExpr, 359 | requestEntryExpr, 360 | templateExpr, 361 | identifierExpr, 362 | drillIdentifierExpr, 363 | selectorExpr, 364 | modifierExpr, 365 | moduleExpr, 366 | sliceExpr, 367 | objectEntryExpr, 368 | objectLiteralExpr, 369 | subqueryExpr, 370 | drillExpr, 371 | literalExpr, 372 | } 373 | -------------------------------------------------------------------------------- /packages/get/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # @getlang/get 2 | 3 | ## 0.3.3 4 | 5 | ### Patch Changes 6 | 7 | - [#55](https://github.com/getlang-dev/get/pull/55) [`194676e`](https://github.com/getlang-dev/get/commit/194676e34bd4311af898565e045aa88a4eb5292a) Thanks [@mattfysh](https://github.com/mattfysh)! - fix modifier context (ii) 8 | 9 | - Updated dependencies [[`194676e`](https://github.com/getlang-dev/get/commit/194676e34bd4311af898565e045aa88a4eb5292a)]: 10 | - @getlang/parser@0.4.3 11 | 12 | ## 0.3.2 13 | 14 | ### Patch Changes 15 | 16 | - [#53](https://github.com/getlang-dev/get/pull/53) [`ecd87ab`](https://github.com/getlang-dev/get/commit/ecd87ab290fbccc7d61bc82950f82d8ed8b974d5) Thanks [@mattfysh](https://github.com/mattfysh)! - fix modifier context 17 | 18 | - Updated dependencies [[`ecd87ab`](https://github.com/getlang-dev/get/commit/ecd87ab290fbccc7d61bc82950f82d8ed8b974d5)]: 19 | - @getlang/parser@0.4.2 20 | 21 | ## 0.3.1 22 | 23 | ### Patch Changes 24 | 25 | - [#51](https://github.com/getlang-dev/get/pull/51) [`e9df7de`](https://github.com/getlang-dev/get/commit/e9df7dec6fc3f5424a680146d881938cea4a249a) Thanks [@mattfysh](https://github.com/mattfysh)! - modifier flags: `useContext` and `materialize` 26 | 27 | - Updated dependencies [[`e9df7de`](https://github.com/getlang-dev/get/commit/e9df7dec6fc3f5424a680146d881938cea4a249a)]: 28 | - @getlang/parser@0.4.1 29 | - @getlang/lib@0.2.1 30 | 31 | ## 0.3.0 32 | 33 | ### Minor Changes 34 | 35 | - [#43](https://github.com/getlang-dev/get/pull/43) [`0f1b076`](https://github.com/getlang-dev/get/commit/0f1b076e347b6ea8d36dd6e29af4c70a22f0fd41) Thanks [@mattfysh](https://github.com/mattfysh)! - slice dependencies analysis 36 | 37 | - [`78660db`](https://github.com/getlang-dev/get/commit/78660db7089cebb8715c5389bac81559f2f4f60b) Thanks [@mattfysh](https://github.com/mattfysh)! - reorganize package layout 38 | 39 | - [#42](https://github.com/getlang-dev/get/pull/42) [`b7385e8`](https://github.com/getlang-dev/get/commit/b7385e89ffcbe02e93fde9789d464b094ae4c726) Thanks [@mattfysh](https://github.com/mattfysh)! - module call context 40 | 41 | ### Patch Changes 42 | 43 | - [#46](https://github.com/getlang-dev/get/pull/46) [`fe8b25b`](https://github.com/getlang-dev/get/commit/fe8b25ba179d0a770b5e00275c70e9c7fc0405cf) Thanks [@mattfysh](https://github.com/mattfysh)! - add custom modifiers 44 | 45 | - [#44](https://github.com/getlang-dev/get/pull/44) [`356a09a`](https://github.com/getlang-dev/get/commit/356a09a81cddcba4c788a103451c8c9517ec596d) Thanks [@mattfysh](https://github.com/mattfysh)! - drill-based walker 46 | 47 | - [#40](https://github.com/getlang-dev/get/pull/40) [`70cb95b`](https://github.com/getlang-dev/get/commit/70cb95be361513925eb9881ee2988787a98dcf1b) Thanks [@mattfysh](https://github.com/mattfysh)! - upgrade deps 48 | 49 | - [#47](https://github.com/getlang-dev/get/pull/47) [`a2304c8`](https://github.com/getlang-dev/get/commit/a2304c824175bb75be82f08a61a7a04880fb137f) Thanks [@mattfysh](https://github.com/mattfysh)! - flexible urls with implied params 50 | 51 | - [#48](https://github.com/getlang-dev/get/pull/48) [`3e8b1c8`](https://github.com/getlang-dev/get/commit/3e8b1c8a591b5de149c6cc7556900a250273c9a5) Thanks [@mattfysh](https://github.com/mattfysh)! - primitive literals 52 | 53 | - Updated dependencies [[`fe8b25b`](https://github.com/getlang-dev/get/commit/fe8b25ba179d0a770b5e00275c70e9c7fc0405cf), [`0f1b076`](https://github.com/getlang-dev/get/commit/0f1b076e347b6ea8d36dd6e29af4c70a22f0fd41), [`356a09a`](https://github.com/getlang-dev/get/commit/356a09a81cddcba4c788a103451c8c9517ec596d), [`70cb95b`](https://github.com/getlang-dev/get/commit/70cb95be361513925eb9881ee2988787a98dcf1b), [`a2304c8`](https://github.com/getlang-dev/get/commit/a2304c824175bb75be82f08a61a7a04880fb137f), [`78660db`](https://github.com/getlang-dev/get/commit/78660db7089cebb8715c5389bac81559f2f4f60b), [`3e8b1c8`](https://github.com/getlang-dev/get/commit/3e8b1c8a591b5de149c6cc7556900a250273c9a5), [`b7385e8`](https://github.com/getlang-dev/get/commit/b7385e89ffcbe02e93fde9789d464b094ae4c726)]: 54 | - @getlang/parser@0.4.0 55 | - @getlang/walker@0.1.0 56 | - @getlang/ast@0.1.0 57 | - @getlang/lib@0.2.0 58 | 59 | ## 0.2.5 60 | 61 | ### Patch Changes 62 | 63 | - [#38](https://github.com/getlang-dev/get/pull/38) [`f0f00d3`](https://github.com/getlang-dev/get/commit/f0f00d33a75c830105f093f07c9bdeeec333df25) Thanks [@mattfysh](https://github.com/mattfysh)! - add `executeModule` api 64 | 65 | ## 0.2.4 66 | 67 | ### Patch Changes 68 | 69 | - [#35](https://github.com/getlang-dev/get/pull/35) [`0c806e0`](https://github.com/getlang-dev/get/commit/0c806e092fc1d87696fb820ac7246b5d3afb63ca) Thanks [@mattfysh](https://github.com/mattfysh)! - bugbash 70 | 71 | - Updated dependencies [[`0c806e0`](https://github.com/getlang-dev/get/commit/0c806e092fc1d87696fb820ac7246b5d3afb63ca)]: 72 | - @getlang/parser@0.3.4 73 | - @getlang/lib@0.1.5 74 | 75 | ## 0.2.3 76 | 77 | ### Patch Changes 78 | 79 | - [#31](https://github.com/getlang-dev/get/pull/31) [`1a395f9`](https://github.com/getlang-dev/get/commit/1a395f9df71d3507bc5b3841eddc9336db3a69ee) Thanks [@mattfysh](https://github.com/mattfysh)! - request parsers 80 | 81 | - Updated dependencies [[`1a395f9`](https://github.com/getlang-dev/get/commit/1a395f9df71d3507bc5b3841eddc9336db3a69ee)]: 82 | - @getlang/parser@0.3.3 83 | - @getlang/utils@0.1.6 84 | 85 | ## 0.2.2 86 | 87 | ### Patch Changes 88 | 89 | - [`aca9d92`](https://github.com/getlang-dev/get/commit/aca9d929e4877fc614b66740c9415f5bcd083c24) Thanks [@mattfysh](https://github.com/mattfysh)! - fix publish 90 | 91 | - Updated dependencies [[`aca9d92`](https://github.com/getlang-dev/get/commit/aca9d929e4877fc614b66740c9415f5bcd083c24)]: 92 | - @getlang/lib@0.1.4 93 | - @getlang/parser@0.3.2 94 | - @getlang/utils@0.1.5 95 | 96 | ## 0.2.1 97 | 98 | ### Patch Changes 99 | 100 | - [`3660548`](https://github.com/getlang-dev/get/commit/3660548d79dd24e13a74bf0d6f24c1fec512062d) Thanks [@mattfysh](https://github.com/mattfysh)! - prevent tsconfig publish 101 | 102 | - Updated dependencies [[`3660548`](https://github.com/getlang-dev/get/commit/3660548d79dd24e13a74bf0d6f24c1fec512062d)]: 103 | - @getlang/parser@0.3.1 104 | - @getlang/utils@0.1.4 105 | - @getlang/lib@0.1.3 106 | 107 | ## 0.2.0 108 | 109 | ### Minor Changes 110 | 111 | - [#28](https://github.com/getlang-dev/get/pull/28) [`f1bdc2c`](https://github.com/getlang-dev/get/commit/f1bdc2c8433a942f84503606790e8bcf4fb37477) Thanks [@mattfysh](https://github.com/mattfysh)! - page links 112 | 113 | ### Patch Changes 114 | 115 | - [#26](https://github.com/getlang-dev/get/pull/26) [`58bb154`](https://github.com/getlang-dev/get/commit/58bb1540f1fa50093b3a72ffb2446e7b1c6aacb0) Thanks [@mattfysh](https://github.com/mattfysh)! - interpolated template optionals 116 | 117 | - Updated dependencies [[`f1bdc2c`](https://github.com/getlang-dev/get/commit/f1bdc2c8433a942f84503606790e8bcf4fb37477), [`58bb154`](https://github.com/getlang-dev/get/commit/58bb1540f1fa50093b3a72ffb2446e7b1c6aacb0)]: 118 | - @getlang/parser@0.3.0 119 | - @getlang/utils@0.1.3 120 | 121 | ## 0.1.2 122 | 123 | ### Patch Changes 124 | 125 | - [#23](https://github.com/getlang-dev/get/pull/23) [`28d0ca8`](https://github.com/getlang-dev/get/commit/28d0ca8dcf840cfc70f002d06a48cace834edcf9) Thanks [@mattfysh](https://github.com/mattfysh)! - workspace layout updates 126 | 127 | - Updated dependencies [[`28d0ca8`](https://github.com/getlang-dev/get/commit/28d0ca8dcf840cfc70f002d06a48cace834edcf9)]: 128 | - @getlang/parser@0.2.2 129 | - @getlang/utils@0.1.2 130 | - @getlang/lib@0.1.2 131 | 132 | ## 0.1.1 133 | 134 | ### Patch Changes 135 | 136 | - [#12](https://github.com/getlang-dev/get/pull/12) [`4b81eda`](https://github.com/getlang-dev/get/commit/4b81eda1e71727f59fe7a0d26abde186ed78c876) Thanks [@mattfysh](https://github.com/mattfysh)! - rename function expressions to subqueries 137 | 138 | - Updated dependencies [[`58e9988`](https://github.com/getlang-dev/get/commit/58e99887e39956ee1e3eaf669cb92fbfa188a022), [`4b81eda`](https://github.com/getlang-dev/get/commit/4b81eda1e71727f59fe7a0d26abde186ed78c876)]: 139 | - @getlang/parser@0.2.0 140 | - @getlang/utils@0.1.1 141 | - @getlang/lib@0.1.1 142 | 143 | ## 0.1.0 144 | 145 | ### Minor Changes 146 | 147 | - [`57ff46d`](https://github.com/getlang-dev/get/commit/57ff46d904484e3277ee7a8481cdf4cee4c3deb2) Thanks [@mattfysh](https://github.com/mattfysh)! - Release v0.1.0 148 | 149 | ### Patch Changes 150 | 151 | - Updated dependencies [[`57ff46d`](https://github.com/getlang-dev/get/commit/57ff46d904484e3277ee7a8481cdf4cee4c3deb2)]: 152 | - @getlang/lib@0.1.0 153 | - @getlang/parser@0.1.0 154 | - @getlang/utils@0.1.0 155 | -------------------------------------------------------------------------------- /packages/parser/src/passes/inference/typeinfo.ts: -------------------------------------------------------------------------------- 1 | import type { Expr, Node, Program, TypeInfo } from '@getlang/ast' 2 | import { Type, t } from '@getlang/ast' 3 | import { invariant } from '@getlang/lib' 4 | import { QuerySyntaxError, ValueReferenceError } from '@getlang/lib/errors' 5 | import type { Path, TransformVisitor } from '@getlang/walker' 6 | import { ScopeTracker, transform } from '@getlang/walker' 7 | import { toPath } from 'lodash-es' 8 | import { render, tx } from '../../utils.js' 9 | 10 | function unwrap(typeInfo: TypeInfo) { 11 | switch (typeInfo.type) { 12 | case Type.List: 13 | return unwrap(typeInfo.of) 14 | case Type.Maybe: 15 | return unwrap(typeInfo.option) 16 | default: 17 | return structuredClone(typeInfo) 18 | } 19 | } 20 | 21 | function rewrap( 22 | typeInfo: TypeInfo | undefined, 23 | itemTypeInfo: TypeInfo, 24 | optional?: boolean, 25 | ): TypeInfo { 26 | switch (typeInfo?.type) { 27 | case Type.List: 28 | return { ...typeInfo, of: rewrap(typeInfo.of, itemTypeInfo, optional) } 29 | case Type.Maybe: { 30 | const option = rewrap(typeInfo.option, itemTypeInfo, optional) 31 | if (option.type === Type.Maybe || !optional) { 32 | return option 33 | } 34 | return { ...typeInfo, option } 35 | } 36 | default: 37 | return structuredClone(itemTypeInfo) 38 | } 39 | } 40 | 41 | function specialize(macroType: TypeInfo, contextType?: TypeInfo) { 42 | function walk(ti: TypeInfo): TypeInfo { 43 | switch (ti.type) { 44 | case Type.Context: 45 | invariant(contextType, 'Specialize requires context type') 46 | return contextType 47 | case Type.Maybe: 48 | return { ...ti, option: walk(ti.option) } 49 | case Type.List: 50 | return { ...ti, of: walk(ti.of) } 51 | case Type.Struct: { 52 | const schema = Object.fromEntries( 53 | Object.entries(ti.schema).map(e => [e[0], walk(e[1])]), 54 | ) 55 | return { ...ti, schema } 56 | } 57 | default: 58 | return ti 59 | } 60 | } 61 | return walk(macroType) 62 | } 63 | 64 | class ItemScopeTracker extends ScopeTracker { 65 | optional = [false] 66 | 67 | override enter(node: Node) { 68 | super.enter(node) 69 | if (this.context && 'typeInfo' in node) { 70 | this.push({ 71 | ...this.context, 72 | typeInfo: unwrap(this.context.typeInfo), 73 | }) 74 | } 75 | } 76 | 77 | override exit(node: Node, path: Path) { 78 | if (this.context && 'typeInfo' in node) { 79 | this.pop() 80 | node.typeInfo = rewrap( 81 | this.context.typeInfo, 82 | structuredClone(node.typeInfo), 83 | this.optional.at(-1), 84 | ) 85 | } 86 | super.exit(node, path) 87 | } 88 | } 89 | 90 | type ResolveTypeOptions = { 91 | returnTypes: { [module: string]: TypeInfo } 92 | contextType?: TypeInfo 93 | } 94 | 95 | export function resolveTypes(ast: Program, options: ResolveTypeOptions) { 96 | const { returnTypes, contextType = { type: Type.Context } } = options 97 | const scope = new ItemScopeTracker() 98 | let ex: Expr | undefined 99 | 100 | const visitor: TransformVisitor = { 101 | Program: { 102 | enter() { 103 | scope.context = { 104 | ...t.InputExpr(tx.token(''), false), 105 | typeInfo: contextType, 106 | } 107 | }, 108 | exit() { 109 | ex = scope.extracted 110 | }, 111 | }, 112 | 113 | InputExpr(node) { 114 | let typeInfo: TypeInfo = { type: Type.Value } 115 | if (node.optional) { 116 | typeInfo = { type: Type.Maybe, option: typeInfo } 117 | } 118 | return { ...node, typeInfo } 119 | }, 120 | 121 | AssignmentStmt: { 122 | enter(node) { 123 | scope.optional.push(node.optional) 124 | }, 125 | exit() { 126 | scope.optional.pop() 127 | }, 128 | }, 129 | 130 | IdentifierExpr(node) { 131 | const id = node.id.value 132 | const value = scope.lookup(id) 133 | invariant(value, new ValueReferenceError(id)) 134 | const typeInfo = structuredClone(value.typeInfo) 135 | return { ...node, typeInfo } 136 | }, 137 | 138 | DrillIdentifierExpr(node) { 139 | const id = node.id.value 140 | const value = scope.lookup(id) 141 | invariant(value, new ValueReferenceError(id)) 142 | let typeInfo = structuredClone(value.typeInfo) 143 | if (node.expand) { 144 | typeInfo = { type: Type.List, of: typeInfo } 145 | } 146 | return { ...node, typeInfo } 147 | }, 148 | 149 | RequestExpr(node) { 150 | return { 151 | ...node, 152 | typeInfo: { 153 | type: Type.Struct, 154 | schema: { 155 | url: { type: Type.Value }, 156 | status: { type: Type.Value }, 157 | headers: { type: Type.Headers }, 158 | body: { type: Type.Value }, 159 | }, 160 | }, 161 | } 162 | }, 163 | 164 | SliceExpr(node) { 165 | let typeInfo: TypeInfo = { type: Type.Value } 166 | if (scope.optional.at(-1)) { 167 | typeInfo = { type: Type.Maybe, option: typeInfo } 168 | } 169 | return { ...node, typeInfo } 170 | }, 171 | 172 | SelectorExpr(node) { 173 | function selectorTypeInfo(): TypeInfo { 174 | invariant( 175 | node.selector.kind === 'TemplateExpr', 176 | new QuerySyntaxError('Selector requires template'), 177 | ) 178 | const scopeT = scope.context!.typeInfo 179 | switch (scopeT.type) { 180 | case Type.Headers: 181 | case Type.Cookies: 182 | return { type: Type.Value } 183 | case Type.Struct: { 184 | const sel = render(node.selector) 185 | return toPath(sel).reduce( 186 | (acc, cur) => 187 | (acc.type === Type.Struct && acc.schema[cur]) || { 188 | type: Type.Value, 189 | }, 190 | scopeT, 191 | ) 192 | } 193 | default: 194 | return scopeT 195 | } 196 | } 197 | 198 | let typeInfo: TypeInfo = structuredClone(selectorTypeInfo()) 199 | if (node.expand) { 200 | typeInfo = { type: Type.List, of: typeInfo } 201 | } else if (scope.optional.at(-1)) { 202 | typeInfo = { type: Type.Maybe, option: typeInfo } 203 | } 204 | return { ...node, typeInfo } 205 | }, 206 | 207 | ModifierExpr(node) { 208 | const modTypeMap: Record = { 209 | html: { type: Type.Html }, 210 | js: { type: Type.Js }, 211 | json: { type: Type.Value }, 212 | link: { type: Type.Value }, 213 | headers: { type: Type.Headers }, 214 | cookies: { type: Type.Cookies }, 215 | } 216 | const mod = node.modifier.value 217 | const typeInfo = modTypeMap[mod] || returnTypes[mod] 218 | invariant(typeInfo, 'Modifier type lookup failed') 219 | return { ...node, typeInfo } 220 | }, 221 | 222 | ModuleExpr(node) { 223 | let typeInfo: TypeInfo = { type: Type.Value } 224 | if (node.call) { 225 | const returnType = returnTypes[node.module.value] 226 | invariant(returnType, 'Module return type lookup failed') 227 | typeInfo = specialize(returnType, scope.context?.typeInfo) 228 | } 229 | return { ...node, typeInfo } 230 | }, 231 | 232 | SubqueryExpr(node) { 233 | const typeInfo = scope.extracted?.typeInfo || { type: Type.Never } 234 | return { ...node, typeInfo: structuredClone(typeInfo) } 235 | }, 236 | 237 | DrillExpr(node) { 238 | const typeInfo = structuredClone(node.body.at(-1)!.typeInfo) 239 | return { ...node, typeInfo } 240 | }, 241 | 242 | ObjectEntryExpr: { 243 | enter(node) { 244 | scope.optional.push(node.optional) 245 | }, 246 | exit() { 247 | scope.optional.pop() 248 | }, 249 | }, 250 | 251 | ObjectLiteralExpr(node) { 252 | const typeInfo: TypeInfo = { 253 | type: Type.Struct, 254 | schema: Object.fromEntries( 255 | node.entries.map(e => { 256 | const key = render(e.key) 257 | invariant( 258 | key, 259 | new QuerySyntaxError('Object keys must be string literals'), 260 | ) 261 | const value = structuredClone(e.value.typeInfo) 262 | return [key, value] 263 | }), 264 | ), 265 | } 266 | return { ...node, typeInfo } 267 | }, 268 | } 269 | 270 | const program: Program = transform(ast, { scope, ...visitor }) 271 | const returnType = ex?.typeInfo ?? { type: Type.Never } 272 | 273 | return { program, returnType: returnType } 274 | } 275 | -------------------------------------------------------------------------------- /packages/get/src/execute.ts: -------------------------------------------------------------------------------- 1 | import type { Expr, TypeInfo } from '@getlang/ast' 2 | import { isToken, Type } from '@getlang/ast' 3 | import type { Hooks, Inputs } from '@getlang/lib' 4 | import * as lib from '@getlang/lib' 5 | import * as errors from '@getlang/lib/errors' 6 | import type { ReduceVisitor } from '@getlang/walker' 7 | import { reduce, ScopeTracker } from '@getlang/walker' 8 | import type { Execute } from './calls.js' 9 | import { callModifier, callModule } from './calls.js' 10 | import { Registry } from './registry.js' 11 | import type { RuntimeValue } from './value.js' 12 | import { assert, materialize } from './value.js' 13 | 14 | const { 15 | NullInputError, 16 | QuerySyntaxError, 17 | SliceError, 18 | UnknownInputsError, 19 | ValueTypeError, 20 | ValueReferenceError, 21 | } = errors 22 | 23 | export async function execute( 24 | rootModule: string, 25 | rootInputs: Inputs, 26 | hooks: Required, 27 | ) { 28 | const scope = new ScopeTracker() 29 | 30 | const executeModule: Execute = async (entry, inputs) => { 31 | const provided = new Set(Object.keys(inputs)) 32 | const unknown = provided.difference(entry.inputs) 33 | lib.invariant(unknown.size === 0, new UnknownInputsError([...unknown])) 34 | 35 | async function withItemContext(expr: Expr): Promise { 36 | const ctx = scope.context 37 | if (ctx?.typeInfo.type !== Type.List) { 38 | const { data } = await reduce(expr, options) 39 | return data 40 | } 41 | const list = [] 42 | for (const data of ctx.data) { 43 | scope.push({ data, typeInfo: ctx.typeInfo.of }) 44 | const item = await withItemContext(expr) 45 | list.push(item) 46 | scope.pop() 47 | } 48 | return list 49 | } 50 | 51 | function lookup(id: string, typeInfo: TypeInfo) { 52 | const value = scope.lookup(id) 53 | lib.invariant(value, new ValueReferenceError(id)) 54 | return { data: value.data, typeInfo } 55 | } 56 | 57 | let ex: RuntimeValue | undefined 58 | 59 | const visitor: ReduceVisitor = { 60 | InputExpr(node) { 61 | const name = node.id.value 62 | let data = inputs[name] 63 | if (data === undefined) { 64 | if (!node.optional) { 65 | throw new NullInputError(name) 66 | } else if (node.defaultValue) { 67 | data = node.defaultValue.data 68 | } else { 69 | data = new lib.NullSelection(`input:${name}`) 70 | } 71 | } 72 | return { data, typeInfo: node.typeInfo } 73 | }, 74 | 75 | AssignmentStmt(node) { 76 | assert(node.value) 77 | }, 78 | 79 | LiteralExpr(node) { 80 | return { data: node.value, typeInfo: node.typeInfo } 81 | }, 82 | 83 | Program: { 84 | enter() { 85 | scope.extracted = { data: null, typeInfo: { type: Type.Value } } 86 | }, 87 | exit() { 88 | ex = scope.extracted 89 | }, 90 | }, 91 | 92 | TemplateExpr(node, path) { 93 | const firstNull = node.elements.find( 94 | el => 'data' in el && el.data instanceof lib.NullSelection, 95 | ) 96 | if (firstNull) { 97 | const isRoot = path.parent?.node.kind !== 'TemplateExpr' 98 | return isRoot ? firstNull : '' 99 | } 100 | const els = node.elements.map(el => { 101 | return isToken(el) ? el.value : materialize(el) 102 | }) 103 | const data = els.join('') 104 | return { data, typeInfo: node.typeInfo } 105 | }, 106 | 107 | async SliceExpr({ slice, typeInfo }) { 108 | try { 109 | const ctx = scope.context 110 | const deps = ctx ? materialize(ctx) : {} 111 | const ret = await hooks.slice(slice.value, deps) 112 | const data = 113 | ret === undefined ? new lib.NullSelection('') : ret 114 | return { data, typeInfo } 115 | } catch (e) { 116 | throw new SliceError({ cause: e }) 117 | } 118 | }, 119 | 120 | IdentifierExpr(node) { 121 | return lookup(node.id.value, node.typeInfo) 122 | }, 123 | 124 | DrillIdentifierExpr(node) { 125 | return lookup(node.id.value, node.typeInfo) 126 | }, 127 | 128 | SelectorExpr(node) { 129 | lib.invariant(scope.context, 'Unresolved context') 130 | 131 | const selector = node.selector.data 132 | lib.invariant( 133 | typeof selector === 'string', 134 | new ValueTypeError('Expected selector string'), 135 | ) 136 | 137 | const args = [scope.context.data, selector, node.expand] as const 138 | 139 | function select(typeInfo: TypeInfo) { 140 | switch (typeInfo.type) { 141 | case Type.Maybe: 142 | return select(typeInfo.option) 143 | case Type.Html: 144 | return lib.html.select(...args) 145 | case Type.Js: 146 | return lib.js.select(...args) 147 | case Type.Headers: 148 | return lib.headers.select(...args) 149 | case Type.Cookies: 150 | return lib.cookies.select(...args) 151 | default: 152 | return lib.json.select(...args) 153 | } 154 | } 155 | 156 | const data = select(scope.context.typeInfo) 157 | return { data, typeInfo: node.typeInfo } 158 | }, 159 | 160 | async ModifierExpr(node) { 161 | const mod = node.modifier.value 162 | const args = materialize(node.args) 163 | const data = await callModifier(registry, mod, args, scope.context) 164 | return { data, typeInfo: node.typeInfo } 165 | }, 166 | 167 | ModuleExpr(node) { 168 | return node.call 169 | ? callModule( 170 | registry, 171 | executeModule, 172 | hooks, 173 | node.module.value, 174 | node.args, 175 | scope.context?.typeInfo, 176 | ) 177 | : { 178 | data: materialize(node.args), 179 | typeInfo: node.typeInfo, 180 | } 181 | }, 182 | 183 | ObjectEntryExpr(node) { 184 | assert(node.value) 185 | const data = [node.key.data, node.value.data] 186 | return { data, typeInfo: { type: Type.Value } } 187 | }, 188 | 189 | ObjectLiteralExpr(node) { 190 | const data = Object.fromEntries( 191 | node.entries 192 | .map(e => e.data) 193 | .filter(e => !(e[1] instanceof lib.NullSelection)), 194 | ) 195 | return { data, typeInfo: node.typeInfo } 196 | }, 197 | 198 | SubqueryExpr() { 199 | const ex = scope.extracted 200 | lib.invariant(ex, new QuerySyntaxError('Subquery must extract a value')) 201 | return ex 202 | }, 203 | 204 | DrillExpr: { 205 | async enter(node, path) { 206 | for (const expr of node.body) { 207 | if (!(scope.context?.data instanceof lib.NullSelection)) { 208 | const data = await withItemContext(expr) 209 | scope.context = { data, typeInfo: expr.typeInfo } 210 | } 211 | } 212 | path.replace(scope.context) 213 | }, 214 | }, 215 | 216 | async RequestExpr(node) { 217 | const method = node.method.value 218 | const url = node.url.data 219 | const body = node.body?.data ?? '' 220 | 221 | const headers = node.headers.data[1] 222 | const blocks = Object.fromEntries(node.blocks.map(v => v.data)) 223 | 224 | const data = await lib.http.request( 225 | method, 226 | url, 227 | headers, 228 | blocks, 229 | body, 230 | hooks.request, 231 | ) 232 | return { data, typeInfo: node.typeInfo } 233 | }, 234 | 235 | RequestBlockExpr(node) { 236 | const value = Object.fromEntries( 237 | node.entries 238 | .map(e => e.data) 239 | .filter(e => !(e[1] instanceof lib.NullSelection)), 240 | ) 241 | const data = [node.name.value, value] 242 | return { data, typeInfo: node.typeInfo } 243 | }, 244 | 245 | RequestEntryExpr(node) { 246 | const data = [node.key.data, node.value.data] 247 | return { data, typeInfo: { type: Type.Value } } 248 | }, 249 | } 250 | 251 | const options = { scope, ...visitor } 252 | await reduce(entry.program, options) 253 | return ex 254 | } 255 | 256 | const registry = new Registry(hooks) 257 | const rootEntry = await registry.import(rootModule) 258 | const ex = await executeModule(rootEntry, rootInputs) 259 | return ex && assert(ex) && materialize(ex) 260 | } 261 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2024 Matthew Fysh 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /test/modules.spec.ts: -------------------------------------------------------------------------------- 1 | import { describe, expect, test } from 'bun:test' 2 | import { 3 | NullInputError, 4 | RecursiveCallError, 5 | UnknownInputsError, 6 | } from '@getlang/lib/errors' 7 | import { execute } from './helpers.js' 8 | 9 | describe('modules', () => { 10 | test('extract', async () => { 11 | const src = 'extract `501`' 12 | const result = await execute(src) 13 | expect(result).toEqual(501) 14 | }) 15 | 16 | test('syntax error', () => { 17 | const result = execute( 18 | ` 19 | GET https://test.com 20 | 21 | extrct { title } 22 | `, 23 | {}, 24 | { willThrow: true }, 25 | ) 26 | return expect(result).rejects.toThrow( 27 | 'SyntaxError: Invalid token at line 3 col 1:\n\n1 GET https://test.com\n2 \n3 extrct { title }\n ^', 28 | ) 29 | }) 30 | 31 | describe('inputs', () => { 32 | test('single', async () => { 33 | const src = ` 34 | inputs { value } 35 | extract $value 36 | ` 37 | const result = await execute(src, { value: 'ping' }) 38 | expect(result).toEqual('ping') 39 | }) 40 | 41 | test('multiple', async () => { 42 | const src = ` 43 | inputs { x, y } 44 | extract $y 45 | ` 46 | const result = await execute(src, { x: 10, y: 20 }) 47 | expect(result).toEqual(20) 48 | }) 49 | 50 | test('required input missing', () => { 51 | const result = execute(` 52 | inputs { value } 53 | extract $value 54 | `) 55 | return expect(result).rejects.toThrow(new NullInputError('value')) 56 | }) 57 | 58 | test('unknown input provided', () => { 59 | const result = execute('extract `123`', { x: 1 }) 60 | return expect(result).rejects.toThrow(new UnknownInputsError(['x'])) 61 | }) 62 | 63 | test('unknown inputs provided', () => { 64 | const result = execute('extract `123`', { x: 1, y: 2 }) 65 | return expect(result).rejects.toThrow(new UnknownInputsError(['x', 'y'])) 66 | }) 67 | 68 | test('optional input', async () => { 69 | const src = ` 70 | inputs { value? } 71 | extract |12| 72 | ` 73 | const result = await execute(src) 74 | // does not throw error 75 | expect(result).toEqual(12) 76 | }) 77 | 78 | test('default value', async () => { 79 | const src = ` 80 | inputs { stopA = |'big sur'|, stopB = |'carmel'| } 81 | 82 | extract { $stopA, $stopB } 83 | ` 84 | const result = await execute(src, { stopB: 'monterey' }) 85 | expect(result).toEqual({ 86 | stopA: 'big sur', 87 | stopB: 'monterey', 88 | }) 89 | }) 90 | 91 | test('falsy default value', async () => { 92 | const src = ` 93 | inputs { offset = |0| } 94 | extract { $offset } 95 | ` 96 | const result = await execute(src, {}) 97 | expect(result).toEqual({ offset: 0 }) 98 | }) 99 | }) 100 | 101 | test('variables', async () => { 102 | const result = await execute(` 103 | set x = |{ test: true }| 104 | extract $x 105 | `) 106 | expect(result).toEqual({ test: true }) 107 | }) 108 | 109 | test('subquery scope with context', async () => { 110 | const result = await execute(` 111 | set x = |{ test: true }| 112 | extract $x -> ( extract $ ) 113 | `) 114 | expect(result).toEqual({ test: true }) 115 | }) 116 | 117 | test('subquery scope with closures', async () => { 118 | const result = await execute(` 119 | set x = |{ test: true }| 120 | extract ( 121 | extract $x 122 | ) 123 | `) 124 | expect(result).toEqual({ test: true }) 125 | }) 126 | 127 | describe('calls', () => { 128 | test('modules', async () => { 129 | const result = await execute({ 130 | Auth: 'extract { token: `"abc"` }', 131 | Home: `extract { 132 | auth: @Auth -> token 133 | }`, 134 | }) 135 | expect(result).toEqual({ 136 | auth: 'abc', 137 | }) 138 | }) 139 | 140 | describe('semantics', () => { 141 | const Call = ` 142 | inputs { called } 143 | extract { called: |true| } 144 | ` 145 | 146 | test('select', async () => { 147 | const modules = { 148 | Call, 149 | Home: ` 150 | set called = |false| 151 | extract { 152 | select: @Call({ $called }) -> called 153 | } 154 | `, 155 | } 156 | const result = await execute(modules) 157 | expect(result).toEqual({ select: true }) 158 | }) 159 | 160 | test('from var', async () => { 161 | const modules = { 162 | Call, 163 | Home: ` 164 | set called = |false| 165 | set as_var = @Call({ $called }) 166 | extract { 167 | from_var: $as_var -> called 168 | } 169 | `, 170 | } 171 | const result = await execute(modules) 172 | expect(result).toEqual({ from_var: true }) 173 | }) 174 | 175 | test('subquery', async () => { 176 | const modules = { 177 | Call, 178 | Home: ` 179 | set called = |false| 180 | extract { 181 | subquery: ( 182 | extract @Call({ $called }) 183 | ) -> called 184 | } 185 | `, 186 | } 187 | const result = await execute(modules) 188 | expect(result).toEqual({ subquery: true }) 189 | }) 190 | 191 | test('from subquery', async () => { 192 | const modules = { 193 | Call, 194 | Home: ` 195 | set called = |false| 196 | set as_subquery = ( extract @Call({ $called }) ) 197 | 198 | extract { 199 | from_subquery: $as_subquery -> called 200 | } 201 | `, 202 | } 203 | const result = await execute(modules) 204 | expect(result).toEqual({ from_subquery: true }) 205 | }) 206 | 207 | test.skip('object key', async () => { 208 | const modules = { 209 | Call, 210 | Home: ` 211 | set called = |false| 212 | extract { 213 | object: as_entry = { key: @Call({ $called }) } -> key -> called 214 | } 215 | `, 216 | } 217 | const result = await execute(modules) 218 | expect(result).toEqual({ object: true }) 219 | }) 220 | }) 221 | 222 | test('paint inputs', async () => { 223 | const modules = { 224 | Reverse: ` 225 | inputs { list } 226 | set result = | list.map(x => Number(x) * 10).reverse() | 227 | extract { $result } 228 | `, 229 | Home: ` 230 | set list = "
  • 1
  • 2
  • 3
" -> @html => li 231 | extract @Reverse({ $list }) -> result 232 | `, 233 | } 234 | 235 | const result = await execute(modules) 236 | expect(result).toEqual([30, 20, 10]) 237 | }) 238 | 239 | test('drill return value', async () => { 240 | const modules = { 241 | Req: ` 242 | GET http://stub 243 | 244 | extract @html 245 | `, 246 | Home: ` 247 | set req = @Req 248 | extract $req -> { div, span } 249 | `, 250 | } 251 | 252 | const result = await execute( 253 | modules, 254 | {}, 255 | { 256 | fetch: () => 257 | new Response(`
x
y`), 258 | }, 259 | ) 260 | expect(result).toEqual({ div: 'x', span: 'y' }) 261 | }) 262 | 263 | test.skip('drill returned request', async () => { 264 | const modules = { 265 | Req: ` 266 | GET http://stub 267 | 268 | extract $ 269 | `, 270 | Home: ` 271 | set req = @Req 272 | extract $req -> { div, span } 273 | `, 274 | } 275 | 276 | const result = await execute( 277 | modules, 278 | {}, 279 | { 280 | fetch: () => 281 | new Response(`
x
y`), 282 | }, 283 | ) 284 | expect(result).toEqual({ div: 'x', span: 'y' }) 285 | }) 286 | 287 | test('links', async () => { 288 | const modules = { 289 | Search: 'extract `1`', 290 | Product: 'extract `2`', 291 | Home: ` 292 | inputs { query, page? } 293 | 294 | GET https://search.com/ 295 | [query] 296 | s: $query 297 | page: $page 298 | 299 | set results = => li.result -> @Product({ 300 | @link: a 301 | name: a 302 | desc: p.description 303 | }) 304 | 305 | extract { 306 | items: $results 307 | pager: .pager -> { 308 | next: @Search) a.next 309 | prev: @Search) a.prev 310 | } 311 | } 312 | `, 313 | } 314 | 315 | const result = await execute( 316 | modules, 317 | { query: 'gifts' }, 318 | { 319 | fetch: () => 320 | new Response(` 321 | 322 |
    323 |
  • 324 | Deck o cards 325 |

    Casino grade playing cards

    326 |
    • 327 |
328 |
329 | 330 |
331 | `), 332 | }, 333 | ) 334 | expect(result).toEqual({ 335 | items: [ 336 | { 337 | '@link': 'https://search.com/products/1', 338 | name: 'Deck o cards', 339 | desc: 'Casino grade playing cards', 340 | }, 341 | ], 342 | pager: { 343 | next: { 344 | '@link': 'https://search.com/?s=gifts&page=2', 345 | }, 346 | prev: {}, 347 | }, 348 | }) 349 | }) 350 | 351 | test('links pre/post-amble', async () => { 352 | const modules = { 353 | Home: ` 354 | GET http://stub/x/y/z 355 | 356 | extract #a 357 | -> :scope > #b 358 | -> @Link) :scope > #c 359 | -> :scope > #d 360 | `, 361 | Link: ` 362 | extract { 363 | _module: |'Link'| 364 | } 365 | `, 366 | } 367 | const result = await execute( 368 | modules, 369 | {}, 370 | { 371 | fetch: () => 372 | new Response(` 373 | 374 | 375 |
376 |
377 |
378 | link 379 |
380 |
381 | 382 | `), 383 | }, 384 | ) 385 | 386 | expect(result).toEqual({ 387 | '@link': 'http://stub/a/b/c/d', 388 | }) 389 | }) 390 | 391 | test('context propagation', async () => { 392 | const modules = { 393 | Home: ` 394 | GET http://stub 395 | 396 | extract { 397 | a: @Data({ text: |'first'| }) 398 | b: @Data({ text: |'second'| }) 399 | } 400 | `, 401 | Data: ` 402 | inputs { text } 403 | 404 | extract //div[p[contains(text(), '$text')]] 405 | -> ./@data-json 406 | -> @json 407 | `, 408 | } 409 | 410 | const result = await execute( 411 | modules, 412 | {}, 413 | { 414 | fetch: () => 415 | new Response(` 416 | 417 |

first

418 |

second

419 | `), 420 | }, 421 | ) 422 | 423 | expect(result).toEqual({ 424 | a: { x: 1 }, 425 | b: { y: 2 }, 426 | }) 427 | }) 428 | 429 | test('drill macro return types', async () => { 430 | const modules = { 431 | Home: ` 432 | GET http://stub 433 | 434 | extract @Data({text: |'first'|}) 435 | -> ./@data-json 436 | -> @json 437 | `, 438 | Data: ` 439 | inputs { text } 440 | 441 | extract //div[p[contains(text(), '$text')]] 442 | `, 443 | } 444 | 445 | const result = await execute( 446 | modules, 447 | {}, 448 | { 449 | fetch: () => 450 | new Response(`

first

`), 451 | }, 452 | ) 453 | 454 | expect(result).toEqual({ x: 1 }) 455 | }) 456 | 457 | test('recursive', async () => { 458 | const modules = { 459 | Home: ` 460 | extract { 461 | value: @Page -> value 462 | } 463 | `, 464 | Page: ` 465 | extract { 466 | value: @Home -> value 467 | } 468 | `, 469 | } 470 | 471 | const result = execute(modules) 472 | return expect(result).rejects.toThrow( 473 | new RecursiveCallError(['Home', 'Page', 'Home']), 474 | ) 475 | }) 476 | 477 | test('recursive link not called', async () => { 478 | const modules = { 479 | Home: ` 480 | extract { 481 | page: @Page -> value 482 | } 483 | `, 484 | Page: ` 485 | extract { 486 | value: @Home({ 487 | called: |false| 488 | }) 489 | } 490 | `, 491 | } 492 | 493 | const result = await execute(modules) 494 | expect(result).toEqual({ 495 | page: { called: false }, 496 | }) 497 | }) 498 | 499 | test('non-macro not called', async () => { 500 | const modules = { 501 | NotMacro: ` 502 | extract "

100

" -> @html -> p 503 | `, 504 | Home: ` 505 | extract @NotMacro({ num: 0 }) 506 | `, 507 | } 508 | 509 | const result = await execute(modules) 510 | expect(result).toEqual({ num: 0 }) 511 | }) 512 | }) 513 | }) 514 | -------------------------------------------------------------------------------- /packages/parser/src/grammar.ts: -------------------------------------------------------------------------------- 1 | // Generated automatically by nearley, version 2.20.1 2 | // http://github.com/Hardmath123/nearley 3 | // Bypasses TS6133. Allow declared but unused functions. 4 | // @ts-ignore 5 | function id(d: any[]): any { return d[0]; } 6 | declare var identifier: any; 7 | declare var request_verb: any; 8 | declare var request_block_name: any; 9 | declare var request_block_body: any; 10 | declare var request_block_body_end: any; 11 | declare var drill_arrow: any; 12 | declare var link: any; 13 | declare var identifier_expr: any; 14 | declare var call: any; 15 | declare var str: any; 16 | declare var interpvar: any; 17 | declare var bool: any; 18 | declare var num: any; 19 | declare var slice: any; 20 | declare var ws: any; 21 | declare var comment: any; 22 | declare var nl: any; 23 | 24 | import lexer from './grammar/lexer.js' 25 | import * as p from './grammar/parse.js' 26 | 27 | interface NearleyToken { 28 | value: any; 29 | [key: string]: any; 30 | }; 31 | 32 | interface NearleyLexer { 33 | reset: (chunk: string, info: any) => void; 34 | next: () => NearleyToken | undefined; 35 | save: () => any; 36 | formatError: (token: never) => string; 37 | has: (tokenType: string) => boolean; 38 | }; 39 | 40 | interface NearleyRule { 41 | name: string; 42 | symbols: NearleySymbol[]; 43 | postprocess?: (d: any[], loc?: number, reject?: {}) => any; 44 | }; 45 | 46 | type NearleySymbol = string | { literal: any } | { test: (token: any) => boolean }; 47 | 48 | interface Grammar { 49 | Lexer: NearleyLexer | undefined; 50 | ParserRules: NearleyRule[]; 51 | ParserStart: string; 52 | }; 53 | 54 | const grammar: Grammar = { 55 | Lexer: lexer, 56 | ParserRules: [ 57 | {"name": "program$ebnf$1$subexpression$1", "symbols": ["inputs", "line_sep"]}, 58 | {"name": "program$ebnf$1", "symbols": ["program$ebnf$1$subexpression$1"], "postprocess": id}, 59 | {"name": "program$ebnf$1", "symbols": [], "postprocess": () => null}, 60 | {"name": "program", "symbols": ["_", "program$ebnf$1", "statements", "_"], "postprocess": p.program}, 61 | {"name": "statements$ebnf$1", "symbols": []}, 62 | {"name": "statements$ebnf$1$subexpression$1", "symbols": ["line_sep", "statement"]}, 63 | {"name": "statements$ebnf$1", "symbols": ["statements$ebnf$1", "statements$ebnf$1$subexpression$1"], "postprocess": (d) => d[0].concat([d[1]])}, 64 | {"name": "statements", "symbols": ["statement", "statements$ebnf$1"], "postprocess": p.statements}, 65 | {"name": "statement$subexpression$1", "symbols": ["request"]}, 66 | {"name": "statement$subexpression$1", "symbols": ["assignment"]}, 67 | {"name": "statement$subexpression$1", "symbols": ["extract"]}, 68 | {"name": "statement", "symbols": ["statement$subexpression$1"], "postprocess": p.idd}, 69 | {"name": "inputs$ebnf$1", "symbols": []}, 70 | {"name": "inputs$ebnf$1$subexpression$1", "symbols": ["_", {"literal":","}, "_", "input_decl"]}, 71 | {"name": "inputs$ebnf$1", "symbols": ["inputs$ebnf$1", "inputs$ebnf$1$subexpression$1"], "postprocess": (d) => d[0].concat([d[1]])}, 72 | {"name": "inputs", "symbols": [{"literal":"inputs"}, "__", {"literal":"{"}, "_", "input_decl", "inputs$ebnf$1", "_", {"literal":"}"}], "postprocess": p.declInputs}, 73 | {"name": "assignment$ebnf$1", "symbols": [{"literal":"?"}], "postprocess": id}, 74 | {"name": "assignment$ebnf$1", "symbols": [], "postprocess": () => null}, 75 | {"name": "assignment", "symbols": [{"literal":"set"}, "__", (lexer.has("identifier") ? {type: "identifier"} : identifier), "assignment$ebnf$1", "_", {"literal":"="}, "_", "expression"], "postprocess": p.assignment}, 76 | {"name": "extract", "symbols": [{"literal":"extract"}, "__", "expression"], "postprocess": p.extract}, 77 | {"name": "input_decl$ebnf$1", "symbols": [{"literal":"?"}], "postprocess": id}, 78 | {"name": "input_decl$ebnf$1", "symbols": [], "postprocess": () => null}, 79 | {"name": "input_decl$ebnf$2$subexpression$1", "symbols": ["_", {"literal":"="}, "_", "input_default"]}, 80 | {"name": "input_decl$ebnf$2", "symbols": ["input_decl$ebnf$2$subexpression$1"], "postprocess": id}, 81 | {"name": "input_decl$ebnf$2", "symbols": [], "postprocess": () => null}, 82 | {"name": "input_decl", "symbols": [(lexer.has("identifier") ? {type: "identifier"} : identifier), "input_decl$ebnf$1", "input_decl$ebnf$2"], "postprocess": p.inputDecl}, 83 | {"name": "input_default", "symbols": ["slice"], "postprocess": id}, 84 | {"name": "request$ebnf$1$subexpression$1", "symbols": ["line_sep", "request_block"]}, 85 | {"name": "request$ebnf$1", "symbols": ["request$ebnf$1$subexpression$1"], "postprocess": id}, 86 | {"name": "request$ebnf$1", "symbols": [], "postprocess": () => null}, 87 | {"name": "request", "symbols": [(lexer.has("request_verb") ? {type: "request_verb"} : request_verb), "template", "request$ebnf$1", "request_blocks"], "postprocess": p.request}, 88 | {"name": "request_blocks$ebnf$1", "symbols": []}, 89 | {"name": "request_blocks$ebnf$1$subexpression$1", "symbols": ["line_sep", "request_block_named"]}, 90 | {"name": "request_blocks$ebnf$1", "symbols": ["request_blocks$ebnf$1", "request_blocks$ebnf$1$subexpression$1"], "postprocess": (d) => d[0].concat([d[1]])}, 91 | {"name": "request_blocks$ebnf$2$subexpression$1", "symbols": ["line_sep", "request_block_body"]}, 92 | {"name": "request_blocks$ebnf$2", "symbols": ["request_blocks$ebnf$2$subexpression$1"], "postprocess": id}, 93 | {"name": "request_blocks$ebnf$2", "symbols": [], "postprocess": () => null}, 94 | {"name": "request_blocks", "symbols": ["request_blocks$ebnf$1", "request_blocks$ebnf$2"], "postprocess": p.requestBlocks}, 95 | {"name": "request_block_named", "symbols": [(lexer.has("request_block_name") ? {type: "request_block_name"} : request_block_name), "line_sep", "request_block"], "postprocess": p.requestBlockNamed}, 96 | {"name": "request_block$ebnf$1", "symbols": []}, 97 | {"name": "request_block$ebnf$1$subexpression$1", "symbols": ["line_sep", "request_entry"]}, 98 | {"name": "request_block$ebnf$1", "symbols": ["request_block$ebnf$1", "request_block$ebnf$1$subexpression$1"], "postprocess": (d) => d[0].concat([d[1]])}, 99 | {"name": "request_block", "symbols": ["request_entry", "request_block$ebnf$1"], "postprocess": p.requestBlock}, 100 | {"name": "request_entry$ebnf$1$subexpression$1", "symbols": ["__", "template"]}, 101 | {"name": "request_entry$ebnf$1", "symbols": ["request_entry$ebnf$1$subexpression$1"], "postprocess": id}, 102 | {"name": "request_entry$ebnf$1", "symbols": [], "postprocess": () => null}, 103 | {"name": "request_entry", "symbols": ["template", {"literal":":"}, "request_entry$ebnf$1"], "postprocess": p.requestEntry}, 104 | {"name": "request_block_body", "symbols": [(lexer.has("request_block_body") ? {type: "request_block_body"} : request_block_body), "template", (lexer.has("request_block_body_end") ? {type: "request_block_body_end"} : request_block_body_end)], "postprocess": p.requestBlockBody}, 105 | {"name": "expression", "symbols": ["drill"], "postprocess": id}, 106 | {"name": "expression$ebnf$1$subexpression$1", "symbols": ["drill", "_", (lexer.has("drill_arrow") ? {type: "drill_arrow"} : drill_arrow), "_"]}, 107 | {"name": "expression$ebnf$1", "symbols": ["expression$ebnf$1$subexpression$1"], "postprocess": id}, 108 | {"name": "expression$ebnf$1", "symbols": [], "postprocess": () => null}, 109 | {"name": "expression", "symbols": ["expression$ebnf$1", (lexer.has("link") ? {type: "link"} : link), "_", "drill"], "postprocess": p.link}, 110 | {"name": "drill$ebnf$1$subexpression$1", "symbols": [(lexer.has("drill_arrow") ? {type: "drill_arrow"} : drill_arrow), "_"]}, 111 | {"name": "drill$ebnf$1", "symbols": ["drill$ebnf$1$subexpression$1"], "postprocess": id}, 112 | {"name": "drill$ebnf$1", "symbols": [], "postprocess": () => null}, 113 | {"name": "drill$ebnf$2", "symbols": []}, 114 | {"name": "drill$ebnf$2$subexpression$1", "symbols": ["_", (lexer.has("drill_arrow") ? {type: "drill_arrow"} : drill_arrow), "_", "bit"]}, 115 | {"name": "drill$ebnf$2", "symbols": ["drill$ebnf$2", "drill$ebnf$2$subexpression$1"], "postprocess": (d) => d[0].concat([d[1]])}, 116 | {"name": "drill", "symbols": ["drill$ebnf$1", "bit", "drill$ebnf$2"], "postprocess": p.drill}, 117 | {"name": "bit$subexpression$1", "symbols": ["literal"]}, 118 | {"name": "bit$subexpression$1", "symbols": ["slice"]}, 119 | {"name": "bit$subexpression$1", "symbols": ["call"]}, 120 | {"name": "bit$subexpression$1", "symbols": ["object"]}, 121 | {"name": "bit$subexpression$1", "symbols": ["subquery"]}, 122 | {"name": "bit", "symbols": ["bit$subexpression$1"], "postprocess": p.idd}, 123 | {"name": "bit", "symbols": ["template"], "postprocess": p.selector}, 124 | {"name": "bit", "symbols": [(lexer.has("identifier_expr") ? {type: "identifier_expr"} : identifier_expr)], "postprocess": p.idbit}, 125 | {"name": "subquery", "symbols": [{"literal":"("}, "_", "statements", "_", {"literal":")"}], "postprocess": p.subquery}, 126 | {"name": "call$ebnf$1$subexpression$1", "symbols": [{"literal":"("}, "object", {"literal":")"}]}, 127 | {"name": "call$ebnf$1", "symbols": ["call$ebnf$1$subexpression$1"], "postprocess": id}, 128 | {"name": "call$ebnf$1", "symbols": [], "postprocess": () => null}, 129 | {"name": "call", "symbols": [(lexer.has("call") ? {type: "call"} : call), "call$ebnf$1"], "postprocess": p.call}, 130 | {"name": "object$ebnf$1", "symbols": []}, 131 | {"name": "object$ebnf$1$subexpression$1$ebnf$1$subexpression$1", "symbols": ["_", {"literal":","}]}, 132 | {"name": "object$ebnf$1$subexpression$1$ebnf$1", "symbols": ["object$ebnf$1$subexpression$1$ebnf$1$subexpression$1"], "postprocess": id}, 133 | {"name": "object$ebnf$1$subexpression$1$ebnf$1", "symbols": [], "postprocess": () => null}, 134 | {"name": "object$ebnf$1$subexpression$1", "symbols": ["object_entry", "object$ebnf$1$subexpression$1$ebnf$1", "_"]}, 135 | {"name": "object$ebnf$1", "symbols": ["object$ebnf$1", "object$ebnf$1$subexpression$1"], "postprocess": (d) => d[0].concat([d[1]])}, 136 | {"name": "object", "symbols": [{"literal":"{"}, "_", "object$ebnf$1", {"literal":"}"}], "postprocess": p.object}, 137 | {"name": "object_entry$ebnf$1", "symbols": [{"literal":"@"}], "postprocess": id}, 138 | {"name": "object_entry$ebnf$1", "symbols": [], "postprocess": () => null}, 139 | {"name": "object_entry$ebnf$2", "symbols": [{"literal":"?"}], "postprocess": id}, 140 | {"name": "object_entry$ebnf$2", "symbols": [], "postprocess": () => null}, 141 | {"name": "object_entry", "symbols": ["object_entry$ebnf$1", (lexer.has("identifier") ? {type: "identifier"} : identifier), "object_entry$ebnf$2", {"literal":":"}, "_", "expression"], "postprocess": p.objectEntry}, 142 | {"name": "object_entry$ebnf$3", "symbols": [{"literal":"?"}], "postprocess": id}, 143 | {"name": "object_entry$ebnf$3", "symbols": [], "postprocess": () => null}, 144 | {"name": "object_entry", "symbols": [(lexer.has("identifier") ? {type: "identifier"} : identifier), "object_entry$ebnf$3"], "postprocess": p.objectEntryShorthandSelect}, 145 | {"name": "object_entry$ebnf$4", "symbols": [{"literal":"?"}], "postprocess": id}, 146 | {"name": "object_entry$ebnf$4", "symbols": [], "postprocess": () => null}, 147 | {"name": "object_entry", "symbols": [(lexer.has("identifier_expr") ? {type: "identifier_expr"} : identifier_expr), "object_entry$ebnf$4"], "postprocess": p.objectEntryShorthandIdent}, 148 | {"name": "template$ebnf$1$subexpression$1", "symbols": [(lexer.has("str") ? {type: "str"} : str)]}, 149 | {"name": "template$ebnf$1$subexpression$1", "symbols": [(lexer.has("interpvar") ? {type: "interpvar"} : interpvar)]}, 150 | {"name": "template$ebnf$1$subexpression$1", "symbols": ["interp_expr"]}, 151 | {"name": "template$ebnf$1$subexpression$1", "symbols": ["interp_tmpl"]}, 152 | {"name": "template$ebnf$1", "symbols": ["template$ebnf$1$subexpression$1"]}, 153 | {"name": "template$ebnf$1$subexpression$2", "symbols": [(lexer.has("str") ? {type: "str"} : str)]}, 154 | {"name": "template$ebnf$1$subexpression$2", "symbols": [(lexer.has("interpvar") ? {type: "interpvar"} : interpvar)]}, 155 | {"name": "template$ebnf$1$subexpression$2", "symbols": ["interp_expr"]}, 156 | {"name": "template$ebnf$1$subexpression$2", "symbols": ["interp_tmpl"]}, 157 | {"name": "template$ebnf$1", "symbols": ["template$ebnf$1", "template$ebnf$1$subexpression$2"], "postprocess": (d) => d[0].concat([d[1]])}, 158 | {"name": "template", "symbols": ["template$ebnf$1"], "postprocess": p.template}, 159 | {"name": "interp_expr", "symbols": [{"literal":"${"}, "_", (lexer.has("identifier") ? {type: "identifier"} : identifier), "_", {"literal":"}"}], "postprocess": p.interpExpr}, 160 | {"name": "interp_tmpl", "symbols": [{"literal":"$["}, "_", "template", "_", {"literal":"]"}], "postprocess": p.interpTmpl}, 161 | {"name": "literal$subexpression$1", "symbols": [(lexer.has("bool") ? {type: "bool"} : bool)]}, 162 | {"name": "literal$subexpression$1", "symbols": [(lexer.has("num") ? {type: "num"} : num)]}, 163 | {"name": "literal", "symbols": ["literal$subexpression$1"], "postprocess": p.literal}, 164 | {"name": "literal", "symbols": [{"literal":"'"}, "template", {"literal":"'"}], "postprocess": p.string}, 165 | {"name": "literal", "symbols": [{"literal":"\""}, "template", {"literal":"\""}], "postprocess": p.string}, 166 | {"name": "slice", "symbols": [(lexer.has("slice") ? {type: "slice"} : slice)], "postprocess": p.slice}, 167 | {"name": "line_sep$ebnf$1", "symbols": []}, 168 | {"name": "line_sep$ebnf$1$subexpression$1", "symbols": [(lexer.has("ws") ? {type: "ws"} : ws)]}, 169 | {"name": "line_sep$ebnf$1$subexpression$1", "symbols": [(lexer.has("comment") ? {type: "comment"} : comment)]}, 170 | {"name": "line_sep$ebnf$1", "symbols": ["line_sep$ebnf$1", "line_sep$ebnf$1$subexpression$1"], "postprocess": (d) => d[0].concat([d[1]])}, 171 | {"name": "line_sep", "symbols": ["line_sep$ebnf$1", (lexer.has("nl") ? {type: "nl"} : nl), "_"], "postprocess": p.ws}, 172 | {"name": "__$ebnf$1", "symbols": ["ws"]}, 173 | {"name": "__$ebnf$1", "symbols": ["__$ebnf$1", "ws"], "postprocess": (d) => d[0].concat([d[1]])}, 174 | {"name": "__", "symbols": ["__$ebnf$1"], "postprocess": p.ws}, 175 | {"name": "_$ebnf$1", "symbols": []}, 176 | {"name": "_$ebnf$1", "symbols": ["_$ebnf$1", "ws"], "postprocess": (d) => d[0].concat([d[1]])}, 177 | {"name": "_", "symbols": ["_$ebnf$1"], "postprocess": p.ws}, 178 | {"name": "ws$subexpression$1", "symbols": [(lexer.has("ws") ? {type: "ws"} : ws)]}, 179 | {"name": "ws$subexpression$1", "symbols": [(lexer.has("comment") ? {type: "comment"} : comment)]}, 180 | {"name": "ws$subexpression$1", "symbols": [(lexer.has("nl") ? {type: "nl"} : nl)]}, 181 | {"name": "ws", "symbols": ["ws$subexpression$1"], "postprocess": p.ws} 182 | ], 183 | ParserStart: "program", 184 | }; 185 | 186 | export default grammar; 187 | -------------------------------------------------------------------------------- /test/request.spec.ts: -------------------------------------------------------------------------------- 1 | import { beforeEach, describe, expect, mock, test } from 'bun:test' 2 | import type { Inputs } from '@getlang/lib' 3 | import type { Fetch } from './helpers.js' 4 | import { execute as _exec } from './helpers.js' 5 | 6 | const mockFetch = mock( 7 | () => 8 | new Response('

test

', { 9 | headers: { 10 | 'content-type': 'text/html', 11 | }, 12 | }), 13 | ) 14 | 15 | const execute = (src: string, inputs: Inputs = {}, fetch: Fetch = mockFetch) => 16 | _exec(src, inputs, { fetch }) 17 | 18 | beforeEach(() => { 19 | mockFetch.mockClear() 20 | }) 21 | 22 | describe('request', () => { 23 | describe('verbs', () => { 24 | test('get', async () => { 25 | const result = await execute(` 26 | GET http://get.com 27 | 28 | extract -> h1 29 | `) 30 | expect(mockFetch).toHaveBeenCalledTimes(1) 31 | await expect(mockFetch).toHaveServed('http://get.com/', { method: 'GET' }) 32 | expect(result).toEqual('test') 33 | }) 34 | 35 | test('post', async () => { 36 | await execute('POST http://post.com') 37 | await expect(mockFetch).toHaveServed('http://post.com/', { 38 | method: 'POST', 39 | }) 40 | }) 41 | 42 | test('put', async () => { 43 | await execute('PUT http://put.com') 44 | await expect(mockFetch).toHaveServed('http://put.com/', { method: 'PUT' }) 45 | }) 46 | 47 | test('patch', async () => { 48 | await execute('PATCH http://patch.com') 49 | await expect(mockFetch).toHaveServed('http://patch.com/', { 50 | method: 'PATCH', 51 | }) 52 | }) 53 | 54 | test('delete', async () => { 55 | await execute('DELETE http://delete.com') 56 | await expect(mockFetch).toHaveServed('http://delete.com/', { 57 | method: 'DELETE', 58 | }) 59 | }) 60 | }) 61 | 62 | describe('urls', () => { 63 | test('literal', async () => { 64 | await execute('GET http://get.com') 65 | await expect(mockFetch).toHaveServed('http://get.com/', { 66 | method: 'GET', 67 | }) 68 | }) 69 | 70 | test('identifier', async () => { 71 | await execute(` 72 | set ident = |'http://ident.com'| 73 | GET $ident 74 | `) 75 | await expect(mockFetch).toHaveServed('http://ident.com/', { 76 | method: 'GET', 77 | }) 78 | }) 79 | 80 | test('interpolated', async () => { 81 | await execute(` 82 | set query = |'monterey'| 83 | GET https://boogle.com/search/$query 84 | `) 85 | await expect(mockFetch).toHaveServed( 86 | 'https://boogle.com/search/monterey', 87 | { 88 | method: 'GET', 89 | }, 90 | ) 91 | }) 92 | 93 | test('interpolated expression', async () => { 94 | await execute(` 95 | set query = |'big sur'| 96 | GET https://ging.com/\${query}_results 97 | `) 98 | await expect(mockFetch).toHaveServed( 99 | 'https://ging.com/big%20sur_results', 100 | { 101 | method: 'GET', 102 | }, 103 | ) 104 | }) 105 | 106 | test('interpolated value', async () => { 107 | await execute(` 108 | set loc = |'
sea ranch
'| -> @html 109 | GET https://goto.ca/:loc 110 | `) 111 | await expect(mockFetch).toHaveServed('https://goto.ca/sea%20ranch', { 112 | method: 'GET', 113 | }) 114 | }) 115 | 116 | test('implicit params', async () => { 117 | await execute('GET http://implied.com/projects/:projectId', { 118 | projectId: 12, 119 | }) 120 | await expect(mockFetch).toHaveServed('http://implied.com/projects/12', { 121 | method: 'GET', 122 | }) 123 | }) 124 | 125 | test('non-conforming', async () => { 126 | await execute(`GET example.com`) 127 | expect(mockFetch).toHaveServed('example.com/', { method: 'GET' }) 128 | }) 129 | }) 130 | 131 | test('headers', async () => { 132 | await execute(` 133 | set token = |123| 134 | 135 | GET http://api.unweb.com 136 | Authorization: Bearer $token 137 | Accept: application/json 138 | `) 139 | 140 | await expect(mockFetch).toHaveServed('http://api.unweb.com/', { 141 | method: 'GET', 142 | headers: new Headers({ 143 | Authorization: 'Bearer 123', 144 | Accept: 'application/json', 145 | }), 146 | }) 147 | }) 148 | 149 | describe('blocks', () => { 150 | test('querystring', async () => { 151 | await execute(` 152 | set ident = |"b"| 153 | set interp = |"olated"| 154 | 155 | GET https://example.com 156 | X-Test: true 157 | [query] 158 | a: literal 159 | b: $ident 160 | c: interp$interp 161 | `) 162 | 163 | await expect(mockFetch).toHaveServed( 164 | 'https://example.com/?a=literal&b=b&c=interpolated', 165 | { 166 | method: 'GET', 167 | headers: new Headers({ 168 | 'X-Test': 'true', 169 | }), 170 | }, 171 | ) 172 | }) 173 | 174 | test('querystring merge', async () => { 175 | await execute(` 176 | GET http://example.com/?a=1 177 | [query] 178 | a: 2 179 | b: 4 180 | `) 181 | await expect(mockFetch).toHaveServed('http://example.com/?a=1&a=2&b=4', { 182 | method: 'GET', 183 | }) 184 | }) 185 | 186 | test('cookies, encoded', async () => { 187 | await execute(` 188 | GET https://example.com 189 | [cookies] 190 | a: A 191 | b: 123 192 | c: /here&we!are? 193 | `) 194 | 195 | await expect(mockFetch).toHaveServed('https://example.com/', { 196 | method: 'GET', 197 | headers: new Headers({ 198 | Cookie: 'a=A; b=123; c=%2Fhere%26we%21are%3F', 199 | }), 200 | }) 201 | }) 202 | 203 | test('json body', async () => { 204 | await execute(` 205 | POST https://example.com/login 206 | [json] 207 | username: admin 208 | password: test 209 | `) 210 | 211 | await expect(mockFetch).toHaveServed('https://example.com/login', { 212 | method: 'POST', 213 | body: '{"username":"admin","password":"test"}', 214 | }) 215 | }) 216 | 217 | test('raw body', async () => { 218 | await execute(` 219 | set hello = |"hi there"| 220 | 221 | POST https://example.com 222 | [body] 223 | hello 224 | g'day 225 | welcome 226 | 227 | [/body] 228 | `) 229 | 230 | await expect(mockFetch).toHaveServed('https://example.com/', { 231 | method: 'POST', 232 | headers: new Headers(), 233 | body: "hello\n g'day\n welcome\n", 234 | }) 235 | }) 236 | 237 | test('omits undefined', async () => { 238 | await execute(` 239 | set foo? = |undefined| 240 | 241 | POST https://example.com 242 | X-Foo: $foo 243 | X-Bar: bar 244 | [query] 245 | foo: $foo 246 | bar: bar 247 | [cookies] 248 | foo: $foo 249 | bar: bar 250 | [json] 251 | foo: $foo 252 | bar: bar 253 | `) 254 | 255 | await expect(mockFetch).toHaveServed('https://example.com/?bar=bar', { 256 | method: 'POST', 257 | headers: new Headers({ 258 | 'X-Bar': 'bar', 259 | Cookie: 'bar=bar', 260 | }), 261 | body: '{"bar":"bar"}', 262 | }) 263 | }) 264 | 265 | test('optional template groups', async () => { 266 | await execute(` 267 | set foo? = |undefined| 268 | 269 | GET https://example.com/pre$[/:foo]/post 270 | X-Foo: $foo 271 | X-Bar: bar 272 | X-Baz: baz$[$foo]zza 273 | `) 274 | 275 | await expect(mockFetch).toHaveServed('https://example.com/pre/post', { 276 | method: 'GET', 277 | headers: new Headers({ 278 | 'X-Bar': 'bar', 279 | 'X-Baz': 'bazzza', 280 | }), 281 | }) 282 | }) 283 | 284 | test('nested template parts', async () => { 285 | const src = ` 286 | inputs { x?, y? } 287 | 288 | GET https://getlang.dev 289 | Header: aa$[bb\${x}cc$[dd\${y}ee]ff]gg 290 | ` 291 | 292 | await execute(src) 293 | await execute(src, { x: '0x0' }) 294 | await execute(src, { y: '0y0' }) 295 | await execute(src, { x: '0x0', y: '0y0' }) 296 | 297 | await expect(mockFetch).toHaveServed('https://getlang.dev/', { 298 | method: 'GET', 299 | headers: new Headers({ Header: 'aagg' }), 300 | }) 301 | 302 | await expect(mockFetch).toHaveServed('https://getlang.dev/', { 303 | method: 'GET', 304 | headers: new Headers({ Header: 'aabb0x0ccffgg' }), 305 | }) 306 | 307 | await expect(mockFetch).toHaveServed('https://getlang.dev/', { 308 | method: 'GET', 309 | headers: new Headers({ Header: 'aagg' }), 310 | }) 311 | 312 | await expect(mockFetch).toHaveServed('https://getlang.dev/', { 313 | method: 'GET', 314 | headers: new Headers({ Header: 'aabb0x0ccdd0y0eeffgg' }), 315 | }) 316 | }) 317 | }) 318 | 319 | describe('context switching', () => { 320 | test('updates context variable ($) dynamically', async () => { 321 | let fetched = 0 322 | const mockFetch = mock(() => { 323 | const which = ++fetched 324 | return new Response(JSON.stringify({ which })) 325 | }) 326 | 327 | const result = await execute( 328 | ` 329 | GET https://example.com/api 330 | Accept: application/json 331 | 332 | set whicha = $ -> body -> @json -> which 333 | 334 | GET https://example.com/api 335 | Accept: application/json 336 | 337 | set whichb = $ -> body -> @json -> which 338 | 339 | extract { $whicha, $whichb } 340 | `, 341 | {}, 342 | mockFetch, 343 | ) 344 | 345 | expect(result).toEqual({ whicha: 1, whichb: 2 }) 346 | }) 347 | 348 | test('updates subquery context', async () => { 349 | const result = await execute(` 350 | set x? = |'

'| -> @html -> ( 351 | GET http://example.com 352 | 353 | extract h1 354 | ) 355 | 356 | extract $x 357 | `) 358 | 359 | expect(result).toEqual('test') 360 | }) 361 | }) 362 | 363 | describe('inference', () => { 364 | test('examines accept header', async () => { 365 | const result = await execute( 366 | ` 367 | GET https://example.com/api 368 | Accept: application/json 369 | 370 | extract -> works 371 | `, 372 | {}, 373 | () => new Response('{"works":true}'), 374 | ) 375 | expect(result).toEqual(true) 376 | }) 377 | 378 | test('can be overridden manually with explicit modifiers', async () => { 379 | const result = await execute(` 380 | GET https://example.com/api 381 | Accept: application/json 382 | 383 | extract @html -> h1 384 | `) 385 | expect(result).toEqual('test') 386 | }) 387 | }) 388 | 389 | describe('url resolution', () => { 390 | test('resolves urls against request context', async () => { 391 | const result = await execute( 392 | ` 393 | GET https://base.com/a/b/c 394 | Accept: application/json 395 | 396 | extract { 397 | link1: link -> @link 398 | link2: anchor -> @html -> a -> @link 399 | link3: attr -> @html -> i -> ./@data-url -> @link 400 | link4: img -> @html -> img -> @link 401 | } 402 | `, 403 | {}, 404 | () => 405 | new Response( 406 | JSON.stringify({ 407 | link: '../xyz.html', 408 | anchor: "

", 409 | attr: "
click here
", 410 | img: "
", 411 | }), 412 | ), 413 | ) 414 | 415 | expect(result).toEqual({ 416 | link1: 'https://base.com/a/xyz.html', 417 | link2: 'https://base.com/from/anchor', 418 | link3: 'https://base.com/from/attr', 419 | link4: 'https://base.com/from/img', 420 | }) 421 | }) 422 | 423 | test('resolves nested urls', async () => { 424 | const result = await execute( 425 | ` 426 | GET https://base.com/a/b/c 427 | 428 | extract => a -> { 429 | text: $ 430 | link: @link 431 | } 432 | `, 433 | {}, 434 | () => 435 | new Response(`
436 | first 437 | second 438 |
`), 439 | ) 440 | 441 | expect(result).toEqual([ 442 | { text: 'first', link: 'https://base.com/a/xyz.html' }, 443 | { text: 'second', link: 'https://base.com/from/root' }, 444 | ]) 445 | }) 446 | 447 | test('infers base inside list context', async () => { 448 | const result = await execute( 449 | ` 450 | GET https://bar.com/with/links 451 | 452 | extract => a -> @link 453 | `, 454 | {}, 455 | () => new Response(`click here`), 456 | ) 457 | 458 | expect(result).toEqual(['https://bar.com/foo']) 459 | }) 460 | 461 | test('resolved standalone links to context url', async () => { 462 | const src = ` 463 | inputs { query, page? } 464 | 465 | GET https://example.com/search/:query 466 | [query] 467 | page: $page 468 | 469 | extract { 470 | url: @link 471 | } 472 | ` 473 | 474 | const page1 = await execute(src, { query: 'any' }) 475 | expect(page1).toEqual({ 476 | url: 'https://example.com/search/any', 477 | }) 478 | 479 | const page2 = await execute(src, { query: 'any', page: 2 }) 480 | expect(page2).toEqual({ 481 | url: 'https://example.com/search/any?page=2', 482 | }) 483 | }) 484 | }) 485 | 486 | describe('non-body selectors', () => { 487 | test('header', async () => { 488 | const result = await execute(` 489 | GET https://example.com 490 | 491 | extract @headers -> content-type 492 | `) 493 | 494 | expect(result).toEqual('text/html') 495 | }) 496 | 497 | test('cookie', async () => { 498 | const result = await execute( 499 | ` 500 | GET https://example.com 501 | 502 | extract @cookies -> session 503 | `, 504 | {}, 505 | () => 506 | new Response('

test

', { 507 | headers: { 508 | 'set-cookie': 509 | 'session=jZDE5MDBhNzczNDMzMTk4; Domain=.example.com; Path=/; Expires=Tue, 16 Jun 2026 07:31:59 GMT; Secure', 510 | }, 511 | }), 512 | ) 513 | 514 | expect(result).toEqual('jZDE5MDBhNzczNDMzMTk4') 515 | }) 516 | }) 517 | 518 | test('selector object shorthand', async () => { 519 | const result = await execute(` 520 | GET http://get.com 521 | 522 | extract { h1 } 523 | `) 524 | expect(result).toEqual({ h1: 'test' }) 525 | }) 526 | }) 527 | --------------------------------------------------------------------------------