├── src
    ├── rpc
    │   ├── index.ts
    │   ├── types.ts
    │   ├── procedures.ts
    │   └── server.ts
    ├── index.rpc.ts
    ├── parser
    │   ├── utils
    │   │   ├── regex.ts
    │   │   ├── full_char_code_at.ts
    │   │   ├── acorn.ts
    │   │   ├── bracket.ts
    │   │   └── html.ts
    │   ├── state
    │   │   ├── multi_line_comment.ts
    │   │   ├── config.ts
    │   │   ├── fragment.ts
    │   │   └── text.ts
    │   ├── read
    │   │   ├── expression.ts
    │   │   └── context.ts
    │   ├── interfaces.ts
    │   └── index.ts
    ├── types
    │   ├── acorn.d.ts
    │   ├── index.ts
    │   ├── customTypes.ts
    │   └── message.ts
    ├── compiler
    │   ├── base
    │   │   ├── nodes
    │   │   │   ├── comment.ts
    │   │   │   ├── text.ts
    │   │   │   ├── config.ts
    │   │   │   ├── fragment.ts
    │   │   │   ├── tags
    │   │   │   │   ├── scope.ts
    │   │   │   │   ├── ref.ts
    │   │   │   │   ├── step.ts
    │   │   │   │   ├── message.ts
    │   │   │   │   ├── scope.test.ts
    │   │   │   │   ├── content.ts
    │   │   │   │   └── step.test.ts
    │   │   │   ├── if.ts
    │   │   │   ├── mustache.ts
    │   │   │   ├── for.ts
    │   │   │   ├── for.test.ts
    │   │   │   ├── if.test.ts
    │   │   │   └── tag.ts
    │   │   └── types.ts
    │   ├── logic
    │   │   ├── nodes
    │   │   │   ├── literal.ts
    │   │   │   ├── chainExpression.ts
    │   │   │   ├── sequenceExpression.ts
    │   │   │   ├── conditionalExpression.ts
    │   │   │   ├── identifier.ts
    │   │   │   ├── memberExpression.ts
    │   │   │   ├── unaryExpression.ts
    │   │   │   ├── binaryExpression.ts
    │   │   │   ├── arrayExpression.ts
    │   │   │   ├── callExpression.ts
    │   │   │   ├── objectExpression.ts
    │   │   │   ├── updateExpression.ts
    │   │   │   ├── index.ts
    │   │   │   └── assignmentExpression.ts
    │   │   ├── index.ts
    │   │   ├── types.ts
    │   │   └── operators.ts
    │   ├── types.ts
    │   ├── deserializeChain.test.ts
    │   ├── test
    │   │   └── helpers.ts
    │   ├── index.ts
    │   ├── deserializeChain.ts
    │   ├── utils.test.ts
    │   ├── scope.ts
    │   ├── chain-serialize.test.ts
    │   ├── errors.test.ts
    │   └── utils.ts
    ├── index.ts
    ├── test
    │   └── helpers.ts
    ├── providers
    │   ├── adapter.ts
    │   ├── openai-responses
    │   │   ├── utils
    │   │   │   └── parsers
    │   │   │   │   ├── reasoning.ts
    │   │   │   │   ├── webSearch.ts
    │   │   │   │   ├── fileSearch.ts
    │   │   │   │   ├── inputMessage.ts
    │   │   │   │   ├── functionCall.ts
    │   │   │   │   ├── parseSimpleContent.ts
    │   │   │   │   └── outputMessage.ts
    │   │   └── types.ts
    │   ├── index.ts
    │   ├── openai
    │   │   ├── types.ts
    │   │   └── adapter.test.ts
    │   ├── anthropic
    │   │   ├── types.ts
    │   │   └── adapter.test.ts
    │   ├── vercel-ai
    │   │   ├── types.ts
    │   │   └── adapter.test.ts
    │   └── utils
    │   │   └── getMimeType.ts
    ├── utils
    │   └── names.ts
    ├── constants.ts
    ├── error
    │   └── error.ts
    └── build.test.ts
├── .prettierignore
├── .prettierrc
├── vitest.config.ts
├── examples
    ├── lib.ts
    └── rpc.ts
├── .gitignore
├── LICENSE
├── tsconfig.json
├── .github
    └── workflows
    │   ├── test.yml
    │   ├── lint.yml
    │   └── publish.yml
├── eslint.config.mjs
├── rollup.config.rpc.mjs
├── rollup.config.mjs
├── package.json
└── README.md


/src/rpc/index.ts:
--------------------------------------------------------------------------------
1 | export { serve } from './server'
2 | 


--------------------------------------------------------------------------------
/.prettierignore:
--------------------------------------------------------------------------------
1 | .*
2 | dist
3 | build
4 | **/tests/fixtures
5 | 


--------------------------------------------------------------------------------
/src/index.rpc.ts:
--------------------------------------------------------------------------------
1 | import { serve } from './rpc'
2 | 
3 | serve()
4 | 


--------------------------------------------------------------------------------
/.prettierrc:
--------------------------------------------------------------------------------
1 | {
2 |   "semi": false,
3 |   "jsxSingleQuote": true,
4 |   "singleQuote": true,
5 |   "trailingComma": "all"
6 | }
7 | 


--------------------------------------------------------------------------------
/src/parser/utils/regex.ts:
--------------------------------------------------------------------------------
1 | import { RESERVED_TAGS } from "$promptl/constants";
2 | 
3 | export const RESERVED_TAG_REGEX = new RegExp(`^</?(${RESERVED_TAGS.join('|')})(\\s|/?>|$)`)


--------------------------------------------------------------------------------
/src/types/acorn.d.ts:
--------------------------------------------------------------------------------
1 | declare module 'acorn' {
2 |   export function isIdentifierStart(code: number, astral?: boolean): boolean
3 |   export function isIdentifierChar(code: number, astral?: boolean): boolean
4 | }
5 | 


--------------------------------------------------------------------------------
/src/compiler/base/nodes/comment.ts:
--------------------------------------------------------------------------------
1 | import { Comment } from '$promptl/parser/interfaces'
2 | 
3 | import { CompileNodeContext } from '../types'
4 | 
5 | export async function compile(_: CompileNodeContext<Comment>) {
6 |   /* do nothing */
7 | }
8 | 


--------------------------------------------------------------------------------
/src/compiler/base/nodes/text.ts:
--------------------------------------------------------------------------------
 1 | import { Text } from '$promptl/parser/interfaces'
 2 | 
 3 | import { CompileNodeContext } from '../types'
 4 | 
 5 | export async function compile({
 6 |   node,
 7 |   addStrayText,
 8 | }: CompileNodeContext<Text>) {
 9 |   addStrayText(node.data)
10 | }
11 | 


--------------------------------------------------------------------------------
/src/compiler/base/nodes/config.ts:
--------------------------------------------------------------------------------
 1 | import { Config as ConfigNode } from '$promptl/parser/interfaces'
 2 | import yaml from 'yaml'
 3 | 
 4 | import { CompileNodeContext } from '../types'
 5 | 
 6 | export async function compile({
 7 |   node,
 8 |   setConfig,
 9 | }: CompileNodeContext<ConfigNode>): Promise<void> {
10 |   setConfig(yaml.parse(node.value))
11 | }
12 | 


--------------------------------------------------------------------------------
/src/index.ts:
--------------------------------------------------------------------------------
 1 | export * from './types'
 2 | export * from './compiler'
 3 | export * from './parser'
 4 | export * from './providers'
 5 | 
 6 | import * as parserInterfaces from './parser/interfaces'
 7 | type Fragment = parserInterfaces.Fragment
 8 | export type { Fragment, parserInterfaces as IParser }
 9 | 
10 | export { default as CompileError } from './error/error'
11 | 


--------------------------------------------------------------------------------
/src/compiler/logic/nodes/literal.ts:
--------------------------------------------------------------------------------
 1 | import { type ResolveNodeProps } from '$promptl/compiler/logic/types'
 2 | import { type Literal } from 'estree'
 3 | 
 4 | /**
 5 |  * ### Literal
 6 |  * Represents a literal value.
 7 |  */
 8 | export async function resolve({ node }: ResolveNodeProps<Literal>) {
 9 |   return node.value
10 | }
11 | 
12 | export function updateScopeContext() {
13 |   // Do nothing
14 | }
15 | 


--------------------------------------------------------------------------------
/src/test/helpers.ts:
--------------------------------------------------------------------------------
 1 | import { expect } from 'vitest'
 2 | 
 3 | export async function getExpectedError<T extends Error>(
 4 |   action: () => unknown,
 5 |   errorClass: new (...args: any[]) => T,
 6 | ): Promise<T> {
 7 |   try {
 8 |     await action()
 9 |   } catch (err) {
10 |     expect(err).toBeInstanceOf(errorClass)
11 |     return err as T
12 |   }
13 |   throw new Error('Expected an error to be thrown')
14 | }
15 | 


--------------------------------------------------------------------------------
/src/parser/utils/full_char_code_at.ts:
--------------------------------------------------------------------------------
 1 | // Adapted from https://github.com/acornjs/acorn/blob/6584815dca7440e00de841d1dad152302fdd7ca5/src/tokenize.js
 2 | 
 3 | export default function fullCharCodeAt(str: string, i: number): number {
 4 |   const code = str.charCodeAt(i)
 5 |   if (code <= 0xd7ff || code >= 0xe000) return code
 6 | 
 7 |   const next = str.charCodeAt(i + 1)
 8 |   return (code << 10) + next - 0x35fdc00
 9 | }
10 | 


--------------------------------------------------------------------------------
/src/parser/utils/acorn.ts:
--------------------------------------------------------------------------------
 1 | import * as code_red from 'code-red'
 2 | 
 3 | export const parse = (source: string) =>
 4 |   code_red.parse(source, {
 5 |     sourceType: 'module',
 6 |     ecmaVersion: 13,
 7 |     locations: true,
 8 |   })
 9 | 
10 | export const parseExpressionAt = (source: string, index: number) =>
11 |   code_red.parseExpressionAt(source, index, {
12 |     sourceType: 'module',
13 |     ecmaVersion: 13,
14 |     locations: true,
15 |   })
16 | 


--------------------------------------------------------------------------------
/vitest.config.ts:
--------------------------------------------------------------------------------
 1 | import { dirname } from 'path'
 2 | import { fileURLToPath } from 'url'
 3 | 
 4 | import { defineConfig } from 'vitest/config'
 5 | 
 6 | const filename = fileURLToPath(import.meta.url)
 7 | const root = dirname(filename)
 8 | 
 9 | export default defineConfig({
10 |   resolve: {
11 |     alias: {
12 |       $promptl: `${root}/src`,
13 |     },
14 |   },
15 |   test: {
16 |     globals: true,
17 |     environment: 'node',
18 |   },
19 | })
20 | 


--------------------------------------------------------------------------------
/src/compiler/base/nodes/fragment.ts:
--------------------------------------------------------------------------------
 1 | import { Fragment } from '$promptl/parser/interfaces'
 2 | 
 3 | import { CompileNodeContext } from '../types'
 4 | 
 5 | export async function compile({
 6 |   node,
 7 |   scope,
 8 |   isInsideStepTag,
 9 |   isInsideContentTag,
10 |   isInsideMessageTag,
11 |   fullPath,
12 |   resolveBaseNode,
13 | }: CompileNodeContext<Fragment>) {
14 |   for await (const childNode of node.children ?? []) {
15 |     await resolveBaseNode({
16 |       node: childNode,
17 |       scope,
18 |       isInsideStepTag,
19 |       isInsideMessageTag,
20 |       isInsideContentTag,
21 |       fullPath,
22 |     })
23 |   }
24 | }
25 | 


--------------------------------------------------------------------------------
/src/compiler/base/nodes/tags/scope.ts:
--------------------------------------------------------------------------------
 1 | import Scope from '$promptl/compiler/scope'
 2 | import { ScopeTag } from '$promptl/parser/interfaces'
 3 | 
 4 | import { CompileNodeContext } from '../../types'
 5 | 
 6 | export async function compile(
 7 |   props: CompileNodeContext<ScopeTag>,
 8 |   attributes: Record<string, unknown>,
 9 | ) {
10 |   const {
11 |     node,
12 |     resolveBaseNode,
13 |   } = props
14 | 
15 |   const childScope = new Scope(attributes)
16 | 
17 |   for await (const childNode of node.children) {
18 |     await resolveBaseNode({
19 |       ...props,
20 |       node: childNode,
21 |       scope: childScope,
22 |     })
23 |   }
24 | }
25 | 


--------------------------------------------------------------------------------
/src/providers/adapter.ts:
--------------------------------------------------------------------------------
 1 | import { AdapterKey } from '$promptl/providers'
 2 | import { Message, Conversation as PromptlConversation } from '$promptl/types'
 3 | 
 4 | export type ProviderConversation<M extends object> = {
 5 |   config: PromptlConversation['config']
 6 |   messages: M[]
 7 | }
 8 | 
 9 | export type ProviderAdapter<M extends object> = {
10 |   type: AdapterKey
11 |   toPromptl(conversation: ProviderConversation<M>): PromptlConversation
12 |   fromPromptl(conversation: PromptlConversation): ProviderConversation<M>
13 | }
14 | 
15 | export const defaultAdapter: ProviderAdapter<Message> = {
16 |   type: 'default',
17 |   toPromptl: (c) => c,
18 |   fromPromptl: (c) => c,
19 | }
20 | 


--------------------------------------------------------------------------------
/examples/lib.ts:
--------------------------------------------------------------------------------
 1 | // Run `pnpm build:lib` before running this example
 2 | 
 3 | import assert from 'node:assert'
 4 | import { inspect } from 'node:util'
 5 | import { Chain } from '../dist/index.js'
 6 | 
 7 | const prompt = `
 8 | <step>
 9 |   <system>
10 |     You are a helpful assistant.
11 |   </system>
12 |   <user>
13 |     Say hello.
14 |   </user>
15 | </step>
16 | <step>
17 |   <user>
18 |     Now say goodbye.
19 |   </user>
20 | </step>
21 | `
22 | 
23 | const chain = new Chain({ prompt })
24 | let conversation = await chain.step()
25 | conversation = await chain.step('Hello!')
26 | conversation = await chain.step('Goodbye!')
27 | 
28 | assert(chain.completed)
29 | assert(conversation.completed)
30 | 
31 | console.log(inspect(conversation.messages, { depth: null }))
32 | 


--------------------------------------------------------------------------------
/src/types/index.ts:
--------------------------------------------------------------------------------
 1 | import CompileError from '$promptl/error/error'
 2 | import { Fragment } from '$promptl/index'
 3 | 
 4 | import { Message } from './message'
 5 | 
 6 | export type Config = Record<string, unknown>
 7 | 
 8 | export type Conversation = {
 9 |   config: Config
10 |   messages: Message[]
11 | }
12 | 
13 | export type ConversationMetadata = {
14 |   hash: string
15 |   resolvedPrompt: string
16 |   ast: Fragment
17 |   config: Config
18 |   errors: CompileError[]
19 |   parameters: Set<string> // Variables used in the prompt that have not been defined in runtime
20 |   isChain: boolean
21 |   setConfig: (config: Config) => string
22 |   includedPromptPaths: Set<string>
23 | }
24 | 
25 | export { type SerializedChain } from '$promptl/compiler'
26 | export * from './message'
27 | export * from './customTypes'
28 | 


--------------------------------------------------------------------------------
/src/compiler/types.ts:
--------------------------------------------------------------------------------
 1 | import { TemplateNode } from '$promptl/parser/interfaces'
 2 | import { MessageRole } from '$promptl/types'
 3 | 
 4 | import type Scope from './scope'
 5 | 
 6 | export type Document = {
 7 |   path: string
 8 |   content: string
 9 | }
10 | export type ReferencePromptFn = (
11 |   path: string,
12 |   from?: string,
13 | ) => Promise<Document | undefined>
14 | 
15 | export type ResolveBaseNodeProps<N extends TemplateNode> = {
16 |   node: N
17 |   scope: Scope
18 |   isInsideStepTag: boolean
19 |   isInsideMessageTag: boolean
20 |   isInsideContentTag: boolean
21 |   completedValue?: unknown
22 |   fullPath?: string | undefined
23 | }
24 | 
25 | export type CompileOptions = {
26 |   referenceFn?: ReferencePromptFn
27 |   fullPath?: string
28 |   defaultRole?: MessageRole
29 |   includeSourceMap?: boolean
30 | }
31 | 


--------------------------------------------------------------------------------
/src/compiler/logic/nodes/chainExpression.ts:
--------------------------------------------------------------------------------
 1 | import type {
 2 |   ResolveNodeProps,
 3 |   UpdateScopeContextProps,
 4 | } from '$promptl/compiler/logic/types'
 5 | import type { ChainExpression } from 'estree'
 6 | 
 7 | import { resolveLogicNode, updateScopeContextForNode } from '..'
 8 | 
 9 | /**
10 |  * ### Chain Expression
11 |  * Represents a chain of operations. This is only being used for optional member expressions '?.'
12 |  */
13 | export async function resolve({
14 |   node,
15 |   ...props
16 | }: ResolveNodeProps<ChainExpression>) {
17 |   return resolveLogicNode({
18 |     node: node.expression,
19 |     ...props,
20 |   })
21 | }
22 | 
23 | export function updateScopeContext({
24 |   node,
25 |   ...props
26 | }: UpdateScopeContextProps<ChainExpression>) {
27 |   updateScopeContextForNode({ node: node.expression, ...props })
28 | }
29 | 


--------------------------------------------------------------------------------
/src/compiler/base/nodes/if.ts:
--------------------------------------------------------------------------------
 1 | import { IfBlock } from '$promptl/parser/interfaces'
 2 | 
 3 | import { CompileNodeContext } from '../types'
 4 | 
 5 | export async function compile({
 6 |   node,
 7 |   scope,
 8 |   isInsideStepTag,
 9 |   isInsideContentTag,
10 |   isInsideMessageTag,
11 |   fullPath,
12 |   resolveBaseNode,
13 |   resolveExpression,
14 | }: CompileNodeContext<IfBlock>) {
15 |   const condition = await resolveExpression(node.expression, scope)
16 |   const children = (condition ? node.children : node.else?.children) ?? []
17 |   const childScope = scope.copy()
18 |   for await (const childNode of children ?? []) {
19 |     await resolveBaseNode({
20 |       node: childNode,
21 |       scope: childScope,
22 |       isInsideStepTag,
23 |       isInsideMessageTag,
24 |       isInsideContentTag,
25 |       fullPath,
26 |     })
27 |   }
28 | }
29 | 


--------------------------------------------------------------------------------
/src/providers/openai-responses/utils/parsers/reasoning.ts:
--------------------------------------------------------------------------------
 1 | import { MessageInputItem } from '$promptl/providers/openai-responses/types'
 2 | import { AssistantMessage, ContentType, MessageRole } from '$promptl/types'
 3 | import { ResponseReasoningItem } from 'openai/resources/responses/responses'
 4 | 
 5 | export function isReasoning(
 6 |   message: MessageInputItem,
 7 | ): message is ResponseReasoningItem {
 8 |   return message.type === 'reasoning'
 9 | }
10 | 
11 | export function parseReasoning(message: ResponseReasoningItem) {
12 |   return {
13 |     id: message.id,
14 |     role: MessageRole.assistant,
15 |     content: message.summary.map((summary) => ({
16 |       type: ContentType.text,
17 |       text: summary.text,
18 |     })),
19 |     status: message.status,
20 |     encrypted_content: message.encrypted_content,
21 |   } satisfies AssistantMessage
22 | }
23 | 


--------------------------------------------------------------------------------
/src/utils/names.ts:
--------------------------------------------------------------------------------
 1 | export const reserved = new Set([
 2 |   'arguments',
 3 |   'as',
 4 |   'await',
 5 |   'break',
 6 |   'case',
 7 |   'catch',
 8 |   'class',
 9 |   'const',
10 |   'continue',
11 |   'debugger',
12 |   'default',
13 |   'delete',
14 |   'do',
15 |   'else',
16 |   'endfor',
17 |   'endif',
18 |   'enum',
19 |   'eval',
20 |   'export',
21 |   'extends',
22 |   'false',
23 |   'finally',
24 |   'for',
25 |   'function',
26 |   'if',
27 |   'implements',
28 |   'import',
29 |   'in',
30 |   'instanceof',
31 |   'interface',
32 |   'let',
33 |   'new',
34 |   'null',
35 |   'package',
36 |   'private',
37 |   'protected',
38 |   'public',
39 |   'return',
40 |   'static',
41 |   'super',
42 |   'switch',
43 |   'this',
44 |   'throw',
45 |   'true',
46 |   'try',
47 |   'typeof',
48 |   'var',
49 |   'void',
50 |   'while',
51 |   'with',
52 |   'yield',
53 | ])
54 | 


--------------------------------------------------------------------------------
/src/parser/state/multi_line_comment.ts:
--------------------------------------------------------------------------------
 1 | import PARSER_ERRORS from '$promptl/error/errors'
 2 | import type { Comment } from '$promptl/parser/interfaces'
 3 | 
 4 | import { Parser } from '..'
 5 | 
 6 | export function multiLineComment(parser: Parser) {
 7 |   if (parser.match('*/')) {
 8 |     parser.error(PARSER_ERRORS.unexpectedEndOfComment)
 9 |   }
10 | 
11 |   const start = parser.index
12 | 
13 |   while (parser.index < parser.template.length) {
14 |     if (parser.matchRegex(/\*\//)) {
15 |       parser.index += 2
16 |       break
17 |     }
18 |     parser.index++
19 |   }
20 | 
21 |   const data = parser.template.substring(start, parser.index)
22 | 
23 |   const node = {
24 |     start,
25 |     end: parser.index,
26 |     type: 'Comment',
27 |     raw: data,
28 |     data: data.substring(2, data.length - 2),
29 |   } as Comment
30 | 
31 |   parser.current().children!.push(node)
32 | }
33 | 


--------------------------------------------------------------------------------
/src/parser/state/config.ts:
--------------------------------------------------------------------------------
 1 | import PARSER_ERRORS from '$promptl/error/errors'
 2 | import { Parser } from '$promptl/parser'
 3 | import type { Config } from '$promptl/parser/interfaces'
 4 | 
 5 | export function config(parser: Parser) {
 6 |   const start = parser.index
 7 |   parser.eat('---')
 8 | 
 9 |   // Read until there is a line break followed by a triple dash
10 |   const currentIndex = parser.index
11 |   const data = parser.readUntil(/\n\s*---\s*/)
12 |   if (parser.index === parser.template.length) {
13 |     parser.error(PARSER_ERRORS.unexpectedToken('---'), currentIndex + 1)
14 |   }
15 | 
16 |   parser.allowWhitespace()
17 |   parser.eat('---', true)
18 |   parser.eat('\n')
19 | 
20 |   const node = {
21 |     start,
22 |     end: parser.index,
23 |     type: 'Config',
24 |     raw: data,
25 |     value: data,
26 |   } as Config
27 | 
28 |   parser.current().children!.push(node)
29 | }
30 | 


--------------------------------------------------------------------------------
/src/parser/state/fragment.ts:
--------------------------------------------------------------------------------
 1 | import {
 2 |   CUSTOM_TAG_END,
 3 |   CUSTOM_TAG_START,
 4 | } from '$promptl/constants'
 5 | 
 6 | import { Parser } from '..'
 7 | import { RESERVED_TAG_REGEX } from '../utils/regex'
 8 | import { config } from './config'
 9 | import { multiLineComment } from './multi_line_comment'
10 | import { mustache } from './mustache'
11 | import { tag } from './tag'
12 | import { text } from './text'
13 | 
14 | export default function fragment(parser: Parser): (parser: Parser) => void {
15 |   if (
16 |     parser.matchRegex(RESERVED_TAG_REGEX) ||
17 |     parser.match('<!--')
18 |   ) {
19 |     return tag
20 |   }
21 |   if (parser.match(CUSTOM_TAG_START) || parser.match(CUSTOM_TAG_END)) {
22 |     return mustache
23 |   }
24 |   if (parser.match('/*') || parser.match('*/')) {
25 |     return multiLineComment
26 |   }
27 |   if (parser.matchRegex(/-{3}(?!-)/)) {
28 |     return config
29 |   }
30 | 
31 |   return text
32 | }
33 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
 2 | 
 3 | # Dependencies
 4 | node_modules
 5 | .pnp
 6 | .pnp.js
 7 | 
 8 | # Local env files
 9 | .env
10 | .env.local
11 | .env.development
12 | .env.development.local
13 | .env.test.local
14 | .env.production.local
15 | 
16 | # Testing
17 | coverage
18 | 
19 | # Turbo
20 | .turbo
21 | 
22 | # Vercel
23 | .vercel
24 | 
25 | # Build Outputs
26 | .next/
27 | out/
28 | build
29 | dist
30 | dist-rpc
31 | 
32 | # Debug
33 | npm-debug.log*
34 | yarn-debug.log*
35 | yarn-error.log*
36 | 
37 | # Misc
38 | .DS_Store
39 | 
40 | # postgres data in development. Ignore
41 | docker/pgdata/
42 | 
43 | # Next.js
44 | next-env.d.ts
45 | apps/web/scripts-dist
46 | 
47 | # File uploads in development
48 | tmp
49 | 
50 | # Misc
51 | TODO.md
52 | .cursorrules
53 | .vscode/settings.json
54 | 
55 | # Sentry
56 | .sentryclirc
57 | .env.sentry-build-plugin
58 | bin
59 | 
60 | AGENTS.md
61 | 
62 | # Vim crap
63 | *-E
64 | 


--------------------------------------------------------------------------------
/src/compiler/logic/nodes/sequenceExpression.ts:
--------------------------------------------------------------------------------
 1 | import {
 2 |   resolveLogicNode,
 3 |   updateScopeContextForNode,
 4 | } from '$promptl/compiler/logic'
 5 | import type {
 6 |   ResolveNodeProps,
 7 |   UpdateScopeContextProps,
 8 | } from '$promptl/compiler/logic/types'
 9 | import type { SequenceExpression } from 'estree'
10 | 
11 | /**
12 |  * ### SequenceExpression
13 |  * Represents a sequence of expressions. It is only used to evaluate ?. operators.
14 |  */
15 | export async function resolve({
16 |   node,
17 |   ...props
18 | }: ResolveNodeProps<SequenceExpression>) {
19 |   return await Promise.all(
20 |     node.expressions.map((expression) =>
21 |       resolveLogicNode({ node: expression, ...props }),
22 |     ),
23 |   )
24 | }
25 | 
26 | export function updateScopeContext({
27 |   node,
28 |   ...props
29 | }: UpdateScopeContextProps<SequenceExpression>) {
30 |   for (const expression of node.expressions) {
31 |     updateScopeContextForNode({ node: expression, ...props })
32 |   }
33 | }
34 | 


--------------------------------------------------------------------------------
/src/parser/utils/bracket.ts:
--------------------------------------------------------------------------------
 1 | const SQUARE_BRACKET_OPEN = '['.charCodeAt(0)
 2 | const SQUARE_BRACKET_CLOSE = ']'.charCodeAt(0)
 3 | const CURLY_BRACKET_OPEN = '{'.charCodeAt(0)
 4 | const CURLY_BRACKET_CLOSE = '}'.charCodeAt(0)
 5 | 
 6 | export function isBracketOpen(code: number) {
 7 |   return code === SQUARE_BRACKET_OPEN || code === CURLY_BRACKET_OPEN
 8 | }
 9 | 
10 | export function isBracketClose(code: number) {
11 |   return code === SQUARE_BRACKET_CLOSE || code === CURLY_BRACKET_CLOSE
12 | }
13 | 
14 | export function isBracketPair(open: number, close: number) {
15 |   return (
16 |     (open === SQUARE_BRACKET_OPEN && close === SQUARE_BRACKET_CLOSE) ||
17 |     (open === CURLY_BRACKET_OPEN && close === CURLY_BRACKET_CLOSE)
18 |   )
19 | }
20 | 
21 | export function getBracketClose(open: number): number | undefined {
22 |   if (open === SQUARE_BRACKET_OPEN) {
23 |     return SQUARE_BRACKET_CLOSE
24 |   }
25 |   if (open === CURLY_BRACKET_OPEN) {
26 |     return CURLY_BRACKET_CLOSE
27 |   }
28 | }
29 | 


--------------------------------------------------------------------------------
/src/parser/read/expression.ts:
--------------------------------------------------------------------------------
 1 | import CompileError from '$promptl/error/error'
 2 | import PARSER_ERRORS from '$promptl/error/errors'
 3 | import { Parser } from '$promptl/parser'
 4 | import { parseExpressionAt } from '$promptl/parser/utils/acorn'
 5 | 
 6 | export default function readExpression(parser: Parser) {
 7 |   try {
 8 |     const node = parseExpressionAt(parser.template, parser.index)
 9 | 
10 |     let numParenthesis = 0
11 | 
12 |     for (let i = parser.index; i < node.start; i += 1) {
13 |       if (parser.template[i] === '(') numParenthesis += 1
14 |     }
15 | 
16 |     let index = node.end
17 |     while (numParenthesis > 0) {
18 |       const char = parser.template[index]
19 | 
20 |       if (char === ')') {
21 |         numParenthesis -= 1
22 |       } else if (!/\s/.test(char!)) {
23 |         parser.error(PARSER_ERRORS.unexpectedToken(')'), index)
24 |       }
25 | 
26 |       index += 1
27 |     }
28 | 
29 |     parser.index = index
30 | 
31 |     return node
32 |   } catch (err) {
33 |     parser.acornError(err as CompileError)
34 |   }
35 | }
36 | 


--------------------------------------------------------------------------------
/src/compiler/logic/nodes/conditionalExpression.ts:
--------------------------------------------------------------------------------
 1 | import type {
 2 |   ResolveNodeProps,
 3 |   UpdateScopeContextProps,
 4 | } from '$promptl/compiler/logic/types'
 5 | import type { ConditionalExpression } from 'estree'
 6 | 
 7 | import { resolveLogicNode, updateScopeContextForNode } from '..'
 8 | 
 9 | /**
10 |  * ### ConditionalExpression
11 |  * Represents a ternary operation.
12 |  *
13 |  * Example: `a ? b : c`
14 |  */
15 | export async function resolve({
16 |   node,
17 |   ...props
18 | }: ResolveNodeProps<ConditionalExpression>) {
19 |   const condition = await resolveLogicNode({ node: node.test, ...props })
20 |   return await resolveLogicNode({
21 |     node: condition ? node.consequent : node.alternate,
22 |     ...props,
23 |   })
24 | }
25 | 
26 | export function updateScopeContext({
27 |   node,
28 |   ...props
29 | }: UpdateScopeContextProps<ConditionalExpression>) {
30 |   updateScopeContextForNode({ node: node.test, ...props })
31 |   updateScopeContextForNode({ node: node.consequent, ...props })
32 |   updateScopeContextForNode({ node: node.alternate, ...props })
33 | }
34 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (C) 2024 Latitude Data SL
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 6 | 
 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
 8 | 
 9 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
10 | 


--------------------------------------------------------------------------------
/src/providers/openai-responses/utils/parsers/webSearch.ts:
--------------------------------------------------------------------------------
 1 | import {
 2 |   MessageInputItem,
 3 |   WebSearchCall,
 4 | } from '$promptl/providers/openai-responses/types'
 5 | import {
 6 |   AssistantMessage,
 7 |   ContentType,
 8 |   MessageRole,
 9 |   ToolCallContent,
10 | } from '$promptl/types'
11 | 
12 | export function isWebsearchCall(
13 |   message: MessageInputItem,
14 | ): message is WebSearchCall {
15 |   return message.type === 'web_search_call'
16 | }
17 | 
18 | export function parseWebsearchCall({
19 |   message: inputMessage,
20 |   webSearch,
21 | }: {
22 |   message: WebSearchCall
23 |   webSearch: Map<string, string>
24 | }) {
25 |   const callId = inputMessage.id.replace(/^ws_/, '')
26 |   webSearch.set(callId, callId)
27 | 
28 |   return {
29 |     role: MessageRole.assistant,
30 |     id: inputMessage.id,
31 |     status: inputMessage.status,
32 |     content: [
33 |       {
34 |         type: ContentType.toolCall,
35 |         toolCallId: callId,
36 |         toolName: inputMessage.type,
37 |         toolArguments: {},
38 |       } satisfies ToolCallContent,
39 |     ],
40 |   } satisfies AssistantMessage
41 | }
42 | 


--------------------------------------------------------------------------------
/src/providers/index.ts:
--------------------------------------------------------------------------------
 1 | import { OpenAIResponsesAdapter } from '$promptl/providers/openai-responses/adapter'
 2 | import { Message } from '$promptl/types'
 3 | import { defaultAdapter, ProviderAdapter } from './adapter'
 4 | import { AnthropicAdapter } from './anthropic/adapter'
 5 | import { OpenAIAdapter } from './openai/adapter'
 6 | import { VercelAIAdapter } from './vercel-ai/adapter'
 7 | 
 8 | export type { ProviderAdapter } from './adapter'
 9 | 
10 | export const Adapters = {
11 |   default: defaultAdapter,
12 |   anthropic: AnthropicAdapter,
13 |   openai: OpenAIAdapter,
14 |   openaiResponses: OpenAIResponsesAdapter,
15 |   vercel: VercelAIAdapter,
16 | } as const
17 | 
18 | export type AdapterKey = keyof typeof Adapters
19 | export type AdapterMessageType<
20 |   T extends keyof typeof Adapters = keyof typeof Adapters,
21 | > = ReturnType<(typeof Adapters)[T]['fromPromptl']>['messages'][number]
22 | 
23 | export function getAdapter<M extends Message>(adapterType: AdapterKey) {
24 |   const adapter = Adapters[adapterType]
25 |   if (!adapter) throw new Error(`Adapter not found: ${adapterType}`)
26 |   return adapter as ProviderAdapter<M>
27 | }
28 | 


--------------------------------------------------------------------------------
/src/compiler/logic/index.ts:
--------------------------------------------------------------------------------
 1 | import type { Node } from 'estree'
 2 | 
 3 | import { nodeResolvers, updateScopeContextResolvers } from './nodes'
 4 | import type {
 5 |   NodeType,
 6 |   ResolveNodeProps,
 7 |   UpdateScopeContextProps,
 8 | } from './types'
 9 | 
10 | /**
11 |  * Given a node, calculates the resulting value.
12 |  */
13 | export async function resolveLogicNode(props: ResolveNodeProps<Node>) {
14 |   const type = props.node.type as NodeType
15 |   if (!nodeResolvers[type]) {
16 |     throw new Error(`Unknown node type: ${type}`)
17 |   }
18 | 
19 |   const nodeResolver = nodeResolvers[props.node.type as NodeType]
20 |   return nodeResolver(props)
21 | }
22 | 
23 | /**
24 |  * Given a node, keeps track of the defined variables.
25 |  */
26 | export async function updateScopeContextForNode(
27 |   props: UpdateScopeContextProps<Node>,
28 | ) {
29 |   const type = props.node.type as NodeType
30 |   if (!updateScopeContextResolvers[type]) {
31 |     throw new Error(`Unknown node type: ${type}`)
32 |   }
33 | 
34 |   const updateScopeContextFn =
35 |     updateScopeContextResolvers[props.node.type as NodeType]
36 |   return updateScopeContextFn(props)
37 | }
38 | 


--------------------------------------------------------------------------------
/src/rpc/types.ts:
--------------------------------------------------------------------------------
 1 | declare global {
 2 |   const Javy: {
 3 |     IO: {
 4 |       readSync: (fd: number, buffer: Uint8Array) => number
 5 |       writeSync: (fd: number, buffer: Uint8Array) => void
 6 |     }
 7 |   }
 8 | }
 9 | 
10 | export namespace RPC {
11 |   export const enum Procedure {
12 |     ScanPrompt = 'scanPrompt',
13 |     RenderPrompt = 'renderPrompt',
14 |     CreateChain = 'createChain',
15 |     StepChain = 'stepChain',
16 |   }
17 | 
18 |   export type Call<Parameters> = {
19 |     procedure: Procedure
20 |     parameters: Parameters
21 |   }
22 | 
23 |   export const enum ErrorCode {
24 |     ReceiveError = 'RECEIVE_ERROR',
25 |     ExecuteError = 'EXECUTE_ERROR',
26 |     SendError = 'SEND_ERROR',
27 |     ProcedureError = 'PROCEDURE_ERROR',
28 |     UnknownError = 'UNKNOWN_ERROR',
29 |   }
30 | 
31 |   export type Error = {
32 |     code: ErrorCode
33 |     message: string
34 |     details?: Record<string, any>
35 |   }
36 | 
37 |   export type Result<Value> = {
38 |     value?: Value
39 |     error?: Error
40 |   }
41 | 
42 |   // Errors are thrown in order to pass them
43 |   export type Handler<Parameters, Value> = (parameters: Parameters) => Value
44 | }
45 | 


--------------------------------------------------------------------------------
/src/providers/openai-responses/utils/parsers/fileSearch.ts:
--------------------------------------------------------------------------------
 1 | import {
 2 |   FileSearchCall,
 3 |   MessageInputItem,
 4 | } from '$promptl/providers/openai-responses/types'
 5 | import {
 6 |   AssistantMessage,
 7 |   ContentType,
 8 |   MessageRole,
 9 |   ToolCallContent,
10 | } from '$promptl/types'
11 | 
12 | export function isFileSearchCall(
13 |   message: MessageInputItem,
14 | ): message is FileSearchCall {
15 |   return message.type === 'file_search_call'
16 | }
17 | 
18 | export function parseFileSearch({
19 |   message,
20 |   fileSearch,
21 | }: {
22 |   message: FileSearchCall
23 |   fileSearch: Map<string, string>
24 | }) {
25 |   const callId = message.id.replace(/^fs_/, '')
26 |   fileSearch.set(callId, callId)
27 |   return {
28 |     role: MessageRole.assistant,
29 |     id: message.id,
30 |     status: message.status,
31 |     content: [
32 |       {
33 |         type: ContentType.toolCall,
34 |         toolCallId: callId,
35 |         toolName: message.type,
36 |         toolArguments: {
37 |           queries: message.queries,
38 |           results: message.results,
39 |         },
40 |       } satisfies ToolCallContent,
41 |     ],
42 |   } satisfies AssistantMessage
43 | }
44 | 


--------------------------------------------------------------------------------
/src/providers/openai-responses/utils/parsers/inputMessage.ts:
--------------------------------------------------------------------------------
 1 | import {
 2 |   MessageInputItem,
 3 |   ResponseInputMessage,
 4 | } from '$promptl/providers/openai-responses/types'
 5 | import { parseSimpleContent } from '$promptl/providers/openai-responses/utils/parsers/parseSimpleContent'
 6 | import {
 7 |   MessageRole,
 8 |   SystemMessage,
 9 |   DeveloperMessage,
10 |   UserMessage,
11 | } from '$promptl/types'
12 | 
13 | type InputMessageRole = ResponseInputMessage['role']
14 | 
15 | type PromptlMessage = SystemMessage | DeveloperMessage | UserMessage
16 | function fromOpenAIRoleToPromptlRole(role: InputMessageRole) {
17 |   return MessageRole[role]
18 | }
19 | 
20 | export function isInputMessage(
21 |   message: MessageInputItem,
22 | ): message is ResponseInputMessage {
23 |   return (
24 |     message.type === 'message' &&
25 |     message.role !== 'assistant' &&
26 |     typeof message.content !== 'string'
27 |   )
28 | }
29 | 
30 | export function parseInputMessage(message: ResponseInputMessage) {
31 |   return {
32 |     role: fromOpenAIRoleToPromptlRole(message.role),
33 |     status: message.status,
34 |     content: message.content.map(parseSimpleContent),
35 |   } satisfies PromptlMessage
36 | }
37 | 


--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "$schema": "https://json.schemastore.org/tsconfig",
 3 |   "display": "Default",
 4 |   "compilerOptions": {
 5 |     "allowJs": true,
 6 |     "declaration": false,
 7 |     "declarationMap": false,
 8 |     "esModuleInterop": true,
 9 |     "forceConsistentCasingInFileNames": true,
10 |     "incremental": false,
11 |     "isolatedModules": true,
12 |     "lib": ["es2022", "DOM", "DOM.Iterable", "esnext.asynciterable"],
13 |     "module": "ESNext",
14 |     "moduleDetection": "force",
15 |     "moduleResolution": "node",
16 |     "noFallthroughCasesInSwitch": true,
17 |     "noUncheckedIndexedAccess": true,
18 |     "noUnusedLocals": true,
19 |     "resolveJsonModule": true,
20 |     "skipLibCheck": true,
21 |     "strict": true,
22 |     "target": "ES2022",
23 |     "jsx": "react-jsx",
24 |     "typeRoots": [
25 |       "node_modules/@types",
26 |       "node_modules/@latitude-data/typescript-config/types"
27 |     ],
28 |     "baseUrl": ".",
29 |     "rootDir": "./src",
30 |     "outDir": "dist",
31 |     "noEmit": false,
32 |     "paths": {
33 |       "$promptl/*": ["./src/*"],
34 |       "acorn": ["src/types/acorn"]
35 |     }
36 |   },
37 |   "include": ["src/**/*"],
38 |   "exclude": ["node_modules", "dist"]
39 | }


--------------------------------------------------------------------------------
/src/compiler/logic/nodes/identifier.ts:
--------------------------------------------------------------------------------
 1 | import type {
 2 |   ResolveNodeProps,
 3 |   UpdateScopeContextProps,
 4 | } from '$promptl/compiler/logic/types'
 5 | import errors from '$promptl/error/errors'
 6 | import type { Identifier } from 'estree'
 7 | 
 8 | /**
 9 |  * ### Identifier
10 |  * Represents a variable from the scope.
11 |  */
12 | export async function resolve({
13 |   node,
14 |   scope,
15 |   builtins,
16 | }: ResolveNodeProps<Identifier>) {
17 |   if (node.name in builtins) {
18 |     return builtins[node.name]!()
19 |   }
20 |   if (!scope.exists(node.name)) {
21 |     return undefined
22 |   }
23 |   return scope.get(node.name)
24 | }
25 | 
26 | export function updateScopeContext({
27 |   node,
28 |   scopeContext,
29 |   builtins,
30 |   raiseError,
31 | }: UpdateScopeContextProps<Identifier>) {
32 |   if (node.name in builtins) {
33 |     return
34 |   }
35 |   if (!scopeContext.definedVariables.has(node.name)) {
36 |     if (scopeContext.onlyPredefinedVariables === undefined) {
37 |       scopeContext.usedUndefinedVariables.add(node.name)
38 |       return
39 |     }
40 |     if (!scopeContext.onlyPredefinedVariables.has(node.name)) {
41 |       raiseError(errors.variableNotDefined(node.name), node)
42 |     }
43 |   }
44 | }
45 | 


--------------------------------------------------------------------------------
/src/compiler/logic/types.ts:
--------------------------------------------------------------------------------
 1 | import Scope, { ScopeContext } from '$promptl/compiler/scope'
 2 | import { Node } from 'estree'
 3 | 
 4 | export enum NodeType {
 5 |   Literal = 'Literal',
 6 |   Identifier = 'Identifier',
 7 |   ObjectExpression = 'ObjectExpression',
 8 |   ArrayExpression = 'ArrayExpression',
 9 |   SequenceExpression = 'SequenceExpression',
10 |   LogicalExpression = 'LogicalExpression',
11 |   BinaryExpression = 'BinaryExpression',
12 |   UnaryExpression = 'UnaryExpression',
13 |   AssignmentExpression = 'AssignmentExpression',
14 |   UpdateExpression = 'UpdateExpression',
15 |   MemberExpression = 'MemberExpression',
16 |   ConditionalExpression = 'ConditionalExpression',
17 |   CallExpression = 'CallExpression',
18 |   ChainExpression = 'ChainExpression',
19 | }
20 | 
21 | type RaiseErrorFn<T = void | never> = (
22 |   { code, message }: { code: string; message: string },
23 |   node: Node,
24 | ) => T
25 | 
26 | export type ResolveNodeProps<N extends Node> = {
27 |   node: N
28 |   scope: Scope
29 |   builtins: Record<string, () => any>
30 |   raiseError: RaiseErrorFn<never>
31 | }
32 | 
33 | export type UpdateScopeContextProps<N extends Node> = {
34 |   node: N
35 |   scopeContext: ScopeContext
36 |   builtins: Record<string, () => any>
37 |   raiseError: RaiseErrorFn<void>
38 | }
39 | 


--------------------------------------------------------------------------------
/src/compiler/deserializeChain.test.ts:
--------------------------------------------------------------------------------
 1 | import { describe, expect, it } from 'vitest'
 2 | 
 3 | import { removeCommonIndent } from './utils'
 4 | import { Adapters } from '$promptl/providers'
 5 | import { Chain } from './chain'
 6 | 
 7 | describe('deserialize chain', async () => {
 8 |   it('get final step from serialized chain', async () => {
 9 |     const prompt = removeCommonIndent(`
10 |       <step>
11 |         {{foo = 5}}
12 |       </step>
13 |       <step>
14 |         {{foo += 1}}
15 |       </step>
16 |       <step>
17 |         <assistant>
18 |           The final result is {{foo}}
19 |         </assistant>
20 |       </step>
21 |     `)
22 | 
23 |     const chain = new Chain({ prompt, adapter: Adapters.openai })
24 | 
25 |     await chain.step()
26 |     await chain.step('First step response')
27 |     const serializedChain = chain.serialize()
28 |     const serialized = JSON.stringify(serializedChain)
29 | 
30 |     // In another context we deserialize existing chain
31 |     const deserializedChain = Chain.deserialize({ serialized })
32 |     const { messages } = await deserializedChain!.step('Last step')
33 |     expect(messages[messages.length - 1]).toEqual({
34 |       role: 'assistant',
35 |       tool_calls: undefined,
36 |       content: [{ text: 'The final result is 6', type: 'text' }],
37 |     })
38 |   })
39 | })
40 | 


--------------------------------------------------------------------------------
/.github/workflows/test.yml:
--------------------------------------------------------------------------------
 1 | name: Tests
 2 | 
 3 | on:
 4 |   pull_request:
 5 |     branches:
 6 |       - main
 7 | jobs:
 8 |   test:
 9 |     runs-on: ubuntu-latest
10 |     strategy:
11 |       matrix:
12 |         node-version: [20.x]
13 | 
14 |     steps:
15 |       - name: Checkout repository
16 |         uses: actions/checkout@v3
17 |         with:
18 |           fetch-depth: 2
19 | 
20 |       - name: Setup Node.js
21 |         uses: actions/setup-node@v3
22 |         with:
23 |           node-version: ${{ matrix.node-version }}
24 | 
25 |       - name: Install pnpm
26 |         uses: pnpm/action-setup@v3
27 |         with:
28 |           version: 9
29 |           run_install: false
30 | 
31 |       - name: Get pnpm store directory
32 |         shell: bash
33 |         run: |
34 |           echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
35 | 
36 |       - name: Setup pnpm cache
37 |         uses: actions/cache@v3
38 |         with:
39 |           path: ${{ env.STORE_PATH }}
40 |           key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
41 |           restore-keys: |
42 |             ${{ runner.os }}-pnpm-store-
43 | 
44 |       - name: Install Node.js dependencies
45 |         run: pnpm install
46 | 
47 |       - name: Node.js Test
48 |         env:
49 |           NODE_ENV: test
50 |         run: pnpm test
51 | 


--------------------------------------------------------------------------------
/src/compiler/logic/nodes/memberExpression.ts:
--------------------------------------------------------------------------------
 1 | import { MEMBER_EXPRESSION_METHOD } from '$promptl/compiler/logic/operators'
 2 | import type {
 3 |   ResolveNodeProps,
 4 |   UpdateScopeContextProps,
 5 | } from '$promptl/compiler/logic/types'
 6 | import type { Identifier, MemberExpression } from 'estree'
 7 | 
 8 | import { resolveLogicNode, updateScopeContextForNode } from '..'
 9 | 
10 | /**
11 |  * ### MemberExpression
12 |  * Represents a property from an object. If the property does not exist in the object, it will return undefined.
13 |  */
14 | export async function resolve({
15 |   node,
16 |   ...props
17 | }: ResolveNodeProps<MemberExpression>) {
18 |   const object = await resolveLogicNode({
19 |     node: node.object,
20 |     ...props,
21 |   })
22 | 
23 |   // Accessing to the property can be optional (?.)
24 |   if (object == null && node.optional) return undefined
25 | 
26 |   const property = node.computed
27 |     ? await resolveLogicNode({
28 |         node: node.property,
29 |         ...props,
30 |       })
31 |     : (node.property as Identifier).name
32 | 
33 |   return MEMBER_EXPRESSION_METHOD(object, property)
34 | }
35 | 
36 | export function updateScopeContext({
37 |   node,
38 |   ...props
39 | }: UpdateScopeContextProps<MemberExpression>) {
40 |   updateScopeContextForNode({ node: node.object, ...props })
41 |   if (node.computed) {
42 |     updateScopeContextForNode({ node: node.property, ...props })
43 |   }
44 | }
45 | 


--------------------------------------------------------------------------------
/.github/workflows/lint.yml:
--------------------------------------------------------------------------------
 1 | name: Linter & Types
 2 | 
 3 | on:
 4 |   pull_request:
 5 |     branches:
 6 |       - main
 7 | jobs:
 8 |   lint:
 9 |     runs-on: ubuntu-latest
10 | 
11 |     strategy:
12 |       matrix:
13 |         node-version: [20.x]
14 | 
15 |     steps:
16 |       - name: Checkout repository
17 |         uses: actions/checkout@v3
18 |         with:
19 |           fetch-depth: 2
20 | 
21 |       - name: Setup Node.js
22 |         uses: actions/setup-node@v3
23 |         with:
24 |           node-version: ${{ matrix.node-version }}
25 | 
26 |       - name: Install pnpm
27 |         uses: pnpm/action-setup@v3
28 |         with:
29 |           version: 9
30 |           run_install: false
31 | 
32 |       - name: Get pnpm store directory
33 |         shell: bash
34 |         run: |
35 |           echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
36 | 
37 |       - name: Setup pnpm cache
38 |         uses: actions/cache@v3
39 |         with:
40 |           path: ${{ env.STORE_PATH }}
41 |           key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
42 |           restore-keys: |
43 |             ${{ runner.os }}-pnpm-store-
44 | 
45 |       - name: Install Node.js dependencies
46 |         run: pnpm install
47 | 
48 |       - name: Prettier
49 |         run: pnpm prettier:check
50 | 
51 |       - name: Node.js Lint
52 |         run: pnpm lint
53 | 
54 |       - name: TypeScript
55 |         run: pnpm tc
56 | 
57 | 


--------------------------------------------------------------------------------
/src/constants.ts:
--------------------------------------------------------------------------------
 1 | import { ContentTypeTagName, MessageRole } from './types'
 2 | 
 3 | export const CUSTOM_TAG_START = '{{'
 4 | export const CUSTOM_TAG_END = '}}'
 5 | 
 6 | export enum TAG_NAMES {
 7 |   message = 'message',
 8 |   system = MessageRole.system,
 9 |   user = MessageRole.user,
10 |   assistant = MessageRole.assistant,
11 |   tool = MessageRole.tool,
12 |   content = 'content',
13 |   text = ContentTypeTagName.text,
14 |   image = ContentTypeTagName.image,
15 |   file = ContentTypeTagName.file,
16 |   toolCall = ContentTypeTagName.toolCall,
17 |   prompt = 'prompt',
18 |   scope = 'scope',
19 |   step = 'step',
20 | }
21 | 
22 | export const CUSTOM_MESSAGE_ROLE_ATTR = 'role' as const
23 | export const CUSTOM_CONTENT_TYPE_ATTR = 'type' as const
24 | export const REFERENCE_PATH_ATTR = 'path' as const
25 | export const REFERENCE_DEPTH_LIMIT = 50
26 | export const CHAIN_STEP_ISOLATED_ATTR = 'isolated' as const
27 | 
28 | export enum KEYWORDS {
29 |   if = 'if',
30 |   endif = 'endif',
31 |   else = 'else',
32 |   for = 'for',
33 |   endfor = 'endfor',
34 |   as = 'as',
35 |   in = 'in',
36 |   true = 'true',
37 |   false = 'false',
38 |   null = 'null',
39 | }
40 | 
41 | export const RESERVED_KEYWORDS = Object.values(KEYWORDS)
42 | export const RESERVED_TAGS = Object.values(TAG_NAMES)
43 | export const SPECIAL_IDENTIFIERS = new Set(['$now'])
44 | 
45 | export const SPECIAL_RESOLVERS: Record<string, () => unknown> = {
46 |   $now: () => new Date(),
47 | }
48 | 


--------------------------------------------------------------------------------
/src/compiler/logic/nodes/unaryExpression.ts:
--------------------------------------------------------------------------------
 1 | import { UNARY_OPERATOR_METHODS } from '$promptl/compiler/logic/operators'
 2 | import type {
 3 |   ResolveNodeProps,
 4 |   UpdateScopeContextProps,
 5 | } from '$promptl/compiler/logic/types'
 6 | import errors from '$promptl/error/errors'
 7 | import type { UnaryExpression } from 'estree'
 8 | 
 9 | import { resolveLogicNode, updateScopeContextForNode } from '..'
10 | 
11 | /**
12 |  * ### UnaryExpression
13 |  * Represents a simple operation on a single operand, either as a prefix or suffix.
14 |  *
15 |  * Example: `{!a}`
16 |  */
17 | export async function resolve({
18 |   node,
19 |   raiseError,
20 |   ...props
21 | }: ResolveNodeProps<UnaryExpression>) {
22 |   const unaryOperator = node.operator
23 |   if (!(unaryOperator in UNARY_OPERATOR_METHODS)) {
24 |     raiseError(errors.unsupportedOperator(unaryOperator), node)
25 |   }
26 | 
27 |   const unaryArgument = await resolveLogicNode({
28 |     node: node.argument,
29 |     raiseError,
30 |     ...props,
31 |   })
32 |   const unaryPrefix = node.prefix
33 |   return UNARY_OPERATOR_METHODS[unaryOperator]?.(unaryArgument, unaryPrefix)
34 | }
35 | 
36 | export function updateScopeContext({
37 |   node,
38 |   scopeContext,
39 |   ...props
40 | }: UpdateScopeContextProps<UnaryExpression>) {
41 |   const unaryOperator = node.operator
42 |   if (!(unaryOperator in UNARY_OPERATOR_METHODS)) {
43 |     props.raiseError(errors.unsupportedOperator(unaryOperator), node)
44 |   }
45 | 
46 |   updateScopeContextForNode({ node: node.argument, scopeContext, ...props })
47 | }
48 | 


--------------------------------------------------------------------------------
/src/compiler/test/helpers.ts:
--------------------------------------------------------------------------------
 1 | import {
 2 |   AssistantMessage,
 3 |   Config,
 4 |   ContentType,
 5 |   Conversation,
 6 |   Message,
 7 |   MessageContent,
 8 | } from '$promptl/types'
 9 | import { expect } from 'vitest'
10 | 
11 | import { Chain } from '../chain'
12 | 
13 | export async function getExpectedError<T>(
14 |   action: () => Promise<unknown>,
15 |   errorClass: new () => T,
16 | ): Promise<T> {
17 |   try {
18 |     await action()
19 |   } catch (err) {
20 |     expect(err).toBeInstanceOf(errorClass)
21 |     return err as T
22 |   }
23 |   throw new Error('Expected an error to be thrown')
24 | }
25 | 
26 | export async function complete({
27 |   chain,
28 |   callback,
29 |   maxSteps = 50,
30 | }: {
31 |   chain: Chain
32 |   callback?: (convo: Conversation) => Promise<string>
33 |   maxSteps?: number
34 | }): Promise<{
35 |   response: MessageContent[]
36 |   messages: Message[]
37 |   config: Config
38 |   steps: number
39 | }> {
40 |   let steps = 0
41 |   let responseMessage: Omit<AssistantMessage, 'role'> | undefined
42 | 
43 |   while (true) {
44 |     const { completed, messages, config } = await chain.step(responseMessage)
45 | 
46 |     if (completed) {
47 |       return {
48 |         messages,
49 |         config,
50 |         steps,
51 |         response: responseMessage!.content as MessageContent[],
52 |       }
53 |     }
54 | 
55 |     const response = callback
56 |       ? await callback({ messages, config })
57 |       : 'RESPONSE'
58 |     responseMessage = { content: [{ type: ContentType.text, text: response }] }
59 |     steps++
60 | 
61 |     if (steps > maxSteps) throw new Error('too many chain steps')
62 |   }
63 | }
64 | 


--------------------------------------------------------------------------------
/src/compiler/logic/nodes/binaryExpression.ts:
--------------------------------------------------------------------------------
 1 | import { BINARY_OPERATOR_METHODS } from '$promptl/compiler/logic/operators'
 2 | import type {
 3 |   ResolveNodeProps,
 4 |   UpdateScopeContextProps,
 5 | } from '$promptl/compiler/logic/types'
 6 | import errors from '$promptl/error/errors'
 7 | import type { BinaryExpression, LogicalExpression } from 'estree'
 8 | 
 9 | import { resolveLogicNode, updateScopeContextForNode } from '..'
10 | 
11 | /**
12 |  * ### BinaryExpression
13 |  * Represents a simple operation between two operands.
14 |  *
15 |  * Example: `{a > b}`
16 |  */
17 | export async function resolve({
18 |   node,
19 |   raiseError,
20 |   ...props
21 | }: ResolveNodeProps<BinaryExpression | LogicalExpression>) {
22 |   const binaryOperator = node.operator
23 |   if (!(binaryOperator in BINARY_OPERATOR_METHODS)) {
24 |     raiseError(errors.unsupportedOperator(binaryOperator), node)
25 |   }
26 |   const leftOperand = await resolveLogicNode({
27 |     node: node.left,
28 |     raiseError,
29 |     ...props,
30 |   })
31 |   const rightOperand = await resolveLogicNode({
32 |     node: node.right,
33 |     raiseError,
34 |     ...props,
35 |   })
36 | 
37 |   return BINARY_OPERATOR_METHODS[binaryOperator]?.(leftOperand, rightOperand)
38 | }
39 | 
40 | export function updateScopeContext({
41 |   node,
42 |   ...props
43 | }: UpdateScopeContextProps<BinaryExpression | LogicalExpression>) {
44 |   const binaryOperator = node.operator
45 |   if (!(binaryOperator in BINARY_OPERATOR_METHODS)) {
46 |     props.raiseError(errors.unsupportedOperator(binaryOperator), node)
47 |   }
48 | 
49 |   updateScopeContextForNode({ node: node.left, ...props })
50 |   updateScopeContextForNode({ node: node.right, ...props })
51 | }
52 | 


--------------------------------------------------------------------------------
/src/compiler/base/nodes/mustache.ts:
--------------------------------------------------------------------------------
 1 | import errors from '$promptl/error/errors'
 2 | import { MustacheTag } from '$promptl/parser/interfaces'
 3 | import {
 4 |   isPromptLFile,
 5 |   PromptLFile,
 6 |   promptLFileToMessageContent,
 7 | } from '$promptl/types'
 8 | 
 9 | import { CompileNodeContext } from '../types'
10 | 
11 | export async function compile({
12 |   node,
13 |   scope,
14 |   addStrayText,
15 |   groupStrayText,
16 |   isInsideContentTag,
17 |   addContent,
18 |   resolveExpression,
19 |   baseNodeError,
20 | }: CompileNodeContext<MustacheTag>) {
21 |   const expression = node.expression
22 |   const value = await resolveExpression(expression, scope)
23 |   if (value === undefined) return
24 | 
25 |   const files = promptLFileArray(value)
26 |   if (files) {
27 |     if (isInsideContentTag) {
28 |       if (files.length > 1) {
29 |         baseNodeError(errors.multipleFilesInContentTag, node)
30 |         return
31 |       }
32 | 
33 |       const file = files[0]!
34 |       addStrayText(String(file.url), node)
35 |       return
36 |     }
37 | 
38 |     groupStrayText()
39 | 
40 |     files.forEach((file) => {
41 |       addContent({
42 |         node,
43 |         content: promptLFileToMessageContent(file),
44 |       })
45 |     })
46 | 
47 |     return
48 |   }
49 | 
50 |   if (typeof value === 'object' && value !== null) {
51 |     addStrayText(JSON.stringify(value), node)
52 |     return
53 |   }
54 | 
55 |   addStrayText(String(value), node)
56 | }
57 | 
58 | function promptLFileArray(value: unknown): PromptLFile[] | undefined {
59 |   if (isPromptLFile(value)) return [value]
60 |   if (Array.isArray(value) && value.length && value.every(isPromptLFile)) {
61 |     return value
62 |   }
63 |   return undefined
64 | }
65 | 


--------------------------------------------------------------------------------
/eslint.config.mjs:
--------------------------------------------------------------------------------
 1 | import typescriptEslintEslintPlugin from '@typescript-eslint/eslint-plugin'
 2 | import prettier from 'eslint-plugin-prettier'
 3 | import globals from 'globals'
 4 | import tsParser from '@typescript-eslint/parser'
 5 | import path from 'node:path'
 6 | import { fileURLToPath } from 'node:url'
 7 | import js from '@eslint/js'
 8 | import { FlatCompat } from '@eslint/eslintrc'
 9 | 
10 | const __filename = fileURLToPath(import.meta.url)
11 | const __dirname = path.dirname(__filename)
12 | const compat = new FlatCompat({
13 |   baseDirectory: __dirname,
14 |   recommendedConfig: js.configs.recommended,
15 |   allConfig: js.configs.all,
16 | })
17 | 
18 | export default [
19 |   {
20 |     ignores: ['**/.*.js', '**/node_modules/', '**/dist/'],
21 |   },
22 |   ...compat.extends('eslint:recommended'),
23 |   {
24 |     plugins: {
25 |       '@typescript-eslint': typescriptEslintEslintPlugin,
26 |       'prettier': prettier,
27 |     },
28 | 
29 |     languageOptions: {
30 |       globals: {
31 |         ...globals.node,
32 |         ...globals.browser,
33 |         Javy: 'readonly',
34 |       },
35 | 
36 |       parser: tsParser,
37 |     },
38 | 
39 |     settings: {
40 |       'import/resolver': {
41 |         typescript: {
42 |           project: './tsconfig.json',
43 |         },
44 |       },
45 |     },
46 | 
47 |     rules: {
48 |       'no-constant-condition': 'off',
49 |       'no-unused-vars': 'off',
50 | 
51 |       '@typescript-eslint/no-unused-vars': [
52 |         'error',
53 |         {
54 |           args: 'all',
55 |           argsIgnorePattern: '^_',
56 |           varsIgnorePattern: '^_',
57 |         },
58 |       ],
59 |     },
60 |   },
61 |   {
62 |     files: ['**/*.js', '**/*.jsx', '**/*.ts', '**/*.tsx'],
63 |   },
64 | ]
65 | 


--------------------------------------------------------------------------------
/src/providers/openai-responses/utils/parsers/functionCall.ts:
--------------------------------------------------------------------------------
 1 | import {
 2 |   MessageInputItem,
 3 |   ToolCallRequest,
 4 |   ToolCallResponse,
 5 | } from '$promptl/providers/openai-responses/types'
 6 | import {
 7 |   AssistantMessage,
 8 |   ContentType,
 9 |   MessageRole,
10 |   ToolMessage,
11 | } from '$promptl/types'
12 | 
13 | export function isFunctionCall(
14 |   message: MessageInputItem,
15 | ): message is ToolCallRequest {
16 |   return message.type === 'function_call'
17 | }
18 | 
19 | export function isFunctionCallOutput(
20 |   message: MessageInputItem,
21 | ): message is ToolCallResponse {
22 |   return message.type === 'function_call_output'
23 | }
24 | 
25 | export function parseFunctionCall(message: ToolCallRequest) {
26 |   return {
27 |     role: MessageRole.assistant,
28 |     id: message.id,
29 |     status: message.status,
30 |     content: [
31 |       {
32 |         type: ContentType.toolCall,
33 |         toolCallId: message.call_id,
34 |         toolName: message.name,
35 |         toolArguments: JSON.parse(message.arguments),
36 |       },
37 |     ],
38 |   } satisfies AssistantMessage
39 | }
40 | 
41 | export function parseFunctionCallOutput({
42 |   message,
43 |   toolNameMap,
44 | }: {
45 |   message: ToolCallResponse
46 |   toolNameMap: Map<string, string>
47 | }) {
48 |   // Tool name map should include the name for this tool response
49 |   return {
50 |     role: MessageRole.tool,
51 |     toolId: message.call_id,
52 |     toolName: toolNameMap.get(message.call_id) ?? message.call_id,
53 |     content: [
54 |       {
55 |         type: ContentType.text,
56 |         text: message.output,
57 |       },
58 |     ],
59 |     // Optional
60 |     status: message.status,
61 |     id: message.id,
62 |   } satisfies ToolMessage
63 | }
64 | 


--------------------------------------------------------------------------------
/src/types/customTypes.ts:
--------------------------------------------------------------------------------
 1 | import { ContentType, FileContent, ImageContent } from './message'
 2 | 
 3 | /**
 4 |  * Custom file type for PromptL.
 5 |  * It contains redundant information to make it easier to use.
 6 |  */
 7 | export type PromptLFile = {
 8 |   type: string
 9 |   mime: string
10 |   mimeType: string
11 | 
12 |   isImage: boolean
13 | 
14 |   name: string
15 | 
16 |   size: number
17 |   bytes: number
18 | 
19 |   url: string
20 | 
21 |   // Used to identify it
22 |   __promptlType: 'file'
23 | }
24 | 
25 | export function toPromptLFile({
26 |   file,
27 |   url,
28 | }: {
29 |   file: { type: string; size: number; name: string }
30 |   url: string
31 | }): PromptLFile {
32 |   const mimeType = file.type
33 |   const fileSize = file.size
34 | 
35 |   const isImage = mimeType.startsWith('image/')
36 | 
37 |   return {
38 |     __promptlType: 'file',
39 |     name: file.name,
40 |     url,
41 |     isImage,
42 | 
43 |     // Redundant type
44 |     type: mimeType,
45 |     mime: mimeType,
46 |     mimeType: mimeType,
47 | 
48 |     // Redundant size
49 |     size: fileSize,
50 |     bytes: fileSize,
51 |   }
52 | }
53 | 
54 | export function isPromptLFile(value: unknown): value is PromptLFile {
55 |   return (
56 |     typeof value === 'object' &&
57 |     value !== null &&
58 |     '__promptlType' in value &&
59 |     (value as Record<string, unknown>).__promptlType === 'file'
60 |   )
61 | }
62 | 
63 | export function promptLFileToMessageContent(
64 |   file: PromptLFile,
65 | ): FileContent | ImageContent {
66 |   if (file.isImage) {
67 |     return {
68 |       type: ContentType.image,
69 |       image: file.url,
70 |     }
71 |   }
72 | 
73 |   return {
74 |     type: ContentType.file,
75 |     mimeType: file.mimeType,
76 |     file: file.url,
77 |   }
78 | }
79 | 


--------------------------------------------------------------------------------
/src/compiler/logic/nodes/arrayExpression.ts:
--------------------------------------------------------------------------------
 1 | import {
 2 |   resolveLogicNode,
 3 |   updateScopeContextForNode,
 4 | } from '$promptl/compiler/logic'
 5 | import type {
 6 |   ResolveNodeProps,
 7 |   UpdateScopeContextProps,
 8 | } from '$promptl/compiler/logic/types'
 9 | import { isIterable } from '$promptl/compiler/utils'
10 | import errors from '$promptl/error/errors'
11 | import type { ArrayExpression } from 'estree'
12 | 
13 | /**
14 |  * ### ArrayExpression
15 |  * Returns an array of values
16 |  */
17 | 
18 | export async function resolve({
19 |   node,
20 |   ...props
21 | }: ResolveNodeProps<ArrayExpression>) {
22 |   const { raiseError } = props
23 |   const resolvedArray = []
24 |   for (const element of node.elements) {
25 |     if (!element) continue
26 |     if (element.type !== 'SpreadElement') {
27 |       const value = await resolveLogicNode({
28 |         node: element,
29 |         ...props,
30 |       })
31 |       resolvedArray.push(value)
32 |       continue
33 |     }
34 | 
35 |     const spreadObject = await resolveLogicNode({
36 |       node: element.argument,
37 |       ...props,
38 |     })
39 | 
40 |     if (!isIterable(spreadObject)) {
41 |       raiseError(errors.invalidSpreadInArray(typeof spreadObject), element)
42 |     }
43 | 
44 |     for await (const value of spreadObject as Iterable<unknown>) {
45 |       resolvedArray.push(value)
46 |     }
47 |   }
48 | 
49 |   return resolvedArray
50 | }
51 | 
52 | export function updateScopeContext({
53 |   node,
54 |   ...props
55 | }: UpdateScopeContextProps<ArrayExpression>) {
56 |   for (const element of node.elements) {
57 |     if (!element) continue
58 | 
59 |     updateScopeContextForNode({
60 |       node: element.type === 'SpreadElement' ? element.argument : element,
61 |       ...props,
62 |     })
63 |   }
64 | }
65 | 


--------------------------------------------------------------------------------
/rollup.config.rpc.mjs:
--------------------------------------------------------------------------------
 1 | import commonjs from '@rollup/plugin-commonjs'
 2 | import nodeResolve from '@rollup/plugin-node-resolve'
 3 | import typescript from '@rollup/plugin-typescript'
 4 | import execute from 'rollup-plugin-execute'
 5 | 
 6 | /**
 7 |  * We have a internal circular dependency in the compiler,
 8 |  * which is intentional. We think in this case Rollup is too noisy.
 9 |  *
10 |  * @param {import('rollup').RollupLog} warning
11 |  * @returns {boolean}
12 |  */
13 | function isInternalCircularDependency(warning) {
14 |   return (
15 |     warning.code == 'CIRCULAR_DEPENDENCY' &&
16 |     warning.message.includes('src/compiler') &&
17 |     !warning.message.includes('node_modules')
18 |   )
19 | }
20 | 
21 | /** @type {import('rollup').RollupOptions} */
22 | export default {
23 |   onwarn: (warning, warn) => {
24 |     if (!isInternalCircularDependency(warning)) warn(warning)
25 |   },
26 |   input: 'src/index.rpc.ts',
27 |   output: [
28 |     {
29 |       file: 'dist-rpc/promptl.js',
30 |       format: 'es',
31 |     },
32 |   ],
33 |   plugins: [
34 |     nodeResolve({
35 |       preferBuiltins: true,
36 |     }),
37 |     commonjs(),
38 |     typescript({
39 |       noEmit: true,
40 |       tsconfig: './tsconfig.json',
41 |       exclude: ['**/__tests__', '**/*.test.ts'],
42 |     }),
43 |     execute([
44 |       [
45 |         'javy --version |',
46 |         'grep -q "6.0.0" ||',
47 |         '(echo "\x1b[0;31mJavy 6.0.0 required\x1b[0m"',
48 |         '&& exit 1)',
49 |       ].join(' '),
50 |       [
51 |         'javy build',
52 |         '-C dynamic=n',
53 |         '-C source=compressed',
54 |         '-J javy-stream-io=y',
55 |         '-J simd-json-builtins=y',
56 |         '-J text-encoding=y',
57 |         '-J event-loop=y',
58 |         '-o dist-rpc/promptl.wasm',
59 |         'dist-rpc/promptl.js',
60 |       ].join(' '),
61 |     ]),
62 |   ],
63 | }
64 | 


--------------------------------------------------------------------------------
/src/compiler/base/nodes/tags/ref.ts:
--------------------------------------------------------------------------------
 1 | import Scope from '$promptl/compiler/scope'
 2 | import errors from '$promptl/error/errors'
 3 | import { parse } from '$promptl/parser'
 4 | import { Fragment, ReferenceTag } from '$promptl/parser/interfaces'
 5 | 
 6 | import { CompileNodeContext, TemplateNodeWithStatus } from '../../types'
 7 | 
 8 | type ForNodeWithStatus = TemplateNodeWithStatus & {
 9 |   status: TemplateNodeWithStatus['status'] & {
10 |     refAst: Fragment
11 |     refFullPath: string
12 |   }
13 | }
14 | 
15 | export async function compile(
16 |   props: CompileNodeContext<ReferenceTag>,
17 |   attributes: Record<string, unknown>,
18 | ) {
19 |   const {
20 |     node,
21 |     scope,
22 |     fullPath,
23 |     referencePromptFn,
24 |     baseNodeError,
25 |     resolveBaseNode,
26 |   } = props
27 | 
28 |   const nodeWithStatus = node as ForNodeWithStatus
29 |   nodeWithStatus.status = {
30 |     ...nodeWithStatus.status,
31 |     scopePointers: scope.getPointers(),
32 |   }
33 | 
34 |   const { path, ...refParameters } = attributes
35 |   if (!path) baseNodeError(errors.referenceTagWithoutPath, node)
36 |   if (typeof path !== 'string') baseNodeError(errors.invalidReferencePath, node)
37 | 
38 |   if (!nodeWithStatus.status.refAst || !nodeWithStatus.status.refFullPath) {
39 |     if (!referencePromptFn) baseNodeError(errors.missingReferenceFunction, node)
40 | 
41 |     const prompt = await referencePromptFn!(path as string, fullPath)
42 |     if (!prompt) baseNodeError(errors.referenceNotFound, node)
43 |     const ast = parse(prompt!.content)
44 |     nodeWithStatus.status.refAst = ast
45 |     nodeWithStatus.status.refFullPath = prompt!.path
46 |   }
47 | 
48 |   const refScope = new Scope(refParameters)
49 |   await resolveBaseNode({
50 |     ...props,
51 |     node: nodeWithStatus.status.refAst,
52 |     scope: refScope,
53 |     fullPath: nodeWithStatus.status.refFullPath,
54 |   })
55 | }
56 | 


--------------------------------------------------------------------------------
/src/compiler/logic/nodes/callExpression.ts:
--------------------------------------------------------------------------------
 1 | import type {
 2 |   ResolveNodeProps,
 3 |   UpdateScopeContextProps,
 4 | } from '$promptl/compiler/logic/types'
 5 | import CompileError from '$promptl/error/error'
 6 | import errors from '$promptl/error/errors'
 7 | import type { SimpleCallExpression } from 'estree'
 8 | 
 9 | import { resolveLogicNode, updateScopeContextForNode } from '..'
10 | 
11 | /**
12 |  * ### CallExpression
13 |  * Represents a method call.
14 |  *
15 |  * Examples: `foo()` `foo.bar()`
16 |  */
17 | export async function resolve(props: ResolveNodeProps<SimpleCallExpression>) {
18 |   const { node, raiseError } = props
19 |   const method = (await resolveLogicNode({
20 |     ...props,
21 |     node: node.callee,
22 |   })) as Function
23 | 
24 |   if (typeof method !== 'function') {
25 |     raiseError(errors.notAFunction(typeof method), node)
26 |   }
27 | 
28 |   const args = await resolveArgs(props)
29 |   return await runMethod({ ...props, method, args })
30 | }
31 | 
32 | function resolveArgs(
33 |   props: ResolveNodeProps<SimpleCallExpression>,
34 | ): Promise<unknown[]> {
35 |   const { node } = props
36 |   return Promise.all(
37 |     node.arguments.map((arg) =>
38 |       resolveLogicNode({
39 |         ...props,
40 |         node: arg,
41 |       }),
42 |     ),
43 |   )
44 | }
45 | 
46 | async function runMethod({
47 |   method,
48 |   args,
49 |   node,
50 |   raiseError,
51 | }: ResolveNodeProps<SimpleCallExpression> & {
52 |   method: Function
53 |   args: unknown[]
54 | }) {
55 |   try {
56 |     return await method(...args)
57 |   } catch (error: unknown) {
58 |     if (error instanceof CompileError) throw error
59 |     raiseError(errors.functionCallError(error), node)
60 |   }
61 | }
62 | 
63 | export function updateScopeContext({
64 |   node,
65 |   ...props
66 | }: UpdateScopeContextProps<SimpleCallExpression>) {
67 |   updateScopeContextForNode({ node: node.callee, ...props })
68 |   for (const arg of node.arguments) {
69 |     updateScopeContextForNode({ node: arg, ...props })
70 |   }
71 | }
72 | 


--------------------------------------------------------------------------------
/src/providers/openai-responses/types.ts:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * OpenAI API reference for messages
 3 |  * Reference for https://api.openai.com/v1/responses
 4 |  * Source: https://platform.openai.com/docs/api-reference/responses
 5 |  */
 6 | 
 7 | import type {
 8 |   EasyInputMessage,
 9 |   ResponseCreateParamsBase,
10 |   ResponseFunctionToolCall,
11 |   ResponseInputContent,
12 |   ResponseInputItem,
13 |   ResponseOutputMessage as OpenAIResponseOutputMessage,
14 |   ResponseOutputText as OpenAIResponseOutputText,
15 |   ResponseOutputRefusal as OpenAIResponseOutputRefusal,
16 |   ResponseReasoningItem as OpenAIResponseReasoningItem,
17 |   ResponseFunctionWebSearch,
18 |   ResponseFileSearchToolCall,
19 | } from 'openai/resources/responses/responses'
20 | 
21 | export type MessageInputItem = ResponseInputItem
22 | export type MessageRole = EasyInputMessage['role']
23 | export type MessageContentSimple = ResponseInputContent
24 | export type MessageContentInputItem = ResponseInputItem.Message['content']
25 | 
26 | // Output message
27 | export type ResponseOutputMessage = OpenAIResponseOutputMessage
28 | export type ResponseOutputText = OpenAIResponseOutputText
29 | export type ResponseOutputRefusal = OpenAIResponseOutputRefusal
30 | 
31 | // Input Message
32 | export type ResponseInputMessage = ResponseInputItem.Message
33 | 
34 | // Function Call
35 | export type ToolCallRequest = ResponseFunctionToolCall
36 | export type ToolCallResponse = ResponseInputItem.FunctionCallOutput
37 | 
38 | // Reasoning
39 | export type ResponseReasoningItem = OpenAIResponseReasoningItem
40 | 
41 | // Simple Input
42 | export type SimpleInputMessage = EasyInputMessage
43 | 
44 | // Web Search
45 | export type WebSearchCall = ResponseFunctionWebSearch
46 | 
47 | // File Search
48 | export type FileSearchCall = ResponseFileSearchToolCall
49 | 
50 | export type MessageContent =
51 |   | EasyInputMessage['content']
52 |   | ResponseInputItem.Message['content']
53 |   | ResponseOutputMessage['content']
54 | export type Message = ResponseCreateParamsBase['input']
55 | 


--------------------------------------------------------------------------------
/rollup.config.mjs:
--------------------------------------------------------------------------------
 1 | import * as path from 'path'
 2 | import * as url from 'url'
 3 | 
 4 | import alias from '@rollup/plugin-alias'
 5 | import typescript from '@rollup/plugin-typescript'
 6 | import { dts } from 'rollup-plugin-dts'
 7 | import { nodeResolve } from '@rollup/plugin-node-resolve'
 8 | 
 9 | /**
10 |  * We have a internal circular dependency in the compiler,
11 |  * which is intentional. We think in this case Rollup is too noisy.
12 |  *
13 |  * @param {import('rollup').RollupLog} warning
14 |  * @returns {boolean}
15 |  */
16 | function isInternalCircularDependency(warning) {
17 |   return (
18 |     warning.code == 'CIRCULAR_DEPENDENCY' &&
19 |     warning.message.includes('src/compiler') &&
20 |     !warning.message.includes('node_modules')
21 |   )
22 | }
23 | 
24 | const __dirname = path.dirname(url.fileURLToPath(import.meta.url))
25 | const aliasEntries = {
26 |   entries: [{ find: '$promptl', replacement: path.resolve(__dirname, 'src') }],
27 | }
28 | 
29 | /** @type {import('rollup').RollupOptions} */
30 | export default [
31 |   {
32 |     onwarn: (warning, warn) => {
33 |       if (isInternalCircularDependency(warning)) return
34 | 
35 |       warn(warning)
36 |     },
37 |     input: 'src/index.ts',
38 |     output: [
39 |       { file: 'dist/index.js' },
40 |       { file: 'dist/index.cjs', format: 'cjs' },
41 |     ],
42 |     plugins: [
43 |       nodeResolve(),
44 |       typescript({
45 |         noEmit: true,
46 |         tsconfig: './tsconfig.json',
47 |         exclude: ['**/__tests__', '**/*.test.ts'],
48 |       }),
49 |     ],
50 |     external: [
51 |       'openai',
52 |       'acorn',
53 |       'node:crypto',
54 |       'yaml',
55 |       'crypto',
56 |       'zod',
57 |       'fast-sha256',
58 |     ],
59 |   },
60 |   {
61 |     input: 'src/index.ts',
62 |     output: [{ file: 'dist/index.d.ts', format: 'es' }],
63 |     plugins: [
64 |       alias(aliasEntries),
65 |       dts({
66 |         tsconfig: './tsconfig.json',
67 |         exclude: ['**/__tests__', '**/*.test.ts'],
68 |       }),
69 |     ],
70 |   },
71 | ]
72 | 


--------------------------------------------------------------------------------
/src/providers/openai/types.ts:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * OpenAI API reference for messages
 3 |  * Reference for https://api.openai.com/v1/chat/completions
 4 |  * Source: https://platform.openai.com/docs/api-reference/chat/create#chat-create-messages
 5 |  */
 6 | 
 7 | import { MessageRole } from '$promptl/types'
 8 | 
 9 | export enum ContentType {
10 |   text = 'text',
11 |   image = 'image',
12 |   input_audio = 'input_audio',
13 | }
14 | 
15 | interface IMessageContent {
16 |   type: ContentType
17 |   [key: string]: unknown
18 | }
19 | 
20 | export type TextContent = IMessageContent & {
21 |   type: ContentType.text
22 |   text: string
23 | }
24 | 
25 | export type ImageContent = IMessageContent & {
26 |   type: ContentType.image
27 |   image: string | Uint8Array | Buffer | ArrayBuffer | URL
28 | }
29 | 
30 | export type AudioContent = IMessageContent & {
31 |   type: ContentType.input_audio
32 |   data: string
33 |   format: string
34 | }
35 | 
36 | export type MessageContent = TextContent | ImageContent | AudioContent
37 | 
38 | export type ToolCall = {
39 |   id: string
40 |   type: 'function'
41 |   function: {
42 |     name: string
43 |     arguments: string
44 |   }
45 | }
46 | 
47 | interface IMessage {
48 |   role: MessageRole
49 |   content: MessageContent[]
50 | }
51 | 
52 | export type SystemMessage = IMessage & {
53 |   role: MessageRole.system
54 |   name?: string
55 |   content: string | TextContent[]
56 | }
57 | 
58 | export type UserMessage = IMessage & {
59 |   role: MessageRole.user
60 |   name?: string
61 |   content: string | MessageContent[]
62 | }
63 | 
64 | export type AssistantMessage = IMessage & {
65 |   role: MessageRole.assistant
66 |   content: string
67 |   refusal?: string
68 |   name?: string
69 |   audio?: { id: string }
70 |   tool_calls?: ToolCall[]
71 | }
72 | 
73 | export type ToolMessage = IMessage & {
74 |   role: MessageRole.tool
75 |   tool_call_id: string
76 |   content: string | MessageContent[]
77 | }
78 | 
79 | export type Message =
80 |   | SystemMessage
81 |   | UserMessage
82 |   | AssistantMessage
83 |   | ToolMessage
84 | 


--------------------------------------------------------------------------------
/src/compiler/index.ts:
--------------------------------------------------------------------------------
 1 | import {
 2 |   AdapterMessageType,
 3 |   Adapters,
 4 |   ProviderAdapter,
 5 | } from '$promptl/providers'
 6 | import { ProviderConversation } from '$promptl/providers/adapter'
 7 | import { ConversationMetadata, Message } from '$promptl/types'
 8 | 
 9 | import { Chain } from './chain'
10 | import { Scan, ScanSchema } from './scan'
11 | import type { CompileOptions, Document, ReferencePromptFn } from './types'
12 | import { Fragment } from '$promptl/parser/interfaces'
13 | 
14 | export async function render<M extends AdapterMessageType = Message>({
15 |   prompt,
16 |   parameters = {},
17 |   adapter = Adapters.openai as ProviderAdapter<M>,
18 |   ...compileOptions
19 | }: {
20 |   prompt: string
21 |   parameters?: Record<string, unknown>
22 |   adapter?: ProviderAdapter<M>
23 | } & CompileOptions): Promise<ProviderConversation<M>> {
24 |   const iterator = new Chain({ prompt, parameters, adapter, ...compileOptions })
25 |   const { messages, config } = await iterator.step()
26 |   return { messages, config }
27 | }
28 | 
29 | export function createChain({
30 |   prompt,
31 |   parameters,
32 |   ...compileOptions
33 | }: {
34 |   prompt: string
35 |   parameters: Record<string, unknown>
36 | } & CompileOptions): Chain {
37 |   return new Chain({ prompt, parameters, ...compileOptions })
38 | }
39 | 
40 | export function scan({
41 |   prompt,
42 |   serialized,
43 |   fullPath,
44 |   referenceFn,
45 |   withParameters,
46 |   configSchema,
47 |   requireConfig,
48 | }: {
49 |   prompt: string
50 |   serialized?: Fragment
51 |   fullPath?: string
52 |   referenceFn?: ReferencePromptFn
53 |   withParameters?: string[]
54 |   configSchema?: ScanSchema
55 |   requireConfig?: boolean
56 | }): Promise<ConversationMetadata> {
57 |   return new Scan({
58 |     document: { path: fullPath ?? '', content: prompt },
59 |     serialized,
60 |     referenceFn,
61 |     withParameters,
62 |     configSchema,
63 |     requireConfig,
64 |   }).run()
65 | }
66 | 
67 | type SerializedChain = ReturnType<Chain['serialize']>
68 | 
69 | export { Chain, type SerializedChain, type Document, type ReferencePromptFn }
70 | 


--------------------------------------------------------------------------------
/src/providers/anthropic/types.ts:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * Anthropic API reference for messages
 3 |  * Reference for https://api.anthropic.com/v1/messages
 4 |  * Source: https://docs.anthropic.com/en/api/messages
 5 |  */
 6 | 
 7 | import { MessageRole as PromptlMessageRole } from '$promptl/types'
 8 | 
 9 | export enum MessageRole {
10 |   user = PromptlMessageRole.user,
11 |   assistant = PromptlMessageRole.assistant,
12 | }
13 | 
14 | export enum ContentType {
15 |   text = 'text',
16 |   image = 'image',
17 |   tool_use = 'tool_use',
18 |   tool_result = 'tool_result',
19 |   document = 'document',
20 | }
21 | 
22 | interface IMessageContent {
23 |   type: ContentType
24 |   cache_control?: {
25 |     type: string
26 |   }
27 |   [key: string]: unknown
28 | }
29 | 
30 | export type TextContent = IMessageContent & {
31 |   type: ContentType.text
32 |   text: string
33 | }
34 | 
35 | export type ImageContent = IMessageContent & {
36 |   type: ContentType.image
37 |   source: {
38 |     type: string
39 |     media_type: string
40 |     data: string
41 |   }
42 | }
43 | 
44 | export type ToolUseContent = IMessageContent & {
45 |   type: ContentType.tool_use
46 |   id: string
47 |   name: string
48 |   input: object
49 | }
50 | 
51 | export type ToolResultContent = IMessageContent & {
52 |   type: ContentType.tool_result
53 |   tool_use_id: string
54 |   is_error?: boolean
55 |   content?: string | (TextContent | ImageContent)[]
56 | }
57 | 
58 | export type DocumentContent = IMessageContent & {
59 |   type: ContentType.document
60 |   source: {
61 |     type: string
62 |     media_type: string
63 |     data: string
64 |   }
65 | }
66 | 
67 | export type MessageContent =
68 |   | TextContent
69 |   | ImageContent
70 |   | DocumentContent
71 |   | ToolUseContent
72 |   | ToolResultContent
73 | 
74 | interface IMessage {
75 |   role: MessageRole
76 |   content: string | MessageContent[]
77 | }
78 | 
79 | export type UserMessage = IMessage & {
80 |   role: MessageRole.user
81 | }
82 | 
83 | export type AssistantMessage = IMessage & {
84 |   role: MessageRole.assistant
85 | }
86 | 
87 | export type Message = UserMessage | AssistantMessage
88 | 


--------------------------------------------------------------------------------
/src/compiler/logic/nodes/objectExpression.ts:
--------------------------------------------------------------------------------
 1 | import {
 2 |   resolveLogicNode,
 3 |   updateScopeContextForNode,
 4 | } from '$promptl/compiler/logic'
 5 | import {
 6 |   UpdateScopeContextProps,
 7 |   type ResolveNodeProps,
 8 | } from '$promptl/compiler/logic/types'
 9 | import errors from '$promptl/error/errors'
10 | import { type Identifier, type ObjectExpression } from 'estree'
11 | 
12 | /**
13 |  * ### ObjectExpression
14 |  * Represents a javascript Object
15 |  */
16 | export async function resolve({
17 |   node,
18 |   scope,
19 |   raiseError,
20 |   ...props
21 | }: ResolveNodeProps<ObjectExpression>) {
22 |   const resolvedObject: { [key: string]: any } = {}
23 |   for (const prop of node.properties) {
24 |     if (prop.type === 'SpreadElement') {
25 |       const spreadObject = await resolveLogicNode({
26 |         node: prop.argument,
27 |         scope,
28 |         raiseError,
29 |         ...props,
30 |       })
31 |       if (typeof spreadObject !== 'object') {
32 |         raiseError(errors.invalidSpreadInObject(typeof spreadObject), prop)
33 |       }
34 |       Object.entries(spreadObject as object).forEach(([key, value]) => {
35 |         resolvedObject[key] = value
36 |       })
37 |       continue
38 |     }
39 |     if (prop.type === 'Property') {
40 |       const key = prop.key as Identifier
41 |       const value = await resolveLogicNode({
42 |         node: prop.value,
43 |         scope,
44 |         raiseError,
45 |         ...props,
46 |       })
47 |       resolvedObject[key.name] = value
48 |       continue
49 |     }
50 |     throw raiseError(errors.invalidObjectKey, prop)
51 |   }
52 |   return resolvedObject
53 | }
54 | 
55 | export function updateScopeContext({
56 |   node,
57 |   ...props
58 | }: UpdateScopeContextProps<ObjectExpression>) {
59 |   for (const prop of node.properties) {
60 |     if (prop.type === 'SpreadElement') {
61 |       updateScopeContextForNode({ node: prop.argument, ...props })
62 |       continue
63 |     }
64 |     if (prop.type === 'Property') {
65 |       updateScopeContextForNode({ node: prop.value, ...props })
66 |       continue
67 |     }
68 |     props.raiseError(errors.invalidObjectKey, prop)
69 |   }
70 | }
71 | 


--------------------------------------------------------------------------------
/src/providers/vercel-ai/types.ts:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * Vercel AI SDK message types
 3 |  * Based on the AI SDK Core message structure
 4 |  */
 5 | 
 6 | import { MessageRole } from '$promptl/types'
 7 | 
 8 | export enum ContentType {
 9 |   text = 'text',
10 |   image = 'image',
11 |   file = 'file',
12 |   toolCall = 'tool-call',
13 |   toolResult = 'tool-result',
14 |   reasoning = 'reasoning',
15 | }
16 | 
17 | interface IMessageContent {
18 |   type: ContentType
19 |   [key: string]: unknown
20 | }
21 | 
22 | export type TextPart = IMessageContent & {
23 |   type: ContentType.text
24 |   text: string
25 | }
26 | 
27 | export type ImagePart = IMessageContent & {
28 |   type: ContentType.image
29 |   image: string | Uint8Array | Buffer | ArrayBuffer | URL
30 |   mediaType?: string
31 | }
32 | 
33 | export type FilePart = IMessageContent & {
34 |   type: ContentType.file
35 |   data: string | Uint8Array | Buffer | ArrayBuffer | URL
36 |   mediaType: string
37 |   filename?: string
38 | }
39 | 
40 | export type ToolCallPart = IMessageContent & {
41 |   type: ContentType.toolCall
42 |   toolCallId: string
43 |   toolName: string
44 |   args: Record<string, unknown>
45 | }
46 | 
47 | export type ToolResultPart = IMessageContent & {
48 |   type: ContentType.toolResult
49 |   toolCallId: string
50 |   toolName: string
51 |   result: unknown
52 |   isError?: boolean
53 | }
54 | 
55 | export type ReasoningPart = IMessageContent & {
56 |   type: ContentType.reasoning
57 |   text: string
58 | }
59 | 
60 | export type MessageContent =
61 |   | TextPart
62 |   | ImagePart
63 |   | FilePart
64 |   | ToolCallPart
65 |   | ToolResultPart
66 |   | ReasoningPart
67 | 
68 | export type SystemMessage = {
69 |   role: MessageRole.system
70 |   content: string
71 | }
72 | 
73 | export type UserMessage = {
74 |   role: MessageRole.user
75 |   content: string | MessageContent[]
76 | }
77 | 
78 | export type AssistantMessage = {
79 |   role: MessageRole.assistant
80 |   content: string | MessageContent[]
81 | }
82 | 
83 | export type ToolMessage = {
84 |   role: MessageRole.tool
85 |   content: MessageContent[]
86 | }
87 | 
88 | export type Message =
89 |   | SystemMessage
90 |   | UserMessage
91 |   | AssistantMessage
92 |   | ToolMessage
93 | 


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "promptl-ai",
 3 |   "version": "0.9.4",
 4 |   "author": "Latitude Data",
 5 |   "license": "MIT",
 6 |   "description": "Compiler for PromptL, the prompt language",
 7 |   "type": "module",
 8 |   "main": "./dist/index.cjs",
 9 |   "module": "./dist/index.js",
10 |   "types": "./dist/index.d.ts",
11 |   "exports": {
12 |     ".": {
13 |       "import": {
14 |         "types": "./dist/index.d.ts",
15 |         "default": "./dist/index.js"
16 |       },
17 |       "require": {
18 |         "types": "./dist/index.d.ts",
19 |         "default": "./dist/index.cjs"
20 |       }
21 |     }
22 |   },
23 |   "files": [
24 |     "dist"
25 |   ],
26 |   "scripts": {
27 |     "prepare": "npm run build",
28 |     "dev": "rollup -c -w",
29 |     "build": "rollup -c",
30 |     "build:lib": "rollup -c rollup.config.mjs",
31 |     "build:rpc": "rollup -c rollup.config.rpc.mjs",
32 |     "test": "vitest run",
33 |     "test:watch": "vitest",
34 |     "prettier": "prettier --write src/**/*.ts",
35 |     "prettier:check": "prettier --check src/**/*.ts --ignore-path .prettierrcignore",
36 |     "lint": "eslint src",
37 |     "tc": "tsc --noEmit"
38 |   },
39 |   "dependencies": {
40 |     "acorn": "^8.9.0",
41 |     "code-red": "^1.0.3",
42 |     "fast-sha256": "^1.3.0",
43 |     "locate-character": "^3.0.0",
44 |     "openai": "^4.98.0",
45 |     "yaml": "^2.4.5",
46 |     "zod": "^4.1.8"
47 |   },
48 |   "devDependencies": {
49 |     "@eslint/eslintrc": "^3.2.0",
50 |     "@eslint/js": "^9.17.0",
51 |     "@rollup/plugin-alias": "^5.1.0",
52 |     "@rollup/plugin-commonjs": "^25.0.7",
53 |     "@rollup/plugin-node-resolve": "^15.3.1",
54 |     "@rollup/plugin-typescript": "^11.1.6",
55 |     "@types/estree": "^1.0.1",
56 |     "@types/node": "^20.12.12",
57 |     "@typescript-eslint/eslint-plugin": "^8.19.0",
58 |     "eslint": "^9.17.0",
59 |     "eslint-plugin-prettier": "^5.2.1",
60 |     "globals": "^15.14.0",
61 |     "prettier": "^3.4.2",
62 |     "rollup": "^4.10.0",
63 |     "rollup-plugin-dts": "^6.1.1",
64 |     "rollup-plugin-execute": "^1.1.1",
65 |     "tslib": "^2.8.1",
66 |     "tsx": "^4.19.2",
67 |     "typescript": "^5.6.3",
68 |     "vitest": "^1.2.2"
69 |   },
70 |   "packageManager": "pnpm@9.8.0"
71 | }
72 | 


--------------------------------------------------------------------------------
/src/providers/utils/getMimeType.ts:
--------------------------------------------------------------------------------
 1 | const DEFAULT_MIME_TYPE = 'application/octet-stream'
 2 | export function getMimeType(filename: string | undefined): string {
 3 |   if (!filename) return DEFAULT_MIME_TYPE
 4 | 
 5 |   const extension = filename.split('.').pop()?.toLowerCase()
 6 | 
 7 |   const mimeTypes: Record<string, string> = {
 8 |     // Text
 9 |     txt: 'text/plain',
10 |     md: 'text/markdown',
11 |     html: 'text/html',
12 |     htm: 'text/html',
13 |     css: 'text/css',
14 |     csv: 'text/csv',
15 |     tsv: 'text/tab-separated-values',
16 |     xml: 'application/xml',
17 |     json: 'application/json',
18 |     js: 'application/javascript',
19 |     ts: 'application/typescript',
20 | 
21 |     // Images
22 |     jpg: 'image/jpeg',
23 |     jpeg: 'image/jpeg',
24 |     png: 'image/png',
25 |     gif: 'image/gif',
26 |     bmp: 'image/bmp',
27 |     webp: 'image/webp',
28 |     svg: 'image/svg+xml',
29 |     ico: 'image/vnd.microsoft.icon',
30 | 
31 |     // Audio
32 |     mp3: 'audio/mpeg',
33 |     wav: 'audio/wav',
34 |     ogg: 'audio/ogg',
35 |     flac: 'audio/flac',
36 |     m4a: 'audio/mp4',
37 | 
38 |     // Video
39 |     mp4: 'video/mp4',
40 |     mov: 'video/quicktime',
41 |     avi: 'video/x-msvideo',
42 |     webm: 'video/webm',
43 |     mkv: 'video/x-matroska',
44 | 
45 |     // Fonts
46 |     ttf: 'font/ttf',
47 |     otf: 'font/otf',
48 |     woff: 'font/woff',
49 |     woff2: 'font/woff2',
50 | 
51 |     // Documents
52 |     pdf: 'application/pdf',
53 |     doc: 'application/msword',
54 |     docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
55 |     xls: 'application/vnd.ms-excel',
56 |     xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
57 |     ppt: 'application/vnd.ms-powerpoint',
58 |     pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
59 | 
60 |     // Archives
61 |     zip: 'application/zip',
62 |     tar: 'application/x-tar',
63 |     gz: 'application/gzip',
64 |     rar: 'application/vnd.rar',
65 |     '7z': 'application/x-7z-compressed',
66 | 
67 |     // Others
68 |     apk: 'application/vnd.android.package-archive',
69 |     exe: 'application/vnd.microsoft.portable-executable',
70 |   }
71 | 
72 |   return mimeTypes[extension || ''] ?? DEFAULT_MIME_TYPE
73 | }
74 | 


--------------------------------------------------------------------------------
/src/parser/interfaces.ts:
--------------------------------------------------------------------------------
 1 | import { TAG_NAMES } from '$promptl/constants'
 2 | import { ContentTypeTagName, MessageRole } from '$promptl/types'
 3 | import { Identifier, type Node as LogicalExpression } from 'estree'
 4 | 
 5 | export type BaseNode = {
 6 |   start: number | null
 7 |   end: number | null
 8 |   type: string
 9 |   children?: TemplateNode[]
10 |   [propName: string]: any
11 | }
12 | 
13 | export type Fragment = BaseNode & {
14 |   type: 'Fragment'
15 |   children: TemplateNode[]
16 | }
17 | 
18 | export type Config = BaseNode & {
19 |   type: 'Config'
20 |   value: string
21 | }
22 | 
23 | export type Text = BaseNode & {
24 |   type: 'Text'
25 |   data: string
26 | }
27 | 
28 | export type Attribute = BaseNode & {
29 |   type: 'Attribute'
30 |   name: string
31 |   value: TemplateNode[] | true
32 | }
33 | 
34 | type IElementTag<T extends string> = BaseNode & {
35 |   type: 'ElementTag'
36 |   name: T
37 |   attributes: Attribute[]
38 |   children: TemplateNode[]
39 | }
40 | 
41 | export type MessageTag =
42 |   | IElementTag<MessageRole>
43 |   | IElementTag<typeof TAG_NAMES.message>
44 | export type ContentTag =
45 |   | IElementTag<ContentTypeTagName>
46 |   | IElementTag<typeof TAG_NAMES.content>
47 | 
48 | export type ReferenceTag = IElementTag<typeof TAG_NAMES.prompt>
49 | export type ScopeTag = IElementTag<typeof TAG_NAMES.scope>
50 | export type ChainStepTag = IElementTag<typeof TAG_NAMES.step>
51 | export type ElementTag =
52 |   | ContentTag
53 |   | MessageTag
54 |   | ReferenceTag
55 |   | ScopeTag
56 |   | ChainStepTag
57 |   | IElementTag<string>
58 | 
59 | export type MustacheTag = BaseNode & {
60 |   type: 'MustacheTag'
61 |   expression: LogicalExpression
62 | }
63 | 
64 | export type Comment = BaseNode & {
65 |   type: 'Comment'
66 |   data: string
67 | }
68 | 
69 | export type ElseBlock = BaseNode & {
70 |   type: 'ElseBlock'
71 | }
72 | 
73 | export type IfBlock = BaseNode & {
74 |   type: 'IfBlock'
75 |   expression: LogicalExpression
76 |   else: ElseBlock | null
77 | }
78 | 
79 | export type ForBlock = BaseNode & {
80 |   type: 'ForBlock'
81 |   expression: LogicalExpression
82 |   context: Identifier
83 |   index: Identifier | null
84 |   else: ElseBlock | null
85 | }
86 | 
87 | export type TemplateNode =
88 |   | Fragment
89 |   | Config
90 |   | Text
91 |   | ElementTag
92 |   | MustacheTag
93 |   | Comment
94 |   | IfBlock
95 |   | ForBlock
96 | 


--------------------------------------------------------------------------------
/src/compiler/deserializeChain.ts:
--------------------------------------------------------------------------------
 1 | import { SerializedChain } from '$promptl/compiler'
 2 | import { Chain } from '$promptl/compiler/chain'
 3 | import Scope from '$promptl/compiler/scope'
 4 | import { getAdapter } from '$promptl/providers'
 5 | 
 6 | function safeSerializedData(data: string | SerializedChain): SerializedChain {
 7 |   try {
 8 |     const serialized =
 9 |       typeof data === 'string'
10 |         ? JSON.parse(data)
11 |         : typeof data === 'object'
12 |           ? data
13 |           : {}
14 | 
15 |     const compilerOptions = serialized.compilerOptions || {}
16 |     const globalConfig = serialized.globalConfig
17 |     const globalMessages = serialized.globalMessages || []
18 | 
19 |     if (
20 |       typeof serialized !== 'object' ||
21 |       typeof serialized.ast !== 'object' ||
22 |       typeof serialized.scope !== 'object' ||
23 |       typeof serialized.didStart !== 'boolean' ||
24 |       typeof serialized.completed !== 'boolean' ||
25 |       typeof serialized.adapterType !== 'string' ||
26 |       typeof serialized.rawText !== 'string'
27 |     ) {
28 |       throw new Error()
29 |     }
30 |     return {
31 |       rawText: serialized.rawText,
32 |       ast: serialized.ast,
33 |       scope: serialized.scope,
34 |       didStart: serialized.didStart,
35 |       completed: serialized.completed,
36 |       adapterType: serialized.adapterType,
37 |       compilerOptions,
38 |       globalConfig,
39 |       globalMessages,
40 |     }
41 |   } catch {
42 |     throw new Error('Invalid serialized chain data')
43 |   }
44 | }
45 | 
46 | export type SerializedProps = {
47 |   serialized: string | SerializedChain | undefined | null
48 | }
49 | export function deserializeChain({
50 |   serialized,
51 | }: SerializedProps): Chain | undefined {
52 |   if (!serialized) return undefined
53 | 
54 |   const {
55 |     rawText,
56 |     ast,
57 |     scope: serializedScope,
58 |     didStart,
59 |     completed,
60 |     adapterType,
61 |     compilerOptions,
62 |     globalConfig,
63 |     globalMessages,
64 |   } = safeSerializedData(serialized)
65 | 
66 |   const adapter = getAdapter(adapterType)
67 |   const scope = new Scope()
68 |   scope.setStash(serializedScope.stash)
69 |   scope.setPointers(serializedScope.pointers)
70 | 
71 |   return new Chain({
72 |     prompt: rawText,
73 |     serialized: {
74 |       ast,
75 |       scope,
76 |       didStart,
77 |       completed,
78 |       globalConfig,
79 |       globalMessages,
80 |     },
81 |     adapter,
82 |     ...compilerOptions,
83 |   })
84 | }
85 | 


--------------------------------------------------------------------------------
/src/types/message.ts:
--------------------------------------------------------------------------------
  1 | /* Message Content */
  2 | 
  3 | export enum ContentType {
  4 |   text = 'text',
  5 |   image = 'image',
  6 |   file = 'file',
  7 |   toolCall = 'tool-call',
  8 | }
  9 | 
 10 | export enum ContentTypeTagName {
 11 |   // This is used to translate between the tag name and the actual tag value
 12 |   text = 'content-text',
 13 |   image = 'content-image',
 14 |   file = 'content-file',
 15 |   toolCall = 'tool-call',
 16 | }
 17 | 
 18 | export type PromptlSourceRef = {
 19 |   start: number
 20 |   end: number
 21 |   identifier?: string
 22 | }
 23 | 
 24 | interface IMessageContent {
 25 |   type: ContentType
 26 |   _promptlSourceMap?: PromptlSourceRef[]
 27 |   [key: string]: unknown
 28 | }
 29 | 
 30 | export type TextContent = IMessageContent & {
 31 |   type: ContentType.text
 32 |   text: string
 33 | }
 34 | 
 35 | export type ImageContent = IMessageContent & {
 36 |   type: ContentType.image
 37 |   image: string | Uint8Array | Buffer | ArrayBuffer | URL
 38 | }
 39 | 
 40 | export type FileContent = IMessageContent & {
 41 |   type: ContentType.file
 42 |   file: string | Uint8Array | Buffer | ArrayBuffer | URL
 43 |   mimeType: string
 44 | }
 45 | 
 46 | export type ToolCallContent = {
 47 |   type: ContentType.toolCall
 48 |   toolCallId: string
 49 |   toolName: string
 50 |   toolArguments: Record<string, unknown>
 51 | }
 52 | 
 53 | export type MessageContent =
 54 |   | TextContent
 55 |   | ImageContent
 56 |   | FileContent
 57 |   | ToolCallContent
 58 | 
 59 | /* Message */
 60 | 
 61 | export enum MessageRole {
 62 |   system = 'system',
 63 |   developer = 'developer',
 64 |   user = 'user',
 65 |   assistant = 'assistant',
 66 |   tool = 'tool',
 67 | }
 68 | 
 69 | interface IMessage {
 70 |   role: MessageRole
 71 |   content: MessageContent[]
 72 |   [key: string]: unknown
 73 | }
 74 | 
 75 | export type SystemMessage = IMessage & {
 76 |   role: MessageRole.system
 77 | }
 78 | 
 79 | export type DeveloperMessage = IMessage & {
 80 |   role: MessageRole.developer
 81 | }
 82 | 
 83 | export type UserMessage = IMessage & {
 84 |   role: MessageRole.user
 85 |   name?: string
 86 | }
 87 | 
 88 | export type AssistantMessage = IMessage & {
 89 |   role: MessageRole.assistant
 90 | }
 91 | 
 92 | export type ToolMessage = IMessage & {
 93 |   role: MessageRole.tool
 94 |   toolName: string
 95 |   toolId: string
 96 | }
 97 | 
 98 | export type Message =
 99 |   | SystemMessage
100 |   | DeveloperMessage
101 |   | UserMessage
102 |   | AssistantMessage
103 |   | ToolMessage
104 | 


--------------------------------------------------------------------------------
/src/compiler/logic/nodes/updateExpression.ts:
--------------------------------------------------------------------------------
 1 | import {
 2 |   resolveLogicNode,
 3 |   updateScopeContextForNode,
 4 | } from '$promptl/compiler/logic'
 5 | import type {
 6 |   ResolveNodeProps,
 7 |   UpdateScopeContextProps,
 8 | } from '$promptl/compiler/logic/types'
 9 | import errors from '$promptl/error/errors'
10 | import type { AssignmentExpression, UpdateExpression } from 'estree'
11 | 
12 | /**
13 |  * ### UpdateExpression
14 |  * Represents a javascript update expression.
15 |  * Depending on the operator, it can increment or decrement a value.
16 |  * Depending on the position of the operator, the return value can be resolved before or after the operation.
17 |  *
18 |  * Examples: `{--foo}` `{bar++}`
19 |  */
20 | export async function resolve({
21 |   node,
22 |   scope,
23 |   raiseError,
24 |   ...props
25 | }: ResolveNodeProps<UpdateExpression>) {
26 |   const updateOperator = node.operator
27 | 
28 |   if (!['++', '--'].includes(updateOperator)) {
29 |     raiseError(errors.unsupportedOperator(updateOperator), node)
30 |   }
31 | 
32 |   const assignmentOperators = {
33 |     '++': '+=',
34 |     '--': '-=',
35 |   }
36 | 
37 |   const originalValue = await resolveLogicNode({
38 |     node: node.argument,
39 |     scope,
40 |     raiseError,
41 |     ...props,
42 |   })
43 | 
44 |   if (typeof originalValue !== 'number') {
45 |     raiseError(errors.invalidUpdate(updateOperator, typeof originalValue), node)
46 |   }
47 | 
48 |   // Simulate an AssignmentExpression with the same operation
49 |   const assignmentNode = {
50 |     ...node,
51 |     type: 'AssignmentExpression',
52 |     left: node.argument,
53 |     operator: assignmentOperators[updateOperator],
54 |     right: {
55 |       type: 'Literal',
56 |       value: 1,
57 |     },
58 |   } as AssignmentExpression
59 | 
60 |   // Perform the assignment
61 |   await resolveLogicNode({
62 |     node: assignmentNode,
63 |     scope,
64 |     raiseError,
65 |     ...props,
66 |   })
67 | 
68 |   const updatedValue = await resolveLogicNode({
69 |     node: node.argument,
70 |     scope,
71 |     raiseError,
72 |     ...props,
73 |   })
74 | 
75 |   return node.prefix ? updatedValue : originalValue
76 | }
77 | 
78 | export function updateScopeContext({
79 |   node,
80 |   ...props
81 | }: UpdateScopeContextProps<UpdateExpression>) {
82 |   const updateOperator = node.operator
83 |   if (!['++', '--'].includes(updateOperator)) {
84 |     props.raiseError(errors.unsupportedOperator(updateOperator), node)
85 |   }
86 | 
87 |   updateScopeContextForNode({ node: node.argument, ...props })
88 | }
89 | 


--------------------------------------------------------------------------------
/.github/workflows/publish.yml:
--------------------------------------------------------------------------------
 1 | name: Publish
 2 | 
 3 | on:
 4 |   push:
 5 |     branches:
 6 |       - main
 7 | 
 8 | jobs:
 9 |   publish:
10 |     name: Build and Publish
11 |     runs-on: ubuntu-latest
12 | 
13 |     steps:
14 |       - name: Checkout repository
15 |         uses: actions/checkout@v4
16 | 
17 |       - name: Setup Node.js
18 |         uses: actions/setup-node@v4
19 |         with:
20 |           node-version: '22'
21 |           registry-url: 'https://registry.npmjs.org'
22 | 
23 |       - name: Install pnpm
24 |         uses: pnpm/action-setup@v2
25 |         with:
26 |           version: 9
27 | 
28 |       - name: Get pnpm store directory
29 |         shell: bash
30 |         run: |
31 |           echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
32 | 
33 |       - name: Setup pnpm cache
34 |         uses: actions/cache@v3
35 |         with:
36 |           path: ${{ env.STORE_PATH }}
37 |           key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
38 |           restore-keys: |
39 |             ${{ runner.os }}-pnpm-store-
40 | 
41 |       - name: Get package version
42 |         id: get_version
43 |         run: |
44 |           CURRENT_VERSION=$(node -p "require('./package.json').version")
45 |           echo "version=$CURRENT_VERSION" >> $GITHUB_OUTPUT
46 | 
47 |       - name: Check version on npm
48 |         id: check_version
49 |         run: |
50 |           NPM_VERSION=$(npm view promptl-ai version 2>/dev/null || echo "0.0.0")
51 |           if [ "${{ steps.get_version.outputs.version }}" != "$NPM_VERSION" ]; then
52 |             echo "should_publish=true" >> $GITHUB_OUTPUT
53 |           else
54 |             echo "should_publish=false" >> $GITHUB_OUTPUT
55 |           fi
56 | 
57 |       - name: Install dependencies
58 |         if: steps.check_version.outputs.should_publish == 'true'
59 |         run: pnpm install
60 | 
61 |       - name: Build package (with workspace dependencies)
62 |         if: steps.check_version.outputs.should_publish == 'true'
63 |         run: pnpm run build
64 | 
65 |       - name: Run linter
66 |         if: steps.check_version.outputs.should_publish == 'true'
67 |         run: pnpm run lint
68 | 
69 |       - name: Run typescript checker
70 |         if: steps.check_version.outputs.should_publish == 'true'
71 |         run: pnpm run tc
72 | 
73 |       - name: Run tests
74 |         if: steps.check_version.outputs.should_publish == 'true'
75 |         run: pnpm run test
76 | 
77 |       - name: Publish to npm
78 |         if: steps.check_version.outputs.should_publish == 'true'
79 |         run: pnpm publish --access public --no-git-checks
80 | 
81 |         env:
82 |           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
83 | 


--------------------------------------------------------------------------------
/src/compiler/base/types.ts:
--------------------------------------------------------------------------------
 1 | import Scope, { ScopePointers } from '$promptl/compiler/scope'
 2 | import { TemplateNode } from '$promptl/parser/interfaces'
 3 | import {
 4 |   AssistantMessage,
 5 |   Config,
 6 |   Message,
 7 |   MessageContent,
 8 |   PromptlSourceRef,
 9 | } from '$promptl/types'
10 | import type { Node as LogicalExpression } from 'estree'
11 | 
12 | import { ReferencePromptFn, ResolveBaseNodeProps } from '../types'
13 | 
14 | export enum NodeType {
15 |   Literal = 'Literal',
16 |   Identifier = 'Identifier',
17 |   ObjectExpression = 'ObjectExpression',
18 |   ArrayExpression = 'ArrayExpression',
19 |   SequenceExpression = 'SequenceExpression',
20 |   LogicalExpression = 'LogicalExpression',
21 |   BinaryExpression = 'BinaryExpression',
22 |   UnaryExpression = 'UnaryExpression',
23 |   AssignmentExpression = 'AssignmentExpression',
24 |   UpdateExpression = 'UpdateExpression',
25 |   MemberExpression = 'MemberExpression',
26 |   ConditionalExpression = 'ConditionalExpression',
27 |   CallExpression = 'CallExpression',
28 |   ChainExpression = 'ChainExpression',
29 | }
30 | 
31 | type RaiseErrorFn<T = void | never, N = TemplateNode | LogicalExpression> = (
32 |   { code, message }: { code: string; message: string },
33 |   node: N,
34 | ) => T
35 | 
36 | type NodeStatus = {
37 |   completedAs?: unknown
38 |   scopePointers?: ScopePointers | undefined
39 |   loopIterationIndex?: number
40 | }
41 | 
42 | export type TemplateNodeWithStatus = TemplateNode & {
43 |   status?: NodeStatus
44 | }
45 | 
46 | export type CompileNodeContext<N extends TemplateNode> = {
47 |   node: N
48 |   scope: Scope
49 |   resolveExpression: (
50 |     expression: LogicalExpression,
51 |     scope: Scope,
52 |   ) => Promise<unknown>
53 |   resolveBaseNode: (props: ResolveBaseNodeProps<TemplateNode>) => Promise<void>
54 |   baseNodeError: RaiseErrorFn<never, TemplateNode>
55 |   expressionError: RaiseErrorFn<never, LogicalExpression>
56 | 
57 |   isInsideStepTag: boolean
58 |   isInsideMessageTag: boolean
59 |   isInsideContentTag: boolean
60 | 
61 |   fullPath: string | undefined
62 |   referencePromptFn: ReferencePromptFn | undefined
63 | 
64 |   setConfig: (config: Config) => void
65 |   addMessage: (message: Message, global?: boolean) => void
66 |   addStrayText: (text: string, node?: TemplateNode) => void
67 |   popStrayText: () => { text: string; sourceMap: PromptlSourceRef[] }
68 |   groupStrayText: () => void
69 |   addContent: (item: { node?: TemplateNode; content: MessageContent }) => void
70 |   popContent: () => { node?: TemplateNode; content: MessageContent }[]
71 |   groupContent: () => void
72 |   popStepResponse: () => AssistantMessage | undefined
73 | 
74 |   stop: (config?: Config) => void
75 | }
76 | 


--------------------------------------------------------------------------------
/src/rpc/procedures.ts:
--------------------------------------------------------------------------------
 1 | import { render, scan, SerializedChain } from '../compiler'
 2 | import { Chain, StepResponse } from '../compiler/chain'
 3 | import { AdapterKey, getAdapter } from '../providers'
 4 | import { Message, MessageRole } from '../types'
 5 | import { RPC } from './types'
 6 | 
 7 | export default {
 8 |   [RPC.Procedure.ScanPrompt]: async ({ prompt }: { prompt: string }) => {
 9 |     const result = await scan({
10 |       prompt: prompt,
11 |     })
12 | 
13 |     return {
14 |       hash: result.hash,
15 |       resolvedPrompt: result.resolvedPrompt,
16 |       config: result.config,
17 |       errors: result.errors,
18 |       parameters: Array.from(result.parameters),
19 |       isChain: result.isChain,
20 |       includedPromptPaths: Array.from(result.includedPromptPaths),
21 |     }
22 |   },
23 | 
24 |   [RPC.Procedure.RenderPrompt]: async ({
25 |     prompt,
26 |     parameters,
27 |     adapter,
28 |     defaultRole,
29 |     includeSourceMap,
30 |   }: {
31 |     prompt: string
32 |     parameters?: Record<string, unknown>
33 |     adapter?: AdapterKey
34 |     defaultRole?: MessageRole
35 |     includeSourceMap?: boolean
36 |   }) => {
37 |     const result = await render({
38 |       prompt: prompt,
39 |       parameters: parameters,
40 |       adapter: adapter ? getAdapter(adapter) : undefined,
41 |       defaultRole: defaultRole,
42 |       includeSourceMap: includeSourceMap,
43 |     })
44 | 
45 |     return {
46 |       messages: result.messages,
47 |       config: result.config,
48 |     }
49 |   },
50 | 
51 |   [RPC.Procedure.CreateChain]: async ({
52 |     prompt,
53 |     parameters,
54 |     adapter,
55 |     defaultRole,
56 |     includeSourceMap,
57 |   }: {
58 |     prompt: string
59 |     parameters?: Record<string, unknown>
60 |     adapter?: AdapterKey
61 |     defaultRole?: MessageRole
62 |     includeSourceMap?: boolean
63 |   }) => {
64 |     return new Chain({
65 |       prompt: prompt,
66 |       parameters: parameters,
67 |       adapter: adapter ? getAdapter(adapter) : undefined,
68 |       defaultRole: defaultRole,
69 |       includeSourceMap: includeSourceMap,
70 |     }).serialize()
71 |   },
72 | 
73 |   [RPC.Procedure.StepChain]: async ({
74 |     chain: fromChain,
75 |     response,
76 |   }: {
77 |     chain: SerializedChain
78 |     response?: StepResponse<Message>
79 |   }) => {
80 |     const chain = Chain.deserialize({ serialized: fromChain })!
81 |     const result = await chain.step(response)
82 | 
83 |     return {
84 |       chain: chain.serialize(),
85 |       messages: result.messages,
86 |       config: result.config,
87 |       completed: result.completed,
88 |     }
89 |   },
90 | } as Record<RPC.Procedure, RPC.Handler<any, Promise<any>>>
91 | 


--------------------------------------------------------------------------------
/src/compiler/base/nodes/for.ts:
--------------------------------------------------------------------------------
 1 | import { hasContent, isIterable } from '$promptl/compiler/utils'
 2 | import errors from '$promptl/error/errors'
 3 | import { ForBlock } from '$promptl/parser/interfaces'
 4 | 
 5 | import { CompileNodeContext, TemplateNodeWithStatus } from '../types'
 6 | 
 7 | type ForNodeWithStatus = TemplateNodeWithStatus & {
 8 |   status: TemplateNodeWithStatus['status'] & {
 9 |     loopIterationIndex: number
10 |   }
11 | }
12 | 
13 | export async function compile({
14 |   node,
15 |   scope,
16 |   isInsideStepTag,
17 |   isInsideContentTag,
18 |   isInsideMessageTag,
19 |   resolveBaseNode,
20 |   resolveExpression,
21 |   expressionError,
22 |   fullPath,
23 | }: CompileNodeContext<ForBlock>) {
24 |   const nodeWithStatus = node as ForNodeWithStatus
25 |   nodeWithStatus.status = {
26 |     ...nodeWithStatus.status,
27 |     scopePointers: scope.getPointers(),
28 |   }
29 | 
30 |   const iterableElement = await resolveExpression(node.expression, scope)
31 |   if (!isIterable(iterableElement) || !(await hasContent(iterableElement))) {
32 |     const childScope = scope.copy()
33 |     for await (const childNode of node.else?.children ?? []) {
34 |       await resolveBaseNode({
35 |         node: childNode,
36 |         scope: childScope,
37 |         isInsideStepTag,
38 |         isInsideMessageTag,
39 |         isInsideContentTag,
40 |         fullPath,
41 |       })
42 |     }
43 |     return
44 |   }
45 | 
46 |   const contextVarName = node.context.name
47 |   const indexVarName = node.index?.name
48 |   if (scope.exists(contextVarName)) {
49 |     throw expressionError(
50 |       errors.variableAlreadyDeclared(contextVarName),
51 |       node.context,
52 |     )
53 |   }
54 | 
55 |   if (indexVarName && scope.exists(indexVarName)) {
56 |     throw expressionError(
57 |       errors.variableAlreadyDeclared(indexVarName),
58 |       node.index!,
59 |     )
60 |   }
61 | 
62 |   let i = 0
63 | 
64 |   for await (const element of iterableElement) {
65 |     if (i < (nodeWithStatus.status.loopIterationIndex ?? 0)) {
66 |       i++
67 |       continue
68 |     }
69 |     nodeWithStatus.status.loopIterationIndex = i
70 | 
71 |     const localScope = scope.copy()
72 |     localScope.set(contextVarName, element)
73 |     if (indexVarName) localScope.set(indexVarName, i)
74 | 
75 |     for await (const childNode of node.children ?? []) {
76 |       await resolveBaseNode({
77 |         node: childNode,
78 |         scope: localScope,
79 |         isInsideStepTag,
80 |         isInsideMessageTag,
81 |         isInsideContentTag,
82 |         fullPath,
83 |         completedValue: `step_${i}`,
84 |       })
85 |     }
86 | 
87 |     i++
88 |   }
89 | 
90 |   nodeWithStatus.status = {
91 |     ...nodeWithStatus.status,
92 |     loopIterationIndex: 0,
93 |   }
94 | }
95 | 


--------------------------------------------------------------------------------
/src/compiler/base/nodes/tags/step.ts:
--------------------------------------------------------------------------------
 1 | import { tagAttributeIsLiteral } from '$promptl/compiler/utils'
 2 | import errors from '$promptl/error/errors'
 3 | import { ChainStepTag } from '$promptl/parser/interfaces'
 4 | import { Config, ContentType } from '$promptl/types'
 5 | 
 6 | import { CompileNodeContext } from '../../types'
 7 | 
 8 | function isValidConfig(value: unknown): value is Config | undefined {
 9 |   if (value === undefined) return true
10 |   if (Array.isArray(value)) return false
11 |   return typeof value === 'object'
12 | }
13 | 
14 | export async function compile(
15 |   {
16 |     node,
17 |     scope,
18 |     isInsideStepTag,
19 |     isInsideMessageTag,
20 |     isInsideContentTag,
21 |     fullPath,
22 |     popStepResponse,
23 |     groupContent,
24 |     resolveBaseNode,
25 |     baseNodeError,
26 |     stop,
27 |   }: CompileNodeContext<ChainStepTag>,
28 |   attributes: Record<string, unknown>,
29 | ) {
30 |   if (isInsideStepTag) {
31 |     baseNodeError(errors.stepTagInsideStep, node)
32 |   }
33 | 
34 |   const stepResponse = popStepResponse()
35 | 
36 |   const { as: responseVarName, raw: messageVarName, ...config } = attributes
37 | 
38 |   // The step must be processed.
39 |   if (stepResponse === undefined) {
40 |     if (!isValidConfig(config)) {
41 |       baseNodeError(errors.invalidStepConfig, node)
42 |     }
43 | 
44 |     for await (const childNode of node.children ?? []) {
45 |       await resolveBaseNode({
46 |         node: childNode,
47 |         scope,
48 |         isInsideStepTag: true,
49 |         isInsideMessageTag,
50 |         isInsideContentTag,
51 |         fullPath,
52 |       })
53 |     }
54 | 
55 |     // Stop the compiling process up to this point.
56 |     stop(config as Config)
57 |   }
58 | 
59 |   // The step has already been process, this is the continuation of the chain.
60 | 
61 |   if ('raw' in attributes) {
62 |     if (!tagAttributeIsLiteral(node, 'raw')) {
63 |       baseNodeError(errors.invalidStaticAttribute('raw'), node)
64 |     }
65 | 
66 |     scope.set(String(messageVarName), stepResponse)
67 |   }
68 | 
69 |   if ('as' in attributes) {
70 |     if (!tagAttributeIsLiteral(node, 'as')) {
71 |       baseNodeError(errors.invalidStaticAttribute('as'), node)
72 |     }
73 | 
74 |     const textResponse = (stepResponse?.content ?? []).filter(c => c.type === ContentType.text).map(c => c.text).join('')
75 |     let responseVarValue = textResponse
76 | 
77 |     if ("schema" in config) {
78 |       try {
79 |         responseVarValue = JSON.parse(responseVarValue.trim())
80 |       } catch (error) {
81 |         baseNodeError(errors.invalidStepResponseFormat(error as Error), node)
82 |       }
83 |     }
84 | 
85 |     scope.set(String(responseVarName), responseVarValue)
86 |   }
87 | 
88 |   groupContent()
89 | }
90 | 


--------------------------------------------------------------------------------
/src/compiler/logic/operators.ts:
--------------------------------------------------------------------------------
 1 | // https://github.com/estree/estree/blob/master/es5.md#binary-operations
 2 | export const BINARY_OPERATOR_METHODS: {
 3 |   [operator: string]: (left: any, right: any) => unknown
 4 | } = {
 5 |   // BinaryExpression
 6 |   '==': (left, right) => left == right,
 7 |   '!=': (left, right) => left != right,
 8 |   '===': (left, right) => left === right,
 9 |   '!==': (left, right) => left !== right,
10 |   '<': (left, right) => left < right,
11 |   '<=': (left, right) => left <= right,
12 |   '>': (left, right) => left > right,
13 |   '>=': (left, right) => left >= right,
14 |   '<<': (left, right) => left << right,
15 |   '>>': (left, right) => left >> right,
16 |   '>>>': (left, right) => left >>> right,
17 |   '+': (left, right) => left + right,
18 |   '-': (left, right) => left - right,
19 |   '*': (left, right) => left * right,
20 |   '/': (left, right) => left / right,
21 |   '%': (left, right) => left % right,
22 |   '|': (left, right) => left | right,
23 |   '^': (left, right) => left ^ right,
24 |   '&': (left, right) => left & right,
25 |   in: (left, right) => left in right,
26 |   instanceof: (left, right) => (left as object) instanceof right,
27 | 
28 |   // LogicalExpression
29 |   '||': (left, right) => left || right,
30 |   '&&': (left, right) => left && right,
31 |   '??': (left, right) => left ?? right,
32 | }
33 | 
34 | // https://github.com/estree/estree/blob/master/es5.md#unary-operations
35 | export const UNARY_OPERATOR_METHODS: {
36 |   [operator: string]: (value: any, prefix: any) => unknown
37 | } = {
38 |   // UnaryExpression
39 |   '-': (value, prefix) => (prefix ? -value : value),
40 |   '+': (value, prefix) => (prefix ? +value : value),
41 |   '!': (value, _) => !value,
42 |   '~': (value, _) => ~value,
43 |   typeof: (value, _) => typeof value,
44 |   void: (value, _) => void value,
45 | }
46 | 
47 | // https://github.com/estree/estree/blob/master/es5.md#memberexpression
48 | export const MEMBER_EXPRESSION_METHOD = (
49 |   object: any,
50 |   property: any,
51 | ): unknown => {
52 |   const value = object[property]
53 |   return typeof value === 'function' ? value.bind(object) : value
54 | }
55 | 
56 | // https://github.com/estree/estree/blob/master/es5.md#assignmentexpression
57 | export const ASSIGNMENT_OPERATOR_METHODS: {
58 |   [operator: string]: (left: any, right: any) => unknown
59 | } = {
60 |   '=': (_, right) => right,
61 |   '+=': (left, right) => left + right,
62 |   '-=': (left, right) => left - right,
63 |   '*=': (left, right) => left * right,
64 |   '/=': (left, right) => left / right,
65 |   '%=': (left, right) => left % right,
66 |   '<<=': (left, right) => left << right,
67 |   '>>=': (left, right) => left >> right,
68 |   '>>>=': (left, right) => left >>> right,
69 |   '|=': (left, right) => left | right,
70 |   '^=': (left, right) => left ^ right,
71 |   '&=': (left, right) => left & right,
72 | }
73 | 


--------------------------------------------------------------------------------
/src/parser/read/context.ts:
--------------------------------------------------------------------------------
 1 | import type CompileError from '$promptl/error/error'
 2 | import PARSER_ERRORS from '$promptl/error/errors'
 3 | import { parseExpressionAt } from '$promptl/parser/utils/acorn'
 4 | import {
 5 |   getBracketClose,
 6 |   isBracketClose,
 7 |   isBracketOpen,
 8 |   isBracketPair,
 9 | } from '$promptl/parser/utils/bracket'
10 | import fullCharCodeAt from '$promptl/parser/utils/full_char_code_at'
11 | import { isIdentifierStart } from 'acorn'
12 | import { Pattern } from 'estree'
13 | 
14 | import { Parser } from '..'
15 | 
16 | export default function readContext(
17 |   parser: Parser,
18 | ): Pattern & { start: number; end: number } {
19 |   const start = parser.index
20 |   let i = parser.index
21 | 
22 |   const code = fullCharCodeAt(parser.template, i)
23 |   if (isIdentifierStart(code, true)) {
24 |     return {
25 |       type: 'Identifier',
26 |       name: parser.readIdentifier()!,
27 |       start,
28 |       end: parser.index,
29 |     }
30 |   }
31 | 
32 |   if (!isBracketOpen(code)) {
33 |     parser.error(PARSER_ERRORS.unexpectedTokenDestructure)
34 |   }
35 | 
36 |   const bracketStack: number[] = [code]
37 |   i += code <= 0xffff ? 1 : 2
38 | 
39 |   while (i < parser.template.length) {
40 |     const code = fullCharCodeAt(parser.template, i)
41 |     if (isBracketOpen(code)) {
42 |       bracketStack.push(code)
43 |     } else if (isBracketClose(code)) {
44 |       if (!isBracketPair(bracketStack[bracketStack.length - 1]!, code)) {
45 |         parser.error(
46 |           PARSER_ERRORS.unexpectedToken(
47 |             String.fromCharCode(
48 |               getBracketClose(bracketStack[bracketStack.length - 1]!) ?? 0,
49 |             ),
50 |           ),
51 |         )
52 |       }
53 |       bracketStack.pop()
54 |       if (bracketStack.length === 0) {
55 |         i += code <= 0xffff ? 1 : 2
56 |         break
57 |       }
58 |     }
59 |     i += code <= 0xffff ? 1 : 2
60 |   }
61 | 
62 |   parser.index = i
63 | 
64 |   const patternString = parser.template.slice(start, i)
65 |   try {
66 |     // the length of the `space_with_newline` has to be start - 1
67 |     // because we added a `(` in front of the pattern_string,
68 |     // which shifted the entire string to right by 1
69 |     // so we offset it by removing 1 character in the `space_with_newline`
70 |     // to achieve that, we remove the 1st space encountered,
71 |     // so it will not affect the `column` of the node
72 |     let spaceWithNewLine = parser.template
73 |       .slice(0, start)
74 |       .replace(/[^\n]/g, ' ')
75 |     const firstSpace = spaceWithNewLine.indexOf(' ')
76 |     spaceWithNewLine =
77 |       spaceWithNewLine.slice(0, firstSpace) +
78 |       spaceWithNewLine.slice(firstSpace + 1)
79 | 
80 |     return parseExpressionAt(
81 |       `${spaceWithNewLine}(${patternString} = 1)`,
82 |       start - 1,
83 |     ).left
84 |   } catch (error) {
85 |     parser.acornError(error as CompileError)
86 |   }
87 | 
88 |   return {
89 |     type: 'Identifier',
90 |     name: '',
91 |     start: parser.index,
92 |     end: parser.index,
93 |   }
94 | }
95 | 


--------------------------------------------------------------------------------
/src/providers/openai-responses/utils/parsers/parseSimpleContent.ts:
--------------------------------------------------------------------------------
  1 | import {
  2 |   MessageContent,
  3 |   MessageContentSimple,
  4 |   MessageInputItem,
  5 |   SimpleInputMessage,
  6 | } from '$promptl/providers/openai-responses/types'
  7 | import { getMimeType } from '$promptl/providers/utils/getMimeType'
  8 | import {
  9 |   ContentType,
 10 |   FileContent,
 11 |   ImageContent,
 12 |   MessageRole,
 13 |   TextContent,
 14 |   AssistantMessage,
 15 |   SystemMessage,
 16 |   DeveloperMessage,
 17 |   UserMessage,
 18 | } from '$promptl/types'
 19 | 
 20 | type PromptlMessage =
 21 |   | SystemMessage
 22 |   | DeveloperMessage
 23 |   | AssistantMessage
 24 |   | UserMessage
 25 | 
 26 | const SIMPLE_CONTENT_TYPES: MessageContentSimple['type'][] = [
 27 |   'input_text',
 28 |   'input_image',
 29 |   'input_file',
 30 | ]
 31 | 
 32 | export function isSimpleContent(
 33 |   content: MessageContent[0],
 34 | ): content is MessageContentSimple {
 35 |   if (typeof content === 'string') return false
 36 |   if (!('type' in content)) return false
 37 |   if (content.type === 'refusal') return false
 38 |   if (content.type === 'output_text') return false
 39 | 
 40 |   return SIMPLE_CONTENT_TYPES.includes(content.type)
 41 | }
 42 | 
 43 | export function parseSimpleContent(content: MessageContentSimple) {
 44 |   if (content.type === 'input_text') {
 45 |     return {
 46 |       type: ContentType.text,
 47 |       text: content.text,
 48 |     } satisfies TextContent
 49 |   }
 50 | 
 51 |   if (content.type === 'input_image' && content.image_url) {
 52 |     return {
 53 |       type: ContentType.image,
 54 |       image: content.image_url,
 55 |       detail: content.detail,
 56 |     } satisfies ImageContent
 57 |   }
 58 | 
 59 |   if (content.type === 'input_file' && content.file_data) {
 60 |     return {
 61 |       type: ContentType.file,
 62 |       file: content.file_data,
 63 |       filename: content.filename,
 64 |       file_id: content.file_id,
 65 |       mimeType: getMimeType(content.filename),
 66 |     } satisfies FileContent
 67 |   }
 68 |   throw new Error(
 69 |     `Unknown type ${content.type} in OpenAIResponse message input content`,
 70 |   )
 71 | }
 72 | 
 73 | function isAnyRole(role: SimpleInputMessage['role']): boolean {
 74 |   return (
 75 |     role === 'user' ||
 76 |     role === 'assistant' ||
 77 |     role === 'system' ||
 78 |     role === 'developer'
 79 |   )
 80 | }
 81 | 
 82 | export function isSimpleInputMessage(
 83 |   message: MessageInputItem,
 84 | ): message is SimpleInputMessage {
 85 |   return message.type === 'message' && isAnyRole(message.role)
 86 | }
 87 | 
 88 | export function parseSimpleInputMessage(message: SimpleInputMessage) {
 89 |   if (typeof message.content === 'string') {
 90 |     return {
 91 |       role: MessageRole[message.role],
 92 |       content: [
 93 |         {
 94 |           type: ContentType.text,
 95 |           text: message.content,
 96 |         } satisfies TextContent,
 97 |       ],
 98 |     } satisfies PromptlMessage
 99 |   }
100 | 
101 |   return {
102 |     role: MessageRole[message.role],
103 |     content: message.content.map(parseSimpleContent),
104 |   } satisfies PromptlMessage
105 | }
106 | 


--------------------------------------------------------------------------------
/src/compiler/base/nodes/for.test.ts:
--------------------------------------------------------------------------------
 1 | import { Message, MessageContent, TextContent } from '$promptl/types'
 2 | import { describe, expect, it } from 'vitest'
 3 | 
 4 | import { render } from '../../index'
 5 | import { removeCommonIndent } from '../../utils'
 6 | 
 7 | async function getCompiledText(
 8 |   prompt: string,
 9 |   parameters: Record<string, any> = {},
10 | ) {
11 |   const result = await render({
12 |     prompt: removeCommonIndent(prompt),
13 |     parameters,
14 |   })
15 | 
16 |   return result.messages.reduce((acc: string, message: Message) => {
17 |     const content =
18 |       typeof message.content === 'string'
19 |         ? message.content
20 |         : (message.content as MessageContent[])
21 |             .map((c) => (c as TextContent).text)
22 |             .join('')
23 | 
24 |     return acc + content
25 |   }, '')
26 | }
27 | 
28 | describe('each loops', async () => {
29 |   it('iterates over any iterable object', async () => {
30 |     const prompt1 = `{{ for element in  [1, 2, 3] }} {{element}} {{ endfor }}`
31 |     const prompt2 = `{{ for element in "foo" }} {{element}} {{ endfor }}`
32 | 
33 |     const result1 = await getCompiledText(prompt1)
34 |     const result2 = await getCompiledText(prompt2)
35 | 
36 |     expect(result1).toBe('123')
37 |     expect(result2).toBe('foo')
38 |   })
39 | 
40 |   it('computes the else block when the element is not iterable', async () => {
41 |     const prompt1 = `{{ for element in 5}} {{element}} {{ else }} FOO {{ endfor }}`
42 |     const prompt2 = `{{ for element in { a: 1, b: 2, c: 3 } }} {{element}} {{ else }} FOO {{ endfor }}`
43 | 
44 |     const result1 = await getCompiledText(prompt1)
45 |     const result2 = await getCompiledText(prompt2)
46 | 
47 |     expect(result1).toBe('FOO')
48 |     expect(result2).toBe('FOO')
49 |   })
50 | 
51 |   it('computes the else block when the iterable object is empty', async () => {
52 |     const prompt = `{{ for element in [] }} {{element}} {{ else }} FOO {{ endfor }}`
53 |     const result = await getCompiledText(prompt)
54 |     expect(result).toBe('FOO')
55 |   })
56 | 
57 |   it('does not do anything when the iterable object is not iterable and there is no else block', async () => {
58 |     const prompt = `{{ for element in 5 }} {{element}} {{ endfor }}`
59 |     expect(render({ prompt, parameters: {} })).resolves
60 |   })
61 | 
62 |   it('gives access to the index of the element', async () => {
63 |     const prompt = `{{ for element, index in ['a', 'b', 'c'] }} {{index}} {{ endfor }}`
64 |     const result = await getCompiledText(prompt)
65 |     expect(result).toBe('012')
66 |   })
67 | 
68 |   it('respects variable scope', async () => {
69 |     const prompt1 = `{{ for elemenet in ['a', 'b', 'c'] }} {{foo = 5}} {{ endfor }} {{foo}}`
70 |     const prompt2 = `{{foo = 5}} {{ for element in ['a', 'b', 'c'] }} {{foo = 7}} {{ endfor }} {{foo}}`
71 |     const prompt3 = `{{foo = 5}} {{ for element in [1, 2, 3] }} {{foo += element}} {{ endfor }} {{foo}}`
72 |     const result1 = await getCompiledText(prompt1)
73 |     const result2 = await getCompiledText(prompt2)
74 |     const result3 = await getCompiledText(prompt3)
75 | 
76 |     expect(result1).toBe('')
77 |     expect(result2).toBe('7')
78 |     expect(result3).toBe('11')
79 |   })
80 | })
81 | 


--------------------------------------------------------------------------------
/src/parser/state/text.ts:
--------------------------------------------------------------------------------
  1 | import { CUSTOM_TAG_START, RESERVED_TAGS } from '$promptl/constants'
  2 | import { type Parser } from '$promptl/parser'
  3 | import type { Text } from '$promptl/parser/interfaces'
  4 | 
  5 | const RESERVED_DELIMITERS = [CUSTOM_TAG_START, '/*', '<!--']
  6 | const WHITESPACE_CHARS = [' ', '\n', '\r', '\t']
  7 | 
  8 | function matchesReservedTag(template: string, index: number): boolean {
  9 |   if (template[index] !== '<') return false
 10 | 
 11 |   const isClosingTag = template[index + 1] === '/'
 12 |   let tagStart = index + (isClosingTag ? 2 : 1)
 13 | 
 14 |   for (const tag of RESERVED_TAGS) {
 15 |     if (
 16 |       template.startsWith(tag, tagStart) &&
 17 |       (WHITESPACE_CHARS.includes(template[tagStart + tag.length] as string) ||
 18 |         template[tagStart + tag.length] === '/' ||
 19 |         template[tagStart + tag.length] === '>' ||
 20 |         tagStart + tag.length === template.length)
 21 |     ) {
 22 |       return true
 23 |     }
 24 |   }
 25 | 
 26 |   return false
 27 | }
 28 | 
 29 | export function text(parser: Parser) {
 30 |   const start = parser.index
 31 |   let data = ''
 32 |   const template = parser.template
 33 |   const len = template.length
 34 | 
 35 |   // Track escape state with a boolean instead of recounting backslashes
 36 |   let isEscaped = false
 37 | 
 38 |   while (parser.index < len) {
 39 |     const char = template[parser.index]
 40 | 
 41 |     // Handle backslash: toggle escape state and only add it if already escaped
 42 |     if (char === '\\' && !isEscaped) {
 43 |       isEscaped = true
 44 |       parser.index++
 45 |       continue
 46 |     }
 47 | 
 48 |     // If we're in escaped mode, add the current character regardless of what it is
 49 |     if (isEscaped) {
 50 |       data += char
 51 |       parser.index++
 52 |       isEscaped = false
 53 |       continue
 54 |     }
 55 | 
 56 |     // Check break conditions (only when not escaped)
 57 |     if (
 58 |       char === '-' &&
 59 |       template[parser.index + 1] === '-' &&
 60 |       template[parser.index + 2] === '-' &&
 61 |       template[parser.index + 3] !== '-'
 62 |     ) {
 63 |       break
 64 |     }
 65 | 
 66 |     let delimiterMatched = false
 67 |     for (const delim of RESERVED_DELIMITERS) {
 68 |       if (template.startsWith(delim, parser.index)) {
 69 |         delimiterMatched = true
 70 |         break
 71 |       }
 72 |     }
 73 |     if (delimiterMatched) break
 74 | 
 75 |     if (matchesReservedTag(template, parser.index)) break
 76 | 
 77 |     // Handle dashes more efficiently
 78 |     if (char === '-') {
 79 |       let dashEnd = parser.index + 1
 80 |       while (dashEnd < len && template[dashEnd] === '-') dashEnd++
 81 |       const dashCount = dashEnd - parser.index
 82 |       data += '-'.repeat(dashCount)
 83 |       parser.index = dashEnd
 84 |     } else {
 85 |       // Normal character processing
 86 |       data += char
 87 |       parser.index++
 88 |     }
 89 |   }
 90 | 
 91 |   // Create the text node with optimized data processing
 92 |   // Since we're handling escape characters differently now, we can simplify this replacement
 93 |   const node = {
 94 |     start,
 95 |     end: parser.index,
 96 |     type: 'Text',
 97 |     raw: data,
 98 |     data: data, // The escaping is already handled correctly during parsing
 99 |   } as Text
100 | 
101 |   parser.current().children!.push(node)
102 | }
103 | 


--------------------------------------------------------------------------------
/src/parser/utils/html.ts:
--------------------------------------------------------------------------------
 1 | import entities from './entities'
 2 | 
 3 | const windows_1252 = [
 4 |   8364, 129, 8218, 402, 8222, 8230, 8224, 8225, 710, 8240, 352, 8249, 338, 141,
 5 |   381, 143, 144, 8216, 8217, 8220, 8221, 8226, 8211, 8212, 732, 8482, 353, 8250,
 6 |   339, 157, 382, 376,
 7 | ]
 8 | 
 9 | function reg_exp_entity(entity_name: string, is_attribute_value: boolean) {
10 |   // https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state
11 |   // doesn't decode the html entity which not ends with ; and next character is =, number or alphabet in attribute value.
12 |   if (is_attribute_value && !entity_name.endsWith(';')) {
13 |     return `${entity_name}\\b(?!=)`
14 |   }
15 |   return entity_name
16 | }
17 | 
18 | function get_entity_pattern(is_attribute_value: boolean) {
19 |   const reg_exp_num = '#(?:x[a-fA-F\\d]+|\\d+)(?:;)?'
20 |   const reg_exp_entities = Object.keys(entities).map((entity_name) =>
21 |     reg_exp_entity(entity_name, is_attribute_value),
22 |   )
23 |   const entity_pattern = new RegExp(
24 |     `&(${reg_exp_num}|${reg_exp_entities.join('|')})`,
25 |     'g',
26 |   )
27 |   return entity_pattern
28 | }
29 | const entity_pattern_content = get_entity_pattern(false)
30 | const entity_pattern_attr_value = get_entity_pattern(true)
31 | 
32 | export function decode_character_references(
33 |   html: string,
34 |   is_attribute_value: boolean,
35 | ) {
36 |   const entity_pattern = is_attribute_value
37 |     ? entity_pattern_attr_value
38 |     : entity_pattern_content
39 |   return html.replace(entity_pattern, (match, entity) => {
40 |     let code
41 |     // Handle named entities
42 |     if (entity[0] !== '#') {
43 |       code = entities[entity]
44 |     } else if (entity[1] === 'x') {
45 |       code = parseInt(entity.substring(2), 16)
46 |     } else {
47 |       code = parseInt(entity.substring(1), 10)
48 |     }
49 |     if (!code) {
50 |       return match
51 |     }
52 |     return String.fromCodePoint(validate_code(code))
53 |   })
54 | }
55 | const NUL = 0
56 | // some code points are verboten. If we were inserting HTML, the browser would replace the illegal
57 | // code points with alternatives in some cases - since we're bypassing that mechanism, we need
58 | // to replace them ourselves
59 | //
60 | // Source: http://en.wikipedia.org/wiki/Character_encodings_in_HTML#Illegal_characters
61 | 
62 | function validate_code(code: number): number {
63 |   // line feed becomes generic whitespace
64 |   if (code === 10) {
65 |     return 32
66 |   }
67 |   // ASCII range. (Why someone would use HTML entities for ASCII characters I don't know, but...)
68 |   if (code < 128) {
69 |     return code
70 |   }
71 |   // code points 128-159 are dealt with leniently by browsers, but they're incorrect. We need
72 |   // to correct the mistake or we'll end up with missing € signs and so on
73 |   if (code <= 159) {
74 |     return windows_1252[code - 128]!
75 |   }
76 |   // basic multilingual plane
77 |   if (code < 55296) {
78 |     return code
79 |   }
80 |   // UTF-16 surrogate halves
81 |   if (code <= 57343) {
82 |     return NUL
83 |   }
84 |   // rest of the basic multilingual plane
85 |   if (code <= 65535) {
86 |     return code
87 |   }
88 |   // supplementary multilingual plane 0x10000 - 0x1ffff
89 |   if (code >= 65536 && code <= 131071) {
90 |     return code
91 |   }
92 |   // supplementary ideographic plane 0x20000 - 0x2ffff
93 |   if (code >= 131072 && code <= 196607) {
94 |     return code
95 |   }
96 |   return NUL
97 | }
98 | 


--------------------------------------------------------------------------------
/src/compiler/base/nodes/tags/message.ts:
--------------------------------------------------------------------------------
  1 | import { CUSTOM_MESSAGE_ROLE_ATTR, TAG_NAMES } from '$promptl/constants'
  2 | import errors from '$promptl/error/errors'
  3 | import { MessageTag, TemplateNode } from '$promptl/parser/interfaces'
  4 | import {
  5 |   ContentType,
  6 |   Message,
  7 |   MessageContent,
  8 |   MessageRole,
  9 | } from '$promptl/types'
 10 | 
 11 | import { CompileNodeContext } from '../../types'
 12 | 
 13 | export async function compile(
 14 |   props: CompileNodeContext<MessageTag>,
 15 |   attributes: Record<string, unknown>,
 16 | ) {
 17 |   const {
 18 |     node,
 19 |     scope,
 20 |     isInsideStepTag,
 21 |     isInsideMessageTag,
 22 |     isInsideContentTag,
 23 |     fullPath,
 24 |     resolveBaseNode,
 25 |     baseNodeError,
 26 |     groupContent,
 27 |     groupStrayText,
 28 |     popContent,
 29 |     addMessage,
 30 |   } = props
 31 | 
 32 |   if (isInsideContentTag || isInsideMessageTag) {
 33 |     baseNodeError(errors.messageTagInsideMessage, node)
 34 |   }
 35 | 
 36 |   groupContent()
 37 | 
 38 |   let role = node.name as MessageRole
 39 |   if (node.name === TAG_NAMES.message) {
 40 |     if (attributes[CUSTOM_MESSAGE_ROLE_ATTR] === undefined) {
 41 |       baseNodeError(errors.messageTagWithoutRole, node)
 42 |     }
 43 |     role = attributes[CUSTOM_MESSAGE_ROLE_ATTR] as MessageRole
 44 |     delete attributes[CUSTOM_MESSAGE_ROLE_ATTR]
 45 |   }
 46 | 
 47 |   for await (const childNode of node.children ?? []) {
 48 |     await resolveBaseNode({
 49 |       node: childNode,
 50 |       scope,
 51 |       isInsideStepTag,
 52 |       isInsideMessageTag: true,
 53 |       isInsideContentTag,
 54 |       fullPath,
 55 |     })
 56 |   }
 57 | 
 58 |   groupStrayText()
 59 |   const content = popContent()
 60 | 
 61 |   const message = buildMessage(props as CompileNodeContext<MessageTag>, {
 62 |     role,
 63 |     attributes,
 64 |     content,
 65 |   })!
 66 |   addMessage(message)
 67 | }
 68 | 
 69 | type BuildProps<R extends MessageRole> = {
 70 |   role: R
 71 |   attributes: Record<string, unknown>
 72 |   content: { node?: TemplateNode; content: MessageContent }[]
 73 | }
 74 | 
 75 | function buildMessage<R extends MessageRole>(
 76 |   { node, baseNodeError }: CompileNodeContext<MessageTag>,
 77 |   { role, attributes, content }: BuildProps<R>,
 78 | ): Message | undefined {
 79 |   if (!Object.values(MessageRole).includes(role)) {
 80 |     baseNodeError(errors.invalidMessageRole(role), node)
 81 |   }
 82 | 
 83 |   if (role !== MessageRole.assistant) {
 84 |     content.forEach((item) => {
 85 |       if (item.content.type === ContentType.toolCall) {
 86 |         baseNodeError(errors.invalidToolCallPlacement, item.node ?? node)
 87 |       }
 88 |     })
 89 |   }
 90 | 
 91 |   const message = {
 92 |     ...attributes,
 93 |     role,
 94 |     content: content.map((item) => item.content),
 95 |   } as Message
 96 | 
 97 |   if (role === MessageRole.user) {
 98 |     message.name = attributes.name ? String(attributes.name) : undefined
 99 |   }
100 | 
101 |   if (role === MessageRole.tool) {
102 |     if (attributes.id === undefined) {
103 |       baseNodeError(errors.toolMessageWithoutId, node)
104 |     }
105 | 
106 |     if (attributes.name === undefined) {
107 |       baseNodeError(errors.toolMessageWithoutName, node)
108 |     }
109 | 
110 |     message.toolId = String(attributes.id)
111 |     message.toolName = String(attributes.name)
112 |     delete message['id']
113 |     delete message['name']
114 |   }
115 | 
116 |   return message
117 | }
118 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # PromptL
 2 | 
 3 | ```sh
 4 | npm install promptl-ai
 5 | ```
 6 | 
 7 | ## What is PromptL?
 8 | 
 9 | [PromptL](https://promptl.ai/) offers a common, easy-to-use syntax for defining dynamic prompts for LLMs. It is a simple, yet powerful language that allows you to define prompts in a human-readable format, while still being able to leverage the full power of LLMs.
10 | 
11 | ## PromptL in other languages
12 | 
13 | Thanks to our [universal WASM module with RPC](examples/rpc.ts), you can use PromptL in any language that can run WASM natively or through a library. These are the official bindings:
14 | 
15 | - [Python](https://github.com/latitude-dev/promptl-py)
16 | 
17 | ## Why PromptL?
18 | 
19 | While LLMs are becoming more powerful and popular by the day, defining prompts for them can be a daunting task. All main LLM providers, despite their differences, have adopted a similar structure for their prompting. It consists of a conversation between the user and assistant, which is defined by a list of messages and a series of configuration options. In response, it will return an assistant message as a reply.
20 | 
21 | This structure looks something like this:
22 | 
23 | ```json
24 | {
25 |   "model": "<your-model>",
26 |   "temperature": 0.6,
27 |   "messages": [
28 |     {
29 |       "type": "system",
30 |       "content": "You are a useful AI assistant expert in geography."
31 |     },
32 |     {
33 |       "type": "user",
34 |       "content": "Hi! What's the capital of Spain?"
35 |     }
36 |   ]
37 | }
38 | ```
39 | 
40 | This structure may be simple, but it can be tough for non-techy users to grasp or write it from scratch. In addition to this, creating a single static prompt is not that useful. Typically, users need to define conversations dynamically, where the flow changes based on user input or event parameters. The problem is, adding code to modify the conversation based on these parameters can get confusing and repetitive – it needs to be done for each prompt individually.
41 | 
42 | This is how the PromptL syntax steps in. It defines a language simple enough for any user to use and understand. And, at the same time, it offers immense power for users who want to maximize its potential. It allows users to define the same structure they would build before, but in a more readable way. Plus, they can add custom dynamic logic to create anything they need, all in just a single file.
43 | 
44 | Take a look at the same prompt as before, using the PromptL syntax:
45 | 
46 | ```plaintext
47 | ---
48 | model: <your-model>
49 | temperature: 0.6
50 | ---
51 | 
52 | You are a useful AI assistant expert in geography.
53 | 
54 | <user>
55 |   Hi! What's the capital of {{ country_name }}?
56 | </user>
57 | ```
58 | 
59 | In this case, not only the syntax is way more readable and maintainable, but it also allows for dynamic generation of prompts by using variables like `{{ country_name }}`.
60 | 
61 | This is just a small example of what PromptL can do. It is a powerful tool that can help you define dynamic prompts for your LLMs in a simple and easy way, without giving up any feature or functionality from the original structure.
62 | 
63 | ## Links
64 | 
65 | [Website](https://promptl.ai/) | [Documentation](https://docs.latitude.so/promptl/getting-started/introduction)
66 | 
67 | ## Development
68 | 
69 | To build the JavaScript library, run `pnpm build:lib`.
70 | 
71 | To build the universal WASM module with RPC, first install [`javy`](https://github.com/bytecodealliance/javy/releases) `6.0.0` or higher, and then run `pnpm build:rpc`.
72 | 


--------------------------------------------------------------------------------
/src/providers/anthropic/adapter.test.ts:
--------------------------------------------------------------------------------
  1 | import { render } from '$promptl/compiler'
  2 | import { removeCommonIndent } from '$promptl/compiler/utils'
  3 | import { MessageRole } from '$promptl/types'
  4 | import { describe, expect, it } from 'vitest'
  5 | 
  6 | import { Adapters } from '..'
  7 | 
  8 | describe('Anthropic adapter', async () => {
  9 |   it('moves top system messages to config', async () => {
 10 |     const prompt = `Hello <system>World</system>`
 11 |     const { config, messages } = await render({
 12 |       prompt,
 13 |       adapter: Adapters.anthropic,
 14 |     })
 15 |     expect(messages).toEqual([])
 16 | 
 17 |     expect(config.system).toEqual([
 18 |       {
 19 |         type: 'text',
 20 |         text: 'Hello',
 21 |       },
 22 |       {
 23 |         type: 'text',
 24 |         text: 'World',
 25 |       },
 26 |     ])
 27 |   })
 28 | 
 29 |   it('adapts user messages', async () => {
 30 |     const prompt = removeCommonIndent(`
 31 |       <user>
 32 |         Hello world!
 33 |         <content-image>https://image.source/</content-image>
 34 |         <content-file mime="text/plain">text content</content-file>
 35 |         <content-file mime="application/pdf">pdf content</content-file>
 36 |       </user>
 37 |     `)
 38 | 
 39 |     const { messages } = await render({ prompt, adapter: Adapters.anthropic })
 40 |     expect(messages).toEqual([
 41 |       {
 42 |         role: MessageRole.user,
 43 |         content: [
 44 |           { type: 'text', text: 'Hello world!' },
 45 |           {
 46 |             type: 'image',
 47 |             source: {
 48 |               type: 'base64',
 49 |               media_type: 'image/png',
 50 |               data: 'https://image.source/',
 51 |             },
 52 |           },
 53 |           { type: 'text', text: 'text content' },
 54 |           {
 55 |             type: 'document',
 56 |             source: {
 57 |               type: 'base64',
 58 |               media_type: 'application/pdf',
 59 |               data: 'pdf content',
 60 |             },
 61 |           },
 62 |         ],
 63 |       },
 64 |     ])
 65 |   })
 66 | 
 67 |   it('adapts assistant messages', async () => {
 68 |     const prompt = removeCommonIndent(`
 69 |       <assistant>
 70 |         Hello world!
 71 |         <tool-call id="1234" name="get_weather" arguments={{ { location: "Barcelona" } }} />
 72 |       </assistant>
 73 |     `)
 74 | 
 75 |     const { messages } = await render({ prompt, adapter: Adapters.anthropic })
 76 |     expect(messages).toEqual([
 77 |       {
 78 |         role: MessageRole.assistant,
 79 |         content: [
 80 |           { type: 'text', text: 'Hello world!' },
 81 |           {
 82 |             type: 'tool_use',
 83 |             id: '1234',
 84 |             name: 'get_weather',
 85 |             input: { location: 'Barcelona' },
 86 |           },
 87 |         ],
 88 |       },
 89 |     ])
 90 |   })
 91 | 
 92 |   it('adapts tool messages', async () => {
 93 |     const prompt = removeCommonIndent(`
 94 |       <tool id="1234" name="temperature">
 95 |         17ºC
 96 |       </tool>
 97 |     `)
 98 | 
 99 |     const { messages } = await render({ prompt, adapter: Adapters.anthropic })
100 |     expect(messages).toEqual([
101 |       {
102 |         toolName: 'temperature',
103 |         role: MessageRole.user,
104 |         content: [
105 |           {
106 |             type: 'tool_result',
107 |             tool_use_id: '1234',
108 |             content: [{ type: 'text', text: '17ºC' }],
109 |           },
110 |         ],
111 |       },
112 |     ])
113 |   })
114 | })
115 | 


--------------------------------------------------------------------------------
/src/error/error.ts:
--------------------------------------------------------------------------------
  1 | import { Fragment } from '$promptl/parser/interfaces'
  2 | import { locate } from 'locate-character'
  3 | 
  4 | export interface Position {
  5 |   line: number
  6 |   column: number
  7 | }
  8 | 
  9 | type CompileErrorProps = {
 10 |   name: string
 11 |   code: string
 12 |   source: string
 13 |   start: number
 14 |   end?: number
 15 |   fragment?: Fragment
 16 | }
 17 | 
 18 | export default class CompileError extends Error {
 19 |   startIndex: number
 20 |   endIndex: number
 21 |   name: string
 22 |   code: string
 23 |   frame: string
 24 |   start?: Position
 25 |   end?: Position
 26 |   pos?: number
 27 |   fragment?: Fragment
 28 | 
 29 |   constructor({
 30 |     message,
 31 |     name,
 32 |     code,
 33 |     startIndex,
 34 |     endIndex,
 35 |     start,
 36 |     end,
 37 |     frame,
 38 |     fragment,
 39 |   }: {
 40 |     message: string
 41 |     name: string
 42 |     code: string
 43 |     startIndex: number
 44 |     endIndex: number
 45 |     frame: string
 46 |     start?: Position
 47 |     end?: Position
 48 |     fragment?: Fragment
 49 |   }) {
 50 |     super(message)
 51 | 
 52 |     this.name = name
 53 |     this.code = code
 54 |     this.startIndex = startIndex
 55 |     // Legacy alias for startIndex
 56 |     this.pos = this.startIndex
 57 |     this.endIndex = endIndex
 58 |     this.start = start
 59 |     this.end = end
 60 |     this.frame = frame
 61 |     this.fragment = fragment
 62 |   }
 63 | 
 64 |   toString() {
 65 |     if (!this.start) return this.message
 66 |     return `${this.message} (${this.start.line}:${this.start.column})\n${this.frame}`
 67 |   }
 68 | }
 69 | 
 70 | function tabsToSpaces(str: string) {
 71 |   return str.replace(/^\t+/, (match) => match.split('\t').join('  '))
 72 | }
 73 | 
 74 | function getCodeFrame(
 75 |   source: string,
 76 |   line: number,
 77 |   startColumn: number,
 78 |   endColumn: number | undefined,
 79 | ): string {
 80 |   const lines = source.split('\n')
 81 |   const frameStart = Math.max(0, line - 2)
 82 |   const frameEnd = Math.min(line + 3, lines.length)
 83 |   const digits = String(frameEnd + 1).length
 84 |   return lines
 85 |     .slice(frameStart, frameEnd)
 86 |     .map((str, i) => {
 87 |       const isErrorLine = frameStart + i === line
 88 |       const lineNum = String(i + frameStart + 1).padStart(digits, ' ')
 89 |       if (isErrorLine) {
 90 |         const indicator =
 91 |           ' '.repeat(
 92 |             digits + 2 + tabsToSpaces(str.slice(0, startColumn)).length,
 93 |           ) +
 94 |           '^' +
 95 |           '~'.repeat(endColumn ? Math.max(0, endColumn - startColumn - 1) : 0)
 96 |         return `${lineNum}: ${tabsToSpaces(str)}\n\n${indicator}`
 97 |       }
 98 |       return `${lineNum}: ${tabsToSpaces(str)}`
 99 |     })
100 |     .join('\n')
101 | }
102 | 
103 | export function error(message: string, props: CompileErrorProps): never {
104 |   const start = locate(props.source, props.start, {
105 |     offsetLine: 1,
106 |     offsetColumn: 1,
107 |   })
108 |   const end = locate(props.source, props.end ?? props.start, {
109 |     offsetLine: 1,
110 |     offsetColumn: 1,
111 |   })
112 |   const endIndex = props.end ?? props.start
113 |   throw new CompileError({
114 |     message,
115 |     name: props.name,
116 |     code: props.code,
117 |     startIndex: props.start,
118 |     endIndex,
119 |     start,
120 |     end,
121 |     frame: getCodeFrame(
122 |       props.source,
123 |       (start?.line ?? 1) - 1,
124 |       start?.column ?? 0,
125 |       end?.column,
126 |     ),
127 |     fragment: props.fragment,
128 |   })
129 | }
130 | 


--------------------------------------------------------------------------------
/src/compiler/base/nodes/if.test.ts:
--------------------------------------------------------------------------------
  1 | import { AssistantMessage, MessageRole, UserMessage } from '$promptl/types'
  2 | import { describe, expect, it, vi } from 'vitest'
  3 | 
  4 | import { render } from '../..'
  5 | import { removeCommonIndent } from '../../utils'
  6 | 
  7 | describe('conditional expressions', async () => {
  8 |   it('only evaluates the content inside the correct branch', async () => {
  9 |     const prompt = `
 10 |       {{ if foo }}
 11 |         {{ whenTrue() }}
 12 |       {{ else }}
 13 |         {{ whenFalse() }}
 14 |       {{ endif }}
 15 |     `
 16 |     const whenTrue = vi.fn()
 17 |     const whenFalse = vi.fn()
 18 | 
 19 |     await render({
 20 |       prompt: removeCommonIndent(prompt),
 21 |       parameters: {
 22 |         foo: true,
 23 |         whenTrue,
 24 |         whenFalse,
 25 |       },
 26 |     })
 27 | 
 28 |     expect(whenTrue).toHaveBeenCalled()
 29 |     expect(whenFalse).not.toHaveBeenCalled()
 30 | 
 31 |     whenTrue.mockClear()
 32 |     whenFalse.mockClear()
 33 | 
 34 |     await render({
 35 |       prompt: removeCommonIndent(prompt),
 36 |       parameters: {
 37 |         foo: false,
 38 |         whenTrue,
 39 |         whenFalse,
 40 |       },
 41 |     })
 42 | 
 43 |     expect(whenTrue).not.toHaveBeenCalled()
 44 |     expect(whenFalse).toHaveBeenCalled()
 45 |   })
 46 | 
 47 |   it('adds messages conditionally', async () => {
 48 |     const prompt = `
 49 |       {{ if foo }}
 50 |         <user>Foo!</user>
 51 |       {{ else }}
 52 |         <assistant>Bar!</assistant>
 53 |       {{ endif }}
 54 |     `
 55 |     const result1 = await render({
 56 |       prompt: removeCommonIndent(prompt),
 57 |       parameters: {
 58 |         foo: true,
 59 |       },
 60 |     })
 61 |     const result2 = await render({
 62 |       prompt: removeCommonIndent(prompt),
 63 |       parameters: {
 64 |         foo: false,
 65 |       },
 66 |     })
 67 | 
 68 |     expect(result1.messages.length).toBe(1)
 69 |     const message1 = result1.messages[0]! as UserMessage
 70 |     expect(message1.role).toBe('user')
 71 |     expect(message1.content.length).toBe(1)
 72 |     expect(message1.content[0]!.type).toBe('text')
 73 |     expect(message1.content).toEqual([{ type: 'text', text: 'Foo!' }])
 74 | 
 75 |     expect(result2.messages.length).toBe(1)
 76 |     const message2 = result2.messages[0]! as AssistantMessage
 77 |     expect(message2.role).toBe('assistant')
 78 |     expect(message2.content).toEqual([{ type: 'text', text: 'Bar!' }])
 79 |   })
 80 | 
 81 |   it('adds message contents conditionally', async () => {
 82 |     const prompt = `
 83 |       <user>
 84 |         {{ if foo }}
 85 |           Foo!
 86 |         {{ else }}
 87 |           Bar!
 88 |         {{ endif }}
 89 |       </user>
 90 |     `
 91 | 
 92 |     const result1 = await render({
 93 |       prompt: removeCommonIndent(prompt),
 94 |       parameters: { foo: true },
 95 |     })
 96 |     const result2 = await render({
 97 |       prompt: removeCommonIndent(prompt),
 98 |       parameters: { foo: false },
 99 |     })
100 | 
101 |     expect(result1.messages).toEqual([
102 |       {
103 |         role: MessageRole.user,
104 |         name: undefined,
105 |         content: [
106 |           {
107 |             type: 'text',
108 |             text: 'Foo!',
109 |           },
110 |         ],
111 |       },
112 |     ])
113 |     expect(result2.messages).toEqual([
114 |       {
115 |         role: MessageRole.user,
116 |         name: undefined,
117 |         content: [
118 |           {
119 |             type: 'text',
120 |             text: 'Bar!',
121 |           },
122 |         ],
123 |       },
124 |     ])
125 |   })
126 | })
127 | 


--------------------------------------------------------------------------------
/src/compiler/utils.test.ts:
--------------------------------------------------------------------------------
  1 | import { describe, it, expect } from 'vitest'
  2 | import { ZodError, core, z } from 'zod'
  3 | import { getMostSpecificError } from './utils'
  4 | 
  5 | type ZodIssue = core.$ZodIssue
  6 | 
  7 | function makeZodError(issues: ZodIssue[]): ZodError {
  8 |   return new ZodError(issues)
  9 | }
 10 | 
 11 | describe('getMostSpecificError', () => {
 12 |   it('returns the message and path for a simple error', () => {
 13 |     const error = makeZodError([
 14 |       {
 15 |         code: 'invalid_type',
 16 |         expected: 'string',
 17 |         input: 'number',
 18 |         path: ['foo'],
 19 |         message: 'Expected string',
 20 |       },
 21 |     ])
 22 |     const result = getMostSpecificError(error.issues[0]!)
 23 |     expect(result.message).toMatch('Expected type')
 24 |     expect(result.path).toEqual(['foo'])
 25 |   })
 26 | 
 27 |   it('returns the most specific (deepest) error in a nested structure', () => {
 28 |     const unionError = makeZodError([
 29 |       {
 30 |         code: 'invalid_union',
 31 |         errors: [
 32 |           [
 33 |             {
 34 |               code: 'invalid_type',
 35 |               expected: 'string',
 36 |               input: 'number',
 37 |               path: ['foo', 'bar'],
 38 |               message: 'Expected string',
 39 |             },
 40 |           ],
 41 |           [
 42 |             {
 43 |               code: 'invalid_type',
 44 |               expected: 'number',
 45 |               input: 'string',
 46 |               path: ['foo'],
 47 |               message: 'Expected number',
 48 |             },
 49 |           ],
 50 |         ],
 51 |         path: ['foo'],
 52 |         message: 'Invalid union',
 53 |       },
 54 |     ])
 55 |     const result = getMostSpecificError(unionError.issues[0]!)
 56 |     expect(result.path).toEqual(['foo', 'bar'])
 57 |     expect(result.message).toMatch('Expected type')
 58 |   })
 59 | 
 60 |   it('returns the error message and empty path if no issues', () => {
 61 |     const error = makeZodError([
 62 |       {
 63 |         code: 'custom',
 64 |         path: [],
 65 |         message: 'Custom error',
 66 |       },
 67 |     ])
 68 |     const result = getMostSpecificError(error.issues[0]!)
 69 |     expect(result.message).toMatch('Custom error')
 70 |     expect(result.path).toEqual([])
 71 |   })
 72 | 
 73 |   it('handles errors with multiple paths and picks the deepest', () => {
 74 |     const error = makeZodError([
 75 |       {
 76 |         code: 'invalid_type',
 77 |         expected: 'string',
 78 |         input: 'number',
 79 |         path: ['a'],
 80 |         message: 'Expected string',
 81 |       },
 82 |       {
 83 |         code: 'invalid_type',
 84 |         expected: 'number',
 85 |         input: 'string',
 86 |         path: ['a', 'b', 'c'],
 87 |         message: 'Expected number',
 88 |       },
 89 |     ])
 90 |     const result = getMostSpecificError(error.issues[1]!) // The deepest path is at index 1
 91 |     expect(result.path).toEqual(['a', 'b', 'c'])
 92 |     expect(result.message).toMatch('Expected type')
 93 |   })
 94 | 
 95 |   it('handles ZodError thrown by zod schema', () => {
 96 |     const schema = z.object({ foo: z.string() })
 97 |     let error: ZodError | undefined
 98 |     try {
 99 |       schema.parse({ foo: 123 })
100 |     } catch (e) {
101 |       error = e as ZodError
102 |     }
103 |     expect(error).toBeDefined()
104 |     expect(error!.issues.length).toBeGreaterThan(0)
105 |     const result = getMostSpecificError(error!.issues[0]!)
106 |     expect(result.path).toEqual(['foo'])
107 |     expect(result.message).toMatch('Expected type')
108 |   })
109 | })
110 | 


--------------------------------------------------------------------------------
/src/providers/openai/adapter.test.ts:
--------------------------------------------------------------------------------
  1 | import { render } from '$promptl/compiler'
  2 | import { removeCommonIndent } from '$promptl/compiler/utils'
  3 | import { MessageRole } from '$promptl/types'
  4 | import { describe, expect, it } from 'vitest'
  5 | 
  6 | import { Adapters } from '..'
  7 | import { AssistantMessage as OpenAiAssistantMessage } from './types'
  8 | 
  9 | describe('OpenAI adapter', async () => {
 10 |   it('adapts system messages', async () => {
 11 |     const prompt = `Hello world!`
 12 |     const { messages } = await render({ prompt, adapter: Adapters.openai })
 13 |     expect(messages).toEqual([
 14 |       {
 15 |         role: MessageRole.system,
 16 |         content: 'Hello world!', // System messages are defined as a string in OpenAI
 17 |       },
 18 |     ])
 19 |   })
 20 | 
 21 |   it('adapts user messages', async () => {
 22 |     const prompt = removeCommonIndent(`
 23 |       <user name="Image master">
 24 |         Hello world!
 25 |         <content-image>https://image.source/</content-image>
 26 |         <content-file mime="text/plain">text content</content-file>
 27 |         <content-file mime="audio/wav">audio content</content-file>
 28 |       </user>
 29 |     `)
 30 | 
 31 |     const { messages } = await render({ prompt, adapter: Adapters.openai })
 32 |     expect(messages).toEqual([
 33 |       {
 34 |         role: MessageRole.user,
 35 |         name: 'Image master',
 36 |         content: [
 37 |           { type: 'text', text: 'Hello world!' },
 38 |           { type: 'image', image: 'https://image.source/' },
 39 |           { type: 'text', text: 'text content' },
 40 |           {
 41 |             type: 'input_audio',
 42 |             data: 'audio content',
 43 |             format: 'wav',
 44 |           },
 45 |         ],
 46 |       },
 47 |     ])
 48 |   })
 49 | 
 50 |   it('adapts assistant messages', async () => {
 51 |     const prompt = removeCommonIndent(`
 52 |       <assistant>
 53 |         Hello world!
 54 |         <tool-call id="1234" name="get_weather" arguments={{ { location: "Barcelona" } }} />
 55 |       </assistant>
 56 |     `)
 57 | 
 58 |     const { messages } = await render({ prompt, adapter: Adapters.openai })
 59 |     expect(messages).toEqual([
 60 |       {
 61 |         role: MessageRole.assistant,
 62 |         content: [{ type: 'text', text: 'Hello world!' }],
 63 |         tool_calls: [
 64 |           {
 65 |             id: '1234',
 66 |             type: 'function',
 67 |             function: {
 68 |               name: 'get_weather',
 69 |               arguments: JSON.stringify({ location: 'Barcelona' }),
 70 |             },
 71 |           },
 72 |         ],
 73 |       },
 74 |     ])
 75 | 
 76 |     const promptWithoutToolCalls = removeCommonIndent(`
 77 |       <assistant>
 78 |         Hello world!
 79 |       </assistant>
 80 |     `)
 81 | 
 82 |     const { messages: messagesWithoutToolCalls } = await render({
 83 |       prompt: promptWithoutToolCalls,
 84 |       adapter: Adapters.openai,
 85 |     })
 86 | 
 87 |     expect(messagesWithoutToolCalls[0]!.role).toBe(MessageRole.assistant)
 88 |     expect(
 89 |       (messagesWithoutToolCalls[0] as OpenAiAssistantMessage).tool_calls,
 90 |     ).not.toBeDefined()
 91 |   })
 92 | 
 93 |   it('adapts tool messages', async () => {
 94 |     const prompt = removeCommonIndent(`
 95 |       <tool id="1234" name="temperature">
 96 |         17ºC
 97 |       </tool>
 98 |     `)
 99 | 
100 |     const { messages } = await render({ prompt, adapter: Adapters.openai })
101 |     expect(messages).toEqual([
102 |       {
103 |         role: MessageRole.tool,
104 |         tool_call_id: '1234',
105 |         toolName: 'temperature',
106 |         content: [{ type: 'text', text: '17ºC' }],
107 |       },
108 |     ])
109 |   })
110 | })
111 | 


--------------------------------------------------------------------------------
/src/providers/openai-responses/utils/parsers/outputMessage.ts:
--------------------------------------------------------------------------------
  1 | import {
  2 |   MessageInputItem,
  3 |   ResponseOutputMessage,
  4 |   ResponseOutputText,
  5 |   ResponseOutputRefusal,
  6 | } from '$promptl/providers/openai-responses/types'
  7 | import {
  8 |   ContentType,
  9 |   Message as PromptlMessage,
 10 |   MessageRole,
 11 |   TextContent,
 12 |   AssistantMessage,
 13 |   ToolCallContent,
 14 | } from '$promptl/types'
 15 | 
 16 | export function isOutputMessage(
 17 |   message: MessageInputItem,
 18 | ): message is ResponseOutputMessage {
 19 |   return (
 20 |     message.type === 'message' &&
 21 |     'status' in message &&
 22 |     message.role === 'assistant' &&
 23 |     'id' in message
 24 |   )
 25 | }
 26 | 
 27 | function isOutputTextContent(
 28 |   content: ResponseOutputMessage['content'][0],
 29 | ): content is ResponseOutputText {
 30 |   return content.type === 'output_text'
 31 | }
 32 | 
 33 | function isOutputRefusalContent(
 34 |   content: ResponseOutputMessage['content'][0],
 35 | ): content is ResponseOutputRefusal {
 36 |   return content.type === 'refusal'
 37 | }
 38 | 
 39 | function getOpenAIBuiltinFunctionCallId({
 40 |   messageId,
 41 |   webSearch,
 42 |   fileSearch,
 43 | }: {
 44 |   messageId: string
 45 |   webSearch: Map<string, string>
 46 |   fileSearch: Map<string, string>
 47 | }) {
 48 |   const id = messageId.replace(/^msg_/, '')
 49 |   const callWebSearchId = webSearch.get(id)
 50 |   if (callWebSearchId) {
 51 |     return { toolId: callWebSearchId, toolName: 'web_search_call' }
 52 |   }
 53 | 
 54 |   const callFileSearchId = fileSearch.get(id)
 55 | 
 56 |   if (callFileSearchId) {
 57 |     return { toolId: callFileSearchId, toolName: 'file_search_call' }
 58 |   }
 59 | 
 60 |   return null
 61 | }
 62 | 
 63 | export function parseOutputMessage({
 64 |   message,
 65 |   webSearch,
 66 |   fileSearch,
 67 | }: {
 68 |   message: ResponseOutputMessage
 69 |   webSearch: Map<string, string>
 70 |   fileSearch: Map<string, string>
 71 | }) {
 72 |   const builtinTool = getOpenAIBuiltinFunctionCallId({
 73 |     messageId: message.id,
 74 |     webSearch,
 75 |     fileSearch,
 76 |   })
 77 |   if (builtinTool) {
 78 |     return {
 79 |       id: message.id,
 80 |       role: MessageRole.assistant,
 81 |       status: message.status,
 82 |       content: message.content
 83 |         .map((content) => {
 84 |           if (!isOutputTextContent(content)) return null
 85 | 
 86 |           return {
 87 |             type: ContentType.toolCall,
 88 |             toolCallId: builtinTool.toolId,
 89 |             toolName: builtinTool.toolName,
 90 |             toolArguments: {
 91 |               text: content.text,
 92 |               annotations: content.annotations,
 93 |             },
 94 |           } satisfies ToolCallContent
 95 |         })
 96 |         .filter((c) => c !== null),
 97 |     } satisfies AssistantMessage
 98 |   }
 99 | 
100 |   return {
101 |     id: message.id,
102 |     role: MessageRole.assistant,
103 |     status: message.status,
104 |     content: message.content.map((content) => {
105 |       if (isOutputTextContent(content)) {
106 |         return {
107 |           type: ContentType.text,
108 |           annotations: content.annotations,
109 |           text: content.text,
110 |         } satisfies TextContent
111 |       }
112 | 
113 |       if (isOutputRefusalContent(content)) {
114 |         return {
115 |           type: ContentType.text,
116 |           text: content.refusal,
117 |         } satisfies TextContent
118 |       }
119 | 
120 |       // @ts-expect-error - Should not happen
121 |       const type = content.type
122 |       throw new Error(
123 |         `Unknown type ${type} in OpenAIResponse message output content`,
124 |       )
125 |     }),
126 |   } satisfies PromptlMessage
127 | }
128 | 


--------------------------------------------------------------------------------
/src/compiler/base/nodes/tag.ts:
--------------------------------------------------------------------------------
  1 | import {
  2 |   isChainStepTag,
  3 |   isContentTag,
  4 |   isMessageTag,
  5 |   isRefTag,
  6 |   isScopeTag,
  7 | } from '$promptl/compiler/utils'
  8 | import errors from '$promptl/error/errors'
  9 | import {
 10 |   ChainStepTag,
 11 |   ContentTag,
 12 |   ElementTag,
 13 |   MessageTag,
 14 |   ReferenceTag,
 15 |   ScopeTag,
 16 | } from '$promptl/parser/interfaces'
 17 | 
 18 | import { CompileNodeContext } from '../types'
 19 | import { compile as resolveContent } from './tags/content'
 20 | import { compile as resolveMessage } from './tags/message'
 21 | import { compile as resolveRef } from './tags/ref'
 22 | import { compile as resolveChainStep } from './tags/step'
 23 | import { compile as resolveScope } from './tags/scope'
 24 | 
 25 | async function resolveTagAttributes({
 26 |   node: tagNode,
 27 |   scope,
 28 |   resolveExpression,
 29 | }: CompileNodeContext<ElementTag>): Promise<Record<string, unknown>> {
 30 |   const attributeNodes = tagNode.attributes
 31 |   if (attributeNodes.length === 0) return {}
 32 | 
 33 |   const attributes: Record<string, unknown> = {}
 34 |   for (const attributeNode of attributeNodes) {
 35 |     const { name, value } = attributeNode
 36 |     if (value === true) {
 37 |       attributes[name] = true
 38 |       continue
 39 |     }
 40 | 
 41 |     const accumulatedValue: unknown[] = []
 42 |     for await (const node of value) {
 43 |       if (node.type === 'Text') {
 44 |         if (node.data) {
 45 |           accumulatedValue.push(node.data)
 46 |         }
 47 |         continue
 48 |       }
 49 | 
 50 |       if (node.type === 'MustacheTag') {
 51 |         const expression = node.expression
 52 |         const resolvedValue = await resolveExpression(expression, scope)
 53 |         if (resolvedValue === undefined) continue
 54 |         accumulatedValue.push(resolvedValue)
 55 |         continue
 56 |       }
 57 |     }
 58 | 
 59 |     const finalValue =
 60 |       accumulatedValue.length > 1
 61 |         ? accumulatedValue.map(String).join('')
 62 |         : accumulatedValue[0]
 63 | 
 64 |     attributes[name] = finalValue
 65 |   }
 66 | 
 67 |   return attributes
 68 | }
 69 | 
 70 | export async function compile(props: CompileNodeContext<ElementTag>) {
 71 |   const {
 72 |     node,
 73 |     scope,
 74 |     isInsideStepTag,
 75 |     isInsideContentTag,
 76 |     isInsideMessageTag,
 77 |     fullPath,
 78 |     resolveBaseNode,
 79 |     baseNodeError,
 80 |     groupStrayText,
 81 |   } = props
 82 |   groupStrayText()
 83 | 
 84 |   const attributes = await resolveTagAttributes(props)
 85 | 
 86 |   if (isContentTag(node)) {
 87 |     await resolveContent(props as CompileNodeContext<ContentTag>, attributes)
 88 |     return
 89 |   }
 90 | 
 91 |   if (isMessageTag(node)) {
 92 |     await resolveMessage(props as CompileNodeContext<MessageTag>, attributes)
 93 |     return
 94 |   }
 95 | 
 96 |   if (isRefTag(node)) {
 97 |     await resolveRef(props as CompileNodeContext<ReferenceTag>, attributes)
 98 |     return
 99 |   }
100 | 
101 |   if (isScopeTag(node)) {
102 |     await resolveScope(props as CompileNodeContext<ScopeTag>, attributes)
103 |     return
104 |   }
105 | 
106 |   if (isChainStepTag(node)) {
107 |     await resolveChainStep(
108 |       props as CompileNodeContext<ChainStepTag>,
109 |       attributes,
110 |     )
111 |     return
112 |   }
113 | 
114 |   //@ts-ignore - Linter knows there *should* not be another type of tag.
115 |   baseNodeError(errors.unknownTag(node.name), node)
116 | 
117 |   //@ts-ignore - ditto
118 |   for await (const childNode of node.children ?? []) {
119 |     await resolveBaseNode({
120 |       node: childNode,
121 |       scope,
122 |       isInsideStepTag,
123 |       isInsideMessageTag,
124 |       isInsideContentTag,
125 |       fullPath,
126 |     })
127 |   }
128 | }
129 | 


--------------------------------------------------------------------------------
/src/compiler/scope.ts:
--------------------------------------------------------------------------------
  1 | export type ScopePointers = { [key: string]: number }
  2 | export type ScopeStash = unknown[]
  3 | 
  4 | export default class Scope {
  5 |   /**
  6 |    * Global stash
  7 |    * All variable values are stored in a single global array. This is done to allow multiple
  8 |    * scopes to share the same variable values and be able to modify them.
  9 |    *
 10 |    * For example:
 11 |    * ```md
 12 |    * {var1 = 1}
 13 |    * {#if <condition>}
 14 |    *  {var1 = 2}
 15 |    *  {var2 = 3}
 16 |    * {/if}
 17 |    * ```
 18 |    * In this case, there are two scopes: root and if. Both scopes share the same variable `var1`,
 19 |    * and modifying it in the if scope should also modify it in the root scope. But `var2` is only
 20 |    * defined in the if scope and should not be accessible in the root scope.
 21 |    *
 22 |    * Local pointers
 23 |    * Every scope has its own local pointers that contains the indexes of the variables in the global stash.
 24 |    */
 25 | 
 26 |   // Stash of every variable value in the global scope
 27 |   private globalStash: ScopeStash = []
 28 |   // Index of every variable in the stash in the current scope
 29 |   private localPointers: ScopePointers = {}
 30 | 
 31 |   constructor(initialState: Record<string, unknown> = {}) {
 32 |     for (const [key, value] of Object.entries(initialState)) {
 33 |       this.localPointers[key] = this.addToStash(value)
 34 |     }
 35 |   }
 36 | 
 37 |   static withStash(stash: ScopeStash): Scope {
 38 |     const scope = new Scope()
 39 |     scope.globalStash = stash
 40 |     return scope
 41 |   }
 42 | 
 43 |   exists(name: string): boolean {
 44 |     return name in this.localPointers
 45 |   }
 46 | 
 47 |   get(name: string): unknown {
 48 |     const index = this.localPointers[name] ?? undefined
 49 | 
 50 |     if (index === undefined) {
 51 |       throw new Error(`Variable '${name}' does not exist`)
 52 |     }
 53 | 
 54 |     return this.readFromStash(index)
 55 |   }
 56 | 
 57 |   set(name: string, value: unknown): void {
 58 |     if (!this.exists(name)) {
 59 |       this.localPointers[name] = this.addToStash(value)
 60 |       return
 61 |     }
 62 |     const index = this.localPointers[name]!
 63 |     this.modifyStash(index, value)
 64 |   }
 65 | 
 66 |   copy(localPointers?: ScopePointers): Scope {
 67 |     const scope = new Scope()
 68 |     scope.globalStash = this.globalStash
 69 |     scope.localPointers = { ...(localPointers ?? this.localPointers) }
 70 |     return scope
 71 |   }
 72 | 
 73 |   getStash(): ScopeStash {
 74 |     return this.globalStash
 75 |   }
 76 | 
 77 |   getPointers(): ScopePointers {
 78 |     return this.localPointers
 79 |   }
 80 | 
 81 |   setPointers(pointers: ScopePointers): void {
 82 |     this.localPointers = pointers
 83 |   }
 84 | 
 85 |   setStash(stash: ScopeStash): void {
 86 |     this.globalStash = stash
 87 |   }
 88 | 
 89 |   serialize(): { stash: ScopeStash; pointers: ScopePointers } {
 90 |     return {
 91 |       stash: this.globalStash,
 92 |       pointers: this.localPointers,
 93 |     }
 94 |   }
 95 | 
 96 |   private readFromStash(index: number): unknown {
 97 |     return this.globalStash[index]
 98 |   }
 99 | 
100 |   private addToStash(value: unknown): number {
101 |     this.globalStash.push(value)
102 |     return this.globalStash.length - 1
103 |   }
104 | 
105 |   private modifyStash(index: number, value: unknown): void {
106 |     this.globalStash[index] = value
107 |   }
108 | }
109 | 
110 | export type ScopeContext = {
111 |   // If defined, all usedUndefinedVariables that are not in this set will return an error
112 |   onlyPredefinedVariables?: Set<string>
113 |   // Variables that are not in current scope but have been used
114 |   usedUndefinedVariables: Set<string>
115 |   // Variables that are in current scope
116 |   definedVariables: Set<string>
117 | }
118 | 


--------------------------------------------------------------------------------
/src/compiler/base/nodes/tags/scope.test.ts:
--------------------------------------------------------------------------------
  1 | import { Adapters, Chain, render } from '$promptl/index'
  2 | import { complete } from '$promptl/compiler/test/helpers'
  3 | import { removeCommonIndent } from '$promptl/compiler/utils'
  4 | import { MessageRole, SystemMessage, UserMessage } from '$promptl/types'
  5 | import { describe, expect, it, vi } from 'vitest'
  6 | 
  7 | describe('scope tags', async () => {
  8 |   it('returns contents as usual', async () => {
  9 |     const prompt = removeCommonIndent(`
 10 |       <user>
 11 |         <scope>
 12 |           <content-text>This is a text content!</content-text>
 13 |         </scope>
 14 |       </user>
 15 |     `)
 16 | 
 17 |     const result = await render({ prompt, adapter: Adapters.default })
 18 | 
 19 |     expect(result.messages.length).toBe(1)
 20 |     expect(result.messages[0]!.role).toBe(MessageRole.user)
 21 |     const message = result.messages[0]! as UserMessage
 22 |     expect(message.content).toEqual([
 23 |       {
 24 |         type: 'text',
 25 |         text: 'This is a text content!',
 26 |       },
 27 |     ])
 28 |   })
 29 | 
 30 |   it('contains its own scope', async () => {
 31 |     const prompt = removeCommonIndent(`
 32 |       {{ foo = 'foo' }}
 33 |       {{ bar = 'bar' }}
 34 | 
 35 |       <scope baz={{ bar }}>
 36 |         {{ baz = 'baz' }}
 37 |         {{ foo = 'new foo' }}
 38 |       </scope>
 39 | 
 40 |       <content-text>{{ foo }}</content-text>
 41 |       <content-text>{{ bar }}</content-text>
 42 |     `)
 43 | 
 44 |     const result = await render({ prompt, adapter: Adapters.default })
 45 | 
 46 |     expect(result.messages.length).toBe(1)
 47 |     const message = result.messages[0]! as SystemMessage
 48 |     expect(message.content).toEqual([
 49 |       {
 50 |         type: 'text',
 51 |         text: 'foo',
 52 |       },
 53 |       {
 54 |         type: 'text',
 55 |         text: 'bar',
 56 |       },
 57 |     ])
 58 |   })
 59 | 
 60 |   it('does not automatically inherit variables or parameters from parents', async () => {
 61 |     const prompt = removeCommonIndent(`
 62 |       {{ foo = 'bar' }}
 63 |       <scope>
 64 |         {{ foo == 'bar' }}
 65 |       </scope>
 66 |     `)
 67 | 
 68 |     const result = await render({ prompt, adapter: Adapters.default })
 69 |     expect(result.messages.length).toBe(1)
 70 |     const message = result.messages[0]! as SystemMessage
 71 |     expect(message.content).toEqual([
 72 |       {
 73 |         type: 'text',
 74 |         text: 'false',
 75 |       },
 76 |     ])
 77 |   })
 78 | 
 79 |   it('can inherit parameters from parents if explicitly passed', async () => {
 80 |     const prompt = removeCommonIndent(`
 81 |       <scope foo={{ foo }}>
 82 |         {{ foo }}
 83 |       </scope>
 84 |     `)
 85 | 
 86 |     const result = await render({
 87 |       prompt,
 88 |       parameters: { foo: 'bar' },
 89 |       adapter: Adapters.default,
 90 |     })
 91 | 
 92 |     expect(result.messages.length).toBe(1)
 93 |     const message = result.messages[0]! as SystemMessage
 94 |     expect(message.content).toEqual([
 95 |       {
 96 |         type: 'text',
 97 |         text: 'bar',
 98 |       },
 99 |     ])
100 |   })
101 | 
102 |   it('node state from scope is correctly cached during steps', async () => {
103 |     const func = vi.fn()
104 | 
105 |     const prompt = removeCommonIndent(`
106 |       <scope func={{ func }}>
107 |         <step>
108 |           {{ func() }}
109 |         </step>
110 |         {{ for i in [1, 2] }}
111 |           <step>
112 |             {{ func() }}
113 |           </step>
114 |         {{ endfor }}
115 |       </scope>
116 |     `)
117 | 
118 |     const chain = new Chain({
119 |       prompt,
120 |       parameters: { func },
121 |     })
122 | 
123 |     await complete({ chain })
124 | 
125 |     expect(func).toHaveBeenCalledTimes(3)
126 |   })
127 | })
128 | 


--------------------------------------------------------------------------------
/src/rpc/server.ts:
--------------------------------------------------------------------------------
  1 | import procedures from './procedures'
  2 | import { RPC } from './types'
  3 | 
  4 | const enum Fd {
  5 |   StdIn = 0,
  6 |   StdOut = 1,
  7 |   StdErr = 2,
  8 | }
  9 | 
 10 | const CHUNK_SIZE = 1024
 11 | 
 12 | const decoder = new TextDecoder()
 13 | const encoder = new TextEncoder()
 14 | 
 15 | function receive(): RPC.Call<any>[] {
 16 |   const chunks = []
 17 |   let size = 0
 18 | 
 19 |   while (true) {
 20 |     const chunk = new Uint8Array(CHUNK_SIZE)
 21 | 
 22 |     const bytes = Javy.IO.readSync(Fd.StdIn, chunk)
 23 |     if (bytes === 0) {
 24 |       break
 25 |     }
 26 | 
 27 |     size += bytes
 28 |     chunks.push(chunk.subarray(0, bytes))
 29 |   }
 30 | 
 31 |   const { calls } = chunks.reduce(
 32 |     ({ offset, calls }, chunk) => {
 33 |       calls.set(chunk, offset)
 34 |       offset += chunk.length
 35 | 
 36 |       return { offset, calls }
 37 |     },
 38 |     { offset: 0, calls: new Uint8Array(size) },
 39 |   )
 40 | 
 41 |   const payload = JSON.parse(decoder.decode(calls).trim())
 42 | 
 43 |   return payload
 44 | }
 45 | 
 46 | function jsonReplacer(_key: string, value: any): any {
 47 |   if (value instanceof Error) {
 48 |     const error: Record<string, any> = {}
 49 | 
 50 |     for (const name of Object.getOwnPropertyNames(value)) {
 51 |       const property = (value as any)[name]
 52 | 
 53 |       if (property instanceof Error) {
 54 |         error[name] = jsonReplacer(name, property)
 55 |       } else {
 56 |         error[name] = property
 57 |       }
 58 |     }
 59 | 
 60 |     return error
 61 |   }
 62 | 
 63 |   return value
 64 | }
 65 | 
 66 | function send(results: RPC.Result<any>[]) {
 67 |   const payload = encoder.encode(JSON.stringify(results, jsonReplacer) + '\n')
 68 | 
 69 |   Javy.IO.writeSync(Fd.StdOut, payload)
 70 | }
 71 | 
 72 | async function execute(calls: RPC.Call<any>[]): Promise<RPC.Result<any>[]> {
 73 |   const results: RPC.Result<any>[] = []
 74 | 
 75 |   for (const call of calls) {
 76 |     const handler = procedures[call.procedure]
 77 |     if (!handler) {
 78 |       results.push({
 79 |         error: {
 80 |           code: RPC.ErrorCode.ExecuteError,
 81 |           message: `Unknown RPC procedure: ${call.procedure}`,
 82 |         },
 83 |       })
 84 |       continue
 85 |     }
 86 | 
 87 |     try {
 88 |       results.push({
 89 |         value: await handler(call.parameters),
 90 |       })
 91 |     } catch (error: any) {
 92 |       results.push({
 93 |         error: {
 94 |           code: RPC.ErrorCode.ProcedureError,
 95 |           message: error.message || String(error),
 96 |           details: error,
 97 |         },
 98 |       })
 99 |     }
100 |   }
101 | 
102 |   return results
103 | }
104 | 
105 | export function serve() {
106 |   let calls: RPC.Call<any>[]
107 | 
108 |   try {
109 |     calls = receive()
110 |   } catch (error: any) {
111 |     send([
112 |       {
113 |         error: {
114 |           code: RPC.ErrorCode.ReceiveError,
115 |           message: `Failed to unmarshal RPC calls: ${error.message || String(error)}`,
116 |           details: error,
117 |         },
118 |       },
119 |     ])
120 |     return
121 |   }
122 | 
123 |   execute(calls)
124 |     .then((results) => {
125 |       try {
126 |         send(results)
127 |       } catch (error: any) {
128 |         send([
129 |           {
130 |             error: {
131 |               code: RPC.ErrorCode.SendError,
132 |               message: `Failed to marshal RPC results: ${error.message || String(error)}`,
133 |               details: error,
134 |             },
135 |           },
136 |         ])
137 |       }
138 |     })
139 |     .catch((error: any) => {
140 |       send([
141 |         {
142 |           error: {
143 |             code: RPC.ErrorCode.ExecuteError,
144 |             message: `Failed to execute RPC procedures: ${error.message || String(error)}`,
145 |             details: error,
146 |           },
147 |         },
148 |       ])
149 |     })
150 | }
151 | 


--------------------------------------------------------------------------------
/src/compiler/base/nodes/tags/content.ts:
--------------------------------------------------------------------------------
  1 | import { CUSTOM_CONTENT_TYPE_ATTR, TAG_NAMES } from '$promptl/constants'
  2 | import errors from '$promptl/error/errors'
  3 | import { ContentTag } from '$promptl/parser/interfaces'
  4 | import { ContentType, ContentTypeTagName } from '$promptl/types'
  5 | 
  6 | import { CompileNodeContext } from '../../types'
  7 | 
  8 | export async function compile(
  9 |   {
 10 |     node,
 11 |     scope,
 12 |     isInsideStepTag,
 13 |     isInsideMessageTag,
 14 |     isInsideContentTag,
 15 |     fullPath,
 16 |     resolveBaseNode,
 17 |     baseNodeError,
 18 |     popStrayText,
 19 |     addContent,
 20 |   }: CompileNodeContext<ContentTag>,
 21 |   attributes: Record<string, unknown>,
 22 | ) {
 23 |   if (isInsideContentTag) {
 24 |     baseNodeError(errors.contentTagInsideContent, node)
 25 |   }
 26 | 
 27 |   for await (const childNode of node.children ?? []) {
 28 |     await resolveBaseNode({
 29 |       node: childNode,
 30 |       scope,
 31 |       isInsideStepTag,
 32 |       isInsideMessageTag,
 33 |       isInsideContentTag: true,
 34 |       fullPath,
 35 |     })
 36 |   }
 37 | 
 38 |   let type: ContentType
 39 |   if (node.name === TAG_NAMES.content) {
 40 |     if (attributes[CUSTOM_CONTENT_TYPE_ATTR] === undefined) {
 41 |       baseNodeError(errors.messageTagWithoutRole, node)
 42 |     }
 43 |     type = attributes[CUSTOM_CONTENT_TYPE_ATTR] as ContentType
 44 |     delete attributes[CUSTOM_CONTENT_TYPE_ATTR]
 45 |   } else {
 46 |     const contentTypeKeysFromTagName = Object.fromEntries(
 47 |       Object.entries(ContentTypeTagName).map(([k, v]) => [v, k]),
 48 |     )
 49 |     type =
 50 |       ContentType[
 51 |         contentTypeKeysFromTagName[node.name] as keyof typeof ContentType
 52 |       ]
 53 |   }
 54 | 
 55 |   const stray = popStrayText()
 56 | 
 57 |   if (type === ContentType.text && stray.text.length > 0) {
 58 |     addContent({
 59 |       node,
 60 |       content: {
 61 |         ...attributes,
 62 |         type: ContentType.text,
 63 |         text: stray.text,
 64 |         _promptlSourceMap: stray.sourceMap,
 65 |       },
 66 |     })
 67 |     return
 68 |   }
 69 | 
 70 |   if (type === ContentType.image) {
 71 |     if (!stray.text.length) {
 72 |       baseNodeError(errors.emptyContentTag, node)
 73 |     }
 74 | 
 75 |     addContent({
 76 |       node,
 77 |       content: {
 78 |         ...attributes,
 79 |         type: ContentType.image,
 80 |         image: stray.text,
 81 |         _promptlSourceMap: stray.sourceMap,
 82 |       },
 83 |     })
 84 |     return
 85 |   }
 86 | 
 87 |   if (type === ContentType.file) {
 88 |     if (!stray.text.length) {
 89 |       baseNodeError(errors.emptyContentTag, node)
 90 |     }
 91 | 
 92 |     const { mime: mimeType, ...rest } = attributes
 93 |     if (!mimeType) baseNodeError(errors.fileTagWithoutMimeType, node)
 94 | 
 95 |     addContent({
 96 |       node,
 97 |       content: {
 98 |         ...rest,
 99 |         type: ContentType.file,
100 |         file: stray.text,
101 |         mimeType: String(mimeType),
102 |         _promptlSourceMap: stray.sourceMap,
103 |       },
104 |     })
105 |     return
106 |   }
107 | 
108 |   if (type == ContentType.toolCall) {
109 |     const { id, name, ...rest } = attributes
110 |     if (!id) baseNodeError(errors.toolCallTagWithoutId, node)
111 |     if (!name) baseNodeError(errors.toolCallWithoutName, node)
112 | 
113 |     let toolArguments = rest['arguments']
114 |     delete rest['arguments']
115 |     if (toolArguments && typeof toolArguments === 'string') {
116 |       try {
117 |         rest['arguments'] = JSON.parse(toolArguments)
118 |       } catch {
119 |         baseNodeError(errors.invalidToolCallArguments, node)
120 |       }
121 |     }
122 | 
123 |     addContent({
124 |       node,
125 |       content: {
126 |         ...rest,
127 |         type: ContentType.toolCall,
128 |         toolCallId: String(id),
129 |         toolName: String(name),
130 |         toolArguments: (toolArguments ?? {}) as Record<string, unknown>,
131 |       },
132 |     })
133 |     return
134 |   }
135 | 
136 |   baseNodeError(errors.invalidContentType(type), node)
137 | }
138 | 


--------------------------------------------------------------------------------
/src/compiler/chain-serialize.test.ts:
--------------------------------------------------------------------------------
  1 | import { describe, expect, it } from 'vitest'
  2 | 
  3 | import { Adapters } from '$promptl/providers'
  4 | import { MessageRole } from '$promptl/types'
  5 | import { Chain } from './chain'
  6 | import { removeCommonIndent } from './utils'
  7 | 
  8 | describe('serialize chain', async () => {
  9 |   it('serialize without running step', async () => {
 10 |     const prompt = removeCommonIndent(`
 11 |       <step>
 12 |         Before step
 13 |       </step>
 14 |       <step>
 15 |         After step
 16 |       </step>
 17 |     `)
 18 | 
 19 |     const chain = new Chain({ prompt, adapter: Adapters.default })
 20 |     const serialized = chain.serialize()
 21 | 
 22 |     expect(serialized).toEqual({
 23 |       rawText: prompt,
 24 |       scope: {
 25 |         pointers: {},
 26 |         stash: [],
 27 |       },
 28 |       completed: false,
 29 |       didStart: false,
 30 |       adapterType: 'default',
 31 |       compilerOptions: {},
 32 |       globalConfig: undefined,
 33 |       ast: expect.any(Object),
 34 |       globalMessages: [],
 35 |     })
 36 |   })
 37 | 
 38 |   it('serialize with single step', async () => {
 39 |     const prompt = removeCommonIndent(`
 40 |       ---
 41 |       provider: OpenAI_PATATA
 42 |       model: gpt-4
 43 |       ---
 44 |       {{ foo = 'foo' }}
 45 |       A message
 46 |     `)
 47 | 
 48 |     const chain = new Chain({
 49 |       prompt: removeCommonIndent(prompt),
 50 |       parameters: {},
 51 |       adapter: Adapters.default,
 52 |     })
 53 |     await chain.step()
 54 |     const serialized = chain.serialize()
 55 | 
 56 |     expect(serialized).toEqual({
 57 |       rawText: prompt,
 58 |       scope: {
 59 |         pointers: { foo: 0 },
 60 |         stash: ['foo'],
 61 |       },
 62 |       completed: false,
 63 |       didStart: true,
 64 |       adapterType: 'default',
 65 |       compilerOptions: {},
 66 |       globalConfig: {
 67 |         provider: 'OpenAI_PATATA',
 68 |         model: 'gpt-4',
 69 |       },
 70 |       ast: expect.any(Object),
 71 |       globalMessages: [
 72 |         {
 73 |           role: 'system',
 74 |           content: [{ type: 'text', text: 'A message' }],
 75 |         },
 76 |       ],
 77 |     })
 78 |   })
 79 | 
 80 |   it('serialize 2 steps', async () => {
 81 |     const prompt = removeCommonIndent(`
 82 |       <step>
 83 |         {{foo = 5}}
 84 |       </step>
 85 |       <step>
 86 |         {{foo += 1}}
 87 |       </step>
 88 |       <step>
 89 |         {{foo}}
 90 |       </step>
 91 |     `)
 92 | 
 93 |     const chain = new Chain({ prompt, adapter: Adapters.openai })
 94 |     await chain.step()
 95 |     await chain.step('First step response')
 96 |     const serialized = chain.serialize()
 97 | 
 98 |     expect(serialized).toEqual({
 99 |       rawText: prompt,
100 |       scope: {
101 |         pointers: { foo: 0 },
102 |         stash: [6],
103 |       },
104 |       completed: false,
105 |       didStart: true,
106 |       adapterType: 'openai',
107 |       compilerOptions: { includeSourceMap: false },
108 |       globalConfig: undefined,
109 |       ast: expect.any(Object),
110 |       globalMessages: [
111 |         {
112 |           role: 'assistant',
113 |           content: [{ type: 'text', text: 'First step response' }],
114 |         },
115 |       ],
116 |     })
117 |   })
118 | 
119 |   it('serialize parameters', async () => {
120 |     const prompt = removeCommonIndent(`
121 |        Hello {{name}}
122 |     `)
123 | 
124 |     const chain = new Chain({
125 |       prompt,
126 |       parameters: { name: 'Paco' },
127 |       adapter: Adapters.default,
128 |       defaultRole: MessageRole.user,
129 |     })
130 |     await chain.step()
131 |     const serialized = chain.serialize()
132 | 
133 |     expect(serialized).toEqual({
134 |       rawText: prompt,
135 |       adapterType: 'default',
136 |       scope: { pointers: { name: 0 }, stash: ['Paco'] },
137 |       completed: false,
138 |       didStart: true,
139 |       compilerOptions: { defaultRole: 'user' },
140 |       ast: expect.any(Object),
141 |       globalConfig: undefined,
142 |       globalMessages: [
143 |         {
144 |           role: 'user',
145 |           content: [{ type: 'text', text: 'Hello Paco' }],
146 |         },
147 |       ],
148 |     })
149 |   })
150 | })
151 | 


--------------------------------------------------------------------------------
/src/providers/vercel-ai/adapter.test.ts:
--------------------------------------------------------------------------------
  1 | import { render } from '$promptl/compiler'
  2 | import { removeCommonIndent } from '$promptl/compiler/utils'
  3 | import { MessageRole, ContentType } from '$promptl/types'
  4 | import { describe, it, expect } from 'vitest'
  5 | 
  6 | import { Adapters } from '..'
  7 | 
  8 | describe('VercelAIAdapter', () => {
  9 |   it('adapts system messages', async () => {
 10 |     const prompt = `You are a helpful assistant`
 11 |     const { messages } = await render({
 12 |       prompt,
 13 |       adapter: Adapters['vercel'],
 14 |     })
 15 |     expect(messages).toEqual([
 16 |       {
 17 |         role: MessageRole.system,
 18 |         content: 'You are a helpful assistant',
 19 |       },
 20 |     ])
 21 |   })
 22 | 
 23 |   it('adapts user messages', async () => {
 24 |     const prompt = removeCommonIndent(`
 25 |       <user>
 26 |         Hello world!
 27 |         <content-image>https://image.source/</content-image>
 28 |         <content-file mime="text/plain">text content</content-file>
 29 |       </user>
 30 |     `)
 31 | 
 32 |     const { messages } = await render({
 33 |       prompt,
 34 |       adapter: Adapters['vercel'],
 35 |     })
 36 |     expect(messages).toEqual([
 37 |       {
 38 |         role: MessageRole.user,
 39 |         content: [
 40 |           { type: ContentType.text, text: 'Hello world!' },
 41 |           { type: ContentType.image, image: 'https://image.source/' },
 42 |           {
 43 |             type: ContentType.file,
 44 |             data: 'text content',
 45 |             mediaType: 'text/plain',
 46 |           },
 47 |         ],
 48 |       },
 49 |     ])
 50 |   })
 51 | 
 52 |   it('adapts assistant messages', async () => {
 53 |     const prompt = removeCommonIndent(`
 54 |       <assistant>
 55 |         I will help you with that.
 56 |       </assistant>
 57 |     `)
 58 | 
 59 |     const { messages } = await render({
 60 |       prompt,
 61 |       adapter: Adapters['vercel'],
 62 |     })
 63 |     expect(messages).toEqual([
 64 |       {
 65 |         role: MessageRole.assistant,
 66 |         content: [
 67 |           { type: ContentType.text, text: 'I will help you with that.' },
 68 |         ],
 69 |       },
 70 |     ])
 71 |   })
 72 | 
 73 |   it('adapts tool messages', async () => {
 74 |     const prompt = removeCommonIndent(`
 75 |       <tool id="call_123" name="get_weather">
 76 |         The weather is sunny.
 77 |       </tool>
 78 |     `)
 79 | 
 80 |     const { messages } = await render({
 81 |       prompt,
 82 |       adapter: Adapters['vercel'],
 83 |     })
 84 |     expect(messages).toEqual([
 85 |       {
 86 |         role: MessageRole.tool,
 87 |         content: [
 88 |           {
 89 |             type: 'tool-result',
 90 |             toolCallId: 'call_123',
 91 |             toolName: 'get_weather',
 92 |             result: 'The weather is sunny.',
 93 |           },
 94 |         ],
 95 |       },
 96 |     ])
 97 |   })
 98 | 
 99 |   it('adapts system messages with config', async () => {
100 |     const prompt = removeCommonIndent(`
101 |       ---
102 |       temperature: 0.8
103 |       maxTokens: 1000
104 |       ---
105 |       You are a helpful assistant
106 |     `)
107 | 
108 |     const { messages, config } = await render({
109 |       prompt,
110 |       adapter: Adapters['vercel'],
111 |     })
112 |     expect(config).toEqual({ temperature: 0.8, maxTokens: 1000 })
113 |     expect(messages).toEqual([
114 |       {
115 |         role: MessageRole.system,
116 |         content: 'You are a helpful assistant',
117 |       },
118 |     ])
119 |   })
120 | 
121 |   it('handles assistant messages with tool calls', async () => {
122 |     const prompt = removeCommonIndent(`
123 |       <assistant>
124 |         I will help you with that.
125 |         <tool-call id="call_123" name="get_weather">
126 |           {"location": "San Francisco"}
127 |         </tool-call>
128 |       </assistant>
129 |     `)
130 | 
131 |     const { messages } = await render({
132 |       prompt,
133 |       adapter: Adapters['vercel'],
134 |     })
135 | 
136 |     expect(messages).toHaveLength(1)
137 |     expect(messages[0]?.role).toBe(MessageRole.assistant)
138 |     expect(messages[0]?.content).toHaveLength(2)
139 |     expect(messages[0]?.content[0]).toEqual({
140 |       type: ContentType.text,
141 |       text: 'I will help you with that.',
142 |     })
143 |     expect(messages[0]?.content[1]).toMatchObject({
144 |       type: ContentType.toolCall,
145 |       toolCallId: 'call_123',
146 |       toolName: 'get_weather',
147 |     })
148 |     expect(messages[0]?.content[1]).toHaveProperty('args')
149 |   })
150 | })
151 | 


--------------------------------------------------------------------------------
/examples/rpc.ts:
--------------------------------------------------------------------------------
  1 | // Run `pnpm build:rpc` before running this example
  2 | 
  3 | import assert from 'node:assert'
  4 | import { type FileHandle, mkdir, open, readFile, writeFile } from 'node:fs/promises'
  5 | import { tmpdir } from 'node:os'
  6 | import { join } from 'node:path'
  7 | import { inspect } from 'node:util'
  8 | import { WASI } from 'node:wasi'
  9 | 
 10 | const PROMPTL_WASM_PATH = './dist/promptl.wasm'
 11 | 
 12 | const prompt = `
 13 | <step>
 14 |   <system>
 15 |     You are a helpful assistant.
 16 |   </system>
 17 |   <user>
 18 |     Say hello.
 19 |   </user>
 20 | </step>
 21 | <step>
 22 |   <user>
 23 |     Now say goodbye.
 24 |   </user>
 25 | </step>
 26 | `
 27 | let chain, conversation
 28 | chain = await createChain(prompt);
 29 | ({ chain, ...conversation } = await stepChain(chain));
 30 | ({ chain, ...conversation } = await stepChain(chain, 'Hello!'));
 31 | ({ chain, ...conversation } = await stepChain(chain, 'Goodbye!'));
 32 | 
 33 | assert(chain.completed)
 34 | assert(conversation.completed)
 35 | 
 36 | console.log(inspect(conversation.messages, { depth: null }))
 37 | 
 38 | // Utility functions
 39 | 
 40 | async function createChain(prompt: string): Promise<any> {
 41 |   return await execute([
 42 |     {
 43 |       procedure: 'createChain',
 44 |       parameters: {
 45 |         prompt: prompt,
 46 |       },
 47 |     },
 48 |   ]).then((result) => {
 49 |     if (result[0]!.error) {
 50 |       console.log('Error dump: ', inspect(result[0]!.error, { depth: null }))
 51 |       throw new Error(result[0]!.error.message)
 52 |     }
 53 |     return result[0]!.value
 54 |   })
 55 | }
 56 | 
 57 | async function stepChain(chain: any, response?: any): Promise<any> {
 58 |   return await execute([
 59 |     {
 60 |       procedure: 'stepChain',
 61 |       parameters: {
 62 |         chain: chain,
 63 |         response: response,
 64 |       },
 65 |     },
 66 |   ]).then((result) => {
 67 |     if (result[0]!.error) {
 68 |       console.log('Error dump: ', inspect(result[0]!.error, { depth: null }))
 69 |       throw new Error(result[0]!.error.message)
 70 |     }
 71 |     return result[0]!.value
 72 |   })
 73 | }
 74 | 
 75 | async function execute(data: any): Promise<any> {
 76 |   const dir = join(tmpdir(), 'promptl')
 77 |   const stdin_path = join(dir, 'stdin')
 78 |   const stdout_path = join(dir, 'stdout')
 79 |   const stderr_path = join(dir, 'stderr')
 80 | 
 81 |   await mkdir(dir, { recursive: true })
 82 |   await writeFile(stdin_path, '')
 83 |   await writeFile(stdout_path, '')
 84 |   await writeFile(stderr_path, '')
 85 | 
 86 |   let stdin: FileHandle | undefined
 87 |   let stdout: FileHandle | undefined
 88 |   let stderr: FileHandle | undefined
 89 | 
 90 |   let wasmStdin: FileHandle | undefined
 91 |   let wasmStdout: FileHandle | undefined
 92 |   let wasmStderr: FileHandle | undefined
 93 | 
 94 |   try {
 95 |     stdin = await open(stdin_path, 'w')
 96 |     stdout = await open(stdout_path, 'r')
 97 |     stderr = await open(stderr_path, 'r')
 98 | 
 99 |     wasmStdin = await open(stdin_path, 'r')
100 |     wasmStdout = await open(stdout_path, 'w')
101 |     wasmStderr = await open(stderr_path, 'w')
102 | 
103 |     const wasi = new WASI({
104 |       version: 'preview1',
105 |       args: [],
106 |       env: {},
107 |       stdin: wasmStdin.fd,
108 |       stdout: wasmStdout.fd,
109 |       stderr: wasmStderr.fd,
110 |       returnOnExit: true,
111 |     })
112 | 
113 |     const bytes = await readFile(PROMPTL_WASM_PATH)
114 |     WebAssembly.validate(bytes)
115 |     const module = await WebAssembly.compile(bytes)
116 |     const instance = await WebAssembly.instantiate(
117 |       module,
118 |       wasi.getImportObject(),
119 |     )
120 | 
121 |     await send(stdin, data)
122 | 
123 |     wasi.start(instance)
124 | 
125 |     const [out, err] = await Promise.all([receive(stdout), receive(stderr)])
126 |     if (err) throw new Error(err)
127 | 
128 |     return out
129 |   } finally {
130 |     await Promise.all([
131 |       stdin?.close(),
132 |       stdout?.close(),
133 |       stderr?.close(),
134 |       wasmStdin?.close(),
135 |       wasmStdout?.close(),
136 |       wasmStderr?.close(),
137 |     ])
138 |   }
139 | }
140 | 
141 | async function send(file: FileHandle, data: any) {
142 |   await writeFile(file, JSON.stringify(data) + '\n', {
143 |     encoding: 'utf8',
144 |     flush: true,
145 |   })
146 | }
147 | 
148 | async function receive(file: FileHandle): Promise<any> {
149 |   const data = await readFile(file, {
150 |     encoding: 'utf8',
151 |   }).then((data) => data.trim())
152 | 
153 |   try {
154 |     return JSON.parse(data)
155 |   } catch {
156 |     return data
157 |   }
158 | }
159 | 


--------------------------------------------------------------------------------
/src/compiler/logic/nodes/index.ts:
--------------------------------------------------------------------------------
  1 | import {
  2 |   NodeType,
  3 |   ResolveNodeProps,
  4 |   UpdateScopeContextProps,
  5 | } from '$promptl/compiler/logic/types'
  6 | import { Node } from 'estree'
  7 | 
  8 | import {
  9 |   resolve as resolveArrayExpression,
 10 |   updateScopeContext as updateArrayScopeContext,
 11 | } from './arrayExpression'
 12 | import {
 13 |   resolve as resolveAssignmentExpression,
 14 |   updateScopeContext as updateAssignmentScopeContext,
 15 | } from './assignmentExpression'
 16 | import {
 17 |   resolve as resolveBinaryExpression,
 18 |   updateScopeContext as updateBinaryScopeContext,
 19 | } from './binaryExpression'
 20 | import {
 21 |   resolve as resolveCallExpression,
 22 |   updateScopeContext as updateCallScopeContext,
 23 | } from './callExpression'
 24 | import {
 25 |   resolve as resolveChainExpression,
 26 |   updateScopeContext as updateChainScopeContext,
 27 | } from './chainExpression'
 28 | import {
 29 |   resolve as resolveConditionalExpression,
 30 |   updateScopeContext as updateConditionalScopeContext,
 31 | } from './conditionalExpression'
 32 | import {
 33 |   resolve as resolveIdentifier,
 34 |   updateScopeContext as updateIdentifierScopeContext,
 35 | } from './identifier'
 36 | import {
 37 |   resolve as resolveLiteral,
 38 |   updateScopeContext as updateLiteralScopeContext,
 39 | } from './literal'
 40 | import {
 41 |   resolve as resolveMemberExpression,
 42 |   updateScopeContext as updateMemberScopeContext,
 43 | } from './memberExpression'
 44 | import {
 45 |   resolve as resolveObjectExpression,
 46 |   updateScopeContext as updateObjectScopeContext,
 47 | } from './objectExpression'
 48 | import {
 49 |   resolve as resolveSequenceExpression,
 50 |   updateScopeContext as updateSequenceScopeContext,
 51 | } from './sequenceExpression'
 52 | import {
 53 |   resolve as resolveUnaryExpression,
 54 |   updateScopeContext as updateUnaryScopeContext,
 55 | } from './unaryExpression'
 56 | import {
 57 |   resolve as resolveUpdateExpression,
 58 |   updateScopeContext as updateUpdateScopeContext,
 59 | } from './updateExpression'
 60 | 
 61 | type ResolveNodeFn = (props: ResolveNodeProps<Node>) => Promise<unknown>
 62 | type UpdateScopeContextFn = (props: UpdateScopeContextProps<Node>) => void
 63 | 
 64 | export const nodeResolvers: Record<NodeType, ResolveNodeFn> = {
 65 |   [NodeType.ArrayExpression]: resolveArrayExpression as ResolveNodeFn,
 66 |   [NodeType.AssignmentExpression]: resolveAssignmentExpression as ResolveNodeFn,
 67 |   [NodeType.BinaryExpression]: resolveBinaryExpression as ResolveNodeFn,
 68 |   [NodeType.CallExpression]: resolveCallExpression as ResolveNodeFn,
 69 |   [NodeType.ChainExpression]: resolveChainExpression as ResolveNodeFn,
 70 |   [NodeType.ConditionalExpression]:
 71 |     resolveConditionalExpression as ResolveNodeFn,
 72 |   [NodeType.Identifier]: resolveIdentifier as ResolveNodeFn,
 73 |   [NodeType.Literal]: resolveLiteral as ResolveNodeFn,
 74 |   [NodeType.LogicalExpression]: resolveBinaryExpression as ResolveNodeFn,
 75 |   [NodeType.ObjectExpression]: resolveObjectExpression as ResolveNodeFn,
 76 |   [NodeType.MemberExpression]: resolveMemberExpression as ResolveNodeFn,
 77 |   [NodeType.SequenceExpression]: resolveSequenceExpression as ResolveNodeFn,
 78 |   [NodeType.UnaryExpression]: resolveUnaryExpression as ResolveNodeFn,
 79 |   [NodeType.UpdateExpression]: resolveUpdateExpression as ResolveNodeFn,
 80 | }
 81 | 
 82 | export const updateScopeContextResolvers: Record<
 83 |   NodeType,
 84 |   UpdateScopeContextFn
 85 | > = {
 86 |   [NodeType.ArrayExpression]: updateArrayScopeContext as UpdateScopeContextFn,
 87 |   [NodeType.AssignmentExpression]:
 88 |     updateAssignmentScopeContext as UpdateScopeContextFn,
 89 |   [NodeType.BinaryExpression]: updateBinaryScopeContext as UpdateScopeContextFn,
 90 |   [NodeType.CallExpression]: updateCallScopeContext as UpdateScopeContextFn,
 91 |   [NodeType.ChainExpression]: updateChainScopeContext as UpdateScopeContextFn,
 92 |   [NodeType.ConditionalExpression]:
 93 |     updateConditionalScopeContext as UpdateScopeContextFn,
 94 |   [NodeType.Identifier]: updateIdentifierScopeContext as UpdateScopeContextFn,
 95 |   [NodeType.Literal]: updateLiteralScopeContext as UpdateScopeContextFn,
 96 |   [NodeType.LogicalExpression]:
 97 |     updateBinaryScopeContext as UpdateScopeContextFn,
 98 |   [NodeType.ObjectExpression]: updateObjectScopeContext as UpdateScopeContextFn,
 99 |   [NodeType.MemberExpression]: updateMemberScopeContext as UpdateScopeContextFn,
100 |   [NodeType.SequenceExpression]:
101 |     updateSequenceScopeContext as UpdateScopeContextFn,
102 |   [NodeType.UnaryExpression]: updateUnaryScopeContext as UpdateScopeContextFn,
103 |   [NodeType.UpdateExpression]: updateUpdateScopeContext as UpdateScopeContextFn,
104 | }
105 | 


--------------------------------------------------------------------------------
/src/compiler/errors.test.ts:
--------------------------------------------------------------------------------
  1 | import CompileError from '$promptl/error/error'
  2 | import { describe, expect, it } from 'vitest'
  3 | 
  4 | import { scan, render } from '.'
  5 | import { removeCommonIndent } from './utils'
  6 | 
  7 | const getExpectedError = async (
  8 |   fn: () => Promise<void>,
  9 |   errorMessage: string,
 10 | ): Promise<CompileError> => {
 11 |   try {
 12 |     await fn()
 13 |   } catch (err) {
 14 |     expect(err).toBeInstanceOf(CompileError)
 15 |     return err as CompileError
 16 |   }
 17 |   throw new Error(errorMessage)
 18 | }
 19 | 
 20 | const expectBothErrors = async ({
 21 |   code,
 22 |   prompt,
 23 | }: {
 24 |   code: string
 25 |   prompt: string
 26 | }) => {
 27 |   const compileError = await getExpectedError(async () => {
 28 |     await render({
 29 |       prompt: removeCommonIndent(prompt),
 30 |       parameters: {},
 31 |     })
 32 |   }, `Expected compile to throw '${code}'`)
 33 |   expect(compileError.code).toBe(code)
 34 | 
 35 |   const metadata = await scan({
 36 |     prompt: removeCommonIndent(prompt),
 37 |   })
 38 |   if (metadata.errors.length === 0) {
 39 |     throw new Error(`Expected scan to throw '${code}'`)
 40 |   }
 41 |   const metadataError = metadata.errors[0]!
 42 | 
 43 |   expect(metadataError.code).toBe(code)
 44 |   expect(compileError.code).toBe(metadataError.code)
 45 | }
 46 | 
 47 | describe(`all compilation errors that don't require value resolution are caught both in compile and scan`, () => {
 48 |   it.todo('unsupported-base-node-type') // This one requires the parser to return an unsupported node type, which is not reproducible
 49 | 
 50 |   it('variable-already-declared', async () => {
 51 |     const prompt = `
 52 |       {{foo = 5}}
 53 |       {{ for foo in [1, 2, 3] }}
 54 |         {{foo}}
 55 |       {{ endfor }}
 56 |     `
 57 | 
 58 |     await expectBothErrors({
 59 |       code: 'variable-already-declared',
 60 |       prompt,
 61 |     })
 62 |   })
 63 | 
 64 |   it.todo('invalid-object-key', async () => {
 65 |     // Parser does not even parse this prompt
 66 |     const prompt = `
 67 |       {{ foo = 5 }}
 68 |       {{ { [1, 2, 3]: 'bar' } }}
 69 |     `
 70 | 
 71 |     await expectBothErrors({
 72 |       code: 'invalid-object-key',
 73 |       prompt,
 74 |     })
 75 |   })
 76 | 
 77 |   it.todo('unsupported-operator', async () => {
 78 |     // Parser does not even parse this prompt
 79 |     const prompt = `
 80 |       {{ foo = 5 ++ 2 }}
 81 |     `
 82 | 
 83 |     await expectBothErrors({
 84 |       code: 'unsupported-operator',
 85 |       prompt,
 86 |     })
 87 |   })
 88 | 
 89 |   it('invalid-assignment', async () => {
 90 |     const prompt = `
 91 |       {{ [ foo ] = [ 5 ] }}
 92 |     `
 93 | 
 94 |     await expectBothErrors({
 95 |       code: 'invalid-assignment',
 96 |       prompt,
 97 |     })
 98 |   })
 99 | 
100 |   it('invalid-tool-call-placement', async () => {
101 |     const prompt = `
102 |       <user>
103 |         <tool-call id="foo" name="bar" />
104 |       </user>
105 |     `
106 | 
107 |     await expectBothErrors({
108 |       code: 'invalid-tool-call-placement',
109 |       prompt,
110 |     })
111 |   })
112 | 
113 |   it('step-tag-inside-step', async () => {
114 |     const prompt = `
115 |       <step>
116 |         <step>
117 |           Foo
118 |         </step>
119 |       </step>
120 |     `
121 | 
122 |     await expectBothErrors({
123 |       code: 'step-tag-inside-step',
124 |       prompt,
125 |     })
126 |   })
127 | 
128 |   it('message-tag-inside-message', async () => {
129 |     const prompt = `
130 |       <system>
131 |         <user>
132 |           Foo
133 |         </user>
134 |       </system>
135 |     `
136 | 
137 |     await expectBothErrors({
138 |       code: 'message-tag-inside-message',
139 |       prompt,
140 |     })
141 |   })
142 | 
143 |   it('content-tag-inside-content', async () => {
144 |     const prompt = `
145 |       <content-text>
146 |         <content-text>
147 |           Foo
148 |         </content-text>
149 |       </content-text>
150 |     `
151 | 
152 |     await expectBothErrors({
153 |       code: 'content-tag-inside-content',
154 |       prompt,
155 |     })
156 |   })
157 | 
158 |   it('tool-call-tag-without-id', async () => {
159 |     const prompt = `
160 |       <assistant>
161 |         <tool-call name="bar" />
162 |       </assistant>
163 |     `
164 | 
165 |     await expectBothErrors({
166 |       code: 'tool-call-tag-without-id',
167 |       prompt,
168 |     })
169 |   })
170 | 
171 |   it('tool-message-without-id', async () => {
172 |     const prompt = `
173 |       <tool>
174 |         Foo
175 |       </tool>
176 |     `
177 | 
178 |     await expectBothErrors({
179 |       code: 'tool-message-without-id',
180 |       prompt,
181 |     })
182 |   })
183 | 
184 |   it('tool-call-without-name', async () => {
185 |     const prompt = `
186 |       <assistant>
187 |         <tool-call id="foo" />
188 |       </assistant>
189 |     `
190 | 
191 |     await expectBothErrors({
192 |       code: 'tool-call-without-name',
193 |       prompt,
194 |     })
195 |   })
196 | 
197 |   it('message-tag-without-role', async () => {
198 |     const prompt = `
199 |       <message>
200 |         Foo
201 |       </message>
202 |     `
203 | 
204 |     await expectBothErrors({
205 |       code: 'message-tag-without-role',
206 |       prompt,
207 |     })
208 |   })
209 | })
210 | 


--------------------------------------------------------------------------------
/src/build.test.ts:
--------------------------------------------------------------------------------
  1 | import { execSync } from 'child_process'
  2 | import { existsSync, readFileSync } from 'fs'
  3 | import { join } from 'path'
  4 | import { describe, expect, it, beforeAll } from 'vitest'
  5 | 
  6 | const DIST_DIR = join(process.cwd(), 'dist')
  7 | const CJS_FILE = join(DIST_DIR, 'index.cjs')
  8 | const ESM_FILE = join(DIST_DIR, 'index.js')
  9 | const DTS_FILE = join(DIST_DIR, 'index.d.ts')
 10 | 
 11 | describe('Build Output Verification', () => {
 12 |   beforeAll(() => {
 13 |     // Ensure build exists
 14 |     if (!existsSync(DIST_DIR)) {
 15 |       execSync('pnpm build', { stdio: 'inherit' })
 16 |     }
 17 |   })
 18 | 
 19 |   it('should generate all required build files', () => {
 20 |     expect(existsSync(CJS_FILE)).toBe(true)
 21 |     expect(existsSync(ESM_FILE)).toBe(true)
 22 |     expect(existsSync(DTS_FILE)).toBe(true)
 23 |   })
 24 | 
 25 |   it('should generate valid CommonJS format', () => {
 26 |     const cjsContent = readFileSync(CJS_FILE, 'utf8')
 27 |     
 28 |     // Should start with 'use strict'
 29 |     expect(cjsContent).toMatch(/^'use strict'/)
 30 |     
 31 |     // Should use require() for dependencies
 32 |     expect(cjsContent).toMatch(/require\(['"]/)
 33 |     
 34 |     // Should not contain ES6 import statements
 35 |     expect(cjsContent).not.toMatch(/^import\s+/)
 36 |   })
 37 | 
 38 |   it('should generate valid ES module format', () => {
 39 |     const esmContent = readFileSync(ESM_FILE, 'utf8')
 40 |     
 41 |     // Should use import statements
 42 |     expect(esmContent).toMatch(/^import\s+/)
 43 |     
 44 |     // Should not contain require() calls
 45 |     expect(esmContent).not.toMatch(/require\(['"]/)
 46 |   })
 47 | 
 48 |   it('should generate TypeScript definitions', () => {
 49 |     const dtsContent = readFileSync(DTS_FILE, 'utf8')
 50 |     
 51 |     // Should contain export declarations
 52 |     expect(dtsContent).toMatch(/export\s+/)
 53 |     
 54 |     // Should contain key function types
 55 |     expect(dtsContent).toMatch(/parse/)
 56 |     expect(dtsContent).toMatch(/render/)
 57 |   })
 58 | 
 59 |   it('should load CJS build correctly', async () => {
 60 |     // Use dynamic require to avoid bundler issues
 61 |     const cjsModule = await import(CJS_FILE)
 62 |     
 63 |     expect(typeof cjsModule.parse).toBe('function')
 64 |     expect(typeof cjsModule.render).toBe('function')
 65 |     expect(typeof cjsModule.createChain).toBe('function')
 66 |     expect(cjsModule.CompileError).toBeDefined()
 67 |     
 68 |     // Should have all expected exports
 69 |     const exports = Object.keys(cjsModule)
 70 |     expect(exports).toContain('parse')
 71 |     expect(exports).toContain('render')
 72 |     expect(exports).toContain('CompileError')
 73 |     expect(exports).toContain('ContentType')
 74 |     expect(exports).toContain('MessageRole')
 75 |   })
 76 | 
 77 |   it('should load ESM build correctly', async () => {
 78 |     const esmModule = await import(ESM_FILE)
 79 |     
 80 |     expect(typeof esmModule.parse).toBe('function')
 81 |     expect(typeof esmModule.render).toBe('function')
 82 |     expect(typeof esmModule.createChain).toBe('function')
 83 |     expect(esmModule.CompileError).toBeDefined()
 84 |     
 85 |     // Should have all expected exports
 86 |     const exports = Object.keys(esmModule)
 87 |     expect(exports).toContain('parse')
 88 |     expect(exports).toContain('render')
 89 |     expect(exports).toContain('CompileError')
 90 |     expect(exports).toContain('ContentType')
 91 |     expect(exports).toContain('MessageRole')
 92 |   })
 93 | 
 94 |   it('should have functional parse and render in CJS build', async () => {
 95 |     const { parse, render } = await import(CJS_FILE)
 96 |     
 97 |     const template = 'Hello {{name}}!'
 98 |     const parsed = parse(template)
 99 |     
100 |     expect(parsed).toBeDefined()
101 |     expect(parsed.type).toBe('Fragment')
102 |     
103 |     const rendered = await render({ 
104 |       prompt: template, 
105 |       parameters: { name: 'World' } 
106 |     })
107 |     
108 |     expect(rendered).toBeDefined()
109 |     expect(rendered.messages).toBeDefined()
110 |     expect(rendered.config).toBeDefined()
111 |     expect(Array.isArray(rendered.messages)).toBe(true)
112 |   })
113 | 
114 |   it('should have functional parse and render in ESM build', async () => {
115 |     const { parse, render } = await import(ESM_FILE)
116 |     
117 |     const template = 'Hello {{name}}!'
118 |     const parsed = parse(template)
119 |     
120 |     expect(parsed).toBeDefined()
121 |     expect(parsed.type).toBe('Fragment')
122 |     
123 |     const rendered = await render({ 
124 |       prompt: template, 
125 |       parameters: { name: 'World' } 
126 |     })
127 |     
128 |     expect(rendered).toBeDefined()
129 |     expect(rendered.messages).toBeDefined()
130 |     expect(rendered.config).toBeDefined()
131 |     expect(Array.isArray(rendered.messages)).toBe(true)
132 |   })
133 | 
134 |   it('should have identical exports between CJS and ESM builds', async () => {
135 |     const cjsModule = await import(CJS_FILE)
136 |     const esmModule = await import(ESM_FILE)
137 |     
138 |     // Filter out 'default' export which is expected to differ between CJS and ESM
139 |     const cjsExports = Object.keys(cjsModule).filter(key => key !== 'default').sort()
140 |     const esmExports = Object.keys(esmModule).filter(key => key !== 'default').sort()
141 |     
142 |     expect(cjsExports).toEqual(esmExports)
143 |     expect(cjsExports.length).toBeGreaterThan(10) // Ensure we have substantial exports
144 |   })
145 | })


--------------------------------------------------------------------------------
/src/compiler/logic/nodes/assignmentExpression.ts:
--------------------------------------------------------------------------------
  1 | import { ASSIGNMENT_OPERATOR_METHODS } from '$promptl/compiler/logic/operators'
  2 | import type {
  3 |   ResolveNodeProps,
  4 |   UpdateScopeContextProps,
  5 | } from '$promptl/compiler/logic/types'
  6 | import errors from '$promptl/error/errors'
  7 | import type {
  8 |   AssignmentExpression,
  9 |   AssignmentOperator,
 10 |   Identifier,
 11 |   MemberExpression,
 12 | } from 'estree'
 13 | 
 14 | import { resolveLogicNode, updateScopeContextForNode } from '..'
 15 | 
 16 | /**
 17 |  * ### AssignmentExpression
 18 |  * Represents an assignment or update to a variable or property. Returns the newly assigned value.
 19 |  * The assignment can be made to an existing variable or property, or to a new one. Assignments to constants are not allowed.
 20 |  *
 21 |  * Examples: `foo = 1` `obj.foo = 'bar'` `foo += 1`
 22 |  */
 23 | export async function resolve({
 24 |   node,
 25 |   scope,
 26 |   raiseError,
 27 |   ...props
 28 | }: ResolveNodeProps<AssignmentExpression>) {
 29 |   const assignmentOperator = node.operator
 30 |   if (!(assignmentOperator in ASSIGNMENT_OPERATOR_METHODS)) {
 31 |     raiseError(errors.unsupportedOperator(assignmentOperator), node)
 32 |   }
 33 |   const assignmentMethod = ASSIGNMENT_OPERATOR_METHODS[assignmentOperator]!
 34 | 
 35 |   const assignmentValue = await resolveLogicNode({
 36 |     node: node.right,
 37 |     scope,
 38 |     raiseError,
 39 |     ...props,
 40 |   })
 41 | 
 42 |   if (node.left.type === 'Identifier') {
 43 |     await assignToVariable({
 44 |       assignmentOperator,
 45 |       assignmentMethod,
 46 |       assignmentValue,
 47 |       node: node.left,
 48 |       scope,
 49 |       raiseError,
 50 |       ...props,
 51 |     })
 52 |     return
 53 |   }
 54 | 
 55 |   if (node.left.type === 'MemberExpression') {
 56 |     await assignToProperty({
 57 |       assignmentOperator,
 58 |       assignmentMethod,
 59 |       assignmentValue,
 60 |       node: node.left,
 61 |       scope,
 62 |       raiseError,
 63 |       ...props,
 64 |     })
 65 |     return
 66 |   }
 67 | 
 68 |   raiseError(errors.invalidAssignment, node)
 69 | }
 70 | 
 71 | async function assignToVariable({
 72 |   assignmentOperator,
 73 |   assignmentMethod,
 74 |   assignmentValue,
 75 |   node,
 76 |   scope,
 77 |   raiseError,
 78 | }: ResolveNodeProps<Identifier> & {
 79 |   assignmentOperator: AssignmentOperator
 80 |   assignmentMethod: (typeof ASSIGNMENT_OPERATOR_METHODS)[keyof typeof ASSIGNMENT_OPERATOR_METHODS]
 81 |   assignmentValue: unknown
 82 | }) {
 83 |   const assignedVariableName = node.name
 84 | 
 85 |   if (assignmentOperator != '=' && !scope.exists(assignedVariableName)) {
 86 |     raiseError(errors.variableNotDeclared(assignedVariableName), node)
 87 |   }
 88 | 
 89 |   const updatedValue = assignmentMethod(
 90 |     scope.exists(assignedVariableName)
 91 |       ? scope.get(assignedVariableName)
 92 |       : undefined,
 93 |     assignmentValue,
 94 |   )
 95 | 
 96 |   scope.set(assignedVariableName, updatedValue)
 97 |   return updatedValue
 98 | }
 99 | 
100 | async function assignToProperty({
101 |   assignmentOperator,
102 |   assignmentMethod,
103 |   assignmentValue,
104 |   node,
105 |   ...props
106 | }: ResolveNodeProps<MemberExpression> & {
107 |   assignmentOperator: AssignmentOperator
108 |   assignmentMethod: (typeof ASSIGNMENT_OPERATOR_METHODS)[keyof typeof ASSIGNMENT_OPERATOR_METHODS]
109 |   assignmentValue: unknown
110 | }) {
111 |   const { raiseError } = props
112 |   const object = (await resolveLogicNode({
113 |     node: node.object,
114 |     ...props,
115 |   })) as { [key: string]: any }
116 | 
117 |   const property = (
118 |     node.computed
119 |       ? await resolveLogicNode({
120 |           node: node.property,
121 |           ...props,
122 |         })
123 |       : (node.property as Identifier).name
124 |   ) as string
125 | 
126 |   if (assignmentOperator != '=' && !(property in object)) {
127 |     raiseError(errors.propertyNotExists(property), node)
128 |   }
129 | 
130 |   const originalValue = object[property]
131 |   const updatedValue = assignmentMethod(originalValue, assignmentValue)
132 |   object[property] = updatedValue
133 |   return updatedValue
134 | }
135 | 
136 | export function updateScopeContext({
137 |   node,
138 |   scopeContext,
139 |   builtins,
140 |   raiseError,
141 | }: UpdateScopeContextProps<AssignmentExpression>) {
142 |   const assignmentOperator = node.operator
143 |   if (!(assignmentOperator in ASSIGNMENT_OPERATOR_METHODS)) {
144 |     raiseError(errors.unsupportedOperator(assignmentOperator), node)
145 |   }
146 | 
147 |   updateScopeContextForNode({
148 |     node: node.right,
149 |     scopeContext,
150 |     builtins,
151 |     raiseError,
152 |   })
153 | 
154 |   if (node.left.type === 'Identifier') {
155 |     // Variable assignment
156 |     const assignedVariableName = (node.left as Identifier).name
157 |     if (assignedVariableName in builtins) {
158 |       raiseError(errors.assignmentToBuiltin(assignedVariableName), node)
159 |     }
160 |     if (assignmentOperator != '=') {
161 |       // Update an existing variable
162 |       if (!scopeContext.definedVariables.has(assignedVariableName)) {
163 |         scopeContext.usedUndefinedVariables.add(assignedVariableName)
164 |       }
165 |     }
166 |     scopeContext.definedVariables.add(assignedVariableName)
167 |     return
168 |   }
169 | 
170 |   if (node.left.type === 'MemberExpression') {
171 |     updateScopeContextForNode({
172 |       node: node.left,
173 |       scopeContext,
174 |       builtins,
175 |       raiseError,
176 |     })
177 |     return
178 |   }
179 | 
180 |   raiseError(errors.invalidAssignment, node)
181 | }
182 | 


--------------------------------------------------------------------------------
/src/compiler/base/nodes/tags/step.test.ts:
--------------------------------------------------------------------------------
  1 | import CompileError from '$promptl/error/error'
  2 | import { complete } from '$promptl/compiler/test/helpers'
  3 | import { removeCommonIndent } from '$promptl/compiler/utils'
  4 | import { Chain } from '$promptl/index'
  5 | import { describe, expect, it, vi } from 'vitest'
  6 | 
  7 | describe('step tags', async () => {
  8 |   it('does not create a variable from response if not specified', async () => {
  9 |     const mock = vi.fn()
 10 |     const prompt = removeCommonIndent(`
 11 |       <step>
 12 |         Ensure truthfulness of the following statement, give a reason and a confidence score.
 13 |         Statement: fake statement
 14 |       </step>
 15 |       <step>
 16 |         Now correct the statement if it is not true.
 17 |       </step>
 18 |     `)
 19 | 
 20 |     const chain = new Chain({ prompt, parameters: { mock } })
 21 |     await complete({
 22 |       chain,
 23 |       callback: async () =>
 24 |         `
 25 |       The statement is not true because it is fake. My confidence score is 100.
 26 |     `.trim(),
 27 |     })
 28 | 
 29 |     expect(mock).not.toHaveBeenCalled()
 30 |   })
 31 | 
 32 |   it('creates a text variable from response if specified', async () => {
 33 |     const mock = vi.fn()
 34 |     const prompt = removeCommonIndent(`
 35 |       <step as="analysis">
 36 |         Ensure truthfulness of the following statement, give a reason and a confidence score.
 37 |         Statement: fake statement
 38 |       </step>
 39 |       <step>
 40 |         {{ mock(analysis) }}
 41 |         Now correct the statement if it is not true.
 42 |       </step>
 43 |     `)
 44 | 
 45 |     const chain = new Chain({ prompt, parameters: { mock } })
 46 |     await complete({
 47 |       chain,
 48 |       callback: async () =>
 49 |         `
 50 |       The statement is not true because it is fake. My confidence score is 100.
 51 |     `.trim(),
 52 |     })
 53 | 
 54 |     expect(mock).toHaveBeenCalledWith(
 55 |       'The statement is not true because it is fake. My confidence score is 100.',
 56 |     )
 57 |   })
 58 | 
 59 |   it('creates an object variable from response if specified and schema is provided', async () => {
 60 |     const mock = vi.fn()
 61 |     const prompt = removeCommonIndent(`
 62 |       <step as="analysis" schema={{{type: "object", properties: {truthful: {type: "boolean"}, reason: {type: "string"}, confidence: {type: "integer"}}, required: ["truthful", "reason", "confidence"]}}}>
 63 |         Ensure truthfulness of the following statement, give a reason and a confidence score.
 64 |         Statement: fake statement
 65 |       </step>
 66 |       <step>
 67 |         {{ mock(analysis) }}
 68 |         {{ if !analysis.truthful && analysis.confidence > 50 }}
 69 |           Correct the statement taking into account the reason: '{{ analysis.reason }}'.
 70 |         {{ endif }}
 71 |       </step>
 72 |     `)
 73 | 
 74 |     const chain = new Chain({ prompt, parameters: { mock } })
 75 |     const { messages } = await complete({
 76 |       chain,
 77 |       callback: async () =>
 78 |         `
 79 |       {
 80 |         "truthful": false,
 81 |         "reason": "It is fake",
 82 |         "confidence": 100
 83 |       }
 84 |     `.trim(),
 85 |     })
 86 | 
 87 |     expect(mock).toHaveBeenCalledWith({
 88 |       truthful: false,
 89 |       reason: 'It is fake',
 90 |       confidence: 100,
 91 |     })
 92 |     expect(messages[2]!.content).toEqual(
 93 |       "Correct the statement taking into account the reason: 'It is fake'.",
 94 |     )
 95 |   })
 96 | 
 97 |   it('fails creating an object variable from response if specified and schema is provided but response is invalid', async () => {
 98 |     const mock = vi.fn()
 99 |     const prompt = removeCommonIndent(`
100 |       <step as="analysis" schema={{{type: "object", properties: {truthful: {type: "boolean"}, reason: {type: "string"}, confidence: {type: "integer"}}, required: ["truthful", "reason", "confidence"]}}}>
101 |         Ensure truthfulness of the following statement, give a reason and a confidence score.
102 |         Statement: fake statement
103 |       </step>
104 |       <step>
105 |         {{ mock(analysis) }}
106 |         {{ if !analysis.truthful && analysis.confidence > 50 }}
107 |           Correct the statement taking into account the reason: '{{ analysis.reason }}'.
108 |         {{ endif }}
109 |       </step>
110 |     `)
111 | 
112 |     const chain = new Chain({ prompt, parameters: { mock } })
113 |     let error: CompileError
114 |     try {
115 |       await complete({
116 |         chain,
117 |         callback: async () =>
118 |           `
119 |       Bad JSON.
120 |     `.trim(),
121 |       })
122 |     } catch (e) {
123 |       error = e as CompileError
124 |       expect(e).toBeInstanceOf(CompileError)
125 |     }
126 | 
127 |     expect(error!.code).toBe('invalid-step-response-format')
128 |     expect(mock).not.toHaveBeenCalled()
129 |   })
130 | 
131 |   it('creates a raw variable from response if specified', async () => {
132 |     const mock = vi.fn()
133 |     const prompt = removeCommonIndent(`
134 |       <step raw="analysis">
135 |         Ensure truthfulness of the following statement, give a reason and a confidence score.
136 |         Statement: fake statement
137 |       </step>
138 |       <step>
139 |         {{ mock(analysis) }}
140 |         Now correct the statement if it is not true.
141 |       </step>
142 |     `)
143 | 
144 |     const chain = new Chain({ prompt, parameters: { mock } })
145 |     await complete({
146 |       chain,
147 |       callback: async () =>
148 |         `
149 |       The statement is not true because it is fake. My confidence score is 100.
150 |     `.trim(),
151 |     })
152 | 
153 |     expect(mock).toHaveBeenCalledWith({
154 |       role: 'assistant',
155 |       content: [
156 |         {
157 |           type: 'text',
158 |           text: 'The statement is not true because it is fake. My confidence score is 100.',
159 |         },
160 |       ],
161 |     })
162 |   })
163 | })
164 | 


--------------------------------------------------------------------------------
/src/parser/index.ts:
--------------------------------------------------------------------------------
  1 | import CompileError, { error } from '$promptl/error/error'
  2 | import PARSER_ERRORS from '$promptl/error/errors'
  3 | import { reserved } from '$promptl/utils/names'
  4 | import { isIdentifierChar, isIdentifierStart } from 'acorn'
  5 | 
  6 | import type { BaseNode, Fragment } from './interfaces'
  7 | import fragment from './state/fragment'
  8 | import fullCharCodeAt from './utils/full_char_code_at'
  9 | 
 10 | export function parse(template: string) {
 11 |   return new Parser(template).parse()
 12 | }
 13 | 
 14 | type ParserState = (parser: Parser) => void | ParserState
 15 | type AutoClosedTag = { tag: string; reason: string; depth: number }
 16 | 
 17 | export class Parser {
 18 |   index: number = 0
 19 |   stack: BaseNode[] = []
 20 |   lastAutoClosedTag: AutoClosedTag | null = null
 21 |   fragment: Fragment
 22 | 
 23 |   constructor(public template: string) {
 24 |     this.fragment = {
 25 |       start: 0,
 26 |       end: this.template.length,
 27 |       type: 'Fragment',
 28 |       children: [],
 29 |     }
 30 |   }
 31 | 
 32 |   parse(): Fragment {
 33 |     try {
 34 |       return this._parse()
 35 |     } catch (err) {
 36 |       if (err instanceof CompileError) {
 37 |         throw err
 38 |       }
 39 |       this.error({
 40 |         code: 'parse-error',
 41 |         message: 'Syntax error',
 42 |       })
 43 |     }
 44 |   }
 45 | 
 46 |   _parse(): Fragment {
 47 |     this.stack.push(this.fragment)
 48 | 
 49 |     let state: ParserState = fragment
 50 |     while (this.index < this.template.length) {
 51 |       state = state(this) || fragment
 52 |     }
 53 |     if (this.stack.length > 1) {
 54 |       const current = this.current()
 55 |       this.error(
 56 |         {
 57 |           code: `unclosed-block`,
 58 |           message: `Block was left open`,
 59 |         },
 60 |         current.start! + 1,
 61 |       )
 62 |     }
 63 |     if (state !== fragment) {
 64 |       this.error({
 65 |         code: `unexpected-eof`,
 66 |         message: `Unexpected end of input`,
 67 |       })
 68 |     }
 69 |     if (this.fragment.children.length) {
 70 |       let start = this.fragment.children[0]!.start!
 71 |       while (/\s/.test(this.fragment[start])) start += 1
 72 |       let end = this.fragment.children[this.fragment.children.length - 1]!.end!
 73 |       while (/\s/.test(this.fragment[end - 1])) end -= 1
 74 |       this.fragment.start = start
 75 |       this.fragment.end = end
 76 |     } else {
 77 |       this.fragment.start = this.fragment.end = null
 78 |     }
 79 | 
 80 |     return this.fragment
 81 |   }
 82 | 
 83 |   current(): BaseNode {
 84 |     return this.stack[this.stack.length - 1]!
 85 |   }
 86 | 
 87 |   match(str: string) {
 88 |     return this.template.slice(this.index, this.index + str.length) === str
 89 |   }
 90 | 
 91 |   allowWhitespace() {
 92 |     while (
 93 |       this.index < this.template.length &&
 94 |       /\s/.test(this.template[this.index] || '')
 95 |     ) {
 96 |       this.index++
 97 |     }
 98 |   }
 99 | 
100 |   requireWhitespace() {
101 |     if (!/\s/.test(this.template[this.index]!)) {
102 |       this.error({
103 |         code: 'missing-whitespace',
104 |         message: 'Expected whitespace',
105 |       })
106 |     }
107 |     this.allowWhitespace()
108 |   }
109 | 
110 |   eat(
111 |     str: string,
112 |     required: boolean = false,
113 |     error?: { code: string; message: string },
114 |   ) {
115 |     if (this.match(str)) {
116 |       this.index += str.length
117 |       return true
118 |     }
119 |     if (required) {
120 |       this.error(
121 |         error ||
122 |           (this.index === this.template.length
123 |             ? PARSER_ERRORS.unexpectedEofToken(str)
124 |             : PARSER_ERRORS.unexpectedToken(str)),
125 |       )
126 |     }
127 |     return false
128 |   }
129 | 
130 |   error(
131 |     { code, message }: { code: string; message: string },
132 |     index = this.index,
133 |   ): never {
134 |     error(message, {
135 |       name: 'ParseError',
136 |       code,
137 |       source: this.template,
138 |       start: index - 1,
139 |       end: this.template.length,
140 |       fragment: this.fragment,
141 |     })
142 |   }
143 | 
144 |   acornError(err: CompileError) {
145 |     this.error(
146 |       {
147 |         code: 'parse-error',
148 |         message: err.message.replace(/ \(\d+:\d+\)$/, ''),
149 |       },
150 |       err.pos,
151 |     )
152 |   }
153 | 
154 |   matchRegex(pattern: RegExp) {
155 |     const match = pattern.exec(this.template.slice(this.index))
156 |     if (!match || match.index !== 0) return null
157 |     return match[0]
158 |   }
159 | 
160 |   read(pattern: RegExp) {
161 |     const result = this.matchRegex(pattern)
162 |     if (result) this.index += result.length
163 |     return result
164 |   }
165 | 
166 |   readIdentifier(allowReserved: boolean = false) {
167 |     const start = this.index
168 |     let i = this.index
169 |     const code = fullCharCodeAt(this.template, i)
170 |     if (!isIdentifierStart(code, true)) return null
171 |     i += code <= 0xffff ? 1 : 2
172 |     while (i < this.template.length) {
173 |       const code = fullCharCodeAt(this.template, i)
174 |       if (!isIdentifierChar(code, true)) break
175 |       i += code <= 0xffff ? 1 : 2
176 |     }
177 |     const identifier = this.template.slice(this.index, (this.index = i))
178 |     if (!allowReserved && reserved.has(identifier)) {
179 |       this.error(
180 |         {
181 |           code: 'unexpected-reserved-word',
182 |           message: `'${identifier}' is a reserved word in JavaScript and cannot be used here`,
183 |         },
184 |         start,
185 |       )
186 |     }
187 |     return identifier
188 |   }
189 | 
190 |   readUntil(pattern: RegExp) {
191 |     if (this.index >= this.template.length) {
192 |       this.error(PARSER_ERRORS.unexpectedEof)
193 |     }
194 |     const start = this.index
195 |     const match = pattern.exec(this.template.slice(this.index))
196 |     if (match) {
197 |       this.index = start + match.index
198 |       return this.template.slice(start, this.index)
199 |     }
200 |     this.index = this.template.length
201 |     return this.template.slice(start)
202 |   }
203 | }
204 | 


--------------------------------------------------------------------------------
/src/compiler/utils.ts:
--------------------------------------------------------------------------------
  1 | import { ZodError, core } from 'zod'
  2 | import { TAG_NAMES } from '$promptl/constants'
  3 | import {
  4 |   ChainStepTag,
  5 |   ContentTag,
  6 |   ElementTag,
  7 |   MessageTag,
  8 |   ReferenceTag,
  9 |   ScopeTag,
 10 | } from '$promptl/parser/interfaces'
 11 | import { ContentTypeTagName, MessageRole } from '$promptl/types'
 12 | import { Scalar, Node as YAMLItem, YAMLMap, YAMLSeq } from 'yaml'
 13 | 
 14 | type ZodIssue = core.$ZodIssue
 15 | 
 16 | export function isIterable(obj: unknown): obj is Iterable<unknown> {
 17 |   return (obj as Iterable<unknown>)?.[Symbol.iterator] !== undefined
 18 | }
 19 | 
 20 | export async function hasContent(iterable: Iterable<unknown>) {
 21 |   for await (const _ of iterable) {
 22 |     return true
 23 |   }
 24 |   return false
 25 | }
 26 | 
 27 | export function getCommonIndent(text: string): number {
 28 |   return (
 29 |     text.split('\n').reduce((acc: number | null, line: string) => {
 30 |       if (line.trim() === '') return acc
 31 |       const indent = line.match(/^\s*/)![0]
 32 |       if (acc === null) return indent.length
 33 |       return indent.length < acc ? indent.length : acc
 34 |     }, null) ?? 0
 35 |   )
 36 | }
 37 | 
 38 | export function removeCommonIndent(text: string): string {
 39 |   const indent = getCommonIndent(text)
 40 |   return text
 41 |     .split('\n')
 42 |     .map((line) => line.slice(indent))
 43 |     .join('\n')
 44 |     .trim()
 45 | }
 46 | 
 47 | export function isMessageTag(tag: ElementTag): tag is MessageTag {
 48 |   if (tag.name === TAG_NAMES.message) return true
 49 |   return Object.values(MessageRole).includes(tag.name as MessageRole)
 50 | }
 51 | 
 52 | export function isContentTag(tag: ElementTag): tag is ContentTag {
 53 |   if (tag.name === TAG_NAMES.content) return true
 54 |   return Object.values(ContentTypeTagName).includes(
 55 |     tag.name as ContentTypeTagName,
 56 |   )
 57 | }
 58 | 
 59 | export function isRefTag(tag: ElementTag): tag is ReferenceTag {
 60 |   return tag.name === TAG_NAMES.prompt
 61 | }
 62 | 
 63 | export function isChainStepTag(tag: ElementTag): tag is ChainStepTag {
 64 |   return tag.name === TAG_NAMES.step
 65 | }
 66 | 
 67 | export function isScopeTag(tag: ElementTag): tag is ScopeTag {
 68 |   return tag.name === TAG_NAMES.scope
 69 | }
 70 | 
 71 | export function tagAttributeIsLiteral(tag: ElementTag, name: string): boolean {
 72 |   const attr = tag.attributes.find((attr) => attr.name === name)
 73 |   if (!attr) return false
 74 |   if (attr.value === true) return true
 75 |   return attr.value.every((v) => v.type === 'Text')
 76 | }
 77 | 
 78 | type YAMLItemRange = [number, number] | undefined
 79 | export function findYAMLItemPosition(
 80 |   parent: YAMLItem,
 81 |   path: PropertyKey[],
 82 | ): YAMLItemRange {
 83 |   const parentRange: YAMLItemRange = parent?.range
 84 |     ? [parent.range[0], parent.range[1]]
 85 |     : undefined
 86 | 
 87 |   if (!parentRange || path.length === 0 || !('items' in parent)) {
 88 |     return parentRange
 89 |   }
 90 | 
 91 |   let child: YAMLItem | undefined
 92 |   if (parent instanceof YAMLMap) {
 93 |     child = parent.items.find((i) => {
 94 |       return (i.key as Scalar)?.value === path[0]!
 95 |     })?.value as YAMLItem | undefined
 96 |   }
 97 |   if (parent instanceof YAMLSeq && typeof path[0] === 'number') {
 98 |     child = parent.items[Number(path[0])] as YAMLItem | undefined
 99 |   }
100 | 
101 |   if (!child) return parentRange
102 |   return findYAMLItemPosition(child, path.slice(1)) ?? parentRange
103 | }
104 | 
105 | export function isZodError(error: unknown): error is ZodError {
106 |   return error instanceof ZodError
107 | }
108 | 
109 | function collectAllLeafIssues(issue: ZodIssue): ZodIssue[] {
110 |   if (issue.code === 'invalid_union') {
111 |     issue.errors
112 |     return issue.errors.flatMap((issues) =>
113 |       issues.flatMap(collectAllLeafIssues),
114 |     )
115 |   }
116 |   return [issue]
117 | }
118 | 
119 | function getZodIssueMessage(issue: ZodIssue): string {
120 |   switch (issue.code) {
121 |     case 'invalid_type': {
122 |       const attr = issue.path.at(-1)
123 |       return typeof attr === 'string'
124 |         ? `Expected type \`${issue.expected}\` for attribute "${attr}", but received \`${issue.input}\`.`
125 |         : `Expected type \`${issue.expected}\`, but received \`${issue.input}\`.`
126 |     }
127 | 
128 |     case 'invalid_value': {
129 |       const attr = issue.path.at(-1)
130 |       const literalValues = issue.values.join(', ')
131 |       return typeof attr === 'string'
132 |         ? `Expected literal \`${literalValues}\` for attribute "${attr}", but received \`${issue.input}\`.`
133 |         : `Expected literal \`${literalValues}\`, but received \`${issue.input}\`.`
134 |     }
135 | 
136 |     case 'unrecognized_keys':
137 |       return `Unrecognized keys: ${issue.keys.join(', ')}.`
138 | 
139 |     case 'invalid_union':
140 |       return `Invalid union: ${issue.message}`
141 | 
142 |     case 'too_small':
143 |       return `Value is too small: ${issue.message || 'Value does not meet minimum size.'}`
144 | 
145 |     case 'too_big':
146 |       return `Value is too big: ${issue.message || 'Value exceeds maximum size.'}`
147 | 
148 |     case 'not_multiple_of':
149 |       return `Value is not a multiple of ${issue.divisor}: ${issue.message || 'Value does not meet multiple of condition.'}`
150 | 
151 |     case 'custom':
152 |       return `Custom validation error: ${issue.message || 'No additional message provided.'}`
153 | 
154 |     case 'invalid_key':
155 |       return `Invalid key: ${issue.message || 'Key validation failed.'}`
156 | 
157 |     case 'invalid_format':
158 |       return `Invalid format: ${issue.message || `Expected format ${issue.format}.`}`
159 | 
160 |     case 'invalid_element':
161 |       return `Invalid element: ${issue.message || 'Element validation failed.'}`
162 | 
163 |     default:
164 |       // The types are exhaustive, but we don't want to miss any new ones
165 |       return 'Unknown validation error.'
166 |   }
167 | }
168 | 
169 | export function getMostSpecificError(error: ZodIssue): {
170 |   message: string
171 |   path: PropertyKey[]
172 | } {
173 |   const allIssues = collectAllLeafIssues(error)
174 |   const mostSpecific = allIssues.reduce(
175 |     (acc, cur) => (cur.path.length > acc.path.length ? cur : acc),
176 |     allIssues[0]!,
177 |   )
178 |   return {
179 |     message: getZodIssueMessage(mostSpecific),
180 |     path: mostSpecific.path,
181 |   }
182 | }
183 | 


--------------------------------------------------------------------------------