├── .env.example
├── .github
└── workflows
│ ├── publish.yml
│ └── test.yml
├── .gitignore
├── .gitmodules
├── .npmignore
├── CHANGELOG.md
├── CONTRIBUTING.md
├── LICENSE
├── README.md
├── jest.config.js
├── package-lock.json
├── package.json
├── src
├── converters.ts
├── dev
│ ├── regenerateGroundTruth.ts
│ ├── runLocalTest.ts
│ ├── testHtmlToMarkdown.ts
│ └── testUsage.ts
├── example.ts
├── extractors.ts
├── index.ts
├── types.ts
└── utils
│ └── schemaUtils.ts
├── tests
├── fixtures
│ ├── article-with-images.html
│ ├── blog-post.html
│ └── product-list.html
├── integration
│ ├── extract.test.ts
│ ├── html-to-markdown.test.ts
│ └── processedContent.test.ts
├── setup.ts
└── unit
│ ├── converters.test.ts
│ ├── extractors.test.ts
│ └── schemaUtils.test.ts
└── tsconfig.json
/.env.example:
--------------------------------------------------------------------------------
1 | # API Keys for testing
2 | GOOGLE_API_KEY=your_google_api_key_here
3 | OPENAI_API_KEY=your_openai_api_key_here
4 |
5 | # Test configuration
6 | TEST_TIMEOUT=30000
7 | LOG_LEVEL=info
--------------------------------------------------------------------------------
/.github/workflows/publish.yml:
--------------------------------------------------------------------------------
1 | name: Publish Package to NPM
2 |
3 | on:
4 | push:
5 | tags:
6 | - 'v*' # Run workflow on version tags, e.g. v1.0.0
7 |
8 | permissions:
9 | contents: write
10 | packages: write
11 |
12 | jobs:
13 | build-and-publish:
14 | runs-on: ubuntu-latest
15 | steps:
16 | - uses: actions/checkout@v3
17 | with:
18 | fetch-depth: 0 # Fetch all history for proper versioning
19 |
20 | - name: Use Node.js
21 | uses: actions/setup-node@v3
22 | with:
23 | node-version: '20.x'
24 | registry-url: 'https://registry.npmjs.org/'
25 | cache: 'npm'
26 |
27 | - name: Install dependencies
28 | run: npm ci
29 |
30 | - name: Build package
31 | run: npm run build
32 |
33 | - name: Run unit tests
34 | run: npm run test:unit
35 |
36 | - name: Update test data submodule
37 | run: npm run test:html2md:update
38 |
39 | - name: Run integration tests
40 | run: npm run test:integration
41 | env:
42 | OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
43 | GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
44 |
45 | - name: Generate release notes
46 | id: release
47 | run: |
48 | VERSION=${GITHUB_REF#refs/tags/}
49 | echo "version=$VERSION" >> $GITHUB_OUTPUT
50 | # Extract changes from git log or CHANGELOG if available
51 | CHANGES=$(git log --pretty=format:"* %s (%h)" $(git describe --tags --abbrev=0 HEAD^)..HEAD || echo "Initial release")
52 | echo "CHANGES<> $GITHUB_ENV
53 | echo "$CHANGES" >> $GITHUB_ENV
54 | echo "EOF" >> $GITHUB_ENV
55 |
56 | - name: Create GitHub Release
57 | uses: softprops/action-gh-release@v1
58 | with:
59 | name: Release ${{ steps.release.outputs.version }}
60 | body: |
61 | ## Changes in this release
62 |
63 | ${{ env.CHANGES }}
64 |
65 | For full details, see the [CHANGELOG](https://github.com/lightfeed/extractor/blob/main/CHANGELOG.md).
66 | draft: false
67 | prerelease: false
68 |
69 | - name: Publish to NPM
70 | run: npm publish --access public
71 | env:
72 | NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
--------------------------------------------------------------------------------
/.github/workflows/test.yml:
--------------------------------------------------------------------------------
1 | name: Test
2 |
3 | on:
4 | push:
5 | branches: [main]
6 | pull_request:
7 | branches: [main]
8 | schedule:
9 | - cron: '0 0 * * 1' # Run weekly on Monday at midnight UTC
10 |
11 | jobs:
12 | unit-tests:
13 | runs-on: ubuntu-latest
14 | strategy:
15 | matrix:
16 | node-version: [18.x, 20.x]
17 |
18 | steps:
19 | - uses: actions/checkout@v3
20 |
21 | - name: Use Node.js ${{ matrix.node-version }}
22 | uses: actions/setup-node@v3
23 | with:
24 | node-version: ${{ matrix.node-version }}
25 | cache: 'npm'
26 |
27 | - name: Install dependencies
28 | run: npm ci
29 |
30 | - name: Run unit tests
31 | run: npm run test:unit
32 |
33 | integration-tests:
34 | runs-on: ubuntu-latest
35 | needs: unit-tests
36 | strategy:
37 | matrix:
38 | node-version: [20.x]
39 |
40 | steps:
41 | - uses: actions/checkout@v3
42 |
43 | - name: Use Node.js ${{ matrix.node-version }}
44 | uses: actions/setup-node@v3
45 | with:
46 | node-version: ${{ matrix.node-version }}
47 | cache: 'npm'
48 |
49 | - name: Install dependencies
50 | run: npm ci
51 |
52 | - name: Update test data submodule
53 | run: npm run test:html2md:update
54 |
55 | - name: Run integration tests
56 | run: npm run test:integration
57 | env:
58 | OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
59 | GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Dependencies
2 | node_modules/
3 | yarn.lock
4 |
5 | # Build output
6 | dist/
7 | build/
8 | lib/
9 |
10 | # Environment variables
11 | .env
12 | .env.local
13 | .env.*.local
14 |
15 | # Logs
16 | logs
17 | *.log
18 | npm-debug.log*
19 | yarn-debug.log*
20 | yarn-error.log*
21 |
22 | # IDE and editors
23 | .idea/
24 | .vscode/
25 | *.swp
26 | *.swo
27 | .DS_Store
28 |
29 | # Test coverage
30 | coverage/
31 |
32 | # Temporary files
33 | tmp/
34 | temp/
35 |
36 | # Optionally fetched test data submodule
37 | /test-data/
38 |
39 | # Dev test output
40 | /dev-output/
41 |
42 | # Keep the .gitmodules file that defines the submodule
43 | !/.gitmodules
--------------------------------------------------------------------------------
/.gitmodules:
--------------------------------------------------------------------------------
1 | [submodule "test-data"]
2 | path = test-data
3 | url = https://github.com/lightfeed/extractor-test-data.git
4 |
--------------------------------------------------------------------------------
/.npmignore:
--------------------------------------------------------------------------------
1 | # Source
2 | src/
3 | tests/
4 | test-data/
5 |
6 | # Config files
7 | .github/
8 | .git/
9 | .gitignore
10 | .gitmodules
11 | .editorconfig
12 | .prettierrc
13 | .eslintrc
14 | .env*
15 | .vscode/
16 | tsconfig.json
17 | tslint.json
18 | jest.config.js
19 |
20 | # Build artifacts
21 | coverage/
22 | node_modules/
23 |
24 | # Development files
25 | *.log
26 | .DS_Store
27 | examples/
28 | src/dev/
29 | src/example.ts
--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
1 | # Changelog
2 |
3 | All notable changes to this project will be documented in this file.
4 |
5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7 |
8 | ## [Unreleased]
9 |
10 | ## [0.1.7] - 2025-06-07
11 |
12 | ## Changed
13 | - Updte README to use @lightfeed/extractor as new npm project
14 |
15 | ## [0.1.6] - 2025-06-07
16 |
17 | ## Changed
18 | - Update project name to lightfeed/extractor and publish to npm project @lightfeed/extractor
19 |
20 | ## [0.1.5] - 2025-05-14
21 |
22 | ## Fixed
23 | - Improve main html content extraction - preserve option, label and select (can be important for product detail pages)
24 |
25 | ## [0.1.4] - 2025-05-13
26 |
27 | ## Fixed
28 | - Fixed schema conversion bug when input zod schema is from a different zod version
29 |
30 | ## [0.1.3] - 2025-05-13
31 |
32 | ### Added
33 | - Use processedContent instead of markdown in response
34 | - Improve enrich prompt to not remove any fields from the original JSON object
35 |
36 | ## [0.1.2] - 2025-05-12
37 |
38 | ### Added
39 | - Support enriching data
40 | - Handle nullable instead of optional in schema. This is required for schema in OpenAI models
41 |
42 | ## [0.1.1] - 2025-05-11
43 |
44 | ### Added
45 | - Initial release with core functionality
46 | - HTML to Markdown conversion with main content extraction
47 | - Structured data extraction with LLM support
48 | - Support for OpenAI and Google Gemini API
49 | - URL validation and fixing
50 | - Comprehensive test suite
51 |
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # Contributing to lightfeed/extractor
2 |
3 | Thank you for considering contributing to lightfeed/extractor! This document outlines the process for contributing to the project and releasing new versions.
4 |
5 | ## Development Workflow
6 |
7 | 1. Fork the repository
8 | 2. Create a feature branch (`git checkout -b feature/amazing-feature`)
9 | 3. Make your changes
10 | 4. Run tests to ensure everything works:
11 | - `npm run test:unit` - Run unit tests
12 | - `npm run test:integration` - Run integration tests (requires API keys)
13 | - `npm run test:html2md` - Run HTML to Markdown tests
14 | 5. Commit your changes (`git commit -m 'Add some amazing feature'`)
15 | 6. Push to the branch (`git push origin feature/amazing-feature`)
16 | 7. Open a Pull Request
17 |
18 | ## CI/CD Pipeline
19 |
20 | This project uses GitHub Actions for continuous integration and deployment:
21 |
22 | ### Testing Workflow
23 |
24 | The testing workflow runs automatically:
25 | - On each push to the `main` branch
26 | - On each pull request to the `main` branch
27 | - Weekly on Monday at midnight UTC
28 |
29 | The workflow includes:
30 | 1. Unit tests - Run across multiple Node.js versions (18.x, 20.x)
31 | 2. Integration tests - Run on Node.js 20.x using provided API secrets
32 |
33 | ### Setting up API keys for CI
34 |
35 | To enable integration tests in CI, add your API keys as secrets in your GitHub repository:
36 |
37 | 1. Go to your GitHub repository
38 | 2. Click on "Settings" > "Secrets and variables" > "Actions"
39 | 3. Add the following secrets:
40 | - `OPENAI_API_KEY` - Your OpenAI API key
41 | - `GOOGLE_API_KEY` - Your Google API key
42 |
43 | ## Release Process
44 |
45 | This project uses semantic versioning. To create a new release:
46 |
47 | 1. Update the version in `package.json`
48 | 2. Update the `CHANGELOG.md` with details of the changes
49 | 3. Commit these changes with a message like "Bump version to x.y.z"
50 | 4. Create and push a new tag:
51 | ```
52 | git tag -a vx.y.z -m "Release version x.y.z"
53 | git push origin vx.y.z
54 | ```
55 |
56 | When you push a new tag prefixed with "v" (e.g., v1.0.0), GitHub Actions will automatically:
57 | 1. Build the package
58 | 2. Run unit tests
59 | 3. Create a GitHub Release with notes from your git history
60 | 4. Publish the package to npm
61 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright [2025] [Revar Immersive Technology Inc.]
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 | Lightfeed Extractor
4 |
5 |
6 |
7 | Use LLMs to robustly extract structured data from HTML and markdown
8 |
31 |
32 | ## How It Works
33 |
34 | 1. **HTML to Markdown Conversion**: If the input is HTML, it's first converted to clean, LLM-friendly markdown. This step can optionally extract only the main content and include images. See [HTML to Markdown Conversion](#html-to-markdown-conversion) section for details. The `convertHtmlToMarkdown` function can also be used standalone.
35 |
36 | 2. **LLM Processing**: The markdown is sent to an LLM in JSON mode (Google Gemini 2.5 flash or OpenAI GPT-4o mini by default) with a prompt to extract structured data according to your Zod schema or enrich existing data objects. You can set a maximum input token limit to control costs or avoid exceeding the model's context window, and the function will return token usage metrics for each LLM call.
37 |
38 | 3. **JSON Sanitization**: If the LLM structured output fails or doesn't fully match your schema, a sanitization process attempts to recover and fix the data. This makes complex schema extraction much more robust, especially with deeply nested objects and arrays. See [JSON Sanitization](#json-sanitization) for details.
39 |
40 | 4. **URL Validation**: All extracted URLs are validated - handling relative URLs, removing invalid ones, and repairing markdown-escaped links. See [URL Validation](#url-validation) section for details.
41 |
42 | ## Why use an LLM extractor?
43 | 💡 Understands natural language criteria and context to extract the data you need, not just raw content as displayed
44 |
45 | ⚡️ No need to manually create custom scraper code for each site
46 |
47 | 🔁 Resilient to website changes, e.g., HTML structure, CSS selectors, or page layout
48 |
49 | ✅ LLMs are becoming more accurate and cost-effective
50 |
51 | ## Installation
52 |
53 | ```bash
54 | npm install @lightfeed/extractor
55 | ```
56 |
57 | ## Hosted Version
58 |
59 | While this library provides a robust foundation for data extraction, you might want to consider [lightfeed.ai](https://lightfeed.ai) if you need:
60 |
61 | - **Persistent Searchable Databases**: Automatically store and manage extracted data in a production-ready vector database
62 | - **Scheduled Runs, Deduplication and Tracking**: Smart detection and handling of duplicate content across your sources, with automated change tracking
63 | - **Deep Link Extraction**: Follow links to collect complete data from connected pages
64 | - **Real-time API and Integration**: Query your extracted data through robust API endpoints and integrations
65 | - **Research Portal**: Explore and analyze your data through an intuitive interface
66 |
67 | ## Usage
68 |
69 | ### Basic Example
70 |
71 | ```typescript
72 | import { extract, ContentFormat, LLMProvider } from "@lightfeed/extractor";
73 | import { z } from "zod";
74 |
75 | async function main() {
76 | // Define your schema. We will run one more sanitization process to recover imperfect, failed, or partial LLM outputs into this schema
77 | const schema = z.object({
78 | title: z.string(),
79 | author: z.string().optional(),
80 | date: z.string().optional(),
81 | tags: z.array(z.string()),
82 | summary: z.string().describe("A brief summary of the article content within 500 characters"),
83 | // Use .url() to fix and validate URL field
84 | links: z.array(z.string().url()).describe("All URLs mentioned in the article")
85 | });
86 |
87 | // Extract from HTML
88 | const result = await extract({
89 | content: `
90 |
91 |
Understanding Async/Await in JavaScript
92 |
93 | John Doe |
94 | January 15, 2023 |
95 | #JavaScript #Programming
96 |
97 |
This article explains how async/await works in modern JavaScript.
100 |
101 | `,
102 | format: ContentFormat.HTML,
103 | schema,
104 | sourceUrl: "https://example.com/blog/async-await", // Required for HTML format to handle relative URLs
105 | googleApiKey: "your-google-gemini-api-key",
106 | });
107 |
108 | console.log("Extracted Data:", result.data);
109 | console.log("Token Usage:", result.usage);
110 | }
111 |
112 | main().catch(console.error);
113 | ```
114 |
115 | ### Extracting from Markdown or Plain Text
116 |
117 | You can also extract structured data directly from Markdown string:
118 |
119 | ```typescript
120 | const result = await extract({
121 | content: markdownContent,
122 | // Specify that content is Markdown. In addition to HTML and Markdown, you can also extract plain text by ContentFormat.TXT
123 | format: ContentFormat.MARKDOWN,
124 | schema: mySchema,
125 | googleApiKey: "your-google-gemini-api-key",
126 | });
127 | ```
128 |
129 | ### Custom Extraction Prompts
130 |
131 | You can provide a custom prompt to guide the extraction process:
132 |
133 | ```typescript
134 | const result = await extract({
135 | content: htmlContent,
136 | format: ContentFormat.HTML,
137 | schema: mySchema,
138 | sourceUrl: "https://example.com/products",
139 | // In custom prompt, defined what data should be retrieved
140 | prompt: "Extract ONLY products that are on sale or have special discounts. Include their original prices, discounted prices, and product URL.",
141 | googleApiKey: "your-google-gemini-api-key",
142 | });
143 | ```
144 |
145 | If no prompt is provided, a default extraction prompt will be used.
146 |
147 | ### Data Enrichment
148 |
149 | You can use the `dataToEnrich` option to provide an existing data object that will be enriched with additional information from the content. This is particularly useful for:
150 |
151 | - Updating incomplete records with missing information
152 | - Enhancing existing data with new details from content
153 | - Merging data from multiple sources
154 |
155 | The LLM will be instructed to enrich the provided object rather than creating a completely new one:
156 |
157 | ```typescript
158 | // Example of enriching a product record with missing information
159 | const productToEnrich = {
160 | productUrl: "https://example.com/products/smart-security-camera",
161 | name: "",
162 | price: 0,
163 | reviews: [],
164 | };
165 |
166 | const result = await extract({
167 | content: htmlContent,
168 | format: ContentFormat.HTML,
169 | schema: productSchema,
170 | sourceUrl: "https://example.com/products/smart-security-camera",
171 | prompt: "Enrich the product data with complete details from the product page.",
172 | dataToEnrich: productToEnrich,
173 | googleApiKey: "your-google-gemini-api-key",
174 | });
175 |
176 | // Result will contain the original data enriched with information from the content
177 | console.log(result.data);
178 | // {
179 | // productUrl: "https://example.com/products/smart-security-camera" // Preserved from original object
180 | // name: "Smart Security Camera", // Enriched from the product page
181 | // price: 74.50, // Enriched from the product page
182 | // reviews: ["I really like this camera", ...] // Reviews enriched from the product page
183 | // }
184 | ```
185 |
186 | ### Customizing LLM Provider and Managing Token Limits
187 |
188 | You can customize LLM and manage token limits to control costs and ensure your content fits within the model's maximum context window:
189 |
190 | ```typescript
191 | // Extract from Markdown with token limit
192 | const result = await extract({
193 | content: markdownContent,
194 | format: ContentFormat.MARKDOWN,
195 | schema,
196 | // Provide model provider and model name
197 | provider: LLMProvider.OPENAI,
198 | modelName: "gpt-4o-mini",
199 | openaiApiKey: "your-openai-api-key",
200 | // Limit to roughly 128K tokens (max input for gpt-4o-mini)
201 | maxInputTokens: 128000,
202 | });
203 | ```
204 |
205 | > [!WARNING]
206 | > For OpenAI models, optional schema is not supported. You need to change `.optional()` to `.nullable()`.
207 |
208 | ### Extracting from Main HTML
209 |
210 | For blog posts or articles with lots of navigation elements, headers, and footers, you can use the `extractMainHtml` option to focus on just the main content:
211 |
212 | ```typescript
213 | const result = await extract({
214 | content: htmlContent,
215 | format: ContentFormat.HTML,
216 | schema: mySchema,
217 | htmlExtractionOptions: {
218 | extractMainHtml: true // Uses heuristics to remove navigation, headers, footers, etc.
219 | },
220 | sourceUrl,
221 | });
222 | ```
223 |
224 | > [!NOTE]
225 | > The `extractMainHtml` option only applies to HTML format. It uses heuristics to identify and extract what appears to be the main content area (like article or main tags). It's recommended to keep this option off (false) when extracting details about a single item (like detail page for a product) as it might remove important contextual elements.
226 |
227 | ### Extracting Images from HTML
228 |
229 | By default, images are excluded from the HTML extraction process to simplify the output. If you need to extract image URLs or references, you can enable the `includeImages` option:
230 |
231 | ```typescript
232 | // Define a schema that includes product images
233 | const productListSchema = z.object({
234 | products: z.array(
235 | z.object({
236 | name: z.string(),
237 | price: z.number(),
238 | description: z.string().optional(),
239 | // Include an array of images for each product
240 | image: z.object({
241 | url: z.string().url(),
242 | alt: z.string().optional(),
243 | }).optional(),
244 | })
245 | ),
246 | });
247 |
248 | const result = await extract({
249 | content: htmlContent,
250 | format: ContentFormat.HTML,
251 | schema: mySchema,
252 | htmlExtractionOptions: {
253 | includeImages: true // Includes images in the generated markdown
254 | },
255 | sourceUrl: sourceUrl,
256 | });
257 | ```
258 |
259 | ## API Keys
260 |
261 | The library will check for API keys in the following order:
262 |
263 | 1. Directly provided API key parameter (`googleApiKey` or `openaiApiKey`)
264 | 2. Environment variables (`GOOGLE_API_KEY` or `OPENAI_API_KEY`)
265 |
266 | While the library can use environment variables, it's recommended to explicitly provide API keys in production code for better control and transparency.
267 |
268 | ## API Reference
269 |
270 | ### `extract(options: ExtractorOptions): Promise>`
271 |
272 | Main function to extract structured data from content.
273 |
274 | #### Options
275 |
276 | | Option | Type | Description | Default |
277 | |--------|------|-------------|---------|
278 | | `content` | `string` | HTML, markdown, or plain text content to extract from | Required |
279 | | `format` | `ContentFormat` | Content format (HTML, MARKDOWN, or TXT) | Required |
280 | | `schema` | `z.ZodTypeAny` | Zod schema defining the structure to extract | Required |
281 | | `prompt` | `string` | Custom prompt to guide the extraction process | Internal default prompt |
282 | | `provider` | `LLMProvider` | LLM provider (GOOGLE_GEMINI or OPENAI) | `LLMProvider.GOOGLE_GEMINI` |
283 | | `modelName` | `string` | Model name to use | Provider-specific default, Google Gemini 2.5 flash or OpenAI GPT-4o mini |
284 | | `googleApiKey` | `string` | Google Gemini API key (if using Google Gemini provider) | From env `GOOGLE_API_KEY` |
285 | | `openaiApiKey` | `string` | OpenAI API key (if using OpenAI provider) | From env `OPENAI_API_KEY` |
286 | | `temperature` | `number` | Temperature for the LLM (0-1) | `0` |
287 | | `htmlExtractionOptions` | `HTMLExtractionOptions` | HTML-specific options for content extraction (see below) | `{}` |
288 | | `sourceUrl` | `string` | URL of the HTML content, required when format is HTML to properly handle relative URLs | Required for HTML format |
289 | | `maxInputTokens` | `number` | Maximum number of input tokens to send to the LLM. Uses a rough conversion of 4 characters per token. When specified, content will be truncated if the total prompt size exceeds this limit. | `undefined` |
290 | | `dataToEnrich` | `Record` | Original data object to enrich with information from the content. When provided, the LLM will be instructed to update this object rather than creating a new one from scratch. | `undefined` |
291 |
292 | #### HTML Extraction Options
293 |
294 | | Option | Type | Description | Default |
295 | |--------|------|-------------|---------|
296 | | `extractMainHtml` | `boolean` | When enabled for HTML content, attempts to extract the main content area, removing navigation bars, headers, footers, sidebars etc. using heuristics. Should be kept off when extracting details about a single item. | `false` |
297 | | `includeImages` | `boolean` | When enabled, images in the HTML will be included in the markdown output. Enable this when you need to extract image URLs or related content. | `false` |
298 |
299 | #### Return Value
300 |
301 | The function returns a Promise that resolves to an `ExtractorResult` object:
302 |
303 | ```typescript
304 | interface ExtractorResult {
305 | data: T; // Extracted structured data
306 | processedContent: string; // Processed content that was sent to the LLM. Markdown if the input was HTM (after conversion)
307 | usage: { // Token usage statistics
308 | inputTokens?: number;
309 | outputTokens?: number;
310 | };
311 | }
312 | ```
313 |
314 | ### HTML to Markdown Conversion
315 |
316 | The `convertHtmlToMarkdown` utility function allows you to convert HTML content to markdown without performing extraction.
317 |
318 | **Function signature:**
319 | ```typescript
320 | convertHtmlToMarkdown(html: string, options?: HTMLExtractionOptions, sourceUrl?: string): string
321 | ```
322 |
323 | #### Parameters
324 |
325 | | Parameter | Type | Description | Default |
326 | |-----------|------|-------------|---------|
327 | | `html` | `string` | HTML content to convert to markdown | Required |
328 | | `options` | `HTMLExtractionOptions` | See [HTML Extraction Options](#html-extraction-options) | `undefined` |
329 | | `sourceUrl` | `string` | URL of the HTML content, used to properly convert relative URLs to absolute URLs | `undefined` |
330 |
331 | #### Return Value
332 |
333 | The function returns a string containing the markdown conversion of the HTML content.
334 |
335 | #### Example
336 |
337 | ```typescript
338 | import { convertHtmlToMarkdown, HTMLExtractionOptions } from "@lightfeed/extractor";
339 |
340 | // Basic conversion
341 | const markdown = convertHtmlToMarkdown("
Hello World
This is a test
");
342 | console.log(markdown);
343 | // Output: "Hello World\n===========\n\nThis is a test"
344 |
345 | // With options to extract main content and include images
346 | const options: HTMLExtractionOptions = {
347 | extractMainHtml: true,
348 | includeImages: true
349 | };
350 |
351 | // With source URL to handle relative links
352 | const markdownWithOptions = convertHtmlToMarkdown(
353 | `
354 |
355 | Header
356 |
360 |
361 |
362 | `,
363 | options,
364 | "https://example.com"
365 | );
366 | console.log(markdownWithOptions);
367 | // Output: "[About](https://example.com/about)"
368 | ```
369 |
370 | ### JSON Sanitization
371 |
372 | The `safeSanitizedParser` utility function helps sanitize and recover partial data from LLM outputs that may not perfectly conform to your schema.
373 |
374 | **Function signature:**
375 | ```typescript
376 | safeSanitizedParser(schema: ZodTypeAny, rawObject: unknown): z.infer | null
377 | ```
378 |
379 | ```typescript
380 | import { safeSanitizedParser } from "@lightfeed/extractor";
381 | import { z } from "zod";
382 |
383 | // Define a product catalog schema
384 | const productSchema = z.object({
385 | products: z.array(
386 | z.object({
387 | id: z.number(),
388 | name: z.string(), // Required field
389 | price: z.number().optional(), // Optional number
390 | inStock: z.boolean().optional(),
391 | category: z.string().optional(),
392 | })
393 | ),
394 | storeInfo: z.object({
395 | name: z.string(),
396 | location: z.string().optional(),
397 | rating: z.number().optional(),
398 | })
399 | });
400 |
401 | // Example LLM output with realistic validation issues
402 | const rawLLMOutput = {
403 | products: [
404 | {
405 | id: 1,
406 | name: "Laptop",
407 | price: 999,
408 | inStock: true,
409 | }, // Valid product
410 | {
411 | id: 2,
412 | name: "Headphones",
413 | price: "N/A", // Non-convertible string for optional number
414 | inStock: true,
415 | category: "Audio",
416 | },
417 | {
418 | id: 3,
419 | // Missing required "name" field
420 | price: 45.99,
421 | inStock: false
422 | },
423 | {
424 | id: 4,
425 | name: "Keyboard",
426 | price: 59.99,
427 | inStock: true
428 | } // Valid product
429 | ],
430 | storeInfo: {
431 | name: "TechStore",
432 | location: "123 Main St",
433 | rating: "N/A" // Invalid: rating is not a number
434 | }
435 | };
436 |
437 | // Sanitize the data to recover what's valid
438 | const sanitizedData = safeSanitizedParser(productSchema, rawLLMOutput);
439 |
440 | // Result:
441 | // {
442 | // products: [
443 | // {
444 | // id: 1,
445 | // name: "Laptop",
446 | // price: 999,
447 | // inStock: true,
448 | // },
449 | // {
450 | // id: 2,
451 | // name: "Headphones",
452 | // inStock: true,
453 | // category: "Audio",
454 | // },
455 | // {
456 | // id: 4,
457 | // name: "Keyboard",
458 | // price: 59.99,
459 | // inStock: true,
460 | // }
461 | // ],
462 | // storeInfo: {
463 | // name: "TechStore",
464 | // location: "123 Main St",
465 | // }
466 | // }
467 | ```
468 |
469 | This utility is especially useful when:
470 | - LLMs return non-convertible data for optional fields (like "N/A" for numbers)
471 | - Some objects in arrays are missing required fields
472 | - Objects contain invalid values that don't match constraints
473 | - You want to recover as much valid data as possible while safely removing problematic parts
474 |
475 | ### URL Validation
476 |
477 | The library provides robust URL validation and handling through Zod's `z.string().url()` validator:
478 |
479 | ```typescript
480 | const schema = z.object({
481 | title: z.string(),
482 | link: z.string().url(), // Full URL validation works!
483 | sources: z.array(z.string().url()) // Also works with arrays of URLs
484 | });
485 |
486 | const result = await extract({
487 | content: markdownContent,
488 | format: ContentFormat.MARKDOWN,
489 | schema,
490 | // ... other options
491 | });
492 | ```
493 |
494 | #### How URL Validation Works
495 |
496 | Our URL validation system provides several key benefits:
497 |
498 | 1. **Validation**: Uses Zod's built-in `url()` validator to ensure URLs are properly formatted
499 | 2. **Special Character Handling**: Automatically fixes URLs with escaped special characters in markdown (e.g., `https://example.com/meeting-\(2023\)` becomes `https://example.com/meeting-(2023)`)
500 | 3. **Relative URL Resolution**: Converts relative URLs to absolute URLs when `sourceUrl` is provided
501 | 4. **Invalid URL Handling**: Skips invalid URLs rather than failing the entire extraction using our `safeSanitizedParser`
502 |
503 | This approach ensures reliable URL extraction while maintaining the full power of Zod's schema validation.
504 |
505 | ## Development
506 |
507 | ### Setup
508 |
509 | 1. Clone the repository
510 | 2. Install dependencies with `npm install`
511 | 3. Create a `.env` file in the root directory with your API keys (see `.env.example`)
512 |
513 | ### Scripts
514 |
515 | - `npm run build` - Build the library
516 | - `npm run clean` - Remove build artifacts
517 | - `npm run test` - Run all tests (requires API keys for integration tests)
518 | - `npm run dev` - Run the example file
519 |
520 | ### Running Local Tests
521 |
522 | You can test the library with real API calls and sample HTML files:
523 |
524 | ```bash
525 | # Run all local tests with both providers
526 | npm run test:local
527 |
528 | # Run specific test type with both providers
529 | npm run test:local -- blog
530 | npm run test:local -- product
531 |
532 | # Run tests with a specific provider
533 | npm run test:local -- blog openai # Test blog extraction with OpenAI
534 | npm run test:local -- product gemini # Test product extraction with Google Gemini
535 | ```
536 |
537 | ### Testing
538 |
539 | The library includes both unit tests and integration tests:
540 |
541 | - **Unit tests**: Test individual components without making API calls
542 | - **Integration tests**: Test full extraction pipeline with real API calls
543 |
544 | Integration tests require valid API keys to be provided in your `.env` file or environment variables. Tests will fail if required API keys are not available.
545 |
546 | Each integration test runs with both Google Gemini and OpenAI to ensure compatibility across providers.
547 |
548 | #### HTML to Markdown Integration Tests
549 |
550 | This project includes comprehensive integration tests for the HTML to Markdown converter using real-world HTML samples. The tests validate three conversion types:
551 |
552 | 1. Basic conversion (no images)
553 | 2. Main content extraction (no images)
554 | 3. Conversion with images included
555 |
556 | These tests use a Git submodule with HTML files and groundtruth markdown files. The submodule is not downloaded by default to keep the repository lightweight. To run these tests:
557 |
558 | ```bash
559 | # First time: Initialize and download the test data submodule
560 | npm run test:html2md:update
561 |
562 | # Run the HTML to Markdown integration tests
563 | npm run test:html2md
564 |
565 | # Update test data if new test files are available
566 | npm run test:html2md:sync
567 | ```
568 |
569 | The test suite automatically discovers all available test files and creates test cases for each conversion type that has a corresponding groundtruth file.
570 |
571 | #### Running Specific Tests
572 |
573 | You can run individual tests by using the `-t` flag with a pattern that matches the test description:
574 |
575 | ```bash
576 | # Run a specific test by exact description
577 | npm run test -- -t "should extract blog post data using Google Gemini default model"
578 |
579 | # Run all tests that include a specific keyword
580 | npm run test -- -t "blog post"
581 |
582 | # Run all tests for a specific provider
583 | npm run test -- -t "OpenAI"
584 |
585 | # Run all unit tests for a specific utility
586 | npm run test -- -t "safeSanitizedParser"
587 |
588 | # Run specific HTML to Markdown tests
589 | npm run test -- -t "should convert forum/tech-0 to markdown"
590 | ```
591 |
592 | The `-t` flag uses pattern matching, so you can be as specific or general as needed to select the tests you want to run.
593 |
594 | ## Support
595 |
596 | If you need direct assistance with your implementation:
597 | - Email us at support@lightfeed.ai
598 | - Open an issue in this repository
599 | - Post your question in our [Discord community](https://discord.gg/txZ2s4pgQJ)
600 |
601 | ## License
602 |
603 | Apache 2.0
604 |
--------------------------------------------------------------------------------
/jest.config.js:
--------------------------------------------------------------------------------
1 | /** @type {import('ts-jest').JestConfigWithTsJest} */
2 | module.exports = {
3 | preset: "ts-jest",
4 | testEnvironment: "node",
5 | roots: ["/src", "/tests"],
6 | testMatch: ["**/__tests__/**/*.ts?(x)", "**/?(*.)+(spec|test).ts?(x)"],
7 | collectCoverage: true,
8 | coverageDirectory: "coverage",
9 | collectCoverageFrom: [
10 | "src/**/*.ts",
11 | "!src/dev/**/*.ts",
12 | "!src/**/*.d.ts",
13 | "!src/types.ts",
14 | "!src/example.ts",
15 | "!**/node_modules/**",
16 | "!**/vendor/**",
17 | ],
18 | transform: {
19 | "^.+\\.tsx?$": "ts-jest",
20 | },
21 | setupFiles: ["/tests/setup.ts"],
22 | watchman: false,
23 | };
24 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "@lightfeed/extractor",
3 | "version": "0.1.7",
4 | "description": "Use LLMs to robustly extract and enrich structured data from HTML and markdown",
5 | "main": "dist/index.js",
6 | "types": "dist/index.d.ts",
7 | "files": [
8 | "dist"
9 | ],
10 | "engines": {
11 | "node": ">=18"
12 | },
13 | "scripts": {
14 | "build": "tsc",
15 | "clean": "rimraf dist",
16 | "prepare": "npm run clean && npm run build",
17 | "prepublishOnly": "npm run test:unit",
18 | "test": "jest",
19 | "test:unit": "jest tests/unit",
20 | "test:integration": "jest tests/integration",
21 | "test:watch": "jest --watch",
22 | "test:cov": "jest --coverage",
23 | "test:local": "ts-node src/dev/runLocalTest.ts",
24 | "test:usage": "ts-node src/dev/testUsage.ts",
25 | "test:html2md": "jest tests/integration/html-to-markdown.test.ts",
26 | "test:html2md:update": "git submodule update --init --recursive test-data",
27 | "test:html2md:sync": "cd test-data && git pull origin main && cd ..",
28 | "test:html2md:regenerate": "ts-node src/dev/regenerateGroundTruth.ts",
29 | "lint": "tslint -p tsconfig.json",
30 | "dev": "ts-node src/example.ts",
31 | "dev:html2md": "ts-node src/dev/testHtmlToMarkdown.ts"
32 | },
33 | "repository": {
34 | "type": "git",
35 | "url": "git+https://github.com/lightfeed/extractor.git"
36 | },
37 | "keywords": [
38 | "llm",
39 | "extraction",
40 | "web-scraping",
41 | "html",
42 | "markdown",
43 | "structured-data",
44 | "openai",
45 | "gemini"
46 | ],
47 | "author": "Lightfeed",
48 | "license": "Apache-2.0",
49 | "bugs": {
50 | "url": "https://github.com/lightfeed/extractor/issues"
51 | },
52 | "homepage": "https://github.com/lightfeed/extractor#readme",
53 | "dependencies": {
54 | "@langchain/google-genai": "^0.2.5",
55 | "@langchain/openai": "^0.5.10",
56 | "cheerio": "^1.0.0",
57 | "jsonrepair": "^3.12.0",
58 | "langchain": "^0.3.24",
59 | "turndown": "^7.2.0",
60 | "xmldom": "^0.6.0",
61 | "xpath": "^0.0.34",
62 | "zod": "^3.24.3"
63 | },
64 | "devDependencies": {
65 | "@types/jest": "^29.5.12",
66 | "@types/node": "^22.15.3",
67 | "@types/turndown": "^5.0.5",
68 | "@types/xmldom": "^0.1.34",
69 | "dotenv": "^16.3.1",
70 | "jest": "^29.7.0",
71 | "rimraf": "^5.0.10",
72 | "ts-jest": "^29.1.2",
73 | "ts-node": "^10.9.2",
74 | "typescript": "^5.8.3"
75 | },
76 | "publishConfig": {
77 | "access": "public"
78 | }
79 | }
80 |
--------------------------------------------------------------------------------
/src/converters.ts:
--------------------------------------------------------------------------------
1 | import TurndownService from "turndown";
2 | import { HTMLExtractionOptions } from "./types";
3 | import { DOMParser, XMLSerializer } from "xmldom";
4 | import { isNodeLike } from "xpath";
5 | import * as url from "url";
6 |
7 | var xpath = require("xpath");
8 | const cheerio = require("cheerio");
9 |
10 | /**
11 | * Extract the main content from an HTML string if requested
12 | */
13 | function extractMainHtml(html: string): string {
14 | try {
15 | const bodyDoc = new DOMParser().parseFromString(html, "text/html");
16 |
17 | [...OVERALL_DISCARD_XPATH, ...PRECISION_DISCARD_XPATH].forEach((xPath) => {
18 | const result = xpath.parse(xPath).select({ node: bodyDoc, isHtml: true });
19 |
20 | // Ensure result is an array before calling forEach
21 | const nodes = Array.isArray(result) ? result : [result];
22 |
23 | nodes.forEach((node) => {
24 | if (isNodeLike(node) && node.parentNode) {
25 | node.parentNode.removeChild(node);
26 | }
27 | });
28 | });
29 |
30 | const refinedHtml = new XMLSerializer().serializeToString(bodyDoc);
31 | return refinedHtml == "" ? html : refinedHtml;
32 | } catch (error) {
33 | console.error("error extracting main html", error);
34 | return "";
35 | }
36 | }
37 |
38 | /**
39 | * Convert HTML to Markdown
40 | */
41 | export function htmlToMarkdown(
42 | html: string,
43 | options?: HTMLExtractionOptions,
44 | sourceUrl?: string
45 | ): string {
46 | // First clean up the html
47 | const tidiedHtml = tidyHtml(html, options?.includeImages ?? false);
48 |
49 | // Turndown config
50 | // Reference: https://github.com/jina-ai/reader/blob/1e3bae6aad9cf0005c14f0036b46b49390e63203/backend/functions/src/cloud-functions/crawler.ts#L134
51 | const turnDownService = new TurndownService();
52 |
53 | // Define elements to remove - conditionally include or exclude images
54 | const elementsToRemove: any[] = [
55 | "meta",
56 | "style",
57 | "script",
58 | "noscript",
59 | "link",
60 | "textarea",
61 | ];
62 |
63 | // Only remove image elements if includeImages is not enabled
64 | if (!options?.includeImages) {
65 | elementsToRemove.push("img", "picture", "figure");
66 | }
67 |
68 | turnDownService.addRule("remove-irrelevant", {
69 | filter: elementsToRemove,
70 | replacement: () => "",
71 | });
72 |
73 | turnDownService.addRule("truncate-svg", {
74 | filter: "svg" as any,
75 | replacement: () => "",
76 | });
77 |
78 | turnDownService.addRule("title-as-h1", {
79 | filter: ["title"],
80 | replacement: (innerText: string) => `${innerText}\n===============\n`,
81 | });
82 |
83 | turnDownService.addRule("improved-paragraph", {
84 | filter: "p",
85 | replacement: (innerText: string) => {
86 | const trimmed = innerText.trim();
87 | if (!trimmed) {
88 | return "";
89 | }
90 |
91 | return `${trimmed.replace(/\n{3,}/g, "\n\n")}\n\n`;
92 | },
93 | });
94 |
95 | turnDownService.addRule("improved-inline-link", {
96 | filter: function (node: any, options: any) {
97 | return Boolean(
98 | options.linkStyle === "inlined" &&
99 | node.nodeName === "A" &&
100 | node.getAttribute("href")
101 | );
102 | },
103 |
104 | replacement: function (content: string, node: any) {
105 | let href = node.getAttribute("href");
106 | if (href) {
107 | // Convert relative URLs to absolute if sourceUrl is provided
108 | if (
109 | sourceUrl &&
110 | !href.startsWith("http") &&
111 | !href.startsWith("mailto:")
112 | ) {
113 | try {
114 | href = url.resolve(sourceUrl, href);
115 | } catch (error) {
116 | console.warn(
117 | `Failed to resolve URL ${href} against ${sourceUrl}:`,
118 | error
119 | );
120 | }
121 | }
122 | href = href.replace(/([()])/g, "\\$1");
123 | }
124 | let title = cleanAttribute(node.getAttribute("title"));
125 | if (title) title = ' "' + title.replace(/"/g, '\\"') + '"';
126 |
127 | const fixedContent = content.replace(/\s+/g, " ").trim();
128 | const fixedHref = href.replace(/\s+/g, "").trim();
129 |
130 | return `[${fixedContent}](${fixedHref}${title || ""})`;
131 | },
132 | });
133 |
134 | turnDownService.addRule("images", {
135 | filter: "img",
136 |
137 | replacement: function (content: string, node: any) {
138 | let src = node.getAttribute("src");
139 | if (src) {
140 | // Convert relative URLs to absolute if sourceUrl is provided
141 | if (sourceUrl && !src.startsWith("http") && !src.startsWith("data:")) {
142 | try {
143 | src = url.resolve(sourceUrl, src);
144 | } catch (error) {
145 | console.warn(
146 | `Failed to resolve URL ${src} against ${sourceUrl}:`,
147 | error
148 | );
149 | }
150 | }
151 | src = src.replace(/([()])/g, "\\$1");
152 | } else {
153 | return ""; // No source, no image
154 | }
155 |
156 | let alt = cleanAttribute(node.getAttribute("alt") || "");
157 | let title = cleanAttribute(node.getAttribute("title"));
158 |
159 | if (title) title = ' "' + title.replace(/"/g, '\\"') + '"';
160 |
161 | const fixedSrc = src.replace(/\s+/g, "").trim();
162 |
163 | return ``;
164 | },
165 | });
166 |
167 | const fullMarkdown = turnDownService.turndown(tidiedHtml).trim();
168 | if (options?.extractMainHtml) {
169 | const mainHtml = extractMainHtml(tidiedHtml);
170 | const mainMarkdown = turnDownService.turndown(mainHtml).trim();
171 | // Heristics:
172 | // If main content is empty or is less than 20% of full content and not too short, use full content
173 | if (
174 | mainMarkdown.length == 0 ||
175 | (mainMarkdown.length < fullMarkdown.length * 0.2 &&
176 | mainMarkdown.length < 500)
177 | ) {
178 | return fullMarkdown;
179 | } else {
180 | return mainMarkdown;
181 | }
182 | } else {
183 | return fullMarkdown;
184 | }
185 | }
186 |
187 | // Clean up the html
188 | function tidyHtml(html: string, includeImages: boolean): string {
189 | const $ = cheerio.load(html);
190 | $("*").each(function (this: any) {
191 | const element = $(this);
192 | const attributes = Object.keys(this.attribs);
193 |
194 | for (let i = 0; i < attributes.length; i++) {
195 | let attr = attributes[i];
196 | // Check if the attribute value has an odd number of quotes
197 | // If the attribute name has a quote, it might be a broken attribute. Remove it completely.
198 | // (this occured at dealnews.com)
199 | if (attr.includes('"')) {
200 | element.remove();
201 | }
202 | }
203 | });
204 |
205 | // Adatpted from https://github.com/adbar/trafilatura/blob/c7e00f3a31e436c7b6ce666b44712e16e30908c0/trafilatura/settings.py#L55
206 | // Removed (because user might want to extract them):
207 | // - form
208 | // - fieldset
209 | // - footer (might contain company info)
210 | // - img, picture, figure (if includeImages is false)
211 | // - option, label, select (this can present product options and titles)
212 | const manuallyCleanedElements = [
213 | // important
214 | "aside",
215 | "embed",
216 | // "footer",
217 | // "form",
218 | "head",
219 | "iframe",
220 | "menu",
221 | "object",
222 | "script",
223 | // other content
224 | "applet",
225 | "audio",
226 | "canvas",
227 | "map",
228 | "svg",
229 | "video",
230 | // secondary
231 | "area",
232 | "blink",
233 | "button",
234 | "datalist",
235 | "dialog",
236 | "frame",
237 | "frameset",
238 | // "fieldset",
239 | "link",
240 | "input",
241 | "ins",
242 | // "label",
243 | "legend",
244 | "marquee",
245 | "math",
246 | "menuitem",
247 | "nav",
248 | "noscript",
249 | "optgroup",
250 | // "option",
251 | "output",
252 | "param",
253 | "progress",
254 | "rp",
255 | "rt",
256 | "rtc",
257 | // "select",
258 | "source",
259 | "style",
260 | "track",
261 | "textarea",
262 | "time",
263 | "use",
264 | ];
265 |
266 | if (!includeImages) {
267 | manuallyCleanedElements.push("img", "picture", "figure");
268 | }
269 |
270 | // Further clean html
271 | manuallyCleanedElements.forEach((element) => {
272 | $(element).remove();
273 | });
274 | return $("body").html();
275 | }
276 |
277 | function cleanAttribute(attribute: string) {
278 | return attribute ? attribute.replace(/(\n+\s*)+/g, "\n") : "";
279 | }
280 |
281 | // Adapted from https://github.com/adbar/trafilatura/blob/c7e00f3a31e436c7b6ce666b44712e16e30908c0/trafilatura/xpaths.py#L100
282 | // Added:
283 | // - Add contains(@id, "filter") to remove filter menus
284 | // - footer
285 | // Removed (because user might want to extract them):
286 | // - Commented out tags
287 | // - Commented out sidebar (sidebar sometimes can be too aggressive and can remove main content)
288 | // - Commented out author
289 | // - Commented out rating
290 | // - Commented out attachment
291 | // - Commented out timestamp
292 | // - Commented out user-info and user-profile
293 | // - Commented out comment or hidden section
294 | // - Not including @data-testid (it can remove dynamic product listings)
295 | // - Commented out options
296 | const OVERALL_DISCARD_XPATH = [
297 | // navigation + footers, news outlets related posts, sharing, jp-post-flair jp-relatedposts
298 | `.//*[(self::div or self::item or self::list
299 | or self::p or self::section or self::span)][
300 | contains(translate(@id, "F","f"), "footer") or contains(translate(@class, "F","f"), "footer")
301 | or contains(@id, "related") or contains(translate(@class, "R", "r"), "related") or
302 | contains(@id, "viral") or contains(@class, "viral") or
303 | contains(@id, "filter") or
304 | starts-with(@id, "shar") or starts-with(@class, "shar") or
305 | contains(@class, "share-") or
306 | contains(translate(@id, "S", "s"), "share") or
307 | contains(@id, "social") or contains(@class, "social") or contains(@class, "sociable") or
308 | contains(@id, "syndication") or contains(@class, "syndication") or
309 | starts-with(@id, "jp-") or starts-with(@id, "dpsp-content") or
310 | contains(@class, "embedded") or contains(@class, "embed")
311 | or contains(@id, "newsletter") or contains(@class, "newsletter")
312 | or contains(@class, "subnav") or
313 | contains(@id, "cookie") or contains(@class, "cookie") or ` +
314 | // `contains(@id, "tags") or contains(@class, "tags") or ` +
315 | // `contains(@id, "sidebar") or contains(@class, "sidebar") or ` +
316 | `contains(@id, "banner") or contains(@class, "banner")
317 | or contains(@class, "meta") or
318 | contains(@id, "menu") or contains(@class, "menu") or
319 | contains(translate(@id, "N", "n"), "nav") or contains(translate(@role, "N", "n"), "nav")
320 | or starts-with(@class, "nav") or contains(translate(@class, "N", "n"), "navigation") or
321 | contains(@class, "navbar") or contains(@class, "navbox") or starts-with(@class, "post-nav")
322 | or contains(@id, "breadcrumb") or contains(@class, "breadcrumb") or
323 | contains(@id, "bread-crumb") or contains(@class, "bread-crumb") or ` +
324 | // `contains(@id, "author") or contains(@class, "author") or ` +
325 | `contains(@id, "button") or contains(@class, "button")
326 | or contains(translate(@class, "B", "b"), "byline") or ` +
327 | // contains(@class, "rating") or ` +
328 | `starts-with(@class, "widget") or ` +
329 | // contains(@class, "attachment") or contains(@class, "timestamp") or
330 | // contains(@class, "user-info") or contains(@class, "user-profile") or
331 | `contains(@class, "-ad-") or contains(@class, "-icon")
332 | or contains(@class, "article-infos") or
333 | contains(translate(@class, "I", "i"), "infoline")
334 | or contains(@data-component, "MostPopularStories")
335 | or contains(@class, "outbrain") or contains(@class, "taboola")
336 | or contains(@class, "criteo") ` +
337 | // or contains(@class, "options")
338 | `or contains(@class, "consent") or contains(@class, "modal-content")
339 | or contains(@class, "paid-content") or contains(@class, "paidcontent")
340 | or contains(@id, "premium-") or contains(@id, "paywall")
341 | or contains(@class, "obfuscated") or contains(@class, "blurred")
342 | or contains(@class, " ad ")
343 | or contains(@class, "next-post") or contains(@class, "side-stories")
344 | or contains(@class, "related-stories") or contains(@class, "most-popular")
345 | or contains(@class, "mol-factbox") or starts-with(@class, "ZendeskForm")
346 | or contains(@class, "message-container") or contains(@id, "message_container")
347 | or contains(@class, "yin") or contains(@class, "zlylin") or
348 | contains(@class, "xg1") or contains(@id, "bmdh")
349 | or @data-lp-replacement-content]`,
350 | ".//footer",
351 |
352 | // comment debris + hidden parts
353 | // `.//*[@class="comments-title" or contains(@class, "comments-title") or
354 | // contains(@class, "nocomments") or starts-with(@id, "reply-") or starts-with(@class, "reply-") or
355 | // contains(@class, "-reply-") or contains(@class, "message")
356 | // or contains(@id, "akismet") or contains(@class, "akismet") or
357 | // starts-with(@class, "hide-") or contains(@class, "hide-print") or contains(@id, "hidden")
358 | // or contains(@style, "hidden") or contains(@hidden, "hidden") or contains(@class, "noprint")
359 | // or contains(@style, "display:none") or contains(@class, " hidden") or @aria-hidden="true"
360 | // or contains(@class, "notloaded")]`,
361 | ];
362 |
363 | // Adapted from https://github.com/adbar/trafilatura/blob/c7e00f3a31e436c7b6ce666b44712e16e30908c0/trafilatura/xpaths.py#L179
364 | // Removed:
365 | // - contains(@style, "border")
366 | const PRECISION_DISCARD_XPATH = [
367 | ".//header",
368 | `.//*[(self::div or self::item or self::list
369 | or self::p or self::section or self::span)][
370 | contains(@id, "bottom") or contains(@class, "bottom") or
371 | contains(@id, "link") or contains(@class, "link")
372 | ]`,
373 | ];
374 |
--------------------------------------------------------------------------------
/src/dev/regenerateGroundTruth.ts:
--------------------------------------------------------------------------------
1 | import * as fs from "fs";
2 | import * as path from "path";
3 | import { htmlToMarkdown } from "../converters";
4 | import { HTMLExtractionOptions } from "../types";
5 | import * as cheerio from "cheerio";
6 |
7 | // Function to sanitize HTML content
8 | function sanitizeHTML(html: string, originalSource: string): string {
9 | const $ = cheerio.load(html);
10 |
11 | // Remove scripts and event handlers
12 | $("script").remove();
13 | $("[onclick]").removeAttr("onclick");
14 | $("[onload]").removeAttr("onload");
15 | // Find all elements with attributes starting with "on" and remove them
16 | $("*").each(function () {
17 | const el = $(this);
18 | const node = el[0];
19 |
20 | // Skip if not an element node or has no attributes
21 | if (!node || node.type !== "tag" || !("attribs" in node)) return;
22 |
23 | // Now TypeScript knows node.attribs exists
24 | Object.keys(node.attribs)
25 | .filter((attr) => attr.startsWith("on"))
26 | .forEach((attr) => el.removeAttr(attr));
27 | });
28 | // Remove styles
29 | $("style").remove();
30 | $("[style]").removeAttr("style");
31 |
32 | // Replace text content with placeholder
33 | $("p, h1, h2, h3, h4, h5, span, div").each(function () {
34 | const el = $(this);
35 | if (el.children().length === 0) {
36 | // Only replace text in leaf nodes
37 | const originalText = el.text();
38 | const length = originalText.length;
39 |
40 | if (length > 0) {
41 | // Generate placeholder text with exactly the same length
42 | const loremIpsumBase =
43 | "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. ";
44 |
45 | // Create deterministic placeholder based on original length and first character
46 | let placeholder = "";
47 | // Repeat the base text as many times as needed
48 | while (placeholder.length < length) {
49 | placeholder += loremIpsumBase;
50 | }
51 |
52 | // Trim to exact length of original text
53 | placeholder = placeholder.substring(0, length);
54 |
55 | el.text(placeholder);
56 | }
57 | }
58 | });
59 |
60 | // Replace links
61 | $("a").each(function () {
62 | const el = $(this);
63 | const isEmail = el.attr("href") && el.attr("href")!.startsWith("mailto:");
64 | const isExternal =
65 | el.attr("href") &&
66 | (el.attr("href")!.startsWith("http") ||
67 | el.attr("href")!.startsWith("www"));
68 |
69 | // Replace with appropriate placeholder based on link type
70 | if (isEmail) {
71 | // Replace email links
72 | el.attr("href", "mailto:example@example.com");
73 | } else if (isExternal) {
74 | // Replace external links
75 | el.attr("href", "https://example.com/external-link");
76 | } else {
77 | // Replace internal/relative links
78 | el.attr("href", "/placeholder-page");
79 | }
80 |
81 | const originalLinkText = el.text().trim();
82 | const textLength = originalLinkText.length;
83 | if (textLength > 0) {
84 | // Base text patterns for different link types
85 | let placeholderBase = "Link Text";
86 |
87 | if (isEmail) {
88 | placeholderBase = "Email Link";
89 | } else if (isExternal) {
90 | placeholderBase = "External Link";
91 | } else {
92 | placeholderBase = "Page Link";
93 | }
94 |
95 | // Replace the link text
96 | el.text(placeholderBase);
97 | }
98 | });
99 |
100 | // Replace images with real placeholder services
101 | $("img").each(function () {
102 | const el = $(this);
103 | const width = el.attr("width") || 300;
104 | const height = el.attr("height") || 200;
105 |
106 | // Use a real placeholder image service
107 | el.attr("src", `https://picsum.photos/${width}/${height}`);
108 |
109 | // Add generic alt text if none exists
110 | if (!el.attr("alt")) {
111 | el.attr("alt", "Placeholder image");
112 | }
113 | });
114 |
115 | // Add attribution header
116 | const sanitizedHTML = $.html();
117 |
118 | return sanitizedHTML;
119 | }
120 |
121 | // Function to convert HTML to Markdown and save as ground truth
122 | async function generateGroundTruth(
123 | htmlFilePath: string,
124 | groundtruthDir: string,
125 | options?: HTMLExtractionOptions,
126 | variant: string = ""
127 | ) {
128 | try {
129 | // Read and sanitize the HTML file
130 | const originalHtml = fs.readFileSync(htmlFilePath, "utf8");
131 | const sanitizedHtml = sanitizeHTML(originalHtml, htmlFilePath);
132 |
133 | // Save sanitized HTML back to the original file
134 | fs.writeFileSync(htmlFilePath, sanitizedHtml);
135 | console.log(`✅ Sanitized HTML: ${htmlFilePath}`);
136 |
137 | // Convert to Markdown
138 | const markdown = htmlToMarkdown(sanitizedHtml, options);
139 |
140 | // Create groundtruth directory if it doesn't exist
141 | if (!fs.existsSync(groundtruthDir)) {
142 | fs.mkdirSync(groundtruthDir, { recursive: true });
143 | }
144 |
145 | // Generate output filename
146 | const baseName = path.basename(htmlFilePath, ".html");
147 | const outputFilename = variant
148 | ? `${baseName}.${variant}.md`
149 | : `${baseName}.md`;
150 | const outputPath = path.join(groundtruthDir, outputFilename);
151 |
152 | // Save the markdown
153 | fs.writeFileSync(outputPath, markdown);
154 | console.log(`✅ Generated ground truth: ${outputPath}`);
155 |
156 | return outputPath;
157 | } catch (error) {
158 | console.error("❌ Error generating ground truth:", error);
159 | throw error;
160 | }
161 | }
162 |
163 | // Main function to regenerate all ground truth files
164 | async function main() {
165 | const testDataDir = path.join(process.cwd(), "test-data");
166 |
167 | // Check if test-data directory exists
168 | if (!fs.existsSync(testDataDir)) {
169 | console.error(
170 | "❌ test-data directory not found. Please run 'npm run test:html2md:update' first."
171 | );
172 | process.exit(1);
173 | }
174 |
175 | const htmlDir = path.join(testDataDir, "html");
176 | const groundtruthDir = path.join(testDataDir, "groundtruth");
177 |
178 | // Get all categories (subdirectories under html/)
179 | const categories = fs
180 | .readdirSync(htmlDir, { withFileTypes: true })
181 | .filter((dirent) => dirent.isDirectory())
182 | .map((dirent) => dirent.name);
183 |
184 | console.log("\n🔍 Regenerating ground truth files...\n");
185 |
186 | // Process each category
187 | for (const category of categories) {
188 | console.log(`\n📁 Processing category: ${category}`);
189 |
190 | const categoryHtmlDir = path.join(htmlDir, category);
191 | const categoryGroundtruthDir = path.join(groundtruthDir, category);
192 |
193 | // Create category directory in groundtruth if it doesn't exist
194 | if (!fs.existsSync(categoryGroundtruthDir)) {
195 | fs.mkdirSync(categoryGroundtruthDir, { recursive: true });
196 | }
197 |
198 | // Get all HTML files in this category
199 | const htmlFiles = fs
200 | .readdirSync(categoryHtmlDir)
201 | .filter((file) => file.endsWith(".html"))
202 | .map((file) => file.replace(".html", ""));
203 |
204 | // Process each HTML file
205 | for (const filename of htmlFiles) {
206 | const htmlFilePath = path.join(categoryHtmlDir, `${filename}.html`);
207 |
208 | // Generate ground truth files with different options
209 | await generateGroundTruth(htmlFilePath, categoryGroundtruthDir); // Basic conversion
210 | await generateGroundTruth(
211 | htmlFilePath,
212 | categoryGroundtruthDir,
213 | { includeImages: true },
214 | "images"
215 | );
216 | await generateGroundTruth(
217 | htmlFilePath,
218 | categoryGroundtruthDir,
219 | { extractMainHtml: true },
220 | "main"
221 | );
222 | }
223 | }
224 |
225 | console.log("\n✨ All ground truth files have been regenerated!");
226 | }
227 |
228 | // Run the main function
229 | main().catch(console.error);
230 |
--------------------------------------------------------------------------------
/src/dev/runLocalTest.ts:
--------------------------------------------------------------------------------
1 | import * as fs from "fs";
2 | import * as path from "path";
3 | import { config } from "dotenv";
4 | import { z } from "zod";
5 | import { extract, ContentFormat, LLMProvider } from "../index";
6 |
7 | // Load environment variables from .env file
8 | config({ path: path.resolve(process.cwd(), ".env") });
9 |
10 | // Helper to load HTML test fixtures
11 | function loadFixture(filename: string): string {
12 | return fs.readFileSync(
13 | path.resolve(__dirname, "../../tests/fixtures", filename),
14 | "utf8"
15 | );
16 | }
17 |
18 | // Example schemas for different content types
19 | const blogSchema = z.object({
20 | title: z.string(),
21 | author: z.string().optional(),
22 | date: z.string().optional(),
23 | tags: z
24 | .array(z.string())
25 | .optional()
26 | .describe("Tags appear after the date. Do not include the # symbol."),
27 | summary: z.string(),
28 | content: z.string().optional(),
29 | });
30 |
31 | // OpenAI version with nullable instead of optional
32 | const blogSchemaOpenAI = z.object({
33 | title: z.string(),
34 | author: z.string().nullable(),
35 | date: z.string().nullable(),
36 | tags: z
37 | .array(z.string())
38 | .nullable()
39 | .describe("Tags appear after the date. Do not include the # symbol."),
40 | summary: z.string(),
41 | content: z.string().nullable(),
42 | });
43 |
44 | const productSchema = z.object({
45 | products: z.array(
46 | z.object({
47 | name: z.string(),
48 | price: z.string(),
49 | rating: z.string().optional(),
50 | description: z.string().optional(),
51 | features: z.array(z.string()).optional(),
52 | })
53 | ),
54 | });
55 |
56 | // OpenAI version with nullable instead of optional
57 | const productSchemaOpenAI = z.object({
58 | products: z.array(
59 | z.object({
60 | name: z.string(),
61 | price: z.string(),
62 | rating: z.string().nullable(),
63 | description: z.string().nullable(),
64 | features: z.array(z.string()).nullable(),
65 | })
66 | ),
67 | });
68 |
69 | // Test functions
70 | async function testBlogExtraction(provider = LLMProvider.GOOGLE_GEMINI) {
71 | console.log(`Testing blog post extraction with ${provider}...`);
72 |
73 | try {
74 | const html = loadFixture("blog-post.html");
75 |
76 | // Check for required API key
77 | if (provider === LLMProvider.GOOGLE_GEMINI && !process.env.GOOGLE_API_KEY) {
78 | console.error("Error: GOOGLE_API_KEY environment variable is required");
79 | process.exit(1);
80 | } else if (provider === LLMProvider.OPENAI && !process.env.OPENAI_API_KEY) {
81 | console.error("Error: OPENAI_API_KEY environment variable is required");
82 | process.exit(1);
83 | }
84 |
85 | const apiKey =
86 | provider === LLMProvider.GOOGLE_GEMINI
87 | ? process.env.GOOGLE_API_KEY
88 | : process.env.OPENAI_API_KEY;
89 |
90 | const result = await extract({
91 | content: html,
92 | format: ContentFormat.HTML,
93 | schema:
94 | provider === LLMProvider.GOOGLE_GEMINI ? blogSchema : blogSchemaOpenAI,
95 | provider,
96 | googleApiKey: provider === LLMProvider.GOOGLE_GEMINI ? apiKey : undefined,
97 | openaiApiKey: provider === LLMProvider.OPENAI ? apiKey : undefined,
98 | htmlExtractionOptions: {
99 | extractMainHtml: false,
100 | },
101 | sourceUrl: "https://www.example.com/blog/blog-post",
102 | });
103 |
104 | console.log("Extracted data:");
105 | console.log(JSON.stringify(result.data, null, 2));
106 | console.log("\nToken usage:");
107 | console.log(result.usage);
108 |
109 | return result;
110 | } catch (error) {
111 | console.error(`Blog extraction error with ${provider}:`, error);
112 | throw error;
113 | }
114 | }
115 |
116 | async function testProductExtraction(provider = LLMProvider.GOOGLE_GEMINI) {
117 | console.log(`Testing product listing extraction with ${provider}...`);
118 |
119 | try {
120 | const html = loadFixture("product-list.html");
121 |
122 | // Check for required API key
123 | if (provider === LLMProvider.GOOGLE_GEMINI && !process.env.GOOGLE_API_KEY) {
124 | console.error("Error: GOOGLE_API_KEY environment variable is required");
125 | process.exit(1);
126 | } else if (provider === LLMProvider.OPENAI && !process.env.OPENAI_API_KEY) {
127 | console.error("Error: OPENAI_API_KEY environment variable is required");
128 | process.exit(1);
129 | }
130 |
131 | const apiKey =
132 | provider === LLMProvider.GOOGLE_GEMINI
133 | ? process.env.GOOGLE_API_KEY
134 | : process.env.OPENAI_API_KEY;
135 |
136 | const result = await extract({
137 | content: html,
138 | format: ContentFormat.HTML,
139 | schema:
140 | provider === LLMProvider.GOOGLE_GEMINI
141 | ? productSchema
142 | : productSchemaOpenAI,
143 | provider,
144 | googleApiKey: provider === LLMProvider.GOOGLE_GEMINI ? apiKey : undefined,
145 | openaiApiKey: provider === LLMProvider.OPENAI ? apiKey : undefined,
146 | htmlExtractionOptions: {
147 | extractMainHtml: true,
148 | },
149 | sourceUrl: "https://www.example.com/product/product-list",
150 | });
151 |
152 | console.log("Extracted data:");
153 | console.log(JSON.stringify(result.data, null, 2));
154 | console.log("\nToken usage:");
155 | console.log(result.usage);
156 |
157 | return result;
158 | } catch (error) {
159 | console.error(`Product extraction error with ${provider}:`, error);
160 | throw error;
161 | }
162 | }
163 |
164 | // Run tests based on command line arguments
165 | async function main() {
166 | // Parse arguments: content type and provider
167 | const args = process.argv.slice(2);
168 | const contentType = args[0] || "all"; // 'blog', 'product', or 'all'
169 | const provider =
170 | args[1]?.toUpperCase() === "OPENAI"
171 | ? LLMProvider.OPENAI
172 | : args[1]?.toUpperCase() === "GEMINI"
173 | ? LLMProvider.GOOGLE_GEMINI
174 | : "all"; // 'OPENAI', 'GEMINI', or 'all'
175 |
176 | console.log("API Keys available:");
177 | console.log(`- GOOGLE_API_KEY: ${process.env.GOOGLE_API_KEY ? "Yes" : "No"}`);
178 | console.log(`- OPENAI_API_KEY: ${process.env.OPENAI_API_KEY ? "Yes" : "No"}`);
179 | console.log("");
180 |
181 | // Run blog tests
182 | if (contentType === "blog" || contentType === "all") {
183 | if (provider === LLMProvider.GOOGLE_GEMINI || provider === "all") {
184 | await testBlogExtraction(LLMProvider.GOOGLE_GEMINI);
185 | }
186 | if (provider === LLMProvider.OPENAI || provider === "all") {
187 | await testBlogExtraction(LLMProvider.OPENAI);
188 | }
189 | }
190 |
191 | // Run product tests
192 | if (contentType === "product" || contentType === "all") {
193 | if (provider === LLMProvider.GOOGLE_GEMINI || provider === "all") {
194 | await testProductExtraction(LLMProvider.GOOGLE_GEMINI);
195 | }
196 | if (provider === LLMProvider.OPENAI || provider === "all") {
197 | await testProductExtraction(LLMProvider.OPENAI);
198 | }
199 | }
200 | }
201 |
202 | // Only run if directly executed
203 | if (require.main === module) {
204 | console.log("Starting local extraction test...");
205 | console.log("Make sure you have set up your .env file with API keys.");
206 | console.log("Usage: npm run test:local -- [contentType] [provider]");
207 | console.log(" contentType: 'blog', 'product', or 'all' (default)");
208 | console.log(" provider: 'openai', 'gemini', or 'all' (default)");
209 |
210 | main()
211 | .then(() => {
212 | console.log("All tests completed successfully.");
213 | })
214 | .catch((error) => {
215 | console.error("Test failed:", error);
216 | process.exit(1);
217 | });
218 | }
219 |
--------------------------------------------------------------------------------
/src/dev/testHtmlToMarkdown.ts:
--------------------------------------------------------------------------------
1 | import * as fs from "fs";
2 | import * as path from "path";
3 | import { htmlToMarkdown } from "../converters";
4 | import { HTMLExtractionOptions } from "../types";
5 |
6 | // Function to convert HTML to Markdown and save the result
7 | async function testConvertHtmlToMarkdown(
8 | htmlFilePath: string,
9 | outputDir: string,
10 | options?: HTMLExtractionOptions
11 | ) {
12 | try {
13 | // Read the HTML file
14 | const html = fs.readFileSync(htmlFilePath, "utf8");
15 |
16 | // Convert to Markdown
17 | const markdown = htmlToMarkdown(html, options);
18 |
19 | // Create output directory if it doesn't exist
20 | if (!fs.existsSync(outputDir)) {
21 | fs.mkdirSync(outputDir, { recursive: true });
22 | }
23 |
24 | // Generate output filename
25 | const baseName = path.basename(htmlFilePath, ".html");
26 | const optionsSuffix = options?.includeImages
27 | ? ".with-images"
28 | : options?.extractMainHtml
29 | ? ".main-content"
30 | : "";
31 | const outputPath = path.join(outputDir, `${baseName}${optionsSuffix}.md`);
32 |
33 | // Save the markdown
34 | fs.writeFileSync(outputPath, markdown);
35 | console.log(`✅ Converted ${htmlFilePath} to ${outputPath}`);
36 |
37 | return outputPath;
38 | } catch (error) {
39 | console.error("❌ Error converting HTML to Markdown:", error);
40 | throw error;
41 | }
42 | }
43 |
44 | // Main function to run the test
45 | async function main() {
46 | // Get the HTML file path from command line arguments
47 | const htmlFilePath = process.argv[2];
48 | if (!htmlFilePath) {
49 | console.error("❌ Please provide an HTML file path as an argument");
50 | console.log("Usage: npm run dev:html2md ");
51 | process.exit(1);
52 | }
53 |
54 | // Create output directory
55 | const outputDir = path.join(process.cwd(), "dev-output", "markdown");
56 |
57 | // Test different conversion options
58 | console.log(
59 | "\n🔍 Testing HTML to Markdown conversion with different options...\n"
60 | );
61 |
62 | // 1. Basic conversion
63 | await testConvertHtmlToMarkdown(htmlFilePath, outputDir);
64 |
65 | // 2. Conversion with images
66 | await testConvertHtmlToMarkdown(htmlFilePath, outputDir, {
67 | includeImages: true,
68 | });
69 |
70 | // 3. Main content extraction
71 | await testConvertHtmlToMarkdown(htmlFilePath, outputDir, {
72 | extractMainHtml: true,
73 | });
74 |
75 | // 4. Both images and main content
76 | await testConvertHtmlToMarkdown(htmlFilePath, outputDir, {
77 | includeImages: true,
78 | extractMainHtml: true,
79 | });
80 |
81 | console.log(
82 | "\n✨ All conversions completed! Check the output in:",
83 | outputDir
84 | );
85 | }
86 |
87 | // Run the main function
88 | main().catch(console.error);
89 |
--------------------------------------------------------------------------------
/src/dev/testUsage.ts:
--------------------------------------------------------------------------------
1 | import { config } from "dotenv";
2 | import * as path from "path";
3 | import { z } from "zod";
4 | import { extract, ContentFormat, LLMProvider } from "../index";
5 |
6 | // Load environment variables from .env file
7 | config({ path: path.resolve(process.cwd(), ".env") });
8 |
9 | // A simple test script to verify usage tracking works
10 | async function testUsageTracking() {
11 | console.log("Testing usage tracking with OpenAI...");
12 |
13 | // Check if API keys are available
14 | if (!process.env.OPENAI_API_KEY) {
15 | console.error("Error: OPENAI_API_KEY environment variable is required");
16 | process.exit(1);
17 | }
18 |
19 | // Simple schema to test extraction
20 | const schema = z.object({
21 | title: z.string(),
22 | description: z.string(),
23 | });
24 |
25 | // Simple markdown content
26 | const markdown = `
27 | # Hello World
28 |
29 | This is a test of the usage tracking system.
30 | `;
31 |
32 | try {
33 | // Run extraction
34 | const result = await extract({
35 | content: markdown,
36 | format: ContentFormat.MARKDOWN,
37 | schema,
38 | provider: LLMProvider.OPENAI,
39 | openaiApiKey: process.env.OPENAI_API_KEY,
40 | });
41 |
42 | // Log the results
43 | console.log("\nExtracted data:");
44 | console.log(JSON.stringify(result.data, null, 2));
45 |
46 | console.log("\nToken usage:");
47 | console.log(result.usage);
48 |
49 | // Check if usage was captured
50 | if (result.usage.inputTokens && result.usage.outputTokens) {
51 | console.log("\n✅ Usage tracking is working correctly!");
52 | } else {
53 | console.log("\n❌ Usage tracking failed!");
54 | }
55 | } catch (error) {
56 | console.error("Error testing usage tracking:", error);
57 | }
58 | }
59 |
60 | // Run the test if executed directly
61 | if (require.main === module) {
62 | testUsageTracking()
63 | .then(() => console.log("Test completed"))
64 | .catch(console.error);
65 | }
66 |
--------------------------------------------------------------------------------
/src/example.ts:
--------------------------------------------------------------------------------
1 | import { extract, ContentFormat, LLMProvider } from "./index";
2 | import { z } from "zod";
3 | import { config } from "dotenv";
4 | import * as path from "path";
5 | import * as fs from "fs";
6 | import { htmlToMarkdown } from "./converters";
7 |
8 | // Load environment variables from .env file
9 | config({ path: path.resolve(process.cwd(), ".env") });
10 |
11 | async function example() {
12 | try {
13 | // Check if API key is available
14 | if (!process.env.GOOGLE_API_KEY) {
15 | console.error("Error: GOOGLE_API_KEY environment variable is required");
16 | return;
17 | }
18 |
19 | // Define a schema for blog post extraction
20 | const schema = z.object({
21 | title: z.string(),
22 | author: z.string().optional(),
23 | date: z.string().optional(),
24 | summary: z.string(),
25 | categories: z.array(z.string()).optional(),
26 | });
27 |
28 | const htmlContent = fs.readFileSync(
29 | path.resolve(__dirname, "../tests/fixtures", "blog-post.html"),
30 | "utf8"
31 | );
32 | const sourceUrl = "https://www.example.com/blog/async-await";
33 |
34 | const markdown = htmlToMarkdown(
35 | htmlContent,
36 | {
37 | extractMainHtml: true,
38 | includeImages: true,
39 | },
40 | sourceUrl
41 | );
42 |
43 | // fs.writeFileSync("test.md", markdown);
44 |
45 | console.log("Running extraction example...");
46 |
47 | // Extract data from HTML
48 | const result = await extract({
49 | content: htmlContent,
50 | format: ContentFormat.HTML,
51 | schema,
52 | // Using Google Gemini by default
53 | openaiApiKey: process.env.OPENAI_API_KEY,
54 | provider: LLMProvider.OPENAI,
55 | sourceUrl,
56 | });
57 |
58 | console.log("Extracted Data:");
59 | console.log(JSON.stringify(result.data, null, 2));
60 |
61 | console.log("\nMarkdown Content:");
62 | console.log(result.processedContent.slice(0, 1000) + "\n...");
63 |
64 | console.log("\nToken Usage:");
65 | console.log(result.usage);
66 | } catch (error) {
67 | console.error("Error in example:", error);
68 | }
69 | }
70 |
71 | // Only run if directly executed
72 | if (require.main === module) {
73 | example();
74 | }
75 |
--------------------------------------------------------------------------------
/src/extractors.ts:
--------------------------------------------------------------------------------
1 | import { ChatOpenAI } from "@langchain/openai";
2 | import { ChatGoogleGenerativeAI } from "@langchain/google-genai";
3 | import { z } from "zod";
4 | import { LLMProvider, Usage, ContentFormat } from "./types";
5 | import { AIMessage } from "@langchain/core/messages";
6 | import {
7 | safeSanitizedParser,
8 | transformSchemaForLLM,
9 | fixUrlEscapeSequences,
10 | } from "./utils/schemaUtils";
11 | import { jsonrepair } from "jsonrepair";
12 |
13 | // Define LLMResult type here since direct import is problematic
14 | interface TokenUsage {
15 | promptTokens?: number;
16 | completionTokens?: number;
17 | totalTokens?: number;
18 | }
19 |
20 | interface LLMOutput {
21 | tokenUsage?: TokenUsage;
22 | }
23 |
24 | interface LLMResult {
25 | llmOutput?: LLMOutput;
26 | }
27 |
28 | /**
29 | * Get usage statistics from LLM output
30 | */
31 | export function getUsage(output: LLMResult): Usage {
32 | const usage: Usage = {};
33 |
34 | if (output.llmOutput && output.llmOutput.tokenUsage) {
35 | usage.inputTokens = output.llmOutput.tokenUsage.promptTokens;
36 | usage.outputTokens = output.llmOutput.tokenUsage.completionTokens;
37 | }
38 |
39 | return usage;
40 | }
41 |
42 | /**
43 | * Create LLM instance based on provider and configuration
44 | */
45 | export function createLLM(
46 | provider: LLMProvider,
47 | modelName: string,
48 | apiKey: string,
49 | temperature: number = 0
50 | ) {
51 | switch (provider) {
52 | case LLMProvider.OPENAI:
53 | return new ChatOpenAI({
54 | apiKey,
55 | modelName,
56 | temperature,
57 | });
58 |
59 | case LLMProvider.GOOGLE_GEMINI:
60 | return new ChatGoogleGenerativeAI({
61 | apiKey,
62 | model: modelName,
63 | temperature,
64 | });
65 |
66 | default:
67 | throw new Error(`Unsupported LLM provider: ${provider}`);
68 | }
69 | }
70 |
71 | interface ExtractionPromptOptions {
72 | format: string;
73 | content: string;
74 | customPrompt?: string;
75 | dataToEnrich?: Record;
76 | }
77 |
78 | interface TruncateContentOptions extends ExtractionPromptOptions {
79 | maxTokens: number;
80 | }
81 |
82 | /**
83 | * Generate the extraction prompt with or without a custom query
84 | */
85 | export function generateExtractionPrompt({
86 | format,
87 | content,
88 | customPrompt,
89 | dataToEnrich,
90 | }: ExtractionPromptOptions): string {
91 | // Base prompt structure that's shared between default and custom prompts
92 | const extractionTask = customPrompt
93 | ? `${customPrompt}`
94 | : "Please extract structured information from the provided context.";
95 |
96 | // If dataToEnrich is provided, include it in the prompt for enrichment
97 | let promptTemplate = `Context information is below:
98 | ------
99 | Format: ${format}
100 | ---
101 | ${content}
102 | ------
103 |
104 | `;
105 |
106 | if (dataToEnrich) {
107 | promptTemplate += `Original JSON object:
108 | ---
109 | ${JSON.stringify(dataToEnrich, null, 2)}
110 | ------
111 |
112 | You are a data extraction assistant that extracts structured information from the above context in ${format} and JSON.
113 |
114 | Your task is: ${extractionTask}
115 |
116 | ## Guidelines:
117 | 1. Extract ONLY information explicitly stated in the context
118 | 2. Enrich the original JSON object with information from the context
119 | 3. Do not remove any fields from the original JSON object
120 | 4. Only update existing fields and fill in additional fields if new and relevant information is available in the context
121 | 5. Do not make assumptions or infer missing data
122 | 6. Do not include information that appears incomplete or truncated
123 |
124 | `;
125 | } else {
126 | promptTemplate += `You are a data extraction assistant that extracts structured information from the above context.
127 |
128 | Your task is: ${extractionTask}
129 |
130 | ## Guidelines:
131 | 1. Extract ONLY information explicitly stated in the context
132 | 2. Do not make assumptions or infer missing data
133 | 3. Leave fields empty when information is not present or you are uncertain
134 | 4. Do not include information that appears incomplete or truncated
135 | 5. Follow the required schema exactly
136 |
137 | `;
138 | }
139 |
140 | promptTemplate += `Return only the structured data in valid JSON format and nothing else.`;
141 |
142 | return promptTemplate;
143 | }
144 |
145 | /**
146 | * Truncate content to fit within token limit
147 | * Uses a rough conversion of 4 characters per token
148 | */
149 | export function truncateContent({
150 | format,
151 | content,
152 | customPrompt,
153 | dataToEnrich,
154 | maxTokens,
155 | }: TruncateContentOptions): string {
156 | const maxChars = maxTokens * 4;
157 |
158 | // First generate the full prompt
159 | const fullPrompt = generateExtractionPrompt({
160 | format,
161 | content,
162 | customPrompt,
163 | dataToEnrich,
164 | });
165 |
166 | // If the full prompt is within limits, return original content
167 | if (fullPrompt.length <= maxChars) {
168 | return content;
169 | }
170 |
171 | // Calculate how much we need to reduce the content
172 | const excessChars = fullPrompt.length - maxChars;
173 |
174 | // Truncate content by the excess amount
175 | return content.slice(0, content.length - excessChars);
176 | }
177 |
178 | /**
179 | * Extract structured data from markdown using an LLM
180 | */
181 | export async function extractWithLLM(
182 | content: string,
183 | schema: T,
184 | provider: LLMProvider,
185 | modelName: string,
186 | apiKey: string,
187 | temperature: number = 0,
188 | customPrompt?: string,
189 | format: string = ContentFormat.MARKDOWN,
190 | maxInputTokens?: number,
191 | dataToEnrich?: Record
192 | ): Promise<{ data: z.infer; usage: Usage }> {
193 | const llm = createLLM(provider, modelName, apiKey, temperature);
194 | let usage: Usage = {};
195 |
196 | // Truncate content if maxInputTokens is specified
197 | const truncatedContent = maxInputTokens
198 | ? truncateContent({
199 | format,
200 | content,
201 | customPrompt,
202 | dataToEnrich,
203 | maxTokens: maxInputTokens,
204 | })
205 | : content;
206 |
207 | // Generate the prompt using the unified template function
208 | const prompt = generateExtractionPrompt({
209 | format,
210 | content: truncatedContent,
211 | customPrompt,
212 | dataToEnrich,
213 | });
214 |
215 | try {
216 | // Transform schema to be compatible with LLM output (converting url() to string())
217 | const llmSchema = transformSchemaForLLM(schema);
218 |
219 | // Extract structured data with a withStructuredOutput chain
220 | const structuredOutputLLM = llm.withStructuredOutput(llmSchema, {
221 | includeRaw: true,
222 | });
223 |
224 | // Create a callback handler for usage tracking
225 | const callbacks = [
226 | {
227 | handleLLMEnd: (output: any) => {
228 | usage = getUsage(output);
229 | },
230 | },
231 | ];
232 |
233 | // Invoke the LLM with callbacks to track usage
234 | const response = await structuredOutputLLM.invoke(prompt, { callbacks });
235 | const raw = response.raw as AIMessage;
236 |
237 | let data = response.parsed;
238 |
239 | // If structured output is not successful, try to parse the raw object.
240 | if (data == null) {
241 | // Note: this only works for OpenAI models.
242 | if (raw.tool_calls && raw.tool_calls.length > 0) {
243 | // This is the raw object in JSON mode before structured output tool call.
244 | const rawObject = raw.tool_calls[0].args;
245 | // Manually sanitize the object and remove any unsafe but optional fields or unsafe items in arrays.
246 | data = safeSanitizedParser(llmSchema, rawObject);
247 | }
248 |
249 | // Note: this only works for Google Gemini models.
250 | if (raw.lc_kwargs && raw.lc_kwargs.content) {
251 | // Gemini does not return a JSON object, it returns a string that is a JSON object.
252 | // We use jsonrepair to fix the JSON string and then parse it.
253 | const rawJson = raw.lc_kwargs.content;
254 | const rawObject = JSON.parse(jsonrepair(rawJson));
255 | data = safeSanitizedParser(llmSchema, rawObject);
256 | }
257 | if (data == null) {
258 | throw new Error("No valid data was extracted");
259 | }
260 | }
261 |
262 | // If structured output worked, we still need to fix URL escape sequences
263 | // and validate against the original schema
264 | const fixedData = fixUrlEscapeSequences(data, schema);
265 | const validatedData = safeSanitizedParser(schema, fixedData);
266 | // If validation fails, something went wrong with the URL validation
267 | if (validatedData === null) {
268 | throw new Error(
269 | "Extracted data failed validation against original schema"
270 | );
271 | }
272 |
273 | data = validatedData;
274 |
275 | // Return the parsed data and usage statistics
276 | return {
277 | data,
278 | usage,
279 | };
280 | } catch (error) {
281 | console.error("Error during LLM extraction:", error);
282 | throw error;
283 | }
284 | }
285 |
--------------------------------------------------------------------------------
/src/index.ts:
--------------------------------------------------------------------------------
1 | import { z } from "zod";
2 | import { htmlToMarkdown } from "./converters";
3 | import { extractWithLLM } from "./extractors";
4 | import {
5 | ContentFormat,
6 | LLMProvider,
7 | ExtractorOptions,
8 | ExtractorResult,
9 | HTMLExtractionOptions,
10 | } from "./types";
11 |
12 | // Default model names
13 | const DEFAULT_MODELS = {
14 | [LLMProvider.GOOGLE_GEMINI]: "gemini-2.5-flash-preview-04-17",
15 | [LLMProvider.OPENAI]: "gpt-4o-mini",
16 | };
17 |
18 | /**
19 | * Extract structured data from HTML, markdown, or plain text content using an LLM
20 | *
21 | * @param options Configuration options for extraction
22 | * @param options.content HTML, markdown, or plain text content to extract from
23 | * @param options.format Content format (HTML, MARKDOWN, or TXT)
24 | * @param options.schema Zod schema defining the structure to extract
25 | * @param options.provider LLM provider (GOOGLE_GEMINI or OPENAI)
26 | * @param options.modelName Model name to use (provider-specific)
27 | * @param options.googleApiKey Google API key (if using Google Gemini provider)
28 | * @param options.openaiApiKey OpenAI API key (if using OpenAI provider)
29 | * @param options.temperature Temperature for the LLM (0-1)
30 | * @param options.prompt Custom prompt to guide the extraction process
31 | * @param options.sourceUrl URL of the HTML content (required for HTML format)
32 | * @param options.htmlExtractionOptions HTML-specific options for content extraction
33 | * @param options.maxInputTokens Maximum number of input tokens to send to the LLM
34 | * @param options.dataToEnrich Original data object to enrich with information from the content
35 | * @returns The extracted data, original content, and usage statistics
36 | */
37 | export async function extract(
38 | options: ExtractorOptions
39 | ): Promise>> {
40 | // Validate required parameters
41 | const provider = options.provider ?? LLMProvider.GOOGLE_GEMINI;
42 | let apiKey: string;
43 |
44 | if (provider === LLMProvider.GOOGLE_GEMINI) {
45 | apiKey = options.googleApiKey ?? process.env.GOOGLE_API_KEY ?? "";
46 | if (!apiKey) {
47 | throw new Error(
48 | "Google API key is required. Provide googleApiKey option or set GOOGLE_API_KEY environment variable."
49 | );
50 | }
51 | } else if (provider === LLMProvider.OPENAI) {
52 | apiKey = options.openaiApiKey ?? process.env.OPENAI_API_KEY ?? "";
53 | if (!apiKey) {
54 | throw new Error(
55 | "OpenAI API key is required. Provide openaiApiKey option or set OPENAI_API_KEY environment variable."
56 | );
57 | }
58 | } else {
59 | throw new Error(`Unsupported LLM provider: ${provider}`);
60 | }
61 |
62 | // Validate sourceUrl for HTML format
63 | if (options.format === ContentFormat.HTML && !options.sourceUrl) {
64 | throw new Error(
65 | "sourceUrl is required when format is HTML to properly handle relative URLs"
66 | );
67 | }
68 |
69 | // Get model name (use defaults if not provided)
70 | const modelName = options.modelName ?? DEFAULT_MODELS[provider];
71 |
72 | // Convert HTML to markdown if needed
73 | let content = options.content;
74 | let formatToUse = options.format;
75 |
76 | if (options.format === ContentFormat.HTML) {
77 | content = htmlToMarkdown(
78 | options.content,
79 | options.htmlExtractionOptions,
80 | options.sourceUrl
81 | );
82 | // For the LLM, the content is now markdown
83 | formatToUse = ContentFormat.MARKDOWN;
84 | }
85 |
86 | // Extract structured data using LLM
87 | const { data, usage } = await extractWithLLM(
88 | content,
89 | options.schema,
90 | provider,
91 | modelName,
92 | apiKey,
93 | options.temperature ?? 0,
94 | options.prompt,
95 | formatToUse.toString(), // Pass the correct format based on actual content
96 | options.maxInputTokens,
97 | options.dataToEnrich
98 | );
99 |
100 | // Return the full result
101 | return {
102 | data,
103 | processedContent: content,
104 | usage,
105 | };
106 | }
107 |
108 | /**
109 | * Convert HTML to markdown
110 | *
111 | * @param html HTML content to convert
112 | * @param options HTML extraction options
113 | * @param sourceUrl Source URL for resolving relative links
114 | * @returns Markdown content
115 | */
116 | export function convertHtmlToMarkdown(
117 | html: string,
118 | options?: HTMLExtractionOptions,
119 | sourceUrl?: string
120 | ): string {
121 | return htmlToMarkdown(html, options, sourceUrl);
122 | }
123 |
124 | // Re-export types and enums
125 | export * from "./types";
126 |
127 | // Utils
128 | export { safeSanitizedParser } from "./utils/schemaUtils";
129 |
--------------------------------------------------------------------------------
/src/types.ts:
--------------------------------------------------------------------------------
1 | import { z } from "zod";
2 |
3 | /**
4 | * Represents the format of the input content
5 | */
6 | export enum ContentFormat {
7 | HTML = "html",
8 | MARKDOWN = "markdown",
9 | TXT = "txt",
10 | }
11 |
12 | /**
13 | * Supported LLM providers
14 | */
15 | export enum LLMProvider {
16 | OPENAI = "openai",
17 | GOOGLE_GEMINI = "google_gemini",
18 | }
19 |
20 | /**
21 | * Options for HTML content processing
22 | */
23 | export interface HTMLExtractionOptions {
24 | /**
25 | * When enabled, attempts to extract the main content from HTML, removing navigation bars, headers, footers, etc.
26 | * This uses heuristics to identify the main content area.
27 | *
28 | * Should be kept off (false) when extracting specific details about a single item,
29 | * as it might remove important contextual elements.
30 | *
31 | * Only applies to HTML format, not markdown.
32 | */
33 | extractMainHtml?: boolean;
34 |
35 | /**
36 | * When enabled, images in the HTML will be included in the markdown output.
37 | * By default, images are excluded to simplify the extraction process.
38 | *
39 | * Enable this option when you need to extract image information or URLs.
40 | */
41 | includeImages?: boolean;
42 | }
43 |
44 | /**
45 | * Options for the extractor
46 | */
47 | export interface ExtractorOptions {
48 | /** Content to extract from (HTML, Markdown, or plain text) */
49 | content: string;
50 |
51 | /** Format of the content */
52 | format: ContentFormat;
53 |
54 | /** Schema for structured extraction */
55 | schema: T;
56 |
57 | /** LLM Provider (OpenAI or Google Gemini) */
58 | provider?: LLMProvider;
59 |
60 | /** Model name to use */
61 | modelName?: string;
62 |
63 | /** OpenAI API key */
64 | openaiApiKey?: string;
65 |
66 | /** Google API key */
67 | googleApiKey?: string;
68 |
69 | /** Temperature for the LLM (0-1), defaults to 0 */
70 | temperature?: number;
71 |
72 | /** HTML-specific extraction options (only applies when format is HTML) */
73 | htmlExtractionOptions?: HTMLExtractionOptions;
74 |
75 | /** Custom prompt for extraction (if not provided, a default prompt will be used) */
76 | prompt?: string;
77 |
78 | /** URL of the HTML content, required only for HTML format */
79 | sourceUrl?: string;
80 |
81 | /** Maximum number of input tokens to send to the LLM. Uses a rough conversion of 4 characters per token. */
82 | maxInputTokens?: number;
83 |
84 | /** Original data object to enrich with extracted information. When provided, the LLM will be instructed to enrich this object with additional information from the content. */
85 | dataToEnrich?: Record;
86 | }
87 |
88 | /**
89 | * Usage statistics for LLM calls
90 | */
91 | export interface Usage {
92 | inputTokens?: number;
93 | outputTokens?: number;
94 | }
95 |
96 | /**
97 | * Result of the extraction process
98 | */
99 | export interface ExtractorResult {
100 | /** Extracted data according to the schema */
101 | data: T;
102 |
103 | /**
104 | * Processed content that was sent to the LLM.
105 | * This will be markdown if the input was HTML (after conversion),
106 | * or the original content if the input was already markdown or plain text.
107 | */
108 | processedContent: string;
109 |
110 | /** Usage statistics */
111 | usage: Usage;
112 | }
113 |
--------------------------------------------------------------------------------
/src/utils/schemaUtils.ts:
--------------------------------------------------------------------------------
1 | import {
2 | z,
3 | ZodArray,
4 | ZodObject,
5 | ZodOptional,
6 | ZodTypeAny,
7 | ZodNullable,
8 | ZodFirstPartyTypeKind,
9 | } from "zod";
10 |
11 | /**
12 | * Checks if a schema is a ZodString with URL validation
13 | */
14 | export function isUrlSchema(schema: ZodTypeAny): boolean {
15 | if (!isZodType(schema, ZodFirstPartyTypeKind.ZodString)) return false;
16 |
17 | // Check if schema has URL validation by checking for internal checks property
18 | // This is a bit of a hack but necessary since Zod doesn't expose validation info
19 | const checks = (schema as any)._def.checks;
20 | if (!checks || !Array.isArray(checks)) return false;
21 |
22 | return checks.some((check) => check.kind === "url");
23 | }
24 |
25 | /**
26 | * Helper function to check schema type without using instanceof (can fail due to zod version differences)
27 | */
28 | function isZodType(schema: ZodTypeAny, type: ZodFirstPartyTypeKind): boolean {
29 | return (schema as any)._def.typeName === type;
30 | }
31 |
32 | /**
33 | * Transforms a schema, replacing any URL validations with string validations
34 | * for compatibility with LLM output
35 | */
36 | export function transformSchemaForLLM(
37 | schema: T
38 | ): ZodTypeAny {
39 | // For URL string schemas, remove the URL check but preserve everything else
40 | if (isUrlSchema(schema)) {
41 | const originalDef = { ...(schema as any)._def };
42 |
43 | // Filter out only URL checks, keep all other checks
44 | if (originalDef.checks && Array.isArray(originalDef.checks)) {
45 | originalDef.checks = originalDef.checks.filter(
46 | (check: any) => check.kind !== "url"
47 | );
48 | }
49 |
50 | // Create a new string schema with the modified definition
51 | return new z.ZodString({
52 | ...originalDef,
53 | typeName: z.ZodFirstPartyTypeKind.ZodString,
54 | });
55 | }
56 |
57 | // For object schemas, transform each property
58 | if (isZodType(schema, ZodFirstPartyTypeKind.ZodObject)) {
59 | const originalDef = { ...(schema as any)._def };
60 | const newShape: Record = {};
61 |
62 | // Transform each property in the shape
63 | for (const [key, propertySchema] of Object.entries((schema as any).shape)) {
64 | newShape[key] = transformSchemaForLLM(propertySchema as ZodTypeAny);
65 | }
66 |
67 | // Create a new object with the same definition but transformed shape
68 | return new z.ZodObject({
69 | ...originalDef,
70 | shape: () => newShape,
71 | typeName: z.ZodFirstPartyTypeKind.ZodObject,
72 | });
73 | }
74 |
75 | // For array schemas, transform the element schema
76 | if (isZodType(schema, ZodFirstPartyTypeKind.ZodArray)) {
77 | const originalDef = { ...(schema as any)._def };
78 | const transformedElement = transformSchemaForLLM(
79 | (schema as any).element as ZodTypeAny
80 | );
81 |
82 | // Create a new array with the same definition but transformed element
83 | return new z.ZodArray({
84 | ...originalDef,
85 | type: transformedElement,
86 | typeName: z.ZodFirstPartyTypeKind.ZodArray,
87 | });
88 | }
89 |
90 | // For optional schemas, transform the inner schema
91 | if (isZodType(schema, ZodFirstPartyTypeKind.ZodOptional)) {
92 | const originalDef = { ...(schema as any)._def };
93 | const transformedInner = transformSchemaForLLM(
94 | (schema as any).unwrap() as ZodTypeAny
95 | );
96 |
97 | // Create a new optional with the same definition but transformed inner type
98 | return new z.ZodOptional({
99 | ...originalDef,
100 | innerType: transformedInner,
101 | typeName: z.ZodFirstPartyTypeKind.ZodOptional,
102 | });
103 | }
104 |
105 | // For nullable schemas, transform the inner schema
106 | if (isZodType(schema, ZodFirstPartyTypeKind.ZodNullable)) {
107 | const originalDef = { ...(schema as any)._def };
108 | const transformedInner = transformSchemaForLLM(
109 | (schema as any).unwrap() as ZodTypeAny
110 | );
111 |
112 | // Create a new nullable with the same definition but transformed inner type
113 | return new z.ZodNullable({
114 | ...originalDef,
115 | innerType: transformedInner,
116 | typeName: z.ZodFirstPartyTypeKind.ZodNullable,
117 | });
118 | }
119 |
120 | // Return the original schema for all other types
121 | return schema;
122 | }
123 |
124 | /**
125 | * Fix URL escape sequences in the object based on the original schema
126 | */
127 | export function fixUrlEscapeSequences(data: any, schema: ZodTypeAny): any {
128 | if (data === null || data === undefined) return data;
129 |
130 | if (isUrlSchema(schema)) {
131 | if (typeof data === "string") {
132 | // Replace escaped parentheses with unescaped versions
133 | return data.replace(/\\\(/g, "(").replace(/\\\)/g, ")");
134 | }
135 | return data;
136 | }
137 |
138 | if (
139 | isZodType(schema, ZodFirstPartyTypeKind.ZodObject) &&
140 | typeof data === "object" &&
141 | !Array.isArray(data)
142 | ) {
143 | const shape = (schema as any).shape;
144 | const result: Record = {};
145 |
146 | for (const [key, propertySchema] of Object.entries(shape)) {
147 | if (key in data) {
148 | result[key] = fixUrlEscapeSequences(
149 | data[key],
150 | propertySchema as ZodTypeAny
151 | );
152 | } else {
153 | result[key] = data[key];
154 | }
155 | }
156 |
157 | return result;
158 | }
159 |
160 | if (
161 | isZodType(schema, ZodFirstPartyTypeKind.ZodArray) &&
162 | Array.isArray(data)
163 | ) {
164 | const elementSchema = (schema as any).element as ZodTypeAny;
165 | return data.map((item) => fixUrlEscapeSequences(item, elementSchema));
166 | }
167 |
168 | if (isZodType(schema, ZodFirstPartyTypeKind.ZodOptional)) {
169 | const innerSchema = (schema as any).unwrap() as ZodTypeAny;
170 | return fixUrlEscapeSequences(data, innerSchema);
171 | }
172 |
173 | if (isZodType(schema, ZodFirstPartyTypeKind.ZodNullable)) {
174 | const innerSchema = (schema as any).unwrap() as ZodTypeAny;
175 | return fixUrlEscapeSequences(data, innerSchema);
176 | }
177 |
178 | return data;
179 | }
180 |
181 | /**
182 | * Sanitizes an object to conform to a Zod schema by removing invalid optional fields or array items.
183 | * If the object can't be sanitized to match the schema, returns null.
184 | *
185 | * @param schema The Zod schema to validate against
186 | * @param rawObject The raw object to sanitize
187 | * @returns The sanitized object or null if it can't be sanitized
188 | */
189 | export function safeSanitizedParser(
190 | schema: T,
191 | rawObject: unknown
192 | ): z.infer | null {
193 | try {
194 | // If the raw object is null or undefined, just validate it directly
195 | if (rawObject === null || rawObject === undefined) {
196 | return schema.parse(rawObject);
197 | }
198 |
199 | // Handle different schema types
200 | if (isZodType(schema, ZodFirstPartyTypeKind.ZodObject)) {
201 | return sanitizeObject(schema as any, rawObject);
202 | } else if (isZodType(schema, ZodFirstPartyTypeKind.ZodArray)) {
203 | return sanitizeArray(schema as any, rawObject);
204 | } else if (isZodType(schema, ZodFirstPartyTypeKind.ZodOptional)) {
205 | return sanitizeOptional(schema as any, rawObject);
206 | } else if (isZodType(schema, ZodFirstPartyTypeKind.ZodNullable)) {
207 | return sanitizeNullable(schema as any, rawObject);
208 | } else {
209 | // For primitive values, try to parse directly
210 | return schema.parse(rawObject);
211 | }
212 | } catch (error) {
213 | // If any error occurs during sanitization, return null
214 | return null;
215 | }
216 | }
217 |
218 | /**
219 | * Sanitizes an object against a Zod object schema
220 | */
221 | function sanitizeObject(schema: ZodObject, rawObject: unknown): any {
222 | if (
223 | typeof rawObject !== "object" ||
224 | rawObject === null ||
225 | Array.isArray(rawObject)
226 | ) {
227 | throw new Error("Expected an object");
228 | }
229 |
230 | const shape = schema.shape;
231 | const result: Record = {};
232 | const rawObjectRecord = rawObject as Record;
233 |
234 | // Process each property in the schema
235 | for (const [key, propertySchema] of Object.entries(shape)) {
236 | // Skip if the property doesn't exist in the raw object
237 | if (!(key in rawObjectRecord)) {
238 | continue;
239 | }
240 |
241 | // If property is optional, try to sanitize it
242 | if (
243 | isZodType(propertySchema as ZodTypeAny, ZodFirstPartyTypeKind.ZodOptional)
244 | ) {
245 | const sanitized = safeSanitizedParser(
246 | propertySchema as ZodTypeAny,
247 | rawObjectRecord[key]
248 | );
249 | if (sanitized !== null) {
250 | result[key] = sanitized;
251 | }
252 | // If sanitization fails, just skip the optional property
253 | } else if (
254 | isZodType(propertySchema as ZodTypeAny, ZodFirstPartyTypeKind.ZodNullable)
255 | ) {
256 | // For nullable properties, try to sanitize or set to null
257 | try {
258 | const sanitized = safeSanitizedParser(
259 | propertySchema as ZodTypeAny,
260 | rawObjectRecord[key]
261 | );
262 | result[key] = sanitized;
263 | } catch {
264 | // If sanitization fails, set to null for nullable properties
265 | result[key] = null;
266 | }
267 | } else {
268 | // For required properties, try to sanitize and throw if it fails
269 | const sanitized = safeSanitizedParser(
270 | propertySchema as ZodTypeAny,
271 | rawObjectRecord[key]
272 | );
273 | if (sanitized === null) {
274 | throw new Error(`Required property ${key} could not be sanitized`);
275 | }
276 | result[key] = sanitized;
277 | }
278 | }
279 |
280 | // Validate the final object to ensure it matches the schema
281 | return schema.parse(result);
282 | }
283 |
284 | /**
285 | * Sanitizes an array against a Zod array schema
286 | */
287 | function sanitizeArray(schema: ZodArray, rawValue: unknown): any {
288 | if (!Array.isArray(rawValue)) {
289 | throw new Error("Expected an array");
290 | }
291 |
292 | const elementSchema = schema.element as ZodTypeAny;
293 | const sanitizedArray = [];
294 |
295 | // Process each item in the array
296 | for (const item of rawValue) {
297 | try {
298 | const sanitizedItem = safeSanitizedParser(elementSchema, item);
299 | if (sanitizedItem !== null) {
300 | sanitizedArray.push(sanitizedItem);
301 | }
302 | // If an item can't be sanitized, just skip it
303 | } catch {
304 | // Skip invalid array items
305 | }
306 | }
307 |
308 | // Validate the final array to ensure it matches the schema
309 | return schema.parse(sanitizedArray);
310 | }
311 |
312 | /**
313 | * Sanitizes a value against an optional Zod schema
314 | */
315 | function sanitizeOptional(schema: ZodOptional, rawValue: unknown): any {
316 | try {
317 | // Try to sanitize using the inner schema
318 | const innerSchema = schema.unwrap();
319 | const parsed = safeSanitizedParser(innerSchema, rawValue);
320 | // If the parsed value is not valid, return undefined for optional values
321 | if (parsed === null) {
322 | return undefined;
323 | }
324 | return parsed;
325 | } catch {
326 | // If sanitization fails, return undefined for optional values
327 | return undefined;
328 | }
329 | }
330 |
331 | /**
332 | * Sanitizes a value against a nullable Zod schema
333 | */
334 | function sanitizeNullable(schema: ZodNullable, rawValue: unknown): any {
335 | // If the value is null, return null directly
336 | if (rawValue === null) {
337 | return null;
338 | }
339 |
340 | try {
341 | // Try to sanitize using the inner schema
342 | const innerSchema = schema.unwrap();
343 | const sanitized = safeSanitizedParser(innerSchema, rawValue);
344 |
345 | // If sanitization of inner schema fails, return null
346 | if (sanitized === null) {
347 | return null;
348 | }
349 |
350 | return sanitized;
351 | } catch {
352 | // If sanitization fails, return null for nullable values
353 | return null;
354 | }
355 | }
356 |
--------------------------------------------------------------------------------
/tests/fixtures/article-with-images.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | Modern Web Development with React and Node.js
7 |
8 |
9 |
10 |
17 |
Modern Web Development with React and Node.js
18 |
19 | Jane Smith
20 | March 20, 2023
21 |
22 | #React
23 | #Node.js
24 | #JavaScript
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 | The Modern Web Stack: React for front-end and Node.js for back-end
33 |
34 |
35 |
Building modern web applications requires a solid understanding of both front-end and back-end technologies. React has become the industry standard for building interactive user interfaces, while Node.js powers the server-side of many applications.
36 |
37 |
React: Building User Interfaces
38 |
React is a JavaScript library for building user interfaces, particularly single-page applications. It allows developers to create reusable UI components and manage application state efficiently.
Node.js allows JavaScript to be used for server-side programming. It uses an event-driven, non-blocking I/O model that makes it lightweight and efficient for data-intensive real-time applications.
When combined, React and Node.js create a powerful full-stack JavaScript environment. The front-end is handled by React components, while the back-end API is managed by Node.js.
59 |
60 |
A typical architecture might look like this:
61 |
62 |
React components for the user interface
63 |
Redux or Context API for state management
64 |
Express.js (Node.js framework) for the API layer
65 |
MongoDB or another database for data persistence
66 |
67 |
68 |
69 |
Popular Tools in the React/Node.js Ecosystem
70 |
71 |
72 |
73 |
Webpack for module bundling
74 |
75 |
76 |
77 |
Express.js for API routing
78 |
79 |
80 |
81 |
MongoDB for database
82 |
83 |
84 |
85 |
86 |
Conclusion
87 |
The combination of React and Node.js provides a consistent development experience across the stack, as both use JavaScript. This allows for better code reuse and a more streamlined development process.
88 |
89 |
Whether you're building a simple web application or a complex enterprise system, the React and Node.js stack offers flexibility, performance, and scalability.
Async/await is a modern way to handle asynchronous operations in JavaScript. It was introduced in ES2017 and has since become the preferred method for handling promises.
23 |
24 |
What is Async/Await?
25 |
The async keyword is used to declare an asynchronous function. An async function automatically returns a promise, and the value returned by the function will be resolved with the returned promise.
26 |
27 |
The await keyword can only be used inside an async function. It pauses the execution of the function until the promise is resolved or rejected.
In this example, the function will wait for the fetch operation to complete before moving to the next line. This makes asynchronous code look and behave more like synchronous code, making it easier to understand and maintain.
39 |
40 |
Error Handling
41 |
With async/await, you can use try/catch blocks for error handling, which is more intuitive than promise chaining with .catch().
Async/await makes asynchronous code more readable and maintainable. It's built on promises, so you can still use all the promise methods when needed, but the syntax is cleaner and more intuitive.
",
332 | customPrompt,
333 | dataToEnrich,
334 | });
335 |
336 | expect(prompt).toContain(customPrompt);
337 | expect(prompt).toContain("Enrich the original JSON object");
338 | expect(prompt).toContain(JSON.stringify(dataToEnrich, null, 2));
339 | });
340 | });
341 | });
342 |
--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | /* Visit https://aka.ms/tsconfig to read more about this file */
4 |
5 | /* Projects */
6 | // "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
7 | // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */
8 | // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */
9 | // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */
10 | // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */
11 | // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
12 |
13 | /* Language and Environment */
14 | "target": "es2018", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
15 | // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
16 | // "jsx": "preserve", /* Specify what JSX code is generated. */
17 | // "libReplacement": true, /* Enable lib replacement. */
18 | // "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */
19 | // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */
20 | // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
21 | // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
22 | // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
23 | // "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
24 | // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */
25 | // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */
26 | // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */
27 |
28 | /* Modules */
29 | "module": "commonjs", /* Specify what module code is generated. */
30 | "rootDir": "./src",
31 | "moduleResolution": "node",
32 | // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
33 | // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
34 | // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
35 | // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */
36 | // "types": [], /* Specify type package names to be included without being referenced in a source file. */
37 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
38 | // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */
39 | // "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */
40 | // "rewriteRelativeImportExtensions": true, /* Rewrite '.ts', '.tsx', '.mts', and '.cts' file extensions in relative import paths to their JavaScript equivalent in output files. */
41 | // "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */
42 | // "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */
43 | // "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */
44 | // "noUncheckedSideEffectImports": true, /* Check side effect imports. */
45 | // "resolveJsonModule": true, /* Enable importing .json files. */
46 | // "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */
47 | // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */
48 |
49 | /* JavaScript Support */
50 | // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
51 | // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */
52 | // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
53 |
54 | /* Emit */
55 | "declaration": true,
56 | // "declarationMap": true, /* Create sourcemaps for d.ts files. */
57 | // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */
58 | "sourceMap": true,
59 | // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */
60 | // "noEmit": true, /* Disable emitting files from a compilation. */
61 | // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
62 | "outDir": "./dist",
63 | // "removeComments": true, /* Disable emitting comments. */
64 | // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
65 | // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
66 | // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */
67 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
68 | // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */
69 | // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
70 | // "newLine": "crlf", /* Set the newline character for emitting files. */
71 | // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
72 | // "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */
73 | // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */
74 | // "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */
75 | // "declarationDir": "./", /* Specify the output directory for generated declaration files. */
76 |
77 | /* Interop Constraints */
78 | // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */
79 | // "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */
80 | // "isolatedDeclarations": true, /* Require sufficient annotation on exports so other tools can trivially generate declaration files. */
81 | // "erasableSyntaxOnly": true, /* Do not allow runtime constructs that are not part of ECMAScript. */
82 | // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */
83 | "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */
84 | // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
85 | "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */
86 |
87 | /* Type Checking */
88 | "strict": true, /* Enable all strict type-checking options. */
89 | // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */
90 | // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */
91 | // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
92 | // "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
93 | // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */
94 | // "strictBuiltinIteratorReturn": true, /* Built-in iterators are instantiated with a 'TReturn' type of 'undefined' instead of 'any'. */
95 | // "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */
96 | // "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */
97 | // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */
98 | // "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */
99 | // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */
100 | // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */
101 | // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */
102 | // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */
103 | // "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */
104 | // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */
105 | // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */
106 | // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */
107 | // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */
108 |
109 | /* Completeness */
110 | // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */
111 | "skipLibCheck": true /* Skip type checking all .d.ts files. */
112 | },
113 | "include": ["src/**/*"],
114 | "exclude": ["node_modules", "dist", "**/*.test.ts"]
115 | }
116 |
--------------------------------------------------------------------------------