├── .gitignore
├── LICENSE
├── README.md
├── extract_from_cc
├── configs
│ └── randomized_all.yaml
├── extract_from_warc.py
├── spark_extract_dataset.py
├── spark_session_builder.py
├── text_normalizer.py
└── warcs
│ └── shard_0.txt
├── filtering
└── filter_dataset.py
├── imgs
├── OpenWebMath-left.png
├── openwebmath_logo.png
└── pipeline.png
└── text_extraction
├── setup.py
└── text_extract
├── __init__.py
├── banned_selectors.txt
├── boilerplate_words.txt
├── extract.py
├── latex_processing.py
├── line_processing.py
├── mmltex
├── README
├── cmarkup.xsl
├── entities.xsl
├── glayout.xsl
├── mmltex.xsl
├── scripts.xsl
├── tables.xsl
└── tokens.xsl
├── tree_processing.py
└── utils.py
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | share/python-wheels/
24 | *.egg-info/
25 | .installed.cfg
26 | *.egg
27 | MANIFEST
28 |
29 | # PyInstaller
30 | # Usually these files are written by a python script from a template
31 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
32 | *.manifest
33 | *.spec
34 |
35 | # Installer logs
36 | pip-log.txt
37 | pip-delete-this-directory.txt
38 |
39 | # Unit test / coverage reports
40 | htmlcov/
41 | .tox/
42 | .nox/
43 | .coverage
44 | .coverage.*
45 | .cache
46 | nosetests.xml
47 | coverage.xml
48 | *.cover
49 | *.py,cover
50 | .hypothesis/
51 | .pytest_cache/
52 | cover/
53 |
54 | # Translations
55 | *.mo
56 | *.pot
57 |
58 | # Django stuff:
59 | *.log
60 | local_settings.py
61 | db.sqlite3
62 | db.sqlite3-journal
63 |
64 | # Flask stuff:
65 | instance/
66 | .webassets-cache
67 |
68 | # Scrapy stuff:
69 | .scrapy
70 |
71 | # Sphinx documentation
72 | docs/_build/
73 |
74 | # PyBuilder
75 | .pybuilder/
76 | target/
77 |
78 | # Jupyter Notebook
79 | .ipynb_checkpoints
80 |
81 | # IPython
82 | profile_default/
83 | ipython_config.py
84 |
85 | # pyenv
86 | # For a library or package, you might want to ignore these files since the code is
87 | # intended to run in multiple environments; otherwise, check them in:
88 | # .python-version
89 |
90 | # pipenv
91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
94 | # install all needed dependencies.
95 | #Pipfile.lock
96 |
97 | # poetry
98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
99 | # This is especially recommended for binary packages to ensure reproducibility, and is more
100 | # commonly ignored for libraries.
101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
102 | #poetry.lock
103 |
104 | # pdm
105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
106 | #pdm.lock
107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
108 | # in version control.
109 | # https://pdm.fming.dev/#use-with-ide
110 | .pdm.toml
111 |
112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
113 | __pypackages__/
114 |
115 | # Celery stuff
116 | celerybeat-schedule
117 | celerybeat.pid
118 |
119 | # SageMath parsed files
120 | *.sage.py
121 |
122 | # Environments
123 | .env
124 | .venv
125 | env/
126 | venv/
127 | ENV/
128 | env.bak/
129 | venv.bak/
130 |
131 | # Spyder project settings
132 | .spyderproject
133 | .spyproject
134 |
135 | # Rope project settings
136 | .ropeproject
137 |
138 | # mkdocs documentation
139 | /site
140 |
141 | # mypy
142 | .mypy_cache/
143 | .dmypy.json
144 | dmypy.json
145 |
146 | # Pyre type checker
147 | .pyre/
148 |
149 | # pytype static type analyzer
150 | .pytype/
151 |
152 | # Cython debug symbols
153 | cython_debug/
154 |
155 | # PyCharm
156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
158 | # and can be added to the global gitignore or merged into this file. For a more nuclear
159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder.
160 | #.idea/
161 |
162 | # Mac
163 | .DS_Store
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright [yyyy] [name of copyright owner]
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # 
OpenWebMath: An Open Dataset of High-Quality Mathematical Web Text
2 |
3 | [Keiran Paster](https://keirp.com)\*, [Marco Dos Santos](https://marco-dossantos.github.io/)\*, [Zhangir Azerbayev](https://zhangir-azerbayev.github.io/), [Jimmy Ba](https://jimmylba.github.io/)
4 |
5 | [🤗 Download OpenWebMath ](https://huggingface.co/datasets/open-web-math/open-web-math) | [ArXiv](https://arxiv.org/abs/2310.06786)
6 | | [PDF](https://arxiv.org/pdf/2310.06786.pdf)
7 |
8 | # About OpenWebMath
9 |
10 | **OpenWebMath** is a dataset containing the majority of the high-quality, mathematical text from the internet. It is filtered and extracted from over 200B HTML files on Common Crawl down to a set of **6.3 million documents** containing a total of **14.7B tokens**. OpenWebMath is intended for use in _pretraining_ and _finetuning_ large language models.
11 |
12 | You can download the dataset using Hugging Face:
13 |
14 | ```python
15 | from datasets import load_dataset
16 | ds = load_dataset("open-web-math/open-web-math")
17 | ```
18 |
19 | # Code Structure
20 |
21 | We provide code in this repository to reproduce our processing pipeline. The code is organized into three separate folders:
22 |
23 | 1. `text_extraction` contains the code for extracting text and LaTeX from HTML documents.
24 | 2. `extract_from_cc` contains the code for extracting the dataset from Common Crawl, including prefiltering, language identification, MathScore filtering, and perplexity filtering.
25 | 3. `filtering` includes many of the manual filtering steps, including blacklisted domains.
26 |
27 | In order to run the `extract_from_cc` code, you either need to run it in Apache Spark or manually run `extract_from_warc.py` by passing in a WARC file as an argument.
28 |
29 | For deduplication, please use the [text-dedup](https://github.com/ChenghaoMou/text-dedup) library.
30 |
31 | Finally, for filtering, `filter.py` contains the code to load a Hugging Face dataset and filter it based on our heuristics.
32 |
33 | The _MathScore_ model and KenLM model will be released in the near future.
34 |
35 | # OpenWebMath Pipeline
36 |
37 | 
38 |
39 | OpenWebMath builds on the massive [Common Crawl](https://commoncrawl.org/) dataset, which contains over 200B HTML documents. We filtered the data to only include documents that are: (1) in English, (2) contain mathematical content, and (3) are of high quality. We also put a strong emphasis on extracting LaTeX content from the HTML documents as well as reducing boilerplate in comparison to other web datasets.
40 |
41 | The OpenWebMath pipeline consists of five steps:
42 |
43 | 1. **Prefiltering HTML Documents**:
44 | - We apply a simple prefilter to all HTML documents in Common Crawl in order to skip documents without mathematical content to unnecessary processing time.
45 | 2. **Text Extraction**:
46 | - Extract text, including LaTeX content, from the HTML documents while removing boilerplate.
47 | 3. **Content Classification and Filtering**:
48 | - Apply a [FastText language identification model](https://fasttext.cc/docs/en/language-identification.html) to keep only English documents.
49 | - Filter high perplexity documents using a [KenLM](https://github.com/kpu/kenlm) model trained on [Proof-Pile](https://huggingface.co/datasets/hoskinson-center/proof-pile).
50 | - Filter non-mathematical documents using our own _MathScore_ model.
51 | 4. **Deduplication**:
52 | - Deduplicate the dataset using SimHash in [text-dedup](https://github.com/ChenghaoMou/text-dedup).
53 | 5. **Manual Inspection**:
54 | - Inspect the documents gathered from previous steps and remove low quality pages.
55 |
56 | For a detailed discussion on the processing pipeline, please refer to our paper.
57 |
58 | # OpenWebMath Contents
59 |
60 | The dataset is structured as follows:
61 |
62 | ```python
63 | {
64 | "text": ..., # document text.
65 | "url": ..., # document url.
66 | "date": ..., # date the page was crawled.
67 | "metadata": ..., # JSON containing information from the extraction process.
68 | }
69 | ```
70 |
71 | OpenWebMath contains documents from over 130k different domains, including data from forums, educational pages, and blogs. The dataset contains documents covering mathematics, physics, statistics, computer science, and more. The following table shows the most common domains in OpenWebMath by character count.
72 |
73 | | Domain | # Characters | % Characters |
74 | | ----------------- | ------------- | ------------ |
75 | | stackexchange.com | 4,655,132,784 | 9.55% |
76 | | nature.com | 1,529,935,838 | 3.14% |
77 | | wordpress.com | 1,294,166,938 | 2.66% |
78 | | physicsforums.com | 1,160,137,919 | 2.38% |
79 | | github.io | 725,689,722 | 1.49% |
80 | | zbmath.org | 620,019,503 | 1.27% |
81 | | wikipedia.org | 618,024,754 | 1.27% |
82 | | groundai.com | 545,214,990 | 1.12% |
83 | | blogspot.com | 520,392,333 | 1.07% |
84 | | mathoverflow.net | 499,102,560 | 1.02% |
85 |
86 | # License
87 |
88 | OpenWebMath is made available under an ODC-By 1.0 license; users should also abide by the CommonCrawl ToU: [https://commoncrawl.org/terms-of-use/](https://commoncrawl.org/terms-of-use/). We do not alter the license of any of the underlying data.
89 |
90 | # Citation Information
91 |
92 | ```
93 | @misc{paster2023openwebmath,
94 | title={OpenWebMath: An Open Dataset of High-Quality Mathematical Web Text},
95 | author={Keiran Paster and Marco Dos Santos and Zhangir Azerbayev and Jimmy Ba},
96 | year={2023},
97 | eprint={2310.06786},
98 | archivePrefix={arXiv},
99 | primaryClass={cs.AI}
100 | }
101 | ```
102 |
--------------------------------------------------------------------------------
/extract_from_cc/configs/randomized_all.yaml:
--------------------------------------------------------------------------------
1 | markdown_headings: [
2 | [0.9, True],
3 | [0.1, False],
4 | ]
5 | markdown_code: [
6 | [0.95, True],
7 | [0.05, False],
8 | ]
9 | boilerplate_config:
10 | ratio_threshold: [
11 | [0.9, 0.18],
12 | [0.1, 0.30],
13 | ]
14 | absolute_threshold: [
15 | [0.9, 10],
16 | [0.1, 20],
17 | ]
18 | end_threshold: [
19 | [0.95, 15],
20 | [0.05, 5],
21 | ]
22 | enable: [
23 | [0.95, True],
24 | [0.05, False],
25 | ]
26 | remove_buttons: True
27 | remove_image_figures: True
28 | remove_link_clusters: True
29 | table_config:
30 | min_rows: 2
31 | min_cols: 3
32 | format: 'plain'
33 | remove_chinese: True
34 | remove_edit_buttons: True
35 | extract_latex: True
--------------------------------------------------------------------------------
/extract_from_cc/extract_from_warc.py:
--------------------------------------------------------------------------------
1 | import shutil
2 | import traceback
3 | from fastwarc.warc import ArchiveIterator
4 | import argparse
5 | from tqdm import tqdm
6 | from resiliparse.parse.html import HTMLTree
7 | from resiliparse.extract.html2text import extract_plain_text
8 | from resiliparse.parse.encoding import detect_encoding, bytes_to_str
9 | import fsspec
10 | from time import sleep
11 | import random
12 | from io import BytesIO
13 | from resiliparse.process_guard import time_guard, ExecutionTimeout
14 | import json
15 | import os
16 | import re
17 | import fasttext
18 | import kenlm
19 |
20 | # Temporary debugging imports
21 | import uuid
22 | from collections import defaultdict
23 |
24 | from text_extract.extract import extract_text
25 | from text_extract.utils import Config
26 | from text_extract.latex_processing import get_math_config
27 | from text_normalizer import normalize
28 |
29 | # Load the fasttext model
30 | if os.path.isdir('../models'):
31 | MODEL_PATH = '../models/math_score.bin'
32 | else:
33 | MODEL_PATH = 'math_score.bin'
34 |
35 | def score_text(text):
36 | normalized_text = normalize(text).replace('\n', ' ')
37 | # Remove any [EQUATION] tokens
38 | normalized_text = normalized_text.replace('[EQUATION]', '')
39 | pred = score_model.predict(normalized_text, k=2)
40 | if pred[0][0] == '__label__positive':
41 | prob = pred[1][0]
42 | else:
43 | prob = pred[1][1]
44 |
45 | return prob
46 |
47 | # Load the kenlm model
48 | LM_PATH = '../models/lm-v2.binary'
49 |
50 | lm = kenlm.Model(LM_PATH)
51 |
52 | def document_perplexity(text):
53 | text = normalize(text)
54 | score = lm.score(text)
55 | return 10 ** (-score / len(text.split()))
56 |
57 | # Load the model
58 | score_model = fasttext.load_model(MODEL_PATH)
59 |
60 | def is_english(text):
61 | normalized_text = normalize(text).replace('\n', ' ')
62 | pred = lid_model.predict(normalized_text, k=1)
63 | if pred[0][0] == "__label__en" and pred[1][0] >= 0.5:
64 | return True
65 | return False
66 |
67 | MODEL_BIN = '../models/lid.176.bin'
68 | lid_model = fasttext.load_model(MODEL_BIN)
69 |
70 | randomized_config = Config('configs/randomized_all.yaml')
71 |
72 | MATH_KEYWORDS = [
73 | 'MathJax',
74 | 'mathjax',
75 | ' 0.8 and len(text) > 500:
136 | return True
137 |
138 | return False
139 |
140 | def is_html(record):
141 | """Check that the record is an HTML record."""
142 | if record.headers is None:
143 | return False
144 | if record.http_headers is None:
145 | return False
146 | content_type = str(record.http_content_type)
147 | if content_type.startswith('text/html') or content_type.startswith('application/xhtml+xml'):
148 | return True
149 | return False
150 |
151 | def extract(html, config):
152 | res = extract_text(html, config, fast=True)
153 | if res is None:
154 | return None
155 | text, info = res
156 | metadata = {
157 | 'extraction_info': info,
158 | 'config': config,
159 | }
160 | return text, metadata
161 |
162 | def load_warc(warc_file):
163 | """Loads a WARC file with fsspec. Retries if it fails."""
164 | for i in range(10):
165 | try:
166 | with fsspec.open(warc_file, 'rb') as f:
167 | return f.read()
168 | except:
169 | if i == 9:
170 | raise Exception('Failed to read {}'.format(warc_file))
171 | print('Retrying to read {}'.format(warc_file))
172 | # Sleep a random amount of time
173 | sleep(random.random())
174 |
175 | def process_warc(warc_file):
176 | """Yields extracted text from a WARC file."""
177 |
178 | doc_counter = defaultdict(int)
179 |
180 | # Error if it takes more than 20 minutes
181 | with time_guard(timeout=60*20):
182 | try:
183 | f = load_warc(warc_file)
184 | for i in range(10):
185 | try:
186 | stream = BytesIO(f)
187 | break
188 | except:
189 | if i == 9:
190 | print('Failed to read {}'.format(warc_file))
191 | return
192 | print('Retrying to read {}'.format(warc_file))
193 |
194 | # We only want to extract text from the response records
195 | total_parsed = 0
196 | total_has_math = 0
197 | for record in tqdm(ArchiveIterator(stream)):
198 | try:
199 | doc_counter['records'] += 1
200 | if record.headers is None: continue
201 | if record.http_headers is None: continue
202 | if record.headers['WARC-Type'] != 'response': continue
203 | if not is_html(record): continue
204 | doc_counter['html'] += 1
205 | # Extract text from the payload
206 | html = record.reader.read()
207 | html = decode_html(html)
208 | url = record.headers.get('WARC-Target-URI')
209 | record_date = record.record_date
210 | if html is None: continue
211 | if not contains_math(html): continue
212 | doc_counter['passes prefilter'] += 1
213 | randomized_config_sample = randomized_config.sample()
214 | res = extract(html, randomized_config_sample)
215 | total_parsed += 1
216 | print(f'Running percentage: {total_has_math / total_parsed:.2f}, total parsed: {total_parsed}, total has math: {total_has_math}')
217 | if res is None: continue
218 | randomized_text, metadata = res
219 |
220 | found_math = metadata['extraction_info']['found_math']
221 | if not is_english(randomized_text): continue
222 | doc_counter['is english'] += 1
223 | score = score_text(randomized_text)
224 | if found_math:
225 | if score < 0.15: continue
226 | else:
227 | if score < 0.8: continue
228 | doc_counter['passes score'] += 1
229 |
230 | metadata['extraction_info']['math_score'] = score
231 | perplexity = document_perplexity(randomized_text)
232 | metadata['extraction_info']['perplexity'] = perplexity
233 |
234 | if perplexity > 30_000: continue
235 | doc_counter['passes perplexity'] += 1
236 | total_has_math += 1
237 |
238 | yield (url, (randomized_text, html, warc_file, str(metadata), str(record_date)))
239 | except Exception as e:
240 | print(f'Execution timeout for {warc_file}')
241 | print(e)
242 | # Print a trace
243 | traceback.print_exc()
244 | except:
245 | print(f'Execution (probably) timed out for {warc_file}')
246 | return
247 |
248 | print(f'Finished processing {warc_file}'
249 | f' with {total_has_math} math-containing pages out of {total_parsed} parsed pages')
250 |
251 | # Save the doc counter as a text file with a uuid name
252 | with open(f'{uuid.uuid4()}.txt', 'w') as f:
253 | f.write(json.dumps(doc_counter))
254 |
255 |
256 | def main(warc_file, output_dir):
257 | data = []
258 | print('Extracting text from {}'.format(warc_file))
259 | for url, (text, html, _, metadata, _) in process_warc(warc_file):
260 | data.append({'url': url, 'value': {'text': text, 'no_latex': '', 'no_randomization': '', 'html': html, 'warc_path': warc_file, 'metadata': metadata}})
261 | # Remove directories if they exist
262 | if os.path.exists(output_dir):
263 | shutil.rmtree(output_dir)
264 | os.makedirs(output_dir)
265 | # Save the data as md files
266 | for i, d in enumerate(data):
267 | text = d['value']['text']
268 | html = d['value']['html']
269 | url = d['url']
270 | with open(os.path.join(output_dir, '{}.md'.format(i)), 'w') as f:
271 | f.write('# {}\n\n'.format(url))
272 | f.write(text)
273 | with open(os.path.join(output_dir, '{}.html'.format(i)), 'w') as f:
274 | f.write(html)
275 |
276 |
277 | if __name__ == '__main__':
278 | parser = argparse.ArgumentParser()
279 | parser.add_argument('--warc_file', help='WARC file to extract text from')
280 | parser.add_argument('--output_dir', help='Output dir')
281 | args = parser.parse_args()
282 | main(args.warc_file, args.output_dir)
--------------------------------------------------------------------------------
/extract_from_cc/spark_extract_dataset.py:
--------------------------------------------------------------------------------
1 | import argparse
2 | from spark_session_builder import build_spark_session
3 | from pyspark.sql import SparkSession
4 | from pyspark import SparkContext
5 | from extract_from_warc import process_warc
6 | import jsonlines
7 | import os
8 | from pyspark.sql.types import StructType, StructField, StringType
9 | import uuid
10 |
11 | output_schema = StructType([
12 | StructField('url', StringType(), True),
13 | StructField('value', StructType([
14 | StructField('text', StringType(), True),
15 | StructField('html', StringType(), True),
16 | StructField('warc_path', StringType(), True),
17 | StructField('metadata', StringType(), True),
18 | StructField('date', StringType(), True),
19 | ]), True),
20 | ])
21 |
22 | def process_filename(filename):
23 | if filename.startswith('file:'):
24 | filename = filename[5:]
25 | return filename
26 |
27 | def test_process_warc(warc_file):
28 | yield ('test_url', ('test_html', 'test_text'))
29 |
30 | def main(warc_file_list,
31 | output_dir,
32 | master='local',
33 | driver_memory=33,
34 | driver_cores=1,
35 | executor_memory=33,
36 | num_executors=29,
37 | executor_cores=5,
38 | num_cpus_per_task=5,
39 | num_output_partitions=1,
40 | output_format='parquet',
41 | output_compression='gzip'):
42 | spark = SparkSession.getActiveSession()
43 | if spark is not None:
44 | spark.stop()
45 | spark = build_spark_session(master=master,
46 | driver_memory=driver_memory,
47 | executor_memory=executor_memory,
48 | num_cpus_per_task=num_cpus_per_task,
49 | executor_cores=executor_cores,
50 | driver_cores=driver_cores)
51 | sc = SparkContext.getOrCreate()
52 | # warc_file_list is a text file with one warc file per line
53 | # Get the filename of warc_file_list
54 | warc_file_list_name = os.path.basename(warc_file_list).split('.')[0]
55 | warc_files = spark.read.text(warc_file_list).rdd.map(lambda r: r[0]).collect()
56 | warc_files = [process_filename(w) for w in warc_files]
57 | warc_count = len(warc_files)
58 | print('Found {} warc files'.format(warc_count))
59 | warc_rdd = sc.parallelize(warc_files, warc_count)
60 |
61 | df = warc_rdd.flatMap(process_warc).toDF(output_schema)
62 |
63 | # Split into partitions
64 | df = df.repartition(num_output_partitions)
65 | # Write the output
66 | unique_id = str(uuid.uuid4())
67 | output_file = os.path.join(output_dir, f'math_{warc_file_list_name}_{unique_id}')
68 | df.write.format(output_format).option('compression', output_compression).save(output_file)
69 |
70 | if __name__ == '__main__':
71 | parser = argparse.ArgumentParser()
72 | parser.add_argument('--warc_file_list', type=str)
73 | parser.add_argument('--output_dir', type=str)
74 | parser.add_argument('--master', type=str, default='local')
75 | parser.add_argument('--driver_memory', type=int, default=33)
76 | parser.add_argument('--driver_cores', type=int, default=1)
77 | parser.add_argument('--executor_memory', type=int, default=33)
78 | parser.add_argument('--num_executors', type=int, default=29)
79 | parser.add_argument('--executor_cores', type=int, default=5)
80 | parser.add_argument('--num_cpus_per_task', type=int, default=2)
81 | parser.add_argument('--mem_gb', type=int, default=16)
82 | parser.add_argument('--num_output_partitions', type=int, default=1)
83 | parser.add_argument('--output_format', type=str, default='json')
84 | parser.add_argument('--output_compression', type=str, default='gzip')
85 | args = parser.parse_args()
86 | main(warc_file_list=args.warc_file_list,
87 | output_dir=args.output_dir,
88 | master=args.master,
89 | driver_memory=args.driver_memory,
90 | driver_cores=args.driver_cores,
91 | executor_memory=args.executor_memory,
92 | num_executors=args.num_executors,
93 | executor_cores=args.executor_cores,
94 | num_cpus_per_task=args.num_cpus_per_task,
95 | num_output_partitions=args.num_output_partitions,
96 | output_format=args.output_format,
97 | output_compression=args.output_compression)
--------------------------------------------------------------------------------
/extract_from_cc/spark_session_builder.py:
--------------------------------------------------------------------------------
1 | from pyspark.sql import SparkSession
2 | import os
3 | import sys
4 |
5 |
6 | def build_spark_session(master, driver_memory, executor_memory, num_cpus_per_task, executor_cores, driver_cores):
7 | """Build a spark session based on the master url and the number of cores and memory to use"""
8 | if master == "local":
9 | spark = local_session(executor_cores, driver_memory)
10 | else:
11 | spark = aws_ec2_s3_spark_session(driver_memory, executor_memory, num_cpus_per_task, executor_cores, driver_cores)
12 |
13 | return spark
14 |
15 |
16 | def local_session(num_cores=4, mem_gb=16):
17 | """Build a local spark session"""
18 | spark = (
19 | SparkSession.builder.config("spark.driver.memory", str(mem_gb) + "G")
20 | .master("local[" + str(num_cores) + "]")
21 | .appName("extract_math")
22 | .getOrCreate()
23 | )
24 | return spark
25 |
26 | def aws_ec2_s3_spark_session(driver_memory, executor_memory, num_cpus_per_task, executor_cores, driver_cores):
27 | """Build a spark session on AWS EC2"""
28 | driver_memory = str(int(driver_memory)) + 'g'
29 | # executor_memory = str(int(executor_memory)) + 'g'
30 | main_memory = str(int(executor_memory * 0.9)) + 'g'
31 | memory_overhead = str(executor_memory - int(executor_memory * 0.9)) + 'g'
32 | spark = (
33 | SparkSession.builder.appName("extractmath")
34 | .config("spark.executor.memory", main_memory)
35 | .config("spark.driver.memory", driver_memory)
36 | # .config("spark.driver.cores", str(driver_cores))
37 | .config("spark.executor.memoryOverhead", memory_overhead)
38 | .config("spark.executor.cores", str(executor_cores)) # Number of cpu cores available basically
39 | .config("spark.task.cpus", str(num_cpus_per_task))
40 | .config("spark.task.maxFailures", "10")
41 | .config("spark.dynamicAllocation.enabled", "true")
42 | .config("spark.shuffle.service.enabled", "true")
43 | .getOrCreate()
44 | )
45 | return spark
--------------------------------------------------------------------------------
/extract_from_cc/text_normalizer.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Facebook, Inc. and its affiliates.
2 | #
3 | # This source code is licensed under the MIT license found in the
4 | # LICENSE file in the root directory of this source tree.
5 | #
6 | # From https://github.com/facebookresearch/cc_net/blob/main/cc_net/text_normalizer.py
7 |
8 | import re
9 | import unicodedata
10 |
11 | UNICODE_PUNCT = {
12 | "，": ",",
13 | "。": ".",
14 | "、": ",",
15 | "„": '"',
16 | "”": '"',
17 | "“": '"',
18 | "«": '"',
19 | "»": '"',
20 | "１": '"',
21 | "」": '"',
22 | "「": '"',
23 | "《": '"',
24 | "》": '"',
25 | "´": "'",
26 | "∶": ":",
27 | "：": ":",
28 | "？": "?",
29 | "！": "!",
30 | "（": "(",
31 | "）": ")",
32 | "；": ";",
33 | "–": "-",
34 | "—": " - ",
35 | "．": ". ",
36 | "～": "~",
37 | "’": "'",
38 | "…": "...",
39 | "━": "-",
40 | "〈": "<",
41 | "〉": ">",
42 | "【": "[",
43 | "】": "]",
44 | "％": "%",
45 | "►": "-",
46 | }
47 |
48 | UNICODE_PUNCT_RE = re.compile(f"[{''.join(UNICODE_PUNCT.keys())}]")
49 |
50 | MATH_RE = r"(? str:
55 | return "".join((UNICODE_PUNCT.get(c, c) for c in text))
56 |
57 |
58 | def remove_unicode_punct(text: str) -> str:
59 | """More aggressive version of replace_unicode_punct but also faster."""
60 | return UNICODE_PUNCT_RE.sub("", text)
61 |
62 |
63 | def strip_accents(line: str) -> str:
64 | """Strips accents from a piece of text."""
65 | nfd = unicodedata.normalize("NFD", line)
66 | output = [c for c in nfd if unicodedata.category(c) != "Mn"]
67 | if len(output) == line:
68 | return line
69 | return "".join(output)
70 |
71 |
72 | # Build a regex matching all control characters.
73 | NON_PRINTING_CHARS_RE = re.compile(
74 | f"[{''.join(map(chr, list(range(0,32)) + list(range(127,160))))}]"
75 | )
76 | DIGIT_RE = re.compile(r"\d")
77 | PUNCT_OR_NON_PRINTING_CHARS_RE = re.compile(
78 | (UNICODE_PUNCT_RE.pattern + NON_PRINTING_CHARS_RE.pattern).replace("][", "")
79 | )
80 |
81 |
82 | def remove_non_printing_char(text: str) -> str:
83 | return NON_PRINTING_CHARS_RE.sub("", text)
84 |
85 |
86 | def normalize_spacing_for_tok(text: str, language: str = "en") -> str:
87 | res = (
88 | text.replace("\r", "")
89 | # remove extra spaces
90 | .replace("(", " (")
91 | .replace(")", ") ")
92 | .replace(" +", " ")
93 | )
94 | res = re.sub(r"\) ([\.\!\:\?\;\,])", r"\)\1", res)
95 | res = res.replace("( ", "(").replace(" )", ")")
96 | res = re.sub(r"(\d) \%", r"\1\%", res)
97 | res = res.replace(" :", ":").replace(" ;", ";")
98 | res = res.replace("`", "'").replace("''", ' " ')
99 |
100 | res = (
101 | res.replace("„", '"')
102 | .replace("“", '"')
103 | .replace("”", '"')
104 | .replace("–", "-")
105 | .replace("—", " - ")
106 | .replace(" +", " ")
107 | .replace("´", "'")
108 | .replace("([a-z])‘([a-z])", r"\1'\2/")
109 | .replace("([a-z])’([a-z])", r"\1'\2/")
110 | .replace("‘", '"')
111 | .replace("‚", '"')
112 | .replace("’", '"')
113 | .replace("''", '"')
114 | .replace("´´", '"')
115 | .replace("…", "...")
116 | # French quotes
117 | .replace(" « ", ' "')
118 | .replace("« ", '"')
119 | .replace("«", '"')
120 | .replace(" » ", '" ')
121 | .replace(" »", '"')
122 | .replace("»", '"')
123 | # handle pseudo-spaces
124 | .replace(" %", "%")
125 | .replace("nº ", "nº ")
126 | .replace(" :", ":")
127 | .replace(" ºC", " ºC")
128 | .replace(" cm", " cm")
129 | .replace(" ?", "?")
130 | .replace(" !", "!")
131 | .replace(" ;", ";")
132 | .replace(", ", ", ")
133 | .replace(" +", " ")
134 | .replace("．", ". ")
135 | )
136 | # English "quotation," followed by comma, style
137 | if language == "en":
138 | res = re.sub(r"\"([,\.]+)", r"\1\"", res)
139 | # Czech is confused
140 | elif language == "cs" or language == "cz":
141 | pass
142 | # German/Spanish/French "quotation", followed by comma, style
143 | else:
144 | res = res.replace(',"', '",')
145 | res = re.sub(
146 | r"(\.+)\"(\s*[^<])", r"\"\1\2", res
147 | ) # don't fix period at end of sentence
148 |
149 | if (
150 | language == "de"
151 | or language == "es"
152 | or language == "cz"
153 | or language == "cs"
154 | or language == "fr"
155 | ):
156 | res = re.sub(r"(\d) (\d)", r"\1,\2", res)
157 | else:
158 | res = re.sub(r"(\d) (\d)", r"\1.\2", res)
159 | return res
160 |
161 |
162 | def normalize(line: str, accent=True, case=True, numbers=True, math=True, code=True, punct=1) -> str:
163 | line = line.strip()
164 | if not line:
165 | return line
166 | if case:
167 | line = line.lower()
168 | if accent:
169 | line = strip_accents(line)
170 | if numbers:
171 | line = DIGIT_RE.sub("0", line)
172 | if punct == 1:
173 | line = replace_unicode_punct(line)
174 | elif punct == 2:
175 | line = remove_unicode_punct(line)
176 | if math:
177 | line = re.sub(MATH_RE, "[EQUATION]", line, flags=re.DOTALL)
178 | if code:
179 | line = re.sub(CODE_RE, "[CODE]", line, flags=re.DOTALL)
180 | # Replace any or explicitly written in the text with nothing
181 | line = line.replace("", "").replace("", "")
182 | line = remove_non_printing_char(line)
183 | return line
184 |
185 |
186 | def slow_normalize_for_dedup(line: str) -> str:
187 | return normalize(line, accent=False, case=True, numbers=True, punct=2)
188 |
189 |
190 | def normalize_for_dedup(line: str) -> str:
191 | line = line.strip()
192 | if not line:
193 | return line
194 | # case
195 | line = line.lower()
196 | # numbers
197 | line = DIGIT_RE.sub("0", line)
198 | line = PUNCT_OR_NON_PRINTING_CHARS_RE.sub("", line)
199 | return line
--------------------------------------------------------------------------------
/filtering/filter_dataset.py:
--------------------------------------------------------------------------------
1 | # coding=utf-8
2 | import argparse
3 | from datasets import load_dataset, Features
4 | from tqdm import tqdm
5 | import json
6 | import matplotlib.pyplot as plt
7 | import re
8 |
9 | BAD_URLS = [
10 | 'worldwidescience',
11 | 'science.gov',
12 | 'archive.org',
13 | 'scribd.com',
14 | 'unz.com',
15 | '/profile/',
16 | '/researcher',
17 | 'noobstarter.com',
18 | 'philpapers.org',
19 | 'thesa.com',
20 | 'beyondhighbrow.com',
21 | 'careyoukeep.com',
22 | 'eevblog.com',
23 | 'happyslide.net',
24 | 'issuu.com',
25 | 'zh-cn.unz.com',
26 | 'vixra.org',
27 | 'medcraveonline.com',
28 | 'sciendo.com',
29 | 'open.library.ubc.ca',
30 | 'eurotrib.com',
31 | 'postthreads.org',
32 | 'jim.bmj.com',
33 | 'wanweibaike.com',
34 | 'hzdr.de',
35 | '/joursearch/',
36 | 'docplayer.net',
37 | 'bookofmormonblog.org',
38 | 'bradford-delong.com',
39 | 'profiles.stanford.edu',
40 | 'vo.astronet.ru',
41 | 'homainstallationen.at',
42 | '/author/',
43 | '/authors/'
44 | '/serials/'
45 | 'read.dukeupress.edu',
46 | 'thewikipost.org',
47 | 'is.tuebingen.mpg.de',
48 | 'discourse.darkjedibrotherhood.com',
49 | 'springermedizin.de',
50 | 'materials-chain.com',
51 | 'www.unzmag.net',
52 | 'is.mpg.de',
53 | 'hobby8.5ch.net',
54 | 'forums.penny-arcade.com',
55 | 'wowwiki.com',
56 | '8chan.moe',
57 | 'plosone.org',
58 | 'www.is.mpg.de',
59 | 'feeds.churchturing.org',
60 | 'learn.gcs.edu',
61 | 'mobinuke.com',
62 | 'judithcurry.com',
63 | 'tek-tips.com',
64 | 'skepticforum.com',
65 | 'all_publications',
66 | '.de/publications',
67 | 'nih.gov',
68 | 'lastfm.it',
69 | '/commit',
70 | 'vitaminstore',
71 | 'studylib.net',
72 | 'dokumen.pub',
73 | 'manualzz.com',
74 | 'fraser.stlouisfed.org'
75 | ]
76 |
77 | libretext_good = [
78 | 'math',
79 | 'phys',
80 | 'stats',
81 | ]
82 |
83 | accented_chars = ['ü', 'ï', 'ö', 'ê', 'ä', 'â', 'ê', 'î', 'û', 'ô', 'è', 'é', 'à']
84 | accented_chars = set(accented_chars)
85 | def has_accented_char(text):
86 | num_accent = sum([c in accented_chars for c in text.lower()])
87 | if len(text) == 0:
88 | return False
89 | return num_accent / len(text) > 0.015
90 |
91 | def count_latex_formulas(text):
92 | # Remove unwanted patterns
93 | cleaned_text = re.sub(r'\$\$\\PageIndex\{[^\}]*\}\$\$', '', text)
94 | cleaned_text = re.sub(r'\$\\PageIndex\{[^\}]*\}\$', '', cleaned_text)
95 |
96 | # Pattern for inline and display math
97 | pattern = r'\$\$[^\$]*\$\$|\$[^\$]*\$'
98 |
99 | matches = re.findall(pattern, cleaned_text)
100 |
101 | return len(matches)
102 |
103 | def filter_data(data):
104 | metadata = json.loads(data['metadata'])
105 | perplexity = metadata['extraction_info']['perplexity']
106 | math_score = metadata['extraction_info']['math_score']
107 | if perplexity > 15_000:
108 | return False
109 | if math_score < 0.17:
110 | return False
111 | if 'arxiv-vanity' in data['url']:
112 | return False
113 | # Check if /search is in the path
114 | if '/search' in data['url'] and '//search' not in data['url']:
115 | return False
116 | if 'proceedings' in data['url']:
117 | return False
118 | if 'bibbase' in data['url']:
119 | return False
120 | if 'nrsworld.com' in data['url']:
121 | return False
122 | if 'bibtex' in data['url']:
123 | return False
124 | if 'issn' in data['url']:
125 | return False
126 | if 'arxiv-export' in data['url']:
127 | return False
128 | if 'bmjopen' in data['url']:
129 | return False
130 | if 'stackexchange.com/users' in data['url']:
131 | return False
132 | if 'mathoverflow.net/users' in data['url']:
133 | return False
134 | return True
135 |
136 | def process_data(datas):
137 | # Convert datas (which is keys with lists) to a list of dicts
138 | datas = [dict(zip(datas, t)) for t in zip(*datas.values())]
139 | new_datas = []
140 | for data in datas:
141 | url = data['url']
142 | text = data['text']
143 |
144 | should_filter = not filter_data(data)
145 |
146 | # Filter out bad urls
147 | for bad_url in BAD_URLS:
148 | if bad_url in url:
149 | should_filter = True
150 | break
151 |
152 | # Remove any line that has more than one "newcommand"
153 | lines = text.split('\n')
154 | new_lines = []
155 | for line in lines:
156 | if line.count('newcommand') > 1:
157 | continue
158 | new_lines.append(line)
159 | text = '\n'.join(new_lines)
160 | data['text'] = text
161 |
162 | # Filter less than 100 characters
163 | if len(text) < 100:
164 | should_filter = True
165 |
166 | if 'libretexts' in url:
167 | # Check if the url is part of the whitelist
168 | is_whitelist = False
169 | for good in libretext_good:
170 | if good in url:
171 | is_whitelist = True
172 | break
173 | if not is_whitelist:
174 | # Throw out if 0 math formulas
175 | if count_latex_formulas(text) == 0:
176 | should_filter = True
177 |
178 | # Filter out accents
179 | if has_accented_char(text):
180 | should_filter = True
181 |
182 | if not should_filter:
183 | new_datas.append(data)
184 |
185 | # Transform back to a dict of lists
186 | new_datas = {k: [d[k] for d in new_datas] for k in new_datas[0]}
187 | return new_datas
188 |
189 | def main(args):
190 | dataset = load_dataset(args.input, split='train')
191 | print(dataset)
192 |
193 | # Filter the dataset
194 | filtered_dataset = dataset.map(process_data, num_proc=args.n_processes, batched=True)
195 | print(filtered_dataset)
196 |
197 | # Save the dataset
198 | filtered_dataset.save_to_disk(args.output_dataset)
199 |
200 | if __name__ == '__main__':
201 | parser = argparse.ArgumentParser()
202 | parser.add_argument("--input", type=str, help="Huggingface dataset name")
203 | parser.add_argument("--output_dataset", type=str, required=True, help="path to save dataset")
204 | parser.add_argument("--n_processes", type=int, default=32, help="Number of processes to use")
205 |
206 | args = parser.parse_args()
207 | main(args)
--------------------------------------------------------------------------------
/imgs/OpenWebMath-left.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/keirp/OpenWebMath/98f81d0ae14ff8dd877b84226b69c825b64da498/imgs/OpenWebMath-left.png
--------------------------------------------------------------------------------
/imgs/openwebmath_logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/keirp/OpenWebMath/98f81d0ae14ff8dd877b84226b69c825b64da498/imgs/openwebmath_logo.png
--------------------------------------------------------------------------------
/imgs/pipeline.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/keirp/OpenWebMath/98f81d0ae14ff8dd877b84226b69c825b64da498/imgs/pipeline.png
--------------------------------------------------------------------------------
/text_extraction/setup.py:
--------------------------------------------------------------------------------
1 | from setuptools import setup
2 |
3 | setup(
4 | name='text_extract',
5 | version='0.1',
6 | description='Text extraction tools',
7 | author='',
8 | packages=['text_extract'],
9 | )
--------------------------------------------------------------------------------
/text_extraction/text_extract/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/keirp/OpenWebMath/98f81d0ae14ff8dd877b84226b69c825b64da498/text_extraction/text_extract/__init__.py
--------------------------------------------------------------------------------
/text_extraction/text_extract/banned_selectors.txt:
--------------------------------------------------------------------------------
1 | .breadcrumb
2 | #popup
3 | #flyout
4 | #site-slogan
5 | #site-name
6 | #menu
7 | .nav
8 | .login
9 | .dropdown
10 | .dropdown-menu
11 | #login
12 | .vote
13 | .form-item
14 | .user_pic_popup
15 | #post-editor
16 | .post-form
17 | .bottom-notice
18 | #sidebar
19 | #copyright
20 | #footer
21 | .footer
22 | .site-navigation
23 | .popupgroup
24 | .posthead
25 | .signaturecontainer
26 | .after_content
27 | .userinfo
28 | #similar_threads
29 | .toplinks
30 | .user-info
31 | .post-header
32 | .widget_archive
33 | .widget_categories
34 | .widget_meta
35 | .widget_recent_entries
36 | .widget_rss
37 | .wp_widget_tag_cloud
38 | .widget_calendar
39 | .navbox
40 | #mw-hidden-catlinks
41 | .above_postlist
42 | #navigation
43 | .threadtools
44 | .socialbuttons
45 | #respond
46 | .menu
47 | .WikiaHeader
48 | .buttons
49 | #WikiaRecentActivity
50 | #WikiaRandomWiki
51 | .loggedout-follow-normal
52 | #blurb
53 | #banner-top
54 | .topbar
55 | .topbar-dialog
56 | .related-links
57 | .votecell
58 | .comment-actions
59 | .d-none
60 | .Tooltip
61 | .Notices
62 | .likes-other-gravatars
63 | #logo_and_banner
64 | #pmmcrumb2
65 | .qa-notice
66 | .qa-nav-user
67 | .trackbacks
68 | #further_reading
69 | .topbar-links
70 | #your-communities-section
71 | .links-container
72 | #herobox
73 | .qa-voting-container
74 | .qa-post-when-container
75 | .qa-q-view-who
76 | .qa-q-item-meta
77 | .post-menu
78 | #vbseo-likes
79 | #side_one
80 | #side_two
81 | #feed_bar
82 | .author
83 | #likes-other-gravatars
84 | .pageInfo
85 | .ka-video-player
86 | .mw-editsection
87 | .mw-ui-icon
88 | #mw-revision-info
89 | #siteSub
90 | .heading--main
91 | #loginBarHandle
92 | .medalsrest
93 | .diff-otitle
94 | .diff-ntitle
95 | .diff-currentversion-title
96 | .diff-contentalign-left
97 | [class*="promo"]
98 | [class*="button"]
99 | [class*="upsell"]
100 | .expert-reply-overlay
101 | .PreviewContents
102 | .solutionHeader__isbn
103 | .cta
104 | .update-header
105 | .best-answer-selected
106 | .medal-info
107 | #profile-tooltip
108 | .update-info
109 | .google-search-openstudy
110 | .attachments
111 | button
112 | .delete
113 | .editor-actions
114 | .editor
115 | .files-attached
116 | .call-to-action
117 | .group-info
118 | .top-online-users
119 | .message-userExtras
120 | .message-attribution-opposite
121 | .u-srOnly
122 | .block--similarContents
123 | .u-concealed
124 | .similarThreads
125 | .breadcrumbs
126 | .courseHeader
127 | .impactSection
128 | .creativeCommons
129 | #clear
--------------------------------------------------------------------------------
/text_extraction/text_extract/boilerplate_words.txt:
--------------------------------------------------------------------------------
1 | ©
2 | updates
3 | join our
4 | buy
5 | sign up
6 | no results
7 | search images
8 | all rights reserved
9 | was this
10 | please
11 | visit our
12 | download for free
13 | retrieved from
14 | home page
15 | jump to
16 | notification switch
17 | your email address
18 | view answer
19 | no label found
20 | is licensed under
21 | regular updates
22 | copyright
23 | have access to this article
24 | youtube
25 | advertisment
26 | password
27 | login
28 | learn more
29 | cookie
30 | jump to navigation
31 | download
32 | table of contents
33 | leave a reply
34 | leave a message
35 | skip to
36 | stay updated
37 | contact us
38 | twitter
39 | from wikibooks, open books for an open world
40 | last modified
41 | from wikipedia, the free encyclopedia
42 | more info
43 | terms of use
44 | terms of service
45 | privacy policy
46 | navigation
47 | sign in
48 | report error
49 | newest
50 | under license
51 | follow
52 | newer
53 | notification
54 | post a comment
55 | click here
56 | leave a comment
57 | google
58 | free account
59 | for free
60 | alert
61 | receive update
62 | share this
63 | report ad
64 | more posts
65 | date of creation
66 | link
67 | powered by
68 | receive
69 | newsletter
70 | pdf version
71 | ask
72 | your
73 | facebook
74 | jump to search
75 | required fields
76 | back to top
77 | published by
78 | pdf article
79 | accessload
80 | start with
81 | loading
82 | username
83 | helpful
84 | log in
85 | license
86 | get the best
87 | join us
88 | full article
89 | attribution
90 | main content
91 | printed from
92 | distributed under
93 | rss
94 | 24/7
95 | your service
96 | please contact
97 | captcha
98 | might be incomplete
99 | about this
100 | lifetime
101 | access to
102 | this article is
103 | not found
104 | show more
105 | about
106 | business
107 | interested in joining
108 | wikipedia page
109 | gift
110 | premium
111 | purchase this
112 | purchasing
113 | access denied
114 | wims
115 | latest version
116 | this page
117 | your web browser
118 | recent version
119 | this article
120 | please help
121 | help you
122 | discard
123 | view tag cloud
124 | reply
125 | sponsor
126 | return to
127 | physicsoverflow is an open platform for community peer review
128 | comments
129 | trackback
130 | show menu
131 | add comment
132 | printable view
133 | advertisement
134 | join now
135 | from proofwiki
136 | energy points
137 | at the top of this image page
138 | all products
139 | maplesim
140 | online help
141 | see also
142 | all lesson plans
143 | menu
144 | check out more articles
145 | ad
146 | votes
147 | answer
148 | question you clicked on
149 | this question is closed
--------------------------------------------------------------------------------
/text_extraction/text_extract/extract.py:
--------------------------------------------------------------------------------
1 | from resiliparse.parse.html import HTMLTree
2 | from resiliparse.extract.html2text import extract_plain_text
3 | import os
4 | import re
5 |
6 | from text_extract.latex_processing import extract_math, extract_delimited_math, get_math_config, replace_math_tags_with_dollar_signs
7 | from text_extract.tree_processing import remove_buttons, remove_image_figures, extract_code, extract_tables, extract_headings, remove_dense_links, add_se_separators, wikipedia_preprocess, remove_display_none, main_content_preprocess, post_process_headings
8 | from text_extract.line_processing import remove_empty_headers, remove_edit_buttons, remove_chinese_characters, remove_boilerplate
9 | from text_extract.utils import ReplacementManager
10 |
11 | import faulthandler
12 | faulthandler.enable()
13 |
14 |
15 | selectors_path = os.path.join(os.path.dirname(__file__), "banned_selectors.txt")
16 | with open(selectors_path, "r") as f:
17 | selectors = [line.replace('\n', '').strip() for line in f]
18 | # Remove empty lines
19 | selectors = [line for line in selectors if line]
20 |
21 | def filter_tree(tree, replacement_manager, config):
22 | """Filters the HTML tree to remove unwanted elements."""
23 |
24 | # Remove display none elements
25 | remove_display_none(tree)
26 |
27 | # Remove the wikipedia footer
28 | wikipedia_preprocess(tree)
29 |
30 | if config['remove_buttons']:
31 | # Remove any bootstrap buttons
32 | remove_buttons(tree)
33 |
34 | if config['remove_image_figures']:
35 | # Remove any figures that only contain images
36 | remove_image_figures(tree)
37 |
38 | if config['markdown_code']:
39 | # Wrap the code in markdown code blocks
40 | extract_code(tree, replacement_manager)
41 |
42 | # Record the location of headings and format them
43 | extract_headings(tree, replacement_manager, config['markdown_headings'])
44 |
45 | # Remove link lists
46 | remove_dense_links(tree)
47 |
48 | # Format tables
49 | extract_tables(tree.document, replacement_manager, config['table_config'])
50 |
51 | # Process stack exchange separators
52 | add_se_separators(tree)
53 |
54 | # Preprocess main content
55 | main_content_preprocess(tree)
56 |
57 | return tree
58 |
59 | def html_preprocessing(html):
60 | html = html.replace("<math>", "[itex]")
61 | html = html.replace("</math>", "[/itex]")
62 | return html
63 |
64 | def replace_tags(html, old, new):
65 | pattern = re.compile(old, re.IGNORECASE)
66 | return pattern.sub(new, html)
67 |
68 | def extract_text(html, config, fast=False):
69 | """Extracts plain text from an HTML string."""
70 | html = replace_tags(html, '', '')
74 | html = html_preprocessing(html)
75 | tree = HTMLTree.parse(html)
76 | replacement_manager = ReplacementManager()
77 |
78 | if fast:
79 | links = tree.document.query_selector_all('a')
80 | span_links = tree.document.query_selector_all('span a')
81 | if len(links) > 3000 or len(span_links) > 3000:
82 | print("Too many links, skipping")
83 | return None
84 |
85 | if config['extract_latex']:
86 | math_config = get_math_config(tree.document.html)
87 | tree, info = extract_math(tree, replacement_manager)
88 | else:
89 | info = {}
90 | tree = filter_tree(tree, replacement_manager, config)
91 |
92 | # Disable their filters because we use our own.
93 | text = extract_plain_text(tree,
94 | main_content=True,
95 | alt_texts=False,
96 | skip_elements=selectors)
97 |
98 | if config['extract_latex']:
99 | text = extract_delimited_math(text, math_config, info, replacement_manager)
100 |
101 | text = post_process_headings(text)
102 |
103 | lines = text.split("\n")
104 |
105 | if config['remove_chinese']:
106 | # Remove Chinese characters
107 | lines = remove_chinese_characters(lines)
108 |
109 | if config['boilerplate_config']['enable']:
110 | # Remove boilerplate
111 | lines = remove_boilerplate(lines, config['boilerplate_config'], replacement_manager)
112 |
113 | # Remove headings with nothing (or only other headings) after
114 | lines = remove_empty_headers(lines, replacement_manager)
115 |
116 | # Strip lines
117 | lines = [line.strip() for line in lines]
118 |
119 | # Create the final string
120 | text = "\n".join(lines)
121 |
122 | # Escape any dollar signs in the text
123 | text = text.replace("$", "\\$")
124 |
125 | # Now, add the dollar signs for math
126 | text = replace_math_tags_with_dollar_signs(text)
127 |
128 | if config['remove_edit_buttons']:
129 | # Remove edit buttons
130 | lines = text.split("\n")
131 | lines = remove_edit_buttons(lines)
132 | text = "\n".join(lines)
133 |
134 | # If there are over two newlines in a row, replace with two
135 | text = re.sub(r'\n{3,}', '\n\n', text)
136 |
137 | text = replacement_manager.remove_tags(text)
138 |
139 | text = text.strip()
140 |
141 | return text, info
--------------------------------------------------------------------------------
/text_extraction/text_extract/latex_processing.py:
--------------------------------------------------------------------------------
1 | from py_asciimath.translator.translator import ASCIIMath2Tex
2 | import logging
3 | from lxml import etree as ET
4 | import re
5 | import html
6 | import os
7 | import json
8 | from resiliparse.parse.html import traverse_dom
9 | import urllib3
10 | from urllib.parse import unquote
11 | logging.getLogger().setLevel(logging.ERROR)
12 |
13 | color_regex = re.compile(r'\\textcolor\[.*?\]\{.*?\}')
14 |
15 | asciimath2tex = ASCIIMath2Tex(log=False)
16 |
17 | PARAGRAPH_TAGS = frozenset({
18 | 'body', 'blockquote', 'caption', 'center', 'col', 'colgroup', 'dd',
19 | 'div', 'dl', 'dt', 'fieldset', 'form', 'legend', 'optgroup', 'option',
20 | 'p', 'pre', 'table', 'td', 'textarea', 'tfoot', 'th', 'thead', 'tr',
21 | 'ul', 'li', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
22 | })
23 |
24 | latex_math_commands = [
25 | "\\end", "\\begin", "\\ref", "\\frac", "\\label", "\\bf", "\\right", "\\left",
26 | "\\rm", "\\alpha", "\\mu", "\\def", "\\it", "\\pi", "\\sigma", "\\sum", "\\lambda",
27 | "\\beta", "\\nu", "\\partial", "\\int", "\\delta", "\\rho", "\\phi", "\\gamma",
28 | "\\omega", "\\over", "\\nonumber", "\\bar", "\\sqrt", "\\theta", "\\tau", "\\em",
29 | "\\rangle", "\\hat", "\\tilde", "\\cal", "\\hline", "\\item", "\\psi", "\\vec",
30 | "\\langle", "\\epsilon", "\\eta", "\\cdot", "\\in", "\\xi", "\\infty", "\\quad",
31 | "\\mathcal", "\\times", "\\emph", "\\mathbf", "\\prime", "\\be", "\\mathrm", "\\ee",
32 | "\\vspace", "\\pm", "\\chi", "\\ell", "\\text", "\\qquad", "\\noindent", "\\to",
33 | "\\varphi", "\\hspace", "\\leq", "\\cos", "\\eqref", "\\overline", "\\sin", "\\kappa",
34 | "\\hbox", "\\rightarrow", "\\varepsilon", "\\textit", "\\dagger", "\\big", "\\otimes",
35 | "\\equiv", "\\zeta", "\\dot", "\\ln"
36 | ]
37 |
38 | latex_image_class_names = [
39 | 'latexcenter',
40 | 'latex',
41 | 'tex',
42 | 'latexdisplay',
43 | 'latexblock',
44 | 'latexblockcenter',
45 | ]
46 |
47 |
48 | latex_math_commands = [re.escape(term) for term in latex_math_commands]
49 | latex_math_commands = [x + '(?![a-zA-Z])' for x in latex_math_commands]
50 | latex_regex = re.compile('|'.join(latex_math_commands))
51 |
52 | def extract_asciimath(s):
53 | parsed = asciimath2tex.translate(s)
54 | return parsed
55 |
56 | cur_file = os.path.abspath(__file__)
57 | xsl_path = os.path.join(os.path.dirname(cur_file), 'mmltex/mmltex.xsl')
58 |
59 | xslt = ET.parse(xsl_path)
60 | transform = ET.XSLT(xslt)
61 |
62 | def mml_to_latex(mml_code):
63 | # Remove any attibutes from the math tag
64 | mml_code = re.sub(r'()', r'\1', mml_code)
65 | mml_ns = mml_code.replace('