├── .clang-format
├── .gitignore
├── .gitmodules
├── .travis.yml
├── CHANGES.md
├── CONTRIBUTING.md
├── COPYING
├── DEBUGGING.md
├── Doxyfile
├── Makefile.am
├── README.md
├── THANKS
├── appveyor.yml
├── autogen.sh
├── benchmarks
    ├── arabic_newspapers.html
    ├── baidu.html
    ├── bbc.html
    ├── benchmark.cc
    ├── google.html
    ├── hacker_news.html
    ├── html5_spec.html
    ├── wikipedia.html
    ├── xinhua.html
    └── yahoo.html
├── configure.ac
├── examples
    ├── clean_text.cc
    ├── find_links.cc
    ├── get_title.c
    ├── positions_of_class.cc
    ├── prettyprint.cc
    └── serialize.cc
├── genperf.py
├── gentags.py
├── gtest.gyp
├── gumbo.pc.in
├── gumbo_parser.gyp
├── original-README.md
├── python
    └── gumbo
    │   ├── __init__.py
    │   ├── gumboc.py
    │   ├── gumboc_tags.py
    │   ├── gumboc_test.py
    │   ├── html5lib_adapter.py
    │   ├── html5lib_adapter_test.py
    │   ├── soup_adapter.py
    │   └── soup_adapter_test.py
├── setup.py
├── src
    ├── attribute.c
    ├── attribute.h
    ├── char_ref.c
    ├── char_ref.h
    ├── char_ref.rl
    ├── error.c
    ├── error.h
    ├── gumbo.h
    ├── insertion_mode.h
    ├── parser.c
    ├── parser.h
    ├── string_buffer.c
    ├── string_buffer.h
    ├── string_piece.c
    ├── string_piece.h
    ├── tag.c
    ├── tag.in
    ├── tag_enum.h
    ├── tag_gperf.h
    ├── tag_sizes.h
    ├── tag_strings.h
    ├── token_type.h
    ├── tokenizer.c
    ├── tokenizer.h
    ├── tokenizer_states.h
    ├── utf8.c
    ├── utf8.h
    ├── util.c
    ├── util.h
    ├── vector.c
    └── vector.h
├── tests
    ├── attribute.cc
    ├── char_ref.cc
    ├── parser.cc
    ├── string_buffer.cc
    ├── string_piece.cc
    ├── test_utils.cc
    ├── test_utils.h
    ├── tokenizer.cc
    ├── utf8.cc
    └── vector.cc
└── visualc
    ├── gumbo.vcxproj
    ├── gumbo.vcxproj.filters
    ├── gumbo.vcxproj.user
    └── include
        └── strings.h


/.clang-format:
--------------------------------------------------------------------------------
 1 | ---
 2 | Language:        Cpp
 3 | # BasedOnStyle:  Google
 4 | AccessModifierOffset: -1
 5 | AlignAfterOpenBracket: false
 6 | AlignEscapedNewlinesLeft: true
 7 | AlignOperands:   true
 8 | AlignTrailingComments: true
 9 | AllowAllParametersOfDeclarationOnNextLine: true
10 | AllowShortBlocksOnASingleLine: false
11 | AllowShortCaseLabelsOnASingleLine: false
12 | AllowShortIfStatementsOnASingleLine: true
13 | AllowShortLoopsOnASingleLine: true
14 | AllowShortFunctionsOnASingleLine: All
15 | AlwaysBreakAfterDefinitionReturnType: false
16 | AlwaysBreakTemplateDeclarations: true
17 | AlwaysBreakBeforeMultilineStrings: true
18 | BreakBeforeBinaryOperators: None
19 | BreakBeforeTernaryOperators: true
20 | BreakConstructorInitializersBeforeComma: false
21 | BinPackParameters: true
22 | BinPackArguments: true
23 | ColumnLimit:     80
24 | ConstructorInitializerAllOnOneLineOrOnePerLine: true
25 | ConstructorInitializerIndentWidth: 4
26 | DerivePointerAlignment: true
27 | ExperimentalAutoDetectBinPacking: false
28 | IndentCaseLabels: true
29 | IndentWrappedFunctionNames: false
30 | IndentFunctionDeclarationAfterType: false
31 | MaxEmptyLinesToKeep: 1
32 | KeepEmptyLinesAtTheStartOfBlocks: false
33 | NamespaceIndentation: None
34 | ObjCBlockIndentWidth: 2
35 | ObjCSpaceAfterProperty: false
36 | ObjCSpaceBeforeProtocolList: false
37 | PenaltyBreakBeforeFirstCallParameter: 1
38 | PenaltyBreakComment: 300
39 | PenaltyBreakString: 1000
40 | PenaltyBreakFirstLessLess: 120
41 | PenaltyExcessCharacter: 1000000
42 | PenaltyReturnTypeOnItsOwnLine: 200
43 | PointerAlignment: Left
44 | SpacesBeforeTrailingComments: 2
45 | Cpp11BracedListStyle: true
46 | Standard:        Auto
47 | IndentWidth:     2
48 | TabWidth:        8
49 | UseTab:          Never
50 | BreakBeforeBraces: Attach
51 | SpacesInParentheses: false
52 | SpacesInSquareBrackets: false
53 | SpacesInAngles:  false
54 | SpaceInEmptyParentheses: false
55 | SpacesInCStyleCastParentheses: false
56 | SpaceAfterCStyleCast: true
57 | SpacesInContainerLiterals: true
58 | SpaceBeforeAssignmentOperators: true
59 | ContinuationIndentWidth: 4
60 | CommentPragmas:  '^ IWYU pragma:'
61 | ForEachMacros:   [ foreach, Q_FOREACH, BOOST_FOREACH ]
62 | SpaceBeforeParens: ControlStatements
63 | DisableFormat:   false
64 | ...
65 | 
66 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # Compilation artifacts
 2 | *.o
 3 | *.lo
 4 | *.la
 5 | 
 6 | # Editor swap files
 7 | *.swp
 8 | *.swo
 9 | *.swn
10 | 
11 | #emacs editor leftovers
12 | *.*~
13 | 
14 | #diff leftovers
15 | *.orig
16 | 
17 | # gtest pieces
18 | gtest
19 | gtest-1.7.0
20 | 
21 | # Other build artifacts
22 | /Debug
23 | /visualc/Debug
24 | /visualc/Release
25 | /visualc/gumbo.sdf
26 | /visualc/gumbo.opensdf
27 | /build
28 | .log
29 | .sdf
30 | .opensdf
31 | .deps
32 | .dirstamp
33 | .libs
34 | Makefile
35 | Makefile.in
36 | aclocal.m4
37 | autom4te.cache
38 | compile
39 | config.guess
40 | config.log
41 | config.status
42 | config.sub
43 | configure
44 | depcomp
45 | gumbo.pc
46 | gumbo_test
47 | gumbo_test.log
48 | gumbo_test.trs
49 | install-sh
50 | libtool
51 | ltmain.sh
52 | m4/
53 | missing
54 | test-driver
55 | test-suite.log
56 | 
57 | # gyp android artifacts
58 | gumbo_parser.target.mk
59 | 
60 | # `make dist` artifacts
61 | /gumbo-[0-9].[0-9].tar.gz
62 | /gumbo-[0-9].[0-9]/
63 | 
64 | # Python dist artifacts
65 | *.pyc
66 | *.dylib
67 | dist
68 | build
69 | python/gumbo.egg-info
70 | python/gumbo/libgumbo.so
71 | 
72 | # Example binaries
73 | benchmark
74 | clean_text
75 | find_links
76 | get_title
77 | positions_of_class
78 | prettyprint
79 | serialize
80 | 


--------------------------------------------------------------------------------
/.gitmodules:
--------------------------------------------------------------------------------
1 | [submodule "third_party/gtest"]
2 | 	path = third_party/gtest
3 | 	url = https://chromium.googlesource.com/external/googletest/
4 | [submodule "testdata"]
5 | 	path = testdata
6 | 	url = https://github.com/html5lib/html5lib-tests.git
7 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | language: c++
 2 | 
 3 | compiler:
 4 |   - gcc
 5 |   - clang
 6 | 
 7 | os:
 8 |   - linux
 9 |   - osx
10 | 
11 | install:
12 |   - wget 'https://googletest.googlecode.com/files/gtest-1.7.0.zip'
13 |   - unzip gtest-1.7.0.zip
14 |   - ln -s gtest-1.7.0 gtest
15 |   - sudo pip install BeautifulSoup
16 |   - sudo pip install html5lib==0.95
17 | 
18 | script:
19 |   - ./autogen.sh && ./configure && make && make check
20 |   - python python/gumbo/gumboc_test.py
21 |   - python python/gumbo/html5lib_adapter_test.py
22 |   - python python/gumbo/soup_adapter_test.py
23 |   - sudo make install
24 |   - g++ examples/clean_text.cc `pkg-config --cflags --libs gumbo`
25 |   - sudo python setup.py sdist install
26 |   - python -c 'import gumbo; gumbo.parse("Foo")'
27 | 


--------------------------------------------------------------------------------
/CHANGES.md:
--------------------------------------------------------------------------------
 1 | ## Gumbo 0.10.1 (2015-04-30)
 2 | 
 3 | Same as 0.10.0, but with the version number bumped because the last version-number commit to v0.9.4 makes GitHub think that v0.9.4 is the latest version and so it's not highlighted on the webpage.
 4 | 
 5 | ## Gumbo 0.10.0 (2015-04-30)
 6 | 
 7 | * Full support for `<template>` tag (kevinhendricks, nostrademons).
 8 | * Some fixes for `<rtc>`/`<rt>` handling (kevinhendricks, vmg).
 9 | * All html5lib-trunk tests pass now! (kevinhendricks, vmg, nostrademons)
10 | * Support for fragment parsing (vmg)
11 | * A couple additional example programs (kevinhendricks)
12 | * Performance improvements totaling an estimated 30-40% total improvement (vmg, nostrademons).
13 | 
14 | ## Gumbo 0.9.4 (2015-04-30)
15 | 
16 | * Additional Visual Studio fixes (lowjoel, nostrademons)
17 | * Fixed some unused variable warnings.
18 | * Fix for glibtoolize vs. libtoolize build errors on Mac.
19 | * Fixed `CDATA` end tag handling.
20 | 
21 | ## Gumbo 0.9.3 (2015-02-17)
22 | 
23 | * Bugfix for `&AElig;` entities (rgrove)
24 | * Fix `CDATA` handling; `CDATA` sections now generate a `GUMBO_NODE_CDATA` node rather
25 | than plain text.
26 | * Fix `get_title example` to handle whitespace nodes (gsnedders)
27 | * Visual Studio compilation fixes (fishioon)
28 | * Take the namespace into account when determining whether a node matches a
29 | certain tag (aroben)
30 | * Replace the varargs tag functions with a tagset bytevector, for a 20-30%
31 | speedup in overall parse time (kevinhendricks, vmg)
32 | * Add MacOS X support to Travis CI, and fix the deployment/DLL issues this
33 | uncovered (nostrademons, kevinhendricks, vmg)
34 | 
35 | ## Gumbo 0.9.2 (2014-09-21)
36 | 
37 | * Performance improvements: Ragel-based char ref decoder and DFA-based UTF8
38 | decoder, totaling speedups of up to 300%.
39 | * Added benchmarking program and some sample data.
40 | * Fixed a compiler error under Visual Studio.
41 | * Fix an error in the ctypes bindings that could lead to memory corruption in
42 | the Python bindings.
43 | * Fix duplicate attributes when parsing `<isindex>` tags.
44 | * Don't leave semicolons behind when consuming entity references (rgrove)
45 | * Internally rename some functions in preparation for an amalgamation file
46 | (jdeng)
47 | * Add proper cflags for gyp builds (skabbes)
48 | 
49 | ## Gumbo 0.9.1 (2014-08-07)
50 | 
51 | * First version listed on PyPi.
52 | * Autotools files excluded from GitHub and generated via autogen.sh. (endgame)
53 | * Numerous compiler warnings fixed. (bnoordhuis, craigbarnes)
54 | * Google security audit passed.
55 | * Gyp support (tfarina)
56 | * Naming convention for structs changed to avoid C reserved words.
57 | * Fix several integer and buffer overflows (Maxime2)
58 | * Some Visual Studio compiler support (bugparty)
59 | * Python3 compatibility for the ctypes bindings.
60 | 
61 | ## Gumbo 0.9.0 (2013-08-13)
62 | 
63 | * Initial release open-sourced by Google.
64 | 


--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
 1 | Contributing
 2 | ===========
 3 | Bug reports are very much welcome.  Please use GitHub's issue-tracking feature, as it makes it easier to keep track of bugs and makes it possible for other project watchers to view the existing issues.
 4 | 
 5 | Patches and pull requests are also welcome, but before accepting patches, I need you to sign the Google Contributor License Agreement:
 6 | 
 7 | https://developers.google.com/open-source/cla/individual
 8 | https://developers.google.com/open-source/cla/corporate
 9 | 
10 | (Electronic signatures are fine for individual contributors.)
11 | 
12 | If you're unwilling to do this, it would be most helpful if you could file bug reports that include detailed prose about where in the code the error is and how to fix it, but leave out exact source code.
13 | 
14 | Project priorities
15 | ==================
16 | 
17 | Gumbo's priorities are, in rough order:
18 | 
19 | 1. Conformance to the HTML5 spec
20 | 2. Security & stability
21 | 3. Compatibility, both with previous versions and with different platforms (Visual Studio, Linux, Mac, other language bindings)
22 | 4. API simplicity
23 | 5. Performance
24 | 6. Features
25 | 
26 | Patches are much more likely to be accepted if they don't jeopardize values higher in the list for the sake of ones lower in the list.  So, we will happily take performance improvements that we can get for free, but not at the expense of complicating the API or reducing conformance.  We take patches to improve simplicity, but not at the expense of backwards compatibility.  We take new features only if they don't jeopardize any of the other traits (and are often quite conservative with them because of the backwards-compatibility and simplicity guarantees).
27 | 
28 | If you have a need for additional features beyond Gumbo's basic API, one option is to wrap Gumbo with another library, translating its data structures into ones more appropriate for your own use-case and then throwing away the original parse tree.  Gumbo was built for this; it's why most of the data structures are simple structs and the parse tree is intended to be immutable.  Tree traversal overhead is measured as negligible (~2-3%) compared to parsing time.  See eg. gumbo-libxml or gumbo-query for examples.
29 | 
30 | Code hygiene
31 | ============
32 | 
33 | We accept bugfixes even without this, but here are things you can do to make our lives as maintainers easier:
34 | 
35 | 1. Write unit tests.  When you discover a bug, write a test case that exposes the bug first, and then fix the bug.  We use both Travis CI (Mac/Linux) and AppVeyor (Visual Studio) for continuous integration, and both automatically run against pull requests; this makes it much easier to verify that the patch is correct.
36 | 2. Break big changes up into smaller, atomic pull requests, each of which fixes one bug or adds one feature.  In particular, separate out performance improvements from bug/correctness fixes from new features, and separate out pull requests that need to go into different branches.
37 | 3. Work off the correct branch, and submit the pull request against it.  We follow Semantic Versioning, which means that we bump the patch number for internal-only bugfixes and performance improvements, the minor number for backwards-compatible API changes, and the major number for backwards-incompatible API changes.  There are branches for all of these.
38 | 4. For performance improvements, run benchmarks before and after, and include the numbers in the commit message.  There is a benchmarking program at ./benchmark that runs the parser against some common webpages.
39 | 5. Run clang-format after making your changes to make sure your code conforms to the existing style.  There is a .clang-format file already in the codebase; run it with 'clang-format -i src/*.{h,c}'.
40 | 


--------------------------------------------------------------------------------
/DEBUGGING.md:
--------------------------------------------------------------------------------
  1 | These are a couple of debugging notes that may be helpful for anyone developing
  2 | Gumbo or trying to diagnose a tricky problem.  They will probably not be
  3 | necessary for normal clients of this library - Gumbo is relatively stable, and
  4 | bugs are often rare and obscure.  However, they're handy to have as a reference,
  5 | and may also provide useful Google fodder to people searching for these tools.
  6 | 
  7 | Standard disclaimer: I use all of these techniques on my Ubuntu 14.04 computer
  8 | with gcc 4.8.2, clang 3.4, and gtest 1.6.0, but make no warranty about them
  9 | working on other systems.  In particular, they're almost certain not to work on
 10 | Windows.
 11 | 
 12 | Debug output
 13 | ============
 14 | 
 15 | Gumbo has a compile-time switch to dump lots of debug output onto stdout.
 16 | Compile with the GUMBO_DEBUG define enabled:
 17 | 
 18 | ```bash
 19 | $ make CFLAGS='-DGUMBO_DEBUG'
 20 | ```
 21 | 
 22 | Note that this spits *a lot* of debug information to the console and makes the
 23 | program run significantly slower, so it's usually helpful to isolate only the
 24 | specific HTML file or fragment that causes the bug.  It lets us trace the
 25 | operation of each of the tokenizer & parser's state machines in depth, though.
 26 | 
 27 | Unit tests
 28 | ==========
 29 | 
 30 | As mentioned in the README, Gumbo relies on [googletest][] for unit tests.
 31 | Unzip the gtest ZIP distribution inside the Gumbo root and rename it 'gtest'.
 32 | 'make check' runs the tests, as normal.
 33 | 
 34 | ```bash
 35 | $ make check
 36 | $ cat test-suite.log
 37 | ```
 38 | 
 39 | If you need to debug a core dump, you'll probably want to run the test binary
 40 | directly:
 41 | 
 42 | ```bash
 43 | $ ulimit -c unlimited
 44 | $ make check
 45 | $ .libs/lt-gumbo_test
 46 | $ gdb .libs/lt-gumbo_test core
 47 | ```
 48 | 
 49 | The same goes for core dumps in other example binaries.
 50 | 
 51 | To run only a single unit test, pass the --gtest_filter='TestName' flag to the
 52 | lt-gumbo_test binary.
 53 | 
 54 | Assertions
 55 | ==========
 56 | 
 57 | Gumbo relies pretty heavily on assertions.  By default they're enabled at
 58 | run-time: to turn them off, define NDEBUG:
 59 | 
 60 | ```bash
 61 | $ make CFLAGS='-DNDEBUG'
 62 | ```
 63 | 
 64 | ASAN
 65 | ====
 66 | 
 67 | Google's [address-sanitizer][] is a helpful tool that lets you find memory
 68 | errors with relatively low overhead: enough that you can often run it in
 69 | production.  Enabling it for C/C++ binaries is pretty standard and described on
 70 | the ASAN documentation pages.  It requires Clang >=3.1 or GCC >= 4.8.
 71 | 
 72 | ```bash
 73 | $ make \
 74 |     CFLAGS='-fsanitize=address -fno-omit-frame-pointer -fno-inline' \
 75 |     LDFLAGS='-fsanitize=address'
 76 | ```
 77 | 
 78 | ASAN can also be used when Gumbo is compiled as a shared library and linked into
 79 | a scripting language via FFI, but this use-case is unsupported by the ASAN
 80 | authors.  To do it, use LD_PRELOAD to ensure the ASAN runtime support is
 81 | included in the process:
 82 | 
 83 | ```bash
 84 | $ LD_PRELOAD=libasan.so.0 python -c 'import gumbo; gumbo.parse(problem_text)'
 85 | ```
 86 | 
 87 | Getting clean stack traces from this requires the use of the llvm-symbolizer
 88 | binary, included with clang:
 89 | 
 90 | ```bash
 91 | $ export ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer-3.4
 92 | $ export ASAN_OPTIONS=symbolize=1
 93 | $ LD_PRELOAD=libasan.so.0 python -c \
 94 |   'import gumbo; gumbo.parse(problem_text)' 2>&1 | head -100
 95 | $ killall llvm-symbolizer-3.4
 96 | $ killall llvm-symbolizer-3.4
 97 | $ killall llvm-symbolizer-3.4
 98 | ```
 99 | 
100 | This use case is even less officially supported than using it with dynamic
101 | shared objects; on my machine, it led to a recursive ASAN error about a
102 | use-after-free in llvm-symbolizer, effectively fork-bombing the machine.  Have
103 | the killalls ready, and avoid letting the process run for too long (eg. piping
104 | it to 'less').
105 | 
106 | [googletest]: https://code.google.com/p/googletest/
107 | [address-sanitizer]: https://code.google.com/p/address-sanitizer/
108 | 


--------------------------------------------------------------------------------
/Makefile.am:
--------------------------------------------------------------------------------
  1 | # Googletest magic.  Gumbo relies on Googletest for its unittests, and
  2 | # Googletest (by design) does not provide a "make install" option.  Instead,
  3 | # we'll assume you copy (or symlink) the Googletest distribution into a 'gtest'
  4 | # directory inside the main library directory, and then provide rules to build
  5 | # it automatically.  This approach (and these rules) are copied from the
  6 | # protobuf distribution.
  7 | 
  8 | ACLOCAL_AMFLAGS = -I m4
  9 | 
 10 | if !HAVE_SHARED_LIBGTEST
 11 | 
 12 | # Build gtest before we build protobuf tests.  We don't add gtest to SUBDIRS
 13 | # because then "make check" would also build and run all of gtest's own tests,
 14 | # which takes a lot of time and is generally not useful to us.  Also, we don't
 15 | # want "make install" to recurse into gtest since we don't want to overwrite
 16 | # the installed version of gtest if there is one.
 17 | check-local: gtest/lib/libgtest.la gtest/lib/libgtest_main.la
 18 | 
 19 | gtest/lib/libgtest.la gtest/lib/libgtest_main.la: gtest/Makefile
 20 | 	@echo "Making lib/libgtest.a lib/libgtest_main.a in gtest"
 21 | 	@cd gtest && $(MAKE) $(AM_MAKEFLAGS) lib/libgtest.la lib/libgtest_main.la
 22 | 
 23 | gtest/Makefile: gtest/configure
 24 | 	@cd gtest && ./configure
 25 | 
 26 | # We would like to clean gtest when "make clean" is invoked.  But we have to
 27 | # be careful because clean-local is also invoked during "make distclean", but
 28 | # "make distclean" already recurses into gtest because it's listed among the
 29 | # DIST_SUBDIRS.  distclean will delete gtest/Makefile, so if we then try to
 30 | # cd to the directory again and "make clean" it will fail.  So, check that the
 31 | # Makefile exists before recursing.
 32 | clean-local:
 33 | 	@if test -e gtest/Makefile; then \
 34 | 		echo "Making clean in gtest"; \
 35 | 		cd gtest && $(MAKE) $(AM_MAKEFLAGS) clean; \
 36 | 	fi
 37 | 
 38 | endif !HAVE_SHARED_LIBGTEST
 39 | 
 40 | gentags: src/tag.in
 41 | 	@python gentags.py $<
 42 | 	@gperf -LANSI-C --ignore-case -m200 $< |python genperf.py >src/tag_gperf.h
 43 | 
 44 | lib_LTLIBRARIES = libgumbo.la
 45 | libgumbo_la_CFLAGS = -Wall
 46 | libgumbo_la_LDFLAGS = -version-info 1:0:0 -no-undefined
 47 | libgumbo_la_SOURCES = \
 48 | 				src/attribute.c \
 49 | 				src/attribute.h \
 50 | 				src/char_ref.c \
 51 | 				src/char_ref.h \
 52 | 				src/error.c \
 53 | 				src/error.h \
 54 | 				src/insertion_mode.h \
 55 | 				src/parser.c \
 56 | 				src/parser.h \
 57 | 				src/string_buffer.c \
 58 | 				src/string_buffer.h \
 59 | 				src/string_piece.c \
 60 | 				src/string_piece.h \
 61 | 				src/tag.c \
 62 | 				src/tag_enum.h \
 63 | 				src/tag_gperf.h \
 64 | 				src/tag_strings.h \
 65 | 				src/tag_sizes.h \
 66 | 				src/token_type.h \
 67 | 				src/tokenizer.c \
 68 | 				src/tokenizer.h \
 69 | 				src/tokenizer_states.h \
 70 | 				src/utf8.c \
 71 | 				src/utf8.h \
 72 | 				src/util.c \
 73 | 				src/util.h \
 74 | 				src/vector.c \
 75 | 				src/vector.h
 76 | include_HEADERS = src/gumbo.h src/tag_enum.h
 77 | 
 78 | pkgconfigdir = $(libdir)/pkgconfig
 79 | pkgconfig_DATA = gumbo.pc
 80 | 
 81 | check_PROGRAMS = gumbo_test
 82 | TESTS = gumbo_test
 83 | gumbo_test_CPPFLAGS = \
 84 | 				-I"$(srcdir)/." \
 85 | 				-I"$(srcdir)/src" \
 86 | 				-I"$(srcdir)/gtest/include"
 87 | gumbo_test_SOURCES = \
 88 | 				tests/attribute.cc \
 89 | 				tests/char_ref.cc \
 90 | 				tests/parser.cc \
 91 | 				tests/string_buffer.cc \
 92 | 				tests/string_piece.cc \
 93 | 				tests/tokenizer.cc \
 94 | 				tests/test_utils.cc \
 95 | 				tests/utf8.cc \
 96 | 				tests/vector.cc
 97 | gumbo_test_DEPENDENCIES = libgumbo.la
 98 | gumbo_test_LDADD = libgumbo.la
 99 | 
100 | if HAVE_SHARED_LIBGTEST
101 | # FIXME(bnoordhuis) Should be configurable by the user.
102 | gumbo_test_LDADD += -lgtest -lgtest_main
103 | else
104 | gumbo_test_DEPENDENCIES += check-local
105 | gumbo_test_LDADD += gtest/lib/libgtest.la gtest/lib/libgtest_main.la
106 | endif
107 | 
108 | noinst_PROGRAMS = clean_text find_links get_title positions_of_class benchmark serialize prettyprint
109 | LDADD = libgumbo.la
110 | AM_CPPFLAGS = -I"$(srcdir)/src"
111 | 
112 | clean_text_SOURCES = examples/clean_text.cc
113 | find_links_SOURCES = examples/find_links.cc
114 | get_title_SOURCES = examples/get_title.c
115 | positions_of_class_SOURCES = examples/positions_of_class.cc
116 | benchmark_SOURCES = benchmarks/benchmark.cc
117 | serialize_SOURCES = examples/serialize.cc
118 | prettyprint_SOURCES = examples/prettyprint.cc
119 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | Gumbo - A pure-C HTML5 parser.
2 | ============
3 | 
4 | This project has been unmaintained since 2016 and should not be used.
5 | 
6 | The [original README](original-README.md) is available for historical reference.


--------------------------------------------------------------------------------
/THANKS:
--------------------------------------------------------------------------------
 1 | Gumbo HTML parser THANKS file
 2 | 
 3 | Gumbo was originally written by Jonathan Tang, but many people helped out through suggestions, question-answering, code reviews, bugfixes, and organizational support.  Here is a list of these people.  Help me keep it complete and exempt of errors.
 4 | 
 5 | Adam Barth
 6 | Adam Roben
 7 | Ben Noordhuis
 8 | Bowen Han
 9 | Constantinos Michael
10 | Craig Barnes
11 | Geoffrey Sneddon
12 | Ian Hickson
13 | Jack Deng
14 | Joel Low
15 | Jonathan Shneier
16 | Kevin Hendricks
17 | Mason Tang
18 | Maxim Zakharov
19 | Michal Zalewski
20 | Neal Norwitz
21 | Othar Hansson
22 | Ryan Grove
23 | Stefan Haustein
24 | Steffen Meschkat
25 | Steven Kabbes
26 | Thiago Farina
27 | Vicent Marti
28 | 


--------------------------------------------------------------------------------
/appveyor.yml:
--------------------------------------------------------------------------------
1 | version: 1.0.{build}
2 | build:
3 |   project: visualc/gumbo.vcxproj
4 |   verbosity: minimal
5 | 


--------------------------------------------------------------------------------
/autogen.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/sh
 2 | 
 3 | # Copyright (c) 2013, Ben Noordhuis <info@bnoordhuis.nl>
 4 | #
 5 | # Permission to use, copy, modify, and/or distribute this software for any
 6 | # purpose with or without fee is hereby granted, provided that the above
 7 | # copyright notice and this permission notice appear in all copies.
 8 | #
 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 | # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 | # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 | # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 | # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 | # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 | # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16 | 
17 | if test -z "$LIBTOOLIZE" -a "`uname`" = "Darwin"; then
18 | 	if command -v "glibtoolize" >/dev/null; then
19 | 		LIBTOOLIZE=glibtoolize
20 | 	elif command -v "libtoolize" >/dev/null; then
21 | 		LIBTOOLIZE=libtoolize
22 | 	else
23 | 		echo "autogen.sh: line $LINENO: command glibtoolize or libtoolize not found"
24 | 		exit 1
25 | 	fi
26 | fi
27 | 
28 | set -ex
29 | ${LIBTOOLIZE:-libtoolize}
30 | ${ACLOCAL:-aclocal -I m4}
31 | ${AUTOCONF:-autoconf}
32 | ${AUTOMAKE:-automake} --add-missing
33 | 


--------------------------------------------------------------------------------
/benchmarks/arabic_newspapers.html:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/google/gumbo-parser/a0f9059087c690e700cf42df4cfa58a63c1dae95/benchmarks/arabic_newspapers.html


--------------------------------------------------------------------------------
/benchmarks/benchmark.cc:
--------------------------------------------------------------------------------
 1 | // Copyright 2013 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | //
15 | // Author: jdtang@google.com (Jonathan Tang)
16 | //
17 | 
18 | #include <dirent.h>
19 | #include <fstream>
20 | #include <iostream>
21 | #include <stdint.h>
22 | #include <stdlib.h>
23 | #include <string>
24 | #include <time.h>
25 | 
26 | #include "gumbo.h"
27 | 
28 | static const int kNumReps = 10;
29 | 
30 | int main(int argc, char** argv) {
31 |   if (argc != 1) {
32 |     std::cout << "Usage: benchmarks\n";
33 |     exit(EXIT_FAILURE);
34 |   }
35 | 
36 |   DIR* dir;
37 |   struct dirent* file;
38 | 
39 |   if ((dir = opendir("benchmarks")) == NULL) {
40 |     std::cout << "Couldn't find 'benchmarks' directory.  "
41 |               << "Run from root of distribution.\n";
42 |     exit(EXIT_FAILURE);
43 |   }
44 | 
45 |   while ((file = readdir(dir)) != NULL) {
46 |     std::string filename(file->d_name);
47 |     if (filename.length() > 5 && filename.compare(filename.length() - 5, 5, ".html") == 0) {
48 |       std::string full_filename = "benchmarks/" + filename;
49 |       std::ifstream in(full_filename.c_str(), std::ios::in | std::ios::binary);
50 |       if (!in) {
51 |         std::cout << "File " << full_filename << " couldn't be read!\n";
52 |         exit(EXIT_FAILURE);
53 |       }
54 | 
55 |       std::string contents;
56 |       in.seekg(0, std::ios::end);
57 |       contents.resize(in.tellg());
58 |       in.seekg(0, std::ios::beg);
59 |       in.read(&contents[0], contents.size());
60 |       in.close();
61 | 
62 |       clock_t start_time = clock();
63 |       for (int i = 0; i < kNumReps; ++i) {
64 |         GumboOutput* output = gumbo_parse(contents.c_str());
65 |         gumbo_destroy_output(&kGumboDefaultOptions, output);
66 |       }
67 |       clock_t end_time = clock();
68 |       std::cout << filename << ": "
69 |           << (1000000 * (end_time - start_time) / (kNumReps * CLOCKS_PER_SEC))
70 |           << " microseconds.\n";
71 |     }
72 |   }
73 |   closedir(dir);
74 | }
75 | 


--------------------------------------------------------------------------------
/configure.ac:
--------------------------------------------------------------------------------
 1 | #                                               -*- Autoconf -*-
 2 | # Process this file with autoconf to produce a configure script.
 3 | 
 4 | AC_PREREQ([2.65])
 5 | AC_INIT([gumbo], [0.10.1], [jonathan.d.tang@gmail.com])
 6 | AC_CONFIG_MACRO_DIR([m4])
 7 | AC_CONFIG_SRCDIR([src/parser.c])
 8 | #AC_CONFIG_HEADERS([config.h])
 9 | AC_CONFIG_FILES([Makefile gumbo.pc])
10 | 
11 | # Checks for programs.
12 | AC_PROG_CXX
13 | AC_PROG_CC_C99
14 | 
15 | # Checks for libraries.
16 | 
17 | # Checks for header files.
18 | AC_CHECK_HEADERS([stddef.h stdlib.h string.h strings.h])
19 | 
20 | # Checks for typedefs, structures, and compiler characteristics.
21 | AC_C_INLINE
22 | AC_TYPE_SIZE_T
23 | 
24 | # Checks for library functions.
25 | AC_CHECK_LIB([gtest_main],
26 |              [main],
27 |              AM_CONDITIONAL(HAVE_SHARED_LIBGTEST, [true]),
28 |              AM_CONDITIONAL(HAVE_SHARED_LIBGTEST, [false]))
29 | 
30 | # Init Automake & libtool
31 | AM_INIT_AUTOMAKE([foreign subdir-objects])
32 | m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
33 | LT_INIT
34 | 
35 | AC_OUTPUT
36 | 


--------------------------------------------------------------------------------
/examples/clean_text.cc:
--------------------------------------------------------------------------------
 1 | // Copyright 2013 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | //
15 | // Author: jdtang@google.com (Jonathan Tang)
16 | //
17 | // Gets the cleantext of a page.
18 | 
19 | #include <fstream>
20 | #include <iostream>
21 | #include <stdlib.h>
22 | #include <string>
23 | 
24 | #include "gumbo.h"
25 | 
26 | static std::string cleantext(GumboNode* node) {
27 |   if (node->type == GUMBO_NODE_TEXT) {
28 |     return std::string(node->v.text.text);
29 |   } else if (node->type == GUMBO_NODE_ELEMENT &&
30 |              node->v.element.tag != GUMBO_TAG_SCRIPT &&
31 |              node->v.element.tag != GUMBO_TAG_STYLE) {
32 |     std::string contents = "";
33 |     GumboVector* children = &node->v.element.children;
34 |     for (unsigned int i = 0; i < children->length; ++i) {
35 |       const std::string text = cleantext((GumboNode*) children->data[i]);
36 |       if (i != 0 && !text.empty()) {
37 |         contents.append(" ");
38 |       }
39 |       contents.append(text);
40 |     }
41 |     return contents;
42 |   } else {
43 |     return "";
44 |   }
45 | }
46 | 
47 | int main(int argc, char** argv) {
48 |   if (argc != 2) {
49 |     std::cout << "Usage: clean_text <html filename>\n";
50 |     exit(EXIT_FAILURE);
51 |   }
52 |   const char* filename = argv[1];
53 | 
54 |   std::ifstream in(filename, std::ios::in | std::ios::binary);
55 |   if (!in) {
56 |     std::cout << "File " << filename << " not found!\n";
57 |     exit(EXIT_FAILURE);
58 |   }
59 | 
60 |   std::string contents;
61 |   in.seekg(0, std::ios::end);
62 |   contents.resize(in.tellg());
63 |   in.seekg(0, std::ios::beg);
64 |   in.read(&contents[0], contents.size());
65 |   in.close();
66 | 
67 |   GumboOutput* output = gumbo_parse(contents.c_str());
68 |   std::cout << cleantext(output->root) << std::endl;
69 |   gumbo_destroy_output(&kGumboDefaultOptions, output);
70 | }
71 | 


--------------------------------------------------------------------------------
/examples/find_links.cc:
--------------------------------------------------------------------------------
 1 | // Copyright 2013 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | //
15 | // Author: jdtang@google.com (Jonathan Tang)
16 | //
17 | // Finds the URLs of all links in the page.
18 | 
19 | #include <stdlib.h>
20 | 
21 | #include <fstream>
22 | #include <iostream>
23 | #include <string>
24 | 
25 | #include "gumbo.h"
26 | 
27 | static void search_for_links(GumboNode* node) {
28 |   if (node->type != GUMBO_NODE_ELEMENT) {
29 |     return;
30 |   }
31 |   GumboAttribute* href;
32 |   if (node->v.element.tag == GUMBO_TAG_A &&
33 |       (href = gumbo_get_attribute(&node->v.element.attributes, "href"))) {
34 |     std::cout << href->value << std::endl;
35 |   }
36 | 
37 |   GumboVector* children = &node->v.element.children;
38 |   for (unsigned int i = 0; i < children->length; ++i) {
39 |     search_for_links(static_cast<GumboNode*>(children->data[i]));
40 |   }
41 | }
42 | 
43 | int main(int argc, char** argv) {
44 |   if (argc != 2) {
45 |     std::cout << "Usage: find_links <html filename>.\n";
46 |     exit(EXIT_FAILURE);
47 |   }
48 |   const char* filename = argv[1];
49 | 
50 |   std::ifstream in(filename, std::ios::in | std::ios::binary);
51 |   if (!in) {
52 |     std::cout << "File " << filename << " not found!\n";
53 |     exit(EXIT_FAILURE);
54 |   }
55 | 
56 |   std::string contents;
57 |   in.seekg(0, std::ios::end);
58 |   contents.resize(in.tellg());
59 |   in.seekg(0, std::ios::beg);
60 |   in.read(&contents[0], contents.size());
61 |   in.close();
62 | 
63 |   GumboOutput* output = gumbo_parse(contents.c_str());
64 |   search_for_links(output->root);
65 |   gumbo_destroy_output(&kGumboDefaultOptions, output);
66 | }
67 | 


--------------------------------------------------------------------------------
/examples/get_title.c:
--------------------------------------------------------------------------------
 1 | // Copyright 2013 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | //
15 | // Author: jdtang@google.com (Jonathan Tang)
16 | //
17 | // Retrieves the title of a page.
18 | 
19 | #include <assert.h>
20 | #include <stdio.h>
21 | #include <stdlib.h>
22 | #include <sys/stat.h>
23 | 
24 | #include "gumbo.h"
25 | 
26 | static void read_file(FILE* fp, char** output, int* length) {
27 |   struct stat filestats;
28 |   int fd = fileno(fp);
29 |   fstat(fd, &filestats);
30 |   *length = filestats.st_size;
31 |   *output = malloc(*length + 1);
32 |   int start = 0;
33 |   int bytes_read;
34 |   while ((bytes_read = fread(*output + start, 1, *length - start, fp))) {
35 |     start += bytes_read;
36 |   }
37 | }
38 | 
39 | static const char* find_title(const GumboNode* root) {
40 |   assert(root->type == GUMBO_NODE_ELEMENT);
41 |   assert(root->v.element.children.length >= 2);
42 | 
43 |   const GumboVector* root_children = &root->v.element.children;
44 |   GumboNode* head = NULL;
45 |   for (int i = 0; i < root_children->length; ++i) {
46 |     GumboNode* child = root_children->data[i];
47 |     if (child->type == GUMBO_NODE_ELEMENT &&
48 |         child->v.element.tag == GUMBO_TAG_HEAD) {
49 |       head = child;
50 |       break;
51 |     }
52 |   }
53 |   assert(head != NULL);
54 | 
55 |   GumboVector* head_children = &head->v.element.children;
56 |   for (int i = 0; i < head_children->length; ++i) {
57 |     GumboNode* child = head_children->data[i];
58 |     if (child->type == GUMBO_NODE_ELEMENT &&
59 |         child->v.element.tag == GUMBO_TAG_TITLE) {
60 |       if (child->v.element.children.length != 1) {
61 |         return "<empty title>";
62 |       }
63 |       GumboNode* title_text = child->v.element.children.data[0];
64 |       assert(title_text->type == GUMBO_NODE_TEXT || title_text->type == GUMBO_NODE_WHITESPACE);
65 |       return title_text->v.text.text;
66 |     }
67 |   }
68 |   return "<no title found>";
69 | }
70 | 
71 | int main(int argc, const char** argv) {
72 |   if (argc != 2) {
73 |     printf("Usage: get_title <html filename>.\n");
74 |     exit(EXIT_FAILURE);
75 |   }
76 |   const char* filename = argv[1];
77 | 
78 |   FILE* fp = fopen(filename, "r");
79 |   if (!fp) {
80 |     printf("File %s not found!\n", filename);
81 |     exit(EXIT_FAILURE);
82 |   }
83 | 
84 |   char* input;
85 |   int input_length;
86 |   read_file(fp, &input, &input_length);
87 |   GumboOutput* output = gumbo_parse_with_options(
88 |       &kGumboDefaultOptions, input, input_length);
89 |   const char* title = find_title(output->root);
90 |   printf("%s\n", title);
91 |   gumbo_destroy_output(&kGumboDefaultOptions, output);
92 |   free(input);
93 | }
94 | 


--------------------------------------------------------------------------------
/examples/positions_of_class.cc:
--------------------------------------------------------------------------------
 1 | // Copyright 2013 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | //
15 | // Author: jdtang@google.com (Jonathan Tang)
16 | //
17 | // Print out the positions of all elements with a certain CSS class.
18 | 
19 | #include <stdlib.h>
20 | #include <string.h>
21 | 
22 | #include <algorithm>
23 | #include <fstream>
24 | #include <iostream>
25 | #include <string>
26 | 
27 | #include "gumbo.h"
28 | 
29 | static std::string find_line(
30 |     const std::string& original_text, const GumboAttribute& attr) {
31 |   size_t attr_index = attr.original_value.data - original_text.data();
32 |   size_t begin = original_text.rfind("\n", attr_index) + 1;
33 |   size_t end = original_text.find("\n", attr_index);
34 |   if (end != std::string::npos) {
35 |     end--;
36 |   } else {
37 |     end = (size_t) original_text.length() - 1;
38 |   }
39 |   end = std::min(end, attr_index + 40);
40 |   begin = std::max(begin, attr_index - 40);
41 |   return original_text.substr(begin, end - begin);
42 | }
43 | 
44 | static void search_for_class(
45 |     GumboNode* node, const std::string& original_text, const char* cls_name) {
46 |   if (node->type != GUMBO_NODE_ELEMENT) {
47 |     return;
48 |   }
49 |   GumboAttribute* cls_attr;
50 |   if ((cls_attr = gumbo_get_attribute(&node->v.element.attributes, "class")) &&
51 |       strstr(cls_attr->value, cls_name) != NULL) {
52 |     std::cout << cls_attr->value_start.line << ":"
53 |               << cls_attr->value_start.column << " - "
54 |               << find_line(original_text, *cls_attr) << std::endl;
55 |   }
56 | 
57 |   GumboVector* children = &node->v.element.children;
58 |   for (int i = 0; i < children->length; ++i) {
59 |     search_for_class(
60 |         static_cast<GumboNode*>(children->data[i]), original_text, cls_name);
61 |   }
62 | }
63 | 
64 | int main(int argc, char** argv) {
65 |   if (argc != 3) {
66 |     std::cout << "Usage: positions_of_class <html filename> <CSS classname>.\n";
67 |     exit(EXIT_FAILURE);
68 |   }
69 |   const char* filename = argv[1];
70 |   const char* cls = argv[2];
71 | 
72 |   std::ifstream in(filename, std::ios::in | std::ios::binary);
73 |   if (!in) {
74 |     std::cout << "File " << filename << " not found!\n";
75 |     exit(EXIT_FAILURE);
76 |   }
77 | 
78 |   std::string contents;
79 |   in.seekg(0, std::ios::end);
80 |   contents.resize(in.tellg());
81 |   in.seekg(0, std::ios::beg);
82 |   in.read(&contents[0], contents.size());
83 |   in.close();
84 | 
85 |   // If you used contents.c_str(), it'd be harder to match up original
86 |   // positions, because c_str() creates a copy of the string and you can't do
87 |   // pointer arithmetic betweent contents.data() and the original_* pointers.
88 |   GumboOutput* output = gumbo_parse_with_options(
89 |       &kGumboDefaultOptions, contents.data(), contents.length());
90 |   search_for_class(output->root, contents, cls);
91 |   gumbo_destroy_output(&kGumboDefaultOptions, output);
92 | }
93 | 


--------------------------------------------------------------------------------
/examples/serialize.cc:
--------------------------------------------------------------------------------
  1 | // Copyright 2015 Kevin B. Hendricks, Stratford, Ontario,  All Rights Reserved.
  2 | // loosely based on a greatly simplified version of BeautifulSoup4 decode() routine
  3 | //
  4 | // Licensed under the Apache License, Version 2.0 (the "License");
  5 | // you may not use this file except in compliance with the License.
  6 | // You may obtain a copy of the License at
  7 | //
  8 | //     http://www.apache.org/licenses/LICENSE-2.0
  9 | //
 10 | // Unless required by applicable law or agreed to in writing, software
 11 | // distributed under the License is distributed on an "AS IS" BASIS,
 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 | // See the License for the specific language governing permissions and
 14 | // limitations under the License.
 15 | //
 16 | // Author: Kevin Hendricks
 17 | //
 18 | // Serialize back to html / xhtml making as few changes as possible (even in whitespace)
 19 | 
 20 | #include <fstream>
 21 | #include <iostream>
 22 | #include <stdlib.h>
 23 | #include <string>
 24 | 
 25 | #include "gumbo.h"
 26 | 
 27 | static std::string nonbreaking_inline  = "|a|abbr|acronym|b|bdo|big|cite|code|dfn|em|font|i|img|kbd|nobr|s|small|span|strike|strong|sub|sup|tt|";
 28 | static std::string empty_tags          = "|area|base|basefont|bgsound|br|command|col|embed|event-source|frame|hr|image|img|input|keygen|link|menuitem|meta|param|source|spacer|track|wbr|";
 29 | static std::string preserve_whitespace = "|pre|textarea|script|style|";
 30 | static std::string special_handling    = "|html|body|";
 31 | static std::string no_entity_sub       = "|script|style|";
 32 | 
 33 | 
 34 | static inline void rtrim(std::string &s) 
 35 | {
 36 |   s.erase(s.find_last_not_of(" \n\r\t")+1);
 37 | }
 38 | 
 39 | 
 40 | static inline void ltrim(std::string &s)
 41 | {
 42 |   s.erase(0,s.find_first_not_of(" \n\r\t"));
 43 | }
 44 | 
 45 | 
 46 | static void replace_all(std::string &s, const char * s1, const char * s2)
 47 | {
 48 |   std::string t1(s1);
 49 |   size_t len = t1.length();
 50 |   size_t pos = s.find(t1);
 51 |   while (pos != std::string::npos) {
 52 |     s.replace(pos, len, s2);
 53 |     pos = s.find(t1, pos + len);
 54 |   }
 55 | }
 56 | 
 57 | 
 58 | static std::string substitute_xml_entities_into_text(const std::string &text)
 59 | {
 60 |   std::string result = text;
 61 |   // replacing & must come first 
 62 |   replace_all(result, "&", "&amp;");
 63 |   replace_all(result, "<", "&lt;");
 64 |   replace_all(result, ">", "&gt;");
 65 |   return result;
 66 | }
 67 | 
 68 | 
 69 | static std::string substitute_xml_entities_into_attributes(char quote, const std::string &text)
 70 | {
 71 |   std::string result = substitute_xml_entities_into_text(text);
 72 |   if (quote == '"') {
 73 |     replace_all(result,"\"","&quot;");
 74 |   }    
 75 |   else if (quote == '\'') {
 76 |     replace_all(result,"'","&apos;");
 77 |   }
 78 |  return result;
 79 | }
 80 | 
 81 | 
 82 | static std::string handle_unknown_tag(GumboStringPiece *text)
 83 | {
 84 |   std::string tagname = "";
 85 |   if (text->data == NULL) {
 86 |     return tagname;
 87 |   }
 88 |   // work with copy GumboStringPiece to prevent asserts 
 89 |   // if try to read same unknown tag name more than once
 90 |   GumboStringPiece gsp = *text;
 91 |   gumbo_tag_from_original_text(&gsp);
 92 |   tagname = std::string(gsp.data, gsp.length);
 93 |   return tagname; 
 94 | }
 95 | 
 96 | 
 97 | static std::string get_tag_name(GumboNode *node)
 98 | {
 99 |   std::string tagname;
100 |   // work around lack of proper name for document node
101 |   if (node->type == GUMBO_NODE_DOCUMENT) {
102 |     tagname = "document";
103 |   } else {
104 |     tagname = gumbo_normalized_tagname(node->v.element.tag);
105 |   }
106 |   if (tagname.empty()) {
107 |     tagname = handle_unknown_tag(&node->v.element.original_tag);
108 |   }
109 |   return tagname;
110 | }
111 | 
112 | 
113 | static std::string build_doctype(GumboNode *node)
114 | {
115 |   std::string results = "";
116 |   if (node->v.document.has_doctype) {
117 |     results.append("<!DOCTYPE ");
118 |     results.append(node->v.document.name);
119 |     std::string pi(node->v.document.public_identifier);
120 |     if ((node->v.document.public_identifier != NULL) && !pi.empty() ) {
121 |         results.append(" PUBLIC \"");
122 |         results.append(node->v.document.public_identifier);
123 |         results.append("\" \"");
124 |         results.append(node->v.document.system_identifier);
125 |         results.append("\"");
126 |     }
127 |     results.append(">\n");
128 |   }
129 |   return results;
130 | }
131 | 
132 | 
133 | static std::string build_attributes(GumboAttribute * at, bool no_entities)
134 | {
135 |   std::string atts = " ";
136 |   atts.append(at->name);
137 | 
138 |   // how do we want to handle attributes with empty values
139 |   // <input type="checkbox" checked />  or <input type="checkbox" checked="" /> 
140 | 
141 |   if ( (!std::string(at->value).empty())   || 
142 |        (at->original_value.data[0] == '"') || 
143 |        (at->original_value.data[0] == '\'') ) {
144 | 
145 |     // determine original quote character used if it exists
146 |     char quote = at->original_value.data[0];
147 |     std::string qs = "";
148 |     if (quote == '\'') qs = std::string("'");
149 |     if (quote == '"') qs = std::string("\"");
150 |     atts.append("=");
151 |     atts.append(qs);
152 |     if (no_entities) {
153 |       atts.append(at->value);
154 |     } else {
155 |       atts.append(substitute_xml_entities_into_attributes(quote, std::string(at->value)));
156 |     }
157 |     atts.append(qs);
158 |   }
159 |   return atts;
160 | }
161 | 
162 | 
163 | // forward declaration
164 | static std::string serialize(GumboNode*);
165 | 
166 | 
167 | // serialize children of a node
168 | // may be invoked recursively
169 | 
170 | static std::string serialize_contents(GumboNode* node) {
171 |   std::string contents        = "";
172 |   std::string tagname         = get_tag_name(node);
173 |   std::string key             = "|" + tagname + "|";
174 |   bool no_entity_substitution = no_entity_sub.find(key) != std::string::npos;
175 |   bool keep_whitespace        = preserve_whitespace.find(key) != std::string::npos;
176 |   bool is_inline              = nonbreaking_inline.find(key) != std::string::npos;
177 | 
178 |   // build up result for each child, recursively if need be
179 |   GumboVector* children = &node->v.element.children;
180 | 
181 |   for (unsigned int i = 0; i < children->length; ++i) {
182 |     GumboNode* child = static_cast<GumboNode*> (children->data[i]);
183 | 
184 |     if (child->type == GUMBO_NODE_TEXT) {
185 |       if (no_entity_substitution) {
186 |         contents.append(std::string(child->v.text.text));
187 |       } else {
188 |         contents.append(substitute_xml_entities_into_text(std::string(child->v.text.text)));
189 |       }
190 | 
191 |     } else if (child->type == GUMBO_NODE_ELEMENT || child->type == GUMBO_NODE_TEMPLATE) {
192 |       contents.append(serialize(child));
193 | 
194 |     } else if (child->type == GUMBO_NODE_WHITESPACE) {
195 |       // keep all whitespace to keep as close to original as possible
196 |       contents.append(std::string(child->v.text.text));
197 | 
198 |     } else if (child->type != GUMBO_NODE_COMMENT) {
199 |       // Does this actually exist: (child->type == GUMBO_NODE_CDATA)
200 |       fprintf(stderr, "unknown element of type: %d\n", child->type); 
201 |     }
202 |   }
203 |   return contents;
204 | }
205 | 
206 | 
207 | // serialize a GumboNode back to html/xhtml
208 | // may be invoked recursively
209 | 
210 | static std::string serialize(GumboNode* node) {
211 |   // special case the document node
212 |   if (node->type == GUMBO_NODE_DOCUMENT) {
213 |     std::string results = build_doctype(node);
214 |     results.append(serialize_contents(node));
215 |     return results;
216 |   }
217 | 
218 |   std::string close = "";
219 |   std::string closeTag = "";
220 |   std::string atts = "";
221 |   std::string tagname            = get_tag_name(node);
222 |   std::string key                = "|" + tagname + "|";
223 |   bool need_special_handling     =  special_handling.find(key) != std::string::npos;
224 |   bool is_empty_tag              = empty_tags.find(key) != std::string::npos;
225 |   bool no_entity_substitution    = no_entity_sub.find(key) != std::string::npos;
226 |   bool is_inline                 = nonbreaking_inline.find(key) != std::string::npos;
227 | 
228 |   // build attr string  
229 |   const GumboVector * attribs = &node->v.element.attributes;
230 |   for (int i=0; i< attribs->length; ++i) {
231 |     GumboAttribute* at = static_cast<GumboAttribute*>(attribs->data[i]);
232 |     atts.append(build_attributes(at, no_entity_substitution));
233 |   }
234 | 
235 |   // determine closing tag type
236 |   if (is_empty_tag) {
237 |       close = "/";
238 |   } else {
239 |       closeTag = "</" + tagname + ">";
240 |   }
241 | 
242 |   // serialize your contents
243 |   std::string contents = serialize_contents(node);
244 | 
245 |   if (need_special_handling) {
246 |     ltrim(contents);
247 |     rtrim(contents);
248 |     contents.append("\n");
249 |   }
250 | 
251 |   // build results
252 |   std::string results;
253 |   results.append("<"+tagname+atts+close+">");
254 |   if (need_special_handling) results.append("\n");
255 |   results.append(contents);
256 |   results.append(closeTag);
257 |   if (need_special_handling) results.append("\n");
258 |   return results;
259 | }
260 | 
261 | 
262 | int main(int argc, char** argv) {
263 |   if (argc != 2) {
264 |       std::cout << "serialize <html filename>\n";
265 |       exit(EXIT_FAILURE);
266 |   }
267 |   const char* filename = argv[1];
268 | 
269 |   std::ifstream in(filename, std::ios::in | std::ios::binary);
270 |   if (!in) {
271 |     std::cout << "File " << filename << " not found!\n";
272 |     exit(EXIT_FAILURE);
273 |   }
274 | 
275 |   std::string contents;
276 |   in.seekg(0, std::ios::end);
277 |   contents.resize(in.tellg());
278 |   in.seekg(0, std::ios::beg);
279 |   in.read(&contents[0], contents.size());
280 |   in.close();
281 |  
282 |   GumboOptions options = kGumboDefaultOptions;
283 | 
284 |   GumboOutput* output = gumbo_parse_with_options(&options, contents.data(), contents.length());
285 |   std::cout << serialize(output->document) << std::endl;
286 |   gumbo_destroy_output(&kGumboDefaultOptions, output);
287 | }
288 | 


--------------------------------------------------------------------------------
/genperf.py:
--------------------------------------------------------------------------------
 1 | import sys, re
 2 | 
 3 | gperf_output = sys.stdin.read()
 4 | 
 5 | hash_func = re.search(
 6 |     "static unsigned int\s+hash\s+\((.*?)\)\s{(.*?)\n}",
 7 |     gperf_output, re.DOTALL)
 8 | 
 9 | if not hash_func:
10 |     raise "Failed to detect hash function in GPerf output"
11 | 
12 | wordlist = re.search(
13 |     "wordlist\[\]\s+=\s+{(.*?)}",
14 |     gperf_output, re.DOTALL)
15 | 
16 | if not wordlist:
17 |     raise "Failed to detect word list in GPerf output"
18 | 
19 | def process_wordlist(text):
20 |     wordlist = [w.strip().replace('"', '') for w in text.split(',')]
21 |     taglist = [
22 |         "\tGUMBO_TAG_" + (w.upper().replace('-', '_') if w else 'LAST')
23 |         for w in wordlist]
24 |     return taglist
25 | 
26 | print "static unsigned int tag_hash(%s)\n{%s\n}" % (
27 |         hash_func.group(1), hash_func.group(2))
28 | print ""
29 | print "static const unsigned char kGumboTagMap[] = {\n%s\n};" % (
30 |         ',\n'.join(process_wordlist(wordlist.group(1))))
31 | 


--------------------------------------------------------------------------------
/gentags.py:
--------------------------------------------------------------------------------
 1 | import sys
 2 | 
 3 | def open_and_write_header(filename, comment_prefix):
 4 |   f = open(filename, 'w')
 5 |   f.write(comment_prefix + ' Generated via `gentags.py src/tag.in`.\n')
 6 |   f.write(comment_prefix + ' Do not edit; edit src/tag.in instead.\n')
 7 |   f.write(comment_prefix + ' clang-format off\n')
 8 |   return f
 9 | 
10 | tag_strings = open_and_write_header('src/tag_strings.h', '//')
11 | tag_enum = open_and_write_header('src/tag_enum.h', '//')
12 | tag_sizes = open_and_write_header('src/tag_sizes.h', '//')
13 | 
14 | tag_py = open_and_write_header('python/gumbo/gumboc_tags.py', '#')
15 | tag_py.write('TagNames = [\n')
16 | 
17 | tagfile = open(sys.argv[1])
18 | 
19 | for tag in tagfile:
20 |     tag = tag.strip()
21 |     tag_upper = tag.upper().replace('-', '_')
22 |     tag_strings.write('"%s",\n' % tag)
23 |     tag_enum.write('GUMBO_TAG_%s,\n' % tag_upper)
24 |     tag_sizes.write('%d, ' % len(tag))
25 |     tag_py.write('  "%s",\n' % tag_upper)
26 | 
27 | tagfile.close()
28 | 
29 | tag_strings.close()
30 | tag_enum.close()
31 | tag_sizes.close()
32 | 
33 | tag_py.write(']\n')
34 | tag_py.close()
35 | 


--------------------------------------------------------------------------------
/gtest.gyp:
--------------------------------------------------------------------------------
  1 | # Copyright 2013 Google Inc. All Rights Reserved.
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | #     http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | 
 15 | {
 16 |   'targets': [
 17 |     {
 18 |       'target_name': 'gtest',
 19 |       'type': 'static_library',
 20 |       'include_dirs': [
 21 |         'third_party/gtest',
 22 |         'third_party/gtest/include',
 23 |       ],
 24 |       'link_settings': {
 25 |         'ldflags': ['-lpthread'],
 26 |       },
 27 |       'dependencies': [
 28 |         'gtest_prod',
 29 |       ],
 30 |       'sources': [
 31 |         'third_party/gtest/include/gtest/gtest-death-test.h',
 32 |         'third_party/gtest/include/gtest/gtest-message.h',
 33 |         'third_party/gtest/include/gtest/gtest-param-test.h',
 34 |         'third_party/gtest/include/gtest/gtest-printers.h',
 35 |         'third_party/gtest/include/gtest/gtest-spi.h',
 36 |         'third_party/gtest/include/gtest/gtest-test-part.h',
 37 |         'third_party/gtest/include/gtest/gtest-typed-test.h',
 38 |         'third_party/gtest/include/gtest/gtest.h',
 39 |         'third_party/gtest/include/gtest/gtest_pred_impl.h',
 40 |         'third_party/gtest/include/gtest/internal/gtest-death-test-internal.h',
 41 |         'third_party/gtest/include/gtest/internal/gtest-filepath.h',
 42 |         'third_party/gtest/include/gtest/internal/gtest-internal.h',
 43 |         'third_party/gtest/include/gtest/internal/gtest-linked_ptr.h',
 44 |         'third_party/gtest/include/gtest/internal/gtest-param-util-generated.h',
 45 |         'third_party/gtest/include/gtest/internal/gtest-param-util.h',
 46 |         'third_party/gtest/include/gtest/internal/gtest-port.h',
 47 |         'third_party/gtest/include/gtest/internal/gtest-string.h',
 48 |         'third_party/gtest/include/gtest/internal/gtest-tuple.h',
 49 |         'third_party/gtest/include/gtest/internal/gtest-type-util.h',
 50 |         'third_party/gtest/src/gtest-all.cc',
 51 |         'third_party/gtest/src/gtest-death-test.cc',
 52 |         'third_party/gtest/src/gtest-filepath.cc',
 53 |         'third_party/gtest/src/gtest-internal-inl.h',
 54 |         'third_party/gtest/src/gtest-port.cc',
 55 |         'third_party/gtest/src/gtest-printers.cc',
 56 |         'third_party/gtest/src/gtest-test-part.cc',
 57 |         'third_party/gtest/src/gtest-typed-test.cc',
 58 |         'third_party/gtest/src/gtest.cc',
 59 |       ],
 60 |       'sources!': [
 61 |         'third_party/gtest/src/gtest-all.cc',  # Not needed by our build.
 62 |       ],
 63 |       'conditions': [
 64 |         ['OS == "mac" or OS == "ios"', {
 65 |           'sources': [
 66 |             'gtest_mac.h',
 67 |             'gtest_mac.mm',
 68 |             'platform_test_mac.mm'
 69 |           ],
 70 |           'link_settings': {
 71 |             'libraries': [
 72 |               '$(SDKROOT)/System/Library/Frameworks/Foundation.framework',
 73 |             ],
 74 |           },
 75 |         }],
 76 |         ['OS=="win" and (MSVS_VERSION=="2012" or MSVS_VERSION=="2012e")', {
 77 |           'defines': [
 78 |             '_VARIADIC_MAX=10',
 79 |           ],
 80 |           'direct_dependent_settings': {
 81 |             'defines': [
 82 |               '_VARIADIC_MAX=10',
 83 |             ],
 84 |           },
 85 |         }],
 86 |       ],
 87 |       'direct_dependent_settings': {
 88 |         'defines': [
 89 |           'UNIT_TEST',
 90 |         ],
 91 |         'include_dirs': [
 92 |           'third_party/gtest/include',  # So that gtest headers can find themselves.
 93 |         ],
 94 |         'target_conditions': [
 95 |           ['_type=="executable"', {
 96 |             'test': 1,
 97 |             'conditions': [
 98 |               ['OS=="mac"', {
 99 |                 'run_as': {
100 |                   'action????': ['${BUILT_PRODUCTS_DIR}/${PRODUCT_NAME}'],
101 |                 },
102 |               }],
103 |               ['OS=="win"', {
104 |                 'run_as': {
105 |                   'action????': ['$(TargetPath)', '--gtest_print_time'],
106 |                 },
107 |               }],
108 |             ],
109 |           }],
110 |         ],
111 |         'msvs_disabled_warnings': [4800],
112 |       },
113 |     },
114 |     {
115 |       'target_name': 'gtest_main',
116 |       'type': 'static_library',
117 |       'dependencies': [
118 |         'gtest',
119 |       ],
120 |       'sources': [
121 |         'third_party/gtest/src/gtest_main.cc',
122 |       ],
123 |     },
124 |     {
125 |       'target_name': 'gtest_prod',
126 |       'toolsets': ['host', 'target'],
127 |       'type': 'none',
128 |       'sources': [
129 |         'third_party/gtest/include/gtest/gtest_prod.h',
130 |       ],
131 |     },
132 |   ],
133 | }
134 | 


--------------------------------------------------------------------------------
/gumbo.pc.in:
--------------------------------------------------------------------------------
 1 | prefix=@prefix@
 2 | exec_prefix=@exec_prefix@
 3 | libdir=@libdir@
 4 | includedir=@includedir@
 5 | 
 6 | Name: Gumbo
 7 | Description: A fully-compliant HTML5 parser.
 8 | Version: @PACKAGE_VERSION@
 9 | Libs: -L${libdir} -lgumbo
10 | Libs.private: @LIBS@
11 | Cflags: -I${includedir}
12 | 


--------------------------------------------------------------------------------
/gumbo_parser.gyp:
--------------------------------------------------------------------------------
 1 | # Copyright 2013 Google Inc. All Rights Reserved.
 2 | #
 3 | # Licensed under the Apache License, Version 2.0 (the "License");
 4 | # you may not use this file except in compliance with the License.
 5 | # You may obtain a copy of the License at
 6 | #
 7 | #     http://www.apache.org/licenses/LICENSE-2.0
 8 | #
 9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | 
15 | {
16 |   'targets': [
17 |     {
18 |       'target_name': 'gumbo_parser',
19 |       'type': 'static_library',
20 |       'cflags': ['-std=c99', '-Wall'],
21 |       'sources': [
22 |         'src/attribute.c',
23 |         'src/attribute.h',
24 |         'src/char_ref.c',
25 |         'src/char_ref.h',
26 |         'src/error.c',
27 |         'src/error.h',
28 |         'src/gumbo.h',
29 |         'src/insertion_mode.h',
30 |         'src/parser.c',
31 |         'src/parser.h',
32 |         'src/string_buffer.c',
33 |         'src/string_buffer.h',
34 |         'src/string_piece.c',
35 |         'src/string_piece.h',
36 |         'src/tag.c',
37 |         'src/token_type.h',
38 |         'src/tokenizer.c',
39 |         'src/tokenizer.h',
40 |         'src/tokenizer_states.h',
41 |         'src/utf8.c',
42 |         'src/utf8.h',
43 |         'src/util.c',
44 |         'src/util.h',
45 |         'src/vector.c',
46 |         'src/vector.h',
47 |       ],
48 |     },
49 |     {
50 |       'target_name': 'gumbo_parser_unittests',
51 |       'type': 'executable',
52 |       'dependencies': [
53 |         'gtest.gyp:gtest',
54 |         'gtest.gyp:gtest_main',
55 |         'gumbo_parser',
56 |       ],
57 |       'include_dirs': [
58 |         '.',
59 |         '..',
60 |         'src',
61 |       ],
62 |       'sources': [
63 |         'tests/attribute.cc',
64 |         'tests/char_ref.cc',
65 |         'tests/parser.cc',
66 |         'tests/string_buffer.cc',
67 |         'tests/string_piece.cc',
68 |         'tests/test_utils.cc',
69 |         'tests/test_utils.h',
70 |         'tests/tokenizer.cc',
71 |         'tests/utf8.cc',
72 |         'tests/vector.cc',
73 |       ],
74 |     },
75 |   ],
76 | }
77 | 


--------------------------------------------------------------------------------
/original-README.md:
--------------------------------------------------------------------------------
  1 | Gumbo - A pure-C HTML5 parser.
  2 | ============
  3 | 
  4 | [![Build Status](https://travis-ci.org/google/gumbo-parser.svg?branch=master)](https://travis-ci.org/google/gumbo-parser) [![Build status](https://ci.appveyor.com/api/projects/status/k5xxn4bxf62ao2cp?svg=true)](https://ci.appveyor.com/project/nostrademons/gumbo-parser)
  5 | 
  6 | Gumbo is an implementation of the [HTML5 parsing algorithm][] implemented
  7 | as a pure C99 library with no outside dependencies.  It's designed to serve
  8 | as a building block for other tools and libraries such as linters,
  9 | validators, templating languages, and refactoring and analysis tools.
 10 | 
 11 | Goals & features:
 12 | 
 13 | * Fully conformant with the [HTML5 spec][].
 14 | * Robust and resilient to bad input.
 15 | * Simple API that can be easily wrapped by other languages.
 16 | * Support for source locations and pointers back to the original text.
 17 | * Support for fragment parsing.
 18 | * Relatively lightweight, with no outside dependencies.
 19 | * Passes all [html5lib tests][], including the template tag.
 20 | * Tested on over 2.5 billion pages from Google's index.
 21 | 
 22 | Non-goals:
 23 | 
 24 | * Execution speed.  Gumbo gains some of this by virtue of being written in
 25 |   C, but it is not an important consideration for the intended use-case, and
 26 |   was not a major design factor.
 27 | * Support for encodings other than UTF-8.  For the most part, client code
 28 |   can convert the input stream to UTF-8 text using another library before
 29 |   processing.
 30 | * Mutability.  Gumbo is intentionally designed to turn an HTML document into a
 31 |   parse tree, and free that parse tree all at once.  It's not designed to
 32 |   persistently store nodes or subtrees outside of the parse tree, or to perform
 33 |   arbitrary DOM mutations within your program.  If you need this functionality,
 34 |   we recommend translating the Gumbo parse tree into a mutable DOM
 35 |   representation more suited for the particular needs of your program before
 36 |   operating on it.
 37 | * C89 support.  Most major compilers support C99 by now; the major exception
 38 |   (Microsoft Visual Studio) should be able to compile this in C++ mode with
 39 |   relatively few changes.  (Bug reports welcome.)
 40 | * ~~Security.  Gumbo was initially designed for a product that worked with
 41 |   trusted input files only.  We're working to harden this and make sure that it
 42 |   behaves as expected even on malicious input, but for now, Gumbo should only be
 43 |   run on trusted input or within a sandbox.~~ Gumbo underwent a number of
 44 |   security fixes and passed Google's security review as of version 0.9.1.
 45 | 
 46 | Wishlist (aka "We couldn't get these into the original release, but are
 47 | hoping to add them soon"):
 48 | 
 49 | * Full-featured error reporting.
 50 | * Additional performance improvements.
 51 | * DOM wrapper library/libraries (possibly within other language bindings)
 52 | * Query libraries, to extract information from parse trees using CSS or XPATH.
 53 | 
 54 | Installation
 55 | ============
 56 | 
 57 | To build and install the library, issue the standard UNIX incantation from
 58 | the root of the distribution:
 59 | 
 60 | ```bash
 61 | $ ./autogen.sh
 62 | $ ./configure
 63 | $ make
 64 | $ sudo make install
 65 | ```
 66 | 
 67 | Gumbo comes with full pkg-config support, so you can use the pkg-config to
 68 | print the flags needed to link your program against it:
 69 | 
 70 | ```bash
 71 | $ pkg-config --cflags gumbo         # print compiler flags
 72 | $ pkg-config --libs gumbo           # print linker flags
 73 | $ pkg-config --cflags --libs gumbo  # print both
 74 | ```
 75 | 
 76 | For example:
 77 | 
 78 | ```bash
 79 | $ gcc my_program.c `pkg-config --cflags --libs gumbo`
 80 | ```
 81 | 
 82 | See the pkg-config man page for more info.
 83 | 
 84 | There are a number of sample programs in the examples/ directory.  They're
 85 | built automatically by 'make', but can also be made individually with
 86 | `make <programname>` (eg. `make clean_text`).
 87 | 
 88 | To run the unit tests, you'll need to have [googletest][] downloaded and
 89 | unzipped.  The googletest maintainers recommend against using
 90 | `make install`; instead, symlink the root googletest directory to 'gtest'
 91 | inside gumbo's root directory, and then `make check`:
 92 | 
 93 | ```bash
 94 | $ unzip gtest-1.6.0.zip
 95 | $ cd gumbo-*
 96 | $ ln -s ../gtest-1.6.0 gtest
 97 | $ make check
 98 | ```
 99 | 
100 | Gumbo's `make check` has code to automatically configure & build gtest and
101 | then link in the library.
102 | 
103 | Debian and Fedora users can install libgtest with:
104 | 
105 | ```bash
106 | $ apt-get install libgtest-dev  # Debian/Ubuntu
107 | $ yum install gtest-devel       # CentOS/Fedora
108 | ```
109 | 
110 | Note for Ubuntu users: libgtest-dev package only install source files.
111 | You have to make libraries yourself using cmake:
112 | 
113 |     $ sudo apt-get install cmake
114 |     $ cd /usr/src/gtest
115 |     $ sudo cmake CMakeLists.txt
116 |     $ sudo make
117 |     $ sudo cp *.a /usr/lib
118 | 
119 | The configure script will detect the presence of the library and use that
120 | instead.
121 | 
122 | Note that you need to have super user privileges to execute these commands.
123 | On most distros, you can prefix the commands above with `sudo` to execute
124 | them as the super user.
125 | 
126 | Debian installs usually don't have `sudo` installed (Ubuntu however does.)
127 | Switch users first with `su -`, then run `apt-get`.
128 | 
129 | Basic Usage
130 | ===========
131 | 
132 | Within your program, you need to include "gumbo.h" and then issue a call to
133 | `gumbo_parse`:
134 | 
135 | ```C
136 | #include "gumbo.h"
137 | 
138 | int main() {
139 |   GumboOutput* output = gumbo_parse("<h1>Hello, World!</h1>");
140 |   // Do stuff with output->root
141 |   gumbo_destroy_output(&kGumboDefaultOptions, output);
142 | }
143 | ```
144 | 
145 | See the API documentation and sample programs for more details.
146 | 
147 | A note on API/ABI compatibility
148 | ===============================
149 | 
150 | We'll make a best effort to preserve API compatibility between releases.
151 | The initial release is a 0.9 (beta) release to solicit comments from early
152 | adopters, but if no major problems are found with the API, a 1.0 release
153 | will follow shortly, and the API of that should be considered stable.  If
154 | changes are necessary, we follow [semantic versioning][].
155 | 
156 | We make no such guarantees about the ABI, and it's very likely that
157 | subsequent versions may require a recompile of client code.  For this
158 | reason, we recommend NOT using Gumbo data structures throughout a program,
159 | and instead limiting them to a translation layer that picks out whatever
160 | data is needed from the parse tree and then converts that to persistent
161 | data structures more appropriate for the application.  The API is
162 | structured to encourage this use, with a single delete function for the
163 | whole parse tree, and is not designed with mutation in mind.
164 | 
165 | Python usage
166 | ============
167 | 
168 | To install the python bindings, make sure that the
169 | C library is installed first, and then `sudo python setup.py install` from
170 | the root of the distro.  This installs a 'gumbo' module; `pydoc gumbo`
171 | should tell you about it.
172 | 
173 | Recommended best-practice for Python usage is to use one of the adapters to
174 | an existing API (personally, I prefer BeautifulSoup) and write your program
175 | in terms of those.  The raw CTypes bindings should be considered building
176 | blocks for higher-level libraries and rarely referenced directly.
177 | 
178 | External Bindings and other wrappers
179 | ====================================
180 | 
181 | The following language bindings or other tools/wrappers are maintained by
182 | various contributors in other repositories:
183 | 
184 | * C++: [gumbo-query] by lazytiger
185 | * Ruby:
186 |   * [ruby-gumbo] by Nicolas Martyanoff
187 |   * [nokogumbo] by Sam Ruby
188 | * Node.js: [node-gumbo-parser] by Karl Westin
189 | * D: [gumbo-d] by Christopher Bertels
190 | * Lua: [lua-gumbo] by Craig Barnes
191 | * Objective-C:
192 |   * [ObjectiveGumbo] by Programming Thomas
193 |   * [OCGumbo] by TracyYih
194 | * C#: [GumboBindings] by Vladimir Zotov
195 | * PHP: [GumboPHP] by Paul Preece
196 | * Perl: [HTML::Gumbo] by Ruslan Zakirov
197 | * Julia: [Gumbo.jl] by James Porter
198 | * C/Libxml: [gumbo-libxml] by Jonathan Tang
199 | 
200 | [gumbo-query]: https://github.com/lazytiger/gumbo-query
201 | [ruby-gumbo]: https://github.com/nevir/ruby-gumbo
202 | [nokogumbo]: https://github.com/rubys/nokogumbo
203 | [node-gumbo-parser]: https://github.com/karlwestin/node-gumbo-parser
204 | [gumbo-d]: https://github.com/bakkdoor/gumbo-d
205 | [lua-gumbo]: https://github.com/craigbarnes/lua-gumbo
206 | [OCGumbo]: https://github.com/tracy-e/OCGumbo
207 | [ObjectiveGumbo]: https://github.com/programmingthomas/ObjectiveGumbo
208 | [GumboBindings]: https://github.com/rgripper/GumboBindings
209 | [GumboPHP]: https://github.com/BipSync/gumbo
210 | [Gumbo.jl]: https://github.com/porterjamesj/Gumbo.jl
211 | [gumbo-libxml]: https://github.com/nostrademons/gumbo-libxml
212 | 
213 | [HTML5 parsing algorithm]: http://www.whatwg.org/specs/web-apps/current-work/multipage/#auto-toc-12
214 | [HTML5 spec]: http://www.whatwg.org/specs/web-apps/current-work/multipage/
215 | [html5lib tests]: https://github.com/html5lib/html5lib-tests
216 | [googletest]: https://code.google.com/p/googletest/
217 | [semantic versioning]: http://semver.org/
218 | [HTML::Gumbo]: https://metacpan.org/pod/HTML::Gumbo
219 | 


--------------------------------------------------------------------------------
/python/gumbo/__init__.py:
--------------------------------------------------------------------------------
 1 | """Gumbo HTML parser.
 2 | 
 3 | These are the Python bindings for Gumbo.  All public API classes and functions
 4 | are exported from this module.  They include:
 5 | 
 6 | - CTypes representations of all structs and enums defined in gumbo.h.  The
 7 |   naming convention is to take the C name and strip off the "Gumbo" prefix.
 8 | 
 9 | - A low-level wrapper around the gumbo_parse function, returning the classes
10 |   exposed above.  Usage:
11 | 
12 |   import gumbo
13 |   with gumboc.parse(text, **options) as output:
14 |     do_stuff_with_doctype(output.document)
15 |     do_stuff_with_parse_tree(output.root)
16 | 
17 | - Higher-level bindings that mimic the API provided by html5lib.  Usage:
18 | 
19 |   from gumbo import html5lib
20 | 
21 |   This requires that html5lib be installed (it uses their treebuilders), and is
22 |   intended as a drop-in replacement.
23 | 
24 | - Similarly, higher-level bindings that mimic BeautifulSoup and return
25 |   BeautifulSoup objects.  For this, use:
26 | 
27 |   import gumbo
28 |   soup = gumbo.soup_parse(text, **options)
29 | 
30 |   It will give you back a soup object like BeautifulSoup.BeautifulSoup(text).
31 | """
32 | 
33 | from gumbo.gumboc import *
34 | 
35 | try:
36 |   from gumbo import html5lib_adapter as html5lib
37 | except ImportError:
38 |   # html5lib not installed
39 |   pass
40 | 
41 | try:
42 |   from gumbo.soup_adapter import parse as soup_parse
43 | except ImportError:
44 |   # BeautifulSoup not installed
45 |   pass
46 | 


--------------------------------------------------------------------------------
/python/gumbo/gumboc_tags.py:
--------------------------------------------------------------------------------
  1 | # Generated via `gentags.py src/tag.in`.
  2 | # Do not edit; edit src/tag.in instead.
  3 | # clang-format off
  4 | TagNames = [
  5 |   "HTML",
  6 |   "HEAD",
  7 |   "TITLE",
  8 |   "BASE",
  9 |   "LINK",
 10 |   "META",
 11 |   "STYLE",
 12 |   "SCRIPT",
 13 |   "NOSCRIPT",
 14 |   "TEMPLATE",
 15 |   "BODY",
 16 |   "ARTICLE",
 17 |   "SECTION",
 18 |   "NAV",
 19 |   "ASIDE",
 20 |   "H1",
 21 |   "H2",
 22 |   "H3",
 23 |   "H4",
 24 |   "H5",
 25 |   "H6",
 26 |   "HGROUP",
 27 |   "HEADER",
 28 |   "FOOTER",
 29 |   "ADDRESS",
 30 |   "P",
 31 |   "HR",
 32 |   "PRE",
 33 |   "BLOCKQUOTE",
 34 |   "OL",
 35 |   "UL",
 36 |   "LI",
 37 |   "DL",
 38 |   "DT",
 39 |   "DD",
 40 |   "FIGURE",
 41 |   "FIGCAPTION",
 42 |   "MAIN",
 43 |   "DIV",
 44 |   "A",
 45 |   "EM",
 46 |   "STRONG",
 47 |   "SMALL",
 48 |   "S",
 49 |   "CITE",
 50 |   "Q",
 51 |   "DFN",
 52 |   "ABBR",
 53 |   "DATA",
 54 |   "TIME",
 55 |   "CODE",
 56 |   "VAR",
 57 |   "SAMP",
 58 |   "KBD",
 59 |   "SUB",
 60 |   "SUP",
 61 |   "I",
 62 |   "B",
 63 |   "U",
 64 |   "MARK",
 65 |   "RUBY",
 66 |   "RT",
 67 |   "RP",
 68 |   "BDI",
 69 |   "BDO",
 70 |   "SPAN",
 71 |   "BR",
 72 |   "WBR",
 73 |   "INS",
 74 |   "DEL",
 75 |   "IMAGE",
 76 |   "IMG",
 77 |   "IFRAME",
 78 |   "EMBED",
 79 |   "OBJECT",
 80 |   "PARAM",
 81 |   "VIDEO",
 82 |   "AUDIO",
 83 |   "SOURCE",
 84 |   "TRACK",
 85 |   "CANVAS",
 86 |   "MAP",
 87 |   "AREA",
 88 |   "MATH",
 89 |   "MI",
 90 |   "MO",
 91 |   "MN",
 92 |   "MS",
 93 |   "MTEXT",
 94 |   "MGLYPH",
 95 |   "MALIGNMARK",
 96 |   "ANNOTATION_XML",
 97 |   "SVG",
 98 |   "FOREIGNOBJECT",
 99 |   "DESC",
100 |   "TABLE",
101 |   "CAPTION",
102 |   "COLGROUP",
103 |   "COL",
104 |   "TBODY",
105 |   "THEAD",
106 |   "TFOOT",
107 |   "TR",
108 |   "TD",
109 |   "TH",
110 |   "FORM",
111 |   "FIELDSET",
112 |   "LEGEND",
113 |   "LABEL",
114 |   "INPUT",
115 |   "BUTTON",
116 |   "SELECT",
117 |   "DATALIST",
118 |   "OPTGROUP",
119 |   "OPTION",
120 |   "TEXTAREA",
121 |   "KEYGEN",
122 |   "OUTPUT",
123 |   "PROGRESS",
124 |   "METER",
125 |   "DETAILS",
126 |   "SUMMARY",
127 |   "MENU",
128 |   "MENUITEM",
129 |   "APPLET",
130 |   "ACRONYM",
131 |   "BGSOUND",
132 |   "DIR",
133 |   "FRAME",
134 |   "FRAMESET",
135 |   "NOFRAMES",
136 |   "ISINDEX",
137 |   "LISTING",
138 |   "XMP",
139 |   "NEXTID",
140 |   "NOEMBED",
141 |   "PLAINTEXT",
142 |   "RB",
143 |   "STRIKE",
144 |   "BASEFONT",
145 |   "BIG",
146 |   "BLINK",
147 |   "CENTER",
148 |   "FONT",
149 |   "MARQUEE",
150 |   "MULTICOL",
151 |   "NOBR",
152 |   "SPACER",
153 |   "TT",
154 |   "RTC",
155 | ]
156 | 


--------------------------------------------------------------------------------
/python/gumbo/gumboc_test.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2012 Google Inc. All Rights Reserved.
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | #     http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | #
 15 | 
 16 | """Tests for Gumbo CTypes bindings."""
 17 | 
 18 | __author__ = 'jdtang@google.com (Jonathan Tang)'
 19 | 
 20 | import StringIO
 21 | 
 22 | import unittest
 23 | 
 24 | import gumboc
 25 | 
 26 | 
 27 | class CtypesTest(unittest.TestCase):
 28 |   def testWordParse(self):
 29 |     with gumboc.parse('Test') as output:
 30 |       doctype_node = output.contents.document.contents
 31 |       self.assertEquals(gumboc.NodeType.DOCUMENT, doctype_node.type)
 32 |       document = doctype_node.v.document
 33 |       self.assertEquals('', document.name)
 34 |       self.assertEquals('', document.public_identifier)
 35 |       self.assertEquals('', document.system_identifier)
 36 | 
 37 |       root = output.contents.root.contents
 38 |       self.assertEquals(gumboc.NodeType.ELEMENT, root.type)
 39 |       self.assertEquals(gumboc.Tag.HTML, root.tag)
 40 |       self.assertEquals(gumboc.Namespace.HTML, root.tag_namespace)
 41 |       self.assertEquals(2, len(root.children))
 42 | 
 43 |       head = root.children[0]
 44 |       self.assertEquals(gumboc.NodeType.ELEMENT, head.type)
 45 |       self.assertEquals(gumboc.Tag.HEAD, head.tag)
 46 |       self.assertEquals('head', head.tag_name)
 47 |       self.assertEquals(gumboc.Namespace.HTML, head.tag_namespace)
 48 |       self.assertEquals(0, len(head.original_tag))
 49 |       self.assertEquals('', str(head.original_end_tag))
 50 |       self.assertEquals(0, head.children.length)
 51 | 
 52 |       body = root.children[1]
 53 |       self.assertNotEquals(body, doctype_node)
 54 |       self.assertEquals(gumboc.NodeType.ELEMENT, body.type)
 55 |       self.assertEquals(gumboc.Tag.BODY, body.tag)
 56 |       self.assertEquals('body', body.tag_name)
 57 |       self.assertEquals(1, len(body.children))
 58 | 
 59 |       text_node = body.children[0]
 60 |       self.assertEquals(gumboc.NodeType.TEXT, text_node.type)
 61 |       self.assertEquals('Test', text_node.text)
 62 | 
 63 |   def testBufferThatGoesAway(self):
 64 |     for i in range(10):
 65 |       source = StringIO.StringIO('<foo bar=quux>1<p>2</foo>')
 66 |       parse_tree = gumboc.parse(source.read())
 67 |       source.close()
 68 |     with parse_tree as output:
 69 |       root = output.contents.root.contents
 70 |       body = root.children[1]
 71 |       foo = body.children[0]
 72 |       self.assertEquals(gumboc.NodeType.ELEMENT, foo.type)
 73 |       self.assertEquals(gumboc.Tag.UNKNOWN, foo.tag)
 74 |       self.assertEquals('<foo bar=quux>', str(foo.original_tag))
 75 |       self.assertEquals('', str(foo.original_end_tag))
 76 |       self.assertEquals('foo', foo.tag_name.decode('utf-8'))
 77 |       self.assertEquals('bar', foo.attributes[0].name)
 78 |       self.assertEquals('quux', foo.attributes[0].value)
 79 | 
 80 |   def testUnknownTag(self):
 81 |     with gumboc.parse('<foo bar=quux>1<p>2</foo>') as output:
 82 |       root = output.contents.root.contents
 83 |       body = root.children[1]
 84 |       foo = body.children[0]
 85 |       self.assertEquals(gumboc.NodeType.ELEMENT, foo.type)
 86 |       self.assertEquals(gumboc.Tag.UNKNOWN, foo.tag)
 87 |       self.assertEquals('<foo bar=quux>', str(foo.original_tag))
 88 |       self.assertEquals('', str(foo.original_end_tag))
 89 |       self.assertEquals('foo', foo.tag_name.decode('utf-8'))
 90 |       self.assertEquals('bar', foo.attributes[0].name)
 91 |       self.assertEquals('quux', foo.attributes[0].value)
 92 | 
 93 |   def testSarcasm(self):
 94 |     with gumboc.parse('<div><sarcasm><div></div></sarcasm></div>') as output:
 95 |       root = output.contents.root.contents
 96 |       body = root.children[1]
 97 |       div = body.children[0]
 98 |       sarcasm = div.children[0]
 99 |       self.assertEquals(gumboc.NodeType.ELEMENT, sarcasm.type)
100 |       self.assertEquals(gumboc.Tag.UNKNOWN, sarcasm.tag)
101 |       self.assertEquals('<sarcasm>', str(sarcasm.original_tag))
102 |       self.assertEquals('</sarcasm>', str(sarcasm.original_end_tag))
103 |       self.assertEquals('sarcasm', sarcasm.tag_name.decode('utf-8'))
104 | 
105 |   def testEnums(self):
106 |     self.assertEquals(gumboc.Tag.A, gumboc.Tag.A)
107 |     self.assertEquals(hash(gumboc.Tag.A.value), hash(gumboc.Tag.A))
108 | 
109 |   def testFragment(self):
110 |     with gumboc.parse(
111 |         '<div></div>',
112 |         fragment_context=gumboc.Tag.TITLE,
113 |         fragment_namespace=gumboc.Namespace.SVG) as output:
114 |       root = output.contents.root.contents
115 |       self.assertEquals(1, len(root.children))
116 |       div = root.children[0]
117 |       self.assertEquals(gumboc.NodeType.ELEMENT, div.type)
118 |       self.assertEquals(gumboc.Tag.DIV, div.tag)
119 |       self.assertEquals(gumboc.Namespace.HTML, div.tag_namespace)
120 | 
121 | 
122 | 
123 | 
124 | if __name__ == '__main__':
125 |   unittest.main()
126 | 


--------------------------------------------------------------------------------
/python/gumbo/html5lib_adapter.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2012 Google Inc. All Rights Reserved.
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | #     http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | #
 15 | """Adapter between Gumbo and html5lib.
 16 | 
 17 | This exports one method, parse, with the same signature as html5lib.parse.  It
 18 | takes the text to parse, and optionally an html5lib TreeBuilder to build the
 19 | tree, and gives back a DOM tree in that format.  Example:
 20 | 
 21 |   doc = parse(text, treebuilder='lxml')
 22 | """
 23 | 
 24 | __author__ = 'jdtang@google.com (Jonathan Tang)'
 25 | 
 26 | import gumboc
 27 | 
 28 | # These should match html5lib.constants.namespaces, and be indexed by the enum
 29 | # values of gumboc.Namespace
 30 | _NAMESPACES = [
 31 |     'http://www.w3.org/1999/xhtml',
 32 |     'http://www.w3.org/2000/svg',
 33 |     'http://www.w3.org/1998/Math/MathML',
 34 |     ]
 35 | 
 36 | 
 37 | def _convert_doctype(treebuilder, source_node):
 38 |   if not source_node.has_doctype:
 39 |     # Mimic html5lib behavior: if no doctype token, no doctype node.
 40 |     return
 41 |   treebuilder.insertDoctype({
 42 |       'name': source_node.name.decode('utf-8'),
 43 |       'publicId': source_node.public_identifier.decode('utf-8'),
 44 |       'systemId': source_node.system_identifier.decode('utf-8'),
 45 |       })
 46 | 
 47 | 
 48 | def _convert_attributes(source_node):
 49 |   def maybe_namespace(attr):
 50 |     if attr.namespace != gumboc.AttributeNamespace.NONE:
 51 |       return (repr(attr.namespace).lower() if attr.name != 'xmlns' else None,
 52 |               attr.name.decode('utf-8'),
 53 |               attr.namespace.to_url())
 54 |     else:
 55 |       return attr.name.decode('utf-8')
 56 |   return dict((maybe_namespace(attr), attr.value.decode('utf-8'))
 57 |               for attr in source_node.attributes)
 58 | 
 59 | 
 60 | def _convert_element(source_node):
 61 |   if source_node.type not in ( gumboc.NodeType.ELEMENT, gumboc.NodeType.TEMPLATE):
 62 |     # If-statement instead of assert so it runs with -O
 63 |     raise AssertionError(
 64 |         '_convert_element only works with elements; found %r' %
 65 |         source_node.type)
 66 |   return {
 67 |       'name': source_node.v.element.tag_name.decode('utf-8'),
 68 |       'namespace': _NAMESPACES[source_node.v.element.tag_namespace.value],
 69 |       'data': _convert_attributes(source_node),
 70 |       }
 71 | 
 72 | 
 73 | def _insert_root(treebuilder, source_node, pop_element = True):
 74 |   treebuilder.insertRoot(_convert_element(source_node))
 75 |   for child_node in source_node.children:
 76 |     _insert_node(treebuilder, child_node)
 77 |   if pop_element:
 78 |     treebuilder.openElements.pop()
 79 | 
 80 | def _insert_node(treebuilder, source_node):
 81 |   assert source_node.type != gumboc.NodeType.DOCUMENT
 82 |   if source_node.type == gumboc.NodeType.COMMENT:
 83 |     treebuilder.insertComment({'data': source_node.v.text.text.decode('utf-8')})
 84 |   elif source_node.type in (
 85 |       gumboc.NodeType.TEXT,
 86 |       gumboc.NodeType.WHITESPACE,
 87 |       gumboc.NodeType.CDATA):
 88 |     treebuilder.insertText(source_node.v.text.text.decode('utf-8'))
 89 |   else:
 90 |     treebuilder.insertElementNormal(_convert_element(source_node))
 91 |     for child_node in source_node.v.element.children:
 92 |       _insert_node(treebuilder, child_node)
 93 |     treebuilder.openElements.pop()
 94 | 
 95 | 
 96 | class HTMLParser(object):
 97 |   def __init__(self, tree):
 98 |     self.tree = tree
 99 | 
100 |   def parse(self, text_or_file, **kwargs):
101 |     try:
102 |       text = text_or_file.read()
103 |     except AttributeError:
104 |       # Assume a string.
105 |       text = text_or_file
106 | 
107 |     with gumboc.parse(text, **kwargs) as output:
108 |       _convert_doctype(self.tree, output.contents.document.contents)
109 |       for node in output.contents.document.contents.children:
110 |         if node.type == gumboc.NodeType.COMMENT:
111 |           self.tree.insertComment({'data': node.v.text.text.decode('utf-8')},
112 |                                   self.tree.document)
113 |         elif node.type in (gumboc.NodeType.ELEMENT, gumboc.NodeType.TEMPLATE):
114 |           _insert_root(self.tree, output.contents.root.contents)
115 |         else:
116 |           assert 'Only comments and <html> nodes allowed at the root'
117 |       return self.tree.getDocument()
118 | 
119 |   def parseFragment(self, text_or_file, container, **kwargs):
120 |     try:
121 |       text = text_or_file.read()
122 |     except AttributeError:
123 |       # Assume a string.
124 |       text = text_or_file
125 |     if ' ' in container:
126 |       container_ns, container = container.split(' ')
127 |     else:
128 |       container_ns = "html"
129 | 
130 |     with gumboc.parse(
131 |         text,
132 |         fragment_context=gumboc.Tag.from_str(container),
133 |         fragment_namespace=getattr(gumboc.Namespace, container_ns.upper()),
134 |         **kwargs) as output:
135 |       for node in output.contents.document.contents.children:
136 |         if node.type in (gumboc.NodeType.ELEMENT, gumboc.NodeType.TEMPLATE):
137 |           _insert_root(self.tree, output.contents.root.contents, False)
138 |         else:
139 |           assert 'Malformed fragment parse (??)'
140 |       return self.tree.getFragment()
141 | 


--------------------------------------------------------------------------------
/python/gumbo/html5lib_adapter_test.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2012 Google Inc. All Rights Reserved.
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | #     http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | #
 15 | """Tests for the Gumbo => Html5lib adapter."""
 16 | import codecs
 17 | import collections
 18 | import glob
 19 | import os
 20 | import re
 21 | import StringIO
 22 | import unittest
 23 | import warnings
 24 | 
 25 | from html5lib import treebuilders
 26 | 
 27 | import html5lib_adapter
 28 | 
 29 | 
 30 | TREEBUILDER = treebuilders.getTreeBuilder('dom')
 31 | TESTDATA_BASE_PATH = os.path.join(
 32 |     os.path.split(__file__)[0], '..', '..')
 33 | 
 34 | 
 35 | # Copied from html5lib.tests/test_parser.py
 36 | def convertTreeDump(data):
 37 |   return "\n".join(convertExpected(data, 3).split("\n")[1:])
 38 | 
 39 | 
 40 | # Copied/adapted/simplified from html5lib.tests/support.py
 41 | def html5lib_test_files():
 42 |   return glob.glob(os.path.join(
 43 |       TESTDATA_BASE_PATH, 'testdata', 'tree-construction', '*.dat'))
 44 | 
 45 | 
 46 | class TestData(object):
 47 |   def __init__(self, filename):
 48 |     self.f = codecs.open(filename, encoding="utf8")
 49 | 
 50 |   def __iter__(self):
 51 |     data = collections.defaultdict(lambda: None)
 52 |     key=None
 53 |     for line in self.f:
 54 |       heading = self.isSectionHeading(line)
 55 |       if heading:
 56 |         if data and heading == 'data':
 57 |           #Remove trailing newline
 58 |           data[key] = data[key][:-1]
 59 |           yield self.normaliseOutput(data)
 60 |           data = collections.defaultdict(lambda: None)
 61 |         key = heading
 62 |         data[key] = ''
 63 |       elif key is not None:
 64 |           data[key] += line
 65 |     if data:
 66 |       yield self.normaliseOutput(data)
 67 | 
 68 |   def isSectionHeading(self, line):
 69 |     """If the current heading is a test section heading return the heading,
 70 |     otherwise return False"""
 71 |     if line.startswith("#"):
 72 |       return line[1:].strip()
 73 |     else:
 74 |       return False
 75 | 
 76 |   def normaliseOutput(self, data):
 77 |     # Remove trailing newlines
 78 |     for key, value in data.iteritems():
 79 |       if value.endswith("\n"):
 80 |         data[key] = value[:-1]
 81 |     return data
 82 | 
 83 | def convertExpected(data, stripChars):
 84 |   """convert the output of str(document) to the format used in the testcases"""
 85 |   data = data.split("\n")
 86 |   rv = []
 87 |   for line in data:
 88 |     if line.startswith("|"):
 89 |       rv.append(line[stripChars:])
 90 |     else:
 91 |       rv.append(line)
 92 |   return "\n".join(rv)
 93 | 
 94 | def reformatTemplateContents(expected):
 95 |  lines = expected.split('\n')
 96 |  retval = []
 97 |  template_indents = []
 98 |  for line in lines:
 99 |    line_stripped = line.strip()
100 |    indent = len(line) - len(line_stripped)
101 |    if line_stripped == 'content':
102 |      template_indents.append(indent)
103 |      continue
104 |    while template_indents and indent <= template_indents[-1]:
105 |        template_indents.pop()
106 |    if template_indents:
107 |      line = line[2 * len(template_indents):]
108 |    retval.append(line)
109 |  return '\n'.join(retval)
110 | 
111 | class Html5libAdapterTest(unittest.TestCase):
112 |   """Adapter between Gumbo and the html5lib tests.
113 | 
114 |   This works through a bit of magic.  It's an empty class at first, but then
115 |   buildTestCases runs through the test files in html5lib, and adds a
116 |   method to this class for each one.  That method acts like
117 |   test_parser.TestCase.runParserTest, running a parse, serializing the tree, and
118 |   comparing it to the expected output.
119 | 
120 |   The vague name is so nosetests doesn't try to run it as a test.
121 |   """
122 |   def impl(self, inner_html, input, expected, errors):
123 |     p = html5lib_adapter.HTMLParser(
124 |             tree=TREEBUILDER(namespaceHTMLElements=True))
125 | 
126 |     if inner_html:
127 |       document = p.parseFragment(
128 |           StringIO.StringIO(input), inner_html.replace('math ', 'mathml '))
129 |     else:
130 |       document = p.parse(StringIO.StringIO(input))
131 | 
132 |     with warnings.catch_warnings():
133 |       # Etree serializer in html5lib uses a deprecated getchildren() API.
134 |       warnings.filterwarnings('ignore', category=DeprecationWarning)
135 |       output = convertTreeDump(p.tree.testSerializer(document))
136 | 
137 |     expected = re.compile(r'^(\s*)<(\S+)>', re.M).sub(
138 |         r'\1<html \2>', convertExpected(expected, 2))
139 |     # html5lib doesn't yet support the template tag, but it appears in the
140 |     # tests with the expectation that the template contents will be under the
141 |     # word 'contents', so we need to reformat that string a bit.
142 |     expected = reformatTemplateContents(expected)
143 | 
144 |     error_msg = '\n'.join(['\n\nInput:', input, '\nExpected:', expected,
145 |                            '\nReceived:', output])
146 |     self.assertEquals(expected, output,
147 |                       error_msg.encode('ascii', 'xmlcharrefreplace') + '\n')
148 |     # TODO(jdtang): Check error messages, when there's full error support.
149 | 
150 | 
151 | def BuildTestCases(cls):
152 |   for filename in html5lib_test_files():
153 |     test_name = os.path.basename(filename).replace('.dat', '')
154 |     for i, test in enumerate(TestData(filename)):
155 |       # html5lib parses <noscript> tags as if the scripting-enabled flag is
156 |       # set, while we parse as if the scripting-disabled flag is set (since we
157 |       # don't really support scripting and the resulting parse tree is often
158 |       # more useful for toolsmiths).  That means our output will differ by
159 |       # design from html5lib's, so we disable any of their tests that involve
160 |       # <noscript>
161 |       if '<noscript>' in test['data']:
162 |         continue
163 | 
164 |       # <command> has been renamed to <menuitem> in recent versions of the spec.
165 |       # html5lib 0.95 does not include this yet, and so we disable tests that
166 |       # include the old tag.
167 |       if '<command>' in test['data']:
168 |         continue
169 | 
170 |       def test_func(
171 |           self,
172 |           inner_html=test['document-fragment'],
173 |           input=test['data'],
174 |           expected=test['document'],
175 |           errors=test.get('errors', '').split('\n')):
176 |         return self.impl(inner_html, input, expected, errors)
177 |       test_func.__name__ = 'test_%s_%d' % (test_name, i + 1)
178 |       setattr(cls, test_func.__name__, test_func)
179 | 
180 | 
181 | if __name__ == '__main__':
182 |   BuildTestCases(Html5libAdapterTest)
183 |   unittest.main()
184 | 


--------------------------------------------------------------------------------
/python/gumbo/soup_adapter.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2012 Google Inc. All Rights Reserved.
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | #     http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | #
 15 | 
 16 | """Adapter between Gumbo and BeautifulSoup.
 17 | 
 18 | This parses an HTML document and gives back a BeautifulSoup object, which you
 19 | can then manipulate like a normal BeautifulSoup parse tree.
 20 | """
 21 | 
 22 | __author__ = 'jdtang@google.com (Jonathan Tang)'
 23 | 
 24 | import BeautifulSoup
 25 | 
 26 | import gumboc
 27 | 
 28 | 
 29 | def _utf8(text):
 30 |   return text.decode('utf-8', 'replace')
 31 | 
 32 | 
 33 | def _add_source_info(obj, original_text, start_pos, end_pos):
 34 |   obj.original = str(original_text)
 35 |   obj.line = start_pos.line
 36 |   obj.col = start_pos.column
 37 |   obj.offset = start_pos.offset
 38 |   if end_pos:
 39 |     obj.end_line = end_pos.line
 40 |     obj.end_col = end_pos.column
 41 |     obj.end_offset = end_pos.offset
 42 | 
 43 | 
 44 | def _convert_attrs(attrs):
 45 |   # TODO(jdtang): Ideally attributes would pass along their positions as well,
 46 |   # but I can't extend the built in str objects with new attributes.  Maybe work
 47 |   # around this with a subclass in some way...
 48 |   return [(_utf8(attr.name), _utf8(attr.value)) for attr in attrs]
 49 | 
 50 | 
 51 | def _add_document(soup, element):
 52 |   # Currently ignored, since there's no real place for this in the BeautifulSoup
 53 |   # API.
 54 |   pass
 55 | 
 56 | 
 57 | def _add_element(soup, element):
 58 |   # TODO(jdtang): Expose next/previous in gumbo so they can be passed along to
 59 |   # BeautifulSoup.
 60 |   tag = BeautifulSoup.Tag(
 61 |       soup, _utf8(element.tag_name), _convert_attrs(element.attributes))
 62 |   for child in element.children:
 63 |     tag.append(_add_node(soup, child))
 64 |   _add_source_info(
 65 |       tag, element.original_tag, element.start_pos, element.end_pos)
 66 |   tag.original_end_tag = str(element.original_end_tag)
 67 |   return tag
 68 | 
 69 | 
 70 | def _add_text(cls):
 71 |   def add_text_internal(soup, element):
 72 |     text = cls(_utf8(element.text))
 73 |     _add_source_info(text, element.original_text, element.start_pos, None)
 74 |     return text
 75 |   return add_text_internal
 76 | 
 77 | 
 78 | _HANDLERS = [
 79 |     _add_document,
 80 |     _add_element,
 81 |     _add_text(BeautifulSoup.NavigableString),
 82 |     _add_text(BeautifulSoup.CData),
 83 |     _add_text(BeautifulSoup.Comment),
 84 |     _add_text(BeautifulSoup.NavigableString),
 85 |     _add_element,
 86 |     ]
 87 | 
 88 | 
 89 | def _add_node(soup, node):
 90 |   return _HANDLERS[node.type.value](soup, node.contents)
 91 | 
 92 | 
 93 | def _add_next_prev_pointers(soup):
 94 |   def _traverse(node):
 95 |     # .findAll requires the .next pointer, which is what we're trying to add
 96 |     # when we call this, and so we manually supply a generator to yield the
 97 |     # nodes in DOM order.
 98 |     yield node
 99 |     try:
100 |       for child in node.contents:
101 |         for descendant in _traverse(child):
102 |           yield descendant
103 |     except AttributeError:
104 |       # Not an element.
105 |       return
106 | 
107 |   nodes = sorted(_traverse(soup), key=lambda node: node.offset)
108 |   if nodes:
109 |     nodes[0].previous = None
110 |     nodes[-1].next = None
111 |   for i, node in enumerate(nodes[1:-1], 1):
112 |     nodes[i-1].next = node
113 |     node.previous = nodes[i-1]
114 | 
115 | 
116 | def parse(text, **kwargs):
117 |   with gumboc.parse(text, **kwargs) as output:
118 |     soup = BeautifulSoup.BeautifulSoup()
119 |     soup.append(_add_node(soup, output.contents.root.contents))
120 |     _add_next_prev_pointers(soup)
121 |     return soup
122 | 


--------------------------------------------------------------------------------
/python/gumbo/soup_adapter_test.py:
--------------------------------------------------------------------------------
 1 | # Copyright 2012 Google Inc. All Rights Reserved.
 2 | #
 3 | # Licensed under the Apache License, Version 2.0 (the "License");
 4 | # you may not use this file except in compliance with the License.
 5 | # You may obtain a copy of the License at
 6 | #
 7 | #     http://www.apache.org/licenses/LICENSE-2.0
 8 | #
 9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | #
15 | """Tests for the Gumbo's BeautifulSoup Python adapter."""
16 | 
17 | __author__ = 'jdtang@google.com (Jonathan Tang)'
18 | 
19 | import unittest
20 | 
21 | import soup_adapter
22 | 
23 | 
24 | class SoupAdapterTest(unittest.TestCase):
25 | 
26 |   def testSimpleParse(self):
27 |     soup = soup_adapter.parse(
28 |         """
29 |         <ul>
30 |           <li class=odd><a href="one.html">One</a>
31 |           <li class="even"><a href="two.html">Two</a>
32 |           <li class='odd'><a href="three.html">Three</a>
33 |           <li class="even"><a href="four.html">Four</a>
34 |         </ul>
35 |         """)
36 | 
37 |     head = soup.head
38 |     self.assertEquals(soup, head.parent.parent)
39 |     self.assertEquals(u'head', head.name)
40 |     self.assertEquals(0, len(head))
41 | 
42 |     body = soup.body
43 |     self.assertEquals(head, body.previousSibling)
44 |     self.assertEquals(2, len(body))  # <ul> + trailing whitespace
45 |     self.assertEquals(u'ul', body.contents[0].name)
46 |     self.assertEquals(body, head.next)
47 |     self.assertEquals(head, body.previous)
48 | 
49 |     list_items = body.findAll('li')
50 |     self.assertEquals(4, len(list_items))
51 | 
52 |     evens = body('li', 'even')
53 |     self.assertEquals(2, len(evens))
54 | 
55 |     a2 = body.find('a', href='two.html')
56 |     self.assertEquals(u'a', a2.name)
57 |     self.assertEquals(u'Two', a2.contents[0])
58 |     self.assertEquals(a2, evens[0].next)
59 |     self.assertEquals(evens[0], a2.previous)
60 | 
61 |     li2 = a2.parent
62 |     self.assertEquals(u'li', li2.name)
63 |     self.assertEquals(u'even', li2['class'])
64 |     self.assertEquals(list_items[1], li2)
65 |     self.assertEquals(evens[0], li2)
66 | 
67 | if __name__ == '__main__':
68 |   unittest.main()
69 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | import sys
  3 | from setuptools import setup
  4 | from setuptools.command.sdist import sdist
  5 | 
  6 | _name_of_lib = 'libgumbo.so'
  7 | if sys.platform.startswith('darwin'):
  8 |     _name_of_lib = 'libgumbo.dylib'
  9 | elif sys.platform.startswith('win'):
 10 |     _name_of_lib = 'gumbo.dll'
 11 | 
 12 | class CustomSdistCommand(sdist):
 13 |     """Customized Sdist command, to copy libgumbo.so into the Python directory
 14 |     so that it can be installed with `pip install`."""
 15 |     def run(self):
 16 |         try:
 17 |             import shutil
 18 |             shutil.copyfile('.libs/' + _name_of_lib,
 19 |                 'python/gumbo/' + _name_of_lib)
 20 |             sdist.run(self)
 21 |         except IOError as e:
 22 |             print(e)
 23 | 
 24 | 
 25 | README = '''Gumbo - A pure-C HTML5 parser.
 26 | ==============================
 27 | 
 28 | Gumbo is an implementation of the `HTML5 parsing algorithm <http://www.whatwg.org/specs/web-apps/current-work/multipage/#auto-toc-12>`_ implemented
 29 | as a pure C99 library with no outside dependencies.  It's designed to serve
 30 | as a building block for other tools and libraries such as linters,
 31 | validators, templating languages, and refactoring and analysis tools.  This
 32 | package contains the library itself, Python ctypes bindings for the library, and
 33 | adapters for html5lib and BeautifulSoup (3.2) that give it the same API as those
 34 | libaries.
 35 | 
 36 | Goals & features:
 37 | -----------------
 38 | 
 39 | - Robust and resilient to bad input.
 40 | 
 41 | - Simple API that can be easily wrapped by other languages.
 42 | 
 43 | - Support for source locations and pointers back to the original text.
 44 | 
 45 | - Relatively lightweight, with no outside dependencies.
 46 | 
 47 | - Passes all `html5lib-0.95 tests <https://github.com/html5lib/html5lib-tests>`_.
 48 | 
 49 | - Tested on over 2.5 billion pages from Google's index.
 50 | 
 51 | Non-goals:
 52 | ----------
 53 | 
 54 | - Execution speed.  Gumbo gains some of this by virtue of being written in
 55 |   C, but it is not an important consideration for the intended use-case, and
 56 |   was not a major design factor.
 57 | 
 58 | - Support for encodings other than UTF-8.  For the most part, client code
 59 |   can convert the input stream to UTF-8 text using another library before
 60 |   processing.
 61 | 
 62 | - Security.  Gumbo was initially designed for a product that worked with
 63 |   trusted input files only.  We're working to harden this and make sure that it
 64 |   behaves as expected even on malicious input, but for now, Gumbo should only be
 65 |   run on trusted input or within a sandbox.
 66 | 
 67 | - C89 support.  Most major compilers support C99 by now; the major exception
 68 |   (Microsoft Visual Studio) should be able to compile this in C++ mode with
 69 |   relatively few changes.  (Bug reports welcome.)
 70 | 
 71 | Wishlist (aka "We couldn't get these into the original release, but are
 72 | hoping to add them soon"):
 73 | 
 74 | - Support for recent HTML5 spec changes to support the template tag.
 75 | 
 76 | - Support for fragment parsing.
 77 | 
 78 | - Full-featured error reporting.
 79 | 
 80 | - Bindings in other languages.
 81 | 
 82 | Installation
 83 | ------------
 84 | 
 85 | ```pip install gumbo``` should do it.  If you have a local copy, ```python
 86 | setup.py install``` from the root directory.
 87 | 
 88 | The `html5lib <https://pypi.python.org/pypi/html5lib/0.999>`_ and
 89 | `BeautifulSoup <https://pypi.python.org/pypi/BeautifulSoup/3.2.1>`_ adapters
 90 | require that their respective libraries be installed separately to work.
 91 | 
 92 | Basic Usage
 93 | -----------
 94 | 
 95 | For the ctypes bindings:
 96 | 
 97 | .. code-block:: python
 98 | 
 99 |     import gumbo
100 |     
101 |     with gumbo.parse(text) as output:
102 |         root = output.contents.root.contents
103 |         # root is a Node object representing the root of the parse tree
104 |         # tree-walk over it as necessary.
105 | 
106 | For the BeautifulSoup bindings:
107 | 
108 | .. code-block:: python
109 | 
110 |     import gumbo
111 | 
112 |     soup = gumbo.soup_parse(text)
113 |     # soup is a BeautifulSoup object representing the parse tree.
114 | 
115 | For the html5lib bindings:
116 | 
117 | .. code-block:: python
118 | 
119 |     from gumbo import html5lib
120 | 
121 |     doc = html5lib.parse(text[, treebuilder='lxml'])
122 | 
123 | Recommended best-practice for Python usage is to use one of the adapters to
124 | an existing API (personally, I prefer BeautifulSoup) and write your program
125 | in terms of those.  The raw CTypes bindings should be considered building
126 | blocks for higher-level libraries and rarely referenced directly.
127 | 
128 | See the source code, Pydoc, and implementation of soup_adapter and
129 | html5lib_adapter for more information.
130 | 
131 | A note on API/ABI compatibility
132 | -------------------------------
133 | 
134 | We'll make a best effort to preserve API compatibility between releases.
135 | The initial release is a 0.9 (beta) release to solicit comments from early
136 | adopters, but if no major problems are found with the API, a 1.0 release
137 | will follow shortly, and the API of that should be considered stable.  If
138 | changes are necessary, we follow [semantic versioning][].
139 | 
140 | We make no such guarantees about the ABI, and it's very likely that
141 | subsequent versions may require a recompile of client code.  For this
142 | reason, we recommend NOT using Gumbo data structures throughout a program,
143 | and instead limiting them to a translation layer that picks out whatever
144 | data is needed from the parse tree and then converts that to persistent
145 | data structures more appropriate for the application.  The API is
146 | structured to encourage this use, with a single delete function for the
147 | whole parse tree, and is not designed with mutation in mind.
148 | 
149 | Most of this is transparent to Python usage, as the Python adapters are all
150 | built with this in mind.  However, since ctypes requires ABI compatibility, it
151 | does mean you'll have to re-deploy the gumboc library and C extension when
152 | upgrading to a new version.
153 | '''
154 | 
155 | CLASSIFIERS = [
156 |     'Development Status :: 4 - Beta',
157 |     'Intended Audience :: Developers',
158 |     'License :: OSI Approved :: Apache Software License',
159 |     'Operating System :: Unix',
160 |     'Operating System :: POSIX :: Linux',
161 |     'Programming Language :: C',
162 |     'Programming Language :: Python',
163 |     'Programming Language :: Python :: 2',
164 |     'Programming Language :: Python :: 2.7',
165 |     'Programming Language :: Python :: 3',
166 |     'Programming Language :: Python :: 3.4',
167 |     'Topic :: Software Development :: Libraries :: Python Modules',
168 |     'Topic :: Text Processing :: Markup :: HTML'
169 | ]
170 | 
171 | setup(name='gumbo',
172 |       version='0.10.1',
173 |       description='Python bindings for Gumbo HTML parser',
174 |       long_description=README,
175 |       url='http://github.com/google/gumbo-parser',
176 |       keywords='gumbo html html5 parser google html5lib beautifulsoup',
177 |       author='Jonathan Tang',
178 |       author_email='jonathan.d.tang@gmail.com',
179 |       license='Apache 2.0',
180 |       classifiers=CLASSIFIERS,
181 |       packages=['gumbo'],
182 |       package_dir={'': 'python'},
183 |       package_data={'gumbo': [_name_of_lib]},
184 |       cmdclass={ 'sdist': CustomSdistCommand },
185 |       zip_safe=False)
186 | 


--------------------------------------------------------------------------------
/src/attribute.c:
--------------------------------------------------------------------------------
 1 | // Copyright 2010 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | //
15 | // Author: jdtang@google.com (Jonathan Tang)
16 | 
17 | #include "attribute.h"
18 | 
19 | #include <assert.h>
20 | #include <stdlib.h>
21 | #include <string.h>
22 | #include <strings.h>
23 | 
24 | #include "util.h"
25 | 
26 | struct GumboInternalParser;
27 | 
28 | GumboAttribute* gumbo_get_attribute(
29 |     const GumboVector* attributes, const char* name) {
30 |   for (unsigned int i = 0; i < attributes->length; ++i) {
31 |     GumboAttribute* attr = attributes->data[i];
32 |     if (!strcasecmp(attr->name, name)) {
33 |       return attr;
34 |     }
35 |   }
36 |   return NULL;
37 | }
38 | 
39 | void gumbo_destroy_attribute(
40 |     struct GumboInternalParser* parser, GumboAttribute* attribute) {
41 |   gumbo_parser_deallocate(parser, (void*) attribute->name);
42 |   gumbo_parser_deallocate(parser, (void*) attribute->value);
43 |   gumbo_parser_deallocate(parser, (void*) attribute);
44 | }
45 | 


--------------------------------------------------------------------------------
/src/attribute.h:
--------------------------------------------------------------------------------
 1 | // Copyright 2010 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | //
15 | // Author: jdtang@google.com (Jonathan Tang)
16 | 
17 | #ifndef GUMBO_ATTRIBUTE_H_
18 | #define GUMBO_ATTRIBUTE_H_
19 | 
20 | #include "gumbo.h"
21 | 
22 | #ifdef __cplusplus
23 | extern "C" {
24 | #endif
25 | 
26 | struct GumboInternalParser;
27 | 
28 | // Release the memory used for an GumboAttribute, including the attribute
29 | // itself.
30 | void gumbo_destroy_attribute(
31 |     struct GumboInternalParser* parser, GumboAttribute* attribute);
32 | 
33 | #ifdef __cplusplus
34 | }
35 | #endif
36 | 
37 | #endif  // GUMBO_ATTRIBUTE_H_
38 | 


--------------------------------------------------------------------------------
/src/char_ref.h:
--------------------------------------------------------------------------------
 1 | // Copyright 2011 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | //
15 | // Author: jdtang@google.com (Jonathan Tang)
16 | //
17 | // Internal header for character reference handling; this should not be exposed
18 | // transitively by any public API header.  This is why the functions aren't
19 | // namespaced.
20 | 
21 | #ifndef GUMBO_CHAR_REF_H_
22 | #define GUMBO_CHAR_REF_H_
23 | 
24 | #include <stdbool.h>
25 | 
26 | #ifdef __cplusplus
27 | extern "C" {
28 | #endif
29 | 
30 | struct GumboInternalParser;
31 | struct GumboInternalUtf8Iterator;
32 | 
33 | // Value that indicates no character was produced.
34 | extern const int kGumboNoChar;
35 | 
36 | // Certain named character references generate two codepoints, not one, and so
37 | // the consume_char_ref subroutine needs to return this instead of an int.  The
38 | // first field will be kGumboNoChar if no character reference was found; the
39 | // second field will be kGumboNoChar if that is the case or if the character
40 | // reference returns only a single codepoint.
41 | typedef struct {
42 |   int first;
43 |   int second;
44 | } OneOrTwoCodepoints;
45 | 
46 | // Implements the "consume a character reference" section of the spec.
47 | // This reads in characters from the input as necessary, and fills in a
48 | // OneOrTwoCodepoints struct containing the characters read.  It may add parse
49 | // errors to the GumboParser's errors vector, if the spec calls for it.  Pass a
50 | // space for the "additional allowed char" when the spec says "with no
51 | // additional allowed char".  Returns false on parse error, true otherwise.
52 | bool consume_char_ref(struct GumboInternalParser* parser,
53 |     struct GumboInternalUtf8Iterator* input, int additional_allowed_char,
54 |     bool is_in_attribute, OneOrTwoCodepoints* output);
55 | 
56 | #ifdef __cplusplus
57 | }
58 | #endif
59 | 
60 | #endif  // GUMBO_CHAR_REF_H_
61 | 


--------------------------------------------------------------------------------
/src/error.h:
--------------------------------------------------------------------------------
  1 | // Copyright 2010 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | //
 15 | // Author: jdtang@google.com (Jonathan Tang)
 16 | //
 17 | // Error types, enums, and handling functions.
 18 | 
 19 | #ifndef GUMBO_ERROR_H_
 20 | #define GUMBO_ERROR_H_
 21 | #ifdef _MSC_VER
 22 | #define _CRT_SECURE_NO_WARNINGS
 23 | #endif
 24 | #include <stdint.h>
 25 | 
 26 | #include "gumbo.h"
 27 | #include "insertion_mode.h"
 28 | #include "string_buffer.h"
 29 | #include "token_type.h"
 30 | 
 31 | #ifdef __cplusplus
 32 | extern "C" {
 33 | #endif
 34 | 
 35 | struct GumboInternalParser;
 36 | 
 37 | typedef enum {
 38 |   GUMBO_ERR_UTF8_INVALID,
 39 |   GUMBO_ERR_UTF8_TRUNCATED,
 40 |   GUMBO_ERR_UTF8_NULL,
 41 |   GUMBO_ERR_NUMERIC_CHAR_REF_NO_DIGITS,
 42 |   GUMBO_ERR_NUMERIC_CHAR_REF_WITHOUT_SEMICOLON,
 43 |   GUMBO_ERR_NUMERIC_CHAR_REF_INVALID,
 44 |   GUMBO_ERR_NAMED_CHAR_REF_WITHOUT_SEMICOLON,
 45 |   GUMBO_ERR_NAMED_CHAR_REF_INVALID,
 46 |   GUMBO_ERR_TAG_STARTS_WITH_QUESTION,
 47 |   GUMBO_ERR_TAG_EOF,
 48 |   GUMBO_ERR_TAG_INVALID,
 49 |   GUMBO_ERR_CLOSE_TAG_EMPTY,
 50 |   GUMBO_ERR_CLOSE_TAG_EOF,
 51 |   GUMBO_ERR_CLOSE_TAG_INVALID,
 52 |   GUMBO_ERR_SCRIPT_EOF,
 53 |   GUMBO_ERR_ATTR_NAME_EOF,
 54 |   GUMBO_ERR_ATTR_NAME_INVALID,
 55 |   GUMBO_ERR_ATTR_DOUBLE_QUOTE_EOF,
 56 |   GUMBO_ERR_ATTR_SINGLE_QUOTE_EOF,
 57 |   GUMBO_ERR_ATTR_UNQUOTED_EOF,
 58 |   GUMBO_ERR_ATTR_UNQUOTED_RIGHT_BRACKET,
 59 |   GUMBO_ERR_ATTR_UNQUOTED_EQUALS,
 60 |   GUMBO_ERR_ATTR_AFTER_EOF,
 61 |   GUMBO_ERR_ATTR_AFTER_INVALID,
 62 |   GUMBO_ERR_DUPLICATE_ATTR,
 63 |   GUMBO_ERR_SOLIDUS_EOF,
 64 |   GUMBO_ERR_SOLIDUS_INVALID,
 65 |   GUMBO_ERR_DASHES_OR_DOCTYPE,
 66 |   GUMBO_ERR_COMMENT_EOF,
 67 |   GUMBO_ERR_COMMENT_INVALID,
 68 |   GUMBO_ERR_COMMENT_BANG_AFTER_DOUBLE_DASH,
 69 |   GUMBO_ERR_COMMENT_DASH_AFTER_DOUBLE_DASH,
 70 |   GUMBO_ERR_COMMENT_SPACE_AFTER_DOUBLE_DASH,
 71 |   GUMBO_ERR_COMMENT_END_BANG_EOF,
 72 |   GUMBO_ERR_DOCTYPE_EOF,
 73 |   GUMBO_ERR_DOCTYPE_INVALID,
 74 |   GUMBO_ERR_DOCTYPE_SPACE,
 75 |   GUMBO_ERR_DOCTYPE_RIGHT_BRACKET,
 76 |   GUMBO_ERR_DOCTYPE_SPACE_OR_RIGHT_BRACKET,
 77 |   GUMBO_ERR_DOCTYPE_END,
 78 |   GUMBO_ERR_PARSER,
 79 |   GUMBO_ERR_UNACKNOWLEDGED_SELF_CLOSING_TAG,
 80 | } GumboErrorType;
 81 | 
 82 | // Additional data for duplicated attributes.
 83 | typedef struct GumboInternalDuplicateAttrError {
 84 |   // The name of the attribute.  Owned by this struct.
 85 |   const char* name;
 86 | 
 87 |   // The (0-based) index within the attributes vector of the original
 88 |   // occurrence.
 89 |   unsigned int original_index;
 90 | 
 91 |   // The (0-based) index where the new occurrence would be.
 92 |   unsigned int new_index;
 93 | } GumboDuplicateAttrError;
 94 | 
 95 | // A simplified representation of the tokenizer state, designed to be more
 96 | // useful to clients of this library than the internal representation.  This
 97 | // condenses the actual states used in the tokenizer state machine into a few
 98 | // values that will be familiar to users of HTML.
 99 | typedef enum {
100 |   GUMBO_ERR_TOKENIZER_DATA,
101 |   GUMBO_ERR_TOKENIZER_CHAR_REF,
102 |   GUMBO_ERR_TOKENIZER_RCDATA,
103 |   GUMBO_ERR_TOKENIZER_RAWTEXT,
104 |   GUMBO_ERR_TOKENIZER_PLAINTEXT,
105 |   GUMBO_ERR_TOKENIZER_SCRIPT,
106 |   GUMBO_ERR_TOKENIZER_TAG,
107 |   GUMBO_ERR_TOKENIZER_SELF_CLOSING_TAG,
108 |   GUMBO_ERR_TOKENIZER_ATTR_NAME,
109 |   GUMBO_ERR_TOKENIZER_ATTR_VALUE,
110 |   GUMBO_ERR_TOKENIZER_MARKUP_DECLARATION,
111 |   GUMBO_ERR_TOKENIZER_COMMENT,
112 |   GUMBO_ERR_TOKENIZER_DOCTYPE,
113 |   GUMBO_ERR_TOKENIZER_CDATA,
114 | } GumboTokenizerErrorState;
115 | 
116 | // Additional data for tokenizer errors.
117 | // This records the current state and codepoint encountered - this is usually
118 | // enough to reconstruct what went wrong and provide a friendly error message.
119 | typedef struct GumboInternalTokenizerError {
120 |   // The bad codepoint encountered.
121 |   int codepoint;
122 | 
123 |   // The state that the tokenizer was in at the time.
124 |   GumboTokenizerErrorState state;
125 | } GumboTokenizerError;
126 | 
127 | // Additional data for parse errors.
128 | typedef struct GumboInternalParserError {
129 |   // The type of input token that resulted in this error.
130 |   GumboTokenType input_type;
131 | 
132 |   // The HTML tag of the input token.  TAG_UNKNOWN if this was not a tag token.
133 |   GumboTag input_tag;
134 | 
135 |   // The insertion mode that the parser was in at the time.
136 |   GumboInsertionMode parser_state;
137 | 
138 |   // The tag stack at the point of the error.  Note that this is an GumboVector
139 |   // of GumboTag's *stored by value* - cast the void* to an GumboTag directly to
140 |   // get at the tag.
141 |   GumboVector /* GumboTag */ tag_stack;
142 | } GumboParserError;
143 | 
144 | // The overall error struct representing an error in decoding/tokenizing/parsing
145 | // the HTML.  This contains an enumerated type flag, a source position, and then
146 | // a union of fields containing data specific to the error.
147 | typedef struct GumboInternalError {
148 |   // The type of error.
149 |   GumboErrorType type;
150 | 
151 |   // The position within the source file where the error occurred.
152 |   GumboSourcePosition position;
153 | 
154 |   // A pointer to the byte within the original source file text where the error
155 |   // occurred (note that this is not the same as position.offset, as that gives
156 |   // character-based instead of byte-based offsets).
157 |   const char* original_text;
158 | 
159 |   // Type-specific error information.
160 |   union {
161 |     // The code point we encountered, for:
162 |     // * GUMBO_ERR_UTF8_INVALID
163 |     // * GUMBO_ERR_UTF8_TRUNCATED
164 |     // * GUMBO_ERR_NUMERIC_CHAR_REF_WITHOUT_SEMICOLON
165 |     // * GUMBO_ERR_NUMERIC_CHAR_REF_INVALID
166 |     uint64_t codepoint;
167 | 
168 |     // Tokenizer errors.
169 |     GumboTokenizerError tokenizer;
170 | 
171 |     // Short textual data, for:
172 |     // * GUMBO_ERR_NAMED_CHAR_REF_WITHOUT_SEMICOLON
173 |     // * GUMBO_ERR_NAMED_CHAR_REF_INVALID
174 |     GumboStringPiece text;
175 | 
176 |     // Duplicate attribute data, for GUMBO_ERR_DUPLICATE_ATTR.
177 |     GumboDuplicateAttrError duplicate_attr;
178 | 
179 |     // Parser state, for GUMBO_ERR_PARSER and
180 |     // GUMBO_ERR_UNACKNOWLEDGE_SELF_CLOSING_TAG.
181 |     struct GumboInternalParserError parser;
182 |   } v;
183 | } GumboError;
184 | 
185 | // Adds a new error to the parser's error list, and returns a pointer to it so
186 | // that clients can fill out the rest of its fields.  May return NULL if we're
187 | // already over the max_errors field specified in GumboOptions.
188 | GumboError* gumbo_add_error(struct GumboInternalParser* parser);
189 | 
190 | // Initializes the errors vector in the parser.
191 | void gumbo_init_errors(struct GumboInternalParser* errors);
192 | 
193 | // Frees all the errors in the 'errors_' field of the parser.
194 | void gumbo_destroy_errors(struct GumboInternalParser* errors);
195 | 
196 | // Frees the memory used for a single GumboError.
197 | void gumbo_error_destroy(struct GumboInternalParser* parser, GumboError* error);
198 | 
199 | // Prints an error to a string.  This fills an empty GumboStringBuffer with a
200 | // freshly-allocated buffer containing the error message text.  The caller is
201 | // responsible for deleting the buffer.  (Note that the buffer is allocated with
202 | // the allocator specified in the GumboParser config and hence should be freed
203 | // by gumbo_parser_deallocate().)
204 | void gumbo_error_to_string(struct GumboInternalParser* parser,
205 |     const GumboError* error, GumboStringBuffer* output);
206 | 
207 | // Prints a caret diagnostic to a string.  This fills an empty GumboStringBuffer
208 | // with a freshly-allocated buffer containing the error message text.  The
209 | // caller is responsible for deleting the buffer.  (Note that the buffer is
210 | // allocated with the allocator specified in the GumboParser config and hence
211 | // should be freed by gumbo_parser_deallocate().)
212 | void gumbo_caret_diagnostic_to_string(struct GumboInternalParser* parser,
213 |     const GumboError* error, const char* source_text,
214 |     GumboStringBuffer* output);
215 | 
216 | // Like gumbo_caret_diagnostic_to_string, but prints the text to stdout instead
217 | // of writing to a string.
218 | void gumbo_print_caret_diagnostic(struct GumboInternalParser* parser,
219 |     const GumboError* error, const char* source_text);
220 | 
221 | #ifdef __cplusplus
222 | }
223 | #endif
224 | 
225 | #endif  // GUMBO_ERROR_H_
226 | 


--------------------------------------------------------------------------------
/src/insertion_mode.h:
--------------------------------------------------------------------------------
 1 | // Copyright 2011 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | //
15 | // Author: jdtang@google.com (Jonathan Tang)
16 | 
17 | #ifndef GUMBO_INSERTION_MODE_H_
18 | #define GUMBO_INSERTION_MODE_H_
19 | 
20 | #ifdef __cplusplus
21 | extern "C" {
22 | #endif
23 | 
24 | // http://www.whatwg.org/specs/web-apps/current-work/complete/parsing.html#insertion-mode
25 | // If new enum values are added, be sure to update the kTokenHandlers dispatch
26 | // table in parser.c.
27 | typedef enum {
28 |   GUMBO_INSERTION_MODE_INITIAL,
29 |   GUMBO_INSERTION_MODE_BEFORE_HTML,
30 |   GUMBO_INSERTION_MODE_BEFORE_HEAD,
31 |   GUMBO_INSERTION_MODE_IN_HEAD,
32 |   GUMBO_INSERTION_MODE_IN_HEAD_NOSCRIPT,
33 |   GUMBO_INSERTION_MODE_AFTER_HEAD,
34 |   GUMBO_INSERTION_MODE_IN_BODY,
35 |   GUMBO_INSERTION_MODE_TEXT,
36 |   GUMBO_INSERTION_MODE_IN_TABLE,
37 |   GUMBO_INSERTION_MODE_IN_TABLE_TEXT,
38 |   GUMBO_INSERTION_MODE_IN_CAPTION,
39 |   GUMBO_INSERTION_MODE_IN_COLUMN_GROUP,
40 |   GUMBO_INSERTION_MODE_IN_TABLE_BODY,
41 |   GUMBO_INSERTION_MODE_IN_ROW,
42 |   GUMBO_INSERTION_MODE_IN_CELL,
43 |   GUMBO_INSERTION_MODE_IN_SELECT,
44 |   GUMBO_INSERTION_MODE_IN_SELECT_IN_TABLE,
45 |   GUMBO_INSERTION_MODE_IN_TEMPLATE,
46 |   GUMBO_INSERTION_MODE_AFTER_BODY,
47 |   GUMBO_INSERTION_MODE_IN_FRAMESET,
48 |   GUMBO_INSERTION_MODE_AFTER_FRAMESET,
49 |   GUMBO_INSERTION_MODE_AFTER_AFTER_BODY,
50 |   GUMBO_INSERTION_MODE_AFTER_AFTER_FRAMESET
51 | } GumboInsertionMode;
52 | 
53 | #ifdef __cplusplus
54 | }  // extern C
55 | #endif
56 | 
57 | #endif  // GUMBO_INSERTION_MODE_H_
58 | 


--------------------------------------------------------------------------------
/src/parser.h:
--------------------------------------------------------------------------------
 1 | // Copyright 2010 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | //
15 | // Author: jdtang@google.com (Jonathan Tang)
16 | //
17 | // Contains the definition of the top-level GumboParser structure that's
18 | // threaded through basically every internal function in the library.
19 | 
20 | #ifndef GUMBO_PARSER_H_
21 | #define GUMBO_PARSER_H_
22 | 
23 | #ifdef __cplusplus
24 | extern "C" {
25 | #endif
26 | 
27 | struct GumboInternalParserState;
28 | struct GumboInternalOutput;
29 | struct GumboInternalOptions;
30 | struct GumboInternalTokenizerState;
31 | 
32 | // An overarching struct that's threaded through (nearly) all functions in the
33 | // library, OOP-style.  This gives each function access to the options and
34 | // output, along with any internal state needed for the parse.
35 | typedef struct GumboInternalParser {
36 |   // Settings for this parse run.
37 |   const struct GumboInternalOptions* _options;
38 | 
39 |   // Output for the parse.
40 |   struct GumboInternalOutput* _output;
41 | 
42 |   // The internal tokenizer state, defined as a pointer to avoid a cyclic
43 |   // dependency on html5tokenizer.h.  The main parse routine is responsible for
44 |   // initializing this on parse start, and destroying it on parse end.
45 |   // End-users will never see a non-garbage value in this pointer.
46 |   struct GumboInternalTokenizerState* _tokenizer_state;
47 | 
48 |   // The internal parser state.  Initialized on parse start and destroyed on
49 |   // parse end; end-users will never see a non-garbage value in this pointer.
50 |   struct GumboInternalParserState* _parser_state;
51 | } GumboParser;
52 | 
53 | #ifdef __cplusplus
54 | }
55 | #endif
56 | 
57 | #endif  // GUMBO_PARSER_H_
58 | 


--------------------------------------------------------------------------------
/src/string_buffer.c:
--------------------------------------------------------------------------------
  1 | // Copyright 2010 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | //
 15 | // Author: jdtang@google.com (Jonathan Tang)
 16 | 
 17 | #include "string_buffer.h"
 18 | 
 19 | #include <assert.h>
 20 | #include <stdlib.h>
 21 | #include <string.h>
 22 | #include <strings.h>
 23 | 
 24 | #include "string_piece.h"
 25 | #include "util.h"
 26 | 
 27 | struct GumboInternalParser;
 28 | 
 29 | // Size chosen via statistical analysis of ~60K websites.
 30 | // 99% of text nodes and 98% of attribute names/values fit in this initial size.
 31 | static const size_t kDefaultStringBufferSize = 5;
 32 | 
 33 | static void maybe_resize_string_buffer(struct GumboInternalParser* parser,
 34 |     size_t additional_chars, GumboStringBuffer* buffer) {
 35 |   size_t new_length = buffer->length + additional_chars;
 36 |   size_t new_capacity = buffer->capacity;
 37 |   while (new_capacity < new_length) {
 38 |     new_capacity *= 2;
 39 |   }
 40 |   if (new_capacity != buffer->capacity) {
 41 |     char* new_data = gumbo_parser_allocate(parser, new_capacity);
 42 |     memcpy(new_data, buffer->data, buffer->length);
 43 |     gumbo_parser_deallocate(parser, buffer->data);
 44 |     buffer->data = new_data;
 45 |     buffer->capacity = new_capacity;
 46 |   }
 47 | }
 48 | 
 49 | void gumbo_string_buffer_init(
 50 |     struct GumboInternalParser* parser, GumboStringBuffer* output) {
 51 |   output->data = gumbo_parser_allocate(parser, kDefaultStringBufferSize);
 52 |   output->length = 0;
 53 |   output->capacity = kDefaultStringBufferSize;
 54 | }
 55 | 
 56 | void gumbo_string_buffer_reserve(struct GumboInternalParser* parser,
 57 |     size_t min_capacity, GumboStringBuffer* output) {
 58 |   maybe_resize_string_buffer(parser, min_capacity - output->length, output);
 59 | }
 60 | 
 61 | void gumbo_string_buffer_append_codepoint(
 62 |     struct GumboInternalParser* parser, int c, GumboStringBuffer* output) {
 63 |   // num_bytes is actually the number of continuation bytes, 1 less than the
 64 |   // total number of bytes.  This is done to keep the loop below simple and
 65 |   // should probably change if we unroll it.
 66 |   int num_bytes, prefix;
 67 |   if (c <= 0x7f) {
 68 |     num_bytes = 0;
 69 |     prefix = 0;
 70 |   } else if (c <= 0x7ff) {
 71 |     num_bytes = 1;
 72 |     prefix = 0xc0;
 73 |   } else if (c <= 0xffff) {
 74 |     num_bytes = 2;
 75 |     prefix = 0xe0;
 76 |   } else {
 77 |     num_bytes = 3;
 78 |     prefix = 0xf0;
 79 |   }
 80 |   maybe_resize_string_buffer(parser, num_bytes + 1, output);
 81 |   output->data[output->length++] = prefix | (c >> (num_bytes * 6));
 82 |   for (int i = num_bytes - 1; i >= 0; --i) {
 83 |     output->data[output->length++] = 0x80 | (0x3f & (c >> (i * 6)));
 84 |   }
 85 | }
 86 | 
 87 | void gumbo_string_buffer_append_string(struct GumboInternalParser* parser,
 88 |     GumboStringPiece* str, GumboStringBuffer* output) {
 89 |   maybe_resize_string_buffer(parser, str->length, output);
 90 |   memcpy(output->data + output->length, str->data, str->length);
 91 |   output->length += str->length;
 92 | }
 93 | 
 94 | char* gumbo_string_buffer_to_string(
 95 |     struct GumboInternalParser* parser, GumboStringBuffer* input) {
 96 |   char* buffer = gumbo_parser_allocate(parser, input->length + 1);
 97 |   memcpy(buffer, input->data, input->length);
 98 |   buffer[input->length] = '\0';
 99 |   return buffer;
100 | }
101 | 
102 | void gumbo_string_buffer_clear(
103 |     struct GumboInternalParser* parser, GumboStringBuffer* input) {
104 |   input->length = 0;
105 | }
106 | 
107 | void gumbo_string_buffer_destroy(
108 |     struct GumboInternalParser* parser, GumboStringBuffer* buffer) {
109 |   gumbo_parser_deallocate(parser, buffer->data);
110 | }
111 | 


--------------------------------------------------------------------------------
/src/string_buffer.h:
--------------------------------------------------------------------------------
 1 | // Copyright 2010 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | //
15 | // Author: jdtang@google.com (Jonathan Tang)
16 | //
17 | #ifndef GUMBO_STRING_BUFFER_H_
18 | #define GUMBO_STRING_BUFFER_H_
19 | 
20 | #include <stdbool.h>
21 | #include <stddef.h>
22 | 
23 | #include "gumbo.h"
24 | 
25 | #ifdef __cplusplus
26 | extern "C" {
27 | #endif
28 | 
29 | struct GumboInternalParser;
30 | 
31 | // A struct representing a mutable, growable string.  This consists of a
32 | // heap-allocated buffer that may grow (by doubling) as necessary.  When
33 | // converting to a string, this allocates a new buffer that is only as long as
34 | // it needs to be.  Note that the internal buffer here is *not* nul-terminated,
35 | // so be sure not to use ordinary string manipulation functions on it.
36 | typedef struct {
37 |   // A pointer to the beginning of the string.  NULL iff length == 0.
38 |   char* data;
39 | 
40 |   // The length of the string fragment, in bytes.  May be zero.
41 |   size_t length;
42 | 
43 |   // The capacity of the buffer, in bytes.
44 |   size_t capacity;
45 | } GumboStringBuffer;
46 | 
47 | // Initializes a new GumboStringBuffer.
48 | void gumbo_string_buffer_init(
49 |     struct GumboInternalParser* parser, GumboStringBuffer* output);
50 | 
51 | // Ensures that the buffer contains at least a certain amount of space.  Most
52 | // useful with snprintf and the other length-delimited string functions, which
53 | // may want to write directly into the buffer.
54 | void gumbo_string_buffer_reserve(struct GumboInternalParser* parser,
55 |     size_t min_capacity, GumboStringBuffer* output);
56 | 
57 | // Appends a single Unicode codepoint onto the end of the GumboStringBuffer.
58 | // This is essentially a UTF-8 encoder, and may add 1-4 bytes depending on the
59 | // value of the codepoint.
60 | void gumbo_string_buffer_append_codepoint(
61 |     struct GumboInternalParser* parser, int c, GumboStringBuffer* output);
62 | 
63 | // Appends a string onto the end of the GumboStringBuffer.
64 | void gumbo_string_buffer_append_string(struct GumboInternalParser* parser,
65 |     GumboStringPiece* str, GumboStringBuffer* output);
66 | 
67 | // Converts this string buffer to const char*, alloctaing a new buffer for it.
68 | char* gumbo_string_buffer_to_string(
69 |     struct GumboInternalParser* parser, GumboStringBuffer* input);
70 | 
71 | // Reinitialize this string buffer.  This clears it by setting length=0.  It
72 | // does not zero out the buffer itself.
73 | void gumbo_string_buffer_clear(
74 |     struct GumboInternalParser* parser, GumboStringBuffer* input);
75 | 
76 | // Deallocates this GumboStringBuffer.
77 | void gumbo_string_buffer_destroy(
78 |     struct GumboInternalParser* parser, GumboStringBuffer* buffer);
79 | 
80 | #ifdef __cplusplus
81 | }
82 | #endif
83 | 
84 | #endif  // GUMBO_STRING_BUFFER_H_
85 | 


--------------------------------------------------------------------------------
/src/string_piece.c:
--------------------------------------------------------------------------------
 1 | // Copyright 2010 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | //
15 | // Author: jdtang@google.com (Jonathan Tang)
16 | 
17 | #include "string_piece.h"
18 | 
19 | #include <assert.h>
20 | #include <stdlib.h>
21 | #include <string.h>
22 | #include <strings.h>
23 | 
24 | #include "util.h"
25 | 
26 | struct GumboInternalParser;
27 | 
28 | const GumboStringPiece kGumboEmptyString = {NULL, 0};
29 | 
30 | bool gumbo_string_equals(
31 |     const GumboStringPiece* str1, const GumboStringPiece* str2) {
32 |   return str1->length == str2->length &&
33 |          !memcmp(str1->data, str2->data, str1->length);
34 | }
35 | 
36 | bool gumbo_string_equals_ignore_case(
37 |     const GumboStringPiece* str1, const GumboStringPiece* str2) {
38 |   return str1->length == str2->length &&
39 |          !strncasecmp(str1->data, str2->data, str1->length);
40 | }
41 | 
42 | void gumbo_string_copy(struct GumboInternalParser* parser,
43 |     GumboStringPiece* dest, const GumboStringPiece* source) {
44 |   dest->length = source->length;
45 |   char* buffer = gumbo_parser_allocate(parser, source->length);
46 |   memcpy(buffer, source->data, source->length);
47 |   dest->data = buffer;
48 | }
49 | 


--------------------------------------------------------------------------------
/src/string_piece.h:
--------------------------------------------------------------------------------
 1 | // Copyright 2010 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | //
15 | // Author: jdtang@google.com (Jonathan Tang)
16 | 
17 | #ifndef GUMBO_STRING_PIECE_H_
18 | #define GUMBO_STRING_PIECE_H_
19 | 
20 | #include "gumbo.h"
21 | 
22 | #ifdef __cplusplus
23 | extern "C" {
24 | #endif
25 | 
26 | struct GumboInternalParser;
27 | 
28 | // Performs a deep-copy of an GumboStringPiece, allocating a fresh buffer in the
29 | // destination and copying over the characters from source.  Dest should be
30 | // empty, with no buffer allocated; otherwise, this leaks it.
31 | void gumbo_string_copy(struct GumboInternalParser* parser,
32 |     GumboStringPiece* dest, const GumboStringPiece* source);
33 | 
34 | #ifdef __cplusplus
35 | }
36 | #endif
37 | 
38 | #endif  // GUMBO_STRING_PIECE_H_
39 | 


--------------------------------------------------------------------------------
/src/tag.c:
--------------------------------------------------------------------------------
 1 | // Copyright 2011 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | //
15 | // Author: jdtang@google.com (Jonathan Tang)
16 | 
17 | #include "gumbo.h"
18 | 
19 | #include <assert.h>
20 | #include <ctype.h>
21 | #include <string.h>
22 | 
23 | const char* kGumboTagNames[] = {
24 | #include "tag_strings.h"
25 |     "",  // TAG_UNKNOWN
26 |     "",  // TAG_LAST
27 | };
28 | 
29 | static const unsigned char kGumboTagSizes[] = {
30 | #include "tag_sizes.h"
31 |     0,  // TAG_UNKNOWN
32 |     0,  // TAG_LAST
33 | };
34 | 
35 | const char* gumbo_normalized_tagname(GumboTag tag) {
36 |   assert(tag <= GUMBO_TAG_LAST);
37 |   return kGumboTagNames[tag];
38 | }
39 | 
40 | void gumbo_tag_from_original_text(GumboStringPiece* text) {
41 |   if (text->data == NULL) {
42 |     return;
43 |   }
44 | 
45 |   assert(text->length >= 2);
46 |   assert(text->data[0] == '<');
47 |   assert(text->data[text->length - 1] == '>');
48 |   if (text->data[1] == '/') {
49 |     // End tag.
50 |     assert(text->length >= 3);
51 |     text->data += 2;  // Move past </
52 |     text->length -= 3;
53 |   } else {
54 |     // Start tag.
55 |     text->data += 1;  // Move past <
56 |     text->length -= 2;
57 |     // strnchr is apparently not a standard C library function, so I loop
58 |     // explicitly looking for whitespace or other illegal tag characters.
59 |     for (const char* c = text->data; c != text->data + text->length; ++c) {
60 |       if (isspace(*c) || *c == '/') {
61 |         text->length = c - text->data;
62 |         break;
63 |       }
64 |     }
65 |   }
66 | }
67 | 
68 | static int case_memcmp(const char* s1, const char* s2, unsigned int n) {
69 |   while (n--) {
70 |     unsigned char c1 = tolower(*s1++);
71 |     unsigned char c2 = tolower(*s2++);
72 |     if (c1 != c2) return (int) c1 - (int) c2;
73 |   }
74 |   return 0;
75 | }
76 | 
77 | #include "tag_gperf.h"
78 | #define TAG_MAP_SIZE (sizeof(kGumboTagMap) / sizeof(kGumboTagMap[0]))
79 | 
80 | GumboTag gumbo_tagn_enum(const char* tagname, unsigned int length) {
81 |   if (length) {
82 |     unsigned int key = tag_hash(tagname, length);
83 |     if (key < TAG_MAP_SIZE) {
84 |       GumboTag tag = kGumboTagMap[key];
85 |       if (length == kGumboTagSizes[(int) tag] &&
86 |           !case_memcmp(tagname, kGumboTagNames[(int) tag], length))
87 |         return tag;
88 |     }
89 |   }
90 |   return GUMBO_TAG_UNKNOWN;
91 | }
92 | 
93 | GumboTag gumbo_tag_enum(const char* tagname) {
94 |   return gumbo_tagn_enum(tagname, strlen(tagname));
95 | }
96 | 


--------------------------------------------------------------------------------
/src/tag.in:
--------------------------------------------------------------------------------
  1 | html
  2 | head
  3 | title
  4 | base
  5 | link
  6 | meta
  7 | style
  8 | script
  9 | noscript
 10 | template
 11 | body
 12 | article
 13 | section
 14 | nav
 15 | aside
 16 | h1
 17 | h2
 18 | h3
 19 | h4
 20 | h5
 21 | h6
 22 | hgroup
 23 | header
 24 | footer
 25 | address
 26 | p
 27 | hr
 28 | pre
 29 | blockquote
 30 | ol
 31 | ul
 32 | li
 33 | dl
 34 | dt
 35 | dd
 36 | figure
 37 | figcaption
 38 | main
 39 | div
 40 | a
 41 | em
 42 | strong
 43 | small
 44 | s
 45 | cite
 46 | q
 47 | dfn
 48 | abbr
 49 | data
 50 | time
 51 | code
 52 | var
 53 | samp
 54 | kbd
 55 | sub
 56 | sup
 57 | i
 58 | b
 59 | u
 60 | mark
 61 | ruby
 62 | rt
 63 | rp
 64 | bdi
 65 | bdo
 66 | span
 67 | br
 68 | wbr
 69 | ins
 70 | del
 71 | image
 72 | img
 73 | iframe
 74 | embed
 75 | object
 76 | param
 77 | video
 78 | audio
 79 | source
 80 | track
 81 | canvas
 82 | map
 83 | area
 84 | math
 85 | mi
 86 | mo
 87 | mn
 88 | ms
 89 | mtext
 90 | mglyph
 91 | malignmark
 92 | annotation-xml
 93 | svg
 94 | foreignobject
 95 | desc
 96 | table
 97 | caption
 98 | colgroup
 99 | col
100 | tbody
101 | thead
102 | tfoot
103 | tr
104 | td
105 | th
106 | form
107 | fieldset
108 | legend
109 | label
110 | input
111 | button
112 | select
113 | datalist
114 | optgroup
115 | option
116 | textarea
117 | keygen
118 | output
119 | progress
120 | meter
121 | details
122 | summary
123 | menu
124 | menuitem
125 | applet
126 | acronym
127 | bgsound
128 | dir
129 | frame
130 | frameset
131 | noframes
132 | isindex
133 | listing
134 | xmp
135 | nextid
136 | noembed
137 | plaintext
138 | rb
139 | strike
140 | basefont
141 | big
142 | blink
143 | center
144 | font
145 | marquee
146 | multicol
147 | nobr
148 | spacer
149 | tt
150 | rtc
151 | 


--------------------------------------------------------------------------------
/src/tag_enum.h:
--------------------------------------------------------------------------------
  1 | // Generated via `gentags.py src/tag.in`.
  2 | // Do not edit; edit src/tag.in instead.
  3 | // clang-format off
  4 | GUMBO_TAG_HTML,
  5 | GUMBO_TAG_HEAD,
  6 | GUMBO_TAG_TITLE,
  7 | GUMBO_TAG_BASE,
  8 | GUMBO_TAG_LINK,
  9 | GUMBO_TAG_META,
 10 | GUMBO_TAG_STYLE,
 11 | GUMBO_TAG_SCRIPT,
 12 | GUMBO_TAG_NOSCRIPT,
 13 | GUMBO_TAG_TEMPLATE,
 14 | GUMBO_TAG_BODY,
 15 | GUMBO_TAG_ARTICLE,
 16 | GUMBO_TAG_SECTION,
 17 | GUMBO_TAG_NAV,
 18 | GUMBO_TAG_ASIDE,
 19 | GUMBO_TAG_H1,
 20 | GUMBO_TAG_H2,
 21 | GUMBO_TAG_H3,
 22 | GUMBO_TAG_H4,
 23 | GUMBO_TAG_H5,
 24 | GUMBO_TAG_H6,
 25 | GUMBO_TAG_HGROUP,
 26 | GUMBO_TAG_HEADER,
 27 | GUMBO_TAG_FOOTER,
 28 | GUMBO_TAG_ADDRESS,
 29 | GUMBO_TAG_P,
 30 | GUMBO_TAG_HR,
 31 | GUMBO_TAG_PRE,
 32 | GUMBO_TAG_BLOCKQUOTE,
 33 | GUMBO_TAG_OL,
 34 | GUMBO_TAG_UL,
 35 | GUMBO_TAG_LI,
 36 | GUMBO_TAG_DL,
 37 | GUMBO_TAG_DT,
 38 | GUMBO_TAG_DD,
 39 | GUMBO_TAG_FIGURE,
 40 | GUMBO_TAG_FIGCAPTION,
 41 | GUMBO_TAG_MAIN,
 42 | GUMBO_TAG_DIV,
 43 | GUMBO_TAG_A,
 44 | GUMBO_TAG_EM,
 45 | GUMBO_TAG_STRONG,
 46 | GUMBO_TAG_SMALL,
 47 | GUMBO_TAG_S,
 48 | GUMBO_TAG_CITE,
 49 | GUMBO_TAG_Q,
 50 | GUMBO_TAG_DFN,
 51 | GUMBO_TAG_ABBR,
 52 | GUMBO_TAG_DATA,
 53 | GUMBO_TAG_TIME,
 54 | GUMBO_TAG_CODE,
 55 | GUMBO_TAG_VAR,
 56 | GUMBO_TAG_SAMP,
 57 | GUMBO_TAG_KBD,
 58 | GUMBO_TAG_SUB,
 59 | GUMBO_TAG_SUP,
 60 | GUMBO_TAG_I,
 61 | GUMBO_TAG_B,
 62 | GUMBO_TAG_U,
 63 | GUMBO_TAG_MARK,
 64 | GUMBO_TAG_RUBY,
 65 | GUMBO_TAG_RT,
 66 | GUMBO_TAG_RP,
 67 | GUMBO_TAG_BDI,
 68 | GUMBO_TAG_BDO,
 69 | GUMBO_TAG_SPAN,
 70 | GUMBO_TAG_BR,
 71 | GUMBO_TAG_WBR,
 72 | GUMBO_TAG_INS,
 73 | GUMBO_TAG_DEL,
 74 | GUMBO_TAG_IMAGE,
 75 | GUMBO_TAG_IMG,
 76 | GUMBO_TAG_IFRAME,
 77 | GUMBO_TAG_EMBED,
 78 | GUMBO_TAG_OBJECT,
 79 | GUMBO_TAG_PARAM,
 80 | GUMBO_TAG_VIDEO,
 81 | GUMBO_TAG_AUDIO,
 82 | GUMBO_TAG_SOURCE,
 83 | GUMBO_TAG_TRACK,
 84 | GUMBO_TAG_CANVAS,
 85 | GUMBO_TAG_MAP,
 86 | GUMBO_TAG_AREA,
 87 | GUMBO_TAG_MATH,
 88 | GUMBO_TAG_MI,
 89 | GUMBO_TAG_MO,
 90 | GUMBO_TAG_MN,
 91 | GUMBO_TAG_MS,
 92 | GUMBO_TAG_MTEXT,
 93 | GUMBO_TAG_MGLYPH,
 94 | GUMBO_TAG_MALIGNMARK,
 95 | GUMBO_TAG_ANNOTATION_XML,
 96 | GUMBO_TAG_SVG,
 97 | GUMBO_TAG_FOREIGNOBJECT,
 98 | GUMBO_TAG_DESC,
 99 | GUMBO_TAG_TABLE,
100 | GUMBO_TAG_CAPTION,
101 | GUMBO_TAG_COLGROUP,
102 | GUMBO_TAG_COL,
103 | GUMBO_TAG_TBODY,
104 | GUMBO_TAG_THEAD,
105 | GUMBO_TAG_TFOOT,
106 | GUMBO_TAG_TR,
107 | GUMBO_TAG_TD,
108 | GUMBO_TAG_TH,
109 | GUMBO_TAG_FORM,
110 | GUMBO_TAG_FIELDSET,
111 | GUMBO_TAG_LEGEND,
112 | GUMBO_TAG_LABEL,
113 | GUMBO_TAG_INPUT,
114 | GUMBO_TAG_BUTTON,
115 | GUMBO_TAG_SELECT,
116 | GUMBO_TAG_DATALIST,
117 | GUMBO_TAG_OPTGROUP,
118 | GUMBO_TAG_OPTION,
119 | GUMBO_TAG_TEXTAREA,
120 | GUMBO_TAG_KEYGEN,
121 | GUMBO_TAG_OUTPUT,
122 | GUMBO_TAG_PROGRESS,
123 | GUMBO_TAG_METER,
124 | GUMBO_TAG_DETAILS,
125 | GUMBO_TAG_SUMMARY,
126 | GUMBO_TAG_MENU,
127 | GUMBO_TAG_MENUITEM,
128 | GUMBO_TAG_APPLET,
129 | GUMBO_TAG_ACRONYM,
130 | GUMBO_TAG_BGSOUND,
131 | GUMBO_TAG_DIR,
132 | GUMBO_TAG_FRAME,
133 | GUMBO_TAG_FRAMESET,
134 | GUMBO_TAG_NOFRAMES,
135 | GUMBO_TAG_ISINDEX,
136 | GUMBO_TAG_LISTING,
137 | GUMBO_TAG_XMP,
138 | GUMBO_TAG_NEXTID,
139 | GUMBO_TAG_NOEMBED,
140 | GUMBO_TAG_PLAINTEXT,
141 | GUMBO_TAG_RB,
142 | GUMBO_TAG_STRIKE,
143 | GUMBO_TAG_BASEFONT,
144 | GUMBO_TAG_BIG,
145 | GUMBO_TAG_BLINK,
146 | GUMBO_TAG_CENTER,
147 | GUMBO_TAG_FONT,
148 | GUMBO_TAG_MARQUEE,
149 | GUMBO_TAG_MULTICOL,
150 | GUMBO_TAG_NOBR,
151 | GUMBO_TAG_SPACER,
152 | GUMBO_TAG_TT,
153 | GUMBO_TAG_RTC,
154 | 


--------------------------------------------------------------------------------
/src/tag_gperf.h:
--------------------------------------------------------------------------------
  1 | static unsigned int tag_hash(
  2 |     register const char *str, register unsigned int len) {
  3 |   static unsigned short asso_values[] = {296, 296, 296, 296, 296, 296, 296, 296,
  4 |       296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296,
  5 |       296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296,
  6 |       296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 6, 4, 3, 1, 1, 0,
  7 |       1, 0, 0, 296, 296, 296, 296, 296, 296, 296, 22, 73, 151, 4, 13, 59, 65, 2,
  8 |       69, 0, 134, 9, 16, 52, 55, 28, 101, 0, 1, 6, 63, 126, 104, 93, 124, 296,
  9 |       296, 296, 296, 296, 296, 296, 22, 73, 151, 4, 13, 59, 65, 2, 69, 0, 134,
 10 |       9, 16, 52, 55, 28, 101, 0, 1, 6, 63, 126, 104, 93, 124, 296, 296, 296,
 11 |       296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296,
 12 |       296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296,
 13 |       296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296,
 14 |       296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296,
 15 |       296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296,
 16 |       296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296,
 17 |       296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296,
 18 |       296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296,
 19 |       296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, 296};
 20 |   register unsigned int hval = len;
 21 | 
 22 |   switch (hval) {
 23 |     default:
 24 |       hval += asso_values[(unsigned char) str[1] + 3];
 25 |     /*FALLTHROUGH*/
 26 |     case 1:
 27 |       hval += asso_values[(unsigned char) str[0]];
 28 |       break;
 29 |   }
 30 |   return hval + asso_values[(unsigned char) str[len - 1]];
 31 | }
 32 | 
 33 | static const unsigned char kGumboTagMap[] = {GUMBO_TAG_LAST, GUMBO_TAG_LAST,
 34 |     GUMBO_TAG_LAST, GUMBO_TAG_S, GUMBO_TAG_H6, GUMBO_TAG_H5, GUMBO_TAG_H4,
 35 |     GUMBO_TAG_H3, GUMBO_TAG_SPACER, GUMBO_TAG_H2, GUMBO_TAG_HEADER,
 36 |     GUMBO_TAG_H1, GUMBO_TAG_HEAD, GUMBO_TAG_LAST, GUMBO_TAG_DETAILS,
 37 |     GUMBO_TAG_SELECT, GUMBO_TAG_DIR, GUMBO_TAG_LAST, GUMBO_TAG_DEL,
 38 |     GUMBO_TAG_LAST, GUMBO_TAG_SOURCE, GUMBO_TAG_LEGEND, GUMBO_TAG_DATALIST,
 39 |     GUMBO_TAG_METER, GUMBO_TAG_MGLYPH, GUMBO_TAG_LAST, GUMBO_TAG_MATH,
 40 |     GUMBO_TAG_LABEL, GUMBO_TAG_TABLE, GUMBO_TAG_TEMPLATE, GUMBO_TAG_LAST,
 41 |     GUMBO_TAG_RP, GUMBO_TAG_TIME, GUMBO_TAG_TITLE, GUMBO_TAG_DATA,
 42 |     GUMBO_TAG_APPLET, GUMBO_TAG_HGROUP, GUMBO_TAG_SAMP, GUMBO_TAG_TEXTAREA,
 43 |     GUMBO_TAG_ABBR, GUMBO_TAG_MARQUEE, GUMBO_TAG_LAST, GUMBO_TAG_MENUITEM,
 44 |     GUMBO_TAG_SMALL, GUMBO_TAG_META, GUMBO_TAG_A, GUMBO_TAG_LAST,
 45 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_EMBED,
 46 |     GUMBO_TAG_MAP, GUMBO_TAG_LAST, GUMBO_TAG_PARAM, GUMBO_TAG_LAST,
 47 |     GUMBO_TAG_LAST, GUMBO_TAG_NOBR, GUMBO_TAG_P, GUMBO_TAG_SPAN, GUMBO_TAG_EM,
 48 |     GUMBO_TAG_LAST, GUMBO_TAG_NOFRAMES, GUMBO_TAG_SECTION, GUMBO_TAG_NOEMBED,
 49 |     GUMBO_TAG_NEXTID, GUMBO_TAG_FOOTER, GUMBO_TAG_NOSCRIPT, GUMBO_TAG_HR,
 50 |     GUMBO_TAG_LAST, GUMBO_TAG_FONT, GUMBO_TAG_DL, GUMBO_TAG_TR,
 51 |     GUMBO_TAG_SCRIPT, GUMBO_TAG_MO, GUMBO_TAG_LAST, GUMBO_TAG_DD,
 52 |     GUMBO_TAG_MAIN, GUMBO_TAG_TD, GUMBO_TAG_FOREIGNOBJECT, GUMBO_TAG_FORM,
 53 |     GUMBO_TAG_OBJECT, GUMBO_TAG_LAST, GUMBO_TAG_FIELDSET, GUMBO_TAG_LAST,
 54 |     GUMBO_TAG_BGSOUND, GUMBO_TAG_MENU, GUMBO_TAG_TFOOT, GUMBO_TAG_FIGURE,
 55 |     GUMBO_TAG_RB, GUMBO_TAG_LI, GUMBO_TAG_LISTING, GUMBO_TAG_BASEFONT,
 56 |     GUMBO_TAG_OPTGROUP, GUMBO_TAG_LAST, GUMBO_TAG_BASE, GUMBO_TAG_ADDRESS,
 57 |     GUMBO_TAG_MI, GUMBO_TAG_LAST, GUMBO_TAG_PLAINTEXT, GUMBO_TAG_LAST,
 58 |     GUMBO_TAG_PROGRESS, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
 59 |     GUMBO_TAG_ACRONYM, GUMBO_TAG_ARTICLE, GUMBO_TAG_LAST, GUMBO_TAG_PRE,
 60 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_AREA,
 61 |     GUMBO_TAG_RT, GUMBO_TAG_LAST, GUMBO_TAG_OPTION, GUMBO_TAG_IMAGE,
 62 |     GUMBO_TAG_DT, GUMBO_TAG_LAST, GUMBO_TAG_TT, GUMBO_TAG_HTML, GUMBO_TAG_WBR,
 63 |     GUMBO_TAG_OL, GUMBO_TAG_LAST, GUMBO_TAG_STYLE, GUMBO_TAG_STRIKE,
 64 |     GUMBO_TAG_SUP, GUMBO_TAG_MULTICOL, GUMBO_TAG_U, GUMBO_TAG_DFN, GUMBO_TAG_UL,
 65 |     GUMBO_TAG_FIGCAPTION, GUMBO_TAG_MTEXT, GUMBO_TAG_LAST, GUMBO_TAG_VAR,
 66 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_FRAMESET, GUMBO_TAG_LAST,
 67 |     GUMBO_TAG_BR, GUMBO_TAG_I, GUMBO_TAG_FRAME, GUMBO_TAG_LAST, GUMBO_TAG_DIV,
 68 |     GUMBO_TAG_LAST, GUMBO_TAG_TH, GUMBO_TAG_MS, GUMBO_TAG_ANNOTATION_XML,
 69 |     GUMBO_TAG_B, GUMBO_TAG_TBODY, GUMBO_TAG_THEAD, GUMBO_TAG_BIG,
 70 |     GUMBO_TAG_BLOCKQUOTE, GUMBO_TAG_XMP, GUMBO_TAG_LAST, GUMBO_TAG_KBD,
 71 |     GUMBO_TAG_LAST, GUMBO_TAG_LINK, GUMBO_TAG_IFRAME, GUMBO_TAG_MARK,
 72 |     GUMBO_TAG_CENTER, GUMBO_TAG_OUTPUT, GUMBO_TAG_DESC, GUMBO_TAG_CANVAS,
 73 |     GUMBO_TAG_COL, GUMBO_TAG_MALIGNMARK, GUMBO_TAG_IMG, GUMBO_TAG_ASIDE,
 74 |     GUMBO_TAG_LAST, GUMBO_TAG_CODE, GUMBO_TAG_LAST, GUMBO_TAG_SUB, GUMBO_TAG_MN,
 75 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_INS, GUMBO_TAG_AUDIO,
 76 |     GUMBO_TAG_STRONG, GUMBO_TAG_CITE, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
 77 |     GUMBO_TAG_LAST, GUMBO_TAG_INPUT, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
 78 |     GUMBO_TAG_LAST, GUMBO_TAG_NAV, GUMBO_TAG_LAST, GUMBO_TAG_COLGROUP,
 79 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
 80 |     GUMBO_TAG_LAST, GUMBO_TAG_SVG, GUMBO_TAG_KEYGEN, GUMBO_TAG_VIDEO,
 81 |     GUMBO_TAG_BDO, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
 82 |     GUMBO_TAG_LAST, GUMBO_TAG_BODY, GUMBO_TAG_LAST, GUMBO_TAG_Q, GUMBO_TAG_LAST,
 83 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_TRACK,
 84 |     GUMBO_TAG_LAST, GUMBO_TAG_BDI, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
 85 |     GUMBO_TAG_LAST, GUMBO_TAG_CAPTION, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
 86 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
 87 |     GUMBO_TAG_RUBY, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_BUTTON,
 88 |     GUMBO_TAG_SUMMARY, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
 89 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
 90 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
 91 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
 92 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
 93 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
 94 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
 95 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
 96 |     GUMBO_TAG_LAST, GUMBO_TAG_RTC, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
 97 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
 98 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_BLINK, GUMBO_TAG_LAST,
 99 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
100 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
101 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
102 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
103 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
104 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_LAST,
105 |     GUMBO_TAG_LAST, GUMBO_TAG_LAST, GUMBO_TAG_ISINDEX};
106 | 


--------------------------------------------------------------------------------
/src/tag_sizes.h:
--------------------------------------------------------------------------------
1 | // Generated via `gentags.py src/tag.in`.
2 | // Do not edit; edit src/tag.in instead.
3 | // clang-format off
4 | 4, 4, 5, 4, 4, 4, 5, 6, 8, 8, 4, 7, 7, 3, 5, 2, 2, 2, 2, 2, 2, 6, 6, 6, 7, 1, 2, 3, 10, 2, 2, 2, 2, 2, 2, 6, 10, 4, 3, 1, 2, 6, 5, 1, 4, 1, 3, 4, 4, 4, 4, 3, 4, 3, 3, 3, 1, 1, 1, 4, 4, 2, 2, 3, 3, 4, 2, 3, 3, 3, 5, 3, 6, 5, 6, 5, 5, 5, 6, 5, 6, 3, 4, 4, 2, 2, 2, 2, 5, 6, 10, 14, 3, 13, 4, 5, 7, 8, 3, 5, 5, 5, 2, 2, 2, 4, 8, 6, 5, 5, 6, 6, 8, 8, 6, 8, 6, 6, 8, 5, 7, 7, 4, 8, 6, 7, 7, 3, 5, 8, 8, 7, 7, 3, 6, 7, 9, 2, 6, 8, 3, 5, 6, 4, 7, 8, 4, 6, 2, 3, 


--------------------------------------------------------------------------------
/src/tag_strings.h:
--------------------------------------------------------------------------------
  1 | // Generated via `gentags.py src/tag.in`.
  2 | // Do not edit; edit src/tag.in instead.
  3 | // clang-format off
  4 | "html",
  5 | "head",
  6 | "title",
  7 | "base",
  8 | "link",
  9 | "meta",
 10 | "style",
 11 | "script",
 12 | "noscript",
 13 | "template",
 14 | "body",
 15 | "article",
 16 | "section",
 17 | "nav",
 18 | "aside",
 19 | "h1",
 20 | "h2",
 21 | "h3",
 22 | "h4",
 23 | "h5",
 24 | "h6",
 25 | "hgroup",
 26 | "header",
 27 | "footer",
 28 | "address",
 29 | "p",
 30 | "hr",
 31 | "pre",
 32 | "blockquote",
 33 | "ol",
 34 | "ul",
 35 | "li",
 36 | "dl",
 37 | "dt",
 38 | "dd",
 39 | "figure",
 40 | "figcaption",
 41 | "main",
 42 | "div",
 43 | "a",
 44 | "em",
 45 | "strong",
 46 | "small",
 47 | "s",
 48 | "cite",
 49 | "q",
 50 | "dfn",
 51 | "abbr",
 52 | "data",
 53 | "time",
 54 | "code",
 55 | "var",
 56 | "samp",
 57 | "kbd",
 58 | "sub",
 59 | "sup",
 60 | "i",
 61 | "b",
 62 | "u",
 63 | "mark",
 64 | "ruby",
 65 | "rt",
 66 | "rp",
 67 | "bdi",
 68 | "bdo",
 69 | "span",
 70 | "br",
 71 | "wbr",
 72 | "ins",
 73 | "del",
 74 | "image",
 75 | "img",
 76 | "iframe",
 77 | "embed",
 78 | "object",
 79 | "param",
 80 | "video",
 81 | "audio",
 82 | "source",
 83 | "track",
 84 | "canvas",
 85 | "map",
 86 | "area",
 87 | "math",
 88 | "mi",
 89 | "mo",
 90 | "mn",
 91 | "ms",
 92 | "mtext",
 93 | "mglyph",
 94 | "malignmark",
 95 | "annotation-xml",
 96 | "svg",
 97 | "foreignobject",
 98 | "desc",
 99 | "table",
100 | "caption",
101 | "colgroup",
102 | "col",
103 | "tbody",
104 | "thead",
105 | "tfoot",
106 | "tr",
107 | "td",
108 | "th",
109 | "form",
110 | "fieldset",
111 | "legend",
112 | "label",
113 | "input",
114 | "button",
115 | "select",
116 | "datalist",
117 | "optgroup",
118 | "option",
119 | "textarea",
120 | "keygen",
121 | "output",
122 | "progress",
123 | "meter",
124 | "details",
125 | "summary",
126 | "menu",
127 | "menuitem",
128 | "applet",
129 | "acronym",
130 | "bgsound",
131 | "dir",
132 | "frame",
133 | "frameset",
134 | "noframes",
135 | "isindex",
136 | "listing",
137 | "xmp",
138 | "nextid",
139 | "noembed",
140 | "plaintext",
141 | "rb",
142 | "strike",
143 | "basefont",
144 | "big",
145 | "blink",
146 | "center",
147 | "font",
148 | "marquee",
149 | "multicol",
150 | "nobr",
151 | "spacer",
152 | "tt",
153 | "rtc",
154 | 


--------------------------------------------------------------------------------
/src/token_type.h:
--------------------------------------------------------------------------------
 1 | // Copyright 2011 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | //
15 | // Author: jdtang@google.com (Jonathan Tang)
16 | 
17 | #ifndef GUMBO_TOKEN_TYPE_H_
18 | #define GUMBO_TOKEN_TYPE_H_
19 | 
20 | #ifdef __cplusplus
21 | extern "C" {
22 | #endif
23 | 
24 | // An enum representing the type of token.
25 | typedef enum {
26 |   GUMBO_TOKEN_DOCTYPE,
27 |   GUMBO_TOKEN_START_TAG,
28 |   GUMBO_TOKEN_END_TAG,
29 |   GUMBO_TOKEN_COMMENT,
30 |   GUMBO_TOKEN_WHITESPACE,
31 |   GUMBO_TOKEN_CHARACTER,
32 |   GUMBO_TOKEN_CDATA,
33 |   GUMBO_TOKEN_NULL,
34 |   GUMBO_TOKEN_EOF
35 | } GumboTokenType;
36 | 
37 | #ifdef __cplusplus
38 | }  // extern C
39 | #endif
40 | 
41 | #endif  // GUMBO_TOKEN_TYPE_H_
42 | 


--------------------------------------------------------------------------------
/src/tokenizer.h:
--------------------------------------------------------------------------------
  1 | // Copyright 2010 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | //
 15 | // Author: jdtang@google.com (Jonathan Tang)
 16 | //
 17 | // This contains an implementation of a tokenizer for HTML5.  It consumes a
 18 | // buffer of UTF-8 characters, and then emits a stream of tokens.
 19 | 
 20 | #ifndef GUMBO_TOKENIZER_H_
 21 | #define GUMBO_TOKENIZER_H_
 22 | 
 23 | #include <stdbool.h>
 24 | #include <stddef.h>
 25 | 
 26 | #include "gumbo.h"
 27 | #include "token_type.h"
 28 | #include "tokenizer_states.h"
 29 | 
 30 | #ifdef __cplusplus
 31 | extern "C" {
 32 | #endif
 33 | 
 34 | struct GumboInternalParser;
 35 | 
 36 | // Struct containing all information pertaining to doctype tokens.
 37 | typedef struct GumboInternalTokenDocType {
 38 |   const char* name;
 39 |   const char* public_identifier;
 40 |   const char* system_identifier;
 41 |   bool force_quirks;
 42 |   // There's no way to tell a 0-length public or system ID apart from the
 43 |   // absence of a public or system ID, but they're handled different by the
 44 |   // spec, so we need bool flags for them.
 45 |   bool has_public_identifier;
 46 |   bool has_system_identifier;
 47 | } GumboTokenDocType;
 48 | 
 49 | // Struct containing all information pertaining to start tag tokens.
 50 | typedef struct GumboInternalTokenStartTag {
 51 |   GumboTag tag;
 52 |   GumboVector /* GumboAttribute */ attributes;
 53 |   bool is_self_closing;
 54 | } GumboTokenStartTag;
 55 | 
 56 | // A data structure representing a single token in the input stream.  This
 57 | // contains an enum for the type, the source position, a GumboStringPiece
 58 | // pointing to the original text, and then a union for any parsed data.
 59 | typedef struct GumboInternalToken {
 60 |   GumboTokenType type;
 61 |   GumboSourcePosition position;
 62 |   GumboStringPiece original_text;
 63 |   union {
 64 |     GumboTokenDocType doc_type;
 65 |     GumboTokenStartTag start_tag;
 66 |     GumboTag end_tag;
 67 |     const char* text;  // For comments.
 68 |     int character;     // For character, whitespace, null, and EOF tokens.
 69 |   } v;
 70 | } GumboToken;
 71 | 
 72 | // Initializes the tokenizer state within the GumboParser object, setting up a
 73 | // parse of the specified text.
 74 | void gumbo_tokenizer_state_init(
 75 |     struct GumboInternalParser* parser, const char* text, size_t text_length);
 76 | 
 77 | // Destroys the tokenizer state within the GumboParser object, freeing any
 78 | // dynamically-allocated structures within it.
 79 | void gumbo_tokenizer_state_destroy(struct GumboInternalParser* parser);
 80 | 
 81 | // Sets the tokenizer state to the specified value.  This is needed by some
 82 | // parser states, which alter the state of the tokenizer in response to tags
 83 | // seen.
 84 | void gumbo_tokenizer_set_state(
 85 |     struct GumboInternalParser* parser, GumboTokenizerEnum state);
 86 | 
 87 | // Flags whether the current node is a foreign content element.  This is
 88 | // necessary for the markup declaration open state, where the tokenizer must be
 89 | // aware of the state of the parser to properly tokenize bad comment tags.
 90 | // http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#markup-declaration-open-state
 91 | void gumbo_tokenizer_set_is_current_node_foreign(
 92 |     struct GumboInternalParser* parser, bool is_foreign);
 93 | 
 94 | // Lexes a single token from the specified buffer, filling the output with the
 95 | // parsed GumboToken data structure.  Returns true for a successful
 96 | // tokenization, false if a parse error occurs.
 97 | //
 98 | // Example:
 99 | //   struct GumboInternalParser parser;
100 | //   GumboToken output;
101 | //   gumbo_tokenizer_state_init(&parser, text, strlen(text));
102 | //   while (gumbo_lex(&parser, &output)) {
103 | //     ...do stuff with output.
104 | //     gumbo_token_destroy(&parser, &token);
105 | //   }
106 | //   gumbo_tokenizer_state_destroy(&parser);
107 | bool gumbo_lex(struct GumboInternalParser* parser, GumboToken* output);
108 | 
109 | // Frees the internally-allocated pointers within an GumboToken.  Note that this
110 | // doesn't free the token itself, since oftentimes it will be allocated on the
111 | // stack.  A simple call to free() (or GumboParser->deallocator, if
112 | // appropriate) can handle that.
113 | //
114 | // Note that if you are handing over ownership of the internal strings to some
115 | // other data structure - for example, a parse tree - these do not need to be
116 | // freed.
117 | void gumbo_token_destroy(struct GumboInternalParser* parser, GumboToken* token);
118 | 
119 | #ifdef __cplusplus
120 | }
121 | #endif
122 | 
123 | #endif  // GUMBO_TOKENIZER_H_
124 | 


--------------------------------------------------------------------------------
/src/tokenizer_states.h:
--------------------------------------------------------------------------------
  1 | // Copyright 2011 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | //
 15 | // Author: jdtang@google.com (Jonathan Tang)
 16 | //
 17 | // This contains the list of states used in the tokenizer.  Although at first
 18 | // glance it seems like these could be kept internal to the tokenizer, several
 19 | // of the actions in the parser require that it reach into the tokenizer and
 20 | // reset the tokenizer state.  For that to work, it needs to have the
 21 | // definitions of individual states available.
 22 | //
 23 | // This may also be useful for providing more detailed error messages for parse
 24 | // errors, as we can match up states and inputs in a table without having to
 25 | // clutter the tokenizer code with lots of precise error messages.
 26 | 
 27 | #ifndef GUMBO_TOKENIZER_STATES_H_
 28 | #define GUMBO_TOKENIZER_STATES_H_
 29 | 
 30 | // The ordering of this enum is also used to build the dispatch table for the
 31 | // tokenizer state machine, so if it is changed, be sure to update that too.
 32 | typedef enum {
 33 |   GUMBO_LEX_DATA,
 34 |   GUMBO_LEX_CHAR_REF_IN_DATA,
 35 |   GUMBO_LEX_RCDATA,
 36 |   GUMBO_LEX_CHAR_REF_IN_RCDATA,
 37 |   GUMBO_LEX_RAWTEXT,
 38 |   GUMBO_LEX_SCRIPT,
 39 |   GUMBO_LEX_PLAINTEXT,
 40 |   GUMBO_LEX_TAG_OPEN,
 41 |   GUMBO_LEX_END_TAG_OPEN,
 42 |   GUMBO_LEX_TAG_NAME,
 43 |   GUMBO_LEX_RCDATA_LT,
 44 |   GUMBO_LEX_RCDATA_END_TAG_OPEN,
 45 |   GUMBO_LEX_RCDATA_END_TAG_NAME,
 46 |   GUMBO_LEX_RAWTEXT_LT,
 47 |   GUMBO_LEX_RAWTEXT_END_TAG_OPEN,
 48 |   GUMBO_LEX_RAWTEXT_END_TAG_NAME,
 49 |   GUMBO_LEX_SCRIPT_LT,
 50 |   GUMBO_LEX_SCRIPT_END_TAG_OPEN,
 51 |   GUMBO_LEX_SCRIPT_END_TAG_NAME,
 52 |   GUMBO_LEX_SCRIPT_ESCAPED_START,
 53 |   GUMBO_LEX_SCRIPT_ESCAPED_START_DASH,
 54 |   GUMBO_LEX_SCRIPT_ESCAPED,
 55 |   GUMBO_LEX_SCRIPT_ESCAPED_DASH,
 56 |   GUMBO_LEX_SCRIPT_ESCAPED_DASH_DASH,
 57 |   GUMBO_LEX_SCRIPT_ESCAPED_LT,
 58 |   GUMBO_LEX_SCRIPT_ESCAPED_END_TAG_OPEN,
 59 |   GUMBO_LEX_SCRIPT_ESCAPED_END_TAG_NAME,
 60 |   GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_START,
 61 |   GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED,
 62 |   GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_DASH,
 63 |   GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_DASH_DASH,
 64 |   GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_LT,
 65 |   GUMBO_LEX_SCRIPT_DOUBLE_ESCAPED_END,
 66 |   GUMBO_LEX_BEFORE_ATTR_NAME,
 67 |   GUMBO_LEX_ATTR_NAME,
 68 |   GUMBO_LEX_AFTER_ATTR_NAME,
 69 |   GUMBO_LEX_BEFORE_ATTR_VALUE,
 70 |   GUMBO_LEX_ATTR_VALUE_DOUBLE_QUOTED,
 71 |   GUMBO_LEX_ATTR_VALUE_SINGLE_QUOTED,
 72 |   GUMBO_LEX_ATTR_VALUE_UNQUOTED,
 73 |   GUMBO_LEX_CHAR_REF_IN_ATTR_VALUE,
 74 |   GUMBO_LEX_AFTER_ATTR_VALUE_QUOTED,
 75 |   GUMBO_LEX_SELF_CLOSING_START_TAG,
 76 |   GUMBO_LEX_BOGUS_COMMENT,
 77 |   GUMBO_LEX_MARKUP_DECLARATION,
 78 |   GUMBO_LEX_COMMENT_START,
 79 |   GUMBO_LEX_COMMENT_START_DASH,
 80 |   GUMBO_LEX_COMMENT,
 81 |   GUMBO_LEX_COMMENT_END_DASH,
 82 |   GUMBO_LEX_COMMENT_END,
 83 |   GUMBO_LEX_COMMENT_END_BANG,
 84 |   GUMBO_LEX_DOCTYPE,
 85 |   GUMBO_LEX_BEFORE_DOCTYPE_NAME,
 86 |   GUMBO_LEX_DOCTYPE_NAME,
 87 |   GUMBO_LEX_AFTER_DOCTYPE_NAME,
 88 |   GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_KEYWORD,
 89 |   GUMBO_LEX_BEFORE_DOCTYPE_PUBLIC_ID,
 90 |   GUMBO_LEX_DOCTYPE_PUBLIC_ID_DOUBLE_QUOTED,
 91 |   GUMBO_LEX_DOCTYPE_PUBLIC_ID_SINGLE_QUOTED,
 92 |   GUMBO_LEX_AFTER_DOCTYPE_PUBLIC_ID,
 93 |   GUMBO_LEX_BETWEEN_DOCTYPE_PUBLIC_SYSTEM_ID,
 94 |   GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_KEYWORD,
 95 |   GUMBO_LEX_BEFORE_DOCTYPE_SYSTEM_ID,
 96 |   GUMBO_LEX_DOCTYPE_SYSTEM_ID_DOUBLE_QUOTED,
 97 |   GUMBO_LEX_DOCTYPE_SYSTEM_ID_SINGLE_QUOTED,
 98 |   GUMBO_LEX_AFTER_DOCTYPE_SYSTEM_ID,
 99 |   GUMBO_LEX_BOGUS_DOCTYPE,
100 |   GUMBO_LEX_CDATA
101 | } GumboTokenizerEnum;
102 | 
103 | #endif  // GUMBO_TOKENIZER_STATES_H_
104 | 


--------------------------------------------------------------------------------
/src/utf8.h:
--------------------------------------------------------------------------------
  1 | // Copyright 2010 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | //
 15 | // Author: jdtang@google.com (Jonathan Tang)
 16 | //
 17 | // This contains an implementation of a UTF8 iterator and decoder suitable for
 18 | // an HTML5 parser.  This does a bit more than straight UTF-8 decoding.  The
 19 | // HTML5 spec specifies that:
 20 | // 1. Decoding errors are parse errors.
 21 | // 2. Certain other codepoints (eg. control characters) are parse errors.
 22 | // 3. Carriage returns and CR/LF groups are converted to line feeds.
 23 | // http://www.whatwg.org/specs/web-apps/current-work/multipage/infrastructure.html#decoded-as-utf-8,-with-error-handling
 24 | //
 25 | // Also, we want to keep track of source positions for error handling.  As a
 26 | // result, we fold all that functionality into this decoder, and can't use an
 27 | // off-the-shelf library.
 28 | //
 29 | // This header is internal-only, which is why we prefix functions with only
 30 | // utf8_ or utf8_iterator_ instead of gumbo_utf8_.
 31 | 
 32 | #ifndef GUMBO_UTF8_H_
 33 | #define GUMBO_UTF8_H_
 34 | 
 35 | #include <stdbool.h>
 36 | #include <stddef.h>
 37 | 
 38 | #include "gumbo.h"
 39 | 
 40 | #ifdef __cplusplus
 41 | extern "C" {
 42 | #endif
 43 | 
 44 | struct GumboInternalError;
 45 | struct GumboInternalParser;
 46 | 
 47 | // Unicode replacement char.
 48 | extern const int kUtf8ReplacementChar;
 49 | 
 50 | typedef struct GumboInternalUtf8Iterator {
 51 |   // Points at the start of the code point most recently read into 'current'.
 52 |   const char* _start;
 53 | 
 54 |   // Points at the mark.  The mark is initially set to the beginning of the
 55 |   // input.
 56 |   const char* _mark;
 57 | 
 58 |   // Points past the end of the iter, like a past-the-end iterator in the STL.
 59 |   const char* _end;
 60 | 
 61 |   // The code point under the cursor.
 62 |   int _current;
 63 | 
 64 |   // The width in bytes of the current code point.
 65 |   int _width;
 66 | 
 67 |   // The SourcePosition for the current location.
 68 |   GumboSourcePosition _pos;
 69 | 
 70 |   // The SourcePosition for the mark.
 71 |   GumboSourcePosition _mark_pos;
 72 | 
 73 |   // Pointer back to the GumboParser instance, for configuration options and
 74 |   // error recording.
 75 |   struct GumboInternalParser* _parser;
 76 | } Utf8Iterator;
 77 | 
 78 | // Returns true if this Unicode code point is in the list of characters
 79 | // forbidden by the HTML5 spec, such as NUL bytes and undefined control chars.
 80 | bool utf8_is_invalid_code_point(int c);
 81 | 
 82 | // Initializes a new Utf8Iterator from the given byte buffer.  The source does
 83 | // not have to be NUL-terminated, but the length must be passed in explicitly.
 84 | void utf8iterator_init(struct GumboInternalParser* parser, const char* source,
 85 |     size_t source_length, Utf8Iterator* iter);
 86 | 
 87 | // Advances the current position by one code point.
 88 | void utf8iterator_next(Utf8Iterator* iter);
 89 | 
 90 | // Returns the current code point as an integer.
 91 | int utf8iterator_current(const Utf8Iterator* iter);
 92 | 
 93 | // Retrieves and fills the output parameter with the current source position.
 94 | void utf8iterator_get_position(
 95 |     const Utf8Iterator* iter, GumboSourcePosition* output);
 96 | 
 97 | // Retrieves a character pointer to the start of the current character.
 98 | const char* utf8iterator_get_char_pointer(const Utf8Iterator* iter);
 99 | 
100 | // Retrieves a character pointer to 1 past the end of the buffer.  This is
101 | // necessary for certain state machines and string comparisons that would like
102 | // to look directly for ASCII text in the buffer without going through the
103 | // decoder.
104 | const char* utf8iterator_get_end_pointer(const Utf8Iterator* iter);
105 | 
106 | // If the upcoming text in the buffer matches the specified prefix (which has
107 | // length 'length'), consume it and return true.  Otherwise, return false with
108 | // no other effects.  If the length of the string would overflow the buffer,
109 | // this returns false.  Note that prefix should not contain null bytes because
110 | // of the use of strncmp/strncasecmp internally.  All existing use-cases adhere
111 | // to this.
112 | bool utf8iterator_maybe_consume_match(
113 |     Utf8Iterator* iter, const char* prefix, size_t length, bool case_sensitive);
114 | 
115 | // "Marks" a particular location of interest in the input stream, so that it can
116 | // later be reset() to.  There's also the ability to record an error at the
117 | // point that was marked, as oftentimes that's more useful than the last
118 | // character before the error was detected.
119 | void utf8iterator_mark(Utf8Iterator* iter);
120 | 
121 | // Returns the current input stream position to the mark.
122 | void utf8iterator_reset(Utf8Iterator* iter);
123 | 
124 | // Sets the position and original text fields of an error to the value at the
125 | // mark.
126 | void utf8iterator_fill_error_at_mark(
127 |     Utf8Iterator* iter, struct GumboInternalError* error);
128 | 
129 | #ifdef __cplusplus
130 | }
131 | #endif
132 | #endif  // GUMBO_UTF8_H_
133 | 


--------------------------------------------------------------------------------
/src/util.c:
--------------------------------------------------------------------------------
 1 | // Copyright 2010 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | //
15 | // Author: jdtang@google.com (Jonathan Tang)
16 | 
17 | #include "util.h"
18 | 
19 | #include <assert.h>
20 | #include <stdlib.h>
21 | #include <string.h>
22 | #include <strings.h>
23 | #include <stdarg.h>
24 | #include <stdio.h>
25 | 
26 | #include "gumbo.h"
27 | #include "parser.h"
28 | 
29 | // TODO(jdtang): This should be elsewhere, but there's no .c file for
30 | // SourcePositions and yet the constant needs some linkage, so this is as good
31 | // as any.
32 | const GumboSourcePosition kGumboEmptySourcePosition = {0, 0, 0};
33 | 
34 | void* gumbo_parser_allocate(GumboParser* parser, size_t num_bytes) {
35 |   return parser->_options->allocator(parser->_options->userdata, num_bytes);
36 | }
37 | 
38 | void gumbo_parser_deallocate(GumboParser* parser, void* ptr) {
39 |   parser->_options->deallocator(parser->_options->userdata, ptr);
40 | }
41 | 
42 | char* gumbo_copy_stringz(GumboParser* parser, const char* str) {
43 |   char* buffer = gumbo_parser_allocate(parser, strlen(str) + 1);
44 |   strcpy(buffer, str);
45 |   return buffer;
46 | }
47 | 
48 | // Debug function to trace operation of the parser.  Pass --copts=-DGUMBO_DEBUG
49 | // to use.
50 | void gumbo_debug(const char* format, ...) {
51 | #ifdef GUMBO_DEBUG
52 |   va_list args;
53 |   va_start(args, format);
54 |   vprintf(format, args);
55 |   va_end(args);
56 |   fflush(stdout);
57 | #endif
58 | }
59 | 


--------------------------------------------------------------------------------
/src/util.h:
--------------------------------------------------------------------------------
 1 | // Copyright 2010 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | //
15 | // Author: jdtang@google.com (Jonathan Tang)
16 | //
17 | // This contains some utility functions that didn't fit into any of the other
18 | // headers.
19 | 
20 | #ifndef GUMBO_UTIL_H_
21 | #define GUMBO_UTIL_H_
22 | #ifdef _MSC_VER
23 | #define _CRT_SECURE_NO_WARNINGS
24 | #endif
25 | #include <stdbool.h>
26 | #include <stddef.h>
27 | 
28 | #ifdef __cplusplus
29 | extern "C" {
30 | #endif
31 | 
32 | // Forward declaration since it's passed into some of the functions in this
33 | // header.
34 | struct GumboInternalParser;
35 | 
36 | // Utility function for allocating & copying a null-terminated string into a
37 | // freshly-allocated buffer.  This is necessary for proper memory management; we
38 | // have the convention that all const char* in parse tree structures are
39 | // freshly-allocated, so if we didn't copy, we'd try to delete a literal string
40 | // when the parse tree is destroyed.
41 | char* gumbo_copy_stringz(struct GumboInternalParser* parser, const char* str);
42 | 
43 | // Allocate a chunk of memory, using the allocator specified in the Parser's
44 | // config options.
45 | void* gumbo_parser_allocate(
46 |     struct GumboInternalParser* parser, size_t num_bytes);
47 | 
48 | // Deallocate a chunk of memory, using the deallocator specified in the Parser's
49 | // config options.
50 | void gumbo_parser_deallocate(struct GumboInternalParser* parser, void* ptr);
51 | 
52 | // Debug wrapper for printf, to make it easier to turn off debugging info when
53 | // required.
54 | void gumbo_debug(const char* format, ...);
55 | 
56 | #ifdef __cplusplus
57 | }
58 | #endif
59 | 
60 | #endif  // GUMBO_UTIL_H_
61 | 


--------------------------------------------------------------------------------
/src/vector.c:
--------------------------------------------------------------------------------
  1 | // Copyright 2010 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | //
 15 | // Author: jdtang@google.com (Jonathan Tang)
 16 | 
 17 | #include "vector.h"
 18 | 
 19 | #include <assert.h>
 20 | #include <stdlib.h>
 21 | #include <string.h>
 22 | #include <strings.h>
 23 | 
 24 | #include "util.h"
 25 | 
 26 | struct GumboInternalParser;
 27 | 
 28 | const GumboVector kGumboEmptyVector = {NULL, 0, 0};
 29 | 
 30 | void gumbo_vector_init(struct GumboInternalParser* parser,
 31 |     size_t initial_capacity, GumboVector* vector) {
 32 |   vector->length = 0;
 33 |   vector->capacity = initial_capacity;
 34 |   if (initial_capacity > 0) {
 35 |     vector->data =
 36 |         gumbo_parser_allocate(parser, sizeof(void*) * initial_capacity);
 37 |   } else {
 38 |     vector->data = NULL;
 39 |   }
 40 | }
 41 | 
 42 | void gumbo_vector_destroy(
 43 |     struct GumboInternalParser* parser, GumboVector* vector) {
 44 |   if (vector->capacity > 0) {
 45 |     gumbo_parser_deallocate(parser, vector->data);
 46 |   }
 47 | }
 48 | 
 49 | static void enlarge_vector_if_full(
 50 |     struct GumboInternalParser* parser, GumboVector* vector) {
 51 |   if (vector->length >= vector->capacity) {
 52 |     if (vector->capacity) {
 53 |       size_t old_num_bytes = sizeof(void*) * vector->capacity;
 54 |       vector->capacity *= 2;
 55 |       size_t num_bytes = sizeof(void*) * vector->capacity;
 56 |       void** temp = gumbo_parser_allocate(parser, num_bytes);
 57 |       memcpy(temp, vector->data, old_num_bytes);
 58 |       gumbo_parser_deallocate(parser, vector->data);
 59 |       vector->data = temp;
 60 |     } else {
 61 |       // 0-capacity vector; no previous array to deallocate.
 62 |       vector->capacity = 2;
 63 |       vector->data =
 64 |           gumbo_parser_allocate(parser, sizeof(void*) * vector->capacity);
 65 |     }
 66 |   }
 67 | }
 68 | 
 69 | void gumbo_vector_add(
 70 |     struct GumboInternalParser* parser, void* element, GumboVector* vector) {
 71 |   enlarge_vector_if_full(parser, vector);
 72 |   assert(vector->data);
 73 |   assert(vector->length < vector->capacity);
 74 |   vector->data[vector->length++] = element;
 75 | }
 76 | 
 77 | void* gumbo_vector_pop(
 78 |     struct GumboInternalParser* parser, GumboVector* vector) {
 79 |   if (vector->length == 0) {
 80 |     return NULL;
 81 |   }
 82 |   return vector->data[--vector->length];
 83 | }
 84 | 
 85 | int gumbo_vector_index_of(GumboVector* vector, const void* element) {
 86 |   for (unsigned int i = 0; i < vector->length; ++i) {
 87 |     if (vector->data[i] == element) {
 88 |       return i;
 89 |     }
 90 |   }
 91 |   return -1;
 92 | }
 93 | 
 94 | void gumbo_vector_insert_at(struct GumboInternalParser* parser, void* element,
 95 |     unsigned int index, GumboVector* vector) {
 96 |   assert(index >= 0);
 97 |   assert(index <= vector->length);
 98 |   enlarge_vector_if_full(parser, vector);
 99 |   ++vector->length;
100 |   memmove(&vector->data[index + 1], &vector->data[index],
101 |       sizeof(void*) * (vector->length - index - 1));
102 |   vector->data[index] = element;
103 | }
104 | 
105 | void gumbo_vector_remove(
106 |     struct GumboInternalParser* parser, void* node, GumboVector* vector) {
107 |   int index = gumbo_vector_index_of(vector, node);
108 |   if (index == -1) {
109 |     return;
110 |   }
111 |   gumbo_vector_remove_at(parser, index, vector);
112 | }
113 | 
114 | void* gumbo_vector_remove_at(struct GumboInternalParser* parser,
115 |     unsigned int index, GumboVector* vector) {
116 |   assert(index >= 0);
117 |   assert(index < vector->length);
118 |   void* result = vector->data[index];
119 |   memmove(&vector->data[index], &vector->data[index + 1],
120 |       sizeof(void*) * (vector->length - index - 1));
121 |   --vector->length;
122 |   return result;
123 | }
124 | 


--------------------------------------------------------------------------------
/src/vector.h:
--------------------------------------------------------------------------------
 1 | // Copyright 2010 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | //
15 | // Author: jdtang@google.com (Jonathan Tang)
16 | 
17 | #ifndef GUMBO_VECTOR_H_
18 | #define GUMBO_VECTOR_H_
19 | 
20 | #include "gumbo.h"
21 | 
22 | #ifdef __cplusplus
23 | extern "C" {
24 | #endif
25 | 
26 | // Forward declaration since it's passed into some of the functions in this
27 | // header.
28 | struct GumboInternalParser;
29 | 
30 | // Initializes a new GumboVector with the specified initial capacity.
31 | void gumbo_vector_init(struct GumboInternalParser* parser,
32 |     size_t initial_capacity, GumboVector* vector);
33 | 
34 | // Frees the memory used by an GumboVector.  Does not free the contained
35 | // pointers.
36 | void gumbo_vector_destroy(
37 |     struct GumboInternalParser* parser, GumboVector* vector);
38 | 
39 | // Adds a new element to an GumboVector.
40 | void gumbo_vector_add(
41 |     struct GumboInternalParser* parser, void* element, GumboVector* vector);
42 | 
43 | // Removes and returns the element most recently added to the GumboVector.
44 | // Ownership is transferred to caller.  Capacity is unchanged.  If the vector is
45 | // empty, NULL is returned.
46 | void* gumbo_vector_pop(struct GumboInternalParser* parser, GumboVector* vector);
47 | 
48 | // Inserts an element at a specific index.  This is potentially O(N) time, but
49 | // is necessary for some of the spec's behavior.
50 | void gumbo_vector_insert_at(struct GumboInternalParser* parser, void* element,
51 |     unsigned int index, GumboVector* vector);
52 | 
53 | // Removes an element from the vector, or does nothing if the element is not in
54 | // the vector.
55 | void gumbo_vector_remove(
56 |     struct GumboInternalParser* parser, void* element, GumboVector* vector);
57 | 
58 | // Removes and returns an element at a specific index.  Note that this is
59 | // potentially O(N) time and should be used sparingly.
60 | void* gumbo_vector_remove_at(struct GumboInternalParser* parser,
61 |     unsigned int index, GumboVector* vector);
62 | 
63 | #ifdef __cplusplus
64 | }
65 | #endif
66 | 
67 | #endif  // GUMBO_VECTOR_H_
68 | 


--------------------------------------------------------------------------------
/tests/attribute.cc:
--------------------------------------------------------------------------------
 1 | // Copyright 2011 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | //
15 | // Author: jdtang@google.com (Jonathan Tang)
16 | 
17 | #include "attribute.h"
18 | 
19 | #include <stdlib.h>
20 | #include <string.h>
21 | 
22 | #include "gtest/gtest.h"
23 | #include "test_utils.h"
24 | #include "vector.h"
25 | 
26 | namespace {
27 | 
28 | class GumboAttributeTest : public GumboTest {
29 |  protected:
30 |   GumboAttributeTest() { gumbo_vector_init(&parser_, 2, &vector_); }
31 | 
32 |   ~GumboAttributeTest() { gumbo_vector_destroy(&parser_, &vector_); }
33 | 
34 |   GumboVector vector_;
35 | };
36 | 
37 | TEST_F(GumboAttributeTest, GetAttribute) {
38 |   GumboAttribute attr1;
39 |   GumboAttribute attr2;
40 |   attr1.name = "";
41 |   attr2.name = "foo";
42 | 
43 |   gumbo_vector_add(&parser_, &attr1, &vector_);
44 |   gumbo_vector_add(&parser_, &attr2, &vector_);
45 |   EXPECT_EQ(&attr2, gumbo_get_attribute(&vector_, "foo"));
46 |   EXPECT_EQ(NULL, gumbo_get_attribute(&vector_, "bar"));
47 | }
48 | 
49 | }  // namespace
50 | 


--------------------------------------------------------------------------------
/tests/char_ref.cc:
--------------------------------------------------------------------------------
  1 | // Copyright 2011 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | //
 15 | // Author: jdtang@google.com (Jonathan Tang)
 16 | //
 17 | // GUnit char_ref tests.  These are quick smoke tests, mostly to identify
 18 | // crashing bugs so that they can be fixed without having to debug
 19 | // multi-language tests.  As such, they focus on coverage rather than
 20 | // completeness.  For testing the full spec, use char_ref_py_tests, which share
 21 | // their testdata with the Python html5lib library.
 22 | 
 23 | #include "char_ref.h"
 24 | 
 25 | #include <stdio.h>
 26 | #include <string.h>
 27 | 
 28 | #include "gtest/gtest.h"
 29 | #include "test_utils.h"
 30 | #include "utf8.h"
 31 | 
 32 | namespace {
 33 | 
 34 | class CharRefTest : public GumboTest {
 35 |  protected:
 36 |   bool ConsumeCharRef(const char* input) {
 37 |     return ConsumeCharRef(input, ' ', false);
 38 |   }
 39 | 
 40 |   bool ConsumeCharRef(
 41 |       const char* input, int additional_allowed_char, bool is_in_attribute) {
 42 |     text_ = input;
 43 |     utf8iterator_init(&parser_, input, strlen(input), &iter_);
 44 |     bool result = consume_char_ref(
 45 |         &parser_, &iter_, additional_allowed_char, is_in_attribute, &output_);
 46 |     fflush(stdout);
 47 |     return result;
 48 |   }
 49 | 
 50 |   Utf8Iterator iter_;
 51 |   OneOrTwoCodepoints output_;
 52 | };
 53 | 
 54 | TEST_F(CharRefTest, Whitespace) {
 55 |   EXPECT_TRUE(ConsumeCharRef(" &nbsp;"));
 56 |   EXPECT_EQ(kGumboNoChar, output_.first);
 57 |   EXPECT_EQ(kGumboNoChar, output_.second);
 58 | }
 59 | 
 60 | TEST_F(CharRefTest, NumericHex) {
 61 |   EXPECT_TRUE(ConsumeCharRef("&#x12ab;"));
 62 |   EXPECT_EQ(0x12ab, output_.first);
 63 |   EXPECT_EQ(kGumboNoChar, output_.second);
 64 | }
 65 | 
 66 | TEST_F(CharRefTest, NumericDecimal) {
 67 |   EXPECT_TRUE(ConsumeCharRef("&#1234;"));
 68 |   EXPECT_EQ(1234, output_.first);
 69 |   EXPECT_EQ(kGumboNoChar, output_.second);
 70 | }
 71 | 
 72 | TEST_F(CharRefTest, NumericInvalidDigit) {
 73 |   errors_are_expected_ = true;
 74 |   EXPECT_FALSE(ConsumeCharRef("&#google"));
 75 |   EXPECT_EQ(kGumboNoChar, output_.first);
 76 |   EXPECT_EQ(kGumboNoChar, output_.second);
 77 |   EXPECT_EQ('&', utf8iterator_current(&iter_));
 78 | }
 79 | 
 80 | TEST_F(CharRefTest, NumericNoSemicolon) {
 81 |   errors_are_expected_ = true;
 82 |   EXPECT_FALSE(ConsumeCharRef("&#1234google"));
 83 |   EXPECT_EQ(1234, output_.first);
 84 |   EXPECT_EQ(kGumboNoChar, output_.second);
 85 |   EXPECT_EQ('g', utf8iterator_current(&iter_));
 86 | }
 87 | 
 88 | TEST_F(CharRefTest, NumericReplacement) {
 89 |   errors_are_expected_ = true;
 90 |   EXPECT_FALSE(ConsumeCharRef("&#X82"));
 91 |   // Low quotation mark character.
 92 |   EXPECT_EQ(0x201A, output_.first);
 93 |   EXPECT_EQ(kGumboNoChar, output_.second);
 94 | }
 95 | 
 96 | TEST_F(CharRefTest, NumericInvalid) {
 97 |   errors_are_expected_ = true;
 98 |   EXPECT_FALSE(ConsumeCharRef("&#xDA00"));
 99 |   EXPECT_EQ(0xFFFD, output_.first);
100 |   EXPECT_EQ(kGumboNoChar, output_.second);
101 | }
102 | 
103 | TEST_F(CharRefTest, NumericUtfInvalid) {
104 |   errors_are_expected_ = true;
105 |   EXPECT_FALSE(ConsumeCharRef("&#x007"));
106 |   EXPECT_EQ(0x7, output_.first);
107 |   EXPECT_EQ(kGumboNoChar, output_.second);
108 | }
109 | 
110 | TEST_F(CharRefTest, NamedReplacement) {
111 |   EXPECT_TRUE(ConsumeCharRef("&lt;"));
112 |   EXPECT_EQ('<', output_.first);
113 |   EXPECT_EQ(kGumboNoChar, output_.second);
114 | }
115 | 
116 | TEST_F(CharRefTest, NamedReplacementNoSemicolon) {
117 |   errors_are_expected_ = true;
118 |   EXPECT_FALSE(ConsumeCharRef("&gt"));
119 |   EXPECT_EQ('>', output_.first);
120 |   EXPECT_EQ(kGumboNoChar, output_.second);
121 | }
122 | 
123 | TEST_F(CharRefTest, NamedReplacementWithInvalidUtf8) {
124 |   errors_are_expected_ = true;
125 |   EXPECT_TRUE(ConsumeCharRef("&\xc3\xa5"));
126 |   EXPECT_EQ(kGumboNoChar, output_.first);
127 |   EXPECT_EQ(kGumboNoChar, output_.second);
128 | }
129 | 
130 | TEST_F(CharRefTest, NamedReplacementInvalid) {
131 |   errors_are_expected_ = true;
132 |   EXPECT_FALSE(ConsumeCharRef("&google;"));
133 |   EXPECT_EQ(kGumboNoChar, output_.first);
134 |   EXPECT_EQ(kGumboNoChar, output_.second);
135 |   EXPECT_EQ('&', utf8iterator_current(&iter_));
136 | }
137 | 
138 | // TEST_F(CharRefTest, NamedReplacementInvalidNoSemicolon) {
139 | //  EXPECT_FALSE(ConsumeCharRef("&google"));
140 | //  EXPECT_EQ(kGumboNoChar, output_.first);
141 | //  EXPECT_EQ(kGumboNoChar, output_.second);
142 | //  EXPECT_EQ('&', utf8iterator_current(&iter_));
143 | //}
144 | 
145 | TEST_F(CharRefTest, AdditionalAllowedChar) {
146 |   EXPECT_TRUE(ConsumeCharRef("&\"", '"', false));
147 |   EXPECT_EQ(kGumboNoChar, output_.first);
148 |   EXPECT_EQ(kGumboNoChar, output_.second);
149 |   EXPECT_EQ('&', utf8iterator_current(&iter_));
150 | }
151 | 
152 | TEST_F(CharRefTest, InAttribute) {
153 |   EXPECT_TRUE(ConsumeCharRef("&noted", ' ', true));
154 |   EXPECT_EQ(kGumboNoChar, output_.first);
155 |   EXPECT_EQ(kGumboNoChar, output_.second);
156 |   EXPECT_EQ('&', utf8iterator_current(&iter_));
157 | }
158 | 
159 | TEST_F(CharRefTest, MultiChars) {
160 |   EXPECT_TRUE(ConsumeCharRef("&notindot;"));
161 |   EXPECT_EQ(0x22F5, output_.first);
162 |   EXPECT_EQ(0x0338, output_.second);
163 | }
164 | 
165 | TEST_F(CharRefTest, CharAfter) {
166 |   EXPECT_TRUE(ConsumeCharRef("&lt;x"));
167 |   EXPECT_EQ('<', output_.first);
168 |   EXPECT_EQ(kGumboNoChar, output_.second);
169 |   EXPECT_EQ('x', utf8iterator_current(&iter_));
170 | }
171 | 
172 | }  // namespace
173 | 


--------------------------------------------------------------------------------
/tests/string_buffer.cc:
--------------------------------------------------------------------------------
  1 | // Copyright 2011 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | //
 15 | // Author: jdtang@google.com (Jonathan Tang)
 16 | 
 17 | #include "string_buffer.h"
 18 | 
 19 | #include <stdlib.h>
 20 | #include <string.h>
 21 | 
 22 | #include "gtest/gtest.h"
 23 | #include "test_utils.h"
 24 | #include "util.h"
 25 | 
 26 | namespace {
 27 | 
 28 | #define INIT_GUMBO_STRING(varname, literal) \
 29 |   GumboStringPiece varname = {literal, sizeof(literal) - 1}
 30 | 
 31 | class GumboStringBufferTest : public GumboTest {
 32 |  protected:
 33 |   GumboStringBufferTest() { gumbo_string_buffer_init(&parser_, &buffer_); }
 34 | 
 35 |   ~GumboStringBufferTest() { gumbo_string_buffer_destroy(&parser_, &buffer_); }
 36 | 
 37 |   void NullTerminateBuffer() { buffer_.data[buffer_.length++] = 0; }
 38 | 
 39 |   GumboStringBuffer buffer_;
 40 | };
 41 | 
 42 | TEST_F(GumboStringBufferTest, Reserve) {
 43 |   gumbo_string_buffer_reserve(&parser_, 21, &buffer_);
 44 |   EXPECT_EQ(40, buffer_.capacity);
 45 |   strcpy(buffer_.data, "01234567890123456789");
 46 |   buffer_.length = 20;
 47 |   NullTerminateBuffer();
 48 |   EXPECT_EQ(21, buffer_.length);
 49 |   EXPECT_STREQ("01234567890123456789", buffer_.data);
 50 | }
 51 | 
 52 | TEST_F(GumboStringBufferTest, AppendString) {
 53 |   INIT_GUMBO_STRING(str, "01234567");
 54 |   gumbo_string_buffer_append_string(&parser_, &str, &buffer_);
 55 |   NullTerminateBuffer();
 56 |   EXPECT_STREQ("01234567", buffer_.data);
 57 | }
 58 | 
 59 | TEST_F(GumboStringBufferTest, AppendStringWithResize) {
 60 |   INIT_GUMBO_STRING(str, "01234567");
 61 |   gumbo_string_buffer_append_string(&parser_, &str, &buffer_);
 62 |   gumbo_string_buffer_append_string(&parser_, &str, &buffer_);
 63 |   NullTerminateBuffer();
 64 |   EXPECT_STREQ("0123456701234567", buffer_.data);
 65 | }
 66 | 
 67 | TEST_F(GumboStringBufferTest, AppendCodepoint_1Byte) {
 68 |   gumbo_string_buffer_append_codepoint(&parser_, 'a', &buffer_);
 69 |   NullTerminateBuffer();
 70 |   EXPECT_STREQ("a", buffer_.data);
 71 | }
 72 | 
 73 | TEST_F(GumboStringBufferTest, AppendCodepoint_2Bytes) {
 74 |   gumbo_string_buffer_append_codepoint(&parser_, 0xE5, &buffer_);
 75 |   NullTerminateBuffer();
 76 |   EXPECT_STREQ("\xC3\xA5", buffer_.data);
 77 | }
 78 | 
 79 | TEST_F(GumboStringBufferTest, AppendCodepoint_3Bytes) {
 80 |   gumbo_string_buffer_append_codepoint(&parser_, 0x39E7, &buffer_);
 81 |   NullTerminateBuffer();
 82 |   EXPECT_STREQ("\xE3\xA7\xA7", buffer_.data);
 83 | }
 84 | 
 85 | TEST_F(GumboStringBufferTest, AppendCodepoint_4Bytes) {
 86 |   gumbo_string_buffer_append_codepoint(&parser_, 0x679E7, &buffer_);
 87 |   NullTerminateBuffer();
 88 |   EXPECT_STREQ("\xF1\xA7\xA7\xA7", buffer_.data);
 89 | }
 90 | 
 91 | TEST_F(GumboStringBufferTest, ToString) {
 92 |   gumbo_string_buffer_reserve(&parser_, 8, &buffer_);
 93 |   strcpy(buffer_.data, "012345");
 94 |   buffer_.length = 7;
 95 | 
 96 |   char* dest = gumbo_string_buffer_to_string(&parser_, &buffer_);
 97 |   EXPECT_STREQ("012345", dest);
 98 |   gumbo_parser_deallocate(&parser_, dest);
 99 | }
100 | 
101 | }  // namespace
102 | 


--------------------------------------------------------------------------------
/tests/string_piece.cc:
--------------------------------------------------------------------------------
 1 | // Copyright 2011 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | //
15 | // Author: jdtang@google.com (Jonathan Tang)
16 | 
17 | #include "string_piece.h"
18 | 
19 | #include <stdlib.h>
20 | #include <string.h>
21 | 
22 | #include "gtest/gtest.h"
23 | #include "parser.h"
24 | #include "test_utils.h"
25 | #include "util.h"
26 | 
27 | namespace {
28 | 
29 | typedef GumboTest GumboStringPieceTest;
30 | 
31 | #define INIT_GUMBO_STRING(varname, literal) \
32 |   GumboStringPiece varname = {literal, sizeof(literal) - 1}
33 | 
34 | TEST_F(GumboStringPieceTest, Equal) {
35 |   INIT_GUMBO_STRING(str1, "foo");
36 |   INIT_GUMBO_STRING(str2, "foo");
37 |   EXPECT_TRUE(gumbo_string_equals(&str1, &str2));
38 | }
39 | 
40 | TEST_F(GumboStringPieceTest, NotEqual_DifferingCase) {
41 |   INIT_GUMBO_STRING(str1, "foo");
42 |   INIT_GUMBO_STRING(str2, "Foo");
43 |   EXPECT_FALSE(gumbo_string_equals(&str1, &str2));
44 | }
45 | 
46 | TEST_F(GumboStringPieceTest, NotEqual_Str1Shorter) {
47 |   INIT_GUMBO_STRING(str1, "foo");
48 |   INIT_GUMBO_STRING(str2, "foobar");
49 |   EXPECT_FALSE(gumbo_string_equals(&str1, &str2));
50 | }
51 | 
52 | TEST_F(GumboStringPieceTest, NotEqual_Str2Shorter) {
53 |   INIT_GUMBO_STRING(str1, "foobar");
54 |   INIT_GUMBO_STRING(str2, "foo");
55 |   EXPECT_FALSE(gumbo_string_equals(&str1, &str2));
56 | }
57 | 
58 | TEST_F(GumboStringPieceTest, NotEqual_DifferentText) {
59 |   INIT_GUMBO_STRING(str1, "bar");
60 |   INIT_GUMBO_STRING(str2, "foo");
61 |   EXPECT_FALSE(gumbo_string_equals(&str1, &str2));
62 | }
63 | 
64 | TEST_F(GumboStringPieceTest, CaseEqual) {
65 |   INIT_GUMBO_STRING(str1, "foo");
66 |   INIT_GUMBO_STRING(str2, "fOO");
67 |   EXPECT_TRUE(gumbo_string_equals_ignore_case(&str1, &str2));
68 | }
69 | 
70 | TEST_F(GumboStringPieceTest, CaseNotEqual_Str2Shorter) {
71 |   INIT_GUMBO_STRING(str1, "foobar");
72 |   INIT_GUMBO_STRING(str2, "foo");
73 |   EXPECT_FALSE(gumbo_string_equals_ignore_case(&str1, &str2));
74 | }
75 | 
76 | TEST_F(GumboStringPieceTest, Copy) {
77 |   GumboParser parser;
78 |   parser._options = &kGumboDefaultOptions;
79 |   INIT_GUMBO_STRING(str1, "bar");
80 |   GumboStringPiece str2;
81 |   gumbo_string_copy(&parser, &str2, &str1);
82 |   EXPECT_TRUE(gumbo_string_equals(&str1, &str2));
83 |   gumbo_parser_deallocate(&parser, (void*) str2.data);
84 | }
85 | 
86 | }  // namespace
87 | 


--------------------------------------------------------------------------------
/tests/test_utils.cc:
--------------------------------------------------------------------------------
  1 | // Copyright 2013 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | //
 15 | // Author: jdtang@google.com (Jonathan Tang)
 16 | 
 17 | #include "test_utils.h"
 18 | 
 19 | #include "error.h"
 20 | #include "util.h"
 21 | 
 22 | int GetChildCount(GumboNode* node) {
 23 |   if (node->type == GUMBO_NODE_DOCUMENT) {
 24 |     return node->v.document.children.length;
 25 |   } else {
 26 |     return node->v.element.children.length;
 27 |   }
 28 | }
 29 | 
 30 | GumboTag GetTag(GumboNode* node) { return node->v.element.tag; }
 31 | 
 32 | GumboNode* GetChild(GumboNode* parent, int index) {
 33 |   if (parent->type == GUMBO_NODE_DOCUMENT) {
 34 |     return static_cast<GumboNode*>(parent->v.document.children.data[index]);
 35 |   } else {
 36 |     return static_cast<GumboNode*>(parent->v.element.children.data[index]);
 37 |   }
 38 | }
 39 | 
 40 | int GetAttributeCount(GumboNode* node) {
 41 |   return node->v.element.attributes.length;
 42 | }
 43 | 
 44 | GumboAttribute* GetAttribute(GumboNode* node, int index) {
 45 |   return static_cast<GumboAttribute*>(node->v.element.attributes.data[index]);
 46 | }
 47 | 
 48 | // Convenience function to do some basic assertions on the structure of the
 49 | // document (nodes are elements, nodes have the right tags) and then return
 50 | // the body node.
 51 | void GetAndAssertBody(GumboNode* root, GumboNode** body) {
 52 |   GumboNode* html = NULL;
 53 |   for (int i = 0; i < GetChildCount(root); ++i) {
 54 |     GumboNode* child = GetChild(root, i);
 55 |     if (child->type != GUMBO_NODE_ELEMENT) {
 56 |       ASSERT_EQ(GUMBO_NODE_COMMENT, child->type);
 57 |       continue;
 58 |     }
 59 |     ASSERT_TRUE(html == NULL);
 60 |     html = child;
 61 |   }
 62 |   ASSERT_TRUE(html != NULL);
 63 |   ASSERT_EQ(GUMBO_NODE_ELEMENT, html->type);
 64 |   EXPECT_EQ(GUMBO_TAG_HTML, GetTag(html));
 65 | 
 66 |   // There may be comment/whitespace nodes; this walks through the children of
 67 |   // <html> and assigns head/body based on them, or assert-fails if there are
 68 |   // fewer/more than 2 such nodes.
 69 |   GumboNode* head = NULL;
 70 |   *body = NULL;
 71 |   for (int i = 0; i < GetChildCount(html); ++i) {
 72 |     GumboNode* child = GetChild(html, i);
 73 |     if (child->type != GUMBO_NODE_ELEMENT) {
 74 |       continue;
 75 |     }
 76 | 
 77 |     if (!head) {
 78 |       head = child;
 79 |       EXPECT_EQ(GUMBO_TAG_HEAD, GetTag(head));
 80 |     } else if (!(*body)) {
 81 |       *body = child;
 82 |       EXPECT_EQ(GUMBO_TAG_BODY, GetTag(*body));
 83 |     } else {
 84 |       ASSERT_TRUE("More than two elements found inside <html>" != NULL);
 85 |     }
 86 |   }
 87 |   EXPECT_TRUE(head != NULL);
 88 |   ASSERT_TRUE(*body != NULL);
 89 | }
 90 | 
 91 | void SanityCheckPointers(
 92 |     const char* input, size_t input_length, const GumboNode* node, int depth) {
 93 |   ASSERT_GE(input_length, (size_t) 0);
 94 |   ASSERT_TRUE(node != NULL);
 95 |   // There are some truly pathological HTML documents out there - the
 96 |   // integration tests for this include one where the DOM "tree" is actually a
 97 |   // linked list 27,500 nodes deep - and so we need a limit on the recursion
 98 |   // depth here to avoid blowing the stack. Alternatively, we could externalize
 99 |   // the stack and use an iterative algorithm, but that gets us very little for
100 |   // the additional programming complexity.
101 |   if (node->type == GUMBO_NODE_DOCUMENT || depth > 400) {
102 |     // Don't sanity-check the document as well...we start with the root.
103 |     return;
104 |   }
105 |   if (node->type == GUMBO_NODE_ELEMENT) {
106 |     const GumboElement* element = &node->v.element;
107 |     // Sanity checks on original* pointers, making sure they fall within the
108 |     // original input.
109 |     if (element->original_tag.data && element->original_tag.length) {
110 |       EXPECT_GE(element->original_tag.data, input);
111 |       EXPECT_LT(element->original_tag.data, input + input_length);
112 |       EXPECT_LE(element->original_tag.length, input_length);
113 |     }
114 |     if (element->original_end_tag.data && element->original_tag.length) {
115 |       EXPECT_GE(element->original_end_tag.data, input);
116 |       EXPECT_LT(element->original_end_tag.data, input + input_length);
117 |       EXPECT_LE(element->original_end_tag.length, input_length);
118 |     }
119 |     EXPECT_GE(element->start_pos.offset, 0);
120 |     EXPECT_LE(element->start_pos.offset, input_length);
121 |     EXPECT_GE(element->end_pos.offset, 0);
122 |     EXPECT_LE(element->end_pos.offset, input_length);
123 | 
124 |     const GumboVector* children = &element->children;
125 |     for (int i = 0; i < children->length; ++i) {
126 |       const GumboNode* child = static_cast<const GumboNode*>(children->data[i]);
127 |       // Checks on parent/child links.
128 |       ASSERT_TRUE(child != NULL);
129 |       EXPECT_EQ(node, child->parent);
130 |       EXPECT_EQ(i, child->index_within_parent);
131 |       SanityCheckPointers(input, input_length, child, depth + 1);
132 |     }
133 |   } else {
134 |     const GumboText* text = &node->v.text;
135 |     EXPECT_GE(text->original_text.data, input);
136 |     EXPECT_LT(text->original_text.data, input + input_length);
137 |     EXPECT_LE(text->original_text.length, input_length);
138 |     EXPECT_GE(text->start_pos.offset, 0);
139 |     EXPECT_LT(text->start_pos.offset, input_length);
140 |   }
141 | }
142 | 
143 | // Custom allocator machinery to sanity check for memory leaks.  Normally we can
144 | // use heapcheck/valgrind/ASAN for this, but they only give the
145 | // results when the program terminates.  This means that if the parser is run in
146 | // a loop (say, a MapReduce) and there's a leak, it may end up exhausting memory
147 | // before it can catch the particular document responsible for the leak.  These
148 | // allocators let us check each document individually for leaks.
149 | 
150 | static void* LeakDetectingMalloc(void* userdata, size_t size) {
151 |   MallocStats* stats = static_cast<MallocStats*>(userdata);
152 |   stats->bytes_allocated += size;
153 |   ++stats->objects_allocated;
154 |   // Arbitrary limit of 2G on allocation; parsing any reasonable document
155 |   // shouldn't take more than that.
156 |   assert(stats->bytes_allocated < (1 << 31));
157 |   void* obj = malloc(size);
158 |   // gumbo_debug("Allocated %u bytes at %x.\n", size, obj);
159 |   return obj;
160 | }
161 | 
162 | static void LeakDetectingFree(void* userdata, void* ptr) {
163 |   MallocStats* stats = static_cast<MallocStats*>(userdata);
164 |   if (ptr) {
165 |     ++stats->objects_freed;
166 |     // gumbo_debug("Freed %x.\n");
167 |     free(ptr);
168 |   }
169 | }
170 | 
171 | void InitLeakDetection(GumboOptions* options, MallocStats* stats) {
172 |   stats->bytes_allocated = 0;
173 |   stats->objects_allocated = 0;
174 |   stats->objects_freed = 0;
175 | 
176 |   options->allocator = LeakDetectingMalloc;
177 |   options->deallocator = LeakDetectingFree;
178 |   options->userdata = stats;
179 | }
180 | 
181 | GumboTest::GumboTest()
182 |     : options_(kGumboDefaultOptions), errors_are_expected_(false), text_("") {
183 |   InitLeakDetection(&options_, &malloc_stats_);
184 |   options_.max_errors = 100;
185 |   parser_._options = &options_;
186 |   parser_._output = static_cast<GumboOutput*>(
187 |       gumbo_parser_allocate(&parser_, sizeof(GumboOutput)));
188 |   gumbo_init_errors(&parser_);
189 | }
190 | 
191 | GumboTest::~GumboTest() {
192 |   if (!errors_are_expected_) {
193 |     // TODO(jdtang): A googlemock matcher may be a more appropriate solution for
194 |     // this; we only want to pretty-print errors that are not an expected
195 |     // output of the test.
196 |     for (int i = 0; i < parser_._output->errors.length && i < 1; ++i) {
197 |       gumbo_print_caret_diagnostic(&parser_,
198 |           static_cast<GumboError*>(parser_._output->errors.data[i]), text_);
199 |     }
200 |   }
201 |   gumbo_destroy_errors(&parser_);
202 |   gumbo_parser_deallocate(&parser_, parser_._output);
203 |   EXPECT_EQ(malloc_stats_.objects_allocated, malloc_stats_.objects_freed);
204 | }
205 | 


--------------------------------------------------------------------------------
/tests/test_utils.h:
--------------------------------------------------------------------------------
 1 | // Copyright 2011 Google Inc. All Rights Reserved.
 2 | //
 3 | // Licensed under the Apache License, Version 2.0 (the "License");
 4 | // you may not use this file except in compliance with the License.
 5 | // You may obtain a copy of the License at
 6 | //
 7 | //     http://www.apache.org/licenses/LICENSE-2.0
 8 | //
 9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 | //
15 | // Author: jdtang@google.com (Jonathan Tang)
16 | 
17 | #ifndef GUMBO_TEST_UTILS_H_
18 | #define GUMBO_TEST_UTILS_H_
19 | 
20 | #include <assert.h>
21 | #include <stdint.h>
22 | #include <string>
23 | 
24 | #include "gtest/gtest.h"
25 | #include "gumbo.h"
26 | #include "parser.h"
27 | 
28 | inline std::string ToString(const GumboStringPiece& str) {
29 |   return std::string(str.data, str.length);
30 | }
31 | 
32 | int GetChildCount(GumboNode* node);
33 | GumboTag GetTag(GumboNode* node);
34 | GumboNode* GetChild(GumboNode* parent, int index);
35 | int GetAttributeCount(GumboNode* node);
36 | GumboAttribute* GetAttribute(GumboNode* node, int index);
37 | 
38 | // Convenience function to do some basic assertions on the structure of the
39 | // document (nodes are elements, nodes have the right tags) and then return
40 | // the body node.
41 | void GetAndAssertBody(GumboNode* root, GumboNode** body);
42 | void SanityCheckPointers(
43 |     const char* input, size_t input_length, const GumboNode* node, int depth);
44 | 
45 | // Custom allocator machinery to sanity check for memory leaks.  Normally we can
46 | // use heapcheck/valgrind/ASAN for this, but they only give the
47 | // results when the program terminates.  This means that if the parser is run in
48 | // a loop (say, a MapReduce) and there's a leak, it may end up exhausting memory
49 | // before it can catch the particular document responsible for the leak.  These
50 | // allocators let us check each document individually for leaks.
51 | 
52 | typedef struct {
53 |   uint64_t bytes_allocated;
54 |   uint64_t objects_allocated;
55 |   uint64_t objects_freed;
56 | } MallocStats;
57 | 
58 | void InitLeakDetection(GumboOptions* options, MallocStats* stats);
59 | 
60 | // Base class for Gumbo tests.  This provides an GumboParser object that's
61 | // been initialized to sane values, as normally happens in the beginning of
62 | // gumbo_parse, and then a destructor that cleans up after it.
63 | class GumboTest : public ::testing::Test {
64 |  protected:
65 |   GumboTest();
66 |   virtual ~GumboTest();
67 | 
68 |   MallocStats malloc_stats_;
69 |   GumboOptions options_;
70 |   GumboParser parser_;
71 |   bool errors_are_expected_;
72 |   const char* text_;
73 | };
74 | 
75 | #endif  // GUMBO_TEST_UTILS_H_
76 | 


--------------------------------------------------------------------------------
/tests/vector.cc:
--------------------------------------------------------------------------------
  1 | // Copyright 2011 Google Inc. All Rights Reserved.
  2 | //
  3 | // Licensed under the Apache License, Version 2.0 (the "License");
  4 | // you may not use this file except in compliance with the License.
  5 | // You may obtain a copy of the License at
  6 | //
  7 | //     http://www.apache.org/licenses/LICENSE-2.0
  8 | //
  9 | // Unless required by applicable law or agreed to in writing, software
 10 | // distributed under the License is distributed on an "AS IS" BASIS,
 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | // See the License for the specific language governing permissions and
 13 | // limitations under the License.
 14 | //
 15 | // Author: jdtang@google.com (Jonathan Tang)
 16 | 
 17 | #include "vector.h"
 18 | 
 19 | #include <stdlib.h>
 20 | #include <string.h>
 21 | 
 22 | #include "gtest/gtest.h"
 23 | #include "test_utils.h"
 24 | 
 25 | namespace {
 26 | 
 27 | class GumboVectorTest : public GumboTest {
 28 |  protected:
 29 |   GumboVectorTest()
 30 |       : one_(1),
 31 |         two_(2),
 32 |         three_(3),
 33 |         num_allocations_(0),
 34 |         bytes_allocated_(0),
 35 |         num_deallocations_(0) {
 36 |     gumbo_vector_init(&parser_, 2, &vector_);
 37 |   }
 38 | 
 39 |   ~GumboVectorTest() { gumbo_vector_destroy(&parser_, &vector_); }
 40 | 
 41 |   GumboVector vector_;
 42 | 
 43 |   // dummy ints that we can use to take addresses of.
 44 |   int one_;
 45 |   int two_;
 46 |   int three_;
 47 | 
 48 |   // Counters for testing a custom allocator.
 49 |   int num_allocations_;
 50 |   int bytes_allocated_;
 51 |   int num_deallocations_;
 52 | 
 53 |   static void* custom_gumbo_vector_allocator(void* userdata, size_t num_bytes) {
 54 |     GumboVectorTest* test = static_cast<GumboVectorTest*>(userdata);
 55 |     ++test->num_allocations_;
 56 |     test->bytes_allocated_ += num_bytes;
 57 |     return malloc(num_bytes);
 58 |   }
 59 | 
 60 |   static void custom_gumbo_vector_deallocator(void* userdata, void* ptr) {
 61 |     GumboVectorTest* test = static_cast<GumboVectorTest*>(userdata);
 62 |     ++test->num_deallocations_;
 63 |     free(ptr);
 64 |   }
 65 | };
 66 | 
 67 | TEST_F(GumboVectorTest, Init) {
 68 |   EXPECT_EQ(0, vector_.length);
 69 |   EXPECT_EQ(2, vector_.capacity);
 70 | }
 71 | 
 72 | TEST_F(GumboVectorTest, InitZeroCapacity) {
 73 |   gumbo_vector_destroy(&parser_, &vector_);
 74 |   gumbo_vector_init(&parser_, 0, &vector_);
 75 | 
 76 |   gumbo_vector_add(&parser_, &one_, &vector_);
 77 |   EXPECT_EQ(1, vector_.length);
 78 |   EXPECT_EQ(1, *(static_cast<int*>(vector_.data[0])));
 79 | }
 80 | 
 81 | TEST_F(GumboVectorTest, Add) {
 82 |   gumbo_vector_add(&parser_, &one_, &vector_);
 83 |   EXPECT_EQ(1, vector_.length);
 84 |   EXPECT_EQ(1, *(static_cast<int*>(vector_.data[0])));
 85 |   EXPECT_EQ(0, gumbo_vector_index_of(&vector_, &one_));
 86 |   EXPECT_EQ(-1, gumbo_vector_index_of(&vector_, &two_));
 87 | }
 88 | 
 89 | TEST_F(GumboVectorTest, AddMultiple) {
 90 |   gumbo_vector_add(&parser_, &one_, &vector_);
 91 |   gumbo_vector_add(&parser_, &two_, &vector_);
 92 |   EXPECT_EQ(2, vector_.length);
 93 |   EXPECT_EQ(2, *(static_cast<int*>(vector_.data[1])));
 94 |   EXPECT_EQ(1, gumbo_vector_index_of(&vector_, &two_));
 95 | }
 96 | 
 97 | TEST_F(GumboVectorTest, Realloc) {
 98 |   gumbo_vector_add(&parser_, &one_, &vector_);
 99 |   gumbo_vector_add(&parser_, &two_, &vector_);
100 |   gumbo_vector_add(&parser_, &three_, &vector_);
101 |   EXPECT_EQ(3, vector_.length);
102 |   EXPECT_EQ(4, vector_.capacity);
103 |   EXPECT_EQ(3, *(static_cast<int*>(vector_.data[2])));
104 | }
105 | 
106 | TEST_F(GumboVectorTest, Pop) {
107 |   gumbo_vector_add(&parser_, &one_, &vector_);
108 |   int result = *static_cast<int*>(gumbo_vector_pop(&parser_, &vector_));
109 |   EXPECT_EQ(1, result);
110 |   EXPECT_EQ(0, vector_.length);
111 | }
112 | 
113 | TEST_F(GumboVectorTest, PopEmpty) {
114 |   EXPECT_EQ(NULL, gumbo_vector_pop(&parser_, &vector_));
115 | }
116 | 
117 | TEST_F(GumboVectorTest, InsertAtFirst) {
118 |   gumbo_vector_add(&parser_, &one_, &vector_);
119 |   gumbo_vector_add(&parser_, &two_, &vector_);
120 |   gumbo_vector_insert_at(&parser_, &three_, 0, &vector_);
121 |   EXPECT_EQ(3, vector_.length);
122 |   int result = *static_cast<int*>(vector_.data[0]);
123 |   EXPECT_EQ(3, result);
124 | }
125 | 
126 | TEST_F(GumboVectorTest, InsertAtLast) {
127 |   gumbo_vector_add(&parser_, &one_, &vector_);
128 |   gumbo_vector_add(&parser_, &two_, &vector_);
129 |   gumbo_vector_insert_at(&parser_, &three_, 2, &vector_);
130 |   EXPECT_EQ(3, vector_.length);
131 |   int result = *static_cast<int*>(vector_.data[2]);
132 |   EXPECT_EQ(3, result);
133 | }
134 | 
135 | TEST_F(GumboVectorTest, Remove) {
136 |   gumbo_vector_add(&parser_, &one_, &vector_);
137 |   gumbo_vector_add(&parser_, &two_, &vector_);
138 |   gumbo_vector_add(&parser_, &three_, &vector_);
139 |   gumbo_vector_remove(&parser_, &two_, &vector_);
140 |   EXPECT_EQ(2, vector_.length);
141 |   int three = *static_cast<int*>(vector_.data[1]);
142 |   EXPECT_EQ(3, three);
143 | }
144 | 
145 | TEST_F(GumboVectorTest, RemoveAt) {
146 |   gumbo_vector_add(&parser_, &one_, &vector_);
147 |   gumbo_vector_add(&parser_, &two_, &vector_);
148 |   gumbo_vector_add(&parser_, &three_, &vector_);
149 |   int result =
150 |       *static_cast<int*>(gumbo_vector_remove_at(&parser_, 1, &vector_));
151 |   EXPECT_EQ(2, result);
152 |   EXPECT_EQ(2, vector_.length);
153 |   int three = *static_cast<int*>(vector_.data[1]);
154 |   EXPECT_EQ(3, three);
155 | }
156 | 
157 | }  // namespace
158 | 


--------------------------------------------------------------------------------
/visualc/gumbo.vcxproj:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="utf-8"?>
  2 | <Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
  3 |   <ItemGroup Label="ProjectConfigurations">
  4 |     <ProjectConfiguration Include="Debug|Win32">
  5 |       <Configuration>Debug</Configuration>
  6 |       <Platform>Win32</Platform>
  7 |     </ProjectConfiguration>
  8 |     <ProjectConfiguration Include="Release|Win32">
  9 |       <Configuration>Release</Configuration>
 10 |       <Platform>Win32</Platform>
 11 |     </ProjectConfiguration>
 12 |   </ItemGroup>
 13 |   <PropertyGroup Label="Globals">
 14 |     <ProjectGuid>{2661FA5E-ACC3-4D27-8FD2-9DBB0414EEDC}</ProjectGuid>
 15 |     <RootNamespace>gumbo</RootNamespace>
 16 |     <ProjectName>gumbo</ProjectName>
 17 |   </PropertyGroup>
 18 |   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
 19 |   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
 20 |     <ConfigurationType>StaticLibrary</ConfigurationType>
 21 |     <UseDebugLibraries>true</UseDebugLibraries>
 22 |     <PlatformToolset>v120</PlatformToolset>
 23 |     <CharacterSet>MultiByte</CharacterSet>
 24 |   </PropertyGroup>
 25 |   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
 26 |     <ConfigurationType>StaticLibrary</ConfigurationType>
 27 |     <UseDebugLibraries>false</UseDebugLibraries>
 28 |     <PlatformToolset>v120</PlatformToolset>
 29 |     <WholeProgramOptimization>true</WholeProgramOptimization>
 30 |     <CharacterSet>MultiByte</CharacterSet>
 31 |   </PropertyGroup>
 32 |   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
 33 |   <ImportGroup Label="ExtensionSettings">
 34 |   </ImportGroup>
 35 |   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
 36 |     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
 37 |   </ImportGroup>
 38 |   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
 39 |     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
 40 |   </ImportGroup>
 41 |   <PropertyGroup Label="UserMacros" />
 42 |   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
 43 |     <IncludePath>$(VC_IncludePath);$(WindowsSDK_IncludePath);./include</IncludePath>
 44 |   </PropertyGroup>
 45 |   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
 46 |     <IncludePath>./include;$(VC_IncludePath);$(WindowsSDK_IncludePath);</IncludePath>
 47 |   </PropertyGroup>
 48 |   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
 49 |     <ClCompile>
 50 |       <WarningLevel>Level3</WarningLevel>
 51 |       <Optimization>Disabled</Optimization>
 52 |       <SDLCheck>false</SDLCheck>
 53 |       <BufferSecurityCheck>false</BufferSecurityCheck>
 54 |     </ClCompile>
 55 |     <Link>
 56 |       <GenerateDebugInformation>true</GenerateDebugInformation>
 57 |     </Link>
 58 |   </ItemDefinitionGroup>
 59 |   <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
 60 |     <ClCompile>
 61 |       <WarningLevel>Level3</WarningLevel>
 62 |       <Optimization>MaxSpeed</Optimization>
 63 |       <FunctionLevelLinking>true</FunctionLevelLinking>
 64 |       <IntrinsicFunctions>true</IntrinsicFunctions>
 65 |       <SDLCheck>true</SDLCheck>
 66 |     </ClCompile>
 67 |     <Link>
 68 |       <GenerateDebugInformation>true</GenerateDebugInformation>
 69 |       <EnableCOMDATFolding>true</EnableCOMDATFolding>
 70 |       <OptimizeReferences>true</OptimizeReferences>
 71 |     </Link>
 72 |   </ItemDefinitionGroup>
 73 |   <ItemGroup>
 74 |     <ClCompile Include="..\src\attribute.c" />
 75 |     <ClCompile Include="..\src\char_ref.c" />
 76 |     <ClCompile Include="..\src\error.c" />
 77 |     <ClCompile Include="..\src\parser.c" />
 78 |     <ClCompile Include="..\src\string_buffer.c" />
 79 |     <ClCompile Include="..\src\string_piece.c" />
 80 |     <ClCompile Include="..\src\tag.c" />
 81 |     <ClCompile Include="..\src\tokenizer.c" />
 82 |     <ClCompile Include="..\src\utf8.c" />
 83 |     <ClCompile Include="..\src\util.c" />
 84 |     <ClCompile Include="..\src\vector.c" />
 85 |   </ItemGroup>
 86 |   <ItemGroup>
 87 |     <ClInclude Include="include\strings.h" />
 88 |     <ClInclude Include="..\src\attribute.h" />
 89 |     <ClInclude Include="..\src\char_ref.h" />
 90 |     <ClInclude Include="..\src\error.h" />
 91 |     <ClInclude Include="..\src\gumbo.h" />
 92 |     <ClInclude Include="..\src\insertion_mode.h" />
 93 |     <ClInclude Include="..\src\parser.h" />
 94 |     <ClInclude Include="..\src\string_buffer.h" />
 95 |     <ClInclude Include="..\src\string_piece.h" />
 96 |     <ClInclude Include="..\src\tokenizer.h" />
 97 |     <ClInclude Include="..\src\tokenizer_states.h" />
 98 |     <ClInclude Include="..\src\token_type.h" />
 99 |     <ClInclude Include="..\src\utf8.h" />
100 |     <ClInclude Include="..\src\util.h" />
101 |     <ClInclude Include="..\src\vector.h" />
102 |   </ItemGroup>
103 |   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
104 |   <ImportGroup Label="ExtensionTargets">
105 |   </ImportGroup>
106 | </Project>


--------------------------------------------------------------------------------
/visualc/gumbo.vcxproj.filters:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="utf-8"?>
 2 | <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
 3 |   <ItemGroup>
 4 |     <Filter Include="Source Files">
 5 |       <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
 6 |       <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
 7 |     </Filter>
 8 |     <Filter Include="Header Files">
 9 |       <UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
10 |       <Extensions>h;hh;hpp;hxx;hm;inl;inc;xsd</Extensions>
11 |     </Filter>
12 |     <Filter Include="Resource Files">
13 |       <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
14 |       <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
15 |     </Filter>
16 |   </ItemGroup>
17 |   <ItemGroup>
18 |     <ClCompile Include="..\src\attribute.c">
19 |       <Filter>Source Files</Filter>
20 |     </ClCompile>
21 |     <ClCompile Include="..\src\char_ref.c">
22 |       <Filter>Source Files</Filter>
23 |     </ClCompile>
24 |     <ClCompile Include="..\src\error.c">
25 |       <Filter>Source Files</Filter>
26 |     </ClCompile>
27 |     <ClCompile Include="..\src\parser.c">
28 |       <Filter>Source Files</Filter>
29 |     </ClCompile>
30 |     <ClCompile Include="..\src\string_buffer.c">
31 |       <Filter>Source Files</Filter>
32 |     </ClCompile>
33 |     <ClCompile Include="..\src\string_piece.c">
34 |       <Filter>Source Files</Filter>
35 |     </ClCompile>
36 |     <ClCompile Include="..\src\tag.c">
37 |       <Filter>Source Files</Filter>
38 |     </ClCompile>
39 |     <ClCompile Include="..\src\tokenizer.c">
40 |       <Filter>Source Files</Filter>
41 |     </ClCompile>
42 |     <ClCompile Include="..\src\utf8.c">
43 |       <Filter>Source Files</Filter>
44 |     </ClCompile>
45 |     <ClCompile Include="..\src\util.c">
46 |       <Filter>Source Files</Filter>
47 |     </ClCompile>
48 |     <ClCompile Include="..\src\vector.c">
49 |       <Filter>Source Files</Filter>
50 |     </ClCompile>
51 |   </ItemGroup>
52 |   <ItemGroup>
53 |     <ClInclude Include="..\src\attribute.h">
54 |       <Filter>Header Files</Filter>
55 |     </ClInclude>
56 |     <ClInclude Include="..\src\char_ref.h">
57 |       <Filter>Header Files</Filter>
58 |     </ClInclude>
59 |     <ClInclude Include="..\src\error.h">
60 |       <Filter>Header Files</Filter>
61 |     </ClInclude>
62 |     <ClInclude Include="..\src\gumbo.h">
63 |       <Filter>Header Files</Filter>
64 |     </ClInclude>
65 |     <ClInclude Include="..\src\insertion_mode.h">
66 |       <Filter>Header Files</Filter>
67 |     </ClInclude>
68 |     <ClInclude Include="..\src\parser.h">
69 |       <Filter>Header Files</Filter>
70 |     </ClInclude>
71 |     <ClInclude Include="..\src\string_buffer.h">
72 |       <Filter>Header Files</Filter>
73 |     </ClInclude>
74 |     <ClInclude Include="..\src\string_piece.h">
75 |       <Filter>Header Files</Filter>
76 |     </ClInclude>
77 |     <ClInclude Include="..\src\token_type.h">
78 |       <Filter>Header Files</Filter>
79 |     </ClInclude>
80 |     <ClInclude Include="..\src\tokenizer.h">
81 |       <Filter>Header Files</Filter>
82 |     </ClInclude>
83 |     <ClInclude Include="..\src\tokenizer_states.h">
84 |       <Filter>Header Files</Filter>
85 |     </ClInclude>
86 |     <ClInclude Include="..\src\utf8.h">
87 |       <Filter>Header Files</Filter>
88 |     </ClInclude>
89 |     <ClInclude Include="..\src\util.h">
90 |       <Filter>Header Files</Filter>
91 |     </ClInclude>
92 |     <ClInclude Include="..\src\vector.h">
93 |       <Filter>Header Files</Filter>
94 |     </ClInclude>
95 |     <ClInclude Include="include\strings.h">
96 |       <Filter>Header Files</Filter>
97 |     </ClInclude>
98 |   </ItemGroup>
99 | </Project>


--------------------------------------------------------------------------------
/visualc/gumbo.vcxproj.user:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="utf-8"?>
2 | <Project ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3 |   <PropertyGroup />
4 | </Project>


--------------------------------------------------------------------------------
/visualc/include/strings.h:
--------------------------------------------------------------------------------
1 | /*Dummy file to satisfy source file dependencies on Windows platform*/
2 | #define strcasecmp _stricmp
3 | #define strncasecmp _strnicmp
4 | #define inline __inline
5 | 


--------------------------------------------------------------------------------