├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── NEWS ├── README.md ├── emoji_presentation_scanner.c └── emoji_presentation_scanner.rl /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | ## Community Guidelines 26 | 27 | This project follows 28 | [Google's Open Source Community Guidelines](https://opensource.google.com/conduct/). 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | all: emoji_presentation_scanner.c 3 | 4 | %.c: %.rl Makefile 5 | ragel -F1 $< 6 | 7 | CFLAGS= \ 8 | -O2 \ 9 | -Demoji_text_iter_t="char *" \ 10 | -DEMOJI_LINKAGE= \ 11 | $(NULL) 12 | 13 | %.o: %.c Makefile 14 | $(CC) -c -o $@ $< $(CFLAGS) 15 | 16 | size: emoji_presentation_scanner.o 17 | size $< 18 | 19 | clean: 20 | rm -f emoji_presentation_scanner.o 21 | -------------------------------------------------------------------------------- /NEWS: -------------------------------------------------------------------------------- 1 | Overview of changes leading to 0.4.0 2 | Friday, November 15, 2024 3 | ==================================== 4 | 5 | * Add `make size` targe to 6 | determine binary size. 7 | * Set has_vs for 8 | emoji_keycap_sequence 9 | 10 | Overview of changes leading to 0.3.0 11 | Tuesday, September 5, 2024 12 | ==================================== 13 | 14 | * Add segmentation for variation 15 | selector pairs, VS15 and VS16, 16 | needed for font-variant-emoji. 17 | 18 | Overview of changes leading to 0.2.0 19 | Tuesday, Februar 20, 2024 20 | ==================================== 21 | 22 | * Change Ragel mode to -F1 23 | 24 | Overview of changes leading to 0.1.0 25 | Tuesday, January 29, 2019 26 | ==================================== 27 | 28 | * Initial release 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Emoji Segmenter 2 | === 3 | 4 | This repository contains a Ragel grammar and generated C code for segmenting 5 | runs of text into text-presentation and emoji-presentation runs. It is currently 6 | used in projects such as Chromium and Pango for deciding which preferred 7 | presentation, color or text, a run of text should have. 8 | 9 | The goal is to stay very close to the grammar definitions in [Unicode Technical 10 | Standard #51](http://www.unicode.org/reports/tr51/). 11 | 12 | API 13 | === 14 | 15 | By including the `emoji_presentation_scanner.c` file, you will be able to call 16 | the following API 17 | 18 | ``` 19 | scan_emoji_presentation (emoji_text_iter_t p, 20 | const emoji_text_iter_t pe, 21 | bool* is_emoji, 22 | bool* has_vs) 23 | ``` 24 | 25 | This API call will scan `emoji_text_iter_t p` for the next grammar-token and 26 | return an iterator that points to the end of the next token. An end iterator 27 | needs be specified as `pe` so that the scanner can compare against this and 28 | knows where to stop. In the reference parameter `is_emoji` it returns whether 29 | this token has emoji-presentation text-presentation, `has_vs` is set to true 30 | if the token contains a variation selector. 31 | 32 | A grammar token is either a combination of an emoji plus variation selector 15 33 | for text presentation, an emoji presentation sequence (emoji + VS16), an emoji 34 | presentation emoji or emoji sequence, or a single text presentation character. 35 | 36 | `emoji_text_iter_t` is an iterator type over a buffer of the character classes 37 | that are defined at the beginning of the the Ragel file, e.g. `EMOJI`, 38 | `EMOJI_TEXT_PRESENTATION`, `REGIONAL_INDICATOR`, `KEYCAP_BASE`, etc. 39 | 40 | By typedef'ing `emoji_text_iter_t` to your own iterator type, you can implement 41 | an adapter class that iterates over an input text buffer in any encoding, and on 42 | dereferencing returns the correct Ragel class by implementing something similar 43 | to the following Unicode character class to Ragel class mapping, example taken 44 | from Chromium: 45 | 46 | ``` 47 | char EmojiSegmentationCategory(UChar32 codepoint) { 48 | // Specific ones first. 49 | if (codepoint == kCombiningEnclosingKeycapCharacter) 50 | return COMBINING_ENCLOSING_KEYCAP; 51 | if (codepoint == kCombiningEnclosingCircleBackslashCharacter) 52 | return COMBINING_ENCLOSING_CIRCLE_BACKSLASH; 53 | if (codepoint == kZeroWidthJoinerCharacter) 54 | return ZWJ; 55 | if (codepoint == kVariationSelector15Character) 56 | return VS15; 57 | if (codepoint == kVariationSelector16Character) 58 | return VS16; 59 | if (codepoint == 0x1F3F4) 60 | return TAG_BASE; 61 | if ((codepoint >= 0xE0030 && codepoint <= 0xE0039) || 62 | (codepoint >= 0xE0061 && codepoint <= 0xE007A)) 63 | return TAG_SEQUENCE; 64 | if (codepoint == 0xE007F) 65 | return TAG_TERM; 66 | if (Character::IsEmojiModifierBase(codepoint)) 67 | return EMOJI_MODIFIER_BASE; 68 | if (Character::IsModifier(codepoint)) 69 | return EMOJI_MODIFIER; 70 | if (Character::IsRegionalIndicator(codepoint)) 71 | return REGIONAL_INDICATOR; 72 | if (Character::IsEmojiKeycapBase(codepoint)) 73 | return KEYCAP_BASE; 74 | 75 | if (Character::IsEmojiEmojiDefault(codepoint)) 76 | return EMOJI_EMOJI_PRESENTATION; 77 | if (Character::IsEmojiTextDefault(codepoint)) 78 | return EMOJI_TEXT_PRESENTATION; 79 | if (Character::IsEmoji(codepoint)) 80 | return EMOJI; 81 | 82 | // Ragel state machine will interpret unknown category as "any". 83 | return kMaxEmojiScannerCategory; 84 | } 85 | ``` 86 | 87 | Update/Build requisites 88 | === 89 | 90 | You need to have ragel installed if you want to modify the grammar and generate a new C file as output. 91 | 92 | `apt-get install ragel` 93 | 94 | then run 95 | 96 | `make` 97 | 98 | to update the `emoji_presentation_scanner.c` and `emoji_presentation_scanner_vs.c` output C source file. 99 | 100 | Contributing 101 | === 102 | 103 | See the CONTRIBUTING.md file for how to contribute. 104 | -------------------------------------------------------------------------------- /emoji_presentation_scanner.c: -------------------------------------------------------------------------------- 1 | 2 | #line 1 "emoji_presentation_scanner.rl" 3 | /* Copyright 2019 Google LLC 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | 20 | #ifndef EMOJI_LINKAGE 21 | #define EMOJI_LINKAGE static 22 | #endif 23 | 24 | 25 | #line 23 "emoji_presentation_scanner.c" 26 | static const unsigned char _emoji_presentation_trans_keys[] = { 27 | 0u, 13u, 14u, 15u, 0u, 13u, 9u, 12u, 10u, 12u, 10u, 10u, 4u, 12u, 4u, 12u, 28 | 6u, 6u, 9u, 12u, 8u, 8u, 8u, 10u, 9u, 14u, 0 29 | }; 30 | 31 | static const char _emoji_presentation_key_spans[] = { 32 | 14, 2, 14, 4, 3, 1, 9, 9, 33 | 1, 4, 1, 3, 6 34 | }; 35 | 36 | static const char _emoji_presentation_index_offsets[] = { 37 | 0, 15, 18, 33, 38, 42, 44, 54, 38 | 64, 66, 71, 73, 77 39 | }; 40 | 41 | static const char _emoji_presentation_indicies[] = { 42 | 1, 1, 1, 2, 0, 0, 0, 1, 43 | 0, 0, 0, 0, 0, 1, 0, 4, 44 | 5, 3, 6, 6, 7, 8, 9, 9, 45 | 10, 11, 9, 9, 9, 9, 9, 12, 46 | 9, 5, 13, 14, 15, 0, 13, 16, 47 | 17, 16, 13, 0, 17, 16, 16, 16, 48 | 16, 16, 13, 16, 17, 16, 17, 16, 49 | 16, 16, 16, 5, 13, 14, 15, 16, 50 | 5, 18, 5, 13, 19, 20, 18, 14, 51 | 21, 23, 22, 13, 22, 5, 13, 14, 52 | 15, 16, 4, 16, 0 53 | }; 54 | 55 | static const char _emoji_presentation_trans_targs[] = { 56 | 2, 4, 6, 2, 1, 2, 3, 3, 57 | 7, 2, 8, 9, 12, 0, 2, 5, 58 | 2, 5, 2, 10, 11, 2, 2, 2 59 | }; 60 | 61 | static const char _emoji_presentation_trans_actions[] = { 62 | 1, 2, 2, 3, 0, 4, 7, 2, 63 | 2, 8, 0, 7, 2, 0, 9, 10, 64 | 11, 2, 12, 0, 10, 13, 14, 15 65 | }; 66 | 67 | static const char _emoji_presentation_to_state_actions[] = { 68 | 0, 0, 5, 0, 0, 0, 0, 0, 69 | 0, 0, 0, 0, 0 70 | }; 71 | 72 | static const char _emoji_presentation_from_state_actions[] = { 73 | 0, 0, 6, 0, 0, 0, 0, 0, 74 | 0, 0, 0, 0, 0 75 | }; 76 | 77 | static const char _emoji_presentation_eof_trans[] = { 78 | 1, 4, 0, 1, 17, 1, 17, 17, 79 | 19, 19, 22, 23, 17 80 | }; 81 | 82 | static const int emoji_presentation_start = 2; 83 | 84 | static const int emoji_presentation_en_text_and_emoji_run = 2; 85 | 86 | 87 | #line 26 "emoji_presentation_scanner.rl" 88 | 89 | 90 | 91 | #line 100 "emoji_presentation_scanner.rl" 92 | 93 | 94 | EMOJI_LINKAGE emoji_text_iter_t 95 | scan_emoji_presentation (emoji_text_iter_t p, 96 | const emoji_text_iter_t pe, 97 | bool* is_emoji, 98 | bool* has_vs) 99 | { 100 | emoji_text_iter_t ts; 101 | emoji_text_iter_t te; 102 | const emoji_text_iter_t eof = pe; 103 | 104 | (void)ts; 105 | 106 | unsigned act; 107 | int cs; 108 | 109 | 110 | #line 102 "emoji_presentation_scanner.c" 111 | { 112 | cs = emoji_presentation_start; 113 | ts = 0; 114 | te = 0; 115 | act = 0; 116 | } 117 | 118 | #line 108 "emoji_presentation_scanner.c" 119 | { 120 | int _slen; 121 | int _trans; 122 | const unsigned char *_keys; 123 | const char *_inds; 124 | if ( p == pe ) 125 | goto _test_eof; 126 | _resume: 127 | switch ( _emoji_presentation_from_state_actions[cs] ) { 128 | case 6: 129 | #line 1 "NONE" 130 | {ts = p;} 131 | break; 132 | #line 120 "emoji_presentation_scanner.c" 133 | } 134 | 135 | _keys = _emoji_presentation_trans_keys + (cs<<1); 136 | _inds = _emoji_presentation_indicies + _emoji_presentation_index_offsets[cs]; 137 | 138 | _slen = _emoji_presentation_key_spans[cs]; 139 | _trans = _inds[ _slen > 0 && _keys[0] <=(*p) && 140 | (*p) <= _keys[1] ? 141 | (*p) - _keys[0] : _slen ]; 142 | 143 | _eof_trans: 144 | cs = _emoji_presentation_trans_targs[_trans]; 145 | 146 | if ( _emoji_presentation_trans_actions[_trans] == 0 ) 147 | goto _again; 148 | 149 | switch ( _emoji_presentation_trans_actions[_trans] ) { 150 | case 9: 151 | #line 94 "emoji_presentation_scanner.rl" 152 | {te = p+1;{ *is_emoji = false; *has_vs = true; return te; }} 153 | break; 154 | case 15: 155 | #line 95 "emoji_presentation_scanner.rl" 156 | {te = p+1;{ *is_emoji = true; *has_vs = true; return te; }} 157 | break; 158 | case 4: 159 | #line 96 "emoji_presentation_scanner.rl" 160 | {te = p+1;{ *is_emoji = true; *has_vs = false; return te; }} 161 | break; 162 | case 8: 163 | #line 97 "emoji_presentation_scanner.rl" 164 | {te = p+1;{ *is_emoji = false; *has_vs = false; return te; }} 165 | break; 166 | case 13: 167 | #line 94 "emoji_presentation_scanner.rl" 168 | {te = p;p--;{ *is_emoji = false; *has_vs = true; return te; }} 169 | break; 170 | case 14: 171 | #line 95 "emoji_presentation_scanner.rl" 172 | {te = p;p--;{ *is_emoji = true; *has_vs = true; return te; }} 173 | break; 174 | case 11: 175 | #line 96 "emoji_presentation_scanner.rl" 176 | {te = p;p--;{ *is_emoji = true; *has_vs = false; return te; }} 177 | break; 178 | case 12: 179 | #line 97 "emoji_presentation_scanner.rl" 180 | {te = p;p--;{ *is_emoji = false; *has_vs = false; return te; }} 181 | break; 182 | case 3: 183 | #line 96 "emoji_presentation_scanner.rl" 184 | {{p = ((te))-1;}{ *is_emoji = true; *has_vs = false; return te; }} 185 | break; 186 | case 1: 187 | #line 1 "NONE" 188 | { switch( act ) { 189 | case 2: 190 | {{p = ((te))-1;} *is_emoji = true; *has_vs = true; return te; } 191 | break; 192 | case 3: 193 | {{p = ((te))-1;} *is_emoji = true; *has_vs = false; return te; } 194 | break; 195 | case 4: 196 | {{p = ((te))-1;} *is_emoji = false; *has_vs = false; return te; } 197 | break; 198 | } 199 | } 200 | break; 201 | case 10: 202 | #line 1 "NONE" 203 | {te = p+1;} 204 | #line 95 "emoji_presentation_scanner.rl" 205 | {act = 2;} 206 | break; 207 | case 2: 208 | #line 1 "NONE" 209 | {te = p+1;} 210 | #line 96 "emoji_presentation_scanner.rl" 211 | {act = 3;} 212 | break; 213 | case 7: 214 | #line 1 "NONE" 215 | {te = p+1;} 216 | #line 97 "emoji_presentation_scanner.rl" 217 | {act = 4;} 218 | break; 219 | #line 190 "emoji_presentation_scanner.c" 220 | } 221 | 222 | _again: 223 | switch ( _emoji_presentation_to_state_actions[cs] ) { 224 | case 5: 225 | #line 1 "NONE" 226 | {ts = 0;} 227 | break; 228 | #line 197 "emoji_presentation_scanner.c" 229 | } 230 | 231 | if ( ++p != pe ) 232 | goto _resume; 233 | _test_eof: {} 234 | if ( p == eof ) 235 | { 236 | if ( _emoji_presentation_eof_trans[cs] > 0 ) { 237 | _trans = _emoji_presentation_eof_trans[cs] - 1; 238 | goto _eof_trans; 239 | } 240 | } 241 | 242 | } 243 | 244 | #line 120 "emoji_presentation_scanner.rl" 245 | 246 | 247 | /* Should not be reached. */ 248 | *is_emoji = false; 249 | *has_vs = false; 250 | return p; 251 | } 252 | -------------------------------------------------------------------------------- /emoji_presentation_scanner.rl: -------------------------------------------------------------------------------- 1 | /* Copyright 2019 Google LLC 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | #include 17 | 18 | #ifndef EMOJI_LINKAGE 19 | #define EMOJI_LINKAGE static 20 | #endif 21 | 22 | %%{ 23 | machine emoji_presentation; 24 | alphtype unsigned char; 25 | write data noerror nofinal; 26 | }%% 27 | 28 | %%{ 29 | 30 | EMOJI = 0; 31 | EMOJI_TEXT_PRESENTATION = 1; 32 | EMOJI_EMOJI_PRESENTATION = 2; 33 | EMOJI_MODIFIER_BASE = 3; 34 | EMOJI_MODIFIER = 4; 35 | EMOJI_VS_BASE = 5; 36 | REGIONAL_INDICATOR = 6; 37 | KEYCAP_BASE = 7; 38 | COMBINING_ENCLOSING_KEYCAP = 8; 39 | COMBINING_ENCLOSING_CIRCLE_BACKSLASH = 9; 40 | ZWJ = 10; 41 | VS15 = 11; 42 | VS16 = 12; 43 | TAG_BASE = 13; 44 | TAG_SEQUENCE = 14; 45 | TAG_TERM = 15; 46 | 47 | any_emoji = EMOJI_TEXT_PRESENTATION | EMOJI_EMOJI_PRESENTATION | KEYCAP_BASE | 48 | EMOJI_MODIFIER_BASE | TAG_BASE | EMOJI; 49 | 50 | emoji_combining_enclosing_circle_backslash_sequence = any_emoji 51 | COMBINING_ENCLOSING_CIRCLE_BACKSLASH; 52 | 53 | # This could be sharper than any_emoji by restricting this only to valid 54 | # variation sequences: 55 | # https://www.unicode.org/Public/emoji/11.0/emoji-variation-sequences.txt 56 | # However, implementing 57 | # https://www.unicode.org/reports/tr51/#def_emoji_presentation_sequence is 58 | # sufficient for our purposes here. 59 | emoji_presentation_sequence = any_emoji VS16; 60 | 61 | emoji_modifier_sequence = EMOJI_MODIFIER_BASE EMOJI_MODIFIER; 62 | 63 | emoji_flag_sequence = REGIONAL_INDICATOR REGIONAL_INDICATOR; 64 | 65 | # Here we only allow the valid tag sequences 66 | # https://www.unicode.org/reports/tr51/#valid-emoji-tag-sequences, instead of 67 | # all well-formed ones defined in 68 | # https://www.unicode.org/reports/tr51/#def_emoji_tag_sequence 69 | emoji_tag_sequence = TAG_BASE TAG_SEQUENCE+ TAG_TERM; 70 | 71 | emoji_keycap_sequence = KEYCAP_BASE VS16 COMBINING_ENCLOSING_KEYCAP; 72 | 73 | emoji_zwj_element = emoji_presentation_sequence | emoji_modifier_sequence | any_emoji; 74 | 75 | emoji_zwj_sequence = emoji_zwj_element ( ZWJ emoji_zwj_element )+; 76 | 77 | emoji_presentation = EMOJI_EMOJI_PRESENTATION | TAG_BASE | EMOJI_MODIFIER_BASE | 78 | emoji_presentation_sequence | emoji_modifier_sequence | emoji_flag_sequence | 79 | emoji_tag_sequence | emoji_keycap_sequence | emoji_zwj_sequence | 80 | emoji_combining_enclosing_circle_backslash_sequence; 81 | 82 | emoji_run = emoji_presentation; 83 | 84 | text_emoji_keycap_sequence = KEYCAP_BASE VS15 COMBINING_ENCLOSING_KEYCAP; 85 | text_emoji_run_with_vs = any_emoji VS15 | text_emoji_keycap_sequence; 86 | 87 | emoji_run_with_vs = emoji_presentation_sequence | emoji_keycap_sequence; 88 | 89 | text_run = any; 90 | 91 | text_and_emoji_run := |* 92 | # In order to give the the VS15 sequences higher priority than detecting 93 | # emoji sequences they are listed first as scanner token here. 94 | text_emoji_run_with_vs => { *is_emoji = false; *has_vs = true; return te; }; 95 | emoji_run_with_vs => { *is_emoji = true; *has_vs = true; return te; }; 96 | emoji_run => { *is_emoji = true; *has_vs = false; return te; }; 97 | text_run => { *is_emoji = false; *has_vs = false; return te; }; 98 | *|; 99 | 100 | }%% 101 | 102 | EMOJI_LINKAGE emoji_text_iter_t 103 | scan_emoji_presentation (emoji_text_iter_t p, 104 | const emoji_text_iter_t pe, 105 | bool* is_emoji, 106 | bool* has_vs) 107 | { 108 | emoji_text_iter_t ts; 109 | emoji_text_iter_t te; 110 | const emoji_text_iter_t eof = pe; 111 | 112 | (void)ts; 113 | 114 | unsigned act; 115 | int cs; 116 | 117 | %%{ 118 | write init; 119 | write exec; 120 | }%% 121 | 122 | /* Should not be reached. */ 123 | *is_emoji = false; 124 | *has_vs = false; 125 | return p; 126 | } 127 | --------------------------------------------------------------------------------