39 | * Entities in square brackets are optional. Note, it is not a 40 | * named entity detection mechanism. 41 | * 42 | * @alias string#extractPersonsName 43 | * @param {string} str the input string. 44 | * @return {string} extracted name. 45 | * @example 46 | * extractPersonsName( 'Dr. Sarah Connor M. Tech., PhD. - AI' ); 47 | * // -> 'Sarah Connor' 48 | */ 49 | var extractPersonsName = function ( str ) { 50 | // Remove Degrees by making the list of indexes of each degree and subsequently 51 | // finding the minimum and slicing from there! 52 | var indexes = ncrgx.degrees.map( function ( r ) { 53 | var m = r.exec( str ); 54 | return ( m ) ? m.index : 999999; 55 | } ); 56 | var sp = Math.min.apply( null, indexes ); 57 | 58 | // Generate an Array of Every Elelemnt of Name (e.g. title, first name, 59 | // sir name, honours, etc) 60 | var aeen = str.slice( 0, sp ).replace( rgx.notAlpha, ' ').replace( rgx.spaces, ' ').trim().split(' '); 61 | // Remove titles from the beginning. 62 | while ( aeen.length && ncrgx.titles.test( aeen[0] ) ) aeen.shift(); 63 | return aeen.join(' '); 64 | }; // extractPersonsName() 65 | 66 | module.exports = extractPersonsName; 67 | -------------------------------------------------------------------------------- /src/string-song.js: -------------------------------------------------------------------------------- 1 | // wink-nlp-utils 2 | // NLP Functions for amplifying negations, managing elisions, 3 | // creating ngrams, stems, phonetic codes to tokens and more. 4 | // 5 | // Copyright (C) GRAYPE Systems Private Limited 6 | // 7 | // This file is part of “wink-nlp-utils”. 8 | // 9 | // Permission is hereby granted, free of charge, to any person obtaining a 10 | // copy of this software and associated documentation files (the "Software"), 11 | // to deal in the Software without restriction, including without limitation 12 | // the rights to use, copy, modify, merge, publish, distribute, sublicense, 13 | // and/or sell copies of the Software, and to permit persons to whom the 14 | // Software is furnished to do so, subject to the following conditions: 15 | // 16 | // The above copyright notice and this permission notice shall be included 17 | // in all copies or substantial portions of the Software. 18 | // 19 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20 | // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 22 | // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 24 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | // DEALINGS IN THE SOFTWARE. 26 | 27 | // 28 | 29 | // ## string 30 | 31 | // ### setOfNGrams 32 | /** 33 | * 34 | * Generates the set of ngrams of `size` from the input string. The 35 | * default size is 2, which means it will generate set of bigrams by default. 36 | * It also has an alias **`song()`**. 37 | * 38 | * @alias string#setOfNGrams 39 | * @param {string} str the input string. 40 | * @param {number} [size=2] ngram size. 41 | * @param {function} [ifn=undefined] a function to build index; it is called for 42 | * every **unique occurrence of ngram** of `str`; and it receives the ngram and the `idx` 43 | * as input arguments. The `build()` function of [helper.returnIndexer](#helperreturnindexer) 44 | * may be used as `ifn`. If `undefined` then index is not built. 45 | * @param {number} [idx=undefined] the index; passed as the second argument to the `ifn` 46 | * function. 47 | * @return {set} of ngrams of `size` of `str`. 48 | * @example 49 | * setOfNGrams( 'mama' ); 50 | * // -> Set { 'ma', 'am' } 51 | * song( 'mamma' ); 52 | * // -> Set { 'ma', 'am', 'mm' } 53 | */ 54 | var setOfNGrams = function ( str, size, ifn, idx ) { 55 | var ng = ( size || 2 ), 56 | ngSet = new Set(), 57 | tg; 58 | for ( var i = 0, imax = str.length; i < imax; i += 1 ) { 59 | tg = str.slice( i, i + ng ); 60 | if ( tg.length === ng ) { 61 | if ( ( typeof ifn === 'function' ) && !ngSet.has( tg ) ) { 62 | ifn( tg, idx ); 63 | } 64 | ngSet.add( tg ); 65 | } 66 | } 67 | return ( ngSet ); 68 | }; // song() 69 | 70 | module.exports = setOfNGrams; 71 | -------------------------------------------------------------------------------- /src/string-edge-ngrams.js: -------------------------------------------------------------------------------- 1 | // wink-nlp-utils 2 | // NLP Functions for amplifying negations, managing elisions, 3 | // creating ngrams, stems, phonetic codes to tokens and more. 4 | // 5 | // Copyright (C) GRAYPE Systems Private Limited 6 | // 7 | // This file is part of “wink-nlp-utils”. 8 | // 9 | // Permission is hereby granted, free of charge, to any person obtaining a 10 | // copy of this software and associated documentation files (the "Software"), 11 | // to deal in the Software without restriction, including without limitation 12 | // the rights to use, copy, modify, merge, publish, distribute, sublicense, 13 | // and/or sell copies of the Software, and to permit persons to whom the 14 | // Software is furnished to do so, subject to the following conditions: 15 | // 16 | // The above copyright notice and this permission notice shall be included 17 | // in all copies or substantial portions of the Software. 18 | // 19 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20 | // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 22 | // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 24 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | // DEALINGS IN THE SOFTWARE. 26 | 27 | // 28 | 29 | // ## string 30 | 31 | // ### edgeNGrams 32 | /** 33 | * 34 | * Generates the edge ngrams from the input string. 35 | * 36 | * @alias string#edgeNGrams 37 | * @param {string} str the input string. 38 | * @param {number} [min=2] size of ngram generated. 39 | * @param {number} [max=8] size of ngram is generated. 40 | * @param {number} [delta=2] edge ngrams are generated in increments of this value. 41 | * @param {function} [ifn=undefined] a function to build index; it is called for 42 | * every edge ngram of `str`; and it receives the edge ngram and the `idx` 43 | * as input arguments. The `build()` function of [helper.returnIndexer](#helperreturnindexer) 44 | * may be used as `ifn`. If `undefined` then index is not built. 45 | * @param {number} [idx=undefined] the index; passed as the second argument to the `ifn` 46 | * function. 47 | * @return {string[]} of edge ngrams. 48 | * @example 49 | * edgeNGrams( 'decisively' ); 50 | * // -> [ 'de', 'deci', 'decisi', 'decisive' ] 51 | * edgeNGrams( 'decisively', 8, 10, 1 ); 52 | * // -> [ 'decisive', 'decisivel', 'decisively' ] 53 | */ 54 | var edgeNGrams = function ( str, min, max, delta, ifn, idx ) { 55 | var dlta = ( delta || 2 ), 56 | eg, 57 | egs = [], 58 | imax = Math.min( ( max || 8 ), str.length ) + 1, 59 | start = ( min || 2 ); 60 | 61 | // Generate edge ngrams 62 | for ( var i = start; i < imax; i += dlta ) { 63 | eg = str.slice( 0, i ); 64 | egs.push( eg ); 65 | if ( typeof ifn === 'function' ) { 66 | ifn( eg, idx ); 67 | } 68 | } 69 | return ( egs ); 70 | }; // edgeNGrams() 71 | 72 | module.exports = edgeNGrams; 73 | -------------------------------------------------------------------------------- /src/string-compose-corpus.js: -------------------------------------------------------------------------------- 1 | // wink-nlp-utils 2 | // NLP Functions for amplifying negations, managing elisions, 3 | // creating ngrams, stems, phonetic codes to tokens and more. 4 | // 5 | // Copyright (C) GRAYPE Systems Private Limited 6 | // 7 | // This file is part of “wink-nlp-utils”. 8 | // 9 | // Permission is hereby granted, free of charge, to any person obtaining a 10 | // copy of this software and associated documentation files (the "Software"), 11 | // to deal in the Software without restriction, including without limitation 12 | // the rights to use, copy, modify, merge, publish, distribute, sublicense, 13 | // and/or sell copies of the Software, and to permit persons to whom the 14 | // Software is furnished to do so, subject to the following conditions: 15 | // 16 | // The above copyright notice and this permission notice shall be included 17 | // in all copies or substantial portions of the Software. 18 | // 19 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20 | // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 22 | // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 24 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | // DEALINGS IN THE SOFTWARE. 26 | 27 | // 28 | var helpers = require( 'wink-helpers' ); 29 | var returnQuotedTextExtractor = require( './helper-return-quoted-text-extractor.js' ); 30 | var extractQuotedText = returnQuotedTextExtractor( '[', ']' ); 31 | // ## string 32 | 33 | // ### composeCorpus 34 | /** 35 | * 36 | * Generates all possible sentences from the input argument string. 37 | * The string s must follow a special syntax as illustrated in the 38 | * example below:
39 | * `'[I] [am having|have] [a] [problem|question]'`
40 | * 41 | * Each phrase must be quoted between `[ ]` and each possible option of phrases 42 | * (if any) must be separated by a `|` character. The corpus is composed by 43 | * computing the cartesian product of all the phrases. 44 | * 45 | * @alias string#composeCorpus 46 | * @param {string} str the input string. 47 | * @return {string[]} of all possible sentences. 48 | * @example 49 | * composeCorpus( '[I] [am having|have] [a] [problem|question]' ); 50 | * // -> [ 'I am having a problem', 51 | * // 'I am having a question', 52 | * // 'I have a problem', 53 | * // 'I have a question' ] 54 | */ 55 | var composeCorpus = function ( str ) { 56 | if ( !str || ( typeof str !== 'string' ) ) return []; 57 | 58 | var quotedTextElems = extractQuotedText( str ); 59 | var corpus = []; 60 | var finalCorpus = []; 61 | 62 | if ( !quotedTextElems ) return []; 63 | quotedTextElems.forEach( function ( e ) { 64 | corpus.push( e.split( '|' ) ); 65 | } ); 66 | 67 | helpers.array.product( corpus ).forEach( function ( e ) { 68 | finalCorpus.push( e.join( ' ' ) ); 69 | } ); 70 | return ( finalCorpus ); 71 | }; // composeCorpus() 72 | 73 | module.exports = composeCorpus; 74 | -------------------------------------------------------------------------------- /src/tokens-propagate-negations.js: -------------------------------------------------------------------------------- 1 | // wink-nlp-utils 2 | // NLP Functions for amplifying negations, managing elisions, 3 | // creating ngrams, stems, phonetic codes to tokens and more. 4 | // 5 | // Copyright (C) GRAYPE Systems Private Limited 6 | // 7 | // This file is part of “wink-nlp-utils”. 8 | // 9 | // Permission is hereby granted, free of charge, to any person obtaining a 10 | // copy of this software and associated documentation files (the "Software"), 11 | // to deal in the Software without restriction, including without limitation 12 | // the rights to use, copy, modify, merge, publish, distribute, sublicense, 13 | // and/or sell copies of the Software, and to permit persons to whom the 14 | // Software is furnished to do so, subject to the following conditions: 15 | // 16 | // The above copyright notice and this permission notice shall be included 17 | // in all copies or substantial portions of the Software. 18 | // 19 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20 | // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 22 | // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 24 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | // DEALINGS IN THE SOFTWARE. 26 | 27 | // 28 | var rgx = require( './util_regexes.js' ); 29 | 30 | // ## string 31 | 32 | // ### propagateNegations 33 | /** 34 | * 35 | * It looks for negation tokens in the input array of tokens and propagates 36 | * negation to subsequent `upto` tokens by prefixing them by a `!`. It is useful 37 | * in handling text containing negations during tasks like similarity detection, 38 | * classification or search. 39 | * 40 | * @alias tokens#propagateNegations 41 | * @param {string[]} tokens the input tokens. 42 | * @param {number} [upto=2] number of tokens to be negated after the negation 43 | * token. Note, tokens are only negated either `upto` tokens or up to the token 44 | * preceeding the **`, . ; : ! ?`** punctuations. 45 | * @return {string[]} tokens with negation propagated. 46 | * @example 47 | * propagateNegations( [ 'mary', 'is', 'not', 'feeling', 'good', 'today' ] ); 48 | * // -> [ 'mary', 'is', 'not', '!feeling', '!good', 'today' ] 49 | */ 50 | var propagateNegations = function ( tokens, upto ) { 51 | var i, imax, j, jmax; 52 | var tkns = tokens; 53 | var limit = upto || 2; 54 | for ( i = 0, imax = tkns.length; i < imax; i += 1 ) { 55 | if ( rgx.negations.test( tkns[ i ] ) ) { 56 | for ( j = i + 1, jmax = Math.min( imax, i + limit + 1 ); j < jmax; j += 1 ) { 57 | // Hit a punctuation mark, break out of the loop otherwise go *upto the limit*. 58 | // > TODO: promote to utilities regex, after test cases have been added. 59 | if ( ( /[\,\.\;\:\!\?]/ ).test( tkns[ j ] ) ) break; 60 | // Propoage negation: invert the token by prefixing a `!` to it. 61 | tkns[ j ] = '!' + tkns[ j ]; 62 | } 63 | i = j; 64 | } 65 | } 66 | return tkns; 67 | }; // propagateNegations() 68 | 69 | module.exports = propagateNegations; 70 | -------------------------------------------------------------------------------- /src/phonetize_regexes.js: -------------------------------------------------------------------------------- 1 | // wink-nlp-utils 2 | // NLP Functions for amplifying negations, managing elisions, 3 | // creating ngrams, stems, phonetic codes to tokens and more. 4 | // 5 | // Copyright (C) GRAYPE Systems Private Limited 6 | // 7 | // This file is part of “wink-nlp-utils”. 8 | // 9 | // Permission is hereby granted, free of charge, to any person obtaining a 10 | // copy of this software and associated documentation files (the "Software"), 11 | // to deal in the Software without restriction, including without limitation 12 | // the rights to use, copy, modify, merge, publish, distribute, sublicense, 13 | // and/or sell copies of the Software, and to permit persons to whom the 14 | // Software is furnished to do so, subject to the following conditions: 15 | // 16 | // The above copyright notice and this permission notice shall be included 17 | // in all copies or substantial portions of the Software. 18 | // 19 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20 | // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 22 | // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 24 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | // DEALINGS IN THE SOFTWARE. 26 | 27 | // 28 | /* eslint no-underscore-dangle: "off" */ 29 | var rgx = Object.create( null ); 30 | // Remove repeating characters. 31 | rgx.repeatingChars = /([^c])\1/g; 32 | // Drop first character from character pairs, if found in the beginning. 33 | rgx.kngnPairs = /^(kn|gn|pn|ae|wr)/; 34 | // Drop vowels that are not found in the beginning. 35 | rgx.__vowels = /(?!^)[aeiou]/g; 36 | // Replaces `ough` in the end by 'f' 37 | rgx.ough = /ough$/; 38 | // Replace following 3 instances of `dg` by `j`. 39 | rgx.dge = /dge/g; 40 | rgx.dgi = /dgi/g; 41 | rgx.dgy = /dgy/g; 42 | // Replace `sch` by `sk`. 43 | rgx.sch = /sch/g; 44 | // Drop `c` in `sci, sce, scy`. 45 | rgx.sci = /sci/g; 46 | rgx.sce = /sce/g; 47 | rgx.scy = /scy/g; 48 | // Make 'sh' out of `tio & tia`. 49 | rgx.tio = /tio/g; 50 | rgx.tia = /tia/g; 51 | // `t` is silent in `tch`. 52 | rgx.tch = /tch/g; 53 | // Drop `b` in the end if preceeded by `m`. 54 | rgx.mb_ = /mb$/; 55 | // These are pronounced as `k`. 56 | rgx.cq = /cq/g; 57 | rgx.ck = /ck/g; 58 | // Here `c` sounds like `s` 59 | rgx.ce = /ce/g; 60 | rgx.ci = /ci/g; 61 | rgx.cy = /cy/g; 62 | // And this `f`. 63 | rgx.ph = /ph/g; 64 | // The `sh` finally replaced by `x`. 65 | rgx.sh = /sh|sio|sia/g; 66 | // This is open rgx - TODO: need to finalize. 67 | rgx.vrnotvy = /([aeiou])(r)([^aeiouy])/g; 68 | // `th` sounds like theta - make it 0. 69 | rgx.th = /th/g; 70 | // `c` sounds like `k` except when it is followed by `h`. 71 | rgx.cnoth = /(c)([^h])/g; 72 | // Even `q` sounds like `k`. 73 | rgx.q = /q/g; 74 | // The first `x` sounds like `s`. 75 | rgx._x = /^x/; 76 | // Otherwise `x` is more like `ks`. 77 | rgx.x = /x/g; 78 | // Drop `y` if not followed by a vowel or appears in the end. 79 | rgx.ynotv = /(y)([^aeiou])/g; 80 | rgx.y_ = /y$/; 81 | // `z` is `s`. 82 | rgx.z = /z/g; 83 | 84 | // Export rgx. 85 | module.exports = rgx; 86 | -------------------------------------------------------------------------------- /src/string-bong.js: -------------------------------------------------------------------------------- 1 | // wink-nlp-utils 2 | // NLP Functions for amplifying negations, managing elisions, 3 | // creating ngrams, stems, phonetic codes to tokens and more. 4 | // 5 | // Copyright (C) GRAYPE Systems Private Limited 6 | // 7 | // This file is part of “wink-nlp-utils”. 8 | // 9 | // Permission is hereby granted, free of charge, to any person obtaining a 10 | // copy of this software and associated documentation files (the "Software"), 11 | // to deal in the Software without restriction, including without limitation 12 | // the rights to use, copy, modify, merge, publish, distribute, sublicense, 13 | // and/or sell copies of the Software, and to permit persons to whom the 14 | // Software is furnished to do so, subject to the following conditions: 15 | // 16 | // The above copyright notice and this permission notice shall be included 17 | // in all copies or substantial portions of the Software. 18 | // 19 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20 | // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 22 | // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 24 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | // DEALINGS IN THE SOFTWARE. 26 | 27 | // 28 | 29 | // ## string 30 | 31 | // ### bagOfNGrams 32 | /** 33 | * 34 | * Generates the bag of ngrams of `size` from the input string. The 35 | * default size is 2, which means it will generate bag of bigrams by default. It 36 | * also has an alias **`bong()`**. 37 | * 38 | * @alias string#bagOfNGrams 39 | * @param {string} str the input string. 40 | * @param {number} [size=2] ngram size. 41 | * @param {function} [ifn=undefined] a function to build index; it is called for 42 | * every **unique occurrence of ngram** of `str`; and it receives the ngram and the `idx` 43 | * as input arguments. The `build()` function of [helper.returnIndexer](#helperreturnindexer) 44 | * may be used as `ifn`. If `undefined` then index is not built. 45 | * @param {number} [idx=undefined] the index; passed as the second argument to the `ifn` 46 | * function. 47 | * @return {object} bag of ngrams of `size` from `str`. 48 | * @example 49 | * bagOfNGrams( 'mama' ); 50 | * // -> { ma: 2, am: 1 } 51 | * bong( 'mamma' ); 52 | * // -> { ma: 2, am: 1, mm: 1 } 53 | */ 54 | var bagOfNGrams = function ( str, size, ifn, idx ) { 55 | var ng = ( size || 2 ), 56 | ngBOW = Object.create( null ), 57 | tg; 58 | for ( var i = 0, imax = str.length; i < imax; i += 1 ) { 59 | tg = str.slice( i, i + ng ); 60 | if ( tg.length === ng ) { 61 | // Call `ifn` iff its defined and `tg` is appearing for the first time; 62 | // this avoids multiple calls to `ifn`. Strategy applies to `song()`, 63 | // and `bow()`. 64 | if ( ( typeof ifn === 'function' ) && !ngBOW[ tg ] ) { 65 | ifn( tg, idx ); 66 | } 67 | // Now define, if required and then update counts. 68 | ngBOW[ tg ] = 1 + ( ngBOW[ tg ] || 0 ); 69 | } 70 | } 71 | return ( ngBOW ); 72 | }; // bong() 73 | 74 | module.exports = bagOfNGrams; 75 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | ## Our Standards 8 | 9 | Examples of behavior that contributes to creating a positive environment include: 10 | 11 | * Using welcoming and inclusive language 12 | * Being respectful of differing viewpoints and experiences 13 | * Gracefully accepting constructive criticism 14 | * Focusing on what is best for the community 15 | * Showing empathy towards other community members 16 | 17 | Examples of unacceptable behavior by participants include: 18 | 19 | * The use of sexualized language or imagery and unwelcome sexual attention or advances 20 | * Trolling, insulting/derogatory comments, and personal or political attacks 21 | * Public or private harassment 22 | * Publishing others' private information, such as a physical or electronic address, without explicit permission 23 | * Other conduct which could reasonably be considered inappropriate in a professional setting 24 | 25 | ## Our Responsibilities 26 | 27 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 28 | 29 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 30 | 31 | ## Scope 32 | 33 | This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 34 | 35 | ## Enforcement 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at ContactUs@graype.in. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 38 | 39 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. 40 | 41 | ## Attribution 42 | 43 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] 44 | 45 | [homepage]: http://contributor-covenant.org 46 | [version]: http://contributor-covenant.org/version/1/4/ 47 | -------------------------------------------------------------------------------- /src/helper-return-words-filter.js: -------------------------------------------------------------------------------- 1 | // wink-nlp-utils 2 | // NLP Functions for amplifying negations, managing elisions, 3 | // creating ngrams, stems, phonetic codes to tokens and more. 4 | // 5 | // Copyright (C) GRAYPE Systems Private Limited 6 | // 7 | // This file is part of “wink-nlp-utils”. 8 | // 9 | // Permission is hereby granted, free of charge, to any person obtaining a 10 | // copy of this software and associated documentation files (the "Software"), 11 | // to deal in the Software without restriction, including without limitation 12 | // the rights to use, copy, modify, merge, publish, distribute, sublicense, 13 | // and/or sell copies of the Software, and to permit persons to whom the 14 | // Software is furnished to do so, subject to the following conditions: 15 | // 16 | // The above copyright notice and this permission notice shall be included 17 | // in all copies or substantial portions of the Software. 18 | // 19 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20 | // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 22 | // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 24 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | // DEALINGS IN THE SOFTWARE. 26 | 27 | // 28 | 29 | // ## string 30 | 31 | // ### returnWordsFilter 32 | 33 | /** 34 | * 35 | * Returns an object containing the following functions: (a) `set()`, which returns 36 | * a set of mapped words given in the input array `words`. (b) `exclude()` that 37 | * is suitable for array filtering operations. 38 | * 39 | * If the second argument `mappers` is provided as an array of maping functions 40 | * then these are applied on the input array before converting into a set. A 41 | * mapper function must accept a string as argument and return a string as the result. 42 | * Examples of mapper functions are typically **string** functionss of **`wink-nlp-utils`** 43 | * such as `string.lowerCase()`, `string.stem()` and 44 | * `string.soundex()`. 45 | * 46 | * @alias helper#returnWordsFilter 47 | * @param {string[]} words that can be filtered using the returned wordsFilter. 48 | * @param {function[]} [mappers=undefined] optionally used to map each word before creating 49 | * the wordsFilter. 50 | * @return {wordsFilter} object containg `set()` and `exclude()` functions for `words`. 51 | * @example 52 | * var stopWords = [ 'This', 'That', 'Are', 'Is', 'Was', 'Will', 'a' ]; 53 | * var myFilter = returnWordsFilter( stopWords, [ string.lowerCase ] ); 54 | * [ 'this', 'is', 'a', 'cat' ].filter( myFilter.exclude ); 55 | * // -> [ 'cat' ] 56 | */ 57 | var returnWordsFilter = function ( words, mappers ) { 58 | var mappedWords = words; 59 | var givenMappers = mappers || []; 60 | givenMappers.forEach( function ( m ) { 61 | mappedWords = mappedWords.map( m ); 62 | } ); 63 | 64 | mappedWords = new Set( mappedWords ); 65 | 66 | var exclude = function ( t ) { 67 | return ( !( mappedWords.has( t ) ) ); 68 | }; // exclude() 69 | 70 | var set = function () { 71 | return mappedWords; 72 | }; // set() 73 | 74 | return { 75 | set: set, 76 | exclude: exclude 77 | }; 78 | }; // returnWordsFilter() 79 | 80 | module.exports = returnWordsFilter; 81 | -------------------------------------------------------------------------------- /src/tokens-bow.js: -------------------------------------------------------------------------------- 1 | // wink-nlp-utils 2 | // NLP Functions for amplifying negations, managing elisions, 3 | // creating ngrams, stems, phonetic codes to tokens and more. 4 | // 5 | // Copyright (C) GRAYPE Systems Private Limited 6 | // 7 | // This file is part of “wink-nlp-utils”. 8 | // 9 | // Permission is hereby granted, free of charge, to any person obtaining a 10 | // copy of this software and associated documentation files (the "Software"), 11 | // to deal in the Software without restriction, including without limitation 12 | // the rights to use, copy, modify, merge, publish, distribute, sublicense, 13 | // and/or sell copies of the Software, and to permit persons to whom the 14 | // Software is furnished to do so, subject to the following conditions: 15 | // 16 | // The above copyright notice and this permission notice shall be included 17 | // in all copies or substantial portions of the Software. 18 | // 19 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20 | // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 22 | // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 24 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | // DEALINGS IN THE SOFTWARE. 26 | 27 | // 28 | 29 | // ## string 30 | 31 | // ### bagOfWords 32 | /** 33 | * 34 | * Generates the bag of words from the input string. By default it 35 | * uses `word count` as it's frequency; but if `logCounts` parameter is set to true then 36 | * it will use `log2( word counts + 1 )` as it's frequency. It also has an alias **`bow()`**. 37 | * 38 | * @alias tokens#bagOfWords 39 | * @param {string[]} tokens the input tokens. 40 | * @param {number} [logCounts=false] a true value flags the use of `log2( word count + 1 )` 41 | * instead of just `word count` as frequency. 42 | * @param {function} [ifn=undefined] a function to build index; it is called for 43 | * every **unique occurrence of word** in `tokens`; and it receives the word and the `idx` 44 | * as input arguments. The `build()` function of [helper.returnIndexer](#helperreturnindexer) 45 | * may be used as `ifn`. If `undefined` then index is not built. 46 | * @param {number} [idx=undefined] the index; passed as the second argument to the `ifn` 47 | * function. 48 | * @return {object} bag of words from tokens. 49 | * @example 50 | * bagOfWords( [ 'rain', 'rain', 'go', 'away' ] ); 51 | * // -> { rain: 2, go: 1, away: 1 } 52 | * bow( [ 'rain', 'rain', 'go', 'away' ], true ); 53 | * // -> { rain: 1.584962500721156, go: 1, away: 1 } 54 | */ 55 | var bagOfWords = function ( tokens, logCounts, ifn, idx ) { 56 | var bow1 = Object.create( null ), 57 | i, imax, 58 | token, 59 | words; 60 | for ( i = 0, imax = tokens.length; i < imax; i += 1 ) { 61 | token = tokens[ i ]; 62 | if ( ( typeof ifn === 'function' ) && !bow1[ token ] ) { 63 | ifn( token, idx ); 64 | } 65 | bow1[ token ] = 1 + ( bow1[ token ] || 0 ); 66 | } 67 | if ( !logCounts ) return ( bow1 ); 68 | words = Object.keys( bow1 ); 69 | for ( i = 0, imax = words.length; i < imax; i += 1 ) { 70 | // Add `1` to ensure non-zero count! (Note: log2(1) is 0) 71 | bow1[ words[ i ] ] = Math.log2( bow1[ words[ i ] ] + 1 ); 72 | } 73 | return ( bow1 ); 74 | }; // bow() 75 | 76 | module.exports = bagOfWords; 77 | -------------------------------------------------------------------------------- /test/string-edge-ngrams-specs.js: -------------------------------------------------------------------------------- 1 | // wink-nlp-utils 2 | // NLP Functions for amplifying negations, managing elisions, 3 | // creating ngrams, stems, phonetic codes to tokens and more. 4 | // 5 | // Copyright (C) GRAYPE Systems Private Limited 6 | // 7 | // This file is part of “wink-nlp-utils”. 8 | // 9 | // Permission is hereby granted, free of charge, to any person obtaining a 10 | // copy of this software and associated documentation files (the "Software"), 11 | // to deal in the Software without restriction, including without limitation 12 | // the rights to use, copy, modify, merge, publish, distribute, sublicense, 13 | // and/or sell copies of the Software, and to permit persons to whom the 14 | // Software is furnished to do so, subject to the following conditions: 15 | // 16 | // The above copyright notice and this permission notice shall be included 17 | // in all copies or substantial portions of the Software. 18 | // 19 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20 | // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 22 | // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 24 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | // DEALINGS IN THE SOFTWARE. 26 | 27 | // 28 | var chai = require( 'chai' ); 29 | var mocha = require( 'mocha' ); 30 | var edgeGrams = require( '../src/string-edge-ngrams.js' ); 31 | var index = require( '../src/helper-return-indexer.js' ); 32 | 33 | var expect = chai.expect; 34 | var describe = mocha.describe; 35 | var it = mocha.it; 36 | 37 | // ### Define common errors. 38 | // These are common test data for `null`, `undefined`, and `numeric` inputs 39 | // across all the functions included in the script. 40 | // The exception cases specific to the function are part of the test script of the function. 41 | var errors = [ 42 | { whenInputIs: null, expectedOutputIs: /^Cannot read.*/ }, 43 | { whenInputIs: undefined, expectedOutputIs: /^Cannot read.*/ }, 44 | { whenInputIs: 1, expectedOutputIs: /is not a function$/ } 45 | ]; 46 | 47 | 48 | // ### Create bong test cases. 49 | 50 | describe( 'string.edgeGrams()', function () { 51 | var tests = [ 52 | { whenInputIs: [ '' ], expectedOutputIs: [] }, 53 | { whenInputIs: [ 'decisively' ], expectedOutputIs: [ 'de', 'deci', 'decisi', 'decisive' ] }, 54 | { whenInputIs: [ 'decisively', 8, 10, 1 ], expectedOutputIs: [ 'decisive', 'decisivel', 'decisively' ] } 55 | ]; 56 | 57 | tests.forEach( function ( test ) { 58 | it( 'should return ' + JSON.stringify( test.expectedOutputIs ) + ' if the input is ' + JSON.stringify( test.whenInputIs ), function () { 59 | expect( edgeGrams( ...test.whenInputIs ) ).to.deep.equal( test.expectedOutputIs ); 60 | } ); 61 | } ); 62 | 63 | it( 'indexer result should return an index of 2-grams of rachna & archna', function () { 64 | var bongIndex = index(); 65 | edgeGrams( 'decision', 4, 8, 2, bongIndex.build, 'decision' ); 66 | edgeGrams( 'decisive', 4, 8, 2, bongIndex.build, 'decisive' ); 67 | var result = bongIndex.result(); 68 | expect( result ).to.deep.equal( { deci: [ 'decision', 'decisive' ], decisi: [ 'decision', 'decisive' ], decision: [ 'decision' ], decisive: [ 'decisive' ] } ); 69 | } ); 70 | 71 | errors.slice( 0, 2 ).forEach( function ( error ) { 72 | it( 'should throw ' + error.expectedOutputIs + ' if the input is ' + JSON.stringify( error.whenInputIs ), function () { 73 | expect( edgeGrams.bind( null, error.whenInputIs ) ).to.throw( error.expectedOutputIs ); 74 | } ); 75 | } ); 76 | } ); 77 | -------------------------------------------------------------------------------- /src/string-tokenize.js: -------------------------------------------------------------------------------- 1 | // wink-nlp-utils 2 | // NLP Functions for amplifying negations, managing elisions, 3 | // creating ngrams, stems, phonetic codes to tokens and more. 4 | // 5 | // Copyright (C) GRAYPE Systems Private Limited 6 | // 7 | // This file is part of “wink-nlp-utils”. 8 | // 9 | // Permission is hereby granted, free of charge, to any person obtaining a 10 | // copy of this software and associated documentation files (the "Software"), 11 | // to deal in the Software without restriction, including without limitation 12 | // the rights to use, copy, modify, merge, publish, distribute, sublicense, 13 | // and/or sell copies of the Software, and to permit persons to whom the 14 | // Software is furnished to do so, subject to the following conditions: 15 | // 16 | // The above copyright notice and this permission notice shall be included 17 | // in all copies or substantial portions of the Software. 18 | // 19 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20 | // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 22 | // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 24 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | // DEALINGS IN THE SOFTWARE. 26 | 27 | // 28 | // Load wink-nlp package & helpers. 29 | const winkNLP = require( 'wink-nlp' ); 30 | // Load english language model — light version. 31 | const model = require( 'wink-eng-lite-web-model' ); 32 | // Instantiate winkNLP, only use tokenization. 33 | const nlp = winkNLP( model, [] ); 34 | const its = nlp.its; 35 | 36 | // ## string 37 | 38 | // ### tokenize 39 | /** 40 | * 41 | * Tokenizes the input `sentence` according to the value of `detailed` flag. 42 | * Any occurance of `...` in the `sentence` is 43 | * converted to ellipses. In `detailed = true` mode, it 44 | * tags every token with its type; the supported tags are word, number, url, email, 45 | * mention, hashtag, emoji, emoticon, time, ordinal, currency, punctuation, symbol, 46 | * and tabCFLF. 47 | * 48 | * @alias string#tokenize 49 | * @param {string} sentence the input string. 50 | * @param {boolean} [detailed=false] if true, each token is a object cotaining 51 | * `value` and `tag` of each token; otherwise each token is a string. It's default 52 | * value of **false** ensures compatibility with previous version. 53 | * @return {(string[]|object[])} an array of strings if `detailed` is false otherwise 54 | * an array of objects. 55 | * @example 56 | * tokenize( "someone's wallet, isn't it? I'll return!" ); 57 | * // -> [ 'someone', '\'s', 'wallet', ',', 'is', 'n\'t', 'it', '?', 58 | * // 'I', '\'ll', 'return', '!' ] 59 | * 60 | * tokenize( 'For details on wink, check out http://winkjs.org/ URL!', true ); 61 | * // -> [ { value: 'For', tag: 'word' }, 62 | * // { value: 'details', tag: 'word' }, 63 | * // { value: 'on', tag: 'word' }, 64 | * // { value: 'wink', tag: 'word' }, 65 | * // { value: ',', tag: 'punctuation' }, 66 | * // { value: 'check', tag: 'word' }, 67 | * // { value: 'out', tag: 'word' }, 68 | * // { value: 'http://winkjs.org/', tag: 'url' }, 69 | * // { value: 'URL', tag: 'word' }, 70 | * // { value: '!', tag: 'punctuation' } ] 71 | */ 72 | var tokenize = function ( sentence, detailed ) { 73 | const doc = nlp.readDoc( sentence.replace( '...', '…' ) ); 74 | const tokens = []; 75 | 76 | if ( detailed ) { 77 | doc.tokens().each( ( t ) => { 78 | tokens.push( { value: t.out(), tag: t.out( its.type ) } ); 79 | } ); 80 | 81 | return tokens; 82 | } 83 | 84 | return doc.tokens().out(); 85 | }; // tokenize() 86 | 87 | module.exports = tokenize; 88 | -------------------------------------------------------------------------------- /src/helper-return-quoted-text-extractor.js: -------------------------------------------------------------------------------- 1 | // wink-nlp-utils 2 | // NLP Functions for amplifying negations, managing elisions, 3 | // creating ngrams, stems, phonetic codes to tokens and more. 4 | // 5 | // Copyright (C) GRAYPE Systems Private Limited 6 | // 7 | // This file is part of “wink-nlp-utils”. 8 | // 9 | // Permission is hereby granted, free of charge, to any person obtaining a 10 | // copy of this software and associated documentation files (the "Software"), 11 | // to deal in the Software without restriction, including without limitation 12 | // the rights to use, copy, modify, merge, publish, distribute, sublicense, 13 | // and/or sell copies of the Software, and to permit persons to whom the 14 | // Software is furnished to do so, subject to the following conditions: 15 | // 16 | // The above copyright notice and this permission notice shall be included 17 | // in all copies or substantial portions of the Software. 18 | // 19 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20 | // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 22 | // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 24 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | // DEALINGS IN THE SOFTWARE. 26 | 27 | // 28 | 29 | // ## string 30 | 31 | // ### returnQuotedTextExtractor 32 | 33 | /** 34 | * 35 | * Returns a function that extracts all occurrences of every quoted text 36 | * between the `lq` and the `rq` characters from its argument. This argument 37 | * must be of type string. 38 | * 39 | * @alias helper#returnQuotedTextExtractor 40 | * @param {string} [lq='"'] the left quote character. 41 | * @param {string} [rq='"'] the right quote character. 42 | * @return {function} that will accept an input string argument and return an 43 | * array of all substrings that are quoted between `lq` and `rq`. 44 | * @example 45 | * var extractQuotedText = returnQuotedTextExtractor(); 46 | * extractQuotedText( 'Raise 2 issues - "fix a bug" & "run tests"' ); 47 | * // -> [ 'fix a bug', 'run tests' ] 48 | */ 49 | var returnQuotedTextExtractor = function ( lq, rq ) { 50 | var // Index variable for *for-loop* 51 | i, 52 | // Set defaults for left quote, if required. 53 | lq1 = ( ( lq && ( typeof lq === 'string' ) ) ? lq : '"' ), 54 | // Extracts its length 55 | lqLen = lq1.length, 56 | // The regular expression is created here. 57 | regex = null, 58 | // The string containing the regular expression builds here. 59 | rgxStr = '', 60 | // Set defaults for right quote, if required. 61 | rq1 = ( ( rq && ( typeof rq === 'string' ) ) ? rq : lq1 ), 62 | // Extract its length. 63 | rqLen = rq1.length; 64 | 65 | // Build `rgxStr` 66 | for ( i = 0; i < lqLen; i += 1 ) rgxStr += '\\' + lq1.charAt( i ); 67 | rgxStr += '.*?'; 68 | for ( i = 0; i < rqLen; i += 1 ) rgxStr += '\\' + rq1.charAt( i ); 69 | // Create regular expression. 70 | regex = new RegExp( rgxStr, 'g' ); 71 | // Return the extractor function. 72 | return ( function ( s ) { 73 | if ( !s || ( typeof s !== 'string' ) ) return null; 74 | var // Extracted elements are captured here. 75 | elements = [], 76 | // Extract matches with quotes 77 | matches = s.match( regex ); 78 | if ( !matches || ( matches.length === 0 ) ) return null; 79 | // Collect elements after removing the quotes. 80 | for ( var k = 0, kmax = matches.length; k < kmax; k += 1 ) { 81 | elements.push( matches[ k ].substr( lqLen, matches[ k ].length - ( rqLen + lqLen ) ) ); 82 | } 83 | return ( elements ); 84 | } ); 85 | }; // returnQuotedTextExtractor() 86 | 87 | module.exports = returnQuotedTextExtractor; 88 | -------------------------------------------------------------------------------- /src/string-phonetize.js: -------------------------------------------------------------------------------- 1 | // wink-nlp-utils 2 | // NLP Functions for amplifying negations, managing elisions, 3 | // creating ngrams, stems, phonetic codes to tokens and more. 4 | // 5 | // Copyright (C) GRAYPE Systems Private Limited 6 | // 7 | // This file is part of “wink-nlp-utils”. 8 | // 9 | // Permission is hereby granted, free of charge, to any person obtaining a 10 | // copy of this software and associated documentation files (the "Software"), 11 | // to deal in the Software without restriction, including without limitation 12 | // the rights to use, copy, modify, merge, publish, distribute, sublicense, 13 | // and/or sell copies of the Software, and to permit persons to whom the 14 | // Software is furnished to do so, subject to the following conditions: 15 | // 16 | // The above copyright notice and this permission notice shall be included 17 | // in all copies or substantial portions of the Software. 18 | // 19 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20 | // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 22 | // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 24 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | // DEALINGS IN THE SOFTWARE. 26 | 27 | // 28 | var phnrgx = require( './phonetize_regexes.js' ); 29 | /* eslint no-underscore-dangle: "off" */ 30 | 31 | // ## string 32 | 33 | // ### phonetize 34 | /** 35 | * 36 | * Phonetizes the input string using an algorithmic adaptation of Metaphone; It 37 | * is not an exact implementation of Metaphone. 38 | * 39 | * @alias string#phonetize 40 | * @param {string} word the input word. 41 | * @return {string} phonetic code of `word`. 42 | * @example 43 | * phonetize( 'perspective' ); 44 | * // -> 'prspktv' 45 | * phonetize( 'phenomenon' ); 46 | * // -> 'fnmnn' 47 | */ 48 | var phonetize = function ( word ) { 49 | var p = word.toLowerCase(); 50 | // Remove repeating letters. 51 | p = p.replace( phnrgx.repeatingChars, '$1'); 52 | // Drop first character of `kgknPairs`. 53 | if ( phnrgx.kngnPairs.test( p ) ) { 54 | p = p.substr( 1, p.length - 1 ); 55 | } 56 | // Run Regex Express now! 57 | p = p 58 | // Change `ough` in the end as `f`, 59 | .replace( phnrgx.ough, 'f' ) 60 | // Change `dg` to `j`, in `dge, dgi, dgy`. 61 | .replace( phnrgx.dge, 'je' ) 62 | .replace( phnrgx.dgi, 'ji' ) 63 | .replace( phnrgx.dgy, 'jy' ) 64 | // Change `c` to `k` in `sch` 65 | .replace( phnrgx.sch, 'sk' ) 66 | // Drop `c` in `sci, sce, scy`. 67 | .replace( phnrgx.sci, 'si' ) 68 | .replace( phnrgx.sce, 'se' ) 69 | .replace( phnrgx.scy, 'sy' ) 70 | // Drop `t` if it appears as `tch`. 71 | .replace( phnrgx.tch, 'ch' ) 72 | // Replace `tio & tia` by `sh`. 73 | .replace( phnrgx.tio, 'sh' ) 74 | .replace( phnrgx.tia, 'sh' ) 75 | // Drop `b` if it appears as `mb` in the end. 76 | .replace( phnrgx.mb_, 'm' ) 77 | // Drop `r` if it preceeds a vowel and not followed by a vowel or `y` 78 | // .replace( rgx.vrnotvy, '$1$3' ) 79 | // Replace `c` by `s` in `ce, ci, cy`. 80 | .replace( phnrgx.ce, 'se' ) 81 | .replace( phnrgx.ci, 'si' ) 82 | .replace( phnrgx.cy, 'sy' ) 83 | // Replace `cq` by `q`. 84 | .replace( phnrgx.cq, 'q' ) 85 | // Replace `ck` by `k`. 86 | .replace( phnrgx.ck, 'k' ) 87 | // Replace `ph` by `f`. 88 | .replace( phnrgx.ph, 'f' ) 89 | // Replace `th` by `0` (theta look alike!). 90 | .replace( phnrgx.th, '0' ) 91 | // Replace `c` by `k` if it is not followed by `h`. 92 | .replace( phnrgx.cnoth, 'k$2' ) 93 | // Replace `q` by `k`. 94 | .replace( phnrgx.q, 'k' ) 95 | // Replace `x` by `s` if it appears in the beginning. 96 | .replace( phnrgx._x, 's' ) 97 | // Other wise replace `x` by `ks`. 98 | .replace( phnrgx.x, 'ks' ) 99 | // Replace `sh, sia, sio` by `x`. Needs to be done post `x` processing! 100 | .replace( phnrgx.sh, 'x' ) 101 | // Drop `y` if it is now followed by a **vowel**. 102 | .replace( phnrgx.ynotv, '$2' ) 103 | .replace( phnrgx.y_, '' ) 104 | // Replace `z` by `s`. 105 | .replace( phnrgx.z, 's' ) 106 | // Drop all **vowels** excluding the first one. 107 | .replace( phnrgx.__vowels, '' ); 108 | 109 | return ( p ); 110 | }; // phonetize() 111 | 112 | module.exports = phonetize; 113 | -------------------------------------------------------------------------------- /src/util_regexes.js: -------------------------------------------------------------------------------- 1 | // wink-nlp-utils 2 | // NLP Functions for amplifying negations, managing elisions, 3 | // creating ngrams, stems, phonetic codes to tokens and more. 4 | // 5 | // Copyright (C) GRAYPE Systems Private Limited 6 | // 7 | // This file is part of “wink-nlp-utils”. 8 | // 9 | // Permission is hereby granted, free of charge, to any person obtaining a 10 | // copy of this software and associated documentation files (the "Software"), 11 | // to deal in the Software without restriction, including without limitation 12 | // the rights to use, copy, modify, merge, publish, distribute, sublicense, 13 | // and/or sell copies of the Software, and to permit persons to whom the 14 | // Software is furnished to do so, subject to the following conditions: 15 | // 16 | // The above copyright notice and this permission notice shall be included 17 | // in all copies or substantial portions of the Software. 18 | // 19 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20 | // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 22 | // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 24 | // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | // DEALINGS IN THE SOFTWARE. 26 | 27 | // 28 | var rgx = Object.create( null ); 29 | 30 | // Matches standard english punctuations in a text. 31 | rgx.punctuations = /[\’\'\‘\’\`\“\”\"\[\]\(\)\{\}\…\,\.\!\;\?\/\-\:]/ig; 32 | // End Of Sentence Punctuations - useful for splitting text into sentences. 33 | rgx.eosPunctuations = /([\.\?\!])\s*(?=[a-z]|\s+\d)/gi; 34 | 35 | // Matches special characters: `* + % # @ ^ = ~ | \` in a text. 36 | rgx.splChars = /[\*\+\%\#\@\^\=\~\|\\]/ig; 37 | 38 | // Matches common english elisions including n't. 39 | // These are special ones as 's otherwise may be apostrophe! 40 | rgx.elisionsSpl = /(\b)(it|let|that|who|what|here|there|when|where|why|how)(\'s)\b/gi; 41 | // Single (1) character elisions. 42 | rgx.elisions1 = /([a-z])(\'d|\'m)\b/gi; 43 | // Two (2) character elisions. 44 | rgx.elisions2 = /([a-z])(\'ll|\'ve|\'re|n\'t)\b/gi; 45 | // Sperate not elision 'nt. 46 | rgx.notElision = /([a-z])(n\'t)\b/gi; 47 | // Specially handle cannot 48 | rgx.cannot = /\b(can)(not)\b/gi; 49 | 50 | // Matches space, tab, or new line characters in text. 51 | rgx.spaces = /\s+/ig; 52 | // Matches anything other than space, tab, or new line characters. 53 | rgx.notSpace = /\S/g; 54 | // Matches alpha and space characters in a text. 55 | rgx.alphaSpace = /[a-z\s]/ig; 56 | // Matches alphanumerals and space characters in a text. 57 | rgx.alphaNumericSpace = /[a-z0-9\s]/ig; 58 | // Matches non alpha characters in a text. 59 | rgx.notAlpha = /[^a-z]/ig; 60 | // Matches non alphanumerals in a text. 61 | rgx.notAlphaNumeric = /[^a-z0-9]/ig; 62 | // Matches one or more non-words characters. 63 | rgx.nonWords = /\W+/ig; 64 | // Matches complete negation token 65 | rgx.negations = /^(never|none|not|no)$/i; 66 | 67 | // Matches run of capital words in a text. 68 | rgx.rocWords = /(?:\b[A-Z][A-Za-z]*\s*){2,}/g; 69 | 70 | // Matches integer, decimal, JS floating point numbers in a text. 71 | rgx.number = /[0-9]*\.[0-9]+e[\+\-]{1}[0-9]+|[0-9]*\.[0-9]+|[0-9]+/ig; 72 | 73 | // Matches time in 12 hour am/pm format in a text. 74 | rgx.timeIn12HrAMPM = /(?:[0-9]|0[0-9]|1[0-2])((:?:[0-5][0-9])){0,1}\s?(?:[aApP][mM])/ig; 75 | 76 | // Matches HTML tags - in fact any thing enclosed in angular brackets including 77 | // the brackets. 78 | rgx.htmlTags = /(?:<[^>]*>)/g; 79 | // Matches the HTML Esc Sequences 80 | // Esc Seq of type `<` or ` ` 81 | rgx.htmlEscSeq1 = /(?:&[a-z]{2,6};)/gi; 82 | // Esc Seq of type ` ` 83 | rgx.htmlEscSeq2 = /(?:&#[0-9]{2,4};)/gi; 84 | 85 | // Tests if a given string is possibly in the Indian mobile telephone number format. 86 | rgx.mobileIndian = /^(0|\+91)?[789]\d{9}$/; 87 | // Tests if a given string is in the valid email format. 88 | rgx.email = /^(([^<>()[\]\\.,;:\s@"]+(\.[^<>()[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/; 89 | 90 | // Extracts any number and text from a
](http://wink.org.in/)
9 |
10 | Prepare raw text for Natural Language Processing (NLP) using **`wink-nlp-utils`**. It offers a set of [APIs](http://wink.org.in/wink-nlp-utils/) to work on [strings](http://wink.org.in/wink-nlp-utils/#string) such as names, sentences, paragraphs and [tokens](http://wink.org.in/wink-nlp-utils/#tokens) represented as an array of strings/words. They perform the required pre-processing for many ML tasks such as [semantic search](https://www.npmjs.com/package/wink-bm25-text-search), and [classification](https://www.npmjs.com/package/wink-naive-bayes-text-classifier).
11 |
12 | 👉🏽 |
13 | We recommend using winkNLP for core natural language processing tasks. It performs Tokenization, Sentence Boundary Detection, and Named Entity Recognition at a blazing fast speeds. It supports all your text processing needs starting from Sentiment Analysis, POS Tagging, Lemmatization, Stemming, Stop Word Removal, Negation Handling, Bigrams to Frequency Table Creation and more. WinkNLP features user-friendly declarative APIs for Iteration, Filtering, and Text Visualization, and runs on web browsers. 14 | |