29 | // import { ml, } from './ml';
30 | import { nlp, } from './nlp';
31 | // import { util as utils, } from './util';
32 | import { DataSet, } from './DataSet';
33 |
34 | /**
35 | * class creating sparse matrices from a corpus
36 | * @class ColumnVectorizer
37 | * @memberOf nlp
38 | */
39 | export class ColumnVectorizer {
40 | /**
41 | * creates a new instance for classifying text data for machine learning
42 | * @example
43 | * const dataset = new ms.nlp.ColumnVectorizer(csvData);
44 | * @param {Object} [options={}]
45 | * @prop {Object[]} this.data - Array of strings
46 | * @prop {Set} this.tokens - Unique collection of all tokenized strings
47 | * @prop {Object[]} this.vectors - Array of tokenized words with value of count of appreance in string
48 | * @prop {Object} this.wordMap - Object of all unique words, with value of 0
49 | * @prop {Object} this.wordCountMap - Object of all unique words, with value as total count of appearances
50 | * @prop {number} this.maxFeatures - max number of features
51 | * @prop {String[]} this.sortedWordCount - list of words as tokens sorted by total appearances
52 | * @prop {String[]} this.limitedFeatures - subset list of maxFeatures words as tokens sorted by total appearances
53 | * @prop {Array[]} this.matrix - words in sparse matrix
54 | * @prop {Function} this.replacer - clean string function
55 | * @returns {this}
56 | */
57 | constructor(options = {}) {
58 | this.data = options.data || [];
59 | this.tokens = new Set();
60 | this.vectors = [];
61 | this.wordMap = {};
62 | this.wordCountMap = {};
63 | this.maxFeatures = options.maxFeatures;
64 | this.sortedWordCount = [];
65 | this.limitedFeatures = [];
66 | this.matrix = [];
67 | this.replacer = (value='') => {
68 | const cleanedValue = value
69 | .toLowerCase()
70 | .replace(/[^a-zA-Z]/gi, ' ');
71 | return nlp.PorterStemmer
72 | .tokenizeAndStem(cleanedValue)
73 | .join(' ');
74 | };
75 | return this;
76 | }
77 | /**
78 | * Returns a distinct array of all tokens
79 | * @return {String[]} returns a distinct array of all tokens
80 | */
81 | get_tokens() {
82 | return Array.from(this.tokens);
83 | }
84 | /**
85 | * Returns array of arrays of strings for dependent features from sparse matrix word map
86 | * @return {String[]} returns array of dependent features for DataSet column matrics
87 | */
88 | get_vector_array() {
89 | return this.get_tokens().map(tok => [
90 | tok,
91 | ]);
92 | }
93 | /**
94 | * Fits and transforms data by creating column vectors (a sparse matrix where each row has every word in the corpus as a column and the count of appearances in the corpus)
95 | * @param {Object} options
96 | * @param {Object[]} options.data - array of corpus data
97 | */
98 | fit_transform(options = {}) {
99 | const data = options.data || this.data;
100 | data.forEach(datum => {
101 | const datums = {};
102 | this.replacer(datum)
103 | .split(' ')
104 | .forEach(tok => {
105 | const token = tok.toLowerCase();
106 | datums[ token ] = (datums[ token ])
107 | ? datums[ token ] + 1
108 | : 1;
109 | this.wordCountMap[token] = (this.wordCountMap[token])
110 | ? this.wordCountMap[token] + 1
111 | : 1;
112 | this.tokens.add(token);
113 | });
114 | this.vectors.push(datums);
115 | });
116 | this.wordMap = Array.from(this.tokens).reduce((result, value) => {
117 | result[ value ] = 0;
118 | return result;
119 | }, {});
120 | this.sortedWordCount = Object.keys(this.wordCountMap)
121 | .sort((a, b) => this.wordCountMap[ b ] - this.wordCountMap[ a ]);
122 | this.vectors = this.vectors.map(vector => Object.assign({}, this.wordMap, vector));
123 | const vectorData = new DataSet(this.vectors);
124 | this.limitedFeatures = this.get_limited_features(options);
125 | this.matrix = vectorData.columnMatrix(this.limitedFeatures);
126 | return this.matrix;
127 | }
128 | /**
129 | * Returns limited sets of dependent features or all dependent features sorted by word count
130 | * @param {*} options
131 | * @param {number} options.maxFeatures - max number of features
132 | */
133 | get_limited_features(options = {}) {
134 | const maxFeatures = options.maxFeatures || this.maxFeatures || this.tokens.size;
135 |
136 | return this.sortedWordCount
137 | .slice(0, maxFeatures)
138 | .map(feature => [ feature, ]);
139 | }
140 | /**
141 | * returns word map with counts
142 | * @example
143 | ColumnVectorizer.evaluateString('I would rate everything Great, views Great, food Great') => { realli: 0,
144 | good: 0,
145 | definit: 0,
146 | recommend: 0,
147 | wait: 0,
148 | staff: 0,
149 | rude: 0,
150 | great: 3,
151 | view: 1,
152 | food: 1,
153 | not: 0,
154 | cold: 0,
155 | took: 0,
156 | forev: 0,
157 | seat: 0,
158 | time: 0,
159 | prompt: 0,
160 | attent: 0,
161 | bland: 0,
162 | flavor: 0,
163 | kind: 0 }
164 | * @param {String} testString
165 | * @return {Object} object of corpus words with accounts
166 | */
167 | evaluateString(testString = '') {
168 | const evalString = this.replacer(testString);
169 | const evalStringWordMap = evalString.split(' ').reduce((result, value) => {
170 | if (this.tokens.has(value)) {
171 | result[ value ] = (result[ value ]!==undefined)
172 | ? result[ value ] + 1
173 | : 1;
174 | }
175 | return result;
176 | }, {});
177 | return Object.assign({}, this.wordMap, evalStringWordMap);
178 | }
179 | /**
180 | * returns new matrix of words with counts in columns
181 | * @example
182 | ColumnVectorizer.evaluate('I would rate everything Great, views Great, food Great') => [ [ 0, 1, 3, 0, 0, 0, 0, 0, 1 ] ]
183 | * @param {String} testString
184 | * @return {number[][]} sparse matrix row for new classification predictions
185 | */
186 | evaluate(testString='', options) {
187 | const stringObj = this.evaluateString(testString);
188 | const limitedFeatures = this.get_limited_features(options);
189 | const vectorData = new DataSet([
190 | stringObj,
191 | ]);
192 | return vectorData.columnMatrix(limitedFeatures);
193 | }
194 | }
195 |
196 | 
29 |
30 | 