├── .gitignore ├── data └── conceptnet_2014-04-27.sql.bz2 ├── example └── index.js ├── index.js ├── package.json ├── readme.md └── test └── basic.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/* 2 | .DS_Store -------------------------------------------------------------------------------- /data/conceptnet_2014-04-27.sql.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/silentrob/conceptnet/bce5124e767ab3ea852c12016508b6f56bbb3cad/data/conceptnet_2014-04-27.sql.bz2 -------------------------------------------------------------------------------- /example/index.js: -------------------------------------------------------------------------------- 1 | var concept = require("../index")(); 2 | var _ = require("underscore"); 3 | 4 | // concept.constructSurface("hammer", "build", function(err, res){ 5 | // // We are going to cheat and just return the sentense. 6 | // // var x = getRandomInt(0,res.length - 1) 7 | // console.log(err, res); 8 | // }); 9 | 10 | // concept.hasPrerequisiteForward("smoke", function(err, res){ 11 | // console.log("Pre", res); 12 | // }); 13 | 14 | // concept.causesForward("fight", function(err, res){ 15 | // console.log("Cause", res); 16 | // }); 17 | 18 | // Finds the intersection of two concepts 19 | // concept.relatedConcepts("tea", "coffee", function(err, res){ 20 | // console.log("Related", res); 21 | // }); 22 | 23 | // concept.relatedConceptsArray(["tea", "coffee"], function(err, res){ 24 | // console.log("Related", res); 25 | // }); 26 | 27 | // return an array of items 28 | concept.filterConcepts(["tea", "coffee", "sand", "rocks"],"drink", function(err, res){ 29 | console.log("Related", res); 30 | }); 31 | 32 | // // What is the color of the ocean 33 | // concept.isAReverse("plant flower", function(err, res){ 34 | // console.log("animal + milk", res); 35 | // }); 36 | 37 | // concept.assersionTest("fly south", "bird", function(err, res){ 38 | // console.log(res); 39 | // }); 40 | 41 | // concept.isAForward("sunset", function(err, res){ 42 | // console.log(res); 43 | // }); 44 | 45 | // concept.atLocationForward("pacific ocean", function(err, res){ 46 | // console.log("season", res); 47 | // }); 48 | 49 | // concept.conceptLookup("When do birds fly south?", function(err, concepts){ 50 | // console.log("CL: When do birds fly south?") 51 | // console.log(concepts); 52 | // }) 53 | 54 | // concept.isAReverse("cube", function(err, res){ 55 | // console.log("cube", res); 56 | // }); 57 | 58 | // concept.hasPropertyForward("toolbox", function(err, res){ 59 | // console.log("hasPropertyForward"); 60 | // console.log(res); 61 | // }) 62 | 63 | // concept.atLocationForward("hammer", function(err, res){ 64 | // console.log("atLocationForward"); 65 | // console.log(res); 66 | // }) 67 | 68 | // concept.isAForward("toolbox", function(err, res){ 69 | // console.log("isAForward, toolbox"); 70 | // console.log(res); 71 | // }) 72 | 73 | // concept.conceptLookup("What is the capital of spain?", function(err, concepts){ 74 | // console.log("CL: What is the capital of spain?") 75 | // console.log(concepts); 76 | // }) 77 | 78 | // concept.usedForForward("taxi", function(err, result){ 79 | // console.log("UF Taxi", result[0].sentense) 80 | // }) 81 | 82 | // concept.putConcept("car", function(err, res){ 83 | // console.log("Where would I put a car?"); 84 | // console.log(err, res) 85 | // }) 86 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | // This is the new and improved mysql Version, smaller and MUCH faster 2 | var async = require("async"); 3 | var Knex = require("knex"); 4 | var _ = require("underscore"); 5 | var debug = require("debug")("ConceptNet"); 6 | 7 | var stopwords = ["for", "like", "use", "an", "if", "of", "to", "the", "is", "a", "i", "are", "and", "who", "what", "where", "when","how", "would", "which", "or", "do", "my", "bob"]; 8 | 9 | var basicFind = function() { 10 | return knex('assertion') 11 | .join('concept as c1', 'c1.id', '=' ,'assertion.concept1_id', 'left') 12 | .join('concept as c2', 'c2.id', '=' ,'assertion.concept2_id', 'left') 13 | .join('surfaceform as f1', 'f1.id', '=' ,'assertion.best_surface1_id', 'left') 14 | .join('surfaceform as f2', 'f2.id', '=' ,'assertion.best_surface2_id', 'left') 15 | .join('frame as frame', 'frame.id', '=' ,'assertion.best_frame_id', 'left') 16 | .join('rawassertion as raw', 'raw.id', '=' ,'assertion.best_raw_id', 'left') 17 | .join('sentence', 'sentence.id', '=' ,'raw.sentence_id', 'left') 18 | .select('c1.text as c1_text', 'c2.text as c2_text', 'c2.num_assertions', 'f1.text as frame1', 'f2.text as frame2', 'sentence.text as sentense', 'frame.text as frame_text' ) 19 | .where('assertion.score', '>', 0) 20 | .whereNotNull('f1.text') 21 | .whereNotNull('f2.text') 22 | .orderBy('c2.num_assertions', 'desc') 23 | .limit(90).clone(); 24 | }; 25 | 26 | 27 | 28 | 29 | var capableOf = function() { 30 | return basicFind().andWhere('assertion.relation_id', 8).clone(); 31 | } 32 | var hasPrerequisite = function() { 33 | return basicFind().andWhere('assertion.relation_id', 3).clone(); 34 | } 35 | var causes = function() { 36 | return basicFind().andWhere('assertion.relation_id', 18).clone(); 37 | } 38 | var atLocation = function() { 39 | return basicFind().andWhere('assertion.relation_id', 6).clone(); 40 | } 41 | var hasProperty = function() { 42 | return basicFind().andWhere('assertion.relation_id', 20).clone(); 43 | } 44 | var usedFor = function() { 45 | return basicFind().andWhere('assertion.relation_id', 7).clone(); 46 | } 47 | var isA = function() { 48 | return basicFind().andWhere('assertion.relation_id', 5).clone(); 49 | } 50 | 51 | // Generic Fwd / Back 52 | var basicForward = function(term) { 53 | return basicFind().andWhere('c1.text', term).clone(); 54 | } 55 | var basicReverse = function(term) { 56 | return basicFind().andWhere('c2.text', term).clone(); 57 | } 58 | 59 | 60 | // Relationship Fwd / Back 61 | var hasPrerequisiteForward = function(term, callback) { 62 | hasPrerequisite().andWhere('c1.text', term).exec(callback); 63 | } 64 | var hasPrerequisiteReverse = function(term, callback) { 65 | hasPrerequisite().andWhere('c2.text', term).exec(callback); 66 | } 67 | 68 | var causesForward = function(term, callback) { 69 | causes().andWhere('c1.text', term).exec(callback); 70 | } 71 | var causesReverse = function(term, callback) { 72 | causes().andWhere('c1.text', term).exec(callback); 73 | } 74 | 75 | var atLocationForward = function(term, callback) { 76 | atLocation().andWhere('c1.text', term).exec(callback); 77 | } 78 | var atLocationReverse = function(term, callback) { 79 | atLocation().andWhere('c2.text', term).exec(callback); 80 | } 81 | 82 | var hasPropertyForward = function(term, callback) { 83 | hasProperty().andWhere('c1.text', term).exec(callback); 84 | } 85 | 86 | var hasPropertyReverse = function(term, callback) { 87 | hasProperty().andWhere('c2.text', term).exec(callback); 88 | } 89 | 90 | var usedForForward = function(term, callback) { 91 | usedFor().andWhere('c1.text', term).exec(callback); 92 | } 93 | 94 | var usedForReverse = function(term, callback) { 95 | usedFor().andWhere('c2.text', term).exec(callback); 96 | } 97 | 98 | var isAForward = function(term, callback) { 99 | isA().andWhere('c1.text', term).exec(callback); 100 | } 101 | 102 | var isAReverse = function(term, callback) { 103 | isA().andWhere('c2.text', term).exec(callback); 104 | } 105 | 106 | var capableOfForward = function(term, callback) { 107 | capableOf().andWhere('c1.text', term).exec(callback); 108 | } 109 | 110 | var capableOfReverse = function(term, callback) { 111 | capableOf().andWhere('c2.text', term).exec(callback); 112 | } 113 | 114 | 115 | 116 | 117 | var getAssertion = function(term1, term2, callback) { 118 | basicForward(term1).andWhere('c2.text', term2).exec(callback); 119 | } 120 | 121 | var assertionLookupForward = function(term, callback) { 122 | basicForward(term).exec(callback); 123 | } 124 | 125 | // PUT IN or PUT ON 126 | // What do you use to put X in 127 | // food => dish 128 | // nail => hammer 129 | // car => garage 130 | var putConcept = function(term, callback) { 131 | debug("IN Put with", term); 132 | usedForReverse(term, function(err, concepts){ 133 | 134 | var map = {}; 135 | var itor = function(item, cb) { 136 | var concept = item.c1_text; 137 | usedForForward(concept, function(err, concepts2) { 138 | map[concept] = 0; 139 | for (var n = 0; n < concepts2.length; n++) { 140 | if (concepts2[n].c2_text.indexOf(term) !== -1) { 141 | map[concept] += 1; 142 | } 143 | } 144 | cb(null, map); 145 | }); 146 | } 147 | 148 | if (concepts.length != 0) { 149 | async.map(concepts, itor, function(err, result){ 150 | var set = result[0]; 151 | var keysSorted = Object.keys(set).sort(function(a,b){return set[b]-set[a]}); 152 | debug("Sorted Set", keysSorted); 153 | // TODO, if the top items are equal maybe pick one randomly 154 | callback(null, keysSorted[0]); 155 | }); 156 | } else { 157 | callback(null, null); 158 | } 159 | }); 160 | } 161 | 162 | var assersionTest = function(term1, term2, cb ) { 163 | 164 | isAForward(term1, function(err, concepts2) { 165 | var lcount = 0, ecount = 0; 166 | for (var i = 0; i < concepts2.length; i++) { 167 | if (concepts2[i].c2_text == term2) { 168 | ecount++ 169 | } 170 | if (concepts2[i].c2_text.indexOf(term2) !== -1) { 171 | lcount++ 172 | } 173 | } 174 | // console.log(term1, term2, lcount, ecount) 175 | cb(null, (lcount + (ecount * 3 ) / concepts2.length)) 176 | }); 177 | } 178 | 179 | 180 | var resolveFact = function(term1, term2, cb ) { 181 | isAForward(term1, function(err, concepts2) { 182 | 183 | // Remove dups 184 | var uniq = _.uniq(concepts2.map(function(item){return item.c2_text})); 185 | 186 | var map = []; 187 | var itor = function(concept, callback) { 188 | assersionTest(concept, term2, function(err, val){ 189 | if (val > 0.01) { 190 | map.push([concept, val]); 191 | } 192 | callback(null) 193 | }); 194 | } 195 | 196 | async.each(uniq, itor, function() { 197 | var keysSorted = map.sort(function(a,b){return b[1] - a[1]}); 198 | debug("resolveFact", keysSorted); 199 | if (keysSorted.length != 0) 200 | cb(null, keysSorted[0][0]); 201 | else { 202 | cb(null, null); 203 | } 204 | }); 205 | }); 206 | } 207 | 208 | var resolveFacts = function(array, term2, cb) { 209 | debug("ResolveFacts", array, term2); 210 | var itor = function(item, next) { 211 | resolveFact(item, term2, function(err, res){ 212 | if (res) { 213 | next(null, item); 214 | } else { 215 | next(null); 216 | } 217 | }); 218 | } 219 | 220 | async.map(array, itor, function(err, res){ 221 | var arr = _.uniq(res); 222 | arr = arr.filter(function(n){ return n != undefined }); 223 | cb(null, arr); 224 | }); 225 | } 226 | 227 | var conceptLookup = function(msg, callback) { 228 | 229 | var words1 = ngrams(msg, 1); 230 | var words2 = ngrams(msg, 2); 231 | var words3 = ngrams(msg, 3); 232 | 233 | words2 = words2.concat(words1); 234 | words3 = words3.concat(words2); 235 | 236 | words3 = _.map(words3, function(key, item) { return key.join(" "); }); 237 | words3 = _.reject(words3, function(word) { return _.contains(stopwords, word.toLowerCase()) }); 238 | 239 | debug("Searching Concepts for", words3); 240 | 241 | var itor = function(item, cb) { 242 | knex('concept') 243 | .select('text', 'num_assertions', 'visible') 244 | .where('num_assertions', '!=', 0) 245 | .andWhere('text', item ) 246 | .exec(function(err, res){ 247 | cb(err, res); 248 | }); 249 | } 250 | 251 | async.mapSeries(words3, itor, function(err, res){ 252 | var concepts = _.filter(_.flatten(res), Boolean); 253 | var newWords = _.map(_.filter(concepts, Boolean), function(item){ return item.text}); 254 | newWords = _.reject(newWords, function(word) { return _.contains(stopwords, word) }); 255 | callback(null, concepts) 256 | }); 257 | } 258 | 259 | var relatedConceptsArray = function(terms, callback) { 260 | var t = []; 261 | var itor = function(term, done) { 262 | isAForward(term, function(err, result){ 263 | var map = []; 264 | _.each(result, function(item){ 265 | map.push(item.c2_text) 266 | // map.push({text:item.c2_text, num: item.num_assertions}) 267 | }); 268 | done(null, map); 269 | }); 270 | } 271 | 272 | async.map(terms, itor, function(err, res){ 273 | var selectedIntersection = _.intersection.apply(_, res); 274 | callback(null, selectedIntersection); 275 | }); 276 | } 277 | 278 | // How are 2 concepts Related 279 | // Returns an array of objects with num_assersions 280 | var relatedConcepts = function(term1, term2, callback) { 281 | var terms = []; 282 | isAForward(term1, function(err, res1){ 283 | isAForward(term2, function(err, res2){ 284 | var map1 = [], map2 = []; 285 | 286 | _.each(res1, function(item){ 287 | map1.push({text:item.c2_text, num: item.num_assertions}) 288 | }); 289 | 290 | _.each(res2, function(item){ 291 | map2.push({text:item.c2_text, num: item.num_assertions}) 292 | }); 293 | 294 | var results = _.uniq(_.intersect(map1, map2)); 295 | callback(null, results) 296 | }); 297 | }); 298 | } 299 | 300 | // Return an array of items with the filterSet removed 301 | var filterConcepts = function(items, filter, callback) { 302 | 303 | var itor = function(item, done) { 304 | assersionTest(item, filter, done); 305 | } 306 | 307 | async.map(items, itor, function(err, data) { 308 | var newList = _.filter(items, function(k,v) { return (data[v] !== 0 && !isNaN(data[v])) }); 309 | callback(null, newList); 310 | }); 311 | } 312 | 313 | var constructSurface = function(term1, term2, callback) { 314 | getAssertion(term1, term2, function(err, fullconcept){ 315 | var x = getRandomInt(0, fullconcept.length - 1); 316 | callback(null, fullconcept[x].sentense); 317 | }); 318 | }; 319 | 320 | module.exports = function(options) { 321 | options = options || {} 322 | knex = Knex.initialize({ 323 | client: 'mysql', 324 | connection: { 325 | host : options.host || "localhost", 326 | user : options.user || "root", 327 | password : options.pass || "", 328 | database : options.database || "conceptnet" 329 | } 330 | }); 331 | 332 | return { 333 | 334 | basicReverse: basicReverse, 335 | basicForward: basicForward, 336 | 337 | // Forward, Reverse Relations 338 | hasPrerequisiteForward: hasPrerequisiteForward, 339 | hasPrerequisiteReverse: hasPrerequisiteReverse, 340 | 341 | causesForward: causesForward, 342 | causesReverse: causesReverse, 343 | 344 | atLocationForward: atLocationForward, 345 | atLocationReverse: atLocationReverse, 346 | 347 | hasPropertyForward: hasPropertyForward, 348 | hasPropertyReverse: hasPropertyReverse, 349 | 350 | isAForward: isAForward, 351 | isAReverse: isAReverse, 352 | 353 | usedForForward: usedForForward, 354 | usedForReverse: usedForReverse, 355 | 356 | capableOfForward: capableOfForward, 357 | capableOfReverse: capableOfReverse, 358 | 359 | // More complicated things :) 360 | putConcept: putConcept, 361 | 362 | constructSurface : constructSurface, 363 | relatedConcepts: relatedConcepts, 364 | relatedConceptsArray: relatedConceptsArray, 365 | 366 | conceptLookup: conceptLookup, 367 | resolveFact: resolveFact, 368 | resolveFacts: resolveFacts, 369 | 370 | filterConcepts: filterConcepts, 371 | 372 | assersionTest: assersionTest, 373 | assertionLookupForward: assertionLookupForward 374 | } 375 | } 376 | 377 | // Helper, intersect Objects 378 | _.intersect = function(array) { 379 | var slice = Array.prototype.slice; // added this line as a utility 380 | var rest = slice.call(arguments, 1); 381 | return _.filter(_.uniq(array), function(item) { 382 | return _.every(rest, function(other) { 383 | return _.any(other, function(element) { return _.isEqual(element, item); }); 384 | }); 385 | }); 386 | }; 387 | 388 | var getRandomInt = function(min, max) { 389 | return Math.floor(Math.random() * (max - min + 1)) + min; 390 | }; 391 | 392 | var ngrams = function(sequence, n) { 393 | var result = [], words = [], count; 394 | words = sequence.split(/\W+/); 395 | words = _.without(words, '', ' ') 396 | count = _.max([0, words.length - n + 1]); 397 | for (var i = 0; i < count; i++) { 398 | result.push(words.slice(i, i + n)); 399 | } 400 | return result; 401 | }; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "conceptnet", 3 | "version": "0.0.1", 4 | "description": "A Bridge to Conceptnet 4 DB", 5 | "main": "index.js", 6 | "directories": { 7 | "example": "example", 8 | "test": "test" 9 | }, 10 | "scripts": { 11 | "test": "mocha ./test/ -R spec" 12 | }, 13 | "keywords": [ 14 | "conceptnet" 15 | ], 16 | "author": "Rob Ellis", 17 | "license": "MIT", 18 | "dependencies": { 19 | "mysql": "~2.2.0", 20 | "debug": "~0.8.1", 21 | "knex": "~0.5.13", 22 | "async": "~0.7.0", 23 | "underscore": "~1.6.0" 24 | }, 25 | "devDependencies": { 26 | "mocha": "~1.18.2", 27 | "should": "~3.3.1" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Conceptnet 2 | 3 | This projects helps setup Concept 4 Database with MySQL and provides some sugar for getting some data out. 4 | Concept Net provides over 610,000 facts of "common sense knowledge" More information about ConceptNet 4 can be found here: http://csc.media.mit.edu/docs/conceptnet/ 5 | 6 | ConceptNet 5 information can be found here: http://conceptnet5.media.mit.edu/ 7 | 8 | It should be noted, this is not the entire DB only the parts I needed, I have included these tables: 9 | 10 | assertion 11 | concept 12 | frame 13 | rawassertion 14 | relation 15 | sentence 16 | surfaceform 17 | 18 | ConceptNet is a Web providing nodes and relations between the archs, these are the relations: 19 | 20 | IsA | What kind of thing is it? | 94606 21 | HasA | What does it possess? | 22782 22 | PartOf | What is it part of? | 4648 23 | UsedFor | What do you use it for? | 50392 24 | AtLocation | Where would you find it? | 45138 25 | CapableOf | What can it do? | 39237 26 | MadeOf | What is it made of? | 1509 27 | CreatedBy | How do you bring it into existence? | 544 28 | HasSubevent | What do you do to accomplish it? | 25433 29 | HasFirstSubevent | What do you do first to accomplish it? | 4116 30 | HasLastSubevent | What do you do last to accomplish it? | 2968 31 | HasPrerequisite | What do you need to do first? | 23379 32 | MotivatedByGoal | Why would you do it? | 15094 33 | Causes | What does it make happen? | 18211 34 | Desires | What does it want? | 9055 35 | CausesDesire | What does it make you want to do? | 4974 36 | HasProperty | What properties does it have? | 82384 37 | ReceivesAction | What can you do to it? | 10838 38 | DefinedAs | How do you define it? | 6420 39 | SymbolOf | What does it represent? | 166 40 | LocatedNear | What is it typically near? | 5024 41 | ConceptuallyRelatedTo | What is related to it in an unknown way? | 23010 42 | InheritsFrom | (not stored, but used in some applications) 43 | 44 | I have also removed the language relation, and the database only supports English. 45 | 46 | npm install conceptnet 47 | 48 | * UnZip and import the SQL Data from the ./data folder 49 | 50 | -------------------------------------------------------------------------------- /test/basic.js: -------------------------------------------------------------------------------- 1 | 2 | var mocha = require("mocha"); 3 | var should = require("should"); 4 | 5 | var options = {host:'127.0.0.1', user:'root', pass:''} 6 | 7 | var concept = require("../")(options); 8 | 9 | describe("It should connect", function(){ 10 | 11 | it("should have a few functions", function(done){ 12 | concept.usedForForward.should.be.Function; 13 | concept.usedForReverse.should.be.Function; 14 | concept.isAForward.should.be.Function; 15 | concept.isAReverse.should.be.Function; 16 | 17 | concept.assertionLookupForward.should.be.Function; 18 | concept.putConcept.should.be.Function; 19 | concept.assersionTest.should.be.Function; 20 | concept.resolveFact.should.be.Function; 21 | 22 | concept.relatedConcepts.should.be.Function; 23 | concept.constructSurface.should.be.Function; 24 | done() 25 | }); 26 | 27 | it.only("should pull out concepts from string", function(done){ 28 | concept.conceptLookup("Who is the current president?", function(err, reply){ 29 | reply.length.should.eql(2); 30 | done(); 31 | }); 32 | }); 33 | }); --------------------------------------------------------------------------------