├── .babelrc ├── .eslintignore ├── .eslintrc.json ├── .gitignore ├── LICENSE ├── README.md ├── classifier.json ├── client ├── .gitignore ├── package-lock.json ├── package.json ├── public │ ├── favicon.ico │ ├── index.html │ └── manifest.json └── src │ ├── components │ ├── App.js │ ├── Header.js │ ├── Preloader.js │ ├── Sidebar.js │ ├── Tweet.js │ └── styles │ │ ├── App.css │ │ ├── header.css │ │ ├── index.css │ │ ├── preloader.css │ │ ├── sidebar.css │ │ └── tweets.css │ ├── containers │ ├── categories.js │ └── tweets.js │ ├── index.js │ ├── logo.svg │ ├── registerServiceWorker.js │ └── util │ ├── categories.js │ ├── config.js │ └── fetch.js ├── controllers ├── __tests__ │ └── tweets.test.js ├── index.js └── tweets.js ├── lib ├── __mocks__ │ └── redis.js ├── classifier.js ├── logger.js └── redis.js ├── package-lock.json ├── package.json ├── server.js ├── start-client.js ├── template.html ├── training-data.json └── training.js /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["env", "react"], 3 | "plugins": [ 4 | "transform-async-to-generator" 5 | ] 6 | } -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | client/node_modules 3 | client/build/ 4 | client/package.json 5 | client/package-lock.json -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "parser": "babel-eslint", 3 | "parserOptions": { 4 | "ecmaVersion": 6, 5 | "sourceType": "module", 6 | "ecmaFeatures": { 7 | "jsx": true 8 | } 9 | }, 10 | "env": { 11 | "browser": true, 12 | "node": true, 13 | "es6": true, 14 | "jest": true 15 | }, 16 | "plugins": [ 17 | "react" 18 | ], 19 | "rules": { 20 | "jsx-quotes": ["error", "prefer-double"] 21 | }, 22 | "extends": ["standard", "standard-react"] 23 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | .env 3 | test.js 4 | pos.js 5 | search.js 6 | /client/build 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Oluwaseun Omoyajowo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Job Tweets 2 | Find Real time job tweets! 3 | 4 | ## Getting Started 5 | 6 | ### Installion 7 | 1. Clone the repo 8 | 2. Install all server modules using `npm install` 9 | 3. For client modules, navigate to client directory and run `npm install` 10 | 4. Get your API credentials. 11 | To get Consumer Key & Consumer Secret, you have to create an app in Twitter via 12 | https://apps.twitter.com/app/new 13 | Then you'll be taken to a page containing Consumer Key & Consumer Secret. 14 | 15 | 5. In the main directory, Rename the .env-dev file to .env and enter your twitter api credentials. 16 | 6. To start the both servers (Backend and Client dev server), run `npm run dev` from the main server directory. 17 | 18 | ### Urgent ToDo 19 | 20 | 1. Cache tweets for 24 hrs 21 | 2. Filter tweets base on location 22 | 3. Filter tweets base on Job Spec 23 | 24 | All commit in development branch. 25 | 26 | Don't forget to star and contribute! 27 | 28 | -------------------------------------------------------------------------------- /classifier.json: -------------------------------------------------------------------------------- 1 | {"classifier":{"classFeatures":{"SD":{"0":2,"1":2,"2":2,"3":2,"4":2,"5":2,"6":2,"7":2,"8":2,"9":2,"10":2,"11":2,"12":2,"13":2,"14":2,"15":2,"16":2,"17":2,"18":2,"19":2,"20":2,"21":2,"22":2,"23":2,"24":2,"25":2,"26":2,"27":2,"28":2,"29":2,"30":2,"31":2,"32":2,"33":2,"34":2,"35":2,"36":2,"37":2,"38":2,"39":2,"40":2,"41":2,"42":2,"43":2,"44":2,"45":2,"46":2,"47":2,"48":2,"49":2,"50":2,"51":2},"SM":{"52":2,"53":2,"54":2,"55":2,"56":2,"57":2,"58":2,"59":2,"60":2,"61":2,"62":2,"63":2,"64":2,"65":2,"66":2,"67":2},"BDM":{"51":2,"68":2,"69":2,"70":2,"71":2,"72":2,"73":2},"HC":{"74":2,"75":2,"76":2,"77":2,"78":2,"79":2,"80":2,"81":2},"IC":{"82":2,"83":2,"84":2,"85":2},"RS":{"57":2,"86":2,"87":2,"88":2,"89":2,"90":2},"HR":{"91":2,"92":2,"93":2,"94":2},"HT":{"95":2,"96":2,"97":2,"98":2,"99":2,"100":2,"101":2,"102":2,"103":2,"104":2,"105":2},"MP":{"69":2,"73":2,"106":2,"107":2,"108":2},"RE":{"109":2,"110":2,"111":2},"AF":{"107":2,"112":2,"113":2,"114":2,"115":2,"116":2,"117":2},"TL":{"118":2,"119":2,"120":2,"121":2,"122":2,"123":2,"124":2,"125":2,"126":2,"127":2,"128":2},"AD":{"50":2,"114":2,"129":2,"130":2,"131":2,"132":2,"133":2,"134":2,"135":2,"136":2,"137":2,"138":2,"139":2,"140":2},"CW":{"66":2,"140":2,"141":2,"142":2,"143":2,"144":2,"145":2,"146":2,"147":2,"148":2,"149":2},"SA":{"51":2,"73":2,"79":2,"150":2,"151":2,"152":2,"153":2,"154":2,"155":2,"156":2},"MU":{"93":2,"145":2,"157":2,"158":2,"159":2,"160":2,"161":2,"162":2,"163":2},"OF":{"164":2,"165":2,"166":2,"167":2,"168":2,"169":2,"170":2,"171":2},"CS":{"78":2,"172":2,"173":2,"174":2,"175":2},"RF":{"70":2,"165":2,"176":2,"177":2,"178":2,"179":2,"180":2,"181":2,"182":2,"183":2,"184":2,"185":2,"186":2,"187":2,"188":2,"189":2,"190":2},"BF":{"70":2,"72":2,"79":2,"171":2,"172":2,"191":2,"192":2,"193":2,"194":2,"195":2,"196":2,"197":2,"198":2,"199":2,"200":2},"LG":{"165":2,"166":2,"198":2,"201":2,"202":2,"203":2,"204":2,"205":2,"206":2},"EG":{"48":2,"207":2,"208":2,"209":2,"210":2,"211":2,"212":2}},"classTotals":{"SD":2,"SM":2,"BDM":2,"HC":2,"IC":2,"RS":2,"HR":2,"HT":2,"MP":2,"RE":2,"AF":2,"TL":2,"AD":2,"CW":2,"SA":2,"MU":2,"OF":2,"CS":2,"RF":2,"BF":2,"LG":2,"EG":2},"totalExamples":23,"smoothing":1},"docs":[{"label":"SD","text":["io","net","app","angular","vuej","architect","android","actionscript","activex","algol","algorithm","api","applet","asp","aspi","assembl","assembl","babel","backend","back","end","sharp","clojur","compil","css","dart","django","ecmascript","front","end","frontend","full","stack","full","stack","go","languag","haskel","html","java","javascript","lisp","node","js","php","python","react","react","nativ","rubi","rust","scala","softwar","engin","sql","web","develop"]},{"label":"SM","text":["advertis","b2b","b2c","bant","brand","brand","data","entri","field","sale","junior","sale","rep","seo","social","seller","digit","market"]},{"label":"BDM","text":["strategist","product","manag","qualiti","manag","busi","develop","busi","oper"]},{"label":"HC","text":["nurs","doctor","hospit","health","care","specialist","therapist","surgic"]},{"label":"IC","text":["internship","siw","intern","intern","colleg"]},{"label":"RS","text":["analysi","data","mine","research","questionnair","statist","research"]},{"label":"HR","text":["human","resourc","director","hr"]},{"label":"HT","text":["lectur","interventionist","univers","trainer","train","teacher","teach","lectur","instructor","coach","cours","cours","certif"]},{"label":"MP","text":["manufactur","plant","product","oper","machin"]},{"label":"RE","text":["agent","estat","real"]},{"label":"AF","text":["agricultur","farm","anim","haverst","tractor","plant","fish"]},{"label":"TL","text":["driver","drive","uber","lyft","pilot","truck","deliveri","logist","transport","freight","broker"]},{"label":"AD","text":["graphic","design","illustr","ui","ux","web","design","respons","design","3d","artist","anim","caricatur","anim","portrait","cartoon","photoshop"]},{"label":"CW","text":["write","blog","editor","photoshop","content","writer","digit","media","creation","photojournalist","journalist"]},{"label":"SA","text":["linux","system","administr","system","cloud","specialist","azur","develop","oper","aw","azur","linod"]},{"label":"MU","text":["compos","musician","music","studio","produc","video","director","song","writer"]},{"label":"OF","text":["admin","execut","assist","excel","receptionist","recruit","webspher","offic"]},{"label":"CS","text":["custom","servic","care","associ","repres"]},{"label":"RF","text":["cook","restaur","banquet","server","dishwash","cater","manag","benihana","chef","kitchen","manag","sou","execut","chef","sushi","dietari","event","waiter","waitress"]},{"label":"BF","text":["banker","loan","offic","teller","custom","loan","processor","mortag","origin","busi","analyst","financ","manag","auditor","tax","specialist"]},{"label":"LG","text":["legal","commerci","financ","attornei","litig","attornei","execut","assist","clerk","paraleg"]},{"label":"EG","text":["electr","engin","mechan","engin","comput","physic","chemic","electron"]}],"features":{"io":1,"net":1,"app":1,"angular":1,"vuej":1,"architect":1,"android":1,"actionscript":1,"activex":1,"algol":1,"algorithm":1,"api":1,"applet":1,"asp":1,"aspi":1,"assembl":2,"babel":1,"backend":1,"back":1,"end":2,"sharp":1,"clojur":1,"compil":1,"css":1,"dart":1,"django":1,"ecmascript":1,"front":1,"frontend":1,"full":2,"stack":2,"go":1,"languag":1,"haskel":1,"html":1,"java":1,"javascript":1,"lisp":1,"node":1,"js":1,"php":1,"python":1,"react":2,"nativ":1,"rubi":1,"rust":1,"scala":1,"softwar":1,"engin":3,"sql":1,"web":2,"develop":3,"advertis":1,"b2b":1,"b2c":1,"bant":1,"brand":2,"data":2,"entri":1,"field":1,"sale":2,"junior":1,"rep":1,"seo":1,"social":1,"seller":1,"digit":2,"market":1,"strategist":1,"product":2,"manag":5,"qualiti":1,"busi":3,"oper":3,"nurs":1,"doctor":1,"hospit":1,"health":1,"care":2,"specialist":3,"therapist":1,"surgic":1,"internship":1,"siw":1,"intern":2,"colleg":1,"analysi":1,"mine":1,"research":2,"questionnair":1,"statist":1,"human":1,"resourc":1,"director":2,"hr":1,"lectur":2,"interventionist":1,"univers":1,"trainer":1,"train":1,"teacher":1,"teach":1,"instructor":1,"coach":1,"cours":2,"certif":1,"manufactur":1,"plant":2,"machin":1,"agent":1,"estat":1,"real":1,"agricultur":1,"farm":1,"anim":3,"haverst":1,"tractor":1,"fish":1,"driver":1,"drive":1,"uber":1,"lyft":1,"pilot":1,"truck":1,"deliveri":1,"logist":1,"transport":1,"freight":1,"broker":1,"graphic":1,"design":3,"illustr":1,"ui":1,"ux":1,"respons":1,"3d":1,"artist":1,"caricatur":1,"portrait":1,"cartoon":1,"photoshop":2,"write":1,"blog":1,"editor":1,"content":1,"writer":2,"media":1,"creation":1,"photojournalist":1,"journalist":1,"linux":1,"system":2,"administr":1,"cloud":1,"azur":2,"aw":1,"linod":1,"compos":1,"musician":1,"music":1,"studio":1,"produc":1,"video":1,"song":1,"admin":1,"execut":3,"assist":2,"excel":1,"receptionist":1,"recruit":1,"webspher":1,"offic":2,"custom":2,"servic":1,"associ":1,"repres":1,"cook":1,"restaur":1,"banquet":1,"server":1,"dishwash":1,"cater":1,"benihana":1,"chef":2,"kitchen":1,"sou":1,"sushi":1,"dietari":1,"event":1,"waiter":1,"waitress":1,"banker":1,"loan":2,"teller":1,"processor":1,"mortag":1,"origin":1,"analyst":1,"financ":2,"auditor":1,"tax":1,"legal":1,"commerci":1,"attornei":2,"litig":1,"clerk":1,"paraleg":1,"electr":1,"mechan":1,"comput":1,"physic":1,"chemic":1,"electron":1},"stemmer":{},"lastAdded":22,"events":{"domain":null,"_events":{},"_eventsCount":1}} -------------------------------------------------------------------------------- /client/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/ignore-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | 6 | # testing 7 | /coverage 8 | 9 | # misc 10 | .DS_Store 11 | .env.local 12 | .env.development.local 13 | .env.test.local 14 | .env.production.local 15 | 16 | /build 17 | 18 | npm-debug.log* 19 | yarn-debug.log* 20 | yarn-error.log* 21 | -------------------------------------------------------------------------------- /client/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "jobtweets", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "prop-types": "^15.6.1", 7 | "react": "^16.4.1", 8 | "react-content-loader": "^3.1.2", 9 | "react-dom": "^16.4.1", 10 | "react-redux": "^5.0.6", 11 | "react-scripts": "1.1.4", 12 | "react-twitter-widgets": "^1.7.1", 13 | "redux": "^4.0.0", 14 | "redux-thunk": "^2.2.0", 15 | "socket.io": "^2.1.1", 16 | "socket.io-client": "^2.1.1", 17 | "twitter": "^1.7.1" 18 | }, 19 | "scripts": { 20 | "start": "react-scripts start", 21 | "build": "react-scripts build", 22 | "test": "react-scripts test --env=jsdom", 23 | "eject": "react-scripts eject" 24 | }, 25 | "proxy": "https://localhost:8080/" 26 | } 27 | -------------------------------------------------------------------------------- /client/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flickz/jobtweets/aa1f0edb152ac6deb09b747483967f2634452b91/client/public/favicon.ico -------------------------------------------------------------------------------- /client/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Real Time Job Tweets 6 | 7 | 8 | 9 | 10 | 11 | 15 | 16 | 17 | 26 | 27 | React App 28 | 29 | 32 |
33 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /client/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "React App", 3 | "name": "Create React App Sample", 4 | "icons": [ 5 | { 6 | "src": "favicon.ico", 7 | "sizes": "192x192", 8 | "type": "image/png" 9 | } 10 | ], 11 | "start_url": "./index.html", 12 | "display": "standalone", 13 | "theme_color": "#000000", 14 | "background_color": "#ffffff" 15 | } 16 | -------------------------------------------------------------------------------- /client/src/components/App.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import Tweets from '../containers/tweets' 3 | import Header from './Header' 4 | // import Categories from './containers/categories' 5 | import './styles/App.css' 6 | 7 | export default class App extends React.PureComponent { 8 | render () { 9 | return ( 10 | 11 |
12 | 13 | 14 | ) 15 | } 16 | } 17 | 18 | -------------------------------------------------------------------------------- /client/src/components/Header.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import './styles/header.css' 3 | 4 | export default () => { 5 | return ( 6 | 9 | ) 10 | } 11 | -------------------------------------------------------------------------------- /client/src/components/Preloader.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import ContentLoader from 'react-content-loader' 3 | import './styles/preloader.css' 4 | 5 | export default (props) => { 6 | return ( 7 |
8 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 |
24 | ) 25 | } 26 | -------------------------------------------------------------------------------- /client/src/components/Sidebar.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import './styles/sidebar.css' 3 | import {categoriesData} from '../util/categories' 4 | 5 | export default ({onCheck}) => { 6 | const categories = categoriesData.map((category, index) => { 7 | return ( 8 |
9 | 13 |
14 | ) 15 | }) 16 | 17 | return ( 18 |
19 | 25 |
26 | ) 27 | } 28 | -------------------------------------------------------------------------------- /client/src/components/Tweet.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import { Tweet } from 'react-twitter-widgets' 3 | import ContentLoader from './Preloader' 4 | import './styles/tweets.css' 5 | 6 | export default ({tweets, fetching}) => { 7 | if (tweets.length > 0) { 8 | const embededTweets = tweets.map((tweet) => { 9 | let id = tweet.id 10 | return ( 11 |
12 | 13 |
) 14 | }) 15 | const loader = (fetching)? () : '' 16 | return ( 17 |
18 | { embededTweets } 19 | { loader } 20 |
21 | ) 22 | } else { 23 | return ( 24 |
25 | 26 |
) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /client/src/components/styles/App.css: -------------------------------------------------------------------------------- 1 | .App { 2 | text-align: center; 3 | } 4 | 5 | .App-logo { 6 | animation: App-logo-spin infinite 20s linear; 7 | height: 80px; 8 | } 9 | 10 | .App-header { 11 | background-color: #222; 12 | height: 150px; 13 | padding: 20px; 14 | color: white; 15 | } 16 | 17 | .App-intro { 18 | font-size: large; 19 | } 20 | 21 | @keyframes App-logo-spin { 22 | from { transform: rotate(0deg); } 23 | to { transform: rotate(360deg); } 24 | } 25 | -------------------------------------------------------------------------------- /client/src/components/styles/header.css: -------------------------------------------------------------------------------- 1 | .header { 2 | display: flex; 3 | justify-content: center; 4 | align-items: center; 5 | height: 60px; 6 | background-color: #fff; 7 | box-shadow: 0 2px 2px 0 rgba(0,0,0,0.14), 0 1px 5px 0 rgba(0,0,0,0.12), 0 3px 1px -2px rgba(0,0,0,0.2); 8 | } 9 | 10 | a.logo{ 11 | text-align: center !important; 12 | display: inline-block; 13 | text-decoration: none; 14 | color: #55acee; 15 | font-size: 2.1rem; 16 | font-weight: 400; 17 | } 18 | -------------------------------------------------------------------------------- /client/src/components/styles/index.css: -------------------------------------------------------------------------------- 1 | body{ 2 | background: #e6ecf0; 3 | margin: 0; 4 | } 5 | a:hover, a:active{ 6 | text-decoration: none; 7 | color: #55acee; 8 | outline: 0; 9 | } 10 | .tweet-section{ 11 | margin-top: 25px; 12 | } 13 | .spinner{ 14 | color: #55acee; 15 | text-align: center; 16 | margin-top: 80px; 17 | } 18 | .spinner i{ 19 | font-size: 32px; 20 | } -------------------------------------------------------------------------------- /client/src/components/styles/preloader.css: -------------------------------------------------------------------------------- 1 | .content-loader { 2 | background: #fff; 3 | width: 100%; 4 | padding: 10px; 5 | margin-top: 10px; 6 | border: 1px solid rgb(225, 232, 237); 7 | border-radius: 5px; 8 | } 9 | 10 | -------------------------------------------------------------------------------- /client/src/components/styles/sidebar.css: -------------------------------------------------------------------------------- 1 | .filter-sidebar{ 2 | background: #fff; 3 | margin-top: 25px; 4 | padding: 5px; 5 | border: 1px solid #e1e8ed; 6 | } 7 | .filter-sidebar p{ 8 | font-size: 18px; 9 | font-weight: 600; 10 | text-align: center; 11 | } 12 | .filter-form{ 13 | margin-left: 10px; 14 | } -------------------------------------------------------------------------------- /client/src/components/styles/tweets.css: -------------------------------------------------------------------------------- 1 | .tweets-wrapper { 2 | display: flex; 3 | flex-direction: column; 4 | align-items: center; 5 | margin-bottom: 30px; 6 | width: 500px; 7 | } 8 | 9 | .content { 10 | display: flex; 11 | flex-direction: column; 12 | align-items: center; 13 | } -------------------------------------------------------------------------------- /client/src/containers/categories.js: -------------------------------------------------------------------------------- 1 | // import React, {Component} from 'react' 2 | // import {connect} from 'react-redux' 3 | // import {addFilter, removeFilter} from '../actions' 4 | // import SideBar from '../components/sidebar' 5 | 6 | // class Categories extends Component { 7 | // constructor () { 8 | // super() 9 | // this.onCheck = (category) => { 10 | // let filters = this.props.filters 11 | // if (filters.indexOf(category) === -1) { 12 | // this.props.addFilter(category) 13 | // } else { 14 | // this.props.removeFilter(category) 15 | // } 16 | // } 17 | // } 18 | 19 | // render () { 20 | // return () 21 | // } 22 | // } 23 | 24 | // function mapStateToProps (state) { 25 | // return { 26 | // filters: state.filters 27 | // } 28 | // } 29 | 30 | // function mapDispatchToProps (dispatch) { 31 | // return { 32 | // addFilter: (category) => { 33 | // dispatch(addFilter(category)) 34 | // }, 35 | // removeFilter: (category) => { 36 | // dispatch(removeFilter(category)) 37 | // } 38 | // } 39 | // } 40 | // export default connect(mapStateToProps, mapDispatchToProps)(Categories) 41 | -------------------------------------------------------------------------------- /client/src/containers/tweets.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import { fetchTweets } from '../util/fetch' 3 | import Preloader from '../components/Preloader' 4 | import EmbededTweet from '../components/Tweet' 5 | 6 | export default class Tweets extends React.Component { 7 | constructor (props) { 8 | super(props) 9 | this.state = { 10 | fetching: true, 11 | tweets: [], 12 | page: 0 13 | } 14 | } 15 | 16 | componentDidMount () { 17 | window.addEventListener('scroll', this.handleScrollEvent) 18 | this.fetchNextTweets() 19 | } 20 | 21 | componentWillUnmount () { 22 | window.removeEventListener('scroll', this.handleScrollEvent) 23 | } 24 | 25 | updateTweetsList (newTweets) { 26 | const { tweets } = this.state 27 | this.setState({ 28 | tweets: [...tweets, ...newTweets] 29 | }) 30 | } 31 | 32 | setFetchingState (fetching) { 33 | this.setState({ 34 | fetching 35 | }) 36 | } 37 | 38 | updateTweetsPage (page) { 39 | this.setState({ 40 | page 41 | }) 42 | } 43 | 44 | async fetchNextTweets () { 45 | this.setFetchingState(true) 46 | const tweets = await fetchTweets(this.state.page) 47 | if (tweets) { 48 | this.updateTweetsList(tweets) 49 | this.setFetchingState(false) 50 | this.setState((prevState, props) => ({ 51 | page: prevState.page + 1 52 | })) 53 | } 54 | } 55 | 56 | handleScrollEvent = (event) => { 57 | const { fetching } = this.state 58 | const windowHeight = window.innerHeight 59 | const scrollVerticalPos = window.scrollY 60 | const contentHeight = document.body.offsetHeight 61 | if ((windowHeight + scrollVerticalPos) >= (contentHeight)) { 62 | if (!fetching) { 63 | this.fetchNextTweets() 64 | .catch(error => { 65 | throw error 66 | }) 67 | } 68 | } 69 | } 70 | 71 | render () { 72 | const { tweets, fetching } = this.state 73 | 74 | return ( 75 |
76 | 77 |
78 | ) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /client/src/index.js: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import ReactDOM from 'react-dom' 3 | import './components/styles/index.css' 4 | import App from './components/App' 5 | import registerServiceWorker from './registerServiceWorker' 6 | 7 | ReactDOM.render( 8 | , 9 | document.getElementById('root')) 10 | 11 | registerServiceWorker() 12 | -------------------------------------------------------------------------------- /client/src/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /client/src/registerServiceWorker.js: -------------------------------------------------------------------------------- 1 | // In production, we register a service worker to serve assets from local cache. 2 | 3 | // This lets the app load faster on subsequent visits in production, and gives 4 | // it offline capabilities. However, it also means that developers (and users) 5 | // will only see deployed updates on the "N+1" visit to a page, since previously 6 | // cached resources are updated in the background. 7 | 8 | // To learn more about the benefits of this model, read https://goo.gl/KwvDNy. 9 | // This link also includes instructions on opting out of this behavior. 10 | 11 | const isLocalhost = Boolean( 12 | window.location.hostname === 'localhosts' || 13 | // [::1] is the IPv6 localhost address. 14 | window.location.hostname === '[::1]' || 15 | // 127.0.0.1/8 is considered localhost for IPv4. 16 | window.location.hostname.match( 17 | /^127(?:\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}$/ 18 | ) 19 | ) 20 | 21 | export default function register () { 22 | if (process.env.NODE_ENV === 'production' && 'serviceWorker' in navigator) { 23 | // The URL constructor is available in all browsers that support SW. 24 | const publicUrl = new URL(process.env.PUBLIC_URL, window.location) 25 | if (publicUrl.origin !== window.location.origin) { 26 | // Our service worker won't work if PUBLIC_URL is on a different origin 27 | // from what our page is served on. This might happen if a CDN is used to 28 | // serve assets; see https://github.com/facebookincubator/create-react-app/issues/2374 29 | return 30 | } 31 | 32 | window.addEventListener('load', () => { 33 | const swUrl = `${process.env.PUBLIC_URL}/service-worker.js` 34 | 35 | if (!isLocalhost) { 36 | // Is not local host. Just register service worker 37 | registerValidSW(swUrl) 38 | } else { 39 | // This is running on localhost. Lets check if a service worker still exists or not. 40 | checkValidServiceWorker(swUrl) 41 | } 42 | }) 43 | } 44 | } 45 | 46 | function registerValidSW (swUrl) { 47 | navigator.serviceWorker 48 | .register(swUrl) 49 | .then(registration => { 50 | registration.onupdatefound = () => { 51 | const installingWorker = registration.installing 52 | installingWorker.onstatechange = () => { 53 | if (installingWorker.state === 'installed') { 54 | if (navigator.serviceWorker.controller) { 55 | // At this point, the old content will have been purged and 56 | // the fresh content will have been added to the cache. 57 | // It's the perfect time to display a "New content is 58 | // available; please refresh." message in your web app. 59 | console.log('New content is available; please refresh.') 60 | } else { 61 | // At this point, everything has been precached. 62 | // It's the perfect time to display a 63 | // "Content is cached for offline use." message. 64 | console.log('Content is cached for offline use.') 65 | } 66 | } 67 | } 68 | } 69 | }) 70 | .catch(error => { 71 | console.error('Error during service worker registration:', error) 72 | }) 73 | } 74 | 75 | function checkValidServiceWorker (swUrl) { 76 | // Check if the service worker can be found. If it can't reload the page. 77 | fetch(swUrl) 78 | .then(response => { 79 | // Ensure service worker exists, and that we really are getting a JS file. 80 | if ( 81 | response.status === 404 || 82 | response.headers.get('content-type').indexOf('javascript') === -1 83 | ) { 84 | // No service worker found. Probably a different app. Reload the page. 85 | navigator.serviceWorker.ready.then(registration => { 86 | registration.unregister().then(() => { 87 | window.location.reload() 88 | }) 89 | }) 90 | } else { 91 | // Service worker found. Proceed as normal. 92 | registerValidSW(swUrl) 93 | } 94 | }) 95 | .catch(() => { 96 | console.log( 97 | 'No internet connection found. App is running in offline mode.' 98 | ) 99 | }) 100 | } 101 | 102 | export function unregister () { 103 | if ('serviceWorker' in navigator) { 104 | navigator.serviceWorker.ready.then(registration => { 105 | registration.unregister() 106 | }) 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /client/src/util/categories.js: -------------------------------------------------------------------------------- 1 | export const categoriesData = [ 2 | { 3 | acronym: 'AF', 4 | fullMeaning: 'Agriculture & Farming' 5 | }, 6 | { 7 | acronym: 'AD', 8 | fullMeaning: 'Art & Design' 9 | }, 10 | { 11 | acronym: 'BF', 12 | fullMeaning: 'Banking & Finance' 13 | }, 14 | { 15 | acronym: 'BDM', 16 | fullMeaning: 'Business Development & Management' 17 | }, 18 | { 19 | acronym: 'CW', 20 | fullMeaning: 'Content Writing' 21 | }, 22 | { 23 | acronym: 'CS', 24 | fullMeaning: 'Customer Services' 25 | }, 26 | { 27 | acronym: 'ET', 28 | fullMeaning: 'Education & Teaching' 29 | }, 30 | { 31 | acronym: 'EG', 32 | fullMeaning: 'Engineering' 33 | }, 34 | { 35 | acronym: 'HC', 36 | fullMeaning: 'Health Care' 37 | }, 38 | { 39 | acronym: 'HR', 40 | fullMeaning: 'Human Resources' 41 | }, 42 | { 43 | acronym: 'IC', 44 | fullMeaning: 'Internship & College' 45 | }, 46 | { 47 | acronym: 'LG', 48 | fullMeaning: 'Legal' 49 | }, 50 | { 51 | acronym: 'MP', 52 | fullMeaning: 'Manufacturing & Production' 53 | }, 54 | { 55 | acronym: 'MU', 56 | fullMeaning: 'Music' 57 | }, 58 | { 59 | acronym: 'OF', 60 | fullMeaning: 'Office' 61 | }, 62 | { 63 | acronym: 'RE', 64 | fullMeaning: 'Real Estate' 65 | }, 66 | { 67 | acronym: 'RF', 68 | fullMeaning: 'Restaurant & Food Service' 69 | }, 70 | { 71 | acronym: 'RS', 72 | fullMeaning: 'Research' 73 | }, 74 | { 75 | acronym: 'SM', 76 | fullMeaning: 'Sales & Marketing' 77 | }, 78 | { 79 | acronym: 'SD', 80 | fullMeaning: 'Software Development' 81 | }, 82 | { 83 | acronym: 'SA', 84 | fullMeaning: 'System Admin' 85 | }, 86 | { 87 | acronym: 'TL', 88 | fullMeaning: 'Transportation & Logistics' 89 | } 90 | ] 91 | -------------------------------------------------------------------------------- /client/src/util/config.js: -------------------------------------------------------------------------------- 1 | export const API_BASE_URL = 'http://localhost:8080/api/v1' 2 | export const STREAM_SOCKET_URL = 'localhost:8080/' -------------------------------------------------------------------------------- /client/src/util/fetch.js: -------------------------------------------------------------------------------- 1 | import io from 'socket.io-client/lib' 2 | import { STREAM_SOCKET_URL, API_BASE_URL } from './config' 3 | 4 | export const stream = io(STREAM_SOCKET_URL) 5 | 6 | stream.on('error', (error) => { 7 | console.log('Error occoured..', error) 8 | }) 9 | 10 | function receiveTweetsData (data) { 11 | console.log(data) 12 | } 13 | 14 | function handleFetchError (error) { 15 | console.log(error) 16 | } 17 | 18 | export const fetchTweets = (page) => { 19 | const url = `${API_BASE_URL}/tweets?page=${page}` 20 | return window.fetch(url) 21 | .then((response) => { 22 | if(!response.ok) { 23 | throw response 24 | } 25 | return response.json() 26 | }) 27 | .catch(handleFetchError) 28 | } 29 | -------------------------------------------------------------------------------- /controllers/__tests__/tweets.test.js: -------------------------------------------------------------------------------- 1 | const { addTweet, getTweet, getTweetKeys, getTweets } = require('../tweets') 2 | jest.mock('../../lib/redis') 3 | 4 | const dummyTweet = { 5 | id: '1212', 6 | first_class: 'SD', 7 | second_class: 'DO' 8 | } 9 | 10 | describe('#addTweet()', () => { 11 | it('should add tweet to redis', async () => { 12 | const result = await addTweet(dummyTweet) 13 | expect(result).toHaveLength(2) 14 | }) 15 | }) 16 | 17 | describe('#getTweet()', () => { 18 | it('should return tweet with the key', async () => { 19 | const result = await getTweet('tweet:1') 20 | expect(result.id).toEqual(dummyTweet.id) 21 | }) 22 | }) 23 | 24 | describe('#getTweetKeys()', () => { 25 | it('should return list of keys within specified range', async () => { 26 | const results = await getTweetKeys(-1, 0) 27 | expect(results).toHaveLength(1) 28 | }) 29 | }) 30 | 31 | describe('#getTweets()', () => { 32 | it('should return list of tweets within specified range', async () => { 33 | const results = await getTweets(-1, 0) 34 | expect(results).toHaveLength(1) 35 | }) 36 | }) 37 | -------------------------------------------------------------------------------- /controllers/index.js: -------------------------------------------------------------------------------- 1 | module.exports = require('requireindex')(__dirname) 2 | -------------------------------------------------------------------------------- /controllers/tweets.js: -------------------------------------------------------------------------------- 1 | const { isEmpty } = require('lodash') 2 | const logger = require('../lib/logger')() 3 | const { RedisClient } = require('../lib/redis') 4 | 5 | const redis = RedisClient() 6 | const listKey = 'tweets' 7 | 8 | exports.addTweet = async function (tweet) { 9 | let tweetId 10 | try { 11 | tweetId = await redis.incr('tweet_id') 12 | } catch (err) { 13 | logger.error(`Error incrementing tweet in redis`) 14 | logger.error(err) 15 | return err 16 | } 17 | const hashKey = `tweet:${tweetId}` 18 | return redis.multi().hmset(hashKey, tweet).rpush(listKey, hashKey).exec() 19 | } 20 | 21 | exports.getTweet = async function (key) { 22 | return redis.hgetall(key) 23 | } 24 | 25 | exports.getTweetKeys = async function (start, stop) { 26 | return redis.lrange(listKey, start, stop) 27 | } 28 | 29 | const getTweets = async function (start, stop) { 30 | const tweets = [] 31 | const tweetKeys = await redis.lrange(listKey, start, stop) 32 | if (tweetKeys.length >= 1) { 33 | for (let i = 0; i < tweetKeys.length; i++) { 34 | const key = tweetKeys[i] 35 | const tweet = await redis.hgetall(key) 36 | 37 | if (isEmpty(tweet) === false) { 38 | tweets.push(tweet) 39 | } 40 | } 41 | return tweets 42 | } 43 | return tweets 44 | } 45 | 46 | exports.get = async function (req, res) { 47 | const page = parseInt(req.query.page) 48 | 49 | function getRange (page) { 50 | let from 51 | if (page === 0) { 52 | from = page * 10 53 | } else { 54 | from = page * 10 + 1 55 | } 56 | const to = (page + 1) * 10 57 | return [from, to] 58 | } 59 | const [from, to] = getRange(page) 60 | console.log(from, to) 61 | 62 | try { 63 | const tweets = await getTweets(from, to) 64 | if (tweets) { 65 | res.status(200).send(tweets) 66 | return 67 | } 68 | return res.status(204).send() 69 | } catch (err) { 70 | console.error(`Error occur from getting tweets in range (${from}-${to}`) 71 | console.error(err) 72 | res.status(400) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /lib/__mocks__/redis.js: -------------------------------------------------------------------------------- 1 | const Redis = require('ioredis-mock') 2 | 3 | exports.RedisClient = function () { 4 | return new Redis() 5 | } 6 | -------------------------------------------------------------------------------- /lib/classifier.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const { BayesClassifier } = require('natural') 3 | 4 | const classifierPath = path.join(__dirname, '../classifier.json') 5 | 6 | exports.classifyTweetWithBayes = function (text, callback) { 7 | BayesClassifier.load(classifierPath, null, (err, classifier) => { 8 | if (err) { 9 | console.error(err) 10 | callback(err, null) 11 | return 12 | } 13 | 14 | text = removeHash(text) 15 | const classifications = [] 16 | const result = classifier.getClassifications(text) 17 | // Get the 1st two classifications 18 | classifications.push(result[0].label) 19 | classifications.push(result[1].label) 20 | 21 | callback(null, classifications) 22 | }) 23 | } 24 | 25 | const removeHash = (word) => { 26 | return word.replace(/#|RT|rt/g, '') 27 | } 28 | -------------------------------------------------------------------------------- /lib/logger.js: -------------------------------------------------------------------------------- 1 | const winston = require('winston') 2 | 3 | module.exports = function () { 4 | const myLogTransports = [] 5 | 6 | switch (process.env.NODE_ENV) { 7 | case 'production': 8 | myLogTransports.push(new (winston.transports.Console)({ level: 'info' })) 9 | break 10 | case 'test': 11 | myLogTransports.push(new (winston.transports.Console)({ level: 'debug' })) 12 | break 13 | default: 14 | // Dev environments, usually 15 | myLogTransports.push(new (winston.transports.Console)({ level: 'debug' })) 16 | break 17 | } 18 | 19 | let logger = winston.createLogger({ 20 | transports: myLogTransports 21 | }) 22 | 23 | return logger 24 | } 25 | -------------------------------------------------------------------------------- /lib/redis.js: -------------------------------------------------------------------------------- 1 | const Redis = require('ioredis') 2 | 3 | exports.RedisClient = function () { 4 | return new Redis({ 5 | port: process.env.REDIS_PORT, 6 | host: process.env.REDIS_HOST 7 | }) 8 | } 9 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "jobtweets", 3 | "version": "1.0.0", 4 | "description": "\"Find realtime job tweets\"", 5 | "main": "server.js", 6 | "scripts": { 7 | "test": "jest --watch", 8 | "start": "node server.js", 9 | "dev": "concurrently \"npm run server\" \"npm run client\"", 10 | "server": "node server.js", 11 | "client": "node start-client.js", 12 | "lint": "eslint \"**/*.js\" \"**/*.jsx\"", 13 | "lint:fix": "eslint \"**/*.js\" \"**/*.jsx\" --fix" 14 | }, 15 | "author": "", 16 | "license": "ISC", 17 | "dependencies": { 18 | "asn1": "^0.2.4", 19 | "cors": "^2.8.5", 20 | "dotenv": "^4.0.0", 21 | "express": "^4.17.1", 22 | "ioredis": "^3.2.2", 23 | "lodash": "^4.17.15", 24 | "morgan": "^1.10.0", 25 | "natural": "^0.6.3", 26 | "requireindex": "^1.2.0", 27 | "socket.io": "^2.3.0", 28 | "twitter": "^1.7.1", 29 | "winston": "^3.2.1" 30 | }, 31 | "devDependencies": { 32 | "babel-core": "^6.26.3", 33 | "babel-eslint": "^8.2.6", 34 | "babel-jest": "^23.6.0", 35 | "babel-plugin-transform-async-to-generator": "^6.24.1", 36 | "babel-preset-env": "^1.7.0", 37 | "babel-preset-react": "^6.24.1", 38 | "concurrently": "^3.6.1", 39 | "enzyme": "^3.11.0", 40 | "enzyme-adapter-react-16": "^1.15.2", 41 | "eslint": "^4.19.1", 42 | "eslint-config-standard": "^11.0.0", 43 | "eslint-config-standard-react": "^6.0.0", 44 | "eslint-plugin-import": "^2.20.2", 45 | "eslint-plugin-node": "^6.0.1", 46 | "eslint-plugin-promise": "^3.8.0", 47 | "eslint-plugin-react": "^7.20.0", 48 | "eslint-plugin-standard": "^3.1.0", 49 | "ioredis-mock": "^3.14.3", 50 | "jest": "^23.6.0" 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /server.js: -------------------------------------------------------------------------------- 1 | require('dotenv').config() 2 | const util = require('util') 3 | const express = require('express') 4 | const logger = require('morgan') 5 | const Twitter = require('twitter') 6 | const cors = require('cors') 7 | const socket = require('socket.io') 8 | const { Server } = require('http') 9 | const controllers = require('./controllers') 10 | const { classifyTweetWithBayes } = require('./lib/classifier') 11 | 12 | const classifyTweet = util.promisify(classifyTweetWithBayes) 13 | const app = express() 14 | const server = Server(app) 15 | const io = socket(server) 16 | 17 | // Express only serves static assets in production 18 | if (process.env.NODE_ENV === 'production') { 19 | app.use(express.static('client/build')) 20 | } 21 | 22 | app.use(cors()) 23 | app.use(logger('dev')) 24 | app.get('/api/v1/tweets', controllers.tweets.get) 25 | 26 | const client = new Twitter({ 27 | consumer_key: process.env.CONSUMER_KEY, 28 | consumer_secret: process.env.CONSUMER_SECRET, 29 | access_token_key: process.env.ACCESS_TOKEN_KEY, 30 | access_token_secret: process.env.ACCESS_TOKEN_SECRET 31 | }) 32 | 33 | const searchOptions = { 34 | track: 'we are hiring, looking for interns, vacancy, we have job available', 35 | filter_level: 'low' 36 | } 37 | 38 | const stream = client.stream('statuses/filter', searchOptions) 39 | 40 | stream.on('error', (error) => { 41 | console.error('Twitter error occoured..', error) 42 | }) 43 | 44 | io.on('connection', (socket) => { 45 | socket.on('error', (error) => { 46 | console.error('New socket Error', error) 47 | }) 48 | }) 49 | 50 | stream.on('data', async (data) => { 51 | const tweet = {} 52 | if (data.lang === 'en') { 53 | try { 54 | const classifications = await classifyTweet(data.text) 55 | tweet['id'] = data.id_str 56 | tweet['first_class'] = classifications[0] 57 | tweet['second_class'] = classifications[1] 58 | await controllers.tweets.addTweet(tweet) 59 | } catch (err) { 60 | console.error(err) 61 | } 62 | } 63 | }) 64 | 65 | server.listen(process.env.PORT || 8080, () => console.log('Server listening...')) 66 | -------------------------------------------------------------------------------- /start-client.js: -------------------------------------------------------------------------------- 1 | const args = ['start'] 2 | const opts = {stdio: 'inherit', cwd: 'client', shell: true} 3 | require('child_process').spawn('npm', args, opts) 4 | -------------------------------------------------------------------------------- /template.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Real Time Job Tweets 5 | 6 | 7 | 8 | 9 | 10 | 11 | 63 | 64 | 65 |
66 |
67 |
68 | 71 |
72 |
73 |
74 |
75 |
76 |
77 | 215 |
216 |
217 |
218 | 221 |
222 |
223 |
224 |
225 | 226 | 227 | 228 | 229 | 230 | 231 | -------------------------------------------------------------------------------- /training-data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "field": "SD", 4 | "keywords": ["iOS",".NET","App", "Angular", "VueJS" ,"Architect","Android","ActionScript","ActiveX","Algol","Algorithm","API","Applet","ASP","ASPI","Assembler","Assembly","Babel","Backend", "Back-end","C","C sharp","C++","C#","Clojure","Compiler","CSS","Dart","Django","ECMAScript","F#","Front-end","Frontend", "Full-stack", "Full stack", "Go language","Haskell","HTML","Java","JavaScript","LISP","Node.js","PHP","Python","React","React Native","Ruby","Rust","Scala","Software","engineer","SQL","Web", "Developer"] 5 | }, 6 | { 7 | "field": "SM", 8 | "keywords": ["Advertising","B2B","B2C","BANT","Branding","Brand", "Data entry","Field Sales","Junior Sales Rep", "SEO" ,"Social Seller", "Digital Marketing"] 9 | }, 10 | { 11 | "field": "BDM", 12 | "keywords":["Strategist","Product", "Manager","Quality management","Business Developer", "Business", "Operation"] 13 | }, 14 | { 15 | "field": "HC", 16 | "keywords": ["Nurse", "Doctor", "Hospital", "Health Care", "Specialist", "Therapist", "Surgical"] 17 | }, 18 | { 19 | "field": "IC", 20 | "keywords": ["Internship", "IT", "SIWES", "Intern", "Interns", "College"] 21 | }, 22 | { 23 | "field": "RS", 24 | "keywords": ["Analysis","Data Mining","Research","Questionnaire","Statistics","Researchers"] 25 | }, 26 | { 27 | "field": "HR", 28 | "keywords":["Human", "Resources", "Director", "HR"] 29 | }, 30 | { 31 | "field": "HT", 32 | "keywords": ["Lecturer", "interventionist", "University", "Trainer", "Training", "Teacher", "Teaching", "Lecturing", "Instructor", "Coach", "Course", "Courses", "Certificate"] 33 | }, 34 | { 35 | "field": "MP", 36 | "keywords": ["Manufacturing", "Plants", "Production", "Operator", "Machine"] 37 | }, 38 | { 39 | "field": "RE", 40 | "keywords": ["Agent", "Estate", "Real"] 41 | }, 42 | { 43 | "field": "AF", 44 | "keywords": ["Agriculture", "Farming", "Animal", "Haverster", "Tractor", "Plants", "Fishing"] 45 | }, 46 | { 47 | "field": "TL", 48 | "keywords": ["Driver", "Driving", "Uber", "Lyft", "Pilot", "Truck", "Delivery", "Logistics", "Transportation", "Freight Broker"] 49 | }, 50 | { 51 | "field": "AD", 52 | "keywords": ["Art", "Graphic Designer", "Illustrator", "UI", "UX", "Web Designer", "Responsive", "Design", "3D Artist", "Animator", "Caricature", "Animation", "Portraits", "Cartoon", "Photoshop"] 53 | }, 54 | { 55 | "field": "CW", 56 | "keywords": ["Writing", "Blogs", "Editor", "Photoshop", "Content Writer", "Digital Media Creation", "Photojournalist", "Journalist"] 57 | }, 58 | { 59 | "field": "SA", 60 | "keywords":["Linux", "System", "Administrator", "Systems", "Cloud Specialist", "Azure", "Development Operation", "AWS", "Azure", "Linode"] 61 | }, 62 | { 63 | "field": "MU", 64 | "keywords": ["Composer", "Musician", "Music Studio", "Producer", "Video Director", "Song Writer"] 65 | }, 66 | { 67 | "field": "OF", 68 | "keywords": ["Admin", "Executive", "Assistant", "Excel", "Receptionist", "Recruiter", "Websphere", "Office"] 69 | }, 70 | { 71 | "field": "CS", 72 | "keywords": ["Customer Service", "Care", "Associate", "Representative"] 73 | }, 74 | { 75 | "field": "RF", 76 | "keywords": ["Cook", "Restaurant", "Banquet Server", "Dishwasher", "Catering Manager", "Benihana Chef", "Kitchen Manager", "Sous", "Executive Chef", "Sushi", "Dietary","Event", "Waiter Waitress"] 77 | }, 78 | { 79 | "field": "BF", 80 | "keywords": ["Banker", "Loan Officer", "Teller", "Customer", "Loan Processor", "Mortage Originator", "Business Analyst", "Finance Manager", "Auditor", "Tax Specialist"] 81 | }, 82 | { 83 | "field": "LG", 84 | "keywords": ["Legal", "Commercial Finance Attorney", "Litigation Attorney", "Executive Assistant", "Clerk", "Paralegal"] 85 | }, 86 | { 87 | "field": "EG", 88 | "keywords": ["Electrical", "engineer", "Mechanical","Engineering", "Computer", "Physics", "Chemical", "Electronics"] 89 | } 90 | ] -------------------------------------------------------------------------------- /training.js: -------------------------------------------------------------------------------- 1 | const {BayesClassifier} = require('natural') 2 | const {readFileSync} = require('fs') 3 | const {join} = require('lodash') 4 | 5 | const classifier = new BayesClassifier() 6 | 7 | function train () { 8 | var data = readFileSync('training-data.json', 'utf-8') 9 | data = JSON.parse(data) 10 | 11 | for (let i = 0; i < data.length; i++) { 12 | // The keywords is an array but natural classifier works better with a string 13 | // so keywords are turned to string of text. 14 | let text = join(data[i].keywords, ',') 15 | classifier.addDocument(text, data[i].field) 16 | } 17 | } 18 | 19 | train() 20 | 21 | classifier.events.on('trainedWithDocument', function (obj) { 22 | // console.log(obj); 23 | }) 24 | 25 | classifier.train() 26 | 27 | classifier.save('classifier.json', (err, classifier) => { 28 | if (err) throw err 29 | }) 30 | 31 | // function classifyTweet(text){ 32 | // return new Promise((resolve, reject)=>{ 33 | // BayesClassifier.load('classifier.json', null, (err, classifier)=>{ 34 | // if(err)reject(err); 35 | // //text = removeHash(text); 36 | // let classification = classifier.classify(text); 37 | // if(classification.length > 0){ 38 | // resolve(classification); 39 | // } 40 | // resolve(null); 41 | // }); 42 | // }) 43 | // } 44 | 45 | // classifyTweet("RT @rbanks: I\'m looking for two design interns for the summer to work at @MSFTResearchCam. One will focus on UX, working on ").then((data)=>{ 46 | // console.log(data); 47 | // }).catch((err)=>{ 48 | // console.log(err); 49 | // }) 50 | --------------------------------------------------------------------------------