41 |
43 |
44 |
45 |
49 | })}
50 |
51 |
52 | );
53 | }
54 | }
55 |
56 | Scatterplot.defaultProps = {
57 | width: 400,
58 | height: 400,
59 | dotSize: 5,
60 | title: "title"
61 | };
62 |
63 |
64 | export default Scatterplot;
65 |
--------------------------------------------------------------------------------
/docs/create_model.rst:
--------------------------------------------------------------------------------
1 | Create Model
2 | ------------
3 |
4 | DDT incrementally builds a model as the user `annotates {this.props.title}
42 |
71 |
72 |
73 |
74 | );
75 | }
76 | }
77 |
78 | export default Domain;
79 |
--------------------------------------------------------------------------------
/client/src/components/Monitoring.js:
--------------------------------------------------------------------------------
1 | import React, { Component } from 'react';
2 | import {
3 | Table,
4 | TableBody,
5 | TableHeader,
6 | TableHeaderColumn,
7 | TableRow,
8 | TableRowColumn,
9 | } from 'material-ui/Table';
10 |
11 | import RaisedButton from 'material-ui/RaisedButton';
12 | import $ from 'jquery';
13 |
14 | class Monitoring extends Component {
15 |
16 | constructor(props){
17 | super(props);
18 | this.state={
19 | processes:{},
20 | }
21 | }
22 | componentWillMount(){
23 | this.setState({processes: this.props.processes,})
24 | }
25 |
26 | componentWillReceiveProps(nextProps){
27 | this.setState({processes: nextProps.processes, })
28 | }
29 |
30 | stopProcess(process, process_info){
31 | $.post(
32 | '/stopProcess',
33 | {"process": process["process"], "process_info": JSON.stringify(process_info["process_row"])},
34 | function(message) {
35 | if(message.includes("Crawler"))
36 | this.props.updateFilterCrawlerData("stopCrawler");
37 | console.log(message);
38 | }.bind(this)
39 | );
40 | }
41 |
42 | render(){
43 | var rows = Object.keys(this.state.processes).map((process, index)=>{
44 | return this.state.processes[process].map((process_row, index_row)=>{
45 | var disableStop = true;
46 | if(process_row.status === "Running" || process_row.status === "Starting" || process_row.status === "Downloading")
47 | disableStop = false;
48 | if(process_row.status !== "Completed"){
49 | return Domain: {this.props.nameDomain}
100 |Labeled data:
101 |Relevant: {this.state.currentTags["Relevant"]}
102 |Irrelevant: {this.state.currentTags["Irrelevant"]}
103 |Neutral: {this.state.currentTags["Neutral"]}
104 |
87 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
101 |
102 |
103 |
104 | )
105 | }
106 | }
107 |
108 | export default SidebarMenu;
109 |
--------------------------------------------------------------------------------
/server/server.py:
--------------------------------------------------------------------------------
1 | import cherrypy
2 | from domain_discovery_API.server import Page
3 | #import domain_discovery_api.* as dd_api
4 | from ConfigParser import ConfigParser
5 | import json
6 | import os
7 | from threading import Lock
8 | import urlparse
9 | from domain_discovery_API.models.domain_discovery_model import DomainModel
10 | from domain_discovery_API.models.crawler_model import CrawlerModel
11 | from domain_discovery_API.models.RadvizModel import RadvizModel
12 |
13 | class DDTServer(Page):
14 | @staticmethod
15 | def getConfig():
16 | # Parses file to prevent cherrypy from restarting when config.conf changes: after each request
17 | # it restarts saying config.conf changed, when it did not.
18 | config = ConfigParser()
19 | config.read(os.path.join(os.path.dirname(__file__), "config.conf"))
20 |
21 | configMap = {}
22 | for section in config.sections():
23 | configMap[section] = {}
24 | for option in config.options(section):
25 | # Handles specific integer entries.
26 | val = config.get(section, option)
27 | if option == "server.socket_port" or option == "server.thread_pool":
28 | val = int(val)
29 | configMap[section][option] = val
30 |
31 | return configMap
32 |
33 | # Default constructor reading app config file.
34 | def __init__(self):
35 | path = os.path.dirname(os.path.realpath(__file__))
36 | self._ddtModel = DomainModel(path)
37 | self._crawlerModel = CrawlerModel(path)
38 | self._ddtModel.setCrawlerModel(self._crawlerModel)
39 | self._radvizModel = RadvizModel(path)
40 | models = {"domain": self._ddtModel, "crawler": self._crawlerModel, "radviz": self._radvizModel}
41 | super(DDTServer, self).__init__(models, path)
42 |
43 | # Access to seed crawler vis.
44 | @cherrypy.expose
45 | def seedcrawler(self):
46 | # TODO Use SeedCrawlerModelAdapter self._crawler = SeedCrawlerModelAdapter()
47 | return open(os.path.join(self._HTML_DIR, u"index.html"))
48 |
49 | @cherrypy.expose
50 | def release(self):
51 | return open(os.path.join(self._HTML_DIR, u"release.html"))
52 |
53 | @cherrypy.expose
54 | def index(self):
55 | return self.seedcrawler()
56 |
57 | # Submits a web query for a list of terms, e.g. 'ebola disease'
58 | # @cherrypy.expose
59 | # def queryWeb(self, terms, session):
60 | # print "\n\n\n QUERY WEB IN REACT"
61 | # session = json.loads(session)
62 | # cherrypy.response.headers["Content-Type"] = "text/plain;"
63 | # # for res in self._model.queryWeb(terms, 20, session=session):
64 | # # print "\n\n\n SERVER QUERY WEB\n",res,"\n\n\n"
65 |
66 | # return self._model.queryWeb(terms, 20, session=session)
67 | # queryWeb._cp_config ={'response.stream':True}
68 |
69 | @cherrypy.expose
70 | def thing(self):
71 | cherrypy.response.headers['Content-Type'] = 'text/plain'
72 | # if not authorized():
73 | # raise cherrypy.NotFound()
74 | def content():
75 | print "\n\n\nRUNNING CONTENT\n\n\n"
76 | yield json.dumps({"first":"Hello"})
77 | yield json.dumps({"first":"World"})
78 | return content()
79 | thing._cp_config = {'response.stream': True}
80 |
81 | if __name__ == "__main__":
82 | server = DDTServer()
83 |
84 | # CherryPy always starts with app.root when trying to map request URIs
85 | # to objects, so we need to mount a request handler root. A request
86 | # to "/" will be mapped to HelloWorld().index().
87 | app = cherrypy.quickstart(server, config=DDTServer.getConfig())
88 | cherrypy.config.update(server.config)
89 | #app = cherrypy.tree.mount(page, "/", page.config)
90 |
91 | #if hasattr(cherrypy.engine, "signal_handler"):
92 | # cherrypy.engine.signal_handler.subscribe()
93 | #if hasattr(cherrypy.engine, "console_control_handler"):
94 | # cherrypy.engine.console_control_handler.subscribe()
95 | #cherrypy.engine.start()
96 | #cherrypy.engine.block()
97 |
98 | else:
99 | server = DDTServer()
100 | # This branch is for the test suite; you can ignore it.
101 | config = DDTServer.getConfig()
102 | app = cherrypy.tree.mount(server, config=config)
103 |
--------------------------------------------------------------------------------
/docs/visualization_radviz.rst:
--------------------------------------------------------------------------------
1 | Visualization through RadViz
2 | ----------------------------
3 |
4 | Select the **Visualization Tab** in the **Explore Data View** to see the multidimensional scaling visualization using RadViz.
5 |
6 | .. image:: figures/visualization_general.png
7 | :width: 800px
8 | :align: center
9 | :height: 400px
10 | :alt: alternate text
11 |
12 | RadViz is a data visualization that enables users to explore and analyze samples in a data set (such as a corpus of web pages in the case of DDT), represented as points in the visualization, in terms of similarity relations among semantic descriptors (keywords on the pages). Keywords are located along a circle, and pages are represented as a points in the circle. The more similar the pages the closer the distance between them. Also, the greater the proximity of a page to a keyword, the greater the frequency of occurrence of that keyword in that page. This kind of analysis allows users to identify regions of interest in the data set according to the most relevant features of the sample.
13 |
14 | Explore Pages
15 | *************
16 |
17 | In order to explore the pages in the visualization you would need to select the pages that you want to see the various details for.
18 |
19 | .. image:: figures/visualization_toolbar_lassoSelect.png
20 | :width: 400px
21 | :align: center
22 | :height: 410px
23 | :alt: alternate text
24 |
25 | Selection of a group of pages is done using lasso selection. For this, the user simply drags a freehand selection around the pages located in the circle in RadViz, similar to how you would outline something on a piece of paper with a pen or pencil. To start the lasso selection users simply have to click at the spot where you want to begin the selection, then continue holding your mouse button down and drag to draw a freeform selection outline. To complete the selection, simply release your mouse button. You don't necessarily have to return the same spot you started from, but if you don't, RadViz will automatically close the selection for you by drawing a straight line from the point where you released your mouse button to the point where you began, so in most cases, you will want to finish where you started.
26 |
27 | When the pages are selected, you will observe the following:
28 |
29 | - Keywords contained in the selected pages will be highlighted along the circle.
30 | - A WordCloud of all the top keywords contained in the selected pages is generated in the right top corner. The font size of the keyword in the word cloud is proportional to the frequency of occurence of the word
31 | - Snippets of selected pages are shown at the right bottom corner
32 |
33 | Pages can be tagged through RadViz as ’Positive’ and ’Negative’, and even Custom Tag, by drawing lasso around any region of interest, which made the selection of a sub-group of pages very easy, and then users can tag the selected pages as ’Positive’, ’Negative’ and Custom Tag.
34 |
35 | ToolBar RadViz
36 | **************
37 |
38 | .. image:: figures/visualization_toolbar1.png
39 | :width: 700px
40 | :align: center
41 | :height: 80px
42 | :alt: alternate text
43 |
44 | This visualization has five controls to interact with, whose functionlity are described below.
45 |
46 | Showing data in RadViz
47 | <<<<<<<<<<<<<<<<<<<<<<
48 |
49 | .. |toolbar_select| image:: figures/visualization_toolbar_select.png
50 |
51 | |toolbar_select| radio buttons, can be used to show or hide data on RadViz.
52 |
53 | **Show all:** Show all is selected by default in this visualization. It shows all the pages present in the data collection.
54 |
55 | **Hide selected:** This option hides the selected pages of the current view.
56 |
57 | **Hide unselected:** This option hides the pages that are not selected.
58 |
59 |
60 | Translation in RadViz
61 | <<<<<<<<<<<<<<<<<<<<<<
62 |
63 | .. |toolbar_translation| image:: figures/visualization_toolbar_transaltion.png
64 |
65 | |toolbar_translation| slider allows to calibrate the degree of denseness or sparseness of the representations of the pages in the visualization.
66 |
67 | Find Keyword in RadViz
68 | <<<<<<<<<<<<<<<<<<<<<<
69 |
70 | .. |toolbar_find_keyword| image:: figures/visualization_toolbar_find_keyword.png
71 |
72 | |toolbar_find_keyword| auto-complete text-field allows to search a keyword over all keywords in the visualization. Blue font color is used to highlight the keyword (shown below). This functionality is supported by an autocomplete process using all keyword used in th current view of RadViz.
73 |
74 | .. image:: figures/visualization_toolbar_find_keyword_result.png
75 | :width: 400px
76 | :align: center
77 | :height: 400px
78 | :alt: alternate text
79 |
--------------------------------------------------------------------------------
/client/src/components/TermsSnippetViewer.js:
--------------------------------------------------------------------------------
1 | // Filename: TermsSnippetViewer.js
2 | // Purpose: Showing snippets that belong to an specific term.
3 | // Author: Sonia Castelo (scastelo2@gmail.com)
4 |
5 |
6 | import React, { Component } from 'react';
7 | import {scaleLinear} from 'd3-scale';
8 | import {range} from 'd3-array';
9 | import {select} from 'd3-selection';
10 | import ReactFauxDom from 'react-faux-dom';
11 | import $ from "jquery";
12 | //import {select} from 'd3-selection';
13 | //import ReactFauxDom from 'react-faux-dom';
14 |
15 |
16 | class TermsSnippetViewer extends Component {
17 | constructor(props){
18 | super(props);
19 | this.state = {
20 | term: [],
21 | };
22 | this.update = this.update.bind(this);
23 | this.onLoadedTermsSnippets = this.onLoadedTermsSnippets.bind(this);
24 |
25 | //this.drawWordCloud = this.drawWordCloud.bind(this);
26 | }
27 |
28 | componentWillMount(){
29 | this.setState({term:this.props.term});
30 | this.getTermSnippets(this.props.term, this.props.session);
31 | //this.wordCloud = ReactFauxDom.createElement('div');
32 | }
33 |
34 | componentWillReceiveProps(nextProps){
35 | if(nextProps.focusContext && this.state.term == nextProps.focusTermContext){
36 | return;
37 | }
38 | this.setState({term:nextProps.term});
39 | this.getTermSnippets(nextProps.term, nextProps.session);
40 |
41 | }
42 |
43 |
44 | getTermSnippets(term, session){
45 | $.post(
46 | '/getTermSnippets',
47 | {'term': term, 'session': JSON.stringify(session)},
48 | function(data) {
49 | this.onLoadedTermsSnippets(data);
50 | }.bind(this)).fail(function() {
51 | console.log("Something wrong happen. Try again.");
52 | }.bind(this));
53 | }
54 |
55 | // Responds to loaded terms snippets.
56 | onLoadedTermsSnippets(data) {
57 | var term = data.term;
58 | var tags = data.tags;
59 | var context = data.context;
60 |
61 | var termObj = {term: term, tags: tags};
62 |
63 | var termSnippets = [];
64 | $.each(context, function(url, context){
65 | var termSnippet = {};
66 | termSnippet['term'] = termObj;
67 | termSnippet['url'] = url;
68 | termSnippet['snippet'] = context;
69 | termSnippets.push(termSnippet);
70 | });
71 |
72 | //var lazyUpdate = true;
73 | //this.termsSnippetsViewer.clear(lazyUpdate);
74 | //this.termsSnippetsViewer.addItems(termSnippets);
75 | this.update(termSnippets);
76 | };
77 |
78 | /**
79 | * Updates viewer.
80 | */
81 | update(items_) {
82 | // Removes missing items.
83 | select("#termsSnippet").selectAll('*').remove();
84 |
85 | var items = select("#termsSnippet")
86 | .attr("width", 300)
87 | .attr("height", 300)
88 | .selectAll('.item').data(items_, function(item, i) {
89 | //console.log(item.term.term + '-' + i + '-' + item.snippet.substring(0, 30));
90 | return item.term.term + '-' + i + '-' + item.snippet.substring(0, 30);
91 | });
92 |
93 | // New items.
94 | items.enter()
95 | .append('div')
96 | .classed('noselect', true)
97 | .classed('item', true)
98 | .html(function(item, i) {
99 | var snippet = (item.snippet.replace(//g, "")).replace("", "");
100 | return '' + snippet + '
'; 101 | }) 102 | .style('cursor', 'pointer') 103 | .on('click', function(item, i) { 104 | var elem = select(this); 105 | elem.classed('dblclicked', !elem.classed('dblclicked')); 106 | window.open(item.url, '_blank'); 107 | }); 108 | 109 | items.each(function(item, i) { 110 | console.log("---"); 111 | var tags = item.term['tags']; 112 | var isPositive = tags.indexOf('Positive') != -1; 113 | var isNegative = tags.indexOf('Negative') != -1; 114 | /*select(this).selectAll('em') 115 | .classed('Positive', isPositive) 116 | .classed('Negative', isNegative);*/ 117 | }); 118 | }; 119 | 120 | 121 | render() { 122 | if(this.state.term!==""){ 123 | return ( 124 |
125 |
126 | );
127 | }
128 | else {
129 | return();
130 | }
131 |
132 | }
133 | }
134 |
135 | TermsSnippetViewer.propTypes = {
136 | width: React.PropTypes.number.isRequired,
137 | height: React.PropTypes.number.isRequired,
138 | };
139 |
140 | TermsSnippetViewer.defaultProps = {
141 | width: 300,
142 | height: 300,
143 | };
144 |
145 |
146 |
147 | export default TermsSnippetViewer;
148 |
--------------------------------------------------------------------------------
/client/src/components/Filters.js:
--------------------------------------------------------------------------------
1 | import React, {Component} from 'react';
2 | import {Card, CardHeader, CardMedia} from 'material-ui/Card';
3 | import FiltersTabs from './FiltersTabs';
4 | import Avatar from 'material-ui/Avatar';
5 | import CheckList from 'material-ui/svg-icons/av/playlist-add-check';
6 |
7 | const styles = {
8 | card: {
9 | background: 'white',
10 | borderRadius: '0px 0px 0px 0px',
11 | borderStyle: 'solid',
12 | borderColor: '#C09ED7',
13 | borderWidth: '1px 0px 1px 0px'
14 |
15 | },
16 | avatar:{
17 | margin:'-4px 8px 0px 0px',
18 | },
19 | cardHeader:{
20 | background: '#DCCCE7',
21 | padding:'10px 1px 10px 6px',
22 | borderRadius: '0px 0px 0px 0px',
23 | },
24 | cardMedia:{
25 | background: '#DCCCE7',
26 | padding:'0px 4px 2px 4px',
27 | borderRadius: '0px 0px 0px 0px',
28 | height: "500px",
29 | },
30 |
31 | };
32 |
33 | class Filters extends Component{
34 |
35 | constructor(props) {
36 | super(props);
37 | this.state = {
38 | expanded: undefined,
39 | sessionString:"",
40 | session: undefined,
41 | sizeAvatar:undefined,
42 | checked_queries:[],
43 | checked_tags:[],
44 | };
45 | this.queryFromSearch=true;
46 | }
47 |
48 | componentWillMount(){
49 | this.setState({
50 | expanded: this.props.statedCard,
51 | session:this.props.session,
52 | sessionString:JSON.stringify(this.props.session),
53 | sizeAvatar:this.props.sizeAvatar,
54 | });
55 | }
56 |
57 | componentWillReceiveProps(nextProps) {
58 | this.queryFromSearch = (this.props.queryFromSearch ===undefined)?false:true;
59 | // Calculate new state
60 | if(nextProps.statedCard !== this.state.statedCard){
61 | this.setState({expanded: nextProps.statedCard}, function() {
62 | this.setState({expanded: nextProps.statedCard});
63 | });
64 | }
65 | if(JSON.stringify(nextProps.session) !== this.state.sessionString || this.queryFromSearch){
66 | this.setState({
67 | session:nextProps.session,
68 | sessionString:JSON.stringify(this.props.session),
69 | });
70 | }
71 | else{
72 | return;
73 | }
74 | }
75 |
76 | shouldComponentUpdate(nextProps, nextState) {
77 | this.queryFromSearch = (this.props.queryFromSearch ===undefined)?false:true;
78 | //console.log("filter before shouldComponentUpdate");
79 | //console.log(this.props.update);
80 | if(nextProps.updateCrawlerData==="updateCrawler" || nextProps.updateCrawlerData==="stopCrawler" || this.queryFromSearch || this.props.update || JSON.stringify(nextProps.session) !== this.state.sessionString || nextProps.statedCard !== this.state.statedCard || JSON.stringify(nextState.session) !== this.state.sessionString) {
81 | return true;
82 | }
83 | //console.log("filter after shouldComponentUpdate");
84 | return false;
85 | }
86 |
87 |
88 | handleExpandChange = (expanded) => {
89 | this.setState({expanded: expanded});
90 | if(expanded){
91 | this.props.setActiveMenu(expanded, 1);
92 | }
93 | }
94 |
95 | handleToggle = (event, toggle) => {
96 | this.setState({expanded: toggle});
97 | }
98 |
99 | handleExpand = () => {
100 | this.setState({expanded: true});
101 | }
102 |
103 | handleReduce = () => {
104 | this.setState({expanded: false});
105 | }
106 |
107 | //Send information about a query that is running (from FiltersTabs.js to Body.js)
108 | updateStatusMessage(value, term){
109 | this.props.updateStatusMessage(value, term);
110 | }
111 |
112 | updateSession(newSession){
113 | this.setState({
114 | session:newSession,
115 | sessionString:JSON.stringify(newSession),
116 | });
117 | this.props.updateSession(newSession);
118 | }
119 |
120 | deletedFilter(sessionTemp){
121 | this.setState({
122 | session:sessionTemp, sessionString: JSON.stringify(sessionTemp)
123 | });
124 | this.props.deletedFilter(sessionTemp);
125 | }
126 |
127 | /*updateCheckedQueries(checkedQueries){
128 | this.setState({
129 | checked_queries:checkedQueries,
130 | });
131 | }
132 | */
133 |
134 | render(){
135 | return(
136 |
137 |
142 | >
143 |
146 |
147 |
156 |
157 | );
158 | }
159 | }
160 |
161 | export default CrawlingView;
162 |
--------------------------------------------------------------------------------
/docs/install.rst:
--------------------------------------------------------------------------------
1 | Install and Run
2 | ===============
3 |
4 | You can install the system from source or using Docker.
5 |
6 | Docker Version
7 | --------------
8 |
9 | You must have docker installed (`Docker Installation for Mac
148 |
149 |
150 |
151 |
152 |
153 |
154 |
155 |
72 |
80 |
129 |
145 | )
146 | }
147 | }
148 |
149 | MultiselectTable.propTypes = {
150 | rows: PropTypes.array,
151 | columnHeadings: PropTypes.array,
152 | onRowSelection: PropTypes.func,
153 | resetSelection: PropTypes.bool
154 | }
155 |
156 | export default MultiselectTable;
157 |
--------------------------------------------------------------------------------
/client/src/components/Terms.js:
--------------------------------------------------------------------------------
1 | // Filename: Terms.js
2 | // Purpose: This is an intermediate component between Body.js and TermsList.js. It handles the changes in Terms card.
3 | // Author: Sonia Castelo (scastelo2@gmail.com)
4 | import React, {Component} from 'react';
5 | import {Card, CardHeader, CardMedia} from 'material-ui/Card';
6 | import TermsList from './TermsList';
7 | import Avatar from 'material-ui/Avatar';
8 | import Assignment from 'material-ui/svg-icons/action/assignment-returned';
9 | import Divider from 'material-ui/Divider';
10 | import $ from 'jquery';
11 | import CircularProgress from 'material-ui/CircularProgress';
12 |
13 |
14 | class CircularProgressSimple extends React.Component{
15 | render(){
16 | return(
17 |
130 | ...}
135 | breakClassName={"break-me"}
136 | pageCount={(this.props.rows || []).length/this.perPage}
137 | marginPagesDisplayed={1}
138 | pageRangeDisplayed={1}
139 | onPageChange={(page) => {this.setState({currentPage: page.selected})}}
140 | containerClassName={"pagination"}
141 | subContainerClassName={"pages pagination"}
142 | activeClassName={"active"} />
143 |
144 |
18 |
20 | );}
21 | }
22 |
23 |
24 | class Terms extends Component{
25 |
26 | constructor(props) {
27 | super(props);
28 | this.state = {
29 | expanded: this.props.statedCard,
30 | update:true,
31 | listTerms: [],
32 | session:{},
33 | sessionString:"",
34 | fromCrawling:false,
35 | };
36 | };
37 |
38 | componentWillMount = () => {
39 | this.setState({expanded: this.props.statedCard, session:this.props.session, sessionString:JSON.stringify(this.props.session) , fromCrawling:this.props.fromCrawling,});
40 | this.loadTerms();
41 | };
42 |
43 |
44 | //Handling state's changes of search card. (expanded or reduced)
45 | componentWillReceiveProps = (nextProps) => {
46 | // Calculate new state
47 | if(nextProps.statedCard !== this.state.statedCard){
48 | this.setState({expanded: nextProps.statedCard}, function() {
49 | this.setState({expanded: nextProps.statedCard});
50 | });
51 | }
52 |
53 | if(JSON.stringify(nextProps.session) !== this.state.sessionString && nextProps.statedCard){
54 | this.setState({
55 | session:nextProps.session,
56 | sessionString:JSON.stringify(this.props.session),
57 | listTerms: [],
58 | });
59 | this.loadTerms();
60 | }
61 | else{
62 | return;
63 | }
64 |
65 | };
66 |
67 | handleExpandChange = (expanded) => {
68 | this.setState({expanded: expanded});
69 | if(expanded){
70 | this.props.setActiveMenu(expanded, 3);
71 |
72 | }
73 | };
74 |
75 | handleReduce = () => {
76 | this.setState({expanded: false});
77 | };
78 |
79 |
80 | loadTerms(){
81 | var session = this.props.session;
82 | $.post(
83 | '/extractTerms',
84 | {'numberOfTerms': 40, 'session': JSON.stringify(session)},
85 | function(summary) {
86 | var entries = [];
87 | entries = summary.map(function(w) {
88 | return {'word': w[0], 'posFreq': w[1], 'negFreq': w[2], 'tags': w[3]}
89 | });
90 | this.setState({listTerms: entries});
91 | this.updateTerms(entries); //sending new terms to FocusedCrawling component
92 | }.bind(this)).fail(function() {
93 | console.log("Something wrong happen. Try again.");
94 | }.bind(this));
95 | };
96 |
97 | //sending new terms to FocusedCrawling component
98 | updateTerms(updateListTerm){
99 | if(this.props.updateTerms != undefined)
100 | this.props.updateTerms(updateListTerm);
101 | }
102 |
103 | updateListTermParent(updateListTerm){
104 | this.setState({listTerms: updateListTerm});
105 | this.updateTerms(updateListTerm); //sending new terms to FocusedCrawling component
106 | this.forceUpdate();
107 | }
108 |
109 | //Check if the component should be updated or not
110 | shouldComponentUpdate(nextProps, nextState) {
111 | if(JSON.stringify(nextProps.session) !== this.state.sessionString || nextProps.statedCard !== this.state.statedCard || JSON.stringify(nextState.session) !== this.state.sessionString || this.props.focusedCrawlDomains) {
112 | return true;
113 | }
114 | return false;
115 | }
116 |
117 |
118 | render(){
119 |
120 | const styles = {
121 | card: {
122 | borderStyle: 'solid',
123 | borderColor: '#C09ED7',
124 | background: 'white',
125 | borderRadius: '0px 0px 0px 0px',
126 | borderWidth: '0px 0px 0px 0px'
127 | },
128 | avatar:{
129 | margin:'-4px 8px 0px 0px',
130 | },
131 | cardHeader:{
132 | background: this.props.BackgroundColorTerm, //'#DCCCE7',
133 | padding:'10px 1px 10px 6px',
134 | borderRadius: '0px 0px 0px 0px',
135 | },
136 | cardMedia:{
137 | background: this.props.BackgroundColorTerm,
138 | padding:'2px 4px 2px 4px',
139 | borderRadius: '0px 0px 0px 0px',
140 | height: "390px",
141 | },
142 |
143 | };
144 |
145 | let terms = " ";
146 | if(this.state.listTerms.length>0){
147 | terms = this.state.listTerms.map(function(w) {
148 | return {w["word"]}
; 149 | }); 150 | } 151 | var isThereTerms = (this.state.listTerms.length>0)?
167 | {isThereTerms}
168 |
169 |
172 |
174 | );
175 | }
176 | else {
177 | return(
178 |
179 |
180 | );
181 | }
182 | }
183 | }
184 |
185 | export default RadViz;
186 |
--------------------------------------------------------------------------------
/client/src/components/Home.js:
--------------------------------------------------------------------------------
1 | import React, { Component } from 'react';
2 | var ReactRouter = require('react-router');
3 | var Link = ReactRouter.Link;
4 |
5 | import Checkbox from 'material-ui/Checkbox';
6 | import {List, ListItem} from 'material-ui/List';
7 | import Subheader from 'material-ui/Subheader';
8 | import { Row, Col} from 'react-bootstrap';
9 | //import {Toolbar, ToolbarGroup, ToolbarSeparator, ToolbarTitle} from 'material-ui/Toolbar';
10 |
11 | import FlatButton from 'material-ui/FlatButton';
12 | import Forward from 'material-ui/svg-icons/content/forward';
13 | import AddBox from 'material-ui/svg-icons/content/add-box';
14 | import DeleteForever from 'material-ui/svg-icons/action/delete-forever';
15 | import {fullWhite} from 'material-ui/styles/colors';
16 | import $ from 'jquery';
17 |
18 | import AppBar from 'material-ui/AppBar';
19 | import logoNYU from '../images/nyu_logo_purple.png';
20 | import Dialog from 'material-ui/Dialog';
21 | import TextField from 'material-ui/TextField';
22 | import RaisedButton from 'material-ui/RaisedButton';
23 |
24 | const styles = {
25 | listDomains:{
26 | borderStyle: 'solid',
27 | borderColor: '#C09ED7',
28 | background: 'white',
29 | borderRadius: '0px 0px 0px 0px',
30 | borderWidth: '0px 0px 1px 0px',
31 | },
32 | };
33 |
34 |
35 | class Home extends Component {
36 | constructor(props){
37 | super(props);
38 | this.state = {
39 | domains: undefined,
40 | openCreateDomain: false,
41 | openDeleteDomain: false,
42 | openDuplicateDomainName:false,
43 | newNameDomain:"",
44 | delDomains: {}
45 | };
46 | this.focusTextField = this.focusTextField.bind(this);
47 | this.textInput = null;
48 | }
49 |
50 | getAvailableDomains(){
51 | $.post(
52 | '/getAvailableDomains',
53 | {"type": "init"},
54 | function(domains) {
55 | this.setState({domains: domains['crawlers']});
56 | }.bind(this)
57 | );
58 | }
59 | componentWillMount() {
60 | //Get domains.
61 | this.getAvailableDomains();
62 | }
63 |
64 | handleOpenCreateDomain = () => {
65 | this.setState({openCreateDomain: true});
66 | this.focusTextField();
67 | };
68 |
69 | handleCloseCreateDomain = () => {
70 | this.setState({openCreateDomain: false, openDuplicateDomainName:false, newNameDomain:"" });
71 | };
72 |
73 | handleOpenDeleteDomain = () => {
74 | this.setState({openDeleteDomain: true});
75 | };
76 |
77 | handleCloseDeleteDomain = () => {
78 | this.setState({openDeleteDomain: false});
79 | };
80 |
81 | handleCloseDuplicateDomainName = () => {
82 | this.setState({openDuplicateDomainName: false});
83 | };
84 |
85 | //Handling changes into TextField newNameDomain (updating TextField).
86 | handleTextChangeNewNameDomain(e){
87 | this.setState({ "newNameDomain": e.target.value});
88 | }
89 |
90 | // Explicitly focus the text input using the raw DOM API
91 | focusTextField() {
92 | setTimeout(() => this.textInput.focus(), 100);
93 | }
94 |
95 | //Create a new domain
96 | createNewDomain(){
97 | //createNewDomain
98 | var nameDomain= this.state.newNameDomain;
99 | var duplicateDomain = false;
100 | var mydata = this.state.domains;
101 | Object.keys(mydata).map((k, index)=>{ var name = mydata[k].name; if(name.trim().toLowerCase().replace(/\s+/g,"_") === nameDomain.trim().toLowerCase().replace(/\s+/g,"_")){ duplicateDomain = true;} }); // .trim() to remove last and first spaces from a string.
102 | if(!duplicateDomain){
103 | $.post(
104 | '/addDomain',
105 | {'index_name': nameDomain},
106 | function(domains) {
107 | this.setState({openCreateDomain: false, newNameDomain:"", openDuplicateDomainName:false });
108 | this.getAvailableDomains();
109 | this.forceUpdate();
110 | }.bind(this)
111 | );
112 | }
113 | else{
114 | this.setState({openCreateDomain: true, newNameDomain:nameDomain, openDuplicateDomainName:true});
115 | }
116 | };
117 |
118 | //Delete selected domains
119 | deleteDomains(){
120 | var delDomains= this.state.delDomains;
121 | $.post(
122 | '/delDomain',
123 | {'domains': JSON.stringify(delDomains)},
124 | function(domains) {
125 | this.setState({openDeleteDomain: false, delDomains: {}});
126 | this.getAvailableDomains();
127 | this.forceUpdate();
128 | }.bind(this)
129 | );
130 | };
131 |
132 | // Get all the domains selected for deletion
133 | addDelDomains(id,name){
134 | var tempDelDomains = this.state.delDomains;
135 | tempDelDomains[id]=name;
136 | this.setState({delDomains:tempDelDomains});
137 | }
138 |
139 | render(){
140 |
141 | const actionsCreateDomain = [
142 |
181 | Domain Discovery Tool }
184 | //iconElementLeft={
}
186 | //onLeftIconButtonTouchTap={this.removeRecord.bind(this)}
187 | >
188 |
189 |
273 | );
274 | }
275 | return(
276 |
277 | );
278 | }
279 | }
280 |
281 | Home.defaultProps = {
282 | backgroundColor:"#9A7BB0",
283 | };
284 |
285 | export default Home;
286 |
--------------------------------------------------------------------------------
/client/src/components/Body.js:
--------------------------------------------------------------------------------
1 | // Filename: Body.js
2 | // Purpose: Contain filters and view components. If there is some change into filter or view components, body.js should be updated.
3 | // Author: Sonia Castelo (scastelo2@gmail.com)
4 | import React, {Component} from 'react';
5 | import { Row, Col} from 'react-bootstrap';
6 | import DomainInfo from './DomainInfo';
7 | import Search from './Search';
8 | import Filters from './Filters';
9 | import Terms from './Terms';
10 | import Views from './Views';
11 | import CrawlingView from './CrawlingView';
12 | import '../css/Components.css';
13 | import 'react-checkbox-tree/lib/react-checkbox-tree.css';
14 | import 'react-select/dist/react-select.css';
15 | import Sidebar from 'react-sidebar';
16 | import Plus from 'material-ui/svg-icons/action/swap-horiz';
17 | import FloatingActionButton from 'material-ui/FloatingActionButton';
18 | import Snackbar from 'material-ui/Snackbar'; //Adding an indicator which tell us that the pages are being downloaded
19 |
20 | import {Card, CardActions, CardHeader, CardText, CardMedia} from 'material-ui/Card';
21 | import {List, ListItem} from 'material-ui/List';
22 | import Subheader from 'material-ui/Subheader';
23 |
24 | const styles = {
25 | button:{
26 | //hoverColor:"#9c9ca4"
27 | marginTop:20,
28 | paddingBottom:'-145px',
29 | marginBottom:'-545px',
30 | marginRight: 5,
31 | },
32 | contentHeaderMenuLink: {
33 | textDecoration: 'none',
34 | color: 'white',
35 | padding: 8,
36 | },
37 | content: {
38 | marginTop: '68px',
39 | marginRight: '5px',
40 | marginBottom: '8px',
41 | marginLeft: '5px',
42 | backgroundColor: '#FFFFFF',
43 | borderRadius: '10px 10px 10px 10px',
44 | },
45 | avatar:{
46 | margin:'-4px 8px 0px 0px',
47 | },
48 |
49 | //
50 | card: {
51 |
52 | borderStyle: 'solid',
53 | borderColor: '#C09ED7',
54 | background: 'white',
55 | borderRadius: '0px 0px 0px 0px',
56 | borderWidth: '0px 0px 1px 0px'
57 | },
58 | cardHeader:{
59 | background: '#DCCCE7',
60 | padding:'10px 1px 10px 6px',
61 | borderRadius: '0px 0px 0px 0px',
62 | },
63 | cardMedia:{
64 | background: '#DCCCE7',
65 | padding:'2px 4px 2px 4px',
66 | borderRadius: '0px 0px 0px 0px',
67 | height: "200px",
68 | },
69 | };
70 |
71 |
72 |
73 | class Body extends Component{
74 |
75 | constructor(props) {
76 | super(props);
77 | this.state = {
78 | docked: true,
79 | open: true,
80 | transitions: true,
81 | touch: true,
82 | shadow: true,
83 | pullRight: false,
84 | touchHandleWidth: 20,
85 | dragToggleDistance: 30,
86 | size:350,
87 | iconDomainInfo:null,
88 | stateDomainInfoCard:false,
89 | stateSearchCard:false,
90 | offset:0,
91 | currentPagination:0,
92 | stateFiltersCard:false,
93 | stateTermsCard:false,
94 | sizeAvatar:25,
95 | currentDomain:'',
96 | sessionBody:{},
97 | sessionString:"",
98 | pages:{},
99 | update:false,
100 | runCurrentQuery: "*",
101 | intervalFuncId:undefined,
102 | stopApplyQueryOverView:false, //Allow interactions with data (applying filters, tagging, etc) while multiple searches are running.
103 | };
104 | this.sessionB={};
105 | }
106 |
107 | /*consultaQueries: {"search_engine":"GOOG","activeProjectionAlg":"Group by Correlation"
108 | ,"domainId":"AVWjx7ciIf40cqEj1ACn","pagesCap":"100","fromDate":null,"toDate":null,
109 | "filter":null,"pageRetrievalCriteria":"Most Recent","selected_morelike":"",
110 | "model":{"positive":"Relevant","nagative":"Irrelevant"}}*/
111 | createSession(domainId){
112 | var session = {};
113 | session['search_engine'] = "GOOG";
114 | session['activeProjectionAlg'] = "Group by Correlation";
115 | session['domainId'] = domainId;
116 | session['pagesCap'] = "100";
117 | session['fromDate'] = null;
118 | session['toDate'] = null;
119 | session['filter'] = null; //null
120 | session['pageRetrievalCriteria'] = "Most Recent";
121 | session['selected_morelike'] = "";
122 | session['selected_queries']="";
123 | session['selected_tlds']="";
124 | session['selected_aterms']="";
125 | session['selected_tags']="";
126 | session['selected_model_tags']="";
127 | session['selected_crawled_tags']="";
128 | session['model'] = {};
129 | session['model']['positive'] = "Relevant";
130 | session['model']['negative'] = "Irrelevant";
131 |
132 |
133 |
134 | return session;
135 | }
136 |
137 | //Get queries, tags, urls from a speficic domain.
138 | componentWillMount() {
139 | this.setState({currentDomain: this.props.currentDomain, sessionBody: this.createSession(this.props.currentDomain), sessionString: JSON.stringify(this.createSession(this.props.currentDomain)) });
140 | }
141 |
142 | //handling update of props (ex. filters, session, etc)
143 | componentWillReceiveProps = (newProps) => {
144 | if(newProps.reloadBody){
145 | let sessionTemp = this.state.sessionBody;
146 | sessionTemp['filter']= (newProps.filterKeyword === '')?null:newProps.filterKeyword;
147 | this.setState({sessionBody: sessionTemp, sessionString: JSON.stringify(sessionTemp)});
148 | }
149 | if(newProps.currentDomain === this.state.currentDomain){
150 | return;
151 | }
152 |
153 | this.setState({currentDomain: this.props.currentDomain});
154 |
155 | };
156 |
157 | //Verify if it is necessary an update.
158 | shouldComponentUpdate(nextProps, nextState) {
159 | if (nextState.sessionString === this.state.sessionString) {
160 | if(nextProps.updateCrawlerData=="updateCrawler" || nextProps.updateCrawlerData=="stopCrawler" || nextProps.filterKeyword !== null || nextProps.filterKeyword !== "" || nextState.stateDomainInfoCard!==this.state.stateDomainInfoCard || nextState.stateSearchCard!==this.state.stateSearchCard || nextState.stateTermsCard!==this.state.stateTermsCard || nextState.stateFiltersCard!==this.state.stateFiltersCard){
161 | return true;
162 | }
163 | return false;
164 | }
165 | return true;
166 | }
167 |
168 | //Handling menus of DomainInfo, Search, and Filter Cards.
169 | closeMenu(){
170 | this.setState({
171 | size: 60,
172 | //iconDomainInfo:
190 |
272 |
191 |
192 |
270 |
271 | -
194 |
Domains
299 |
301 |
302 |
303 |
304 |
305 |
306 |
307 |
308 |
309 |
310 |
311 |
312 |
313 |
314 |
316 |
317 |
318 |
319 | );
320 |
321 | const sidebarProps = {
322 | sidebar: sidebar,
323 | docked: this.state.docked,
324 | sidebarClassName: 'custom-sidebar-class',
325 | open: this.state.open,
326 | touch: this.state.touch,
327 | shadow: this.state.shadow,
328 | pullRight: this.state.pullRight,
329 | touchHandleWidth: this.state.touchHandleWidth,
330 | dragToggleDistance: this.state.dragToggleDistance,
331 | transitions: this.state.transitions,
332 | onSetOpen: this.onSetOpen,
333 | };
334 |
335 | return (
336 |
338 |
339 |
340 |
341 |
342 |
354 |
356 | )
357 | }
358 |
359 |
360 | }
361 | }
362 |
363 |
364 |
365 | export default Body;
366 |
--------------------------------------------------------------------------------