├── .gitignore ├── LICENSE ├── README.md ├── app-ui ├── .babelrc ├── .gitignore ├── .npmrc ├── package-lock.json ├── package.json ├── public │ ├── bundle.js.LICENSE.txt │ └── bundle.js.map ├── src │ ├── app.js │ ├── components │ │ ├── NotFoundPage.js │ │ ├── SideBar.js │ │ ├── constants.js │ │ ├── file-upload │ │ │ ├── FileUploadMain.js │ │ │ ├── MultipleFileUploadForm.js │ │ │ ├── SingleFileUploadForm.js │ │ │ └── UploadFileButton.js │ │ ├── highlightable │ │ │ ├── Highlightable.js │ │ │ ├── Range.js │ │ │ ├── helpers.js │ │ │ └── nodes │ │ │ │ ├── EmojiNode.js │ │ │ │ ├── Node.js │ │ │ │ └── UrlNode.js │ │ ├── label-page │ │ │ ├── LabelPage.js │ │ │ ├── common │ │ │ │ ├── LabelNavigationSimple.js │ │ │ │ └── TextNavigation.js │ │ │ ├── entity-disambiguation │ │ │ │ ├── EDLabelPage.js │ │ │ │ ├── KbSearch.js │ │ │ │ ├── LabelNavigation.js │ │ │ │ ├── Navigation.js │ │ │ │ └── Text.js │ │ │ └── supervised │ │ │ │ ├── SupervisedLabelPage.js │ │ │ │ ├── classification │ │ │ │ ├── LabelNavigation.js │ │ │ │ ├── MainArea.js │ │ │ │ └── Navigation.js │ │ │ │ └── ner │ │ │ │ ├── HighlightableTable.js │ │ │ │ ├── HighlightableText.js │ │ │ │ ├── LabelNavigation.js │ │ │ │ ├── MainArea.js │ │ │ │ └── Navigation.js │ │ ├── project-summary │ │ │ ├── ProjectMain.js │ │ │ ├── ProjectTitle.js │ │ │ ├── common │ │ │ │ ├── DeleteProjectButton.js │ │ │ │ ├── DownloadUnlabeledDataButton.js │ │ │ │ └── FileDownloadButton.js │ │ │ ├── entity-disambiguation │ │ │ │ └── EntityDisambiguationProjectMain.js │ │ │ └── supervised │ │ │ │ ├── ExportRulesButton.js │ │ │ │ ├── LabelTable.js │ │ │ │ ├── PerformanceTable.js │ │ │ │ ├── ProjectDataDrawers.js │ │ │ │ ├── RuleTable.js │ │ │ │ ├── SupervisedProjectMain.js │ │ │ │ ├── classification │ │ │ │ ├── LabelTableClassification.js │ │ │ │ └── RuleTableClassification.js │ │ │ │ └── ner │ │ │ │ ├── LabelTableNER.js │ │ │ │ └── RuleTableNER.js │ │ ├── projects │ │ │ └── Projects.js │ │ ├── rules │ │ │ ├── main-page │ │ │ │ ├── ImportRuleCard.js │ │ │ │ └── RuleMain.js │ │ │ └── rule-forms │ │ │ │ ├── base │ │ │ │ ├── CaseSensitive.js │ │ │ │ ├── ClassDefinitionBox.js │ │ │ │ ├── ContainOrDoesNotContain.js │ │ │ │ ├── GenericOrderedRule.js │ │ │ │ ├── InputField.js │ │ │ │ ├── OptionSelection.js │ │ │ │ ├── RuleSubmit.js │ │ │ │ ├── RuleSubmitButton.js │ │ │ │ ├── SentenceOrFullText.js │ │ │ │ └── TypeSelector.js │ │ │ │ ├── classification-rules │ │ │ │ ├── NonOrderedRule.js │ │ │ │ ├── OrderedRule.js │ │ │ │ └── SentimentRule.js │ │ │ │ └── ner-rules │ │ │ │ ├── NounPhraseRule.js │ │ │ │ └── RegexRule.js │ │ ├── search │ │ │ ├── LabelComponent.js │ │ │ ├── Result.js │ │ │ ├── ResultwithLabels.js │ │ │ ├── RuleFilters.js │ │ │ ├── Search.js │ │ │ ├── buildRequest.js │ │ │ ├── buildState.js │ │ │ ├── getSearchTextComponent.js │ │ │ └── runRequest.js │ │ ├── set-project-params │ │ │ ├── ClassNames.js │ │ │ └── ProjectParamsPage.js │ │ ├── utils.js │ │ └── welcome-page │ │ │ └── WelcomePage.js │ ├── routers │ │ ├── AppRouter.js │ │ └── ProjectStartPage.js │ ├── styles │ │ └── theme.js │ └── utils.js ├── state-changes.txt ├── webpack.common.js ├── webpack.dev.js ├── webpack.prod.js └── yarn.lock ├── docs ├── Makefile ├── make.bat └── source │ ├── _static │ └── style.css │ ├── _templates │ └── layout.html │ ├── architecture.jpg │ ├── conf.py │ ├── format.json │ ├── howitworks.rst │ ├── index.rst │ ├── installation.md │ └── overview.md ├── pics ├── 0_newproj.png ├── 1_select_label.png ├── 2_anno_span.png ├── 3_save_next.png ├── 4_fin_prox.png ├── download.png └── full-process-color.png ├── requirements.txt ├── setup.py └── src └── famie ├── __init__.py ├── api ├── __init__.py ├── active_learning │ ├── __init__.py │ ├── config.py │ ├── constants.py │ ├── controllers.py │ ├── models.py │ ├── passed_args.json │ ├── trainers.py │ └── utils.py ├── api_fns │ ├── __init__.py │ ├── project_creation │ │ ├── __init__.py │ │ ├── common.py │ │ └── supervised.py │ ├── project_settings │ │ ├── __init__.py │ │ └── supervised.py │ └── utils.py ├── blueprints │ ├── __init__.py │ ├── common.py │ └── supervised.py ├── static │ ├── bundle.js │ ├── bundle.js.LICENSE.txt │ ├── bundle.js.map │ └── index.html └── templates │ └── index.html ├── config ├── __init__.py ├── common_config.ini └── config_reader.py ├── constants.py ├── entry_points ├── __init__.py └── run_app.py └── scripts ├── __init__.py ├── start_app.py └── uninstall_app.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | .idea/ 3 | __pycache__/ 4 | *.py[cod] 5 | *$py.class 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .Python 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | pip-wheel-metadata/ 25 | share/python-wheels/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | MANIFEST 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .nox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | *.py,cover 52 | .hypothesis/ 53 | .pytest_cache/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | db.sqlite3 63 | db.sqlite3-journal 64 | 65 | # Flask stuff: 66 | instance/ 67 | .webassets-cache 68 | 69 | # Scrapy stuff: 70 | .scrapy 71 | 72 | # Sphinx documentation 73 | docs/_build/ 74 | 75 | # PyBuilder 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | .python-version 87 | 88 | # pipenv 89 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 90 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 91 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 92 | # install all needed dependencies. 93 | #Pipfile.lock 94 | 95 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 96 | __pypackages__/ 97 | 98 | # Celery stuff 99 | celerybeat-schedule 100 | celerybeat.pid 101 | 102 | # SageMath parsed files 103 | *.sage.py 104 | 105 | # Environments 106 | .env 107 | .venv 108 | env/ 109 | venv/ 110 | ENV/ 111 | env.bak/ 112 | venv.bak/ 113 | 114 | # Spyder project settings 115 | .spyderproject 116 | .spyproject 117 | 118 | # Rope project settings 119 | .ropeproject 120 | 121 | # mkdocs documentation 122 | /site 123 | 124 | # mypy 125 | .mypy_cache/ 126 | .dmypy.json 127 | dmypy.json 128 | 129 | # Pyre type checker 130 | .pyre/ 131 | src/famie/api/active_learning/database/ 132 | src/famie/api/active_learning/logs/ 133 | src/famie/api/active_learning/passed_args.json 134 | src/famie/api/active_learning/signals/ 135 | .vscode/ 136 | -------------------------------------------------------------------------------- /app-ui/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | [ 4 | "@babel/preset-env", 5 | { 6 | "targets": { 7 | "node": "10" 8 | } 9 | } 10 | ], 11 | "@babel/react" 12 | ], 13 | "plugins": [ 14 | "transform-class-properties", 15 | "emotion", 16 | "recharts", 17 | "lodash" 18 | ] 19 | } -------------------------------------------------------------------------------- /app-ui/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | bundle.js -------------------------------------------------------------------------------- /app-ui/.npmrc: -------------------------------------------------------------------------------- 1 | engine-strict=true -------------------------------------------------------------------------------- /app-ui/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "labelling-app", 3 | "version": "1.0.0", 4 | "main": "index.js", 5 | "license": "MIT", 6 | "scripts": { 7 | "build": "webpack --config ./webpack.prod.js && cp public/bundle.js ../src/famie/api/static/", 8 | "dev-server": "webpack serve --config ./webpack.dev.js" 9 | }, 10 | "dependencies": { 11 | "@babel/core": "7.16.0", 12 | "@babel/preset-env": "7.16.0", 13 | "@babel/preset-react": "7.16.0", 14 | "@elastic/react-search-ui": "1.7.0", 15 | "@elastic/react-search-ui-views": "1.7.0", 16 | "@material-ui/core": "4.11.2", 17 | "@material-ui/icons": "4.9.1", 18 | "@material-ui/lab": "4.0.0-alpha.57", 19 | "babel-cli": "6.24.1", 20 | "babel-loader": "8.2.3", 21 | "babel-plugin-lodash": "3.3.4", 22 | "babel-plugin-recharts": "1.2.1", 23 | "babel-plugin-transform-class-properties": "6.24.1", 24 | "css-loader": "6.5.1", 25 | "emoji-regex": "^6.5.1", 26 | "jquery": "^3.4.1", 27 | "lodash": "4.17.20", 28 | "node-sass": "6.0.1", 29 | "normalize.css": "7.0.0", 30 | "papaparse": "5.3.1", 31 | "query-string": "6.12.0", 32 | "react": "16.8.0", 33 | "react-dom": "16.8.0", 34 | "react-router-dom": "5.1.2", 35 | "react-spinners": "^0.8.1", 36 | "react-uuid": "1.0.2", 37 | "recharts": "1.8.5", 38 | "recompose": "0.30.0", 39 | "sass-loader": "12.3.0", 40 | "style-loader": "3.3.1", 41 | "webpack": "5.62.1", 42 | "webpack-dev-server": "4.4.0", 43 | "webpack-merge": "5.8.0", 44 | "yargs": "^17.4.0" 45 | }, 46 | "devDependencies": { 47 | "webpack-cli": "^4.9.1" 48 | }, 49 | "engines": { 50 | "node": "16.13.0", 51 | "npm": "8.1.0" 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /app-ui/public/bundle.js.LICENSE.txt: -------------------------------------------------------------------------------- 1 | /* 2 | object-assign 3 | (c) Sindre Sorhus 4 | @license MIT 5 | */ 6 | 7 | /* @license 8 | Papa Parse 9 | v5.3.1 10 | https://github.com/mholt/PapaParse 11 | License: MIT 12 | */ 13 | 14 | /*! 15 | Copyright (c) 2018 Jed Watson. 16 | Licensed under the MIT License (MIT), see 17 | http://jedwatson.github.io/classnames 18 | */ 19 | 20 | /*! 21 | * Sizzle CSS Selector Engine v2.3.6 22 | * https://sizzlejs.com/ 23 | * 24 | * Copyright JS Foundation and other contributors 25 | * Released under the MIT license 26 | * https://js.foundation/ 27 | * 28 | * Date: 2021-02-16 29 | */ 30 | 31 | /*! 32 | * jQuery JavaScript Library v3.6.0 33 | * https://jquery.com/ 34 | * 35 | * Includes Sizzle.js 36 | * https://sizzlejs.com/ 37 | * 38 | * Copyright OpenJS Foundation and other contributors 39 | * Released under the MIT license 40 | * https://jquery.org/license 41 | * 42 | * Date: 2021-03-02T17:08Z 43 | */ 44 | 45 | /*! decimal.js-light v2.5.1 https://github.com/MikeMcl/decimal.js-light/LICENCE */ 46 | 47 | /** 48 | * A better abstraction over CSS. 49 | * 50 | * @copyright Oleg Isonen (Slobodskoi) / Isonen 2014-present 51 | * @website https://github.com/cssinjs/jss 52 | * @license MIT 53 | */ 54 | 55 | /** @license React v0.13.6 56 | * scheduler.production.min.js 57 | * 58 | * Copyright (c) Facebook, Inc. and its affiliates. 59 | * 60 | * This source code is licensed under the MIT license found in the 61 | * LICENSE file in the root directory of this source tree. 62 | */ 63 | 64 | /** @license React v16.13.1 65 | * react-is.production.min.js 66 | * 67 | * Copyright (c) Facebook, Inc. and its affiliates. 68 | * 69 | * This source code is licensed under the MIT license found in the 70 | * LICENSE file in the root directory of this source tree. 71 | */ 72 | 73 | /** @license React v16.8.0 74 | * react-dom.production.min.js 75 | * 76 | * Copyright (c) Facebook, Inc. and its affiliates. 77 | * 78 | * This source code is licensed under the MIT license found in the 79 | * LICENSE file in the root directory of this source tree. 80 | */ 81 | 82 | /** @license React v16.8.0 83 | * react.production.min.js 84 | * 85 | * Copyright (c) Facebook, Inc. and its affiliates. 86 | * 87 | * This source code is licensed under the MIT license found in the 88 | * LICENSE file in the root directory of this source tree. 89 | */ 90 | 91 | /**! 92 | * @fileOverview Kickass library to create and place poppers near their reference elements. 93 | * @version 1.16.1-lts 94 | * @license 95 | * Copyright (c) 2016 Federico Zivolo and contributors 96 | * 97 | * Permission is hereby granted, free of charge, to any person obtaining a copy 98 | * of this software and associated documentation files (the "Software"), to deal 99 | * in the Software without restriction, including without limitation the rights 100 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 101 | * copies of the Software, and to permit persons to whom the Software is 102 | * furnished to do so, subject to the following conditions: 103 | * 104 | * The above copyright notice and this permission notice shall be included in all 105 | * copies or substantial portions of the Software. 106 | * 107 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 108 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 109 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 110 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 111 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 112 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 113 | * SOFTWARE. 114 | */ 115 | -------------------------------------------------------------------------------- /app-ui/src/app.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import AppRouter from './routers/AppRouter'; 4 | import 'normalize.css/normalize.css'; 5 | // import './styles/styles.scss'; 6 | // import theme from './styles/theme'; 7 | import CssBaseline from '@material-ui/core/CssBaseline'; 8 | import { ThemeProvider } from '@material-ui/core/styles'; 9 | 10 | // import Router from './playground/Router'; 11 | // import AwesomeComponent from './playground/SpinningWheel'; 12 | 13 | // import FileDownload from './components/project-summary/FileDownloadButton'; 14 | import Search from './components/search/Search'; 15 | import WelcomePage from './components/welcome-page/WelcomePage'; 16 | import RegexRule from './components/rules/rule-forms/ner-rules/RegexRule'; 17 | import NounChunkRule from './components/rules/rule-forms/ner-rules/NounPhraseRule'; 18 | 19 | import ProjectParamsPage from './components/set-project-params/ProjectParamsPage'; 20 | 21 | 22 | ReactDOM.render( 23 | 24 | 25 | 26 | , 27 | document.getElementById('app'), 28 | ); 29 | 30 | // ReactDOM.render( console.log(x)} 34 | // />, document.getElementById('app')) 35 | 36 | 37 | // ReactDOM.render(, document.getElementById('app')); 38 | 39 | // ReactDOM.render(, document.getElementById('app')) 40 | 41 | // ReactDOM.render(, document.getElementById('app')) 42 | // ReactDOM.render(, document.getElementById('app')) 43 | 44 | // ReactDOM.render(, document.getElementById('app')); 45 | // ReactDOM.render(, document.getElementById('app')); 46 | // ReactDOM.render(, document.getElementById('app')); 47 | // ReactDOM.render(, document.getElementById('app')); 48 | 49 | // ReactDOM.render(, 53 | // document.getElementById('app')); 54 | 55 | // ReactDOM.render(, document.getElementById('app')); 56 | // ReactDOM.render(, document.getElementById('app')); 57 | -------------------------------------------------------------------------------- /app-ui/src/components/NotFoundPage.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Link } from 'react-router-dom'; 3 | 4 | const NotFoundPage = () => ( 5 |
6 | 404! - Go home 7 |
8 | ); 9 | 10 | export default NotFoundPage; -------------------------------------------------------------------------------- /app-ui/src/components/SideBar.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Link } from 'react-router-dom'; 3 | import Drawer from '@material-ui/core/Drawer'; 4 | import List from '@material-ui/core/List'; 5 | import ListItem from '@material-ui/core/ListItem'; 6 | import Divider from '@material-ui/core/Divider'; 7 | import { withStyles } from '@material-ui/core/styles'; 8 | import ListItemText from '@material-ui/core/ListItemText'; 9 | import { PROJECT_TYPES } from './constants'; 10 | 11 | const drawerWidth = 150; 12 | 13 | const styles = theme => ({ 14 | drawer: { 15 | width: drawerWidth, 16 | flexShrink: 0 17 | }, 18 | listItemRoot1: { 19 | "&.MuiListItem-root": { 20 | backgroundColor: 'red', 21 | color: 'white' 22 | } 23 | }, 24 | listItemRoot2: { 25 | "&.MuiListItem-root": { 26 | backgroundColor: 'green', 27 | color: 'white' 28 | } 29 | }, 30 | }); 31 | 32 | 33 | class SideBar extends React.Component { 34 | 35 | render() { 36 | const { classes, ...rest } = this.props; 37 | 38 | return ( 39 | 71 | ) 72 | } 73 | } 74 | 75 | export default withStyles(styles)(SideBar); 76 | export { drawerWidth }; -------------------------------------------------------------------------------- /app-ui/src/components/constants.js: -------------------------------------------------------------------------------- 1 | const PROJECT_TYPES = { classification: "classification", 2 | ner: "ner", 3 | entity_disambiguation: "entity_disambiguation"}; 4 | 5 | const PROJECT_TYPES_ABBREV = { classification: "classification", 6 | ner: "NER", 7 | entity_disambiguation: "entity linking"}; 8 | 9 | const WIKI_DOCS_FILE_FORMAT = ""; 10 | const DOCS_TEXT_FILE_FORMAT = ""; 11 | const DOCS_CLASSNAME_FILE_FORMAT = ""; 12 | const DOCS_MENTIONS_FILE_FORMAT = ""; 13 | const DOCS_KB_FILE_FORMAT = ""; 14 | 15 | const FILE_TYPE_DOCUMENTS = "documents"; 16 | const FILE_TYPE_DOCUMENTS_WIKI = "documents_wiki"; 17 | const FILE_TYPE_KB = "kb"; 18 | 19 | const DEFAULT_WIKI_COLUMN = "url"; 20 | const DEFAULT_TEXT_COLUMN = "text"; 21 | const DEFAULT_CLASS_NAME_COLUMN = "label"; 22 | const DEFAULT_MENTIONS_COLUMNS = ["text", "mentions"]; 23 | const DEFAULT_KB_COLUMNS = ["name", "description"]; 24 | 25 | export { PROJECT_TYPES, 26 | PROJECT_TYPES_ABBREV, 27 | DOCS_MENTIONS_FILE_FORMAT, 28 | DOCS_KB_FILE_FORMAT, 29 | DOCS_TEXT_FILE_FORMAT, 30 | DOCS_CLASSNAME_FILE_FORMAT, 31 | FILE_TYPE_DOCUMENTS, 32 | FILE_TYPE_KB, 33 | FILE_TYPE_DOCUMENTS_WIKI, 34 | DEFAULT_TEXT_COLUMN, 35 | DEFAULT_CLASS_NAME_COLUMN, 36 | DEFAULT_MENTIONS_COLUMNS, 37 | DEFAULT_KB_COLUMNS, 38 | DEFAULT_WIKI_COLUMN, 39 | WIKI_DOCS_FILE_FORMAT }; 40 | -------------------------------------------------------------------------------- /app-ui/src/components/file-upload/UploadFileButton.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Button from '@material-ui/core/Button'; 3 | import CircularProgress from '@material-ui/core/CircularProgress'; 4 | 5 | export const UploadFileButton = (props) => { 6 | console.log("Inside UploadFileButton", props); 7 | if(!props.loading){ 8 | return ( 9 | 20 | ) 21 | }else{ 22 | return ( 23 | 26 | ) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /app-ui/src/components/highlightable/Range.js: -------------------------------------------------------------------------------- 1 | function Range(start, end, text, id, entityId){ 2 | return ( 3 | { 4 | start: start, 5 | end: end, 6 | text: text, 7 | id: id, 8 | entityId: entityId 9 | } 10 | ) 11 | } 12 | 13 | export default Range; -------------------------------------------------------------------------------- /app-ui/src/components/highlightable/helpers.js: -------------------------------------------------------------------------------- 1 | export function getUrl(i, text) { 2 | const stringToTest = text.slice(i); 3 | const myRegexp = /^(https?:\/\/(?:www\.|(?!www))[^\s\.]+\.[^\s]{2,}|www\.[^\s]+\.[^\s]{2,})/g; 4 | const match = myRegexp.exec(stringToTest); 5 | 6 | return match && match.length ? match[1] : ''; 7 | } 8 | 9 | export function debounce(func, wait, immediate) { 10 | let timeout; 11 | 12 | return function () { 13 | const context = this, args = arguments; 14 | const later = () => { 15 | timeout = null; 16 | if (!immediate) func.apply(context, args); 17 | }; 18 | const callNow = immediate && !timeout; 19 | 20 | clearTimeout(timeout); 21 | timeout = setTimeout(later, wait); 22 | if (callNow) func.apply(context, args); 23 | }; 24 | } -------------------------------------------------------------------------------- /app-ui/src/components/highlightable/nodes/EmojiNode.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import PropTypes from 'prop-types'; 3 | 4 | import Node from './Node'; 5 | 6 | const EmojiNode = props => { 7 | 8 | return 12 | {`${props.text[props.charIndex]}${props.text[props.charIndex + 1]}`} 13 | ; 14 | }; 15 | 16 | EmojiNode.propTypes = { 17 | highlightStyle: PropTypes.object, 18 | id: PropTypes.string, 19 | charIndex: PropTypes.number, 20 | range: PropTypes.object, 21 | text: PropTypes.string, 22 | children: PropTypes.node 23 | }; 24 | 25 | export default EmojiNode; -------------------------------------------------------------------------------- /app-ui/src/components/highlightable/nodes/Node.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import PropTypes from 'prop-types'; 3 | 4 | const Node = props => { 5 | const getStyle = range => range ? props.highlightStyle : props.style; 6 | const getRangeKey = () => `${props.id}-${props.range.start}-${props.charIndex}`; 7 | const getNormalKey = () => `${props.id}-${props.charIndex}`; 8 | const getKey = range => range ? getRangeKey() : getNormalKey(); 9 | 10 | return ( 13 | {props.children} 14 | ); 15 | }; 16 | 17 | Node.propTypes = { 18 | highlightStyle: PropTypes.object, 19 | style: PropTypes.object, 20 | id: PropTypes.string, 21 | charIndex: PropTypes.number, 22 | range: PropTypes.object, 23 | children: PropTypes.node 24 | }; 25 | 26 | export default Node; -------------------------------------------------------------------------------- /app-ui/src/components/highlightable/nodes/UrlNode.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import PropTypes from 'prop-types'; 3 | 4 | import Node from './Node'; 5 | 6 | const UrlNode = props => { 7 | const style = {wordWrap: 'break-word'}; 8 | 9 | return 14 | 17 | {props.url} 18 | 19 | ; 20 | }; 21 | 22 | UrlNode.propTypes = { 23 | highlightStyle: PropTypes.object, 24 | id: PropTypes.string, 25 | charIndex: PropTypes.number, 26 | range: PropTypes.object, 27 | url: PropTypes.string 28 | }; 29 | 30 | export default UrlNode; -------------------------------------------------------------------------------- /app-ui/src/components/label-page/LabelPage.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { PROJECT_TYPES } from '../constants'; 3 | import SupervisedLabelPage from './supervised/SupervisedLabelPage'; 4 | import EntityDisambiguationLabelPage from './entity-disambiguation/EDLabelPage'; 5 | 6 | const LabelPage = (props) => { 7 | switch(props.projectType){ 8 | case PROJECT_TYPES.classification: 9 | return ; 10 | case PROJECT_TYPES.ner: 11 | return ; 12 | default: 13 | return ; 14 | } 15 | }; 16 | 17 | export default LabelPage; -------------------------------------------------------------------------------- /app-ui/src/components/label-page/common/LabelNavigationSimple.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Button from '@material-ui/core/Button'; 3 | import Grid from '@material-ui/core/Grid'; 4 | import $ from 'jquery'; 5 | 6 | 7 | const Container = (props) => { 8 | return () 13 | } 14 | 15 | const Item = props => { 16 | return() 17 | } 18 | 19 | const SingleLabel = (props) => { 20 | if(props.selected){ 21 | return ( 22 | 29 | ) 30 | }else{ 31 | return ( 32 | 39 | ) 40 | } 41 | } 42 | 43 | class LabelNavigationSimple extends React.Component { 44 | 45 | updateLabel = (manual_label, doc_id) => { 46 | const data = new FormData(); 47 | data.append('project_name', this.props.projectName); 48 | data.append('manual_label', JSON.stringify(manual_label)); 49 | data.append('doc_id', doc_id); 50 | data.append('session_id', this.props.sessionId); 51 | console.log("Sending label ", manual_label); 52 | 53 | $.ajax({ 54 | url : '/api/label-doc', 55 | type : 'POST', 56 | data : data, 57 | processData: false, // tell jQuery not to process the data 58 | contentType: false, // tell jQuery not to set contentType, 59 | success : function(data) { 60 | console.log(`Label modified for doc id ${doc_id}`); 61 | }.bind(this), 62 | error: function (error) { 63 | alert(`Error updating manual label for doc id ${doc_id}`); 64 | }.bind(this) 65 | }); 66 | } 67 | 68 | addLabel = (label) => { 69 | this.props.updateIndexAfterLabelling({label}); 70 | this.updateLabel(label, this.props.docId); 71 | } 72 | 73 | render() { 74 | 75 | return ( 76 | 77 | {this.props.classnames.map((row, index) => { 78 | return ( 79 | 80 | this.addLabel(row)} 82 | className={row.name} 83 | selected={this.props.label == row.label} 84 | /> 85 | 86 | ) 87 | })} 88 | 89 | ) 90 | } 91 | } 92 | 93 | export default LabelNavigationSimple; 94 | -------------------------------------------------------------------------------- /app-ui/src/components/label-page/common/TextNavigation.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Grid from '@material-ui/core/Grid'; 3 | import IconButton from '@material-ui/core/IconButton'; 4 | import ArrowBackIcon from '@material-ui/icons/ArrowBack'; 5 | import ArrowForwardIcon from '@material-ui/icons/ArrowForward'; 6 | import { withStyles } from '@material-ui/core/styles'; 7 | 8 | 9 | const styles = theme => ({ 10 | icon_button: { 11 | borderRadius: 4, 12 | backgroundColor: theme.palette.primary.main, 13 | color: "white", 14 | height: 38 15 | } 16 | }); 17 | 18 | 19 | const Container = (props) => { 20 | const { className, ...rest } = props; 21 | return () 26 | } 27 | 28 | const Item = props => { 29 | return() 30 | } 31 | 32 | const TextNavigation = (props) => { 33 | const classes = props.classes; 34 | 35 | return ( 36 | 37 | 38 | 43 | 44 | 45 | 46 | 47 | 52 | 53 | 54 | 55 | 56 | ) 57 | } 58 | 59 | export default withStyles(styles)(TextNavigation); -------------------------------------------------------------------------------- /app-ui/src/components/label-page/entity-disambiguation/KbSearch.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | import { TextField } from '@material-ui/core'; 4 | import { Autocomplete } from '@material-ui/lab'; 5 | import { withStyles } from '@material-ui/core/styles'; 6 | 7 | import $ from 'jquery'; 8 | import _ from 'lodash'; 9 | 10 | 11 | const styles = theme => ({ 12 | inputRoot: { 13 | marginBottom: 16, 14 | width: 200 15 | }, 16 | }); 17 | 18 | 19 | 20 | class KbSearch extends React.Component { 21 | 22 | state = { 23 | options: [], 24 | inputValue: '', 25 | value: null 26 | } 27 | 28 | componentDidMount = () => { 29 | this.onInputChange({}, '', 'input'); 30 | } 31 | 32 | componentDidUpdate = (prevProps, prevState) => { 33 | const prevDisplayedKbs = new Set(prevProps.displayedKbs.map((x, ind) => x.label)); 34 | const newDisplayedKbs = new Set(this.props.displayedKbs.map((x, ind) => x.label)); 35 | if(!_.isEqual(prevDisplayedKbs, newDisplayedKbs)){ 36 | this.onInputChange({}, '', 'input'); 37 | } 38 | } 39 | 40 | removeDisplayedKbsfromOptions = (displayedKbs, options) => { 41 | const kbIds = displayedKbs.map((x) => x.label); 42 | 43 | // need to filter out the kbs that are already displayed 44 | const displayedOptions = options.filter(x => !kbIds.includes(x.id)) 45 | 46 | return displayedOptions; 47 | } 48 | 49 | onInputChange = (event, inputValue, reason) => { 50 | console.log("setting state to input ", inputValue, reason); 51 | 52 | if(reason == "input"){ 53 | this.setState({inputValue}); 54 | 55 | $.ajax({ 56 | url: '/api/search-kb', 57 | type: 'GET', 58 | data: { 59 | "input": inputValue, 60 | "project_name": this.props.projectName 61 | }, 62 | success: function (data) { 63 | const options = $.parseJSON(data); 64 | const displayedOptions = this.removeDisplayedKbsfromOptions(this.props.displayedKbs, 65 | options); 66 | 67 | this.setState({ options: displayedOptions }); 68 | }.bind(this), 69 | error: function (error) { 70 | console.log("Error in call to server") 71 | }.bind(this) 72 | }); 73 | } 74 | }; 75 | 76 | addSuggestion = (event, input, reason) => { 77 | if(reason == "select-option"){ 78 | this.props.addSuggestion(input); 79 | const displayedOptions = this.state.options.filter(x => x.id != input.id); 80 | 81 | this.setState( {inputValue: '', 82 | selectedValue: null, 83 | options: displayedOptions}); 84 | } 85 | if(reason == 'clear'){ 86 | this.setState( {inputValue: ''} ); 87 | } 88 | } 89 | 90 | render() { 91 | const { classes } = this.props; 92 | 93 | return ( 94 | option.name} 98 | onChange={this.addSuggestion} 99 | renderInput={params => 104 | } 105 | onInputChange={this.onInputChange} 106 | getOptionSelected={(value, option) => value.name == option.name} 107 | autoSelect={true} 108 | inputValue={this.state.inputValue} 109 | disableClearable={true} 110 | value={this.state.value} 111 | /> 112 | ) 113 | }; 114 | }; 115 | 116 | export default withStyles(styles)(KbSearch); 117 | -------------------------------------------------------------------------------- /app-ui/src/components/label-page/entity-disambiguation/LabelNavigation.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import LabelNavigationSimple from '../common/LabelNavigationSimple'; 3 | import KbSearch from './KbSearch'; 4 | import Grid from '@material-ui/core/Grid'; 5 | import _ from 'lodash' 6 | 7 | 8 | 9 | const Container = (props) => { 10 | const { className, ...rest } = props; 11 | return () 17 | } 18 | 19 | const Item = props => { 20 | return() 21 | } 22 | 23 | 24 | class LabelNavigation extends React.Component { 25 | 26 | state = { 27 | kbSuggestions: this.props.kbSuggestions 28 | } 29 | 30 | componentDidUpdate(prevProps, prevState) { 31 | if(!_.isEqual(prevProps.kbSuggestions, this.props.kbSuggestions)){ 32 | this.setState( {kbSuggestions: this.props.kbSuggestions} ); 33 | } 34 | if(prevProps.entityId != this.props.entityId){ 35 | this.setState( {kbSuggestions: this.props.kbSuggestions} ); 36 | } 37 | } 38 | 39 | addSuggestion = (kb) => { 40 | console.log("adding suggestion ", kb); 41 | this.setState((prevState) => { 42 | return { kbSuggestions: 43 | prevState.kbSuggestions.concat({'label': kb.id, 44 | 'name': kb.name})}; 45 | }) 46 | } 47 | 48 | render() { 49 | return ( 50 | 51 | 52 | 60 | 61 | 62 | 67 | 68 | 69 | ) 70 | } 71 | } 72 | 73 | export default LabelNavigation; -------------------------------------------------------------------------------- /app-ui/src/components/label-page/entity-disambiguation/Navigation.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import TextNavigation from '../common/TextNavigation'; 3 | import Grid from '@material-ui/core/Grid'; 4 | import LabelNavigation from './LabelNavigation'; 5 | 6 | 7 | const Container = (props) => { 8 | const { className, ...rest } = props; 9 | return () 15 | } 16 | 17 | const Item = props => { 18 | return() 19 | } 20 | 21 | const Navigation = (props) => { 22 | 23 | return ( 24 | 25 | 26 | 32 | 33 | 34 | 42 | 43 | 44 | ) 45 | } 46 | 47 | export default Navigation; -------------------------------------------------------------------------------- /app-ui/src/components/label-page/entity-disambiguation/Text.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Highlightable from '../../highlightable/Highlightable'; 3 | import { withStyles } from '@material-ui/core/styles'; 4 | import Paper from '@material-ui/core/Paper'; 5 | import Typography from '@material-ui/core/Typography'; 6 | import TextNavigation from '../common/TextNavigation'; 7 | import Box from '@material-ui/core/Box'; 8 | import _ from 'lodash' 9 | 10 | 11 | const styles = theme => ({ 12 | paper: { 13 | minHeight: '200px', 14 | padding: theme.spacing(1), 15 | margin: theme.spacing(5) 16 | }, 17 | icon_button: { 18 | borderRadius: 4, 19 | color: theme.palette.primary.main, 20 | backgroundColor: "white", 21 | height: 38 22 | } 23 | }); 24 | 25 | class Text extends React.Component{ 26 | state = { 27 | currentTextSpans: this.props.currentTextSpans 28 | }; 29 | 30 | compareSpan = (span1, span2) => { 31 | return (span1.start == span2.start) && (span1.end == span2.end); 32 | } 33 | 34 | compareSpans = (x, y) => { 35 | if(x.length !== y.length){ 36 | return false; 37 | } 38 | 39 | const difference = _.differenceWith(x, y, this.compareSpan); 40 | console.log("Comparing spans", x, y, difference); 41 | return difference === undefined || difference.length==0; 42 | }; 43 | 44 | componentDidUpdate(prevProps){ 45 | if(!this.compareSpans(prevProps.currentTextSpans, this.props.currentTextSpans)) { 46 | console.log("props at TextNER have changed"); 47 | this.setState({currentTextSpans: this.props.currentTextSpans}); 48 | } 49 | } 50 | 51 | render() { 52 | console.log("Inside Text ", this.props, this.state); 53 | return ( 54 |
55 | 63 | 64 | 65 | 72 | 73 | 74 | 75 | 82 | 83 | 84 |
85 | ) 86 | } 87 | } 88 | 89 | export default withStyles(styles)(Text); -------------------------------------------------------------------------------- /app-ui/src/components/label-page/supervised/classification/MainArea.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Paper from '@material-ui/core/Paper'; 3 | import Grid from '@material-ui/core/Grid'; 4 | import Typography from '@material-ui/core/Typography'; 5 | import Navigation from './Navigation'; 6 | import { withStyles } from '@material-ui/core/styles'; 7 | 8 | 9 | 10 | const styles = theme => ({ 11 | paper: { 12 | width: 'auto', 13 | minHeight: '200px', 14 | padding: theme.spacing(1), 15 | margin: theme.spacing(5) 16 | }, 17 | container: { 18 | width: '80%', 19 | minHeight: '200px', 20 | padding: theme.spacing(1), 21 | margin: theme.spacing(5) 22 | }, 23 | }); 24 | 25 | // for message and inside main area 26 | const Container = (props) => { 27 | const { classes, ...rest } = props; 28 | return () 35 | } 36 | 37 | const Item = props => { 38 | return() 39 | } 40 | 41 | 42 | const PaperContainer = (props) => { 43 | const { classes, ...rest } = props; 44 | return () 50 | } 51 | 52 | 53 | const MainArea = (props) => { 54 | 55 | const classes = props.classes; 56 | 57 | // get all the unique entity ids in the text 58 | console.log("We will map over currentDisplayedLabels", props.currentDisplayedLabels, 59 | props.classNames 60 | ); 61 | const entities = props.currentDisplayedLabels.map((x, ind) => props.classNames[x.id]); 62 | const EntitySet = new Set(entities.map((x, ind) => x.id)) 63 | 64 | // all the entities that are not in the text and will populate the search 65 | const otherEntities = props.classNames.filter(x => !EntitySet.has(x.id)); 66 | 67 | return ( 68 | 69 | 70 | 71 | 72 | 73 | {props.content.text} 74 | 75 | 76 | 77 | 78 | 79 | 97 | 98 | 99 | ) 100 | } 101 | 102 | export default withStyles(styles)(MainArea); -------------------------------------------------------------------------------- /app-ui/src/components/label-page/supervised/classification/Navigation.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import LabelNavigation from './LabelNavigation'; 3 | import TextNavigation from '../../common/TextNavigation'; 4 | import Grid from '@material-ui/core/Grid'; 5 | import Typography from '@material-ui/core/Typography'; 6 | import { withStyles } from '@material-ui/core/styles'; 7 | 8 | 9 | const styles = theme => ({ 10 | container: {paddingLeft: theme.spacing(5)} 11 | }); 12 | 13 | 14 | const Container = (props) => { 15 | const { className, ...rest } = props; 16 | return () 24 | } 25 | 26 | const Item = props => { 27 | return() 28 | } 29 | 30 | const Navigation = (props) => { 31 | 32 | const { classes } = props; 33 | 34 | return ( 35 | 36 | 37 | {props.groundTruth && 38 | 39 | Ground-truth: {props.groundTruth} 40 | } 41 | 42 | 43 | 44 | 50 | 51 | 52 | 65 | 66 | 67 | ) 68 | } 69 | 70 | export default withStyles(styles)(Navigation); -------------------------------------------------------------------------------- /app-ui/src/components/label-page/supervised/ner/HighlightableTable.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Table from '@material-ui/core/Table'; 3 | import TableBody from '@material-ui/core/TableBody'; 4 | import TableCell from '@material-ui/core/TableCell'; 5 | import TableContainer from '@material-ui/core/TableContainer'; 6 | import TableHead from '@material-ui/core/TableHead'; 7 | import TableRow from '@material-ui/core/TableRow'; 8 | import Paper from '@material-ui/core/Paper'; 9 | import { withStyles } from '@material-ui/core/styles'; 10 | import Highlightable from '../../../highlightable/Highlightable'; 11 | import _ from 'lodash' 12 | 13 | 14 | const styles = theme => ({ 15 | root: { 16 | width: '80%', 17 | minHeight: '200px', 18 | padding: theme.spacing(1), 19 | margin: theme.spacing(5) 20 | }}); 21 | 22 | class HighlightableTable extends React.Component{ 23 | state = { 24 | currentTextSpans: this.props.currentTextSpans 25 | }; 26 | 27 | compareSpan = (span1, span2) => { 28 | return (span1.start == span2.start) && (span1.end == span2.end); 29 | } 30 | 31 | compareSpans = (x, y) => { 32 | if(x.length !== y.length){ 33 | return false; 34 | } 35 | 36 | const difference = _.differenceWith(x, y, this.compareSpan); 37 | console.log("Comparing spans", x, y, difference); 38 | return difference === undefined || difference.length==0; 39 | }; 40 | 41 | componentDidUpdate(prevProps){ 42 | if(!this.compareSpans(prevProps.currentTextSpans, this.props.currentTextSpans)) { 43 | console.log("props at TextNER have changed"); 44 | this.setState({currentTextSpans: this.props.currentTextSpans}); 45 | } 46 | } 47 | 48 | setTextSpans = (range) => { 49 | this.props.setTextSpans(range) 50 | } 51 | 52 | addTextSpan(range){ 53 | if(this.props.currentSelectedEntityId === undefined){ 54 | alert("Please select a type before annotating spans."); 55 | return; 56 | } 57 | range["entityId"] = this.props.currentSelectedEntityId; 58 | this.props.addTextSpan(range); 59 | } 60 | 61 | deleteTextSpan(rangeToDelete){ 62 | this.props.deleteTextSpan(rangeToDelete); 63 | } 64 | 65 | render(){ 66 | const { classes, ...rest } = this.props; 67 | console.log("Style is ", classes); 68 | return ( 69 | 70 | 71 | 72 | 73 | {this.props.content.columnNames.map((row, index) => ( 74 | {row} 75 | ))} 76 | 77 | 78 | 79 | {this.props.content.rows.map((row, index) => ( 80 | 81 | {Object.entries(row).map((singleRow, singleRowIndex) => ( 82 | 83 | this.addTextSpan(range)} 92 | onDeleteRange={(range) => this.deleteTextSpan(range)} 93 | /> 94 | 95 | ))} 96 | 97 | ))} 98 | 99 |
100 |
101 | ); 102 | } 103 | } 104 | 105 | export default withStyles(styles)(HighlightableTable); -------------------------------------------------------------------------------- /app-ui/src/components/label-page/supervised/ner/HighlightableText.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Highlightable from '../../../highlightable/Highlightable'; 3 | import _ from 'lodash' 4 | 5 | 6 | class HighlightableText extends React.Component{ 7 | state = { 8 | currentTextSpans: this.props.currentTextSpans 9 | }; 10 | 11 | compareSpan = (span1, span2) => { 12 | return (span1.start == span2.start) && (span1.end == span2.end); 13 | } 14 | 15 | compareSpans = (x, y) => { 16 | if(x.length !== y.length){ 17 | return false; 18 | } 19 | 20 | const difference = _.differenceWith(x, y, this.compareSpan); 21 | console.log("Comparing spans", x, y, difference); 22 | return difference === undefined || difference.length==0; 23 | }; 24 | 25 | componentDidUpdate(prevProps){ 26 | if(!this.compareSpans(prevProps.currentTextSpans, this.props.currentTextSpans)) { 27 | console.log("props at TextNER have changed from ", prevProps.currentTextSpans, " to ", this.props.currentTextSpans); 28 | this.setState({currentTextSpans: this.props.currentTextSpans}); 29 | } 30 | } 31 | 32 | setTextSpans = (range) => { 33 | this.props.setTextSpans(range) 34 | } 35 | 36 | addTextSpan(range){ 37 | console.log("Inside addTextSpan", range); 38 | if(this.props.currentSelectedEntityId === undefined){ 39 | alert("Please select a type before annotating spans."); 40 | return; 41 | } 42 | range["entityId"] = this.props.currentSelectedEntityId; 43 | this.props.addTextSpan(range); 44 | } 45 | 46 | deleteTextSpan(rangeToDelete){ 47 | this.props.deleteTextSpan(rangeToDelete); 48 | } 49 | 50 | render() { 51 | return ( 52 | this.addTextSpan(range)} 60 | onDeleteRange={(range) => this.deleteTextSpan(range)} 61 | /> 62 | ) 63 | } 64 | } 65 | 66 | export default HighlightableText; 67 | 68 | -------------------------------------------------------------------------------- /app-ui/src/components/label-page/supervised/ner/MainArea.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Paper from '@material-ui/core/Paper'; 3 | import Grid from '@material-ui/core/Grid'; 4 | import HighlightableText from './HighlightableText'; 5 | import HighlightableTable from './HighlightableTable'; 6 | import Typography from '@material-ui/core/Typography'; 7 | import Navigation from './Navigation'; 8 | import { withStyles } from '@material-ui/core/styles'; 9 | import _ from 'lodash' 10 | 11 | 12 | const styles = theme => ({ 13 | paper: { 14 | width: 'auto', 15 | minHeight: '200px', 16 | padding: theme.spacing(1), 17 | margin: theme.spacing(5) 18 | }, 19 | container: { 20 | width: '80%', 21 | minHeight: '200px', 22 | padding: theme.spacing(1), 23 | margin: theme.spacing(5) 24 | }, 25 | container: { 26 | width: '100%', 27 | }, 28 | }); 29 | 30 | 31 | // for message and inside main area 32 | const Container = (props) => { 33 | const { classes, ...rest } = props; 34 | return () 41 | } 42 | 43 | const Item = (props) => { 44 | const { classes, ...rest } = props; 45 | return() 46 | } 47 | 48 | const PaperContainer = (props) => { 49 | const { classes, ...rest } = props; 50 | return () 56 | } 57 | 58 | 59 | const MainArea = (props) => { 60 | 61 | // get all the unique entity ids in the text 62 | console.log("Inside MainArea,", props); 63 | let entities = props.currentDisplayedLabels.map((x, ind) => x && props.classNames[x.entityId]); 64 | console.log("Inside MainArea,", entities); 65 | entities = _.uniqBy(entities, 'id'); 66 | console.log("Inside MainArea,", entities); 67 | const EntitySet = new Set(entities.map((x, ind) => x.id)) 68 | 69 | // all the entities that are not in the text and will populate the search 70 | const otherEntities = props.classNames.filter(x => !EntitySet.has(x.id)); 71 | 72 | const entityColourMap = props.classNames.reduce(function(obj, itm) { 73 | obj[itm['id']] = itm['colour']; 74 | 75 | return obj; 76 | }, {}); 77 | 78 | return ( 79 | 80 | 81 | {props.content.isTable? 82 | 90 | : 91 | 92 | 93 | 101 | 102 | 103 | } 104 | 105 | 106 | 123 | 124 | 125 | ) 126 | } 127 | 128 | export default withStyles(styles)(MainArea); -------------------------------------------------------------------------------- /app-ui/src/components/label-page/supervised/ner/Navigation.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import TextNavigation from '../../common/TextNavigation'; 3 | import { withStyles } from '@material-ui/core/styles'; 4 | import Grid from '@material-ui/core/Grid'; 5 | import LabelNavigation from './LabelNavigation'; 6 | 7 | 8 | const styles = theme => ({ 9 | root: { 10 | maxWidth: 'sm' 11 | }, 12 | container: {paddingLeft: theme.spacing(5)} 13 | }); 14 | 15 | 16 | const Container = (props) => { 17 | const { className, ...rest } = props; 18 | return () 26 | } 27 | 28 | const Item = props => { 29 | return() 30 | } 31 | 32 | const Navigation = (props) => { 33 | 34 | const { classes } = props; 35 | 36 | return ( 37 | 38 | 39 | 45 | 46 | 47 | 59 | 60 | 61 | ) 62 | } 63 | 64 | export default withStyles(styles)(Navigation); -------------------------------------------------------------------------------- /app-ui/src/components/project-summary/ProjectMain.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import SupervisedProjectMain from './supervised/SupervisedProjectMain'; 3 | import EntityDisambiguationProjectMain from './entity-disambiguation/EntityDisambiguationProjectMain'; 4 | import { PROJECT_TYPES } from '../constants'; 5 | 6 | 7 | const ProjectMain = (props) => { 8 | if(props.projectType == PROJECT_TYPES.classification || props.projectType == PROJECT_TYPES.ner){ 9 | return ( 10 | 18 | ) 19 | }else{ 20 | return ( 21 | 28 | ) 29 | } 30 | } 31 | 32 | export default ProjectMain; -------------------------------------------------------------------------------- /app-ui/src/components/project-summary/ProjectTitle.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Typography from '@material-ui/core/Typography'; 3 | 4 | 5 | const ProjectTitle = (props) => { 6 | return ( 7 | 8 | {`Project: ${props.projectName}`} 9 | 10 | ) 11 | } 12 | 13 | export default ProjectTitle; 14 | 15 | -------------------------------------------------------------------------------- /app-ui/src/components/project-summary/common/DeleteProjectButton.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Button from '@material-ui/core/Button'; 3 | 4 | const DeleteProjectButton = (props) => { 5 | return ( 6 | 14 | ); 15 | }; 16 | 17 | export default DeleteProjectButton; 18 | -------------------------------------------------------------------------------- /app-ui/src/components/project-summary/common/DownloadUnlabeledDataButton.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import $ from 'jquery'; 3 | import Button from '@material-ui/core/Button';; 4 | import CircularProgress from '@material-ui/core/CircularProgress'; 5 | import { withStyles } from '@material-ui/core/styles'; 6 | import Container from '@material-ui/core/Container'; 7 | import { PROJECT_TYPES } from '../../constants'; 8 | 9 | 10 | const styles = theme => ({ 11 | button: { 12 | marginTop: "20px", 13 | marginLeft: "10px", 14 | width: "280px" 15 | }, 16 | container: { 17 | width: "280px", 18 | maxWidth: "280px", 19 | display: "inline-block" 20 | } 21 | }); 22 | 23 | function saveAsFile(text, filename) { 24 | // Step 1: Create the blob object with the text you received 25 | const type = 'application/text'; // modify or get it from response 26 | const blob = new Blob([text], {type}); 27 | 28 | // Step 2: Create Blob Object URL for that blob 29 | const url = URL.createObjectURL(blob); 30 | 31 | // Step 3: Trigger downloading the object using that URL 32 | const a = document.createElement('a'); 33 | a.href = url; 34 | a.download = filename; 35 | a.click(); // triggering it manually 36 | } 37 | 38 | 39 | class DownloadUnlabeledDataButton extends React.Component{ 40 | 41 | state = {loading: false} 42 | 43 | downloadLabels = (projectName) => { 44 | const data = new FormData(); 45 | data.append('project_name', projectName); 46 | this.setState({ loading: true }); 47 | 48 | $.ajax({ 49 | url : '/api/download-unlabeled', 50 | type : 'POST', 51 | data : data, 52 | processData: false, // tell jQuery not to process the data 53 | contentType: false, // tell jQuery not to set contentType, 54 | success : function(data) { 55 | saveAsFile(data, `${projectName}.suggested-examples.json`); 56 | this.setState({ loading: false }); 57 | }.bind(this), 58 | error: function (error) { 59 | this.setState({ loading: false }); 60 | alert(error); 61 | } 62 | }); 63 | } 64 | 65 | selectButtonLabel(projectType){ 66 | switch(projectType) { 67 | case PROJECT_TYPES.classification: 68 | return 'Download labels'; 69 | case PROJECT_TYPES.ner: 70 | return 'Download Suggested Examples'; 71 | case PROJECT_TYPES.entity_disambiguation: 72 | return 'Download labels 1'; 73 | default: 74 | return 'Download labels 2' 75 | } 76 | } 77 | 78 | render() { 79 | const { classes } = this.props; 80 | 81 | if(!this.state.loading){ 82 | return ( 83 | ) 91 | } 92 | else{ 93 | return ( 94 | 95 | 98 | 99 | ) 100 | } 101 | } 102 | }; 103 | 104 | export default withStyles(styles)(DownloadUnlabeledDataButton); -------------------------------------------------------------------------------- /app-ui/src/components/project-summary/common/FileDownloadButton.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import $ from 'jquery'; 3 | import Button from '@material-ui/core/Button';; 4 | import CircularProgress from '@material-ui/core/CircularProgress'; 5 | import { withStyles } from '@material-ui/core/styles'; 6 | import Container from '@material-ui/core/Container'; 7 | import { PROJECT_TYPES } from '../../constants'; 8 | 9 | 10 | const styles = theme => ({ 11 | button: { 12 | marginTop: "20px", 13 | width: "250px" 14 | }, 15 | container: { 16 | width: "250px", 17 | maxWidth: "250px", 18 | display: "inline-block" 19 | } 20 | }); 21 | 22 | function saveAsFile(text, filename) { 23 | // Step 1: Create the blob object with the text you received 24 | const type = 'application/text'; // modify or get it from response 25 | const blob = new Blob([text], {type}); 26 | 27 | // Step 2: Create Blob Object URL for that blob 28 | const url = URL.createObjectURL(blob); 29 | 30 | // Step 3: Trigger downloading the object using that URL 31 | const a = document.createElement('a'); 32 | a.href = url; 33 | a.download = filename; 34 | a.click(); // triggering it manually 35 | } 36 | 37 | 38 | class FileDownloadButton extends React.Component{ 39 | 40 | state = {loading: false} 41 | 42 | downloadLabels = (projectName) => { 43 | const data = new FormData(); 44 | data.append('project_name', projectName); 45 | this.setState({ loading: true }); 46 | 47 | $.ajax({ 48 | url : '/api/export-labels', 49 | type : 'POST', 50 | data : data, 51 | processData: false, // tell jQuery not to process the data 52 | contentType: false, // tell jQuery not to set contentType, 53 | success : function(data) { 54 | saveAsFile(data, `${projectName}.labeled-data.json`); 55 | this.setState({ loading: false }); 56 | }.bind(this), 57 | error: function (error) { 58 | this.setState({ loading: false }); 59 | alert(error); 60 | } 61 | }); 62 | } 63 | 64 | selectButtonLabel(projectType){ 65 | switch(projectType) { 66 | case PROJECT_TYPES.classification: 67 | return 'Download labels'; 68 | case PROJECT_TYPES.ner: 69 | return 'Download Labeled Data'; 70 | case PROJECT_TYPES.entity_disambiguation: 71 | return 'Download labels'; 72 | default: 73 | return 'Download labels' 74 | } 75 | } 76 | 77 | render() { 78 | const { classes } = this.props; 79 | 80 | if(!this.state.loading){ 81 | return ( 82 | ) 90 | } 91 | else{ 92 | return ( 93 | 94 | 97 | 98 | ) 99 | } 100 | } 101 | }; 102 | 103 | export default withStyles(styles)(FileDownloadButton); -------------------------------------------------------------------------------- /app-ui/src/components/project-summary/supervised/ExportRulesButton.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import $ from 'jquery'; 3 | import Button from '@material-ui/core/Button';; 4 | import { withStyles } from '@material-ui/core/styles'; 5 | import Container from '@material-ui/core/Container'; 6 | import CircularProgress from '@material-ui/core/CircularProgress'; 7 | 8 | 9 | const styles = theme => ({ 10 | button: { 11 | marginTop: "20px", 12 | marginLeft: "10px", 13 | width: "250px" 14 | }, 15 | container: { 16 | width: "250px", 17 | maxWidth: "250px", 18 | display: "inline-block" 19 | } 20 | }); 21 | 22 | 23 | function saveAsFile(text, filename) { 24 | // Step 1: Create the blob object with the text you received 25 | const type = 'application/text'; // modify or get it from response 26 | const blob = new Blob([text], {type}); 27 | 28 | // Step 2: Create Blob Object URL for that blob 29 | const url = URL.createObjectURL(blob); 30 | 31 | // Step 3: Trigger downloading the object using that URL 32 | const a = document.createElement('a'); 33 | a.href = url; 34 | a.download = filename; 35 | a.click(); // triggering it manually 36 | } 37 | 38 | 39 | class ExportRulesButton extends React.Component{ 40 | 41 | state = {loading: false} 42 | 43 | exportRules = (projectName) => { 44 | const data = new FormData(); 45 | data.append('project_name', projectName); 46 | this.setState({ loading: true }); 47 | 48 | $.ajax({ 49 | url : '/api/export-rules', 50 | type : 'POST', 51 | data : data, 52 | processData: false, // tell jQuery not to process the data 53 | contentType: false, // tell jQuery not to set contentType, 54 | success : function(data) { 55 | saveAsFile(data, `${projectName}.trained-model.ckpt`); 56 | this.setState({ loading: false }); 57 | }.bind(this), 58 | error: function (error) { 59 | this.setState({ loading: false }); 60 | alert(error); 61 | } 62 | }); 63 | } 64 | 65 | render() { 66 | const { classes } = this.props; 67 | 68 | if(!this.state.loading){ 69 | return ( 70 | 78 | ) 79 | } 80 | else{ 81 | return ( 82 | 83 | 86 | 87 | ) 88 | } 89 | } 90 | }; 91 | 92 | export default withStyles(styles)(ExportRulesButton); -------------------------------------------------------------------------------- /app-ui/src/components/project-summary/supervised/LabelTable.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { PROJECT_TYPES } from '../../constants'; 3 | import LabelTableClassification from './classification/LabelTableClassification'; 4 | import LabelTableNER from './ner/LabelTableNER'; 5 | 6 | 7 | const LabelTable = (props) => { 8 | const params = { 9 | totalDocuments: props.docs.totalDocuments, 10 | totalManualDocs: props.docs.totalManualDocs, 11 | totalManualDocsEmpty: props.docs.totalManualDocsEmpty, 12 | totalManualSpans: props.docs.totalManualSpans, 13 | modelClasses: props.docs.classes, 14 | exploreLabelled: props.exploreLabelled, 15 | classes: props.classes 16 | } 17 | 18 | if(props.projectType == PROJECT_TYPES["classification"]){ 19 | return ( 20 | 21 | ) 22 | }else{ 23 | if(props.projectType == PROJECT_TYPES["ner"]){ 24 | return ( 25 | 26 | ) 27 | } 28 | } 29 | } 30 | 31 | export default LabelTable; -------------------------------------------------------------------------------- /app-ui/src/components/project-summary/supervised/PerformanceTable.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { makeStyles } from '@material-ui/core/styles'; 3 | import Table from '@material-ui/core/Table'; 4 | import TableBody from '@material-ui/core/TableBody'; 5 | import TableCell from '@material-ui/core/TableCell'; 6 | import TableContainer from '@material-ui/core/TableContainer'; 7 | import TableHead from '@material-ui/core/TableHead'; 8 | import TableRow from '@material-ui/core/TableRow'; 9 | import Paper from '@material-ui/core/Paper'; 10 | import Typography from '@material-ui/core/Typography'; 11 | 12 | 13 | const useStyles = makeStyles({ 14 | tablecell: { 15 | fontSize: '80%' 16 | } 17 | }); 18 | 19 | const StyledTableCell = (props) => { 20 | const classes = useStyles(); 21 | return ( 22 | 29 | {props.content} 30 | 31 | ) 32 | } 33 | 34 | const RuleTableHead = (props) => ( 35 | 36 | 37 | 38 | 39 | 40 | 41 | ) 42 | 43 | const RuleTableRow = (props) => { 44 | return ( 45 | 46 | 47 | 51 | 52 | )} 53 | 54 | const NumberToString = (number) => { 55 | return `${(number * 100).toFixed(1)}`; 56 | } 57 | 58 | const PerformanceTable = (props) => { 59 | 60 | const modelClasses = props.docs.classes; 61 | 62 | return ( 63 | 64 |
65 |
66 | 67 | 68 | 69 | 70 | {modelClasses.map((row, index) => { 71 | if(row.estimatedPrecision){ 72 | let precisionString = NumberToString(row.estimatedPrecision) + "%"; 73 | let recallString = NumberToString(row.estimatedRecall) + "%"; 74 | 75 | let groundTruthPrecision, groundTruthRecall; 76 | if(row.groundTruthPrecision){ 77 | groundTruthPrecision = ` (true: ${NumberToString(row.groundTruthPrecision)}%)`; 78 | groundTruthRecall = ` (true: ${NumberToString(row.groundTruthRecall)}%)` 79 | } 80 | 81 | if(row.estimatedPrecisionLowerBound){ 82 | precisionString += ` (${NumberToString(row.estimatedPrecisionLowerBound)} - ${NumberToString(row.estimatedPrecisionUpperBound)})` + (groundTruthPrecision || ""); 83 | }else{ 84 | precisionString += (groundTruthPrecision || ""); 85 | } 86 | 87 | if(row.estimatedRecallLowerBound){ 88 | recallString += ` (${NumberToString(row.estimatedRecallLowerBound)} - ${NumberToString(row.estimatedRecallUpperBound)})` + (groundTruthRecall || ""); 89 | } 90 | else{ 91 | recallString += (groundTruthRecall || "") 92 | } 93 | 94 | return ( 95 | 102 | ) 103 | } 104 | })} 105 | 106 |
107 |
108 |
109 | ) 110 | } 111 | 112 | export default PerformanceTable; -------------------------------------------------------------------------------- /app-ui/src/components/project-summary/supervised/ProjectDataDrawers.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import RuleTable from './RuleTable'; 3 | import LabelTable from './LabelTable'; 4 | import PerformanceTable from './PerformanceTable'; 5 | import AppBar from '@material-ui/core/AppBar'; 6 | import Tabs from '@material-ui/core/Tabs'; 7 | import Tab from '@material-ui/core/Tab'; 8 | import CircularProgress from '@material-ui/core/CircularProgress'; 9 | import FileDownloadButton from '../common/FileDownloadButton'; 10 | import DownloadUnlabeledDataButton from '../common/DownloadUnlabeledDataButton'; 11 | import ExportRulesButton from './ExportRulesButton'; 12 | 13 | 14 | class ProjectDataDrawers extends React.Component{ 15 | 16 | state = { 17 | tab: 0 18 | } 19 | 20 | setTabValue = (e, value) => { 21 | this.setState({tab: value}); 22 | } 23 | 24 | render(){ 25 | if(!this.props.loading){ 26 | return ( 27 |
28 | 29 | 30 | 31 | 32 | 33 |
34 | {this.state.tab == 0 && 35 | 42 | } 43 | 47 | 51 | 52 |
53 | ) 54 | } 55 | else{ 56 | return ( 57 |
58 |

Selecting best unlabeled examples for the next annotation iteration...

59 |

60 | 61 |
62 | ) 63 | } 64 | } 65 | } 66 | 67 | export default ProjectDataDrawers; -------------------------------------------------------------------------------- /app-ui/src/components/project-summary/supervised/RuleTable.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { PROJECT_TYPES } from '../../constants'; 3 | import RuleTableClassification from './classification/RuleTableClassification'; 4 | import RuleTableNER from './ner/RuleTableNER'; 5 | 6 | 7 | const RuleTable = (props) => { 8 | if(props.projectType == PROJECT_TYPES["classification"]){ 9 | return ( 10 | 19 | ) 20 | }else{ 21 | if(props.projectType == PROJECT_TYPES["ner"]){ 22 | return ( 23 | 32 | ) 33 | } 34 | } 35 | } 36 | 37 | export default RuleTable; -------------------------------------------------------------------------------- /app-ui/src/components/project-summary/supervised/classification/LabelTableClassification.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { makeStyles } from '@material-ui/core/styles'; 3 | import Table from '@material-ui/core/Table'; 4 | import TableHead from '@material-ui/core/TableHead'; 5 | import TableBody from '@material-ui/core/TableBody'; 6 | import TableCell from '@material-ui/core/TableCell'; 7 | import TableContainer from '@material-ui/core/TableContainer'; 8 | import TableRow from '@material-ui/core/TableRow'; 9 | import Paper from '@material-ui/core/Paper'; 10 | import TouchAppIcon from '@material-ui/icons/TouchApp'; 11 | import IconButton from '@material-ui/core/IconButton'; 12 | import Tooltip from '@material-ui/core/Tooltip'; 13 | 14 | 15 | const useStyles = makeStyles({ 16 | tablecell: { 17 | fontSize: '80%' 18 | } 19 | }); 20 | 21 | const StyledTableCell = (props) => { 22 | const classes = useStyles(); 23 | return ( 24 | 30 | {props.content} 31 | 32 | ) 33 | } 34 | 35 | const RuleTableHead = (props) => ( 36 | 37 | 38 | 39 | 40 | 41 | 42 | ) 43 | 44 | const ExploreButton = (props) => ( 45 | 46 | props.exploreLabelled(e, props.label)} name="explore-label-button"> 47 | 48 | 49 | 50 | ) 51 | 52 | const RuleTableRow = (props) => { 53 | return ( 54 | 55 | 56 | 60 | } 66 | /> 67 | 68 | )} 69 | 70 | const LabelTableClassification = (props) => { 71 | console.log("Inside document summary", props); 72 | 73 | return ( 74 |
75 | 76 | 77 | 78 | 79 | {props.modelClasses.map((row, index) => { 80 | return ( 81 | 89 | ) 90 | })} 91 | 92 | 99 | 100 |
101 |
102 |
103 | ) 104 | } 105 | 106 | export default LabelTableClassification; -------------------------------------------------------------------------------- /app-ui/src/components/project-summary/supervised/ner/LabelTableNER.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { makeStyles } from '@material-ui/core/styles'; 3 | import Table from '@material-ui/core/Table'; 4 | import TableHead from '@material-ui/core/TableHead'; 5 | import TableBody from '@material-ui/core/TableBody'; 6 | import TableCell from '@material-ui/core/TableCell'; 7 | import TableContainer from '@material-ui/core/TableContainer'; 8 | import TableRow from '@material-ui/core/TableRow'; 9 | import Paper from '@material-ui/core/Paper'; 10 | import TouchAppIcon from '@material-ui/icons/TouchApp'; 11 | import IconButton from '@material-ui/core/IconButton'; 12 | import Tooltip from '@material-ui/core/Tooltip'; 13 | 14 | 15 | const useStyles = makeStyles({ 16 | tablecell: { 17 | fontSize: '80%' 18 | } 19 | }); 20 | 21 | const useStyles2 = makeStyles({ 22 | tablecell: { 23 | fontSize: '80%', 24 | fontWeight: 'bold', 25 | color: '#0089de' 26 | } 27 | }); 28 | 29 | const StyledTableCell = (props) => { 30 | const classes = useStyles(); 31 | return ( 32 | 38 | {props.content} 39 | 40 | ) 41 | } 42 | 43 | const StyledTableCell2 = (props) => { 44 | const classes = useStyles2(); 45 | return ( 46 | 52 | {props.content} 53 | 54 | ) 55 | } 56 | 57 | const RuleTableHead = (props) => ( 58 | 59 | 60 | 61 | 62 | 63 | 64 | ) 65 | 66 | const ExploreButton = (props) => ( 67 | 68 | props.exploreLabelled(e, props.label)} name="explore-label-button"> 69 | 70 | 71 | 72 | ) 73 | 74 | const RuleTableRow0 = (props) => { 75 | return ( 76 | 77 | 78 | 82 | 83 | )} 84 | 85 | const RuleTableRow1 = (props) => { 86 | return ( 87 | 88 | 89 | 93 | 94 | )} 95 | 96 | const RuleTableRow2 = (props) => { 97 | return ( 98 | 99 | 100 | 104 | } 110 | /> 111 | 112 | )} 113 | 114 | const LabelTableNER = (props) => { 115 | console.log("Inside document summary", props); 116 | 117 | return ( 118 |
119 | 120 | 121 | 122 | 123 | {props.modelClasses.map((row, index) => { 124 | return ( 125 | 134 | ) 135 | })} 136 | 143 | 150 | 151 |
152 |
153 |
154 | ) 155 | } 156 | 157 | export default LabelTableNER; -------------------------------------------------------------------------------- /app-ui/src/components/projects/Projects.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import AddIcon from '@material-ui/icons/Add'; 3 | import IconButton from '@material-ui/core/IconButton'; 4 | import { withStyles } from '@material-ui/core/styles'; 5 | import { Link } from 'react-router-dom'; 6 | import Card from '@material-ui/core/Card'; 7 | import CardContent from '@material-ui/core/CardContent'; 8 | import Typography from '@material-ui/core/Typography'; 9 | import Grid from '@material-ui/core/Grid'; 10 | import { trimString } from '../../utils'; 11 | import { PROJECT_TYPES_ABBREV } from '../constants'; 12 | 13 | const MAX_PROJECT_NAME_LENGTH = 20; 14 | 15 | const styles = theme => ({ 16 | button_root: { 17 | height: '100%', 18 | width: '100%' 19 | }, 20 | icon_root: { 21 | height: '50%', 22 | width: '50%' 23 | }, 24 | card: { 25 | height: '150px', 26 | width: '150px', 27 | justifyContent: 'center', 28 | display: 'flex', 29 | alignItems: 'center', 30 | position: 'relative' 31 | }, 32 | container: { 33 | marginTop: '20px' 34 | }, 35 | main_div: { 36 | margin: '20px' 37 | }, 38 | new_banner: { position: 'absolute', 39 | top: '10px', 40 | right: '10px' , 41 | color: theme.palette.secondary.main, 42 | fontWeight: 'bold'} 43 | }); 44 | 45 | const Container = (props) => { 46 | const { classes, ...rest } = props; 47 | return () 52 | } 53 | 54 | const Item = props => { 55 | return() 56 | } 57 | 58 | const ProjectCard = (props) => { 59 | const { classes } = props; 60 | const trimmedProjectName = trimString(props.title, MAX_PROJECT_NAME_LENGTH); 61 | return ( 62 | 63 |
64 | {PROJECT_TYPES_ABBREV[props.projectType]} 65 |
66 | 67 | {trimmedProjectName} 68 | 69 |
70 | ) 71 | } 72 | 73 | const AddCard = (props) => { 74 | const { classes } = props; 75 | return ( 76 | 77 | 78 | 85 | 89 | 90 | 91 | 92 | ) 93 | } 94 | 95 | const ProjectCards = (props) => { 96 | const { classes } = props; 97 | return ( 98 | 99 | {props.projects.map((project, index) => { 100 | // console.log("card", project); 101 | return ( 102 | 103 | 107 | 112 | 113 | ) 114 | })} 115 | 116 | 117 | 118 | 119 | ) 120 | } 121 | 122 | 123 | class Projects extends React.Component { 124 | 125 | constructor(props){ 126 | super(props); 127 | this.props.resetCurrentProject(); 128 | } 129 | 130 | render(){ 131 | const classes = this.props.classes; 132 | return ( 133 |
134 | My projects 135 | 136 |
137 | ) 138 | } 139 | } 140 | 141 | export default withStyles(styles)(Projects); -------------------------------------------------------------------------------- /app-ui/src/components/rules/main-page/ImportRuleCard.js: -------------------------------------------------------------------------------- 1 | import Card from '@material-ui/core/Card'; 2 | import CardContent from '@material-ui/core/CardContent'; 3 | import CardActionArea from '@material-ui/core/CardActionArea'; 4 | import Typography from '@material-ui/core/Typography'; 5 | import React from 'react'; 6 | import $ from 'jquery'; 7 | import { Redirect } from 'react-router-dom'; 8 | import uuid from 'react-uuid'; 9 | import CircularProgress from '@material-ui/core/CircularProgress'; 10 | 11 | 12 | 13 | const IMPORT_PARAMS = { 14 | totalAttempts: 16, 15 | timeAttemptms: 15000 16 | } 17 | 18 | class ImportRuleCard extends React.Component{ 19 | state = {toProjectSummary: false, 20 | importingLoading: false} 21 | 22 | importRulesEndPoint = (selectedFile, 23 | projectName, 24 | importId, 25 | attemptNum, 26 | polling) => { 27 | const data = new FormData(); 28 | console.log("Inside importRulesEndPoint", attemptNum, polling, selectedFile); 29 | data.append('file', selectedFile); 30 | data.append('project_name', projectName); 31 | data.append('polling', polling); 32 | data.append('import_id', importId); 33 | 34 | $.ajax({ 35 | url : '/api/import-rules', 36 | type : 'POST', 37 | data : data, 38 | processData: false, // tell jQuery not to process the data 39 | contentType: false, // tell jQuery not to set contentType, 40 | timeout: 60000, 41 | success : function(data) { 42 | console.log("import rules data: ", data); 43 | const jsonData = JSON.parse(data); 44 | console.log("jsonData", jsonData); 45 | const importIdFromServer = jsonData.id; 46 | 47 | if(!importIdFromServer){ 48 | if(attemptNum < IMPORT_PARAMS.totalAttempts){ 49 | setTimeout(() => this.importRulesEndPoint( 50 | selectedFile, 51 | projectName, 52 | importId, 53 | attemptNum+1, 54 | true), 55 | IMPORT_PARAMS.timeAttemptms); 56 | } 57 | else{ 58 | alert("Server timed out"); 59 | this.setState( {importingLoading: false} ); 60 | } 61 | } 62 | else{ 63 | var today = new Date(); 64 | console.log("Rules imported successfully", today.toLocaleString()); 65 | this.setState( {toProjectSummary: true, 66 | importingLoading: false} ); 67 | } 68 | }.bind(this), 69 | error: function (xmlhttprequest, textstatus, message) { 70 | console.log("Error", textstatus, message); 71 | if(textstatus==="timeout" & attemptNum < IMPORT_PARAMS.totalAttempts) { 72 | setTimeout(() => this.importRulesEndPoint( 73 | selectedFile, 74 | projectName, 75 | importId, 76 | attemptNum+1, 77 | true), 78 | IMPORT_PARAMS.timeAttemptms); 79 | } 80 | else{ 81 | alert("Error importing rules"); 82 | this.setState( {importingLoading: false} ); 83 | } 84 | }.bind(this) 85 | }); 86 | 87 | this.setState( {importingLoading: true} ); 88 | } 89 | 90 | importRules = (e, projectName) => { 91 | e.preventDefault; 92 | const selectedFile = e.target.files[0]; 93 | if(!selectedFile){ 94 | alert("Need to select file!"); 95 | }else{ 96 | this.importRulesEndPoint(selectedFile, 97 | projectName, 98 | uuid(), 99 | 0, 100 | false); 101 | } 102 | e.target.value = null; 103 | } 104 | 105 | render() { 106 | 107 | if(this.state.toProjectSummary === true) { 108 | return 109 | } 110 | 111 | return( 112 | 113 |
114 | this.importRules(e, this.props.projectName)} 120 | /> 121 | 125 | 126 | Import rules 127 |
128 | {this.state.importingLoading? 129 | 130 | : 131 | Load a file with rules. 132 | } 133 |
134 |
135 |
136 |
137 | ) 138 | } 139 | } 140 | 141 | export default ImportRuleCard; -------------------------------------------------------------------------------- /app-ui/src/components/rules/main-page/RuleMain.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Link } from 'react-router-dom'; 3 | import { makeStyles } from '@material-ui/core/styles'; 4 | import SideBar from '../../SideBar'; 5 | import Typography from '@material-ui/core/Typography'; 6 | import Box from '@material-ui/core/Box'; 7 | import Card from '@material-ui/core/Card'; 8 | import CardContent from '@material-ui/core/CardContent'; 9 | import Grid from '@material-ui/core/Grid'; 10 | import ImportRuleCard from './ImportRuleCard'; 11 | import { PROJECT_TYPES } from '../../constants'; 12 | 13 | 14 | const useStyles = makeStyles({ 15 | container: {display: 'flex'}, 16 | main_content: {marginLeft: "20px"}, 17 | card: { 18 | height: '200px', 19 | width: '200px', 20 | } 21 | }); 22 | 23 | const Container = (props) => { 24 | const { classes, ...rest } = props; 25 | return () 30 | } 31 | 32 | const Item = props => { 33 | return() 34 | } 35 | 36 | function createData(title, explanation, url) { 37 | return { title, explanation, url }; 38 | } 39 | 40 | const ClassificationRules = [ 41 | createData('Non-ordered series of phrases', 42 | 'Find a sequence of phrases where the order does not matter.', 43 | '/non-ordered'), 44 | createData('Ordered series of phrases', 45 | 'Find a sequence of phrases in a given order.', 46 | '/ordered') 47 | ] 48 | 49 | const NERRules = [ 50 | createData('Regex match on entity', 51 | 'Find a sequence of tokens matching a regular expression.', 52 | '/regex'), 53 | createData('Regex match on noun-phrases', 54 | 'Find matching noun phrases.', 55 | '/noun-phrase'), 56 | ] 57 | 58 | function getListRules(projectType) { 59 | if(projectType == PROJECT_TYPES.classification){ 60 | return ClassificationRules; 61 | } 62 | else{ 63 | return NERRules; 64 | } 65 | } 66 | 67 | 68 | const RuleCard = (props) => { 69 | const { classes } = props; 70 | return ( 71 | 72 | 73 | {props.title} 74 |
75 | {props.explanation} 76 |
77 |
78 | ) 79 | } 80 | 81 | 82 | const RuleMain = (props) => { 83 | const classes = useStyles(); 84 | const rules = getListRules(props.projectType); 85 | 86 | return ( 87 |
88 | 92 |
93 | 94 | Add rules. 95 | 96 | 97 | { 98 | rules.map((rule, index) => ( 99 | 100 | 104 | 109 | 110 | 111 | )) 112 | } 113 | 114 | 119 | 120 | 121 |
122 |
123 | ) 124 | } 125 | 126 | export default RuleMain; -------------------------------------------------------------------------------- /app-ui/src/components/rules/rule-forms/base/CaseSensitive.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import OptionSelection from './OptionSelection'; 3 | 4 | 5 | const CaseSensitive = () => { 6 | return ( 7 | 12 | ) 13 | } 14 | 15 | export default CaseSensitive; -------------------------------------------------------------------------------- /app-ui/src/components/rules/rule-forms/base/ClassDefinitionBox.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Autocomplete from '@material-ui/lab/Autocomplete'; 3 | import Grid from '@material-ui/core/Grid'; 4 | import Typography from '@material-ui/core/Typography'; 5 | import TextField from '@material-ui/core/TextField'; 6 | 7 | 8 | const Container = (props) => { 9 | const { classes, ...rest } = props; 10 | return () 15 | } 16 | 17 | const Item = props => { 18 | return() 19 | } 20 | 21 | const ClassDefinitionBox = (props) => { 22 | return ( 23 |
24 | 25 | {props.addText && 26 | 27 | Then it's most likely class: 28 | 29 | } 30 | 31 | option.name} 35 | onChange={props.setClass} 36 | style={{ width: 300 }} 37 | renderInput={(params) => } 38 | inputValue={props.inputValue} 39 | onInputChange={props.onInputChange} 40 | value={props.value} 41 | /> 42 | 43 | 44 |
45 | ) 46 | } 47 | 48 | export default ClassDefinitionBox; -------------------------------------------------------------------------------- /app-ui/src/components/rules/rule-forms/base/ContainOrDoesNotContain.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import OptionSelection from './OptionSelection'; 3 | 4 | 5 | const ContainOrDoesNotContain = () => { 6 | return ( 7 | 12 | ) 13 | } 14 | 15 | export default ContainOrDoesNotContain -------------------------------------------------------------------------------- /app-ui/src/components/rules/rule-forms/base/InputField.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import TextField from '@material-ui/core/TextField'; 3 | 4 | 5 | const InputField = (props) => { 6 | return ( 7 | 13 | ) 14 | } 15 | 16 | export default InputField; -------------------------------------------------------------------------------- /app-ui/src/components/rules/rule-forms/base/OptionSelection.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | 4 | const OptionSelection = (props) => { 5 | return ( 6 |
7 | { 8 | props.options.map((option, index) => ( 9 |
10 | 16 | 17 |
18 | )) 19 | } 20 |
21 | ) 22 | } 23 | 24 | export default OptionSelection; -------------------------------------------------------------------------------- /app-ui/src/components/rules/rule-forms/base/RuleSubmit.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import InputField from '../base/InputField'; 3 | import RuleSubmitButton from '../base/RuleSubmitButton'; 4 | import Grid from '@material-ui/core/Grid'; 5 | 6 | 7 | const Container = (props) => { 8 | const { classes, ...rest } = props; 9 | return () 14 | } 15 | 16 | const Item = props => { 17 | return() 18 | } 19 | 20 | const RuleSubmit = (props) => { 21 | return ( 22 | 23 | 24 | 28 | 29 | 30 | 32 | 33 | 34 | ); 35 | }; 36 | 37 | export default RuleSubmit; -------------------------------------------------------------------------------- /app-ui/src/components/rules/rule-forms/base/RuleSubmitButton.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Button from '@material-ui/core/Button'; 3 | import CircularProgress from '@material-ui/core/CircularProgress'; 4 | 5 | 6 | const RuleSubmitButton = (props) => { 7 | if(!props.loading){ 8 | return ( 9 | 16 | ) 17 | }else{ 18 | return ( 19 | 20 | ) 21 | } 22 | } 23 | 24 | export default RuleSubmitButton; -------------------------------------------------------------------------------- /app-ui/src/components/rules/rule-forms/base/SentenceOrFullText.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import OptionSelection from './OptionSelection'; 3 | 4 | 5 | const SentenceOrFullText = () => { 6 | return ( 7 | 12 | ) 13 | } 14 | 15 | export default SentenceOrFullText; -------------------------------------------------------------------------------- /app-ui/src/components/rules/rule-forms/base/TypeSelector.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | const TypeSelector = (props) => { 4 | return ( 5 | 20 | ) 21 | } 22 | 23 | export default TypeSelector; -------------------------------------------------------------------------------- /app-ui/src/components/rules/rule-forms/classification-rules/NonOrderedRule.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import GenericOrderedRule from '../base/GenericOrderedRule'; 3 | 4 | const NonOrderedRule = (props) => { 5 | return ( 6 | 12 | ) 13 | } 14 | 15 | export default NonOrderedRule; -------------------------------------------------------------------------------- /app-ui/src/components/rules/rule-forms/classification-rules/OrderedRule.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import GenericOrderedRule from '../base/GenericOrderedRule'; 3 | 4 | const OrderedRule = (props) => { 5 | return ( 6 | 12 | ) 13 | } 14 | 15 | export default OrderedRule; -------------------------------------------------------------------------------- /app-ui/src/components/search/LabelComponent.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ClassificationLabelNavigation from '../label-page/supervised/classification/LabelNavigation'; 3 | import NERLabelNavigation from '../label-page/supervised/ner/LabelNavigation'; 4 | import { PROJECT_TYPES } from '../constants'; 5 | import _ from 'lodash' 6 | 7 | 8 | const LabelComponent = (props) => { 9 | if(props.projectType == PROJECT_TYPES.classification){ 10 | let entities; 11 | if(props.currentDisplayedLabels){ 12 | entities = props.currentDisplayedLabels.map((x, ind) => props.classNames[x.id]); 13 | console.log("Inside MainArea,", entities); 14 | entities = _.uniqBy(entities, 'id'); 15 | }else{ 16 | entities = []; 17 | } 18 | 19 | const EntitySet = new Set(entities.map((x, ind) => x.id)) 20 | 21 | // all the entities that are not in the text and will populate the search 22 | const otherEntities = props.classNames.filter(x => !EntitySet.has(x.id)); 23 | return ( 24 |
25 | 36 |
37 | ); 38 | } 39 | 40 | if(props.projectType == PROJECT_TYPES.ner){ 41 | console.log("Inside LabelComponent", props); 42 | 43 | let entities; 44 | if(props.currentDisplayedLabels){ 45 | entities = props.currentDisplayedLabels.map((x, ind) => x && props.classNames[x.entityId]); 46 | console.log("Inside MainArea,", entities); 47 | entities = _.uniqBy(entities, 'id'); 48 | }else{ 49 | entities = []; 50 | } 51 | 52 | console.log("Inside MainArea,", entities); 53 | const EntitySet = new Set(entities.map((x, ind) => x.id)) 54 | 55 | // all the entities that are not in the text and will populate the search 56 | const otherEntities = props.classNames.filter(x => !EntitySet.has(x.id)); 57 | 58 | return ( 59 |
60 | 72 |
73 | ) 74 | } 75 | }; 76 | 77 | 78 | export default LabelComponent; 79 | -------------------------------------------------------------------------------- /app-ui/src/components/search/Result.js: -------------------------------------------------------------------------------- 1 | import PropTypes from "prop-types"; 2 | import React from "react"; 3 | 4 | import { appendClassName, getUrlSanitizer } from "@elastic/react-search-ui-views/lib/view-helpers"; 5 | import { isFieldValueWrapper } from "@elastic/react-search-ui-views/lib/types/FieldValueWrapper"; 6 | 7 | 8 | const DISPLAYED_FIELDS = ["id", "text"] 9 | 10 | function getFieldType(result, field, type) { 11 | if (result[field]) return result[field][type]; 12 | } 13 | 14 | function getRaw(result, field) { 15 | return getFieldType(result, field, "raw"); 16 | } 17 | 18 | function getSnippet(result, field) { 19 | return getFieldType(result, field, "snippet"); 20 | } 21 | 22 | function htmlEscape(str) { 23 | if (!str) return ""; 24 | 25 | return String(str) 26 | .replace(/&/g, "&") 27 | .replace(/"/g, """) 28 | .replace(/'/g, "'") 29 | .replace(//g, ">"); 31 | } 32 | 33 | function getEscapedField(result, field) { 34 | // Fallback to raw values here, because non-string fields 35 | // will not have a snippet fallback. Raw values MUST be html escaped. 36 | const safeField = 37 | getSnippet(result, field) || htmlEscape(getRaw(result, field)); 38 | return Array.isArray(safeField) ? safeField.join(", ") : safeField; 39 | } 40 | 41 | function getField(result, field) { 42 | // Fallback to raw values here, because non-string fields 43 | // will not have a snippet fallback. Raw values MUST be html escaped. 44 | const safeField = 45 | getSnippet(result, field) || getRaw(result, field); 46 | return Array.isArray(safeField) ? safeField.join(", ") : `${safeField}`; 47 | } 48 | 49 | function getEscapedFields(result) { 50 | return Object.keys(result).reduce((acc, field) => { 51 | // If we receive an arbitrary value from the response, we may not properly 52 | // handle it, so we should filter out arbitrary values here. 53 | // 54 | // I.e., 55 | // Arbitrary value: "_metaField: '1939191'" 56 | // vs. 57 | // FieldValueWrapper: "_metaField: {raw: '1939191'}" 58 | if (!isFieldValueWrapper(result[field])) return acc; 59 | // return { ...acc, [field]: getEscapedField(result, field) }; 60 | return { ...acc, [field]: getField(result, field) }; 61 | }, {}); 62 | } 63 | 64 | function Result({ 65 | className, 66 | result, 67 | onClickLink, 68 | titleField, 69 | urlField, 70 | labelComponent, 71 | getSearchTextComponent, 72 | ...rest 73 | }) { 74 | const fields = Object.fromEntries(DISPLAYED_FIELDS.map(k => [k, getEscapedFields(result)[k]])); 75 | console.log("Inside Result", result, fields); 76 | const title = getEscapedField(result, titleField); 77 | const url = getUrlSanitizer(URL, location)(getRaw(result, urlField)); 78 | 79 | return ( 80 |
  • 81 |
    82 | {title && !url && ( 83 | 87 | )} 88 | {title && url && ( 89 | 97 | )} 98 |
    99 |
    100 |
      101 | {Object.entries(fields).map(([fieldName, fieldValue]) => ( 102 |
    • 103 | {fieldName}{" "} 104 | {getSearchTextComponent(fieldValue)} 105 |
    • 106 | ))} 107 |
    108 |
    109 | {labelComponent} 110 |
  • 111 | ); 112 | } 113 | 114 | Result.propTypes = { 115 | result: PropTypes.object.isRequired, 116 | onClickLink: PropTypes.func.isRequired, 117 | className: PropTypes.string, 118 | titleField: PropTypes.string, 119 | urlField: PropTypes.string 120 | }; 121 | 122 | export default Result; -------------------------------------------------------------------------------- /app-ui/src/components/search/ResultwithLabels.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Result from "./Result"; 3 | import getSearchTextComponent from './getSearchTextComponent'; 4 | import LabelComponent from './LabelComponent'; 5 | import { PROJECT_TYPES } from '../constants'; 6 | 7 | 8 | const getcurrentDisplayedLabels = (classNames, result) => { 9 | // will return the validated spans for NER or label for classification 10 | if(result["manualLabel"]){ 11 | try{ 12 | const manualLabel = result["manualLabel"]["raw"]["label"]; 13 | if(Array.isArray(manualLabel)){ 14 | // for NER 15 | const spans = manualLabel; 16 | return {entities: spans, haveBeenValidated: true}; 17 | }else{ 18 | // for classification 19 | const entities = [classNames[manualLabel]]; 20 | return {entities: entities, haveBeenValidated: true}; 21 | } 22 | } 23 | catch(error){ 24 | console.log("Error getting manual label field from document in ES.", error) 25 | } 26 | }else{ 27 | return {entities: [], haveBeenValidated: false}; 28 | } 29 | } 30 | 31 | function getcurrentSelectedEntityId(projectType, entities) { 32 | if(projectType == PROJECT_TYPES.classification){ 33 | return entities.length > 0 && entities[0].id 34 | }else{ 35 | return undefined; 36 | } 37 | } 38 | 39 | class ResultWithLabels extends React.Component { 40 | 41 | constructor(props) { 42 | super(props); 43 | 44 | const currentDisplayedLabels = getcurrentDisplayedLabels(this.props.classNames, this.props.result); 45 | console.log("Constructor of ResultWithLabels", this.props.classNames, currentDisplayedLabels); 46 | 47 | this.state = { 48 | currentDisplayedLabels: currentDisplayedLabels.entities, // for classification - the labels suggested to user (includes validated label if it exists, otherwise nothing). for NER - the current selection of spans (not necessarily confirmed) shown to user. 49 | currentSelectedEntityId: getcurrentSelectedEntityId(props.projectType, currentDisplayedLabels.entities), //for NER & classification, the currently selected entity id 50 | isCurrentlyDisplayedValidated: currentDisplayedLabels.haveBeenValidated 51 | } 52 | } 53 | 54 | selectEntity = (entity) => { 55 | this.setState((prevState) => { 56 | console.log("Inside ResultWithLabels selectEntity", prevState.currentSelectedEntityId, entity); 57 | return {currentSelectedEntityId: (entity.id==prevState.currentSelectedEntityId)? undefined: entity.id} 58 | }); 59 | } 60 | 61 | updateIndexAfterLabelling = ({label, spans}) => { 62 | if(typeof(spans)!="undefined"){ 63 | this.setState(() => { 64 | return { currentDisplayedLabels: spans, 65 | isCurrentlyDisplayedValidated: true} 66 | }) 67 | } 68 | } 69 | 70 | addTextSpan = (span) => { 71 | console.log("Inside addTextSpan ", span); 72 | this.setState((prevState) => { 73 | let currentDisplayedLabels = prevState.currentDisplayedLabels; 74 | if(currentDisplayedLabels){ 75 | currentDisplayedLabels = currentDisplayedLabels.concat(span); 76 | } 77 | else{ 78 | currentDisplayedLabels = [span] 79 | } 80 | return { currentDisplayedLabels, isCurrentlyDisplayedValidated: false } 81 | }) 82 | } 83 | 84 | deleteTextSpan = (spanToDelete) => { 85 | this.setState((prevState) => { 86 | const currentDisplayedLabels = prevState.currentDisplayedLabels.filter(span => (span.id != spanToDelete.id)); 87 | return { currentDisplayedLabels, 88 | currentSelectedEntityId: undefined, 89 | isCurrentlyDisplayedValidated: false }; 90 | }) 91 | } 92 | 93 | render = () => { 94 | const result = this.props.result; 95 | delete result.manual_label; 96 | 97 | return( 98 | 99 | {}} 102 | labelComponent={this.props.enableLabelling && 103 | 115 | } 116 | getSearchTextComponent={(fieldValue) => 117 | getSearchTextComponent(fieldValue, 118 | this.props.projectType, 119 | this.props.classNames, 120 | this.state.currentSelectedEntityId, 121 | this.state.currentDisplayedLabels, 122 | this.addTextSpan, 123 | this.deleteTextSpan)} 124 | /> 125 | 126 | 127 | ) 128 | } 129 | } 130 | 131 | export default ResultWithLabels; -------------------------------------------------------------------------------- /app-ui/src/components/search/RuleFilters.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import Chip from '@material-ui/core/Chip'; 3 | import { withStyles } from '@material-ui/core/styles'; 4 | import InputLabel from '@material-ui/core/InputLabel'; 5 | import FormControl from '@material-ui/core/FormControl'; 6 | import Select from '@material-ui/core/Select'; 7 | import MenuItem from '@material-ui/core/MenuItem'; 8 | 9 | import * as colors from '@material-ui/core/colors'; 10 | 11 | 12 | const MAX_FILTERS_DISPLAYED_UNDERNEATH = 5; // needs to be lower than MAX_APPLIED_FILTERS in the Search component 13 | 14 | const styles = theme => ({ 15 | main_container: { 16 | marginLeft: 10, 17 | display: 'flex' 18 | }, 19 | chip: { 20 | margin: 5, 21 | color: 'white' 22 | }, 23 | form: { 24 | minWidth: 120, 25 | marginLeft: 5 26 | } 27 | }); 28 | 29 | 30 | const RuleFilters = (props) => { 31 | const { classes, ...rest } = props; 32 | 33 | return ( 34 | (props.rules.length + props.hasTables) > 0 && 35 |
    36 | {props.withText &&

    {"Filters:"}

    } 37 | {props.hasTables && !props.filterTables && } 45 | {props.rules.filter((item, index) => index < MAX_FILTERS_DISPLAYED_UNDERNEATH) 46 | .map((x, ind) => props.addAppliedRuleFilter(x['id']))} 53 | onDelete={props.removeAppliedRuleFilter && (() => props.removeAppliedRuleFilter(x['id']))} 54 | />)} 55 | 56 | {props.rules.length > MAX_FILTERS_DISPLAYED_UNDERNEATH && 57 | 58 | More 59 | 71 | 72 | } 73 |
    74 | 75 | ); 76 | } 77 | 78 | export default withStyles(styles)(RuleFilters); -------------------------------------------------------------------------------- /app-ui/src/components/search/buildRequest.js: -------------------------------------------------------------------------------- 1 | 2 | function buildFrom(current, resultsPerPage) { 3 | if (!current || !resultsPerPage) return; 4 | return (current - 1) * resultsPerPage; 5 | } 6 | 7 | function buildMatch(searchTerm) { 8 | return searchTerm 9 | ? { 10 | multi_match: { 11 | query: searchTerm, 12 | fields: ["text"] 13 | } 14 | } 15 | : { match_all: {} }; 16 | } 17 | 18 | 19 | function buildRuleFilter(appliedRuleFilters) { 20 | return appliedRuleFilters.map((rule) => { 21 | return ({ 22 | "nested": { 23 | "path": "rules", 24 | "query": { 25 | "bool": { 26 | "must": [ 27 | { 28 | "match": { 29 | "rules.rule_id": rule.id 30 | } 31 | } 32 | ] 33 | } 34 | } 35 | } 36 | }) 37 | }) 38 | } 39 | 40 | function buildTableFilter() { 41 | return { 42 | "match": { 43 | "is_table": { 44 | "query": "true" 45 | } 46 | } 47 | } 48 | } 49 | 50 | function buildFilters(appliedRuleFilters, filterTables){ 51 | let ruleFilters = buildRuleFilter(appliedRuleFilters); 52 | if(filterTables){ 53 | return ruleFilters.concat(buildTableFilter()) 54 | } 55 | return ruleFilters; 56 | } 57 | 58 | /* 59 | Converts current application state to an Elasticsearch request. 60 | When implementing an onSearch Handler in Search UI, the handler needs to take the 61 | current state of the application and convert it to an API request. 62 | For instance, there is a "current" property in the application state that you receive 63 | in this handler. The "current" property represents the current page in pagination. This 64 | method converts our "current" property to Elasticsearch's "from" parameter. 65 | This "current" property is a "page" offset, while Elasticsearch's "from" parameter 66 | is a "item" offset. In other words, for a set of 100 results and a page size 67 | of 10, if our "current" value is "4", then the equivalent Elasticsearch "from" value 68 | would be "40". This method does that conversion. 69 | We then do similar things for searchTerm, filters, sort, etc. 70 | */ 71 | export default function buildRequest(state) { 72 | const { 73 | current, 74 | resultsPerPage, 75 | searchTerm, 76 | appliedRuleFilters, 77 | filterTables 78 | } = state; 79 | 80 | const match = buildMatch(searchTerm); 81 | const filters = buildFilters(appliedRuleFilters, filterTables); 82 | const size = resultsPerPage; 83 | const from = buildFrom(current, resultsPerPage); 84 | 85 | const body = { 86 | // Static query Configuration 87 | // -------------------------- 88 | // https://www.elastic.co/guide/en/elasticsearch/reference/7.x/search-request-highlighting.html 89 | highlight: { 90 | number_of_fragments: 0, 91 | fields: { 92 | text: {} 93 | } 94 | }, 95 | //https://www.elastic.co/guide/en/elasticsearch/reference/7.x/search-request-source-filtering.html#search-request-source-filtering 96 | _source: ["id", "text", "manual_label.*"], 97 | 98 | // Dynamic values based on current Search UI state 99 | // -------------------------- 100 | // https://www.elastic.co/guide/en/elasticsearch/reference/7.x/full-text-queries.html 101 | query: { 102 | bool: { 103 | must: filters.concat(match) 104 | } 105 | }, 106 | // https://www.elastic.co/guide/en/elasticsearch/reference/7.x/search-request-from-size.html 107 | ...(size && { size }), 108 | ...(from && { from }) 109 | }; 110 | 111 | console.log("Inside buildRequest", state, body, filters); 112 | 113 | return body; 114 | } -------------------------------------------------------------------------------- /app-ui/src/components/search/buildState.js: -------------------------------------------------------------------------------- 1 | import { renameKeysToCamelCase } from '../utils'; 2 | 3 | 4 | function buildTotalPages(resultsPerPage, totalResults) { 5 | if (!resultsPerPage) return 0; 6 | if (totalResults === 0) return 1; 7 | return Math.ceil(totalResults / resultsPerPage); 8 | } 9 | 10 | function buildTotalResults(hits) { 11 | return hits.total.value; 12 | } 13 | 14 | function getHighlight(hit, fieldName) { 15 | if ( 16 | !hit.highlight || 17 | !hit.highlight[fieldName] || 18 | hit.highlight[fieldName].length < 1 19 | ) { 20 | return; 21 | } 22 | 23 | return hit.highlight[fieldName][0]; 24 | } 25 | 26 | function buildResults(hits) { 27 | const addEachKeyValueToObject = (acc, [key, value]) => ({ 28 | ...acc, 29 | [key]: value 30 | }); 31 | 32 | const toObject = (value, snippet) => { 33 | return { raw: value, ...(snippet && { snippet }) }; 34 | }; 35 | 36 | return hits.map(record => { 37 | return Object.entries(record._source) 38 | .map(([fieldName, fieldValue]) => [ 39 | fieldName, 40 | toObject(fieldValue, getHighlight(record, fieldName)) 41 | ]) 42 | .reduce(addEachKeyValueToObject, {}); 43 | }); 44 | } 45 | 46 | /* 47 | Converts an Elasticsearch response to new application state 48 | When implementing an onSearch Handler in Search UI, the handler needs to convert 49 | search results into a new application state that Search UI understands. 50 | For instance, Elasticsearch returns "hits" for search results. This maps to 51 | the "results" property in application state, which requires a specific format. So this 52 | file iterates through "hits" and reformats them to "results" that Search UI 53 | understands. 54 | We do similar things for facets and totals. 55 | */ 56 | export default function buildState(response, resultsPerPage) { 57 | const results = buildResults(response.hits.hits); 58 | const modifiedResults = renameKeysToCamelCase(results); 59 | const modifiedResultsArray = Object.keys(modifiedResults).map(function (key) { return modifiedResults[key]; }); 60 | 61 | const totalResults = buildTotalResults(response.hits); 62 | const totalPages = buildTotalPages(resultsPerPage, totalResults); 63 | 64 | console.log("Running renameKeysToCamelCase, ", results, modifiedResults, modifiedResultsArray, totalResults, 65 | totalPages); 66 | 67 | return { 68 | results: modifiedResultsArray, 69 | totalPages, 70 | totalResults 71 | }; 72 | } -------------------------------------------------------------------------------- /app-ui/src/components/search/getSearchTextComponent.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import Typography from '@material-ui/core/Typography'; 3 | import HighlightableText from '../label-page/supervised/ner/HighlightableText'; 4 | import { PROJECT_TYPES } from '../constants'; 5 | 6 | 7 | function getSearchTextComponent(fieldValue, 8 | projectType, 9 | classNames, 10 | currentSelectedEntityId, 11 | currentDisplayedLabels, 12 | addTextSpan, 13 | deleteTextSpan) { 14 | if(projectType==PROJECT_TYPES.classification){ 15 | return ( 16 | 20 | ) 21 | } 22 | 23 | if(projectType==PROJECT_TYPES.ner){ 24 | const entityColourMap = classNames.reduce(function(obj, itm) { 25 | obj[itm['id']] = itm['colour']; 26 | 27 | return obj; 28 | }, {}); 29 | 30 | const content = fieldValue.replace(//g, '').replace(/<\/em>/g, ''); 31 | return ( 32 | 33 | 41 | 42 | ) 43 | } 44 | } 45 | 46 | export default getSearchTextComponent; -------------------------------------------------------------------------------- /app-ui/src/components/search/runRequest.js: -------------------------------------------------------------------------------- 1 | export default async function runRequest(projectName, body) { 2 | console.log("Inside runRequest ", projectName, body); 3 | const response = await fetch("/api/search", { 4 | method: "POST", 5 | headers: { "content-type": "application/json" }, 6 | body: JSON.stringify({"body": body, "projectName": projectName}) 7 | }); 8 | return response.json(); 9 | } -------------------------------------------------------------------------------- /app-ui/src/components/set-project-params/ProjectParamsPage.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ClassNames from './ClassNames'; 3 | 4 | 5 | const ProjectParamsPage = (props) => { 6 | return ( 7 | 12 | ) 13 | } 14 | 15 | export default ProjectParamsPage; -------------------------------------------------------------------------------- /app-ui/src/components/utils.js: -------------------------------------------------------------------------------- 1 | import _ from 'lodash'; 2 | 3 | const processValToCamelCase = (val) => { 4 | return ( 5 | ((typeof val !== 'object') || (val === null)) ? val: 6 | Array.isArray(val)? val.map(renameKeysToCamelCase): renameKeysToCamelCase(val)) 7 | }; 8 | 9 | export function renameKeysToCamelCase(obj){ 10 | if((typeof obj !== 'object') || (obj === null)){ 11 | return obj; 12 | } 13 | else{ 14 | return (Object.fromEntries( 15 | Object.entries(obj) 16 | .map(([key, val]) => [ 17 | _.camelCase(key), processValToCamelCase(val) 18 | ]) 19 | ))}}; 20 | 21 | 22 | const processValToSnakeCase = (val) => { 23 | return ( 24 | ((typeof val !== 'object') || (val === null)) ? val: 25 | Array.isArray(val)? val.map(renameKeysToSnakeCase): renameKeysToSnakeCase(val)) 26 | }; 27 | 28 | export function renameKeysToSnakeCase(obj){ 29 | if((typeof obj !== 'object') || (obj === null)){ 30 | return obj; 31 | } 32 | else{ 33 | return (Object.fromEntries( 34 | Object.entries(obj) 35 | .map(([key, val]) => [ 36 | _.snakeCase(key), processValToSnakeCase(val) 37 | ]) 38 | ))}}; -------------------------------------------------------------------------------- /app-ui/src/components/welcome-page/WelcomePage.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Typography from '@material-ui/core/Typography'; 3 | import Box from '@material-ui/core/Box'; 4 | import SideBar from '../SideBar'; 5 | import { withStyles } from '@material-ui/core/styles'; 6 | import Card from '@material-ui/core/Card'; 7 | import CardContent from '@material-ui/core/CardContent'; 8 | import Grid from '@material-ui/core/Grid'; 9 | import { Link } from 'react-router-dom'; 10 | import { PROJECT_TYPES } from '../constants'; 11 | 12 | 13 | const styles = theme => ({ 14 | container: {marginTop: '20px', display: 'flex'}, 15 | card: { 16 | height: '150px', 17 | width: '250px', 18 | textAlign: 'center', 19 | alignItems: 'center', 20 | display: 'flex', 21 | position: 'relative' 22 | }, 23 | main_content: {marginLeft: "20px"}, 24 | new_banner: { position: 'absolute', 25 | top: '10px', 26 | right: '10px' , 27 | color: theme.palette.secondary.main, 28 | fontWeight: 'bold'} 29 | }); 30 | 31 | const projectTypes = [ 32 | {'type': PROJECT_TYPES.ner, 33 | 'explanation': 'Event Detection', 34 | 'needParams': true}, 35 | {'type': PROJECT_TYPES.ner, 36 | 'explanation': 'Event Argument Extraction', 37 | 'needParams': true}, 38 | {'type': PROJECT_TYPES.ner, 39 | 'explanation': 'Named Entity Recognition', 40 | 'needParams': true} 41 | ]; 42 | 43 | const Container = (props) => { 44 | const { classes, ...rest } = props; 45 | return () 50 | } 51 | 52 | const Item = props => { 53 | return() 54 | } 55 | 56 | const ProjectTypeCard = (props) => { 57 | const { classes } = props; 58 | return ( 59 | 60 | {props.isNew && 61 |
    62 | NEW 63 |
    64 | } 65 | 66 | 70 | {props.explanation} 71 | 72 | 73 |
    74 | ) 75 | } 76 | 77 | const ProjectTypeCards = (props) => { 78 | const { classes } = props; 79 | return ( 80 | 81 | {props.projects.map((project, index) => { 82 | return ( 83 | 84 | {props.setProjectType(project.type, !project.needParams, project.wikiData)}} 88 | > 89 | 94 | 95 | ) 96 | })} 97 | 98 | ) 99 | } 100 | 101 | class WelcomePage extends React.Component{ 102 | constructor(props){ 103 | super(props); 104 | this.props.resetCurrentProject(); 105 | } 106 | 107 | render(){ 108 | const { classes } = this.props; 109 | return ( 110 |
    111 | 112 |
    113 | 114 | FAMIE: A Fast Active Learning Framework for Multilingual Information Extraction 115 | 116 | 121 |
    122 |
    123 | ) 124 | } 125 | } 126 | 127 | export default withStyles(styles)(WelcomePage); 128 | -------------------------------------------------------------------------------- /app-ui/src/routers/ProjectStartPage.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ProjectMain from '../components/project-summary/ProjectMain'; 3 | import FileUploadMain from '../components/file-upload/FileUploadMain'; 4 | import NotFoundPage from '../components/NotFoundPage'; 5 | import ProjectParamsPage from '../components/set-project-params/ProjectParamsPage'; 6 | 7 | 8 | class ProjectStartPage extends React.Component { 9 | 10 | constructor(props){ 11 | super(props); 12 | this.props.setAllProjectVars(this.props.uri); 13 | 14 | console.log("Inside ProjectStartPage", this.props); 15 | } 16 | 17 | componentDidUpdate(prevProps, prevState) { 18 | console.log("ProjectStartPage did update", prevState, prevProps, this.props); 19 | } 20 | 21 | render() { 22 | if(!this.props.projectName){ 23 | return 24 | } 25 | 26 | if(this.props.projectUploadFinished && this.props.projectParamsFinished){ 27 | return 35 | } 36 | 37 | if(this.props.projectUploadFinished){ 38 | return 42 | } 43 | 44 | // console.log("Doing file upload", props); 45 | return 55 | } 56 | } 57 | 58 | export default ProjectStartPage; -------------------------------------------------------------------------------- /app-ui/src/styles/theme.js: -------------------------------------------------------------------------------- 1 | import { red } from '@material-ui/core/colors'; 2 | import { createMuiTheme } from '@material-ui/core/styles'; 3 | 4 | // A custom theme for this app 5 | // const theme = createMuiTheme({ 6 | // palette: { 7 | // primary: { 8 | // main: '#556cd6', 9 | // }, 10 | // secondary: { 11 | // main: '#19857b', 12 | // }, 13 | // error: { 14 | // main: red.A400, 15 | // }, 16 | // background: { 17 | // default: '#fff', 18 | // }, 19 | // }, 20 | // props: { 21 | // MuiButton: { 22 | // variant: 'contained', 23 | // color: 'primary' 24 | // }, 25 | // }, 26 | // }); 27 | 28 | // export default theme; -------------------------------------------------------------------------------- /app-ui/src/utils.js: -------------------------------------------------------------------------------- 1 | export function getSlug(projectName) { 2 | return encodeURIComponent(projectName); 3 | }; 4 | 5 | export function trimString(longString, maxLength){ 6 | let displayedName = longString.trim(); 7 | if(longString.length > maxLength){ 8 | let lastSpace = -1; 9 | for(let textCharIndex = 0; textCharIndex < maxLength; textCharIndex++) { 10 | const currentChar = longString[textCharIndex]; 11 | if(currentChar == ' '){ 12 | lastSpace = textCharIndex; 13 | } 14 | } 15 | if((lastSpace > -1)){ 16 | displayedName = displayedName.substring(0, lastSpace) + '...'; 17 | } 18 | else{ 19 | displayedName = displayedName.substring(0, maxLength).trim() + '...'; 20 | } 21 | } 22 | return displayedName; 23 | } -------------------------------------------------------------------------------- /app-ui/state-changes.txt: -------------------------------------------------------------------------------- 1 | Files in this folder are modified from https://github.com/dataqa/dataqa/tree/master/dataqa-ui 2 | -------------------------------------------------------------------------------- /app-ui/webpack.common.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | module.exports = { 4 | entry: './src/app.js', 5 | output: { 6 | path: path.join(__dirname, 'public'), 7 | filename: 'bundle.js' 8 | }, 9 | module: { 10 | rules: [{ 11 | loader: 'babel-loader', 12 | test: /\.js$/, 13 | exclude: /node_modules/, 14 | options: { 15 | plugins: ['recharts', 'lodash'] 16 | } 17 | }, { 18 | test: /\.s?css$/, 19 | use: [ 20 | 'style-loader', 21 | 'css-loader', 22 | 'sass-loader' 23 | ] 24 | }] 25 | } 26 | }; 27 | -------------------------------------------------------------------------------- /app-ui/webpack.dev.js: -------------------------------------------------------------------------------- 1 | const { merge } = require('webpack-merge'); 2 | const common = require('./webpack.common.js'); 3 | const path = require('path'); 4 | 5 | module.exports = merge(common, { 6 | mode: 'development', 7 | devtool: 'eval-cheap-module-source-map', 8 | devServer: { 9 | static: { 10 | directory: path.join(__dirname, 'public'), 11 | }, 12 | proxy: { 13 | '/api': { 14 | target: 'http://localhost:8888', 15 | secure: false 16 | } 17 | }, 18 | historyApiFallback: true, 19 | hot: true 20 | } 21 | }); 22 | -------------------------------------------------------------------------------- /app-ui/webpack.prod.js: -------------------------------------------------------------------------------- 1 | const { merge } = require('webpack-merge'); 2 | const common = require('./webpack.common.js'); 3 | 4 | module.exports = merge(common, { 5 | mode: 'production', 6 | devtool: 'source-map' 7 | }); 8 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/source/_static/style.css: -------------------------------------------------------------------------------- 1 | .wy-nav-content { 2 | max-width: 1500px !important; 3 | } 4 | -------------------------------------------------------------------------------- /docs/source/_templates/layout.html: -------------------------------------------------------------------------------- 1 | {% extends "!layout.html" %} 2 | {% block extrahead %} 3 | 4 | {% endblock %} 5 | -------------------------------------------------------------------------------- /docs/source/architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nlp-uoregon/famie/e771caa31bb6b1ffc27d6c27c4cdb4913ab94aa8/docs/source/architecture.jpg -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | sys.path.insert(0, os.path.abspath('../')) 16 | sys.path.insert(0, os.path.abspath('../..')) 17 | 18 | 19 | # -- Project information ----------------------------------------------------- 20 | 21 | project = 'famie' 22 | copyright = '2022, NLP Group at the University of Oregon' 23 | author = 'NLP Group at the University of Oregon' 24 | 25 | # The full version, including alpha/beta/rc tags 26 | release = '' 27 | 28 | 29 | # -- General configuration --------------------------------------------------- 30 | 31 | # Add any Sphinx extension module names here, as strings. They can be 32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 33 | # ones. 34 | extensions = [ 35 | 'sphinx.ext.autodoc', 36 | 'sphinx.ext.coverage', 37 | 'sphinx.ext.napoleon', 38 | 'recommonmark', 39 | 'sphinx.ext.viewcode' 40 | ] 41 | 42 | source_suffix = ['.rst', '.md'] 43 | 44 | master_doc = 'index' 45 | 46 | 47 | # Add any paths that contain templates here, relative to this directory. 48 | templates_path = ['_templates'] 49 | 50 | # List of patterns, relative to source directory, that match files and 51 | # directories to ignore when looking for source files. 52 | # This pattern also affects html_static_path and html_extra_path. 53 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 54 | 55 | 56 | # -- Options for HTML output ------------------------------------------------- 57 | 58 | # The theme to use for HTML and HTML Help pages. See the documentation for 59 | # a list of builtin themes. 60 | # 61 | html_theme = 'sphinx_rtd_theme' 62 | 63 | # Add any paths that contain custom static files (such as style sheets) here, 64 | # relative to this directory. They are copied after the builtin static files, 65 | # so a file named "default.css" will overwrite the builtin "default.css". 66 | html_static_path = ['_static'] 67 | -------------------------------------------------------------------------------- /docs/source/format.json: -------------------------------------------------------------------------------- 1 | { 2 | 'id': 2, 3 | 'text': 'On\npourra toujours parler à propos d\'Averroès de "décentrement du Sujet".', 4 | 'dspan': (149, 5 | 222), 6 | 'tokens': [ 7 | { 8 | 'id': 1, 9 | 'text': 'On', 10 | 'dspan': (149, 11 | 151), 12 | 'span': (0, 13 | 2) 14 | }, 15 | { 16 | 'id': 2, 17 | 'text': 'pourra', 18 | 'dspan': (152, 19 | 158), 20 | 'span': (3, 21 | 9) 22 | }, 23 | { 24 | 'id': 3, 25 | 'text': 'toujours', 26 | 'dspan': (159, 27 | 167), 28 | 'span': (10, 29 | 18) 30 | }, 31 | { 32 | 'id': 4, 33 | 'text': 'parler', 34 | 'dspan': (168, 35 | 174), 36 | 'span': (19, 37 | 25) 38 | }, 39 | { 40 | 'id': 5, 41 | 'text': 'à', 42 | 'dspan': (175, 43 | 176), 44 | 'span': (26, 45 | 27) 46 | }, 47 | { 48 | 'id': 6, 49 | 'text': 'propos', 50 | 'dspan': (177, 51 | 183), 52 | 'span': (28, 53 | 34) 54 | }, 55 | { 56 | 'id': 7, 57 | 'text': "d'", 58 | 'dspan': (184, 59 | 186), 60 | 'span': (35, 61 | 37) 62 | }, 63 | { 64 | 'id': 8, 65 | 'text': 'Averroès', 66 | 'dspan': (186, 67 | 194), 68 | 'span': (37, 69 | 45) 70 | }, 71 | { 72 | 'id': 9, 73 | 'text': 'de', 74 | 'dspan': (195, 75 | 197), 76 | 'span': (46, 77 | 48) 78 | }, 79 | { 80 | 'id': 10, 81 | 'text': '"', 82 | 'dspan': (198, 83 | 199), 84 | 'span': (49, 85 | 50) 86 | }, 87 | { 88 | 'id': 11, 89 | 'text': 'décentrement', 90 | 'dspan': (199, 91 | 211), 92 | 'span': (50, 93 | 62) 94 | }, 95 | { 96 | 'id': (12, 97 | 13), 98 | 'text': 'du', 99 | 'expanded': [ 100 | { 101 | 'id': 12, 102 | 'text': 'de' 103 | }, 104 | { 105 | 'id': 13, 106 | 'text': 'le' 107 | } 108 | ], 109 | 'span': (63, 110 | 65), 111 | 'dspan': (212, 112 | 214) 113 | }, 114 | { 115 | 'id': 14, 116 | 'text': 'Sujet', 117 | 'dspan': (215, 118 | 220), 119 | 'span': (66, 120 | 71) 121 | }, 122 | { 123 | 'id': 15, 124 | 'text': '"', 125 | 'dspan': (220, 126 | 221), 127 | 'span': (71, 128 | 72) 129 | }, 130 | { 131 | 'id': 16, 132 | 'text': '.', 133 | 'dspan': (221, 134 | 222), 135 | 'span': (72, 136 | 73) 137 | } 138 | ] 139 | } -------------------------------------------------------------------------------- /docs/source/howitworks.rst: -------------------------------------------------------------------------------- 1 | How FaMIE works 2 | ================= 3 | 4 | In this section, we briefly present the most important details of the technologies used by FaMIE. 5 | 6 | .. figure:: ../../pics/full-process-color.png 7 | :width: 500 8 | :alt: Proxy Active Learning process 9 | :align: center 10 | 11 | Incorporating current large-scale language models into traditional AL process would dramatically increase the model training time, thus introducing a long idle time for annotators that might reduce annotation quality and 12 | quantity. To address this issue without sacrificing final performance, FAMIE introduces **Proxy Active Learning**. In particular, a small proxy model is used to unlabeled data selection, while the main model is trained during the long annotation time of the annotators (i.e., main model training and data annotation are done in parallel). Given the main model trained at previous iteration, knowledge distillation will be employed to synchronize the knowledge between the main and proxy models at the current iteration. 13 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. trankit documentation master file, created by 2 | sphinx-quickstart on March 31 10:21:23 2021. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | FaMIE's Documentation 7 | ================================================ 8 | 9 | FAMIE is a comprehensive and efficient **active learning** (AL) toolkit for **multilingual information extraction** (IE). FAMIE is designed to address a fundamental problem in existing AL frameworks where annotators need to wait for a long time between annotation batches due to the time-consuming nature of model training and data selection at each AL iteration. With a novel `proxy AL mechanism `_ and the integration of our SOTA multilingual toolkit `Trankit `_, FAMIE can quickly provide users with a labeled dataset and a ready-to-use model for different IE tasks over `100 languages `_. 10 | 11 | FAMIE's github: https://github.com/nlp-uoregon/famie 12 | 13 | FAMIE's demo website: http://nlp.uoregon.edu:9000/ 14 | 15 | .. toctree:: 16 | :maxdepth: 2 17 | :caption: Introduction 18 | 19 | installation 20 | overview 21 | howitworks 22 | -------------------------------------------------------------------------------- /docs/source/installation.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | Installing *FaMIE* is easily done via one of the following methods: 4 | 5 | ## Using pip 6 | 7 | ``` 8 | pip install famie 9 | ``` 10 | The command would install *FaMIE* and all dependent packages automatically. 11 | 12 | ## From source 13 | ``` 14 | git clone https://github.com/nlp-uoregon/famie 15 | cd famie 16 | pip install -e . 17 | ``` 18 | This would first clone our github repo and install FaMIE. 19 | 20 | If you encounter any other problem with the installation, please raise an issue [here](https://github.com/nlp-uoregon/famie/issues/new) to let us know. Thanks. 21 | -------------------------------------------------------------------------------- /docs/source/overview.md: -------------------------------------------------------------------------------- 1 | # Quick examples 2 | 3 | ## Initialization 4 | To start an annotation session, please use the following command: 5 | ```python 6 | famie start 7 | ``` 8 | This will run a server on users' local machines (no data or models will leave users' local machines), users can access FAMIE's web interface via the URL: http://127.0.0.1:9000/ 9 | 10 | To start a new project, users need to upload an unlabeled dataset file with an entity type file (in text format) to the web interface. After that, they will be directed to a data statistic page. Clicking on the bottom left corner will start the labeling process. 11 | 12 |
    13 | 14 | ## Annotation 15 | 16 | for each data sample, annotators first select a label from dropdown, then proceed to highlight appropriate spans for the corresponding labels. 17 |
    18 |
    19 | 20 | Annotators continue labeling until all entities in the given sentence are covered, from which they can proceed by clicking save button and then next arrow to go to the next example. 21 |
    22 | 23 | After finishing labeled every unlabeled data of the current iteration, clicking on **Finish Iter** will take users to a waiting page for the next iteration (during this time, the proxy model is being retrained with the new labeled data, which usually takes about 3 to 5 minutes). 24 |
    25 | 26 | ## Output 27 | FAMIE allows users to download the trained models and annotated data of the current round via the web interface. 28 |
    29 | 30 | FAMIE also provides a simple and intuitive code 31 | interface for interacting with the resulting labeled 32 | dataset and trained main models after the AL processes. 33 | 34 | ```python 35 | import famie 36 | 37 | # access a project via its name 38 | p = famie.get_project('named-entity-recognition') 39 | 40 | # access the project's labeled data 41 | data = p.get_labeled_data() # a Python dictionary 42 | 43 | # export the project's labeled data to a file 44 | p.export_labeled_data('data.json') 45 | 46 | # export the project's trained model to a file 47 | p.export_trained_model('model.ckpt') 48 | 49 | # access the project's trained model 50 | model = p.get_trained_model() 51 | 52 | # access a trained model from file 53 | model = famie.load_model_from_file('model.ckpt') 54 | 55 | # use the trained model to make predictions 56 | model.predict('Oregon is a beautiful state!') 57 | # ['B-Location', 'O', 'O', 'O', 'O'] -------------------------------------------------------------------------------- /pics/0_newproj.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nlp-uoregon/famie/e771caa31bb6b1ffc27d6c27c4cdb4913ab94aa8/pics/0_newproj.png -------------------------------------------------------------------------------- /pics/1_select_label.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nlp-uoregon/famie/e771caa31bb6b1ffc27d6c27c4cdb4913ab94aa8/pics/1_select_label.png -------------------------------------------------------------------------------- /pics/2_anno_span.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nlp-uoregon/famie/e771caa31bb6b1ffc27d6c27c4cdb4913ab94aa8/pics/2_anno_span.png -------------------------------------------------------------------------------- /pics/3_save_next.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nlp-uoregon/famie/e771caa31bb6b1ffc27d6c27c4cdb4913ab94aa8/pics/3_save_next.png -------------------------------------------------------------------------------- /pics/4_fin_prox.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nlp-uoregon/famie/e771caa31bb6b1ffc27d6c27c4cdb4913ab94aa8/pics/4_fin_prox.png -------------------------------------------------------------------------------- /pics/download.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nlp-uoregon/famie/e771caa31bb6b1ffc27d6c27c4cdb4913ab94aa8/pics/download.png -------------------------------------------------------------------------------- /pics/full-process-color.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nlp-uoregon/famie/e771caa31bb6b1ffc27d6c27c4cdb4913ab94aa8/pics/full-process-color.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Flask>=1.1.1 2 | flask-cors==3.0.10 3 | html5lib==1.1 4 | numpy>=1.19.2 5 | requests>=2.23 6 | scikit-learn>=0.22;python_version>="3.8" 7 | scikit-learn<0.22;python_version<"3.8" 8 | adapter-transformers==2.1.0 9 | langid==1.1.6 10 | trankit==1.1.0 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | import setuptools 3 | from sys import platform 4 | import os 5 | 6 | PKG_DIR = os.path.dirname(os.path.abspath(__file__)) 7 | 8 | required = Path("requirements.txt").read_text().splitlines() 9 | 10 | HERE = Path(__file__).parent 11 | 12 | # The text of the README file 13 | README = (HERE / "README.md").read_text() 14 | 15 | def package_files(directory): 16 | paths = [] 17 | for (path, directories, filenames) in os.walk(directory): 18 | for filename in filenames: 19 | package_relative_path = Path(*Path(path).parts[2:], filename) 20 | paths.append(str(package_relative_path)) 21 | return paths 22 | 23 | 24 | extra_files = [] 25 | extra_files.extend(package_files(Path('', 'src/famie', 'config'))) 26 | 27 | extra_files.extend(["api/static/bundle.js", 28 | "api/templates/index.html"]) 29 | 30 | setuptools.setup( 31 | name="famie", 32 | version="0.3.0", 33 | author="NLP Group at the University of Oregon", 34 | author_email="thien@cs.uoregon.edu", 35 | description="FAMIE: A Fast Active Learning Framework for Multilingual Information Extraction", 36 | long_description=README, 37 | long_description_content_type="text/markdown", 38 | url="https://github.com/nlp-uoregon/famie", 39 | python_requires='>=3.6', 40 | install_requires=required, 41 | package_dir={'': 'src'}, 42 | packages=setuptools.find_packages(where='src'), 43 | package_data={"famie": extra_files}, 44 | entry_points={'console_scripts': 'famie=famie.entry_points.run_app:main'}, 45 | data_files=[('.', ["requirements.txt"])], 46 | license='GPL-3.0 License', 47 | classifiers=[ 48 | 'Development Status :: 4 - Beta', 49 | 50 | 'Intended Audience :: Developers', 51 | 'Intended Audience :: Education', 52 | 'Intended Audience :: Science/Research', 53 | 'Intended Audience :: Information Technology', 54 | 'Topic :: Scientific/Engineering', 55 | 'Topic :: Scientific/Engineering :: Artificial Intelligence', 56 | 'Topic :: Scientific/Engineering :: Information Analysis', 57 | 'Topic :: Text Processing', 58 | 'Topic :: Text Processing :: Linguistic', 59 | 'Topic :: Software Development', 60 | 'Topic :: Software Development :: Libraries', 61 | 62 | 'Programming Language :: Python :: 3.6', 63 | 'Programming Language :: Python :: 3.7', 64 | 'Programming Language :: Python :: 3.8', 65 | ], 66 | ) 67 | -------------------------------------------------------------------------------- /src/famie/api/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Date: Feb 11, 2022 3 | Mofied from:https://github.com/dataqa/dataqa/blob/master/src/dataqa/api/__init__.py 4 | ''' 5 | from flask import Flask 6 | 7 | 8 | def create_app(): 9 | app = Flask(__name__) 10 | return app 11 | -------------------------------------------------------------------------------- /src/famie/api/active_learning/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nlp-uoregon/famie/e771caa31bb6b1ffc27d6c27c4cdb4913ab94aa8/src/famie/api/active_learning/__init__.py -------------------------------------------------------------------------------- /src/famie/api/active_learning/config.py: -------------------------------------------------------------------------------- 1 | import os, json, trankit, torch 2 | from transformers import XLMRobertaTokenizer 3 | from .constants import WORKING_DIR 4 | 5 | 6 | class Config: 7 | def __init__(self, args_fpath): 8 | if os.path.exists(args_fpath): 9 | with open(args_fpath) as f: 10 | passed_args = json.load(f) 11 | else: 12 | passed_args = { 13 | 'selection': 'mnlp', 14 | 'target_embedding': 'nreimers/mMiniLMv2-L12-H384-distilled-from-XLMR-Large', 15 | 'proxy_embedding': 'nreimers/mMiniLMv2-L6-H384-distilled-from-XLMR-Large' 16 | } 17 | # print('passed args: {}'.format(passed_args)) 18 | self.cache_dir = os.path.join(WORKING_DIR, 'resource') 19 | self.target_embedding_name = passed_args['target_embedding'] 20 | self.proxy_embedding_name = passed_args['proxy_embedding'] 21 | 22 | self.trankit_tokenizer = trankit.Pipeline('english', cache_dir=os.path.join(WORKING_DIR, 'cache/trankit')) 23 | self.proxy_tokenizer = XLMRobertaTokenizer.from_pretrained(self.proxy_embedding_name, 24 | cache_dir=os.path.join(self.cache_dir, 25 | self.proxy_embedding_name), 26 | do_lower_case=False) 27 | self.target_tokenizer = XLMRobertaTokenizer.from_pretrained(self.target_embedding_name, 28 | cache_dir=os.path.join(self.cache_dir, 29 | self.target_embedding_name), 30 | do_lower_case=False) 31 | self.max_sent_length = 200 32 | 33 | self.target_reduction_factor = 4 34 | self.proxy_reduction_factor = 2 35 | self.embedding_dropout = 0.4 36 | self.hidden_num = 200 37 | 38 | self.adapter_learning_rate = 2e-4 39 | self.adapter_weight_decay = 2e-4 40 | self.learning_rate = 1e-3 41 | self.weight_decay = 1e-3 42 | 43 | self.batch_size = 16 44 | self.proxy_max_epoch = 20 45 | self.target_max_epoch = 40 46 | self.seed = 3456 47 | self.accumulate_step = 1 48 | self.grad_clipping = 4.5 49 | 50 | self.distill = True 51 | self.selection = passed_args['selection'] 52 | assert self.selection in ['random', 'bertkm', 'badge', 'mnlp'] 53 | 54 | self.num_examples_per_iter = 50 55 | 56 | self.vocabs = {} 57 | 58 | if torch.cuda.is_available(): 59 | self.use_gpu = True 60 | else: 61 | self.use_gpu = False 62 | 63 | 64 | config = Config(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'passed_args.json')) 65 | -------------------------------------------------------------------------------- /src/famie/api/active_learning/constants.py: -------------------------------------------------------------------------------- 1 | import os, json 2 | 3 | 4 | def ensure_dir(dir_path): 5 | os.makedirs(dir_path, exist_ok=True) 6 | 7 | 8 | CODE2LANG = { 9 | 'af': 'afrikaans', 'ar': 'arabic', 'hy': 'armenian', 'eu': 'basque', 'be': 'belarusian', 'bg': 'bulgarian', 10 | 'ca': 'catalan', 'zh': 'chinese', 'hr': 'croatian', 'cs': 'czech', 11 | 'da': 'danish', 'nl': 'dutch', 'en': 'english', 'et': 'estonian', 'fi': 'finnish', 'fr': 'french', 'gl': 'galician', 12 | 'de': 'german', 'el': 'greek', 'he': 'hebrew', 'hi': 'hindi', 13 | 'hu': 'hungarian', 'id': 'indonesian', 'ga': 'irish', 'it': 'italian', 'ja': 'japanese', 'kk': 'kazakh', 14 | 'ko': 'korean', 'ku': 'kurmanji', 'la': 'latin', 'lv': 'latvian', 15 | 'lt': 'lithuanian', 'mr': 'marathi', 'nn': 'norwegian-nynorsk', 'nb': 'norwegian-bokmaal', 'fa': 'persian', 16 | 'pl': 'polish', 'pt': 'portuguese', 'ro': 'romanian', 17 | 'ru': 'russian', 'sr': 'serbian', 'sk': 'slovak', 'sl': 'slovenian', 'es': 'spanish', 18 | 'sv': 'swedish', 'ta': 'tamil', 'te': 'telugu', 'tr': 'turkish', 19 | 'uk': 'ukrainian', 'ur': 'urdu', 'ug': 'uyghur', 'vi': 'vietnamese' 20 | } 21 | 22 | SUPPORTED_TASKS = { 23 | 'ner' 24 | } 25 | 26 | DEBUG = True 27 | 28 | WORKING_DIR = os.path.dirname(os.path.realpath(__file__)) 29 | DATABASE_DIR = os.path.join(WORKING_DIR, 'database') 30 | PROJECT_INFO_FPATH = os.path.join(DATABASE_DIR, 'project2info.json') 31 | LOG_DIR = os.path.join(WORKING_DIR, 'logs') 32 | PRETRAIN_DIR = os.path.join(WORKING_DIR, 'pretrain-dir') 33 | 34 | OUTPUT_DIR = os.path.join(WORKING_DIR, 'famie-output') 35 | 36 | SIGNAL_DIR = {'base': os.path.join(WORKING_DIR, 'signals')} 37 | TASK_NAME_FPATH = os.path.join(SIGNAL_DIR['base'], 'task_name.txt') 38 | 39 | ensure_dir(SIGNAL_DIR['base']) 40 | ensure_dir(DATABASE_DIR) 41 | ensure_dir(PRETRAIN_DIR) 42 | 43 | for task in SUPPORTED_TASKS: 44 | SIGNAL_DIR[task] = os.path.join(SIGNAL_DIR['base'], task) 45 | ensure_dir(SIGNAL_DIR[task]) 46 | 47 | LISTEN_TIME = 1 48 | MAX_EXAMPLES_PER_ITER = 10000 49 | 50 | STOP_CONTROLLER = 'stop-controller' 51 | 52 | PAUSE_MODEL = 'pause-model' 53 | RUN_TARGET = 'run-target' 54 | RUN_PROXY = 'run-proxy' 55 | PROXY_PREDICTS = 'proxy-predicts' 56 | TARGET_PREDICTS = 'target-predicts' 57 | 58 | SIGNALS = { 59 | STOP_CONTROLLER: 'Stop the controller', 60 | PAUSE_MODEL: 'Pause model', 61 | RUN_TARGET: 'Run the target model', 62 | RUN_PROXY: 'Run the proxy model', 63 | PROXY_PREDICTS: 'Use proxy model to make predictions', 64 | TARGET_PREDICTS: 'Use target model to make predictions' 65 | } 66 | 67 | EMBEDDING2DIM = { 68 | 'xlm-roberta-large': 1024, 69 | 'xlm-roberta-base': 768, 70 | 'nreimers/mMiniLMv2-L12-H384-distilled-from-XLMR-Large': 384, 71 | 'nreimers/mMiniLMv2-L6-H384-distilled-from-XLMR-Large': 384 72 | } 73 | 74 | PROXY_PRETRAINED_TRIGGER_MODEL_PATH = os.path.join(PRETRAIN_DIR, 'proxy.pretrained-trigger.ckpt') 75 | PROXY_PRETRAINED_ARGUMENT_MODEL_PATH = os.path.join(PRETRAIN_DIR, 'proxy.pretrained-argument.ckpt') 76 | TARGET_PRETRAINED_TRIGGER_MODEL_PATH = os.path.join(PRETRAIN_DIR, 'target.pretrained-trigger.ckpt') 77 | TARGET_PRETRAINED_ARGUMENT_MODEL_PATH = os.path.join(PRETRAIN_DIR, 'target.pretrained-argument.ckpt') 78 | 79 | if not os.path.exists(PROJECT_INFO_FPATH): 80 | with open(PROJECT_INFO_FPATH, 'w') as f: 81 | json.dump({}, f) 82 | 83 | SPAN_KEYS = ["id", "start", "end", "text", "entity_id"] 84 | 85 | PROXY_BATCH_FIELDS = [ 86 | 'example_ids', 'texts', 'tokens', 'anchors', 'piece_idxs', 'attention_masks', 'token_lens', 87 | 'label_idxs', 'token_nums', 'distill_mask', 88 | 'tch_lbl_dist', 'transitions' 89 | ] 90 | 91 | TARGET_BATCH_FIELDS = [ 92 | 'example_ids', 'texts', 'tokens', 'anchors', 'piece_idxs', 'attention_masks', 'token_lens', 93 | 'label_idxs', 'token_nums', 'distill_mask' 94 | ] 95 | 96 | AL_BATCH_FIELDS = [ 97 | 'example_ids', 'tokens', 'anchors', 'piece_idxs', 'attention_masks', 'token_lens', 98 | 'labels', 'label_idxs', 'token_nums' 99 | ] 100 | 101 | CKPT_KEYS = {'project_name', 'project_task_type', 'lang', 'embedding_name', 'hidden_num', 'vocabs', 'weights'} 102 | 103 | 104 | def convert_ckpt_to_json(ckpt_fpath): 105 | import json, torch 106 | 107 | ckpt = torch.load(ckpt_fpath) if torch.cuda.is_available() else torch.load(ckpt_fpath, map_location=torch.device('cpu')) 108 | assert set(ckpt.keys()) == CKPT_KEYS 109 | 110 | for param_name in ckpt['weights']: 111 | ckpt['weights'][param_name] = ckpt['weights'][param_name].data.cpu().numpy().tolist() 112 | 113 | return ckpt 114 | 115 | 116 | def convert_json_to_ckpt(json_fpath, use_gpu): 117 | import torch 118 | 119 | with open(json_fpath) as f: 120 | ckpt = json.load(f) 121 | 122 | assert set(ckpt.keys()) == CKPT_KEYS 123 | 124 | for param_name in ckpt['weights']: 125 | ckpt['weights'][param_name] = torch.tensor(ckpt['weights'][param_name]).cuda() if use_gpu else torch.tensor( 126 | ckpt['weights'][param_name]) 127 | 128 | return ckpt 129 | -------------------------------------------------------------------------------- /src/famie/api/active_learning/controllers.py: -------------------------------------------------------------------------------- 1 | # define controllers for active learning modules 2 | from .models import SeqLabel, ConditionalSeqLabel 3 | from .trainers import * 4 | from .utils import * 5 | 6 | import time, os 7 | import _thread as thread 8 | 9 | 10 | class Controller: 11 | def __init__(self, config, task): 12 | assert task in SUPPORTED_TASKS 13 | 14 | self.config = config 15 | self.task = task 16 | self.signal_fpath = os.path.join(SIGNAL_DIR[task], 'signal.controller.txt') 17 | 18 | self.reset_signal() 19 | 20 | self.model = None 21 | self.dataset = None 22 | self.trainer = None 23 | self.is_listening = False 24 | 25 | def reset_signal(self): 26 | with open(self.signal_fpath, 'w') as f: 27 | f.write(PAUSE_MODEL) 28 | 29 | def read_signal(self): 30 | with open(self.signal_fpath) as f: 31 | signal = f.read().strip().lower() 32 | 33 | return signal 34 | 35 | def receive_signal(self, signal): 36 | with open(self.signal_fpath, 'w') as f: 37 | f.write(signal) 38 | 39 | def stop(self): 40 | self.receive_signal(STOP_CONTROLLER) 41 | 42 | def run_proxy_model(self, unlabeled_data, project_name): 43 | self.trainer['proxy'].receive_signal(RUN_PROXY, unlabeled_data, project_name) 44 | self.trainer['target'].receive_signal(RUN_PROXY, unlabeled_data, project_name) 45 | 46 | def run_target_model(self, project_name): 47 | self.trainer['proxy'].receive_signal(RUN_TARGET, [], project_name) 48 | self.trainer['target'].receive_signal(RUN_TARGET, [], project_name) 49 | 50 | def proxy_model_predicts(self, unlabeled_data, project_name): 51 | if self.trainer['proxy'].is_trained: 52 | self.trainer['proxy'].receive_signal(PROXY_PREDICTS, unlabeled_data, project_name) 53 | if self.trainer['target'].is_trained: 54 | self.trainer['target'].receive_signal(PROXY_PREDICTS, unlabeled_data, project_name) 55 | 56 | def target_model_predicts(self, unlabeled_data, project_name): 57 | if self.trainer['proxy'].is_trained: 58 | self.trainer['proxy'].receive_signal(TARGET_PREDICTS, unlabeled_data, project_name) 59 | if self.trainer['target'].is_trained: 60 | self.trainer['target'].receive_signal(TARGET_PREDICTS, unlabeled_data, project_name) 61 | 62 | def stop_listening(self): 63 | if self.trainer: 64 | self.trainer['proxy'].receive_signal(STOP_CONTROLLER, [], None) 65 | self.trainer['target'].receive_signal(STOP_CONTROLLER, [], None) 66 | 67 | self.model = None 68 | self.dataset = None 69 | self.trainer = None 70 | self.is_listening = False 71 | # print('{} controller: stopped listening!'.format(self.task)) 72 | 73 | def listen(self, project_state): 74 | project_dir = project_state['project_dir'] 75 | project_id = project_state['project_id'] 76 | project_task_type = project_state['project_task_type'] 77 | project_annotations = project_state['annotations'] 78 | provided_labeled_data = project_state['provided_labeled_data'] 79 | 80 | if not self.is_listening: 81 | print('-' * 50) 82 | print("Loading models for project '{}'...".format(project_id)) 83 | self.is_listening = True 84 | 85 | self.config.vocabs[project_id] = { 86 | 'entity-type': {}, 'entity-label': {'O': 0} 87 | } 88 | with open(os.path.join(project_dir, 'vocabs.json')) as f: 89 | self.config.vocabs[project_id]['entity-type'] = json.load(f) 90 | 91 | for entity_type in self.config.vocabs[project_id]['entity-type']: 92 | self.config.vocabs[project_id]['entity-label']['B-{}'.format(entity_type)] = len( 93 | self.config.vocabs[project_id]['entity-label']) 94 | self.config.vocabs[project_id]['entity-label']['I-{}'.format(entity_type)] = len( 95 | self.config.vocabs[project_id]['entity-label']) 96 | 97 | self.dataset = { 98 | 'proxy': ProxyDataset(self.config, project_id, project_dir, project_annotations, provided_labeled_data), 99 | 'target': TargetDataset(self.config, project_id, project_dir, project_annotations, 100 | provided_labeled_data) 101 | } 102 | self.dataset['target'].lang = self.dataset['proxy'].lang 103 | 104 | if project_task_type == 'conditional': 105 | print('initializating ConditionalSeqLabel models...') 106 | self.model = { 107 | 'proxy': ConditionalSeqLabel(self.config, project_id, model_name='proxy'), 108 | 'target': ConditionalSeqLabel(self.config, project_id, model_name='target') 109 | } 110 | else: 111 | print('initializing SeqLabel models...') 112 | assert project_task_type == 'unconditional' 113 | self.model = { 114 | 'proxy': SeqLabel(self.config, project_id, model_name='proxy'), 115 | 'target': SeqLabel(self.config, project_id, model_name='target') 116 | } 117 | 118 | self.trainer = { 119 | 'proxy': ProxyTrainer(self.config, self.task, self.model['proxy'], self.dataset['proxy'], 120 | project_task_type), 121 | 'target': TargetTrainer(self.config, self.task, self.model['target'], self.dataset['target'], 122 | project_task_type) 123 | } 124 | 125 | thread.start_new_thread(self.trainer['proxy'].start_listening, ()) 126 | thread.start_new_thread(self.trainer['target'].start_listening, ()) 127 | print('-' * 50) 128 | else: 129 | self.dataset['proxy'].update_data(project_annotations, provided_labeled_data) 130 | self.dataset['target'].update_data(project_annotations, provided_labeled_data) 131 | 132 | 133 | if __name__ == '__main__': 134 | controller = Controller({}, 'ner') 135 | controller.listen() 136 | print('Listening ...') 137 | while True: 138 | time.sleep(1) 139 | -------------------------------------------------------------------------------- /src/famie/api/active_learning/passed_args.json: -------------------------------------------------------------------------------- 1 | {"selection": "mnlp", "proxy_embedding": "nreimers/mMiniLMv2-L6-H384-distilled-from-XLMR-Large", "target_embedding": "xlm-roberta-base"} -------------------------------------------------------------------------------- /src/famie/api/api_fns/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nlp-uoregon/famie/e771caa31bb6b1ffc27d6c27c4cdb4913ab94aa8/src/famie/api/api_fns/__init__.py -------------------------------------------------------------------------------- /src/famie/api/api_fns/project_creation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nlp-uoregon/famie/e771caa31bb6b1ffc27d6c27c4cdb4913ab94aa8/src/famie/api/api_fns/project_creation/__init__.py -------------------------------------------------------------------------------- /src/famie/api/api_fns/project_creation/common.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Date: Feb 11, 2022 3 | Mofied from:https://github.com/dataqa/dataqa/blob/master/src/dataqa/api/api_fns/project_creation/common.py 4 | ''' 5 | 6 | from abc import ABC, abstractmethod 7 | import csv 8 | import json 9 | from pathlib import Path 10 | import random 11 | 12 | from famie.api.api_fns.utils import check_file_size, get_column_names, get_decoded_stream 13 | from famie.constants import (ES_GROUND_TRUTH_NAME_FIELD, 14 | FILE_TYPE_DOCUMENTS, 15 | FILE_TYPE_DOCUMENTS_WIKI, 16 | INPUT_FILE_SPECS, 17 | MAPPINGS, 18 | PROJECT_TYPE_CLASSIFICATION, 19 | PROJECT_TYPE_NER) 20 | 21 | 22 | class UploadedFile(ABC): 23 | 24 | def __init__(self, project_type, input_data, file_type): 25 | # run input param checks 26 | if project_type in [PROJECT_TYPE_CLASSIFICATION, PROJECT_TYPE_NER]: 27 | if file_type not in [FILE_TYPE_DOCUMENTS, FILE_TYPE_DOCUMENTS_WIKI]: 28 | raise Exception(f"File type {file_type} is not supported for project type {project_type}.") 29 | 30 | if file_type == FILE_TYPE_DOCUMENTS_WIKI and project_type != PROJECT_TYPE_NER: 31 | raise Exception(f"Using urls is not supported for project type {project_type}.") 32 | 33 | self.project_type = project_type 34 | self.input_data = input_data 35 | if (type(input_data) != list): 36 | self.filename = input_data.filename 37 | self.total_documents = 0 38 | self.file_type = file_type # documents, kb or documents_wiki 39 | self.processed_data = None 40 | 41 | def do_all_file_checks(self): 42 | self.input_data = get_decoded_stream(self.input_data) 43 | 44 | def read_line(self): 45 | num_lines = 0 46 | for line in self.input_data.readlines(): 47 | line = line.strip() 48 | if line: 49 | yield line 50 | num_lines += 1 51 | 52 | def __iter__(self): 53 | for line in self.read_line(): 54 | self.total_documents += 1 55 | yield {'text': line} 56 | 57 | @abstractmethod 58 | def process_file(self, es_uri, index_name, get_row, project_full_path, spacy_binary_filepath): 59 | pass 60 | 61 | 62 | class ES_indexer(object): 63 | 64 | def __init__(self, es_uri, index_name, get_row, mapping_specs): 65 | self.es_uri = es_uri 66 | self.index_name = index_name 67 | self.mapping_es = mapping_specs["mapping_es"] 68 | self.settings_es = mapping_specs.get("settings_es") 69 | self.get_row = get_row 70 | self.bulk_line_size = 100 71 | self.num_read_lines = 0 72 | self.num_indexed_docs = 0 73 | self.current_rows = [] 74 | 75 | def create_new_index(self): 76 | create_new_index(self.es_uri, self.index_name, self.mapping_es, self.settings_es) 77 | 78 | def __enter__(self): 79 | return self 80 | 81 | def index_line(self, line): 82 | if (self.num_read_lines > 0) and (self.num_read_lines % self.bulk_line_size == 0): 83 | try: 84 | bulk_load_documents(self.es_uri, self.index_name, self.current_rows, self.num_indexed_docs) 85 | self.num_indexed_docs = self.num_read_lines 86 | self.current_rows = [] 87 | except Exception as e: 88 | print(f"Error bulk loading lines " 89 | f"{self.num_indexed_docs} to {self.num_indexed_docs + len(self.current_rows) - 1} to elasticsearch") 90 | raise 91 | 92 | new_row = self.get_row(line) 93 | self.current_rows.append(new_row) 94 | self.num_read_lines += 1 95 | 96 | def __exit__(self, type, value, traceback): 97 | # do things at exit time 98 | if self.current_rows: 99 | try: 100 | bulk_load_documents(self.es_uri, self.index_name, self.current_rows, self.num_indexed_docs) 101 | except: 102 | print(f"Error bulk loading lines " 103 | f"{self.num_indexed_docs} to {self.num_indexed_docs + len(self.current_rows) - 1} to elasticsearch") 104 | raise 105 | 106 | 107 | def bulk_load_documents(es_uri, index_name, list_docs, start_doc_ind): 108 | json_data = [] 109 | num_docs = 0 110 | for doc in list_docs: 111 | json_data.append(json.dumps({"index": {"_index": index_name, 112 | "_id": start_doc_ind + num_docs}})) 113 | if "id" not in doc: 114 | doc["id"] = start_doc_ind + num_docs 115 | json_data.append(json.dumps(doc)) 116 | num_docs += 1 117 | request_body = "\n".join(json_data) + "\n" 118 | bulk_upload(es_uri, request_body) 119 | 120 | 121 | def index_df(es_uri, index_name, df, get_row): 122 | num_lines = 100 123 | rows = [] 124 | start_doc_ind = 0 125 | 126 | for ind, row in df.iterrows(): 127 | if (ind > 0) and (ind % num_lines == 0): 128 | try: 129 | bulk_load_documents(es_uri, index_name, rows, start_doc_ind) 130 | start_doc_ind = ind 131 | rows = [] 132 | except: 133 | print(f"Error bulk loading lines " 134 | f"{start_doc_ind} to {start_doc_ind + num_lines - 1} to elasticsearch") 135 | raise 136 | 137 | new_row = get_row(row) 138 | rows.append(new_row) 139 | 140 | if rows: 141 | bulk_load_documents(es_uri, index_name, rows, start_doc_ind) 142 | 143 | 144 | def get_random_int_5_digits(): 145 | return random.randint(10000, 99999) 146 | 147 | 148 | def sanitise_string(s): 149 | return ''.join(e for e in s if (e.isalnum()) or e == '_').lower() 150 | 151 | 152 | def get_random_index_name(prefix): 153 | index_name = sanitise_string(prefix) 154 | suffix = get_random_int_5_digits() 155 | index_name = f"{index_name}_{suffix}" 156 | return index_name 157 | 158 | 159 | def get_upload_key(project_type, file_type): 160 | if file_type not in INPUT_FILE_SPECS[project_type]: 161 | raise Exception(f"File type {file_type} not supported for project of type {project_type}.") 162 | return INPUT_FILE_SPECS[project_type][file_type]['upload_key'] 163 | 164 | 165 | def check_column_names(file, column_names): 166 | actual_column_names = get_column_names(file) 167 | for column_name in column_names: 168 | if not column_name in actual_column_names: 169 | raise Exception(f"File needs to contain a \"{column_name}\" column") 170 | return actual_column_names 171 | -------------------------------------------------------------------------------- /src/famie/api/api_fns/project_settings/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nlp-uoregon/famie/e771caa31bb6b1ffc27d6c27c4cdb4913ab94aa8/src/famie/api/api_fns/project_settings/__init__.py -------------------------------------------------------------------------------- /src/famie/api/api_fns/project_settings/supervised.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Date: Feb 11, 2022 3 | Mofied from:https://github.com/dataqa/dataqa/blob/master/src/dataqa/api/api_fns/project_settings/supervised.py 4 | ''' 5 | from itertools import cycle 6 | from famie.api.api_fns.utils import check_file_size, get_decoded_stream 7 | from famie.constants import COLOURS 8 | import json 9 | 10 | 11 | def add_class_colours(class_names): 12 | for class_colour, class_item in zip(cycle(COLOURS), class_names): 13 | class_item['colour'] = class_colour 14 | 15 | 16 | def get_class_names(file_bytes): 17 | file = get_decoded_stream(file_bytes) 18 | lines = [line.strip() for line in file.readlines() if line.strip()] 19 | class_names = [] 20 | for line_ind, line in enumerate(lines): 21 | class_name = line 22 | if len(class_name) == 0: 23 | raise Exception(f"There is an empty class name on line {line_ind + 1}.") 24 | class_names.append({"id": line_ind, "name": class_name}) 25 | 26 | add_class_colours(class_names) 27 | return class_names 28 | 29 | 30 | def set_class_names(file_bytes): 31 | class_names = get_class_names(file_bytes) 32 | return class_names 33 | 34 | 35 | def convert_to_bio2(ori_tags): 36 | bio2_tags = [] 37 | for i, tag in enumerate(ori_tags): 38 | if tag == 'O': 39 | bio2_tags.append(tag) 40 | elif tag[0] == 'I': 41 | if i == 0 or ori_tags[i - 1] == 'O' or ori_tags[i - 1][1:] != tag[1:]: 42 | bio2_tags.append('B' + tag[1:]) 43 | else: 44 | bio2_tags.append(tag) 45 | else: 46 | bio2_tags.append(tag) 47 | return bio2_tags 48 | 49 | 50 | def get_example_from_lines(sent_lines): 51 | tokens = [] 52 | ner_tags = [] 53 | for line in sent_lines: 54 | array = line.split() 55 | assert len(array) >= 2 56 | tokens.append(array[0]) 57 | ner_tags.append(array[1]) 58 | ner_tags = convert_to_bio2(ner_tags) 59 | return {'tokens': [{'text': t} for t in tokens], 'labels': ner_tags} 60 | 61 | 62 | def get_examples_from_bio_fpath(raw_lines): 63 | sent_lines = [] 64 | bio2_examples = [] 65 | for line in raw_lines: 66 | line = line.strip() 67 | if '-DOCSTART-' in line: 68 | continue 69 | if len(line) > 0: 70 | array = line.split() 71 | if len(array) < 2: 72 | continue 73 | else: 74 | sent_lines.append(line) 75 | elif len(sent_lines) > 0: 76 | example = get_example_from_lines(sent_lines) 77 | bio2_examples.append(example) 78 | bio2_examples[-1]['example_id'] = 'provided-example-{}'.format(len(bio2_examples)) 79 | 80 | sent_lines = [] 81 | 82 | if len(sent_lines) > 0: 83 | bio2_examples.append(get_example_from_lines(sent_lines)) 84 | bio2_examples[-1]['example_id'] = 'provided-example-{}'.format(len(bio2_examples)) 85 | 86 | return bio2_examples 87 | 88 | 89 | def parse_labeled_data(file_bytes, project_task_type): 90 | file = get_decoded_stream(file_bytes) 91 | lines = [line.strip() for line in file.readlines()] 92 | if len(lines) == 0: 93 | raise Exception("Provided data is empty.") 94 | try: 95 | d = json.loads(lines[0]) 96 | data_format = 'json' 97 | except json.decoder.JSONDecodeError: 98 | data_format = 'BIO' 99 | 100 | if data_format == 'BIO': 101 | # labeled data in BIO format 102 | provided_labeled_data = get_examples_from_bio_fpath(lines) 103 | for example in provided_labeled_data: 104 | example['project_task_type'] = 'unconditional' 105 | example['anchor'] = -1 106 | example['anchor_type'] = 'unknown' 107 | else: 108 | if project_task_type == 'unconditional': 109 | provided_labeled_data = [] 110 | for line in lines: 111 | if line: 112 | example = json.loads(line) 113 | inst = { 114 | 'example_id': 'provided-example-{}'.format(len(provided_labeled_data)), 115 | 'text': example['text'], 116 | 'tokens': example['tokens'], 117 | 'anchor': -1, 118 | 'anchor_type': 'unknown' 119 | } 120 | inst['labels'] = ['O'] * len(example['tokens']) 121 | for event in example['event_mentions']: 122 | trigger_start, trigger_end, event_type = event['trigger'] 123 | if trigger_end > len(inst['tokens']): 124 | continue 125 | inst['labels'][trigger_start] = 'B-{}'.format(event_type) 126 | for k in range(trigger_start + 1, trigger_end): 127 | inst['labels'][k] = 'I-{}'.format(event_type) 128 | 129 | provided_labeled_data.append(inst) 130 | else: 131 | assert project_task_type == 'conditional' 132 | provided_labeled_data = [] 133 | for line in lines: 134 | if line: 135 | example = json.loads(line) 136 | 137 | for event in example['event_mentions']: 138 | trigger_start, trigger_end, event_type = event['trigger'] 139 | 140 | inst = { 141 | 'example_id': 'provided-example-{}'.format(len(provided_labeled_data)), 142 | 'text': example['text'], 143 | 'tokens': example['tokens'], 144 | 'anchor': trigger_start, 145 | 'anchor_type': event_type 146 | } 147 | 148 | inst['labels'] = ['O'] * len(example['tokens']) 149 | 150 | for argument in event['arguments']: 151 | arg_start, arg_end, arg_role = argument 152 | if arg_end > len(inst['tokens']): 153 | continue 154 | 155 | inst['labels'][arg_start] = 'B-{}'.format(arg_role) 156 | for k in range(arg_start + 1, arg_end): 157 | inst['labels'][k] = 'I-{}'.format(arg_role) 158 | 159 | provided_labeled_data.append(inst) 160 | 161 | return provided_labeled_data 162 | -------------------------------------------------------------------------------- /src/famie/api/api_fns/utils.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Date: Feb 11, 2022 3 | Mofied from:https://github.com/dataqa/dataqa/blob/master/src/dataqa/api/api_fns/utils.py 4 | ''' 5 | import io 6 | import os 7 | 8 | 9 | def get_column_names(file): 10 | first_line = file.readline() 11 | try: 12 | column_names = first_line.strip("\n").split(',') 13 | except: 14 | raise Exception("Need to load a csv file") 15 | file.seek(0) 16 | return column_names 17 | 18 | 19 | def get_decoded_stream(file_bytes): 20 | file = io.TextIOWrapper(file_bytes, encoding='utf-8') 21 | return file 22 | 23 | 24 | def check_file_size(file): 25 | lines = [line.strip() for line in file.readlines() if line.strip()] 26 | if len(lines) == 0: 27 | raise Exception("File is empty") 28 | -------------------------------------------------------------------------------- /src/famie/api/blueprints/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nlp-uoregon/famie/e771caa31bb6b1ffc27d6c27c4cdb4913ab94aa8/src/famie/api/blueprints/__init__.py -------------------------------------------------------------------------------- /src/famie/api/blueprints/common.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Date: Feb 11, 2022 3 | Mofied from:https://github.com/dataqa/dataqa/blob/master/src/dataqa/api/blueprints/common.py 4 | ''' 5 | from datetime import datetime 6 | from distutils.util import strtobool 7 | import json 8 | import shutil 9 | 10 | from flask import (current_app, 11 | render_template, 12 | request, 13 | Blueprint) 14 | 15 | from famie.api.api_fns.project_creation.common import get_upload_key 16 | 17 | from famie.api.api_fns.project_creation.supervised import create_supervised_project 18 | 19 | from famie.constants import (ALL_PROJECT_TYPES, 20 | PROJECT_TYPE_CLASSIFICATION, 21 | PROJECT_TYPE_ED, 22 | PROJECT_TYPE_NER) 23 | from famie.api.active_learning.controllers import * 24 | from famie.api.active_learning.config import config 25 | 26 | bp = Blueprint('api', __name__) 27 | 28 | al_controllers = {'active_task': 'ner', 'active_project': 'asdf0qweo23904123ieaewklf'} 29 | for supported_task in SUPPORTED_TASKS: 30 | al_controllers[supported_task] = Controller(config, task=supported_task) 31 | 32 | 33 | @bp.route("/") 34 | def index(): 35 | return render_template('index.html') 36 | 37 | 38 | @bp.route('/', defaults={'path': ''}) 39 | @bp.route('/') 40 | def catch_all(path): 41 | return render_template('index.html') 42 | 43 | 44 | @bp.route('/hello') 45 | def hello_world(): 46 | return 'Hello, World!' 47 | 48 | 49 | @bp.route('/api/upload', methods=['POST']) 50 | def upload(): 51 | start_time = datetime.now() 52 | file = request.files['file'] 53 | if not file: 54 | raise Exception("File is undefined") 55 | 56 | project_name = request.form['project_name'] 57 | if not project_name: 58 | raise Exception("Project name undefined") 59 | 60 | try: 61 | column_name_mapping = json.loads(request.form['column_names']) 62 | except: 63 | raise Exception(f"Column name mapping not passed correctly") 64 | if not column_name_mapping: 65 | raise Exception("Column names undefined") 66 | 67 | project_type = request.form['project_type'] 68 | if not project_type: 69 | raise Exception("Project type undefined") 70 | 71 | try: 72 | file_type = request.form['file_type'] 73 | upload_key = get_upload_key(project_type, file_type) 74 | except: 75 | raise Exception(f"File type undefined or incorrect") 76 | 77 | upload_id = request.form[upload_key] 78 | if not upload_id: 79 | raise Exception("Need upload id for polling") 80 | 81 | if project_type not in ALL_PROJECT_TYPES: 82 | raise Exception(f"Non-recognised project type {project_type}. " 83 | f"Needs to be one of: {ALL_PROJECT_TYPES}") 84 | 85 | class_names = None 86 | 87 | input_params = (config, 88 | detect_lang, 89 | file, 90 | project_name, 91 | project_type, 92 | upload_id, 93 | file_type) 94 | 95 | upload_folder = os.path.join(DATABASE_DIR, project_name) 96 | ensure_dir(upload_folder) 97 | project_id, project_task_type = create_supervised_project(*input_params, upload_folder) 98 | 99 | update_project_to_database(project_info={ 100 | 'id': project_id, 'name': project_name, 'type': project_type, 'project_task_type': project_task_type, 101 | 'classes': [], 'index_name': project_name, 'filename': os.path.join(upload_folder, 'unlabeled-data.json') 102 | }) 103 | 104 | if project_id: 105 | end_time = datetime.now() 106 | print(f"Uploading took {(end_time - start_time).seconds / 60} minutes.") 107 | print('Uploading... done!') 108 | return json.dumps({"id": project_id, "class_names": class_names}) 109 | 110 | 111 | @bp.route('/api/delete-project/', methods=['DELETE']) 112 | def delete_project(project_name): 113 | if not project_name: 114 | raise Exception("Project name undefined") 115 | upload_folder = os.path.join(DATABASE_DIR, project_name) 116 | shutil.rmtree(upload_folder, ignore_errors=True) 117 | active_task = al_controllers['active_task'] 118 | if project_name == al_controllers['active_project']: 119 | al_controllers[active_task].stop_listening() 120 | return "success" 121 | 122 | 123 | @bp.route('/api/get-projects', methods=['GET']) 124 | def get_projects(): 125 | # stop all controllers when users see the list of projects 126 | for task in SUPPORTED_TASKS: 127 | al_controllers[task].stop_listening() 128 | 129 | project_list = get_project_list() 130 | return json.dumps(project_list) 131 | 132 | 133 | @bp.route('/api/export-labels', methods=['POST']) 134 | def export_labels_api(): 135 | project_name = request.form['project_name'] 136 | if not project_name: 137 | raise Exception("Project name undefined") 138 | 139 | labeled_fpath = os.path.join(OUTPUT_DIR, project_name, 'labeled-data.json') 140 | if not os.path.exists(labeled_fpath): 141 | print('{} does not exist!'.format(labeled_fpath)) 142 | return '' 143 | else: 144 | with open(labeled_fpath) as f: 145 | data = f.read().strip() 146 | return data 147 | -------------------------------------------------------------------------------- /src/famie/api/static/bundle.js.LICENSE.txt: -------------------------------------------------------------------------------- 1 | /* 2 | object-assign 3 | (c) Sindre Sorhus 4 | @license MIT 5 | */ 6 | 7 | /* @license 8 | Papa Parse 9 | v5.3.1 10 | https://github.com/mholt/PapaParse 11 | License: MIT 12 | */ 13 | 14 | /*! 15 | Copyright (c) 2018 Jed Watson. 16 | Licensed under the MIT License (MIT), see 17 | http://jedwatson.github.io/classnames 18 | */ 19 | 20 | /*! 21 | * Sizzle CSS Selector Engine v2.3.6 22 | * https://sizzlejs.com/ 23 | * 24 | * Copyright JS Foundation and other contributors 25 | * Released under the MIT license 26 | * https://js.foundation/ 27 | * 28 | * Date: 2021-02-16 29 | */ 30 | 31 | /*! 32 | * jQuery JavaScript Library v3.6.0 33 | * https://jquery.com/ 34 | * 35 | * Includes Sizzle.js 36 | * https://sizzlejs.com/ 37 | * 38 | * Copyright OpenJS Foundation and other contributors 39 | * Released under the MIT license 40 | * https://jquery.org/license 41 | * 42 | * Date: 2021-03-02T17:08Z 43 | */ 44 | 45 | /*! decimal.js-light v2.5.1 https://github.com/MikeMcl/decimal.js-light/LICENCE */ 46 | 47 | /** 48 | * A better abstraction over CSS. 49 | * 50 | * @copyright Oleg Isonen (Slobodskoi) / Isonen 2014-present 51 | * @website https://github.com/cssinjs/jss 52 | * @license MIT 53 | */ 54 | 55 | /** @license React v0.13.6 56 | * scheduler.production.min.js 57 | * 58 | * Copyright (c) Facebook, Inc. and its affiliates. 59 | * 60 | * This source code is licensed under the MIT license found in the 61 | * LICENSE file in the root directory of this source tree. 62 | */ 63 | 64 | /** @license React v16.13.1 65 | * react-is.production.min.js 66 | * 67 | * Copyright (c) Facebook, Inc. and its affiliates. 68 | * 69 | * This source code is licensed under the MIT license found in the 70 | * LICENSE file in the root directory of this source tree. 71 | */ 72 | 73 | /** @license React v16.8.0 74 | * react-dom.production.min.js 75 | * 76 | * Copyright (c) Facebook, Inc. and its affiliates. 77 | * 78 | * This source code is licensed under the MIT license found in the 79 | * LICENSE file in the root directory of this source tree. 80 | */ 81 | 82 | /** @license React v16.8.0 83 | * react.production.min.js 84 | * 85 | * Copyright (c) Facebook, Inc. and its affiliates. 86 | * 87 | * This source code is licensed under the MIT license found in the 88 | * LICENSE file in the root directory of this source tree. 89 | */ 90 | 91 | /**! 92 | * @fileOverview Kickass library to create and place poppers near their reference elements. 93 | * @version 1.16.1-lts 94 | * @license 95 | * Copyright (c) 2016 Federico Zivolo and contributors 96 | * 97 | * Permission is hereby granted, free of charge, to any person obtaining a copy 98 | * of this software and associated documentation files (the "Software"), to deal 99 | * in the Software without restriction, including without limitation the rights 100 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 101 | * copies of the Software, and to permit persons to whom the Software is 102 | * furnished to do so, subject to the following conditions: 103 | * 104 | * The above copyright notice and this permission notice shall be included in all 105 | * copies or substantial portions of the Software. 106 | * 107 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 108 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 109 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 110 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 111 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 112 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 113 | * SOFTWARE. 114 | */ 115 | -------------------------------------------------------------------------------- /src/famie/api/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | FaMIE 10 | 11 | 12 | 13 |
    14 | 15 | 16 | -------------------------------------------------------------------------------- /src/famie/api/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | FAMIE's Demo Website 10 | 11 | 12 | 13 |
    14 | 15 | 16 | -------------------------------------------------------------------------------- /src/famie/config/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nlp-uoregon/famie/e771caa31bb6b1ffc27d6c27c4cdb4913ab94aa8/src/famie/config/__init__.py -------------------------------------------------------------------------------- /src/famie/config/common_config.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | ES_HOST = http://localhost 3 | FLASK_PORT = 9000 4 | -------------------------------------------------------------------------------- /src/famie/config/config_reader.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Date: Feb 11, 2022 3 | Mofied from:https://github.com/dataqa/dataqa/blob/master/src/dataqa/config/config_reader.py 4 | ''' 5 | import configparser 6 | from famie.constants import HOME, ROOT_PATH 7 | from pathlib import Path 8 | 9 | APP_CONFIG_FILE = str(Path(ROOT_PATH, "config/common_config.ini")) 10 | 11 | 12 | def read_config(platform_config_path=None): 13 | config = configparser.ConfigParser() 14 | config.optionxform = str 15 | config["DEFAULT"]["home"] = HOME 16 | config["DEFAULT"]["root"] = ROOT_PATH 17 | config.read(APP_CONFIG_FILE) 18 | if platform_config_path is not None: 19 | config.read(platform_config_path) 20 | return config 21 | -------------------------------------------------------------------------------- /src/famie/entry_points/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nlp-uoregon/famie/e771caa31bb6b1ffc27d6c27c4cdb4913ab94aa8/src/famie/entry_points/__init__.py -------------------------------------------------------------------------------- /src/famie/entry_points/run_app.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from famie.scripts import start_app 3 | 4 | def command_start(args): 5 | print("Starting a new session...") 6 | start_app.main(args) 7 | 8 | 9 | def main(): 10 | parser = argparse.ArgumentParser( 11 | description="FAMIE: A Fast Active Learning Framework for Multilingual Information Extraction.") 12 | subparsers = parser.add_subparsers() 13 | parser_start = subparsers.add_parser("start", help="Subparser for creating a new session.") 14 | parser_start.add_argument("--selection", 15 | type=str, 16 | default="mnlp", 17 | help="Data selection strategy", 18 | choices=['mnlp', 'bertkm', 'badge', 'random']) 19 | parser_start.add_argument("--port", 20 | type=str, 21 | default="9000", 22 | help="Port specification") 23 | parser_start.add_argument("--target_embedding", 24 | type=str, 25 | default='xlm-roberta-base', 26 | help="Pretrained language model for the main model, default='xlm-roberta-large'", 27 | choices=['xlm-roberta-base', 'xlm-roberta-large']) 28 | parser_start.add_argument("--proxy_embedding", 29 | type=str, 30 | default='nreimers/mMiniLMv2-L6-H384-distilled-from-XLMR-Large', 31 | help="Pretrained Language Model for the proxy model, default='nreimers/mMiniLMv2-L6-H384-distilled-from-XLMR-Large'", 32 | choices=['nreimers/mMiniLMv2-L6-H384-distilled-from-XLMR-Large', 33 | 'nreimers/mMiniLMv2-L12-H384-distilled-from-XLMR-Large']) 34 | parser_start.set_defaults(handler=command_start) 35 | 36 | ''' 37 | if args.action == 'run': 38 | start_app.main() 39 | elif args.action == 'uninstall': 40 | uninstall_app.main(config) 41 | ''' 42 | args = parser.parse_args() 43 | if hasattr(args, "handler"): 44 | args.handler(args) 45 | 46 | 47 | if __name__ == "__main__": 48 | main() 49 | -------------------------------------------------------------------------------- /src/famie/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nlp-uoregon/famie/e771caa31bb6b1ffc27d6c27c4cdb4913ab94aa8/src/famie/scripts/__init__.py -------------------------------------------------------------------------------- /src/famie/scripts/start_app.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Date: Feb 11, 2022 3 | Mofied from:https://github.com/dataqa/dataqa/blob/master/src/dataqa/scripts/start_app.py 4 | ''' 5 | import argparse 6 | import os, json 7 | import webbrowser 8 | from famie.config.config_reader import read_config 9 | from famie.api.active_learning.constants import WORKING_DIR 10 | from pathlib import Path 11 | from flask_cors import CORS 12 | 13 | 14 | def main(args): 15 | ######### passing arguments ######## 16 | with open(os.path.join(WORKING_DIR, 'passed_args.json'), 'w') as f: 17 | json.dump({ 18 | 'selection': args.selection, 19 | 'proxy_embedding': args.proxy_embedding, 20 | 'target_embedding': args.target_embedding 21 | }, f) 22 | #################################### 23 | 24 | config = read_config() 25 | 26 | from famie.api import create_app 27 | 28 | application = create_app() 29 | 30 | from famie.api.blueprints.common import bp 31 | from famie.api.blueprints.supervised import supervised_bp 32 | 33 | application.register_blueprint(bp) 34 | application.register_blueprint(supervised_bp) 35 | 36 | application.config.from_mapping(config.items("DEFAULT")) 37 | 38 | print('FAMIE`s Web Interface is available at: http://127.0.0.1:{}/'.format(args.port)) 39 | print('-' * 50) 40 | 41 | CORS(application) 42 | 43 | application.run(debug=False, 44 | port=args.port, 45 | host='0.0.0.0') 46 | -------------------------------------------------------------------------------- /src/famie/scripts/uninstall_app.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Date: Feb 11, 2022 3 | Mofied from:https://github.com/dataqa/dataqa/blob/master/src/dataqa/scripts/uninstall_app.py 4 | ''' 5 | import os 6 | import shutil 7 | 8 | 9 | def main(config): 10 | upload_folder = config["DEFAULT"]["UPLOAD_FOLDER"] 11 | 12 | if os.path.exists(upload_folder): 13 | reply = input(f"Delete directory {upload_folder}? [y/[n]] ") 14 | if reply.lower().strip() == "y": 15 | shutil.rmtree(upload_folder) 16 | else: 17 | print("Doing nothing") 18 | 19 | 20 | if __name__ == "__main__": 21 | main() 22 | --------------------------------------------------------------------------------