├── docs ├── .nojekyll ├── public │ ├── .nojekyll │ ├── objects.inv │ ├── _static │ │ ├── up.png │ │ ├── down.png │ │ ├── file.png │ │ ├── plus.png │ │ ├── comment.png │ │ ├── minus.png │ │ ├── ajax-loader.gif │ │ ├── down-pressed.png │ │ ├── up-pressed.png │ │ ├── comment-bright.png │ │ ├── comment-close.png │ │ ├── fonts │ │ │ ├── Lato │ │ │ │ ├── lato-bold.eot │ │ │ │ ├── lato-bold.ttf │ │ │ │ ├── lato-bold.woff │ │ │ │ ├── lato-bold.woff2 │ │ │ │ ├── lato-italic.eot │ │ │ │ ├── lato-italic.ttf │ │ │ │ ├── lato-italic.woff │ │ │ │ ├── lato-italic.woff2 │ │ │ │ ├── lato-regular.eot │ │ │ │ ├── lato-regular.ttf │ │ │ │ ├── lato-regular.woff │ │ │ │ ├── lato-bolditalic.eot │ │ │ │ ├── lato-bolditalic.ttf │ │ │ │ ├── lato-bolditalic.woff │ │ │ │ ├── lato-regular.woff2 │ │ │ │ └── lato-bolditalic.woff2 │ │ │ ├── fontawesome-webfont.eot │ │ │ ├── fontawesome-webfont.ttf │ │ │ ├── fontawesome-webfont.woff │ │ │ ├── fontawesome-webfont.woff2 │ │ │ └── RobotoSlab │ │ │ │ ├── roboto-slab-v7-bold.eot │ │ │ │ ├── roboto-slab-v7-bold.ttf │ │ │ │ ├── roboto-slab-v7-bold.woff │ │ │ │ ├── roboto-slab-v7-bold.woff2 │ │ │ │ ├── roboto-slab-v7-regular.eot │ │ │ │ ├── roboto-slab-v7-regular.ttf │ │ │ │ ├── roboto-slab-v7-regular.woff │ │ │ │ └── roboto-slab-v7-regular.woff2 │ │ ├── css │ │ │ └── badge_only.css │ │ ├── js │ │ │ └── theme.js │ │ ├── pygments.css │ │ └── doctools.js │ ├── .buildinfo │ ├── _sources │ │ ├── package_reference.rst.txt │ │ ├── submission.rst.txt │ │ ├── index.rst.txt │ │ ├── faq.rst.txt │ │ ├── evaluations.rst.txt │ │ └── datasets.rst.txt │ ├── genindex.html │ ├── search.html │ ├── searchindex.js │ ├── _modules │ │ └── index.html │ ├── py-modindex.html │ ├── submission.html │ └── faq.html ├── requirements.txt ├── Makefile └── source │ ├── submission.rst │ ├── faq.rst │ ├── index.rst │ ├── evaluations.rst │ ├── conf.py │ └── datasets.rst ├── noesis ├── __init__.py ├── util │ ├── __init__.py │ └── checkpoint.py ├── dataset │ ├── __init__.py │ ├── utils.py │ ├── vocabulary.py │ └── dataset.py ├── evaluator │ ├── __init__.py │ └── evaluator.py ├── networks │ ├── __init__.py │ └── dual_encoder.py ├── trainers │ ├── __init__.py │ └── supervised_trainer.py └── sample.py ├── evaluations ├── __init__.py └── metrics.py ├── noesis-tf ├── models │ ├── __init__.py │ ├── helpers.py │ └── dual_encoder.py ├── util │ ├── __init__.py │ └── blocks.py ├── images │ └── architecture.png ├── metrics.py ├── inputs.py ├── train.py ├── hparams.py ├── README.md ├── model.py └── scripts │ └── prepare_data.py ├── requirements.txt ├── LICENSE ├── .gitignore ├── README.md └── setup.py /docs/.nojekyll: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /noesis/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/public/.nojekyll: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /evaluations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /noesis/util/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /noesis-tf/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /noesis-tf/util/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /noesis/dataset/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /noesis/evaluator/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /noesis/networks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /noesis/trainers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx 2 | sphinx_rtd_theme 3 | recommonmark -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | torch 2 | dill 3 | ijson 4 | tensorflow 5 | numpy -------------------------------------------------------------------------------- /docs/public/objects.inv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/objects.inv -------------------------------------------------------------------------------- /docs/public/_static/up.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/up.png -------------------------------------------------------------------------------- /docs/public/_static/down.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/down.png -------------------------------------------------------------------------------- /docs/public/_static/file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/file.png -------------------------------------------------------------------------------- /docs/public/_static/plus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/plus.png -------------------------------------------------------------------------------- /docs/public/_static/comment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/comment.png -------------------------------------------------------------------------------- /docs/public/_static/minus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/minus.png -------------------------------------------------------------------------------- /noesis-tf/images/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/noesis-tf/images/architecture.png -------------------------------------------------------------------------------- /docs/public/_static/ajax-loader.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/ajax-loader.gif -------------------------------------------------------------------------------- /docs/public/_static/down-pressed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/down-pressed.png -------------------------------------------------------------------------------- /docs/public/_static/up-pressed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/up-pressed.png -------------------------------------------------------------------------------- /docs/public/_static/comment-bright.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/comment-bright.png -------------------------------------------------------------------------------- /docs/public/_static/comment-close.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/comment-close.png -------------------------------------------------------------------------------- /docs/public/_static/fonts/Lato/lato-bold.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/Lato/lato-bold.eot -------------------------------------------------------------------------------- /docs/public/_static/fonts/Lato/lato-bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/Lato/lato-bold.ttf -------------------------------------------------------------------------------- /docs/public/_static/fonts/Lato/lato-bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/Lato/lato-bold.woff -------------------------------------------------------------------------------- /docs/public/_static/fonts/Lato/lato-bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/Lato/lato-bold.woff2 -------------------------------------------------------------------------------- /docs/public/_static/fonts/Lato/lato-italic.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/Lato/lato-italic.eot -------------------------------------------------------------------------------- /docs/public/_static/fonts/Lato/lato-italic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/Lato/lato-italic.ttf -------------------------------------------------------------------------------- /docs/public/_static/fonts/Lato/lato-italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/Lato/lato-italic.woff -------------------------------------------------------------------------------- /docs/public/_static/fonts/Lato/lato-italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/Lato/lato-italic.woff2 -------------------------------------------------------------------------------- /docs/public/_static/fonts/Lato/lato-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/Lato/lato-regular.eot -------------------------------------------------------------------------------- /docs/public/_static/fonts/Lato/lato-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/Lato/lato-regular.ttf -------------------------------------------------------------------------------- /docs/public/_static/fonts/Lato/lato-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/Lato/lato-regular.woff -------------------------------------------------------------------------------- /docs/public/_static/fonts/Lato/lato-bolditalic.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/Lato/lato-bolditalic.eot -------------------------------------------------------------------------------- /docs/public/_static/fonts/Lato/lato-bolditalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/Lato/lato-bolditalic.ttf -------------------------------------------------------------------------------- /docs/public/_static/fonts/Lato/lato-bolditalic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/Lato/lato-bolditalic.woff -------------------------------------------------------------------------------- /docs/public/_static/fonts/Lato/lato-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/Lato/lato-regular.woff2 -------------------------------------------------------------------------------- /docs/public/_static/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /docs/public/_static/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /docs/public/_static/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /docs/public/_static/fonts/fontawesome-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/fontawesome-webfont.woff2 -------------------------------------------------------------------------------- /docs/public/_static/fonts/Lato/lato-bolditalic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/Lato/lato-bolditalic.woff2 -------------------------------------------------------------------------------- /docs/public/_static/fonts/RobotoSlab/roboto-slab-v7-bold.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/RobotoSlab/roboto-slab-v7-bold.eot -------------------------------------------------------------------------------- /docs/public/_static/fonts/RobotoSlab/roboto-slab-v7-bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/RobotoSlab/roboto-slab-v7-bold.ttf -------------------------------------------------------------------------------- /docs/public/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff -------------------------------------------------------------------------------- /docs/public/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff2 -------------------------------------------------------------------------------- /docs/public/_static/fonts/RobotoSlab/roboto-slab-v7-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/RobotoSlab/roboto-slab-v7-regular.eot -------------------------------------------------------------------------------- /docs/public/_static/fonts/RobotoSlab/roboto-slab-v7-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/RobotoSlab/roboto-slab-v7-regular.ttf -------------------------------------------------------------------------------- /docs/public/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff -------------------------------------------------------------------------------- /docs/public/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/dstc-noesis/HEAD/docs/public/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff2 -------------------------------------------------------------------------------- /docs/public/.buildinfo: -------------------------------------------------------------------------------- 1 | # Sphinx build info version 1 2 | # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. 3 | config: 6c17cb11edf46b86171e238e44a376c7 4 | tags: 645f666f9bcd5a90fca523b33c5a78b7 5 | -------------------------------------------------------------------------------- /noesis-tf/metrics.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import functools 3 | from tensorflow.contrib.learn.python.learn.metric_spec import MetricSpec 4 | 5 | 6 | def create_evaluation_metrics(): 7 | eval_metrics = {} 8 | for k in [1, 2, 5, 10, 50, 100]: 9 | eval_metrics["recall_at_%d" % k] = MetricSpec(metric_fn=functools.partial( 10 | tf.contrib.metrics.streaming_sparse_recall_at_k, 11 | k=k)) 12 | return eval_metrics 13 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = dstc7-noesis 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/public/_sources/package_reference.rst.txt: -------------------------------------------------------------------------------- 1 | Package Reference 2 | ================= 3 | Networks 4 | -------- 5 | .. automodule:: noesis.networks.dual_encoder 6 | :members: 7 | :undoc-members: 8 | 9 | Dataset 10 | ------- 11 | .. automodule:: noesis.dataset.dataset 12 | :members: 13 | :undoc-members: 14 | 15 | Trainers 16 | -------- 17 | .. automodule:: noesis.trainers.supervised_trainer 18 | :members: 19 | :undoc-members: 20 | 21 | Evaluator 22 | --------- 23 | .. automodule:: noesis.evaluator.evaluator 24 | :members: 25 | :undoc-members: 26 | 27 | Vocabulary 28 | ---------- 29 | .. automodule:: noesis.dataset.vocabulary 30 | :members: 31 | :undoc-members: 32 | 33 | Utilities 34 | --------- 35 | .. automodule:: noesis.dataset.utils 36 | :members: 37 | :undoc-members: 38 | 39 | Checkpoint 40 | ---------- 41 | .. automodule:: noesis.util.checkpoint 42 | :members: 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 International Business Machines 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .idea 3 | 4 | # Operating system related file 5 | .DS_Store 6 | 7 | # Byte-compiled / optimized / DLL files 8 | __pycache__/ 9 | *.py[cod] 10 | *$py.class 11 | 12 | # Distribution / packaging 13 | .Python 14 | env/ 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *,cover 49 | .hypothesis/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # IPython Notebook 73 | .ipynb_checkpoints 74 | 75 | # Vagrant 76 | .vagrant 77 | 78 | -------------------------------------------------------------------------------- /noesis-tf/models/helpers.py: -------------------------------------------------------------------------------- 1 | import array 2 | import numpy as np 3 | import tensorflow as tf 4 | from collections import defaultdict 5 | 6 | 7 | def load_vocab(filename): 8 | vocab = None 9 | with open(filename) as f: 10 | vocab = f.read().splitlines() 11 | dct = defaultdict(int) 12 | vocab = set(vocab) 13 | for idx, word in enumerate(vocab): 14 | dct[word] = idx 15 | return [vocab, dct] 16 | 17 | 18 | def load_glove_vectors(filename, vocab): 19 | """ 20 | Load glove vectors from a .txt file. 21 | Optionally limit the vocabulary to save memory. `vocab` should be a set. 22 | """ 23 | dct = {} 24 | vectors = array.array('d') 25 | current_idx = 0 26 | with open(filename, "r", encoding="utf-8") as f: 27 | for _, line in enumerate(f): 28 | tokens = line.split(" ") 29 | word = tokens[0] 30 | entries = tokens[1:] 31 | if not vocab or word in vocab: 32 | dct[word] = current_idx 33 | vectors.extend(float(x) for x in entries) 34 | current_idx += 1 35 | word_dim = len(entries) 36 | num_vectors = len(dct) 37 | tf.logging.info("Found {} out of {} vectors in Glove".format(num_vectors, len(vocab))) 38 | return [np.array(vectors).reshape(num_vectors, word_dim), dct] 39 | 40 | 41 | def build_initial_embedding_matrix(vocab_dict, glove_dict, glove_vectors, embedding_dim): 42 | initial_embeddings = np.random.uniform(-0.25, 0.25, (len(vocab_dict), embedding_dim)).astype("float32") 43 | for word, glove_word_idx in glove_dict.items(): 44 | word_idx = vocab_dict.get(word) 45 | initial_embeddings[word_idx, :] = glove_vectors[glove_word_idx] 46 | return initial_embeddings 47 | -------------------------------------------------------------------------------- /docs/source/submission.rst: -------------------------------------------------------------------------------- 1 | Submission 2 | ========== 3 | 4 | Your submissions should be emailed to chulaka.gunasekara@ibm.com, with the subject line **DSTC7_Track1_Submission**. The results should be submitted from an email address that is registered for Track 1. 5 | 6 | You need to submit a single zipped directory containing the result files for each of the subtasks that you need to be evaluated on. The files should be named in the following format. 7 | ``_subtask_.json`` 8 | 9 | The should be replaced by either ‘Ubuntu’ or ‘Advising’, and the should be replaced by the subtask number(1-5). 10 | For example, the results file for subtask 1 on Ubuntu dataset should be named as Ubuntu_subtask_1.json 11 | 12 | Each results file should follow the following json format. 13 | 14 | .. code-block:: json 15 | 16 | [ 17 | { 18 | "example-id": xxxxxxx, 19 | "candidate-ranking":[ 20 | { 21 | "candidate-id": aaaaaa, 22 | "confidence": b.bbb 23 | }, 24 | { 25 | "candidate-id": cccccc, 26 | "confidence": d.ddd 27 | }, 28 | ... 29 | ] 30 | }, 31 | ... 32 | ] 33 | 34 | 35 | The value for the field "example-id" should contain the corresponding example-id of the test dataset. The candidate ranking field should ONLY include 100 candidates in the order of confidence. 36 | 37 | For subtask 2, where the selection is made from a global list of candidates, candidate-ranking fields should **only include the top 100 candidates** from the global list. 38 | 39 | For subtask 4, when the correct candidate not available in the candidate set, return ``"candidate-id": NONE`` with the confidence score as an item in the candidate-ranking list. -------------------------------------------------------------------------------- /docs/public/_sources/submission.rst.txt: -------------------------------------------------------------------------------- 1 | Submission 2 | ========== 3 | 4 | Your submissions should be emailed to chulaka.gunasekara@ibm.com, with the subject line **DSTC7_Track1_Submission**. The results should be submitted from an email address that is registered for Track 1. 5 | 6 | You need to submit a single zipped directory containing the result files for each of the subtasks that you need to be evaluated on. The files should be named in the following format. 7 | ``_subtask_.json`` 8 | 9 | The should be replaced by either ‘Ubuntu’ or ‘Advising’, and the should be replaced by the subtask number(1-5). 10 | For example, the results file for subtask 1 on Ubuntu dataset should be named as Ubuntu_subtask_1.json 11 | 12 | Each results file should follow the following json format. 13 | 14 | .. code-block:: json 15 | 16 | [ 17 | { 18 | "example-id": xxxxxxx, 19 | "candidate-ranking":[ 20 | { 21 | "candidate-id": aaaaaa, 22 | "confidence": b.bbb 23 | }, 24 | { 25 | "candidate-id": cccccc, 26 | "confidence": d.ddd 27 | }, 28 | ... 29 | ] 30 | }, 31 | ... 32 | ] 33 | 34 | 35 | The value for the field "example-id" should contain the corresponding example-id of the test dataset. The candidate ranking field should ONLY include 100 candidates in the order of confidence. 36 | 37 | For subtask 2, where the selection is made from a global list of candidates, candidate-ranking fields should **only include the top 100 candidates** from the global list. 38 | 39 | For subtask 4, when the correct candidate not available in the candidate set, return ``"candidate-id": NONE`` with the confidence score as an item in the candidate-ranking list. -------------------------------------------------------------------------------- /docs/public/_sources/index.rst.txt: -------------------------------------------------------------------------------- 1 | .. dstc7-noesis documentation master file, created by 2 | sphinx-quickstart on Wed Jun 6 01:03:45 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Noetic End-to-End Response Selection Challenge 7 | ============================================== 8 | 9 | **Update** - A tensorflow based baseline for subtask 1 of the track is available `here `_. 10 | 11 | This challenge is part of dialog state tracking challenge (DSTC 7) series. It provides a partial conversation, and requires participants to select the correct next utterances from a set of candidates. 12 | Unlike previous similar challenges, this task tries to push towards real world problems by introducing: 13 | 14 | - A large number of candidates 15 | - Cases where no candidate is correct 16 | - External data 17 | 18 | This challenge is offered with two goal oriented dialog datasets, used in 5 subtasks. 19 | A participant may participate in one, several, or all the subtasks. 20 | A full description of the track is available `here `_. 21 | 22 | Organizers 23 | ---------- 24 | * `Lazaros Polymenako `_, `Chulaka Gunasekara `_ – IBM Research AI 25 | * `Walter Lasecki `_, `Jonathan K. Kummerfeld `_ – University of Michigan 26 | 27 | 28 | Maintainers 29 | ----------- 30 | * `Chulaka Gunasekara `_ 31 | 32 | To get a guaranteed support you are kindly requested to open an issue. 33 | 34 | Thank you for understanding! 35 | 36 | .. toctree:: 37 | :hidden: 38 | :glob: 39 | 40 | * 41 | -------------------------------------------------------------------------------- /noesis-tf/inputs.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | TEXT_FEATURE_SIZE = 160 4 | 5 | def get_feature_columns(mode): 6 | feature_columns = [] 7 | 8 | feature_columns.append(tf.contrib.layers.real_valued_column( 9 | column_name="context", dimension=TEXT_FEATURE_SIZE, dtype=tf.int64)) 10 | feature_columns.append(tf.contrib.layers.real_valued_column( 11 | column_name="context_len", dimension=1, dtype=tf.int64)) 12 | 13 | feature_columns.append(tf.contrib.layers.real_valued_column( 14 | column_name="target", dimension=1, dtype=tf.int64)) 15 | 16 | for i in range(100): 17 | feature_columns.append(tf.contrib.layers.real_valued_column( 18 | column_name="option_{}".format(i), dimension=TEXT_FEATURE_SIZE, dtype=tf.int64)) 19 | feature_columns.append(tf.contrib.layers.real_valued_column( 20 | column_name="option_{}_len".format(i), dimension=1, dtype=tf.int64)) 21 | 22 | return set(feature_columns) 23 | 24 | 25 | def create_input_fn(mode, input_files, batch_size, num_epochs): 26 | def input_fn(): 27 | features = tf.contrib.layers.create_feature_spec_for_parsing( 28 | get_feature_columns(mode)) 29 | 30 | feature_map = tf.contrib.learn.io.read_batch_features( 31 | file_pattern=input_files, 32 | batch_size=batch_size, 33 | features=features, 34 | reader=tf.TFRecordReader, 35 | randomize_input=True, 36 | num_epochs=num_epochs, 37 | queue_capacity=200000 + batch_size * 10, 38 | name="read_batch_features_{}".format(mode)) 39 | 40 | # This is an ugly hack because of a current bug in tf.learn 41 | # During evaluation TF tries to restore the epoch variable which isn't defined during training 42 | # So we define the variable manually here 43 | if mode == tf.contrib.learn.ModeKeys.TRAIN: 44 | tf.get_variable( 45 | "read_batch_features_eval/file_name_queue/limit_epochs/epochs", 46 | initializer=tf.constant(0, dtype=tf.int64)) 47 | 48 | target = feature_map.pop("target") 49 | 50 | return feature_map, target 51 | 52 | return input_fn 53 | -------------------------------------------------------------------------------- /docs/source/faq.rst: -------------------------------------------------------------------------------- 1 | FAQs 2 | ==== 3 | 4 | 1. **What is the timeline of the competition**? 5 | 6 | +-----------------------------------+-----------------------+ 7 | | Task | Dates | 8 | +===================================+=======================+ 9 | | Development phase (14 weeks) | Jun 1 – Sep 9, 2018 | 10 | +-----------------------------------+-----------------------+ 11 | | Evaluation phase (2 weeks) | Sep 10 – Oct 8, 2018 | 12 | +-----------------------------------+-----------------------+ 13 | | Release of the results | 16th Oct 2018 | 14 | +-----------------------------------+-----------------------+ 15 | | Paper submission deadline | Oct-Nov 2018 | 16 | +-----------------------------------+-----------------------+ 17 | | DSTC7 special session or workshop | Spring 2019 | 18 | +-----------------------------------+-----------------------+ 19 | 20 | | 21 | 22 | 2. **What should we submit**? 23 | 24 | You are required to submit the responses to the test dataset that will be released on the 10th of September. The format of the responses can be found under `submissions` section. 25 | 26 | 3. **Do we need to work on both datasets**? 27 | 28 | Not necessary. You can select one dataset and work on all or a subset of the subtasks. But, submitting results for both datasets and all subtasks will increase your chance of winning the competition. 29 | 30 | 4. **How are we evaluated**? 31 | 32 | For each test instance, we expect you to return a set of 100 choices (candidate ids) from the set candidates and a probability distribution over those 100 choices. For more details please check the `evaluations` section. 33 | 34 | 4. **What do you mean by end-to-end models**? 35 | 36 | We don't need the whole system to be end-to-end trainable. You can have separate components, which are not trained with back-propagation. However, we expect the functionality of each of the components in your system to be learned from the given dataset. We discourage the use of hand-coded features for any component in your system, as one of the focus points of the challenge is automation. 37 | 38 | 5. **Can we use pre-trained word embeddings**? 39 | 40 | You can use any pre-trained embeddings that was publicly available before the 1st of June. -------------------------------------------------------------------------------- /docs/public/_sources/faq.rst.txt: -------------------------------------------------------------------------------- 1 | FAQs 2 | ==== 3 | 4 | 1. **What is the timeline of the competition**? 5 | 6 | +-----------------------------------+-----------------------+ 7 | | Task | Dates | 8 | +===================================+=======================+ 9 | | Development phase (14 weeks) | Jun 1 – Sep 9, 2018 | 10 | +-----------------------------------+-----------------------+ 11 | | Evaluation phase (2 weeks) | Sep 10 – Oct 8, 2018 | 12 | +-----------------------------------+-----------------------+ 13 | | Release of the results | 16th Oct 2018 | 14 | +-----------------------------------+-----------------------+ 15 | | Paper submission deadline | Oct-Nov 2018 | 16 | +-----------------------------------+-----------------------+ 17 | | DSTC7 special session or workshop | Spring 2019 | 18 | +-----------------------------------+-----------------------+ 19 | 20 | | 21 | 22 | 2. **What should we submit**? 23 | 24 | You are required to submit the responses to the test dataset that will be released on the 10th of September. The format of the responses can be found under `submissions` section. 25 | 26 | 3. **Do we need to work on both datasets**? 27 | 28 | Not necessary. You can select one dataset and work on all or a subset of the subtasks. But, submitting results for both datasets and all subtasks will increase your chance of winning the competition. 29 | 30 | 4. **How are we evaluated**? 31 | 32 | For each test instance, we expect you to return a set of 100 choices (candidate ids) from the set candidates and a probability distribution over those 100 choices. For more details please check the `evaluations` section. 33 | 34 | 4. **What do you mean by end-to-end models**? 35 | 36 | We don't need the whole system to be end-to-end trainable. You can have separate components, which are not trained with back-propagation. However, we expect the functionality of each of the components in your system to be learned from the given dataset. We discourage the use of hand-coded features for any component in your system, as one of the focus points of the challenge is automation. 37 | 38 | 5. **Can we use pre-trained word embeddings**? 39 | 40 | You can use any pre-trained embeddings that was publicly available before the 1st of June. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Dialog System Technology Challenges 7 (DSTC 7) 2 | 3 | ## [Track 1 - Sentence Selection](http://workshop.colips.org/dstc7/proposals/Track%201%20Merged%20Challenge%20Extended%20Desscription_v2.pdf) 4 | 5 | This challenge provides a partial conversation, and requires participants to select the correct next utterances from a set of candidates. 6 | Unlike previous similar challenges, this task tries to push towards real world problems by introducing: 7 | 8 | - A large number of candidates 9 | - Cases where no candidate is correct 10 | - External data 11 | 12 | This challenge is offered with two goal oriented dialog datasets, used in 5 tasks. 13 | A participant may participate in one, several, or all the subtasks. 14 | 15 | If you use this data or code in your work, please cite the task description paper: 16 | 17 | ``` 18 | @InProceedings{dstc19task1, 19 | title = {DSTC7 Task 1: Noetic End-to-End Response Selection}, 20 | author = {Chulaka Gunasekara, Jonathan K. Kummerfeld, Lazaros Polymenakos, and Walter S. Lasecki}, 21 | year = {2019}, 22 | booktitle = {7th Edition of the Dialog System Technology Challenges at AAAI 2019}, 23 | url = {http://workshop.colips.org/dstc7/papers/dstc7_task1_final_report.pdf}, 24 | month = {January}, 25 | } 26 | ``` 27 | 28 | If you use the Ubuntu data, please also cite the paper in which we describe its creation: 29 | 30 | ``` 31 | @Article{arxiv18disentangle, 32 | author = {Jonathan K. Kummerfeld, Sai R. Gouravajhala, Joseph Peper, Vignesh Athreya, Chulaka Gunasekara, Jatin Ganhotra, Siva Sankalp Patel, Lazaros Polymenakos, and Walter S. Lasecki}, 33 | title = {Analyzing Assumptions in Conversation Disentanglement Research Through the Lens of a New Dataset and Model}, 34 | journal = {ArXiv e-prints}, 35 | archivePrefix = {arXiv}, 36 | eprint = {1810.11118}, 37 | primaryClass = {cs.CL}, 38 | year = {2018}, 39 | month = {October}, 40 | url = {https://arxiv.org/pdf/1810.11118.pdf}, 41 | } 42 | ``` 43 | 44 | **For more detail, please visit our [website](https://ibm.github.io/dstc7-noesis/public/index.html)**. 45 | 46 | ### Organizers 47 | 48 | [Lazaros Polymenakos](mailto:lcpolyme@us.ibm.com), [Chulaka Gunasekara](mailto:chulaka.gunasekara@ibm.com) – IBM Research AI
49 | [Walter Lasecki](mailto:wlasecki@umich.edu), [Jonathan K. Kummerfeld](mailto:jkummerf@umich.edu) – University of Michigan 50 | -------------------------------------------------------------------------------- /noesis-tf/train.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | 4 | import tensorflow as tf 5 | import model 6 | from hparams import create_hparams 7 | import metrics 8 | import inputs 9 | 10 | from models.dual_encoder import dual_encoder_model 11 | 12 | 13 | tf.flags.DEFINE_string("train_in", None, "Path to input data file") 14 | tf.flags.DEFINE_string("validation_in", None, "Path to validation data file") 15 | 16 | tf.flags.DEFINE_string("model_dir", None, "Directory to store model checkpoints (defaults to ./runs)") 17 | tf.flags.DEFINE_integer("loglevel", 20, "Tensorflow log level") 18 | tf.flags.DEFINE_integer("num_epochs", None, "Number of training Epochs. Defaults to indefinite.") 19 | tf.flags.DEFINE_integer("eval_every", 2000, "Evaluate after this many train steps") 20 | 21 | FLAGS = tf.flags.FLAGS 22 | 23 | TIMESTAMP = int(time.time()) 24 | 25 | if FLAGS.model_dir: 26 | MODEL_DIR = FLAGS.model_dir 27 | else: 28 | MODEL_DIR = os.path.abspath(os.path.join("./runs", str(TIMESTAMP))) 29 | 30 | TRAIN_FILE = os.path.abspath(os.path.join(FLAGS.train_in)) 31 | VALIDATION_FILE = os.path.abspath(os.path.join(FLAGS.validation_in)) 32 | 33 | tf.logging.set_verbosity(FLAGS.loglevel) 34 | 35 | 36 | def main(unused_argv): 37 | config = tf.ConfigProto() 38 | config.gpu_options.allow_growth = True 39 | 40 | hyper_params = create_hparams() 41 | 42 | model_fn = model.create_model_fn( 43 | hyper_params, 44 | model_impl=dual_encoder_model) 45 | 46 | estimator = tf.contrib.learn.Estimator( 47 | model_fn=model_fn, 48 | model_dir=MODEL_DIR, 49 | config=tf.contrib.learn.RunConfig(session_config=config)) 50 | 51 | input_fn_train = inputs.create_input_fn( 52 | mode=tf.contrib.learn.ModeKeys.TRAIN, 53 | input_files=[TRAIN_FILE], 54 | batch_size=hyper_params.batch_size, 55 | num_epochs=FLAGS.num_epochs) 56 | 57 | input_fn_eval = inputs.create_input_fn( 58 | mode=tf.contrib.learn.ModeKeys.EVAL, 59 | input_files=[VALIDATION_FILE], 60 | batch_size=hyper_params.eval_batch_size, 61 | num_epochs=1) 62 | 63 | eval_metrics = metrics.create_evaluation_metrics() 64 | 65 | eval_monitor = tf.contrib.learn.monitors.ValidationMonitor( 66 | input_fn=input_fn_eval, 67 | every_n_steps=FLAGS.eval_every, 68 | metrics=eval_metrics) 69 | 70 | estimator.fit(input_fn=input_fn_train, steps=None, monitors=[eval_monitor]) 71 | 72 | 73 | if __name__ == "__main__": 74 | tf.app.run() 75 | -------------------------------------------------------------------------------- /noesis-tf/hparams.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from collections import namedtuple 3 | 4 | # Model Parameters 5 | tf.flags.DEFINE_integer( 6 | "vocab_size", 7 | 100000, 8 | "The size of the vocabulary. Only change this if you changed the preprocessing") 9 | 10 | # Model Parameters 11 | tf.flags.DEFINE_integer("embedding_dim", 100, "Dimensionality of the embeddings") 12 | tf.flags.DEFINE_integer("rnn_dim", 256, "Dimensionality of the RNN cell") 13 | tf.flags.DEFINE_integer("max_context_len", 160, "Truncate contexts to this length") 14 | tf.flags.DEFINE_integer("max_utterance_len", 160, "Truncate utterance to this length") 15 | 16 | # Pre-trained embeddings 17 | tf.flags.DEFINE_string("glove_path", None, "Path to pre-trained Glove vectors") 18 | tf.flags.DEFINE_string("vocab_path", None, "Path to vocabulary.txt file") 19 | 20 | # Training Parameters 21 | tf.flags.DEFINE_float("learning_rate", 0.001, "Learning rate") 22 | tf.flags.DEFINE_integer("batch_size", 16, "Batch size during training") 23 | tf.flags.DEFINE_integer("eval_batch_size", 20, "Batch size during evaluation") 24 | tf.flags.DEFINE_string("optimizer", "Adam", "Optimizer Name (Adam, Adagrad, etc)") 25 | tf.flags.DEFINE_float("keep_rate", 1.0, "Drop out probability") 26 | tf.flags.DEFINE_float("decay_rate", 0.95, "Exponential decay rate") 27 | tf.flags.DEFINE_integer("decay_steps", 5000, "Decay steps") 28 | tf.flags.DEFINE_bool("staircase", False, "Staircase decay") 29 | 30 | FLAGS = tf.flags.FLAGS 31 | 32 | HParams = namedtuple( 33 | "HParams", 34 | [ 35 | "batch_size", 36 | "embedding_dim", 37 | "eval_batch_size", 38 | "learning_rate", 39 | "max_context_len", 40 | "max_utterance_len", 41 | "optimizer", 42 | "rnn_dim", 43 | "vocab_size", 44 | "glove_path", 45 | "vocab_path", 46 | "keep_rate", 47 | "decay_rate", 48 | "decay_steps", 49 | "staircase" 50 | ]) 51 | 52 | 53 | def create_hparams(): 54 | return HParams( 55 | batch_size=FLAGS.batch_size, 56 | eval_batch_size=FLAGS.eval_batch_size, 57 | vocab_size=FLAGS.vocab_size, 58 | optimizer=FLAGS.optimizer, 59 | learning_rate=FLAGS.learning_rate, 60 | embedding_dim=FLAGS.embedding_dim, 61 | max_context_len=FLAGS.max_context_len, 62 | max_utterance_len=FLAGS.max_utterance_len, 63 | glove_path=FLAGS.glove_path, 64 | vocab_path=FLAGS.vocab_path, 65 | rnn_dim=FLAGS.rnn_dim, 66 | keep_rate=FLAGS.keep_rate, 67 | decay_rate=FLAGS.decay_rate, 68 | decay_steps=FLAGS.decay_steps, 69 | staircase=FLAGS.staircase 70 | ) 71 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | # To use a consistent encoding 3 | from codecs import open 4 | from os import path 5 | 6 | here = path.abspath(path.dirname(__file__)) 7 | 8 | # Get the long description from the README file 9 | with open(path.join(here, 'README.md'), encoding='utf-8') as f: 10 | long_description = f.read() 11 | 12 | setup( 13 | name='noesis', 14 | 15 | # Versions should comply with PEP440. For a discussion on single-sourcing 16 | # the version across setup.py and the project code, see 17 | # https://packaging.python.org/en/latest/single_source_version.html 18 | version='0.0.1', 19 | 20 | description='Dialog State Tracking Challenge 7 - Noetic End-to-End Response Selection', 21 | long_description=long_description, 22 | 23 | # The project's main homepage. 24 | url='https://github.com/IBM/dstc7-noesis', 25 | 26 | # Choose your license 27 | license='Apache License 2.0', 28 | 29 | # See https://pypi.python.org/pypi?%3Aaction=list_classifiers 30 | classifiers=[ 31 | # How mature is this project? Common values are 32 | # 3 - Alpha 33 | # 4 - Beta 34 | # 5 - Production/Stable 35 | 'Development Status :: 3 - Alpha', 36 | 37 | # Indicate who your project is intended for 38 | 'Intended Audience :: Research', 39 | 'Topic :: Software Development', 40 | 41 | # Pick your license as you wish (should match "license" above) 42 | 'License :: Apache License 2.0', 43 | 44 | # Specify the Python versions you support here. In particular, ensure 45 | # that you indicate whether you support Python 2, Python 3 or both. 46 | 'Programming Language :: Python :: 2.7', 47 | 'Programming Language :: Python :: 3.6' 48 | ], 49 | 50 | # What does your project relate to? 51 | keywords='dual-encoder py-torch development', 52 | 53 | # You can just specify the packages manually here if your project is 54 | # simple. Or you can use find_packages(). 55 | packages=find_packages(exclude=['contrib', 'docs', 'tests']), 56 | 57 | # Alternatively, if you want to distribute just a my_module.py, uncomment 58 | # this: 59 | # py_modules=["my_module"], 60 | 61 | # List run-time dependencies here. These will be installed by pip when 62 | # your project is installed. For an analysis of "install_requires" vs pip's 63 | # requirements files see: 64 | # https://packaging.python.org/en/latest/requirements.html 65 | install_requires=['numpy', 'torch', 'dill'], 66 | 67 | # List additional groups of dependencies here (e.g. development 68 | # dependencies). You can install these using the following syntax, 69 | # for example: 70 | # $ pip install -e .[dev,test] 71 | extras_require={ 72 | 'dev': ['check-manifest'], 73 | 'test': ['coverage'], 74 | } 75 | ) 76 | -------------------------------------------------------------------------------- /noesis-tf/README.md: -------------------------------------------------------------------------------- 1 | ## Response Selection for Conversation Systems in Tensorflow 2 | 3 | #### Overview 4 | This code provides a baseline for the subtask 1 of DSTC-7 [Sentence Selection track](https://ibm.github.io/dstc7-noesis/public/index.html). 5 | 6 | This code extends the work from Denny Britz which implements the Dual LSTM Encoder model from [The Ubuntu Dialogue Corpus: A Large Dataset for Research in Unstructured Multi-Turn Dialogue Systems](http://arxiv.org/abs/1506.08909) 7 | 8 | [Refer to the original blog post here](http://www.wildml.com/2016/07/deep-learning-for-chatbots-2-retrieval-based-model-tensorflow) 9 | 10 | #### Setup 11 | 12 | This code uses Python 3.6 and Tensorflow-GPU 1.6. Clone the repository and install all required packages. It is recommended to use the [Anaconda package manager](https://www.anaconda.com/download/#macos). After installing Anaconda - 13 | 14 | ``` 15 | cd noesis-tf 16 | conda create --name dstc7 python=3.6 17 | source activate dstc7 18 | pip install -r ../requirements.txt 19 | ``` 20 | 21 | #### Get the data 22 | 23 | Make sure you register for the track 1 of DSTC7 to download the data and copy it inside the `data` directory. 24 | 25 | #### Prepare the data 26 | 27 | Before training, the data needs to converted to suitable format for tensorflow. The script `prepare_data.py` can be used to convert data from JSON format to TFRecords for subtasks 1 of both datasets. 28 | 29 | ``` 30 | python scripts/prepare_data.py --train_in data/ubuntu_train_subtask_1.json --validation_in data/ubuntu_dev_subtask_1.json --train_out data/ubuntu_subtask_1.tfrecords --validation_out data/ubuntu_dev_subtask_1.tfrecords --vocab_path data/ubuntu_subtask_1.txt --vocab_processor data/ubuntu_subtask_1.bin 31 | ``` 32 | 33 | #### Training 34 | 35 | The following command can be used to train the model for the Ubuntu subtask 1. Similar command works for subtask 1 of Advising data as well. 36 | 37 | ``` 38 | python train.py --train_in data/ubuntu_subtask_1.tfrecords --validation_in data/ubuntu_dev_subtask_1.tfrecords -glove_path data/glove.42B.300d.txt -vocab_path data/ubuntu_subtask_1.txt --embedding_dim=300 --batch_size=16 39 | ``` 40 | 41 | The glove embeddings can be downloaded from [here](https://nlp.stanford.edu/projects/glove/) 42 | 43 | Check `hparams.py` for all command-line arguments. 44 | 45 | #### Model 46 | 47 | This baseline model extends the dual-encoder model used [here](http://www.wildml.com/2016/07/deep-learning-for-chatbots-2-retrieval-based-model-tensorflow). The architecture of the model is shown in the figure below. ![](images/architecture.png) 48 | 49 | 50 | 51 | #### Dual Encoder Baselines (Recall) 52 | 53 | Baselines are reported on validation set. 54 | 55 | | Dataset | 1 in 100 R@1 | 1 in 100 R@2 | 1 in 100 R@5 | 1 in 100 R@10 | 1 in 100 R@50 56 | | :---------------: | :-------------: | :--------------------: |:----------: | :---------: | :---------: | 57 | | Ubuntu - Subtask 1 | 8.32% | 13.36% | 24.26% | 35.98% | 80.04% | 58 | | Advising - Subtask 1 | 6.20% | 9.80% | 18.40% | 29.60% | 72.80% | 59 | 60 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. dstc7-noesis documentation master file, created by 2 | sphinx-quickstart on Wed Jun 6 01:03:45 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Noetic End-to-End Response Selection Challenge 7 | ============================================== 8 | 9 | **Update** - A tensorflow based baseline for subtask 1 of the track is available `here `_. 10 | 11 | This challenge is part of dialog state tracking challenge (DSTC 7) series. It provides a partial conversation, and requires participants to select the correct next utterances from a set of candidates. 12 | Unlike previous similar challenges, this task tries to push towards real world problems by introducing: 13 | 14 | - A large number of candidates 15 | - Cases where no candidate is correct 16 | - External data 17 | 18 | This challenge is offered with two goal oriented dialog datasets, used in 5 subtasks. 19 | A participant may participate in one, several, or all the subtasks. 20 | A full description of the track is available `here `_. 21 | 22 | If you use this data or code in your work, please cite the task description paper:: 23 | 24 | @InProceedings{dstc19task1, 25 | title = {DSTC7 Task 1: Noetic End-to-End Response Selection}, 26 | author = {Chulaka Gunasekara, Jonathan K. Kummerfeld, Lazaros Polymenakos, and Walter S. Lasecki}, 27 | year = {2019}, 28 | booktitle = {7th Edition of the Dialog System Technology Challenges at AAAI 2019}, 29 | url = {http://workshop.colips.org/dstc7/papers/dstc7_task1_final_report.pdf}, 30 | month = {January}, 31 | } 32 | 33 | If you use the Ubuntu data, please also cite the paper in which we describe its creation:: 34 | 35 | @Article{arxiv18disentangle, 36 | author = {Jonathan K. Kummerfeld, Sai R. Gouravajhala, Joseph Peper, Vignesh Athreya, Chulaka Gunasekara, Jatin Ganhotra, Siva Sankalp Patel, Lazaros Polymenakos, and Walter S. Lasecki}, 37 | title = {Analyzing Assumptions in Conversation Disentanglement Research Through the Lens of a New Dataset and Model}, 38 | journal = {ArXiv e-prints}, 39 | archivePrefix = {arXiv}, 40 | eprint = {1810.11118}, 41 | primaryClass = {cs.CL}, 42 | year = {2018}, 43 | month = {October}, 44 | url = {https://arxiv.org/pdf/1810.11118.pdf}, 45 | } 46 | 47 | Organizers 48 | ---------- 49 | * `Lazaros Polymenako `_, `Chulaka Gunasekara `_ – IBM Research AI 50 | * `Walter Lasecki `_, `Jonathan K. Kummerfeld `_ – University of Michigan 51 | 52 | 53 | Maintainers 54 | ----------- 55 | * `Chulaka Gunasekara `_ 56 | 57 | To get a guaranteed support you are kindly requested to open an issue. 58 | 59 | Thank you for understanding! 60 | 61 | .. toctree:: 62 | :hidden: 63 | :glob: 64 | 65 | * 66 | -------------------------------------------------------------------------------- /docs/public/_static/css/badge_only.css: -------------------------------------------------------------------------------- 1 | .fa:before{-webkit-font-smoothing:antialiased}.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;content:""}.clearfix:after{clear:both}@font-face{font-family:FontAwesome;font-weight:normal;font-style:normal;src:url("../fonts/fontawesome-webfont.eot");src:url("../fonts/fontawesome-webfont.eot?#iefix") format("embedded-opentype"),url("../fonts/fontawesome-webfont.woff") format("woff"),url("../fonts/fontawesome-webfont.ttf") format("truetype"),url("../fonts/fontawesome-webfont.svg#FontAwesome") format("svg")}.fa:before{display:inline-block;font-family:FontAwesome;font-style:normal;font-weight:normal;line-height:1;text-decoration:inherit}a .fa{display:inline-block;text-decoration:inherit}li .fa{display:inline-block}li .fa-large:before,li .fa-large:before{width:1.875em}ul.fas{list-style-type:none;margin-left:2em;text-indent:-0.8em}ul.fas li .fa{width:.8em}ul.fas li .fa-large:before,ul.fas li .fa-large:before{vertical-align:baseline}.fa-book:before{content:""}.icon-book:before{content:""}.fa-caret-down:before{content:""}.icon-caret-down:before{content:""}.fa-caret-up:before{content:""}.icon-caret-up:before{content:""}.fa-caret-left:before{content:""}.icon-caret-left:before{content:""}.fa-caret-right:before{content:""}.icon-caret-right:before{content:""}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:"Lato","proxima-nova","Helvetica Neue",Arial,sans-serif;z-index:400}.rst-versions a{color:#2980B9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27AE60;*zoom:1}.rst-versions .rst-current-version:before,.rst-versions .rst-current-version:after{display:table;content:""}.rst-versions .rst-current-version:after{clear:both}.rst-versions .rst-current-version .fa{color:#fcfcfc}.rst-versions .rst-current-version .fa-book{float:left}.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#E74C3C;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#F1C40F;color:#000}.rst-versions.shift-up{height:auto;max-height:100%}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:gray;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:solid 1px #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px}.rst-versions.rst-badge .icon-book{float:none}.rst-versions.rst-badge .fa-book{float:none}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book{float:left}.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge .rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width: 768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}} 2 | -------------------------------------------------------------------------------- /noesis-tf/model.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | 3 | 4 | def get_id_feature(features, key, len_key, max_len): 5 | ids = features[key] 6 | ids_len = tf.squeeze(features[len_key], [1]) 7 | ids_len = tf.minimum(ids_len, tf.constant(max_len, dtype=tf.int64)) 8 | return ids, ids_len 9 | 10 | 11 | def create_train_op(loss, hparams): 12 | def exp_decay(learning_rate, global_step): 13 | return tf.train.exponential_decay(learning_rate, global_step, decay_steps=hparams.decay_steps, decay_rate=hparams.decay_rate, 14 | staircase=hparams.staircase, name="lr_decay") 15 | train_op = tf.contrib.layers.optimize_loss( 16 | loss=loss, 17 | global_step=tf.contrib.framework.get_global_step(), 18 | learning_rate=hparams.learning_rate, 19 | clip_gradients=10.0, 20 | optimizer=hparams.optimizer, 21 | learning_rate_decay_fn=exp_decay 22 | ) 23 | return train_op 24 | 25 | 26 | def create_model_fn(hparams, model_impl): 27 | def model_fn(features, targets, mode): 28 | context, context_len = get_id_feature( 29 | features, "context", "context_len", hparams.max_context_len) 30 | 31 | all_utterances = [] 32 | all_utterances_lens = [] 33 | 34 | for i in range(100): 35 | option, option_len = get_id_feature(features, 36 | "option_{}".format(i), 37 | "option_{}_len".format(i), 38 | hparams.max_utterance_len) 39 | all_utterances.append(option) 40 | all_utterances_lens.append(option_len) 41 | 42 | if mode == tf.contrib.learn.ModeKeys.TRAIN: 43 | probs, loss = model_impl( 44 | hparams, 45 | mode, 46 | context, 47 | context_len, 48 | all_utterances, 49 | tf.transpose(tf.stack(all_utterances_lens, axis=0)), 50 | targets, 51 | hparams.batch_size) 52 | train_op = create_train_op(loss, hparams) 53 | return probs, loss, train_op 54 | 55 | if mode == tf.contrib.learn.ModeKeys.INFER: 56 | 57 | probs, loss = model_impl( 58 | hparams, 59 | mode, 60 | tf.concat(0, context), 61 | tf.concat(0, context_len), 62 | tf.concat(0, all_utterances), 63 | tf.concat(0, all_utterances_lens), 64 | None, 65 | hparams.eval_batch_size) 66 | 67 | split_probs = tf.split(0, features["len"], probs) 68 | probs = tf.concat(1, split_probs) 69 | 70 | return probs, 0.0, None 71 | 72 | if mode == tf.contrib.learn.ModeKeys.EVAL: 73 | probs, loss = model_impl( 74 | hparams, 75 | mode, 76 | context, 77 | context_len, 78 | all_utterances, 79 | tf.transpose(tf.stack(all_utterances_lens, axis=0)), 80 | targets, 81 | hparams.eval_batch_size) 82 | 83 | shaped_probs = probs 84 | 85 | return shaped_probs, loss, None 86 | 87 | return model_fn 88 | -------------------------------------------------------------------------------- /noesis/evaluator/evaluator.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function, division 2 | 3 | import numpy as np 4 | import torch 5 | from torch.nn import CrossEntropyLoss 6 | 7 | 8 | class Evaluator(object): 9 | """ Class to evaluate models with given datasets. 10 | 11 | Args: 12 | loss (torch.NN.CrossEntropyLoss, optional): loss for evaluator (default: torch.NN.CrossEntropyLoss) 13 | batch_size (int, optional): batch size for evaluator (default: 64) 14 | """ 15 | 16 | def __init__(self, loss_func=CrossEntropyLoss(), batch_size=64): 17 | self.loss_func = loss_func 18 | self.batch_size = batch_size 19 | 20 | def evaluate(self, model, data): 21 | """ Evaluate a model on given dataset and return performance. 22 | 23 | Args: 24 | model (models.networks): model to evaluate 25 | data (dataset.dataset.Dataset): dataset to evaluate against 26 | 27 | Returns: 28 | loss (float): loss of the given model on the given dataset 29 | """ 30 | model.eval() 31 | 32 | match = 0 33 | total = 0 34 | recall = {'@1': 0, '@2': 0, '@5': 0, '@10': 0, '@50': 0, '@100': 0} 35 | loss = 0 36 | 37 | # device = None if torch.cuda.is_available() else -1 38 | 39 | with torch.no_grad(): 40 | for batch in data.make_batches(self.batch_size): 41 | if torch.cuda.is_available(): 42 | context_variable = torch.tensor(batch[0]).cuda() 43 | responses_variable = torch.tensor(batch[1]).cuda() 44 | target_variable = torch.tensor(batch[2]).cuda() 45 | context_lengths_variable = torch.tensor(batch[3]).cuda() 46 | responses_lengths_variable = torch.tensor(batch[4]).cuda() 47 | else: 48 | context_variable = torch.tensor(batch[0]) 49 | responses_variable = torch.tensor(batch[1]) 50 | target_variable = torch.tensor(batch[2]) 51 | context_lengths_variable = torch.tensor(batch[3]) 52 | responses_lengths_variable = torch.tensor(batch[4]) 53 | 54 | outputs = model(context_variable, responses_variable, context_lengths_variable, responses_lengths_variable) 55 | 56 | # Get loss 57 | if len(outputs.size()) == 1: 58 | outputs = outputs.unsqueeze(0) 59 | loss += self.loss_func(outputs, target_variable) 60 | 61 | # Evaluation 62 | predictions = np.argsort(outputs.numpy(), axis=1) 63 | num_samples = predictions.shape[0] 64 | 65 | ranks = predictions[np.arange(num_samples), target_variable] 66 | match += sum(ranks == 0) 67 | recall['@1'] = match 68 | recall['@2'] += sum(ranks <= 2) 69 | recall['@5'] += sum(ranks <= 5) 70 | recall['@10'] += sum(ranks <= 10) 71 | recall['@50'] += sum(ranks <= 50) 72 | recall['@100'] += sum(ranks <= 100) 73 | total += num_samples 74 | 75 | if total == 0: 76 | accuracy = float('nan') 77 | else: 78 | accuracy = match / total 79 | 80 | return loss, accuracy, {k: v/total for k, v in recall.items()} 81 | -------------------------------------------------------------------------------- /noesis-tf/util/blocks.py: -------------------------------------------------------------------------------- 1 | """ 2 | Functions and components that can be slotted into tensorflow models. 3 | TODO: Write functions for various types of attention. 4 | """ 5 | 6 | import tensorflow as tf 7 | 8 | 9 | def length(sequence): 10 | """ 11 | Get true length of sequences (without padding), and mask for true-length in max-length. 12 | Input of shape: (batch_size, max_seq_length, hidden_dim) 13 | Output shapes, 14 | length: (batch_size) 15 | mask: (batch_size, max_seq_length, 1) 16 | """ 17 | populated = tf.sign(tf.abs(sequence)) 18 | length = tf.cast(tf.reduce_sum(populated, axis=1), tf.int32) 19 | mask = tf.cast(tf.expand_dims(populated, -1), tf.float32) 20 | return length, mask 21 | 22 | 23 | 24 | def biLSTM(inputs, dim, seq_len, name): 25 | """ 26 | A Bi-Directional LSTM layer. Returns forward and backward hidden states as a tuple, and cell states as a tuple. 27 | Output of hidden states: [(batch_size, max_seq_length, hidden_dim), (batch_size, max_seq_length, hidden_dim)] 28 | Same shape for cell states. 29 | """ 30 | with tf.name_scope(name): 31 | with tf.variable_scope('forward' + name): 32 | lstm_fwd = tf.contrib.rnn.LSTMCell(num_units=dim) 33 | with tf.variable_scope('backward' + name): 34 | lstm_bwd = tf.contrib.rnn.LSTMCell(num_units=dim) 35 | 36 | hidden_states, cell_states = tf.nn.bidirectional_dynamic_rnn(cell_fw=lstm_fwd, cell_bw=lstm_bwd, 37 | inputs=inputs, sequence_length=seq_len, 38 | dtype=tf.float32, scope=name) 39 | 40 | return hidden_states, cell_states 41 | 42 | 43 | def LSTM(inputs, dim, seq_len, name): 44 | """ 45 | An LSTM layer. Returns hidden states and cell states as a tuple. 46 | Output shape of hidden states: (batch_size, max_seq_length, hidden_dim) 47 | Same shape for cell states. 48 | """ 49 | with tf.name_scope(name): 50 | cell = tf.contrib.rnn.LSTMCell(num_units=dim) 51 | hidden_states, cell_states = tf.nn.dynamic_rnn(cell, inputs=inputs, sequence_length=seq_len, 52 | dtype=tf.float32, scope=name) 53 | 54 | return hidden_states, cell_states 55 | 56 | 57 | def last_output(output, true_length): 58 | """ 59 | To get the last hidden layer form a dynamically unrolled RNN. 60 | Input of shape (batch_size, max_seq_length, hidden_dim). 61 | true_length: Tensor of shape (batch_size). Such a tensor is given by the length() function. 62 | Output of shape (batch_size, hidden_dim). 63 | """ 64 | max_length = int(output.get_shape()[1]) 65 | length_mask = tf.expand_dims(tf.one_hot(true_length-1, max_length, on_value=1., off_value=0.), -1) 66 | last_output = tf.reduce_sum(tf.multiply(output, length_mask), 1) 67 | return last_output 68 | 69 | 70 | def masked_softmax(scores, mask): 71 | """ 72 | Used to calculate a softmax score with true sequence length (without padding), rather than max-sequence length. 73 | Input shape: (batch_size, max_seq_length, hidden_dim). 74 | mask parameter: Tensor of shape (batch_size, max_seq_length). Such a mask is given by the length() function. 75 | """ 76 | numerator = tf.exp(tf.subtract(scores, tf.reduce_max(scores, 1, keep_dims=True))) * mask 77 | denominator = tf.reduce_sum(numerator, 1, keep_dims=True) 78 | weights = tf.div(numerator, denominator) 79 | return weights -------------------------------------------------------------------------------- /docs/source/evaluations.rst: -------------------------------------------------------------------------------- 1 | Evaluations 2 | =========== 3 | 4 | Metrics 5 | ------- 6 | 7 | For each test instance, we will expect you to return a set of 100 choices (candidate ids) from the set of possible follow-up sentences and a probability distribution over those 100 choices. 8 | As competition metrics we will compute range of scores, including recall@k, MRR(mean reciprocal rank) and MAP(mean average precision). 9 | 10 | Following evaluation metrics will be used to evaluate your submissions. 11 | 12 | +----------+------------------------------------------+-----------------------------------------------+ 13 | | Sub-Task | Ubuntu | Advising | 14 | +==========+==========================================+===============================================+ 15 | | 1 | Recall @1, Recall @10, Recall @50, MRR | Recall @1, Recall @10, Recall @50, MRR | 16 | +----------+------------------------------------------+-----------------------------------------------+ 17 | | 2 | Recall @1, Recall @10, Recall @50, MRR | | 18 | +----------+------------------------------------------+-----------------------------------------------+ 19 | | 3 | | Recall @1, Recall @10, Recall @50, MRR, MAP | 20 | +----------+------------------------------------------+-----------------------------------------------+ 21 | | 4 | Recall @1, Recall @10, Recall @50, MRR | Recall @1, Recall @10, Recall @50, MRR | 22 | +----------+------------------------------------------+-----------------------------------------------+ 23 | | 5 | Recall @1, Recall @10, Recall @50, MRR | Recall @1, Recall @10, Recall @50, MRR | 24 | +----------+------------------------------------------+-----------------------------------------------+ 25 | 26 | **Note:** 27 | We will evaluate MAP for sub-task 3 with Advising data as the you are supposed to return the correct response and all the paraphrases associated with it. 28 | 29 | Best Scores 30 | ----------- 31 | 32 | The ranking considers the average of Recall\@10 and MRR. Best Recall\@10 and MRR scores for each subtask is is shown in the below table. 33 | 34 | Recall\@10 35 | 36 | +----------+---------+-----------------+-----------------+ 37 | | Sub-Task | Ubuntu | Advising-Case-1 | Advising-Case-2 | 38 | +==========+=========+=================+=================+ 39 | | 1 | 0.902 | 0.85 | 0.63 | 40 | +----------+---------+-----------------+-----------------+ 41 | | 2 | 0.361 | NA | NA | 42 | +----------+---------+-----------------+-----------------+ 43 | | 3 | NA | 0.906 | 0.75 | 44 | +----------+---------+-----------------+-----------------+ 45 | | 4 | 0.739 | 0.652 | 0.508 | 46 | +----------+---------+-----------------+-----------------+ 47 | | 5 | 0.905 | 0.864 | 0.63 | 48 | +----------+---------+-----------------+-----------------+ 49 | 50 | 51 | MRR 52 | 53 | +----------+---------+-----------------+-----------------+ 54 | | Sub-Task | Ubuntu | Advising-Case-1 | Advising-Case-2 | 55 | +==========+=========+=================+=================+ 56 | | 1 | 0.7350 | 0.6078 | 0.3390 | 57 | +----------+---------+-----------------+-----------------+ 58 | | 2 | 0.2528 | NA | NA | 59 | +----------+---------+-----------------+-----------------+ 60 | | 3 | NA | 0.6238 | 0.4341 | 61 | +----------+---------+-----------------+-----------------+ 62 | | 4 | 0.5891 | 0.3495 | 0.2422 | 63 | +----------+---------+-----------------+-----------------+ 64 | | 5 | 0.7399 | 0.6455 | 0.3390 | 65 | +----------+---------+-----------------+-----------------+ -------------------------------------------------------------------------------- /docs/public/_sources/evaluations.rst.txt: -------------------------------------------------------------------------------- 1 | Evaluations 2 | =========== 3 | 4 | Metrics 5 | ------- 6 | 7 | For each test instance, we will expect you to return a set of 100 choices (candidate ids) from the set of possible follow-up sentences and a probability distribution over those 100 choices. 8 | As competition metrics we will compute range of scores, including recall@k, MRR(mean reciprocal rank) and MAP(mean average precision). 9 | 10 | Following evaluation metrics will be used to evaluate your submissions. 11 | 12 | +----------+------------------------------------------+-----------------------------------------------+ 13 | | Sub-Task | Ubuntu | Advising | 14 | +==========+==========================================+===============================================+ 15 | | 1 | Recall @1, Recall @10, Recall @50, MRR | Recall @1, Recall @10, Recall @50, MRR | 16 | +----------+------------------------------------------+-----------------------------------------------+ 17 | | 2 | Recall @1, Recall @10, Recall @50, MRR | | 18 | +----------+------------------------------------------+-----------------------------------------------+ 19 | | 3 | | Recall @1, Recall @10, Recall @50, MRR, MAP | 20 | +----------+------------------------------------------+-----------------------------------------------+ 21 | | 4 | Recall @1, Recall @10, Recall @50, MRR | Recall @1, Recall @10, Recall @50, MRR | 22 | +----------+------------------------------------------+-----------------------------------------------+ 23 | | 5 | Recall @1, Recall @10, Recall @50, MRR | Recall @1, Recall @10, Recall @50, MRR | 24 | +----------+------------------------------------------+-----------------------------------------------+ 25 | 26 | **Note:** 27 | We will evaluate MAP for sub-task 3 with Advising data as the you are supposed to return the correct response and all the paraphrases associated with it. 28 | 29 | Best Scores 30 | ----------- 31 | 32 | The ranking considers the average of Recall\@10 and MRR. Best Recall\@10 and MRR scores for each subtask is is shown in the below table. 33 | 34 | Recall\@10 35 | 36 | +----------+---------+-----------------+-----------------+ 37 | | Sub-Task | Ubuntu | Advising-Case-1 | Advising-Case-2 | 38 | +==========+=========+=================+=================+ 39 | | 1 | 0.902 | 0.85 | 0.63 | 40 | +----------+---------+-----------------+-----------------+ 41 | | 2 | 0.361 | NA | NA | 42 | +----------+---------+-----------------+-----------------+ 43 | | 3 | NA | 0.906 | 0.75 | 44 | +----------+---------+-----------------+-----------------+ 45 | | 4 | 0.739 | 0.652 | 0.508 | 46 | +----------+---------+-----------------+-----------------+ 47 | | 5 | 0.905 | 0.864 | 0.63 | 48 | +----------+---------+-----------------+-----------------+ 49 | 50 | 51 | MRR 52 | 53 | +----------+---------+-----------------+-----------------+ 54 | | Sub-Task | Ubuntu | Advising-Case-1 | Advising-Case-2 | 55 | +==========+=========+=================+=================+ 56 | | 1 | 0.7350 | 0.6078 | 0.3390 | 57 | +----------+---------+-----------------+-----------------+ 58 | | 2 | 0.2528 | NA | NA | 59 | +----------+---------+-----------------+-----------------+ 60 | | 3 | NA | 0.6238 | 0.4341 | 61 | +----------+---------+-----------------+-----------------+ 62 | | 4 | 0.5891 | 0.3495 | 0.2422 | 63 | +----------+---------+-----------------+-----------------+ 64 | | 5 | 0.7399 | 0.6455 | 0.3390 | 65 | +----------+---------+-----------------+-----------------+ -------------------------------------------------------------------------------- /noesis/sample.py: -------------------------------------------------------------------------------- 1 | import os 2 | import argparse 3 | import logging 4 | 5 | import torch 6 | from torch.nn import CrossEntropyLoss 7 | 8 | from networks.dual_encoder import Encoder, DualEncoder 9 | from trainers.supervised_trainer import SupervisedTrainer 10 | from util.checkpoint import Checkpoint 11 | from dataset.dataset import Dataset 12 | from evaluator.evaluator import Evaluator 13 | 14 | # Sample usage: 15 | # # training 16 | # python sample.py --train_path $TRAIN_PATH --dev_path $DEV_PATH --expt_dir $EXPT_PATH 17 | # # resuming from the latest checkpoint of the experiment 18 | # python sample.py --train_path $TRAIN_PATH --dev_path $DEV_PATH --expt_dir $EXPT_PATH --resume 19 | # # resuming from a specific checkpoint 20 | # python sample.py --train_path $TRAIN_PATH --dev_path $DEV_PATH --expt_dir $EXPT_PATH --load_checkpoint $CHECKPOINT_DIR 21 | 22 | parser = argparse.ArgumentParser() 23 | parser.add_argument('--train_path', action='store', dest='train_path', 24 | help='Path to train data') 25 | parser.add_argument('--dev_path', action='store', dest='dev_path', 26 | help='Path to dev data') 27 | parser.add_argument('--test_path', action='store', dest='test_path', 28 | help='Path to test data') 29 | parser.add_argument('--expt_dir', action='store', dest='expt_dir', default='./experiment', 30 | help='Path to experiment directory. If load_checkpoint is True, then path to checkpoint directory has to be provided') 31 | parser.add_argument('--load_checkpoint', action='store', dest='load_checkpoint', 32 | help='The name of the checkpoint to load, usually an encoded time string') 33 | parser.add_argument('--resume', action='store_true', dest='resume', 34 | default=False, 35 | help='Indicates if training has to be resumed from the latest checkpoint') 36 | parser.add_argument('--log-level', dest='log_level', 37 | default='info', 38 | help='Logging level.') 39 | 40 | opt = parser.parse_args() 41 | 42 | LOG_FORMAT = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s' 43 | logging.basicConfig(format=LOG_FORMAT, level=getattr(logging, opt.log_level.upper())) 44 | logging.info(opt) 45 | 46 | if opt.load_checkpoint is not None: 47 | logging.info("loading checkpoint from {}".format(os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint))) 48 | checkpoint_path = os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint) 49 | checkpoint = Checkpoint.load(checkpoint_path) 50 | dual_encoder = checkpoint.model 51 | vocab = checkpoint.vocab 52 | else: 53 | # Prepare dataset 54 | train = Dataset.from_file(opt.train_path) 55 | dev = Dataset.from_file(opt.dev_path, vocab=train.vocab) 56 | vocab = train.vocab 57 | max_len = 500 58 | 59 | # Prepare loss 60 | loss_func = CrossEntropyLoss() 61 | if torch.cuda.is_available(): 62 | loss_func.cuda() 63 | 64 | optimizer = None 65 | if not opt.resume: 66 | # Initialize model 67 | hidden_size = 128 68 | bidirectional = True 69 | context_encoder = Encoder(vocab.get_vocab_size(), max_len, hidden_size, 70 | bidirectional=bidirectional, variable_lengths=True) 71 | response_encoder = Encoder(vocab.get_vocab_size(), max_len, hidden_size, 72 | bidirectional=bidirectional, variable_lengths=True) 73 | 74 | dual_encoder = DualEncoder(context_encoder, response_encoder) 75 | if torch.cuda.is_available(): 76 | dual_encoder.cuda() 77 | 78 | for param in dual_encoder.parameters(): 79 | param.data.uniform_(-0.08, 0.08) 80 | 81 | # train 82 | t = SupervisedTrainer(loss_func=loss_func, batch_size=1, 83 | checkpoint_every=30, 84 | print_every=100, expt_dir=opt.expt_dir) 85 | 86 | t.train(dual_encoder, train, batch_size=1, num_epochs=20, dev_data=dev, optimizer=optimizer, resume=opt.resume) 87 | 88 | evaluator = Evaluator(batch_size=1) 89 | l, precision, recall = evaluator.evaluate(dual_encoder, dev) 90 | print("Precision: {}, Recall: {}".format(precision, recall)) 91 | 92 | 93 | 94 | -------------------------------------------------------------------------------- /docs/public/_static/js/theme.js: -------------------------------------------------------------------------------- 1 | /* sphinx_rtd_theme version 0.4.1 | MIT license */ 2 | /* Built 20180727 10:07 */ 3 | require=function n(e,i,t){function o(s,a){if(!i[s]){if(!e[s]){var l="function"==typeof require&&require;if(!a&&l)return l(s,!0);if(r)return r(s,!0);var c=new Error("Cannot find module '"+s+"'");throw c.code="MODULE_NOT_FOUND",c}var u=i[s]={exports:{}};e[s][0].call(u.exports,function(n){var i=e[s][1][n];return o(i||n)},u,u.exports,n,e,i,t)}return i[s].exports}for(var r="function"==typeof require&&require,s=0;s"),n("table.docutils.footnote").wrap("
"),n("table.docutils.citation").wrap("
"),n(".wy-menu-vertical ul").not(".simple").siblings("a").each(function(){var i=n(this);expand=n(''),expand.on("click",function(n){return e.toggleCurrent(i),n.stopPropagation(),!1}),i.prepend(expand)})},reset:function(){var n=encodeURI(window.location.hash)||"#";try{var e=$(".wy-menu-vertical"),i=e.find('[href="'+n+'"]');if(0===i.length){var t=$('.document [id="'+n.substring(1)+'"]').closest("div.section");0===(i=e.find('[href="#'+t.attr("id")+'"]')).length&&(i=e.find('[href="#"]'))}i.length>0&&($(".wy-menu-vertical .current").removeClass("current"),i.addClass("current"),i.closest("li.toctree-l1").addClass("current"),i.closest("li.toctree-l1").parent().addClass("current"),i.closest("li.toctree-l1").addClass("current"),i.closest("li.toctree-l2").addClass("current"),i.closest("li.toctree-l3").addClass("current"),i.closest("li.toctree-l4").addClass("current"))}catch(o){console.log("Error expanding nav for anchor",o)}},onScroll:function(){this.winScroll=!1;var n=this.win.scrollTop(),e=n+this.winHeight,i=this.navBar.scrollTop()+(n-this.winPosition);n<0||e>this.docHeight||(this.navBar.scrollTop(i),this.winPosition=n)},onResize:function(){this.winResize=!1,this.winHeight=this.win.height(),this.docHeight=$(document).height()},hashChange:function(){this.linkScroll=!0,this.win.one("hashchange",function(){this.linkScroll=!1})},toggleCurrent:function(n){var e=n.closest("li");e.siblings("li.current").removeClass("current"),e.siblings().find("li.current").removeClass("current"),e.find("> ul li.current").removeClass("current"),e.toggleClass("current")}},"undefined"!=typeof window&&(window.SphinxRtdTheme={Navigation:e.exports.ThemeNav,StickyNav:e.exports.ThemeNav}),function(){for(var n=0,e=["ms","moz","webkit","o"],i=0;i