├── .gitignore
├── .travis.yml
├── Datasets.md
├── FAQ.md
├── LICENSE
├── README.rst
├── deepmatcher
    ├── __init__.py
    ├── batch.py
    ├── data
    │   ├── __init__.py
    │   ├── dataset.py
    │   ├── field.py
    │   ├── iterator.py
    │   └── process.py
    ├── models
    │   ├── _utils.py
    │   ├── attr_summarizers.py
    │   ├── core.py
    │   ├── modules.py
    │   ├── word_aggregators.py
    │   ├── word_comparators.py
    │   └── word_contextualizers.py
    ├── optim.py
    ├── runner.py
    └── utils.py
├── docs
    ├── .nojekyll
    ├── Makefile
    ├── html
    │   ├── .buildinfo
    │   ├── _modules
    │   │   ├── deepmatcher
    │   │   │   ├── batch.html
    │   │   │   ├── data
    │   │   │   │   └── process.html
    │   │   │   ├── models
    │   │   │   │   ├── attr_summarizers.html
    │   │   │   │   ├── core.html
    │   │   │   │   ├── modules.html
    │   │   │   │   ├── word_aggregators.html
    │   │   │   │   ├── word_comparators.html
    │   │   │   │   └── word_contextualizers.html
    │   │   │   └── optim.html
    │   │   └── index.html
    │   ├── _sources
    │   │   ├── attr_summarizers.rst.txt
    │   │   ├── batch.rst.txt
    │   │   ├── data.rst.txt
    │   │   ├── deepmatcher.rst.txt
    │   │   ├── index.rst.txt
    │   │   ├── modules.rst.txt
    │   │   ├── notes
    │   │   │   ├── design.rst.txt
    │   │   │   ├── faq.rst.txt
    │   │   │   └── model.rst.txt
    │   │   ├── optim.rst.txt
    │   │   ├── word_aggregators.rst.txt
    │   │   ├── word_comparators.rst.txt
    │   │   └── word_contextualizers.rst.txt
    │   ├── _static
    │   │   ├── ae.png
    │   │   ├── ajax-loader.gif
    │   │   ├── arch.png
    │   │   ├── as.png
    │   │   ├── asr.png
    │   │   ├── basic.css
    │   │   ├── comment-bright.png
    │   │   ├── comment-close.png
    │   │   ├── comment.png
    │   │   ├── css
    │   │   │   ├── badge_only.css
    │   │   │   └── theme.css
    │   │   ├── doctools.js
    │   │   ├── down-pressed.png
    │   │   ├── down.png
    │   │   ├── file.png
    │   │   ├── fonts
    │   │   │   ├── Inconsolata-Bold.ttf
    │   │   │   ├── Inconsolata-Regular.ttf
    │   │   │   ├── Lato-Bold.ttf
    │   │   │   ├── Lato-BoldItalic.ttf
    │   │   │   ├── Lato-Italic.ttf
    │   │   │   ├── Lato-Regular.ttf
    │   │   │   ├── RobotoSlab-Bold.ttf
    │   │   │   ├── RobotoSlab-Regular.ttf
    │   │   │   ├── fontawesome-webfont.eot
    │   │   │   ├── fontawesome-webfont.svg
    │   │   │   ├── fontawesome-webfont.ttf
    │   │   │   ├── fontawesome-webfont.woff
    │   │   │   └── fontawesome-webfont.woff2
    │   │   ├── jquery-3.1.0.js
    │   │   ├── jquery.js
    │   │   ├── js
    │   │   │   ├── modernizr.min.js
    │   │   │   └── theme.js
    │   │   ├── match_input_ex.png
    │   │   ├── minus.png
    │   │   ├── plus.png
    │   │   ├── pygments.css
    │   │   ├── searchtools.js
    │   │   ├── underscore-1.3.1.js
    │   │   ├── underscore.js
    │   │   ├── up-pressed.png
    │   │   ├── up.png
    │   │   └── websupport.js
    │   ├── attr_summarizers.html
    │   ├── batch.html
    │   ├── data.html
    │   ├── deepmatcher.html
    │   ├── genindex.html
    │   ├── index.html
    │   ├── modules.html
    │   ├── notes
    │   │   ├── design.html
    │   │   ├── faq.html
    │   │   └── model.html
    │   ├── objects.inv
    │   ├── optim.html
    │   ├── py-modindex.html
    │   ├── search.html
    │   ├── searchindex.js
    │   ├── word_aggregators.html
    │   ├── word_comparators.html
    │   └── word_contextualizers.html
    ├── make.bat
    └── source
    │   ├── _static
    │       ├── ae.png
    │       ├── arch.png
    │       ├── as.png
    │       ├── asr.png
    │       └── match_input_ex.png
    │   ├── attr_summarizers.rst
    │   ├── batch.rst
    │   ├── conf.py
    │   ├── data.rst
    │   ├── deepmatcher.rst
    │   ├── index.rst
    │   ├── modules.rst
    │   ├── notes
    │       ├── design.rst
    │       ├── faq.rst
    │       └── model.rst
    │   ├── optim.rst
    │   ├── word_aggregators.rst
    │   ├── word_comparators.rst
    │   └── word_contextualizers.rst
├── examples
    ├── data_processing.ipynb
    ├── end_to_end_em.ipynb
    ├── getting_started.ipynb
    ├── matching_models.ipynb
    ├── question_answering.ipynb
    └── sample_data
    │   ├── amz_goog_test.csv
    │   ├── amz_goog_train.csv
    │   ├── amz_goog_unlabeled.csv
    │   ├── amz_goog_validation.csv
    │   └── itunes-amazon
    │       ├── e2e-tutorial
    │           └── gold.csv
    │       ├── tableA.csv
    │       ├── tableB.csv
    │       ├── test.csv
    │       ├── train.csv
    │       ├── unlabeled.csv
    │       └── validation.csv
├── setup.cfg
├── setup.py
└── test
    ├── __init__.py
    ├── test_dataset.py
    ├── test_datasets
        ├── fasttext_sample.vec
        ├── fasttext_sample.vec.zip
        ├── sample_table_large.csv
        ├── sample_table_small.csv
        ├── test_test.csv
        ├── test_train.csv
        ├── test_unlabeled.csv
        └── test_valid.csv
    ├── test_field.py
    ├── test_integration.py
    ├── test_iterator.py
    └── test_process.py


/.gitignore:
--------------------------------------------------------------------------------
1 | *.pyc
2 | *.pth
3 | deepmatcher.egg-info/*
4 | examples/.ipynb_checkpoints/*
5 | docs/build
6 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | matrix:
 2 |   include:
 3 |   - os: linux
 4 |     python: 3.6
 5 |     env: PYTHON_VERSION=3.6
 6 |   - os: linux
 7 |     python: 3.7
 8 |     env: PYTHON_VERSION=3.7
 9 |   - os: osx
10 |     language: generic
11 |     env:
12 |     - PYTHON_VERSION=3.6
13 |   - os: osx
14 |     language: generic
15 |     env:
16 |     - PYTHON_VERSION=3.7
17 | 
18 | notifications:
19 |   email: false
20 | 
21 | before_install:
22 |   - if [[ "$TRAVIS_OS_NAME" == "linux" && "$PYTHON_VERSION" == 2.* ]]; then wget http://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; fi
23 |   - if [[ "$TRAVIS_OS_NAME" == "linux" && "$PYTHON_VERSION" == 3.* ]]; then wget http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; fi
24 |   - if [[ "$TRAVIS_OS_NAME" == "osx" && "$PYTHON_VERSION" == 2.* ]]; then wget https://repo.continuum.io/miniconda/Miniconda2-latest-MacOSX-x86_64.sh -O miniconda.sh; fi
25 |   - if [[ "$TRAVIS_OS_NAME" == "osx" && "$PYTHON_VERSION" == 3.* ]]; then wget https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh; fi
26 |   - bash miniconda.sh -b -p $HOME/miniconda
27 |   - export PATH="$HOME/miniconda/bin:$PATH"
28 |   - conda config --add channels conda-forge
29 |   - echo $PYTHON_VERSION
30 |   - conda create -n testenv python=$PYTHON_VERSION  --yes
31 |   - source activate testenv
32 | 
33 | install:
34 |   - conda install --yes python=$PYTHON_VERSION pip scikit-learn nose
35 |   - pip install -e . | cat
36 |   - python -m nltk.downloader perluniprops nonbreaking_prefixes punkt
37 | 
38 | script:
39 |   - nosetests -v
40 | 


--------------------------------------------------------------------------------
/FAQ.md:
--------------------------------------------------------------------------------
 1 | # FAQ
 2 | 
 3 | ### I get a `TypeError` with a stacktrace similar to [this](https://gist.github.com/sidharthms/2698c2c9fc072036371166c5972b48f9) when I try to train the model.
 4 | 
 5 | This is a known issue and will be fixed in the next release. As a workaround, you can simply change the name of the attribute causing the problem, e.g., you can change "left_type" and "right_type" to "left_entity_type" and "right_entity_type" respectively.
 6 | 
 7 | ### What is `fasttextmirror` and why does `deepmatcher` list it as a dependency?
 8 | 
 9 | This is because the official `fasttext` [release on PyPI](https://pypi.org/project/fasttext/) is out of date and has not been updated in over a year. `fasttextmirror` is a [github fork](https://github.com/sidharthms/fastText) of the official `fasttext` [github repository](https://github.com/facebookresearch/fastText) as of June 5 2018.
10 | 
11 | ### I do not see any output from `dm.data.process` after the very first time I run it.
12 | 
13 | This is because during the first time `dm.data.process` is called on a dataset, it builds a cache of the data. In subsequent calls, it simply reuses the cache so as to load faster.
14 | 
15 | ### Why do I see a message saying "Loading best model..." at the end of training?
16 | 
17 | After each epoch of training `deepmatcher` computes the validation set accuracy of the latest model. If the validation accuracy is better than the best validation accuracy seen so far, the model is saved to disk. After some epochs the accuracy may stop improving and instead may start declining. So at the end of training, `deepmatcher` restores the best model from disk. This is done by modifying the model in-place.
18 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | BSD 3-Clause License
 2 | 
 3 | Copyright (c) Sidharth Mudgal and Han Li 2018,
 4 | All rights reserved.
 5 | 
 6 | Redistribution and use in source and binary forms, with or without
 7 | modification, are permitted provided that the following conditions are met:
 8 | 
 9 | * Redistributions of source code must retain the above copyright notice, this
10 |   list of conditions and the following disclaimer.
11 | 
12 | * Redistributions in binary form must reproduce the above copyright notice,
13 |   this list of conditions and the following disclaimer in the documentation
14 |   and/or other materials provided with the distribution.
15 | 
16 | * Neither the name of the copyright holder nor the names of its
17 |   contributors may be used to endorse or promote products derived from
18 |   this software without specific prior written permission.
19 | 
20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 | 


--------------------------------------------------------------------------------
/README.rst:
--------------------------------------------------------------------------------
  1 | DeepMatcher
  2 | =============
  3 | 
  4 | .. image:: https://travis-ci.org/anhaidgroup/deepmatcher.svg?branch=master
  5 |     :target: https://travis-ci.org/anhaidgroup/deepmatcher
  6 | 
  7 | .. image:: https://img.shields.io/badge/License-BSD%203--Clause-blue.svg
  8 |     :target: https://opensource.org/licenses/BSD-3-Clause
  9 | 
 10 | DeepMatcher is a Python package for performing entity and text matching using deep learning.
 11 | It provides built-in neural networks and utilities that enable you to train and apply
 12 | state-of-the-art deep learning models for entity matching in less than 10 lines of code.
 13 | The models are also easily customizable - the modular design allows any subcomponent to be
 14 | altered or swapped out for a custom implementation.
 15 | 
 16 | As an example, given labeled tuple pairs such as the following:
 17 | 
 18 | .. image:: https://raw.githubusercontent.com/anhaidgroup/deepmatcher/master/docs/source/_static/match_input_ex.png
 19 | 
 20 | DeepMatcher uses labeled tuple pairs and trains a neural network to perform matching, i.e., to
 21 | predict match / non-match labels. The trained network can then be used to obtain labels for
 22 | unlabeled tuple pairs.
 23 | 
 24 | Paper and Data
 25 | ****************
 26 | 
 27 | For details on the architecture of the models used, take a look at our paper `Deep
 28 | Learning for Entity Matching`_ (SIGMOD '18). All public datasets used in
 29 | the paper can be downloaded from the `datasets page <Datasets.md>`__.
 30 | 
 31 | Quick Start: DeepMatcher in 30 seconds
 32 | ******************************************
 33 | 
 34 | There are four main steps in using DeepMatcher:
 35 | 
 36 | 1. Data processing: Load and process labeled training, validation and test CSV data.
 37 | 
 38 | .. code-block:: python
 39 | 
 40 |    import deepmatcher as dm
 41 |    train, validation, test = dm.data.process(path='data_directory',
 42 |        train='train.csv', validation='validation.csv', test='test.csv')
 43 | 
 44 | 2. Model definition: Specify neural network architecture. Uses the built-in hybrid
 45 |    model (as discussed in section 4.4 of `our paper
 46 |    <http://pages.cs.wisc.edu/~anhai/papers1/deepmatcher-sigmod18.pdf>`__) by default. Can
 47 |    be customized to your heart's desire.
 48 | 
 49 | .. code-block:: python
 50 | 
 51 |    model = dm.MatchingModel()
 52 | 
 53 | 3. Model training: Train neural network.
 54 | 
 55 | .. code-block:: python
 56 | 
 57 |    model.run_train(train, validation, best_save_path='best_model.pth')
 58 | 
 59 | 4. Application: Evaluate model on test set and apply to unlabeled data.
 60 | 
 61 | .. code-block:: python
 62 | 
 63 |    model.run_eval(test)
 64 | 
 65 |    unlabeled = dm.data.process_unlabeled(path='data_directory/unlabeled.csv', trained_model=model)
 66 |    model.run_prediction(unlabeled)
 67 | 
 68 | Installation
 69 | **************
 70 | 
 71 | We currently support only Python versions 3.5+. Installing using pip is recommended:
 72 | 
 73 | .. code-block::
 74 | 
 75 |    pip install deepmatcher
 76 | 
 77 | Tutorials
 78 | **********
 79 | 
 80 | **Using DeepMatcher:**
 81 | 
 82 | 1. `Getting Started`_: A more in-depth guide to help you get familiar with the basics of
 83 |    using DeepMatcher.
 84 | 2. `Data Processing`_: Advanced guide on what data processing involves and how to
 85 |    customize it.
 86 | 3. `Matching Models`_: Advanced guide on neural network architecture for entity matching
 87 |    and how to customize it.
 88 | 
 89 | **Entity Matching Workflow:**
 90 | 
 91 | `End to End Entity Matching`_: A guide to develop a complete entity
 92 | matching workflow. The tutorial discusses how to use DeepMatcher with `Magellan`_ to
 93 | perform blocking, sampling, labeling and matching to obtain matching tuple pairs from two
 94 | tables.
 95 | 
 96 | **DeepMatcher for other matching tasks:**
 97 | 
 98 | `Question Answering with DeepMatcher`_: A tutorial on how to use DeepMatcher for question
 99 | answering. Specifically, we will look at `WikiQA`_, a benchmark dataset for the task of
100 | Answer Selection.
101 | 
102 | API Reference
103 | ***************
104 | 
105 | API docs `are here`_.
106 | 
107 | Support
108 | **********
109 | 
110 | Take a look at `the FAQ <FAQ.md>`__ for common issues. If you run into any issues or have questions not answered in the FAQ,
111 | please `file GitHub issues`_ and we will address them asap.
112 | 
113 | The Team
114 | **********
115 | 
116 | DeepMatcher was developed by University of Wisconsin-Madison grad students Sidharth Mudgal
117 | and Han Li, under the supervision of Prof. AnHai Doan and Prof. Theodoros Rekatsinas.
118 | 
119 | .. _`Deep Learning for Entity Matching`: http://pages.cs.wisc.edu/~anhai/papers1/deepmatcher-sigmod18.pdf
120 | .. _`Prof. AnHai Doan's data repository`: https://sites.google.com/site/anhaidgroup/useful-stuff/data
121 | .. _`Magellan`: https://sites.google.com/site/anhaidgroup/projects/magellan
122 | .. _`Getting Started`: https://nbviewer.jupyter.org/github/anhaidgroup/deepmatcher/blob/master/examples/getting_started.ipynb
123 | .. _`Data Processing`: https://nbviewer.jupyter.org/github/anhaidgroup/deepmatcher/blob/master/examples/data_processing.ipynb
124 | .. _`Matching Models`: https://nbviewer.jupyter.org/github/anhaidgroup/deepmatcher/blob/master/examples/matching_models.ipynb
125 | .. _`End to End Entity Matching`: https://nbviewer.jupyter.org/github/anhaidgroup/deepmatcher/blob/master/examples/end_to_end_em.ipynb
126 | .. _`are here`: https://anhaidgroup.github.io/deepmatcher/html/
127 | .. _`Question Answering with DeepMatcher`: https://nbviewer.jupyter.org/github/anhaidgroup/deepmatcher/blob/master/examples/question_answering.ipynb
128 | .. _`WikiQA`: https://aclweb.org/anthology/D15-1237
129 | .. _`file GitHub issues`: https://github.com/anhaidgroup/deepmatcher/issues
130 | 


--------------------------------------------------------------------------------
/deepmatcher/__init__.py:
--------------------------------------------------------------------------------
 1 | r"""
 2 | The deepmatcher package contains high level modules used in the construction of deep
 3 | learning modules for entity matching.
 4 | """
 5 | 
 6 | import logging
 7 | import warnings
 8 | import sys
 9 | 
10 | from .data import process as data_process
11 | from .models import modules
12 | from .models.core import (MatchingModel, AttrSummarizer, WordContextualizer,
13 |                           WordComparator, WordAggregator, Classifier)
14 | from .models import (attr_summarizers, word_aggregators, word_comparators,
15 |                      word_contextualizers)
16 | 
17 | # Register these as submodules of deepmatcher. This helps organize files better while
18 | # permitting an easier way to access these modules.
19 | sys.modules['deepmatcher.attr_summarizers'] = attr_summarizers
20 | sys.modules['deepmatcher.word_contextualizers'] = word_contextualizers
21 | sys.modules['deepmatcher.word_comparators'] = word_comparators
22 | sys.modules['deepmatcher.word_aggregators'] = word_aggregators
23 | sys.modules['deepmatcher.modules'] = modules
24 | 
25 | warnings.filterwarnings('always', module='deepmatcher')
26 | 
27 | logging.basicConfig()
28 | logging.getLogger('deepmatcher.data.field').setLevel(logging.INFO)
29 | 
30 | 
31 | def process(*args, **kwargs):
32 |     warnings.warn('"deepmatcher.process" is deprecated and will be removed in a later '
33 |                   'release, please use "deepmatcher.data.process" instead',
34 |                   DeprecationWarning)
35 |     return data_process(*args, **kwargs)
36 | 
37 | 
38 | __version__ = '0.1.2.post2'
39 | __author__ = 'Sidharth Mudgal, Han Li'
40 | 
41 | __all__ = [
42 |     'attr_summarizers', 'word_aggregators', 'word_comparators', 'word_contextualizers',
43 |     'process', 'MatchingModel', 'AttrSummarizer', 'WordContextualizer', 'WordComparator',
44 |     'WordAggregator', 'Classifier', 'modules'
45 | ]
46 | 
47 | _check_nan = True
48 | 
49 | 
50 | def disable_nan_checks():
51 |     _check_nan = False
52 | 
53 | 
54 | def enable_nan_checks():
55 |     _check_nan = True
56 | 


--------------------------------------------------------------------------------
/deepmatcher/batch.py:
--------------------------------------------------------------------------------
  1 | from collections import namedtuple
  2 | 
  3 | import torch
  4 | 
  5 | AttrTensor_ = namedtuple('AttrTensor', ['data', 'lengths', 'word_probs', 'pc'])
  6 | 
  7 | 
  8 | class AttrTensor(AttrTensor_):
  9 |     """A wrapper around the batch tensor for a specific attribute.
 10 | 
 11 |     The purpose of having a wrapper around the tensor is to include attribute specific
 12 |     metadata along with it. Metadata include the following:
 13 | 
 14 |     * ``lengths``: Lengths of each sequence (attribute value) in the batch.
 15 |     * ``word_probs``: For each sequence in the batch, a list of word probabilities
 16 |       corresponding to words in the sequence.
 17 |     * ``pc``: The first principal component of the sequence embeddings for all values of
 18 |       this attribute. For details on how this is computed refer documentation for
 19 |       :meth:`~deepmatcher.data.MatchingDataset.compute_metadata`. This is used for
 20 |       implementing the SIF model proposed in
 21 |       `this paper <https://openreview.net/pdf?id=SyK00v5xx>`__.
 22 | 
 23 |     This class is essentially a :class:`namedtuple`. The tensor containing the data and
 24 |     the associated metadata described above can be accessed as follows::
 25 | 
 26 |         name_attr = AttrTensor(data, lengths, word_probs, pc)
 27 |         assert(name_attr.data == data)
 28 |         assert(name_attr.lengths == lengths)
 29 |         assert(name_attr.word_probs == word_probs)
 30 |         assert(name_attr.pc == pc)
 31 |     """
 32 | 
 33 |     @staticmethod
 34 |     def __new__(cls, *args, **kwargs):
 35 |         if len(kwargs) == 0:
 36 |             return super(AttrTensor, cls).__new__(cls, *args)
 37 |         else:
 38 |             name = kwargs['name']
 39 |             attr = kwargs['attr']
 40 |             train_info = kwargs['train_info']
 41 |             if isinstance(attr, tuple):
 42 |                 data = attr[0]
 43 |                 lengths = attr[1]
 44 |             else:
 45 |                 data = attr
 46 |                 lengths = None
 47 |             word_probs = None
 48 |             if 'word_probs' in train_info.metadata:
 49 |                 raw_word_probs = train_info.metadata['word_probs'][name]
 50 |                 word_probs = torch.Tensor(
 51 |                     [[raw_word_probs[int(w)] for w in b] for b in data.data])
 52 |                 if data.is_cuda:
 53 |                     word_probs = word_probs.cuda()
 54 |             pc = None
 55 |             if 'pc' in train_info.metadata:
 56 |                 pc = torch.Tensor(train_info.metadata['pc'][name])
 57 |                 if data.is_cuda:
 58 |                     pc = pc.cuda()
 59 |             return AttrTensor(data, lengths, word_probs, pc)
 60 | 
 61 |     @staticmethod
 62 |     def from_old_metadata(data, old_attrtensor):
 63 |         """Wrap a PyTorch :class:`torch.Tensor` into an :class:`AttrTensor`.
 64 | 
 65 |         The metadata information is (shallow) copied from a pre-existing
 66 |         :class:`AttrTensor`. This is useful when the data for an attribute is
 67 |         transformed by a neural network and we wish the wrap the result into an
 68 |         :class:`AttrTensor` for further processing by another module that requires
 69 |         access to metadata.
 70 | 
 71 |         Args:
 72 |             old_attrtensor (:class:`AttrTensor`):
 73 |                 The pre-existing :class:`AttrTensor` to copy metadata from.
 74 |         """
 75 |         return AttrTensor(data, *old_attrtensor[1:])
 76 | 
 77 | 
 78 | class MatchingBatch(object):
 79 |     """A batch of data and associated metadata for a text matching task.
 80 | 
 81 |     Consists of one :class:`AttrTensor` (containing the data and metadata) for each
 82 |     attribute. For example, the :class:`AttrTensor` s of a :class:`MatchingBatch` object
 83 |     ``mbatch`` for a matching task with two attribtues ``name`` and ``category``, can be
 84 |     accessed as follows::
 85 | 
 86 |         name_attr = mbatch.name
 87 |         category_attr = mbatch.category
 88 |     """
 89 | 
 90 |     def __init__(self, input, train_info):
 91 |         copy_fields = train_info.all_text_fields
 92 |         for name in copy_fields:
 93 |             setattr(self, name,
 94 |                     AttrTensor(
 95 |                         name=name, attr=getattr(input, name),
 96 |                         train_info=train_info))
 97 |         for name in [train_info.label_field, train_info.id_field]:
 98 |             if name is not None and hasattr(input, name):
 99 |                 setattr(self, name, getattr(input, name))
100 | 


--------------------------------------------------------------------------------
/deepmatcher/data/__init__.py:
--------------------------------------------------------------------------------
 1 | from .field import MatchingField, reset_vector_cache
 2 | from .dataset import MatchingDataset
 3 | from .iterator import MatchingIterator
 4 | from .process import process, process_unlabeled
 5 | from .dataset import split
 6 | 
 7 | __all__ = [
 8 |     'MatchingField', 'MatchingDataset', 'MatchingIterator', 'process', 'process_unlabeled', 'split',
 9 |     'reset_vector_cache'
10 | ]
11 | 


--------------------------------------------------------------------------------
/deepmatcher/data/field.py:
--------------------------------------------------------------------------------
  1 | import gzip
  2 | import logging
  3 | import os
  4 | import shutil
  5 | import tarfile
  6 | import zipfile
  7 | 
  8 | import nltk
  9 | import six
 10 | 
 11 | import fasttext
 12 | import torch
 13 | from torchtext.legacy import data, vocab
 14 | from torchtext.utils import download_from_url
 15 | from urllib.request import urlretrieve
 16 | 
 17 | logger = logging.getLogger(__name__)
 18 | 
 19 | 
 20 | class FastText(vocab.Vectors):
 21 | 
 22 |     def __init__(self,
 23 |                  suffix='wiki-news-300d-1M.vec.zip',
 24 |                  url_base='https://dl.fbaipublicfiles.com/fasttext/vectors-english/',
 25 |                  **kwargs):
 26 |         url = url_base + suffix
 27 |         base, ext = os.path.splitext(suffix)
 28 |         name = suffix if ext == '.vec' else base
 29 |         print('Fasttext url b', url_base)
 30 |         super(FastText, self).__init__(name, url=url, **kwargs)
 31 | 
 32 | 
 33 | class FastTextBinary(vocab.Vectors):
 34 | 
 35 |     name_base = 'wiki.{}.bin'
 36 |     _direct_en_url = 'https://drive.google.com/uc?export=download&id=1Vih8gAmgBnuYDxfblbT94P6WjB7s1ZSh'
 37 | 
 38 |     def __init__(self, language='en', url_base=None, cache=None):
 39 |         """
 40 |         Arguments:
 41 |            language: Language of fastText pre-trained embedding model
 42 |            cache: directory for cached model
 43 |          """
 44 |         cache = os.path.expanduser(cache)
 45 |         base = url_base or 'https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/wiki.{}.zip'
 46 |         url = base.format(language)
 47 |         backup_url = None
 48 |         self.destination = os.path.join(cache, 'wiki.{}.zip'.format(language))
 49 |         if language == 'en' and url_base is None:
 50 |             backup_url = url
 51 |             self.backup_destination = self.destination
 52 |             url = FastTextBinary._direct_en_url
 53 |             self.destination = os.path.join(cache, 'wiki.en.bin')
 54 |         name = FastTextBinary.name_base.format(language)
 55 | 
 56 |         self.cache(name, cache, url=url, backup_url=backup_url)
 57 | 
 58 |     def __getitem__(self, token):
 59 |         return torch.Tensor(self.model.get_word_vector(token))
 60 | 
 61 |     def cache(self, name, cache, url=None, backup_url=None):
 62 |         path = os.path.join(cache, name)
 63 |         if not os.path.isfile(path) and url:
 64 |             if not os.path.exists(self.destination):
 65 |                 logger.info('Downloading vectors from {} to {}'.format(url, self.destination))
 66 |                 if not os.path.exists(cache):
 67 |                     os.makedirs(cache)
 68 |                 if not os.path.isfile(self.destination):
 69 |                     if 'drive.google.com' in url:
 70 |                         try:
 71 |                             download_from_url(url, self.destination)
 72 |                         except RuntimeError as err:
 73 |                             if 'quota was exceeded' in str(err):
 74 |                                 logger.info('Unable to fetch cached English Word Embeddings'
 75 |                                         ' from {}\nDownloading embeddings from {} to {}'.format(
 76 |                                         url, backup_url, self.backup_destination))
 77 |                                 if not os.path.exists(self.backup_destination):
 78 |                                     urlretrieve(backup_url, self.backup_destination)
 79 |                                 self.destination = self.backup_destination
 80 |                     else:
 81 |                         urlretrieve(url, self.destination)
 82 |             logger.info('Extracting vectors into {}'.format(cache))
 83 |             ext = os.path.splitext(self.destination)[1][1:]
 84 |             if ext == 'zip':
 85 |                 with zipfile.ZipFile(self.destination, "r") as zf:
 86 |                     zf.extractall(cache)
 87 |             elif ext == 'gz':
 88 |                 try:
 89 |                     with tarfile.open(self.destination, 'r:gz') as tar:
 90 |                         tar.extractall(path=cache)
 91 |                 except:
 92 |                     with gzip.open(self.destination, 'rb') as infile:
 93 |                         with open(path, 'wb') as outfile:
 94 |                             shutil.copyfileobj(infile, outfile)
 95 |         if not os.path.isfile(path):
 96 |             raise RuntimeError('no vectors found at {}'.format(path))
 97 | 
 98 |         self.model = fasttext.load_model(path)
 99 |         self.dim = len(self['a'])
100 | 
101 | 
102 | class MatchingVocab(vocab.Vocab):
103 | 
104 |     def extend_vectors(self, tokens, vectors):
105 |         tot_dim = sum(v.dim for v in vectors)
106 |         prev_len = len(self.itos)
107 | 
108 |         new_tokens = []
109 |         for token in tokens:
110 |             if token not in self.stoi:
111 |                 self.itos.append(token)
112 |                 self.stoi[token] = len(self.itos) - 1
113 |                 new_tokens.append(token)
114 |         self.vectors.resize_(len(self.itos), tot_dim)
115 | 
116 |         for i in range(prev_len, prev_len + len(new_tokens)):
117 |             token = self.itos[i]
118 |             assert token == new_tokens[i - prev_len]
119 | 
120 |             start_dim = 0
121 |             for v in vectors:
122 |                 end_dim = start_dim + v.dim
123 |                 self.vectors[i][start_dim:end_dim] = v[token.strip()]
124 |                 start_dim = end_dim
125 |             assert (start_dim == tot_dim)
126 | 
127 | 
128 | class MatchingField(data.Field):
129 |     vocab_cls = MatchingVocab
130 | 
131 |     _cached_vec_data = {}
132 | 
133 |     def __init__(self, tokenize='nltk', id=False, **kwargs):
134 |         self.tokenizer_arg = tokenize
135 |         self.is_id = id
136 |         tokenize = MatchingField._get_tokenizer(tokenize)
137 |         super(MatchingField, self).__init__(tokenize=tokenize, **kwargs)
138 | 
139 |     @staticmethod
140 |     def _get_tokenizer(tokenizer):
141 |         if tokenizer == 'nltk':
142 |             return nltk.word_tokenize
143 |         return tokenizer
144 | 
145 |     def preprocess_args(self):
146 |         attrs = [
147 |             'sequential', 'init_token', 'eos_token', 'unk_token', 'preprocessing',
148 |             'lower', 'tokenizer_arg'
149 |         ]
150 |         args_dict = {attr: getattr(self, attr) for attr in attrs}
151 |         for param, arg in list(six.iteritems(args_dict)):
152 |             if six.callable(arg):
153 |                 del args_dict[param]
154 |         return args_dict
155 | 
156 |     @classmethod
157 |     def _get_vector_data(cls, vecs, cache):
158 |         if not isinstance(vecs, list):
159 |             vecs = [vecs]
160 | 
161 |         vec_datas = []
162 |         for vec in vecs:
163 |             if not isinstance(vec, vocab.Vectors):
164 |                 vec_name = vec
165 |                 vec_data = cls._cached_vec_data.get(vec_name)
166 |                 if vec_data is None:
167 |                     parts = vec_name.split('.')
168 |                     if parts[0] == 'fasttext':
169 |                         if parts[2] == 'bin':
170 |                             vec_data = FastTextBinary(language=parts[1], cache=cache)
171 |                         elif parts[2] == 'vec' and parts[1] == 'wiki':
172 |                             vec_data = FastText(
173 |                                 suffix='wiki-news-300d-1M.vec.zip', cache=cache)
174 |                         elif parts[2] == 'vec' and parts[1] == 'crawl':
175 |                             vec_data = FastText(
176 |                                 suffix='crawl-300d-2M.vec.zip', cache=cache)
177 |                 if vec_data is None:
178 |                     vec_data = vocab.pretrained_aliases[vec_name](cache=cache)
179 |                 cls._cached_vec_data[vec_name] = vec_data
180 |                 vec_datas.append(vec_data)
181 |             else:
182 |                 vec_datas.append(vec)
183 | 
184 |         return vec_datas
185 | 
186 |     def build_vocab(self, *args, vectors=None, cache=None, **kwargs):
187 |         if cache is not None:
188 |             cache = os.path.expanduser(cache)
189 |         if vectors is not None:
190 |             vectors = MatchingField._get_vector_data(vectors, cache)
191 |         super(MatchingField, self).build_vocab(*args, vectors=vectors, **kwargs)
192 | 
193 |     def extend_vocab(self, *args, vectors=None, cache=None):
194 |         sources = []
195 |         for arg in args:
196 |             if isinstance(arg, data.Dataset):
197 |                 sources += [
198 |                     getattr(arg, name)
199 |                     for name, field in arg.fields.items()
200 |                     if field is self
201 |                 ]
202 |             else:
203 |                 sources.append(arg)
204 | 
205 |         tokens = set()
206 |         for source in sources:
207 |             for x in source:
208 |                 if not self.sequential:
209 |                     tokens.add(x)
210 |                 else:
211 |                     tokens.update(x)
212 | 
213 |         if self.vocab.vectors is not None:
214 |             vectors = MatchingField._get_vector_data(vectors, cache)
215 |             self.vocab.extend_vectors(tokens, vectors)
216 | 
217 |     def numericalize(self, arr, *args, **kwargs):
218 |         if not self.is_id:
219 |             return super(MatchingField, self).numericalize(arr, *args, **kwargs)
220 |         return arr
221 | 
222 | 
223 | def reset_vector_cache():
224 |     MatchingField._cached_vec_data = {}
225 | 


--------------------------------------------------------------------------------
/deepmatcher/data/iterator.py:
--------------------------------------------------------------------------------
 1 | from __future__ import division
 2 | 
 3 | import logging
 4 | 
 5 | from torchtext.legacy import data
 6 | 
 7 | from ..batch import MatchingBatch
 8 | 
 9 | logger = logging.getLogger(__name__)
10 | 
11 | 
12 | class MatchingIterator(data.BucketIterator):
13 | 
14 |     def __init__(self,
15 |                  dataset,
16 |                  train_info,
17 |                  train,
18 |                  batch_size,
19 |                  sort_in_buckets=None,
20 |                  device=None,
21 |                  **kwargs):
22 |         if sort_in_buckets is None:
23 |             sort_in_buckets = train
24 |         self.sort_in_buckets = sort_in_buckets
25 |         self.train_info = train_info
26 |         super(MatchingIterator, self).__init__(
27 |             dataset, batch_size, train=train, repeat=False, sort=False, device=device, **kwargs)
28 | 
29 |     @classmethod
30 |     def splits(cls, datasets, batch_sizes=None, **kwargs):
31 |         """Create Iterator objects for multiple splits of a dataset.
32 | 
33 |         Args:
34 |             datasets: Tuple of Dataset objects corresponding to the splits. The
35 |                 first such object should be the train set.
36 |             batch_sizes: Tuple of batch sizes to use for the different splits,
37 |                 or None to use the same batch_size for all splits.
38 |             Remaining keyword arguments: Passed to the constructor of the
39 |                 iterator class being used.
40 |         """
41 |         if batch_sizes is None:
42 |             batch_sizes = [kwargs.pop('batch_size')] * len(datasets)
43 |         ret = []
44 |         for i in range(len(datasets)):
45 |             ret.append(
46 |                 cls(datasets[i],
47 |                     train_info=datasets[0],
48 |                     train=i==0,
49 |                     batch_size=batch_sizes[i],
50 |                     **kwargs))
51 |         return tuple(ret)
52 | 
53 |     def __iter__(self):
54 |         for batch in super(MatchingIterator, self).__iter__():
55 |             yield MatchingBatch(batch, self.train_info)
56 | 
57 |     def create_batches(self):
58 |         if self.sort_in_buckets:
59 |             return data.BucketIterator.create_batches(self)
60 |         else:
61 |             return data.Iterator.create_batches(self)
62 | 


--------------------------------------------------------------------------------
/deepmatcher/models/_utils.py:
--------------------------------------------------------------------------------
 1 | import pdb
 2 | 
 3 | import six
 4 | 
 5 | import deepmatcher as dm
 6 | import torch
 7 | 
 8 | from ..batch import AttrTensor
 9 | 
10 | 
11 | # From onmt-py
12 | def sequence_mask(lengths, max_len=None):
13 |     """
14 |     Creates a boolean mask from sequence lengths.
15 |     """
16 |     batch_size = lengths.numel()
17 |     max_len = max_len or lengths.max()
18 |     return (torch.arange(0, max_len).type_as(lengths).repeat(batch_size, 1).lt(
19 |         lengths.unsqueeze(1)))
20 | 
21 | 
22 | def get_module(cls, op, required=False, op_kwarg=None, **kwargs):
23 |     if op is None and not required or isinstance(op, cls):
24 |         return op
25 |     elif required:
26 |         return cls(**kwargs)
27 |     elif isinstance(op, six.string_types):
28 |         if op_kwarg is not None:
29 |             kwargs[op_kwarg] = op
30 |             return cls(**kwargs)
31 |         else:
32 |             return cls(op, **kwargs)
33 |     elif six.callable(op) and not isinstance(op, torch.nn.Module):
34 |         return dm.modules.LazyModuleFn(op)
35 |     else:
36 |         raise ValueError(
37 |             str(cls) + ' arg must be a valid string, a ' + str(cls) + ' object, or a '
38 |             'callable.')
39 | 
40 | 
41 | def check_nan(*values):
42 |     for value in values:
43 |         if isinstance(value, AttrTensor):
44 |             value = value.data
45 |         if isinstance(value, torch.Tensor) and (value != value).any():
46 |             print('NaN detected!!!')
47 |             pdb.set_trace()
48 | 


--------------------------------------------------------------------------------
/deepmatcher/models/attr_summarizers.py:
--------------------------------------------------------------------------------
  1 | """Defines built-in attribute summarizers."""
  2 | 
  3 | import deepmatcher as dm
  4 | 
  5 | 
  6 | class SIF(dm.AttrSummarizer):
  7 |     """__init__(word_contextualizer=None, word_comparator=None, word_aggregator=None, \
  8 |         hidden_size=None)
  9 | 
 10 |     The attribute summarizer for the SIF (Smooth Inverse Frequency) model.
 11 | 
 12 |     Args:
 13 |         word_contextualizer (string or :class:`~deepmatcher.WordContextualizer` or callable): The
 14 |             word contextualizer module (refer to :class:`~deepmatcher.WordContextualizer` for
 15 |             details) to use for attribute summarization. The SIF model does not take word
 16 |             context information into account, hence this defaults to None.
 17 |         word_comparator (string or :class:`~deepmatcher.WordComparator` or callable): The word
 18 |             comparator module (refer to :class:`~deepmatcher.WordComparator` for details) to use
 19 |             for attribute summarization. The SIF model does not perform word by word
 20 |             comparisons, hence this defaults to None.
 21 |         word_aggregator (string or :class:`~deepmatcher.WordAggregator` or callable): The word
 22 |             aggregator module (refer to :class:`~deepmatcher.WordAggregator` for details) to use
 23 |             for attribute summarization. This model uses SIF-based weighted average
 24 |             aggregation over the  word embeddings of an input sequence, hence this
 25 |             defaults to 'sif-pool'.
 26 |         hidden_size (int): The hidden size to use for all 3 attribute summarization
 27 |             sub-modules (i.e., word contextualizer, word comparator, and word aggregator),
 28 |             if they are customized. By default, the SIF model does not use this parameter.
 29 |     """
 30 | 
 31 |     def _init(self,
 32 |               word_contextualizer=None,
 33 |               word_comparator=None,
 34 |               word_aggregator=None,
 35 |               hidden_size=None):
 36 |         word_aggregator = word_aggregator or 'sif-pool'
 37 |         super(SIF, self)._init(
 38 |             word_contextualizer=word_contextualizer,
 39 |             word_comparator=word_comparator,
 40 |             word_aggregator=word_aggregator,
 41 |             hidden_size=hidden_size)
 42 | 
 43 | 
 44 | class RNN(dm.AttrSummarizer):
 45 |     r"""The attribute summarizer for the RNN model.
 46 | 
 47 |     Args:
 48 |         word_contextualizer (string or :class:`~deepmatcher.WordContextualizer` or callable): The
 49 |             word contextualizer module (refer to :class:`~deepmatcher.WordContextualizer` for
 50 |             details) to use for attribute summarization. This model uses RNN to take into
 51 |             account the context information, and the default value is 'gru' (i.e., uses
 52 |             the bidirectional GRU model as the specific RNN instantiation.) Other options
 53 |             are 'rnn' (the vanilla bi-RNN) and 'lstm' (the bi-LSTM model).
 54 |         word_comparator (string or :class:`~deepmatcher.WordComparator` or callable): The word
 55 |             comparator module (refer to :class:`~deepmatcher.WordComparator` for details) to use
 56 |             for attribute summarization. The RNN model does not perform word by word
 57 |             comparisons, hence this defaults to None.
 58 |         word_aggregator (string or :class:`~deepmatcher.WordAggregator` or callable): The word
 59 |             aggregator module (refer to :class:`~deepmatcher.WordAggregator` for details) to use
 60 |             for attribute summarization. The RNN model uses bi-directional RNN and
 61 |             concatenates the last ouputs of the forward and backward RNNs, hence the
 62 |             default value is 'birnn-last-pool'.
 63 |         hidden_size (int): The hidden size to use for the word contextualizer. This value
 64 |             will also be used as the hidden size for the other 2 attribute summarization
 65 |             sub-modules (i.e., word comparator, and word aggregator), if they are
 66 |             customized. If not specified, the hidden size for each component will be set
 67 |             to be the same as its input size. E.g. if the word embedding dimension is 300
 68 |             and hidden_size is None, the word contextualizer's hidden size will be 300.
 69 |     """
 70 | 
 71 |     def _init(self,
 72 |               word_contextualizer=None,
 73 |               word_comparator=None,
 74 |               word_aggregator=None,
 75 |               hidden_size=None):
 76 |         word_contextualizer = word_contextualizer or 'gru'
 77 |         word_aggregator = word_aggregator or 'birnn-last-pool'
 78 |         super(RNN, self)._init(
 79 |             word_contextualizer=word_contextualizer,
 80 |             word_comparator=word_comparator,
 81 |             word_aggregator=word_aggregator,
 82 |             hidden_size=hidden_size)
 83 | 
 84 | 
 85 | class Attention(dm.AttrSummarizer):
 86 |     r"""The attribute summarizer for the attention-based model.
 87 | 
 88 |     Args:
 89 |         word_contextualizer (string or :class:`~deepmatcher.WordContextualizer` or callable): The
 90 |             word contextualizer module (refer to :class:`~deepmatcher.WordContextualizer` for
 91 |             details) to use for attribute summarization. The attention model does not take
 92 |             word context information into account, hence this defaults to None.
 93 |         word_comparator (string or :class:`~deepmatcher.WordComparator` or callable): The word
 94 |             comparator module (refer to :class:`~deepmatcher.WordComparator` for details) to use
 95 |             for attribute summarization. The attention model performs word by word
 96 |             comparison with the decomposable attention mechanism, hence this defaults to
 97 |             'decomposable-attention'.
 98 |         word_aggregator (string or :class:`~deepmatcher.WordAggregator` or callable): The word
 99 |             aggregator module (refer to :class:`~deepmatcher.WordAggregator` for details) to use
100 |             for attribute summarization. The Attention model performs the aggregation by
101 |             summing over the comparison results from the word comparator, divided by the
102 |             length of the input sequence (to get constant variance through the network
103 |             flow). Hence this defaults to 'divsqrt-pool'.
104 |         hidden_size (int): The hidden size to use for the word comparator. This value
105 |             will also be used as the hidden size for the other 2 attribute summarization
106 |             sub-modules (i.e., word contextualizer, and word aggregator), if they are
107 |             customized. If not specified, the hidden size for each component will be set
108 |             to be the same as its input size. E.g. if the word embedding dimension is 300
109 |             and hidden_size is None, the word contextualizer's hidden size will be 300.
110 |     """
111 | 
112 |     def _init(self,
113 |               word_contextualizer=None,
114 |               word_comparator=None,
115 |               word_aggregator=None,
116 |               hidden_size=None):
117 |         word_comparator = word_comparator or 'decomposable-attention'
118 |         word_aggregator = word_aggregator or 'divsqrt-pool'
119 |         super(Attention, self)._init(
120 |             word_contextualizer=word_contextualizer,
121 |             word_comparator=word_comparator,
122 |             word_aggregator=word_aggregator,
123 |             hidden_size=hidden_size)
124 | 
125 | 
126 | class Hybrid(dm.AttrSummarizer):
127 |     r"""The attribute summarizer for the hybrid model.
128 | 
129 |     Args:
130 |         word_contextualizer (string or :class:`~deepmatcher.WordContextualizer` or callable): The
131 |             word contextualizer module (refer to :class:`~deepmatcher.WordContextualizer` for
132 |             details) to use for attribute summarization. The hybrid model uses
133 |             bidirectional GRU(a specific type of RNN) to take into account the context
134 |             information. The default value is 'gru'.
135 |         word_comparator (string or :class:`~deepmatcher.WordComparator` or callable): The word
136 |             comparator module (refer to :class:`~deepmatcher.WordComparator` for details) to use
137 |             for attribute summarization. The hybrid model performs word by word comparison
138 |             over the raw input word embeddings (rather than the RNN hiddens states), hence
139 |             this defaults to an Attention object with 'decomposable' as the attention
140 |             mechanism on the raw input embeddings.
141 |         word_aggregator (string or :class:`~deepmatcher.WordAggregator` or callable): The word
142 |             aggregator module (refer to :class:`~deepmatcher.WordAggregator` for details) to use
143 |             for attribute summarization. A second layer of attention has been used for the
144 |             aggregation. Please consult the paper for more information. The default value
145 |             is 'concat-attention-with-rnn'.
146 |         hidden_size (int): The hidden size to use for all 3 attribute summarization
147 |             sub-modules (i.e., word contextualizer, word comparator, and word aggregator),
148 |             if they are customized.
149 |     """
150 | 
151 |     def _init(self,
152 |               word_contextualizer=None,
153 |               word_comparator=None,
154 |               word_aggregator=None,
155 |               hidden_size=None):
156 |         word_contextualizer = word_contextualizer or 'gru'
157 |         word_comparator = word_comparator or dm.word_comparators.Attention(
158 |             alignment_network='decomposable', raw_alignment=True)
159 |         word_aggregator = word_aggregator or 'attention-with-rnn'
160 |         super(Hybrid, self)._init(
161 |             word_contextualizer=word_contextualizer,
162 |             word_comparator=word_comparator,
163 |             word_aggregator=word_aggregator,
164 |             hidden_size=hidden_size)
165 | 


--------------------------------------------------------------------------------
/deepmatcher/models/word_aggregators.py:
--------------------------------------------------------------------------------
  1 | import pdb
  2 | 
  3 | import six
  4 | 
  5 | import deepmatcher as dm
  6 | import torch
  7 | import torch.nn as nn
  8 | from torch.autograd import Variable
  9 | 
 10 | from . import _utils
 11 | from ..batch import AttrTensor
 12 | 
 13 | 
 14 | class Pool(dm.modules.Pool, dm.WordAggregator):
 15 |     """Pooling based Word Aggregator.
 16 | 
 17 |     Takes the same parameters as the :class:`~deepmatcher.modules.Pool` module.
 18 |     """
 19 |     pass
 20 | 
 21 | 
 22 | class AttentionWithRNN(dm.WordAggregator):
 23 |     r"""__init__(hidden_size=None, input_dropout=0, rnn='gru', rnn_pool_style='birnn-last', score_dropout=0, input_context_comparison_network='1-layer-highway', value_transform_network=None, transform_dropout=0, input_size=None)
 24 | 
 25 |     Attention and RNN based Word Aggregator. This class can be used when the
 26 |     aggregation on the primary input also needs the information from the context.
 27 |     Specifically, the :class:`~deepmatcher.attr_summarizers.Hybrid` attribute summarizer
 28 |     uses this aggregation approach by default. This module takes a primary input sequence
 29 |     and a context input sequence and computes a single summary vector for the primary
 30 |     input sequence based on the information in the context input. To do so, it does the
 31 |     following:
 32 | 
 33 |     1. Applies an :ref:`rnn-op` over the context input.
 34 |     2. Uses a :ref:`pool-op` operation over the RNN to obtain a single vector summarizing
 35 |        the information in the context input.
 36 |     3. Based on this context summary vector, uses attention to score the relevance of each
 37 |        vector in the primary input sequence.
 38 |     4. Performs a weighted average of the vectors in the primary input sequence based on
 39 |        the computed scores to obtain a context dependent summary of the primary input
 40 |        sequence.
 41 | 
 42 |     Args:
 43 |         hidden_size (int):
 44 |             The default hidden size to use for the RNN,
 45 |             `input_context_comparison_network`, and the `value_transform_network`.
 46 |         input_dropout (float):
 47 |             If non-zero, applies dropout to the input to this module. Dropout probability
 48 |             must be between 0 and 1.
 49 |         rnn (string or :class:`~deepmatcher.modules.RNN` or callable):
 50 |             The RNN used in Step 1 described above. Argument must specify an
 51 |             :ref:`rnn-op` operation.
 52 |         rnn_pool_style (string):
 53 |             The pooling operation used in Step 2 described above. Argument must specify a
 54 |             :ref:`pool-op` operation.
 55 |         score_dropout (float):
 56 |             If non-zero, applies dropout to the attention scores computed in Step 3
 57 |             described above. Dropout probability must be between 0 and 1.
 58 |         input_context_comparison_network (string or :class:`~deepmatcher.modules.Transform` or callable):
 59 |             The neural network that takes each vector in the primary input sequence
 60 |             concatenated with the context summary vector to obtain a hidden vector
 61 |             representing the primary vector's relevance to the context input.
 62 |             Argument must specify a :ref:`transform-op` operation.
 63 |         value_transform_network (string or :class:`~deepmatcher.modules.Transform` or callable):
 64 |             The neural network to transform the primary input sequence before taking its
 65 |             weighted average in Step 4 described above. Argument must be None or specify a
 66 |             :ref:`transform-op` operation.
 67 |         transform_dropout (float):
 68 |             If non-zero, applies dropout to the output of the `value_transform_network`,
 69 |             if applicable. Dropout probability must be between 0 and 1.
 70 |         input_size (int):
 71 |             The number of features in the input to the module. This parameter will be
 72 |             automatically specified by :class:`LazyModule`.
 73 |     """
 74 | 
 75 |     def _init(self,
 76 |               hidden_size=None,
 77 |               input_dropout=0,
 78 |               rnn='gru',
 79 |               rnn_pool_style='birnn-last',
 80 |               score_dropout=0,
 81 |               input_context_comparison_network='1-layer-highway',
 82 |               value_transform_network=None,
 83 |               transform_dropout=0,
 84 |               input_size=None):
 85 | 
 86 |         # self.alignment_network = dm.modules._alignment_module(
 87 |         #     alignment_network, hidden_size=hidden_size)
 88 | 
 89 |         assert rnn is not None
 90 |         self.rnn = _utils.get_module(dm.modules.RNN, rnn, hidden_size=hidden_size)
 91 |         self.rnn.expect_signature('[AxBxC] -> [AxBx{D}]'.format(D=hidden_size))
 92 | 
 93 |         self.rnn_pool = dm.modules.Pool(rnn_pool_style)
 94 | 
 95 |         self.input_context_comparison_network = dm.modules._transform_module(
 96 |             input_context_comparison_network, hidden_size=hidden_size)
 97 |         self.scoring_network = dm.modules._transform_module('1-layer', hidden_size=1)
 98 |         self.value_transform_network = dm.modules._transform_module(
 99 |             value_transform_network, hidden_size=hidden_size)
100 | 
101 |         self.input_dropout = nn.Dropout(input_dropout)
102 |         self.transform_dropout = nn.Dropout(transform_dropout)
103 |         self.score_dropout = nn.Dropout(score_dropout)
104 | 
105 |         self.softmax = nn.Softmax(dim=1)
106 | 
107 |     def _forward(self, input_with_meta, context_with_meta):
108 |         r"""
109 |         The forward function of the attention-with-RNN netowrk.
110 | 
111 |         Args:
112 |         input_with_meta ():
113 |             The input sequence with metadata information.
114 |         context_with_meta ():
115 |             The context sequence with metadata information.
116 |         """
117 |         input = self.input_dropout(input_with_meta.data)
118 |         context = self.input_dropout(context_with_meta.data)
119 | 
120 |         context_rnn_output = self.rnn(
121 |             AttrTensor.from_old_metadata(context, context_with_meta))
122 | 
123 |         # Dims: batch x 1 x hidden_size
124 |         context_pool_output = self.rnn_pool(context_rnn_output).data.unsqueeze(1)
125 | 
126 |         # Dims: batch x len1 x hidden_size
127 |         context_pool_repeated = context_pool_output.repeat(1, input.size(1), 1)
128 | 
129 |         # Dims: batch x len1 x (hidden_size * 2)
130 |         concatenated = torch.cat((input, context_pool_repeated), dim=2)
131 | 
132 |         # Dims: batch x len1
133 |         raw_scores = self.scoring_network(
134 |             self.input_context_comparison_network(concatenated)).squeeze(2)
135 | 
136 |         alignment_scores = self.score_dropout(raw_scores)
137 | 
138 |         if input_with_meta.lengths is not None:
139 |             mask = _utils.sequence_mask(input_with_meta.lengths)
140 |             alignment_scores.data.masked_fill_(~mask, -float('inf'))
141 | 
142 |         # Make values along dim 2 sum to 1.
143 |         normalized_scores = self.softmax(alignment_scores)
144 | 
145 |         transformed = input
146 |         if self.value_transform_network is not None:
147 |             transformed = self.transform_dropout(self.value_transform_network(input))
148 | 
149 |         weighted_sum = torch.bmm(normalized_scores.unsqueeze(1), transformed).squeeze(1)
150 |         return AttrTensor.from_old_metadata(weighted_sum, input_with_meta)
151 | 


--------------------------------------------------------------------------------
/deepmatcher/models/word_comparators.py:
--------------------------------------------------------------------------------
  1 | import pdb
  2 | 
  3 | import six
  4 | 
  5 | import deepmatcher as dm
  6 | import torch
  7 | import torch.nn as nn
  8 | from torch.autograd import Variable
  9 | 
 10 | from . import _utils
 11 | from ..batch import AttrTensor
 12 | 
 13 | 
 14 | class Attention(dm.WordComparator):
 15 |     r"""__init__(heads=1, hidden_size=None, raw_alignment=False, input_dropout=0, alignment_network='decomposable', scale=False, score_dropout=0, value_transform_network=None, input_transform_network=None, value_merge='concat', transform_dropout=0, comparison_merge='concat', comparison_network='2-layer-highway', input_size=None)
 16 | 
 17 |     Attention based Word Comparator with `multi-head <https://arxiv.org/abs/1706.03762>`__
 18 |     support. This module does the following:
 19 | 
 20 |     1. Computes an alignment matrix between the primary input sequence and the context
 21 |        input sequence.
 22 |     2. For each vector in the primary input sequence, takes a weighted average over all
 23 |        vectors in the context input sequence, where weights are given by the alignment
 24 |        matrix. Intuitively, for each word / phrase vector in the primary input sequence,
 25 |        this represents the aligning word / phrase vector in the context input sequence.
 26 |     3. Compares the vectors in the primary input sequence with its aligning vectors.
 27 | 
 28 |     Args:
 29 |         heads (int):
 30 |             Number of attention heads to use. Defaults to 1.
 31 |         hidden_size (int):
 32 |             The default hidden size of the `alignment_network`, transform networks (if
 33 |             applicable), and comparison network.
 34 |         raw_alignment (bool):
 35 |             If True, uses the contextualized version (transformed by the Word
 36 |             Contextualizer module) of the input and context sequences for computing
 37 |             alignment in Step 1 described above. If False, uses the raw
 38 |             (non-contextualized) word embedding sequences for computing alignment in Step
 39 |             1 described above. For step 2, the Word Contextualizer's version of the
 40 |             context sequence is used for computing the weighted averages in both cases.
 41 |             Raw alignment has been shown to `perform better and speed up convergence
 42 |             <http://pages.cs.wisc.edu/~anhai/papers1/deepmatcher-sigmod18.pdf>`__,
 43 |             especially in cases of limited training data.
 44 |         input_dropout (float):
 45 |             If non-zero, applies dropout to the input to this module. Dropout probability
 46 |             must be between 0 and 1.
 47 |         alignment_network (string or :class:`deepmatcher.modules.AlignmentNetwork` or callable):
 48 |             The neural network that takes the primary input sequence, aligns the word /
 49 |             phrase vectors in this sequence with word / phrase vector in the context
 50 |             sequence, and returns the corresponding alignment score matrix. Argument must
 51 |             specify a :ref:`align-op` operation.
 52 |         scale (bool):
 53 |             Whether to scale the alignment scores by the square root of the
 54 |             `hidden_size` parameter. Based on `scaled dot-product attention
 55 |             <https://arxiv.org/abs/1706.03762>`__
 56 |         score_dropout (float):
 57 |             If non-zero, applies dropout to the alignment score matrix. Dropout
 58 |             probability must be between 0 and 1.
 59 |         value_transform_network (string or :class:`~deepmatcher.modules.Transform` or callable):
 60 |             The neural network to transform the context input sequence before taking the
 61 |             weighted averages in Step 2 described above. Argument must be None or specify
 62 |             a :ref:`transform-op` operation. If the argument is a string, the hidden size
 63 |             of the transform operation is computed as :code:`hidden_size // heads`. If
 64 |             argument is None, and `heads` is 1, then the values are not transformed. If
 65 |             argument is None and `heads` is > 1, then a 1 layer highway network without
 66 |             any non-linearity is used. The hidden size for this is computed as mentioned
 67 |             above.
 68 |         input_transform_network (string or :class:`~deepmatcher.modules.Transform` or callable):
 69 |             The neural network to transform the primary input sequence before it is
 70 |             compared with the aligning vectors in the context sequence in Step 3 described
 71 |             above. Argument must be None or specify a :ref:`transform-op` operation. If
 72 |             None, uses the same neural network as the value transform network (sharing not
 73 |             just the structure but also weight parameters).
 74 |         value_merge (string or :class:`~deepmatcher.modules.Merge` or callable):
 75 |             Specifies how to merge the outputs of all attention heads for each vector
 76 |             in the primary input sequence. Concatenates the outputs of all heads by
 77 |             default. Argument must specify a :ref:`merge-op` operation.
 78 |         transform_dropout (float):
 79 |             If non-zero, applies dropout to the outputs of the `value_transform_network`
 80 |             and `input_transform_network`, if applicable. Dropout probability must be
 81 |             between 0 and 1.
 82 |         comparison_merge (string or :class:`~deepmatcher.modules.Merge` or callable):
 83 |             For each vector in the primary input sequence, specifies how to merge it with
 84 |             its aligning vector in the context input sequence, to obtain a single vector.
 85 |             The resulting sequence vectors forms the input to the `comparison_network`.
 86 |             Concatenates each primary input vector with its aligning vector by default.
 87 |             Argument must specify a :ref:`merge-op` operation.
 88 |         comparison_network (string or :class:`~deepmatcher.modules.Transform` or callable):
 89 |             The neural network to compare the vectors in the primary input sequence and
 90 |             their aligning vectors in the context input. Input to this module is produced
 91 |             by the `comparison_merge` operation. Argument must specify a
 92 |             :ref:`transform-op` operation.
 93 |         input_size (int):
 94 |             The number of features in the input to the module. This parameter will be
 95 |             automatically specified by :class:`LazyModule`.
 96 |     """
 97 | 
 98 |     def _init(self,
 99 |               heads=1,
100 |               hidden_size=None,
101 |               raw_alignment=False,
102 |               input_dropout=0,
103 |               alignment_network='decomposable',
104 |               scale=False,
105 |               score_dropout=0,
106 |               value_transform_network=None,
107 |               input_transform_network=None,
108 |               value_merge='concat',
109 |               transform_dropout=0,
110 |               comparison_merge='concat',
111 |               comparison_network='2-layer-highway',
112 |               input_size=None):
113 |         hidden_size = hidden_size if hidden_size is not None else input_size[0]
114 | 
115 |         self.alignment_networks = nn.ModuleList()
116 |         for head in range(heads):
117 |             self.alignment_networks.append(
118 |                 dm.modules._alignment_module(alignment_network, hidden_size))
119 | 
120 |         if value_transform_network is None and heads > 1:
121 |             value_transform_network = dm.modules.Transform(
122 |                 '1-layer-highway', non_linearity=None, hidden_size=hidden_size // heads)
123 |         self.value_transform_network = dm.modules._transform_module(
124 |             value_transform_network, hidden_size // heads)
125 | 
126 |         if input_transform_network is None:
127 |             self.input_transform_network = self.value_transform_network
128 |         else:
129 |             self.input_transform_network = dm.modules._transform_module(
130 |                 input_transform_network, hidden_size // heads)
131 | 
132 |         self.value_merge = dm.modules._merge_module(value_merge)
133 |         self.comparison_merge = dm.modules._merge_module(comparison_merge)
134 |         self.comparison_network = dm.modules._transform_module(comparison_network,
135 |                                                                hidden_size)
136 | 
137 |         self.input_dropout = nn.Dropout(input_dropout)
138 |         self.transform_dropout = nn.Dropout(transform_dropout)
139 |         self.score_dropout = nn.Dropout(score_dropout)
140 |         self.softmax = nn.Softmax(dim=2)
141 | 
142 |         self.raw_alignment = raw_alignment
143 |         self.heads = heads
144 |         self.scale = scale
145 | 
146 |     def _forward(self,
147 |                  input_with_meta,
148 |                  context_with_meta,
149 |                  raw_input_with_meta=None,
150 |                  raw_context_with_meta=None):
151 |         input = self.input_dropout(input_with_meta.data)
152 |         context = self.input_dropout(context_with_meta.data)
153 |         raw_input = self.input_dropout(raw_input_with_meta.data)
154 |         raw_context = self.input_dropout(raw_context_with_meta.data)
155 | 
156 |         queries = input
157 |         keys = context
158 |         values = context
159 |         if self.raw_alignment:
160 |             queries = raw_input
161 |             keys = raw_context
162 | 
163 |         inputs_transformed = []
164 |         values_aligned = []
165 |         for head in range(self.heads):
166 |             # Dims: batch x len1 x len2
167 |             alignment_scores = self.score_dropout(self.alignment_networks[head](queries,
168 |                                                                                 keys))
169 |             if self.scale:
170 |                 alignment_scores = alignment_scores / torch.sqrt(hidden_size)
171 | 
172 |             if context_with_meta.lengths is not None:
173 |                 mask = _utils.sequence_mask(context_with_meta.lengths)
174 |                 mask = mask.unsqueeze(1)  # Make it broadcastable.
175 |                 alignment_scores.data.masked_fill_(~mask, -float('inf'))
176 | 
177 |             # Make values along dim 2 sum to 1.
178 |             normalized_scores = self.softmax(alignment_scores)
179 | 
180 |             if self.input_transform_network is not None:
181 |                 inputs_transformed.append(
182 |                     self.transform_dropout(self.input_transform_network(input)))
183 |             if self.value_transform_network is not None:
184 |                 values_transformed = self.transform_dropout(
185 |                     self.value_transform_network(values))
186 |             else:
187 |                 values_transformed = values
188 | 
189 |             # Dims: batch x len1 x channels
190 |             values_aligned.append(torch.bmm(normalized_scores, values_transformed))
191 | 
192 |         inputs_merged = input
193 |         if inputs_transformed:
194 |             inputs_merged = self.value_merge(*inputs_transformed)
195 |         values_merged = self.value_merge(*values_aligned)
196 | 
197 |         comparison_merged = self.comparison_merge(inputs_merged, values_merged)
198 |         comparison = self.comparison_network(comparison_merged)
199 | 
200 |         return AttrTensor.from_old_metadata(comparison, input_with_meta)
201 | 


--------------------------------------------------------------------------------
/deepmatcher/models/word_contextualizers.py:
--------------------------------------------------------------------------------
  1 | import six
  2 | 
  3 | import deepmatcher as dm
  4 | import torch
  5 | import torch.nn as nn
  6 | from torch.autograd import Variable
  7 | 
  8 | from . import _utils
  9 | from ..batch import AttrTensor
 10 | 
 11 | 
 12 | class RNN(dm.modules.RNN, dm.WordContextualizer):
 13 |     """Multi layered RNN based Word Contextualizer.
 14 | 
 15 |     Supports dropout and residual / highway connections. Takes the same parameters as the
 16 |     :class:`~deepmatcher.modules.RNN` module.
 17 |     """
 18 |     pass
 19 | 
 20 | 
 21 | # class CNN(dm.WordContextualizer):
 22 | #     pass
 23 | 
 24 | 
 25 | class SelfAttention(dm.WordContextualizer):
 26 |     """__init__(heads=1, hidden_size=None, input_dropout=0, alignment_network='decomposable', scale=False, score_dropout=0, value_transform_network=None, value_merge='concat', transform_dropout=0, output_transform_network=None, output_dropout=0, bypass_network='highway', input_size=None)
 27 | 
 28 |     Self Attention based Word Contextualizer.
 29 | 
 30 |     Supports `vanilla self attention <https://arxiv.org/abs/1606.01933>`__ and `multi-head
 31 |     self attention <https://arxiv.org/abs/1706.03762>`__.
 32 | 
 33 |     Args:
 34 |         heads (int):
 35 |             Number of attention heads to use. Defaults to 1.
 36 |         hidden_size (int):
 37 |             The default hidden size of the `alignment_network` and transform networks, if
 38 |             they are not disabled.
 39 |         input_dropout (float):
 40 |             If non-zero, applies dropout to the input to this module. Dropout probability
 41 |             must be between 0 and 1.
 42 |         alignment_network (string or :class:`deepmatcher.modules.AlignmentNetwork` or callable):
 43 |             The neural network takes the input sequence, aligns the words in the sequence
 44 |             with other words in the sequence, and returns the corresponding alignment
 45 |             score matrix. Argument must specify a :ref:`align-op` operation.
 46 |         scale (bool):
 47 |             Whether to scale the alignment scores by the square root of the
 48 |             `hidden_size` parameter. Based on `scaled dot-product attention
 49 |             <https://arxiv.org/abs/1706.03762>`__
 50 |         score_dropout (float):
 51 |             If non-zero, applies dropout to the alignment score matrix. Dropout
 52 |             probability must be between 0 and 1.
 53 |         value_transform_network (string or :class:`~deepmatcher.modules.Transform` or callable):
 54 |             For each word embedding in the input sequence, SelfAttention takes a weighted
 55 |             average of the aligning values, i.e., the aligning word embeddings based on
 56 |             the alignment scores. This parameter specifies the neural network to transform
 57 |             the values (word embeddings) before taking the weighted average. Argument must
 58 |             be None or specify a :ref:`transform-op` operation. If the argument is a
 59 |             string, the hidden size of the transform operation is computed as
 60 |             :code:`hidden_size // heads`. If argument is None, and `heads` is 1, then the
 61 |             values are not transformed. If argument is None and `heads` is > 1, then a 1
 62 |             layer highway network without any non-linearity is used. The hidden size for
 63 |             this is computed as mentioned above.
 64 |         value_merge (string or :class:`~deepmatcher.modules.Merge` or callable):
 65 |             For each word embedding in the input sequence, each SelfAttention head
 66 |             produces one corresponding vector as output. This parameter specifies how
 67 |             to merge the outputs of all attention heads for each word embedding.
 68 |             Concatenates the outputs of all heads by default. Argument must specify a
 69 |             :ref:`merge-op` operation.
 70 |         transform_dropout (float):
 71 |             If non-zero, applies dropout to the output of the `value_transform_network`,
 72 |             if applicable. Dropout probability must be between 0 and 1.
 73 |         output_transform_network (string or :class:`~deepmatcher.modules.Transform` or callable):
 74 |             For each word embedding in the input sequence, SelfAttention produces one
 75 |             corresponding vector as output. This neural network specifies how to transform
 76 |             each of these output vectors to obtain a hidden representation of size
 77 |             `hidden_size`. Argument must be None or specify a :ref:`transform-op`
 78 |             operation. If argument is None, and `heads` is 1, then the output vectors are
 79 |             not transformed. If argument is None and `heads` is > 1, then a 1 layer
 80 |             highway network without any non-linearity is used.
 81 |         output_dropout (float):
 82 |             If non-zero, applies dropout to the output of the `output_transform_network`,
 83 |             if applicable. Dropout probability must be between 0 and 1.
 84 |         bypass_network (string or :class:`Bypass` or callable):
 85 |             The bypass network (e.g. residual or highway network) to use. The input word
 86 |             embedding sequence to this module is considered as the raw input to the bypass
 87 |             network and the final output vector sequence (output of `value_merge` or
 88 |             `output_transform_network` if applicable) is considered as the transformed
 89 |             input. Argument must specify a :ref:`bypass-op` operation. If None, does not
 90 |             use a bypass network.
 91 |         input_size (int):
 92 |             The number of features in the input to the module. This parameter will be
 93 |             automatically specified by :class:`LazyModule`.
 94 |     """
 95 | 
 96 |     def _init(self,
 97 |               heads=1,
 98 |               hidden_size=None,
 99 |               input_dropout=0,
100 |               alignment_network='decomposable',
101 |               scale=False,
102 |               score_dropout=0,
103 |               value_transform_network=None,
104 |               value_merge='concat',
105 |               transform_dropout=0,
106 |               output_transform_network=None,
107 |               output_dropout=0,
108 |               bypass_network='highway',
109 |               input_size=None):
110 |         hidden_size = hidden_size if hidden_size is not None else input_size
111 | 
112 |         self.alignment_networks = nn.ModuleList()
113 |         for head in range(heads):
114 |             self.alignment_networks.append(
115 |                 dm.modules._alignment_module(alignment_network, hidden_size))
116 | 
117 |         if value_transform_network is None and heads > 1:
118 |             value_transform_network = dm.modules.Transform(
119 |                 '1-layer-highway', non_linearity=None, hidden_size=hidden_size // heads)
120 |         self.value_transform_network = dm.modules._transform_module(
121 |             value_transform_network, hidden_size // heads)
122 | 
123 |         self.value_merge = dm.modules._merge_module(value_merge)
124 | 
125 |         self.softmax = nn.Softmax(dim=2)
126 | 
127 |         if output_transform_network is None and heads > 1:
128 |             output_transform_network = dm.modules.Transform(
129 |                 '1-layer-highway', non_linearity=None, hidden_size=hidden_size)
130 |         self.output_transform_network = dm.modules._transform_module(
131 |             output_transform_network, hidden_size)
132 | 
133 |         self.input_dropout = nn.Dropout(input_dropout)
134 |         self.transform_dropout = nn.Dropout(transform_dropout)
135 |         self.score_dropout = nn.Dropout(output_dropout)
136 |         self.output_dropout = nn.Dropout(output_dropout)
137 | 
138 |         self.bypass_network = dm.modules._bypass_module(bypass_network)
139 | 
140 |         self.heads = heads
141 |         self.scale = scale
142 |         self.hidden_size = hidden_size
143 | 
144 |     def _forward(self, input_with_meta):
145 |         input = self.input_dropout(input_with_meta.data)
146 | 
147 |         values_aligned = []
148 |         for head in range(self.heads):
149 |             # Dims: batch x len1 x len2
150 |             alignment_scores = self.score_dropout(self.alignment_networks[head](input,
151 |                                                                                 input))
152 | 
153 |             if self.scale:
154 |                 alignment_scores = alignment_scores / torch.sqrt(self.hidden_size)
155 | 
156 |             if input_with_meta.lengths is not None:
157 |                 mask = _utils.sequence_mask(input_with_meta.lengths)
158 |                 mask = mask.unsqueeze(1)  # Make it broadcastable.
159 |                 alignment_scores.data.masked_fill_(~mask, -float('inf'))
160 | 
161 |             normalized_scores = self.softmax(alignment_scores)
162 | 
163 |             if self.value_transform_network is not None:
164 |                 values_transformed = self.transform_dropout(
165 |                     self.value_transform_network(input))
166 |             else:
167 |                 values_transformed = input
168 | 
169 |             # Dims: batch x len1 x channels
170 |             values_aligned.append(torch.bmm(normalized_scores, values_transformed))
171 | 
172 |         values_merged = self.value_merge(*values_aligned)
173 | 
174 |         output = values_merged
175 |         if self.output_transform_network:
176 |             output = self.output_transform_network(output)
177 |         output = self.output_dropout(output)
178 | 
179 |         final_output = self.bypass_network(output, input)
180 | 
181 |         return AttrTensor.from_old_metadata(final_output, input_with_meta)
182 | 


--------------------------------------------------------------------------------
/deepmatcher/optim.py:
--------------------------------------------------------------------------------
  1 | import logging
  2 | 
  3 | import torch
  4 | import torch.nn as nn
  5 | import torch.optim as optim
  6 | from torch.autograd import Variable
  7 | from torch.nn.utils import clip_grad_norm_
  8 | 
  9 | logger = logging.getLogger('deepmatcher.optim')
 10 | 
 11 | 
 12 | class SoftNLLLoss(nn.NLLLoss):
 13 |     """A soft version of negative log likelihood loss with support for label smoothing.
 14 | 
 15 |     Effectively equivalent to PyTorch's :class:`torch.nn.NLLLoss`, if `label_smoothing`
 16 |     set to zero. While the numerical loss values will be different compared to
 17 |     :class:`torch.nn.NLLLoss`, this loss results in the same gradients. This is because
 18 |     the implementation uses :class:`torch.nn.KLDivLoss` to support multi-class label
 19 |     smoothing.
 20 | 
 21 |     Args:
 22 |         label_smoothing (float):
 23 |             The smoothing parameter :math:`epsilon` for label smoothing. For details on
 24 |             label smoothing refer `this paper <https://arxiv.org/abs/1512.00567v1>`__.
 25 |         weight (:class:`torch.Tensor`):
 26 |             A 1D tensor of size equal to the number of classes. Specifies the manual
 27 |             weight rescaling applied to each class. Useful in cases when there is severe
 28 |             class imbalance in the training set.
 29 |         num_classes (int):
 30 |             The number of classes.
 31 |         size_average (bool):
 32 |             By default, the losses are averaged for each minibatch over observations **as
 33 |             well as** over dimensions. However, if ``False`` the losses are instead
 34 |             summed. This is a keyword only parameter.
 35 |     """
 36 | 
 37 |     def __init__(self, label_smoothing=0, weight=None, num_classes=2, **kwargs):
 38 |         super(SoftNLLLoss, self).__init__(**kwargs)
 39 |         self.label_smoothing = label_smoothing
 40 |         self.confidence = 1 - self.label_smoothing
 41 |         self.num_classes = num_classes
 42 |         self.register_buffer('weight', Variable(weight))
 43 | 
 44 |         assert label_smoothing >= 0.0 and label_smoothing <= 1.0
 45 | 
 46 |         self.criterion = nn.KLDivLoss(**kwargs)
 47 | 
 48 |     def forward(self, input, target):
 49 |         one_hot = torch.zeros_like(input)
 50 |         one_hot.fill_(self.label_smoothing / (self.num_classes - 1))
 51 |         one_hot.scatter_(1, target.unsqueeze(1).long(), self.confidence)
 52 | 
 53 |         if self.weight is not None:
 54 |             one_hot.mul_(self.weight)
 55 | 
 56 |         return self.criterion(input, one_hot)
 57 | 
 58 | 
 59 | # This class is based on the Optimizer class in the ONMT-py project.
 60 | class Optimizer(object):
 61 |     """Controller class for optimization.
 62 | 
 63 |     Mostly a thin wrapper for `optim`, but also useful for implementing learning rate
 64 |     scheduling beyond what is currently available. Also implements necessary methods for
 65 |     training RNNs such as grad manipulations.
 66 | 
 67 |     Args:
 68 |         method (string):
 69 |             One of [sgd, adagrad, adadelta, adam].
 70 |         lr (float):
 71 |             Learning rate.
 72 |         lr_decay (float):
 73 |             Learning rate decay multiplier.
 74 |         start_decay_at (int):
 75 |             Epoch to start learning rate decay. If None, starts decay when the validation
 76 |             accuracy stops improving. Defaults to 1.
 77 |         beta1, beta2 (float):
 78 |             Hyperarameters for adam.
 79 |         adagrad_accum (float, optional):
 80 |             Initialization hyperparameter for adagrad.
 81 |     """
 82 | 
 83 |     def __init__(self,
 84 |                  method='adam',
 85 |                  lr=0.001,
 86 |                  max_grad_norm=5,
 87 |                  start_decay_at=1,
 88 |                  beta1=0.9,
 89 |                  beta2=0.999,
 90 |                  adagrad_accum=0.0,
 91 |                  lr_decay=0.8):
 92 |         self.last_acc = None
 93 |         self.lr = lr
 94 |         self.original_lr = lr
 95 |         self.max_grad_norm = max_grad_norm
 96 |         self.method = method
 97 |         self.lr_decay = lr_decay
 98 |         self.start_decay_at = start_decay_at
 99 |         self.start_decay = False
100 |         self._step = 0
101 |         self.betas = [beta1, beta2]
102 |         self.adagrad_accum = adagrad_accum
103 |         self.params = None
104 |         logger.info('Initial learning rate: {:0.3e}'.format(self.lr))
105 | 
106 |     def set_parameters(self, params):
107 |         """Sets the model parameters and initializes the base optimizer.
108 | 
109 |         Args:
110 |             params:
111 |                 Dictionary of named model parameters. Parameters that do not require
112 |                 gradients will be filtered out for optimization.
113 |         """
114 |         self.params = []
115 |         for k, p in params:
116 |             if p.requires_grad:
117 |                 self.params.append(p)
118 |         if self.method == 'sgd':
119 |             self.base_optimizer = optim.SGD(self.params, lr=self.lr)
120 |         elif self.method == 'adagrad':
121 |             self.base_optimizer = optim.Adagrad(self.params, lr=self.lr)
122 |             for group in self.base_optimizer.param_groups:
123 |                 for p in group['params']:
124 |                     self.base_optimizer.state[p]['sum'] = self.base_optimizer\
125 |                         .state[p]['sum'].fill_(self.adagrad_accum)
126 |         elif self.method == 'adadelta':
127 |             self.base_optimizer = optim.Adadelta(self.params, lr=self.lr)
128 |         elif self.method == 'adam':
129 |             self.base_optimizer = optim.Adam(
130 |                 self.params, lr=self.lr, betas=self.betas, eps=1e-9)
131 |         else:
132 |             raise RuntimeError("Invalid optim method: " + self.method)
133 | 
134 |     def _set_rate(self, lr):
135 |         for param_group in self.base_optimizer.param_groups:
136 |             param_group['lr'] = self.lr
137 | 
138 |     def step(self):
139 |         """Update the model parameters based on current gradients.
140 | 
141 |         Optionally, will employ gradient clipping.
142 |         """
143 |         self._step += 1
144 | 
145 |         if self.max_grad_norm:
146 |             clip_grad_norm_(self.params, self.max_grad_norm)
147 |         self.base_optimizer.step()
148 | 
149 |     def update_learning_rate(self, acc, epoch):
150 |         """Decay learning rate.
151 | 
152 |         Decays lerning rate if val perf does not improve or we hit the `start_decay_at`
153 |         limit.
154 | 
155 |         Args:
156 |             acc:
157 |                 The accuracy score on the validation set.
158 |             epoch:
159 |                 The current epoch number.
160 |         """
161 | 
162 |         if self.start_decay_at is not None and epoch >= self.start_decay_at:
163 |             self.start_decay = True
164 |         if self.last_acc is not None and acc < self.last_acc:
165 |             self.start_decay = True
166 | 
167 |         if self.start_decay:
168 |             self.lr = self.lr * self.lr_decay
169 |             logger.info('Setting learning rate to {:0.3e} for next epoch'.format(self.lr))
170 | 
171 |         self.last_acc = acc
172 |         self._set_rate(self.lr)
173 | 


--------------------------------------------------------------------------------
/deepmatcher/utils.py:
--------------------------------------------------------------------------------
1 | class Bunch:
2 | 
3 |     def __init__(self, **kwds):
4 |         self.__dict__.update(kwds)
5 | 
6 | 
7 | def tally_parameters(model):
8 |     return sum([p.nelement() for p in model.parameters() if p.requires_grad])
9 | 


--------------------------------------------------------------------------------
/docs/.nojekyll:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/.nojekyll


--------------------------------------------------------------------------------
/docs/Makefile:
--------------------------------------------------------------------------------
 1 | # Minimal makefile for Sphinx documentation
 2 | #
 3 | 
 4 | # You can set these variables from the command line.
 5 | SPHINXOPTS    = -v
 6 | SPHINXBUILD   = sphinx-build
 7 | SPHINXPROJ    = DeepMatcher
 8 | SOURCEDIR     = source
 9 | BUILDDIR      = .
10 | 
11 | # Put it first so that "make" without argument is like "make help".
12 | help:
13 | 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
14 | 
15 | .PHONY: help Makefile
16 | 
17 | # Catch-all target: route all unknown targets to Sphinx using the new
18 | # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
19 | %: Makefile
20 | 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
21 | 


--------------------------------------------------------------------------------
/docs/html/.buildinfo:
--------------------------------------------------------------------------------
1 | # Sphinx build info version 1
2 | # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
3 | config: 15e75be89226dbf5aa5de2a2c3469209
4 | tags: 645f666f9bcd5a90fca523b33c5a78b7
5 | 


--------------------------------------------------------------------------------
/docs/html/_sources/attr_summarizers.rst.txt:
--------------------------------------------------------------------------------
 1 | .. role:: hidden
 2 |     :class: hidden-section
 3 | 
 4 | deepmatcher.attr_summarizers
 5 | ============================
 6 | 
 7 | .. automodule:: deepmatcher.attr_summarizers
 8 | .. currentmodule:: deepmatcher.attr_summarizers
 9 | 
10 | :hidden:`SIF`
11 | ~~~~~~~~~~~~~~~~~~
12 | .. autoclass:: SIF
13 |     :members:
14 | 
15 | :hidden:`RNN`
16 | ~~~~~~~~~~~~~~~~~~
17 | .. autoclass:: RNN
18 |     :members:
19 | 
20 | :hidden:`Attention`
21 | ~~~~~~~~~~~~~~~~~~
22 | .. autoclass:: Attention
23 |     :members:
24 | 
25 | :hidden:`Hybrid`
26 | ~~~~~~~~~~~~~~~~~~
27 | .. autoclass:: Hybrid
28 |     :members:
29 | 


--------------------------------------------------------------------------------
/docs/html/_sources/batch.rst.txt:
--------------------------------------------------------------------------------
 1 | .. role:: hidden
 2 |     :class: hidden-section
 3 | 
 4 | deepmatcher.batch
 5 | =========================
 6 | 
 7 | .. automodule:: deepmatcher.batch
 8 | .. currentmodule:: deepmatcher.batch
 9 | 
10 | :hidden:`AttrTensor`
11 | ~~~~~~~~~~~~~~~~~~~~~
12 | .. autoclass:: AttrTensor
13 |     :members:
14 | 
15 | :hidden:`MatchingBatch`
16 | ~~~~~~~~~~~~~~~~~~~~~~~~
17 | .. autoclass:: MatchingBatch
18 |     :members:
19 | 


--------------------------------------------------------------------------------
/docs/html/_sources/data.rst.txt:
--------------------------------------------------------------------------------
 1 | .. role:: hidden
 2 |     :meth: hidden-section
 3 | 
 4 | deepmatcher.data
 5 | =========================
 6 | 
 7 | .. automodule:: deepmatcher.data
 8 | .. currentmodule:: deepmatcher.data
 9 | 
10 | :hidden:`process`
11 | ~~~~~~~~~~~~~~~~~~
12 | .. autofunction:: deepmatcher.data.process
13 | 
14 | :hidden:`process_unlabeled`
15 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
16 | .. autofunction:: deepmatcher.data.process_unlabeled
17 | 


--------------------------------------------------------------------------------
/docs/html/_sources/deepmatcher.rst.txt:
--------------------------------------------------------------------------------
 1 | .. role:: hidden
 2 |     :class: hidden-section
 3 | 
 4 | deepmatcher
 5 | ==================================
 6 | 
 7 | .. automodule:: deepmatcher
 8 | 
 9 | Main Modules
10 | ----------------------------------
11 | 
12 | :hidden:`MatchingModel`
13 | ~~~~~~~~~~~~~~~~~~
14 | .. autoclass:: MatchingModel
15 |     :members:
16 | 
17 | :hidden:`AttrSummarizer`
18 | ~~~~~~~~~~~~~~~~~~
19 | .. autoclass:: deepmatcher.AttrSummarizer
20 |     :members:
21 | 
22 | :hidden:`Classifier`
23 | ~~~~~~~~~~~~~~~~~~
24 | .. autoclass:: deepmatcher.Classifier
25 |     :members:
26 | 
27 | 
28 | Components of Attribute Summarizer
29 | ----------------------------------
30 | 
31 | :hidden:`WordContextualizer`
32 | ~~~~~~~~~~~~~~~~~~
33 | .. autoclass:: deepmatcher.WordContextualizer
34 |     :members:
35 | 
36 | :hidden:`WordComparator`
37 | ~~~~~~~~~~~~~~~~~~
38 | .. autoclass:: deepmatcher.WordComparator
39 |     :members:
40 | 
41 | :hidden:`WordAggregator`
42 | ~~~~~~~~~~~~~~~~~~
43 | .. autoclass:: deepmatcher.WordAggregator
44 |     :members:
45 | 


--------------------------------------------------------------------------------
/docs/html/_sources/index.rst.txt:
--------------------------------------------------------------------------------
 1 | .. DeepMatcher documentation master file, created by
 2 |    sphinx-quickstart on Sat Mar 31 15:32:50 2018.
 3 |    You can adapt this file completely to your liking, but it should at least
 4 |    contain the root `toctree` directive.
 5 | 
 6 | :github_url: https://github.com/sidharthms/deepmatcher
 7 | 
 8 | DeepMatcher documentation
 9 | =======================================
10 | 
11 | DeepMatcher is a deep learning library for entity matching.
12 | 
13 | Tutorials
14 | ---------
15 | 
16 | Tutorials are located `here <https://github.com/sidharthms/deepmatcher#tutorials>`_
17 | 
18 | .. toctree::
19 |    :maxdepth: 1
20 |    :caption: Package Reference
21 | 
22 |    deepmatcher
23 |    deepmatcher.attr_summarizers <attr_summarizers>
24 |    deepmatcher.word_contextualizers <word_contextualizers>
25 |    deepmatcher.word_comparators <word_comparators>
26 |    deepmatcher.word_aggregators <word_aggregators>
27 |    deepmatcher.modules <modules>
28 |    deepmatcher.optim <optim>
29 |    deepmatcher.data <data>
30 |    deepmatcher.batch <batch>
31 | 
32 | .. automodule:: deepmatcher
33 |    :members:
34 | 
35 | Indices and tables
36 | ==================
37 | 
38 | * :ref:`genindex`
39 | * :ref:`modindex`
40 | * :ref:`search`
41 | 


--------------------------------------------------------------------------------
/docs/html/_sources/modules.rst.txt:
--------------------------------------------------------------------------------
  1 | .. role:: hidden
  2 |     :class: hidden-section
  3 | 
  4 | deepmatcher.modules
  5 | =========================
  6 | 
  7 | .. automodule:: deepmatcher.modules
  8 | .. currentmodule:: deepmatcher.modules
  9 | 
 10 | Standard Operations
 11 | -------------------------------
 12 | 
 13 | Many components in DeepMatcher, e.g. AttentionWithRNN word aggregator, allow users to
 14 | customize the behavior of operations performed by the component, such as alignment, vector
 15 | transformation, pooling, etc., by setting a parameter that specifies the operation. Here
 16 | we describe operations commonly used across the package, and show how to specify them.
 17 | 
 18 | .. _transform-op:
 19 | 
 20 | Transform
 21 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 22 | 
 23 | The transform operation takes a single vector and performs transforms it to produce
 24 | another vector as output. The transformation may be non-linear. A transform operation can
 25 | be specified by using one of the following:
 26 | 
 27 | * A string: One of the `styles` supported by the :class:`Transform` module.
 28 | * An instance of :class:`Transform`.
 29 | * A :attr:`callable`: A function that returns a PyTorch :class:`~torch.nn.Module`. This
 30 |   module must have the same input and output shape signature as the :class:`Transform`
 31 |   module.
 32 | 
 33 | This operation is implemented by the :class:`Transform` module:
 34 | 
 35 | .. autoclass:: Transform
 36 |     :members:
 37 | 
 38 | .. _pool-op:
 39 | 
 40 | Pool
 41 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 42 | 
 43 | The Pool operation takes a sequence of vectors and aggregates this sequence to produce a
 44 | single vector as output. A Pool operation can be specified using one of the
 45 | following:
 46 | 
 47 | * A string: One of the `styles` supported by the :class:`Pool` module.
 48 | * An instance of :class:`Pool`.
 49 | * A :attr:`callable`: A function that returns a PyTorch :class:`~torch.nn.Module`. This
 50 |   module must have the same input and output shape signature as the :class:`Pool` module.
 51 | 
 52 | This operation is implemented by the :class:`Pool` module:
 53 | 
 54 | .. autoclass:: Pool
 55 |     :members:
 56 | 
 57 | .. _merge-op:
 58 | 
 59 | Merge
 60 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 61 | 
 62 | The Merge operation takes two or more vectors and aggregates the information in them to
 63 | produce a single vector as output. Unlike the case of :ref:`pool-op` operation, the input
 64 | vectors here are not considered to be sequential in nature. A Merge operation can be
 65 | specified using one of the following:
 66 | 
 67 | * A string: One of the `styles` supported by the :class:`Merge` module. Note that some
 68 |   styles only support two input vectors to be merged, while others allow multiple inputs.
 69 | * An instance of :class:`Merge`.
 70 | * A :attr:`callable`: A function that returns a PyTorch :class:`~torch.nn.Module`. This
 71 |   module must have the same input and output shape signature as the
 72 |   :class:`Merge` module.
 73 | 
 74 | This operation is implemented by the :class:`Merge` module:
 75 | 
 76 | .. autoclass:: Merge
 77 |     :members:
 78 | 
 79 | .. _align-op:
 80 | 
 81 | Align
 82 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 83 | 
 84 | The Align operation takes two sequences of vectors, aligns the words in them, and returns
 85 | the corresponding alignment score matrix. For each word in the first sequence, the
 86 | alignment matrix contains unnormalized scores indicating the degree to which each word in
 87 | the second sequence aligns with it. For an example of one way to do this, take a look `at
 88 | this paper <https://arxiv.org/abs/1606.01933>`__. An Align operation can be specified
 89 | using one of the following:
 90 | 
 91 | * A string: One of the `styles` supported by the :class:`AlignmentNetwork` module.
 92 | * An instance of :class:`AlignmentNetwork`.
 93 | * A :attr:`callable`: A function that returns a PyTorch :class:`~torch.nn.Module`. This
 94 |   module must have the same input and output shape signature as the
 95 |   :class:`AlignmentNetwork` module.
 96 | 
 97 | This operation is implemented by the :class:`AlignmentNetwork` module:
 98 | 
 99 | .. autoclass:: AlignmentNetwork
100 |     :members:
101 | 
102 | .. _bypass-op:
103 | 
104 | Bypass
105 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
106 | 
107 | The Bypass operation takes two tensors, one corresponding to an input tensor and the other
108 | corresponding to a transformed version of the first tensor, applies a bypass network
109 | and returns one tensor of the same size as the transformed tensor. Examples of bypass
110 | networks include `residual networks <https://arxiv.org/abs/1512.03385>`__ and
111 | `highway networks <https://arxiv.org/abs/1505.00387>`__.
112 | 
113 | * A string: One of the `styles` supported by the :class:`Bypass` module.
114 | * An instance of :class:`Bypass`.
115 | * A :attr:`callable`: A function that returns a PyTorch :class:`~torch.nn.Module`. This
116 |   module must have the same input and output shape signature as the
117 |   :class:`Bypass` module.
118 | 
119 | This operation is implemented by the :class:`Bypass` module:
120 | 
121 | .. autoclass:: Bypass
122 |     :members:
123 | 
124 | .. _rnn-op:
125 | 
126 | RNN
127 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
128 | 
129 | This operation takes a sequence of vectors and produces a context-aware transformation of
130 | the input sequence as output. For an intro to RNNs, take a look at
131 | `this article <http://www.wildml.com/2015/09/recurrent-neural-networks-tutorial-part-1-introduction-to-rnns/>`__.
132 | An RNN operation can be specified using one of the following:
133 | 
134 | * A string: One of the `unit_types` supported by the :class:`RNN` module.
135 | * An instance of :class:`RNN`.
136 | * A :attr:`callable`: A function that returns a PyTorch :class:`~torch.nn.Module`. This
137 |   module must have the same input and output shape signature as the
138 |   :class:`RNN` module.
139 | 
140 | This operation is implemented by the :class:`RNN` module:
141 | 
142 | .. autoclass:: RNN
143 |     :members:
144 | 
145 | Utility Modules
146 | -------------------
147 | 
148 | Apart from standard operations, DeepMatcher also contains several utility modules to help
149 | glue together various components. These are listed below.
150 | 
151 | :hidden:`Lambda`
152 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
153 | .. autoclass:: Lambda
154 |     :members:
155 | 
156 | :hidden:`MultiSequential`
157 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
158 | .. autoclass:: MultiSequential
159 |     :members:
160 | 
161 | :hidden:`NoMeta`
162 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
163 | .. autoclass:: NoMeta
164 |     :members:
165 | 
166 | :hidden:`ModuleMap`
167 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
168 | .. autoclass:: ModuleMap
169 |     :members:
170 | 
171 | :hidden:`LazyModule`
172 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
173 | .. autoclass:: LazyModule
174 |     :members: __init__, forward, expect_signature
175 | 
176 | :hidden:`LazyModuleFn`
177 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
178 | .. autoclass:: LazyModuleFn
179 |     :members:
180 | 


--------------------------------------------------------------------------------
/docs/html/_sources/notes/design.rst.txt:
--------------------------------------------------------------------------------
1 | Design Notes about DeepMatcher package
2 | =======================================
3 | 


--------------------------------------------------------------------------------
/docs/html/_sources/notes/faq.rst.txt:
--------------------------------------------------------------------------------
1 | FAQ
2 | ===
3 | 


--------------------------------------------------------------------------------
/docs/html/_sources/notes/model.rst.txt:
--------------------------------------------------------------------------------
1 | DL Architecture Template
2 | =========================
3 | 


--------------------------------------------------------------------------------
/docs/html/_sources/optim.rst.txt:
--------------------------------------------------------------------------------
 1 | .. role:: hidden
 2 |     :class: hidden-section
 3 | 
 4 | deepmatcher.optim
 5 | =========================
 6 | 
 7 | .. automodule:: deepmatcher.optim
 8 | .. currentmodule:: deepmatcher.optim
 9 | 
10 | :hidden:`SoftNLLLoss`
11 | ~~~~~~~~~~~~~~~~~~~~~~~
12 | .. autoclass:: SoftNLLLoss
13 |     :members:
14 | 
15 | :hidden:`Optimizer`
16 | ~~~~~~~~~~~~~~~~~~~~~~~
17 | .. autoclass:: Optimizer
18 |     :members:
19 | 


--------------------------------------------------------------------------------
/docs/html/_sources/word_aggregators.rst.txt:
--------------------------------------------------------------------------------
 1 | .. role:: hidden
 2 |     :class: hidden-section
 3 | 
 4 | deepmatcher.word_aggregators
 5 | ==============================
 6 | 
 7 | .. automodule:: deepmatcher.word_aggregators
 8 | .. currentmodule:: deepmatcher.word_aggregators
 9 | 
10 | :hidden:`Pool`
11 | ~~~~~~~~~~~~~~~~~~
12 | .. autoclass:: Pool
13 |     :members:
14 | 
15 | :hidden:`AttentionWithRNN`
16 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~
17 | .. autoclass:: AttentionWithRNN
18 |     :members:
19 | 


--------------------------------------------------------------------------------
/docs/html/_sources/word_comparators.rst.txt:
--------------------------------------------------------------------------------
1 | deepmatcher.word_comparators
2 | ==============================
3 | 
4 | .. automodule:: deepmatcher.word_comparators
5 | .. currentmodule:: deepmatcher.word_comparators
6 | 
7 | .. autoclass:: Attention
8 |     :members:
9 | 


--------------------------------------------------------------------------------
/docs/html/_sources/word_contextualizers.rst.txt:
--------------------------------------------------------------------------------
 1 | .. role:: hidden
 2 |     :class: hidden-section
 3 | 
 4 | deepmatcher.word_contextualizers
 5 | ==================================
 6 | 
 7 | .. automodule:: deepmatcher.word_contextualizers
 8 | .. currentmodule:: deepmatcher.word_contextualizers
 9 | 
10 | :hidden:`RNN`
11 | ~~~~~~~~~~~~~~~~~~
12 | .. autoclass:: RNN
13 |     :members:
14 | 
15 | :hidden:`SelfAttention`
16 | ~~~~~~~~~~~~~~~~~~~~~~~~~
17 | .. autoclass:: SelfAttention
18 |     :members:
19 | 


--------------------------------------------------------------------------------
/docs/html/_static/ae.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/ae.png


--------------------------------------------------------------------------------
/docs/html/_static/ajax-loader.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/ajax-loader.gif


--------------------------------------------------------------------------------
/docs/html/_static/arch.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/arch.png


--------------------------------------------------------------------------------
/docs/html/_static/as.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/as.png


--------------------------------------------------------------------------------
/docs/html/_static/asr.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/asr.png


--------------------------------------------------------------------------------
/docs/html/_static/comment-bright.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/comment-bright.png


--------------------------------------------------------------------------------
/docs/html/_static/comment-close.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/comment-close.png


--------------------------------------------------------------------------------
/docs/html/_static/comment.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/comment.png


--------------------------------------------------------------------------------
/docs/html/_static/css/badge_only.css:
--------------------------------------------------------------------------------
1 | .fa:before{-webkit-font-smoothing:antialiased}.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;content:""}.clearfix:after{clear:both}@font-face{font-family:FontAwesome;font-weight:normal;font-style:normal;src:url("../fonts/fontawesome-webfont.eot");src:url("../fonts/fontawesome-webfont.eot?#iefix") format("embedded-opentype"),url("../fonts/fontawesome-webfont.woff") format("woff"),url("../fonts/fontawesome-webfont.ttf") format("truetype"),url("../fonts/fontawesome-webfont.svg#FontAwesome") format("svg")}.fa:before{display:inline-block;font-family:FontAwesome;font-style:normal;font-weight:normal;line-height:1;text-decoration:inherit}a .fa{display:inline-block;text-decoration:inherit}li .fa{display:inline-block}li .fa-large:before,li .fa-large:before{width:1.875em}ul.fas{list-style-type:none;margin-left:2em;text-indent:-0.8em}ul.fas li .fa{width:.8em}ul.fas li .fa-large:before,ul.fas li .fa-large:before{vertical-align:baseline}.fa-book:before{content:""}.icon-book:before{content:""}.fa-caret-down:before{content:""}.icon-caret-down:before{content:""}.fa-caret-up:before{content:""}.icon-caret-up:before{content:""}.fa-caret-left:before{content:""}.icon-caret-left:before{content:""}.fa-caret-right:before{content:""}.icon-caret-right:before{content:""}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:"Lato","proxima-nova","Helvetica Neue",Arial,sans-serif;z-index:400}.rst-versions a{color:#2980B9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27AE60;*zoom:1}.rst-versions .rst-current-version:before,.rst-versions .rst-current-version:after{display:table;content:""}.rst-versions .rst-current-version:after{clear:both}.rst-versions .rst-current-version .fa{color:#fcfcfc}.rst-versions .rst-current-version .fa-book{float:left}.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#E74C3C;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#F1C40F;color:#000}.rst-versions.shift-up{height:auto;max-height:100%}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:gray;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:solid 1px #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px}.rst-versions.rst-badge .icon-book{float:none}.rst-versions.rst-badge .fa-book{float:none}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book{float:left}.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge .rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width: 768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}}
2 | 


--------------------------------------------------------------------------------
/docs/html/_static/doctools.js:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * doctools.js
  3 |  * ~~~~~~~~~~~
  4 |  *
  5 |  * Sphinx JavaScript utilities for all documentation.
  6 |  *
  7 |  * :copyright: Copyright 2007-2018 by the Sphinx team, see AUTHORS.
  8 |  * :license: BSD, see LICENSE for details.
  9 |  *
 10 |  */
 11 | 
 12 | /**
 13 |  * select a different prefix for underscore
 14 |  */
 15 | $u = _.noConflict();
 16 | 
 17 | /**
 18 |  * make the code below compatible with browsers without
 19 |  * an installed firebug like debugger
 20 | if (!window.console || !console.firebug) {
 21 |   var names = ["log", "debug", "info", "warn", "error", "assert", "dir",
 22 |     "dirxml", "group", "groupEnd", "time", "timeEnd", "count", "trace",
 23 |     "profile", "profileEnd"];
 24 |   window.console = {};
 25 |   for (var i = 0; i < names.length; ++i)
 26 |     window.console[names[i]] = function() {};
 27 | }
 28 |  */
 29 | 
 30 | /**
 31 |  * small helper function to urldecode strings
 32 |  */
 33 | jQuery.urldecode = function(x) {
 34 |   return decodeURIComponent(x).replace(/\+/g, ' ');
 35 | };
 36 | 
 37 | /**
 38 |  * small helper function to urlencode strings
 39 |  */
 40 | jQuery.urlencode = encodeURIComponent;
 41 | 
 42 | /**
 43 |  * This function returns the parsed url parameters of the
 44 |  * current request. Multiple values per key are supported,
 45 |  * it will always return arrays of strings for the value parts.
 46 |  */
 47 | jQuery.getQueryParameters = function(s) {
 48 |   if (typeof s === 'undefined')
 49 |     s = document.location.search;
 50 |   var parts = s.substr(s.indexOf('?') + 1).split('&');
 51 |   var result = {};
 52 |   for (var i = 0; i < parts.length; i++) {
 53 |     var tmp = parts[i].split('=', 2);
 54 |     var key = jQuery.urldecode(tmp[0]);
 55 |     var value = jQuery.urldecode(tmp[1]);
 56 |     if (key in result)
 57 |       result[key].push(value);
 58 |     else
 59 |       result[key] = [value];
 60 |   }
 61 |   return result;
 62 | };
 63 | 
 64 | /**
 65 |  * highlight a given string on a jquery object by wrapping it in
 66 |  * span elements with the given class name.
 67 |  */
 68 | jQuery.fn.highlightText = function(text, className) {
 69 |   function highlight(node, addItems) {
 70 |     if (node.nodeType === 3) {
 71 |       var val = node.nodeValue;
 72 |       var pos = val.toLowerCase().indexOf(text);
 73 |       if (pos >= 0 && !jQuery(node.parentNode).hasClass(className)) {
 74 |         var span;
 75 |         var isInSVG = jQuery(node).closest("body, svg, foreignObject").is("svg");
 76 |         if (isInSVG) {
 77 |           span = document.createElementNS("http://www.w3.org/2000/svg", "tspan");
 78 |         } else {
 79 |           span = document.createElement("span");
 80 |           span.className = className;
 81 |         }
 82 |         span.appendChild(document.createTextNode(val.substr(pos, text.length)));
 83 |         node.parentNode.insertBefore(span, node.parentNode.insertBefore(
 84 |           document.createTextNode(val.substr(pos + text.length)),
 85 |           node.nextSibling));
 86 |         node.nodeValue = val.substr(0, pos);
 87 |         if (isInSVG) {
 88 |           var bbox = span.getBBox();
 89 |           var rect = document.createElementNS("http://www.w3.org/2000/svg", "rect");
 90 |        	  rect.x.baseVal.value = bbox.x;
 91 |           rect.y.baseVal.value = bbox.y;
 92 |           rect.width.baseVal.value = bbox.width;
 93 |           rect.height.baseVal.value = bbox.height;
 94 |           rect.setAttribute('class', className);
 95 |           var parentOfText = node.parentNode.parentNode;
 96 |           addItems.push({
 97 |               "parent": node.parentNode,
 98 |               "target": rect});
 99 |         }
100 |       }
101 |     }
102 |     else if (!jQuery(node).is("button, select, textarea")) {
103 |       jQuery.each(node.childNodes, function() {
104 |         highlight(this, addItems);
105 |       });
106 |     }
107 |   }
108 |   var addItems = [];
109 |   var result = this.each(function() {
110 |     highlight(this, addItems);
111 |   });
112 |   for (var i = 0; i < addItems.length; ++i) {
113 |     jQuery(addItems[i].parent).before(addItems[i].target);
114 |   }
115 |   return result;
116 | };
117 | 
118 | /*
119 |  * backward compatibility for jQuery.browser
120 |  * This will be supported until firefox bug is fixed.
121 |  */
122 | if (!jQuery.browser) {
123 |   jQuery.uaMatch = function(ua) {
124 |     ua = ua.toLowerCase();
125 | 
126 |     var match = /(chrome)[ \/]([\w.]+)/.exec(ua) ||
127 |       /(webkit)[ \/]([\w.]+)/.exec(ua) ||
128 |       /(opera)(?:.*version|)[ \/]([\w.]+)/.exec(ua) ||
129 |       /(msie) ([\w.]+)/.exec(ua) ||
130 |       ua.indexOf("compatible") < 0 && /(mozilla)(?:.*? rv:([\w.]+)|)/.exec(ua) ||
131 |       [];
132 | 
133 |     return {
134 |       browser: match[ 1 ] || "",
135 |       version: match[ 2 ] || "0"
136 |     };
137 |   };
138 |   jQuery.browser = {};
139 |   jQuery.browser[jQuery.uaMatch(navigator.userAgent).browser] = true;
140 | }
141 | 
142 | /**
143 |  * Small JavaScript module for the documentation.
144 |  */
145 | var Documentation = {
146 | 
147 |   init : function() {
148 |     this.fixFirefoxAnchorBug();
149 |     this.highlightSearchWords();
150 |     this.initIndexTable();
151 |     
152 |   },
153 | 
154 |   /**
155 |    * i18n support
156 |    */
157 |   TRANSLATIONS : {},
158 |   PLURAL_EXPR : function(n) { return n === 1 ? 0 : 1; },
159 |   LOCALE : 'unknown',
160 | 
161 |   // gettext and ngettext don't access this so that the functions
162 |   // can safely bound to a different name (_ = Documentation.gettext)
163 |   gettext : function(string) {
164 |     var translated = Documentation.TRANSLATIONS[string];
165 |     if (typeof translated === 'undefined')
166 |       return string;
167 |     return (typeof translated === 'string') ? translated : translated[0];
168 |   },
169 | 
170 |   ngettext : function(singular, plural, n) {
171 |     var translated = Documentation.TRANSLATIONS[singular];
172 |     if (typeof translated === 'undefined')
173 |       return (n == 1) ? singular : plural;
174 |     return translated[Documentation.PLURALEXPR(n)];
175 |   },
176 | 
177 |   addTranslations : function(catalog) {
178 |     for (var key in catalog.messages)
179 |       this.TRANSLATIONS[key] = catalog.messages[key];
180 |     this.PLURAL_EXPR = new Function('n', 'return +(' + catalog.plural_expr + ')');
181 |     this.LOCALE = catalog.locale;
182 |   },
183 | 
184 |   /**
185 |    * add context elements like header anchor links
186 |    */
187 |   addContextElements : function() {
188 |     $('div[id] > :header:first').each(function() {
189 |       $('<a class="headerlink">\u00B6</a>').
190 |       attr('href', '#' + this.id).
191 |       attr('title', _('Permalink to this headline')).
192 |       appendTo(this);
193 |     });
194 |     $('dt[id]').each(function() {
195 |       $('<a class="headerlink">\u00B6</a>').
196 |       attr('href', '#' + this.id).
197 |       attr('title', _('Permalink to this definition')).
198 |       appendTo(this);
199 |     });
200 |   },
201 | 
202 |   /**
203 |    * workaround a firefox stupidity
204 |    * see: https://bugzilla.mozilla.org/show_bug.cgi?id=645075
205 |    */
206 |   fixFirefoxAnchorBug : function() {
207 |     if (document.location.hash && $.browser.mozilla)
208 |       window.setTimeout(function() {
209 |         document.location.href += '';
210 |       }, 10);
211 |   },
212 | 
213 |   /**
214 |    * highlight the search words provided in the url in the text
215 |    */
216 |   highlightSearchWords : function() {
217 |     var params = $.getQueryParameters();
218 |     var terms = (params.highlight) ? params.highlight[0].split(/\s+/) : [];
219 |     if (terms.length) {
220 |       var body = $('div.body');
221 |       if (!body.length) {
222 |         body = $('body');
223 |       }
224 |       window.setTimeout(function() {
225 |         $.each(terms, function() {
226 |           body.highlightText(this.toLowerCase(), 'highlighted');
227 |         });
228 |       }, 10);
229 |       $('<p class="highlight-link"><a href="javascript:Documentation.' +
230 |         'hideSearchWords()">' + _('Hide Search Matches') + '</a></p>')
231 |           .appendTo($('#searchbox'));
232 |     }
233 |   },
234 | 
235 |   /**
236 |    * init the domain index toggle buttons
237 |    */
238 |   initIndexTable : function() {
239 |     var togglers = $('img.toggler').click(function() {
240 |       var src = $(this).attr('src');
241 |       var idnum = $(this).attr('id').substr(7);
242 |       $('tr.cg-' + idnum).toggle();
243 |       if (src.substr(-9) === 'minus.png')
244 |         $(this).attr('src', src.substr(0, src.length-9) + 'plus.png');
245 |       else
246 |         $(this).attr('src', src.substr(0, src.length-8) + 'minus.png');
247 |     }).css('display', '');
248 |     if (DOCUMENTATION_OPTIONS.COLLAPSE_INDEX) {
249 |         togglers.click();
250 |     }
251 |   },
252 | 
253 |   /**
254 |    * helper function to hide the search marks again
255 |    */
256 |   hideSearchWords : function() {
257 |     $('#searchbox .highlight-link').fadeOut(300);
258 |     $('span.highlighted').removeClass('highlighted');
259 |   },
260 | 
261 |   /**
262 |    * make the url absolute
263 |    */
264 |   makeURL : function(relativeURL) {
265 |     return DOCUMENTATION_OPTIONS.URL_ROOT + '/' + relativeURL;
266 |   },
267 | 
268 |   /**
269 |    * get the current relative url
270 |    */
271 |   getCurrentURL : function() {
272 |     var path = document.location.pathname;
273 |     var parts = path.split(/\//);
274 |     $.each(DOCUMENTATION_OPTIONS.URL_ROOT.split(/\//), function() {
275 |       if (this === '..')
276 |         parts.pop();
277 |     });
278 |     var url = parts.join('/');
279 |     return path.substring(url.lastIndexOf('/') + 1, path.length - 1);
280 |   },
281 | 
282 |   initOnKeyListeners: function() {
283 |     $(document).keyup(function(event) {
284 |       var activeElementType = document.activeElement.tagName;
285 |       // don't navigate when in search box or textarea
286 |       if (activeElementType !== 'TEXTAREA' && activeElementType !== 'INPUT' && activeElementType !== 'SELECT') {
287 |         switch (event.keyCode) {
288 |           case 37: // left
289 |             var prevHref = $('link[rel="prev"]').prop('href');
290 |             if (prevHref) {
291 |               window.location.href = prevHref;
292 |               return false;
293 |             }
294 |           case 39: // right
295 |             var nextHref = $('link[rel="next"]').prop('href');
296 |             if (nextHref) {
297 |               window.location.href = nextHref;
298 |               return false;
299 |             }
300 |         }
301 |       }
302 |     });
303 |   }
304 | };
305 | 
306 | // quick alias for translations
307 | _ = Documentation.gettext;
308 | 
309 | $(document).ready(function() {
310 |   Documentation.init();
311 | });


--------------------------------------------------------------------------------
/docs/html/_static/down-pressed.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/down-pressed.png


--------------------------------------------------------------------------------
/docs/html/_static/down.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/down.png


--------------------------------------------------------------------------------
/docs/html/_static/file.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/file.png


--------------------------------------------------------------------------------
/docs/html/_static/fonts/Inconsolata-Bold.ttf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/fonts/Inconsolata-Bold.ttf


--------------------------------------------------------------------------------
/docs/html/_static/fonts/Inconsolata-Regular.ttf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/fonts/Inconsolata-Regular.ttf


--------------------------------------------------------------------------------
/docs/html/_static/fonts/Lato-Bold.ttf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/fonts/Lato-Bold.ttf


--------------------------------------------------------------------------------
/docs/html/_static/fonts/Lato-BoldItalic.ttf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/fonts/Lato-BoldItalic.ttf


--------------------------------------------------------------------------------
/docs/html/_static/fonts/Lato-Italic.ttf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/fonts/Lato-Italic.ttf


--------------------------------------------------------------------------------
/docs/html/_static/fonts/Lato-Regular.ttf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/fonts/Lato-Regular.ttf


--------------------------------------------------------------------------------
/docs/html/_static/fonts/RobotoSlab-Bold.ttf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/fonts/RobotoSlab-Bold.ttf


--------------------------------------------------------------------------------
/docs/html/_static/fonts/RobotoSlab-Regular.ttf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/fonts/RobotoSlab-Regular.ttf


--------------------------------------------------------------------------------
/docs/html/_static/fonts/fontawesome-webfont.eot:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/fonts/fontawesome-webfont.eot


--------------------------------------------------------------------------------
/docs/html/_static/fonts/fontawesome-webfont.ttf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/fonts/fontawesome-webfont.ttf


--------------------------------------------------------------------------------
/docs/html/_static/fonts/fontawesome-webfont.woff:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/fonts/fontawesome-webfont.woff


--------------------------------------------------------------------------------
/docs/html/_static/fonts/fontawesome-webfont.woff2:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/fonts/fontawesome-webfont.woff2


--------------------------------------------------------------------------------
/docs/html/_static/js/theme.js:
--------------------------------------------------------------------------------
1 | require=function r(s,a,l){function c(i,n){if(!a[i]){if(!s[i]){var e="function"==typeof require&&require;if(!n&&e)return e(i,!0);if(u)return u(i,!0);var t=new Error("Cannot find module '"+i+"'");throw t.code="MODULE_NOT_FOUND",t}var o=a[i]={exports:{}};s[i][0].call(o.exports,function(n){var e=s[i][1][n];return c(e||n)},o,o.exports,r,s,a,l)}return a[i].exports}for(var u="function"==typeof require&&require,n=0;n<l.length;n++)c(l[n]);return c}({"sphinx-rtd-theme":[function(n,e,i){var jQuery="undefined"!=typeof window?window.jQuery:n("jquery");e.exports.ThemeNav={navBar:null,win:null,winScroll:!1,winResize:!1,linkScroll:!1,winPosition:0,winHeight:null,docHeight:null,isRunning:!1,enable:function(e){var i=this;"undefined"==typeof withStickNav&&(e=!0),i.isRunning||(i.isRunning=!0,jQuery(function(n){i.init(n),i.reset(),i.win.on("hashchange",i.reset),e&&i.win.on("scroll",function(){i.linkScroll||i.winScroll||(i.winScroll=!0,requestAnimationFrame(function(){i.onScroll()}))}),i.win.on("resize",function(){i.winResize||(i.winResize=!0,requestAnimationFrame(function(){i.onResize()}))}),i.onResize()}))},enableSticky:function(){this.enable(!0)},init:function(i){i(document);var t=this;this.navBar=i("div.wy-side-scroll:first"),this.win=i(window),i(document).on("click","[data-toggle='wy-nav-top']",function(){i("[data-toggle='wy-nav-shift']").toggleClass("shift"),i("[data-toggle='rst-versions']").toggleClass("shift")}).on("click",".wy-menu-vertical .current ul li a",function(){var n=i(this);i("[data-toggle='wy-nav-shift']").removeClass("shift"),i("[data-toggle='rst-versions']").toggleClass("shift"),t.toggleCurrent(n),t.hashChange()}).on("click","[data-toggle='rst-current-version']",function(){i("[data-toggle='rst-versions']").toggleClass("shift-up")}),i("table.docutils:not(.field-list,.footnote,.citation)").wrap("<div class='wy-table-responsive'></div>"),i("table.docutils.footnote").wrap("<div class='wy-table-responsive footnote'></div>"),i("table.docutils.citation").wrap("<div class='wy-table-responsive citation'></div>"),i(".wy-menu-vertical ul").not(".simple").siblings("a").each(function(){var e=i(this);expand=i('<span class="toctree-expand"></span>'),expand.on("click",function(n){return t.toggleCurrent(e),n.stopPropagation(),!1}),e.prepend(expand)})},reset:function(){var n=encodeURI(window.location.hash)||"#";try{var e=$(".wy-menu-vertical"),i=e.find('[href="'+n+'"]');if(0===i.length){var t=$('.document [id="'+n.substring(1)+'"]').closest("div.section");0===(i=e.find('[href="#'+t.attr("id")+'"]')).length&&(i=e.find('[href="#"]'))}0<i.length&&($(".wy-menu-vertical .current").removeClass("current"),i.addClass("current"),i.closest("li.toctree-l1").addClass("current"),i.closest("li.toctree-l1").parent().addClass("current"),i.closest("li.toctree-l1").addClass("current"),i.closest("li.toctree-l2").addClass("current"),i.closest("li.toctree-l3").addClass("current"),i.closest("li.toctree-l4").addClass("current"))}catch(o){console.log("Error expanding nav for anchor",o)}},onScroll:function(){this.winScroll=!1;var n=this.win.scrollTop(),e=n+this.winHeight,i=this.navBar.scrollTop()+(n-this.winPosition);n<0||e>this.docHeight||(this.navBar.scrollTop(i),this.winPosition=n)},onResize:function(){this.winResize=!1,this.winHeight=this.win.height(),this.docHeight=$(document).height()},hashChange:function(){this.linkScroll=!0,this.win.one("hashchange",function(){this.linkScroll=!1})},toggleCurrent:function(n){var e=n.closest("li");e.siblings("li.current").removeClass("current"),e.siblings().find("li.current").removeClass("current"),e.find("> ul li.current").removeClass("current"),e.toggleClass("current")}},"undefined"!=typeof window&&(window.SphinxRtdTheme={Navigation:e.exports.ThemeNav,StickyNav:e.exports.ThemeNav}),function(){for(var r=0,n=["ms","moz","webkit","o"],e=0;e<n.length&&!window.requestAnimationFrame;++e)window.requestAnimationFrame=window[n[e]+"RequestAnimationFrame"],window.cancelAnimationFrame=window[n[e]+"CancelAnimationFrame"]||window[n[e]+"CancelRequestAnimationFrame"];window.requestAnimationFrame||(window.requestAnimationFrame=function(n,e){var i=(new Date).getTime(),t=Math.max(0,16-(i-r)),o=window.setTimeout(function(){n(i+t)},t);return r=i+t,o}),window.cancelAnimationFrame||(window.cancelAnimationFrame=function(n){clearTimeout(n)})}()},{jquery:"jquery"}]},{},["sphinx-rtd-theme"]);


--------------------------------------------------------------------------------
/docs/html/_static/match_input_ex.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/match_input_ex.png


--------------------------------------------------------------------------------
/docs/html/_static/minus.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/minus.png


--------------------------------------------------------------------------------
/docs/html/_static/plus.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/plus.png


--------------------------------------------------------------------------------
/docs/html/_static/pygments.css:
--------------------------------------------------------------------------------
 1 | .highlight .hll { background-color: #ffffcc }
 2 | .highlight  { background: #eeffcc; }
 3 | .highlight .c { color: #408090; font-style: italic } /* Comment */
 4 | .highlight .err { border: 1px solid #FF0000 } /* Error */
 5 | .highlight .k { color: #007020; font-weight: bold } /* Keyword */
 6 | .highlight .o { color: #666666 } /* Operator */
 7 | .highlight .ch { color: #408090; font-style: italic } /* Comment.Hashbang */
 8 | .highlight .cm { color: #408090; font-style: italic } /* Comment.Multiline */
 9 | .highlight .cp { color: #007020 } /* Comment.Preproc */
10 | .highlight .cpf { color: #408090; font-style: italic } /* Comment.PreprocFile */
11 | .highlight .c1 { color: #408090; font-style: italic } /* Comment.Single */
12 | .highlight .cs { color: #408090; background-color: #fff0f0 } /* Comment.Special */
13 | .highlight .gd { color: #A00000 } /* Generic.Deleted */
14 | .highlight .ge { font-style: italic } /* Generic.Emph */
15 | .highlight .gr { color: #FF0000 } /* Generic.Error */
16 | .highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */
17 | .highlight .gi { color: #00A000 } /* Generic.Inserted */
18 | .highlight .go { color: #333333 } /* Generic.Output */
19 | .highlight .gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */
20 | .highlight .gs { font-weight: bold } /* Generic.Strong */
21 | .highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */
22 | .highlight .gt { color: #0044DD } /* Generic.Traceback */
23 | .highlight .kc { color: #007020; font-weight: bold } /* Keyword.Constant */
24 | .highlight .kd { color: #007020; font-weight: bold } /* Keyword.Declaration */
25 | .highlight .kn { color: #007020; font-weight: bold } /* Keyword.Namespace */
26 | .highlight .kp { color: #007020 } /* Keyword.Pseudo */
27 | .highlight .kr { color: #007020; font-weight: bold } /* Keyword.Reserved */
28 | .highlight .kt { color: #902000 } /* Keyword.Type */
29 | .highlight .m { color: #208050 } /* Literal.Number */
30 | .highlight .s { color: #4070a0 } /* Literal.String */
31 | .highlight .na { color: #4070a0 } /* Name.Attribute */
32 | .highlight .nb { color: #007020 } /* Name.Builtin */
33 | .highlight .nc { color: #0e84b5; font-weight: bold } /* Name.Class */
34 | .highlight .no { color: #60add5 } /* Name.Constant */
35 | .highlight .nd { color: #555555; font-weight: bold } /* Name.Decorator */
36 | .highlight .ni { color: #d55537; font-weight: bold } /* Name.Entity */
37 | .highlight .ne { color: #007020 } /* Name.Exception */
38 | .highlight .nf { color: #06287e } /* Name.Function */
39 | .highlight .nl { color: #002070; font-weight: bold } /* Name.Label */
40 | .highlight .nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */
41 | .highlight .nt { color: #062873; font-weight: bold } /* Name.Tag */
42 | .highlight .nv { color: #bb60d5 } /* Name.Variable */
43 | .highlight .ow { color: #007020; font-weight: bold } /* Operator.Word */
44 | .highlight .w { color: #bbbbbb } /* Text.Whitespace */
45 | .highlight .mb { color: #208050 } /* Literal.Number.Bin */
46 | .highlight .mf { color: #208050 } /* Literal.Number.Float */
47 | .highlight .mh { color: #208050 } /* Literal.Number.Hex */
48 | .highlight .mi { color: #208050 } /* Literal.Number.Integer */
49 | .highlight .mo { color: #208050 } /* Literal.Number.Oct */
50 | .highlight .sa { color: #4070a0 } /* Literal.String.Affix */
51 | .highlight .sb { color: #4070a0 } /* Literal.String.Backtick */
52 | .highlight .sc { color: #4070a0 } /* Literal.String.Char */
53 | .highlight .dl { color: #4070a0 } /* Literal.String.Delimiter */
54 | .highlight .sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */
55 | .highlight .s2 { color: #4070a0 } /* Literal.String.Double */
56 | .highlight .se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */
57 | .highlight .sh { color: #4070a0 } /* Literal.String.Heredoc */
58 | .highlight .si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */
59 | .highlight .sx { color: #c65d09 } /* Literal.String.Other */
60 | .highlight .sr { color: #235388 } /* Literal.String.Regex */
61 | .highlight .s1 { color: #4070a0 } /* Literal.String.Single */
62 | .highlight .ss { color: #517918 } /* Literal.String.Symbol */
63 | .highlight .bp { color: #007020 } /* Name.Builtin.Pseudo */
64 | .highlight .fm { color: #06287e } /* Name.Function.Magic */
65 | .highlight .vc { color: #bb60d5 } /* Name.Variable.Class */
66 | .highlight .vg { color: #bb60d5 } /* Name.Variable.Global */
67 | .highlight .vi { color: #bb60d5 } /* Name.Variable.Instance */
68 | .highlight .vm { color: #bb60d5 } /* Name.Variable.Magic */
69 | .highlight .il { color: #208050 } /* Literal.Number.Integer.Long */


--------------------------------------------------------------------------------
/docs/html/_static/up-pressed.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/up-pressed.png


--------------------------------------------------------------------------------
/docs/html/_static/up.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/_static/up.png


--------------------------------------------------------------------------------
/docs/html/notes/faq.html:
--------------------------------------------------------------------------------
  1 | 
  2 | 
  3 | <!DOCTYPE html>
  4 | <!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
  5 | <!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
  6 | <head>
  7 |   <meta charset="utf-8">
  8 |   
  9 |   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 10 |   
 11 |   <title>FAQ &mdash; DeepMatcher 0.1.0rc1 documentation</title>
 12 |   
 13 | 
 14 |   
 15 |   
 16 |   
 17 |   
 18 | 
 19 |   
 20 | 
 21 |   
 22 |   
 23 |     
 24 | 
 25 |   
 26 | 
 27 |   
 28 |     <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
 29 |   <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
 30 |     <link rel="index" title="Index" href="../genindex.html" />
 31 |     <link rel="search" title="Search" href="../search.html" /> 
 32 | 
 33 |   
 34 |   <script src="../_static/js/modernizr.min.js"></script>
 35 | 
 36 | </head>
 37 | 
 38 | <body class="wy-body-for-nav">
 39 | 
 40 |    
 41 |   <div class="wy-grid-for-nav">
 42 | 
 43 |     
 44 |     <nav data-toggle="wy-nav-shift" class="wy-nav-side">
 45 |       <div class="wy-side-scroll">
 46 |         <div class="wy-side-nav-search">
 47 |           
 48 | 
 49 |           
 50 |             <a href="../index.html" class="icon icon-home"> DeepMatcher
 51 |           
 52 | 
 53 |           
 54 |           </a>
 55 | 
 56 |           
 57 |             
 58 |             
 59 |               <div class="version">
 60 |                 0.1
 61 |               </div>
 62 |             
 63 |           
 64 | 
 65 |           
 66 | <div role="search">
 67 |   <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
 68 |     <input type="text" name="q" placeholder="Search docs" />
 69 |     <input type="hidden" name="check_keywords" value="yes" />
 70 |     <input type="hidden" name="area" value="default" />
 71 |   </form>
 72 | </div>
 73 | 
 74 |           
 75 |         </div>
 76 | 
 77 |         <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
 78 |           
 79 |             
 80 |             
 81 |               
 82 |             
 83 |             
 84 |               <p class="caption"><span class="caption-text">Package Reference</span></p>
 85 | <ul>
 86 | <li class="toctree-l1"><a class="reference internal" href="../deepmatcher.html">deepmatcher</a><ul>
 87 | <li class="toctree-l2"><a class="reference internal" href="../deepmatcher.html#main-modules">Main Modules</a><ul>
 88 | <li class="toctree-l3"><a class="reference internal" href="../deepmatcher.html#matchingmodel"><span class="hidden-section">MatchingModel</span></a></li>
 89 | <li class="toctree-l3"><a class="reference internal" href="../deepmatcher.html#attrsummarizer"><span class="hidden-section">AttrSummarizer</span></a></li>
 90 | <li class="toctree-l3"><a class="reference internal" href="../deepmatcher.html#classifier"><span class="hidden-section">Classifier</span></a></li>
 91 | </ul>
 92 | </li>
 93 | <li class="toctree-l2"><a class="reference internal" href="../deepmatcher.html#components-of-attribute-summarizer">Components of Attribute Summarizer</a><ul>
 94 | <li class="toctree-l3"><a class="reference internal" href="../deepmatcher.html#wordcontextualizer"><span class="hidden-section">WordContextualizer</span></a></li>
 95 | <li class="toctree-l3"><a class="reference internal" href="../deepmatcher.html#wordcomparator"><span class="hidden-section">WordComparator</span></a></li>
 96 | <li class="toctree-l3"><a class="reference internal" href="../deepmatcher.html#wordaggregator"><span class="hidden-section">WordAggregator</span></a></li>
 97 | </ul>
 98 | </li>
 99 | </ul>
100 | </li>
101 | <li class="toctree-l1"><a class="reference internal" href="../attr_summarizers.html">deepmatcher.attr_summarizers</a><ul>
102 | <li class="toctree-l2"><a class="reference internal" href="../attr_summarizers.html#sif"><span class="hidden-section">SIF</span></a></li>
103 | <li class="toctree-l2"><a class="reference internal" href="../attr_summarizers.html#rnn"><span class="hidden-section">RNN</span></a></li>
104 | <li class="toctree-l2"><a class="reference internal" href="../attr_summarizers.html#attention"><span class="hidden-section">Attention</span></a></li>
105 | <li class="toctree-l2"><a class="reference internal" href="../attr_summarizers.html#hybrid"><span class="hidden-section">Hybrid</span></a></li>
106 | </ul>
107 | </li>
108 | <li class="toctree-l1"><a class="reference internal" href="../word_contextualizers.html">deepmatcher.word_contextualizers</a><ul>
109 | <li class="toctree-l2"><a class="reference internal" href="../word_contextualizers.html#rnn"><span class="hidden-section">RNN</span></a></li>
110 | <li class="toctree-l2"><a class="reference internal" href="../word_contextualizers.html#selfattention"><span class="hidden-section">SelfAttention</span></a></li>
111 | </ul>
112 | </li>
113 | <li class="toctree-l1"><a class="reference internal" href="../word_comparators.html">deepmatcher.word_comparators</a></li>
114 | <li class="toctree-l1"><a class="reference internal" href="../word_aggregators.html">deepmatcher.word_aggregators</a><ul>
115 | <li class="toctree-l2"><a class="reference internal" href="../word_aggregators.html#pool"><span class="hidden-section">Pool</span></a></li>
116 | <li class="toctree-l2"><a class="reference internal" href="../word_aggregators.html#attentionwithrnn"><span class="hidden-section">AttentionWithRNN</span></a></li>
117 | </ul>
118 | </li>
119 | <li class="toctree-l1"><a class="reference internal" href="../modules.html">deepmatcher.modules</a><ul>
120 | <li class="toctree-l2"><a class="reference internal" href="../modules.html#standard-operations">Standard Operations</a><ul>
121 | <li class="toctree-l3"><a class="reference internal" href="../modules.html#transform">Transform</a></li>
122 | <li class="toctree-l3"><a class="reference internal" href="../modules.html#pool">Pool</a></li>
123 | <li class="toctree-l3"><a class="reference internal" href="../modules.html#merge">Merge</a></li>
124 | <li class="toctree-l3"><a class="reference internal" href="../modules.html#align">Align</a></li>
125 | <li class="toctree-l3"><a class="reference internal" href="../modules.html#bypass">Bypass</a></li>
126 | <li class="toctree-l3"><a class="reference internal" href="../modules.html#rnn">RNN</a></li>
127 | </ul>
128 | </li>
129 | <li class="toctree-l2"><a class="reference internal" href="../modules.html#utility-modules">Utility Modules</a><ul>
130 | <li class="toctree-l3"><a class="reference internal" href="../modules.html#lambda"><span class="hidden-section">Lambda</span></a></li>
131 | <li class="toctree-l3"><a class="reference internal" href="../modules.html#multisequential"><span class="hidden-section">MultiSequential</span></a></li>
132 | <li class="toctree-l3"><a class="reference internal" href="../modules.html#nometa"><span class="hidden-section">NoMeta</span></a></li>
133 | <li class="toctree-l3"><a class="reference internal" href="../modules.html#modulemap"><span class="hidden-section">ModuleMap</span></a></li>
134 | <li class="toctree-l3"><a class="reference internal" href="../modules.html#lazymodule"><span class="hidden-section">LazyModule</span></a></li>
135 | <li class="toctree-l3"><a class="reference internal" href="../modules.html#lazymodulefn"><span class="hidden-section">LazyModuleFn</span></a></li>
136 | </ul>
137 | </li>
138 | </ul>
139 | </li>
140 | <li class="toctree-l1"><a class="reference internal" href="../optim.html">deepmatcher.optim</a><ul>
141 | <li class="toctree-l2"><a class="reference internal" href="../optim.html#softnllloss"><span class="hidden-section">SoftNLLLoss</span></a></li>
142 | <li class="toctree-l2"><a class="reference internal" href="../optim.html#optimizer"><span class="hidden-section">Optimizer</span></a></li>
143 | </ul>
144 | </li>
145 | <li class="toctree-l1"><a class="reference internal" href="../data.html">deepmatcher.data</a><ul>
146 | <li class="toctree-l2"><a class="reference internal" href="../data.html#process"><span class="hidden-section">process</span></a></li>
147 | <li class="toctree-l2"><a class="reference internal" href="../data.html#process-unlabeled"><span class="hidden-section">process_unlabeled</span></a></li>
148 | </ul>
149 | </li>
150 | <li class="toctree-l1"><a class="reference internal" href="../batch.html">deepmatcher.batch</a><ul>
151 | <li class="toctree-l2"><a class="reference internal" href="../batch.html#attrtensor"><span class="hidden-section">AttrTensor</span></a></li>
152 | <li class="toctree-l2"><a class="reference internal" href="../batch.html#matchingbatch"><span class="hidden-section">MatchingBatch</span></a></li>
153 | </ul>
154 | </li>
155 | </ul>
156 | 
157 |             
158 |           
159 |         </div>
160 |       </div>
161 |     </nav>
162 | 
163 |     <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
164 | 
165 |       
166 |       <nav class="wy-nav-top" aria-label="top navigation">
167 |         
168 |           <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
169 |           <a href="../index.html">DeepMatcher</a>
170 |         
171 |       </nav>
172 | 
173 | 
174 |       <div class="wy-nav-content">
175 |         
176 |         <div class="rst-content">
177 |         
178 |           
179 | 
180 | 
181 | 
182 | 
183 | 
184 | 
185 | 
186 | 
187 | 
188 | 
189 | 
190 | 
191 | 
192 | 
193 | 
194 | <div role="navigation" aria-label="breadcrumbs navigation">
195 | 
196 |   <ul class="wy-breadcrumbs">
197 |     
198 |       <li><a href="../index.html">Docs</a> &raquo;</li>
199 |         
200 |       <li>FAQ</li>
201 |     
202 |     
203 |       <li class="wy-breadcrumbs-aside">
204 |         
205 |             
206 |             <a href="../_sources/notes/faq.rst.txt" rel="nofollow"> View page source</a>
207 |           
208 |         
209 |       </li>
210 |     
211 |   </ul>
212 | 
213 |   
214 |   <hr/>
215 | </div>
216 |           <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
217 |            <div itemprop="articleBody">
218 |             
219 |   <div class="section" id="faq">
220 | <h1>FAQ<a class="headerlink" href="#faq" title="Permalink to this headline">¶</a></h1>
221 | </div>
222 | 
223 | 
224 |            </div>
225 |            
226 |           </div>
227 |           <footer>
228 |   
229 | 
230 |   <hr/>
231 | 
232 |   <div role="contentinfo">
233 |     <p>
234 |         &copy; Copyright 2018, Sidharth Mudgal, Han Li.
235 | 
236 |     </p>
237 |   </div>
238 |   Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
239 | 
240 | </footer>
241 | 
242 |         </div>
243 |       </div>
244 | 
245 |     </section>
246 | 
247 |   </div>
248 |   
249 | 
250 | 
251 |   
252 | 
253 |     <script type="text/javascript">
254 |         var DOCUMENTATION_OPTIONS = {
255 |             URL_ROOT:'../',
256 |             VERSION:'0.1.0rc1',
257 |             LANGUAGE:'None',
258 |             COLLAPSE_INDEX:false,
259 |             FILE_SUFFIX:'.html',
260 |             HAS_SOURCE:  true,
261 |             SOURCELINK_SUFFIX: '.txt'
262 |         };
263 |     </script>
264 |       <script type="text/javascript" src="../_static/jquery.js"></script>
265 |       <script type="text/javascript" src="../_static/underscore.js"></script>
266 |       <script type="text/javascript" src="../_static/doctools.js"></script>
267 |       <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
268 | 
269 |   
270 | 
271 |   
272 |   
273 |     <script type="text/javascript" src="../_static/js/theme.js"></script>
274 |   
275 | 
276 |   <script type="text/javascript">
277 |       jQuery(function () {
278 |           SphinxRtdTheme.Navigation.enable(true);
279 |       });
280 |   </script> 
281 | 
282 | </body>
283 | </html>


--------------------------------------------------------------------------------
/docs/html/objects.inv:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/html/objects.inv


--------------------------------------------------------------------------------
/docs/make.bat:
--------------------------------------------------------------------------------
 1 | @ECHO OFF
 2 | 
 3 | pushd %~dp0
 4 | 
 5 | REM Command file for Sphinx documentation
 6 | 
 7 | if "%SPHINXBUILD%" == "" (
 8 | 	set SPHINXBUILD=sphinx-build
 9 | )
10 | set SOURCEDIR=source
11 | set BUILDDIR=build
12 | set SPHINXPROJ=DeepMatcher
13 | 
14 | if "%1" == "" goto help
15 | 
16 | %SPHINXBUILD% >NUL 2>NUL
17 | if errorlevel 9009 (
18 | 	echo.
19 | 	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
20 | 	echo.installed, then set the SPHINXBUILD environment variable to point
21 | 	echo.to the full path of the 'sphinx-build' executable. Alternatively you
22 | 	echo.may add the Sphinx directory to PATH.
23 | 	echo.
24 | 	echo.If you don't have Sphinx installed, grab it from
25 | 	echo.http://sphinx-doc.org/
26 | 	exit /b 1
27 | )
28 | 
29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
30 | goto end
31 | 
32 | :help
33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
34 | 
35 | :end
36 | popd
37 | 


--------------------------------------------------------------------------------
/docs/source/_static/ae.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/source/_static/ae.png


--------------------------------------------------------------------------------
/docs/source/_static/arch.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/source/_static/arch.png


--------------------------------------------------------------------------------
/docs/source/_static/as.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/source/_static/as.png


--------------------------------------------------------------------------------
/docs/source/_static/asr.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/source/_static/asr.png


--------------------------------------------------------------------------------
/docs/source/_static/match_input_ex.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/docs/source/_static/match_input_ex.png


--------------------------------------------------------------------------------
/docs/source/attr_summarizers.rst:
--------------------------------------------------------------------------------
 1 | .. role:: hidden
 2 |     :class: hidden-section
 3 | 
 4 | deepmatcher.attr_summarizers
 5 | ============================
 6 | 
 7 | .. automodule:: deepmatcher.attr_summarizers
 8 | .. currentmodule:: deepmatcher.attr_summarizers
 9 | 
10 | :hidden:`SIF`
11 | ~~~~~~~~~~~~~~~~~~
12 | .. autoclass:: SIF
13 |     :members:
14 | 
15 | :hidden:`RNN`
16 | ~~~~~~~~~~~~~~~~~~
17 | .. autoclass:: RNN
18 |     :members:
19 | 
20 | :hidden:`Attention`
21 | ~~~~~~~~~~~~~~~~~~
22 | .. autoclass:: Attention
23 |     :members:
24 | 
25 | :hidden:`Hybrid`
26 | ~~~~~~~~~~~~~~~~~~
27 | .. autoclass:: Hybrid
28 |     :members:
29 | 


--------------------------------------------------------------------------------
/docs/source/batch.rst:
--------------------------------------------------------------------------------
 1 | .. role:: hidden
 2 |     :class: hidden-section
 3 | 
 4 | deepmatcher.batch
 5 | =========================
 6 | 
 7 | .. automodule:: deepmatcher.batch
 8 | .. currentmodule:: deepmatcher.batch
 9 | 
10 | :hidden:`AttrTensor`
11 | ~~~~~~~~~~~~~~~~~~~~~
12 | .. autoclass:: AttrTensor
13 |     :members:
14 | 
15 | :hidden:`MatchingBatch`
16 | ~~~~~~~~~~~~~~~~~~~~~~~~
17 | .. autoclass:: MatchingBatch
18 |     :members:
19 | 


--------------------------------------------------------------------------------
/docs/source/conf.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python3
  2 | # -*- coding: utf-8 -*-
  3 | #
  4 | # DeepMatcher documentation build configuration file, created by
  5 | # sphinx-quickstart on Sat Mar 31 15:32:50 2018.
  6 | #
  7 | # This file is execfile()d with the current directory set to its
  8 | # containing dir.
  9 | #
 10 | # Note that not all possible configuration values are present in this
 11 | # autogenerated file.
 12 | #
 13 | # All configuration values have a default; values that are commented out
 14 | # serve to show the default.
 15 | 
 16 | # If extensions (or modules to document with autodoc) are in another directory,
 17 | # add these directories to sys.path here. If the directory is relative to the
 18 | # documentation root, use os.path.abspath to make it absolute, like shown here.
 19 | #
 20 | 
 21 | import deepmatcher
 22 | import sphinx_rtd_theme
 23 | 
 24 | # -- General configuration ------------------------------------------------
 25 | 
 26 | # If your documentation needs a minimal Sphinx version, state it here.
 27 | #
 28 | # needs_sphinx = '1.0'
 29 | 
 30 | # Add any Sphinx extension module names here, as strings. They can be
 31 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 32 | # ones.
 33 | extensions = [
 34 |     'sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.doctest',
 35 |     'sphinx.ext.napoleon', 'sphinx.ext.intersphinx', 'sphinx.ext.todo',
 36 |     'sphinx.ext.coverage', 'sphinx.ext.mathjax', 'sphinx.ext.viewcode'
 37 | ]
 38 | napoleon_use_ivar = True
 39 | 
 40 | # Add any paths that contain templates here, relative to this directory.
 41 | templates_path = ['_templates']
 42 | 
 43 | # The suffix(es) of source filenames.
 44 | # You can specify multiple suffix as a list of string:
 45 | #
 46 | # source_suffix = ['.rst', '.md']
 47 | source_suffix = '.rst'
 48 | 
 49 | # The master toctree document.
 50 | master_doc = 'index'
 51 | 
 52 | # General information about the project.
 53 | project = 'DeepMatcher'
 54 | copyright = '2018, Sidharth Mudgal, Han Li'
 55 | author = 'Sidharth Mudgal, Han Li'
 56 | 
 57 | # The version info for the project you're documenting, acts as replacement for
 58 | # |version| and |release|, also used in various other places throughout the
 59 | # built documents.
 60 | #
 61 | # The short X.Y version.
 62 | version = '0.1'
 63 | # The full version, including alpha/beta/rc tags.
 64 | release = '0.1.0rc1'
 65 | 
 66 | # The language for content autogenerated by Sphinx. Refer to documentation
 67 | # for a list of supported languages.
 68 | #
 69 | # This is also used if you do content translation via gettext catalogs.
 70 | # Usually you set "language" from the command line for these cases.
 71 | language = None
 72 | 
 73 | # List of patterns, relative to source directory, that match files and
 74 | # directories to ignore when looking for source files.
 75 | # This patterns also effect to html_static_path and html_extra_path
 76 | exclude_patterns = []
 77 | 
 78 | # The name of the Pygments (syntax highlighting) style to use.
 79 | pygments_style = 'sphinx'
 80 | 
 81 | # If true, `todo` and `todoList` produce output, else they produce nothing.
 82 | todo_include_todos = True
 83 | 
 84 | # -- Options for HTML output ----------------------------------------------
 85 | 
 86 | # The theme to use for HTML and HTML Help pages.  See the documentation for
 87 | # a list of builtin themes.
 88 | #
 89 | html_theme = 'sphinx_rtd_theme'
 90 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 91 | 
 92 | # Theme options are theme-specific and customize the look and feel of a theme
 93 | # further.  For a list of options available for each theme, see the
 94 | # documentation.
 95 | #
 96 | html_theme_options = {
 97 |     'collapse_navigation': False,
 98 |     'display_version': True,
 99 |     'logo_only': True,
100 |     'navigation_depth': 4,
101 | }
102 | 
103 | # Add any paths that contain custom static files (such as style sheets) here,
104 | # relative to this directory. They are copied after the builtin static files,
105 | # so a file named "default.css" will overwrite the builtin "default.css".
106 | html_static_path = ['_static']
107 | 
108 | # Custom sidebar templates, must be a dictionary that maps document names
109 | # to template names.
110 | #
111 | # This is required for the alabaster theme
112 | # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
113 | html_sidebars = {
114 |     '**': [
115 |         'relations.html',  # needs 'show_related': True theme option to display
116 |         'searchbox.html',
117 |     ]
118 | }
119 | 
120 | # -- Options for HTMLHelp output ------------------------------------------
121 | 
122 | # Output file base name for HTML help builder.
123 | htmlhelp_basename = 'DeepMatcherdoc'
124 | 
125 | # -- Options for LaTeX output ---------------------------------------------
126 | 
127 | latex_elements = {
128 |     # The paper size ('letterpaper' or 'a4paper').
129 |     #
130 |     # 'papersize': 'letterpaper',
131 | 
132 |     # The font size ('10pt', '11pt' or '12pt').
133 |     #
134 |     # 'pointsize': '10pt',
135 | 
136 |     # Additional stuff for the LaTeX preamble.
137 |     #
138 |     # 'preamble': '',
139 | 
140 |     # Latex figure (float) alignment
141 |     #
142 |     # 'figure_align': 'htbp',
143 | }
144 | 
145 | # Grouping the document tree into LaTeX files. List of tuples
146 | # (source start file, target name, title,
147 | #  author, documentclass [howto, manual, or own class]).
148 | latex_documents = [
149 |     (master_doc, 'DeepMatcher.tex', 'DeepMatcher Documentation',
150 |      'Sidharth Mudgal, Han Li', 'manual'),
151 | ]
152 | 
153 | # -- Options for manual page output ---------------------------------------
154 | 
155 | # One entry per manual page. List of tuples
156 | # (source start file, name, description, authors, manual section).
157 | man_pages = [(master_doc, 'deepmatcher', 'DeepMatcher Documentation', [author], 1)]
158 | 
159 | # -- Options for Texinfo output -------------------------------------------
160 | 
161 | # Grouping the document tree into Texinfo files. List of tuples
162 | # (source start file, target name, title, author,
163 | #  dir menu entry, description, category)
164 | texinfo_documents = [
165 |     (master_doc, 'DeepMatcher', 'DeepMatcher Documentation', author, 'DeepMatcher',
166 |      'One line description of project.', 'Miscellaneous'),
167 | ]
168 | 
169 | # Example configuration for intersphinx: refer to the Python standard library.
170 | intersphinx_mapping = {
171 |     'python': ('https://docs.python.org/', None),
172 |     'torch': ('http://pytorch.org/docs/0.3.1/', None),
173 |     'torchtext': ('http://torchtext.readthedocs.io/en/latest/', None),
174 |     'pandas': ('https://pandas.pydata.org/pandas-docs/stable/', None)
175 | }
176 | 


--------------------------------------------------------------------------------
/docs/source/data.rst:
--------------------------------------------------------------------------------
 1 | .. role:: hidden
 2 |     :meth: hidden-section
 3 | 
 4 | deepmatcher.data
 5 | =========================
 6 | 
 7 | .. automodule:: deepmatcher.data
 8 | .. currentmodule:: deepmatcher.data
 9 | 
10 | :hidden:`process`
11 | ~~~~~~~~~~~~~~~~~~
12 | .. autofunction:: deepmatcher.data.process
13 | 
14 | :hidden:`process_unlabeled`
15 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
16 | .. autofunction:: deepmatcher.data.process_unlabeled
17 | 


--------------------------------------------------------------------------------
/docs/source/deepmatcher.rst:
--------------------------------------------------------------------------------
 1 | .. role:: hidden
 2 |     :class: hidden-section
 3 | 
 4 | deepmatcher
 5 | ==================================
 6 | 
 7 | .. automodule:: deepmatcher
 8 | 
 9 | Main Modules
10 | ----------------------------------
11 | 
12 | :hidden:`MatchingModel`
13 | ~~~~~~~~~~~~~~~~~~
14 | .. autoclass:: MatchingModel
15 |     :members:
16 | 
17 | :hidden:`AttrSummarizer`
18 | ~~~~~~~~~~~~~~~~~~
19 | .. autoclass:: deepmatcher.AttrSummarizer
20 |     :members:
21 | 
22 | :hidden:`Classifier`
23 | ~~~~~~~~~~~~~~~~~~
24 | .. autoclass:: deepmatcher.Classifier
25 |     :members:
26 | 
27 | 
28 | Components of Attribute Summarizer
29 | ----------------------------------
30 | 
31 | :hidden:`WordContextualizer`
32 | ~~~~~~~~~~~~~~~~~~
33 | .. autoclass:: deepmatcher.WordContextualizer
34 |     :members:
35 | 
36 | :hidden:`WordComparator`
37 | ~~~~~~~~~~~~~~~~~~
38 | .. autoclass:: deepmatcher.WordComparator
39 |     :members:
40 | 
41 | :hidden:`WordAggregator`
42 | ~~~~~~~~~~~~~~~~~~
43 | .. autoclass:: deepmatcher.WordAggregator
44 |     :members:
45 | 


--------------------------------------------------------------------------------
/docs/source/index.rst:
--------------------------------------------------------------------------------
 1 | .. DeepMatcher documentation master file, created by
 2 |    sphinx-quickstart on Sat Mar 31 15:32:50 2018.
 3 |    You can adapt this file completely to your liking, but it should at least
 4 |    contain the root `toctree` directive.
 5 | 
 6 | :github_url: https://github.com/sidharthms/deepmatcher
 7 | 
 8 | DeepMatcher documentation
 9 | =======================================
10 | 
11 | DeepMatcher is a deep learning library for entity matching.
12 | 
13 | Tutorials
14 | ---------
15 | 
16 | Tutorials are located `here <https://github.com/sidharthms/deepmatcher#tutorials>`_
17 | 
18 | .. toctree::
19 |    :maxdepth: 1
20 |    :caption: Package Reference
21 | 
22 |    deepmatcher
23 |    deepmatcher.attr_summarizers <attr_summarizers>
24 |    deepmatcher.word_contextualizers <word_contextualizers>
25 |    deepmatcher.word_comparators <word_comparators>
26 |    deepmatcher.word_aggregators <word_aggregators>
27 |    deepmatcher.modules <modules>
28 |    deepmatcher.optim <optim>
29 |    deepmatcher.data <data>
30 |    deepmatcher.batch <batch>
31 | 
32 | .. automodule:: deepmatcher
33 |    :members:
34 | 
35 | Indices and tables
36 | ==================
37 | 
38 | * :ref:`genindex`
39 | * :ref:`modindex`
40 | * :ref:`search`
41 | 


--------------------------------------------------------------------------------
/docs/source/modules.rst:
--------------------------------------------------------------------------------
  1 | .. role:: hidden
  2 |     :class: hidden-section
  3 | 
  4 | deepmatcher.modules
  5 | =========================
  6 | 
  7 | .. automodule:: deepmatcher.modules
  8 | .. currentmodule:: deepmatcher.modules
  9 | 
 10 | Standard Operations
 11 | -------------------------------
 12 | 
 13 | Many components in DeepMatcher, e.g. AttentionWithRNN word aggregator, allow users to
 14 | customize the behavior of operations performed by the component, such as alignment, vector
 15 | transformation, pooling, etc., by setting a parameter that specifies the operation. Here
 16 | we describe operations commonly used across the package, and show how to specify them.
 17 | 
 18 | .. _transform-op:
 19 | 
 20 | Transform
 21 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 22 | 
 23 | The transform operation takes a single vector and performs transforms it to produce
 24 | another vector as output. The transformation may be non-linear. A transform operation can
 25 | be specified by using one of the following:
 26 | 
 27 | * A string: One of the `styles` supported by the :class:`Transform` module.
 28 | * An instance of :class:`Transform`.
 29 | * A :attr:`callable`: A function that returns a PyTorch :class:`~torch.nn.Module`. This
 30 |   module must have the same input and output shape signature as the :class:`Transform`
 31 |   module.
 32 | 
 33 | This operation is implemented by the :class:`Transform` module:
 34 | 
 35 | .. autoclass:: Transform
 36 |     :members:
 37 | 
 38 | .. _pool-op:
 39 | 
 40 | Pool
 41 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 42 | 
 43 | The Pool operation takes a sequence of vectors and aggregates this sequence to produce a
 44 | single vector as output. A Pool operation can be specified using one of the
 45 | following:
 46 | 
 47 | * A string: One of the `styles` supported by the :class:`Pool` module.
 48 | * An instance of :class:`Pool`.
 49 | * A :attr:`callable`: A function that returns a PyTorch :class:`~torch.nn.Module`. This
 50 |   module must have the same input and output shape signature as the :class:`Pool` module.
 51 | 
 52 | This operation is implemented by the :class:`Pool` module:
 53 | 
 54 | .. autoclass:: Pool
 55 |     :members:
 56 | 
 57 | .. _merge-op:
 58 | 
 59 | Merge
 60 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 61 | 
 62 | The Merge operation takes two or more vectors and aggregates the information in them to
 63 | produce a single vector as output. Unlike the case of :ref:`pool-op` operation, the input
 64 | vectors here are not considered to be sequential in nature. A Merge operation can be
 65 | specified using one of the following:
 66 | 
 67 | * A string: One of the `styles` supported by the :class:`Merge` module. Note that some
 68 |   styles only support two input vectors to be merged, while others allow multiple inputs.
 69 | * An instance of :class:`Merge`.
 70 | * A :attr:`callable`: A function that returns a PyTorch :class:`~torch.nn.Module`. This
 71 |   module must have the same input and output shape signature as the
 72 |   :class:`Merge` module.
 73 | 
 74 | This operation is implemented by the :class:`Merge` module:
 75 | 
 76 | .. autoclass:: Merge
 77 |     :members:
 78 | 
 79 | .. _align-op:
 80 | 
 81 | Align
 82 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 83 | 
 84 | The Align operation takes two sequences of vectors, aligns the words in them, and returns
 85 | the corresponding alignment score matrix. For each word in the first sequence, the
 86 | alignment matrix contains unnormalized scores indicating the degree to which each word in
 87 | the second sequence aligns with it. For an example of one way to do this, take a look `at
 88 | this paper <https://arxiv.org/abs/1606.01933>`__. An Align operation can be specified
 89 | using one of the following:
 90 | 
 91 | * A string: One of the `styles` supported by the :class:`AlignmentNetwork` module.
 92 | * An instance of :class:`AlignmentNetwork`.
 93 | * A :attr:`callable`: A function that returns a PyTorch :class:`~torch.nn.Module`. This
 94 |   module must have the same input and output shape signature as the
 95 |   :class:`AlignmentNetwork` module.
 96 | 
 97 | This operation is implemented by the :class:`AlignmentNetwork` module:
 98 | 
 99 | .. autoclass:: AlignmentNetwork
100 |     :members:
101 | 
102 | .. _bypass-op:
103 | 
104 | Bypass
105 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
106 | 
107 | The Bypass operation takes two tensors, one corresponding to an input tensor and the other
108 | corresponding to a transformed version of the first tensor, applies a bypass network
109 | and returns one tensor of the same size as the transformed tensor. Examples of bypass
110 | networks include `residual networks <https://arxiv.org/abs/1512.03385>`__ and
111 | `highway networks <https://arxiv.org/abs/1505.00387>`__.
112 | 
113 | * A string: One of the `styles` supported by the :class:`Bypass` module.
114 | * An instance of :class:`Bypass`.
115 | * A :attr:`callable`: A function that returns a PyTorch :class:`~torch.nn.Module`. This
116 |   module must have the same input and output shape signature as the
117 |   :class:`Bypass` module.
118 | 
119 | This operation is implemented by the :class:`Bypass` module:
120 | 
121 | .. autoclass:: Bypass
122 |     :members:
123 | 
124 | .. _rnn-op:
125 | 
126 | RNN
127 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
128 | 
129 | This operation takes a sequence of vectors and produces a context-aware transformation of
130 | the input sequence as output. For an intro to RNNs, take a look at
131 | `this article <http://www.wildml.com/2015/09/recurrent-neural-networks-tutorial-part-1-introduction-to-rnns/>`__.
132 | An RNN operation can be specified using one of the following:
133 | 
134 | * A string: One of the `unit_types` supported by the :class:`RNN` module.
135 | * An instance of :class:`RNN`.
136 | * A :attr:`callable`: A function that returns a PyTorch :class:`~torch.nn.Module`. This
137 |   module must have the same input and output shape signature as the
138 |   :class:`RNN` module.
139 | 
140 | This operation is implemented by the :class:`RNN` module:
141 | 
142 | .. autoclass:: RNN
143 |     :members:
144 | 
145 | Utility Modules
146 | -------------------
147 | 
148 | Apart from standard operations, DeepMatcher also contains several utility modules to help
149 | glue together various components. These are listed below.
150 | 
151 | :hidden:`Lambda`
152 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
153 | .. autoclass:: Lambda
154 |     :members:
155 | 
156 | :hidden:`MultiSequential`
157 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
158 | .. autoclass:: MultiSequential
159 |     :members:
160 | 
161 | :hidden:`NoMeta`
162 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
163 | .. autoclass:: NoMeta
164 |     :members:
165 | 
166 | :hidden:`ModuleMap`
167 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
168 | .. autoclass:: ModuleMap
169 |     :members:
170 | 
171 | :hidden:`LazyModule`
172 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
173 | .. autoclass:: LazyModule
174 |     :members: __init__, forward, expect_signature
175 | 
176 | :hidden:`LazyModuleFn`
177 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
178 | .. autoclass:: LazyModuleFn
179 |     :members:
180 | 


--------------------------------------------------------------------------------
/docs/source/notes/design.rst:
--------------------------------------------------------------------------------
1 | Design Notes about DeepMatcher package
2 | =======================================
3 | 


--------------------------------------------------------------------------------
/docs/source/notes/faq.rst:
--------------------------------------------------------------------------------
1 | FAQ
2 | ===
3 | 


--------------------------------------------------------------------------------
/docs/source/notes/model.rst:
--------------------------------------------------------------------------------
1 | DL Architecture Template
2 | =========================
3 | 


--------------------------------------------------------------------------------
/docs/source/optim.rst:
--------------------------------------------------------------------------------
 1 | .. role:: hidden
 2 |     :class: hidden-section
 3 | 
 4 | deepmatcher.optim
 5 | =========================
 6 | 
 7 | .. automodule:: deepmatcher.optim
 8 | .. currentmodule:: deepmatcher.optim
 9 | 
10 | :hidden:`SoftNLLLoss`
11 | ~~~~~~~~~~~~~~~~~~~~~~~
12 | .. autoclass:: SoftNLLLoss
13 |     :members:
14 | 
15 | :hidden:`Optimizer`
16 | ~~~~~~~~~~~~~~~~~~~~~~~
17 | .. autoclass:: Optimizer
18 |     :members:
19 | 


--------------------------------------------------------------------------------
/docs/source/word_aggregators.rst:
--------------------------------------------------------------------------------
 1 | .. role:: hidden
 2 |     :class: hidden-section
 3 | 
 4 | deepmatcher.word_aggregators
 5 | ==============================
 6 | 
 7 | .. automodule:: deepmatcher.word_aggregators
 8 | .. currentmodule:: deepmatcher.word_aggregators
 9 | 
10 | :hidden:`Pool`
11 | ~~~~~~~~~~~~~~~~~~
12 | .. autoclass:: Pool
13 |     :members:
14 | 
15 | :hidden:`AttentionWithRNN`
16 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~
17 | .. autoclass:: AttentionWithRNN
18 |     :members:
19 | 


--------------------------------------------------------------------------------
/docs/source/word_comparators.rst:
--------------------------------------------------------------------------------
1 | deepmatcher.word_comparators
2 | ==============================
3 | 
4 | .. automodule:: deepmatcher.word_comparators
5 | .. currentmodule:: deepmatcher.word_comparators
6 | 
7 | .. autoclass:: Attention
8 |     :members:
9 | 


--------------------------------------------------------------------------------
/docs/source/word_contextualizers.rst:
--------------------------------------------------------------------------------
 1 | .. role:: hidden
 2 |     :class: hidden-section
 3 | 
 4 | deepmatcher.word_contextualizers
 5 | ==================================
 6 | 
 7 | .. automodule:: deepmatcher.word_contextualizers
 8 | .. currentmodule:: deepmatcher.word_contextualizers
 9 | 
10 | :hidden:`RNN`
11 | ~~~~~~~~~~~~~~~~~~
12 | .. autoclass:: RNN
13 |     :members:
14 | 
15 | :hidden:`SelfAttention`
16 | ~~~~~~~~~~~~~~~~~~~~~~~~~
17 | .. autoclass:: SelfAttention
18 |     :members:
19 | 


--------------------------------------------------------------------------------
/setup.cfg:
--------------------------------------------------------------------------------
1 | [bdist_wheel]
2 | # This flag says to generate wheels that support both Python 2 and Python
3 | # 3. If your code will not run unchanged on both Python 2 and 3, you will
4 | # need to generate separate wheels for each Python version that you
5 | # support.
6 | universal=1
7 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | import io
 2 | import os
 3 | import re
 4 | 
 5 | from setuptools import setup
 6 | 
 7 | 
 8 | def read(*names, **kwargs):
 9 |     with open(os.path.join(os.path.dirname(__file__), *names)) as fp:
10 |         return fp.read()
11 | 
12 | 
13 | def find_version(*file_paths):
14 |     version_file = read(*file_paths)
15 |     version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M)
16 |     if version_match:
17 |         return version_match.group(1)
18 |     raise RuntimeError("Unable to find version string.")
19 | 
20 | 
21 | VERSION = find_version('deepmatcher', '__init__.py')
22 | long_description = read('README.rst')
23 | 
24 | # Deepmatcher lists "fasttextmirror" as a dependency because the official "fasttext"
25 | # release on PyPI is out of date and has not been updated in over a year.
26 | # "fasttextmirror" is a clone of the "fasttext" repository as of June 5 2018 and is hosted
27 | # at https://github.com/sidharthms/fastText.
28 | setup(
29 |     name='deepmatcher',
30 |     description='A deep learning package for entity matching',
31 |     long_description=long_description,
32 |     version=VERSION,
33 |     author='Sidharth Mudgal, Han Li',
34 |     author_email='uwmagellan@gmail.com',
35 |     url='http://deepmatcher.ml',
36 |     license='BSD',
37 |     packages=['deepmatcher', 'deepmatcher.data', 'deepmatcher.models'],
38 |     python_requires='>=3.5',
39 |     install_requires=[
40 |         'torch>=1.0', 'tqdm', 'pyprind', 'six', 'Cython', 'torchtext>=0.9',
41 |         'nltk>=3.2.5', 'fasttext', 'pandas', 'dill', 'scikit-learn'
42 |     ])
43 | 


--------------------------------------------------------------------------------
/test/__init__.py:
--------------------------------------------------------------------------------
1 | import os
2 | 
3 | test_dir_path = os.path.dirname(os.path.realpath(__file__))
4 | 


--------------------------------------------------------------------------------
/test/test_dataset.py:
--------------------------------------------------------------------------------
  1 | import io
  2 | import os
  3 | import shutil
  4 | import unittest
  5 | from test import test_dir_path
  6 | 
  7 | import pandas as pd
  8 | from nose.tools import *
  9 | 
 10 | import torch
 11 | from deepmatcher.data.dataset import *
 12 | from deepmatcher.data.field import FastText, MatchingField
 13 | from deepmatcher.data.process import _make_fields, process
 14 | from torchtext.utils import unicode_csv_reader
 15 | from urllib.parse import urljoin
 16 | from urllib.request import pathname2url
 17 | 
 18 | 
 19 | # import nltk
 20 | # nltk.download('perluniprops')
 21 | # nltk.download('nonbreaking_prefixes')
 22 | 
 23 | 
 24 | class ClassMatchingDatasetTestCases(unittest.TestCase):
 25 | 
 26 |     def test_init_1(self):
 27 |         fields = [('left_a', MatchingField()), ('right_a', MatchingField())]
 28 |         col_naming = {'id': 'id', 'label': 'label', 'left': 'left', 'right': 'right'}
 29 |         path = os.path.join(test_dir_path, 'test_datasets', 'sample_table_small.csv')
 30 |         md = MatchingDataset(fields, col_naming, path=path)
 31 |         self.assertEqual(md.id_field, 'id')
 32 |         self.assertEqual(md.label_field, 'label')
 33 |         self.assertEqual(md.all_left_fields, ['left_a'])
 34 |         self.assertEqual(md.all_right_fields, ['right_a'])
 35 |         self.assertEqual(md.all_text_fields, ['left_a', 'right_a'])
 36 |         self.assertEqual(md.canonical_text_fields, ['_a'])
 37 | 
 38 | 
 39 | class MatchingDatasetSplitsTestCases(unittest.TestCase):
 40 | 
 41 |     def setUp(self):
 42 |         self.data_dir = os.path.join(test_dir_path, 'test_datasets')
 43 |         self.train = 'test_train.csv'
 44 |         self.validation = 'test_valid.csv'
 45 |         self.test = 'test_test.csv'
 46 |         self.cache_name = 'test_cacheddata.pth'
 47 |         with io.open(
 48 |                 os.path.expanduser(os.path.join(self.data_dir, self.train)),
 49 |                 encoding="utf8") as f:
 50 |             header = next(unicode_csv_reader(f))
 51 | 
 52 |         id_attr = 'id'
 53 |         label_attr = 'label'
 54 |         ignore_columns = ['left_id', 'right_id']
 55 |         self.fields = _make_fields(header, id_attr, label_attr, ignore_columns, True,
 56 |                                    'nltk', False)
 57 | 
 58 |         self.column_naming = {
 59 |             'id': id_attr,
 60 |             'left': 'left_',
 61 |             'right': 'right_',
 62 |             'label': label_attr
 63 |         }
 64 | 
 65 |     def tearDown(self):
 66 |         cache_name = os.path.join(self.data_dir, self.cache_name)
 67 |         if os.path.exists(cache_name):
 68 |             os.remove(cache_name)
 69 | 
 70 |     def test_splits_1(self):
 71 |         datasets = MatchingDataset.splits(
 72 |             self.data_dir,
 73 |             self.train,
 74 |             self.validation,
 75 |             self.test,
 76 |             self.fields,
 77 |             None,
 78 |             None,
 79 |             self.column_naming,
 80 |             self.cache_name,
 81 |             train_pca=False)
 82 | 
 83 |     @raises(MatchingDataset.CacheStaleException)
 84 |     def test_splits_2(self):
 85 |         datasets = MatchingDataset.splits(
 86 |             self.data_dir,
 87 |             self.train,
 88 |             self.validation,
 89 |             self.test,
 90 |             self.fields,
 91 |             None,
 92 |             None,
 93 |             self.column_naming,
 94 |             self.cache_name,
 95 |             train_pca=False)
 96 | 
 97 |         datasets_2 = MatchingDataset.splits(
 98 |             self.data_dir,
 99 |             'sample_table_small.csv',
100 |             self.validation,
101 |             self.test,
102 |             self.fields,
103 |             None,
104 |             None,
105 |             self.column_naming,
106 |             self.cache_name,
107 |             True,
108 |             False,
109 |             train_pca=False)
110 | 
111 |     def test_splits_3(self):
112 |         datasets = MatchingDataset.splits(
113 |             self.data_dir,
114 |             self.train,
115 |             self.validation,
116 |             self.test,
117 |             self.fields,
118 |             None,
119 |             None,
120 |             self.column_naming,
121 |             self.cache_name,
122 |             train_pca=False)
123 | 
124 |         datasets_2 = MatchingDataset.splits(
125 |             self.data_dir,
126 |             self.train,
127 |             self.validation,
128 |             self.test,
129 |             self.fields,
130 |             None,
131 |             None,
132 |             self.column_naming,
133 |             self.cache_name,
134 |             False,
135 |             False,
136 |             train_pca=False)
137 | 
138 | 
139 | class DataframeSplitTestCases(unittest.TestCase):
140 | 
141 |     def test_split_1(self):
142 |         labeled_path = os.path.join(test_dir_path, 'test_datasets',
143 |                                     'sample_table_large.csv')
144 |         labeled_table = pd.read_csv(labeled_path)
145 |         ori_cols = list(labeled_table.columns)
146 |         out_path = os.path.join(test_dir_path, 'test_datasets')
147 |         train_prefix = 'train.csv'
148 |         valid_prefix = 'valid.csv'
149 |         test_prefix = 'test.csv'
150 |         split(labeled_table, out_path, train_prefix, valid_prefix, test_prefix)
151 | 
152 |         train_path = os.path.join(out_path, train_prefix)
153 |         valid_path = os.path.join(out_path, valid_prefix)
154 |         test_path = os.path.join(out_path, test_prefix)
155 | 
156 |         train = pd.read_csv(train_path)
157 |         valid = pd.read_csv(valid_path)
158 |         test = pd.read_csv(test_path)
159 | 
160 |         self.assertEqual(list(train.columns), ori_cols)
161 |         self.assertEqual(list(valid.columns), ori_cols)
162 |         self.assertEqual(list(test.columns), ori_cols)
163 | 
164 |         if os.path.exists(train_path):
165 |             os.remove(train_path)
166 |         if os.path.exists(valid_path):
167 |             os.remove(valid_path)
168 |         if os.path.exists(test_path):
169 |             os.remove(test_path)
170 | 
171 |     def test_split_2(self):
172 |         labeled_path = os.path.join(test_dir_path, 'test_datasets',
173 |                                     'sample_table_large.csv')
174 |         labeled_table = pd.read_csv(labeled_path)
175 |         ori_cols = list(labeled_table.columns)
176 |         out_path = os.path.join(test_dir_path, 'test_datasets')
177 |         train_prefix = 'train.csv'
178 |         valid_prefix = 'valid.csv'
179 |         test_prefix = 'test.csv'
180 |         split(labeled_path, out_path, train_prefix, valid_prefix, test_prefix)
181 | 
182 |         train_path = os.path.join(out_path, train_prefix)
183 |         valid_path = os.path.join(out_path, valid_prefix)
184 |         test_path = os.path.join(out_path, test_prefix)
185 | 
186 |         train = pd.read_csv(train_path)
187 |         valid = pd.read_csv(valid_path)
188 |         test = pd.read_csv(test_path)
189 | 
190 |         self.assertEqual(list(train.columns), ori_cols)
191 |         self.assertEqual(list(valid.columns), ori_cols)
192 |         self.assertEqual(list(test.columns), ori_cols)
193 | 
194 |         if os.path.exists(train_path):
195 |             os.remove(train_path)
196 |         if os.path.exists(valid_path):
197 |             os.remove(valid_path)
198 |         if os.path.exists(test_path):
199 |             os.remove(test_path)
200 | 
201 | 
202 | class GetRawTableTestCases(unittest.TestCase):
203 | 
204 |     def test_get_raw_table(self):
205 |         vectors_cache_dir = '.cache'
206 |         if os.path.exists(vectors_cache_dir):
207 |             shutil.rmtree(vectors_cache_dir)
208 | 
209 |         data_cache_path = os.path.join(test_dir_path, 'test_datasets', 'cacheddata.pth')
210 |         if os.path.exists(data_cache_path):
211 |             os.remove(data_cache_path)
212 | 
213 |         vec_dir = os.path.abspath(os.path.join(test_dir_path, 'test_datasets'))
214 |         filename = 'fasttext_sample.vec.zip'
215 |         url_base = urljoin('file:', pathname2url(vec_dir)) + os.path.sep
216 |         ft = FastText(filename, url_base=url_base, cache=vectors_cache_dir)
217 | 
218 |         train = process(
219 |             path=os.path.join(test_dir_path, 'test_datasets'),
220 |             train='sample_table_small.csv',
221 |             id_attr='id',
222 |             embeddings=ft,
223 |             embeddings_cache_path='',
224 |             pca=False)
225 | 
226 |         train_raw = train.get_raw_table()
227 |         ori_train = pd.read_csv(
228 |             os.path.join(test_dir_path, 'test_datasets', 'sample_table_small.csv'))
229 |         self.assertEqual(set(train_raw.columns), set(ori_train.columns))
230 | 
231 |         if os.path.exists(data_cache_path):
232 |             os.remove(data_cache_path)
233 | 
234 |         if os.path.exists(vectors_cache_dir):
235 |             shutil.rmtree(vectors_cache_dir)
236 | 


--------------------------------------------------------------------------------
/test/test_datasets/fasttext_sample.vec.zip:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/anhaidgroup/deepmatcher/a89ffe6cd246f690afd2772e47eb071741160f16/test/test_datasets/fasttext_sample.vec.zip


--------------------------------------------------------------------------------
/test/test_datasets/sample_table_small.csv:
--------------------------------------------------------------------------------
1 | id,left_a,right_a,label
2 | 0,test,is,1
3 | 1,a,hh,0
4 | 


--------------------------------------------------------------------------------
/test/test_datasets/test_test.csv:
--------------------------------------------------------------------------------
 1 | id,left_id,right_id,label,left_Song_Name,left_Artist_Name,left_Album_Name,left_Genre,left_Price,left_CopyRight,left_Time,left_Released,right_Song_Name,right_Artist_Name,right_Album_Name,right_Genre,right_Price,right_CopyRight,right_Time,right_Released
 2 | 28,311,22308,0,Lifted ( feat . Emeli Sand ' © & Professor Green ),Naughty Boy,Hotel Cabana ( Deluxe Version ),"Pop , Music , Rock , R&B / Soul , Contemporary R&B , Dance,Electronic,Hip-Hop / Rap , Pop/Rock",$ 1.29,2013 Naughty Boy Recordings Ltd under exclusive licence to Virgin Records Ltd,4:15,6-May-14,La La La [ feat . Sam Smith ],Naughty Boy,Hotel Cabana [ Explicit ],Pop,$ 1.29,( C ) 2014 Naughty Boy Recordings Ltd under exclusive licence to Virgin Records Ltd,3:42,"May 6 , 2014"
 3 | 411,4468,31072,1,Better Than Words,One Direction,Midnight Memories,"Pop , Music , Rock , Pop/Rock , Teen Pop , Dance",$ 1.29,2013 Simco Limited under exclusive license to Sony Music Entertainment UK Limited,3:27,25-Nov-13,Better Than Words,One Direction,Midnight Memories ( Deluxe ),Pop,$ 1.29,"( C ) 2014 Disney Enterprises , Inc.",3:28,"November 25 , 2013"
 4 | 440,5001,3099,1,Who I Am With You,Chris Young,A.M.,"Country , Music , Urban Cowboy , Contemporary Country",$ 1.29,‰ ãÑ 2013 Sony Music Entertainment,3:13,16-Sep-13,Who I Am With You,Chris Young,A.M.,Country,$ 1.29,"( C ) 2010 Hollywood Records , Inc.",3:11,"September 16 , 2013"
 5 | 183,1406,48287,1,The Legend Of Zelda ( Electrixx Remix ),Zedd,The Legend Of Zelda - Single,"Dance , Music",$ 0.99,"2011 Bazooka , a division of zooland Music GmbH",6:09,27-May-11,The Legend Of Zelda ( Electrixx Remix ),Zedd,The Legend Of Zelda,"Electronica , Dance & Electronic",$ 0.89,"2011 Bazooka , a division of zooland Music GmbH",6:09,"May 27 , 2011"
 6 | 405,4301,29695,1,See You Again,Elle King,Love Stuff,"Alternative , Music , Rock",$ 1.29,"2014 RCA Records , a division of Sony Music Entertainment",3:26,13-Feb-15,See You Again,Elle King,Love Stuff,Alternative Rock,$ 1.29,( C ) 1996 The Island Def Jam Music Group,3:26,"February 13 , 2015"
 7 | 438,4936,36673,0,Songs Like This,Carrie Underwood,Play On,"Country , Music , Pop , Adult Contemporary , Contemporary Country , Rock",$ 1.29,"‰ ãÑ 2009 19 Recordings Limited , under exclusive license to Arista Nashville",2:37,3-Nov-09,This Time,Carrie Underwood,Play On,Country,$ 1.29,"( P ) 2009 19 Recordings Limited , under exclusive license to Arista Nashville",3:52,"November 3 , 2009"
 8 | 433,4747,5682,1,Skrillex Orchestral Suite By Varien ( Bonus Track ),Skrillex,Bangarang,"Dance , Music , Electronic",$ 1.29,"2012 Big Beat Records , Inc. for the United States and WEA International for the world outside the United States .",6:54,27-Dec-11,Skrillex Orchestral Suite By Varien ( Bonus Track ),Skrillex,Bangarang EP [ Explicit ],"Dance & Electronic , Dubstep",$ 1.29,Elektroshok Records,6:54,"December 27 , 2011"
 9 | 146,1275,28013,0,Toyfriend ( feat . Wynter Gordon ) [ Continuous Mix Version ],David Guetta,One Love ( Deluxe Version ),"Dance , Music",$ 1.29,2010 Gum Prod licence exclusive Parlophone Music France,2:51,21-Aug-09,Dangerous ( feat . Sam Martin ),David Guetta,Listen ( Deluxe ),Dance & Electronic,$ 1.29,2015 If Moving Your Mine,3:23,"November 24 , 2014"
10 | 258,2459,10377,0,O Little Town of Bethlehem,Kenny Chesney,All I Want for Christmas Is a Real Good Tan,"Holiday , Music , Country , Honky Tonk , Contemporary Country , Christmas",$ 0.99,2003 BMG Music,4:18,4-Oct-03,Sweet Little Jesus Boy,Kenny Rogers,Christmas Greetings,"Holiday , Christmas , Miscellaneous",$ 1.29,"( C ) 2000 Capitol Records , LLC",3:04,"August 17 , 2000"
11 | 266,2696,17160,0,Whateva ( feat . Monty ),Fetty Wap,Fetty Wap ( Deluxe ),"Hip-Hop/Rap , Music , Rap",Album Only,2015 300 Entertainment/RGF Productions,5:11,,For My Team ( feat . Monty ) [ Clean ],Fetty Wap,Fetty Wap ( Deluxe ) [ Clean ],Rap & Hip-Hop,$ 1.29,"( C ) 2000 Mute Records Ltd. , a BMG Company , under exclusive license to INgrooves",3:28,"September 25 , 2015"
12 | 237,2022,28157,0,If I Am a Stranger ( Live In Amsterdam ),Ryan Adams,Live After Deaf ( Live ) [ Complete Collection ],"Singer/Songwriter , Music , Rock",$ 0.99,2012 PAXAM RECORDS,5:10,2-Jul-12,Firecracker ( Live in Amsterdam ),Ryan Adams,Live After Deaf ( Amsterdam ),"Folk , Rock",$ 0.99,2012 PAXAM RECORDS,3:17,"July 2 , 2012"
13 | 205,1583,41583,0,Second Chance ( Live Acoustic from Kansas City ),Shinedown,Somewhere In the Stratosphere ( Live ),"Rock , Music , Metal , Alternative , Hard Rock",$ 1.29,2011 Atlantic Recording Corporation for the United States and WEA International Inc. for the world outside of the United States .,5:06,18-Apr-11,Heroes ( Live Acoustic From Kansas City ),Shinedown,Somewhere In The Stratosphere,Rock,$ 1.29,( C ) 2005 MCA Nashville,8:09,"April 18 , 2011"
14 | 324,3324,7403,1,We Dem Boyz,Wiz Khalifa,Blacc Hollywood ( Deluxe Version ),"Hip-Hop/Rap , Music , Rap , Hardcore Rap , East Coast Rap , Rock",$ 1.99,2014 Atlantic Recording Corporation for the United States and WEA International Inc. for the world outside of the United States . A Warner Music Group Company,3:56,19-Aug-14,We Dem Boyz [ Explicit ],Wiz Khalifa,Blacc Hollywood [ Explicit ],Rap & Hip-Hop,$ 1.29,2014 Atlantic Recording Corporation for the United States and WEA International Inc. for the world outside of the United States . A Warner Music Group Company,3:45,"August 19 , 2014"
15 | 10,252,53036,0,VHS Outro ( Interlude ),X Ambassadors,VHS,"Alternative , Music , Rock , Adult Alternative",$ 1.29,2015 KIDinaKORNER/Interscope Records,1:25,30-Jun-15,Smoke ( Interlude ) [ Explicit ],X Ambassadors,VHS [ Explicit ],Alternative Rock,$ 1.29,( C ) 2015 KIDinaKORNER/Interscope Records,0:24,"June 30 , 2015"
16 | 7,251,52967,1,VHS Outro ( Interlude ),X Ambassadors,VHS,"Alternative , Music , Rock , Adult Alternative",$ 1.29,2015 KIDinaKORNER/Interscope Records,1:25,30-Jun-15,VHS Outro ( Interlude ) [ Clean ],X Ambassadors,VHS [ Clean ],Alternative Rock,$ 1.29,( C ) 2015 KIDinaKORNER/Interscope Records,1:26,"June 30 , 2015"
17 | 144,1274,9947,0,Toyfriend ( feat . Wynter Gordon ) [ Continuous Mix Version ],David Guetta,One Love ( Deluxe Version ),"Dance , Music",$ 1.29,2010 Gum Prod licence exclusive Parlophone Music France,2:51,21-Aug-09,Crossroads ( Album Version ( Edited ) ) [ feat . Grout ] [ Clean ],David Banner,Certified ( Edited Version ),Rap & Hip-Hop,$ 0.99,"( C ) 2014 Island Records , a division of UMG Recordings , Inc.",3:45,"September 20 , 2005"
18 | 523,6592,38375,1,I Almost Do,Taylor Swift,Red,"Country , Music , Rock , Contemporary Country , Honky Tonk , Pop , Teen Pop",$ 1.29,"‰ ãÑ 2012 Big Machine Records , LLC .",4:04,22-Oct-12,I Almost Do,Taylor Swift,Red,Country,$ 1.29,"( C ) 2012 Big Machine Records , LLC .",4:02,"October 30 , 2012"
19 | 201,1570,27494,0,Breaking Inside ( feat . Lzzy Hale of Halestorm ) [ Bonus Track ],Shinedown,The Sound of Madness ( Deluxe Edition ),"Rock , Music , Alternative , Grunge , Metal , Hard Rock",$ 1.29,2010 Atlantic Recording Corporation for the United States and WEA International Inc. for the world outside of the United States .,3:47,24-Jun-08,Fly From The Inside,Shinedown,Leave A Whisper ( Deluxe ),Rock,$ 1.29,2015 Mad Decent,3:55,"May 27 , 2003"
20 | 318,3318,34341,0,Medicated ( feat . Chevy Woods & Juicy J ),Wiz Khalifa,O.N.I.F.C.,"Hip-Hop/Rap , Music , Rap , East Coast Rap , Hardcore Rap",$ 1.29,2012 Atlantic Recording Corporation for the United States and WEA International Inc. for the world outside of the United States,5:30,4-Dec-12,Bluffin ( feat . Berner ) [ Explicit ],Wiz Khalifa,O.N.I.F.C. [ Explicit ],Rap & Hip-Hop,$ 1.29,2012 Atlantic Recording Corporation for the United States and WEA International Inc. for the world outside of the United States,5:32,"December 4 , 2012"
21 | 25,303,22290,0,Lifted ( feat . Emeli Sand ' © & Professor Green ),Naughty Boy,Hotel Cabana ( Deluxe Version ),"Pop , Music , Rock , R&B / Soul , Contemporary R&B , Dance,Electronic,Hip-Hop / Rap , Pop/Rock",$ 1.29,2013 Naughty Boy Recordings Ltd under exclusive licence to Virgin Records Ltd,4:15,6-May-14,One Way [ feat . Mic Righteous ],Naughty Boy,Hotel Cabana ( Deluxe Version ) [ Explicit ],Pop,$ 1.29,( C ) 2014 Naughty Boy Recordings Ltd under exclusive licence to Virgin Records Ltd,3:26,"May 6 , 2014"
22 | 133,1264,34920,0,Toyfriend ( feat . Wynter Gordon ) [ Continuous Mix Version ],David Guetta,One Love ( Deluxe Version ),"Dance , Music",$ 1.29,2010 Gum Prod licence exclusive Parlophone Music France,2:51,21-Aug-09,Gettin ' Over ( Feat . Chris Willis ; Continuous Mix Version ),David Guetta,One Love ( Deluxe Version ),Dance & Electronic,$ 1.29,( C ) 2014 Swedish House Mafia Holdings Ltd ( BVI ) under exclusive license to Virgin Records Ltd,3:31,"August 21 , 2009"
23 | 154,1280,9941,0,Toyfriend ( feat . Wynter Gordon ) [ Continuous Mix Version ],David Guetta,One Love ( Deluxe Version ),"Dance , Music",$ 1.29,2010 Gum Prod licence exclusive Parlophone Music France,2:51,21-Aug-09,Bloody War ( Album Version ( Edited ) ) [ feat . BG ] [ Clean ],David Banner,Certified ( Edited Version ),Rap & Hip-Hop,$ 0.99,"( C ) 2014 Island Records , a division of UMG Recordings , Inc.",4:27,"September 20 , 2005"
24 | 524,6621,784,0,Bad Blood ( Karaoke Version ),Taylor Swift,Taylor Swift Karaoke : 1989,"Pop , Music",$ 1.29,"‰ ãÑ 2014 Big Machine Records , LLC",3:29,3-Mar-15,Bad Blood,Taylor Swift,1989,Pop,$ 1.29,"( C ) 2014 Big Machine Records , LLC .",3:31,"October 27 , 2014"
25 | 529,6762,51437,0,"The Birds , Pt. 1",The Weeknd,Trilogy,"R&B / Soul , Music , Rock , Contemporary R&B",$ 1.29,"‰ ãÑ 2012 XO & co. , Inc.",3:34,13-Nov-12,The Birds Pt. 2 ( Album Version ( Edited ) ) [ Clean ],The Weeknd,Trilogy ( Edited Version ) [ Clean ],R&B,$ 1.29,"( C ) 2013 XO & co. , Inc.",5:53,"September 18 , 2015"
26 | 


--------------------------------------------------------------------------------
/test/test_datasets/test_unlabeled.csv:
--------------------------------------------------------------------------------
 1 | id,left_id,right_id,left_Song_Name,left_Artist_Name,left_Album_Name,left_Genre,left_Price,left_CopyRight,left_Time,left_Released,right_Song_Name,right_Artist_Name,right_Album_Name,right_Genre,right_Price,right_CopyRight,right_Time,right_Released
 2 | 28,311,22308,Lifted ( feat . Emeli Sand ' © & Professor Green ),Naughty Boy,Hotel Cabana ( Deluxe Version ),"Pop , Music , Rock , R&B / Soul , Contemporary R&B , Dance,Electronic,Hip-Hop / Rap , Pop/Rock",$ 1.29,2013 Naughty Boy Recordings Ltd under exclusive licence to Virgin Records Ltd,4:15,6-May-14,La La La [ feat . Sam Smith ],Naughty Boy,Hotel Cabana [ Explicit ],Pop,$ 1.29,( C ) 2014 Naughty Boy Recordings Ltd under exclusive licence to Virgin Records Ltd,3:42,"May 6 , 2014"
 3 | 411,4468,31072,Better Than Words,One Direction,Midnight Memories,"Pop , Music , Rock , Pop/Rock , Teen Pop , Dance",$ 1.29,2013 Simco Limited under exclusive license to Sony Music Entertainment UK Limited,3:27,25-Nov-13,Better Than Words,One Direction,Midnight Memories ( Deluxe ),Pop,$ 1.29,"( C ) 2014 Disney Enterprises , Inc.",3:28,"November 25 , 2013"
 4 | 440,5001,3099,Who I Am With You,Chris Young,A.M.,"Country , Music , Urban Cowboy , Contemporary Country",$ 1.29,‰ ãÑ 2013 Sony Music Entertainment,3:13,16-Sep-13,Who I Am With You,Chris Young,A.M.,Country,$ 1.29,"( C ) 2010 Hollywood Records , Inc.",3:11,"September 16 , 2013"
 5 | 183,1406,48287,The Legend Of Zelda ( Electrixx Remix ),Zedd,The Legend Of Zelda - Single,"Dance , Music",$ 0.99,"2011 Bazooka , a division of zooland Music GmbH",6:09,27-May-11,The Legend Of Zelda ( Electrixx Remix ),Zedd,The Legend Of Zelda,"Electronica , Dance & Electronic",$ 0.89,"2011 Bazooka , a division of zooland Music GmbH",6:09,"May 27 , 2011"
 6 | 405,4301,29695,See You Again,Elle King,Love Stuff,"Alternative , Music , Rock",$ 1.29,"2014 RCA Records , a division of Sony Music Entertainment",3:26,13-Feb-15,See You Again,Elle King,Love Stuff,Alternative Rock,$ 1.29,( C ) 1996 The Island Def Jam Music Group,3:26,"February 13 , 2015"
 7 | 438,4936,36673,Songs Like This,Carrie Underwood,Play On,"Country , Music , Pop , Adult Contemporary , Contemporary Country , Rock",$ 1.29,"‰ ãÑ 2009 19 Recordings Limited , under exclusive license to Arista Nashville",2:37,3-Nov-09,This Time,Carrie Underwood,Play On,Country,$ 1.29,"( P ) 2009 19 Recordings Limited , under exclusive license to Arista Nashville",3:52,"November 3 , 2009"
 8 | 433,4747,5682,Skrillex Orchestral Suite By Varien ( Bonus Track ),Skrillex,Bangarang,"Dance , Music , Electronic",$ 1.29,"2012 Big Beat Records , Inc. for the United States and WEA International for the world outside the United States .",6:54,27-Dec-11,Skrillex Orchestral Suite By Varien ( Bonus Track ),Skrillex,Bangarang EP [ Explicit ],"Dance & Electronic , Dubstep",$ 1.29,Elektroshok Records,6:54,"December 27 , 2011"
 9 | 146,1275,28013,Toyfriend ( feat . Wynter Gordon ) [ Continuous Mix Version ],David Guetta,One Love ( Deluxe Version ),"Dance , Music",$ 1.29,2010 Gum Prod licence exclusive Parlophone Music France,2:51,21-Aug-09,Dangerous ( feat . Sam Martin ),David Guetta,Listen ( Deluxe ),Dance & Electronic,$ 1.29,2015 If Moving Your Mine,3:23,"November 24 , 2014"
10 | 258,2459,10377,O Little Town of Bethlehem,Kenny Chesney,All I Want for Christmas Is a Real Good Tan,"Holiday , Music , Country , Honky Tonk , Contemporary Country , Christmas",$ 0.99,2003 BMG Music,4:18,4-Oct-03,Sweet Little Jesus Boy,Kenny Rogers,Christmas Greetings,"Holiday , Christmas , Miscellaneous",$ 1.29,"( C ) 2000 Capitol Records , LLC",3:04,"August 17 , 2000"
11 | 266,2696,17160,Whateva ( feat . Monty ),Fetty Wap,Fetty Wap ( Deluxe ),"Hip-Hop/Rap , Music , Rap",Album Only,2015 300 Entertainment/RGF Productions,5:11,,For My Team ( feat . Monty ) [ Clean ],Fetty Wap,Fetty Wap ( Deluxe ) [ Clean ],Rap & Hip-Hop,$ 1.29,"( C ) 2000 Mute Records Ltd. , a BMG Company , under exclusive license to INgrooves",3:28,"September 25 , 2015"
12 | 237,2022,28157,If I Am a Stranger ( Live In Amsterdam ),Ryan Adams,Live After Deaf ( Live ) [ Complete Collection ],"Singer/Songwriter , Music , Rock",$ 0.99,2012 PAXAM RECORDS,5:10,2-Jul-12,Firecracker ( Live in Amsterdam ),Ryan Adams,Live After Deaf ( Amsterdam ),"Folk , Rock",$ 0.99,2012 PAXAM RECORDS,3:17,"July 2 , 2012"
13 | 205,1583,41583,Second Chance ( Live Acoustic from Kansas City ),Shinedown,Somewhere In the Stratosphere ( Live ),"Rock , Music , Metal , Alternative , Hard Rock",$ 1.29,2011 Atlantic Recording Corporation for the United States and WEA International Inc. for the world outside of the United States .,5:06,18-Apr-11,Heroes ( Live Acoustic From Kansas City ),Shinedown,Somewhere In The Stratosphere,Rock,$ 1.29,( C ) 2005 MCA Nashville,8:09,"April 18 , 2011"
14 | 324,3324,7403,We Dem Boyz,Wiz Khalifa,Blacc Hollywood ( Deluxe Version ),"Hip-Hop/Rap , Music , Rap , Hardcore Rap , East Coast Rap , Rock",$ 1.99,2014 Atlantic Recording Corporation for the United States and WEA International Inc. for the world outside of the United States . A Warner Music Group Company,3:56,19-Aug-14,We Dem Boyz [ Explicit ],Wiz Khalifa,Blacc Hollywood [ Explicit ],Rap & Hip-Hop,$ 1.29,2014 Atlantic Recording Corporation for the United States and WEA International Inc. for the world outside of the United States . A Warner Music Group Company,3:45,"August 19 , 2014"
15 | 10,252,53036,VHS Outro ( Interlude ),X Ambassadors,VHS,"Alternative , Music , Rock , Adult Alternative",$ 1.29,2015 KIDinaKORNER/Interscope Records,1:25,30-Jun-15,Smoke ( Interlude ) [ Explicit ],X Ambassadors,VHS [ Explicit ],Alternative Rock,$ 1.29,( C ) 2015 KIDinaKORNER/Interscope Records,0:24,"June 30 , 2015"
16 | 7,251,52967,VHS Outro ( Interlude ),X Ambassadors,VHS,"Alternative , Music , Rock , Adult Alternative",$ 1.29,2015 KIDinaKORNER/Interscope Records,1:25,30-Jun-15,VHS Outro ( Interlude ) [ Clean ],X Ambassadors,VHS [ Clean ],Alternative Rock,$ 1.29,( C ) 2015 KIDinaKORNER/Interscope Records,1:26,"June 30 , 2015"
17 | 144,1274,9947,Toyfriend ( feat . Wynter Gordon ) [ Continuous Mix Version ],David Guetta,One Love ( Deluxe Version ),"Dance , Music",$ 1.29,2010 Gum Prod licence exclusive Parlophone Music France,2:51,21-Aug-09,Crossroads ( Album Version ( Edited ) ) [ feat . Grout ] [ Clean ],David Banner,Certified ( Edited Version ),Rap & Hip-Hop,$ 0.99,"( C ) 2014 Island Records , a division of UMG Recordings , Inc.",3:45,"September 20 , 2005"
18 | 523,6592,38375,I Almost Do,Taylor Swift,Red,"Country , Music , Rock , Contemporary Country , Honky Tonk , Pop , Teen Pop",$ 1.29,"‰ ãÑ 2012 Big Machine Records , LLC .",4:04,22-Oct-12,I Almost Do,Taylor Swift,Red,Country,$ 1.29,"( C ) 2012 Big Machine Records , LLC .",4:02,"October 30 , 2012"
19 | 201,1570,27494,Breaking Inside ( feat . Lzzy Hale of Halestorm ) [ Bonus Track ],Shinedown,The Sound of Madness ( Deluxe Edition ),"Rock , Music , Alternative , Grunge , Metal , Hard Rock",$ 1.29,2010 Atlantic Recording Corporation for the United States and WEA International Inc. for the world outside of the United States .,3:47,24-Jun-08,Fly From The Inside,Shinedown,Leave A Whisper ( Deluxe ),Rock,$ 1.29,2015 Mad Decent,3:55,"May 27 , 2003"
20 | 318,3318,34341,Medicated ( feat . Chevy Woods & Juicy J ),Wiz Khalifa,O.N.I.F.C.,"Hip-Hop/Rap , Music , Rap , East Coast Rap , Hardcore Rap",$ 1.29,2012 Atlantic Recording Corporation for the United States and WEA International Inc. for the world outside of the United States,5:30,4-Dec-12,Bluffin ( feat . Berner ) [ Explicit ],Wiz Khalifa,O.N.I.F.C. [ Explicit ],Rap & Hip-Hop,$ 1.29,2012 Atlantic Recording Corporation for the United States and WEA International Inc. for the world outside of the United States,5:32,"December 4 , 2012"
21 | 25,303,22290,Lifted ( feat . Emeli Sand ' © & Professor Green ),Naughty Boy,Hotel Cabana ( Deluxe Version ),"Pop , Music , Rock , R&B / Soul , Contemporary R&B , Dance,Electronic,Hip-Hop / Rap , Pop/Rock",$ 1.29,2013 Naughty Boy Recordings Ltd under exclusive licence to Virgin Records Ltd,4:15,6-May-14,One Way [ feat . Mic Righteous ],Naughty Boy,Hotel Cabana ( Deluxe Version ) [ Explicit ],Pop,$ 1.29,( C ) 2014 Naughty Boy Recordings Ltd under exclusive licence to Virgin Records Ltd,3:26,"May 6 , 2014"
22 | 133,1264,34920,Toyfriend ( feat . Wynter Gordon ) [ Continuous Mix Version ],David Guetta,One Love ( Deluxe Version ),"Dance , Music",$ 1.29,2010 Gum Prod licence exclusive Parlophone Music France,2:51,21-Aug-09,Gettin ' Over ( Feat . Chris Willis ; Continuous Mix Version ),David Guetta,One Love ( Deluxe Version ),Dance & Electronic,$ 1.29,( C ) 2014 Swedish House Mafia Holdings Ltd ( BVI ) under exclusive license to Virgin Records Ltd,3:31,"August 21 , 2009"
23 | 154,1280,9941,Toyfriend ( feat . Wynter Gordon ) [ Continuous Mix Version ],David Guetta,One Love ( Deluxe Version ),"Dance , Music",$ 1.29,2010 Gum Prod licence exclusive Parlophone Music France,2:51,21-Aug-09,Bloody War ( Album Version ( Edited ) ) [ feat . BG ] [ Clean ],David Banner,Certified ( Edited Version ),Rap & Hip-Hop,$ 0.99,"( C ) 2014 Island Records , a division of UMG Recordings , Inc.",4:27,"September 20 , 2005"
24 | 524,6621,784,Bad Blood ( Karaoke Version ),Taylor Swift,Taylor Swift Karaoke : 1989,"Pop , Music",$ 1.29,"‰ ãÑ 2014 Big Machine Records , LLC",3:29,3-Mar-15,Bad Blood,Taylor Swift,1989,Pop,$ 1.29,"( C ) 2014 Big Machine Records , LLC .",3:31,"October 27 , 2014"
25 | 529,6762,51437,"The Birds , Pt. 1",The Weeknd,Trilogy,"R&B / Soul , Music , Rock , Contemporary R&B",$ 1.29,"‰ ãÑ 2012 XO & co. , Inc.",3:34,13-Nov-12,The Birds Pt. 2 ( Album Version ( Edited ) ) [ Clean ],The Weeknd,Trilogy ( Edited Version ) [ Clean ],R&B,$ 1.29,"( C ) 2013 XO & co. , Inc.",5:53,"September 18 , 2015"
26 | 


--------------------------------------------------------------------------------
/test/test_datasets/test_valid.csv:
--------------------------------------------------------------------------------
 1 | _id,ltable_id,rtable_id,label,ltable_Song_Name,ltable_Artist_Name,ltable_Album_Name,ltable_Genre,ltable_Price,ltable_CopyRight,ltable_Time,ltable_Released,rtable_Song_Name,rtable_Artist_Name,rtable_Album_Name,rtable_Genre,rtable_Price,rtable_CopyRight,rtable_Time,rtable_Released
 2 | 262,2653,44051,0,From This Dream,Little Big Town,Little Big Town,"Country , Music , Urban Cowboy , Honky Tonk , Contemporary Country",$ 0.99,2002 SONY BMG MUSIC ENTERTAINMENT,4:27,7-May-02,Tattoos On This Town [ Explicit ],Like Tattoos On This Town,Tattoos On This Town - Single ( Jason Aldean Tribute ) [ Explicit ],Country,$ 0.99,2012 Music Club,3:28,"June 14 , 2012"
 3 | 1,147,50730,1,Credit,Meghan Trainor,Title ( Deluxe ),"Pop , Music , Rock , Pop/Rock , Dance , Teen Pop",$ 1.29,"2014 , 2015 Epic Records , a division of Sony Music Entertainment",2:51,9-Jan-15,Credit,Meghan Trainor,Title ( Deluxe ),Pop,$ 1.29,"2011 What A Music Ltd , licence exclusive Parlophone Music France",2:51,"January 9 , 2015"
 4 | 15,264,53050,0,VHS Outro ( Interlude ),X Ambassadors,VHS,"Alternative , Music , Rock , Adult Alternative",$ 1.29,2015 KIDinaKORNER/Interscope Records,1:25,30-Jun-15,Moving Day ( Interlude ),X Ambassadors,VHS [ Explicit ],Alternative Rock,$ 1.29,( C ) 2015 KIDinaKORNER/Interscope Records,0:19,"June 30 , 2015"
 5 | 98,1183,34915,0,Toyfriend ( feat . Wynter Gordon ) [ Continuous Mix Version ],David Guetta,One Love ( Deluxe Version ),"Dance , Music",$ 1.29,2010 Gum Prod licence exclusive Parlophone Music France,2:51,21-Aug-09,I Wan na Go Crazy ( Feat . Will.I.Am ; Extended ; Continuous Mix Version ),David Guetta,One Love ( Deluxe Version ),Dance & Electronic,$ 1.29,( C ) 2014 Swedish House Mafia Holdings Ltd ( BVI ) under exclusive license to Virgin Records Ltd,4:54,"August 21 , 2009"
 6 | 308,3255,4532,0,Applause ( Goldhouse Remix ),Lady Gaga,Applause ( Remixes ),"Pop , Music , Electronic , Rock",$ 1.29,2013 Interscope Records,4:34,30-Sep-13,Applause ( Empire Of The Sun Remix ),Lady Gaga,Applause ( Remixes ),Pop,$ 1.29,( C ) 2013 Interscope Records,4:08,"September 30 , 2013"
 7 | 471,5207,22901,0,Popular ( feat . Lil ' Twist ),Lil Wayne & Lil ' Twist,I Am Not a Human Being,"Hip-Hop/Rap , Music , Rap , Hardcore Rap , Dirty South",$ 1.29,‰ ãÑ 2010 Cash Money Records Inc.,4:40,15-Nov-10,No Worries ( Album Version ( Explicit ) ) [ feat . Detail ] [ Explicit ],Lil Wayne,I Am Not A Human Being II ( Explicit Version ) [ + digital booklet ],Rap & Hip-Hop,$ 1.29,2005 Secretly Canadian,3:42,"March 26 , 2013"
 8 | 212,1636,27530,1,45,Shinedown,Leave a Whisper,"Rock , Music , Hard Rock , Alternative",$ 1.29,2003 Atlantic Recording Corporation for the United States and WEA International for the world outside of the United States,4:09,27-May-03,45,Shinedown,Leave A Whisper,Rock,$ 1.29,2015 Mad Decent,4:09,"May 27 , 2003"
 9 | 420,4698,30404,0,Make It Bun Dem ( Alvin Risk Remix ),Skrillex & Damian `` Jr. . Gong '' Marley,Make It Bun Dem After Hours ( Remixes ),"Dance , Music , Electronic",$ 1.29,2012 OWSLA and Big Beat Records for the United States and WEA International for the world outside the United States .,3:58,28-Aug-12,Make It Bun Dem ( David Heartbreak Remix ),Skrillex & Damian Jr Gong '' Marley '',Make It Bun Dem After Hours EP,Dance & Electronic,$ 1.29,2015 RGK,3:29,"August 28 , 2012"
10 | 527,6679,44061,0,You Belong With Me ( Instrumental With Background Vocals ),Taylor Swift,Taylor Swift Karaoke : Fearless ( Instrumentals With Background Vocals ),"Country , Music",$ 0.99,"‰ ãÑ 2009 Big Machine Records , LLC",3:57,7-Apr-09,How You Get The Girl ( Karaoke Version ),Taylor Swift,Taylor Swift Karaoke : 1989,Pop,$ 1.29,"( C ) 2014 Big Machine Records , LLC",4:05,"March 3 , 2015"
11 | 462,5130,37223,0,Money to Blow,"Birdman , Lil Wayne & Drake",Pricele $ $,"Hip-Hop/Rap , Music , Dirty South",$ 1.29,‰ ãÑ 2009 Cash Money Records Inc.,4:18,23-Nov-09,Been About Money ( Album Version ( Edited ) ),Birdman,Pricele $ $ ( Edited Version ),Rap & Hip-Hop,$ 0.99,( C ) 2009 Cash Money Records Inc.,4:12,"November 23 , 2009"
12 | 513,6314,20460,0,Ca n't Stop Now ( feat . Jovi Rockwell and Mr. Vegas ),Major Lazer,Guns Do n't Kill People ... Lazers Do,"Electronic,Music,Hip-Hop / Rap , Rap , Alternative , Reggae , Dance , Modern Dancehall , Rock",$ 1.29,"‰ ãÑ 2009 Downtown Music , LLC .",4:03,2009,Cash Flow [ feat . Jah Dan ],Major Lazer,Guns Do n't Kill People ... Lazers Do ( Deluxe ) [ Explicit ],"Electronica , Dance & Electronic",$ 1.29,"( C ) 2009 Downtown Music , LLC .",4:06,"June 3 , 2015"
13 | 405,4301,29695,1,See You Again,Elle King,Love Stuff,"Alternative , Music , Rock",$ 1.29,"2014 RCA Records , a division of Sony Music Entertainment",3:26,13-Feb-15,See You Again,Elle King,Love Stuff,Alternative Rock,$ 1.29,( C ) 1996 The Island Def Jam Music Group,3:26,"February 13 , 2015"
14 | 102,1193,28008,1,The Whisperer ( feat . Sia ),David Guetta,Listen,"Dance , Music , Electronic , House , Rock , French Pop",$ 1.29,"2014 What A Music Ltd. under exclusive license to Parlophone/Warner Music France , under exclusive license to Atlantic Recording Corporation for the United States . All rights reserved",3:54,24-Nov-14,The Whisperer ( feat . Sia ),David Guetta,Listen ( Deluxe ),Dance & Electronic,$ 1.29,2015 If Moving Your Mine,3:54,"November 24 , 2014"
15 | 100,1193,27999,0,The Whisperer ( feat . Sia ),David Guetta,Listen,"Dance , Music , Electronic , House , Rock , French Pop",$ 1.29,"2014 What A Music Ltd. under exclusive license to Parlophone/Warner Music France , under exclusive license to Atlantic Recording Corporation for the United States . All rights reserved",3:54,24-Nov-14,Goodbye Friend ( feat . The Script ),David Guetta,Listen ( Deluxe ),Dance & Electronic,$ 1.29,2015 If Moving Your Mine,3:49,"November 24 , 2014"
16 | 57,667,24620,0,Thriller ( Live From Hammersmith Palais ),Fall Out Boy,Infinity On High ( Deluxe Edition ),"Alternative , Music , Rock , Adult Alternative , Punk",$ 0.99,2007 The Island Def Jam Music Group,3:32,20-Nov-07,"This Ai n't A Scene , It 's An Arms Race ( Live From Hammersmith Palais )",Fall Out Boy,Infinity On High - Deluxe Bonus EP,Alternative Rock,$ 1.29,( C ) 2007 The Island Def Jam Music Group,3:23,"November 20 , 2007"
17 | 112,1204,28025,0,Dangerous ( feat . Sam Martin ) [ Robin Schulz Remix ] [ Radio Edit ],David Guetta,Listen ( Deluxe Version ),"Dance , Music , Rock , House , Electronic , French Pop",$ 1.29,"2014 What A Music Ltd. under exclusive license to Parlophone/Warner Music France , under exclusive license to Atlantic Recording Corporation for the United States . All rights reserved .",3:20,24-Nov-14,I 'll Keep Loving you ( feat . Birdy & Jaymes Young ),David Guetta,Listen ( Deluxe ),Dance & Electronic,$ 1.29,2015 If Moving Your Mine,3:08,"November 24 , 2014"
18 | 305,3104,22995,1,We 're All the Same,Calvin Harris,I Created Disco,"Dance , Music , R&B / Soul , Disco , Electronic , Rock",$ 0.99,2007 SONY BMG MUSIC ENTERTAINMENT ( UK ) Limited,3:56,18-Jun-07,We 're All The Same,Calvin Harris,I Created Disco,"Dance & Electronic , Pop , House",$ 0.99,2002 Defected Records Ltd,3:56,"February 22 , 2012"
19 | 162,1299,34007,0,I 'm a Machine ( feat . Crystal Nicole and Tyrese Gibson ),David Guetta,Nothing But the Beat,"Dance , Music , Rock , House , Electronic",$ 1.29,"2011 What A Music Ltd , Licence exclusive Parlophone Music France",3:34,26-Aug-11,I Can Only Imagine ( Feat . Chris Brown & Lil Wayne ),David Guetta,Nothing But The Beat,Dance & Electronic,$ 1.29,2009 Andy Henningsen,3:29,"August 26 , 2011"
20 | 196,1548,41583,0,Second Chance ( Live Acoustic from Kansas City ),Shinedown,Somewhere In the Stratosphere ( Live ),"Rock , Music , Metal , Alternative , Hard Rock",$ 1.29,2011 Atlantic Recording Corporation for the United States and WEA International Inc. for the world outside of the United States .,5:06,18-Apr-11,Heroes ( Live Acoustic From Kansas City ),Shinedown,Somewhere In The Stratosphere,Rock,$ 1.29,( C ) 2005 MCA Nashville,8:09,"April 18 , 2011"
21 | 63,896,44693,0,Thank Me Now,Drake,Thank Me Later,"Hip-Hop/Rap , Music , Hardcore Rap , Rock , Rap , R&B / Soul , Contemporary R&B",$ 1.29,2010 Young Money Entertainment/Cash Money Records/Universal Motown,5:28,16-Jun-10,Show Me A Good Time ( Explicit ),Drake,Thank Me Later ( Explicit Version ) [ Explicit ],Rap & Hip-Hop,$ 1.29,( C ) 2010 Young Money Entertainment/Cash Money Records,3:30,"June 15 , 2010"
22 | 124,1260,54194,0,Toyfriend ( feat . Wynter Gordon ) [ Continuous Mix Version ],David Guetta,One Love ( Deluxe Version ),"Dance , Music",$ 1.29,2010 Gum Prod licence exclusive Parlophone Music France,2:51,21-Aug-09,When Love Takes Over ( feat . Kelly Rowland ) [ Electro Radio Edit ],David Guetta,When Love Takes Over ( feat . Kelly Rowland ),Dance & Electronic,$ 0.99,"( C ) 2015 The Valory Music Co. , LLC",3:02,"May 5 , 2009"
23 | 3,210,19800,0,Slow Down ( Sure Shot Rockers Reggae Dub Remix ),Selena Gomez,Slow Down ( Reggae Remixes ) - Single,"Reggae , Music , Electronic , Dance , Rock",$ 1.29,"2013 Hollywood Records , Inc.",3:15,20-Aug-13,Good For You ( Yellow Claw & Cesqeaux Remix ) [ feat . A$ AP Rocky ],Selena Gomez,Good For You ( Remixes ),Pop,$ 1.29,( C ) 2015 Interscope Records,3:01,"September 4 , 2015"
24 | 137,1268,34910,0,Toyfriend ( feat . Wynter Gordon ) [ Continuous Mix Version ],David Guetta,One Love ( Deluxe Version ),"Dance , Music",$ 1.29,2010 Gum Prod licence exclusive Parlophone Music France,2:51,21-Aug-09,Sound Of Letting Go ( Feat . Chris Willis ),David Guetta,One Love ( Deluxe Version ),Dance & Electronic,$ 1.29,( C ) 2014 Swedish House Mafia Holdings Ltd ( BVI ) under exclusive license to Virgin Records Ltd,3:47,"August 21 , 2009"
25 | 386,3762,38431,1,Brave Honest Beautiful ( feat . Meghan Trainor ),Fifth Harmony,Reflection ( Deluxe ),"Pop , Music , R&B / Soul , Dance , Teen Pop , Rock",$ 1.29,"2014 , 2015 Simco Ltd. under exclusive license to Epic Records , a division of Sony Music Entertainment",3:28,30-Jan-15,Brave Honest Beautiful,Fifth Harmony,Reflection ( Deluxe ),Pop,$ 1.29,"( C ) 2012 Big Machine Records , LLC .",3:28,"January 30 , 2015"
26 | 


--------------------------------------------------------------------------------
/test/test_field.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import shutil
  3 | import unittest
  4 | from collections import Counter
  5 | from test import test_dir_path
  6 | 
  7 | from nose.tools import *
  8 | 
  9 | import torch
 10 | from deepmatcher.data.dataset import MatchingDataset
 11 | from deepmatcher.data.field import (FastText, FastTextBinary, MatchingField,
 12 |                                     MatchingVocab, reset_vector_cache)
 13 | from torchtext.vocab import Vectors
 14 | from urllib.parse import urljoin
 15 | from urllib.request import pathname2url
 16 | 
 17 | # import nltk
 18 | # nltk.download('perluniprops')
 19 | # nltk.download('nonbreaking_prefixes')
 20 | 
 21 | 
 22 | class ClassFastTextTestCases(unittest.TestCase):
 23 | 
 24 |     def test_init_1(self):
 25 |         vectors_cache_dir = '.cache'
 26 |         if os.path.exists(vectors_cache_dir):
 27 |             shutil.rmtree(vectors_cache_dir)
 28 | 
 29 |         pathdir = os.path.abspath(os.path.join(test_dir_path, 'test_datasets'))
 30 |         filename = 'fasttext_sample.vec.zip'
 31 |         url_base = urljoin('file:', pathname2url(pathdir)) + os.path.sep
 32 |         mft = FastText(filename, url_base=url_base, cache=vectors_cache_dir)
 33 |         self.assertEqual(mft.dim, 300)
 34 |         self.assertEqual(mft.vectors.size(), torch.Size([100, 300]))
 35 | 
 36 |         if os.path.exists(vectors_cache_dir):
 37 |             shutil.rmtree(vectors_cache_dir)
 38 | 
 39 | 
 40 | class ClassFastTextBinaryTestCases(unittest.TestCase):
 41 | 
 42 |     @raises(RuntimeError)
 43 |     def test_init_1(self):
 44 |         vectors_cache_dir = '.cache'
 45 |         if os.path.exists(vectors_cache_dir):
 46 |             shutil.rmtree(vectors_cache_dir)
 47 | 
 48 |         pathdir = os.path.abspath(os.path.join(test_dir_path, 'test_datasets'))
 49 |         filename = 'fasttext_sample.vec.zip'
 50 |         url_base = urljoin('file:', pathname2url(os.path.join(pathdir, filename)))
 51 |         mftb = FastTextBinary(filename, url_base=url_base, cache=vectors_cache_dir)
 52 | 
 53 |         if os.path.exists(vectors_cache_dir):
 54 |             shutil.rmtree(vectors_cache_dir)
 55 | 
 56 |     @raises(OSError)
 57 |     def test_init_2(self):
 58 |         vectors_cache_dir = '.cache'
 59 |         if os.path.exists(vectors_cache_dir):
 60 |             shutil.rmtree(vectors_cache_dir)
 61 | 
 62 |         pathdir = os.path.abspath(os.path.join(test_dir_path, 'test_datasets'))
 63 |         filename = 'fasttext_sample_not_exist.vec.zip'
 64 |         url_base = urljoin('file:', pathname2url(os.path.join(pathdir, filename)))
 65 |         mftb = FastTextBinary(filename, url_base=url_base, cache=vectors_cache_dir)
 66 | 
 67 |         if os.path.exists(vectors_cache_dir):
 68 |             shutil.rmtree(vectors_cache_dir)
 69 | 
 70 |     @raises(OSError)
 71 |     def test_init_3(self):
 72 |         vectors_cache_dir = '.cache'
 73 |         if os.path.exists(vectors_cache_dir):
 74 |             shutil.rmtree(vectors_cache_dir)
 75 | 
 76 |         pathdir = os.path.abspath(os.path.join(test_dir_path, 'test_datasets'))
 77 |         filename = 'fasttext_sample_not_exist.gz'
 78 |         url_base = urljoin('file:', pathname2url(os.path.join(pathdir, filename)))
 79 |         mftb = FastTextBinary(filename, url_base=url_base, cache=vectors_cache_dir)
 80 | 
 81 |         if os.path.exists(vectors_cache_dir):
 82 |             shutil.rmtree(vectors_cache_dir)
 83 | 
 84 | 
 85 | class ClassMatchingFieldTestCases(unittest.TestCase):
 86 | 
 87 |     def test_init_1(self):
 88 |         mf = MatchingField()
 89 |         self.assertTrue(mf.sequential)
 90 | 
 91 |     def test_init_2(self):
 92 |         mf = MatchingField()
 93 |         seq = 'Hello, This is a test sequence for tokenizer.'
 94 |         tok_seq = [
 95 |             'Hello', ',', 'This', 'is', 'a', 'test', 'sequence', 'for', 'tokenizer', '.'
 96 |         ]
 97 |         self.assertEqual(mf.tokenize(seq), tok_seq)
 98 | 
 99 |     @raises(ValueError)
100 |     def test_init_3(self):
101 |         mf = MatchingField(tokenize='random string')
102 | 
103 |     def test_preprocess_args_1(self):
104 |         mf = MatchingField()
105 |         arg_dict = mf.preprocess_args()
106 |         res_dict = {
107 |             'sequential': True,
108 |             'init_token': None,
109 |             'eos_token': None,
110 |             'init_token': None,
111 |             'lower': False,
112 |             'preprocessing': None,
113 |             'sequential': True,
114 |             'tokenizer_arg': 'nltk',
115 |             'unk_token': '<unk>'
116 |         }
117 |         self.assertEqual(arg_dict, res_dict)
118 | 
119 |     def test_build_vocab_1(self):
120 |         mf = MatchingField()
121 |         mf.build_vocab()
122 | 
123 |     @raises(KeyError)
124 |     def test_build_vocab_2(self):
125 |         mf = MatchingField()
126 |         vector_file_name = 'fasttext.wiki_test.vec'
127 |         cache_dir = os.path.join(test_dir_path, 'test_datasets')
128 |         vec_data = mf.build_vocab(vectors=vector_file_name, cache=cache_dir)
129 | 
130 |     @raises(KeyError)
131 |     def test_build_vocab_3(self):
132 |         mf = MatchingField()
133 |         vector_file_name = 'fasttext.crawl_test.vec'
134 |         cache_dir = os.path.join(test_dir_path, 'test_datasets')
135 |         vec_data = mf.build_vocab(vectors=vector_file_name, cache=cache_dir)
136 |         self.assertIsNone(vec_data)
137 | 
138 |     def test_get_vector_data(self):
139 |         vectors_cache_dir = '.cache'
140 |         if os.path.exists(vectors_cache_dir):
141 |             shutil.rmtree(vectors_cache_dir)
142 | 
143 |         pathdir = os.path.abspath(os.path.join(test_dir_path, 'test_datasets'))
144 |         filename = 'fasttext_sample.vec'
145 |         file = os.path.join(pathdir, filename)
146 |         url_base = urljoin('file:', pathname2url(file))
147 |         vecs = Vectors(name=filename, cache=vectors_cache_dir, url=url_base)
148 |         self.assertIsInstance(vecs, Vectors)
149 | 
150 |         vec_data = MatchingField._get_vector_data(vecs, vectors_cache_dir)
151 |         self.assertEqual(len(vec_data), 1)
152 |         self.assertEqual(vec_data[0].vectors.size(), torch.Size([100, 300]))
153 |         self.assertEqual(vec_data[0].dim, 300)
154 | 
155 |         if os.path.exists(vectors_cache_dir):
156 |             shutil.rmtree(vectors_cache_dir)
157 | 
158 |     def test_numericalize_1(self):
159 |         mf = MatchingField(id=True)
160 |         arr = [[1], [2], [3]]
161 |         mf.numericalize(arr)
162 |         self.assertEqual(arr, [[1], [2], [3]])
163 | 
164 |     @raises(AttributeError)
165 |     def test_numericalize_2(self):
166 |         mf = MatchingField()
167 |         arr = [['a'], ['b'], ['c']]
168 |         mf.numericalize(arr)
169 | 
170 |     def test_extend_vocab_1(self):
171 |         vectors_cache_dir = '.cache'
172 |         if os.path.exists(vectors_cache_dir):
173 |             shutil.rmtree(vectors_cache_dir)
174 | 
175 |         mf = MatchingField()
176 |         lf = MatchingField(id=True, sequential=False)
177 |         fields = [('id', lf), ('left_a', mf), ('right_a', mf), ('label', lf)]
178 |         col_naming = {'id': 'id', 'label': 'label', 'left': 'left_', 'right': 'right_'}
179 | 
180 |         pathdir = os.path.abspath(os.path.join(test_dir_path, 'test_datasets'))
181 |         filename = 'fasttext_sample.vec'
182 |         file = os.path.join(pathdir, filename)
183 |         url_base = urljoin('file:', pathname2url(file))
184 |         vecs = Vectors(name=filename, cache=vectors_cache_dir, url=url_base)
185 | 
186 |         data_path = os.path.join(test_dir_path, 'test_datasets', 'sample_table_small.csv')
187 |         md = MatchingDataset(fields, col_naming, path=data_path)
188 | 
189 |         mf.build_vocab()
190 |         mf.vocab.vectors = torch.Tensor(len(mf.vocab.itos), 300)
191 |         mf.extend_vocab(md, vectors=vecs)
192 |         self.assertEqual(len(mf.vocab.itos), 6)
193 |         self.assertEqual(mf.vocab.vectors.size(), torch.Size([6, 300]))
194 | 
195 | 
196 | class TestResetVectorCache(unittest.TestCase):
197 | 
198 |     def test_reset_vector_cache_1(self):
199 |         mf = MatchingField()
200 |         reset_vector_cache()
201 |         self.assertDictEqual(mf._cached_vec_data, {})
202 | 
203 | 
204 | class ClassMatchingVocabTestCases(unittest.TestCase):
205 | 
206 |     def test_extend_vectors_1(self):
207 |         vectors_cache_dir = '.cache'
208 |         if os.path.exists(vectors_cache_dir):
209 |             shutil.rmtree(vectors_cache_dir)
210 | 
211 |         pathdir = os.path.abspath(os.path.join(test_dir_path, 'test_datasets'))
212 |         filename = 'fasttext_sample.vec'
213 |         file = os.path.join(pathdir, filename)
214 |         url_base = urljoin('file:', pathname2url(file))
215 |         vecs = Vectors(name=filename, cache=vectors_cache_dir, url=url_base)
216 |         self.assertIsInstance(vecs, Vectors)
217 | 
218 |         vec_data = MatchingField._get_vector_data(vecs, vectors_cache_dir)
219 |         v = MatchingVocab(Counter())
220 |         v.vectors = torch.Tensor(1, vec_data[0].dim)
221 |         v.unk_init = torch.Tensor.zero_
222 |         tokens = {'hello', 'world'}
223 |         v.extend_vectors(tokens, vec_data)
224 |         self.assertEqual(len(v.itos), 4)
225 |         self.assertEqual(v.vectors.size(), torch.Size([4, 300]))
226 |         self.assertEqual(list(v.vectors[2][0:10]), [0.0] * 10)
227 |         self.assertEqual(list(v.vectors[3][0:10]), [0.0] * 10)
228 | 
229 |         if os.path.exists(vectors_cache_dir):
230 |             shutil.rmtree(vectors_cache_dir)
231 | 


--------------------------------------------------------------------------------
/test/test_integration.py:
--------------------------------------------------------------------------------
  1 | from nose.tools import *
  2 | 
  3 | import io
  4 | import os
  5 | import shutil
  6 | import pandas as pd
  7 | import torch
  8 | import unittest
  9 | 
 10 | from deepmatcher import attr_summarizers
 11 | from deepmatcher.data.field import MatchingField, FastText
 12 | from deepmatcher.data.process import process, process_unlabeled
 13 | from deepmatcher import MatchingModel
 14 | from urllib.parse import urljoin
 15 | from urllib.request import pathname2url
 16 | 
 17 | from test import test_dir_path
 18 | 
 19 | class ModelTrainSaveLoadTest(unittest.TestCase):
 20 |     def setUp(self):
 21 |         self.vectors_cache_dir = '.cache'
 22 |         if os.path.exists(self.vectors_cache_dir):
 23 |             shutil.rmtree(self.vectors_cache_dir)
 24 | 
 25 |         self.data_cache_path = os.path.join(test_dir_path, 'test_datasets',
 26 |             'train_cache.pth')
 27 |         if os.path.exists(self.data_cache_path):
 28 |             os.remove(self.data_cache_path)
 29 | 
 30 |         vec_dir = os.path.abspath(os.path.join(test_dir_path, 'test_datasets'))
 31 |         filename = 'fasttext_sample.vec.zip'
 32 |         url_base = urljoin('file:', pathname2url(vec_dir)) + os.path.sep
 33 |         ft = FastText(filename, url_base=url_base, cache=self.vectors_cache_dir)
 34 | 
 35 |         self.train, self.valid, self.test = process(
 36 |             path=os.path.join(test_dir_path, 'test_datasets'),
 37 |             cache='train_cache.pth',
 38 |             train='test_train.csv',
 39 |             validation='test_valid.csv',
 40 |             test='test_test.csv',
 41 |             embeddings=ft,
 42 |             embeddings_cache_path='',
 43 |             ignore_columns=('left_id', 'right_id'))
 44 | 
 45 |     def tearDown(self):
 46 |         if os.path.exists(self.data_cache_path):
 47 |             os.remove(self.data_cache_path)
 48 | 
 49 |         if os.path.exists(self.vectors_cache_dir):
 50 |             shutil.rmtree(self.vectors_cache_dir)
 51 | 
 52 |     def test_sif(self):
 53 |         model_save_path = 'sif_model.pth'
 54 |         model = MatchingModel(attr_summarizer='sif')
 55 |         model.run_train(
 56 |             self.train,
 57 |             self.valid,
 58 |             epochs=1,
 59 |             batch_size=8,
 60 |             best_save_path= model_save_path,
 61 |             pos_neg_ratio=3)
 62 |         s1 = model.run_eval(self.test)
 63 | 
 64 |         model2 = MatchingModel(attr_summarizer='sif')
 65 |         model2.load_state(model_save_path)
 66 |         s2 = model2.run_eval(self.test)
 67 | 
 68 |         self.assertEqual(s1, s2)
 69 | 
 70 |         if os.path.exists(model_save_path):
 71 |             os.remove(model_save_path)
 72 | 
 73 |     def test_rnn(self):
 74 |         model_save_path = 'rnn_model.pth'
 75 |         model = MatchingModel(attr_summarizer='rnn')
 76 |         model.run_train(
 77 |             self.train,
 78 |             self.valid,
 79 |             epochs=1,
 80 |             batch_size=8,
 81 |             best_save_path= model_save_path,
 82 |             pos_neg_ratio=3)
 83 |         s1 = model.run_eval(self.test)
 84 | 
 85 |         model2 = MatchingModel(attr_summarizer='rnn')
 86 |         model2.load_state(model_save_path)
 87 |         s2 = model2.run_eval(self.test)
 88 | 
 89 |         self.assertEqual(s1, s2)
 90 | 
 91 |         if os.path.exists(model_save_path):
 92 |             os.remove(model_save_path)
 93 | 
 94 |     def test_attention(self):
 95 |         model_save_path = 'attention_model.pth'
 96 |         model = MatchingModel(attr_summarizer='attention')
 97 |         model.run_train(
 98 |             self.train,
 99 |             self.valid,
100 |             epochs=1,
101 |             batch_size=8,
102 |             best_save_path= model_save_path,
103 |             pos_neg_ratio=3)
104 | 
105 |         s1 = model.run_eval(self.test)
106 | 
107 |         model2 = MatchingModel(attr_summarizer='attention')
108 |         model2.load_state(model_save_path)
109 |         s2 = model2.run_eval(self.test)
110 | 
111 |         self.assertEqual(s1, s2)
112 | 
113 |         if os.path.exists(model_save_path):
114 |             os.remove(model_save_path)
115 | 
116 |     def test_hybrid(self):
117 |         model_save_path = 'hybrid_model.pth'
118 |         model = MatchingModel(attr_summarizer='hybrid')
119 |         model.run_train(
120 |             self.train,
121 |             self.valid,
122 |             epochs=1,
123 |             batch_size=8,
124 |             best_save_path= model_save_path,
125 |             pos_neg_ratio=3)
126 | 
127 |         s1 = model.run_eval(self.test)
128 | 
129 |         model2 = MatchingModel(attr_summarizer='hybrid')
130 |         model2.load_state(model_save_path)
131 |         s2 = model2.run_eval(self.test)
132 | 
133 |         self.assertEqual(s1, s2)
134 | 
135 |         if os.path.exists(model_save_path):
136 |             os.remove(model_save_path)
137 | 
138 |     def test_hybrid_self_attention(self):
139 |         model_save_path = 'self_att_hybrid_model.pth'
140 |         model = MatchingModel(
141 |             attr_summarizer=attr_summarizers.Hybrid(
142 |                 word_contextualizer='self-attention'))
143 | 
144 |         model.run_train(
145 |             self.train,
146 |             self.valid,
147 |             epochs=1,
148 |             batch_size=8,
149 |             best_save_path= model_save_path,
150 |             pos_neg_ratio=3)
151 | 
152 |         s1 = model.run_eval(self.test)
153 | 
154 |         model2 = MatchingModel(
155 |             attr_summarizer=attr_summarizers.Hybrid(
156 |                 word_contextualizer='self-attention'))
157 |         model2.load_state(model_save_path)
158 |         s2 = model2.run_eval(self.test)
159 | 
160 |         self.assertEqual(s1, s2)
161 | 
162 |         if os.path.exists(model_save_path):
163 |             os.remove(model_save_path)
164 | 
165 | 
166 | class ModelPredictUnlabeledTest(unittest.TestCase):
167 |     def setUp(self):
168 |         self.vectors_cache_dir = '.cache'
169 |         if os.path.exists(self.vectors_cache_dir):
170 |             shutil.rmtree(self.vectors_cache_dir)
171 | 
172 |         self.data_cache_path = os.path.join(test_dir_path, 'test_datasets',
173 |             'train_cache.pth')
174 |         if os.path.exists(self.data_cache_path):
175 |             os.remove(self.data_cache_path)
176 | 
177 |         vec_dir = os.path.abspath(os.path.join(test_dir_path, 'test_datasets'))
178 |         filename = 'fasttext_sample.vec.zip'
179 |         url_base = urljoin('file:', pathname2url(vec_dir)) + os.path.sep
180 |         ft = FastText(filename, url_base=url_base, cache=self.vectors_cache_dir)
181 | 
182 |         self.train, self.valid, self.test = process(
183 |             path=os.path.join(test_dir_path, 'test_datasets'),
184 |             cache='train_cache.pth',
185 |             train='test_train.csv',
186 |             validation='test_valid.csv',
187 |             test='test_test.csv',
188 |             embeddings=ft,
189 |             embeddings_cache_path='',
190 |             ignore_columns=('left_id', 'right_id'))
191 | 
192 |     def tearDown(self):
193 |         if os.path.exists(self.data_cache_path):
194 |             os.remove(self.data_cache_path)
195 | 
196 |         if os.path.exists(self.vectors_cache_dir):
197 |             shutil.rmtree(self.vectors_cache_dir)
198 | 
199 |     def test_sif(self):
200 |         model_save_path = 'sif_model.pth'
201 |         model = MatchingModel(attr_summarizer='sif')
202 |         model.run_train(
203 |             self.train,
204 |             self.valid,
205 |             epochs=1,
206 |             batch_size=8,
207 |             best_save_path= model_save_path,
208 |             pos_neg_ratio=3)
209 | 
210 |         unlabeled = process_unlabeled(
211 |             path=os.path.join(test_dir_path, 'test_datasets', 'test_unlabeled.csv'),
212 |             trained_model=model,
213 |             ignore_columns=('left_id', 'right_id'))
214 | 
215 |         pred_test = model.run_eval(self.test, return_predictions=True)
216 |         pred_unlabeled = model.run_prediction(unlabeled)
217 | 
218 |         self.assertEqual(sorted([tup[1] for tup in pred_test]),
219 |                          sorted(list(pred_unlabeled['match_score'])))
220 | 
221 |         if os.path.exists(model_save_path):
222 |             os.remove(model_save_path)
223 | 
224 |     def test_rnn(self):
225 |         model_save_path = 'rnn_model.pth'
226 |         model = MatchingModel(attr_summarizer='rnn')
227 |         model.run_train(
228 |             self.train,
229 |             self.valid,
230 |             epochs=1,
231 |             batch_size=8,
232 |             best_save_path= model_save_path,
233 |             pos_neg_ratio=3)
234 | 
235 |         unlabeled = process_unlabeled(
236 |             path=os.path.join(test_dir_path, 'test_datasets', 'test_test.csv'),
237 |             trained_model=model,
238 |             ignore_columns=('left_id', 'right_id', 'label'))
239 | 
240 |         pred_test = model.run_eval(self.test, return_predictions=True)
241 |         pred_unlabeled = model.run_prediction(unlabeled)
242 | 
243 |         self.assertEqual(sorted([tup[1] for tup in pred_test]),
244 |                          sorted(list(pred_unlabeled['match_score'])))
245 | 
246 |         if os.path.exists(model_save_path):
247 |             os.remove(model_save_path)
248 | 
249 |     def test_attention(self):
250 |         model_save_path = 'attention_model.pth'
251 |         model = MatchingModel(attr_summarizer='attention')
252 |         model.run_train(
253 |             self.train,
254 |             self.valid,
255 |             epochs=1,
256 |             batch_size=8,
257 |             best_save_path= model_save_path,
258 |             pos_neg_ratio=3)
259 | 
260 |         unlabeled = process_unlabeled(
261 |             path=os.path.join(test_dir_path, 'test_datasets', 'test_unlabeled.csv'),
262 |             trained_model=model,
263 |             ignore_columns=('left_id', 'right_id'))
264 | 
265 |         pred_test = model.run_eval(self.test, return_predictions=True)
266 |         pred_unlabeled = model.run_prediction(unlabeled)
267 | 
268 |         self.assertEqual(sorted([tup[1] for tup in pred_test]),
269 |                          sorted(list(pred_unlabeled['match_score'])))
270 | 
271 |         if os.path.exists(model_save_path):
272 |             os.remove(model_save_path)
273 | 
274 |     def test_hybrid(self):
275 |         model_save_path = 'hybrid_model.pth'
276 |         model = MatchingModel(attr_summarizer='hybrid')
277 |         model.run_train(
278 |             self.train,
279 |             self.valid,
280 |             epochs=1,
281 |             batch_size=8,
282 |             best_save_path= model_save_path,
283 |             pos_neg_ratio=3)
284 | 
285 |         unlabeled = process_unlabeled(
286 |             path=os.path.join(test_dir_path, 'test_datasets', 'test_unlabeled.csv'),
287 |             trained_model=model,
288 |             ignore_columns=('left_id', 'right_id'))
289 | 
290 |         pred_test = model.run_eval(self.test, return_predictions=True)
291 |         pred_unlabeled = model.run_prediction(unlabeled)
292 | 
293 |         self.assertEqual(sorted([tup[1] for tup in pred_test]),
294 |                          sorted(list(pred_unlabeled['match_score'])))
295 | 
296 |         if os.path.exists(model_save_path):
297 |             os.remove(model_save_path)
298 | 


--------------------------------------------------------------------------------
/test/test_iterator.py:
--------------------------------------------------------------------------------
 1 | from nose.tools import *
 2 | 
 3 | import os
 4 | import torch
 5 | import shutil
 6 | import unittest
 7 | from deepmatcher.data.field import FastText
 8 | from deepmatcher.data.process import process
 9 | from deepmatcher.data.iterator import MatchingIterator
10 | 
11 | from test import test_dir_path
12 | from urllib.parse import urljoin
13 | from urllib.request import pathname2url
14 | 
15 | class ClassMatchingIteratorTestCases(unittest.TestCase):
16 |     def test_splits_1(self):
17 |         vectors_cache_dir = '.cache'
18 |         if os.path.exists(vectors_cache_dir):
19 |             shutil.rmtree(vectors_cache_dir)
20 | 
21 |         data_dir = os.path.join(test_dir_path, 'test_datasets')
22 |         train_path = 'sample_table_large.csv'
23 |         valid_path = 'sample_table_large.csv'
24 |         test_path = 'sample_table_large.csv'
25 |         cache_file = 'cache.pth'
26 |         cache_path = os.path.join(data_dir, cache_file)
27 |         if os.path.exists(cache_path):
28 |             os.remove(cache_path)
29 | 
30 |         pathdir = os.path.abspath(os.path.join(test_dir_path, 'test_datasets'))
31 |         filename = 'fasttext_sample.vec.zip'
32 |         url_base = urljoin('file:', pathname2url(pathdir)) + os.path.sep
33 |         ft = FastText(filename, url_base=url_base, cache=vectors_cache_dir)
34 | 
35 |         datasets = process(data_dir, train=train_path, validation=valid_path,
36 |                            test=test_path, cache=cache_file, embeddings=ft,
37 |                            id_attr='_id', left_prefix='ltable_', right_prefix='rtable_',
38 |                            embeddings_cache_path='',pca=False)
39 | 
40 |         splits = MatchingIterator.splits(datasets, batch_size=16)
41 |         self.assertEqual(splits[0].batch_size, 16)
42 |         self.assertEqual(splits[1].batch_size, 16)
43 |         self.assertEqual(splits[2].batch_size, 16)
44 |         splits_sorted = MatchingIterator.splits(datasets,
45 |                         batch_sizes=[16, 32, 64], sort_in_buckets=False)
46 |         self.assertEqual(splits_sorted[0].batch_size, 16)
47 |         self.assertEqual(splits_sorted[1].batch_size, 32)
48 |         self.assertEqual(splits_sorted[2].batch_size, 64)
49 | 
50 |         if os.path.exists(vectors_cache_dir):
51 |             shutil.rmtree(vectors_cache_dir)
52 | 
53 |         if os.path.exists(cache_path):
54 |             os.remove(cache_path)
55 | 
56 |     def test_create_batches_1(self):
57 |         vectors_cache_dir = '.cache'
58 |         if os.path.exists(vectors_cache_dir):
59 |             shutil.rmtree(vectors_cache_dir)
60 | 
61 |         data_dir = os.path.join(test_dir_path, 'test_datasets')
62 |         train_path = 'sample_table_large.csv'
63 |         valid_path = 'sample_table_large.csv'
64 |         test_path = 'sample_table_large.csv'
65 |         cache_file = 'cache.pth'
66 |         cache_path = os.path.join(data_dir, cache_file)
67 |         if os.path.exists(cache_path):
68 |             os.remove(cache_path)
69 | 
70 |         pathdir = os.path.abspath(os.path.join(test_dir_path, 'test_datasets'))
71 |         filename = 'fasttext_sample.vec.zip'
72 |         url_base = urljoin('file:', pathname2url(pathdir)) + os.path.sep
73 |         ft = FastText(filename, url_base=url_base, cache=vectors_cache_dir)
74 | 
75 |         datasets = process(data_dir, train=train_path, validation=valid_path,
76 |                            test=test_path, cache=cache_file, embeddings=ft,
77 |                            id_attr='_id', left_prefix='ltable_', right_prefix='rtable_',
78 |                            embeddings_cache_path='',pca=False)
79 | 
80 |         splits = MatchingIterator.splits(datasets, batch_size=16)
81 |         batch_splits = [split.create_batches() for split in splits]
82 | 
83 |         sorted_splits = MatchingIterator.splits(datasets,
84 |                         batch_sizes=[16, 32, 64], sort_in_buckets=False)
85 |         batch_sorted_splits = [sorted_split.create_batches()
86 |                                 for sorted_split in sorted_splits]
87 |         if os.path.exists(vectors_cache_dir):
88 |             shutil.rmtree(vectors_cache_dir)
89 | 
90 |         if os.path.exists(cache_path):
91 |             os.remove(cache_path)
92 | 


--------------------------------------------------------------------------------
/test/test_process.py:
--------------------------------------------------------------------------------
  1 | import io
  2 | import os
  3 | import shutil
  4 | import unittest
  5 | from test import test_dir_path
  6 | 
  7 | from nose.tools import *
  8 | 
  9 | from deepmatcher.data.field import FastText
 10 | from deepmatcher.data.process import _check_header, _make_fields, process, process_unlabeled
 11 | from torchtext.utils import unicode_csv_reader
 12 | from deepmatcher import MatchingModel
 13 | from urllib.parse import urljoin
 14 | from urllib.request import pathname2url
 15 | 
 16 | 
 17 | class CheckHeaderTestCases(unittest.TestCase):
 18 | 
 19 |     def test_check_header_1(self):
 20 |         path = os.path.join(test_dir_path, 'test_datasets')
 21 |         a_dataset = 'sample_table_small.csv'
 22 |         with io.open(
 23 |                 os.path.expanduser(os.path.join(path, a_dataset)), encoding="utf8") as f:
 24 |             header = next(unicode_csv_reader(f))
 25 |         self.assertEqual(header, ['id', 'left_a', 'right_a', 'label'])
 26 |         id_attr = 'id'
 27 |         label_attr = 'label'
 28 |         left_prefix = 'left'
 29 |         right_prefix = 'right'
 30 |         _check_header(header, id_attr, left_prefix, right_prefix, label_attr, [])
 31 | 
 32 |     @raises(ValueError)
 33 |     def test_check_header_2(self):
 34 |         path = os.path.join(test_dir_path, 'test_datasets')
 35 |         a_dataset = 'sample_table_small.csv'
 36 |         with io.open(
 37 |                 os.path.expanduser(os.path.join(path, a_dataset)), encoding="utf8") as f:
 38 |             header = next(unicode_csv_reader(f))
 39 |         self.assertEqual(header, ['id', 'left_a', 'right_a', 'label'])
 40 |         id_attr = 'id'
 41 |         label_attr = 'label'
 42 |         left_prefix = 'left'
 43 |         right_prefix = 'bb'
 44 |         _check_header(header, id_attr, left_prefix, right_prefix, label_attr, [])
 45 | 
 46 |     @raises(ValueError)
 47 |     def test_check_header_3(self):
 48 |         path = os.path.join(test_dir_path, 'test_datasets')
 49 |         a_dataset = 'sample_table_small.csv'
 50 |         with io.open(
 51 |                 os.path.expanduser(os.path.join(path, a_dataset)), encoding="utf8") as f:
 52 |             header = next(unicode_csv_reader(f))
 53 |         self.assertEqual(header, ['id', 'left_a', 'right_a', 'label'])
 54 |         id_attr = 'id'
 55 |         label_attr = 'label'
 56 |         left_prefix = 'aa'
 57 |         right_prefix = 'right'
 58 |         _check_header(header, id_attr, left_prefix, right_prefix, label_attr, [])
 59 | 
 60 |     @raises(ValueError)
 61 |     def test_check_header_5(self):
 62 |         path = os.path.join(test_dir_path, 'test_datasets')
 63 |         a_dataset = 'sample_table_small.csv'
 64 |         with io.open(
 65 |                 os.path.expanduser(os.path.join(path, a_dataset)), encoding="utf8") as f:
 66 |             header = next(unicode_csv_reader(f))
 67 |         self.assertEqual(header, ['id', 'left_a', 'right_a', 'label'])
 68 |         id_attr = 'id'
 69 |         label_attr = ''
 70 |         left_prefix = 'left'
 71 |         right_prefix = 'right'
 72 |         _check_header(header, id_attr, left_prefix, right_prefix, label_attr, [])
 73 | 
 74 |     @raises(AssertionError)
 75 |     def test_check_header_6(self):
 76 |         path = os.path.join(test_dir_path, 'test_datasets')
 77 |         a_dataset = 'sample_table_small.csv'
 78 |         with io.open(
 79 |                 os.path.expanduser(os.path.join(path, a_dataset)), encoding="utf8") as f:
 80 |             header = next(unicode_csv_reader(f))
 81 |         self.assertEqual(header, ['id', 'left_a', 'right_a', 'label'])
 82 |         header.pop(1)
 83 |         id_attr = 'id'
 84 |         label_attr = 'label'
 85 |         left_prefix = 'left'
 86 |         right_prefix = 'right'
 87 |         _check_header(header, id_attr, left_prefix, right_prefix, label_attr, [])
 88 | 
 89 | 
 90 | class MakeFieldsTestCases(unittest.TestCase):
 91 | 
 92 |     def test_make_fields_1(self):
 93 |         path = os.path.join(test_dir_path, 'test_datasets')
 94 |         a_dataset = 'sample_table_large.csv'
 95 |         with io.open(
 96 |                 os.path.expanduser(os.path.join(path, a_dataset)), encoding="utf8") as f:
 97 |             header = next(unicode_csv_reader(f))
 98 |         self.assertEqual(header, [
 99 |             '_id', 'ltable_id', 'rtable_id', 'label', 'ltable_Song_Name',
100 |             'ltable_Artist_Name', 'ltable_Price', 'ltable_Released', 'rtable_Song_Name',
101 |             'rtable_Artist_Name', 'rtable_Price', 'rtable_Released'
102 |         ])
103 |         id_attr = '_id'
104 |         label_attr = 'label'
105 |         fields = _make_fields(header, id_attr, label_attr, ['ltable_id', 'rtable_id'],
106 |                               True, 'nltk', True)
107 |         self.assertEqual(len(fields), 12)
108 |         counter = {}
109 |         for tup in fields:
110 |             if tup[1] not in counter:
111 |                 counter[tup[1]] = 0
112 |             counter[tup[1]] += 1
113 |         self.assertEqual(sorted(list(counter.values())), [1, 1, 2, 8])
114 | 
115 | 
116 | class ProcessTestCases(unittest.TestCase):
117 | 
118 |     def test_process_1(self):
119 |         vectors_cache_dir = '.cache'
120 |         if os.path.exists(vectors_cache_dir):
121 |             shutil.rmtree(vectors_cache_dir)
122 | 
123 |         data_dir = os.path.join(test_dir_path, 'test_datasets')
124 |         train_path = 'sample_table_large.csv'
125 |         valid_path = 'sample_table_large.csv'
126 |         test_path = 'sample_table_large.csv'
127 |         cache_file = 'cache.pth'
128 |         cache_path = os.path.join(data_dir, cache_file)
129 |         if os.path.exists(cache_path):
130 |             os.remove(cache_path)
131 | 
132 |         pathdir = os.path.abspath(os.path.join(test_dir_path, 'test_datasets'))
133 |         filename = 'fasttext_sample.vec.zip'
134 |         url_base = urljoin('file:', pathname2url(pathdir)) + os.path.sep
135 |         ft = FastText(filename, url_base=url_base, cache=vectors_cache_dir)
136 | 
137 |         process(
138 |             data_dir,
139 |             train=train_path,
140 |             validation=valid_path,
141 |             test=test_path,
142 |             id_attr='_id',
143 |             left_prefix='ltable_',
144 |             right_prefix='rtable_',
145 |             cache=cache_file,
146 |             embeddings=ft,
147 |             embeddings_cache_path='')
148 | 
149 |         if os.path.exists(vectors_cache_dir):
150 |             shutil.rmtree(vectors_cache_dir)
151 | 
152 |         if os.path.exists(cache_path):
153 |             os.remove(cache_path)
154 | 
155 | class ProcessUnlabeledTestCases(unittest.TestCase):
156 |     def test_process_unlabeled_1(self):
157 |         vectors_cache_dir = '.cache'
158 |         if os.path.exists(vectors_cache_dir):
159 |             shutil.rmtree(vectors_cache_dir)
160 | 
161 |         data_cache_path = os.path.join(test_dir_path, 'test_datasets',
162 |             'cacheddata.pth')
163 |         if os.path.exists(data_cache_path):
164 |             os.remove(data_cache_path)
165 | 
166 |         vec_dir = os.path.abspath(os.path.join(test_dir_path, 'test_datasets'))
167 |         filename = 'fasttext_sample.vec.zip'
168 |         url_base = urljoin('file:', pathname2url(vec_dir)) + os.path.sep
169 |         ft = FastText(filename, url_base=url_base, cache=vectors_cache_dir)
170 | 
171 |         train, valid, test = process(
172 |             path=os.path.join(test_dir_path, 'test_datasets'),
173 |             train='test_train.csv',
174 |             validation='test_valid.csv',
175 |             test='test_test.csv',
176 |             id_attr='id',
177 |             ignore_columns=('left_id', 'right_id'),
178 |             embeddings=ft,
179 |             embeddings_cache_path='',
180 |             pca=True)
181 | 
182 |         model_save_path = 'sif_model.pth'
183 |         model = MatchingModel(attr_summarizer='sif')
184 |         model.run_train(
185 |             train,
186 |             valid,
187 |             epochs=1,
188 |             batch_size=8,
189 |             best_save_path= model_save_path,
190 |             pos_neg_ratio=3)
191 | 
192 |         test_unlabeled = process_unlabeled(
193 |             path=os.path.join(test_dir_path, 'test_datasets', 'test_test.csv'),
194 |             trained_model=model,
195 |             ignore_columns=('left_id', 'right_id'))
196 | 
197 |         self.assertEqual(test_unlabeled.all_text_fields, test.all_text_fields)
198 | 
199 |         if os.path.exists(model_save_path):
200 |             os.remove(model_save_path)
201 | 
202 |         if os.path.exists(data_cache_path):
203 |             os.remove(data_cache_path)
204 | 
205 |         if os.path.exists(vectors_cache_dir):
206 |             shutil.rmtree(vectors_cache_dir)
207 | 


--------------------------------------------------------------------------------