├── .github
└── workflows
│ └── pythonpackage.yml
├── .gitignore
├── Dockerfile
├── LICENSE
├── Makefile
├── README.md
├── bin
├── fetch_lample_embedding.py
├── parse_CoNLL2003.py
└── prepare_embeddings.py
├── config
├── external
│ ├── conll2003.glove.yaml
│ ├── conll2003.skipngram.yaml
│ ├── conll2003.yaml
│ └── cpc1.0.yaml
├── iteration
│ ├── long.yaml
│ ├── mid.yaml
│ └── short.yaml
├── model
│ ├── lample.yaml
│ └── simple-lstm-crf.yaml
├── optimizer
│ ├── adabound.yaml
│ ├── adam.yaml
│ └── sgd_with_clipping.yaml
├── preprocessing
│ └── znorm.yaml
└── training
│ ├── conll2003.lample.glove.yaml
│ ├── conll2003.lample.skipngram.adabound.yaml
│ ├── conll2003.lample.skipngram.yaml
│ ├── conll2003.lample.yaml
│ ├── conll2003.simple-lstm-crf.yaml
│ └── cpc1.0.lample.yaml
├── data
└── .gitkeep
├── docs
└── CPC1.0.md
├── model
└── .gitkeep
├── poetry.lock
├── pyner
├── __init__.py
├── named_entity
│ ├── __init__.py
│ ├── corpus.py
│ ├── dataset.py
│ ├── evaluator.py
│ ├── inference.py
│ ├── nn.py
│ ├── recognizer.py
│ ├── tagger.py
│ └── train.py
└── util
│ ├── __init__.py
│ ├── config.py
│ ├── deterministic.py
│ ├── iterator.py
│ ├── metric.py
│ ├── optimizer.py
│ └── vocab.py
├── pyproject.toml
├── setup.cfg
├── static
└── image
│ └── pyner.png
└── tests
├── named_entity
├── test_corpus.py
└── test_dataset.py
└── util
├── test_data
└── v1
│ └── vocab.words.txt
└── test_vocab.py
/.github/workflows/pythonpackage.yml:
--------------------------------------------------------------------------------
1 | name: Python package
2 |
3 | on: [push]
4 |
5 | jobs:
6 | build:
7 |
8 | runs-on: ubuntu-latest
9 | strategy:
10 | max-parallel: 4
11 | matrix:
12 | python-version: [3.6, 3.7]
13 |
14 | steps:
15 | - uses: actions/checkout@v1
16 | - name: Set up Python ${{ matrix.python-version }}
17 | uses: actions/setup-python@v1
18 | with:
19 | python-version: ${{ matrix.python-version }}
20 | - name: Install dependencies
21 | run: |
22 | python -m pip install --upgrade pip
23 | pip install .
24 | - name: Lint with flake8
25 | run: |
26 | pip install flake8
27 | # stop the build if there are Python syntax errors or undefined names
28 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
29 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
30 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
31 | - name: Test with pytest
32 | run: |
33 | pip install pytest
34 | pytest
35 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # emacs
2 | *~
3 |
4 | # compiled
5 | *.pyc
6 |
7 | # IntelliJ
8 | .idea
9 | *.iml
10 |
11 | # model
12 | model
13 |
14 | # data
15 | data
16 |
17 | # notebook
18 | notebook/.ipynb_checkpoints
19 |
20 | # kytea model, which are now included
21 | # *.knm
22 |
23 | /data/external
24 | /prediction
25 |
26 | .*
27 | !.gitignore
28 |
29 | build
30 | dist
31 | *.egg-info
32 |
33 | /script/conlleval
34 |
35 | /cover
36 |
37 | /tool/conlleval
38 |
39 | /model
40 | /model.*
41 |
--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM nvidia/cuda:10.2-cudnn8-devel-ubuntu18.04
2 |
3 | ENV LANG C.UTF-8
4 | ENV LC_ALL C.UTF-8
5 | ENV PYTHONIOENCODING UTF-8
6 |
7 | RUN apt-get update && apt-get install -y --no-install-recommends \
8 | build-essential language-pack-ja \
9 | python3 python3-dev python3-pip python3-setuptools \
10 | curl tmux sudo git && \
11 | apt-get clean && \
12 | rm -rf /var/lib/apt/lists/*
13 |
14 | RUN pip3 install pip --upgrade
15 | RUN pip3 install setuptools --upgrade
16 | RUN pip3 install poetry
17 |
18 | WORKDIR /work
19 | COPY pyproject.toml .
20 | COPY poetry.lock .
21 | COPY pyner ./pyner
22 | RUN poetry install
23 |
24 | WORKDIR /tmp
25 | RUN curl https://www.clips.uantwerpen.be/conll2000/chunking/conlleval.txt > conlleval
26 | RUN chmod 777 conlleval
27 |
28 | WORKDIR /work
29 | COPY data ./data
30 | COPY config ./config
31 |
32 | COPY bin ./bin
33 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2018-2019 himkt
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | export TAG='himkt/pyner:latest'
2 |
3 |
4 | .PHONY: build start test lint
5 |
6 | build:
7 | docker build -t $(TAG) .
8 |
9 | start:
10 | docker run --gpus all --rm -it $(TAG)
11 |
12 | test:
13 | poetry run python -m pytest tests
14 |
15 | lint:
16 | poetry run flake8 pyner
17 |
18 | get-glove:
19 | mkdir -p data/external/GloveEmbeddings
20 | mkdir -p data/processed/GloveEmbeddings
21 | cd data/external/GloveEmbeddings && wget http://nlp.stanford.edu/data/glove.6B.zip
22 | cd data/external/GloveEmbeddings && unzip glove.6B.zip
23 | cd data/external/GloveEmbeddings && rm glove.6B.zip
24 | poetry run python bin/prepare_embeddings.py \
25 | data/external/GloveEmbeddings/glove.6B.100d.txt \
26 | data/processed/GloveEmbeddings/glove.6B.100d \
27 | --format glove
28 |
29 | get-lample:
30 | rm -rf data/external/GloveEmbeddings
31 | mkdir -p data/external/LampleEmbeddings
32 | mkdir -p data/processed/LampleEmbeddings
33 | poetry run python bin/fetch_lample_embedding.py
34 | poetry run python bin/prepare_embeddings.py \
35 | data/external/LampleEmbeddings/skipngram_100d.txt \
36 | data/processed/LampleEmbeddings/skipngram_100d \
37 | --format word2vec
38 |
39 | get-conlleval:
40 | curl https://www.clips.uantwerpen.be/conll2000/chunking/conlleval.txt > conlleval
41 | chmod 777 conlleval
42 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | 