lookup(byte[] text, int offset);
29 |
30 | int getWordId(String headword, short posId, String readingForm);
31 |
32 | /**
33 | * Returns the left-ID of the morpheme specified by the word ID.
34 | *
35 | *
36 | * when the word ID is out of range, the behavior is undefined.
37 | *
38 | * @param wordId
39 | * the word ID of the morpheme
40 | * @return the left-ID of the morpheme
41 | */
42 | short getLeftId(int wordId);
43 |
44 | /**
45 | * Returns the right-ID of the morpheme specified by the word ID.
46 | *
47 | *
48 | * when the word ID is out of range, the behavior is undefined.
49 | *
50 | * @param wordId
51 | * the word ID of the morpheme
52 | * @return the right-ID of the morpheme.
53 | */
54 | short getRightId(int wordId);
55 |
56 | /**
57 | * Returns the word occurrence cost of the morpheme specified by the word ID.
58 | *
59 | *
60 | * when the word ID is out of range, the behavior is undefined.
61 | *
62 | * @param wordId
63 | * the word ID of the morpheme
64 | * @return the word occurrence cost
65 | */
66 | short getCost(int wordId);
67 |
68 | /**
69 | * Returns the informations of the morpheme specified by the word ID.
70 | *
71 | *
72 | * when the word ID is out of range, the behavior is undefined.
73 | *
74 | * @param wordId
75 | * the word ID of the morpheme
76 | * @return the informations of the morpheme
77 | * @see WordInfo
78 | */
79 | WordInfo getWordInfo(int wordId);
80 |
81 | /**
82 | * Returns the ID of the dictionary containing the morpheme specified by the
83 | * word ID.
84 | *
85 | * If the morpheme is in the system dictionary, it returns {@code 0}.
86 | *
87 | * @param wordId
88 | * the word ID of the morpheme
89 | * @return the dictionary ID
90 | * @deprecated use {@link WordId#dic(int)}
91 | */
92 | @Deprecated
93 | default int getDictionaryId(int wordId) {
94 | return WordId.dic(wordId);
95 | }
96 |
97 | /**
98 | * Returns the number of morphemes in the dictionary.
99 | *
100 | * @return the number of morphemes
101 | */
102 | int size();
103 | }
104 |
--------------------------------------------------------------------------------
/src/main/java/com/worksap/nlp/sudachi/StringUtil.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2017-2022 Works Applications Co., Ltd.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.worksap.nlp.sudachi;
18 |
19 | import java.io.IOException;
20 | import java.io.InputStream;
21 | import java.io.InputStreamReader;
22 | import java.net.URL;
23 | import java.nio.ByteBuffer;
24 | import java.nio.ByteOrder;
25 | import java.nio.CharBuffer;
26 | import java.nio.charset.StandardCharsets;
27 | import java.nio.file.Files;
28 | import java.nio.file.Path;
29 | import java.util.Arrays;
30 |
31 | public class StringUtil {
32 | private StringUtil() {
33 | }
34 |
35 | public static String readFully(URL url) throws IOException {
36 | try (InputStream inputStream = url.openStream()) {
37 | return readFully(inputStream);
38 | }
39 | }
40 |
41 | public static String readFully(Path path) throws IOException {
42 | try (InputStream is = Files.newInputStream(path)) {
43 | return readFully(is);
44 | }
45 | }
46 |
47 | public static String readFully(InputStream stream) throws IOException {
48 | InputStreamReader isr = new InputStreamReader(stream, StandardCharsets.UTF_8);
49 | StringBuilder sb = new StringBuilder();
50 | CharBuffer cb = CharBuffer.allocate(1024);
51 | while (isr.read(cb) != -1) {
52 | cb.flip();
53 | sb.append(cb);
54 | cb.clear();
55 | }
56 | return sb.toString();
57 | }
58 |
59 | public static ByteBuffer readAllBytes(URL url) throws IOException {
60 | return readAllBytes(url, ByteOrder.LITTLE_ENDIAN);
61 | }
62 |
63 | public static ByteBuffer readAllBytes(URL url, ByteOrder order) throws IOException {
64 | try (InputStream is = url.openStream()) {
65 | return readAllBytes(is, order);
66 | }
67 | }
68 |
69 | public static ByteBuffer readAllBytes(InputStream inputStream) throws IOException {
70 | return readAllBytes(inputStream, ByteOrder.LITTLE_ENDIAN);
71 | }
72 |
73 | public static ByteBuffer readAllBytes(InputStream inputStream, ByteOrder order) throws IOException {
74 | byte[] buffer = new byte[inputStream.available() + 1024];
75 | int offset = 0;
76 |
77 | while (true) {
78 | int nread = inputStream.read(buffer, offset, buffer.length - offset);
79 | if (nread >= 0) {
80 | offset += nread;
81 | if (offset == buffer.length) {
82 | buffer = Arrays.copyOf(buffer, buffer.length * 2);
83 | }
84 | } else {
85 | break;
86 | }
87 | }
88 | ByteBuffer bbuf = ByteBuffer.wrap(buffer);
89 | bbuf.limit(offset);
90 | bbuf.order(order);
91 | return bbuf;
92 | }
93 | }
94 |
--------------------------------------------------------------------------------
/src/test/dict/lex.csv:
--------------------------------------------------------------------------------
1 | た,1,1,8729,た,助動詞,*,*,*,助動詞-タ,終止形-一般,タ,た,*,A,*,*,*,*
2 | に,2,2,11406,に,助詞,接続助詞,*,*,*,*,ニ,に,*,A,*,*,*,*
3 | に,3,3,4481,に,助詞,格助詞,*,*,*,*,ニ,に,*,A,*,*,*,*
4 | 京都,6,6,5293,京都,名詞,固有名詞,地名,一般,*,*,キョウト,京都,*,A,*,*,*,1/5
5 | 東,7,7,4675,東,名詞,普通名詞,一般,*,*,*,ヒガシ,東,*,A,*,*,*,*
6 | 東京,6,6,2816,東京,名詞,固有名詞,地名,一般,*,*,トウキョウ,東京,*,A,*,*,*,*
7 | 東京都,6,8,5320,東京都,名詞,固有名詞,地名,一般,*,*,トウキョウト,東京都,*,B,5/9,*,5/9,*
8 | 行く,4,4,5105,行く,動詞,非自立可能,*,*,五段-カ行,終止形-一般,イク,行く,*,A,*,*,*,*
9 | 行っ,5,5,5122,行っ,動詞,非自立可能,*,*,五段-カ行,連用形-促音便,イッ,行く,7,A,*,*,*,*
10 | 都,8,8,2914,都,名詞,普通名詞,一般,*,*,*,ト,都,*,A,*,*,*,*
11 | アイ,7,7,4675,アイ,名詞,普通名詞,一般,*,*,*,アイ,アイ,*,A,*,*,*,*
12 | アイウ,7,7,4675,アイウ,名詞,普通名詞,一般,*,*,*,アイウ,アイウ,*,A,*,*,*,*
13 | アイアイウ,6,6,32766,アイウ,名詞,固有名詞,地名,一般,*,*,アイアイウ,アイアイウ,*,A,*,*,*,*
14 | 0,9,9,2478,0,名詞,数詞,*,*,*,*,ゼロ,0,*,A,*,*,*,*
15 | 1,9,9,2478,1,名詞,数詞,*,*,*,*,イチ,1,*,A,*,*,*,*
16 | 2,9,9,2478,2,名詞,数詞,*,*,*,*,ニ,2,*,A,*,*,*,*
17 | 3,9,9,2478,3,名詞,数詞,*,*,*,*,サン,3,*,A,*,*,*,*
18 | 4,9,9,2478,4,名詞,数詞,*,*,*,*,ヨン,4,*,A,*,*,*,*
19 | 5,9,9,2478,5,名詞,数詞,*,*,*,*,ゴ,5,*,A,*,*,*,*
20 | 6,9,9,2478,6,名詞,数詞,*,*,*,*,ロク,6,*,A,*,*,*,*
21 | 7,9,9,2478,7,名詞,数詞,*,*,*,*,ナナ,7,*,A,*,*,*,*
22 | 8,9,9,2478,8,名詞,数詞,*,*,*,*,ハチ,8,*,A,*,*,*,*
23 | 9,9,9,2478,9,名詞,数詞,*,*,*,*,キュウ,9,*,A,*,*,*,*
24 | 〇,9,9,2478,〇,名詞,数詞,*,*,*,*,ゼロ,〇,*,A,*,*,*,*
25 | 一,9,9,2478,一,名詞,数詞,*,*,*,*,イチ,一,*,A,*,*,*,*
26 | 二,9,9,2478,二,名詞,数詞,*,*,*,*,ニ,二,*,A,*,*,*,*
27 | 三,9,9,2478,三,名詞,数詞,*,*,*,*,サン,三,*,A,*,*,*,*
28 | 四,9,9,2478,四,名詞,数詞,*,*,*,*,ヨン,四,*,A,*,*,*,*
29 | 五,9,9,2478,五,名詞,数詞,*,*,*,*,ゴ,五,*,A,*,*,*,*
30 | 六,9,9,2478,六,名詞,数詞,*,*,*,*,ロク,六,*,A,*,*,*,*
31 | 七,9,9,2478,七,名詞,数詞,*,*,*,*,ナナ,七,*,A,*,*,*,*
32 | 八,9,9,2478,八,名詞,数詞,*,*,*,*,ハチ,八,*,A,*,*,*,*
33 | 九,9,9,2478,九,名詞,数詞,*,*,*,*,キュウ,九,*,A,*,*,*,*
34 | 六三四,6,6,0,六三四,名詞,固有名詞,地名,一般,*,*,ムサシ,六三四,*,A,*,*,*,*
35 | いく,4,4,5105,いく,動詞,非自立可能,*,*,五段-カ行,終止形-一般,イク,行く,*,A,*,*,*,*
36 | いっ,5,5,5122,いっ,動詞,非自立可能,*,*,五段-カ行,連用形-促音便,イッ,行く,34,A,*,*,*,*
37 | 012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789,9,9,2478,012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789,名詞,数詞,*,*,*,*,ゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウ,012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789,*,A,*,*,*,*
38 | 特a,8,8,2914,特A,名詞,普通名詞,一般,*,*,*,トクエー,特A,*,A,*,*,*,*
39 | な。な,8,8,2914,な。な,名詞,普通名詞,一般,*,*,*,ナナ,な。な,*,A,*,*,*,*
--------------------------------------------------------------------------------
/src/main/java/com/worksap/nlp/sudachi/dictionary/POS.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2021 Works Applications Co., Ltd.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.worksap.nlp.sudachi.dictionary;
18 |
19 | import java.util.AbstractList;
20 | import java.util.Arrays;
21 | import java.util.List;
22 |
23 | /**
24 | * Part-of-Speech
25 | *
26 | * Sudachi POS are 6-component and consist of: 4 layers of POS tags, conjugation
27 | * type, conjugation form.
28 | */
29 | public final class POS extends AbstractList {
30 | public final static int DEPTH = 6;
31 | public final static int MAX_COMPONENT_LENGTH = 127;
32 | private final String[] elems;
33 |
34 | /**
35 | * @param elems
36 | * non-null string array of exactly six elements
37 | */
38 | public POS(String... elems) {
39 | if (elems == null) {
40 | throw new IllegalArgumentException("pos must not be null");
41 | }
42 | if (elems.length != DEPTH) {
43 | throw new IllegalArgumentException(String.format("pos must have exactly 6 elements, was %s: %s",
44 | elems.length, String.join(",", elems)));
45 | }
46 | for (String e : elems) {
47 | if (e == null) {
48 | throw new IllegalArgumentException("POS components can't be null");
49 | }
50 |
51 | if (e.length() > MAX_COMPONENT_LENGTH) {
52 | throw new IllegalArgumentException(
53 | String.format("POS component had length (%d) > %d: %s", e.length(), MAX_COMPONENT_LENGTH, e));
54 | }
55 | }
56 | this.elems = elems;
57 | }
58 |
59 | /**
60 | * Creates new POS instance from elements. Elements must be 6-length string
61 | * list.
62 | *
63 | * @param elems
64 | * POS object elements
65 | */
66 | public POS(List elems) {
67 | this(elems.toArray(new String[0]));
68 | }
69 |
70 | @Override
71 | public String get(int i) {
72 | return elems[i];
73 | }
74 |
75 | @Override
76 | public int size() {
77 | return DEPTH;
78 | }
79 |
80 | @Override
81 | public boolean equals(Object o) {
82 | if (this == o)
83 | return true;
84 | if (o instanceof POS) {
85 | POS strings = (POS) o;
86 | return Arrays.equals(elems, strings.elems);
87 | }
88 | return super.equals(o);
89 | }
90 |
91 | @Override
92 | public int hashCode() {
93 | int result = 0xfeed;
94 | result = 31 * result + Arrays.hashCode(elems);
95 | return result;
96 | }
97 |
98 | @Override
99 | public String toString() {
100 | return String.join(",", elems);
101 | }
102 | }
103 |
--------------------------------------------------------------------------------
/src/main/java/com/worksap/nlp/sudachi/dictionary/build/Progress.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2021 Works Applications Co., Ltd.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.worksap.nlp.sudachi.dictionary.build;
18 |
19 | import java.time.Duration;
20 |
21 | public class Progress {
22 | private final static long MS_100 = 100_000_000L; // 100ms in nanos
23 | private final int maxUpdates;
24 | private final Callback callback;
25 | private float currentProgress;
26 | private long lastUpdate;
27 |
28 | public Progress(int maxUpdates, Callback callback) {
29 | this.maxUpdates = maxUpdates;
30 | this.callback = callback;
31 | }
32 |
33 | public void startBlock(String name, long start, Kind kind) {
34 | lastUpdate = start;
35 | callback.start(name, kind);
36 | currentProgress = step();
37 | }
38 |
39 | private float step() {
40 | return 1.0f / maxUpdates - 1e-6f;
41 | }
42 |
43 | /**
44 | * This function limits calls to the progress function
45 | *
46 | * @param cur
47 | * current state
48 | * @param max
49 | * maximum state
50 | */
51 | public void progress(long cur, long max) {
52 | double ratio = cur / (double) max;
53 | if (ratio > currentProgress) {
54 | if (ratio >= 1.0) {
55 | callback.progress(1.0f);
56 | currentProgress = Float.MAX_VALUE;
57 | }
58 |
59 | long curTime = System.nanoTime();
60 | if (curTime - lastUpdate > MS_100) {
61 | callback.progress((float) ratio);
62 | float step = step();
63 | double nsteps = ratio / step;
64 | currentProgress += Math.floor(nsteps) * step;
65 | assert ratio < currentProgress;
66 | lastUpdate = curTime;
67 | }
68 | }
69 | }
70 |
71 | public void endBlock(long size, long time) {
72 | callback.end(size, Duration.ofNanos(time));
73 | }
74 |
75 | public enum Kind {
76 | INPUT, OUTPUT
77 | }
78 |
79 | /**
80 | * Progress callback
81 | */
82 | public interface Callback {
83 | /**
84 | * This function will be called for each step at the beginning
85 | *
86 | * @param name
87 | * step name
88 | */
89 | default void start(String name, Kind kind) {
90 | }
91 |
92 | /**
93 | * This function will be called as progress is happening
94 | *
95 | * @param progress
96 | * ratio of the progress
97 | */
98 | void progress(float progress);
99 |
100 | default void end(long size, Duration time) {
101 | }
102 | }
103 | }
104 |
--------------------------------------------------------------------------------
/src/test/resources/dict/lex.csv:
--------------------------------------------------------------------------------
1 | た,1,1,8729,た,助動詞,*,*,*,助動詞-タ,終止形-一般,タ,た,*,A,*,*,*,*
2 | に,2,2,11406,に,助詞,接続助詞,*,*,*,*,ニ,に,*,A,*,*,*,*
3 | に,3,3,4481,に,助詞,格助詞,*,*,*,*,ニ,に,*,A,*,*,*,*
4 | 京都,6,6,5293,京都,名詞,固有名詞,地名,一般,*,*,キョウト,京都,*,A,*,*,*,1/5
5 | 東,7,7,4675,東,名詞,普通名詞,一般,*,*,*,ヒガシ,東,*,A,*,*,*,*
6 | 東京,6,6,2816,東京,名詞,固有名詞,地名,一般,*,*,トウキョウ,東京,*,A,*,*,*,*
7 | 東京都,6,8,5320,東京都,名詞,固有名詞,地名,一般,*,*,トウキョウト,東京都,*,B,5/9,*,5/9,*
8 | 行く,4,4,5105,行く,動詞,非自立可能,*,*,五段-カ行,終止形-一般,イク,行く,*,A,*,*,*,*
9 | 行っ,5,5,5122,行っ,動詞,非自立可能,*,*,五段-カ行,連用形-促音便,イッ,行く,7,A,*,*,*,*
10 | 都,8,8,2914,都,名詞,普通名詞,一般,*,*,*,ト,都,*,A,*,*,*,*
11 | アイ,7,7,4675,アイ,名詞,普通名詞,一般,*,*,*,アイ,アイ,*,A,*,*,*,*
12 | アイウ,7,7,4675,アイウ,名詞,普通名詞,一般,*,*,*,アイウ,アイウ,*,A,*,*,*,*
13 | アイアイウ,6,6,32766,アイアイウ,名詞,固有名詞,地名,一般,*,*,アイアイウ,アイアイウ,*,A,*,*,*,*
14 | 0,9,9,2478,0,名詞,数詞,*,*,*,*,ゼロ,0,*,A,*,*,*,*
15 | 1,9,9,2478,1,名詞,数詞,*,*,*,*,イチ,1,*,A,*,*,*,*
16 | 2,9,9,2478,2,名詞,数詞,*,*,*,*,ニ,2,*,A,*,*,*,*
17 | 3,9,9,2478,3,名詞,数詞,*,*,*,*,サン,3,*,A,*,*,*,*
18 | 4,9,9,2478,4,名詞,数詞,*,*,*,*,ヨン,4,*,A,*,*,*,*
19 | 5,9,9,2478,5,名詞,数詞,*,*,*,*,ゴ,5,*,A,*,*,*,*
20 | 6,9,9,2478,6,名詞,数詞,*,*,*,*,ロク,6,*,A,*,*,*,*
21 | 7,9,9,2478,7,名詞,数詞,*,*,*,*,ナナ,7,*,A,*,*,*,*
22 | 8,9,9,2478,8,名詞,数詞,*,*,*,*,ハチ,8,*,A,*,*,*,*
23 | 9,9,9,2478,9,名詞,数詞,*,*,*,*,キュウ,9,*,A,*,*,*,*
24 | 〇,9,9,2478,〇,名詞,数詞,*,*,*,*,ゼロ,〇,*,A,*,*,*,*
25 | 一,9,9,2478,一,名詞,数詞,*,*,*,*,イチ,一,*,A,*,*,*,*
26 | 二,9,9,2478,二,名詞,数詞,*,*,*,*,ニ,二,*,A,*,*,*,*
27 | 三,9,9,2478,三,名詞,数詞,*,*,*,*,サン,三,*,A,*,*,*,*
28 | 四,9,9,2478,四,名詞,数詞,*,*,*,*,ヨン,四,*,A,*,*,*,*
29 | 五,9,9,2478,五,名詞,数詞,*,*,*,*,ゴ,五,*,A,*,*,*,*
30 | 六,9,9,2478,六,名詞,数詞,*,*,*,*,ロク,六,*,A,*,*,*,*
31 | 七,9,9,2478,七,名詞,数詞,*,*,*,*,ナナ,七,*,A,*,*,*,*
32 | 八,9,9,2478,八,名詞,数詞,*,*,*,*,ハチ,八,*,A,*,*,*,*
33 | 九,9,9,2478,九,名詞,数詞,*,*,*,*,キュウ,九,*,A,*,*,*,*
34 | 六三四,6,6,0,六三四,名詞,固有名詞,地名,一般,*,*,ムサシ,六三四,*,A,*,*,*,*
35 | いく,4,4,5105,いく,動詞,非自立可能,*,*,五段-カ行,終止形-一般,イク,行く,*,A,*,*,*,*
36 | いっ,5,5,5122,いっ,動詞,非自立可能,*,*,五段-カ行,連用形-促音便,イッ,行く,34,A,*,*,*,*
37 | 012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789,9,9,-9000,012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789,名詞,数詞,*,*,*,*,ゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウゼロイチニサンヨンゴロクナナハチキュウ,012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789,*,A,*,*,*,*
38 | 特a,8,8,2914,特A,名詞,普通名詞,一般,*,*,*,トクエー,特A,*,A,*,*,*,*
39 | 隠し,-1,-1,0,隠し,名詞,普通名詞,一般,*,*,*,カクシ,隠し,*,A,*,*,*,*
40 | な。な,8,8,2914,な。な,名詞,普通名詞,一般,*,*,*,ナナ,な。な,*,C,11,11,*,*
41 |
--------------------------------------------------------------------------------
/src/main/java/com/worksap/nlp/sudachi/SentenceSplittingAnalysis.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2023 Works Applications Co., Ltd.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.worksap.nlp.sudachi;
18 |
19 | import com.worksap.nlp.sudachi.dictionary.LexiconSet;
20 | import com.worksap.nlp.sudachi.sentdetect.SentenceDetector;
21 |
22 | import java.util.ArrayList;
23 | import java.util.Iterator;
24 |
25 | /*internal*/ class SentenceSplittingAnalysis implements SentenceDetector.NonBreakCheker {
26 | private final SentenceDetector detector = new SentenceDetector();
27 |
28 | private final Tokenizer.SplitMode mode;
29 | private final JapaneseTokenizer tokenizer;
30 | final ArrayList result = new ArrayList<>();
31 |
32 | SentenceSplittingAnalysis(Tokenizer.SplitMode mode, JapaneseTokenizer tokenizer) {
33 | this.mode = mode;
34 | this.tokenizer = tokenizer;
35 | }
36 |
37 | UTF8InputText input;
38 | int bos;
39 |
40 | int tokenizeBuffer(CharSequence buffer) {
41 | UTF8InputText input = tokenizer.buildInputText(buffer);
42 | String normalized = input.getText();
43 | this.input = input;
44 |
45 | int bos = 0;
46 | int length;
47 |
48 | this.bos = bos;
49 | while ((length = detector.getEos(normalized, this)) > 0) {
50 | int eos = bos + length;
51 | if (eos < normalized.length()) {
52 | eos = input.getNextInOriginal(eos - 1);
53 | length = eos - bos;
54 | }
55 | UTF8InputText sentence = input.slice(bos, eos);
56 | result.add(tokenizer.tokenizeSentence(mode, sentence));
57 | normalized = normalized.substring(length);
58 | bos = eos;
59 | this.bos = bos;
60 | }
61 |
62 | // buffer is full, need to clean it up
63 | if (length < 0 && buffer.length() == -length) {
64 | result.add(tokenizer.tokenizeSentence(mode, input));
65 | return -length;
66 | }
67 |
68 | return length;
69 | }
70 |
71 | int bosPosition() {
72 | return input.textIndexToOriginalTextIndex(bos);
73 | }
74 |
75 | @Override
76 | public boolean hasNonBreakWord(int length) {
77 | UTF8InputText inp = input;
78 | int byteEOS = inp.getCodePointsOffsetLength(0, bos + length);
79 | byte[] bytes = inp.getByteText();
80 | LexiconSet lexicon = tokenizer.lexicon;
81 | for (int i = Math.max(0, byteEOS - 64); i < byteEOS; i++) {
82 | Iterator iterator = lexicon.lookup(bytes, i);
83 | while (iterator.hasNext()) {
84 | int[] r = iterator.next();
85 | int l = r[1];
86 | if (l > byteEOS || (l == byteEOS && bos + length - inp.modifiedOffset(i) > 1)) {
87 | return true;
88 | }
89 | }
90 | }
91 | return false;
92 | }
93 | }
94 |
--------------------------------------------------------------------------------
/src/test/java/com/worksap/nlp/sudachi/JoinKatakanaOovPluginTest.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2017-2022 Works Applications Co., Ltd.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.worksap.nlp.sudachi;
18 |
19 | import static org.junit.Assert.assertEquals;
20 | import static org.junit.Assert.assertFalse;
21 |
22 | import java.io.IOException;
23 | import java.util.List;
24 |
25 | import org.junit.Before;
26 | import org.junit.Test;
27 |
28 | public class JoinKatakanaOovPluginTest {
29 | JapaneseTokenizer tokenizer;
30 | JoinKatakanaOovPlugin plugin;
31 |
32 | @Before
33 | public void setUp() throws IOException {
34 | Dictionary dict = TestDictionary.INSTANCE.user1();
35 | tokenizer = (JapaneseTokenizer) dict.create();
36 | plugin = new JoinKatakanaOovPlugin();
37 | }
38 |
39 | @Test
40 | public void testKatakanaLength() {
41 | // アイ, アイウ in the dictionary
42 |
43 | plugin.minLength = 0;
44 | List path = getPath("アイアイウ");
45 | assertEquals(2, path.size());
46 |
47 | plugin.minLength = 1;
48 | path = getPath("アイアイウ");
49 | assertEquals(2, path.size());
50 |
51 | plugin.minLength = 2;
52 | path = getPath("アイアイウ");
53 | assertEquals(2, path.size());
54 |
55 | plugin.minLength = 3;
56 | path = getPath("アイアイウ");
57 | assertEquals(1, path.size());
58 | }
59 |
60 | @Test
61 | public void testPOS() {
62 | // アイアイウ is 名詞-固有名詞-地名-一般 in the dictionary
63 | plugin.minLength = 3;
64 | List path = getPath("アイアイウ");
65 | assertEquals(1, path.size());
66 | assertFalse(path.get(0).isOOV()); // use the word in dictionary
67 | }
68 |
69 | @Test
70 | public void testStartWithMiddle() {
71 | plugin.minLength = 3;
72 | List path = getPath("アイウアイアイウ");
73 | assertEquals(1, path.size());
74 | }
75 |
76 | @Test
77 | public void testStartWithTail() {
78 | plugin.minLength = 3;
79 | List path = getPath("アイウアイウアイ");
80 | assertEquals(1, path.size());
81 | }
82 |
83 | @Test
84 | public void testWithNOOOVBOW() {
85 | plugin.minLength = 3;
86 | List path = getPath("ァアイアイウ");
87 | assertEquals(2, path.size());
88 | assertEquals("ァ", path.get(0).getWordInfo().getSurface());
89 |
90 | path = getPath("アイウァアイウ");
91 | assertEquals(1, path.size());
92 | }
93 |
94 | private List getPath(String text) {
95 | UTF8InputText input = new UTF8InputTextBuilder(text, tokenizer.grammar).build();
96 | LatticeImpl lattice = tokenizer.buildLattice(input);
97 | List path = lattice.getBestPath();
98 | plugin.rewrite(input, path, lattice);
99 | lattice.clear();
100 | return path;
101 | }
102 |
103 | }
104 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 |
2 | # Created by https://www.gitignore.io/api/osx,java,linux,maven,windows,eclipse
3 |
4 | ### Eclipse ###
5 |
6 | .metadata
7 | bin/
8 | tmp/
9 | *.tmp
10 | *.bak
11 | *.swp
12 | *~.nib
13 | local.properties
14 | .settings/
15 | .loadpath
16 | .recommenders
17 |
18 | # External tool builders
19 | .externalToolBuilders/
20 |
21 | # Locally stored "Eclipse launch configurations"
22 | *.launch
23 |
24 | # PyDev specific (Python IDE for Eclipse)
25 | *.pydevproject
26 |
27 | # CDT-specific (C/C++ Development Tooling)
28 | .cproject
29 |
30 | # Java annotation processor (APT)
31 | .factorypath
32 |
33 | # PDT-specific (PHP Development Tools)
34 | .buildpath
35 |
36 | # sbteclipse plugin
37 | .target
38 |
39 | # Tern plugin
40 | .tern-project
41 |
42 | # TeXlipse plugin
43 | .texlipse
44 |
45 | # STS (Spring Tool Suite)
46 | .springBeans
47 |
48 | # Code Recommenders
49 | .recommenders/
50 |
51 | # Scala IDE specific (Scala & Java development for Eclipse)
52 | .cache-main
53 | .scala_dependencies
54 | .worksheet
55 |
56 | ### Eclipse Patch ###
57 | # Eclipse Core
58 | .project
59 |
60 | # JDT-specific (Eclipse Java Development Tools)
61 | .classpath
62 |
63 | ### IntelliJ-based IDEs
64 |
65 | .idea/
66 |
67 | ### Java ###
68 | # Compiled class file
69 | *.class
70 |
71 | # Log file
72 | *.log
73 |
74 | # BlueJ files
75 | *.ctxt
76 |
77 | # Mobile Tools for Java (J2ME)
78 | .mtj.tmp/
79 |
80 | # Package Files #
81 | *.jar
82 | *.war
83 | *.ear
84 | *.zip
85 | *.tar.gz
86 | *.rar
87 |
88 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
89 | hs_err_pid*
90 |
91 | ### Linux ###
92 | *~
93 |
94 | # temporary files which can be created if a process still has a handle open of a deleted file
95 | .fuse_hidden*
96 |
97 | # KDE directory preferences
98 | .directory
99 |
100 | # Linux trash folder which might appear on any partition or disk
101 | .Trash-*
102 |
103 | # .nfs files are created when an open file is removed but is still being accessed
104 | .nfs*
105 |
106 | ### Maven ###
107 | target/
108 | pom.xml.tag
109 | pom.xml.releaseBackup
110 | pom.xml.versionsBackup
111 | pom.xml.next
112 | release.properties
113 | dependency-reduced-pom.xml
114 | buildNumber.properties
115 | .mvn/timing.properties
116 | settings.xml
117 |
118 | # Avoid ignoring Maven wrapper jar file (.jar files are usually ignored)
119 | !/.mvn/wrapper/maven-wrapper.jar
120 |
121 | ### GRADLE ###
122 |
123 | build/
124 | .gradle/
125 | !gradle/wrapper/gradle-wrapper.jar
126 | out/
127 |
128 | ### OSX ###
129 | *.DS_Store
130 | .AppleDouble
131 | .LSOverride
132 |
133 | # Icon must end with two \r
134 | Icon
135 |
136 | # Thumbnails
137 | ._*
138 |
139 | # Files that might appear in the root of a volume
140 | .DocumentRevisions-V100
141 | .fseventsd
142 | .Spotlight-V100
143 | .TemporaryItems
144 | .Trashes
145 | .VolumeIcon.icns
146 | .com.apple.timemachine.donotpresent
147 |
148 | # Directories potentially created on remote AFP share
149 | .AppleDB
150 | .AppleDesktop
151 | Network Trash Folder
152 | Temporary Items
153 | .apdisk
154 |
155 | ### Windows ###
156 | # Windows thumbnail cache files
157 | Thumbs.db
158 | ehthumbs.db
159 | ehthumbs_vista.db
160 |
161 | # Folder config file
162 | Desktop.ini
163 |
164 | # Recycle Bin used on file shares
165 | $RECYCLE.BIN/
166 |
167 | # Windows Installer files
168 | *.cab
169 | *.msi
170 | *.msm
171 | *.msp
172 |
173 | # Windows shortcuts
174 | *.lnk
175 |
176 | # End of https://www.gitignore.io/api/osx,java,linux,maven,windows,eclipse
177 |
--------------------------------------------------------------------------------
/src/main/java/com/worksap/nlp/sudachi/ProlongedSoundMarkInputTextPlugin.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2021 Works Applications Co., Ltd.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.worksap.nlp.sudachi;
18 |
19 | import java.io.IOException;
20 | import java.util.HashSet;
21 | import java.util.Set;
22 | import java.util.List;
23 |
24 | import com.worksap.nlp.sudachi.dictionary.Grammar;
25 |
26 | /**
27 | * A plugin that rewrites the Katakana-Hiragana Prolonged Sound Mark (Chōonpu)
28 | * and similar symbols.
29 | *
30 | *
31 | * This plugin combines the continuous sequence of prolonged sound marks to 1
32 | * character.
33 | *
34 | *
35 | * {@link Dictionary} initialize this plugin with {@link Settings}. It can be
36 | * referred as {@link Plugin#settings}.
37 | *
38 | *
39 | * The following is an example of settings.
40 | *
41 | *
42 | * {@code
43 | * {
44 | * "class" : "com.worksap.nlp.sudachi.ProlongedSoundMarkInputTextPlugin",
45 | "prolongedSoundMarks": ["ー", "〜", "〰"],
46 | "replacementSymbol": "ー"
47 | * }
48 | * }
49 | *
50 | *
51 | * {@code prolongedSoundMarks} is the list of symbols to be combined.
52 | * {@code replacementSymbol} is the symbol for replacement, after combining
53 | * prolonged sound mark sequences.
54 | *
55 | *
56 | * With above setting example, the plugin rewrites input "エーービ〜〜〜シ〰〰〰〰" to
57 | * "エービーシー".
58 | */
59 | class ProlongedSoundMarkInputTextPlugin extends InputTextPlugin {
60 |
61 | private Set prolongedSoundMarkSet = new HashSet<>();
62 | private String replacementSymbol;
63 |
64 | @Override
65 | public void setUp(Grammar Grammar) throws IOException {
66 | List prolongedSoundMarkStrings = settings.getStringList("prolongedSoundMarks");
67 | for (String s : prolongedSoundMarkStrings) {
68 | prolongedSoundMarkSet.add(s.codePointAt(0));
69 | }
70 | replacementSymbol = settings.getString("replacementSymbol");
71 | }
72 |
73 | @Override
74 | public void rewrite(InputTextBuilder builder) {
75 | String text = builder.getText();
76 |
77 | int n = text.length();
78 | int offset = 0;
79 | int markStartIndex = n;
80 | boolean isProlongedSoundMark = false;
81 | for (int i = 0; i < n; i++) {
82 | int cp = text.codePointAt(i);
83 | if (!isProlongedSoundMark && prolongedSoundMarkSet.contains(cp)) {
84 | isProlongedSoundMark = true;
85 | markStartIndex = i;
86 | } else if (isProlongedSoundMark && !prolongedSoundMarkSet.contains(cp)) {
87 | if ((i - markStartIndex) > 1) {
88 | builder.replace(markStartIndex - offset, i - offset, replacementSymbol);
89 | offset += i - markStartIndex - 1;
90 | }
91 | isProlongedSoundMark = false;
92 | }
93 | }
94 | if (isProlongedSoundMark && (n - markStartIndex) > 1) {
95 | builder.replace(markStartIndex - offset, n - offset, replacementSymbol);
96 | }
97 | }
98 | }
99 |
--------------------------------------------------------------------------------
/src/main/java/com/worksap/nlp/sudachi/IOTools.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2023-2024 Works Applications Co., Ltd.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.worksap.nlp.sudachi;
18 |
19 | import java.io.IOException;
20 | import java.nio.CharBuffer;
21 |
22 | public class IOTools {
23 | private IOTools() {
24 | // forbid instantiation
25 | }
26 |
27 | /**
28 | * Read as much as possible from the readable to the result buffer. Use this to
29 | * make sure that the buffer is fulfilled or no text left unread.
30 | *
31 | * @param readable
32 | * input readable
33 | * @param result
34 | * buffer to read into
35 | * @return number of read characters
36 | * @throws IOException
37 | * when read operation fails
38 | */
39 | public static int readAsMuchAsCan(Readable readable, CharBuffer result) throws IOException {
40 | int totalRead = 0;
41 | while (result.hasRemaining()) {
42 | int read = readable.read(result);
43 | if (read < 0) {
44 | if (totalRead == 0) {
45 | return -1;
46 | } else {
47 | return totalRead;
48 | }
49 | }
50 | totalRead += read;
51 | }
52 | return totalRead;
53 | }
54 |
55 | /**
56 | * Wrapper class for Readable, that uses {@link #readAsMuchAsCan} to read and
57 | * guarantees that the last character read is not a high surrogate unless it is
58 | * the last one in the readable.
59 | */
60 | public static class SurrogateAwareReadable implements Readable {
61 | private Readable readable;
62 | char lastTrailingHighSurrogate;
63 |
64 | SurrogateAwareReadable(Readable input) {
65 | this.readable = input;
66 | }
67 |
68 | @Override
69 | public int read(CharBuffer cb) throws IOException {
70 | boolean trailingKept = false;
71 | if (lastTrailingHighSurrogate != 0) {
72 | cb.append(lastTrailingHighSurrogate);
73 | lastTrailingHighSurrogate = 0;
74 | trailingKept = true;
75 | }
76 |
77 | int nread = IOTools.readAsMuchAsCan(readable, cb);
78 | if (nread < 0) {
79 | if (!trailingKept) {
80 | return -1;
81 | }
82 | // the last char in the readable is a high surrogate and there is nothing we can
83 | // do.
84 | return 1;
85 | }
86 | if (trailingKept) {
87 | nread += 1;
88 | }
89 |
90 | char lastChar = cb.get(cb.position() - 1);
91 | if (Character.isHighSurrogate(lastChar)) {
92 | lastTrailingHighSurrogate = lastChar;
93 | cb.position(cb.position() - 1);
94 | nread -= 1;
95 | }
96 | return nread;
97 | }
98 | }
99 | }
100 |
--------------------------------------------------------------------------------
/src/main/java/com/worksap/nlp/sudachi/dictionary/build/Index.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2021 Works Applications Co., Ltd.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.worksap.nlp.sudachi.dictionary.build;
18 |
19 | import com.worksap.nlp.dartsclone.DoubleArray;
20 |
21 | import java.io.IOException;
22 | import java.nio.ByteBuffer;
23 | import java.nio.ByteOrder;
24 | import java.nio.charset.StandardCharsets;
25 | import java.util.*;
26 |
27 | /**
28 | * Dictionary Parts: Trie index and entry offsets
29 | */
30 | public class Index implements WriteDictionary {
31 | private final SortedMap> elements = new TreeMap<>((byte[] l, byte[] r) -> {
32 | int llen = l.length;
33 | int rlen = r.length;
34 | for (int i = 0; i < Math.min(llen, rlen); i++) {
35 | if (l[i] != r[i]) {
36 | return (l[i] & 0xff) - (r[i] & 0xff);
37 | }
38 | }
39 | return l.length - r.length;
40 | });
41 |
42 | private int count = 0;
43 |
44 | public int add(String key, int wordId) {
45 | byte[] bytes = key.getBytes(StandardCharsets.UTF_8);
46 | List entries = elements.computeIfAbsent(bytes, k -> new ArrayList<>());
47 | if (entries.size() >= 255) {
48 | throw new IllegalArgumentException(String.format("key %s has >= 255 entries in the dictionary", key));
49 | }
50 | entries.add(wordId);
51 | count += 1;
52 | return bytes.length;
53 | }
54 |
55 | public void writeTo(ModelOutput output) throws IOException {
56 | DoubleArray trie = new DoubleArray();
57 |
58 | int size = this.elements.size();
59 |
60 | byte[][] keys = new byte[size][];
61 | int[] values = new int[size];
62 | ByteBuffer wordIdTable = ByteBuffer.allocate(count * (4 + 2));
63 | wordIdTable.order(ByteOrder.LITTLE_ENDIAN);
64 |
65 | output.withSizedPart("WordId table", () -> {
66 | int i = 0;
67 | int numEntries = this.elements.entrySet().size();
68 | for (Map.Entry> entry : this.elements.entrySet()) {
69 | keys[i] = entry.getKey();
70 | values[i] = wordIdTable.position();
71 | i++;
72 | List wordIds = entry.getValue();
73 | wordIdTable.put((byte) wordIds.size());
74 | for (int wid : wordIds) {
75 | wordIdTable.putInt(wid);
76 | }
77 | output.progress(i, numEntries);
78 | }
79 | return wordIdTable.position() + 4;
80 | });
81 |
82 | DicBuffer buffer = new DicBuffer(4);
83 | output.withPart("double array Trie", () -> {
84 | trie.build(keys, values, output::progress);
85 | buffer.putInt(trie.size());
86 | buffer.consume(output::write);
87 | output.write(trie.byteArray());
88 | });
89 |
90 | buffer.putInt(wordIdTable.position());
91 | buffer.consume(output::write);
92 |
93 | wordIdTable.flip();
94 | output.write(wordIdTable);
95 | }
96 | }
97 |
--------------------------------------------------------------------------------
/src/main/java/com/worksap/nlp/sudachi/MorphemeImpl.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2017-2022 Works Applications Co., Ltd.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.worksap.nlp.sudachi;
18 |
19 | import java.util.List;
20 |
21 | import com.worksap.nlp.sudachi.dictionary.WordInfo;
22 |
23 | class MorphemeImpl implements Morpheme {
24 |
25 | final MorphemeList list;
26 | final int index;
27 | WordInfo wordInfo;
28 |
29 | MorphemeImpl(MorphemeList list, int index) {
30 | this.list = list;
31 | this.index = index;
32 | }
33 |
34 | @Override
35 | public int begin() {
36 | return list.getBegin(index);
37 | }
38 |
39 | @Override
40 | public int end() {
41 | return list.getEnd(index);
42 | }
43 |
44 | @Override
45 | public String surface() {
46 | return list.getSurface(index);
47 | }
48 |
49 | @Override
50 | public List partOfSpeech() {
51 | WordInfo wi = getWordInfo();
52 | return list.grammar.getPartOfSpeechString(wi.getPOSId());
53 | }
54 |
55 | @Override
56 | public short partOfSpeechId() {
57 | WordInfo wi = getWordInfo();
58 | return wi.getPOSId();
59 | }
60 |
61 | @Override
62 | public String dictionaryForm() {
63 | WordInfo wi = getWordInfo();
64 | return wi.getDictionaryForm();
65 | }
66 |
67 | @Override
68 | public String normalizedForm() {
69 | WordInfo wi = getWordInfo();
70 | return wi.getNormalizedForm();
71 | }
72 |
73 | @Override
74 | public String readingForm() {
75 | WordInfo wi = getWordInfo();
76 | return wi.getReadingForm();
77 | }
78 |
79 | @Override
80 | public List split(Tokenizer.SplitMode mode) {
81 | return list.split(mode, index);
82 | }
83 |
84 | @Override
85 | public boolean isOOV() {
86 | return list.isOOV(index);
87 | }
88 |
89 | @Override
90 | public int getWordId() {
91 | return list.getWordId(index);
92 | }
93 |
94 | @Override
95 | public int getDictionaryId() {
96 | return list.getDictionaryId(index);
97 | }
98 |
99 | @Override
100 | public int[] getSynonymGroupIds() {
101 | WordInfo wi = getWordInfo();
102 | return wi.getSynonymGoupIds();
103 | }
104 |
105 | WordInfo getWordInfo() {
106 | if (wordInfo == null) {
107 | wordInfo = list.getWordInfo(index);
108 | }
109 | return wordInfo;
110 | }
111 |
112 | @Override
113 | public String toString() {
114 | final StringBuilder sb = new StringBuilder("MorphemeImpl{");
115 | sb.append("begin=").append(begin());
116 | sb.append(", end=").append(end());
117 | sb.append(", surface=").append(surface());
118 | sb.append(", pos=").append(partOfSpeechId()).append('/').append(partOfSpeech());
119 | int wordId = getWordId();
120 | sb.append(", wid=(").append(WordId.dic(wordId)).append(',').append(WordId.word(wordId));
121 | sb.append(")}");
122 | return sb.toString();
123 | }
124 | }
125 |
--------------------------------------------------------------------------------
/src/test/java/com/worksap/nlp/sudachi/JapaneseDictionaryTest.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2017-2022 Works Applications Co., Ltd.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.worksap.nlp.sudachi;
18 |
19 | import static org.hamcrest.CoreMatchers.is;
20 | import static org.hamcrest.CoreMatchers.isA;
21 | import static org.hamcrest.CoreMatchers.notNullValue;
22 | import static org.hamcrest.MatcherAssert.assertThat;
23 |
24 | import java.io.IOException;
25 | import java.util.List;
26 |
27 | import org.junit.After;
28 | import org.junit.Before;
29 | import org.junit.Test;
30 |
31 | public class JapaneseDictionaryTest {
32 | Dictionary dict;
33 |
34 | @Before
35 | public void setUp() throws IOException {
36 | dict = TestDictionary.INSTANCE.user0();
37 | }
38 |
39 | @After
40 | public void tearDown() throws IOException {
41 | dict.close();
42 | }
43 |
44 | @Test
45 | public void create() {
46 | assertThat(dict.create(), isA(Tokenizer.class));
47 | }
48 |
49 | @Test
50 | public void getPartOfSpeechSize() {
51 | assertThat(dict.getPartOfSpeechSize(), is(8));
52 | }
53 |
54 | @Test
55 | public void getPartOfSpeechString() {
56 | List pos = dict.getPartOfSpeechString((short) 0);
57 | assertThat(pos, notNullValue());
58 | assertThat(pos.get(0), is("助動詞"));
59 | }
60 |
61 | @Test
62 | public void instantiateConfigWithoutCharDef() throws IOException {
63 | Config cfg = Config.fromClasspath("sudachi_minimum.json");
64 | cfg.systemDictionary(TestDictionary.INSTANCE.getSystemDict());
65 | try (JapaneseDictionary jd = (JapaneseDictionary) new DictionaryFactory().create(cfg)) {
66 | assertThat(jd, notNullValue());
67 | assertThat(jd.create(), notNullValue());
68 | }
69 | }
70 |
71 | private JapaneseDictionary makeDictionaryIncorrectly() throws IOException {
72 | Config cfg = Config.fromClasspath("sudachi_minimum.json");
73 | cfg.systemDictionary(TestDictionary.INSTANCE.getSystemDict());
74 | try (JapaneseDictionary jd = (JapaneseDictionary) new DictionaryFactory().create(cfg)) {
75 | return jd;
76 | }
77 | }
78 |
79 | @Test(expected = IllegalStateException.class)
80 | public void throwExceptionOnDictionaryUsageAfterClose() throws IOException {
81 | JapaneseDictionary dic = makeDictionaryIncorrectly();
82 | Tokenizer ignored = dic.create();
83 | }
84 |
85 | private Tokenizer makeTokenizerIncorrectly() throws IOException {
86 | Config cfg = Config.fromClasspath("sudachi_minimum.json");
87 | cfg.systemDictionary(TestDictionary.INSTANCE.getSystemDict());
88 | try (JapaneseDictionary jd = (JapaneseDictionary) new DictionaryFactory().create(cfg)) {
89 | return jd.create();
90 | }
91 | }
92 |
93 | @Test(expected = IllegalStateException.class)
94 | public void throwExceptionOnTokenizerUsageAfterClose() throws IOException {
95 | Tokenizer tok = makeTokenizerIncorrectly();
96 | tok.tokenize("a");
97 | }
98 |
99 | }
100 |
--------------------------------------------------------------------------------
/src/main/java/com/worksap/nlp/sudachi/dictionary/DoubleArrayLookup.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2022 Works Applications Co., Ltd.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.worksap.nlp.sudachi.dictionary;
18 |
19 | import java.nio.IntBuffer;
20 |
21 | /**
22 | * This class implements common prefix lookup in the double array with a
23 | * different API. It uses fields to return current values of end offset and a
24 | * value stored in trie to reduce GC pressure. It also modifies the hot loop to
25 | * reduce the number of non-elidable field writes.
26 | */
27 | public final class DoubleArrayLookup {
28 | private IntBuffer array;
29 | private byte[] key;
30 | private int limit;
31 | private int startOffset;
32 | private int offset;
33 | private int nodePos;
34 | private int nodeValue;
35 |
36 | public DoubleArrayLookup() {
37 | this(null);
38 | }
39 |
40 | public DoubleArrayLookup(IntBuffer array) {
41 | this.array = array;
42 | }
43 |
44 | public DoubleArrayLookup(IntBuffer array, byte[] key, int offset, int limit) {
45 | this(array);
46 | reset(key, offset, limit);
47 | }
48 |
49 | private static boolean hasLeaf(int unit) {
50 | return ((unit >>> 8) & 1) == 1;
51 | }
52 |
53 | private static int value(int unit) {
54 | return unit & ((1 << 31) - 1);
55 | }
56 |
57 | private static int label(int unit) {
58 | return unit & ((1 << 31) | 0xFF);
59 | }
60 |
61 | private static int offset(int unit) {
62 | return ((unit >>> 10) << ((unit & (1 << 9)) >>> 6));
63 | }
64 |
65 | public void setArray(IntBuffer array) {
66 | this.array = array;
67 | reset(this.key, this.startOffset, this.limit);
68 | }
69 |
70 | public void reset(byte[] key, int offset, int limit) {
71 | this.key = key;
72 | this.offset = offset;
73 | this.startOffset = offset;
74 | this.limit = limit;
75 | nodePos = 0;
76 | int unit = array.get(nodePos);
77 | nodePos ^= offset(unit);
78 | }
79 |
80 | public boolean next() {
81 | IntBuffer array = this.array;
82 | byte[] key = this.key;
83 | int nodePos = this.nodePos;
84 | int limit = this.limit;
85 |
86 | for (int offset = this.offset; offset < limit; ++offset) {
87 | int k = Byte.toUnsignedInt(key[offset]);
88 | nodePos ^= k;
89 | int unit = array.get(nodePos);
90 | if (label(unit) != k) {
91 | this.offset = limit; // no more loop
92 | this.nodePos = nodePos;
93 | return false;
94 | }
95 |
96 | nodePos ^= offset(unit);
97 | if (hasLeaf(unit)) {
98 | nodeValue = value(array.get(nodePos));
99 | this.offset = offset + 1;
100 | this.nodePos = nodePos;
101 | return true;
102 | }
103 | }
104 | return false;
105 | }
106 |
107 | public int getValue() {
108 | return nodeValue;
109 | }
110 |
111 | public int getOffset() {
112 | return offset;
113 | }
114 | }
115 |
--------------------------------------------------------------------------------
/src/main/java/com/worksap/nlp/sudachi/Dictionary.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2021 Works Applications Co., Ltd.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.worksap.nlp.sudachi;
18 |
19 | import com.worksap.nlp.sudachi.dictionary.POS;
20 |
21 | import java.io.IOException;
22 | import java.util.Arrays;
23 | import java.util.List;
24 | import java.util.function.Predicate;
25 |
26 | /**
27 | * A lexicon and a grammar for morphological analysis.
28 | *
29 | * This class requires a lot of memory. When using multiple analyzers, it is
30 | * recommended to generate only one instance of this class, and generate
31 | * multiple tokenizers.
32 | *
33 | * @see DictionaryFactory
34 | * @see Tokenizer
35 | * @see AutoCloseable
36 | */
37 | public interface Dictionary extends AutoCloseable {
38 |
39 | /**
40 | * Creates a tokenizer instance.
41 | *
42 | * @return a tokenizer
43 | */
44 | public Tokenizer create();
45 |
46 | @Override
47 | public void close() throws IOException;
48 |
49 | /**
50 | * Returns the number of types of part-of-speech.
51 | *
52 | * The IDs of part-of-speech are within the range of 0 to
53 | * {@code getPartOfSpeechSize() - 1}.
54 | *
55 | * @return the number of types of part-of-speech
56 | */
57 | public int getPartOfSpeechSize();
58 |
59 | /**
60 | * Returns the array of strings of part-of-speech name.
61 | *
62 | * The name is divided into layers.
63 | *
64 | * @param posId
65 | * the ID of the part-of-speech
66 | * @return the list of strings of part-of-speech name
67 | * @throws IndexOutOfBoundsException
68 | * if {@code posId} is out of the range
69 | */
70 | public List getPartOfSpeechString(short posId);
71 |
72 | /**
73 | * Create a POS matcher that will match any of POS for which the passed
74 | * predicate returns true. PosMatcher will be much faster than doing string
75 | * comparison on POS objects.
76 | *
77 | * @param predicate
78 | * returns true if the POS is needed
79 | * @return PosMatcher object that mirrors behavior of the predicate
80 | */
81 | PosMatcher posMatcher(Predicate predicate);
82 |
83 | /**
84 | * Create a POS matcher that will mirror matching behavior of passed list of
85 | * partially-defined POS.
86 | *
87 | * @param posList
88 | * list of partially defined part-of-speech objects
89 | * @return mirroring PosMatcher object
90 | * @see PartialPOS
91 | */
92 | default PosMatcher posMatcher(Iterable posList) {
93 | return posMatcher(posRepr -> {
94 | for (PartialPOS p : posList) {
95 | if (p.matches(posRepr)) {
96 | return true;
97 | }
98 | }
99 | return false;
100 | });
101 | }
102 |
103 | /**
104 | * Create a POS matcher that will mirror matching behavior of passed list of
105 | * partially-defined POS.
106 | *
107 | * @param posList
108 | * list of partially defined part-of-speech objects
109 | * @return mirroring PosMatcher object
110 | * @see PartialPOS
111 | */
112 | default PosMatcher posMatcher(PartialPOS... posList) {
113 | return posMatcher(Arrays.asList(posList));
114 | }
115 | }
116 |
--------------------------------------------------------------------------------
/src/main/java/com/worksap/nlp/sudachi/dictionary/WordInfoList.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2021 Works Applications Co., Ltd.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.worksap.nlp.sudachi.dictionary;
18 |
19 | import java.nio.Buffer;
20 | import java.nio.ByteBuffer;
21 |
22 | class WordInfoList {
23 |
24 | private final ByteBuffer bytes;
25 | private final int offset;
26 | private final int wordSize;
27 | private final boolean hasSynonymGid;
28 |
29 | WordInfoList(ByteBuffer bytes, int offset, int wordSize, boolean hasSysnoymGid) {
30 | this.bytes = bytes;
31 | this.offset = offset;
32 | this.wordSize = wordSize;
33 | this.hasSynonymGid = hasSysnoymGid;
34 | }
35 |
36 | WordInfo getWordInfo(int wordId) {
37 | ByteBuffer buf = bytes.asReadOnlyBuffer();
38 | buf.order(bytes.order());
39 | ((Buffer) buf).position(wordIdToOffset(wordId)); // a kludge for Java 9
40 |
41 | String surface = bufferToString(buf);
42 | short headwordLength = (short) bufferToStringLength(buf);
43 | short posId = buf.getShort();
44 | String normalizedForm = bufferToString(buf);
45 | if (normalizedForm.isEmpty()) {
46 | normalizedForm = surface;
47 | }
48 | int dictionaryFormWordId = buf.getInt();
49 | String readingForm = bufferToString(buf);
50 | if (readingForm.isEmpty()) {
51 | readingForm = surface;
52 | }
53 | int[] aUnitSplit = bufferToIntArray(buf);
54 | int[] bUnitSplit = bufferToIntArray(buf);
55 | int[] wordStructure = bufferToIntArray(buf);
56 |
57 | int[] synonymGids = new int[0];
58 | if (hasSynonymGid) {
59 | synonymGids = bufferToIntArray(buf);
60 | }
61 |
62 | String dictionaryForm = surface;
63 | if (dictionaryFormWordId >= 0 && dictionaryFormWordId != wordId) {
64 | WordInfo wi = getWordInfo(dictionaryFormWordId);
65 | dictionaryForm = wi.getSurface();
66 | }
67 |
68 | return new WordInfo(surface, headwordLength, posId, normalizedForm, dictionaryFormWordId, dictionaryForm,
69 | readingForm, aUnitSplit, bUnitSplit, wordStructure, synonymGids);
70 | }
71 |
72 | int size() {
73 | return wordSize;
74 | }
75 |
76 | private int wordIdToOffset(int wordId) {
77 | return bytes.getInt(offset + 4 * wordId);
78 | }
79 |
80 | private int bufferToStringLength(ByteBuffer buffer) {
81 | byte length = buffer.get();
82 | if (length < 0) {
83 | int high = Byte.toUnsignedInt(length);
84 | int low = Byte.toUnsignedInt(buffer.get());
85 | return ((high & 0x7F) << 8) | low;
86 | }
87 | return length;
88 | }
89 |
90 | private String bufferToString(ByteBuffer buffer) {
91 | int length = bufferToStringLength(buffer);
92 | char[] str = new char[length];
93 | for (int i = 0; i < length; i++) {
94 | str[i] = buffer.getChar();
95 | }
96 | return new String(str);
97 | }
98 |
99 | private int[] bufferToIntArray(ByteBuffer buffer) {
100 | int length = Byte.toUnsignedInt(buffer.get());
101 | int[] array = new int[length];
102 | for (int i = 0; i < length; i++) {
103 | array[i] = buffer.getInt();
104 | }
105 | return array;
106 | }
107 | }
108 |
--------------------------------------------------------------------------------
/src/main/java/com/worksap/nlp/sudachi/dictionary/BinaryDictionary.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2017-2022 Works Applications Co., Ltd.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.worksap.nlp.sudachi.dictionary;
18 |
19 | import java.io.Closeable;
20 | import java.io.IOException;
21 | import java.nio.ByteBuffer;
22 |
23 | import com.worksap.nlp.sudachi.Config;
24 | import com.worksap.nlp.sudachi.MMap;
25 |
26 | public class BinaryDictionary implements Closeable, DictionaryAccess {
27 |
28 | private final ByteBuffer bytes;
29 | private final DictionaryHeader header;
30 | private final GrammarImpl grammar;
31 | private final DoubleArrayLexicon lexicon;
32 |
33 | public BinaryDictionary(String fileName) throws IOException {
34 | this(MMap.map(fileName));
35 | }
36 |
37 | public BinaryDictionary(ByteBuffer dictionary) throws IOException {
38 | int offset = 0;
39 | bytes = dictionary;
40 |
41 | header = new DictionaryHeader(bytes, offset);
42 | offset += header.storageSize();
43 |
44 | long version = header.getVersion();
45 | if (DictionaryVersion.hasGrammar(version)) {
46 | grammar = new GrammarImpl(bytes, offset);
47 | offset += grammar.storageSize();
48 | } else if (header.isUserDictionary()) {
49 | grammar = new GrammarImpl();
50 | } else {
51 | MMap.unmap(bytes);
52 | throw new IOException("invalid dictionary");
53 | }
54 |
55 | lexicon = new DoubleArrayLexicon(bytes, offset, DictionaryVersion.hasSynonymGroupIds(version));
56 | }
57 |
58 | public static BinaryDictionary loadSystem(String fileName) throws IOException {
59 | return loadSystem(MMap.map(fileName));
60 | }
61 |
62 | public static BinaryDictionary loadUser(String fileName) throws IOException {
63 | return loadUser(MMap.map(fileName));
64 | }
65 |
66 | public static BinaryDictionary loadSystem(ByteBuffer buffer) throws IOException {
67 | BinaryDictionary dict = new BinaryDictionary(buffer);
68 | if (!dict.getDictionaryHeader().isSystemDictionary()) {
69 | dict.close();
70 | throw new IOException("invalid system dictionary");
71 | }
72 | return dict;
73 | }
74 |
75 | public static BinaryDictionary loadUser(ByteBuffer buffer) throws IOException {
76 | BinaryDictionary dict = new BinaryDictionary(buffer);
77 | if (!dict.getDictionaryHeader().isUserDictionary()) {
78 | dict.close();
79 | throw new IOException("invalid user dictionary");
80 | }
81 | return dict;
82 | }
83 |
84 | public static BinaryDictionary loadSystem(Config.Resource resource) throws IOException {
85 | return resource.consume(res -> loadSystem(res.asByteBuffer()));
86 | }
87 |
88 | public static BinaryDictionary loadUser(Config.Resource resource) throws IOException {
89 | return resource.consume(res -> loadUser(res.asByteBuffer()));
90 | }
91 |
92 | @Override
93 | public void close() throws IOException {
94 | MMap.unmap(bytes);
95 | }
96 |
97 | public DictionaryHeader getDictionaryHeader() {
98 | return header;
99 | }
100 |
101 | public GrammarImpl getGrammar() {
102 | return grammar;
103 | }
104 |
105 | public DoubleArrayLexicon getLexicon() {
106 | return lexicon;
107 | }
108 | }
--------------------------------------------------------------------------------
/src/main/java/com/worksap/nlp/sudachi/JoinKatakanaOovPlugin.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2021 Works Applications Co., Ltd.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.worksap.nlp.sudachi;
18 |
19 | import java.util.List;
20 |
21 | import com.worksap.nlp.sudachi.dictionary.CategoryType;
22 | import com.worksap.nlp.sudachi.dictionary.Grammar;
23 |
24 | /**
25 | * A plugin for concatenation of Katakana OOVs.
26 | *
27 | * This plugin concatenate the Katakana OOV and the adjacent Katakana morphemes.
28 | *
29 | *
30 | * The concatenated morpheme is OOV, and its part of speech must be specified in
31 | * the settings.
32 | *
33 | *
34 | * The following is an example of settings.
35 | *
36 | *
37 | * {@code
38 | * {
39 | * "class" : "com.worksap.nlp.sudachi.JoinKatakanaOovPlugin",
40 | * "oovPOS" : [ "POS1", "POS2", ... ],
41 | * "minLength" : 3
42 | * }
43 | * }
44 | *
45 | */
46 | class JoinKatakanaOovPlugin extends PathRewritePlugin {
47 |
48 | short oovPosId;
49 | int minLength;
50 |
51 | @Override
52 | public void setUp(Grammar grammar) {
53 | List pos = settings.getStringList("oovPOS");
54 | if (pos.isEmpty()) {
55 | throw new IllegalArgumentException("oovPOS is undefined");
56 | }
57 | oovPosId = grammar.getPartOfSpeechId(pos);
58 | if (oovPosId < 0) {
59 | throw new IllegalArgumentException("oovPOS is invalid");
60 | }
61 | minLength = settings.getInt("minLength", 1);
62 | if (minLength < 0) {
63 | throw new IllegalArgumentException("minLength is negative");
64 | }
65 | }
66 |
67 | @Override
68 | public void rewrite(InputText text, List path, Lattice lattice) {
69 | for (int i = 0; i < path.size(); i++) {
70 | LatticeNode node = path.get(i);
71 | if ((node.isOOV() || isShorter(minLength, text, node)) && isKatakanaNode(text, node)) {
72 | int begin = i - 1;
73 | for (; begin >= 0; begin--) {
74 | if (!isKatakanaNode(text, path.get(begin))) {
75 | begin++;
76 | break;
77 | }
78 | }
79 | if (begin < 0) {
80 | begin = 0;
81 | }
82 | int end = i + 1;
83 | for (; end < path.size(); end++) {
84 | if (!isKatakanaNode(text, path.get(end))) {
85 | break;
86 | }
87 | }
88 | while (begin != end && !canOovBowNode(text, path.get(begin))) {
89 | begin++;
90 | }
91 | if (end - begin > 1) {
92 | concatenateOov(path, begin, end, oovPosId, lattice);
93 | i = begin + 1;
94 | }
95 | }
96 | }
97 | }
98 |
99 | boolean isKatakanaNode(InputText text, LatticeNode node) {
100 | return getCharCategoryTypes(text, node).contains(CategoryType.KATAKANA);
101 | }
102 |
103 | boolean isShorter(int length, InputText text, LatticeNode node) {
104 | return text.codePointCount(node.getBegin(), node.getEnd()) < length;
105 | }
106 |
107 | boolean canOovBowNode(InputText text, LatticeNode node) {
108 | return !text.getCharCategoryTypes(node.getBegin()).contains(CategoryType.NOOOVBOW);
109 | }
110 | }
111 |
--------------------------------------------------------------------------------
/src/test/java/com/worksap/nlp/sudachi/MockInputText.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2021 Works Applications Co., Ltd.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | package com.worksap.nlp.sudachi;
18 |
19 | import java.util.EnumSet;
20 | import java.util.Set;
21 |
22 | import com.worksap.nlp.sudachi.dictionary.CategoryType;
23 |
24 | class MockInputText implements InputText {
25 |
26 | String text;
27 | EnumSet[] types;
28 |
29 | @SuppressWarnings("unchecked")
30 | MockInputText(String text) {
31 | this.text = text;
32 | types = new EnumSet[text.length()];
33 | for (int i = 0; i < text.length(); i++) {
34 | types[i] = EnumSet.noneOf(CategoryType.class);
35 | }
36 | }
37 |
38 | void setCategoryType(int begin, int end, CategoryType... types) {
39 | for (int i = begin; i < end; i++) {
40 | for (CategoryType type : types) {
41 | this.types[i].add(type);
42 | }
43 | }
44 | }
45 |
46 | @Override
47 | public String getText() {
48 | return text;
49 | }
50 |
51 | @Override
52 | public String getOriginalText() {
53 | return text;
54 | }
55 |
56 | @Override
57 | public String getSubstring(int begin, int end) {
58 | return text.substring(begin, end);
59 | }
60 |
61 | @Override
62 | public InputText slice(int begin, int end) {
63 | return null;
64 | }
65 |
66 | @Override
67 | public int getOriginalIndex(int index) {
68 | return index;
69 | }
70 |
71 | @Override
72 | public Set getCharCategoryTypes(int index) {
73 | return types[index];
74 | }
75 |
76 | @Override
77 | public Set getCharCategoryTypes(int begin, int end) {
78 | Set continuousCategory = types[begin].clone();
79 | for (int i = text.offsetByCodePoints(begin, 1); i < end; i = text.offsetByCodePoints(i, 1)) {
80 | continuousCategory.retainAll(types[i]);
81 | }
82 | return continuousCategory;
83 | }
84 |
85 | @Override
86 | public int getCharCategoryContinuousLength(int index) {
87 | Set continuousCategory = types[index].clone();
88 | for (int i = text.offsetByCodePoints(index, 1); i < text.length(); i = text.offsetByCodePoints(i, 1)) {
89 | continuousCategory.retainAll(types[i]);
90 | if (continuousCategory.isEmpty()) {
91 | return i - index;
92 | }
93 | }
94 | return text.length() - index;
95 | }
96 |
97 | @Override
98 | public int getCodePointsOffsetLength(int index, int codePointOffset) {
99 | return text.offsetByCodePoints(index, codePointOffset) - index;
100 | }
101 |
102 | @Override
103 | public int codePointCount(int begin, int end) {
104 | return Character.codePointCount(text, begin, end);
105 | }
106 |
107 | @Override
108 | public boolean canBow(int index) {
109 | return true;
110 | }
111 |
112 | @Override
113 | public int getWordCandidateLength(int index) {
114 | return 1;
115 | }
116 |
117 | @Override
118 | public int getNextInOriginal(int index) {
119 | return index + 1;
120 | }
121 |
122 | @Override
123 | public int modifiedOffset(int index) {
124 | return 0;
125 | }
126 |
127 | @Override
128 | public byte[] getByteText() {
129 | return new byte[0];
130 | }
131 | }
132 |
--------------------------------------------------------------------------------