test
43 | @ markdown 44 | :-test 45 | 1. Definition point 1. 46 | - Blah 47 | - Blah 48 | 2. Blah blah blah. 49 | 3. Blah *blah* **blah**! 50 | 51 | Blah blah blah.`, 52 | Out: DictFile{ 53 | {Headword: "blank", Variant: []string(nil), NoHeader: false, HeaderInfo: "", RawHTML: false, Definition: "", line: 1}, 54 | {Headword: "headword", Variant: []string{"variant1", "variant2"}, NoHeader: false, HeaderInfo: "info", RawHTML: false, Definition: "test\ntest", line: 3}, 55 | {Headword: "custom", Variant: []string{"NORMALIZEME"}, NoHeader: true, HeaderInfo: "", RawHTML: true, Definition: "custom word:\ntest
", line: 10}, 56 | {Headword: "markdown", Variant: []string(nil), NoHeader: false, HeaderInfo: "-test", RawHTML: false, Definition: "1. Definition point 1.\n - Blah\n - Blah\n2. Blah blah blah.\n3. Blah *blah* **blah**!\n\nBlah blah blah.", line: 16}, 57 | }, 58 | OutDictFile: `@ blank 59 | 60 | @ headword 61 | : info 62 | & variant1 63 | & variant2 64 | test 65 | test 66 | 67 | @ custom 68 | :: 69 | & NORMALIZEME 70 | 71 | custom word: 72 |test
73 | 74 | @ markdown 75 | : -test 76 | 1. Definition point 1. 77 | - Blah 78 | - Blah 79 | 2. Blah blah blah. 80 | 3. Blah *blah* **blah**! 81 | 82 | Blah blah blah. 83 | 84 | `, 85 | OutKoboHTML: `test
test 87 | test
-
88 |
- Definition point 1.
89 |
90 |
-
91 |
- Blah 92 |
- Blah 93 |
94 | - Blah blah blah. 95 |
- Blah blah blah! 96 |
Blah blah blah.
{{.Headword}}{{with .HeaderInfo}} {{.}}{{end}}
137 | {{- end -}} 138 | 139 | {{- range .Variant -}} 140 |This definition contains raw html.
172 | 173 |You can split it into multiple lines for readability.
174 | 175 |-
176 |
- You can also use all HTML tags. 177 |
- This text has a dark background 178 |
- This text is styled with CSS classes. 179 |
61 |
62 |
63 |
64 |
65 |
114 |
--------------------------------------------------------------------------------
/examples/dictzip-decompile/main.go:
--------------------------------------------------------------------------------
1 | // Command dictzip-decompile converts a dictzip into a dictfile. The regenerated
2 | // dictzip from the dictfile may not match exactly, but it will look the same,
3 | // and certain bugs with prefixes and variants will be implicitly fixed by the
4 | // conversion process (i.e. variant in wrong file, incorrect prefix, missing
5 | // words in index file). All output is in raw HTML, not Markdown.
6 | //
7 | // This is an experimental tool, and the output may not be perfect on complex
8 | // dictionaries.
9 | package main
10 |
11 | import (
12 | "fmt"
13 | "io"
14 | "os"
15 |
16 | "github.com/pgaskin/dictutil/kobodict"
17 | "github.com/spf13/pflag"
18 |
19 | _ "github.com/pgaskin/dictutil/kobodict/marisa"
20 | )
21 |
22 | var version = "dev"
23 |
24 | func main() {
25 | pflag.CommandLine.SortFlags = false
26 | output := pflag.StringP("output", "o", "."+string(os.PathSeparator)+"decompiled.df", "The output filename (will be overwritten if it exists) (- is stdout)")
27 | resources := pflag.BoolP("resources", "r", false, "Also extract referenced resources to the current directory (warning: any existing files will be overwritten, so it is recommended to run in an empty directory if enabled)")
28 | help := pflag.BoolP("help", "h", false, "Show this help text")
29 | pflag.Parse()
30 |
31 | if *help || pflag.NArg() != 1 {
32 | fmt.Fprintf(os.Stderr, "Usage: %s [options] dictzip\n\nVersion: dictzip-decompile %s\n\nOptions:\n%s\nArguments:\n dictzip is the path to the dictzip to decompile.\n\nTo convert the resulting dictfile into a dictzip, use dictgen.\n\nNote: The regenerated dictzip from the dictfile may not match exactly, but it will look the same, and certain bugs with prefixes and variants will be implicitly fixed by the conversion process (i.e. variant in wrong file, incorrect prefix, missing words in index file). All output is in raw HTML, not Markdown.\n\nThis is an experimental tool, and the output may not be perfect on complex dictionaries.\n", os.Args[0], version, pflag.CommandLine.FlagUsages())
33 | if pflag.NArg() != 0 {
34 | os.Exit(2)
35 | } else {
36 | os.Exit(0)
37 | }
38 | return
39 | }
40 |
41 | fn := pflag.Args()[0]
42 |
43 | fmt.Fprintf(os.Stderr, "Opening input dictzip.\n")
44 | f, err := os.Open(fn)
45 | if err != nil {
46 | fmt.Fprintf(os.Stderr, "Error: open input file %#v: %v.\n", fn, err)
47 | os.Exit(1)
48 | return
49 | }
50 | defer f.Close()
51 |
52 | s, err := f.Stat()
53 | if err != nil {
54 | fmt.Fprintf(os.Stderr, "Error: stat input file %#v: %v.\n", fn, err)
55 | os.Exit(1)
56 | return
57 | }
58 |
59 | fmt.Fprintf(os.Stderr, "Parsing dictzip.\n")
60 | dr, err := kobodict.NewReader(f, s.Size())
61 | if err != nil {
62 | fmt.Fprintf(os.Stderr, "Error: parse input file %#v: %v.\n", fn, err)
63 | os.Exit(1)
64 | return
65 | }
66 |
67 | fmt.Fprintf(os.Stderr, "Decompiling dictzip.\n")
68 | df, err := decompile(dr)
69 | if err != nil {
70 | fmt.Fprintf(os.Stderr, "Error: decompile dictzip %#v: %v.\n", fn, err)
71 | os.Exit(1)
72 | return
73 | }
74 |
75 | if *resources {
76 | fmt.Fprintf(os.Stderr, "Extracting resources.\n")
77 | for _, f := range dr.File {
78 | fmt.Fprintf(os.Stderr, " ./%s\n", f.Name)
79 | if err := func() error {
80 | rc, err := f.Open()
81 | if err != nil {
82 | return fmt.Errorf("open: %w", err)
83 | }
84 | defer rc.Close()
85 |
86 | f, err := os.OpenFile(f.Name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
87 | if err != nil {
88 | return fmt.Errorf("create output: %w", err)
89 | }
90 | defer f.Close()
91 |
92 | if _, err := io.Copy(f, rc); err != nil {
93 | return fmt.Errorf("copy: %w", err)
94 | }
95 |
96 | if err := f.Close(); err != nil {
97 | return fmt.Errorf("write output: %w", err)
98 | }
99 |
100 | return nil
101 | }(); err != nil {
102 | fmt.Fprintf(os.Stderr, "Error: extract resource %#v: %v.\n", f.Name, err)
103 | os.Exit(1)
104 | return
105 | }
106 | }
107 | } else {
108 | if len(dr.File) != 0 {
109 | fmt.Fprintf(os.Stderr, "Warning: dictfile contains %d resources, but skipping because resource extraction is not enabled (see --help for more details).\n", len(dr.File))
110 | }
111 | }
112 |
113 | fmt.Fprintf(os.Stderr, "Writing dictfile.\n")
114 | switch *output {
115 | case "-":
116 | if err := df.WriteDictFile(os.Stdout); err != nil {
117 | fmt.Fprintf(os.Stderr, "Error: write dictfile: %v\n", err)
118 | os.Exit(1)
119 | return
120 | }
121 | default:
122 | f, err := os.OpenFile(*output, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
123 | if err != nil {
124 | fmt.Fprintf(os.Stderr, "Error: create dictfile: %v\n", err)
125 | os.Exit(1)
126 | return
127 | }
128 |
129 | if err := df.WriteDictFile(f); err != nil {
130 | f.Close()
131 | fmt.Fprintf(os.Stderr, "Error: write dictfile: %v\n", err)
132 | os.Exit(1)
133 | return
134 | }
135 |
136 | if err := f.Close(); err != nil {
137 | fmt.Fprintf(os.Stderr, "Error: write dictfile: %v\n", err)
138 | os.Exit(1)
139 | return
140 | }
141 | }
142 |
143 | fmt.Fprintf(os.Stderr, "Successfully converted %d entries from dictzip %#v to dictfile %s.\n", len(df), fn, *output)
144 | os.Exit(0)
145 | }
146 |
--------------------------------------------------------------------------------
/examples/dictzip-decompile/parse.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bytes"
5 | "crypto/sha1"
6 | "fmt"
7 | "io/ioutil"
8 | "regexp"
9 | "unicode"
10 |
11 | "github.com/pgaskin/dictutil/dictgen"
12 | "github.com/pgaskin/dictutil/kobodict"
13 | )
14 |
15 | // This isn't exposed as a separate package, as it's subject to change and
16 | // highly specific to dictzip-decompile.
17 |
18 | // The regexps used to extract data should have a similar level of strictness as
19 | // the ones used by nickel (for simplicity, compatibility, and predictability).
20 |
21 | // decompile decompiles a dictzip into a dictfile. External resources are not
22 | // extracted, and must be done separately.
23 | //
24 | // Duplicate entries (e.g. the ones added by dictgen for fixing broken variants)
25 | // are collapsed into one. They will be expanded again as necessary when the
26 | // dictfile is compiled by dictgen.
27 | func decompile(r *kobodict.Reader) (dictgen.DictFile, error) {
28 | var df dictgen.DictFile
29 | seenEntries := map[[20]byte]struct{}{}
30 | for _, dh := range r.Dicthtml {
31 | if err := func() error {
32 | rc, err := dh.Open()
33 | if err != nil {
34 | return fmt.Errorf("open: %w", err)
35 | }
36 | defer rc.Close()
37 |
38 | buf, err := ioutil.ReadAll(rc)
39 | if err != nil {
40 | return fmt.Errorf("read: %w", err)
41 | }
42 |
43 | es, err := extractEntries(buf)
44 | if err != nil {
45 | return fmt.Errorf("extract entries: %w", err)
46 | }
47 |
48 | for _, e := range es {
49 | ss := sha1.Sum(e)
50 | if _, ok := seenEntries[ss]; ok {
51 | continue
52 | }
53 | seenEntries[ss] = struct{}{}
54 |
55 | de, err := decompileEntry(e)
56 | if err != nil {
57 | return fmt.Errorf("decompile entry %#v: %w", string(e), err)
58 | }
59 |
60 | df = append(df, de)
61 | }
62 |
63 | return nil
64 | }(); err != nil {
65 | return nil, fmt.Errorf("process dicthtml %#v: %w", dh.Name, err)
66 | }
67 | }
68 | return df, nil
69 | }
70 |
71 | // The regexps/vars used by decompileEntry.
72 | var (
73 | // generator matchers (match the entire entry, split into parts) (match in order) (don't include variants here)
74 | generator1PenelopeRe = regexp.MustCompile(`^(?s)([^<]+)<\/b>
|><\/a>)\s*([^<]+)\s*<\/b>\s*(.*?)
\s*(.+)\s*<\/p>$`) // also: 2nd and 3rd (header) group must not contain "dfgdfgsdfsdf
Penelope
- Élément exprimant la négation ( pas ), ou la privation ( sans ). ⇒anormal, apolitique.
- an abdominal muscle usu. used in pl.
- about
- Etym: [Shortened form of an. AS. an one. See One.] An adjective, commonly called the indefinite article, and signifying one or any, but less emphatically.
- "At a birth"; "In a word"; "At a blow". Shak. Note: It is placed before nouns of the singular number denoting an individual object, or a quality individualized, before collective nouns, and also before plural nouns when the adjective few or the phrase great many or good many is interposed; as, a dog, a house, a man; a color; a sweetness; a hundred, a fleet, a regiment; a few persons, a great many days. It is used for an, for the sake of euphony, before words beginning with a consonant sound [for exception of certain words beginning with h, see An]; as, a table, a woman, a year, a unit, a eulogy, a ewe, a oneness, such a one, etc. Formally an was used both before vowels and consonants.
- Etym: [Originally the preposition a (an, on).] In each; to or for each; as, "twenty leagues a day", "a hundred pounds a year", "a dollar a yard", etc.
dfkgjdlfjglkdfjg
{{.}}
{{end -}} 32 | {{- with .Meanings}}- {{range .}}
- {{.Text}}{{with .Example}}
{{.}}{{end}} {{end}}
{{range $n, $v := .}}{{if $n}} {{end}}{{range $x, $y := (spldc $v)}}{{if $x}}{{$y}}{{else}}{{$y}}{{end}}{{end}}{{end}}
{{end -}} 34 | {{- with .Synonyms}}{{range $n, $v := .}}{{if $n}} {{end}}{{$v}}{{end}}
{{end -}} 35 | {{- with .Extra}}{{.}}
{{end -}} 36 | `)) 37 | 38 | func main() { 39 | pflag.CommandLine.SortFlags = false 40 | output := pflag.StringP("output", "o", "."+string(os.PathSeparator)+"webster1913.df", "The output filename (will be overwritten if it exists) (- is stdout)") 41 | dump := pflag.Bool("dump", false, "Instead of converting, dump the parsed dictionary to stdout as JSON (for debugging)") 42 | help := pflag.BoolP("help", "h", false, "Show this help text") 43 | pflag.Parse() 44 | 45 | if *help || pflag.NArg() != 1 { 46 | fmt.Fprintf(os.Stderr, "Usage: %s [options] gutenberg_webster1913_path\n\nVersion: webster1913-convert %s\n\nOptions:\n%s\nArguments:\n gutenberg_webster1913_path is the path to Project Gutenberg's Webster's 1913 dictionary. Use - to read from stdin.\n\nTo convert the resulting dictfile into a dictzip, use dictgen.\n", os.Args[0], version, pflag.CommandLine.FlagUsages()) 47 | os.Exit(0) 48 | return 49 | } 50 | 51 | fmt.Fprintf(os.Stderr, "Opening input file.\n") 52 | var r io.Reader 53 | switch v := pflag.Args()[0]; v { 54 | case "-": 55 | r = os.Stdin 56 | default: 57 | f, err := os.Open(v) 58 | if err != nil { 59 | fmt.Fprintf(os.Stderr, "Error: open input %#v: %v\n", v, err) 60 | os.Exit(1) 61 | return 62 | } 63 | defer f.Close() 64 | r = f 65 | } 66 | 67 | fmt.Fprintf(os.Stderr, "Parsing dictionary.\n") 68 | wd, err := webster1913.Parse(r, func(i int, word string) { 69 | if i%1000 == 0 { 70 | fmt.Fprintf(os.Stderr, "[% 5d] %s\n", i, word) 71 | } 72 | }) 73 | if err != nil { 74 | fmt.Fprintf(os.Stderr, "Error: parse webster1913: %v\n", err) 75 | os.Exit(1) 76 | return 77 | } 78 | 79 | if *dump { 80 | fmt.Fprintf(os.Stderr, "Dumping JSON to stdout.\n") 81 | enc := json.NewEncoder(os.Stdout) 82 | enc.SetIndent("", " ") 83 | enc.Encode(wd) 84 | os.Exit(0) 85 | return 86 | } 87 | 88 | fmt.Fprintf(os.Stderr, "Transforming definitions.\n") 89 | var df dictgen.DictFile 90 | dbuf := bytes.NewBuffer(nil) 91 | for _, d := range wd { 92 | dbuf.Reset() 93 | if err := deftmpl.Execute(dbuf, d); err != nil { 94 | fmt.Fprintf(os.Stderr, "Error: render definition %#v: %v\n", d, err) 95 | os.Exit(1) 96 | return 97 | } 98 | df = append(df, &dictgen.DictFileEntry{ 99 | Headword: d.Headword, 100 | Variant: d.Variant, 101 | RawHTML: true, 102 | HeaderInfo: d.Info, 103 | Definition: dbuf.String(), 104 | }) 105 | } 106 | 107 | fmt.Fprintf(os.Stderr, "Writing dictfile.\n") 108 | switch *output { 109 | case "-": 110 | if err := df.WriteDictFile(os.Stdout); err != nil { 111 | fmt.Fprintf(os.Stderr, "Error: write dictfile: %v\n", err) 112 | os.Exit(1) 113 | return 114 | } 115 | default: 116 | f, err := os.OpenFile(*output, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) 117 | if err != nil { 118 | fmt.Fprintf(os.Stderr, "Error: create dictfile: %v\n", err) 119 | os.Exit(1) 120 | return 121 | } 122 | 123 | if err := df.WriteDictFile(f); err != nil { 124 | f.Close() 125 | fmt.Fprintf(os.Stderr, "Error: write dictfile: %v\n", err) 126 | os.Exit(1) 127 | return 128 | } 129 | 130 | if err := f.Close(); err != nil { 131 | fmt.Fprintf(os.Stderr, "Error: write dictfile: %v\n", err) 132 | os.Exit(1) 133 | return 134 | } 135 | } 136 | 137 | fmt.Fprintf(os.Stderr, "Successfully converted %d entries from Webster's 1913 dictionary %#v to dictfile %s.\n", len(df), pflag.Args()[0], *output) 138 | os.Exit(0) 139 | } 140 | -------------------------------------------------------------------------------- /examples/webster1913-convert/webster1913/parser.go: -------------------------------------------------------------------------------- 1 | // Package webster1913 parses Project Gutenberg's Webster's 1913 Unabridged 2 | // Dictionary (http://www.gutenberg.org/ebooks/29765.txt.utf-8). 3 | package webster1913 4 | 5 | import ( 6 | "bufio" 7 | "bytes" 8 | "io" 9 | "regexp" 10 | "runtime/debug" 11 | "strings" 12 | ) 13 | 14 | // Dict represents the parsed dictionary. 15 | type Dict []*Entry 16 | 17 | // Entry is a single dictionary entry. 18 | type Entry struct { 19 | Headword string 20 | Variant []string 21 | Info string 22 | Etymology string 23 | Meanings []*EntryMeaning 24 | Synonyms []string 25 | PhraseDefns []string 26 | Extra string // unparseable text 27 | } 28 | 29 | // EntryMeaning is a meaning for a dictionary entry. 30 | type EntryMeaning struct { 31 | Text string 32 | Example string 33 | } 34 | 35 | var ( 36 | entryWordRe = regexp.MustCompile(`^[A-Z_ ;-]+$`) 37 | numberedDefnStartRe = regexp.MustCompile(`^[0-9]+\.\s*`) 38 | singleDefnStartRe = regexp.MustCompile(`^Defn:\s+`) 39 | noteStartRe = regexp.MustCompile(`^\s*Note:\s+`) 40 | synStartRe = regexp.MustCompile(`^Syn.\s*$`) 41 | synItemStartRe = regexp.MustCompile(`^\s+--\s+`) 42 | phraseDefnStartRe = regexp.MustCompile(`^\s+--\s+([A-Za-z ]+?[A-Za-z])\s*(\([^)]+\))?[,.]\s*`) 43 | wordInfoFormRe = regexp.MustCompile(`(?:p\. p\.|vb\. n\.|p\. pr\.) +([A-Z][a-z]+)[:;.,]`) 44 | ) 45 | 46 | type state int 47 | 48 | const ( 49 | // StateNone is before the first entry. 50 | StateNone state = iota 51 | // StateEntryInfo is at the beginning of the entry. 52 | StateEntryInfo 53 | // StateEntryExtra is unclassified text in the entry. 54 | StateEntryExtra 55 | // StateEntryMeaningText is inside an entry's meaning's text. 56 | StateEntryMeaningText 57 | // StateEntryMeaningExample is inside an entry's meaning's example. 58 | StateEntryMeaningExample 59 | // StateEntrySynonym is inside an entry's synonym list. 60 | StateEntrySynonym 61 | // StateEntryPhraseDefn is inside an entry's phrase definition list. 62 | StateEntryPhraseDefn 63 | ) 64 | 65 | // Parse parses Project Gutenberg's Webster's Unabridged Dictionary. 66 | func Parse(r io.Reader, progress func(i int, w string)) (Dict, error) { 67 | var wd Dict 68 | var perr error 69 | sc := bufio.NewScanner(r) 70 | 71 | var state state 72 | var entry *Entry 73 | var meaning *EntryMeaning 74 | var i int 75 | for sc.Scan() { 76 | ln := sc.Bytes() 77 | lnt := bytes.TrimSpace(ln) 78 | blankLine := len(lnt) == 0 79 | 80 | if bytes.HasPrefix(lnt, []byte("*** END")) { 81 | break 82 | } 83 | 84 | if entryWordRe.Match(ln) { 85 | if state == StateNone { 86 | // skip the file header(up to the word "A") 87 | if !bytes.Equal(lnt, []byte{'A'}) { 88 | continue 89 | } 90 | } 91 | if bytes.Count(lnt, []byte{'-'}) != len(lnt) { 92 | // ^ if all dashes, it is a false positive 93 | if entry != nil { 94 | progress(len(wd), entry.Headword) 95 | } 96 | spl := strings.Split(string(bytes.ToLower(ln)), ";") 97 | entry = &Entry{Headword: strings.TrimSpace(spl[0])} 98 | if len(spl) > 1 { 99 | for _, v := range spl[1:] { 100 | if w := strings.TrimSpace(v); w != "" { 101 | entry.Variant = append(entry.Variant, w) 102 | } 103 | } 104 | } 105 | meaning = nil 106 | wd = append(wd, entry) 107 | state = StateEntryInfo 108 | continue 109 | } 110 | } 111 | 112 | switch state { 113 | case StateNone: 114 | // ignore any text before the first entry 115 | case StateEntryInfo: 116 | switch { 117 | case blankLine: 118 | for _, m := range wordInfoFormRe.FindAllStringSubmatch(entry.Info, -1) { 119 | entry.Variant = append(entry.Variant, strings.ToLower(m[1])) 120 | } 121 | // attempt to split into etymology 122 | if spl := strings.SplitN(entry.Info, " Etym: ", 2); len(spl) == 2 { 123 | entry.Info = strings.TrimSpace(spl[0]) 124 | entry.Etymology = strings.TrimSpace(spl[1]) 125 | } 126 | state = StateEntryExtra 127 | default: 128 | entry.Info += " " + string(lnt) 129 | } 130 | case StateEntryExtra: 131 | switch { 132 | case singleDefnStartRe.Match(ln): 133 | meaning = &EntryMeaning{Text: string(singleDefnStartRe.ReplaceAllLiteral(ln, nil))} 134 | entry.Meanings = append(entry.Meanings, meaning) 135 | state = StateEntryMeaningText 136 | case numberedDefnStartRe.Match(ln): 137 | meaning = &EntryMeaning{Text: string(numberedDefnStartRe.ReplaceAllLiteral(ln, nil))} 138 | entry.Meanings = append(entry.Meanings, meaning) 139 | state = StateEntryMeaningText 140 | case phraseDefnStartRe.Match(ln): 141 | meaning = nil 142 | entry.PhraseDefns = append(entry.PhraseDefns, string(bytes.TrimSpace(bytes.Replace(lnt, []byte("--"), nil, 1)))) 143 | entry.Variant = append(entry.Variant, string(bytes.ToLower(phraseDefnStartRe.FindSubmatch(ln)[1]))) 144 | state = StateEntryPhraseDefn 145 | case blankLine: 146 | // ignore 147 | default: 148 | entry.Extra += " " + string(lnt) 149 | } 150 | case StateEntryMeaningText: 151 | switch { 152 | case synStartRe.Match(ln): 153 | meaning = nil 154 | state = StateEntrySynonym 155 | case singleDefnStartRe.Match(ln): 156 | // if it is in any kind of definition (single/numbered), it is part of it. 157 | meaning.Text += " " + string(singleDefnStartRe.ReplaceAllLiteral(lnt, nil)) 158 | case numberedDefnStartRe.Match(ln): 159 | meaning = &EntryMeaning{Text: string(numberedDefnStartRe.ReplaceAllLiteral(ln, nil))} 160 | entry.Meanings = append(entry.Meanings, meaning) 161 | state = StateEntryMeaningText 162 | case phraseDefnStartRe.Match(ln): 163 | meaning = nil 164 | entry.PhraseDefns = append(entry.PhraseDefns, string(bytes.TrimSpace(bytes.Replace(lnt, []byte("--"), nil, 1)))) 165 | entry.Variant = append(entry.Variant, string(bytes.ToLower(phraseDefnStartRe.FindSubmatch(ln)[1]))) 166 | state = StateEntryPhraseDefn 167 | case len(meaning.Text) > 5 && len(lnt) < 55 && bytes.HasSuffix(lnt, []byte{'.'}) && !noteStartRe.Match(ln): 168 | // if there is already some body text, it is not a hard-wrapped 169 | // line, and it ends with a period, and is not a note, then it's 170 | // the last line of the text before the example. 171 | meaning.Text += " " + string(lnt) 172 | state = StateEntryMeaningExample 173 | case blankLine: 174 | // ignore 175 | default: 176 | meaning.Text += " " + string(lnt) 177 | } 178 | case StateEntryMeaningExample: 179 | switch { 180 | case synStartRe.Match(ln): 181 | meaning = nil 182 | state = StateEntrySynonym 183 | case singleDefnStartRe.Match(ln): 184 | meaning = &EntryMeaning{Text: string(singleDefnStartRe.ReplaceAllLiteral(ln, nil))} 185 | entry.Meanings = append(entry.Meanings, meaning) 186 | state = StateEntryMeaningText 187 | case numberedDefnStartRe.Match(ln): 188 | meaning = &EntryMeaning{Text: string(numberedDefnStartRe.ReplaceAllLiteral(ln, nil))} 189 | entry.Meanings = append(entry.Meanings, meaning) 190 | state = StateEntryMeaningText 191 | case phraseDefnStartRe.Match(ln): 192 | meaning = nil 193 | entry.PhraseDefns = append(entry.PhraseDefns, string(bytes.TrimSpace(bytes.Replace(lnt, []byte("--"), nil, 1)))) 194 | entry.Variant = append(entry.Variant, string(bytes.ToLower(phraseDefnStartRe.FindSubmatch(ln)[1]))) 195 | state = StateEntryPhraseDefn 196 | case blankLine: 197 | // ignore 198 | default: 199 | if meaning.Example != "" { 200 | meaning.Example += " " 201 | } 202 | meaning.Example += string(lnt) 203 | } 204 | case StateEntrySynonym: 205 | switch { 206 | case blankLine: 207 | state = StateEntryExtra 208 | case synItemStartRe.Match(ln): 209 | entry.Synonyms = append(entry.Synonyms, string(synItemStartRe.ReplaceAllLiteral(ln, nil))) 210 | case len(entry.Synonyms) == 0: 211 | // there was a "Syn." without any valid synonyms under it 212 | state = StateEntryExtra 213 | case phraseDefnStartRe.Match(ln): 214 | meaning = nil 215 | entry.PhraseDefns = append(entry.PhraseDefns, string(bytes.TrimSpace(bytes.Replace(lnt, []byte("--"), nil, 1)))) 216 | entry.Variant = append(entry.Variant, string(bytes.ToLower(phraseDefnStartRe.FindSubmatch(ln)[1]))) 217 | state = StateEntryPhraseDefn 218 | default: 219 | entry.Synonyms[len(entry.Synonyms)-1] += " " + string(lnt) 220 | } 221 | case StateEntryPhraseDefn: 222 | switch { 223 | case phraseDefnStartRe.Match(ln): 224 | meaning = nil 225 | entry.PhraseDefns = append(entry.PhraseDefns, string(bytes.TrimSpace(bytes.Replace(lnt, []byte("--"), nil, 1)))) 226 | entry.Variant = append(entry.Variant, string(bytes.ToLower(phraseDefnStartRe.FindSubmatch(ln)[1]))) 227 | state = StateEntryPhraseDefn 228 | case blankLine: 229 | // allow a blank line to end it for reducing the chance of bugs. 230 | state = StateEntryExtra 231 | default: 232 | // phrase definitions are always last, so no need for checking 233 | // for any other state changes (e.g. the start of a numbered 234 | // definition) (and the previous case should deal with any 235 | // edge-cases). 236 | entry.PhraseDefns[len(entry.PhraseDefns)-1] += " " + string(lnt) 237 | } 238 | } 239 | 240 | if i%10000 == 0 { 241 | debug.FreeOSMemory() // hack to try and limit memory usage 242 | } 243 | i++ 244 | } 245 | 246 | if serr := sc.Err(); serr != nil { 247 | return nil, serr 248 | } 249 | if perr != nil { 250 | return nil, perr 251 | } 252 | return wd, nil 253 | } 254 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/pgaskin/dictutil 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/disintegration/imaging v1.6.2 7 | github.com/mattn/go-sqlite3 v2.0.3+incompatible 8 | github.com/pgaskin/koboutils/v2 v2.1.0 9 | github.com/pmezard/go-difflib v1.0.0 // indirect 10 | github.com/russross/blackfriday/v2 v2.0.1 11 | github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect 12 | github.com/spf13/pflag v1.0.5 13 | gopkg.in/yaml.v2 v2.2.8 14 | ) 15 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= 2 | github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= 3 | github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= 4 | github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= 5 | github.com/pgaskin/koboutils/v2 v2.1.0 h1:J5KzLWvj0zDvoP5aJ7RxWuzFA32CcnD+hqH6tw/3uRE= 6 | github.com/pgaskin/koboutils/v2 v2.1.0/go.mod h1:wTzkDIlsxmUyfwfspGcm0Ap+HOxSUYV0S8kMYrf+0gM= 7 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 8 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 9 | github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= 10 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 11 | github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= 12 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 13 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 14 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 15 | golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U= 16 | golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= 17 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 18 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 19 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 20 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 21 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 22 | -------------------------------------------------------------------------------- /kobodict/crypt.go: -------------------------------------------------------------------------------- 1 | package kobodict 2 | 3 | import ( 4 | "bytes" 5 | "crypto/aes" 6 | "crypto/cipher" 7 | "fmt" 8 | ) 9 | 10 | // Crypter represents a symmetric dictionary encryption method. 11 | type Crypter interface { 12 | Encrypter 13 | Decrypter 14 | } 15 | 16 | // CryptMethodAES represents AES-128-ECB encryption with PKCS#7 padding. 17 | const CryptMethodAES string = "aes" 18 | 19 | // NewCrypter creates the specified type of Crypter with the specified key. 20 | func NewCrypter(method string, key []byte) (Crypter, error) { 21 | switch method { 22 | case CryptMethodAES: 23 | c, err := newCryptAES(key) 24 | return c, err 25 | default: 26 | return nil, fmt.Errorf("unknown encryption method %#v", method) 27 | } 28 | } 29 | 30 | type cryptAES struct { 31 | b cipher.Block 32 | } 33 | 34 | func newCryptAES(key []byte) (*cryptAES, error) { 35 | if b, err := aes.NewCipher(key); err != nil { 36 | return nil, err 37 | } else { 38 | return &cryptAES{b}, nil 39 | } 40 | } 41 | 42 | // Encrypt implements Encrypter. 43 | func (c *cryptAES) Encrypt(buf []byte) ([]byte, error) { 44 | if dst, err := cryptPKCS7Pad(buf, aes.BlockSize); err != nil { 45 | return nil, err 46 | } else if dst, err = cryptAES128ECBEncrypt(c.b, dst); err != nil { 47 | return nil, err 48 | } else { 49 | return dst, nil 50 | } 51 | } 52 | 53 | // Decrypt implements Decrypter. 54 | func (c *cryptAES) Decrypt(buf []byte) ([]byte, error) { 55 | if dst, err := cryptAES128ECBDecrypt(c.b, buf); err != nil { 56 | return nil, err 57 | } else if dst, err := cryptPKCS7Unpad(dst, aes.BlockSize); err != nil { 58 | return nil, err 59 | } else { 60 | return dst, nil 61 | } 62 | } 63 | 64 | func cryptPKCS7Unpad(src []byte, blockSize int) ([]byte, error) { 65 | if blockSize > 0xFF || blockSize < 0x00 { 66 | return nil, fmt.Errorf("block size %d out of bounds", blockSize) 67 | } else if len(src)%blockSize != 0 || len(src) == 0 { 68 | return nil, fmt.Errorf("data length %d is empty or not a multiple of block size %d", len(src), blockSize) 69 | } 70 | plen := int(src[len(src)-1]) 71 | if len(src) <= plen { 72 | return nil, fmt.Errorf("invalid padding: padding length %d out of bounds", plen) 73 | } 74 | for _, v := range src[len(src)-plen:] { 75 | if int(v) != plen { 76 | return nil, fmt.Errorf("invalid padding: expected %d, got %d", plen, v) 77 | } 78 | } 79 | return src[:len(src)-plen], nil 80 | } 81 | 82 | func cryptPKCS7Pad(src []byte, blockSize int) ([]byte, error) { 83 | if blockSize > 0xFF || blockSize < 0x00 { 84 | return nil, fmt.Errorf("block size %d out of bounds", blockSize) 85 | } 86 | plen := blockSize - len(src)%blockSize 87 | return append(src, bytes.Repeat([]byte{byte(plen)}, plen)...), nil 88 | } 89 | 90 | func cryptAES128ECBDecrypt(cb cipher.Block, src []byte) ([]byte, error) { 91 | if len(src)%aes.BlockSize != 0 { 92 | return nil, fmt.Errorf("src not a multiple of block size %d", aes.BlockSize) 93 | } 94 | dst := make([]byte, len(src)) 95 | for i := aes.BlockSize; i <= len(src); i += aes.BlockSize { 96 | cb.Decrypt(dst[i-aes.BlockSize:i], src[i-aes.BlockSize:i]) 97 | } 98 | return dst, nil 99 | } 100 | 101 | func cryptAES128ECBEncrypt(cb cipher.Block, src []byte) ([]byte, error) { 102 | if len(src)%aes.BlockSize != 0 { 103 | return nil, fmt.Errorf("src not a multiple of block size %d", aes.BlockSize) 104 | } 105 | dst := make([]byte, len(src)) 106 | for i := aes.BlockSize; i <= len(src); i += aes.BlockSize { 107 | cb.Encrypt(dst[i-aes.BlockSize:i], src[i-aes.BlockSize:i]) 108 | } 109 | return dst, nil 110 | } 111 | -------------------------------------------------------------------------------- /kobodict/crypt_test.go: -------------------------------------------------------------------------------- 1 | package kobodict 2 | 3 | // TODO(v1) 4 | -------------------------------------------------------------------------------- /kobodict/fs.go: -------------------------------------------------------------------------------- 1 | package kobodict 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "os" 9 | "path/filepath" 10 | "strings" 11 | "unicode/utf8" 12 | ) 13 | 14 | // Unpack is a helper function to unpack the contents of a Reader to a folder 15 | // on-disk. The provided dir must be non-existent. Unpack will not close the 16 | // reader. 17 | func Unpack(r *Reader, dir string) error { 18 | if _, err := os.Stat(dir); !os.IsNotExist(err) { 19 | return fmt.Errorf("dir %#v already exists", dir) 20 | } 21 | if err := os.Mkdir(dir, 0755); err != nil { 22 | return fmt.Errorf("create dir %#v: %w", dir, err) 23 | } 24 | for _, f := range r.File { 25 | if err := unpackFile(dir, f.Open, f.Name); err != nil { 26 | return fmt.Errorf("unpack file %#v: %w", f.Name, err) 27 | } 28 | } 29 | for _, f := range r.Dicthtml { 30 | if err := unpackFile(dir, f.Open, f.Name); err != nil { 31 | return fmt.Errorf("unpack dicthtml %#v (prefix: %s): %w", f.Name, f.Prefix, err) 32 | } 33 | } 34 | if err := ioutil.WriteFile(filepath.Join(dir, "words"), []byte(strings.Join(r.Word, "\n")), 0644); err != nil { 35 | return fmt.Errorf("write words file: %w", err) 36 | } 37 | return nil 38 | } 39 | 40 | func unpackFile(dir string, open func() (io.ReadCloser, error), name string) error { 41 | fr, err := open() 42 | if err != nil { 43 | return fmt.Errorf("read contents: %w", err) 44 | } 45 | defer fr.Close() 46 | 47 | fw, err := os.OpenFile(filepath.Join(dir, name), os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0644) 48 | if err != nil { 49 | return fmt.Errorf("create output file: %w", err) 50 | } 51 | defer fw.Close() 52 | 53 | if _, err := io.Copy(fw, fr); err != nil { 54 | return fmt.Errorf("write output file: %w", err) 55 | } 56 | 57 | if err := fw.Close(); err != nil { 58 | return fmt.Errorf("write output file: %w", err) 59 | } 60 | 61 | return nil 62 | } 63 | 64 | // Pack is a helper function to pack the contents a folder unpacked using Unpack 65 | // into a Writer. It is assumed that the writer has not been used. The provided 66 | // file will be overwritten if it exists and is a regular file, or created if it 67 | // doesn't exist. Pack will not close the writer. 68 | func Pack(w *Writer, dir string) error { 69 | if fi, err := os.Stat(filepath.Join(dir, "words")); os.IsNotExist(err) || (err == nil && fi.IsDir()) { 70 | return fmt.Errorf("dir %#v is not an unpacked dictzip (no words file)", dir) 71 | } 72 | 73 | fis, err := ioutil.ReadDir(dir) // note: this is sorted 74 | if err != nil { 75 | return fmt.Errorf("read dir %#v: %w", dir, err) 76 | } 77 | 78 | for _, fi := range fis { 79 | switch { 80 | case fi.IsDir(): 81 | return fmt.Errorf("invalid dir %#v: dirs are not supported", fi.Name()) 82 | case fi.Name() == "words": 83 | continue 84 | case strings.HasSuffix(fi.Name(), ".html"): 85 | if err := func() error { 86 | fr, err := os.OpenFile(filepath.Join(dir, fi.Name()), os.O_RDONLY, 0) 87 | if err != nil { 88 | return fmt.Errorf("open file: %w", err) 89 | } 90 | defer fr.Close() 91 | 92 | tmp := make([]byte, 2) 93 | if _, err := fr.Read(tmp); err != nil { 94 | return fmt.Errorf("read file: %w", err) 95 | } else if tmp[0] == 0x1F && tmp[1] == 0x8B { 96 | return fmt.Errorf("invalid unpacked dicthtml file: already compressed") 97 | } else if _, err := fr.Seek(0, os.SEEK_SET); err != nil { 98 | return fmt.Errorf("read file: %w", err) 99 | } 100 | 101 | fw, err := w.CreateDicthtml(strings.TrimSuffix(fi.Name(), ".html")) 102 | if err != nil { 103 | return fmt.Errorf("create dictzip entry: %w", err) 104 | } 105 | 106 | if _, err := io.Copy(fw, fr); err != nil { 107 | return fmt.Errorf("write file: %w", err) 108 | } 109 | 110 | return nil 111 | }(); err != nil { 112 | return fmt.Errorf("add dicthtml %#v: %w", fi.Name(), err) 113 | } 114 | default: 115 | if err := func() error { 116 | fr, err := os.OpenFile(filepath.Join(dir, fi.Name()), os.O_RDONLY, 0) 117 | if err != nil { 118 | return fmt.Errorf("open file: %w", err) 119 | } 120 | defer fr.Close() 121 | 122 | fw, err := w.CreateFile(strings.TrimSuffix(fi.Name(), ".html")) 123 | if err != nil { 124 | return fmt.Errorf("create dictzip entry: %w", err) 125 | } 126 | 127 | if _, err := io.Copy(fw, fr); err != nil { 128 | return fmt.Errorf("write file: %w", err) 129 | } 130 | 131 | return nil 132 | }(); err != nil { 133 | return fmt.Errorf("add file %#v: %w", fi.Name(), err) 134 | } 135 | } 136 | } 137 | 138 | if err := func() error { 139 | fr, err := os.OpenFile(filepath.Join(dir, "words"), os.O_RDONLY, 0) 140 | if err != nil { 141 | return fmt.Errorf("open words file: %w", err) 142 | } 143 | defer fr.Close() 144 | 145 | sc := bufio.NewScanner(fr) 146 | for sc.Scan() { 147 | if !utf8.Valid(sc.Bytes()) { 148 | return fmt.Errorf("invalid word: %#v", sc.Text()) 149 | } 150 | if word := strings.TrimSpace(sc.Text()); len(word) != 0 { 151 | if err := w.AddWord(word); err != nil { 152 | return fmt.Errorf("add word %#v: %s", word, err) 153 | } 154 | } 155 | } 156 | if sc.Err() != nil { 157 | return fmt.Errorf("read words file: %w", err) 158 | } 159 | 160 | return nil 161 | }(); err != nil { 162 | return fmt.Errorf("add words index: %w", err) 163 | } 164 | 165 | return nil 166 | } 167 | -------------------------------------------------------------------------------- /kobodict/fs_test.go: -------------------------------------------------------------------------------- 1 | package kobodict 2 | 3 | // TODO(v1) 4 | -------------------------------------------------------------------------------- /kobodict/marisa.go: -------------------------------------------------------------------------------- 1 | package kobodict 2 | 3 | import "io" 4 | 5 | // Marisa is used by Reader and Writer for reading/writing Marisa tries. It is 6 | // automatically set on supported platforms if 7 | // github.com/pgaskin/dictutil/kobodict/marisa is imported, but can be 8 | // overridden manually. 9 | var Marisa interface { 10 | MarisaReader 11 | MarisaWriter 12 | } 13 | 14 | // MarisaReader represents a simplified abstraction for reading Marisa tries. 15 | type MarisaReader interface { 16 | ReadAll(io.Reader) ([]string, error) 17 | } 18 | 19 | // MarisaWriter represents a simplified abstraction for writing Marisa tries. 20 | type MarisaWriter interface { 21 | WriteAll(io.Writer, []string) error 22 | } 23 | -------------------------------------------------------------------------------- /kobodict/marisa/marisa.go: -------------------------------------------------------------------------------- 1 | // Package marisa is imported with _ to enable marisa for the kobodict, if 2 | // supported. It is in a separate package so functions in kobodict which don't 3 | // require marisa can be used without compiling it. As an alternative to 4 | // importing this package, you can provide your own implementation of marisa in 5 | // kobodict.Marisa. If imported, this package will fail to compile unless marisa 6 | // is available for your GOOS/GOARCH. 7 | package marisa 8 | 9 | import "github.com/pgaskin/dictutil/kobodict" 10 | 11 | // This is done so it can still be instantiated even if not implemented for the 12 | // current platform (it will be caught when assigning it to kobodict.Marisa), 13 | // named platform for better error messages. 14 | 15 | type platform struct{} 16 | 17 | func init() { 18 | kobodict.Marisa = new(platform) // platform-specific implementation 19 | } 20 | -------------------------------------------------------------------------------- /kobodict/marisa/marisa_cgo.go: -------------------------------------------------------------------------------- 1 | //+build cgo 2 | 3 | package marisa 4 | 5 | import ( 6 | "io" 7 | 8 | "github.com/pgaskin/dictutil/marisa" 9 | ) 10 | 11 | func (*platform) ReadAll(r io.Reader) (wd []string, err error) { 12 | return marisa.ReadAll(r) 13 | } 14 | 15 | func (*platform) WriteAll(w io.Writer, wd []string) (err error) { 16 | return marisa.WriteAll(w, wd) 17 | } 18 | -------------------------------------------------------------------------------- /kobodict/marisa/marisa_test.go: -------------------------------------------------------------------------------- 1 | package marisa 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha1" 6 | "encoding/hex" 7 | "io" 8 | "reflect" 9 | "runtime" 10 | "testing" 11 | 12 | "github.com/pgaskin/dictutil/kobodict" 13 | ) 14 | 15 | func TestMarisa(t *testing.T) { 16 | impl, ok := (interface{})(new(platform)).(interface { 17 | kobodict.MarisaReader 18 | kobodict.MarisaWriter 19 | }) 20 | if !ok { 21 | t.Skipf("warning: Marisa not supported on platform GOOS=%s GOARCH=%s and must be provided externally", runtime.GOOS, runtime.GOARCH) 22 | } 23 | 24 | w := []string{ 25 | "asd", 26 | "dfg", 27 | "sdf", 28 | } 29 | 30 | buf := bytes.NewBuffer(nil) 31 | if err := impl.WriteAll(buf, w); err != nil { 32 | t.Fatalf("unexpected error when writing trie: %v", err) 33 | } else if buf.Len() == 0 { 34 | t.Errorf("written trie is empty") 35 | } 36 | 37 | ss := sha1.New() 38 | 39 | nw, err := impl.ReadAll(io.TeeReader(buf, ss)) 40 | if err != nil { 41 | t.Fatalf("unexpected error when reading written trie: %v", err) 42 | } else if len(nw) == 0 { 43 | t.Errorf("read trie is empty") 44 | } else if !reflect.DeepEqual(nw, w) { 45 | t.Errorf("read tree: expected %+s, got %+s", w, nw) 46 | } 47 | 48 | if runtime.GOARCH == "amd64" { 49 | if x, y := hex.EncodeToString(ss.Sum(nil)), "ea7252fc4e86585dea884e4bcb5ce7be90676474"; x != y { 50 | t.Errorf("trie output is incorrect or non-determinstic, expected sha1 %s, got %s", y, x) 51 | } 52 | } else { 53 | t.Logf("skipping sha1 check on non-amd64 architecture, as the correct file differs slightly on each one (usually by ~4 bytes)") 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /kobodict/reader.go: -------------------------------------------------------------------------------- 1 | package kobodict 2 | 3 | import ( 4 | "archive/zip" 5 | "bytes" 6 | "compress/gzip" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "strings" 11 | ) 12 | 13 | // Reader provides access to the contents of a dictzip file. 14 | type Reader struct { 15 | Word []string 16 | Dicthtml []*ReaderDicthtml 17 | File []*ReaderFile 18 | z *zip.Reader 19 | d Decrypter 20 | } 21 | 22 | // ReaderDicthtml represents a dicthtml file from a Reader. 23 | type ReaderDicthtml struct { 24 | Name string 25 | Prefix string 26 | f *zip.File 27 | r *Reader 28 | } 29 | 30 | // ReaderDicthtml represents a raw file from a Reader (e.g. images). 31 | type ReaderFile struct { 32 | Name string 33 | f *zip.File 34 | r *Reader 35 | } 36 | 37 | // Decrypter decrypts dicthtml files. 38 | type Decrypter interface { 39 | // Decrypt decrypts the dicthtml bytes. It will only be called if the 40 | // dicthtml is not otherwise readable. An error should be returned if the 41 | // decryption itself encounters an error; the decryptor should not try to 42 | // judge if the resulting bytes are valid. 43 | Decrypt([]byte) ([]byte, error) 44 | } 45 | 46 | // NewReader returns a new dictzip reader which reads from r, with the given 47 | // file size. 48 | func NewReader(r io.ReaderAt, size int64) (*Reader, error) { 49 | zr, err := zip.NewReader(r, size) 50 | if err != nil { 51 | return nil, fmt.Errorf("open zip: %w", err) 52 | } 53 | 54 | kr := &Reader{ 55 | z: zr, 56 | } 57 | 58 | var found bool 59 | for _, zf := range zr.File { 60 | if zf.Name == "words" { 61 | if fr, err := zf.Open(); err != nil { 62 | return nil, fmt.Errorf("open words index: %w", err) 63 | } else if Marisa == nil { 64 | return nil, fmt.Errorf("no marisa bindings found") 65 | } else if kr.Word, err = Marisa.ReadAll(fr); err != nil { 66 | return nil, fmt.Errorf("read words index: %w", err) 67 | } 68 | found = true 69 | break 70 | } 71 | } 72 | if !found { 73 | return nil, fmt.Errorf("not a dictzip: no words index found") 74 | } 75 | 76 | for _, f := range zr.File { 77 | switch { 78 | case !f.Mode().IsRegular(): 79 | continue 80 | case f.Name == "words": 81 | continue 82 | case strings.Contains(f.Name, "/"): 83 | return nil, fmt.Errorf("read zip: illegal file %#v: contains slash (not in root dir)", f.Name) 84 | case strings.HasSuffix(f.Name, ".html"): 85 | kr.Dicthtml = append(kr.Dicthtml, &ReaderDicthtml{ 86 | Name: f.Name, 87 | Prefix: strings.TrimSuffix(f.Name, ".html"), 88 | f: f, 89 | r: kr, 90 | }) 91 | default: 92 | kr.File = append(kr.File, &ReaderFile{ 93 | Name: f.Name, 94 | f: f, 95 | r: kr, 96 | }) 97 | } 98 | } 99 | 100 | return kr, nil 101 | } 102 | 103 | // SetDecrypter sets the Decrypter used to decrypt encrypted dicthtml files. 104 | func (r *Reader) SetDecrypter(d Decrypter) { 105 | r.d = d 106 | } 107 | 108 | // Open returns an io.ReadCloser which reads the decoded dicthtml file. Multiple 109 | // files can be read at once. 110 | func (f *ReaderDicthtml) Open() (io.ReadCloser, error) { 111 | enc, err := func() (bool, error) { 112 | fr, err := f.f.Open() 113 | if err != nil { 114 | return false, fmt.Errorf("open zip entry: %v", err) 115 | } 116 | defer fr.Close() 117 | 118 | tmp := make([]byte, 2) 119 | if n, err := fr.Read(tmp); err != nil { 120 | return false, fmt.Errorf("read zip entry: %v", err) 121 | } else if n != len(tmp) { 122 | return false, fmt.Errorf("corrupt dicthtml: too short (%d)", n) 123 | } 124 | 125 | if tmp[0] == 0x1F && tmp[1] == 0x8B { 126 | return false, nil 127 | } 128 | 129 | if f.r.d == nil { 130 | return true, fmt.Errorf("corrupt or encrypted dicthtml: invalid header") 131 | } 132 | 133 | // maybe optimize this later? 134 | if buf, err := ioutil.ReadAll(io.MultiReader(bytes.NewReader(tmp), fr)); err != nil { 135 | return true, fmt.Errorf("read zip entry: %v", err) 136 | } else if dec, err := f.r.d.Decrypt(buf); err != nil { 137 | return true, fmt.Errorf("decrypt dicthtml: %v", err) 138 | } else if dec[0] != 0x1F || dec[1] != 0x8B { 139 | return true, fmt.Errorf("corrupt dicthtml or invalid encryption key: invalid header") 140 | } 141 | return true, nil 142 | }() 143 | if err != nil { 144 | return nil, err 145 | } 146 | 147 | fr, err := f.f.Open() 148 | if err != nil { 149 | return nil, fmt.Errorf("open zip entry: %v", err) 150 | } 151 | 152 | var dr io.Reader 153 | if enc { 154 | if buf, err := ioutil.ReadAll(fr); err != nil { 155 | return nil, fmt.Errorf("read zip entry: %v", err) 156 | } else if dec, err := f.r.d.Decrypt(buf); err != nil { 157 | return nil, fmt.Errorf("decrypt dicthtml: %v", err) 158 | } else if dec[0] != 0x1F || dec[1] != 0x8B { 159 | return nil, fmt.Errorf("corrupt dicthtml or invalid encryption key: invalid header") 160 | } else { 161 | dr = bytes.NewReader(dec) 162 | } 163 | } else { 164 | dr = fr 165 | } 166 | 167 | zr, err := gzip.NewReader(dr) 168 | if err != nil { 169 | return nil, fmt.Errorf("decompress dicthtml: %v", err) 170 | } 171 | 172 | return &funcReadCloser{ 173 | Reader: zr, 174 | Closer: func() error { 175 | if err := zr.Close(); err != nil { 176 | fr.Close() 177 | return err 178 | } 179 | return fr.Close() 180 | }, 181 | }, nil 182 | } 183 | 184 | // Open returns an io.ReadCloser which reads the contents of the file. Multiple 185 | // files can be read at once. 186 | func (f *ReaderFile) Open() (io.ReadCloser, error) { 187 | return f.f.Open() 188 | } 189 | 190 | type funcReadCloser struct { 191 | io.Reader 192 | Closer func() error 193 | } 194 | 195 | func (f *funcReadCloser) Close() error { 196 | if f.Closer != nil { 197 | return f.Closer() 198 | } 199 | return nil 200 | } 201 | -------------------------------------------------------------------------------- /kobodict/reader_test.go: -------------------------------------------------------------------------------- 1 | package kobodict 2 | 3 | // TODO(v1) 4 | -------------------------------------------------------------------------------- /kobodict/util.go: -------------------------------------------------------------------------------- 1 | // Package kobodict implements reading, writing, and other utilities for Kobo 2 | // dictionaries (v2). 3 | // 4 | // A marisa implementation must be provided by 5 | // github.com/pgaskin/kobodict/marisa or a custom one if Writer or Reader is 6 | // used. 7 | package kobodict 8 | 9 | import ( 10 | "strings" 11 | "unicode" 12 | ) 13 | 14 | // NormalizeWordReference normalizes a word for use in an dicthtml headword 15 | // (= 2 || c == '\x00' { // limit to 2 chars, also cut at null 49 | pfx = pfx[:i] // trim up to current char 50 | break 51 | } 52 | pfx[i] = unicode.ToLower(c) // this includes accented chars 53 | } 54 | 55 | for len(pfx) != 0 { 56 | if unicode.IsSpace(pfx[0]) { 57 | pfx = pfx[1:] // trim left space 58 | } else { 59 | break 60 | } 61 | } 62 | 63 | for len(pfx) != 0 { 64 | if unicode.IsSpace(pfx[len(pfx)-1]) { 65 | pfx = pfx[:len(pfx)-1] // trim right space 66 | } else { 67 | break 68 | } 69 | } 70 | 71 | if len(pfx) == 0 { 72 | return "11" // if empty, return "11" 73 | } 74 | 75 | if !unicode.Is(unicode.Cyrillic, pfx[0]) { 76 | for len(pfx) < 2 { 77 | pfx = append(pfx, 'a') // pad right with 'a's to 2 chars 78 | } 79 | if !unicode.IsLetter(pfx[0]) || !unicode.IsLetter(pfx[1]) { 80 | return "11" // if neither of the first 2 chars are letters, return "11" 81 | } 82 | } 83 | 84 | return string(pfx) 85 | } 86 | 87 | // wordPrefix gets the prefix of a word for sharding dicthtml files. 88 | // 89 | // This is not to be used with Kanji, as those are handled by a separate 90 | // function for Japanese dictionaries. 91 | // 92 | // The logic is reversed from DictionaryParser::htmlForWord in libnickel. It 93 | // matches it as closely as possible. 94 | func wordPrefix(w string) string { 95 | // w 96 | // QString::toLower() 97 | w = strings.ToLower(w) 98 | 99 | // QString::leftRef(2) 100 | if len(w) > 2 { 101 | w = string([]rune(w)[:2]) 102 | } 103 | 104 | // QString::trimmed() 105 | w = strings.TrimSpace(w) 106 | 107 | // simplify the following code by converting to rune slice 108 | r := []rune(w) 109 | 110 | // A null byte is a valid Unicode character, but in C, it's treated as 111 | // the end of a string. To keep compatibility with libnickel, we need to 112 | // end a string there if necessary. 113 | for i, c := range r { 114 | if c == '\x00' { 115 | r = r[:i] 116 | break 117 | } 118 | } 119 | 120 | // DictionaryParser::isCyrillic(w[0]) 121 | // skip if true 122 | if !(len(r) != 0 && unicode.Is(unicode.Cyrillic, r[0])) { 123 | // add an 'a' for right padding if not 2 chars 124 | if len(r) != 2 { 125 | r = append(r, 'a') 126 | } 127 | } 128 | 129 | // DictionaryParser::isCyrillic(w[0]) 130 | // skip if != false 131 | switch { 132 | case !(len(r) != 0 && unicode.Is(unicode.Cyrillic, r[0])): 133 | // inlined QChar::isLetter(w[0]), QChar::isLetter(w[1]), unnecessary length check 134 | // skip if both true 135 | if (len(r) >= 1 && unicode.IsLetter(r[0])) && (len(r) >= 2 && unicode.IsLetter(r[1])) { 136 | break 137 | } 138 | fallthrough 139 | case len(r) == 0: 140 | // w = QString::fromLatin1_helper("11"..., 2) 141 | return "11" 142 | } 143 | 144 | return string(r) 145 | } 146 | -------------------------------------------------------------------------------- /kobodict/util_test.go: -------------------------------------------------------------------------------- 1 | package kobodict 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | ) 7 | 8 | func TestNormalizeWordReference(t *testing.T) { 9 | for _, tc := range []struct { 10 | v bool 11 | i, o string 12 | }{ 13 | {true, "Asd", "asd"}, 14 | {false, "Asd", "Asd"}, 15 | {true, " Asd", "asd"}, 16 | {false, " Asd", "Asd"}, 17 | {true, " Asd ", "asd"}, 18 | {false, " Asd ", "Asd"}, 19 | {true, " Asd \n", "asd"}, 20 | {false, " Asd \n", "Asd"}, 21 | {true, " Ȃsd \n", "ȃsd"}, 22 | {false, " Ȃsd \n", "Ȃsd"}, 23 | } { 24 | t.Logf("word %#v [variant:%t] (%#v)", tc.i, tc.v, tc.o) 25 | if o := NormalizeWordReference(tc.i, tc.v); o != tc.o { 26 | t.Errorf(" got %#v", o) 27 | } 28 | } 29 | } 30 | 31 | var tcs = []struct{ w, p string }{ 32 | // dicthtml-en 33 | {"test", "te"}, 34 | {"a-", "11"}, 35 | {"-an", "11"}, 36 | {"GB", "gb"}, 37 | 38 | // dicthtml-fr 39 | {"ébahir", "éb"}, 40 | {"à", "àa"}, 41 | {"a1", "11"}, 42 | {"ô", "ôa"}, 43 | {"kébab", "ké"}, 44 | {"aérer", "aé"}, 45 | {"living-room", "li"}, 46 | 47 | // dicthtml-ja 48 | // Note, Kanji not currently implemented, so not testing (note, the logic 49 | // is in a separate function, anyways). 50 | // {"あ", "あ"}, 51 | // {"アークとう", "アー"}, 52 | 53 | // generated by dictword-test: spaces 54 | {" x", "xa"}, 55 | {" ", "11"}, 56 | {"x ", "xa"}, 57 | {" ", "11"}, 58 | {" ", "11"}, 59 | {"\t\t", "11"}, 60 | {"\t\f\t", "11"}, 61 | {"x ", "xa"}, 62 | {" xx", "xa"}, 63 | 64 | // generated by dictword-test: spaces where trim/prefix order matters 65 | {" x", "11"}, 66 | {" xy", "11"}, 67 | {" xyz", "11"}, 68 | {"x z", "xa"}, 69 | {"x z", "xa"}, 70 | 71 | // generated by dictword-test: cyrillic 72 | {" д", "д"}, 73 | {"д ", "д"}, 74 | {" ", "11"}, 75 | {" ", "11"}, 76 | {" ", "11"}, 77 | {" дд", "д"}, 78 | {"д ", "д"}, 79 | {"д", "д"}, 80 | {"aд", "aд"}, 81 | {"дa", "дa"}, 82 | {"aдa", "aд"}, 83 | {"дaд", "дa"}, 84 | 85 | // generated by dictword-test: uppercase accented letters 86 | {"Ȅe", "ȅe"}, 87 | {"eȄ", "eȅ"}, 88 | {"Ȅ", "ȅa"}, 89 | {"Ȅ!", "11"}, 90 | 91 | // generated by dictword-test: cjk 92 | {" 未", "未a"}, 93 | {" 未", "11"}, 94 | {"未", "未a"}, 95 | {"未未", "未未"}, 96 | {"x未", "x未"}, 97 | {"未x", "未x"}, 98 | {"xy未", "xy"}, 99 | {"还没", "还没"}, 100 | 101 | // generated by dictword-test: misc 102 | {"!", "11"}, 103 | {"!!", "11"}, 104 | {"!!!", "11"}, 105 | {"x!", "11"}, 106 | {"x!!", "11"}, 107 | {"xx!", "xx"}, 108 | {"xxx!", "xx"}, 109 | {" !", "11"}, 110 | {" !!", "11"}, 111 | {" !!!", "11"}, 112 | {" !", "11"}, 113 | {" !!", "11"}, 114 | {" !!!", "11"}, 115 | {" x!", "xa"}, 116 | {" x!!", "xa"}, 117 | {" xx!", "xa"}, 118 | {" xxx!", "xa"}, 119 | 120 | // synthetic 121 | {"x\x00y", "xa"}, 122 | {"\x00xy", "11"}, 123 | } 124 | 125 | func TestWordPrefix(t *testing.T) { 126 | for _, tc := range tcs { 127 | t.Logf("word %#v (%#v)", tc.w, tc.p) 128 | if p := wordPrefix(tc.w); p != tc.p { 129 | t.Errorf(" got (original version) %#v", p) 130 | } 131 | if p := WordPrefix(tc.w); p != tc.p { 132 | t.Errorf(" got (simplified version) %#v", p) 133 | } 134 | } 135 | } 136 | 137 | func BenchmarkWordPrefix(b *testing.B) { 138 | for _, tcf := range []struct { 139 | n string 140 | fn func(string) string 141 | }{ 142 | {"Orig/", wordPrefix}, 143 | {"Smpl/", WordPrefix}, 144 | } { 145 | // all test cases 146 | b.Run(tcf.n+"All"+strconv.Itoa(len(tcs)), func(b *testing.B) { 147 | for i := 0; i < b.N; i++ { 148 | for _, tc := range tcs { 149 | tcf.fn(tc.w) 150 | } 151 | } 152 | }) 153 | 154 | // near-worst possible case 155 | b.Run(tcf.n+"Worst", func(b *testing.B) { 156 | for i := 0; i < b.N; i++ { 157 | tcf.fn(" 还д 没") 158 | } 159 | }) 160 | 161 | // normal case 162 | b.Run(tcf.n+"Normal", func(b *testing.B) { 163 | for i := 0; i < b.N; i++ { 164 | tcf.fn("Test") 165 | } 166 | }) 167 | 168 | // best case 169 | b.Run(tcf.n+"Best", func(b *testing.B) { 170 | for i := 0; i < b.N; i++ { 171 | tcf.fn("aa") 172 | } 173 | }) 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /kobodict/writer.go: -------------------------------------------------------------------------------- 1 | package kobodict 2 | 3 | import ( 4 | "archive/zip" 5 | "bytes" 6 | "compress/gzip" 7 | "fmt" 8 | "io" 9 | "sort" 10 | "strings" 11 | ) 12 | 13 | // Writer creates dictzips. It does not do any validation; it only does what it 14 | // is told. It is up to the user to ensure the input is valid. 15 | type Writer struct { 16 | z *zip.Writer 17 | e Encrypter 18 | words map[string]struct{} // doesn't take up space for values 19 | used map[string]struct{} 20 | closed bool 21 | last io.WriteCloser 22 | } 23 | 24 | // Encrypter encrypts dicthtml files. 25 | type Encrypter interface { 26 | // Encrypt encrypts the provided bytes. 27 | Encrypt([]byte) ([]byte, error) 28 | } 29 | 30 | // NewWriter creates a dictzip writer writing to w. 31 | func NewWriter(w io.Writer) *Writer { 32 | return &Writer{ 33 | z: zip.NewWriter(w), 34 | words: map[string]struct{}{}, 35 | used: map[string]struct{}{}, 36 | } 37 | } 38 | 39 | // AddWord normalizes and adds a word to the index. If the word has already been 40 | // added, it does nothing. 41 | func (w *Writer) AddWord(word string) error { 42 | if w.closed { 43 | return fmt.Errorf("write to closed writer") 44 | } 45 | w.words[strings.TrimSpace(word)] = struct{}{} // index words aren't normalized except for trimming spaces 46 | return nil 47 | } 48 | 49 | // CreateDicthtml adds a dicthtml file for the specified prefix and returns a 50 | // writer which is valid until the next file is created. 51 | func (w *Writer) CreateDicthtml(prefix string) (io.Writer, error) { 52 | if strings.Contains(prefix, "/") { 53 | return nil, fmt.Errorf("invalid prefix: must not contain slashes") 54 | } 55 | if w.closed { 56 | return nil, fmt.Errorf("writer already closed") 57 | } 58 | if w.last != nil { 59 | if err := w.last.Close(); err != nil { 60 | return nil, fmt.Errorf("close last file writer: %w", err) 61 | } 62 | w.last = nil 63 | } 64 | 65 | filename := prefix + ".html" 66 | if _, ok := w.used[filename]; ok { 67 | return nil, fmt.Errorf("file %#v already exists in dictzip", filename) 68 | } 69 | 70 | fw, err := w.z.Create(filename) 71 | if err != nil { 72 | return nil, fmt.Errorf("create zip entry: %w", err) 73 | } 74 | 75 | if w.e != nil { 76 | ew := newEncryptWriter(w.e, fw) 77 | zw := gzip.NewWriter(ew) 78 | 79 | w.last = &funcWriteCloser{ 80 | Writer: zw, 81 | Closer: func() error { 82 | if err := zw.Close(); err != nil { 83 | return err 84 | } 85 | return ew.Close() 86 | }, 87 | } 88 | } else { 89 | w.last = gzip.NewWriter(fw) 90 | } 91 | 92 | w.used[filename] = struct{}{} 93 | return w.last, nil 94 | } 95 | 96 | // CreateFile adds a raw file with the specified name. Note that Kobo only 97 | // supports GIF and JPEG files starting with the "GIF" and "JFIF" magic, and the 98 | // treatment of other files is undefined. In addition, subdirectories are not 99 | // supported. The behaviour is undefined if a dicthtml file is added this way. 100 | func (w *Writer) CreateFile(filename string) (io.Writer, error) { 101 | if strings.Contains(filename, "/") || strings.Contains(filename, "\\") { 102 | return nil, fmt.Errorf("invalid filename: must not contain slashes") 103 | } else if strings.Contains(filename, "words") { 104 | return nil, fmt.Errorf("invalid filename: must not be 'words'") 105 | } else if _, ok := w.used[filename]; ok { 106 | return nil, fmt.Errorf("file %#v already exists in dictzip", filename) 107 | } 108 | if w.last != nil { 109 | if err := w.last.Close(); err != nil { 110 | return nil, fmt.Errorf("close last file writer: %w", err) 111 | } 112 | w.last = nil 113 | } 114 | 115 | fw, err := w.z.Create(filename) 116 | if err != nil { 117 | return nil, fmt.Errorf("create zip entry: %w", err) 118 | } 119 | 120 | w.last = &funcWriteCloser{ 121 | Writer: fw, 122 | Closer: nil, 123 | } 124 | w.used[filename] = struct{}{} 125 | return w.last, nil 126 | } 127 | 128 | // Exists checks if a file already exists in the dictzip with the specified name. 129 | func (w *Writer) Exists(fn string) bool { 130 | _, ok := w.used[fn] 131 | return ok 132 | } 133 | 134 | // Close writes the marisa index and the zip footer. The error should not be 135 | // ignored. It does not close the underlying writer. 136 | func (w *Writer) Close() error { 137 | if w.closed { 138 | return fmt.Errorf("writer already closed") 139 | } 140 | if w.last != nil { 141 | if err := w.last.Close(); err != nil { 142 | return fmt.Errorf("close last file writer: %w", err) 143 | } 144 | w.last = nil 145 | } 146 | 147 | var words []string 148 | for word := range w.words { 149 | words = append(words, word) 150 | } 151 | sort.Strings(words) 152 | 153 | if fw, err := w.z.Create("words"); err != nil { 154 | return fmt.Errorf("create index zip entry: %w", err) 155 | } else if Marisa == nil { 156 | return fmt.Errorf("no marisa bindings found") 157 | } else if err := Marisa.WriteAll(fw, words); err != nil { 158 | return fmt.Errorf("write index: %w", err) 159 | } 160 | 161 | if err := w.z.Close(); err != nil { 162 | return fmt.Errorf("close zip: %w", err) 163 | } 164 | return nil 165 | } 166 | 167 | // SetEncrypter sets the Encrypter used to encrypt dicthtml files. This must be 168 | // will only apply to dicthtml files added after the encrypter is set. 169 | func (w *Writer) SetEncrypter(e Encrypter) { 170 | w.e = e 171 | } 172 | 173 | type encryptWriter struct { 174 | e Encrypter 175 | w io.Writer 176 | b *bytes.Buffer 177 | c bool 178 | } 179 | 180 | func newEncryptWriter(e Encrypter, w io.Writer) io.WriteCloser { 181 | return &encryptWriter{ 182 | e: e, 183 | w: w, 184 | b: bytes.NewBuffer(nil), 185 | c: false, 186 | } 187 | } 188 | 189 | func (e encryptWriter) Write(buf []byte) (n int, err error) { 190 | if e.c { 191 | return 0, fmt.Errorf("write to closed writer") 192 | } 193 | return e.b.Write(buf) 194 | } 195 | 196 | // Close encrypts and writes the buffer to the underlying writer. The error 197 | // should be checked. 198 | func (e encryptWriter) Close() error { 199 | if e.c { 200 | return fmt.Errorf("writer already closed") 201 | } 202 | if buf, err := e.e.Encrypt(e.b.Bytes()); err != nil { 203 | return fmt.Errorf("encrypt bytes: %w", err) 204 | } else if _, err := e.w.Write(buf); err != nil { 205 | return fmt.Errorf("write encrypted bytes: %w", err) 206 | } 207 | return nil 208 | } 209 | 210 | type funcWriteCloser struct { 211 | io.Writer 212 | Closer func() error 213 | } 214 | 215 | func (f *funcWriteCloser) Close() error { 216 | if f.Closer != nil { 217 | return f.Closer() 218 | } 219 | return nil 220 | } 221 | -------------------------------------------------------------------------------- /kobodict/writer_test.go: -------------------------------------------------------------------------------- 1 | package kobodict 2 | 3 | // TODO(v1) 4 | -------------------------------------------------------------------------------- /marisa/libmarisa_generate.go: -------------------------------------------------------------------------------- 1 | //+build libmarisa_generate 2 | 3 | package main 4 | 5 | import ( 6 | "archive/tar" 7 | "bytes" 8 | "compress/gzip" 9 | "fmt" 10 | "io" 11 | "io/ioutil" 12 | "net/http" 13 | "os" 14 | "path" 15 | "regexp" 16 | "strings" 17 | ) 18 | 19 | func main() { 20 | url := "https://github.com/s-yata/marisa-trie/archive/970b20c141f11d9d7572a6bb8d0488f2e0520e22.tar.gz" 21 | version := "970b20c" 22 | 23 | if files, err := tarball(url); err != nil { 24 | fmt.Fprintf(os.Stderr, "Error: download tarball %#v: %v\n", url, err) 25 | os.Exit(1) 26 | return 27 | } else if err := func() error { 28 | if mr, err := libmarisa(files, version); err != nil { 29 | return err 30 | } else if mf, err := os.OpenFile("libmarisa.cc", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644); err != nil { 31 | return err 32 | } else if _, err := io.Copy(mf, mr); err != nil { 33 | mf.Close() 34 | return err 35 | } else { 36 | return mf.Close() 37 | } 38 | }(); err != nil { 39 | fmt.Fprintf(os.Stderr, "Error: generate libmarisa.cc: %v\n", err) 40 | os.Exit(1) 41 | return 42 | } else if err := func() error { 43 | if mr, err := hmarisa(files, version); err != nil { 44 | return err 45 | } else if mf, err := os.OpenFile("libmarisa.h", os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644); err != nil { 46 | return err 47 | } else if _, err := io.Copy(mf, mr); err != nil { 48 | mf.Close() 49 | return err 50 | } else { 51 | return mf.Close() 52 | } 53 | }(); err != nil { 54 | fmt.Fprintf(os.Stderr, "Error: generate libmarisa.h: %v\n", err) 55 | os.Exit(1) 56 | return 57 | } 58 | } 59 | func hmarisa(files map[string][]byte, version string) (io.Reader, error) { 60 | marisaH, err := resolve(files, []string{ 61 | "include/marisa.h", 62 | }, "include", "lib") 63 | if err != nil { 64 | return nil, err 65 | } 66 | 67 | fmt.Printf("Generating libmarisa.h\n") 68 | return io.MultiReader( 69 | // A custom header. 70 | strings.NewReader("// AUTOMATICALLY GENERATED, DO NOT EDIT!\n"), 71 | strings.NewReader("// merged from marisa-trie "+version+".\n"), 72 | // Include the license info. 73 | bytes.NewReader([]byte{'\n', '/', '/', ' '}), 74 | bytes.NewReader(bytes.ReplaceAll(files["COPYING.md"], []byte{'\n'}, []byte{'\n', '/', '/', ' '})), 75 | bytes.NewReader([]byte{'\n', '\n'}), 76 | // Include the header. 77 | bytes.NewReader(marisaH), 78 | ), nil 79 | } 80 | 81 | func libmarisa(files map[string][]byte, version string) (io.Reader, error) { 82 | marisaGrimoireIOLib, err := resolve(files, []string{ 83 | "lib/marisa/grimoire/io/mapper.cc", 84 | "lib/marisa/grimoire/io/reader.cc", 85 | "lib/marisa/grimoire/io/writer.cc", 86 | }, "include", "lib") 87 | if err != nil { 88 | return nil, err 89 | } 90 | 91 | marisaGrimoireTrieLib, err := resolve(files, []string{ 92 | "lib/marisa/grimoire/trie/tail.cc", 93 | "lib/marisa/grimoire/trie/louds-trie.cc", 94 | }, "include", "lib") 95 | if err != nil { 96 | return nil, err 97 | } 98 | 99 | marisaGrimoireVectorLib, err := resolve(files, []string{ 100 | "lib/marisa/grimoire/vector/bit-vector.cc", 101 | }, "include", "lib") 102 | if err != nil { 103 | return nil, err 104 | } 105 | 106 | marisaLib, err := resolve(files, []string{ 107 | "lib/marisa/agent.cc", 108 | "lib/marisa/keyset.cc", 109 | "lib/marisa/trie.cc", 110 | }, "include", "lib") 111 | if err != nil { 112 | return nil, err 113 | } 114 | 115 | fmt.Printf("Generating libmarisa.cc\n") 116 | return io.MultiReader( 117 | // A custom header. 118 | strings.NewReader("// AUTOMATICALLY GENERATED, DO NOT EDIT!\n"), 119 | strings.NewReader("// merged from marisa-trie "+version+".\n"), 120 | // Include the license info. 121 | bytes.NewReader([]byte{'\n', '/', '/', ' '}), 122 | bytes.NewReader(bytes.ReplaceAll(files["COPYING.md"], []byte{'\n'}, []byte{'\n', '/', '/', ' '})), 123 | bytes.NewReader([]byte{'\n', '\n'}), 124 | // Include the warnings from the Makefile.am CXXFLAGS. 125 | // - Note that Clang also recognizes the GCC pragmas. 126 | strings.NewReader("#pragma GCC diagnostic warning \"-Wall\"\n"), 127 | strings.NewReader("#pragma GCC diagnostic warning \"-Weffc++\"\n"), 128 | strings.NewReader("#pragma GCC diagnostic warning \"-Wextra\"\n"), 129 | strings.NewReader("#pragma GCC diagnostic warning \"-Wconversion\"\n"), 130 | // Silence a warning. 131 | strings.NewReader("#pragma GCC diagnostic ignored \"-Wimplicit-fallthrough=\"\n"), 132 | // Include the libs themselves. 133 | bytes.NewReader(marisaGrimoireIOLib), 134 | bytes.NewReader(marisaGrimoireTrieLib), 135 | bytes.NewReader(marisaGrimoireVectorLib), 136 | bytes.NewReader(marisaLib), 137 | // Show info about the generated file. 138 | strings.NewReader("#line 1 \"libmarisa_generate.go\"\n"), 139 | strings.NewReader("#pragma GCC warning \"Using generated built-in marisa-trie "+version+".\"\n"), 140 | ), nil 141 | } 142 | 143 | func tarball(url string) (map[string][]byte, error) { 144 | fmt.Printf("Downloading tarball from %s\n", url) 145 | 146 | resp, err := http.Get(url) 147 | if err != nil { 148 | return nil, err 149 | } 150 | defer resp.Body.Close() 151 | 152 | zr, err := gzip.NewReader(resp.Body) 153 | if err != nil { 154 | return nil, err 155 | } 156 | 157 | var pfx string 158 | files := map[string][]byte{} 159 | 160 | tr := tar.NewReader(zr) 161 | for { 162 | fh, err := tr.Next() 163 | if err == io.EOF { 164 | break 165 | } else if err != nil { 166 | return nil, err 167 | } 168 | 169 | if fh.Name == "pax_global_header" || fh.FileInfo().IsDir() { 170 | continue 171 | } 172 | 173 | if pfx == "" { 174 | if strings.HasPrefix(fh.Name, "./") { 175 | pfx = "./" + strings.Split(fh.Name, "/")[1] + "/" 176 | } else { 177 | pfx = strings.Split(fh.Name, "/")[0] + "/" 178 | } 179 | } 180 | 181 | if !strings.HasPrefix(fh.Name, pfx) { 182 | return nil, fmt.Errorf("extract file %#v: doesn't have common prefix %#v", fh.Name, pfx) 183 | } 184 | 185 | buf, err := ioutil.ReadAll(tr) 186 | if err != nil { 187 | return nil, fmt.Errorf("extract file %#v: %w", fh.Name, err) 188 | } 189 | 190 | fn := strings.TrimPrefix(fh.Name, pfx) 191 | files[fn] = buf 192 | 193 | fmt.Printf(" [D] %s\n", fn) // downloaded 194 | } 195 | 196 | return files, nil 197 | } 198 | 199 | func resolve(files map[string][]byte, filenames []string, includePath ...string) (resolvedFile []byte, err error) { 200 | fmt.Printf("Resolving C* source files %s (against:%s) (I = included, S = preserved because not found, R = skipped because already included)\n", filenames, includePath) 201 | 202 | var resolveFn func(indent string, files map[string][]byte, filename string, buf []byte, done []string, includePath []string) (resolvedFile []byte, err error) 203 | resolveFn = func(indent string, files map[string][]byte, filename string, buf []byte, done []string, includePath []string) (resolvedFile []byte, err error) { 204 | defer func() { 205 | if rerr := recover(); rerr != nil { 206 | resolvedFile, err = nil, rerr.(error) 207 | } 208 | }() 209 | 210 | resolvedFile = regexp.MustCompile(`(?m)^\s*#\s*include\s+["'<][^"'>]+["'>]$`).ReplaceAllFunc(buf, func(importBuf []byte) []byte { 211 | fn := string(regexp.MustCompile(`["'<]([^"'>]+)["'>]`).FindSubmatch(importBuf)[1]) 212 | 213 | for _, ip := range includePath { 214 | ifn := path.Join(ip, fn) 215 | for _, dfn := range done { 216 | if m, _ := path.Match(dfn, ifn); m { 217 | fmt.Printf("%s[R] %s\n", indent, fn) // already included 218 | return nil 219 | } 220 | } 221 | 222 | ibuf, ok := files[ifn] 223 | if ok { 224 | fmt.Printf("%s[I] %s => %s\n", indent, fn, ifn) // include 225 | ibuf, err := resolveFn(indent+" ", files, ifn, ibuf, append(done, ifn), append(includePath, path.Dir(ifn))) 226 | if err != nil { 227 | panic(fmt.Errorf("resolve %#v: %w", ifn, err)) 228 | } 229 | return append(append([]byte{'\n', '\n'}, ibuf...), '\n', '\n') 230 | } 231 | } 232 | 233 | fmt.Printf("%s[S] %s\n", indent, fn) // preserve 234 | return importBuf 235 | }) 236 | 237 | return 238 | } 239 | 240 | for _, fn := range filenames { 241 | if buf, ok := files[fn]; !ok { 242 | return nil, fmt.Errorf("file %#v: not found", fn) 243 | } else if buf, err := resolveFn(" ", files, fn, buf, []string{fn}, append(includePath, path.Dir(fn))); err != nil { 244 | return nil, fmt.Errorf("file %v: %w", fn, err) 245 | } else { 246 | resolvedFile = append(resolvedFile, buf...) 247 | resolvedFile = append(resolvedFile, '\n', '\n') 248 | } 249 | } 250 | 251 | return resolvedFile, nil 252 | } 253 | -------------------------------------------------------------------------------- /marisa/marisa.cc: -------------------------------------------------------------------------------- 1 | #include