├── settings.gradle ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── makedictionary ├── src │ └── main │ │ ├── resources │ │ └── META-INF │ │ │ └── gradle-plugins │ │ │ └── com.anysoftkeyboard.tools.makedictionary.properties │ │ └── groovy │ │ └── com │ │ └── anysoftkeyboard │ │ └── tools │ │ └── makedictionary │ │ ├── MakeDictionaryTask.groovy │ │ ├── MakeDictionaryPlugin.groovy │ │ ├── BinaryDictionaryResourceNormalizer.java │ │ ├── MainClass.java │ │ ├── MakeBinaryDictionary.java │ │ └── XmlWriter.java └── build.gradle ├── generatewordslist ├── src │ └── main │ │ ├── resources │ │ └── META-INF │ │ │ └── gradle-plugins │ │ │ └── com.anysoftkeyboard.tools.makedictionary.properties │ │ └── groovy │ │ └── com │ │ └── anysoftkeyboard │ │ └── tools │ │ └── generatewordslist │ │ ├── WordsListGeneratorPlugin.groovy │ │ ├── GenerateWordsListTask.groovy │ │ ├── WordWithCount.java │ │ ├── GenerateWordsListFromAOSPTask.java │ │ ├── Parser.java │ │ ├── MergeWordsListTask.java │ │ └── XmlWriter.java └── build.gradle ├── DictionaryCreator-old └── src │ └── com │ └── anysoftkeyboard │ └── dictionarycreator │ ├── UI.java │ ├── ParserThread.java │ └── MainForm.java ├── .gitignore ├── packverifier-old └── src │ ├── ThemeDeclarationSAXParserHandler.java │ ├── InvalidPackConfiguration.java │ ├── KeyboardDeclarationSAXParserHandler.java │ ├── DictionaryDeclarationSAXParserHandler.java │ ├── FileCheckSumGenerator.java │ ├── BasicDeclarationSAXParserHandler.java │ ├── AndroidManifestSAXParserHandler.java │ └── Verifier.java ├── gradlew.bat ├── gradlew └── README.md /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'tools' 2 | include 'makedictionary' 3 | include 'generatewordslist' 4 | 5 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AnySoftKeyboard/AnySoftKeyboardTools/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /makedictionary/src/main/resources/META-INF/gradle-plugins/com.anysoftkeyboard.tools.makedictionary.properties: -------------------------------------------------------------------------------- 1 | implementation-class=com.anysoftkeyboard.tools.makedictionary.MakeDictionaryPlugin -------------------------------------------------------------------------------- /generatewordslist/src/main/resources/META-INF/gradle-plugins/com.anysoftkeyboard.tools.makedictionary.properties: -------------------------------------------------------------------------------- 1 | implementation-class=com.anysoftkeyboard.tools.generatewordslist.WordsListGeneratorPlugin -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Mon Nov 12 22:28:21 EST 2018 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.2-bin.zip 7 | -------------------------------------------------------------------------------- /makedictionary/build.gradle: -------------------------------------------------------------------------------- 1 | group 'com.anysoftkeyboard' 2 | version '1.0-SNAPSHOT' 3 | 4 | apply plugin: 'groovy' 5 | 6 | repositories { 7 | mavenCentral() 8 | } 9 | 10 | dependencies { 11 | compile 'com.android.tools.build:gradle:1.5.0' 12 | compile gradleApi() 13 | } 14 | -------------------------------------------------------------------------------- /DictionaryCreator-old/src/com/anysoftkeyboard/dictionarycreator/UI.java: -------------------------------------------------------------------------------- 1 | package com.anysoftkeyboard.dictionarycreator; 2 | 3 | public interface UI { 4 | void showErrorMessage(String message); 5 | void updateProgressState(String message, int precentage); 6 | 7 | void onEnded(); 8 | } 9 | -------------------------------------------------------------------------------- /generatewordslist/build.gradle: -------------------------------------------------------------------------------- 1 | group 'com.anysoftkeyboard' 2 | version '1.0-SNAPSHOT' 3 | 4 | apply plugin: 'groovy' 5 | 6 | sourceCompatibility = 1.8 7 | 8 | repositories { 9 | mavenCentral() 10 | } 11 | 12 | dependencies { 13 | compile gradleApi() 14 | 15 | compile 'org.jsoup:jsoup:1.9.1' 16 | } 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # built application files 2 | *.apk 3 | *.ap_ 4 | 5 | # files for the dex VM 6 | *.dex 7 | 8 | # Java class files 9 | *.class 10 | 11 | # generated files 12 | bin/ 13 | gen/ 14 | build/ 15 | 16 | #ide 17 | .idea 18 | .gradle 19 | *.iml 20 | # Local configuration file (sdk path, etc) 21 | local.properties 22 | -------------------------------------------------------------------------------- /packverifier-old/src/ThemeDeclarationSAXParserHandler.java: -------------------------------------------------------------------------------- 1 | import org.xml.sax.Attributes; 2 | 3 | 4 | public class ThemeDeclarationSAXParserHandler extends BasicDeclarationSAXParserHandler { 5 | 6 | 7 | protected ThemeDeclarationSAXParserHandler() { 8 | super("themes.xml", "Theme"); 9 | } 10 | 11 | @Override 12 | protected void verifyNode(Attributes attributes) { 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /packverifier-old/src/InvalidPackConfiguration.java: -------------------------------------------------------------------------------- 1 | import java.io.File; 2 | 3 | 4 | public class InvalidPackConfiguration extends RuntimeException { 5 | 6 | /** 7 | * 8 | */ 9 | private static final long serialVersionUID = -5455725626189958474L; 10 | 11 | public InvalidPackConfiguration(String filename, String error) { 12 | super(String.format("Configuration file '%s' is invalid! Error: %s", filename, error)); 13 | } 14 | 15 | public InvalidPackConfiguration(File filename, String error) { 16 | this(filename.getAbsolutePath(), error); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /makedictionary/src/main/groovy/com/anysoftkeyboard/tools/makedictionary/MakeDictionaryTask.groovy: -------------------------------------------------------------------------------- 1 | package com.anysoftkeyboard.tools.makedictionary 2 | 3 | import org.gradle.api.DefaultTask 4 | import org.gradle.api.tasks.TaskAction 5 | 6 | /** 7 | * Task to create a binary-dictionary readable by AnySoftKeyboard 8 | */ 9 | public class MakeDictionaryTask extends DefaultTask { 10 | 11 | File inputWordsListFile 12 | File resourcesFolder 13 | String prefix 14 | 15 | @TaskAction 16 | def makeDictionary() { 17 | if (resourcesFolder == null) resourcesFolder = new File(project.projectDir, "/src/main/res/") 18 | MainClass.buildDictionary(inputWordsListFile, resourcesFolder, prefix) 19 | } 20 | } -------------------------------------------------------------------------------- /packverifier-old/src/KeyboardDeclarationSAXParserHandler.java: -------------------------------------------------------------------------------- 1 | import org.xml.sax.Attributes; 2 | 3 | 4 | public class KeyboardDeclarationSAXParserHandler extends BasicDeclarationSAXParserHandler { 5 | 6 | protected KeyboardDeclarationSAXParserHandler() { 7 | super("keyboards.xml", "Keyboard"); 8 | } 9 | 10 | @Override 11 | protected void verifyNode(Attributes attributes) { 12 | /* iconResId="@drawable/ic_stat_he" 13 | * layoutResId="@xml/heb_qwerty" 14 | * defaultDictionaryLocale="iw" 15 | */ 16 | verifyValidDrawableResId(attributes, "iconResId"); 17 | verifyValidXmlResId(attributes, "layoutResId"); 18 | verifyValidAttribute(attributes, "defaultDictionaryLocale"); 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /packverifier-old/src/DictionaryDeclarationSAXParserHandler.java: -------------------------------------------------------------------------------- 1 | import org.xml.sax.Attributes; 2 | 3 | public class DictionaryDeclarationSAXParserHandler extends BasicDeclarationSAXParserHandler { 4 | 5 | public DictionaryDeclarationSAXParserHandler() { 6 | super("dictionaries.xml", "Dictionary"); 7 | } 8 | 9 | @Override 10 | protected void verifyNode(Attributes attributes) { 11 | /* 12 | * locale="iw" type="binary_resource" 13 | * dictionaryResourceId="@raw/he_main" 14 | * autoTextResourceId="@xml/he_autotext" -- optional 15 | */ 16 | 17 | verifyValidAttribute(attributes, "locale"); 18 | verifyValidAttribute(attributes, "type", new String[] { 19 | "binary_resource", "assets_file" 20 | }); 21 | if (attributes.getValue("type").equals("binary_resource")) 22 | verifyValidArrayResId(attributes, "dictionaryResourceId"); 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /generatewordslist/src/main/groovy/com/anysoftkeyboard/tools/generatewordslist/WordsListGeneratorPlugin.groovy: -------------------------------------------------------------------------------- 1 | import org.gradle.api.Plugin 2 | import org.gradle.api.Project 3 | 4 | /* 5 | * Copyright (C) 2016 AnySoftKeyboard 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); 8 | * you may not use this file except in compliance with the License. 9 | * You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, 15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * See the License for the specific language governing permissions and 17 | * limitations under the License. 18 | */ 19 | 20 | class WordsListGeneratorPlugin implements Plugin { 21 | 22 | @Override 23 | void apply(Project project) { 24 | } 25 | 26 | } -------------------------------------------------------------------------------- /packverifier-old/src/FileCheckSumGenerator.java: -------------------------------------------------------------------------------- 1 | import java.io.File; 2 | import java.io.FileInputStream; 3 | import java.io.IOException; 4 | import java.security.MessageDigest; 5 | import java.security.NoSuchAlgorithmException; 6 | 7 | 8 | public class FileCheckSumGenerator { 9 | 10 | public static String generateFileCheckSum(File file) throws NoSuchAlgorithmException, IOException { 11 | MessageDigest md = MessageDigest.getInstance("SHA1"); 12 | FileInputStream fis = new FileInputStream(file); 13 | byte[] dataBytes = new byte[1024]; 14 | 15 | int nread = 0; 16 | 17 | while ((nread = fis.read(dataBytes)) != -1) { 18 | md.update(dataBytes, 0, nread); 19 | }; 20 | fis.close(); 21 | 22 | byte[] mdbytes = md.digest(); 23 | 24 | //convert the byte to hex format 25 | StringBuffer sb = new StringBuffer(""); 26 | for (int i = 0; i < mdbytes.length; i++) { 27 | sb.append(Integer.toString((mdbytes[i] & 0xff) + 0x100, 16).substring(1)); 28 | } 29 | 30 | return sb.toString(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /makedictionary/src/main/groovy/com/anysoftkeyboard/tools/makedictionary/MakeDictionaryPlugin.groovy: -------------------------------------------------------------------------------- 1 | package com.anysoftkeyboard.tools.makedictionary 2 | 3 | import com.android.build.gradle.AppPlugin 4 | import org.gradle.api.Plugin 5 | import org.gradle.api.Project 6 | 7 | /* 8 | * Copyright (C) 2016 AnySoftKeyboard 9 | * 10 | * Licensed under the Apache License, Version 2.0 (the "License"); 11 | * you may not use this file except in compliance with the License. 12 | * You may obtain a copy of the License at 13 | * 14 | * http://www.apache.org/licenses/LICENSE-2.0 15 | * 16 | * Unless required by applicable law or agreed to in writing, software 17 | * distributed under the License is distributed on an "AS IS" BASIS, 18 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | * See the License for the specific language governing permissions and 20 | * limitations under the License. 21 | */ 22 | 23 | class MakeDictionaryPlugin implements Plugin { 24 | 25 | @Override 26 | void apply(Project project) { 27 | def hasAppPlugin = project.plugins.find { p -> p instanceof AppPlugin } 28 | if (!hasAppPlugin) { 29 | throw new IllegalStateException("The 'com.android.application' plugin is required.") 30 | } 31 | } 32 | 33 | } -------------------------------------------------------------------------------- /generatewordslist/src/main/groovy/com/anysoftkeyboard/tools/generatewordslist/GenerateWordsListTask.groovy: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 AnySoftKeyboard 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.anysoftkeyboard.tools.generatewordslist 17 | 18 | import org.gradle.api.DefaultTask 19 | import org.gradle.api.tasks.TaskAction 20 | import org.jsoup.Jsoup 21 | 22 | import java.nio.charset.Charset 23 | 24 | /** 25 | * Task to generate words-list XML file from a input 26 | */ 27 | public class GenerateWordsListTask extends DefaultTask { 28 | File[] inputFiles 29 | File outputWordsListFile 30 | 31 | char[] wordCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz".toCharArray() 32 | char[] additionalInnerCharacters = "'".toCharArray() 33 | 34 | Locale locale = Locale.US 35 | 36 | int maxWordsInList = Integer.MAX_VALUE 37 | 38 | @TaskAction 39 | def generateWordsList() { 40 | List inputTextFiles = new ArrayList<>() 41 | inputFiles.each { 42 | if (it.name.endsWith(".html") || it.name.endsWith(".htm")) { 43 | File wordsInputFile = File.createTempFile(it.name + "_stripped_html_", ".txt") 44 | String inputText = Jsoup.parse(it, "UTF-8").text() 45 | Writer writer = new OutputStreamWriter(new FileOutputStream(wordsInputFile), Charset.forName("UTF-8")) 46 | writer.write(inputText) 47 | writer.flush() 48 | writer.close() 49 | inputTextFiles.add(wordsInputFile) 50 | } else if (it.name.endsWith(".txt")) { 51 | inputTextFiles.add(it) 52 | } else { 53 | println("Skipping file %s, since it's not txt or html file.") 54 | } 55 | } 56 | 57 | Parser parser = new Parser(inputTextFiles, outputWordsListFile, wordCharacters, locale, additionalInnerCharacters, maxWordsInList) 58 | parser.parse() 59 | } 60 | } -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /generatewordslist/src/main/groovy/com/anysoftkeyboard/tools/generatewordslist/WordWithCount.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 AnySoftKeyboard 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.anysoftkeyboard.tools.generatewordslist; 17 | 18 | import java.util.HashMap; 19 | import java.util.Map; 20 | 21 | class WordWithCount implements Comparable { 22 | private final String mKey; 23 | private final Map mWordVariants = new HashMap<>(); 24 | private int mFreq; 25 | 26 | public WordWithCount(String word) { 27 | mKey = word.toLowerCase(); 28 | mWordVariants.put(word, 1); 29 | mFreq = 0; 30 | } 31 | 32 | public WordWithCount(String word, int frequency) { 33 | mKey = word.toLowerCase(); 34 | mWordVariants.put(word, 1); 35 | mFreq = frequency; 36 | } 37 | 38 | public String getKey() { 39 | return mKey; 40 | } 41 | 42 | public String getWord() { 43 | String mostUsedWord = mKey; 44 | int mostUsedValue = Integer.MIN_VALUE; 45 | for (Map.Entry variant : mWordVariants.entrySet()) { 46 | if (variant.getValue() > mostUsedValue) { 47 | mostUsedValue = variant.getValue(); 48 | mostUsedWord = variant.getKey(); 49 | } 50 | } 51 | 52 | return mostUsedWord; 53 | } 54 | 55 | public int getFreq() { 56 | return mFreq; 57 | } 58 | 59 | public void addFreq(String word) { 60 | if (mFreq < Integer.MAX_VALUE) mFreq++; 61 | mWordVariants.compute(word, (s, usages) -> usages == null? 1 : usages+1); 62 | } 63 | 64 | public void addOtherWord(WordWithCount wordWithCount) { 65 | mFreq = Math.max(mFreq, wordWithCount.mFreq); 66 | for (Map.Entry variant : mWordVariants.entrySet()) { 67 | mWordVariants.compute(variant.getKey(), (s, usages) -> usages == null? variant.getValue() : usages+variant.getValue()); 68 | } 69 | } 70 | 71 | @Override 72 | public int compareTo(WordWithCount o) { 73 | return o.mFreq - mFreq; 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /makedictionary/src/main/groovy/com/anysoftkeyboard/tools/makedictionary/BinaryDictionaryResourceNormalizer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 AnySoftKeyboard 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.anysoftkeyboard.tools.makedictionary; 18 | 19 | import java.io.*; 20 | 21 | /** 22 | * Compresses a list of words and frequencies into a tree structured binary dictionary. 23 | */ 24 | class BinaryDictionaryResourceNormalizer { 25 | private static final int DICT_FILE_CHUNK_SIZE = 1000 * 1000; 26 | private final File tempOutputFile; 27 | private final File outputFolder; 28 | private final File dict_id_array; 29 | private final String mPrefix; 30 | 31 | public BinaryDictionaryResourceNormalizer(File tempOutputFile, File outputFolder, File dict_id_array, String prefix) { 32 | this.tempOutputFile = tempOutputFile; 33 | this.outputFolder = outputFolder; 34 | this.dict_id_array = dict_id_array; 35 | mPrefix = prefix; 36 | } 37 | 38 | public void writeDictionaryIdsResource() throws IOException { 39 | splitOutputFile(tempOutputFile, outputFolder); 40 | } 41 | 42 | 43 | private int splitOutputFile(final File tempOutputFile, 44 | final File outputFolder) throws IOException { 45 | //output should be words_1.dict....words_n.dict 46 | InputStream inputStream = new FileInputStream(tempOutputFile); 47 | int file_postfix = 0; 48 | int current_output_file_size = 0; 49 | byte[] buffer = new byte[4 * 1024]; 50 | OutputStream outputStream = null; 51 | int read = 0; 52 | XmlWriter xml = new XmlWriter(dict_id_array); 53 | xml.writeEntity("resources"); 54 | xml.writeEntity("array").writeAttribute("name", mPrefix +"_words_dict_array"); 55 | 56 | while ((read = inputStream.read(buffer)) > 0) { 57 | if (outputStream != null && current_output_file_size >= DICT_FILE_CHUNK_SIZE) { 58 | outputStream.flush(); 59 | outputStream.close(); 60 | outputStream = null; 61 | } 62 | 63 | if (outputStream == null) { 64 | file_postfix++; 65 | xml.writeEntity("item").writeText("@raw/" + mPrefix + "_words_" +file_postfix).endEntity(); 66 | current_output_file_size = 0; 67 | File chunkFile = new File(outputFolder, mPrefix + "_words_" + file_postfix + ".dict"); 68 | outputStream = new FileOutputStream(chunkFile); 69 | System.out.println("Writing to dict file " + chunkFile.getAbsolutePath()); 70 | } 71 | 72 | outputStream.write(buffer, 0, read); 73 | current_output_file_size += read; 74 | } 75 | 76 | xml.endEntity(); 77 | xml.endEntity(); 78 | xml.close(); 79 | 80 | inputStream.close(); 81 | if (outputStream != null) { 82 | outputStream.flush(); 83 | outputStream.close(); 84 | } 85 | System.out.println("Done. Wrote " + file_postfix + " files."); 86 | 87 | return file_postfix; 88 | } 89 | } -------------------------------------------------------------------------------- /makedictionary/src/main/groovy/com/anysoftkeyboard/tools/makedictionary/MainClass.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2015 AnySoftKeyboard 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.anysoftkeyboard.tools.makedictionary; 18 | 19 | import org.xml.sax.SAXException; 20 | 21 | import java.io.File; 22 | import java.io.IOException; 23 | 24 | import javax.xml.parsers.ParserConfigurationException; 25 | 26 | public class MainClass { 27 | 28 | public static void main(String[] args) throws IOException, ParserConfigurationException, SAXException { 29 | if (args.length != 3) { 30 | System.out.println("Usage: makedictionary [path-to-input-file] [path-to-pack-resource-folder] [prefix]"); 31 | System.exit(1); 32 | } 33 | 34 | final File inputFile = new File(args[0]); 35 | final File resourcesFolder = new File(args[1]); 36 | 37 | buildDictionary(inputFile, resourcesFolder, args[2]); 38 | } 39 | 40 | public static void buildDictionary(final File inputFile, final File resourcesFolder, final String prefix) throws IOException, ParserConfigurationException, SAXException { 41 | if (!inputFile.isFile() || !inputFile.exists()) { 42 | throw new IllegalArgumentException("Could not find input file " + inputFile); 43 | } 44 | final File tempOutputFile = File.createTempFile("make_dictionary_temp", "bin"); 45 | 46 | if (!resourcesFolder.isDirectory() || !resourcesFolder.exists()) { 47 | throw new IllegalArgumentException("Could not find resource folder " + resourcesFolder); 48 | } 49 | 50 | final File outputFolder = new File(resourcesFolder, "raw/"); 51 | final File dict_id_array = new File(resourcesFolder, "values/" + prefix + "_words_dict_array.xml"); 52 | 53 | System.out.println("Reading words from input " + inputFile.getAbsolutePath()); 54 | System.out.println("Will store output files under " + outputFolder.getAbsolutePath() + ". Created raw folder? " + outputFolder.mkdirs()); 55 | System.out.println("Deleting previous versions..."); 56 | 57 | //deleting current files 58 | tempOutputFile.delete(); 59 | File[] dictFiles = outputFolder.listFiles((dir, name) -> name.endsWith(".dict")); 60 | if (dictFiles != null && dictFiles.length > 0) { 61 | for (File file : dictFiles) { 62 | file.delete(); 63 | } 64 | } 65 | dict_id_array.delete(); 66 | 67 | MakeBinaryDictionary maker = new MakeBinaryDictionary(inputFile.getAbsolutePath(), tempOutputFile.getAbsolutePath()); 68 | maker.makeDictionary(); 69 | 70 | if (!tempOutputFile.exists()) { 71 | throw new IOException("Failed to create binary dictionary file."); 72 | } 73 | 74 | if (tempOutputFile.length() <= 0) { 75 | throw new IOException("Failed to create binary dictionary file. Size zero."); 76 | } 77 | 78 | //now, if the file is larger than 1MB, I'll need to split it to 1MB chunks and rename them. 79 | BinaryDictionaryResourceNormalizer normalizer = new BinaryDictionaryResourceNormalizer(tempOutputFile, outputFolder, dict_id_array, prefix); 80 | normalizer.writeDictionaryIdsResource(); 81 | 82 | System.out.println("Done."); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /generatewordslist/src/main/groovy/com/anysoftkeyboard/tools/generatewordslist/GenerateWordsListFromAOSPTask.java: -------------------------------------------------------------------------------- 1 | package com.anysoftkeyboard.tools.generatewordslist; 2 | 3 | import org.gradle.api.DefaultTask; 4 | import org.gradle.api.tasks.TaskAction; 5 | 6 | import java.io.BufferedReader; 7 | import java.io.File; 8 | import java.io.FileInputStream; 9 | import java.io.FileOutputStream; 10 | import java.io.FileReader; 11 | import java.io.IOException; 12 | import java.io.InputStreamReader; 13 | import java.io.OutputStreamWriter; 14 | import java.nio.charset.Charset; 15 | import java.util.ArrayList; 16 | import java.util.Collections; 17 | import java.util.List; 18 | import java.util.regex.Matcher; 19 | import java.util.regex.Pattern; 20 | 21 | /** 22 | * Task to generate words-list XML file from a AOSP words-list file. 23 | * https://android.googlesource.com/platform/packages/inputmethods/LatinIME/+/master/dictionaries/ 24 | */ 25 | public class GenerateWordsListFromAOSPTask extends DefaultTask { 26 | private static final Pattern mWordLineRegex = Pattern.compile("^\\s*word=([\\w\\p{L}'\"-]+),f=(\\d+).*$"); 27 | 28 | private File inputFile; 29 | private File outputWordsListFile; 30 | private int maxWordsInList = 200000; 31 | 32 | @TaskAction 33 | public void generateWordsList() throws IOException { 34 | if (inputFile == null) 35 | throw new IllegalArgumentException("Please provide inputFile value."); 36 | if (!inputFile.isFile()) throw new IllegalArgumentException("inputFile must be a file!"); 37 | if (outputWordsListFile == null) 38 | throw new IllegalArgumentException("Please provide outputWordsListFile value."); 39 | 40 | final long inputSize = inputFile.length(); 41 | System.out.println("Reading input file " + inputFile.getName() + " (size " + inputSize + ")..."); 42 | BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(inputFile), Charset.forName("UTF-8"))); 43 | String wordDataLine; 44 | List parsedWords = new ArrayList<>(); 45 | int read = 0; 46 | while (null != (wordDataLine = reader.readLine())) { 47 | read += wordDataLine.length(); 48 | //word=heh,f=0,flags=,originalFreq=53,possibly_offensive=true 49 | Matcher matcher = mWordLineRegex.matcher(wordDataLine); 50 | if (matcher.matches()) { 51 | String word = matcher.group(1); 52 | int frequency = Integer.parseInt(matcher.group(2)); 53 | parsedWords.add(new WordWithCount(word, frequency)); 54 | if ((parsedWords.size() % 50000) == 0) { 55 | System.out.print("." + ((100 * read) / inputSize) + "%."); 56 | } 57 | } 58 | } 59 | 60 | System.out.print(".100%."); 61 | 62 | System.out.println("Sorting list of " + parsedWords.size() + " words..."); 63 | Collections.sort(parsedWords); 64 | 65 | OutputStreamWriter output = new OutputStreamWriter(new FileOutputStream(outputWordsListFile), Charset.forName("UTF-8")); 66 | Parser.createXml(parsedWords, output, maxWordsInList, true); 67 | 68 | output.flush(); 69 | output.close(); 70 | 71 | System.out.println("Done."); 72 | } 73 | 74 | public File getInputFile() { 75 | return inputFile; 76 | } 77 | 78 | public void setInputFile(File inputFile) { 79 | this.inputFile = inputFile; 80 | } 81 | 82 | public File getOutputWordsListFile() { 83 | return outputWordsListFile; 84 | } 85 | 86 | public void setOutputWordsListFile(File outputWordsListFile) { 87 | this.outputWordsListFile = outputWordsListFile; 88 | } 89 | 90 | public int getMaxWordsInList() { 91 | return maxWordsInList; 92 | } 93 | 94 | public void setMaxWordsInList(int maxWordsInList) { 95 | this.maxWordsInList = maxWordsInList; 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /packverifier-old/src/BasicDeclarationSAXParserHandler.java: -------------------------------------------------------------------------------- 1 | import org.xml.sax.Attributes; 2 | import org.xml.sax.SAXException; 3 | import org.xml.sax.helpers.DefaultHandler; 4 | 5 | import java.util.UUID; 6 | 7 | public abstract class BasicDeclarationSAXParserHandler extends DefaultHandler { 8 | private final String mNodeName; 9 | private final String mFileName; 10 | 11 | protected BasicDeclarationSAXParserHandler(String fileName, String nodeName) { 12 | mFileName = fileName; 13 | mNodeName = nodeName; 14 | } 15 | 16 | @Override 17 | public void startElement(String uri, String localName, String qName, Attributes attributes) 18 | throws SAXException { 19 | super.startElement(uri, localName, qName, attributes); 20 | if (qName.equals(mNodeName)) { 21 | verifyNodeBasics(attributes); 22 | verifyNode(attributes); 23 | } 24 | } 25 | 26 | protected abstract void verifyNode(Attributes attributes); 27 | 28 | private void verifyNodeBasics(Attributes attributes) { 29 | /* id="7263630a-3284-4f46-8137-eef38adb5649" 30 | * nameResId="@string/keyboard" 31 | * description="Created by Menny Even Danan" 32 | * index="1" 33 | */ 34 | 35 | 36 | String id = attributes.getValue("id"); 37 | if ( id == null || 38 | id.length() == 0 || 39 | id.contains("change_me")) { 40 | try { 41 | UUID uuid = UUID.fromString(id); 42 | if (uuid == null) throw new RuntimeException();//so I'll catch it. 43 | } catch(Exception e) { 44 | throw new InvalidPackConfiguration(mFileName, "Node "+mNodeName+" has an invalid ID!"); 45 | } 46 | } 47 | 48 | verifyValidStringResId(attributes, "nameResId"); 49 | verifyValidAttribute(attributes, "description"); 50 | } 51 | 52 | protected void verifyValidAttribute(Attributes attributes, String attributeName) { 53 | verifyValidAttribute(attributes, attributeName, null); 54 | } 55 | 56 | protected void verifyValidAttribute(Attributes attributes, String attributeName, String[] possibleValues) { 57 | String value = attributes.getValue(attributeName); 58 | if (value == null || value.length() == 0) 59 | throw new InvalidPackConfiguration(mFileName, "Node "+mNodeName+" has an invalid "+attributeName+" value!"); 60 | 61 | if (possibleValues == null || possibleValues.length == 0) 62 | return; 63 | 64 | for (String aPossibleValue : possibleValues) { 65 | if (value.equals(aPossibleValue)) 66 | return; 67 | } 68 | 69 | throw new InvalidPackConfiguration(mFileName, "Node "+mNodeName+" has an invalid "+attributeName+" value! It is not one of the possible values!"); 70 | } 71 | 72 | protected void verifyValidStringResId(Attributes attributes, String attributeName) { 73 | String nameRes = attributes.getValue(attributeName); 74 | verifyResource(attributeName, nameRes, "string"); 75 | } 76 | 77 | protected void verifyValidXmlResId(Attributes attributes, String attributeName) { 78 | String nameRes = attributes.getValue(attributeName); 79 | verifyResource(attributeName, nameRes, "xml"); 80 | } 81 | 82 | protected void verifyValidRawResId(Attributes attributes, String attributeName) { 83 | String nameRes = attributes.getValue(attributeName); 84 | verifyResource(attributeName, nameRes, "raw"); 85 | } 86 | 87 | protected void verifyValidDrawableResId(Attributes attributes, String attributeName) { 88 | String nameRes = attributes.getValue(attributeName); 89 | verifyResource(attributeName, nameRes, "drawable"); 90 | } 91 | 92 | protected void verifyValidArrayResId(Attributes attributes, String attributeName) { 93 | String nameRes = attributes.getValue(attributeName); 94 | verifyResource(attributeName, nameRes, "array"); 95 | } 96 | 97 | private void verifyResource(String attributeName, String nameRes, String resType) { 98 | if (nameRes == null || !nameRes.startsWith("@"+resType+"/")) 99 | throw new InvalidPackConfiguration(mFileName, "Node "+mNodeName+" has an invalid "+attributeName+" "+resType+" resource !"); 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /packverifier-old/src/AndroidManifestSAXParserHandler.java: -------------------------------------------------------------------------------- 1 | import org.xml.sax.Attributes; 2 | import org.xml.sax.SAXException; 3 | import org.xml.sax.helpers.DefaultHandler; 4 | 5 | 6 | public class AndroidManifestSAXParserHandler extends DefaultHandler { 7 | 8 | private static final String FILENAME = "AndroidManifest.xml"; 9 | private static final String CHANGE_ME = "change_me"; 10 | private String mPackageName; 11 | 12 | private boolean mInReceiver = false; 13 | private boolean mInIntentFilter = false; 14 | private String mCurrentRecieverClass; 15 | //com.menny.android.anysoftkeyboard.KEYBOARD 16 | private String mKeyboardClass; 17 | //com.menny.android.anysoftkeyboard.DICTIONARY 18 | private String mDictionaryClass; 19 | //com.menny.android.anysoftkeyboard.THEME 20 | private String mThemeClass; 21 | 22 | //com.menny.android.anysoftkeyboard.keyboards 23 | private boolean mSawKeyboardsMetadata = false; 24 | //com.menny.android.anysoftkeyboard.dictionaries 25 | private boolean mSawDictionariesMetadata = false; 26 | //com.menny.android.anysoftkeyboard.themes 27 | private boolean mSawThemesMetadata = false; 28 | 29 | @Override 30 | public void startElement(String uri, String localName, String qName, Attributes attributes) 31 | throws SAXException { 32 | super.startElement(uri, localName, qName, attributes); 33 | if (qName.equals("manifest")) { 34 | mPackageName = attributes.getValue("package"); 35 | if (mPackageName == null || mPackageName.length() == 0 || mPackageName.contains(CHANGE_ME)) 36 | throw new InvalidPackConfiguration(FILENAME, "Package name is invalid!"); 37 | } 38 | 39 | if (qName.equals("receiver")) { 40 | mInReceiver = true; 41 | mCurrentRecieverClass = attributes.getValue("android:name"); 42 | } 43 | 44 | if (mInReceiver && qName.equals("intent-filter")) { 45 | mInIntentFilter = true; 46 | } 47 | 48 | if (mInReceiver && mInIntentFilter && qName.equals("action")) { 49 | String actionName = attributes.getValue("android:name"); 50 | if (actionName.equals("com.menny.android.anysoftkeyboard.KEYBOARD")) { 51 | mKeyboardClass = mCurrentRecieverClass; 52 | } else if (actionName.equals("com.menny.android.anysoftkeyboard.DICTIONARY")) { 53 | mDictionaryClass = mCurrentRecieverClass; 54 | } else if (actionName.equals("com.menny.android.anysoftkeyboard.THEME")) { 55 | mThemeClass = mCurrentRecieverClass; 56 | } else { 57 | System.out.println("Unknown reciever type found in AndroidManifest! Action: "+actionName); 58 | } 59 | } 60 | 61 | if (mInReceiver && qName.equals("meta-data")) { 62 | final String type = attributes.getValue("android:name"); 63 | if (type.equals("com.menny.android.anysoftkeyboard.keyboards")) 64 | mSawKeyboardsMetadata = true; 65 | else if (type.equals("com.menny.android.anysoftkeyboard.dictionaries")) 66 | mSawDictionariesMetadata = true; 67 | else if (type.equals("com.menny.android.anysoftkeyboard.themes")) 68 | mSawThemesMetadata = true; 69 | else 70 | System.out.println("Unknown reciever meta-data found in AndroidManifest! Name: "+type); 71 | } 72 | } 73 | 74 | @Override 75 | public void endElement(String uri, String localName, String qName) throws SAXException { 76 | super.endElement(uri, localName, qName); 77 | 78 | if (qName.equals("receiver")) { 79 | mInReceiver = false; 80 | mCurrentRecieverClass = null; 81 | } 82 | 83 | if (mInReceiver && qName.equals("intent-filter")) { 84 | mInIntentFilter = false; 85 | } 86 | } 87 | 88 | @Override 89 | public void endDocument() throws SAXException { 90 | super.endDocument(); 91 | if (mPackageName == null || mPackageName.length() == 0) 92 | throw new InvalidPackConfiguration(FILENAME, "No package name defined!"); 93 | 94 | if (!mSawKeyboardsMetadata && !mSawDictionariesMetadata && !mSawThemesMetadata) 95 | throw new InvalidPackConfiguration(FILENAME, "Pack defines no plugins!"); 96 | //last verifications 97 | verifyMetaDataAndReciever(mKeyboardClass, mSawKeyboardsMetadata, "Keyboard"); 98 | verifyMetaDataAndReciever(mDictionaryClass, mSawDictionariesMetadata, "Dictionary"); 99 | verifyMetaDataAndReciever(mThemeClass, mSawThemesMetadata, "Theme"); 100 | } 101 | 102 | private void verifyMetaDataAndReciever(String className, boolean sawMetadata, String humanName) { 103 | if ((className == null || className.length() == 0) && sawMetadata) 104 | throw new InvalidPackConfiguration(FILENAME, humanName+" metadata has been seen, but no receiver intent-filter found!"); 105 | else if ((className != null && className.length() > 0) && !sawMetadata) 106 | throw new InvalidPackConfiguration(FILENAME, humanName+" receiver intent-filter has been seen, but no meta-data found!"); 107 | } 108 | 109 | public Verifier.PackDetails createPackDetails() { 110 | return new Verifier.PackDetails(mPackageName, mKeyboardClass, mDictionaryClass, mThemeClass); 111 | } 112 | 113 | } 114 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Attempt to set APP_HOME 10 | # Resolve links: $0 may be a link 11 | PRG="$0" 12 | # Need this for relative symlinks. 13 | while [ -h "$PRG" ] ; do 14 | ls=`ls -ld "$PRG"` 15 | link=`expr "$ls" : '.*-> \(.*\)$'` 16 | if expr "$link" : '/.*' > /dev/null; then 17 | PRG="$link" 18 | else 19 | PRG=`dirname "$PRG"`"/$link" 20 | fi 21 | done 22 | SAVED="`pwd`" 23 | cd "`dirname \"$PRG\"`/" >/dev/null 24 | APP_HOME="`pwd -P`" 25 | cd "$SAVED" >/dev/null 26 | 27 | APP_NAME="Gradle" 28 | APP_BASE_NAME=`basename "$0"` 29 | 30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 31 | DEFAULT_JVM_OPTS="" 32 | 33 | # Use the maximum available, or set MAX_FD != -1 to use that value. 34 | MAX_FD="maximum" 35 | 36 | warn ( ) { 37 | echo "$*" 38 | } 39 | 40 | die ( ) { 41 | echo 42 | echo "$*" 43 | echo 44 | exit 1 45 | } 46 | 47 | # OS specific support (must be 'true' or 'false'). 48 | cygwin=false 49 | msys=false 50 | darwin=false 51 | nonstop=false 52 | case "`uname`" in 53 | CYGWIN* ) 54 | cygwin=true 55 | ;; 56 | Darwin* ) 57 | darwin=true 58 | ;; 59 | MINGW* ) 60 | msys=true 61 | ;; 62 | NONSTOP* ) 63 | nonstop=true 64 | ;; 65 | esac 66 | 67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 68 | 69 | # Determine the Java command to use to start the JVM. 70 | if [ -n "$JAVA_HOME" ] ; then 71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 72 | # IBM's JDK on AIX uses strange locations for the executables 73 | JAVACMD="$JAVA_HOME/jre/sh/java" 74 | else 75 | JAVACMD="$JAVA_HOME/bin/java" 76 | fi 77 | if [ ! -x "$JAVACMD" ] ; then 78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 79 | 80 | Please set the JAVA_HOME variable in your environment to match the 81 | location of your Java installation." 82 | fi 83 | else 84 | JAVACMD="java" 85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 86 | 87 | Please set the JAVA_HOME variable in your environment to match the 88 | location of your Java installation." 89 | fi 90 | 91 | # Increase the maximum file descriptors if we can. 92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 93 | MAX_FD_LIMIT=`ulimit -H -n` 94 | if [ $? -eq 0 ] ; then 95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 96 | MAX_FD="$MAX_FD_LIMIT" 97 | fi 98 | ulimit -n $MAX_FD 99 | if [ $? -ne 0 ] ; then 100 | warn "Could not set maximum file descriptor limit: $MAX_FD" 101 | fi 102 | else 103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 104 | fi 105 | fi 106 | 107 | # For Darwin, add options to specify how the application appears in the dock 108 | if $darwin; then 109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 110 | fi 111 | 112 | # For Cygwin, switch paths to Windows format before running java 113 | if $cygwin ; then 114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 116 | JAVACMD=`cygpath --unix "$JAVACMD"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Escape application args 158 | save ( ) { 159 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 160 | echo " " 161 | } 162 | APP_ARGS=$(save "$@") 163 | 164 | # Collect all arguments for the java command, following the shell quoting and substitution rules 165 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 166 | 167 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 168 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 169 | cd "$(dirname "$0")" 170 | fi 171 | 172 | exec "$JAVACMD" "$@" 173 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DEPRECATED! 2 | This repo is holding the tools needed to generate words-list and converting XMLs into binary dictionaries. 3 | This is not longer needed - the tools have moved to https://github.com/AnySoftKeyboard/LanguagePack as part of the buildSrc. 4 | 5 | DO NOT USED THIS REPO! 6 | 7 | AnySoftKeyboardTools 8 | ==================== 9 | 10 | # Make-Dictionary (`makedictioanry`) 11 | Builds a binary dictionary from word-list XML file. 12 | 13 | ## Input 14 | Input should be an XML file in the following format: 15 | ``` 16 | 17 | 18 | Hello 19 | Goodbye 20 | Bye 21 | ``` 22 | `f` is the frequency of the word - a value between _255..1_ - where higher value 23 | means more frequent (common) word. 24 | 25 | ## Output 26 | Will produce the followings: 27 | 1. A `resources-array` resource file `words_dict_array.xml`. Should be placed under `/res/values/` 28 | 2. A set of binary files which are readable by [AnySoftKeyboard](https://github.com/AnySoftKeyboard/AnySoftKeyboard). 29 | These files should be placed under `/res/raw/` resources folder. 30 | 31 | Some Android versions had issues with reading `raw` files which are larger than 1MB. 32 | This utility will break your binary dictionary file into multiple files if 33 | the size of the binary data is more than 1MB. 34 | 35 | ## How to use 36 | This is a Gradle Plugin. 37 | First, add this to the build-script class path: 38 | ``` 39 | buildscript { 40 | repositories { 41 | //any other repositories 42 | 43 | //AnySoftKeyboard-Tools' repository 44 | maven { url 'https://jitpack.io' } 45 | } 46 | dependencies { 47 | //any other dependencies 48 | classpath 'com.github.AnySoftKeyboard.AnySoftKeyboardTools:makedictionary:-SNAPSHOT' 49 | } 50 | } 51 | ``` 52 | Then add a `Task` to create the binary-dictionary: 53 | ``` 54 | task makeEnglishDictionary(type: com.anysoftkeyboard.tools.makedictionary.MakeDictionaryTask) { 55 | inputWordsListFile new File(project.getProjectDir(), "english_dictionary/words.xml") 56 | } 57 | ``` 58 | 59 | optionally, if your `res` folder is non-standard, or you have multiple such folders, you may want to specify its location: 60 | ``` 61 | task makeEnglishDictionary(type: com.anysoftkeyboard.tools.makedictionary.MakeDictionaryTask) { 62 | inputWordsListFile new File(project.getProjectDir(), "english_dictionary/words.xml") 63 | resourcesFolder new File(project.getProjectDir(), "src/main/res/") 64 | } 65 | ``` 66 | 67 | To create the dictionary, run the task from command-line: 68 | ``` 69 | ./gradlew makeEnglishDictionary 70 | ``` 71 | 72 | # Generate Words-List (`generatewordslist`) 73 | Generates words-list XML file (which can be used as input to `makedictioanry`) 74 | 75 | ## GenerateWordsListTask 76 | Generates word-list XML file (with frequencies) from a text file. 77 | 78 | ### Input 79 | Non-marked-up text, aka simple text file. 80 | 81 | ### Output 82 | ``` 83 | 84 | 85 | Hello 86 | Goodbye 87 | Bye 88 | ``` 89 | `f` is the frequency of the word - a value between _255..1_ - where higher value 90 | means more frequent (common) word. 91 | 92 | ### How to use 93 | This is a Gradle Plugin. 94 | First, add this to the build-script class path: 95 | ``` 96 | buildscript { 97 | repositories { 98 | //any other repositories 99 | 100 | //AnySoftKeyboard-Tools' repository 101 | maven { url 'https://jitpack.io' } 102 | } 103 | dependencies { 104 | //any other dependencies 105 | classpath 'com.github.AnySoftKeyboard.AnySoftKeyboardTools:generatewordslist:-SNAPSHOT' 106 | } 107 | } 108 | ``` 109 | Then add a `Task` to generate the XML file: 110 | ``` 111 | task parseEbook1(type: com.anysoftkeyboard.tools.generatewordslist.GenerateWordsListTask) { 112 | inputTextFile new File(project.getProjectDir(), "english_dictionary/ebook_1.txt") 113 | outputWordsListFile new File(project.getProjectDir(), "english_dictionary/words_ebook_1.xml") 114 | wordCharacters "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz".toCharArray() 115 | additionalInnerCharacters "'".toCharArray() 116 | locale Locale.US 117 | } 118 | ``` 119 | 120 | To create the word-list, run the task from command-line: 121 | ``` 122 | ./gradlew parseEbook1 123 | ``` 124 | 125 | ## MergeWordsListTask 126 | Merges several words-list XML files into one. 127 | 128 | ### Input 129 | Valid words-list XML files array. 130 | 131 | ### Output 132 | ``` 133 | 134 | 135 | Hello 136 | Goodbye 137 | Bye 138 | ``` 139 | `f` is the frequency of the word - a value between _255..1_ - where higher value 140 | means more frequent (common) word. 141 | 142 | ### How to use 143 | This is a Gradle Plugin. 144 | First, add this to the build-script class path: 145 | ``` 146 | buildscript { 147 | repositories { 148 | //any other repositories 149 | 150 | //AnySoftKeyboard-Tools' repository 151 | maven { url 'https://jitpack.io' } 152 | } 153 | dependencies { 154 | //any other dependencies 155 | classpath 'com.github.AnySoftKeyboard.AnySoftKeyboardTools:generatewordslist:-SNAPSHOT' 156 | } 157 | } 158 | ``` 159 | Then add a `Task` to generate the XML file: 160 | ``` 161 | task mergeAllWordLists(type: com.anysoftkeyboard.tools.generatewordslist.MergeWordsListTask) { 162 | inputWordsListFiles = [ 163 | new File(project.getProjectDir(), "english_dictionary/words.xml"), 164 | new File(project.getProjectDir(), "english_dictionary/words_ebook_1.xml"), 165 | new File(project.getProjectDir(), "english_dictionary/words_ebook_2.xml") 166 | ] as File[] 167 | outputWordsListFile new File(project.getProjectDir(), "english_dictionary/words_merged.xml") 168 | maxWordsInList 100000 169 | } 170 | ``` 171 | 172 | To merge ths words-lists, run the task from command-line: 173 | ``` 174 | ./gradlew mergeAllWordLists 175 | ``` 176 | 177 | 178 | See an actual, real world, usage at [AnySoftKeyboard](https://github.com/AnySoftKeyboard/AnySoftKeyboard/blob/master/build.gradle) 179 | -------------------------------------------------------------------------------- /generatewordslist/src/main/groovy/com/anysoftkeyboard/tools/generatewordslist/Parser.java: -------------------------------------------------------------------------------- 1 | package com.anysoftkeyboard.tools.generatewordslist; 2 | 3 | import java.io.File; 4 | import java.io.FileOutputStream; 5 | import java.io.FileReader; 6 | import java.io.IOException; 7 | import java.io.InputStreamReader; 8 | import java.io.OutputStreamWriter; 9 | import java.io.Writer; 10 | import java.util.ArrayList; 11 | import java.util.Collections; 12 | import java.util.HashMap; 13 | import java.util.HashSet; 14 | import java.util.List; 15 | import java.util.Locale; 16 | 17 | class Parser { 18 | 19 | private final static int LOOKING_FOR_WORD_START = 1; 20 | private final static int LOOKING_FOR_WORD_END = 2; 21 | private final List mInputFiles; 22 | private final OutputStreamWriter mOutput; 23 | private final HashSet mLangChars; 24 | private final HashSet mLangInnerChars; 25 | private final HashMap mWords; 26 | private final int mMaxListSize; 27 | private final Locale mLocale; 28 | 29 | public Parser(List inputFiles, File outputFile, char[] wordCharacters, Locale locale, char[] additionalInnerWordCharacters, int maxListSize) throws IOException { 30 | if (inputFiles.size() == 0) 31 | throw new IllegalArgumentException("Files list should be at least 1 size."); 32 | for (File inputFile : inputFiles) { 33 | if (!inputFile.exists()) 34 | throw new IOException("Could not file input file " + inputFile); 35 | if (!inputFile.isFile()) throw new IOException("Input must be a file."); 36 | } 37 | mInputFiles = Collections.unmodifiableList(inputFiles); 38 | mLocale = locale; 39 | mMaxListSize = maxListSize; 40 | mOutput = new OutputStreamWriter(new FileOutputStream(outputFile)); 41 | 42 | mLangInnerChars = new HashSet<>(additionalInnerWordCharacters.length + wordCharacters.length); 43 | mLangChars = new HashSet<>(wordCharacters.length); 44 | for (char c : wordCharacters) { 45 | mLangChars.add(c); 46 | mLangInnerChars.add(c); 47 | } 48 | 49 | for (char c : additionalInnerWordCharacters) { 50 | mLangInnerChars.add(c); 51 | } 52 | 53 | mWords = new HashMap<>(); 54 | 55 | System.out.println(String.format(Locale.US, "Parsing %d files for a maximum of %d words, and writing into '%s'.", mInputFiles.size(), mMaxListSize, outputFile)); 56 | } 57 | 58 | public void parse() throws IOException { 59 | for (File inputFile : mInputFiles) { 60 | System.out.println(String.format(Locale.US, "Reading input file %s...", inputFile)); 61 | InputStreamReader inputStream = new FileReader(inputFile); 62 | addWordsFromInputStream(inputFile.length(), inputStream); 63 | inputStream.close(); 64 | System.out.println(String.format(Locale.US, "Have %d words so far.", mWords.size())); 65 | } 66 | 67 | System.out.println("Sorting list..."); 68 | List sortedList = new ArrayList<>(mWords.values()); 69 | Collections.sort(sortedList); 70 | 71 | System.out.println("Creating output XML file..."); 72 | createXml(sortedList, mOutput, mMaxListSize, false); 73 | 74 | mOutput.flush(); 75 | mOutput.close(); 76 | 77 | System.out.println("Done."); 78 | } 79 | 80 | public static void createXml(List sortedList, Writer outputWriter, int maxListSize, boolean takeFrequencyFromWordObject) throws IOException { 81 | final int wordsCount = Math.min(maxListSize, sortedList.size()); 82 | 83 | XmlWriter writer = new XmlWriter(outputWriter, false, 0, true); 84 | writer.writeEntity("wordlist"); 85 | for (int wordIndex = 0; wordIndex < wordsCount; wordIndex++) { 86 | WordWithCount word = sortedList.get(wordIndex); 87 | 88 | writer.writeEntity("w").writeAttribute("f", Integer.toString(takeFrequencyFromWordObject? word.getFreq() : calcActualFreq(wordIndex, wordsCount))).writeText(word.getWord()).endEntity(); 89 | } 90 | System.out.println("Wrote " + wordsCount + " words."); 91 | writer.endEntity(); 92 | } 93 | 94 | private static int calcActualFreq(double wordIndex, double wordsCount) { 95 | return Math.min(255, 1 + (int) (255 * (wordsCount - wordIndex) / wordsCount)); 96 | } 97 | 98 | private void addWordsFromInputStream(final long inputSize, InputStreamReader input) throws IOException { 99 | StringBuilder word = new StringBuilder(); 100 | int intChar; 101 | 102 | int state = LOOKING_FOR_WORD_START; 103 | long read = 0; 104 | while ((intChar = input.read()) > 0) { 105 | if ((read % 50000) == 0 || read == inputSize) { 106 | System.out.print("." + ((100 * read) / inputSize) + "%."); 107 | } 108 | char currentChar = (char) intChar; 109 | read++; 110 | switch (state) { 111 | case LOOKING_FOR_WORD_START: 112 | if (mLangChars.contains(currentChar)) { 113 | word.append(currentChar); 114 | state = LOOKING_FOR_WORD_END; 115 | } 116 | break; 117 | case LOOKING_FOR_WORD_END: 118 | if (mLangInnerChars.contains(currentChar)) { 119 | word.append(currentChar); 120 | } else { 121 | addWord(word); 122 | word.setLength(0); 123 | state = LOOKING_FOR_WORD_START; 124 | } 125 | } 126 | } 127 | //last word? 128 | if (word.length() > 0) 129 | addWord(word); 130 | System.out.println("Done."); 131 | } 132 | 133 | private void addWord(StringBuilder word) { 134 | //removing all none chars from the end. 135 | String typedWord = word.toString(); 136 | String wordKey = typedWord.toLowerCase(mLocale); 137 | if (mWords.containsKey(wordKey)) { 138 | mWords.get(wordKey).addFreq(typedWord); 139 | } else { 140 | mWords.put(wordKey, new WordWithCount(typedWord)); 141 | } 142 | } 143 | 144 | } 145 | -------------------------------------------------------------------------------- /DictionaryCreator-old/src/com/anysoftkeyboard/dictionarycreator/ParserThread.java: -------------------------------------------------------------------------------- 1 | package com.anysoftkeyboard.dictionarycreator; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStreamReader; 5 | import java.io.OutputStreamWriter; 6 | import java.util.ArrayList; 7 | import java.util.Collections; 8 | import java.util.HashMap; 9 | import java.util.HashSet; 10 | import java.util.List; 11 | 12 | public class ParserThread extends Thread { 13 | 14 | private static class WordWithCount implements Comparable 15 | { 16 | private final String mWord; 17 | private int mFreq; 18 | private int mCapitalFreq; 19 | 20 | public WordWithCount(String word) 21 | { 22 | mWord = word.toLowerCase(); 23 | mFreq = 0; 24 | mCapitalFreq = 0; 25 | addFreq(word); 26 | } 27 | 28 | public String getWord() 29 | { 30 | //if more than 90% of the word occurrences are capital, 31 | //then use capital style 32 | if ((mFreq * 0.90) < mCapitalFreq) 33 | return Character.toUpperCase(mWord.charAt(0))+mWord.substring(1); 34 | else 35 | return mWord; 36 | } 37 | 38 | public int getFreq() {return mFreq;} 39 | 40 | public void addFreq(String word) 41 | { 42 | mFreq++; 43 | if (Character.isUpperCase(word.charAt(0))) 44 | mCapitalFreq++; 45 | } 46 | 47 | @Override 48 | public int compareTo(WordWithCount o) { 49 | return o.mFreq - mFreq; 50 | } 51 | } 52 | 53 | private final UI mUi; 54 | private final int mTotalChars; 55 | private final InputStreamReader mInput; 56 | private final int mTotalDictionaryChars; 57 | private final InputStreamReader mInputDictionary; 58 | private final OutputStreamWriter mOutput; 59 | private final HashSet mLangChars; 60 | private final HashSet mLangInnerChars; 61 | private final HashMap mWords; 62 | private final int mMaxWords; 63 | 64 | private int mMaxCount = 1; 65 | 66 | public ParserThread(InputStreamReader input, int totalChars, InputStreamReader dictionary, int totalDictionaryChars, OutputStreamWriter output, UI ui, String languageCharacters, String additionalInnerCharacters, int maxWords) { 67 | mUi = ui; 68 | mInput = input; 69 | mTotalChars = totalChars; 70 | mInputDictionary = dictionary; 71 | mTotalDictionaryChars = totalDictionaryChars; 72 | mOutput = output; 73 | mMaxWords = maxWords; 74 | 75 | mLangChars = new HashSet(languageCharacters.length()); 76 | for (int i = 0; i < languageCharacters.length(); ++i) 77 | mLangChars.add(languageCharacters.charAt(i)); 78 | 79 | mLangInnerChars = new HashSet(additionalInnerCharacters.length()); 80 | for (int i = 0; i < additionalInnerCharacters.length(); ++i) 81 | mLangInnerChars.add(additionalInnerCharacters.charAt(i)); 82 | 83 | mWords = new HashMap(); 84 | } 85 | 86 | private final static int LOOKING_FOR_SPACE = 0; 87 | private final static int LOOKING_FOR_WORD_START = 1; 88 | private final static int INSIDE_WORD = 2; 89 | 90 | @Override 91 | public void run() { 92 | super.run(); 93 | mUi.updateProgressState("Parser started", 0); 94 | 95 | try 96 | { 97 | if (mInputDictionary != null) 98 | addWordsFromInputStream(mInputDictionary, true, mTotalDictionaryChars, LOOKING_FOR_WORD_START); 99 | addWordsFromInputStream(mInput, (mInputDictionary == null), mTotalChars, LOOKING_FOR_SPACE); 100 | } catch (IOException e) { 101 | e.printStackTrace(); 102 | mUi.showErrorMessage(e.getMessage()); 103 | } 104 | finally 105 | { 106 | mUi.updateProgressState("Parser ending...", 0); 107 | try { 108 | mInput.close(); 109 | mUi.updateProgressState("Sorting words (have "+mWords.size()+" words)...", 0); 110 | List sortedList = removeNonFrequentWords(); 111 | mUi.updateProgressState("Creating XML (will use "+mMaxWords+" words)...", 0); 112 | createXml(sortedList); 113 | mOutput.flush(); 114 | mOutput.close(); 115 | } catch (IOException e) { 116 | // TODO Auto-generated catch block 117 | e.printStackTrace(); 118 | mUi.showErrorMessage(e.getMessage()); 119 | } 120 | mUi.updateProgressState("Parser ended. Found "+mWords.size()+" words.", 100); 121 | mUi.onEnded(); 122 | } 123 | } 124 | 125 | private List removeNonFrequentWords() { 126 | List sortedList = new ArrayList(mMaxWords); 127 | sortedList.addAll(mWords.values()); 128 | Collections.sort(sortedList); 129 | 130 | return sortedList; 131 | } 132 | 133 | private void createXml(List sortedList) throws IOException { 134 | //refactoring mMaxCount to the value of the word at the 10% place (10% most freq) 135 | int wordsCount = Math.min(mMaxWords, sortedList.size()); 136 | mMaxCount = sortedList.get((int)(0.05f * wordsCount)).getFreq(); 137 | mOutput.write("\n"); 138 | mOutput.write("\n"); 139 | int wordIndex = 0; 140 | for(WordWithCount word : sortedList) 141 | { 142 | if (wordIndex > mMaxWords) 143 | break; 144 | 145 | mOutput.write(""+word.getWord()+"\n"); 146 | wordIndex++; 147 | } 148 | mOutput.write("\n"); 149 | } 150 | 151 | private int calcActualFreq(int freq) { 152 | if (mMaxCount < freq) 153 | freq = mMaxCount; 154 | return 1+(int)((64*1024 - 1) * (((double)freq)/((double)mMaxCount))); 155 | } 156 | 157 | private void addWordsFromInputStream(InputStreamReader input, boolean addFirst, int totalChars, final int startState) throws IOException { 158 | StringBuilder word = new StringBuilder(); 159 | int intChar; 160 | 161 | int state = startState; 162 | int read = 0; 163 | while((intChar = input.read()) > 0) 164 | { 165 | if ((read % 50000) == 0) 166 | { 167 | mUi.updateProgressState("Parsing (have "+mWords.size()+" words)...", (int)((100f * (float)read) / (float)totalChars)); 168 | } 169 | char currentChar = (char)intChar; 170 | read++; 171 | switch(state) 172 | { 173 | case LOOKING_FOR_SPACE: 174 | if (currentChar == ' ') 175 | { 176 | state = LOOKING_FOR_WORD_START; 177 | } 178 | break; 179 | case LOOKING_FOR_WORD_START: 180 | if (mLangChars.contains(currentChar)) 181 | { 182 | word.append(currentChar); 183 | state = INSIDE_WORD; 184 | } 185 | break; 186 | case INSIDE_WORD: 187 | if (mLangChars.contains(currentChar) || mLangInnerChars.contains(currentChar)) 188 | { 189 | word.append(currentChar); 190 | } 191 | else 192 | { 193 | addWord(word, addFirst); 194 | state = startState; 195 | } 196 | } 197 | } 198 | //last word? 199 | if (word.length() > 0) 200 | addWord(word, addFirst); 201 | } 202 | 203 | private void addWord(StringBuilder word, boolean addFirst) { 204 | //removing all none chars from the end. 205 | while(mLangInnerChars.contains(word.charAt(word.length()-1))) 206 | word.setLength(word.length()-1); 207 | String typedWord = word.toString(); 208 | String wordFound = typedWord.toLowerCase(); 209 | if (mWords.containsKey(wordFound)) 210 | {//it must be inside already (from the dictionary) 211 | mWords.get(wordFound).addFreq(typedWord); 212 | } 213 | else if (addFirst) 214 | { 215 | mWords.put(wordFound, new WordWithCount(typedWord)); 216 | } 217 | 218 | word.setLength(0); 219 | } 220 | 221 | } 222 | -------------------------------------------------------------------------------- /generatewordslist/src/main/groovy/com/anysoftkeyboard/tools/generatewordslist/MergeWordsListTask.java: -------------------------------------------------------------------------------- 1 | package com.anysoftkeyboard.tools.generatewordslist; 2 | 3 | import org.gradle.api.DefaultTask; 4 | import org.gradle.api.tasks.TaskAction; 5 | import org.xml.sax.Attributes; 6 | import org.xml.sax.InputSource; 7 | import org.xml.sax.SAXException; 8 | import org.xml.sax.SAXParseException; 9 | import org.xml.sax.helpers.DefaultHandler; 10 | 11 | import java.io.File; 12 | import java.io.FileInputStream; 13 | import java.io.FileNotFoundException; 14 | import java.io.FileOutputStream; 15 | import java.io.IOException; 16 | import java.io.InputStreamReader; 17 | import java.io.OutputStreamWriter; 18 | import java.io.Writer; 19 | import java.nio.charset.Charset; 20 | import java.util.ArrayList; 21 | import java.util.Arrays; 22 | import java.util.Collections; 23 | import java.util.HashMap; 24 | import java.util.List; 25 | 26 | import javax.xml.parsers.ParserConfigurationException; 27 | import javax.xml.parsers.SAXParser; 28 | import javax.xml.parsers.SAXParserFactory; 29 | 30 | /** 31 | * Task to merge several word-list files into one 32 | */ 33 | public class MergeWordsListTask extends DefaultTask { 34 | @TaskAction 35 | public void mergeWordsLists() throws IOException, ParserConfigurationException, SAXException { 36 | if (inputWordsListFiles == null || inputWordsListFiles.length == 0) 37 | throw new IllegalArgumentException("Must specify at least one inputWordsListFiles"); 38 | if (outputWordsListFile == null) 39 | throw new IllegalArgumentException("Must supply outputWordsListFile"); 40 | 41 | System.out.println("Merging " + inputWordsListFiles.length + " files for maximum " + maxWordsInList + " words, and writing into \'" + outputWordsListFile.getName() + "\'. Discarding " + wordsToDiscard.length + " words."); 42 | final HashMap allWords = new HashMap<>(); 43 | 44 | for (File inputFile : inputWordsListFiles) { 45 | System.out.println("Reading " + inputFile.getName() + "..."); 46 | if (!inputFile.exists()) throw new FileNotFoundException(inputFile.getAbsolutePath()); 47 | SAXParserFactory parserFactor = SAXParserFactory.newInstance(); 48 | SAXParser parser = parserFactor.newSAXParser(); 49 | final InputStreamReader inputStream = new InputStreamReader(new FileInputStream(inputFile), Charset.forName("UTF-8")); 50 | InputSource inputSource = new InputSource(inputStream); 51 | parser.parse(inputSource, new MySaxHandler(allWords)); 52 | System.out.println("Loaded " + allWords.size() + " words in total..."); 53 | inputStream.close(); 54 | } 55 | 56 | 57 | //discarding unwanted words 58 | if (wordsToDiscard.length > 0) { 59 | System.out.print("Discarding words..."); 60 | Arrays.stream(wordsToDiscard).forEach(word -> { 61 | if (allWords.remove(word) != null) System.out.print("."); 62 | }); 63 | System.out.println(""); 64 | } 65 | 66 | 67 | System.out.println("Sorting list..."); 68 | List sortedList = new ArrayList(allWords.values()); 69 | Collections.sort(sortedList); 70 | 71 | System.out.println("Creating output XML file..."); 72 | Writer output = new OutputStreamWriter(new FileOutputStream(outputWordsListFile), Charset.forName("UTF-8")); 73 | Parser.createXml(sortedList, output, maxWordsInList, true); 74 | 75 | output.flush(); 76 | output.close(); 77 | 78 | System.out.println("Done."); 79 | } 80 | 81 | public File[] getInputWordsListFiles() { 82 | return inputWordsListFiles; 83 | } 84 | 85 | public void setInputWordsListFiles(File[] inputWordsListFiles) { 86 | this.inputWordsListFiles = inputWordsListFiles; 87 | } 88 | 89 | public File getOutputWordsListFile() { 90 | return outputWordsListFile; 91 | } 92 | 93 | public void setOutputWordsListFile(File outputWordsListFile) { 94 | this.outputWordsListFile = outputWordsListFile; 95 | } 96 | 97 | public String[] getWordsToDiscard() { 98 | return wordsToDiscard; 99 | } 100 | 101 | public void setWordsToDiscard(String[] wordsToDiscard) { 102 | this.wordsToDiscard = wordsToDiscard; 103 | } 104 | 105 | public int getMaxWordsInList() { 106 | return maxWordsInList; 107 | } 108 | 109 | public void setMaxWordsInList(int maxWordsInList) { 110 | this.maxWordsInList = maxWordsInList; 111 | } 112 | 113 | private File[] inputWordsListFiles; 114 | private File outputWordsListFile; 115 | private String[] wordsToDiscard = new String[0]; 116 | private int maxWordsInList = Integer.MAX_VALUE; 117 | 118 | private static class MySaxHandler extends DefaultHandler { 119 | public MySaxHandler(HashMap allWords) { 120 | this.allWords = allWords; 121 | } 122 | 123 | @Override 124 | public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { 125 | super.startElement(uri, localName, qName, attributes); 126 | if (qName.equals("w")) { 127 | inWord = true; 128 | freq = Integer.parseInt(attributes.getValue("f")); 129 | word.setLength(0); 130 | } else { 131 | inWord = false; 132 | } 133 | 134 | } 135 | 136 | @Override 137 | public void characters(char[] ch, int start, int length) throws SAXException { 138 | super.characters(ch, start, length); 139 | if (inWord) { 140 | word.append(ch, start, length); 141 | } 142 | 143 | } 144 | 145 | @Override 146 | public void skippedEntity(String name) throws SAXException { 147 | System.out.print("Skipped " + name); 148 | super.skippedEntity(name); 149 | } 150 | 151 | @Override 152 | public void warning(SAXParseException e) throws SAXException { 153 | System.out.print("Warning! " + e); 154 | super.warning(e); 155 | } 156 | 157 | @Override 158 | public void error(SAXParseException e) throws SAXException { 159 | System.out.print("Error! " + e); 160 | super.error(e); 161 | } 162 | 163 | @Override 164 | public void fatalError(SAXParseException e) throws SAXException { 165 | System.out.print("Fatal-Error! " + e); 166 | super.fatalError(e); 167 | } 168 | 169 | @Override 170 | public void unparsedEntityDecl(String name, String publicId, String systemId, String notationName) throws SAXException { 171 | System.out.print("unparsedEntityDecl! " + name); 172 | super.unparsedEntityDecl(name, publicId, systemId, notationName); 173 | } 174 | 175 | @Override 176 | public void endElement(String uri, String localName, String qName) throws SAXException { 177 | super.endElement(uri, localName, qName); 178 | if (qName.equals("w") && inWord) { 179 | WordWithCount wordWithCount = new WordWithCount(word.toString(), freq); 180 | if (allWords.containsKey(wordWithCount.getKey())) { 181 | allWords.get(wordWithCount.getKey()).addOtherWord(wordWithCount); 182 | } else { 183 | allWords.put(wordWithCount.getKey(), wordWithCount); 184 | } 185 | 186 | } 187 | 188 | inWord = false; 189 | } 190 | 191 | private HashMap allWords; 192 | private boolean inWord; 193 | private StringBuilder word = new StringBuilder(); 194 | private int freq; 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /makedictionary/src/main/groovy/com/anysoftkeyboard/tools/makedictionary/MakeBinaryDictionary.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2009 The Android Open Source Project 3 | * Copyright (C) 2016 AnySoftKeyboard 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package com.anysoftkeyboard.tools.makedictionary; 19 | 20 | import org.xml.sax.Attributes; 21 | import org.xml.sax.SAXException; 22 | import org.xml.sax.helpers.DefaultHandler; 23 | 24 | import javax.xml.parsers.ParserConfigurationException; 25 | import javax.xml.parsers.SAXParser; 26 | import javax.xml.parsers.SAXParserFactory; 27 | import java.io.*; 28 | import java.util.ArrayList; 29 | import java.util.List; 30 | import java.util.Map; 31 | 32 | /** 33 | * Compresses a list of words and frequencies into a tree structured binary dictionary. 34 | */ 35 | class MakeBinaryDictionary { 36 | private static final int ALPHA_SIZE = 256; 37 | private static final String TAG_WORD = "w"; 38 | private static final String ATTR_FREQ = "f"; 39 | private static final CharNode EMPTY_NODE = new CharNode(); 40 | private static final int CHAR_WIDTH = 8; 41 | private static final int FLAGS_WIDTH = 1; // Terminal flag (word end) 42 | private static final int ADDR_WIDTH = 23; // Offset to children 43 | private static final int FREQ_WIDTH_BYTES = 1; 44 | private static final int COUNT_WIDTH_BYTES = 1; 45 | private static final int FLAG_ADDRESS_MASK = 0x400000; 46 | private static final int FLAG_TERMINAL_MASK = 0x800000; 47 | private static final int ADDRESS_MASK = 0x3FFFFF; 48 | private final String srcFilename; 49 | private final String destFilename; 50 | private List roots; 51 | private Map mDictionary; 52 | private int mWordCount; 53 | private byte[] dict; 54 | private int dictSize; 55 | private int nullChildrenCount = 0; 56 | private int notTerminalCount = 0; 57 | 58 | public MakeBinaryDictionary(String srcFilename, String destFilename) { 59 | this.srcFilename = srcFilename; 60 | this.destFilename = destFilename; 61 | } 62 | 63 | public void makeDictionary() throws ParserConfigurationException, SAXException, IOException { 64 | populateDictionary(srcFilename); 65 | writeToDict(destFilename); 66 | } 67 | 68 | private void populateDictionary(String filename) throws IOException, SAXException, ParserConfigurationException { 69 | roots = new ArrayList<>(); 70 | SAXParser parser = SAXParserFactory.newInstance().newSAXParser(); 71 | parser.parse(new File(filename), new DefaultHandler() { 72 | boolean inWord; 73 | int freq; 74 | StringBuilder wordBuilder = new StringBuilder(48); 75 | 76 | @Override 77 | public void startElement(String uri, String localName, 78 | String qName, Attributes attributes) { 79 | if (qName.equals(TAG_WORD)) { 80 | inWord = true; 81 | freq = Integer.parseInt(attributes.getValue(0)); 82 | wordBuilder.setLength(0); 83 | } 84 | } 85 | 86 | @Override 87 | public void characters(char[] data, int offset, int length) { 88 | // Ignore other whitespace 89 | if (!inWord) return; 90 | wordBuilder.append(data, offset, length); 91 | } 92 | 93 | @Override 94 | public void endElement(String uri, String localName, 95 | String qName) { 96 | if (qName.equals(TAG_WORD)) { 97 | if (wordBuilder.length() > 1) { 98 | addWordTop(wordBuilder.toString(), freq); 99 | mWordCount++; 100 | } 101 | inWord = false; 102 | } 103 | } 104 | }); 105 | 106 | System.out.println("Nodes = " + CharNode.sNodes); 107 | } 108 | 109 | private int indexOf(List children, char c) { 110 | if (children == null) { 111 | return -1; 112 | } 113 | for (int i = 0; i < children.size(); i++) { 114 | if (children.get(i).data == c) { 115 | return i; 116 | } 117 | } 118 | return -1; 119 | } 120 | 121 | private void addWordTop(String word, int occur) { 122 | if (occur > 255) occur = 255; 123 | char firstChar = word.charAt(0); 124 | int index = indexOf(roots, firstChar); 125 | if (index == -1) { 126 | CharNode newNode = new CharNode(); 127 | newNode.data = firstChar; 128 | newNode.freq = occur; 129 | index = roots.size(); 130 | roots.add(newNode); 131 | } else { 132 | roots.get(index).freq += occur; 133 | } 134 | if (word.length() > 1) { 135 | addWordRec(roots.get(index), word, 1, occur); 136 | } else { 137 | roots.get(index).terminal = true; 138 | } 139 | } 140 | 141 | private void addWordRec(CharNode parent, String word, int charAt, int occur) { 142 | CharNode child = null; 143 | char data = word.charAt(charAt); 144 | if (parent.children == null) { 145 | parent.children = new ArrayList<>(); 146 | } else { 147 | for (int i = 0; i < parent.children.size(); i++) { 148 | CharNode node = parent.children.get(i); 149 | if (node.data == data) { 150 | child = node; 151 | break; 152 | } 153 | } 154 | } 155 | if (child == null) { 156 | child = new CharNode(); 157 | parent.children.add(child); 158 | } 159 | child.data = data; 160 | if (child.freq == 0) child.freq = occur; 161 | if (word.length() > charAt + 1) { 162 | addWordRec(child, word, charAt + 1, occur); 163 | } else { 164 | child.terminal = true; 165 | child.freq = occur; 166 | } 167 | } 168 | 169 | private void addCount(int count) { 170 | dict[dictSize++] = (byte) (0xFF & count); 171 | } 172 | 173 | private void addNode(CharNode node) { 174 | int charData = 0xFFFF & node.data; 175 | if (charData > 254) { 176 | dict[dictSize++] = (byte) 255; 177 | dict[dictSize++] = (byte) ((node.data >> 8) & 0xFF); 178 | dict[dictSize++] = (byte) (node.data & 0xFF); 179 | } else { 180 | dict[dictSize++] = (byte) (0xFF & node.data); 181 | } 182 | if (node.children != null) { 183 | dictSize += 3; // Space for children address 184 | } else { 185 | dictSize += 1; // Space for just the terminal/address flags 186 | } 187 | if ((0xFFFFFF & node.freq) > 255) { 188 | node.freq = 255; 189 | } 190 | if (node.terminal) { 191 | byte freq = (byte) (0xFF & node.freq); 192 | dict[dictSize++] = freq; 193 | } 194 | } 195 | 196 | private void updateNodeAddress(int nodeAddress, CharNode node, 197 | int childrenAddress) { 198 | if ((dict[nodeAddress] & 0xFF) == 0xFF) { // 3 byte character 199 | nodeAddress += 2; 200 | } 201 | childrenAddress = ADDRESS_MASK & childrenAddress; 202 | if (childrenAddress == 0) { 203 | nullChildrenCount++; 204 | } else { 205 | childrenAddress |= FLAG_ADDRESS_MASK; 206 | } 207 | if (node.terminal) { 208 | childrenAddress |= FLAG_TERMINAL_MASK; 209 | } else { 210 | notTerminalCount++; 211 | } 212 | dict[nodeAddress + 1] = (byte) (childrenAddress >> 16); 213 | if ((childrenAddress & FLAG_ADDRESS_MASK) != 0) { 214 | dict[nodeAddress + 2] = (byte) ((childrenAddress & 0xFF00) >> 8); 215 | dict[nodeAddress + 3] = (byte) ((childrenAddress & 0xFF)); 216 | } 217 | } 218 | 219 | void writeWordsRec(List children) { 220 | if (children == null || children.size() == 0) { 221 | return; 222 | } 223 | final int childCount = children.size(); 224 | addCount(childCount); 225 | //int childrenStart = dictSize; 226 | int[] childrenAddresses = new int[childCount]; 227 | for (int j = 0; j < childCount; j++) { 228 | CharNode node = children.get(j); 229 | childrenAddresses[j] = dictSize; 230 | addNode(node); 231 | } 232 | for (int j = 0; j < childCount; j++) { 233 | CharNode node = children.get(j); 234 | int nodeAddress = childrenAddresses[j]; 235 | int cacheDictSize = dictSize; 236 | writeWordsRec(node.children); 237 | updateNodeAddress(nodeAddress, node, node.children != null 238 | ? cacheDictSize : 0); 239 | } 240 | } 241 | 242 | private void writeToDict(String dictFilename) { 243 | // 4MB max, 22-bit offsets 244 | dict = new byte[4 * 1024 * 1024]; 245 | dictSize = 0; 246 | writeWordsRec(roots); 247 | System.out.println("Dict Size = " + dictSize); 248 | try { 249 | FileOutputStream fos = new FileOutputStream(dictFilename); 250 | fos.write(dict, 0, dictSize); 251 | fos.close(); 252 | } catch (IOException ioe) { 253 | System.err.println("Error writing dict file:" + ioe); 254 | } 255 | } 256 | 257 | private static class CharNode { 258 | static int sNodes; 259 | char data; 260 | int freq; 261 | boolean terminal; 262 | List children; 263 | 264 | CharNode() { 265 | sNodes++; 266 | } 267 | } 268 | } -------------------------------------------------------------------------------- /DictionaryCreator-old/src/com/anysoftkeyboard/dictionarycreator/MainForm.java: -------------------------------------------------------------------------------- 1 | package com.anysoftkeyboard.dictionarycreator; 2 | 3 | import javax.swing.SwingUtilities; 4 | import java.awt.BorderLayout; 5 | import javax.swing.JPanel; 6 | import javax.swing.JFrame; 7 | import java.awt.Dimension; 8 | import javax.swing.BoxLayout; 9 | import javax.swing.JOptionPane; 10 | import javax.swing.JTextPane; 11 | import javax.swing.JLabel; 12 | import java.awt.GridBagLayout; 13 | import java.awt.GridBagConstraints; 14 | import javax.swing.JTextField; 15 | import java.awt.Insets; 16 | import javax.swing.JButton; 17 | import java.awt.Toolkit; 18 | import java.awt.event.ActionEvent; 19 | import java.io.DataInputStream; 20 | import java.io.DataOutputStream; 21 | import java.io.File; 22 | import java.io.FileInputStream; 23 | import java.io.FileOutputStream; 24 | import java.io.InputStreamReader; 25 | import java.io.OutputStreamWriter; 26 | import javax.swing.JSlider; 27 | import javax.swing.event.ChangeEvent; 28 | import javax.swing.event.ChangeListener; 29 | 30 | public class MainForm extends JFrame implements UI { 31 | 32 | private ParserThread mThread; 33 | private static final long serialVersionUID = 1L; 34 | private JPanel jContentPane = null; 35 | private JPanel jPanel = null; 36 | private JLabel jLabel = null; 37 | private JLabel jLabel2 = null; 38 | private JTextField jTextFieldSourceFilename = null; 39 | private JLabel jLabel3 = null; 40 | private JLabel jLabel4 = null; 41 | private JTextField jTextFieldSpellDictionaryFilename = null; 42 | private JTextField jTextFieldOutputfilename = null; 43 | private JButton jButtonStart = null; 44 | private JLabel jLabelProgress = null; 45 | private JLabel jLabel1 = null; 46 | private JTextField jTextFieldPossibleLetters = null; 47 | private JLabel jLabel5 = null; 48 | private JTextField jTextFieldInnerCharacters = null; 49 | private JLabel jLabelMaxWords = null; 50 | private JSlider jSliderWordsCount = null; 51 | /** 52 | * This method initializes jPanel 53 | * 54 | * @return javax.swing.JPanel 55 | */ 56 | private JPanel getJPanel() { 57 | if (jPanel == null) { 58 | jPanel = new JPanel(); 59 | jPanel.setLayout(new BoxLayout(getJPanel(), BoxLayout.X_AXIS)); 60 | } 61 | return jPanel; 62 | } 63 | 64 | /** 65 | * This method initializes jPanel 66 | * 67 | * @return javax.swing.JPanel 68 | */ 69 | private JPanel getJPanel2() { 70 | if (jPanel == null) { 71 | GridBagConstraints gridBagConstraints11 = new GridBagConstraints(); 72 | gridBagConstraints11.gridx = 0; 73 | gridBagConstraints11.fill = GridBagConstraints.HORIZONTAL; 74 | gridBagConstraints11.gridheight = 1; 75 | gridBagConstraints11.anchor = GridBagConstraints.NORTHWEST; 76 | gridBagConstraints11.gridy = 2; 77 | GridBagConstraints gridBagConstraints2 = new GridBagConstraints(); 78 | gridBagConstraints2.gridx = 0; 79 | gridBagConstraints2.anchor = GridBagConstraints.NORTH; 80 | gridBagConstraints2.fill = GridBagConstraints.NONE; 81 | gridBagConstraints2.gridy = 0; 82 | } 83 | return jPanel; 84 | } 85 | 86 | /** 87 | * This method initializes jTextFieldSourceFilename 88 | * 89 | * @return javax.swing.JTextField 90 | */ 91 | private JTextField getJTextFieldSourceFilename() { 92 | if (jTextFieldSourceFilename == null) { 93 | jTextFieldSourceFilename = new JTextField(); 94 | } 95 | return jTextFieldSourceFilename; 96 | } 97 | 98 | /** 99 | * This method initializes jTextFieldSpellDictionaryFilename 100 | * 101 | * @return javax.swing.JTextField 102 | */ 103 | private JTextField getJTextFieldSpellDictionaryFilename() { 104 | if (jTextFieldSpellDictionaryFilename == null) { 105 | jTextFieldSpellDictionaryFilename = new JTextField(); 106 | } 107 | return jTextFieldSpellDictionaryFilename; 108 | } 109 | 110 | /** 111 | * This method initializes jTextFieldOutputfilename 112 | * 113 | * @return javax.swing.JTextField 114 | */ 115 | private JTextField getJTextFieldOutputfilename() { 116 | if (jTextFieldOutputfilename == null) { 117 | jTextFieldOutputfilename = new JTextField(); 118 | } 119 | return jTextFieldOutputfilename; 120 | } 121 | 122 | /** 123 | * This method initializes jButtonStart 124 | * 125 | * @return javax.swing.JButton 126 | */ 127 | private JButton getJButtonStart() { 128 | if (jButtonStart == null) { 129 | jButtonStart = new JButton(); 130 | jButtonStart.setText("Start parsing"); 131 | jButtonStart.setEnabled(true); 132 | jButtonStart.addActionListener(new java.awt.event.ActionListener() { 133 | public void actionPerformed(java.awt.event.ActionEvent e) { 134 | try 135 | { 136 | FileInputStream sourceFile = new FileInputStream(new File(jTextFieldSourceFilename.getText())); 137 | InputStreamReader input = new InputStreamReader(sourceFile, "UTF-8"); 138 | 139 | FileInputStream dictFile = new FileInputStream(new File(jTextFieldSpellDictionaryFilename.getText())); 140 | InputStreamReader dictInput = new InputStreamReader(dictFile, "UTF-8"); 141 | 142 | FileOutputStream outputFile = new FileOutputStream(new File(jTextFieldOutputfilename.getText())); 143 | OutputStreamWriter output = new OutputStreamWriter(outputFile, "UTF-8"); 144 | mThread = new ParserThread(input, sourceFile.available(), dictInput, dictFile.available(), output, MainForm.this, jTextFieldPossibleLetters.getText(), jTextFieldInnerCharacters.getText(), 145 | jSliderWordsCount.getValue()); 146 | jButtonStart.setEnabled(false); 147 | jLabelProgress.setText("Starting parser thread"); 148 | mThread.start(); 149 | } 150 | catch(Exception ex) 151 | { 152 | ex.printStackTrace(); 153 | showAlertDialog("Error", "Failed to start parser: "+ex.getMessage(), true); 154 | } 155 | } 156 | }); 157 | } 158 | return jButtonStart; 159 | } 160 | 161 | private void showAlertDialog(String title, String message, boolean error) 162 | { 163 | JOptionPane.showMessageDialog(null, message, title, error? JOptionPane.ERROR_MESSAGE : JOptionPane.INFORMATION_MESSAGE); 164 | } 165 | 166 | /** 167 | * This method initializes jTextFieldPossibleLetters 168 | * 169 | * @return javax.swing.JTextField 170 | */ 171 | private JTextField getJTextFieldPossibleLetters() { 172 | if (jTextFieldPossibleLetters == null) { 173 | jTextFieldPossibleLetters = new JTextField(); 174 | jTextFieldPossibleLetters.setText("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"); 175 | } 176 | return jTextFieldPossibleLetters; 177 | } 178 | 179 | /** 180 | * This method initializes jTextFieldInnerCharacters 181 | * 182 | * @return javax.swing.JTextField 183 | */ 184 | private JTextField getJTextFieldInnerCharacters() { 185 | if (jTextFieldInnerCharacters == null) { 186 | jTextFieldInnerCharacters = new JTextField(); 187 | jTextFieldInnerCharacters.setText("'"); 188 | } 189 | return jTextFieldInnerCharacters; 190 | } 191 | 192 | /** 193 | * This method initializes jSliderWordsCount 194 | * 195 | * @return javax.swing.JSlider 196 | */ 197 | private JSlider getJSliderWordsCount() { 198 | if (jSliderWordsCount == null) { 199 | jSliderWordsCount = new JSlider(); 200 | jSliderWordsCount.setMinimum(5000); 201 | jSliderWordsCount.setMaximum(150000); 202 | jSliderWordsCount.setValue(50000); 203 | jSliderWordsCount.addChangeListener(new ChangeListener() { 204 | @Override 205 | public void stateChanged(ChangeEvent arg0) { 206 | jLabelMaxWords.setText("Max words: ("+jSliderWordsCount.getValue()+")"); 207 | } 208 | }); 209 | jLabelMaxWords.setText("Max words: ("+jSliderWordsCount.getValue()+")"); 210 | } 211 | return jSliderWordsCount; 212 | } 213 | 214 | /** 215 | * @param args 216 | */ 217 | public static void main(String[] args) { 218 | // TODO Auto-generated method stub 219 | SwingUtilities.invokeLater(new Runnable() { 220 | public void run() { 221 | MainForm thisClass = new MainForm(); 222 | thisClass.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); 223 | thisClass.setVisible(true); 224 | } 225 | }); 226 | } 227 | 228 | /** 229 | * This is the default constructor 230 | */ 231 | public MainForm() { 232 | super(); 233 | initialize(); 234 | } 235 | 236 | /** 237 | * This method initializes this 238 | * 239 | * @return void 240 | */ 241 | private void initialize() { 242 | this.setSize(593, 388); 243 | this.setIconImage(Toolkit.getDefaultToolkit().getImage(getClass().getResource("/icon_8_key.9.png"))); 244 | this.setContentPane(getJContentPane()); 245 | this.setTitle("AnySoftKeyboard Dictionary Creator"); 246 | } 247 | 248 | /** 249 | * This method initializes jContentPane 250 | * 251 | * @return javax.swing.JPanel 252 | */ 253 | private JPanel getJContentPane() { 254 | if (jContentPane == null) { 255 | jLabelMaxWords = new JLabel(); 256 | jLabelMaxWords.setText("Maximum words"); 257 | jLabel5 = new JLabel(); 258 | jLabel5.setText("Additional characters which are possible only inside a word:"); 259 | jLabel1 = new JLabel(); 260 | jLabel1.setText("Possible characters (lower and upper cases):"); 261 | jLabelProgress = new JLabel(); 262 | jLabelProgress.setText("."); 263 | jLabel4 = new JLabel(); 264 | jLabel4.setText("Select output file:"); 265 | jLabel3 = new JLabel(); 266 | jLabel3.setText("Select spell file:"); 267 | jLabel2 = new JLabel(); 268 | jLabel2.setText("Select source text file:"); 269 | jLabel = new JLabel(); 270 | jLabel.setText("Welcome to AnySoftKeyboard Dictionary Creator"); 271 | jContentPane = new JPanel(); 272 | jContentPane.setLayout(new BoxLayout(getJContentPane(), BoxLayout.Y_AXIS)); 273 | jContentPane.add(getJPanel(), null); 274 | jContentPane.add(jLabel, null); 275 | jContentPane.add(jLabel2, null); 276 | jContentPane.add(getJTextFieldSourceFilename(), null); 277 | jContentPane.add(jLabel3, null); 278 | jContentPane.add(getJTextFieldSpellDictionaryFilename(), null); 279 | jContentPane.add(jLabel4, null); 280 | jContentPane.add(getJTextFieldOutputfilename(), null); 281 | jContentPane.add(jLabel1, null); 282 | jContentPane.add(getJTextFieldPossibleLetters(), null); 283 | jContentPane.add(jLabel5, null); 284 | jContentPane.add(getJTextFieldInnerCharacters(), null); 285 | jContentPane.add(jLabelMaxWords, null); 286 | jContentPane.add(getJSliderWordsCount(), null); 287 | jContentPane.add(getJButtonStart(), null); 288 | jContentPane.add(jLabelProgress, null); 289 | } 290 | return jContentPane; 291 | } 292 | 293 | @Override 294 | public void showErrorMessage(String message) { 295 | // TODO Auto-generated method stub 296 | 297 | } 298 | 299 | @Override 300 | public void updateProgressState(String message, int precentage) { 301 | jLabelProgress.setText(message+" "+precentage+"%"); 302 | } 303 | 304 | @Override 305 | public void onEnded() { 306 | jButtonStart.setEnabled(true); 307 | } 308 | 309 | } // @jve:decl-index=0:visual-constraint="64,28" 310 | -------------------------------------------------------------------------------- /makedictionary/src/main/groovy/com/anysoftkeyboard/tools/makedictionary/XmlWriter.java: -------------------------------------------------------------------------------- 1 | package com.anysoftkeyboard.tools.makedictionary;//Taken from: https://github.com/menny/Java-very-tiny-XmlWriter 2 | /* 3 | * Copyright (c) 2003, Henri Yandell 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or 7 | * without modification, are permitted provided that the 8 | * following conditions are met: 9 | * 10 | * + Redistributions of source code must retain the above copyright notice, 11 | * this list of conditions and the following disclaimer. 12 | * 13 | * + Redistributions in binary form must reproduce the above copyright notice, 14 | * this list of conditions and the following disclaimer in the documentation 15 | * and/or other materials provided with the distribution. 16 | * 17 | * + Neither the name of XmlWriter nor the names of its contributors 18 | * may be used to endorse or promote products derived from this software 19 | * without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 | * POSSIBILITY OF SUCH DAMAGE. 32 | */ 33 | 34 | /* 35 | * Copyright (c) 2003, Henri Yandell 36 | * All rights reserved. 37 | * 38 | * Redistribution and use in source and binary forms, with or 39 | * without modification, are permitted provided that the 40 | * following conditions are met: 41 | * 42 | * + Redistributions of source code must retain the above copyright notice, 43 | * this list of conditions and the following disclaimer. 44 | * 45 | * + Redistributions in binary form must reproduce the above copyright notice, 46 | * this list of conditions and the following disclaimer in the documentation 47 | * and/or other materials provided with the distribution. 48 | * 49 | * + Neither the name of XmlWriter nor the names of its contributors 50 | * may be used to endorse or promote products derived from this software 51 | * without specific prior written permission. 52 | * 53 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 54 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 55 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 56 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 57 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 58 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 59 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 60 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 61 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 62 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 63 | * POSSIBILITY OF SUCH DAMAGE. 64 | */ 65 | 66 | /* 67 | * Copyright (c) 2010, Menny Even Danan 68 | * All rights reserved. 69 | * 70 | * Redistribution and use in source and binary forms, with or 71 | * without modification, are permitted provided that the 72 | * following conditions are met: 73 | * 74 | * + Redistributions of source code must retain the above copyright notice, 75 | * this list of conditions and the following disclaimer. 76 | * 77 | * + Redistributions in binary form must reproduce the above copyright notice, 78 | * this list of conditions and the following disclaimer in the documentation 79 | * and/or other materials provided with the distribution. 80 | * 81 | * + Neither the name of XmlWriter nor the names of its contributors 82 | * may be used to endorse or promote products derived from this software 83 | * without specific prior written permission. 84 | * 85 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 86 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 87 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 88 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 89 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 90 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 91 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 92 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 93 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 94 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 95 | * POSSIBILITY OF SUCH DAMAGE. 96 | */ 97 | import java.io.File; 98 | import java.io.FileWriter; 99 | import java.io.IOException; 100 | import java.io.InvalidObjectException; 101 | import java.io.Writer; 102 | import java.util.Stack; 103 | 104 | /** 105 | * Makes writing XML much much easier. 106 | * 107 | * @author Henri Yandell 108 | * @author Menny Even Danan - just 109 | * added some features on Henri's initial version 110 | * @version 0.2 111 | */ 112 | class XmlWriter { 113 | 114 | private static final String INDENT_STRING = " "; 115 | private final boolean thisIsWriterOwner;// is this instance the owner? 116 | private final Writer writer; // underlying writer 117 | private final int indentingOffset; 118 | private final Stack stack; // of xml entity names 119 | private final StringBuffer attrs; // current attribute string 120 | private boolean empty; // is the current node empty 121 | private boolean justWroteText; 122 | private boolean closed; // is the current node closed... 123 | 124 | /** 125 | * Create an XmlWriter on top of an existing java.io.Writer. 126 | * 127 | * @throws IOException 128 | */ 129 | XmlWriter(Writer writer, boolean takeOwnership, int indentingOffset, boolean addXmlPrefix) 130 | throws IOException { 131 | thisIsWriterOwner = takeOwnership; 132 | this.indentingOffset = indentingOffset; 133 | this.writer = writer; 134 | this.closed = true; 135 | this.stack = new Stack<>(); 136 | this.attrs = new StringBuffer(); 137 | if (addXmlPrefix) 138 | this.writer.write("\n"); 139 | } 140 | 141 | XmlWriter(File outputFile) throws IOException { 142 | this(new FileWriter(outputFile), true, 0, true); 143 | } 144 | 145 | /** 146 | * Begin to output an entity. 147 | * 148 | * @param name name of entity. 149 | */ 150 | XmlWriter writeEntity(String name) throws IOException { 151 | closeOpeningTag(true); 152 | this.closed = false; 153 | for (int tabIndex = 0; tabIndex < stack.size() + indentingOffset; tabIndex++) 154 | this.writer.write(INDENT_STRING); 155 | this.writer.write("<"); 156 | this.writer.write(name); 157 | stack.add(name); 158 | this.empty = true; 159 | this.justWroteText = false; 160 | return this; 161 | } 162 | 163 | // close off the opening tag 164 | private void closeOpeningTag(final boolean newLine) throws IOException { 165 | if (!this.closed) { 166 | writeAttributes(); 167 | this.closed = true; 168 | this.writer.write(">"); 169 | if (newLine) 170 | this.writer.write("\n"); 171 | } 172 | } 173 | 174 | // write out all current attributes 175 | private void writeAttributes() throws IOException { 176 | this.writer.write(this.attrs.toString()); 177 | this.attrs.setLength(0); 178 | this.empty = false; 179 | } 180 | 181 | /** 182 | * Write an attribute out for the current entity. Any xml characters in the 183 | * value are escaped. Currently it does not actually throw the exception, 184 | * but the api is set that way for future changes. 185 | * 186 | * @param attr name of attribute. 187 | * @param value value of attribute. 188 | */ 189 | public XmlWriter writeAttribute(String attr, String value) { 190 | this.attrs.append(" "); 191 | this.attrs.append(attr); 192 | this.attrs.append("=\""); 193 | this.attrs.append(escapeXml(value)); 194 | this.attrs.append("\""); 195 | return this; 196 | } 197 | 198 | /** 199 | * End the current entity. This will throw an exception if it is called when 200 | * there is not a currently open entity. 201 | * 202 | * @throws IOException 203 | */ 204 | public XmlWriter endEntity() throws IOException { 205 | if (this.stack.empty()) { 206 | throw new InvalidObjectException("Called endEntity too many times. "); 207 | } 208 | String name = this.stack.pop(); 209 | if (name != null) { 210 | if (this.empty) { 211 | writeAttributes(); 212 | this.writer.write("/>\n"); 213 | } else { 214 | if (!this.justWroteText) 215 | { 216 | for (int tabIndex = 0; tabIndex < stack.size() + indentingOffset; tabIndex++) 217 | this.writer.write(INDENT_STRING); 218 | } 219 | this.writer.write("\n"); 222 | } 223 | this.empty = false; 224 | this.closed = true; 225 | this.justWroteText = false; 226 | } 227 | return this; 228 | } 229 | 230 | /** 231 | * Close this writer. It does not close the underlying writer, but does 232 | * throw an exception if there are as yet unclosed tags. 233 | * 234 | * @throws IOException 235 | */ 236 | public void close() throws IOException { 237 | if (!this.stack.empty()) { 238 | throw new InvalidObjectException("Tags are not all closed. " + 239 | "Possibly, " + this.stack.pop() + " is unclosed. "); 240 | } 241 | if (thisIsWriterOwner) 242 | { 243 | this.writer.flush(); 244 | this.writer.close(); 245 | } 246 | } 247 | 248 | /** 249 | * Output body text. Any xml characters are escaped. 250 | */ 251 | public XmlWriter writeText(String text) throws IOException { 252 | closeOpeningTag(false); 253 | this.empty = false; 254 | this.justWroteText = true; 255 | this.writer.write(escapeXml(text)); 256 | return this; 257 | } 258 | 259 | // Static functions lifted from generationjava helper classes 260 | // to make the jar smaller. 261 | 262 | // from XmlW 263 | static public String escapeXml(String str) { 264 | str = replaceString(str, "&", "&"); 265 | str = replaceString(str, "<", "<"); 266 | str = replaceString(str, ">", ">"); 267 | str = replaceString(str, "\"", """); 268 | str = replaceString(str, "'", "'"); 269 | return str; 270 | } 271 | 272 | // from StringW 273 | static public String replaceString(String text, String repl, String with) { 274 | return replaceString(text, repl, with, -1); 275 | } 276 | 277 | /** 278 | * Replace a string with another string inside a larger string, for the 279 | * first n values of the search string. 280 | * 281 | * @param text String to do search and replace in 282 | * @param repl String to search for 283 | * @param with String to replace with 284 | * @param max int values to replace 285 | * @return String with n values replacEd 286 | */ 287 | static String replaceString(String text, String repl, String with, int max) { 288 | if (text == null) { 289 | return null; 290 | } 291 | 292 | StringBuffer buffer = new StringBuffer(text.length()); 293 | int start = 0; 294 | int end = 0; 295 | while ((end = text.indexOf(repl, start)) != -1) { 296 | buffer.append(text.substring(start, end)).append(with); 297 | start = end + repl.length(); 298 | 299 | if (--max == 0) { 300 | break; 301 | } 302 | } 303 | buffer.append(text.substring(start)); 304 | 305 | return buffer.toString(); 306 | } 307 | // 308 | // static public void test1() throws WritingException { 309 | // Writer writer = new java.io.StringWriter(); 310 | // XmlWriter xmlwriter = new XmlWriter(writer); 311 | // xmlwriter.writeEntity("person").writeAttribute("name", 312 | // "fred").writeAttribute("age", 313 | // "12").writeEntity("phone").writeText("4254343").endEntity().writeEntity("bob").endEntity().endEntity(); 314 | // xmlwriter.close(); 315 | // System.err.println(writer.toString()); 316 | // } 317 | // static public void test2() throws WritingException { 318 | // Writer writer = new java.io.StringWriter(); 319 | // XmlWriter xmlwriter = new XmlWriter(writer); 320 | // xmlwriter.writeEntity("person"); 321 | // xmlwriter.writeAttribute("name", "fred"); 322 | // xmlwriter.writeAttribute("age", "12"); 323 | // xmlwriter.writeEntity("phone"); 324 | // xmlwriter.writeText("4254343"); 325 | // xmlwriter.endEntity(); 326 | // xmlwriter.writeEntity("bob"); 327 | // xmlwriter.endEntity(); 328 | // xmlwriter.endEntity(); 329 | // xmlwriter.close(); 330 | // System.err.println(writer.toString()); 331 | // } 332 | 333 | } 334 | -------------------------------------------------------------------------------- /generatewordslist/src/main/groovy/com/anysoftkeyboard/tools/generatewordslist/XmlWriter.java: -------------------------------------------------------------------------------- 1 | package com.anysoftkeyboard.tools.generatewordslist; 2 | //Taken from: https://github.com/menny/Java-very-tiny-XmlWriter 3 | /* 4 | * Copyright (c) 2003, Henri Yandell 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or 8 | * without modification, are permitted provided that the 9 | * following conditions are met: 10 | * 11 | * + Redistributions of source code must retain the above copyright notice, 12 | * this list of conditions and the following disclaimer. 13 | * 14 | * + Redistributions in binary form must reproduce the above copyright notice, 15 | * this list of conditions and the following disclaimer in the documentation 16 | * and/or other materials provided with the distribution. 17 | * 18 | * + Neither the name of XmlWriter nor the names of its contributors 19 | * may be used to endorse or promote products derived from this software 20 | * without specific prior written permission. 21 | * 22 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 | * POSSIBILITY OF SUCH DAMAGE. 33 | */ 34 | 35 | /* 36 | * Copyright (c) 2003, Henri Yandell 37 | * All rights reserved. 38 | * 39 | * Redistribution and use in source and binary forms, with or 40 | * without modification, are permitted provided that the 41 | * following conditions are met: 42 | * 43 | * + Redistributions of source code must retain the above copyright notice, 44 | * this list of conditions and the following disclaimer. 45 | * 46 | * + Redistributions in binary form must reproduce the above copyright notice, 47 | * this list of conditions and the following disclaimer in the documentation 48 | * and/or other materials provided with the distribution. 49 | * 50 | * + Neither the name of XmlWriter nor the names of its contributors 51 | * may be used to endorse or promote products derived from this software 52 | * without specific prior written permission. 53 | * 54 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 55 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 56 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 57 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 58 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 59 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 60 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 61 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 62 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 63 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 64 | * POSSIBILITY OF SUCH DAMAGE. 65 | */ 66 | 67 | /* 68 | * Copyright (c) 2010, Menny Even Danan 69 | * All rights reserved. 70 | * 71 | * Redistribution and use in source and binary forms, with or 72 | * without modification, are permitted provided that the 73 | * following conditions are met: 74 | * 75 | * + Redistributions of source code must retain the above copyright notice, 76 | * this list of conditions and the following disclaimer. 77 | * 78 | * + Redistributions in binary form must reproduce the above copyright notice, 79 | * this list of conditions and the following disclaimer in the documentation 80 | * and/or other materials provided with the distribution. 81 | * 82 | * + Neither the name of XmlWriter nor the names of its contributors 83 | * may be used to endorse or promote products derived from this software 84 | * without specific prior written permission. 85 | * 86 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 87 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 88 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 89 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 90 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 91 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 92 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 93 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 94 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 95 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 96 | * POSSIBILITY OF SUCH DAMAGE. 97 | */ 98 | import java.io.File; 99 | import java.io.FileWriter; 100 | import java.io.IOException; 101 | import java.io.InvalidObjectException; 102 | import java.io.Writer; 103 | import java.util.Stack; 104 | 105 | /** 106 | * Makes writing XML much much easier. 107 | * 108 | * @author Henri Yandell 109 | * @author Menny Even Danan - just 110 | * added some features on Henri's initial version 111 | * @version 0.2 112 | */ 113 | class XmlWriter { 114 | 115 | private static final String INDENT_STRING = " "; 116 | private final boolean thisIsWriterOwner;// is this instance the owner? 117 | private final Writer writer; // underlying writer 118 | private final int indentingOffset; 119 | private final Stack stack; // of xml entity names 120 | private final StringBuffer attrs; // current attribute string 121 | private boolean empty; // is the current node empty 122 | private boolean justWroteText; 123 | private boolean closed; // is the current node closed... 124 | 125 | /** 126 | * Create an XmlWriter on top of an existing java.io.Writer. 127 | * 128 | * @throws IOException 129 | */ 130 | XmlWriter(Writer writer, boolean takeOwnership, int indentingOffset, boolean addXmlPrefix) 131 | throws IOException { 132 | thisIsWriterOwner = takeOwnership; 133 | this.indentingOffset = indentingOffset; 134 | this.writer = writer; 135 | this.closed = true; 136 | this.stack = new Stack<>(); 137 | this.attrs = new StringBuffer(); 138 | if (addXmlPrefix) 139 | this.writer.write("\n"); 140 | } 141 | 142 | XmlWriter(File outputFile) throws IOException { 143 | this(new FileWriter(outputFile), true, 0, true); 144 | } 145 | 146 | /** 147 | * Begin to output an entity. 148 | * 149 | * @param name name of entity. 150 | */ 151 | XmlWriter writeEntity(String name) throws IOException { 152 | closeOpeningTag(true); 153 | this.closed = false; 154 | for (int tabIndex = 0; tabIndex < stack.size() + indentingOffset; tabIndex++) 155 | this.writer.write(INDENT_STRING); 156 | this.writer.write("<"); 157 | this.writer.write(name); 158 | stack.add(name); 159 | this.empty = true; 160 | this.justWroteText = false; 161 | return this; 162 | } 163 | 164 | // close off the opening tag 165 | private void closeOpeningTag(final boolean newLine) throws IOException { 166 | if (!this.closed) { 167 | writeAttributes(); 168 | this.closed = true; 169 | this.writer.write(">"); 170 | if (newLine) 171 | this.writer.write("\n"); 172 | } 173 | } 174 | 175 | // write out all current attributes 176 | private void writeAttributes() throws IOException { 177 | this.writer.write(this.attrs.toString()); 178 | this.attrs.setLength(0); 179 | this.empty = false; 180 | } 181 | 182 | /** 183 | * Write an attribute out for the current entity. Any xml characters in the 184 | * value are escaped. Currently it does not actually throw the exception, 185 | * but the api is set that way for future changes. 186 | * 187 | * @param attr name of attribute. 188 | * @param value value of attribute. 189 | */ 190 | public XmlWriter writeAttribute(String attr, String value) { 191 | this.attrs.append(" "); 192 | this.attrs.append(attr); 193 | this.attrs.append("=\""); 194 | this.attrs.append(escapeXml(value)); 195 | this.attrs.append("\""); 196 | return this; 197 | } 198 | 199 | /** 200 | * End the current entity. This will throw an exception if it is called when 201 | * there is not a currently open entity. 202 | * 203 | * @throws IOException 204 | */ 205 | public XmlWriter endEntity() throws IOException { 206 | if (this.stack.empty()) { 207 | throw new InvalidObjectException("Called endEntity too many times. "); 208 | } 209 | String name = this.stack.pop(); 210 | if (name != null) { 211 | if (this.empty) { 212 | writeAttributes(); 213 | this.writer.write("/>\n"); 214 | } else { 215 | if (!this.justWroteText) 216 | { 217 | for (int tabIndex = 0; tabIndex < stack.size() + indentingOffset; tabIndex++) 218 | this.writer.write(INDENT_STRING); 219 | } 220 | this.writer.write("\n"); 223 | } 224 | this.empty = false; 225 | this.closed = true; 226 | this.justWroteText = false; 227 | } 228 | return this; 229 | } 230 | 231 | /** 232 | * Close this writer. It does not close the underlying writer, but does 233 | * throw an exception if there are as yet unclosed tags. 234 | * 235 | * @throws IOException 236 | */ 237 | public void close() throws IOException { 238 | if (!this.stack.empty()) { 239 | throw new InvalidObjectException("Tags are not all closed. " + 240 | "Possibly, " + this.stack.pop() + " is unclosed. "); 241 | } 242 | if (thisIsWriterOwner) 243 | { 244 | this.writer.flush(); 245 | this.writer.close(); 246 | } 247 | } 248 | 249 | /** 250 | * Output body text. Any xml characters are escaped. 251 | */ 252 | public XmlWriter writeText(String text) throws IOException { 253 | closeOpeningTag(false); 254 | this.empty = false; 255 | this.justWroteText = true; 256 | this.writer.write(escapeXml(text)); 257 | return this; 258 | } 259 | 260 | // Static functions lifted from generationjava helper classes 261 | // to make the jar smaller. 262 | 263 | // from XmlW 264 | static public String escapeXml(String str) { 265 | str = replaceString(str, "&", "&"); 266 | str = replaceString(str, "<", "<"); 267 | str = replaceString(str, ">", ">"); 268 | str = replaceString(str, "\"", """); 269 | str = replaceString(str, "'", "'"); 270 | return str; 271 | } 272 | 273 | // from StringW 274 | static public String replaceString(String text, String repl, String with) { 275 | return replaceString(text, repl, with, -1); 276 | } 277 | 278 | /** 279 | * Replace a string with another string inside a larger string, for the 280 | * first n values of the search string. 281 | * 282 | * @param text String to do search and replace in 283 | * @param repl String to search for 284 | * @param with String to replace with 285 | * @param max int values to replace 286 | * @return String with n values replacEd 287 | */ 288 | static String replaceString(String text, String repl, String with, int max) { 289 | if (text == null) { 290 | return null; 291 | } 292 | 293 | StringBuffer buffer = new StringBuffer(text.length()); 294 | int start = 0; 295 | int end = 0; 296 | while ((end = text.indexOf(repl, start)) != -1) { 297 | buffer.append(text.substring(start, end)).append(with); 298 | start = end + repl.length(); 299 | 300 | if (--max == 0) { 301 | break; 302 | } 303 | } 304 | buffer.append(text.substring(start)); 305 | 306 | return buffer.toString(); 307 | } 308 | // 309 | // static public void test1() throws WritingException { 310 | // Writer writer = new java.io.StringWriter(); 311 | // XmlWriter xmlwriter = new XmlWriter(writer); 312 | // xmlwriter.writeEntity("person").writeAttribute("name", 313 | // "fred").writeAttribute("age", 314 | // "12").writeEntity("phone").writeText("4254343").endEntity().writeEntity("bob").endEntity().endEntity(); 315 | // xmlwriter.close(); 316 | // System.err.println(writer.toString()); 317 | // } 318 | // static public void test2() throws WritingException { 319 | // Writer writer = new java.io.StringWriter(); 320 | // XmlWriter xmlwriter = new XmlWriter(writer); 321 | // xmlwriter.writeEntity("person"); 322 | // xmlwriter.writeAttribute("name", "fred"); 323 | // xmlwriter.writeAttribute("age", "12"); 324 | // xmlwriter.writeEntity("phone"); 325 | // xmlwriter.writeText("4254343"); 326 | // xmlwriter.endEntity(); 327 | // xmlwriter.writeEntity("bob"); 328 | // xmlwriter.endEntity(); 329 | // xmlwriter.endEntity(); 330 | // xmlwriter.close(); 331 | // System.err.println(writer.toString()); 332 | // } 333 | 334 | } 335 | -------------------------------------------------------------------------------- /packverifier-old/src/Verifier.java: -------------------------------------------------------------------------------- 1 | import org.xml.sax.Attributes; 2 | import org.xml.sax.SAXException; 3 | import org.xml.sax.helpers.DefaultHandler; 4 | 5 | import java.io.BufferedReader; 6 | import java.io.File; 7 | import java.io.FileReader; 8 | import java.io.IOException; 9 | import java.security.NoSuchAlgorithmException; 10 | 11 | import javax.xml.parsers.ParserConfigurationException; 12 | import javax.xml.parsers.SAXParser; 13 | import javax.xml.parsers.SAXParserFactory; 14 | 15 | /* 16 | * Copyright (C) 2012 AnySoftKeyboard. 17 | * 18 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not 19 | * use this file except in compliance with the License. You may obtain a copy of 20 | * the License at 21 | * 22 | * http://www.apache.org/licenses/LICENSE-2.0 23 | * 24 | * Unless required by applicable law or agreed to in writing, software 25 | * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 26 | * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 27 | * License for the specific language governing permissions and limitations under 28 | * the License. 29 | */ 30 | 31 | public class Verifier { 32 | 33 | public static class PackDetails 34 | { 35 | public final String PackageName; 36 | public final String KeyboardSourceCodeFile; 37 | public final String DictionarySourceCodeFile; 38 | public final String ThemeSourceCodeFile; 39 | 40 | public PackDetails(String packName, String keyboardSourceFile, String dictionarySourceFile, 41 | String themeSourceFile) { 42 | PackageName = packName; 43 | KeyboardSourceCodeFile = keyboardSourceFile != null 44 | && keyboardSourceFile.startsWith(".") ? PackageName 45 | + keyboardSourceFile : keyboardSourceFile; 46 | DictionarySourceCodeFile = dictionarySourceFile != null 47 | && dictionarySourceFile.startsWith(".") ? PackageName 48 | + dictionarySourceFile : dictionarySourceFile; 49 | ThemeSourceCodeFile = themeSourceFile != null && themeSourceFile.startsWith(".") ? PackageName 50 | + themeSourceFile 51 | : themeSourceFile; 52 | } 53 | } 54 | 55 | public static void main(String[] args) throws ParserConfigurationException, SAXException, 56 | IOException, NoSuchAlgorithmException { 57 | boolean releaseMode = false; 58 | if (args != null) 59 | { 60 | for (String arg : args) { 61 | if (arg.equals("release")) 62 | releaseMode = true; 63 | } 64 | } 65 | if (releaseMode) 66 | System.out.println("Starting verification of RELEASE mode!"); 67 | // will verify that the project at the current working folder is valid 68 | // tests: 69 | // 1) package name is valid 70 | // 1.1) in AndroidManifest.xml 71 | // 1.2) in any source file 72 | // 2) any declared pack XML is valid 73 | // 2.1) pack ID is unique and valid 74 | 75 | final File currentFolder = new File(System.getProperty("user.dir")); 76 | 77 | final PackDetails packDetails = verifyAndroidManifestAndGetPackageName(currentFolder); 78 | System.out.println("Package name is " + packDetails.PackageName); 79 | 80 | verifyStringsFileIsValid(currentFolder, packDetails); 81 | 82 | verifyAntBuildFile(currentFolder, packDetails); 83 | 84 | verifySourceCodeHasCorrectPackageName(currentFolder, packDetails); 85 | 86 | if (packDetails.KeyboardSourceCodeFile != null) 87 | verifyKeyboardDeclaration(currentFolder, packDetails); 88 | 89 | if (packDetails.DictionarySourceCodeFile != null) 90 | verifyDictionaryDeclaration(currentFolder, packDetails); 91 | 92 | if (packDetails.ThemeSourceCodeFile != null) 93 | verifyThemeDeclaration(currentFolder, packDetails); 94 | 95 | //checking app icons have been changed 96 | verifyFileCheckSumHasChanged(new File(currentFolder, "/res/drawable/app_icon.png"), true, "0e71ddf43d0147f7cacc7e1d154cb2f2a031d804", releaseMode); 97 | verifyFileCheckSumHasChanged(new File(currentFolder, "/res/drawable-hdpi/app_icon.png"), false, "ea1f28b777177aae01fb0f717c4c04c0f72cac71", releaseMode); 98 | verifyFileCheckSumHasChanged(new File(currentFolder, "/res/drawable-xhdpi/app_icon.png"), false, "862166a2f482b0a9422dc4ed8b293f93b04d6e20", releaseMode); 99 | verifyFileCheckSumHasChanged(new File(currentFolder, "/StoreStuff/landscape.png"), true, "0b39e1c3824515ff2f406bd1ad811774306cdfe4", releaseMode); 100 | verifyFileCheckSumHasChanged(new File(currentFolder, "/StoreStuff/portrait.png"), true, "cd995002d2ea98b16d1e1a1b981b0dadd996c6a6", releaseMode); 101 | verifyFileCheckSumHasChanged(new File(currentFolder, "/StoreStuff/store_hi_res_icon.png"), true, "83d31f26cd4bb3dc719aaa739a82ab6fc5af1b82", releaseMode); 102 | } 103 | 104 | private static void verifyFileCheckSumHasChanged(File file, final boolean mustExist, final String invalidCheckSum, boolean releaseMode) throws NoSuchAlgorithmException, IOException { 105 | if (!mustExist && !file.exists()) return; 106 | 107 | final String currentFileCheckSum = FileCheckSumGenerator.generateFileCheckSum(file); 108 | System.out.println("The file '"+file+"' checksum is "+currentFileCheckSum); 109 | if (currentFileCheckSum.equals(invalidCheckSum)) { 110 | if (releaseMode) { 111 | throw new InvalidPackConfiguration(file.getAbsolutePath(), "The file need to be customized for this pack!"); 112 | } else { 113 | System.out.println("The file '"+file+"' need to be customized for this pack!"); 114 | } 115 | } 116 | } 117 | 118 | private static void verifyAntBuildFile(File currentFolder, PackDetails packDetails) 119 | throws ParserConfigurationException, SAXException, IOException { 120 | File antFile = new File(currentFolder, "build.xml"); 121 | System.out.println("Verifying ANT build file for validity..."); 122 | SAXParser parser = SAXParserFactory.newInstance().newSAXParser(); 123 | parser.parse(antFile, new DefaultHandler() { 124 | @Override 125 | public void startElement(String uri, String localName, String qName, 126 | Attributes attributes) throws SAXException { 127 | super.startElement(uri, localName, qName, attributes); 128 | if (qName.equals("project")) { 129 | if (attributes.getValue("name").equals("LanguagePack")) { 130 | throw new InvalidPackConfiguration("build.xml", 131 | "The name of the ant should be changed"); 132 | } 133 | } 134 | } 135 | }); 136 | } 137 | 138 | private static void verifyKeyboardDeclaration(File currentFolder, PackDetails packDetails) 139 | throws ParserConfigurationException, SAXException, IOException { 140 | File declarationFile = new File(currentFolder, "res/xml/keyboards.xml"); 141 | System.out.println("Verifying plugins declaration file " + declarationFile 142 | + " for validity..."); 143 | SAXParser parser = SAXParserFactory.newInstance().newSAXParser(); 144 | BasicDeclarationSAXParserHandler handler = new KeyboardDeclarationSAXParserHandler(); 145 | parser.parse(declarationFile, handler); 146 | } 147 | 148 | private static void verifyDictionaryDeclaration(File currentFolder, PackDetails packDetails) 149 | throws ParserConfigurationException, SAXException, IOException { 150 | File declarationFile = new File(currentFolder, "res/xml/dictionaries.xml"); 151 | System.out.println("Verifying plugins declaration file " + declarationFile 152 | + " for validity..."); 153 | SAXParser parser = SAXParserFactory.newInstance().newSAXParser(); 154 | BasicDeclarationSAXParserHandler handler = new DictionaryDeclarationSAXParserHandler(); 155 | parser.parse(declarationFile, handler); 156 | } 157 | 158 | private static void verifyThemeDeclaration(File currentFolder, PackDetails packDetails) 159 | throws ParserConfigurationException, SAXException, IOException { 160 | File declarationFile = new File(currentFolder, "res/xml/themes.xml"); 161 | System.out.println("Verifying plugins declaration file " + declarationFile 162 | + " for validity..."); 163 | SAXParser parser = SAXParserFactory.newInstance().newSAXParser(); 164 | BasicDeclarationSAXParserHandler handler = new ThemeDeclarationSAXParserHandler(); 165 | parser.parse(declarationFile, handler); 166 | } 167 | 168 | private static void verifySourceCodeHasCorrectPackageName(File currentFolder, 169 | PackDetails packDetails) throws IOException { 170 | final File srcFolder = new File(currentFolder, "src"); 171 | if (packDetails.KeyboardSourceCodeFile != null) 172 | verifySourceCodeFileHasCorrectPackageName(new File(srcFolder, 173 | packDetails.KeyboardSourceCodeFile.replace('.', '/') + ".java"), packDetails); 174 | if (packDetails.DictionarySourceCodeFile != null) 175 | verifySourceCodeFileHasCorrectPackageName(new File(srcFolder, 176 | packDetails.DictionarySourceCodeFile.replace('.', '/') + ".java"), packDetails); 177 | if (packDetails.ThemeSourceCodeFile != null) 178 | verifySourceCodeFileHasCorrectPackageName(new File(srcFolder, 179 | packDetails.ThemeSourceCodeFile.replace('.', '/') + ".java"), packDetails); 180 | } 181 | 182 | private static void verifySourceCodeFileHasCorrectPackageName(File source, 183 | PackDetails packDetails) throws IOException { 184 | final String requiredPackage = "package " + packDetails.PackageName + ";"; 185 | System.out.println("Verifying source file " + source + " for package name validity (" 186 | + requiredPackage + ")..."); 187 | if (!source.exists()) { 188 | throw new InvalidPackConfiguration(source.getAbsolutePath(), 189 | "Source file does not exist!"); 190 | } 191 | if (!source.isFile()) { 192 | throw new InvalidPackConfiguration(source.getAbsolutePath(), 193 | "Specified file is not a file!"); 194 | } 195 | 196 | BufferedReader in = new BufferedReader(new FileReader(source)); 197 | String strLine = null; 198 | boolean found = false; 199 | try { 200 | while (!found && (strLine = in.readLine()) != null) { 201 | if (strLine.equals(requiredPackage)) 202 | found = true; 203 | } 204 | } finally { 205 | in.close(); 206 | } 207 | 208 | if (!found) { 209 | throw new InvalidPackConfiguration(source.getAbsolutePath(), 210 | "Package name is invalid, it should be " + packDetails.PackageName); 211 | } 212 | } 213 | 214 | 215 | 216 | private static void verifyStringsFileIsValid(File currentFolder, PackDetails packDetails) throws IOException, ParserConfigurationException, SAXException { 217 | final File stringFile = new File(currentFolder, "res/values/strings.xml"); 218 | System.out.println("Verifying strings resources file for validity..."); 219 | SAXParser parser = SAXParserFactory.newInstance().newSAXParser(); 220 | parser.parse(stringFile, new DefaultHandler() { 221 | private boolean mInPackName = false; 222 | private boolean mInKeyboardName = false; 223 | private boolean mInDictionaryName = false; 224 | private boolean mInThemeName = false; 225 | 226 | private String mResourceValue = ""; 227 | @Override 228 | public void startElement(String uri, String localName, String qName, 229 | Attributes attributes) throws SAXException { 230 | super.startElement(uri, localName, qName, attributes); 231 | if (qName.equals("string")) { 232 | final String resourceName = attributes.getValue("name"); 233 | mInPackName = resourceName != null && resourceName.equals("app_name"); 234 | mInKeyboardName = resourceName != null && resourceName.equals("keyboard_name"); 235 | mInDictionaryName = resourceName != null && resourceName.equals("dictionary_name"); 236 | mInThemeName = resourceName != null && resourceName.equals("theme_name"); 237 | } 238 | } 239 | 240 | @Override 241 | public void characters(char[] ch, int start, int length) throws SAXException { 242 | super.characters(ch, start, length); 243 | mResourceValue += new String(ch, start, length); 244 | } 245 | 246 | @Override 247 | public void endElement(String uri, String localName, String qName) throws SAXException { 248 | super.endElement(uri, localName, qName); 249 | if (mInPackName) { 250 | if (!mResourceValue.contains("AnySoftKeyboard")) { 251 | throw new InvalidPackConfiguration(stringFile, "Pack name must include 'AnySoftKeyboard'!"); 252 | } 253 | if (mResourceValue.contains("change_me")) { 254 | throw new InvalidPackConfiguration(stringFile, "Pack name must be customized for this new pack!"); 255 | } 256 | } 257 | if (mInKeyboardName) { 258 | if (mResourceValue.contains("change_me")) { 259 | throw new InvalidPackConfiguration(stringFile, "Keyboard name must be customized for this new pack!"); 260 | } 261 | } 262 | if (mInDictionaryName) { 263 | if (mResourceValue.contains("change_me")) { 264 | throw new InvalidPackConfiguration(stringFile, "Dictionary name must be customized for this new pack!"); 265 | } 266 | } 267 | if (mInThemeName) { 268 | if (mResourceValue.contains("change_me")) { 269 | throw new InvalidPackConfiguration(stringFile, "Theme name must be customized for this new pack!"); 270 | } 271 | } 272 | mResourceValue = ""; 273 | } 274 | }); 275 | } 276 | 277 | private static PackDetails verifyAndroidManifestAndGetPackageName(File currentFolder) 278 | throws ParserConfigurationException, SAXException, IOException { 279 | final File inputFile = new File(currentFolder, "AndroidManifest.xml"); 280 | System.out.println("Reading AndroidManifest from input " + inputFile.getAbsolutePath()); 281 | 282 | SAXParser parser = SAXParserFactory.newInstance().newSAXParser(); 283 | AndroidManifestSAXParserHandler handler = new AndroidManifestSAXParserHandler(); 284 | parser.parse(inputFile, handler); 285 | 286 | return handler.createPackDetails(); 287 | } 288 | 289 | } 290 | --------------------------------------------------------------------------------