├── .gitignore ├── AUTHORS ├── LICENSE ├── README.md ├── README_en.md ├── build.sh ├── builder ├── build.sh ├── docformat │ ├── Makefile │ ├── build.sh │ ├── etc │ │ └── docformat │ │ │ └── docformat.json │ ├── script │ │ └── common │ │ │ ├── clean.sh │ │ │ ├── demo.sh │ │ │ ├── dummy-hadoop.sh │ │ │ ├── install.sh │ │ │ ├── local-hadoop.sh │ │ │ └── real-hadoop.sh │ └── src │ │ ├── common │ │ └── common.go │ │ ├── github.com │ │ ├── bitly │ │ │ ├── go-nsq │ │ │ │ ├── .travis.yml │ │ │ │ ├── ChangeLog.md │ │ │ │ ├── LICENSE │ │ │ │ ├── README.md │ │ │ │ ├── UPGRADING.md │ │ │ │ ├── api_request.go │ │ │ │ ├── command.go │ │ │ │ ├── command_test.go │ │ │ │ ├── config.go │ │ │ │ ├── config_flag.go │ │ │ │ ├── config_flag_test.go │ │ │ │ ├── config_test.go │ │ │ │ ├── conn.go │ │ │ │ ├── consumer.go │ │ │ │ ├── consumer_test.go │ │ │ │ ├── delegates.go │ │ │ │ ├── errors.go │ │ │ │ ├── message.go │ │ │ │ ├── mock_test.go │ │ │ │ ├── producer.go │ │ │ │ ├── producer_test.go │ │ │ │ ├── protocol.go │ │ │ │ ├── states.go │ │ │ │ ├── test.sh │ │ │ │ ├── test │ │ │ │ │ ├── ca.pem │ │ │ │ │ ├── server.key │ │ │ │ │ └── server.pem │ │ │ │ └── version.go │ │ │ └── go-simplejson │ │ │ │ ├── .travis.yml │ │ │ │ ├── LICENSE │ │ │ │ ├── README.md │ │ │ │ ├── simplejson.go │ │ │ │ ├── simplejson_go10.go │ │ │ │ ├── simplejson_go10_test.go │ │ │ │ ├── simplejson_go11.go │ │ │ │ ├── simplejson_go11_test.go │ │ │ │ └── simplejson_test.go │ │ ├── donnie4w │ │ │ └── go-logger │ │ │ │ ├── example │ │ │ │ └── logger_test.go │ │ │ │ ├── logger │ │ │ │ ├── logger.go │ │ │ │ └── logger_test.go │ │ │ │ └── readme.txt │ │ ├── garyburd │ │ │ └── redigo │ │ │ │ ├── LICENSE │ │ │ │ ├── README.markdown │ │ │ │ ├── internal │ │ │ │ ├── commandinfo.go │ │ │ │ ├── commandinfo_test.go │ │ │ │ └── redistest │ │ │ │ │ └── testdb.go │ │ │ │ ├── redis │ │ │ │ ├── conn.go │ │ │ │ ├── conn_test.go │ │ │ │ ├── doc.go │ │ │ │ ├── log.go │ │ │ │ ├── pool.go │ │ │ │ ├── pool_test.go │ │ │ │ ├── pubsub.go │ │ │ │ ├── pubsub_test.go │ │ │ │ ├── redis.go │ │ │ │ ├── reply.go │ │ │ │ ├── reply_test.go │ │ │ │ ├── scan.go │ │ │ │ ├── scan_test.go │ │ │ │ ├── script.go │ │ │ │ ├── script_test.go │ │ │ │ ├── test_test.go │ │ │ │ └── zpop_example_test.go │ │ │ │ └── redisx │ │ │ │ ├── connmux.go │ │ │ │ ├── connmux_test.go │ │ │ │ └── doc.go │ │ ├── golang │ │ │ └── protobuf │ │ │ │ ├── .gitignore │ │ │ │ ├── AUTHORS │ │ │ │ ├── CONTRIBUTORS │ │ │ │ ├── LICENSE │ │ │ │ ├── Make.protobuf │ │ │ │ ├── Makefile │ │ │ │ ├── README │ │ │ │ ├── README.md │ │ │ │ ├── jsonpb │ │ │ │ ├── jsonpb.go │ │ │ │ ├── jsonpb_test.go │ │ │ │ └── jsonpb_test_proto │ │ │ │ │ ├── Makefile │ │ │ │ │ ├── more_test_objects.proto │ │ │ │ │ └── test_objects.proto │ │ │ │ ├── proto │ │ │ │ ├── Makefile │ │ │ │ ├── all_test.go │ │ │ │ ├── clone.go │ │ │ │ ├── clone_test.go │ │ │ │ ├── decode.go │ │ │ │ ├── encode.go │ │ │ │ ├── equal.go │ │ │ │ ├── equal_test.go │ │ │ │ ├── extensions.go │ │ │ │ ├── extensions_test.go │ │ │ │ ├── lib.go │ │ │ │ ├── message_set.go │ │ │ │ ├── message_set_test.go │ │ │ │ ├── pointer_reflect.go │ │ │ │ ├── pointer_unsafe.go │ │ │ │ ├── properties.go │ │ │ │ ├── proto3_proto │ │ │ │ │ └── proto3.proto │ │ │ │ ├── proto3_test.go │ │ │ │ ├── size2_test.go │ │ │ │ ├── size_test.go │ │ │ │ ├── testdata │ │ │ │ │ ├── Makefile │ │ │ │ │ ├── golden_test.go │ │ │ │ │ └── test.proto │ │ │ │ ├── text.go │ │ │ │ ├── text_parser.go │ │ │ │ ├── text_parser_test.go │ │ │ │ └── text_test.go │ │ │ │ └── protoc-gen-go │ │ │ │ ├── Makefile │ │ │ │ ├── descriptor │ │ │ │ └── Makefile │ │ │ │ ├── doc.go │ │ │ │ ├── generator │ │ │ │ ├── Makefile │ │ │ │ ├── generator.go │ │ │ │ └── name_test.go │ │ │ │ ├── internal │ │ │ │ └── grpc │ │ │ │ │ └── grpc.go │ │ │ │ ├── link_grpc.go │ │ │ │ ├── main.go │ │ │ │ ├── plugin │ │ │ │ ├── Makefile │ │ │ │ └── plugin.pb.golden │ │ │ │ └── testdata │ │ │ │ ├── Makefile │ │ │ │ ├── extension_base.proto │ │ │ │ ├── extension_extra.proto │ │ │ │ ├── extension_test.go │ │ │ │ ├── extension_user.proto │ │ │ │ ├── grpc.proto │ │ │ │ ├── imp.pb.go.golden │ │ │ │ ├── imp.proto │ │ │ │ ├── imp2.proto │ │ │ │ ├── imp3.proto │ │ │ │ ├── main_test.go │ │ │ │ ├── multi │ │ │ │ ├── multi1.proto │ │ │ │ ├── multi2.proto │ │ │ │ └── multi3.proto │ │ │ │ ├── my_test │ │ │ │ ├── test.pb.go.golden │ │ │ │ └── test.proto │ │ │ │ └── proto3.proto │ │ └── mreiferson │ │ │ └── go-snappystream │ │ │ ├── .travis.yml │ │ │ ├── LICENSE │ │ │ ├── README.md │ │ │ ├── fixturedata_test.go │ │ │ ├── reader.go │ │ │ ├── reader_test.go │ │ │ ├── readwrite_test.go │ │ │ ├── snappy-go │ │ │ ├── decode.go │ │ │ ├── encode.go │ │ │ ├── snappy.go │ │ │ └── snappy_test.go │ │ │ ├── snappystream.go │ │ │ ├── writer.go │ │ │ └── writer_test.go │ │ ├── job │ │ ├── common.go │ │ ├── job.go │ │ ├── logto_hdfs_collector.go │ │ ├── logto_hdfs_processor.go │ │ ├── test_collector.go │ │ └── test_processor.go │ │ ├── main │ │ ├── dm_src_provider.go │ │ ├── main.go │ │ ├── nsq_src_provider.go │ │ ├── redis_src_provider.go │ │ ├── src_provider.go │ │ └── workshop.go │ │ └── poseidon │ │ └── datastruct │ │ ├── Makefile │ │ └── poseidon_if.pb.go ├── docmeta │ ├── build.gradle │ ├── gradle │ │ └── wrapper │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ └── src │ │ └── main │ │ ├── etc │ │ └── test.json │ │ └── java │ │ └── meta │ │ ├── DocMetaConfigured.java │ │ └── DocMetaMapper.java ├── index │ ├── build.gradle │ ├── build.sh │ ├── gradle │ │ └── wrapper │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ ├── gradlew │ ├── gradlew.bat │ ├── libs │ │ ├── commons-codec-1.3.jar │ │ ├── commons-logging-1.0.4.jar │ │ ├── hadoop-0.20.2.1U11-ant.jar │ │ ├── hadoop-0.20.2.1U11-core.jar │ │ ├── hadoop-0.20.2.1U11-examples.jar │ │ ├── hadoop-0.20.2.1U11-tools.jar │ │ ├── hamcrest-core-1.3.jar │ │ ├── ikanalyzer2012ff.jar │ │ ├── json.jar │ │ ├── junit-4.12.jar │ │ ├── juniversalchardet-1.0.3.jar │ │ └── protobuf-java-3.0.0-beta-2.jar │ ├── settings.gradle │ └── src │ │ ├── main │ │ ├── bin │ │ │ ├── demo.sh │ │ │ ├── mock_start.sh │ │ │ └── start.sh │ │ ├── etc │ │ │ ├── IKAnalyzer.cfg.xml │ │ │ ├── test.json │ │ │ └── test_hdfs.json │ │ └── java │ │ │ ├── InvertedIndex │ │ │ ├── HDFSFileWriter.java │ │ │ ├── IndexGzMetaOutputFormat.java │ │ │ ├── InvertedIndexGenerate.java │ │ │ ├── InvertedIndexGenerateCombiner.java │ │ │ ├── InvertedIndexGenerateMapper.java │ │ │ ├── InvertedIndexGenerateReducer.java │ │ │ ├── LogParser.java │ │ │ ├── NLineReader.java │ │ │ ├── NLineRecordReader.java │ │ │ ├── NTextInputFormat.java │ │ │ ├── ReduceGroupData.java │ │ │ └── plugin │ │ │ │ ├── CommonLogParser.java │ │ │ │ ├── Function │ │ │ │ ├── Base64DecodeFunction.java │ │ │ │ ├── FilterFunction.java │ │ │ │ ├── Function.java │ │ │ │ ├── IpFunction.java │ │ │ │ ├── KeywordFunction.java │ │ │ │ ├── LowerCaseFunction.java │ │ │ │ ├── PathFunction.java │ │ │ │ ├── PathWithUrlencodeFunction.java │ │ │ │ ├── RegexCheck.java │ │ │ │ ├── SplitFunction.java │ │ │ │ ├── TextFunction.java │ │ │ │ ├── TokenFilter.java │ │ │ │ ├── TokenParser.java │ │ │ │ ├── UrlDecodeFunction.java │ │ │ │ ├── UrlEncode.java │ │ │ │ ├── UrlEncodeFunction.java │ │ │ │ └── UrlFunction.java │ │ │ │ ├── LogParserFactory.java │ │ │ │ └── Util.java │ │ │ ├── proto │ │ │ └── PoseidonIf.java │ │ │ └── util │ │ │ ├── IniParser.java │ │ │ └── MurmurHash3.java │ │ └── test │ │ ├── java │ │ ├── FilterFunctionTest.java │ │ ├── TokenFilterTest.java │ │ └── UtilTest.java │ │ └── resources │ │ ├── filter1 │ │ └── filter2 └── indexmeta │ ├── build.gradle │ └── src │ └── main │ ├── etc │ ├── test.json │ └── test2.json │ └── java │ └── meta │ ├── IndexMetaConfigured.java │ ├── IndexMetaMapper.java │ ├── MetaConfigured.java │ └── MetaSetter.java ├── common ├── proto │ ├── .gitignore │ ├── build.sh │ └── poseidon_if.proto └── readme.md ├── dist └── .gitignore ├── docs ├── build_inverted_index.md ├── component.md ├── config.md ├── design_detail.pdf ├── doc_builder.md ├── get_started.md ├── hdfs_reader.md ├── id_generator.md ├── meta.md ├── proxy.md ├── run_with_hadoop.md ├── searcher.jpg └── searcher.md ├── service ├── Godeps │ ├── Godeps.json │ └── Readme ├── allinone │ ├── allinone.go │ ├── build.sh │ ├── conf │ │ └── allinone_conf.ini │ └── serverctl ├── build.sh ├── hdfsreader │ ├── README.md │ ├── build.gradle │ ├── build.sh │ ├── gradle │ │ └── wrapper │ │ │ ├── gradle-wrapper.jar │ │ │ └── gradle-wrapper.properties │ ├── gradlew │ ├── gradlew.bat │ ├── settings.gradle │ ├── src │ │ └── main │ │ │ ├── etc │ │ │ ├── core-site.xml │ │ │ └── log4j.properties │ │ │ ├── java │ │ │ └── Reader.java │ │ │ └── script │ │ │ └── hdfsReader.sh │ └── watchdog.sh ├── idgenerator │ ├── build.sh │ ├── conf │ │ └── idgenerator.ini │ ├── main.go │ ├── module │ │ ├── idgenerator.go │ │ └── redis.go │ └── serverctl ├── meta │ ├── build.sh │ ├── conf │ │ └── meta.ini │ ├── main.go │ ├── module │ │ ├── logic.go │ │ └── module.go │ ├── serverctl │ ├── store │ │ ├── memcached │ │ │ └── memcached.go │ │ ├── redis │ │ │ └── redis.go │ │ ├── store.go │ │ └── store_test.go │ └── testclient │ │ └── client.go ├── proxy │ ├── build.sh │ ├── conf │ │ └── app.ini │ ├── main.go │ ├── module │ │ └── proxy.go │ └── serverctl ├── searcher │ ├── build.sh │ ├── conf │ │ └── app.ini │ ├── main.go │ ├── module │ │ └── poseidon.go │ ├── poseidon │ │ ├── RedisPool.go │ │ ├── RedisPool_test.go │ │ ├── Symc.go │ │ ├── Symc_test.go │ │ ├── business_trait.go │ │ ├── doc_gz_client.go │ │ ├── hdfs_reader.go │ │ ├── inverted_index_gz_client.go │ │ └── memcache_pool.go │ ├── proto │ │ └── poseidon_if.pb.go │ └── serverctl └── vendor │ └── github.com │ ├── bradfitz │ └── gomemcache │ │ ├── LICENSE │ │ └── memcache │ │ ├── memcache.go │ │ └── selector.go │ ├── deckarep │ └── golang-set │ │ ├── .gitignore │ │ ├── .travis.yml │ │ ├── LICENSE │ │ ├── README.md │ │ ├── set.go │ │ ├── threadsafe.go │ │ └── threadunsafe.go │ ├── garyburd │ └── redigo │ │ ├── LICENSE │ │ ├── internal │ │ └── commandinfo.go │ │ └── redis │ │ ├── conn.go │ │ ├── doc.go │ │ ├── log.go │ │ ├── pool.go │ │ ├── pubsub.go │ │ ├── redis.go │ │ ├── reply.go │ │ ├── scan.go │ │ └── script.go │ ├── golang │ ├── glog │ │ ├── LICENSE │ │ ├── README │ │ ├── glog.go │ │ └── glog_file.go │ └── protobuf │ │ ├── LICENSE │ │ └── proto │ │ ├── Makefile │ │ ├── clone.go │ │ ├── decode.go │ │ ├── encode.go │ │ ├── equal.go │ │ ├── extensions.go │ │ ├── lib.go │ │ ├── message_set.go │ │ ├── pointer_reflect.go │ │ ├── pointer_unsafe.go │ │ ├── properties.go │ │ ├── text.go │ │ └── text_parser.go │ ├── gorilla │ ├── context │ │ ├── .travis.yml │ │ ├── LICENSE │ │ ├── README.md │ │ ├── context.go │ │ └── doc.go │ └── mux │ │ ├── .travis.yml │ │ ├── LICENSE │ │ ├── README.md │ │ ├── doc.go │ │ ├── mux.go │ │ ├── regexp.go │ │ └── route.go │ ├── spaolacci │ └── murmur3 │ │ ├── .gitignore │ │ ├── LICENSE │ │ ├── README.md │ │ ├── murmur.go │ │ ├── murmur128.go │ │ ├── murmur32.go │ │ └── murmur64.go │ └── zieckey │ ├── dbuf │ ├── .gitignore │ ├── LICENSE │ ├── README.md │ └── double_buffering.go │ ├── goini │ ├── .gitignore │ ├── .travis.yml │ ├── LICENSE │ ├── README.md │ ├── inherited_ini.go │ └── ini.go │ └── simgo │ ├── .gitignore │ ├── LICENSE │ ├── flags.go │ ├── framework.go │ ├── handler.go │ ├── module.go │ ├── module_admin.go │ └── module_monitor.go └── test.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # built application files 2 | *.apk 3 | *.ap_ 4 | 5 | # files for the dex VM 6 | *.dex 7 | 8 | # Java class files 9 | *.class 10 | 11 | # generated files 12 | bin/ 13 | gen/ 14 | out/ 15 | build/ 16 | 17 | # Local configuration file (sdk path, etc) 18 | local.properties 19 | 20 | # Eclipse project files 21 | .classpath 22 | .project 23 | .settings 24 | 25 | # Windows thumbnail db 26 | .DS_Store 27 | 28 | # Windows exe files 29 | *.exe 30 | 31 | # IDEA/Android Studio project files, because 32 | # the project can be imported from settings.gradle 33 | .idea 34 | *.iml 35 | 36 | # Old-style IDEA project files 37 | *.ipr 38 | *.iws 39 | 40 | # Local IDEA workspace 41 | .idea/workspace.xml 42 | 43 | # Gradle cache 44 | .gradle 45 | 46 | # Sandbox stuff 47 | _sandbox 48 | 49 | *.pid 50 | .svn 51 | package/ 52 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the official list of poseidon authors for copyright purposes. 2 | 3 | # Names should be added to this file as one ofdd 4 | # Organization's name 5 | # Individual's name 6 | # Individual's name 7 | 8 | # Please keep the list sorted. 9 | 10 | du xudong 11 | guo jun 12 | li wei 13 | wei zili 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2016, Qihoo 360 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README_en.md: -------------------------------------------------------------------------------- 1 | # Poseidon 2 | 3 | Poseidon is a log searching engine system that can quickly search and retrieve specific strings in hundreds of petabytes and hundreds of trillion lines of log data. 4 | In the past, if we try to find some information in such large set of data, we need to write a Map/Reduce task program and run it in Hadoop platform. 5 | This will cost several hours, maybe more, that greatly restricting the working efficiency. 6 | 7 | The Poseidon system can solve this problem, it builds the inverted index directly on HDFS and does not change the storing mode of the original log data which is still stored on HDFS. 8 | That means the Poseidon system does not need to store the original log data. 9 | We can use Poseidon system to do searching and use Hadoop system to run Map/Reduce task in the same time and on the same data set. 10 | 11 | That is very different with ElasticSearch which stores the index data and original data in its own system and stores the original data on HDFS if we also need to run Map/Reduce task. 12 | And more importantly, ElasticSearch cannot hold such large scale of data set. 13 | 14 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ ! -f build.sh ]; then 4 | echo 'build.sh must be run within its container folder' 1>&2 5 | exit 1 6 | fi 7 | 8 | 9 | mkdir -p dist 10 | 11 | cd builder 12 | sh build.sh 13 | if [ $? -ne 0 ]; then exit -1; fi; 14 | cd .. 15 | 16 | cd service 17 | sh build.sh 18 | if [ $? -ne 0 ]; then exit -1; fi; 19 | cd .. 20 | 21 | -------------------------------------------------------------------------------- /builder/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | dir=$(dirname $0) 4 | 5 | if [ ! -f build.sh ]; then 6 | echo 'build.sh must be run within its container folder' 1>&2 7 | exit 1 8 | fi 9 | 10 | (cd $dir/index; sh ./build.sh) 11 | (cd $dir/docformat; sh ./build.sh) 12 | -------------------------------------------------------------------------------- /builder/docformat/Makefile: -------------------------------------------------------------------------------- 1 | CURDIR:=$(shell pwd) 2 | APP:=docformat 3 | all: 4 | gofmt -w src/main src/job src/common 5 | export GOPATH=$(CURDIR) && go build -o bin/$(APP) main 6 | 7 | clean: 8 | rm -f bin/$(APP) 9 | 10 | test: 11 | export GOPATH=$(CURDIR) && cd src/job && go test -v 12 | 13 | benchmark: 14 | export GOPATH=$(CURDIR) && cd src/job && go test -test.bench=".*" 15 | 16 | .PHONY:all clean test benchmark 17 | -------------------------------------------------------------------------------- /builder/docformat/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ $# -ne 1 ];then 3 | BUSI="docformat" 4 | else 5 | BUSI=$1 6 | fi 7 | 8 | 9 | CURDIR=$(cd $(dirname $0); pwd -P) 10 | cd ${CURDIR} 11 | make -j || exit 1 12 | 13 | mkdir -p package 14 | 15 | VERSION="0.1" 16 | 17 | TARGET_DIR=.pack_tmp/${BUSI}-${VERSION} 18 | rm -rf ${TARGET_DIR} 19 | mkdir -p ${TARGET_DIR}/{bin,etc,data} 20 | 21 | # bin 22 | cp ${CURDIR}/bin/docformat ${TARGET_DIR}/bin 23 | cp -r ${CURDIR}/script/common/* ${TARGET_DIR}/bin/ 24 | # cp -r ${CURDIR}/script/${BUSI}/* ${TARGET_DIR}/bin/ 25 | sed "s/BUSI_DEFAULT/${BUSI}/g" -i ${TARGET_DIR}/bin/install.sh 26 | 27 | # etc 28 | # cp -r ${CURDIR}/etc/common/* ${TARGET_DIR}/etc/ 29 | \cp -rf ${CURDIR}/etc/${BUSI}/* ${TARGET_DIR}/etc/ 30 | (cd ${TARGET_DIR}/etc/ && mv docformat.json ${BUSI}.json) 31 | 32 | # make pkg 33 | (cd .pack_tmp && tar zcf ../package/${BUSI}-${VERSION}.tar.gz ${BUSI}-${VERSION} --exclude=.svn --exclude=.git) 34 | 35 | rm -rf ${TARGET_DIR} 36 | echo "success ${BUSI}-${VERSION}.tar.gz" 37 | cp package/${BUSI}-${VERSION}.tar.gz ../../dist 38 | -------------------------------------------------------------------------------- /builder/docformat/etc/docformat/docformat.json: -------------------------------------------------------------------------------- 1 | { 2 | "src_provider": { 3 | "src_type":"dm", 4 | "dm": { 5 | "monitor_interval_sec":15, 6 | "pop_last_file":false, 7 | "time_out_ms":1000, 8 | "data_dir":"./logs", 9 | "monitor_paths": [ 10 | "/home/poseidon/data" 11 | ] 12 | } 13 | }, 14 | "main": 15 | { 16 | "processor_num":1, 17 | "collector_num":5, 18 | "msg_chan_size":2, 19 | "item_chan_size":2, 20 | 21 | "max_procs":30, 22 | 23 | "tick_sec":20, 24 | "mem_limit":5000 25 | }, 26 | 27 | "processor": { 28 | "LogtoHdfsProcessor":true 29 | }, 30 | 31 | "collector": { 32 | "logto_hdfs":"LogtoHdfsCollector" 33 | }, 34 | "Hdfs": { 35 | "fixed_idc":"idc", 36 | "use_fixed_idc":true, 37 | 38 | "write_dirs": [ 39 | "./data/dest_1", 40 | "./data/dest_2" 41 | ], 42 | "hadoop_remote_dir":"/home/poseidon/src/test", 43 | "hadoop_remote_time_dirs": [ 44 | "Y-m-d" 45 | ], 46 | "hadoop_remote_file_prefix":"", 47 | "hadoop_remote_file_suffix":".gz", 48 | "flush_minute":20, 49 | "max_merge_file_size":50000000, 50 | "retry_times":3, 51 | "hadoop_cmd":"./bin/local-hadoop.sh", 52 | "gather_minute":15, 53 | 54 | "tick_interval":1, 55 | 56 | "poseidon_mode":true, 57 | "doc_lines":128, 58 | "readbuf_size_byte":40960000, 59 | "docid_domain":"127.0.0.1:39360", 60 | "docid_business":"test" 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /builder/docformat/script/common/clean.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | current=$(cd $(dirname $0) && pwd -P) 3 | current=$(dirname $current) 4 | 5 | cd $current && sh bin/install.sh stop 6 | 7 | rm -fv /home/poseidon/data/* 8 | rm -fv $current/logs/last* 9 | -------------------------------------------------------------------------------- /builder/docformat/script/common/demo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 用于将文件按照指定的文件名放入文件夹 4 | # 文件名约定 5 | # 必须包含日期例如：2016-10-10-08[-20] 6 | # 必须以-分割，至少精确到小时，也可以精确到分钟 7 | # 新文件必须按照字典序最大 8 | 9 | if [ $# -lt 1 ]; 10 | then 11 | echo $0 logfile 12 | echo usage: $0 ./weibo.txt 13 | exit 14 | fi 15 | 16 | pwd=$(pwd) 17 | 18 | day=`date -d "24 hours ago" +"%Y-%m-%d"` 19 | hour=`date -d "24 hours ago" +"%H"` 20 | timestamp=`date -d "24 hours ago" +"%Y%m%d%H%M%S"` 21 | 22 | # 这里的timestamp仅仅是为了使最新的文件按照字典序最大 23 | 24 | cp -rf $1 /home/poseidon/data/log${timestamp}_$day-$hour.txt 25 | 26 | echo file has copyed to /home/poseidon/data 27 | echo wait for file in /home/poseidon/src/test/$day 28 | echo detail log see logs/docformat.log 29 | 30 | # echo and then you should go to run: 31 | # echo /bin/bash $pwd/bin/mock_start.sh $day 32 | -------------------------------------------------------------------------------- /builder/docformat/script/common/dummy-hadoop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | exec 1>>logs/dummy-hadoop.log 2>&1 3 | echo "$(date "+%Y/%m/%d %H:%M:%S") this a fake hadoop client for demo" 4 | echo "params is: $*" 5 | echo "now sleep for a while" 6 | sleep 5 7 | echo "now exit" 8 | exit 0 9 | -------------------------------------------------------------------------------- /builder/docformat/script/common/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ -z "$1" || ("$1" != "start" && "$1" != "stop") ]]; then 4 | echo "Usage: $0 " 5 | echo " $0 start" 6 | exit 7 | fi 8 | 9 | BUSI="BUSI_DEFAULT" 10 | 11 | start_cmd="bin/docformat -c etc/${BUSI}.json" 12 | pid=$(ps axu | grep "${start_cmd}$" | awk '{print $2}') 13 | if [[ -z "$pid" && "$1" == "stop" ]]; then 14 | echo "service has NOT been started" 15 | exit 16 | elif [[ -n "$pid" && "$1" == "start" ]]; then 17 | echo "service has been started" 18 | ps axu | grep "${start_cmd}$" 19 | exit 20 | fi 21 | 22 | current=$(cd $(dirname $0) && pwd -P) 23 | current=$(dirname $current) 24 | echo $current 25 | 26 | if [[ "$1" == "stop" || (-n "$pid" && "$1" == "restart") ]]; then 27 | kill $pid 28 | echo "service has start to stop, see ${current}/logs/docformat.log" 29 | sleep 0.1 30 | ps axu | grep "${start_cmd}$" 31 | if [[ "$1" == "stop" ]]; then 32 | exit 33 | fi 34 | fi 35 | 36 | rm -rf $current/logs 37 | mkdir $current/logs 38 | 39 | log_postfix=$(date "+%Y%m%d-%H%M%S") 40 | if [[ "$1" == "start" ]]; then 41 | cd $current 42 | nohup $start_cmd &> logs/docformat.out.${log_postfix} & 43 | echo "service started" 44 | sleep 0.1 45 | ps axu | grep "${start_cmd}$" 46 | fi 47 | -------------------------------------------------------------------------------- /builder/docformat/script/common/local-hadoop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # exec 1>>logs/local-hadoop.log 2>&1 3 | echo "$(date "+%Y/%m/%d %H:%M:%S") this a local hadoop client for demo" 4 | echo "params is: $*" 5 | # local-hadoop.sh fs -mkdir -p /home/poseidon/src/test 6 | if [ $1 = "fs" ]; then 7 | echo "this is fs" 8 | if [ $2 = "-mkdir" ]; then 9 | echo "this is mkdir" 10 | if [ $3 == "-p" ]; then 11 | echo "mkdir -p $4" 12 | mkdir -p $4 13 | exit $? 14 | else 15 | echo "mkdir $3" 16 | mkdir $3 17 | exit $? 18 | fi 19 | elif [ $2 = "-put" -o $2 = "-copyFromLocal" ]; then 20 | echo "this is put" 21 | if [ $3 == "-f" ]; then 22 | echo "cp -f $4 $5" 23 | cp -f $4 $5 24 | exit $? 25 | else 26 | echo "cp $3 $4" 27 | cp $3 $4 28 | exit $? 29 | fi 30 | fi 31 | fi 32 | echo "not processed" 33 | exit 1 34 | -------------------------------------------------------------------------------- /builder/docformat/script/common/real-hadoop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # exec 1>>logs/real-hadoop.log 2>&1 3 | echo "$(date "+%Y/%m/%d %H:%M:%S") this a proxy of hadoop client for demo" 4 | echo "params is: $*" 5 | /usr/local/hadoop/bin/hadoop $* 6 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/bitly/go-nsq/.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.4.2 4 | - 1.5.1 5 | env: 6 | - NSQ_DOWNLOAD=nsq-0.3.0.linux-amd64.go1.3.3 GOARCH=amd64 7 | - NSQ_DOWNLOAD=nsq-0.3.0.linux-amd64.go1.3.3 GOARCH=386 8 | - NSQ_DOWNLOAD=nsq-0.3.1.linux-amd64.go1.4.1 GOARCH=amd64 9 | - NSQ_DOWNLOAD=nsq-0.3.1.linux-amd64.go1.4.1 GOARCH=386 10 | - NSQ_DOWNLOAD=nsq-0.3.2.linux-amd64.go1.4.1 GOARCH=amd64 11 | - NSQ_DOWNLOAD=nsq-0.3.2.linux-amd64.go1.4.1 GOARCH=386 12 | - NSQ_DOWNLOAD=nsq-0.3.5.linux-amd64.go1.4.2 GOARCH=amd64 13 | - NSQ_DOWNLOAD=nsq-0.3.5.linux-amd64.go1.4.2 GOARCH=386 14 | install: 15 | - go get github.com/mreiferson/go-snappystream 16 | script: 17 | - wget http://bitly-downloads.s3.amazonaws.com/nsq/$NSQ_DOWNLOAD.tar.gz 18 | - tar zxvf $NSQ_DOWNLOAD.tar.gz 19 | - export PATH=$NSQ_DOWNLOAD/bin:$PATH 20 | - pushd $TRAVIS_BUILD_DIR 21 | - ./test.sh 22 | - popd 23 | notifications: 24 | email: false 25 | sudo: false 26 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/bitly/go-nsq/LICENSE: -------------------------------------------------------------------------------- 1 | Permission is hereby granted, free of charge, to any person obtaining a copy 2 | of this software and associated documentation files (the "Software"), to deal 3 | in the Software without restriction, including without limitation the rights 4 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 5 | copies of the Software, and to permit persons to whom the Software is 6 | furnished to do so, subject to the following conditions: 7 | 8 | The above copyright notice and this permission notice shall be included in 9 | all copies or substantial portions of the Software. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 12 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 13 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 14 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 15 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 16 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 17 | THE SOFTWARE. 18 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/bitly/go-nsq/README.md: -------------------------------------------------------------------------------- 1 | ## go-nsq 2 | 3 | [![Build Status](https://secure.travis-ci.org/nsqio/go-nsq.svg?branch=master)][travis] [![GoDoc](https://godoc.org/github.com/nsqio/go-nsq?status.svg)](https://godoc.org/github.com/nsqio/go-nsq) [![GitHub release](https://img.shields.io/github/release/nsqio/go-nsq.svg)](https://github.com/nsqio/go-nsq/releases/latest) 4 | 5 | The official Go package for [NSQ][nsq]. 6 | 7 | ### Docs 8 | 9 | See [godoc][nsq_gopkgdoc] and the [main repo apps][apps] directory for examples of clients built 10 | using this package. 11 | 12 | ### Tests 13 | 14 | Tests are run via `./test.sh` (which requires `nsqd` and `nsqlookupd` to be installed). 15 | 16 | [nsq]: https://github.com/nsqio/nsq 17 | [nsq_gopkgdoc]: http://godoc.org/github.com/nsqio/go-nsq 18 | [apps]: https://github.com/nsqio/nsq/tree/master/apps 19 | [travis]: http://travis-ci.org/nsqio/go-nsq 20 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/bitly/go-nsq/command_test.go: -------------------------------------------------------------------------------- 1 | package nsq 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | ) 7 | 8 | func BenchmarkCommand(b *testing.B) { 9 | b.StopTimer() 10 | data := make([]byte, 2048) 11 | cmd := Publish("test", data) 12 | var buf bytes.Buffer 13 | b.StartTimer() 14 | 15 | for i := 0; i < b.N; i++ { 16 | cmd.WriteTo(&buf) 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/bitly/go-nsq/config_flag.go: -------------------------------------------------------------------------------- 1 | package nsq 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | // ConfigFlag wraps a Config and implements the flag.Value interface 8 | type ConfigFlag struct { 9 | *Config 10 | } 11 | 12 | // Set takes a comma separated value and follows the rules in Config.Set 13 | // using the first field as the option key, and the second (if present) as the value 14 | func (c *ConfigFlag) Set(opt string) (err error) { 15 | parts := strings.SplitN(opt, ",", 2) 16 | key := parts[0] 17 | 18 | switch len(parts) { 19 | case 1: 20 | // default options specified without a value to boolean true 21 | err = c.Config.Set(key, true) 22 | case 2: 23 | err = c.Config.Set(key, parts[1]) 24 | } 25 | return 26 | } 27 | 28 | // String implements the flag.Value interface 29 | func (c *ConfigFlag) String() string { 30 | return "" 31 | } 32 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/bitly/go-nsq/config_flag_test.go: -------------------------------------------------------------------------------- 1 | package nsq_test 2 | 3 | import ( 4 | "flag" 5 | 6 | "github.com/nsqio/go-nsq" 7 | ) 8 | 9 | func ExampleConfigFlag() { 10 | cfg := nsq.NewConfig() 11 | flagSet := flag.NewFlagSet("", flag.ExitOnError) 12 | 13 | flagSet.Var(&nsq.ConfigFlag{cfg}, "consumer-opt", "option to pass through to nsq.Consumer (may be given multiple times)") 14 | flagSet.PrintDefaults() 15 | 16 | err := flagSet.Parse([]string{ 17 | "--consumer-opt=heartbeat_interval,1s", 18 | "--consumer-opt=max_attempts,10", 19 | }) 20 | if err != nil { 21 | panic(err.Error()) 22 | } 23 | println("HeartbeatInterval", cfg.HeartbeatInterval) 24 | println("MaxAttempts", cfg.MaxAttempts) 25 | } 26 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/bitly/go-nsq/errors.go: -------------------------------------------------------------------------------- 1 | package nsq 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | ) 7 | 8 | // ErrNotConnected is returned when a publish command is made 9 | // against a Producer that is not connected 10 | var ErrNotConnected = errors.New("not connected") 11 | 12 | // ErrStopped is returned when a publish command is 13 | // made against a Producer that has been stopped 14 | var ErrStopped = errors.New("stopped") 15 | 16 | // ErrClosing is returned when a connection is closing 17 | var ErrClosing = errors.New("closing") 18 | 19 | // ErrAlreadyConnected is returned from ConnectToNSQD when already connected 20 | var ErrAlreadyConnected = errors.New("already connected") 21 | 22 | // ErrOverMaxInFlight is returned from Consumer if over max-in-flight 23 | var ErrOverMaxInFlight = errors.New("over configure max-inflight") 24 | 25 | // ErrIdentify is returned from Conn as part of the IDENTIFY handshake 26 | type ErrIdentify struct { 27 | Reason string 28 | } 29 | 30 | // Error returns a stringified error 31 | func (e ErrIdentify) Error() string { 32 | return fmt.Sprintf("failed to IDENTIFY - %s", e.Reason) 33 | } 34 | 35 | // ErrProtocol is returned from Producer when encountering 36 | // an NSQ protocol level error 37 | type ErrProtocol struct { 38 | Reason string 39 | } 40 | 41 | // Error returns a stringified error 42 | func (e ErrProtocol) Error() string { 43 | return e.Reason 44 | } 45 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/bitly/go-nsq/states.go: -------------------------------------------------------------------------------- 1 | package nsq 2 | 3 | // states 4 | const ( 5 | StateInit = iota 6 | StateDisconnected 7 | StateConnected 8 | StateSubscribed 9 | // StateClosing means CLOSE has started... 10 | // (responses are ok, but no new messages will be sent) 11 | StateClosing 12 | ) 13 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/bitly/go-nsq/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # a helper script to run tests 5 | 6 | if ! which nsqd >/dev/null; then 7 | echo "missing nsqd binary" && exit 1 8 | fi 9 | 10 | if ! which nsqlookupd >/dev/null; then 11 | echo "missing nsqlookupd binary" && exit 1 12 | fi 13 | 14 | # run nsqlookupd 15 | LOOKUP_LOGFILE=$(mktemp -t nsqlookupd.XXXXXXX) 16 | echo "starting nsqlookupd" 17 | echo " logging to $LOOKUP_LOGFILE" 18 | nsqlookupd >$LOOKUP_LOGFILE 2>&1 & 19 | LOOKUPD_PID=$! 20 | 21 | # run nsqd configured to use our lookupd above 22 | rm -f *.dat 23 | NSQD_LOGFILE=$(mktemp -t nsqlookupd.XXXXXXX) 24 | echo "starting nsqd --data-path=/tmp --lookupd-tcp-address=127.0.0.1:4160 --tls-cert=./test/server.pem --tls-key=./test/server.key --tls-root-ca-file=./test/ca.pem" 25 | echo " logging to $NSQD_LOGFILE" 26 | nsqd --data-path=/tmp --lookupd-tcp-address=127.0.0.1:4160 --tls-cert=./test/server.pem --tls-key=./test/server.key --tls-root-ca-file=./test/ca.pem >$NSQD_LOGFILE 2>&1 & 27 | NSQD_PID=$! 28 | 29 | sleep 0.3 30 | 31 | cleanup() { 32 | echo "killing nsqd PID $NSQD_PID" 33 | kill -s TERM $NSQD_PID || cat $NSQD_LOGFILE 34 | echo "killing nsqlookupd PID $LOOKUPD_PID" 35 | kill -s TERM $LOOKUPD_PID || cat $LOOKUP_LOGFILE 36 | } 37 | trap cleanup INT TERM EXIT 38 | 39 | go test -v -timeout 60s 40 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/bitly/go-nsq/test/ca.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID9zCCAt+gAwIBAgIJAPYpAVNDj2lgMA0GCSqGSIb3DQEBBQUAMIGRMQswCQYD 3 | VQQGEwJERTEMMAoGA1UECAwDTlJXMQ4wDAYDVQQHDAVFYXJ0aDEXMBUGA1UECgwO 4 | UmFuZG9tIENvbXBhbnkxCzAJBgNVBAsMAklUMRcwFQYDVQQDDA53d3cucmFuZG9t 5 | LmNvbTElMCMGCSqGSIb3DQEJARYWS3J5cHRvS2luZ3NAcmFuZG9tLmNvbTAeFw0x 6 | NDA0MDIyMTE0NTJaFw0xNTA0MDIyMTE0NTJaMIGRMQswCQYDVQQGEwJERTEMMAoG 7 | A1UECAwDTlJXMQ4wDAYDVQQHDAVFYXJ0aDEXMBUGA1UECgwOUmFuZG9tIENvbXBh 8 | bnkxCzAJBgNVBAsMAklUMRcwFQYDVQQDDA53d3cucmFuZG9tLmNvbTElMCMGCSqG 9 | SIb3DQEJARYWS3J5cHRvS2luZ3NAcmFuZG9tLmNvbTCCASIwDQYJKoZIhvcNAQEB 10 | BQADggEPADCCAQoCggEBAL/sJU6ODQCsdWAmq3Qyp6vCqVFkSIHwR3oH8vPuwwob 11 | IOrx/pXz2LIRekQ4egT8LCH3QDxhEvFhDNXYM4h/mkQ+GpgzynoIqYrw+yF93pik 12 | T9Tpel2IuntThlZrO/4APRbVw4Ihf3zp47AY71J+8usJjmfWsId4dhqa1lTYecXK 13 | Zwxii8RTH/7LsuwIDOW1QJLGGKNzvVYA42Gh8Cw3uHlmqZ2tA/sp5qg1Z3QU5g7y 14 | EzzRybotHaRb5XMUWHAlGbIl/TW4KlFqFZ0kCXJXL1uO3uq2nIS3bG7ryjbobRVn 15 | dZ6sV34eenIeZWu6zlDxQP/EqxAezz5Ndyt9uYWb/JECAwEAAaNQME4wHQYDVR0O 16 | BBYEFI9l/QHE30clqx+1oCR6IhUYEdqLMB8GA1UdIwQYMBaAFI9l/QHE30clqx+1 17 | oCR6IhUYEdqLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAES6GKxL 18 | oeCLZa83TjJXLagcc9mmdQZgfF3/o61+ye7D9BLqBwN5lx4+kIE1LAUI/Id0mCdW 19 | 9uXmPhpCJ926krahNc4Ol+wQTmZ3j7Mn3DCkFufjr64cGPU/UzH4yjMg9wEf84qz 20 | 5oH+dBifwJM8yoRCxbnMqGBu3xY8WCjPlw8E8lizXFk8wUbLZ/EC5Rjm+KmdT5ud 21 | KTEgM+K6RMNo9vLn5ZasrYyhVcHdEKIzo6qLm1ZVIgpi/1WX0m8hACMfEcqee6ot 22 | 76LEyM3kwfqRkWGZWHEF9D4emp3quU+0AmjM57LHrYjidpDJkVTUHDoMBFHl9Uiq 23 | 0O9+azN48F/bVgU= 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/bitly/go-nsq/test/server.key: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDFHWaPfRA5nU/F 3 | E8AVoFj2TAgMRISLduWlbAgDnMtFLSGVwgjxU13Txsv0LgwJgo4A5xpd2WNV0dIQ 4 | brerxvPVJruKO8KxKFS2U58BCFIG0xGrlQSg5wDGyqxEQY80XlrBtxs81v79GYHy 5 | fBhizg7onlmbNZzxPy7idU0a7TpgzakeDrfJHQ7rI3llvR0U0TdOLno82CtPvosY 6 | 6TYZAIFYgH05yN7DWKuDUI8Fa2RFVkbHPUlJVKROw/0n1yWy7XcwTmQQyaodFYgg 7 | KMCdyR0ElPxLv8dKYFjLvef2DTmuYwbalt5hiQqOpY1wm616Xf4ywz2uEU+ooLW4 8 | /Q6DcRUBAgMBAAECggEBAKDUgVc4YuGvkmOp3sOGhq/Hj5V63m7wvyV/BPb1mwFB 9 | drK7lBJbxCXEgaslWxrzVCw2ZFQIyL1AKtbPj7tv5ImZRdHfOtbTonL1vbnY8Ryy 10 | YSuPtiwW75JD0dULbO38wq5mWaoFfh5DDr0sNbWAjbeNZG14jCpnNDkAHr6Gq2hJ 11 | VzwEwy+W7LXn8s8lYJHi3MsxCJqAFN1u2FOkjBtrcVW06VgV87IX59SOFns80x4E 12 | Nn0ZKH7RU2DuJ7Fm4HtaNH+yaDYxUeo2A/2/UoavyYYCgC5gThsNjlp9/R4gtm0h 13 | VO+8cN5a3s7zL+aITIusY7H/rwWc6XpRmxQn+jwqF8ECgYEA5PpAz4fhXA9abFZx 14 | 0XqCgcwTFY5gTX/JDd1qv0b/PueAR7dY5d37bRbTXpzrHiNFVrq3x432V3+KY0b5 15 | 55PEB1YxwBUga5DvTSa5fLfUibvLpdZjganzdTOsG53wMvNwUT8iUzUQDLkyRfIi 16 | mV0r4Sa34RrBZdWJ2Aou9by2SlkCgYEA3GCHTP7nAcuHXRTsXH3eK/HsfwxdwjhA 17 | G5SG7L7KSoMpzCbe90DuYEr6J/O1nnP0QiSQ2uEeTOARzMfio4E16exWlDDtvPBQ 18 | HqSuQKg4M7fMTN1tj95xmk1yGZMyPxgEfCScBeCbYQzOyZ0j93iFjqMnb2mlriq5 19 | MoSPat3BeukCgYEAjSGaFNABnUZxS1k0qhLCodHw6VZqERp0B7Gze9X8uP7jWFCv 20 | 4G6j66cn/KbnXBoNQNmxMLRVY7TezTYQDiZLquH7pBLheqtIc9ssdKyxuXsgmES9 21 | 7EueHV0N9a+xPxZA4jLxqyuHivATBn2pybFdvFaq+3oMPgISBjCwpRH9oXECgYAN 22 | +n16j8ydW4iZieM4Nq+p/+1tXZ5w3FqMpU4tpCh2s30qOuj3rAGyz+6wLBBAHcDH 23 | lUQu7gqa+7eFUsR4dJCz5s7TFYtu6ZtbZjy7UzBFb4og8gaqEoUIMZNkNecBA4f9 24 | S+EtqkKQ1Fwlg7ctUlK+anDs6zmcI4+dubTTJX/JSQKBgQCsu/gCgoOi2GFgebIh 25 | URvEMrhaiHxcw5u30nMNjWUGpDQK3lVTK51+7wj4xmVfiomvUW6M/HaR2+5xF1U1 26 | QV08cKeWCGfGUFetTxjdhsVhMIk84ygF2l9K6jiHqvtd5rIoQ9Lf8XXbYaQVicRg 27 | qmB2iOzmbQQM/GOSofAeUfE7/A== 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/bitly/go-nsq/test/server.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIID7zCCAtegAwIBAgIJAMsErP97ZQmgMA0GCSqGSIb3DQEBBQUAMIGNMQswCQYD 3 | VQQGEwJERTEMMAoGA1UECAwDTlJXMQ4wDAYDVQQHDAVFYXJ0aDEXMBUGA1UECgwO 4 | UmFuZG9tIENvbXBhbnkxCzAJBgNVBAsMAklUMRcwFQYDVQQDDA53d3cucmFuZG9t 5 | LmNvbTEhMB8GCSqGSIb3DQEJARYSZm9vYmFyQGV4YW1wbGUuY29tMB4XDTE0MDQw 6 | MjIxMTQ1MloXDTI0MDMzMDIxMTQ1MlowgY0xCzAJBgNVBAYTAkRFMQwwCgYDVQQI 7 | DANOUlcxDjAMBgNVBAcMBUVhcnRoMRcwFQYDVQQKDA5SYW5kb20gQ29tcGFueTEL 8 | MAkGA1UECwwCSVQxFzAVBgNVBAMMDnd3dy5yYW5kb20uY29tMSEwHwYJKoZIhvcN 9 | AQkBFhJmb29iYXJAZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw 10 | ggEKAoIBAQDFHWaPfRA5nU/FE8AVoFj2TAgMRISLduWlbAgDnMtFLSGVwgjxU13T 11 | xsv0LgwJgo4A5xpd2WNV0dIQbrerxvPVJruKO8KxKFS2U58BCFIG0xGrlQSg5wDG 12 | yqxEQY80XlrBtxs81v79GYHyfBhizg7onlmbNZzxPy7idU0a7TpgzakeDrfJHQ7r 13 | I3llvR0U0TdOLno82CtPvosY6TYZAIFYgH05yN7DWKuDUI8Fa2RFVkbHPUlJVKRO 14 | w/0n1yWy7XcwTmQQyaodFYggKMCdyR0ElPxLv8dKYFjLvef2DTmuYwbalt5hiQqO 15 | pY1wm616Xf4ywz2uEU+ooLW4/Q6DcRUBAgMBAAGjUDBOMB0GA1UdDgQWBBTxyT32 16 | Exu5TuortZY8zkVotLDNDTAfBgNVHSMEGDAWgBTxyT32Exu5TuortZY8zkVotLDN 17 | DTAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4IBAQAu+0B+caaV4HzIHyfX 18 | Zc6BUPcRoTEZIWX/7JLeeOVUztOjl9ExjYTzjo0QEt+PVcOzfQL/hxE2SPG6fRF7 19 | YRZU1h9t5Ti9rTg9myAbGGMo6MdWZULFcxIWjxhv6qnFPk/fF47PvGwjygFNnzv8 20 | FYmrAI99kK0CYolvXZ5ue250dpE/TCIAyk09a3WeBbHU/hMR/mBUNsitphelDbNK 21 | oohrY9D7QR5Mf/NZgx3a0eDH6zoMYDRPARY3M02EuHHiRKmlyfnPv4ns4/0wCarj 22 | pKpds+G80+k2fyiMgQ5bPTw8sfNgq1z0IvIuWB36XSNenTgnnjArbWii+x95jjNw 23 | XcQg 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/bitly/go-nsq/version.go: -------------------------------------------------------------------------------- 1 | // Package nsq is the official Go package for NSQ (http://nsq.io/) 2 | // 3 | // It provides high-level Consumer and Producer types as well as low-level 4 | // functions to communicate over the NSQ protocol 5 | package nsq 6 | 7 | // VERSION 8 | const VERSION = "1.0.5" 9 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/bitly/go-simplejson/.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.0.3 4 | - 1.1.2 5 | - 1.2 6 | - tip 7 | install: 8 | - go get github.com/bmizerany/assert 9 | notifications: 10 | email: false 11 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/bitly/go-simplejson/LICENSE: -------------------------------------------------------------------------------- 1 | Permission is hereby granted, free of charge, to any person obtaining a copy 2 | of this software and associated documentation files (the "Software"), to deal 3 | in the Software without restriction, including without limitation the rights 4 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 5 | copies of the Software, and to permit persons to whom the Software is 6 | furnished to do so, subject to the following conditions: 7 | 8 | The above copyright notice and this permission notice shall be included in 9 | all copies or substantial portions of the Software. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 12 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 13 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 14 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 15 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 16 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 17 | THE SOFTWARE. 18 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/bitly/go-simplejson/README.md: -------------------------------------------------------------------------------- 1 | ### go-simplejson 2 | 3 | a Go package to interact with arbitrary JSON 4 | 5 | [![Build Status](https://secure.travis-ci.org/bitly/go-simplejson.png)](http://travis-ci.org/bitly/go-simplejson) 6 | 7 | ### Importing 8 | 9 | import github.com/bitly/go-simplejson 10 | 11 | ### Documentation 12 | 13 | Visit the docs on [gopkgdoc](http://godoc.org/github.com/bitly/go-simplejson) 14 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/donnie4w/go-logger/example/logger_test.go: -------------------------------------------------------------------------------- 1 | package example 2 | 3 | import ( 4 | "github.com/donnie4w/go-logger/logger" 5 | "runtime" 6 | "strconv" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func log(i int) { 12 | logger.Debug("Debug>>>>>>>>>>>>>>>>>>>>>>" + strconv.Itoa(i)) 13 | logger.Info("Info>>>>>>>>>>>>>>>>>>>>>>>>>" + strconv.Itoa(i)) 14 | logger.Warn("Warn>>>>>>>>>>>>>>>>>>>>>>>>>" + strconv.Itoa(i)) 15 | logger.Error("Error>>>>>>>>>>>>>>>>>>>>>>>>>" + strconv.Itoa(i)) 16 | logger.Fatal("Fatal>>>>>>>>>>>>>>>>>>>>>>>>>" + strconv.Itoa(i)) 17 | } 18 | 19 | func Test(t *testing.T) { 20 | runtime.GOMAXPROCS(runtime.NumCPU()) 21 | 22 | //指定是否控制台打印，默认为true 23 | logger.SetConsole(true) 24 | //指定日志文件备份方式为文件大小的方式 25 | //第一个参数为日志文件存放目录 26 | //第二个参数为日志文件命名 27 | //第三个参数为备份文件最大数量 28 | //第四个参数为备份文件大小 29 | //第五个参数为文件大小的单位 30 | //logger.SetRollingFile("d:/logtest", "test.log", 10, 5, logger.KB) 31 | 32 | //指定日志文件备份方式为日期的方式 33 | //第一个参数为日志文件存放目录 34 | //第二个参数为日志文件命名 35 | logger.SetRollingDaily("d:/logtest", "test.log") 36 | 37 | //指定日志级别 ALL，DEBUG，INFO，WARN，ERROR，FATAL，OFF 级别由低到高 38 | //一般习惯是测试阶段为debug，生成环境为info以上 39 | logger.SetLevel(logger.ERROR) 40 | 41 | for i := 10000; i > 0; i-- { 42 | go log(i) 43 | time.Sleep(1000 * time.Millisecond) 44 | } 45 | time.Sleep(15 * time.Second) 46 | } 47 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/donnie4w/go-logger/logger/logger_test.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | var LOG_NAME = "ut.log" 9 | 10 | func init() { 11 | os.Remove(LOG_NAME) 12 | SetRollingDaily(".", LOG_NAME) 13 | SetLevel(DEBUG) 14 | SetConsole(true) 15 | } 16 | 17 | func TestManual(t *testing.T) { 18 | Debug("this is debug", "really debug") 19 | Info("this is info", "really info") 20 | Warn("this is warn", "really warn") 21 | Error("this is error", "really error") 22 | Fatal("this is fatal", "really fatal") 23 | 24 | Debugf("this is %s, really %s", "debug", "debug") 25 | Infof("this is %s, really %s", "info", "info") 26 | Warnf("this is %s, really %s", "warn", "warn") 27 | Errorf("this is %s, really %s", "error", "error") 28 | Fatalf("this is %s, really %s", "fatal", "fatal") 29 | } 30 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/donnie4w/go-logger/readme.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/poseidon/9be3a58756c2c604573075b6e7531f13f5ba7c1c/builder/docformat/src/github.com/donnie4w/go-logger/readme.txt -------------------------------------------------------------------------------- /builder/docformat/src/github.com/garyburd/redigo/README.markdown: -------------------------------------------------------------------------------- 1 | Redigo 2 | ====== 3 | 4 | [![Build Status](https://travis-ci.org/garyburd/redigo.svg?branch=master)](https://travis-ci.org/garyburd/redigo) 5 | 6 | Redigo is a [Go](http://golang.org/) client for the [Redis](http://redis.io/) database. 7 | 8 | Features 9 | ------- 10 | 11 | * A [Print-like](http://godoc.org/github.com/garyburd/redigo/redis#hdr-Executing_Commands) API with support for all Redis commands. 12 | * [Pipelining](http://godoc.org/github.com/garyburd/redigo/redis#hdr-Pipelining), including pipelined transactions. 13 | * [Publish/Subscribe](http://godoc.org/github.com/garyburd/redigo/redis#hdr-Publish_and_Subscribe). 14 | * [Connection pooling](http://godoc.org/github.com/garyburd/redigo/redis#Pool). 15 | * [Script helper type](http://godoc.org/github.com/garyburd/redigo/redis#Script) with optimistic use of EVALSHA. 16 | * [Helper functions](http://godoc.org/github.com/garyburd/redigo/redis#hdr-Reply_Helpers) for working with command replies. 17 | 18 | Documentation 19 | ------------- 20 | 21 | - [API Reference](http://godoc.org/github.com/garyburd/redigo/redis) 22 | - [FAQ](https://github.com/garyburd/redigo/wiki/FAQ) 23 | 24 | Installation 25 | ------------ 26 | 27 | Install Redigo using the "go get" command: 28 | 29 | go get github.com/garyburd/redigo/redis 30 | 31 | The Go distribution is Redigo's only dependency. 32 | 33 | Contributing 34 | ------------ 35 | 36 | Contributions are welcome. 37 | 38 | Before writing code, send mail to gary@beagledreams.com to discuss what you 39 | plan to do. This gives me a chance to validate the design, avoid duplication of 40 | effort and ensure that the changes fit the goals of the project. Do not start 41 | the discussion with a pull request. 42 | 43 | License 44 | ------- 45 | 46 | Redigo is available under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html). 47 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/garyburd/redigo/internal/commandinfo.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Gary Burd 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package internal // import "github.com/garyburd/redigo/internal" 16 | 17 | import ( 18 | "strings" 19 | ) 20 | 21 | const ( 22 | WatchState = 1 << iota 23 | MultiState 24 | SubscribeState 25 | MonitorState 26 | ) 27 | 28 | type CommandInfo struct { 29 | Set, Clear int 30 | } 31 | 32 | var commandInfos = map[string]CommandInfo{ 33 | "WATCH": {Set: WatchState}, 34 | "UNWATCH": {Clear: WatchState}, 35 | "MULTI": {Set: MultiState}, 36 | "EXEC": {Clear: WatchState | MultiState}, 37 | "DISCARD": {Clear: WatchState | MultiState}, 38 | "PSUBSCRIBE": {Set: SubscribeState}, 39 | "SUBSCRIBE": {Set: SubscribeState}, 40 | "MONITOR": {Set: MonitorState}, 41 | } 42 | 43 | func init() { 44 | for n, ci := range commandInfos { 45 | commandInfos[strings.ToLower(n)] = ci 46 | } 47 | } 48 | 49 | func LookupCommandInfo(commandName string) CommandInfo { 50 | if ci, ok := commandInfos[commandName]; ok { 51 | return ci 52 | } 53 | return commandInfos[strings.ToUpper(commandName)] 54 | } 55 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/garyburd/redigo/internal/commandinfo_test.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import "testing" 4 | 5 | func TestLookupCommandInfo(t *testing.T) { 6 | for _, n := range []string{"watch", "WATCH", "wAtch"} { 7 | if LookupCommandInfo(n) == (CommandInfo{}) { 8 | t.Errorf("LookupCommandInfo(%q) = CommandInfo{}, expected non-zero value", n) 9 | } 10 | } 11 | } 12 | 13 | func benchmarkLookupCommandInfo(b *testing.B, names ...string) { 14 | for i := 0; i < b.N; i++ { 15 | for _, c := range names { 16 | LookupCommandInfo(c) 17 | } 18 | } 19 | } 20 | 21 | func BenchmarkLookupCommandInfoCorrectCase(b *testing.B) { 22 | benchmarkLookupCommandInfo(b, "watch", "WATCH", "monitor", "MONITOR") 23 | } 24 | 25 | func BenchmarkLookupCommandInfoMixedCase(b *testing.B) { 26 | benchmarkLookupCommandInfo(b, "wAtch", "WeTCH", "monItor", "MONiTOR") 27 | } 28 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/garyburd/redigo/internal/redistest/testdb.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Gary Burd 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | // Package redistest contains utilities for writing Redigo tests. 16 | package redistest 17 | 18 | import ( 19 | "errors" 20 | "time" 21 | 22 | "github.com/garyburd/redigo/redis" 23 | ) 24 | 25 | type testConn struct { 26 | redis.Conn 27 | } 28 | 29 | func (t testConn) Close() error { 30 | _, err := t.Conn.Do("SELECT", "9") 31 | if err != nil { 32 | return nil 33 | } 34 | _, err = t.Conn.Do("FLUSHDB") 35 | if err != nil { 36 | return err 37 | } 38 | return t.Conn.Close() 39 | } 40 | 41 | // Dial dials the local Redis server and selects database 9. To prevent 42 | // stomping on real data, DialTestDB fails if database 9 contains data. The 43 | // returned connection flushes database 9 on close. 44 | func Dial() (redis.Conn, error) { 45 | c, err := redis.DialTimeout("tcp", ":6379", 0, 1*time.Second, 1*time.Second) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | _, err = c.Do("SELECT", "9") 51 | if err != nil { 52 | c.Close() 53 | return nil, err 54 | } 55 | 56 | n, err := redis.Int(c.Do("DBSIZE")) 57 | if err != nil { 58 | c.Close() 59 | return nil, err 60 | } 61 | 62 | if n != 0 { 63 | c.Close() 64 | return nil, errors.New("database #9 is not empty, test can not continue") 65 | } 66 | 67 | return testConn{c}, nil 68 | } 69 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/garyburd/redigo/redis/redis.go: -------------------------------------------------------------------------------- 1 | // Copyright 2012 Gary Burd 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package redis 16 | 17 | // Error represents an error returned in a command reply. 18 | type Error string 19 | 20 | func (err Error) Error() string { return string(err) } 21 | 22 | // Conn represents a connection to a Redis server. 23 | type Conn interface { 24 | // Close closes the connection. 25 | Close() error 26 | 27 | // Err returns a non-nil value if the connection is broken. The returned 28 | // value is either the first non-nil value returned from the underlying 29 | // network connection or a protocol parsing error. Applications should 30 | // close broken connections. 31 | Err() error 32 | 33 | // Do sends a command to the server and returns the received reply. 34 | Do(commandName string, args ...interface{}) (reply interface{}, err error) 35 | 36 | // Send writes the command to the client's output buffer. 37 | Send(commandName string, args ...interface{}) error 38 | 39 | // Flush flushes the output buffer to the Redis server. 40 | Flush() error 41 | 42 | // Receive receives a single reply from the Redis server 43 | Receive() (reply interface{}, err error) 44 | } 45 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/garyburd/redigo/redisx/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2012 Gary Burd 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | // Package redisx contains experimental features for Redigo. Features in this 16 | // package may be modified or deleted at any time. 17 | package redisx // import "github.com/garyburd/redigo/redisx" 18 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.[568ao] 3 | *.pb.go 4 | *.ao 5 | *.so 6 | *.pyc 7 | ._* 8 | .nfs.* 9 | [568a].out 10 | *~ 11 | *.orig 12 | core 13 | _obj 14 | _test 15 | _testmain.go 16 | compiler/protoc-gen-go 17 | compiler/testdata/extension_test 18 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/AUTHORS: -------------------------------------------------------------------------------- 1 | # This source code refers to The Go Authors for copyright purposes. 2 | # The master list of authors is in the main Go distribution, 3 | # visible at http://tip.golang.org/AUTHORS. 4 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | # This source code was written by the Go contributors. 2 | # The master list of contributors is in the main Go distribution, 3 | # visible at http://tip.golang.org/CONTRIBUTORS. 4 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/LICENSE: -------------------------------------------------------------------------------- 1 | Go support for Protocol Buffers - Google's data interchange format 2 | 3 | Copyright 2010 The Go Authors. All rights reserved. 4 | https://github.com/golang/protobuf 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are 8 | met: 9 | 10 | * Redistributions of source code must retain the above copyright 11 | notice, this list of conditions and the following disclaimer. 12 | * Redistributions in binary form must reproduce the above 13 | copyright notice, this list of conditions and the following disclaimer 14 | in the documentation and/or other materials provided with the 15 | distribution. 16 | * Neither the name of Google Inc. nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/Make.protobuf: -------------------------------------------------------------------------------- 1 | # Go support for Protocol Buffers - Google's data interchange format 2 | # 3 | # Copyright 2010 The Go Authors. All rights reserved. 4 | # https://github.com/golang/protobuf 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are 8 | # met: 9 | # 10 | # * Redistributions of source code must retain the above copyright 11 | # notice, this list of conditions and the following disclaimer. 12 | # * Redistributions in binary form must reproduce the above 13 | # copyright notice, this list of conditions and the following disclaimer 14 | # in the documentation and/or other materials provided with the 15 | # distribution. 16 | # * Neither the name of Google Inc. nor the names of its 17 | # contributors may be used to endorse or promote products derived from 18 | # this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | # Includable Makefile to add a rule for generating .pb.go files from .proto files 33 | # (Google protocol buffer descriptions). 34 | # Typical use if myproto.proto is a file in package mypackage in this directory: 35 | # 36 | # include $(GOROOT)/src/pkg/github.com/golang/protobuf/Make.protobuf 37 | 38 | %.pb.go: %.proto 39 | protoc --go_out=. $< 40 | 41 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/Makefile: -------------------------------------------------------------------------------- 1 | # Go support for Protocol Buffers - Google's data interchange format 2 | # 3 | # Copyright 2010 The Go Authors. All rights reserved. 4 | # https://github.com/golang/protobuf 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are 8 | # met: 9 | # 10 | # * Redistributions of source code must retain the above copyright 11 | # notice, this list of conditions and the following disclaimer. 12 | # * Redistributions in binary form must reproduce the above 13 | # copyright notice, this list of conditions and the following disclaimer 14 | # in the documentation and/or other materials provided with the 15 | # distribution. 16 | # * Neither the name of Google Inc. nor the names of its 17 | # contributors may be used to endorse or promote products derived from 18 | # this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | 33 | all: install 34 | 35 | install: 36 | go install ./proto 37 | go install ./jsonpb 38 | go install ./protoc-gen-go 39 | 40 | test: 41 | go test ./proto 42 | go test ./jsonpb 43 | make -C protoc-gen-go/testdata test 44 | 45 | clean: 46 | go clean ./... 47 | 48 | nuke: 49 | go clean -i ./... 50 | 51 | regenerate: 52 | make -C protoc-gen-go/descriptor regenerate 53 | make -C protoc-gen-go/plugin regenerate 54 | make -C protoc-gen-go/testdata regenerate 55 | make -C proto/testdata regenerate 56 | make -C jsonpb/jsonpb_test_proto regenerate 57 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/jsonpb/jsonpb_test_proto/Makefile: -------------------------------------------------------------------------------- 1 | # Go support for Protocol Buffers - Google's data interchange format 2 | # 3 | # Copyright 2015 The Go Authors. All rights reserved. 4 | # https://github.com/golang/protobuf 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are 8 | # met: 9 | # 10 | # * Redistributions of source code must retain the above copyright 11 | # notice, this list of conditions and the following disclaimer. 12 | # * Redistributions in binary form must reproduce the above 13 | # copyright notice, this list of conditions and the following disclaimer 14 | # in the documentation and/or other materials provided with the 15 | # distribution. 16 | # * Neither the name of Google Inc. nor the names of its 17 | # contributors may be used to endorse or promote products derived from 18 | # this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | regenerate: 33 | protoc --go_out=. *.proto 34 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/jsonpb/jsonpb_test_proto/more_test_objects.proto: -------------------------------------------------------------------------------- 1 | // Go support for Protocol Buffers - Google's data interchange format 2 | // 3 | // Copyright 2015 The Go Authors. All rights reserved. 4 | // https://github.com/golang/protobuf 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are 8 | // met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above 13 | // copyright notice, this list of conditions and the following disclaimer 14 | // in the documentation and/or other materials provided with the 15 | // distribution. 16 | // * Neither the name of Google Inc. nor the names of its 17 | // contributors may be used to endorse or promote products derived from 18 | // this software without specific prior written permission. 19 | // 20 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | syntax = "proto3"; 33 | 34 | package jsonpb; 35 | 36 | message Simple3 { 37 | double dub = 1; 38 | } 39 | 40 | message Mappy { 41 | map nummy = 1; 42 | map strry = 2; 43 | map objjy = 3; 44 | map buggy = 4; 45 | map booly = 5; 46 | } 47 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/proto/Makefile: -------------------------------------------------------------------------------- 1 | # Go support for Protocol Buffers - Google's data interchange format 2 | # 3 | # Copyright 2010 The Go Authors. All rights reserved. 4 | # https://github.com/golang/protobuf 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are 8 | # met: 9 | # 10 | # * Redistributions of source code must retain the above copyright 11 | # notice, this list of conditions and the following disclaimer. 12 | # * Redistributions in binary form must reproduce the above 13 | # copyright notice, this list of conditions and the following disclaimer 14 | # in the documentation and/or other materials provided with the 15 | # distribution. 16 | # * Neither the name of Google Inc. nor the names of its 17 | # contributors may be used to endorse or promote products derived from 18 | # this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | install: 33 | go install 34 | 35 | test: install generate-test-pbs 36 | go test 37 | 38 | 39 | generate-test-pbs: 40 | make install 41 | make -C testdata 42 | protoc --go_out=Mtestdata/test.proto=github.com/golang/protobuf/proto/testdata:. proto3_proto/proto3.proto 43 | make 44 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/proto/testdata/Makefile: -------------------------------------------------------------------------------- 1 | # Go support for Protocol Buffers - Google's data interchange format 2 | # 3 | # Copyright 2010 The Go Authors. All rights reserved. 4 | # https://github.com/golang/protobuf 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are 8 | # met: 9 | # 10 | # * Redistributions of source code must retain the above copyright 11 | # notice, this list of conditions and the following disclaimer. 12 | # * Redistributions in binary form must reproduce the above 13 | # copyright notice, this list of conditions and the following disclaimer 14 | # in the documentation and/or other materials provided with the 15 | # distribution. 16 | # * Neither the name of Google Inc. nor the names of its 17 | # contributors may be used to endorse or promote products derived from 18 | # this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | 33 | include ../../Make.protobuf 34 | 35 | all: regenerate 36 | 37 | regenerate: 38 | rm -f test.pb.go 39 | make test.pb.go 40 | 41 | # The following rules are just aids to development. Not needed for typical testing. 42 | 43 | diff: regenerate 44 | git diff test.pb.go 45 | 46 | restore: 47 | cp test.pb.go.golden test.pb.go 48 | 49 | preserve: 50 | cp test.pb.go test.pb.go.golden 51 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/protoc-gen-go/Makefile: -------------------------------------------------------------------------------- 1 | # Go support for Protocol Buffers - Google's data interchange format 2 | # 3 | # Copyright 2010 The Go Authors. All rights reserved. 4 | # https://github.com/golang/protobuf 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are 8 | # met: 9 | # 10 | # * Redistributions of source code must retain the above copyright 11 | # notice, this list of conditions and the following disclaimer. 12 | # * Redistributions in binary form must reproduce the above 13 | # copyright notice, this list of conditions and the following disclaimer 14 | # in the documentation and/or other materials provided with the 15 | # distribution. 16 | # * Neither the name of Google Inc. nor the names of its 17 | # contributors may be used to endorse or promote products derived from 18 | # this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | test: 33 | cd testdata && make test 34 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/protoc-gen-go/generator/Makefile: -------------------------------------------------------------------------------- 1 | # Go support for Protocol Buffers - Google's data interchange format 2 | # 3 | # Copyright 2010 The Go Authors. All rights reserved. 4 | # https://github.com/golang/protobuf 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are 8 | # met: 9 | # 10 | # * Redistributions of source code must retain the above copyright 11 | # notice, this list of conditions and the following disclaimer. 12 | # * Redistributions in binary form must reproduce the above 13 | # copyright notice, this list of conditions and the following disclaimer 14 | # in the documentation and/or other materials provided with the 15 | # distribution. 16 | # * Neither the name of Google Inc. nor the names of its 17 | # contributors may be used to endorse or promote products derived from 18 | # this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | include $(GOROOT)/src/Make.inc 33 | 34 | TARG=github.com/golang/protobuf/compiler/generator 35 | GOFILES=\ 36 | generator.go\ 37 | 38 | DEPS=../descriptor ../plugin ../../proto 39 | 40 | include $(GOROOT)/src/Make.pkg 41 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/protoc-gen-go/link_grpc.go: -------------------------------------------------------------------------------- 1 | // Go support for Protocol Buffers - Google's data interchange format 2 | // 3 | // Copyright 2015 The Go Authors. All rights reserved. 4 | // https://github.com/golang/protobuf 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are 8 | // met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above 13 | // copyright notice, this list of conditions and the following disclaimer 14 | // in the documentation and/or other materials provided with the 15 | // distribution. 16 | // * Neither the name of Google Inc. nor the names of its 17 | // contributors may be used to endorse or promote products derived from 18 | // this software without specific prior written permission. 19 | // 20 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | package main 33 | 34 | import _ "github.com/golang/protobuf/protoc-gen-go/internal/grpc" 35 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/protoc-gen-go/testdata/extension_base.proto: -------------------------------------------------------------------------------- 1 | // Go support for Protocol Buffers - Google's data interchange format 2 | // 3 | // Copyright 2010 The Go Authors. All rights reserved. 4 | // https://github.com/golang/protobuf 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are 8 | // met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above 13 | // copyright notice, this list of conditions and the following disclaimer 14 | // in the documentation and/or other materials provided with the 15 | // distribution. 16 | // * Neither the name of Google Inc. nor the names of its 17 | // contributors may be used to endorse or promote products derived from 18 | // this software without specific prior written permission. 19 | // 20 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | syntax = "proto2"; 33 | 34 | package extension_base; 35 | 36 | message BaseMessage { 37 | optional int32 height = 1; 38 | extensions 4 to 9; 39 | extensions 16 to max; 40 | } 41 | 42 | // Another message that may be extended, using message_set_wire_format. 43 | message OldStyleMessage { 44 | option message_set_wire_format = true; 45 | extensions 100 to max; 46 | } 47 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/protoc-gen-go/testdata/extension_extra.proto: -------------------------------------------------------------------------------- 1 | // Go support for Protocol Buffers - Google's data interchange format 2 | // 3 | // Copyright 2011 The Go Authors. All rights reserved. 4 | // https://github.com/golang/protobuf 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are 8 | // met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above 13 | // copyright notice, this list of conditions and the following disclaimer 14 | // in the documentation and/or other materials provided with the 15 | // distribution. 16 | // * Neither the name of Google Inc. nor the names of its 17 | // contributors may be used to endorse or promote products derived from 18 | // this software without specific prior written permission. 19 | // 20 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | syntax = "proto2"; 33 | 34 | package extension_extra; 35 | 36 | message ExtraMessage { 37 | optional int32 width = 1; 38 | } 39 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/protoc-gen-go/testdata/imp2.proto: -------------------------------------------------------------------------------- 1 | // Go support for Protocol Buffers - Google's data interchange format 2 | // 3 | // Copyright 2011 The Go Authors. All rights reserved. 4 | // https://github.com/golang/protobuf 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are 8 | // met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above 13 | // copyright notice, this list of conditions and the following disclaimer 14 | // in the documentation and/or other materials provided with the 15 | // distribution. 16 | // * Neither the name of Google Inc. nor the names of its 17 | // contributors may be used to endorse or promote products derived from 18 | // this software without specific prior written permission. 19 | // 20 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | syntax = "proto2"; 33 | 34 | package imp; 35 | 36 | message PubliclyImportedMessage { 37 | optional int64 field = 1; 38 | } 39 | 40 | enum PubliclyImportedEnum { 41 | GLASSES = 1; 42 | HAIR = 2; 43 | } 44 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/protoc-gen-go/testdata/imp3.proto: -------------------------------------------------------------------------------- 1 | // Go support for Protocol Buffers - Google's data interchange format 2 | // 3 | // Copyright 2012 The Go Authors. All rights reserved. 4 | // https://github.com/golang/protobuf 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are 8 | // met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above 13 | // copyright notice, this list of conditions and the following disclaimer 14 | // in the documentation and/or other materials provided with the 15 | // distribution. 16 | // * Neither the name of Google Inc. nor the names of its 17 | // contributors may be used to endorse or promote products derived from 18 | // this software without specific prior written permission. 19 | // 20 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | syntax = "proto2"; 33 | 34 | package imp; 35 | 36 | message ForeignImportedMessage { 37 | optional string tuber = 1; 38 | } 39 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/protoc-gen-go/testdata/main_test.go: -------------------------------------------------------------------------------- 1 | // Go support for Protocol Buffers - Google's data interchange format 2 | // 3 | // Copyright 2010 The Go Authors. All rights reserved. 4 | // https://github.com/golang/protobuf 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are 8 | // met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above 13 | // copyright notice, this list of conditions and the following disclaimer 14 | // in the documentation and/or other materials provided with the 15 | // distribution. 16 | // * Neither the name of Google Inc. nor the names of its 17 | // contributors may be used to endorse or promote products derived from 18 | // this software without specific prior written permission. 19 | // 20 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | // A simple binary to link together the protocol buffers in this test. 33 | 34 | package testdata 35 | 36 | import ( 37 | "testing" 38 | 39 | mytestpb "./my_test" 40 | multipb "github.com/golang/protobuf/protoc-gen-go/testdata/multi" 41 | ) 42 | 43 | func TestLink(t *testing.T) { 44 | _ = &multipb.Multi1{} 45 | _ = &mytestpb.Request{} 46 | } 47 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/protoc-gen-go/testdata/multi/multi1.proto: -------------------------------------------------------------------------------- 1 | // Go support for Protocol Buffers - Google's data interchange format 2 | // 3 | // Copyright 2010 The Go Authors. All rights reserved. 4 | // https://github.com/golang/protobuf 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are 8 | // met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above 13 | // copyright notice, this list of conditions and the following disclaimer 14 | // in the documentation and/or other materials provided with the 15 | // distribution. 16 | // * Neither the name of Google Inc. nor the names of its 17 | // contributors may be used to endorse or promote products derived from 18 | // this software without specific prior written permission. 19 | // 20 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | syntax = "proto2"; 33 | 34 | import "multi/multi2.proto"; 35 | import "multi/multi3.proto"; 36 | 37 | package multitest; 38 | 39 | message Multi1 { 40 | required Multi2 multi2 = 1; 41 | optional Multi2.Color color = 2; 42 | optional Multi3.HatType hat_type = 3; 43 | } 44 | 45 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/protoc-gen-go/testdata/multi/multi2.proto: -------------------------------------------------------------------------------- 1 | // Go support for Protocol Buffers - Google's data interchange format 2 | // 3 | // Copyright 2010 The Go Authors. All rights reserved. 4 | // https://github.com/golang/protobuf 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are 8 | // met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above 13 | // copyright notice, this list of conditions and the following disclaimer 14 | // in the documentation and/or other materials provided with the 15 | // distribution. 16 | // * Neither the name of Google Inc. nor the names of its 17 | // contributors may be used to endorse or promote products derived from 18 | // this software without specific prior written permission. 19 | // 20 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | syntax = "proto2"; 33 | 34 | package multitest; 35 | 36 | message Multi2 { 37 | required int32 required_value = 1; 38 | 39 | enum Color { 40 | BLUE = 1; 41 | GREEN = 2; 42 | RED = 3; 43 | }; 44 | optional Color color = 2; 45 | } 46 | 47 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/protoc-gen-go/testdata/multi/multi3.proto: -------------------------------------------------------------------------------- 1 | // Go support for Protocol Buffers - Google's data interchange format 2 | // 3 | // Copyright 2010 The Go Authors. All rights reserved. 4 | // https://github.com/golang/protobuf 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are 8 | // met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above 13 | // copyright notice, this list of conditions and the following disclaimer 14 | // in the documentation and/or other materials provided with the 15 | // distribution. 16 | // * Neither the name of Google Inc. nor the names of its 17 | // contributors may be used to endorse or promote products derived from 18 | // this software without specific prior written permission. 19 | // 20 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | syntax = "proto2"; 33 | 34 | package multitest; 35 | 36 | message Multi3 { 37 | enum HatType { 38 | FEDORA = 1; 39 | FEZ = 2; 40 | }; 41 | optional HatType hat_type = 1; 42 | } 43 | 44 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/golang/protobuf/protoc-gen-go/testdata/proto3.proto: -------------------------------------------------------------------------------- 1 | // Go support for Protocol Buffers - Google's data interchange format 2 | // 3 | // Copyright 2014 The Go Authors. All rights reserved. 4 | // https://github.com/golang/protobuf 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are 8 | // met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above 13 | // copyright notice, this list of conditions and the following disclaimer 14 | // in the documentation and/or other materials provided with the 15 | // distribution. 16 | // * Neither the name of Google Inc. nor the names of its 17 | // contributors may be used to endorse or promote products derived from 18 | // this software without specific prior written permission. 19 | // 20 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | syntax = "proto3"; 33 | 34 | package proto3; 35 | 36 | message Request { 37 | enum Flavour { 38 | SWEET = 0; 39 | SOUR = 1; 40 | UMAMI = 2; 41 | GOPHERLICIOUS = 3; 42 | } 43 | string name = 1; 44 | repeated int64 key = 2; 45 | Flavour taste = 3; 46 | Book book = 4; 47 | } 48 | 49 | message Book { 50 | string title = 1; 51 | bytes raw_data = 2; 52 | } 53 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/mreiferson/go-snappystream/.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.2.2 4 | - 1.3.1 5 | env: 6 | - GOARCH=amd64 7 | - GOARCH=386 8 | install: 9 | - go get code.google.com/p/snappy-go/snappy 10 | script: 11 | - go test -v 12 | notifications: 13 | email: false 14 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/mreiferson/go-snappystream/LICENSE: -------------------------------------------------------------------------------- 1 | Permission is hereby granted, free of charge, to any person obtaining a copy 2 | of this software and associated documentation files (the "Software"), to deal 3 | in the Software without restriction, including without limitation the rights 4 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 5 | copies of the Software, and to permit persons to whom the Software is 6 | furnished to do so, subject to the following conditions: 7 | 8 | The above copyright notice and this permission notice shall be included in 9 | all copies or substantial portions of the Software. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 12 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 13 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 14 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 15 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 16 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 17 | THE SOFTWARE. 18 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/mreiferson/go-snappystream/README.md: -------------------------------------------------------------------------------- 1 | ## go-snappystream 2 | 3 | a Go package for framed snappy streams. 4 | 5 | [![Build Status](https://secure.travis-ci.org/mreiferson/go-snappystream.png?branch=master)](http://travis-ci.org/mreiferson/go-snappystream) [![GoDoc](https://godoc.org/github.com/mreiferson/go-snappystream?status.svg)](https://godoc.org/github.com/mreiferson/go-snappystream) 6 | 7 | This package wraps [snappy-go][1] and supplies a `Reader` and `Writer` 8 | for the snappy [framed stream format][2]. 9 | 10 | [1]: https://code.google.com/p/snappy-go/ 11 | [2]: https://snappy.googlecode.com/svn/trunk/framing_format.txt 12 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/mreiferson/go-snappystream/snappy-go/snappy.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 The Snappy-Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package snappy implements the snappy block-based compression format. 6 | // It aims for very high speeds and reasonable compression. 7 | // 8 | // The C++ snappy implementation is at http://code.google.com/p/snappy/ 9 | package snappy 10 | 11 | /* 12 | Each encoded block begins with the varint-encoded length of the decoded data, 13 | followed by a sequence of chunks. Chunks begin and end on byte boundaries. The 14 | first byte of each chunk is broken into its 2 least and 6 most significant bits 15 | called l and m: l ranges in [0, 4) and m ranges in [0, 64). l is the chunk tag. 16 | Zero means a literal tag. All other values mean a copy tag. 17 | 18 | For literal tags: 19 | - If m < 60, the next 1 + m bytes are literal bytes. 20 | - Otherwise, let n be the little-endian unsigned integer denoted by the next 21 | m - 59 bytes. The next 1 + n bytes after that are literal bytes. 22 | 23 | For copy tags, length bytes are copied from offset bytes ago, in the style of 24 | Lempel-Ziv compression algorithms. In particular: 25 | - For l == 1, the offset ranges in [0, 1<<11) and the length in [4, 12). 26 | The length is 4 + the low 3 bits of m. The high 3 bits of m form bits 8-10 27 | of the offset. The next byte is bits 0-7 of the offset. 28 | - For l == 2, the offset ranges in [0, 1<<16) and the length in [1, 65). 29 | The length is 1 + m. The offset is the little-endian unsigned integer 30 | denoted by the next 2 bytes. 31 | - For l == 3, this tag is a legacy format that is no longer supported. 32 | */ 33 | const ( 34 | tagLiteral = 0x00 35 | tagCopy1 = 0x01 36 | tagCopy2 = 0x02 37 | tagCopy4 = 0x03 38 | ) 39 | -------------------------------------------------------------------------------- /builder/docformat/src/github.com/mreiferson/go-snappystream/snappystream.go: -------------------------------------------------------------------------------- 1 | // snappystream wraps snappy-go and supplies a Reader and Writer 2 | // for the snappy framed stream format: 3 | // https://snappy.googlecode.com/svn/trunk/framing_format.txt 4 | package snappystream 5 | 6 | import ( 7 | "hash/crc32" 8 | 9 | "github.com/mreiferson/go-snappystream/snappy-go" 10 | ) 11 | 12 | // Ext is the file extension for files whose content is a snappy framed stream. 13 | const Ext = ".sz" 14 | 15 | // MediaType is the MIME type used to represent snappy framed content. 16 | const MediaType = "application/x-snappy-framed" 17 | 18 | // ContentEncoding is the appropriate HTTP Content-Encoding header value for 19 | // requests containing a snappy framed entity body. 20 | const ContentEncoding = "x-snappy-framed" 21 | 22 | // MaxBlockSize is the maximum number of decoded bytes allowed to be 23 | // represented in a snappy framed block (sections 4.2 and 4.3). 24 | const MaxBlockSize = 65536 25 | 26 | // maxEncodedBlockSize is the maximum number of encoded bytes in a framed 27 | // block. 28 | var maxEncodedBlockSize = uint32(snappy.MaxEncodedLen(MaxBlockSize)) 29 | 30 | const VerifyChecksum = true 31 | const SkipVerifyChecksum = false 32 | 33 | // Block types defined by the snappy framed format specification. 34 | const ( 35 | blockCompressed = 0x00 36 | blockUncompressed = 0x01 37 | blockPadding = 0xfe 38 | blockStreamIdentifier = 0xff 39 | ) 40 | 41 | // streamID is the stream identifier block that begins a valid snappy framed 42 | // stream. 43 | var streamID = []byte{0xff, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, 0x50, 0x70, 0x59} 44 | 45 | // maskChecksum implements the checksum masking algorithm described by the spec. 46 | func maskChecksum(c uint32) uint32 { 47 | return ((c >> 15) | (c << 17)) + 0xa282ead8 48 | } 49 | 50 | var crcTable *crc32.Table 51 | 52 | func init() { 53 | crcTable = crc32.MakeTable(crc32.Castagnoli) 54 | } 55 | -------------------------------------------------------------------------------- /builder/docformat/src/job/job.go: -------------------------------------------------------------------------------- 1 | // Package job includes mainly 2 interface: Processor & Collector. 2 | // Processor is supposed to receive a single string (refered to as a "msg") 3 | // as input, then it outputs one or more semi-finished results(refered to 4 | // as a "item") via channels to Collector. 5 | // Collector receive the items and flush them out when appropriate or do 6 | // anything you like. 7 | package job 8 | 9 | import ( 10 | sj "github.com/bitly/go-simplejson" 11 | ) 12 | 13 | type Item struct { 14 | Category string 15 | Id string 16 | Content string 17 | RawMsg string 18 | } 19 | 20 | type Processor interface { 21 | Init(ctx *sj.Json, id int, itemChans [](chan Item)) error 22 | Process(msg string) error 23 | Tick() error 24 | Destory() error 25 | } 26 | 27 | type Collector interface { 28 | Init(ctx *sj.Json, id int) error 29 | Collect(item Item) error 30 | Tick() error 31 | Destory() error 32 | } 33 | 34 | func NewProcessor(name string) Processor { 35 | switch name { 36 | case "LogtoHdfsProcessor": 37 | return new(LogtoHdfsProcessor) 38 | case "TestProcessor": 39 | return new(TestProcessor) 40 | default: 41 | return nil 42 | } 43 | return nil 44 | } 45 | 46 | func NewCollector(name string) Collector { 47 | switch name { 48 | case "LogtoHdfsCollector": 49 | return new(LogtoHdfsCollector) 50 | case "TestCollector": 51 | return new(TestCollector) 52 | default: 53 | return nil 54 | } 55 | return nil 56 | } 57 | -------------------------------------------------------------------------------- /builder/docformat/src/job/test_collector.go: -------------------------------------------------------------------------------- 1 | package job 2 | 3 | import ( 4 | sj "github.com/bitly/go-simplejson" 5 | "github.com/donnie4w/go-logger/logger" 6 | "math/rand" 7 | "time" 8 | ) 9 | 10 | type TestCollector struct { 11 | id int 12 | } 13 | 14 | func (p *TestCollector) Init(ctx *sj.Json, id int) error { 15 | p.id = id 16 | return nil 17 | } 18 | 19 | func (p *TestCollector) Collect(item Item) error { 20 | logger.Debugf("collector %d get item: %v", p.id, item) 21 | time.Sleep(time.Millisecond * time.Duration(rand.Intn(1000))) 22 | if p.id == 1 || p.id == 2 { 23 | time.Sleep(time.Millisecond * 20000) 24 | } 25 | 26 | return nil 27 | } 28 | 29 | func (p *TestCollector) Destory() error { 30 | logger.Debug("collector destroy", p.id) 31 | return nil 32 | } 33 | 34 | func (p *TestCollector) Tick() error { 35 | logger.Debug("collector tick", p.id) 36 | return nil 37 | } 38 | -------------------------------------------------------------------------------- /builder/docformat/src/job/test_processor.go: -------------------------------------------------------------------------------- 1 | package job 2 | 3 | import ( 4 | "errors" 5 | "math/rand" 6 | "strings" 7 | 8 | sj "github.com/bitly/go-simplejson" 9 | "github.com/donnie4w/go-logger/logger" 10 | ) 11 | 12 | type TestProcessor struct { 13 | id int 14 | 15 | itemChans [](chan Item) 16 | } 17 | 18 | func (p *TestProcessor) Init(ctx *sj.Json, id int, itemChans [](chan Item)) error { 19 | p.id = id 20 | p.itemChans = itemChans 21 | 22 | return nil 23 | } 24 | 25 | func (p *TestProcessor) Process(msg string) error { 26 | msgArr := strings.Split(msg, "\n") 27 | if len(msgArr) <= 0 { 28 | return nil 29 | } 30 | for _, v := range msgArr { 31 | afterTrim := strings.TrimSpace(v) 32 | if len(afterTrim) <= 0 { 33 | logger.Debugf("processor %d empty msg: [%s]", p.id, msg) 34 | continue 35 | } 36 | item, err := p.getItem(afterTrim) 37 | if err != nil || item == nil { 38 | logger.Errorf("processor %d msg fail: [%s], err: %v", p.id, v, err) 39 | logger.Errorf("processor %d msg msgArr: [%v], msg: %s", p.id, msgArr, msg) 40 | continue 41 | } 42 | logger.Debugf("processor %d msg ok: [%s]", p.id, v) 43 | 44 | index := p.getCollectorIndex(BKDRHash(item.Content), len(p.itemChans)) 45 | p.itemChans[index] <- *item 46 | logger.Debugf("processor %d push successful to index: %d", p.id, index) 47 | } 48 | 49 | return nil 50 | } 51 | 52 | func (p *TestProcessor) Tick() error { 53 | logger.Debug("processor Tick", p.id) 54 | return nil 55 | } 56 | 57 | func (p *TestProcessor) Destory() error { 58 | logger.Debug("processor destroy", p.id) 59 | return nil 60 | } 61 | 62 | func (p *TestProcessor) getItem(msg string) (*Item, error) { 63 | if len(msg) <= 0 { 64 | return nil, errors.New("msg empty") 65 | } 66 | 67 | item := Item{ 68 | Category: "test_collector", 69 | Id: msg, 70 | Content: "content: " + msg, 71 | RawMsg: "rawMsg: " + msg, 72 | } 73 | return &item, nil 74 | } 75 | 76 | func (p *TestProcessor) getCollectorIndex(hash int, size int) int { 77 | return (hash + rand.Intn(10)) % size 78 | } 79 | -------------------------------------------------------------------------------- /builder/docformat/src/main/nsq_src_provider.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "time" 6 | 7 | nsq "github.com/bitly/go-nsq" 8 | sj "github.com/bitly/go-simplejson" 9 | ) 10 | 11 | type NsqSrc struct { 12 | consumer *nsq.Consumer 13 | config *nsq.Config 14 | nsqMsgChan chan nsq.Message 15 | timeOutSec int 16 | } 17 | 18 | func (p *NsqSrc) GetNextMsg() ([][]byte, error) { 19 | select { 20 | case nsqMsg := <-p.nsqMsgChan: 21 | byteArrs := make([][]byte, 1) 22 | byteArrs[0] = nsqMsg.Body 23 | return byteArrs, nil 24 | case <-time.After(time.Duration(p.timeOutSec) * time.Second): 25 | 26 | } 27 | return [][]byte{}, nil 28 | } 29 | 30 | func (p *NsqSrc) Ack() error { 31 | return nil 32 | } 33 | 34 | func (p *NsqSrc) Destory() error { 35 | p.consumer.Stop() 36 | return nil 37 | } 38 | 39 | func (p *NsqSrc) Init(ctx *sj.Json) error { 40 | var err error 41 | p.nsqMsgChan = make(chan nsq.Message) // not buffered 42 | 43 | p.timeOutSec = ctx.Get("timeout_sec").MustInt() 44 | nsqLookupdAddrs := ctx.Get("nsq_lookupd_addrs").MustStringArray() 45 | if len(nsqLookupdAddrs) <= 0 { 46 | return errors.New("no nsq_lookupd_addr in provided") 47 | } 48 | topic := ctx.Get("topic").MustString() 49 | channel := ctx.Get("channel").MustString() 50 | 51 | p.config = nsq.NewConfig() 52 | p.consumer, err = nsq.NewConsumer(topic, channel, p.config) 53 | if p.consumer == nil || err != nil { 54 | return err 55 | } 56 | 57 | p.consumer.AddHandler(nsq.HandlerFunc(func(message *nsq.Message) error { 58 | p.nsqMsgChan <- *message 59 | return nil 60 | })) 61 | 62 | err = p.consumer.ConnectToNSQLookupds(nsqLookupdAddrs) // ip:port 63 | if err != nil { 64 | return errors.New("Could not connect to nsqLookupd") 65 | } 66 | 67 | return nil 68 | } 69 | -------------------------------------------------------------------------------- /builder/docformat/src/main/redis_src_provider.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "common" 5 | "errors" 6 | "strings" 7 | 8 | sj "github.com/bitly/go-simplejson" 9 | "github.com/donnie4w/go-logger/logger" 10 | "github.com/garyburd/redigo/redis" 11 | ) 12 | 13 | type RedisSrc struct { 14 | pool *redis.Pool 15 | topic string 16 | } 17 | 18 | func (p *RedisSrc) GetNextMsg() ([][]byte, error) { 19 | conn := p.pool.Get() 20 | defer conn.Close() 21 | s, err := redis.String(conn.Do("RPOP", p.topic)) 22 | byteArr := make([][]byte, 1) 23 | byteArr[0] = []byte(s) 24 | return byteArr, err 25 | } 26 | 27 | func (p *RedisSrc) Ack() error { 28 | return nil 29 | } 30 | 31 | func (p *RedisSrc) Destory() error { 32 | return nil 33 | } 34 | 35 | func (p *RedisSrc) Init(ctx *sj.Json) error { 36 | redisConf := ctx.Get("redis_config").MustString() 37 | if redisConf == "" { 38 | return errors.New("redis config empty") 39 | } 40 | redisConfArr := strings.Split(redisConf, ":") 41 | if len(redisConfArr) < 3 { 42 | return errors.New("redis conf invalid") 43 | } 44 | 45 | redisHost := redisConfArr[0] 46 | redisPort := redisConfArr[1] 47 | redisAuth := redisConfArr[2] 48 | p.pool = common.NewPool(redisHost+":"+redisPort, redisAuth, 3, 240) 49 | conn := p.pool.Get() 50 | defer conn.Close() 51 | response, err := redis.String(conn.Do("PING")) 52 | if err != nil || response != "PONG" { 53 | logger.Error("redis connect fail [", redisHost, " ", redisPort, " ", redisAuth, response, err) 54 | return errors.New("redis connect fail") 55 | } 56 | p.topic = ctx.Get("topic").MustString() 57 | if p.topic == "" { 58 | return errors.New("topic empty") 59 | } 60 | return nil 61 | } 62 | -------------------------------------------------------------------------------- /builder/docformat/src/main/src_provider.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | sj "github.com/bitly/go-simplejson" 5 | ) 6 | 7 | type SrcProvider interface { 8 | Init(ctx *sj.Json) error 9 | GetNextMsg() ([][]byte, error) 10 | Ack() error 11 | Destory() error 12 | } 13 | 14 | func NewSrcProvider(srcType string) SrcProvider { 15 | switch srcType { 16 | case "nsq": 17 | return new(NsqSrc) 18 | case "dm": 19 | return new(DmSrc) 20 | case "redis": 21 | return new(RedisSrc) 22 | default: 23 | return nil 24 | } 25 | return nil 26 | } 27 | -------------------------------------------------------------------------------- /builder/docformat/src/poseidon/datastruct/Makefile: -------------------------------------------------------------------------------- 1 | 2 | OUTPUT_DIR=$(shell pwd) 3 | PROTOC3:=protoc 4 | 5 | PROTOS := $(wildcard *.proto) 6 | #SRCS := $(patsubst %.proto, $(OUTPUT_DIR)/%.pb.cc, $(PROTOS)) 7 | #PYSRCS := $(patsubst %.proto, $(OUTPUT_DIR)/%_pb2.py, $(PROTOS)) 8 | GOSRCS := $(patsubst %.proto, $(OUTPUT_DIR)/%.pb.go, $(PROTOS)) 9 | 10 | 11 | all : $(GOSRCS) 12 | 13 | t : 14 | @echo $(OUTPUT_DIR) 15 | @echo $(PROTOS) 16 | @echo $(GOSRCS) 17 | 18 | clean : 19 | @rm -rf *.pb.go 20 | @rm -rf *.d *.o 21 | 22 | $(OUTPUT_DIR)/%.pb.cc : %.proto 23 | $(PROTOC3) --cpp_out=$(OUTPUT_DIR) $< 24 | 25 | $(OUTPUT_DIR)/%_pb2.py : %.proto 26 | $(PROTOC3) --python_out=$(OUTPUT_DIR) $< 27 | 28 | $(OUTPUT_DIR)/%.pb.go : %.proto 29 | $(PROTOC3) --go_out=$(OUTPUT_DIR) $< 30 | 31 | -------------------------------------------------------------------------------- /builder/docmeta/build.gradle: -------------------------------------------------------------------------------- 1 | group 'docmeta' 2 | version '0.1' 3 | 4 | apply plugin: 'java' 5 | 6 | sourceCompatibility = 1.7 7 | 8 | repositories { 9 | mavenCentral() 10 | } 11 | 12 | ext { 13 | hadoopVersion = "2.7.1" 14 | } 15 | 16 | def var 17 | var = dependencies { 18 | compile group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion 19 | compile group: 'org.apache.hadoop', name: 'hadoop-mapreduce-client-core', version: hadoopVersion 20 | compile group: 'org.apache.hadoop', name: 'hadoop-mapreduce-client-common', version: hadoopVersion 21 | compile group: 'org.apache.hadoop', name: 'hadoop-mapreduce-client-jobclient', version: hadoopVersion 22 | compile group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: '2.7.1' 23 | 24 | 25 | compile group: 'commons-configuration', name: 'commons-configuration', version: '1.9' 26 | compile group: 'org.apache.commons', name: 'commons-compress', version: '1.4.1' 27 | compile group: 'com.google.protobuf', name: 'protobuf-java', version: '3.1.0' 28 | 29 | compile (project(':indexmeta')) 30 | compile fileTree(dir: '../index/libs', include: 'json.jar') 31 | testCompile group: 'junit', name: 'junit', version: '4.12' 32 | } 33 | var 34 | -------------------------------------------------------------------------------- /builder/docmeta/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/poseidon/9be3a58756c2c604573075b6e7531f13f5ba7c1c/builder/docmeta/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /builder/docmeta/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Mon Sep 26 17:32:50 CST 2016 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-3.0-bin.zip 7 | -------------------------------------------------------------------------------- /builder/docmeta/src/main/etc/test.json: -------------------------------------------------------------------------------- 1 | { 2 | "common": { 3 | "bussiness": "test", 4 | "name_node": "127.0.0.1:9000", 5 | "hadoop_user": "root", 6 | "hdpfs_index_base_path": "/home/poseidon/src", 7 | "data_hashid_section": 10000000000, 8 | "line_per_doc": 128, 9 | "indexgzmeta_section": 200, 10 | "indexgz_file_num_per_field": 1000, 11 | "meta_service": "127.0.0.1:39610" 12 | }, 13 | "inverted_index": { 14 | "name": "ck_ad", 15 | "save_line_count_per_map": 100000, 16 | "line_average_length": 850, 17 | "data_format": "TAB", 18 | "daily_index": false, 19 | "daily_data_size": "10TB", 20 | "log_dir_level": 1, 21 | "hdfs_path": "/home/poseidon/src/test", 22 | "hdfs_cluster": "idc-name", 23 | "hdfs_path_date_pattern": "YYYY-MM-DD", 24 | "hdfs_file_pattern": "gz", 25 | "cache_size": 16777216, 26 | "debug": 0, 27 | "is_total": true, 28 | "etcs": "", 29 | "libs": "", 30 | "field_names": [ 31 | "text" 32 | ], 33 | "tokenizer": { 34 | "text": [ 35 | { 36 | "split": "\\|\\|" 37 | }, 38 | "urlencode", 39 | "keyword" 40 | ] 41 | }, 42 | "alias": { 43 | "text": "text" 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /builder/docmeta/src/main/java/meta/DocMetaConfigured.java: -------------------------------------------------------------------------------- 1 | package meta; 2 | 3 | import org.apache.hadoop.conf.Configuration; 4 | import org.apache.hadoop.io.BytesWritable; 5 | import org.apache.hadoop.io.Text; 6 | import org.apache.hadoop.mapreduce.Job; 7 | import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; 8 | import org.apache.hadoop.util.ToolRunner; 9 | 10 | import java.io.IOException; 11 | import java.net.URISyntaxException; 12 | 13 | /** 14 | * Created by liwei on 9/26/16. 15 | */ 16 | 17 | 18 | public class DocMetaConfigured extends MetaConfigured { 19 | public static void main(String[] args) throws IOException, URISyntaxException, ClassNotFoundException, InterruptedException { 20 | int ret = 0; 21 | try { 22 | ret = ToolRunner.run(new DocMetaConfigured(), args); 23 | } catch (Exception e) { 24 | e.printStackTrace(); 25 | } 26 | System.exit(ret); 27 | } 28 | 29 | @Override 30 | public Job GetMapReduceJob(Configuration conf) { 31 | try { 32 | conf.setInt("mapred.reduce.tasks", 1); 33 | Job job = new Job(conf, DocMetaConfigured.class.getSimpleName()); 34 | System.err.println("get doc meta job "); 35 | job.setJarByClass(DocMetaConfigured.class); 36 | job.setInputFormatClass(TextInputFormat.class); 37 | job.setMapperClass(DocMetaMapper.class); 38 | job.setMapOutputKeyClass(Text.class); 39 | job.setMapOutputValueClass(Text.class); 40 | 41 | job.setOutputKeyClass(Text.class); 42 | job.setOutputValueClass(BytesWritable.class); 43 | return job; 44 | } catch (Exception e) { 45 | 46 | } 47 | return null; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /builder/index/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | which gradle 4 | if [ $? -eq 0 ]; 5 | then 6 | gradle distTar 7 | else 8 | ./gradlew distTar 9 | fi 10 | 11 | if [ $? -ne 0 ]; 12 | then 13 | echo pkg index failed 14 | fi 15 | 16 | cp -r build/distributions/index*.tar ../../dist/ 17 | -------------------------------------------------------------------------------- /builder/index/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/poseidon/9be3a58756c2c604573075b6e7531f13f5ba7c1c/builder/index/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /builder/index/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Tue Jan 03 14:55:05 CST 2017 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-3.0-all.zip 7 | -------------------------------------------------------------------------------- /builder/index/libs/commons-codec-1.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/poseidon/9be3a58756c2c604573075b6e7531f13f5ba7c1c/builder/index/libs/commons-codec-1.3.jar -------------------------------------------------------------------------------- /builder/index/libs/commons-logging-1.0.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/poseidon/9be3a58756c2c604573075b6e7531f13f5ba7c1c/builder/index/libs/commons-logging-1.0.4.jar -------------------------------------------------------------------------------- /builder/index/libs/hadoop-0.20.2.1U11-ant.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/poseidon/9be3a58756c2c604573075b6e7531f13f5ba7c1c/builder/index/libs/hadoop-0.20.2.1U11-ant.jar -------------------------------------------------------------------------------- /builder/index/libs/hadoop-0.20.2.1U11-core.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/poseidon/9be3a58756c2c604573075b6e7531f13f5ba7c1c/builder/index/libs/hadoop-0.20.2.1U11-core.jar -------------------------------------------------------------------------------- /builder/index/libs/hadoop-0.20.2.1U11-examples.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/poseidon/9be3a58756c2c604573075b6e7531f13f5ba7c1c/builder/index/libs/hadoop-0.20.2.1U11-examples.jar -------------------------------------------------------------------------------- /builder/index/libs/hadoop-0.20.2.1U11-tools.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/poseidon/9be3a58756c2c604573075b6e7531f13f5ba7c1c/builder/index/libs/hadoop-0.20.2.1U11-tools.jar -------------------------------------------------------------------------------- /builder/index/libs/hamcrest-core-1.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/poseidon/9be3a58756c2c604573075b6e7531f13f5ba7c1c/builder/index/libs/hamcrest-core-1.3.jar -------------------------------------------------------------------------------- /builder/index/libs/ikanalyzer2012ff.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/poseidon/9be3a58756c2c604573075b6e7531f13f5ba7c1c/builder/index/libs/ikanalyzer2012ff.jar -------------------------------------------------------------------------------- /builder/index/libs/json.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/poseidon/9be3a58756c2c604573075b6e7531f13f5ba7c1c/builder/index/libs/json.jar -------------------------------------------------------------------------------- /builder/index/libs/junit-4.12.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/poseidon/9be3a58756c2c604573075b6e7531f13f5ba7c1c/builder/index/libs/junit-4.12.jar -------------------------------------------------------------------------------- /builder/index/libs/juniversalchardet-1.0.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/poseidon/9be3a58756c2c604573075b6e7531f13f5ba7c1c/builder/index/libs/juniversalchardet-1.0.3.jar -------------------------------------------------------------------------------- /builder/index/libs/protobuf-java-3.0.0-beta-2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/poseidon/9be3a58756c2c604573075b6e7531f13f5ba7c1c/builder/index/libs/protobuf-java-3.0.0-beta-2.jar -------------------------------------------------------------------------------- /builder/index/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'index' 2 | includeFlat 'indexmeta' 3 | includeFlat 'docmeta' 4 | 5 | -------------------------------------------------------------------------------- /builder/index/src/main/bin/demo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -lt 1 ]; 4 | then 5 | echo $0 logfile 6 | echo usage: $0 ./weibo.txt 7 | exit 8 | fi 9 | 10 | pwd=$(pwd) 11 | 12 | day=`date -d "24 hours ago" +"%Y-%m-%d"` 13 | hour=`date -d "24 hours ago" +"%H"` 14 | timestamp=`date -d "24 hours ago" +"%Y%m%d%H%M%S"` 15 | 16 | cp -rf $1 /home/poseidon/data/log${timestamp}_$day-$hour.txt 17 | 18 | echo wait for file in /home/poseidon/src/test/$day 19 | echo and then you should run: 20 | echo /bin/bash $pwd/bin/mock_start.sh $day 21 | 22 | -------------------------------------------------------------------------------- /builder/index/src/main/etc/IKAnalyzer.cfg.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | IK Analyzer 扩展配置 5 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /builder/index/src/main/etc/test.json: -------------------------------------------------------------------------------- 1 | { 2 | "common": { 3 | "bussiness": "test", 4 | "name_node": "127.0.0.1:9000", 5 | "hadoop_user": "root", 6 | "hdpfs_index_base_path": "/home/poseidon/src", 7 | "data_hashid_section": 10000000000, 8 | "line_per_doc": 128, 9 | "indexgzmeta_section": 200, 10 | "indexgz_file_num_per_field": 1000, 11 | "meta_service": "127.0.0.1:39610", 12 | "local_mock": true 13 | }, 14 | "inverted_index": { 15 | "save_line_count_per_map": 100000, 16 | "line_average_length": 850, 17 | "data_format": "TAB", 18 | "daily_index": false, 19 | "daily_data_size": "10TB", 20 | "log_dir_level": 1, 21 | "hdfs_path": "/home/poseidon/src/test", 22 | "hdfs_cluster": "idc-name", 23 | "hdfs_path_date_pattern": "YYYY-MM-DD", 24 | "hdfs_file_pattern": "gz", 25 | "cache_size": 16777216, 26 | "debug": 0, 27 | "is_total": true, 28 | "etcs": "IKAnalyzer.cfg.xml", 29 | "libs": "", 30 | "field_names": [ 31 | "text" 32 | ], 33 | "tokenizer": { 34 | "text": [ 35 | { 36 | "split": "\\|\\|" 37 | }, 38 | "text", 39 | "urlencode", 40 | "keyword" 41 | ] 42 | }, 43 | "alias": { 44 | "text": "text", 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /builder/index/src/main/etc/test_hdfs.json: -------------------------------------------------------------------------------- 1 | { 2 | "common": { 3 | "bussiness": "test", 4 | "name_node": "localhost:9000", 5 | "hadoop_user": "root", 6 | "hdpfs_index_base_path": "/home/poseidon/src", 7 | "data_hashid_section": 10000000000, 8 | "line_per_doc": 128, 9 | "indexgzmeta_section": 200, 10 | "indexgz_file_num_per_field": 1000, 11 | "meta_service": "127.0.0.1:39610", 12 | "local_mock": false 13 | }, 14 | "inverted_index": { 15 | "save_line_count_per_map": 100000, 16 | "line_average_length": 850, 17 | "data_format": "TAB", 18 | "daily_index": false, 19 | "daily_data_size": "10TB", 20 | "log_dir_level": 1, 21 | "hdfs_path": "/home/poseidon/src/test", 22 | "hdfs_cluster": "idc-name", 23 | "hdfs_path_date_pattern": "YYYY-MM-DD", 24 | "hdfs_file_pattern": "gz", 25 | "cache_size": 16777216, 26 | "debug": 0, 27 | "is_total": true, 28 | "etcs": "IKAnalyzer.cfg.xml", 29 | "libs": "indexmeta-0.1.jar,json.jar,ikanalyzer2012ff.jar,protobuf-java-3.1.0.jar,index-0.1.jar", 30 | "field_names": [ 31 | "text" 32 | ], 33 | "tokenizer": { 34 | "text": [ 35 | { 36 | "split": "\\|\\|" 37 | }, 38 | "text", 39 | "urlencode", 40 | "keyword" 41 | ] 42 | }, 43 | "alias": { 44 | "text": "text" 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /builder/index/src/main/java/InvertedIndex/plugin/Function/Base64DecodeFunction.java: -------------------------------------------------------------------------------- 1 | package InvertedIndex.plugin.Function; 2 | 3 | import InvertedIndex.plugin.Util; 4 | 5 | import java.util.HashSet; 6 | import java.util.Set; 7 | 8 | /** 9 | * Created by i-xuchongpeng on 2016/8/11. 10 | */ 11 | public class Base64DecodeFunction implements Function { 12 | public Set Process(String input) { 13 | Set set = new HashSet(); 14 | set.add(Util.Base64DecoderStr(input, false)); 15 | return set; 16 | } 17 | 18 | public Set Process(Set input) { 19 | Set set = new HashSet(); 20 | for (String s : input) { 21 | set.add(Util.Base64DecoderStr(s, false)); 22 | } 23 | return set; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /builder/index/src/main/java/InvertedIndex/plugin/Function/FilterFunction.java: -------------------------------------------------------------------------------- 1 | package InvertedIndex.plugin.Function; 2 | 3 | //import InvertedIndex.plugin.Function.*; 4 | 5 | import java.util.HashSet; 6 | import java.util.Set; 7 | 8 | /** 9 | * Created by i-xuchongpeng on 2016/8/11. 10 | */ 11 | public class FilterFunction implements Function { 12 | private TokenFilter filter_; 13 | 14 | public FilterFunction(TokenFilter filters) { 15 | filter_ = filters; 16 | } 17 | 18 | public Set Process(String input) { 19 | Set set = new HashSet<>(); 20 | if (filter_.PassBy(input)) set.add(input); 21 | return set; 22 | } 23 | 24 | public Set Process(Set input) { 25 | Set set = new HashSet<>(); 26 | for (String s : input) { 27 | if (filter_.PassBy(s)) set.add(s); 28 | } 29 | return set; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /builder/index/src/main/java/InvertedIndex/plugin/Function/Function.java: -------------------------------------------------------------------------------- 1 | package InvertedIndex.plugin.Function; 2 | 3 | import java.util.Set; 4 | 5 | /** 6 | * Created by liwei on 6/8/16. 7 | */ 8 | 9 | public interface Function { 10 | public Set Process(String input); 11 | 12 | public Set Process(Set input); 13 | 14 | 15 | } 16 | -------------------------------------------------------------------------------- /builder/index/src/main/java/InvertedIndex/plugin/Function/IpFunction.java: -------------------------------------------------------------------------------- 1 | package InvertedIndex.plugin.Function; 2 | 3 | import InvertedIndex.plugin.Util; 4 | 5 | import java.util.HashSet; 6 | import java.util.Set; 7 | 8 | /** 9 | * Created by i-xuchongpeng on 2016/8/11. 10 | */ 11 | public class IpFunction implements Function { 12 | public Set Process(String input) { 13 | Set set = new HashSet(); 14 | set.addAll(Util.ParseIp(input)); 15 | return set; 16 | } 17 | 18 | public Set Process(Set input) { 19 | Set set = new HashSet(); 20 | for (String s : input) { 21 | set.addAll(Util.ParseIp(s)); 22 | } 23 | return set; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /builder/index/src/main/java/InvertedIndex/plugin/Function/KeywordFunction.java: -------------------------------------------------------------------------------- 1 | package InvertedIndex.plugin.Function; 2 | 3 | 4 | /** 5 | * Created by i-xuchongpeng on 2016/8/11. 6 | */ 7 | public class KeywordFunction extends LowerCaseFunction { 8 | } 9 | -------------------------------------------------------------------------------- /builder/index/src/main/java/InvertedIndex/plugin/Function/LowerCaseFunction.java: -------------------------------------------------------------------------------- 1 | package InvertedIndex.plugin.Function; 2 | 3 | import java.util.HashSet; 4 | import java.util.Set; 5 | 6 | /** 7 | * Created by i-xuchongpeng on 2016/8/11. 8 | */ 9 | public class LowerCaseFunction implements Function { 10 | public Set Process(String input) { 11 | Set set = new HashSet(); 12 | set.add(input.trim().toLowerCase()); 13 | return set; 14 | } 15 | 16 | public Set Process(Set input) { 17 | Set set = new HashSet(); 18 | for (String s : input) { 19 | set.add(s.trim().toLowerCase()); 20 | } 21 | return set; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /builder/index/src/main/java/InvertedIndex/plugin/Function/PathFunction.java: -------------------------------------------------------------------------------- 1 | package InvertedIndex.plugin.Function; 2 | 3 | import InvertedIndex.plugin.Util; 4 | 5 | import java.util.HashSet; 6 | import java.util.Set; 7 | 8 | /** 9 | * Created by i-xuchongpeng on 2016/8/11. 10 | */ 11 | public class PathFunction implements Function { 12 | public Set Process(String input) { 13 | Set set = new HashSet(); 14 | Util.ParsePath(input, set); 15 | return set; 16 | } 17 | 18 | public Set Process(Set input) { 19 | Set set = new HashSet(); 20 | for (String s : input) { 21 | Util.ParsePath(s, set); 22 | } 23 | return set; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /builder/index/src/main/java/InvertedIndex/plugin/Function/PathWithUrlencodeFunction.java: -------------------------------------------------------------------------------- 1 | package InvertedIndex.plugin.Function; 2 | 3 | import java.util.HashSet; 4 | import java.util.Set; 5 | 6 | /** 7 | * Created by i-xuchongpeng on 2016/8/11. 8 | */ 9 | public class PathWithUrlencodeFunction implements Function { 10 | public Set Process(String input) { 11 | Set set = new HashSet(); 12 | if (input == null || input.isEmpty()) { 13 | return set; 14 | } 15 | String fpath = input.trim().toLowerCase(); 16 | if (fpath.isEmpty()) { 17 | return set; 18 | } 19 | 20 | if (!fpath.isEmpty()) { 21 | String[] fpath_parts = fpath.split("%5c"); 22 | for (int i = 0; i < fpath_parts.length; i++) { 23 | if (fpath_parts[i].isEmpty()) { 24 | continue; 25 | } 26 | if (!set.contains(fpath_parts[i])) { 27 | set.add(fpath_parts[i]); 28 | } 29 | } 30 | if (!set.contains(fpath)) { 31 | set.add(fpath); 32 | } 33 | } 34 | return set; 35 | } 36 | 37 | public Set Process(Set input) { 38 | 39 | Set set = new HashSet(); 40 | for (String s : input) { 41 | if (s == null || s.isEmpty()) { 42 | continue; 43 | } 44 | String fpath = s.trim().toLowerCase(); 45 | if (fpath.isEmpty()) { 46 | continue; 47 | } 48 | if (!fpath.isEmpty()) { 49 | String[] fpath_parts = fpath.split("%5c"); 50 | for (int i = 0; i < fpath_parts.length; i++) { 51 | if (fpath_parts[i].isEmpty()) { 52 | continue; 53 | } 54 | if (!set.contains(fpath_parts[i])) { 55 | set.add(fpath_parts[i]); 56 | } 57 | } 58 | if (!set.contains(fpath)) { 59 | set.add(fpath); 60 | } 61 | } 62 | } 63 | 64 | return set; 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /builder/index/src/main/java/InvertedIndex/plugin/Function/RegexCheck.java: -------------------------------------------------------------------------------- 1 | package InvertedIndex.plugin.Function; 2 | 3 | import InvertedIndex.plugin.Util; 4 | 5 | import java.util.HashSet; 6 | import java.util.Set; 7 | 8 | /** 9 | * Created by i-xuchongpeng on 2016/8/11. 10 | */ 11 | public class RegexCheck implements Function { 12 | private String pattern = null; 13 | 14 | public RegexCheck(String pattern) { 15 | if (pattern != null) { 16 | this.pattern = Util.Base64DecoderStr(pattern, false); 17 | } 18 | } 19 | 20 | protected String process(String input) { 21 | if (this.pattern == null) 22 | return input; 23 | if (input.matches(pattern)) 24 | return input; 25 | return null; 26 | } 27 | 28 | public Set Process(String input) { 29 | Set result = new HashSet(); 30 | String s = process(input); 31 | if (s != null) 32 | result.add(s); 33 | return result; 34 | } 35 | 36 | public Set Process(Set inputs) { 37 | Set result = new HashSet(); 38 | for (String input : inputs) { 39 | String s = process(input); 40 | if (s != null) 41 | result.add(s); 42 | } 43 | return result; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /builder/index/src/main/java/InvertedIndex/plugin/Function/SplitFunction.java: -------------------------------------------------------------------------------- 1 | package InvertedIndex.plugin.Function; 2 | 3 | import java.util.HashSet; 4 | import java.util.Set; 5 | 6 | /** 7 | * Created by i-xuchongpeng on 2016/8/11. 8 | */ 9 | public class SplitFunction implements Function { 10 | private String sep_; 11 | 12 | public SplitFunction(String sep) { 13 | sep_ = sep; 14 | } 15 | 16 | public Set Process(String input) { 17 | Set set = new HashSet(); 18 | String[] vals = input.split(sep_); 19 | for (int i = 0; i < vals.length; i++) { 20 | set.add(vals[i]); 21 | } 22 | return set; 23 | } 24 | 25 | public Set Process(Set input) { 26 | Set set = new HashSet(); 27 | for (String s : input) { 28 | String[] vals = s.split(sep_); 29 | for (int i = 0; i < vals.length; i++) { 30 | set.add(vals[i]); 31 | } 32 | } 33 | return set; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /builder/index/src/main/java/InvertedIndex/plugin/Function/TokenFilter.java: -------------------------------------------------------------------------------- 1 | package InvertedIndex.plugin.Function; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.FileReader; 5 | import java.security.PublicKey; 6 | import java.util.HashSet; 7 | import java.util.Set; 8 | 9 | /** 10 | * Created by i-xuchongpeng on 2016/8/11. 11 | */ //过滤器对象，内置黑名单和过滤概率发生器 12 | //0 表示全部过滤 13 | //非0值precision_,表示按1/precision_过滤输出 14 | public class TokenFilter { 15 | public Set blacklist_ = new HashSet<>(); 16 | private int precision_; 17 | private int worker_count_ = 0; 18 | 19 | public boolean Worker() { 20 | 21 | worker_count_ += 1; 22 | worker_count_ %= precision_; 23 | //System.err.println(worker_count_); 24 | return (worker_count_ == 5); 25 | } 26 | 27 | public TokenFilter(String file_name, int precision) { 28 | precision_ = precision; 29 | loadFilterTokenFile(file_name); 30 | } 31 | 32 | public void loadFilterTokenFile(String file_name) { 33 | try { 34 | //System.out.println(TokenFilter.class.getResource("/").getFile()); 35 | BufferedReader fin = new BufferedReader(new FileReader(file_name)); 36 | String ss = fin.readLine(); 37 | while (ss != null) { 38 | blacklist_.add(ss); 39 | ss = fin.readLine(); 40 | } 41 | } catch (Exception e) { 42 | System.err.println("load " + file_name + " failed :" + e.toString()); 43 | e.printStackTrace(); 44 | System.exit(-1); 45 | } 46 | } 47 | 48 | //for test 49 | public void get() { 50 | 51 | } 52 | 53 | public boolean PassBy(String token) { 54 | if (!blacklist_.contains(token)) { 55 | //System.err.println(token + " not in black~~~~~~`"); 56 | return true; 57 | } 58 | if (precision_ == 0) return false; 59 | return Worker(); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /builder/index/src/main/java/InvertedIndex/plugin/Function/UrlDecodeFunction.java: -------------------------------------------------------------------------------- 1 | package InvertedIndex.plugin.Function; 2 | 3 | import java.util.HashSet; 4 | import java.util.Set; 5 | 6 | /** 7 | * Created by i-xuchongpeng on 2016/8/11. 8 | */ 9 | public class UrlDecodeFunction implements Function { 10 | public Set Process(String input) { 11 | Set set = new HashSet(); 12 | try { 13 | set.add(java.net.URLDecoder.decode(input)); 14 | } catch (Exception e) { 15 | } 16 | return set; 17 | } 18 | 19 | public Set Process(Set input) { 20 | Set set = new HashSet(); 21 | for (String s : input) { 22 | try { 23 | set.add(java.net.URLDecoder.decode(s)); 24 | } catch (Exception e) { 25 | } 26 | } 27 | return set; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /builder/index/src/main/java/InvertedIndex/plugin/Function/UrlEncode.java: -------------------------------------------------------------------------------- 1 | package InvertedIndex.plugin.Function; 2 | 3 | import java.util.HashSet; 4 | import java.util.Set; 5 | 6 | /** 7 | * Created by i-xuchongpeng on 2016/8/11. 8 | */ 9 | public class UrlEncode implements Function { 10 | public Set Process(String input) { 11 | Set set = new HashSet(); 12 | set.add(java.net.URLEncoder.encode(input).toLowerCase()); 13 | return set; 14 | } 15 | 16 | public Set Process(Set input) { 17 | Set set = new HashSet(); 18 | for (String s : input) { 19 | set.add(java.net.URLEncoder.encode(s).toLowerCase()); 20 | } 21 | return set; 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /builder/index/src/main/java/InvertedIndex/plugin/Function/UrlEncodeFunction.java: -------------------------------------------------------------------------------- 1 | package InvertedIndex.plugin.Function; 2 | 3 | import java.net.URLEncoder; 4 | import java.util.HashSet; 5 | import java.util.Set; 6 | 7 | /** 8 | * Created by i-xuchongpeng on 2016/8/11. 9 | */ 10 | public class UrlEncodeFunction implements Function { 11 | protected String process(String input) { 12 | try { 13 | return URLEncoder.encode(input, "utf-8"); 14 | } catch (Exception e) { 15 | try { 16 | return URLEncoder.encode(input, "gb18030"); 17 | } catch (Exception e1) { 18 | } 19 | } 20 | return input; 21 | } 22 | 23 | public Set Process(String input) { 24 | Set set = new HashSet(); 25 | String str = process(input); 26 | set.add(str); 27 | return set; 28 | } 29 | 30 | public Set Process(Set input) { 31 | Set set = new HashSet(); 32 | for (String s : input) { 33 | String str = process(s); 34 | set.add(str); 35 | } 36 | return set; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /builder/index/src/main/java/InvertedIndex/plugin/Function/UrlFunction.java: -------------------------------------------------------------------------------- 1 | package InvertedIndex.plugin.Function; 2 | 3 | import InvertedIndex.plugin.Util; 4 | 5 | import java.util.HashSet; 6 | import java.util.Set; 7 | 8 | /** 9 | * Created by i-xuchongpeng on 2016/8/11. 10 | */ 11 | public class UrlFunction implements Function { 12 | public Set Process(String input) { 13 | Set set = new HashSet(); 14 | Util.ParseUrl(input, set); 15 | return set; 16 | } 17 | 18 | public Set Process(Set input) { 19 | Set set = new HashSet(); 20 | for (String s : input) { 21 | Util.ParseUrl(s, set); 22 | } 23 | return set; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /builder/index/src/main/java/InvertedIndex/plugin/LogParserFactory.java: -------------------------------------------------------------------------------- 1 | package InvertedIndex.plugin; 2 | 3 | import InvertedIndex.LogParser; 4 | import org.apache.hadoop.conf.Configuration; 5 | 6 | import java.util.Set; 7 | 8 | /** 9 | * Created by liwei on 6/8/16. 10 | */ 11 | public class LogParserFactory { 12 | 13 | private void init(Configuration conf, CommonLogParser p) { 14 | String fields = new String(); 15 | for (String f : p.fields()) { 16 | if (!fields.isEmpty()) { 17 | fields += ","; 18 | } 19 | fields += f; 20 | } 21 | System.err.println("fileds:" + fields); 22 | conf.set("fields", fields); 23 | } 24 | 25 | public void Init(Configuration conf) { 26 | String json_str = conf.get("json_conf"); 27 | if (json_str == null || json_str.isEmpty()) { 28 | System.err.print("json_conf is null"); 29 | return; 30 | } 31 | CommonLogParser p = new CommonLogParser(conf, json_str, false); 32 | init(conf, p); 33 | conf.set("filterfiles", p.Filterfiles()); 34 | } 35 | 36 | public LogParser Create(Configuration conf) { 37 | LogParser logparser = null; 38 | String log_name = conf.get("log_name", new String("common")); 39 | String json_str = conf.get("json_conf"); 40 | //json_str = Util.Base64DecoderStr(json_str, false); 41 | CommonLogParser p = new CommonLogParser(conf, json_str, true); 42 | p.set_debug(conf.getInt("_debug_", 0)); 43 | init(conf, p); 44 | logparser = p; 45 | 46 | return logparser; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /builder/index/src/test/java/FilterFunctionTest.java: -------------------------------------------------------------------------------- 1 | import InvertedIndex.plugin.Function.FilterFunction; 2 | 3 | import InvertedIndex.plugin.Function.TokenFilter; 4 | 5 | import org.junit.Assert; 6 | import org.junit.Before; 7 | import org.junit.BeforeClass; 8 | import org.junit.Test; 9 | import org.junit.runner.RunWith; 10 | import org.junit.runners.Parameterized; 11 | 12 | import java.util.*; 13 | 14 | /** 15 | * Created by i-xuchongpeng on 2016/8/11. 16 | */ 17 | @RunWith(Parameterized.class) 18 | public class FilterFunctionTest { 19 | 20 | @BeforeClass 21 | public static void setup() { 22 | } 23 | 24 | private FilterFunction Filter; 25 | private Set input_set = new HashSet<>(); 26 | private String input = "token_text1"; 27 | private int result; 28 | 29 | public FilterFunctionTest(String filter_name, int precision, int result) { 30 | this.Filter = new FilterFunction(new TokenFilter(filter_name, precision)); 31 | this.input_set.add("token_text1"); 32 | this.input_set.add("token_text2"); 33 | this.input_set.add("token_text3"); 34 | this.result = result; 35 | //System.out.println(Filter.filter_.blacklist_); 36 | } 37 | 38 | @Parameterized.Parameters 39 | public static Collection data() { 40 | return Arrays.asList(new Object[][]{ 41 | {"src/test/resources/filter1", 10, 1000}, 42 | {"src/test/resources/filter1", 100, 100}, 43 | {"src/test/resources/filter1", 1000, 10}, 44 | {"src/test/resources/filter2", 10, 10000}, 45 | {"src/test/resources/filter2", 100, 10000} 46 | }); 47 | } 48 | 49 | @Test 50 | public void process() throws Exception { 51 | int res = 0; 52 | 53 | for (int i = 1; i < 10001; i++) { 54 | res += Filter.Process(input).size(); 55 | } 56 | 57 | Assert.assertEquals(result, res); 58 | } 59 | 60 | @Test 61 | public void process1() throws Exception { 62 | int res = 0; 63 | 64 | for (int i = 1; i < 10001; i++) { 65 | res += Filter.Process(input_set).size(); 66 | } 67 | Assert.assertEquals(result * 3, res); 68 | } 69 | 70 | } -------------------------------------------------------------------------------- /builder/index/src/test/java/TokenFilterTest.java: -------------------------------------------------------------------------------- 1 | import InvertedIndex.plugin.Function.TokenFilter; 2 | import org.junit.Assert; 3 | import org.junit.Test; 4 | 5 | import static org.junit.Assert.*; 6 | 7 | /** 8 | * Created by i-xuchongpeng on 2016/8/11. 9 | */ 10 | public class TokenFilterTest { 11 | @Test 12 | public void worker() throws Exception { 13 | 14 | } 15 | 16 | @Test 17 | public void loadFilterTokenFile() throws Exception { 18 | TokenFilter test1 = new TokenFilter("src/test/resources/filter1", 10); 19 | /* 20 | System.out.println(test1.blacklist_); 21 | if (test1.PassBy("token_test")) System.out.println("1"); 22 | System.out.println(test1.getworkcount()); 23 | if (test1.PassBy("token_test1")) System.out.println("2"); 24 | System.out.println(test1.getworkcount()); 25 | if (test1.PassBy("token_test1")) System.out.println("3"); 26 | System.out.println(test1.getworkcount()); 27 | if (test1.PassBy("token_test1")) System.out.println("4"); 28 | System.out.println(test1.getworkcount()); 29 | if (test1.PassBy("token_test1")) System.out.println("5"); 30 | System.out.println(test1.getworkcount()); 31 | //Assert.assertEquals(test1.blacklist_.size(),3); 32 | */ 33 | } 34 | 35 | @Test 36 | public void passby() throws Exception { 37 | System.err.println(Test.class.getResource("/").getFile()); 38 | } 39 | 40 | } -------------------------------------------------------------------------------- /builder/index/src/test/resources/filter1: -------------------------------------------------------------------------------- 1 | token_text1 2 | token_text2 3 | token_text3 -------------------------------------------------------------------------------- /builder/index/src/test/resources/filter2: -------------------------------------------------------------------------------- 1 | token_text4 2 | token_text5 -------------------------------------------------------------------------------- /builder/indexmeta/build.gradle: -------------------------------------------------------------------------------- 1 | group 'indexmeta' 2 | version '0.1' 3 | 4 | apply plugin: 'java' 5 | 6 | sourceCompatibility = 1.7 7 | 8 | repositories { 9 | mavenCentral() 10 | } 11 | 12 | def var 13 | var = dependencies { 14 | compile group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion 15 | compile group: 'org.apache.hadoop', name: 'hadoop-mapreduce-client-core', version: hadoopVersion 16 | compile group: 'org.apache.hadoop', name: 'hadoop-mapreduce-client-common', version: hadoopVersion 17 | compile group: 'org.apache.hadoop', name: 'hadoop-mapreduce-client-jobclient', version: hadoopVersion 18 | compile group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: '2.7.1' 19 | 20 | compile group: 'commons-configuration', name: 'commons-configuration', version: '1.9' 21 | compile group: 'org.apache.commons', name: 'commons-compress', version: '1.4.1' 22 | compile group: 'com.google.protobuf', name: 'protobuf-java', version: '3.1.0' 23 | 24 | //compile fileTree(dir: 'libs', include: 'hadoop-0.20.2.1U11-core.jar') 25 | //compile fileTree(dir: '../index/build/libs', include: 'index-0.1-SNAPSHOT.jar') 26 | compile files('../index/build/libs/index-' + version + '.jar') 27 | 28 | compile fileTree(dir: '../index/libs', include: 'json.jar') 29 | 30 | 31 | testCompile group: 'junit', name: 'junit', version: '4.12' 32 | } 33 | var 34 | -------------------------------------------------------------------------------- /builder/indexmeta/src/main/etc/test.json: -------------------------------------------------------------------------------- 1 | { 2 | "common": { 3 | "bussiness": "test", 4 | "name_node": "127.0.0.1:9000", 5 | "hadoop_user": "root", 6 | "hdpfs_index_base_path": "/home/poseidon/src", 7 | "data_hashid_section": 10000000000, 8 | "line_per_doc": 128, 9 | "indexgzmeta_section": 200, 10 | "indexgz_file_num_per_field": 1000, 11 | "meta_service": "127.0.0.1:39610" 12 | }, 13 | "inverted_index": { 14 | "name": "ck_ad", 15 | "save_line_count_per_map": 100000, 16 | "line_average_length": 850, 17 | "data_format": "TAB", 18 | "daily_index": false, 19 | "daily_data_size": "10TB", 20 | "log_dir_level": 1, 21 | "hdfs_path": "/home/poseidon/src/test", 22 | "hdfs_cluster": "idc-name", 23 | "hdfs_path_date_pattern": "YYYY-MM-DD", 24 | "hdfs_file_pattern": "gz", 25 | "cache_size": 16777216, 26 | "debug": 0, 27 | "is_total": true, 28 | "etcs": "", 29 | "libs": "", 30 | "field_names": [ 31 | "text" 32 | ], 33 | "tokenizer": { 34 | "text": [ 35 | { 36 | "split": "\\|\\|" 37 | }, 38 | "urlencode", 39 | "keyword" 40 | ] 41 | }, 42 | "alias": { 43 | "text": "text" 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /builder/indexmeta/src/main/etc/test2.json: -------------------------------------------------------------------------------- 1 | { 2 | "common": { 3 | "bussiness": "test", 4 | "name_node": "127.0.0.1:9000", 5 | "hadoop_user": "root", 6 | "hdpfs_index_base_path": "/home/poseidon/src", 7 | "data_hashid_section": 10000000000, 8 | "line_per_doc": 128, 9 | "indexgzmeta_section": 200, 10 | "indexgz_file_num_per_field": 1000, 11 | "meta_service": "127.0.0.1:39610" 12 | }, 13 | "inverted_index": { 14 | "name": "ck_ad", 15 | "save_line_count_per_map": 100000, 16 | "line_average_length": 850, 17 | "data_format": "TAB", 18 | "daily_index": false, 19 | "daily_data_size": "10TB", 20 | "log_dir_level": 1, 21 | "hdfs_path": "/home/poseidon/src/test", 22 | "hdfs_cluster": "idc-name", 23 | "hdfs_path_date_pattern": "YYYY-MM-DD", 24 | "hdfs_file_pattern": "gz", 25 | "cache_size": 16777216, 26 | "debug": 0, 27 | "is_total": true, 28 | "etcs": "IKAnalyzer.cfg.xml", 29 | "libs": "protobuf-java-3.0.0-beta-2.jar,ikanalyzer2012ff.jar,json.jar", 30 | "field_names": [ 31 | "text" 32 | ], 33 | "tokenizer": { 34 | "text": [ 35 | { 36 | "split": "\\|\\|" 37 | }, 38 | "urlencode", 39 | "keyword" 40 | ] 41 | }, 42 | "alias": { 43 | "text": "text" 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /builder/indexmeta/src/main/java/meta/IndexMetaConfigured.java: -------------------------------------------------------------------------------- 1 | package meta; 2 | 3 | import meta.MetaConfigured; 4 | import meta.IndexMetaMapper; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.io.BytesWritable; 8 | import org.apache.hadoop.io.Text; 9 | import org.apache.hadoop.mapreduce.Job; 10 | import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; 11 | import org.apache.hadoop.util.ToolRunner; 12 | 13 | import java.io.IOException; 14 | import java.net.URISyntaxException; 15 | 16 | 17 | /** 18 | * Created by liwei on 9/22/16. 19 | */ 20 | public class IndexMetaConfigured extends MetaConfigured { 21 | public static void main(String[] args) throws IOException, URISyntaxException, ClassNotFoundException, InterruptedException { 22 | int ret = 0; 23 | try { 24 | ret = ToolRunner.run(new IndexMetaConfigured(), args); 25 | } catch (Exception e) { 26 | e.printStackTrace(); 27 | } 28 | System.exit(ret); 29 | } 30 | 31 | @Override 32 | public Job GetMapReduceJob(Configuration conf) { 33 | try { 34 | Job job = new Job(conf, IndexMetaConfigured.class.getSimpleName()); 35 | System.err.println("get index meta job "); 36 | job.setJarByClass(IndexMetaConfigured.class); 37 | job.setInputFormatClass(TextInputFormat.class); 38 | job.setMapperClass(IndexMetaMapper.class); 39 | job.setMapOutputKeyClass(Text.class); 40 | job.setMapOutputValueClass(Text.class); 41 | 42 | job.setOutputKeyClass(Text.class); 43 | job.setOutputValueClass(BytesWritable.class); 44 | return job; 45 | } catch (Exception e) { 46 | 47 | } 48 | return null; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /builder/indexmeta/src/main/java/meta/MetaSetter.java: -------------------------------------------------------------------------------- 1 | package meta; 2 | 3 | import org.apache.commons.httpclient.params.HttpMethodParams; 4 | import org.apache.hadoop.util.hash.Hash; 5 | import org.apache.http.HttpEntity; 6 | import org.apache.http.HttpResponse; 7 | import org.apache.http.client.ClientProtocolException; 8 | import org.apache.http.client.HttpClient; 9 | import org.apache.http.client.methods.HttpGet; 10 | import org.apache.http.client.methods.HttpPatch; 11 | import org.apache.http.client.methods.HttpPost; 12 | import org.apache.http.impl.client.DefaultHttpClient; 13 | import org.apache.http.entity.StringEntity; 14 | import org.apache.http.util.EntityUtils; 15 | 16 | 17 | import java.io.BufferedOutputStream; 18 | import java.io.InputStream; 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | /** 23 | * Created by liwei on 9/23/16. 24 | */ 25 | public class MetaSetter { 26 | 27 | private String url_; 28 | 29 | public MetaSetter(String url) { 30 | url_ = url; 31 | } 32 | 33 | /* 34 | public bool MetaPost(Map meta) { 35 | for(Map.Entry entry : meta.entrySet()) { 36 | } 37 | } 38 | */ 39 | 40 | 41 | public String Post(String body) { 42 | HttpClient httpclient = new DefaultHttpClient(); 43 | String retbody = null; 44 | try { 45 | HttpPost httpPost = new HttpPost(url_); 46 | System.err.println("executing request " + httpPost.getURI()); 47 | httpPost.setEntity(new StringEntity(body)); 48 | 49 | HttpResponse response = httpclient.execute(httpPost); 50 | HttpEntity entity = response.getEntity(); 51 | if (entity != null) { 52 | //System.err.println("Response content length: " + entity.getContentLength()); 53 | retbody = EntityUtils.toString(entity); 54 | //System.err.println(retbody); 55 | } 56 | } catch (Exception e) { 57 | e.printStackTrace(); 58 | } finally { 59 | httpclient.getConnectionManager().shutdown(); 60 | } 61 | return retbody; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /common/proto/.gitignore: -------------------------------------------------------------------------------- 1 | *.pb.go 2 | -------------------------------------------------------------------------------- /common/proto/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ ! -f build.sh ]; then 4 | echo 'build.sh must be run within its container folder' 1>&2 5 | exit 1 6 | fi 7 | 8 | ROOT_DIR=`pwd` 9 | PROTOC_BIN=protoc 10 | 11 | $PROTOC_BIN --proto_path=. --go_out=../../service/searcher/proto/ poseidon_if.proto 12 | $PROTOC_BIN --proto_path=. --java_out=../../builder/index/src/main/java/ poseidon_if.proto 13 | 14 | -------------------------------------------------------------------------------- /common/readme.md: -------------------------------------------------------------------------------- 1 | 基础数据结构的 *protobuf* 定义文件，依赖 protobuf 3.0 以上版本。 2 | 如果需要修改的话，为了使修改的内容生效，可以在当前目录下执行 `sh build.sh`。 3 | 默认情况下，我们已经将 *proto* 文件生成的 Java、Golang 源文件提交到 *git* 仓库中。 4 | -------------------------------------------------------------------------------- /dist/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /docs/component.md: -------------------------------------------------------------------------------- 1 | # 术语解释 2 | * *Log* ：原始日志。 3 | 4 | * *Doc* ：Document的缩写，称为文档。 5 | 是128条（条目数可以配置）原始日志的集合，仍然是明文、按行排列的原始日志数据。 6 | 类似于搜索引擎中的 *文档* 。 7 | 8 | * *DocGz* ：Doc的gz压缩数据。 9 | 10 | * *Token* ：一个分词。 11 | 按照一定的分词算法进行分词分出来的单个元素。 12 | 例如一个汉语词、或一个英文单词、或一个MD5串、或一个文件名，等等。 13 | 14 | * *Field* ：分词的字段类型。字段类型一般结合业务自行定义。 15 | 搜索的时候选择某种字段类型去搜索某个分词的日志信息。 16 | 17 | * *RawLogHDFSFile* ： 存在于hdfs中的原始日志文件，一般是压缩格式。 18 | 19 | * *DocGzHDFSFile* ：存在于hdfs中的日志文件，该文件是由一组 *DocGz* 直接拼接在一起的文件。 20 | 由于gz格式特性，该文件仍然可以直接通过gunzip解压。 21 | 22 | * *DocGzMeta* ： *DocGz* 的元数据信息，包含下列三个字段： 23 | 24 | 25 | message DocGzMeta { 26 | string path = 1; // HDFS路径，实际离线存储的时候会做一定的压缩，例如不存储公共前缀 27 | uint64 offset = 2; // 数据起始偏移量 28 | uint32 length = 3; // 数据长度 29 | } 30 | 31 | * *DocIdList* ： 32 | 一个分词可能会出现多个文档中，每个文档有多行原始数据组成。 33 | 每个关联数据需要 docId、rawIndex 两个信息来描述。 34 | 35 | * *InvertedIndex* ：倒排索引结构，搜索引擎中的核心数据结构，一般包含1000个 *Token* 及其索引信息。 36 | 37 | map index = 1; 38 | 39 | * *InvertedIndexGz* ： *InvertedIndex* 数据结构序列化之后的数据，然后使用gz压缩。 40 | 41 | * *InvertedIndexGzHDFSFile* ：在hdfs上存储的倒排索引结构文件，该文件是由一组 *InvertedIndexGz* 直接拼接在一起的文件。 42 | 43 | * *InvertedIndexGzMeta* ： *InvertedIndexGz* 文件的元数据信息。包含下列几个字段： 44 | 45 | message InvertedIndexGzMeta { 46 | uint64 offset = 1; // 某一个 *InvertedIndexGzMeta* 所在 hdfs 文件中的起始地址偏移量 47 | uint32 length = 2; // *InvertedIndexGzMeta* 的所占数据长度 48 | string path = 3; // 一般存放的是文件名的关键字段信息，完整路径可以根据时间、索引表名、hashid等信息推断出来 49 | // uint32 hashid = 4; // 可以通过 *Token* 进行hash运算计算出来 50 | } 51 | 52 | * *InvertedIndexGzMetaHDFSFile* ：在hdfs上存储的倒排索引结构meta文件，该文件是由一组 *InvertedIndexGzMeta* 直接拼接在一起的文件。 53 | 一般可以将 *InvertedIndexGzMeta* 从 *InvertedIndexGzMetaHDFSFile* 构建到nosql数据库中。 54 | -------------------------------------------------------------------------------- /docs/config.md: -------------------------------------------------------------------------------- 1 | # 配置文件模板说明 2 | 3 | docmeta,index,indexmeta的配置文件采用json格式： 4 | 参考[test.json](/cloudsafeeng/openposeidon/raw/master/builder/index/src/main/etc/test.json) 5 | 6 | ## common 节点配置项说明 7 | >* *name* : 业务名 8 | >* *data_hashid_section* : token的hashid的映射区间，默认10000000000。 9 | >* *line_per_doc* : 每个doc包含多少行log。 10 | >* *indexgzmeta_section* : InvertedIndexMetaGz区间值，每 *indexgzmeta_section* 组合一个InvertedIndexGZ。 11 | >* *indexgz_file_num_per_field* ： 每种字段分成 *indexgz_file_num_per_field* 个 *InvertedIndexGzHDFSFile* 文件, 12 | 最终所有的InvertedIndexGZ会存放到 *indexgz_file_num_per_field* 个 *InvertedIndexGzHDFSFile* 文件中。 13 | >* *meta_service* : meta服务的地址。 14 | >* *local_mock* : 本地模拟运行hadoop任务，不需要hadoop集群环境。 15 | >* *hdpfs_index_base_path* : 构建索引信息所输出的hdpfs根目录。 16 | 17 | ## inverted_index 节点配置项说明 18 | >* *save_line_count_per_map*: 一个Token 能关联的 docid 数量的最大值。 19 | >* *hdfs_path* : 日志进过doc组件转换后的格式化日志路径。 20 | >* *etcs* : MR任务所需的依赖配置文件；文件在index-1.0-SNAPSHOT.tar中的etc目录中。 21 | >* *libs* : MR任务所需的jar lib文件；文件在index-1.0-SNAPSHOT.tar中的lib目录中。 22 | >* *data_format* : 日志数据格式，现在支持3种： 23 | > 24 | >> JSON 25 | >>> tokenizer 中的每个字段必须具有层级的含义，用 . 来分隔层级 ,如“data.sha1”,先找到data节点，再找sha1节点值。 26 | >> 27 | >> TAB 28 | >>> 每个字段都是按顺序排列,字段间'\t'分隔，字段名需要单独指定 field_names ,字段中不能包含 *_* 29 | >> 30 | >> KV 31 | >>> 日志每行多个key-value 对； 32 | >> 33 | > 34 | >* *field_names* : *data_format*为TAB时有效，字段名列表。 35 | >* *tokenizer* : 分词器，对哪些字段指定 *分词规则*。注意，某个字段含有中文或者特殊字符时，建议加上urlencode和keyword。 36 | > 37 | >>分词规则 38 | >> 39 | >>> *base64decode*: 40 | >>> 将字符串base64 decode后输出 41 | >>> 42 | >>> *urldecode*: 43 | >>> 将字符串url decode后输出 44 | >>> 45 | >>> *urlencode*: 46 | >>> 将字符串url encode后输出 47 | >>> 48 | >>> *keyword*: 49 | >>> 将整个字符串当做一个分词后输出，例如日志里面的mid、md5、sha1等等；字符串中有大写字母的会转成小写字母 50 | >>> 51 | >>> *url*: 52 | >>> 将字符串按url进行分词后输出 53 | >>> 54 | >>> *path*: 55 | >>> 将字符串按path进行分词后输出 56 | >>> 57 | >>> *split*: 58 | >>> 将字符串按指定分隔符切分成多个分词后输出，分隔符支持 正则 表达式，注意在正则表达式中有明确含义的字符，需要用 “\\”转意 59 | >>> 60 | >>> *regexcheck*: 61 | >>> 将字符串进行正则匹配检测，要测试的字符串符合该条件输出，否则丢弃，正则表达式需使用base64编码 62 | >>> 63 | >>> *text*: 64 | >>> 将字符串按普通文本进行分词 65 | >> 66 | > 67 | >* *alias*: 字段别名，没有别名的，value为原字段。例如："text": "text" 或者 "text": "t", 68 | 69 | 70 | 71 | 72 | ## 待补充 73 | 74 | 75 | -------------------------------------------------------------------------------- /docs/design_detail.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/poseidon/9be3a58756c2c604573075b6e7531f13f5ba7c1c/docs/design_detail.pdf -------------------------------------------------------------------------------- /docs/doc_builder.md: -------------------------------------------------------------------------------- 1 | # DocBuilder 2 | 3 | ------ 4 | 功能：将需要建索引的文件转换为DocGzHDFSFile及DocGzMeta，上传至hdfs或者本地目录 5 | 6 | # 文件命名规范 7 | 文件命中必须包含2005-01-02-04这种格式的传，用于识别该文件该入到哪个DocGzHDFSFile 8 | 9 | # src_provider 10 | 数据源抽象成src_provider,相当于一个队列,不断吐出需要建索引的文件path给DocBuilder,目前支持监控目录、redis-list、也可以根据接口自行实现其他数据源 11 | 12 | ## 监控目录 13 | 将需要生成索引的原始文件都放到指定的文件夹下，src_provider会监控新文件生成，将文件path按顺序吐给DocBuilder 14 | 15 | 示例配置 16 | ``` 17 | "src_provider":{ 18 | "src_type":"dm", 19 | "dm":{ 20 | "monitor_interval_sec":15, 21 | "pop_last_file":true, 22 | "time_out_ms":1000, 23 | "data_dir":"/home/poseidon/data/logto_hdfs", 24 | "monitor_paths":[ 25 | "/home/poseidon/apps/data1/access", 26 | "/home/poseidon/apps/data2/access", 27 | "/home/poseidon/apps/data3/access", 28 | "/home/poseidon/apps/data4/access" 29 | ] 30 | } 31 | } 32 | ``` 33 | 34 | 参数： 35 | > * monitor_interval_sec: 多长时间扫描一次目录 36 | > * pop_last_file: 由于最后一个文件可能还没有写完，返回是是否pop最后一个文件 37 | > * time_out_ms: 没有新文件时等待时间 38 | > * data_dir: 记录当前吐出的最新的文件，下一次将只返回比该文件更新的文件 39 | > * monitor_paths: 监控的目录，暂不支持递归 40 | 41 | ## redis-list 42 | 将需要建索引的原始文件path都push到redis-list（right_push），src_provider会将文件path按顺序吐给DocBuider 43 | 44 | 示例配置 45 | ``` 46 | "src_provider":{ 47 | "src_type":"redis", 48 | "redis":{ 49 | "host":"127.0.0.1", 50 | "port":6500, 51 | "passwd":"this_is_passwd", 52 | "key":"file-list" 53 | } 54 | }, 55 | ``` 56 | 57 | 参数： 58 | > * key: redis中的key，该key应该是一个redis的list, 里面每一个key都是一个本地文件的path 59 | 60 | # 配置 61 | 需要如下配置项 62 | ``` 63 | "hadoop_remote_dir":"/home/poseidon/business_name", 64 | "hadoop_remote_time_dirs":[ 65 | "Y-m-d", 66 | "H" 67 | ], 68 | 69 | "flush_minute":20, 70 | "max_merge_file_size":1200000000, 71 | "retry_times":3, 72 | "hadoop_cmd":"/usr/bin/hadoop/bin/hadoop", 73 | "gather_minute":20, 74 | ``` 75 | > * hadoop_remote_dir: 入到hdfs的目录，不置该项则不上传 76 | > * hadoop_remote_time_dirs: 起名 77 | > * gather_minute：合并文件名中日期接近的文件，减少文件数 78 | > * flush_minute：只有当前时间超过(文件名中的时间 + flush_minute)时才开始对该进行处理，目的是等待该时间的文件基本都生成以后，尽量将其合并为一个文件 79 | -------------------------------------------------------------------------------- /docs/hdfs_reader.md: -------------------------------------------------------------------------------- 1 | # hdfsReader 2 | 3 | ------ 4 | 5 | 功能：根据给定offset和length读取给定文件path 6 | 7 | # read-hdfs 8 | 9 | * 调用方式： 10 | http://ip:port/read-hdfs?path=...&offset=0&length=1123 11 | 12 | * 参数： 13 | * path：hdfs文件路径,支持归档文件读取 14 | * offset：从offset开始读取 15 | * length：读取 length 个字节 16 | 17 | * example: 18 | * http://ip:port/read-hdfs?path=/home/poseidon/gpdata/2016-08-25/parr-00930.gz&offset=62628504&length=30929 19 | * http://ip:port/read-hdfs?path=har:///home/archive/poseidon/2016-06-04.har/2016-06-04/00/002000_2016-06-04-00-00.gz&offset=319383859&length=13110 20 | 21 | # read-localfs 22 | 23 | * 调用方式： 24 | * http://ip:port/read-localfs?path=...&offset=0&length=1123 25 | 26 | * 参数： 27 | * path：本地文件路径 28 | * offset：从offset开始读取（从0开始） 29 | * length：读取 length 个字节 30 | 31 | * example: 32 | * http://ip:port/read-localfs?path=/tmp/test.gz&offset=62628504&length=30929 33 | -------------------------------------------------------------------------------- /docs/id_generator.md: -------------------------------------------------------------------------------- 1 | ## 全局docid生成器 2 | 3 | 为DocGz提供唯一id编号。 4 | 海量数据为减小存储每天都从0重新生成id，为减少压力每次请求可获得一个id区间。 5 | 6 | ### HTTP GET 请求参数如下： 7 | 8 | * count : 请求分配多少个id 9 | * day : 日期，为可选参数，默认为当前日期。格式：YYYYMMDD 10 | * business_name : 业务名 11 | 12 | 举例： 13 | > * http://domain:port/service/idgenerator?count=135&day=20160229&business_name=test 14 | 15 | ### 返回值为JSON格式 16 | 17 | ```json 18 | { 19 | "errno":0, #错误码 20 | "errmsg":"", #错误信息 21 | "start_index":1, #开始索引 22 | "count": #返回id数量 23 | "time": #服务端当前时间戳 24 | } 25 | ``` 26 | 27 | * errno定义 28 | > * 0 成功 29 | > * 100 系统错误 30 | > * 101 缺少参数 31 | 32 | ### 压缩 33 | 34 | + 采用无损压缩算法-[差分](https://zh.wikipedia.org/zh-sg/%E5%B7%AE%E5%88%86%E7%B7%A8%E7%A2%BC) 35 | + 查看[build_inverted_index.md](build_inverted_index.md)获取更多信息 36 | -------------------------------------------------------------------------------- /docs/meta.md: -------------------------------------------------------------------------------- 1 | # Meta Service 2 | 3 | 针对存放Meta信息的NoSQL服务提供统一的HTTP接口服务。NoSQL服务目前支持下列几种协议类型： 4 | 5 | 1. memcached 6 | 2. redis 7 | 8 | 这里的NoSQL主要用来存储Poseidon系统中倒排索引数据的 *Meta* 信息和原始数据的 *Meta* 信息。 9 | 另外，如果想要达到本项目的目标数据集（数百万亿条），所需的 *Meta* 信息存储量将会达到数十 TB 大小，这不是普通的内存版本的`redis`或`memcached`可以搞定的，这个时候需要的是符合`redis`或`memcached`接口协议的带持久化功能的NoSQL存储，例如 [pika] 等等。这种带持久化存储的NoSQL组件，会将大部分数据存储在普通硬盘中，少量热数据会cache在内存中，并且自带LRU等淘汰算法。 10 | 11 | 额外需要说明一点的是，一般的NoSQL的写入速度都只有几万QPS，最高不过10万+的QPS。目前我们这边的做法是，在NoSQL数据库内部支持一个额外的指令，例如 “load /the/path/to/binary/key-value.db”这种指令，我们预先将key/value数据以某种二进制格式组织成文件，并分发到NoSQL所在的服务器上，然后发一个`load`指令到NoSQL让NoSQL在其进程中直接读取这个二进制文件数据导入到NoSQL中，这种形式的写入速度可做到50万的QPS，甚至更高。 12 | 13 | 14 | 15 | ## 查询数据接口 16 | 17 | * 使用HTTP POST请求 18 | * URI 19 | * /service/meta/business/doc/get : 查询 *business* 业务 *DocGz* 的 *Meta* 信息 20 | * /service/meta/business/index/get : 查询 *business* 业务 *InvertedIndexGz* 的 *Meta* 信息 21 | - URI 上有一个参数 *business_name* ，用于表示不同的业务。这里可以实现不同业务数据，存放到不同的NoSQL实例中。 22 | - 请求body 23 | 24 | key\nkey\nkey 25 | 26 | 说明：可以同时一次性批量查询多个key，用\n分隔。也就是一行一个key 27 | 28 | - 返回body 29 | 30 | key\tbase64_encode(val)\n 31 | 32 | key\tbase64_encode(val)\n 33 | 34 | key\tbase64_encode(val)\n 35 | 36 | 说明:val表示从NoSQL中查询出的值，base64编码，用\t分隔；一行一个结果。 37 | 38 | ## 更新数据接口 39 | 40 | * URI 41 | * /service/meta/business/doc/set : 更新 *business* 业务 *DocGz* 的 *Meta* 信息 42 | * /service/meta/business/index/set : 更新 *business* 业务 *InvertedIndexGz* 的 *Meta* 信息 43 | - URI 上有一个参数 *business_name* ，用于表示不同的业务。这里可以实现不同业务数据，存放到不同的NoSQL实例中。 44 | - 使用HTTP POST请求 45 | - 请求body的报文格式如下： 46 | 47 | key\tbase64_encode(val)\n 48 | 49 | key\tbase64_encode(val)\n 50 | 51 | key\tbase64_encode(val)\n 52 | 53 | 说明:val表示是需要更新NoSQL中的值，base64编码，用\t分隔；一行一个Key/Value对 54 | 55 | - 返回body 56 | 57 | key\tOK\n 58 | 59 | key\terr\n 60 | 61 | 说明:每行是一个key的插入结果。 62 | 63 | OK表示插入成功； 64 | 65 | 非OK表示插入失败，并将具体的错误信息一起返回 66 | 67 | 68 | [pika]:https://github.com/Qihoo360/pika -------------------------------------------------------------------------------- /docs/proxy.md: -------------------------------------------------------------------------------- 1 | ## Proxy 2 | 3 | *Written By Go Language* 4 | 5 | 将多天/多关键字查询分发给不同searcher,并发处理请求,收集数据,进一步处理数据,返回给客户端 6 | 7 | ### Build 8 | ```bash 9 | cd PROXYPATH && sh ./build.sh #在bin目录下生成编译文件,并更新$ROOT/dist 10 | ``` 11 | 12 | ### Run 13 | 应用程序运行控制脚本在 *serverctl* 脚本中 14 | ```bash 15 | cd PROXYPATH && sh ./serverctl (start|restart|stop) # @todo reload 16 | ``` 17 | 18 | ### 环境初始化 19 | 1.根据机房初始化配置文件 20 | 2.初始化日志目录 21 | 3.修改相关目录读写权限 22 | 别担心,上线时,上线脚本会自动执行 *sh ./serverctl envinit* 来自动完成环境的初始化 23 | 24 | 25 | ### Deploy 26 | 所有的上线部署脚本都在 *$GOPATH/tools* 目录下 27 | 在 *conf/conf.sh* 中配置测试集群和线上集群,每台server按照回车符分割 28 | ```bash 29 | online_cluster="" 30 | beta_cluster="" 31 | ``` 32 | 你的开发机要开通与线上机器的ssh信任，要上线使用的用户名在上线脚本中配置 33 | *conf/conf.sh* 34 | ```bash 35 | # 部署使用的账号 36 | SSH_USER="" 37 | ``` 38 | 脚本会先把要上线的文件拷贝到线上服务器 然后自动 执行 *sh ./serverctl restart* 重启服务，注意这个过程不是热重启，会丢掉部分请求 39 | 40 | ```bash 41 | cd PROXYPATH && sh ./tools/deploy-beta.sh file #上线测试环境 42 | cd PROXYPATH && sh ./tools/deploy-package.sh #第一次上线 43 | cd PROXYPATH && sh ./tools/deploy-release.sh file #第一次上线后其他上线单独的配置文件或者应用程序 44 | ``` 45 | 46 | ### Logs 47 | 你可以通过conf目录下的 *app.conf.base* 配置日志的错误级别和日志路径,日志根路径默认放在项目根目录下logs目录,日志按天自动分割 48 | ```bash 49 | #日志文件根目录 50 | log_root = ../logs 51 | 52 | #日志level, 0:debug 1:access 2:warning 3:error 53 | log_level = 0 54 | ``` -------------------------------------------------------------------------------- /docs/searcher.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/poseidon/9be3a58756c2c604573075b6e7531f13f5ba7c1c/docs/searcher.jpg -------------------------------------------------------------------------------- /docs/searcher.md: -------------------------------------------------------------------------------- 1 | ## 查询引擎 2 | *Written by Go Language* 3 | 4 | ### Step 5 | #### 1.fetch InvertedIndex Meta 6 | #### 2.fetch DocIdList 7 | #### 3.filter docIdlist 8 | #### 4.get DocGzMeta 9 | #### 5.get DocGz 10 | 11 | ### Architecture For Searcher & Proxy 12 | ![arch](searcher.jpg) 13 | -------------------------------------------------------------------------------- /service/Godeps/Godeps.json: -------------------------------------------------------------------------------- 1 | { 2 | "ImportPath": "github.com/Qihoo360/poseidon/service", 3 | "GoVersion": "go1.7", 4 | "GodepVersion": "v74", 5 | "Deps": [ 6 | { 7 | "ImportPath": "github.com/bradfitz/gomemcache/memcache", 8 | "Comment": "release.r60-41-gfb1f79c", 9 | "Rev": "fb1f79c6b65acda83063cbc69f6bba1522558bfc" 10 | }, 11 | { 12 | "ImportPath": "github.com/deckarep/golang-set", 13 | "Comment": "v1.5-4-g1f0f4ff", 14 | "Rev": "1f0f4ff8d3fbef9328522993ce71c35890f67554" 15 | }, 16 | { 17 | "ImportPath": "github.com/garyburd/redigo/internal", 18 | "Rev": "4ed1111375cbeb698249ffe48dd463e9b0a63a7a" 19 | }, 20 | { 21 | "ImportPath": "github.com/garyburd/redigo/redis", 22 | "Rev": "4ed1111375cbeb698249ffe48dd463e9b0a63a7a" 23 | }, 24 | { 25 | "ImportPath": "github.com/golang/glog", 26 | "Rev": "23def4e6c14b4da8ac2ed8007337bc5eb5007998" 27 | }, 28 | { 29 | "ImportPath": "github.com/golang/protobuf/proto", 30 | "Rev": "7cc19b78d562895b13596ddce7aafb59dd789318" 31 | }, 32 | { 33 | "ImportPath": "github.com/gorilla/context", 34 | "Comment": "v1.1-2-ga8d44e7", 35 | "Rev": "a8d44e7d8e4d532b6a27a02dd82abb31cc1b01bd" 36 | }, 37 | { 38 | "ImportPath": "github.com/gorilla/mux", 39 | "Comment": "v1.1-7-g9c19ed5", 40 | "Rev": "9c19ed558d5df4da88e2ade9c8940d742aef0e7e" 41 | }, 42 | { 43 | "ImportPath": "github.com/spaolacci/murmur3", 44 | "Rev": "0d12bf811670bf6a1a63828dfbd003eded177fce" 45 | }, 46 | { 47 | "ImportPath": "github.com/zieckey/dbuf", 48 | "Rev": "406bb753d347836acd203d9f59a427003bd1425b" 49 | }, 50 | { 51 | "ImportPath": "github.com/zieckey/goini", 52 | "Comment": "release-1.1-15-g4a0b7ae", 53 | "Rev": "4a0b7ae0996c59823dfb6d626738733a083da3b1" 54 | }, 55 | { 56 | "ImportPath": "github.com/zieckey/simgo", 57 | "Rev": "a03dd2ed7afe2add806fcb9376d9ca21e9e836e8" 58 | } 59 | ] 60 | } 61 | -------------------------------------------------------------------------------- /service/Godeps/Readme: -------------------------------------------------------------------------------- 1 | This directory tree is generated automatically by godep. 2 | 3 | Please do not edit. 4 | 5 | See https://github.com/tools/godep for more information. 6 | -------------------------------------------------------------------------------- /service/allinone/allinone.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | idgenerator "github.com/Qihoo360/poseidon/service/idgenerator/module" 5 | meta "github.com/Qihoo360/poseidon/service/meta/module" 6 | proxy "github.com/Qihoo360/poseidon/service/proxy/module" 7 | searcher "github.com/Qihoo360/poseidon/service/searcher/module" 8 | "github.com/zieckey/simgo" 9 | ) 10 | 11 | func main() { 12 | fw := simgo.DefaultFramework 13 | fw.RegisterModule("meta", meta.New()) 14 | fw.RegisterModule("idgenerator", idgenerator.New()) 15 | fw.RegisterModule("proxy", proxy.New()) 16 | fw.RegisterModule("searcher", searcher.New()) 17 | err := fw.Initialize() 18 | if err != nil { 19 | panic(err.Error()) 20 | } 21 | 22 | fw.Run() 23 | } 24 | -------------------------------------------------------------------------------- /service/allinone/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ ! -f build.sh ]; then 3 | echo 'build.sh must be run within its container folder' 1>&2 4 | exit 1 5 | fi 6 | 7 | ROOT_DIR=`pwd` 8 | APP="allinone" 9 | VER="0.1" 10 | 11 | mkdir -p logs bin 12 | mkdir allinone 13 | 14 | go build -o bin/$APP 15 | 16 | cp -r bin/ allinone/ 17 | cp -r conf/ allinone/ 18 | cp serverctl allinone/ 19 | cp -r logs allinone/ 20 | 21 | tar -zcvf $ROOT_DIR/../../dist/$APP-$VER.tar.gz allinone/ 22 | 23 | rm -rf ./allinone 24 | -------------------------------------------------------------------------------- /service/allinone/conf/allinone_conf.ini: -------------------------------------------------------------------------------- 1 | # 将各个子service的配置项继承过来 2 | #inherited_from = meta.ini 3 | #inherited_from = idgenerator.ini 4 | #inherited_from = proxy.ini 5 | 6 | [common] 7 | # http服务器监听端口 8 | http_port = 39360 9 | 10 | # http请求读超时和写超时ms 11 | http_read_timeout_ms = 400 12 | http_write_timeout_ms = 400 13 | 14 | # 是否打开pprof 15 | pprof_enable = true 16 | 17 | access_log = true 18 | 19 | monitor_status_file_path = ../conf/status.html 20 | 21 | # debug flag 22 | debug = true 23 | 24 | pid_file=./app.pid 25 | 26 | [proxy] 27 | poseidon_search_url = http://127.0.0.1:39502 28 | 29 | [/service/idgenerator] 30 | redis_address = 127.0.0.1:6379 31 | redis_password = password 32 | 33 | [searcher] 34 | indexStored = http://127.0.0.1:38086/test_indexid 35 | docStored = http://127.0.0.1:38086/symc_server 36 | hdfs = 127.0.0.1:39997 37 | -------------------------------------------------------------------------------- /service/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ ! -f build.sh ]; then 4 | echo 'build.sh must be run within its container folder' 1>&2 5 | exit 1 6 | fi 7 | 8 | mkdir -p ../dist 9 | cd ./hdfsreader && sh ./build.sh && cd - 10 | cd ./idgenerator && sh ./build.sh && cd - 11 | cd ./proxy && sh ./build.sh && cd - 12 | cd ./searcher && sh ./build.sh && cd - 13 | cd ./meta && sh ./build.sh && cd - 14 | cd ./allinone && sh ./build.sh && cd - 15 | 16 | -------------------------------------------------------------------------------- /service/hdfsreader/README.md: -------------------------------------------------------------------------------- 1 | # HDFS 文件读取 2 | 3 | ### 打包 4 | `sh build.sh` 生成结果放到dist目录 5 | 6 | ### 部署 7 | 将打包结果解压，执行`sh bin/hdfsReader.sh `脚本。 8 | 9 | 修改端口号直接修改hdfsReader.sh中的port 10 | 11 | **`记得安装crontab`** 12 | -------------------------------------------------------------------------------- /service/hdfsreader/build.gradle: -------------------------------------------------------------------------------- 1 | group 'hdfsreader' 2 | version '0.1' 3 | 4 | apply plugin: 'java' 5 | apply plugin: 'application' 6 | 7 | sourceCompatibility = 1.7 8 | mainClassName="Reader" 9 | 10 | repositories { 11 | mavenLocal() 12 | mavenCentral() 13 | jcenter() 14 | 15 | maven{ url 'http://repo1.maven.org/maven2'} 16 | } 17 | 18 | def var = dependencies { 19 | compile group: 'org.apache.hadoop', name: 'hadoop-common', version: '2.7.1' 20 | compile group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: '2.7.1' 21 | } 22 | var 23 | 24 | def copyFile(String src, dst) { 25 | def srcFile = new File(src) 26 | def dstFile = new File(dst) 27 | dstFile.withOutputStream{ os-> 28 | srcFile.withInputStream{ ins-> 29 | os << ins 30 | } 31 | } 32 | } 33 | 34 | task createEtc { 35 | def etc = file("$buildDir/etc") 36 | outputs.dir etc 37 | doLast { 38 | etc.mkdirs() 39 | copyFile('src/main/etc/log4j.properties', "$buildDir/etc/log4j.properties") 40 | copyFile('src/main/etc/core-site.xml', "$buildDir/etc/core-site.xml") 41 | } 42 | } 43 | 44 | task createBin { 45 | def bin = file("$buildDir/bin") 46 | outputs.dir bin 47 | doLast { 48 | bin.mkdirs() 49 | copyFile('src/main/script/hdfsReader.sh', "$buildDir/bin/hdfsReader.sh") 50 | } 51 | } 52 | 53 | distributions { 54 | main { 55 | contents { 56 | from(createEtc) { 57 | into "etc" 58 | } 59 | from(createBin) { 60 | into "bin" 61 | } 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /service/hdfsreader/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | which gradle 4 | if [ $? -eq 0 ]; 5 | then 6 | gradle distTar 7 | else 8 | ./gradlew distTar 9 | fi 10 | 11 | if [ $? -ne 0 ]; 12 | then 13 | echo pkg hdfsreader failed 14 | else 15 | echo "Pack Success" 16 | fi 17 | 18 | cp -r build/distributions/hdfsreader*.tar ../../dist/ 19 | -------------------------------------------------------------------------------- /service/hdfsreader/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Qihoo360/poseidon/9be3a58756c2c604573075b6e7531f13f5ba7c1c/service/hdfsreader/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /service/hdfsreader/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Tue Oct 11 19:26:46 GMT+08:00 2016 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-2.13-bin.zip 7 | -------------------------------------------------------------------------------- /service/hdfsreader/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'hdfsreader' 2 | -------------------------------------------------------------------------------- /service/hdfsreader/src/main/etc/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | fs.defaultFS 4 | file:/ 5 | 6 | -------------------------------------------------------------------------------- /service/hdfsreader/src/main/etc/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO,stdout,fout 2 | log4j.logger.Reader=DEBUG 3 | pattern=[%-5p] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %C{2}:%L: %m%n 4 | 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 7 | log4j.appender.stdout.layout.ConversionPattern=${pattern} 8 | 9 | log4j.appender.fout=org.apache.log4j.DailyRollingFileAppender 10 | log4j.appender.fout.File=logs/reader.log 11 | log4j.appender.fout.layout=org.apache.log4j.PatternLayout 12 | log4j.appender.fout.layout.ConversionPattern=${pattern} 13 | -------------------------------------------------------------------------------- /service/hdfsreader/src/main/script/hdfsReader.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | current=$(cd $(dirname $0) && pwd -P) 4 | echo $current 5 | root=$(cd ${current}/.. && pwd) 6 | echo $root 7 | 8 | CLASSPATH="" 9 | export CLASSPATH="$root/etc":"$root/lib/*":${CLASSPATH} 10 | 11 | if [[ -z "$1" || ("$1" != "start" && "$1" != "stop" && "$1" != "restart") ]]; then 12 | echo "Usage: $0 " 13 | echo " $0 start" 14 | exit 15 | fi 16 | 17 | port=39997 18 | start_cmd="java -Xmx16000m Reader $port" 19 | pid=$(ps axu | grep "Reader $port$" | awk '{print $2}') 20 | if [[ -z "$pid" && "$1" == "stop" ]]; then 21 | echo "service has NOT been started" 22 | exit 23 | elif [[ -n "$pid" && "$1" == "start" ]]; then 24 | echo "service has been started" 25 | ps axu | grep "Reader $port$" 26 | exit 27 | fi 28 | 29 | if [[ "$1" == "stop" || (-n "$pid" && "$1" == "restart") ]]; then 30 | kill -9 $pid 31 | echo "service has been killed" 32 | sleep 0.1 33 | ps axu | grep "Reader $port$" 34 | if [[ "$1" == "stop" ]]; then 35 | exit 36 | fi 37 | fi 38 | 39 | mkdir -p $root/logs 40 | 41 | log_postfix=$(date "+%Y%m%d-%H%M%S") 42 | if [[ "$1" == "start" || "$1" == "restart" ]]; then 43 | cd $root 44 | nohup $start_cmd &> logs/reader.out.${log_postfix} & 45 | echo "service started" 46 | sleep 0.1 47 | ps axu | grep "Reader $port$" 48 | fi 49 | exit 50 | -------------------------------------------------------------------------------- /service/hdfsreader/watchdog.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | count=$(ps axu | grep "java .* Reader [0-9]*$" | wc -l) 3 | if [[ "$count" -ge 1 ]]; then 4 | exit 5 | fi 6 | 7 | echo "service has NOT been started" 8 | 9 | sh hdfsReader.sh start 10 | exit 11 | -------------------------------------------------------------------------------- /service/idgenerator/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ ! -f build.sh ]; then 3 | echo 'build.sh must be run within its container folder' 1>&2 4 | exit 1 5 | fi 6 | 7 | ROOT_DIR=`pwd` 8 | APP=`basename $ROOT_DIR` 9 | VER="0.1" 10 | 11 | 12 | mkdir -p bin logs 13 | mkdir idgenerator 14 | 15 | go build -o bin/$APP 16 | 17 | cp -r bin/ idgenerator/ 18 | cp -r conf/ idgenerator/ 19 | cp serverctl idgenerator/ 20 | cp -r logs idgenerator/ 21 | 22 | 23 | tar -zcvf $ROOT_DIR/../../dist/idgenerator-$VER.tar.gz idgenerator/ 24 | 25 | rm -rf ./idgenerator 26 | -------------------------------------------------------------------------------- /service/idgenerator/conf/idgenerator.ini: -------------------------------------------------------------------------------- 1 | ######## 基础配置项 ############## 2 | [common] 3 | # 是否记录类似于nginx的access.log的请求日志 4 | access_log = true 5 | 6 | pid_file = ./app.pid 7 | 8 | # http请求监听端口 9 | http_port = 39360 10 | 11 | # http请求读超时和写超时ms 12 | http_read_timeout_ms = 400 13 | http_write_timeout_ms = 400 14 | 15 | # 是否打开pprof 16 | pprof_enable = true 17 | 18 | # 19 | #monitor_status_file_path = /the/path/to/status.html 20 | monitor_status_file_path = ../conf/status.html 21 | 22 | # debug flag 23 | debug = true 24 | 25 | ############ 业务配置项 ############## 26 | [/service/idgenerator] 27 | redis_address = 127.0.0.1:6379 28 | redis_password = password 29 | 30 | 31 | -------------------------------------------------------------------------------- /service/idgenerator/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | idgen "github.com/Qihoo360/poseidon/service/idgenerator/module" 5 | "github.com/zieckey/simgo" 6 | "io/ioutil" 7 | "os" 8 | "path" 9 | "strconv" 10 | ) 11 | 12 | //@todo pidfile configure 13 | var pidfile string = "./pid" 14 | 15 | func main() { 16 | os.Chdir(path.Dir(os.Args[0])) 17 | fw := simgo.DefaultFramework 18 | fw.RegisterModule("idgen_service", idgen.New()) 19 | managePid(true, pidfile) 20 | err := fw.Initialize() 21 | if err != nil { 22 | panic(err.Error()) 23 | } 24 | 25 | fw.Run() 26 | } 27 | 28 | //生成/删除当前进程id文件 29 | func managePid(create bool, pidfile string) { 30 | if pidfile == "" { 31 | panic("pidfile is null, please configure it") 32 | } 33 | if create { 34 | pid := os.Getpid() 35 | pidString := strconv.Itoa(pid) 36 | ioutil.WriteFile(pidfile, []byte(pidString), 0777) 37 | } else { 38 | os.Remove(pidfile) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /service/idgenerator/module/redis.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | /** 4 | * 底层redis 连接池 5 | * @author guojun-s@360.cn 6 | * 7 | */ 8 | 9 | import ( 10 | "github.com/garyburd/redigo/redis" 11 | "time" 12 | ) 13 | 14 | const ( 15 | PROTOCOL = "tcp" //connection protocol 16 | ) 17 | 18 | var ( 19 | MaxIdle int = 100 20 | MaxActive int = 1000 21 | IdleTimeout time.Duration = time.Duration(28 * time.Second) 22 | ) 23 | 24 | // 25 | //var RedisPool *redis.Pool 26 | //var Rp = &Pool{} 27 | // 28 | //type Pool struct{} 29 | // 30 | //func (p *Pool) Init(server string, password string, IdleTimeout time.Duration, MaxIdle, MaxActive int) { 31 | // RedisPool = NewPool(server, password, IdleTimeout, MaxIdle, MaxActive) 32 | //} 33 | 34 | /** 35 | * Redis Pool 36 | * 37 | * serverAddr the server address 127.0.0.1:6379 38 | * password password 127.0.0.1:6379:password 39 | * IdleTimeout 超时 40 | * MaxIdle 连接池最大容量 41 | * MaxActive 最大活跃数量 42 | * dbno 选择db127.0.0.1:6379:password:1 43 | * 44 | */ 45 | func NewPool(serverAddr string, password string, IdleTimeout time.Duration, MaxIdle, MaxActive int) *redis.Pool { 46 | 47 | return &redis.Pool{ 48 | MaxIdle: MaxIdle, 49 | MaxActive: MaxActive, 50 | IdleTimeout: IdleTimeout, 51 | Dial: func() (redis.Conn, error) { 52 | c, err := redis.Dial(PROTOCOL, serverAddr) 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | //校验密码 58 | if _, err := c.Do("AUTH", password); err != nil { 59 | c.Close() 60 | return nil, err 61 | } 62 | 63 | return c, err 64 | 65 | }, 66 | TestOnBorrow: func(c redis.Conn, t time.Time) error { 67 | _, err := c.Do("PING") 68 | return err 69 | }, 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /service/meta/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ ! -f build.sh ]; then 3 | echo 'build.sh must be run within its container folder' 1>&2 4 | exit 1 5 | fi 6 | 7 | ROOT_DIR=`pwd` 8 | APP="meta" 9 | VER="0.1" 10 | 11 | mkdir -p bin logs 12 | mkdir meta 13 | 14 | go build -o bin/$APP 15 | 16 | cp -r bin/ meta/ 17 | cp -r conf/ meta/ 18 | cp serverctl meta/ 19 | cp -r logs meta/ 20 | 21 | tar -zcvf $ROOT_DIR/../../dist/$APP-$VER.tar.gz meta/ 22 | 23 | rm -rf ./meta 24 | -------------------------------------------------------------------------------- /service/meta/conf/meta.ini: -------------------------------------------------------------------------------- 1 | ######## 基础配置项 ############## 2 | [common] 3 | # 保存进程id的文件路径, 用于进程停止和重启 4 | pid_file = ./app.pid 5 | 6 | # 是否记录类似于nginx的access.log的请求日志 7 | access_log = true 8 | 9 | # http请求监听端口 10 | http_port = 39610 11 | 12 | # http请求读超时和写超时ms 13 | http_read_timeout_ms = 400 14 | http_write_timeout_ms = 400 15 | 16 | # 是否打开pprof 17 | pprof_enable = true 18 | 19 | # 20 | #monitor_status_file_path = /the/path/to/status.html 21 | monitor_status_file_path = ../conf/status.html 22 | 23 | # debug flag 24 | debug = true 25 | 26 | ############ 业务配置项 ############## 27 | [meta] 28 | 29 | # test 业务的 Doc Meta 信息存储的NoSQL配置信息 30 | [/service/meta/backend-store/doc/test] 31 | timeout_ms = 3000 32 | address = 127.0.0.1:11211 33 | store_type = memcached 34 | 35 | # test 业务的 Index Meta 信息存储的NoSQL配置信息 36 | [/service/meta/backend-store/index/test] 37 | timeout_ms = 3000 38 | address = 127.0.0.1:11211 39 | store_type = memcached 40 | 41 | -------------------------------------------------------------------------------- /service/meta/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/Qihoo360/poseidon/service/meta/module" 5 | "github.com/zieckey/simgo" 6 | ) 7 | 8 | func main() { 9 | fw := simgo.DefaultFramework 10 | fw.RegisterModule("meta_service", module.New()) 11 | err := fw.Initialize() 12 | if err != nil { 13 | panic(err.Error()) 14 | } 15 | 16 | fw.Run() 17 | } 18 | -------------------------------------------------------------------------------- /service/meta/store/memcached/memcached.go: -------------------------------------------------------------------------------- 1 | package memcached 2 | 3 | import ( 4 | "github.com/Qihoo360/poseidon/service/meta/store" 5 | "github.com/bradfitz/gomemcache/memcache" 6 | "github.com/golang/glog" 7 | ) 8 | 9 | type Memcached struct { 10 | conn *memcache.Client 11 | config store.Config 12 | } 13 | 14 | func NewMemcachedStore(c store.Config) (store.Store, error) { 15 | rc := &Memcached{ 16 | config: c, 17 | } 18 | 19 | err := rc.connectInit() 20 | if err == nil { 21 | return rc, nil 22 | } 23 | 24 | return nil, err 25 | } 26 | 27 | func (rc *Memcached) Get(key string) (result store.GetResult) { 28 | if item, err := rc.conn.Get(key); err == nil { 29 | result.Value = string(item.Value) 30 | } else { 31 | if err != memcache.ErrCacheMiss { 32 | result.Err = err 33 | } 34 | } 35 | return result 36 | } 37 | 38 | func (rc *Memcached) MultiGet(keys []string) map[string] /*the-key*/ store.GetResult { 39 | size := len(keys) 40 | rv := make(map[string] /*the-key*/ store.GetResult) 41 | mv, err := rc.conn.GetMulti(keys) 42 | if err == nil { 43 | for _, v := range mv { 44 | var r store.GetResult 45 | r.Value = string(v.Value) 46 | glog.Infof("Memcached.MultiGet key=[%v] value=[%v]", v.Key, r.Value) 47 | rv[v.Key] = r 48 | } 49 | return rv 50 | } else { 51 | glog.Errorf("Memcached.MultiGet ERROR : %v", err.Error()) 52 | } 53 | 54 | for i := 0; i < size; i++ { 55 | var r store.GetResult 56 | if err != memcache.ErrCacheMiss { 57 | r.Err = err 58 | } 59 | rv[keys[i]] = r 60 | } 61 | return rv 62 | } 63 | 64 | func (rc *Memcached) Set(key, value string) error { 65 | item := memcache.Item{Key: key, Value: []byte(value)} 66 | return rc.conn.Set(&item) 67 | } 68 | 69 | func (rc *Memcached) Delete(key string) error { 70 | return rc.conn.Delete(key) 71 | } 72 | 73 | func (rc *Memcached) connectInit() error { 74 | rc.conn = memcache.New(rc.config.Addr) 75 | return nil 76 | } 77 | 78 | func init() { 79 | store.Register(store.MEMCACHED, NewMemcachedStore) 80 | } 81 | -------------------------------------------------------------------------------- /service/meta/store/store.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | type GetResult struct { 9 | Err error 10 | Value string 11 | } 12 | 13 | type Store interface { 14 | // Get gets a value by key 15 | // If the key is not found in the store, the Err is nil and Value is empty 16 | Get(key string) GetResult 17 | 18 | // MultiGet is a batch version of Get 19 | MultiGet(keys []string) map[string] /*the-key*/ GetResult 20 | 21 | // Set sets a value with key where the key exists or not 22 | Set(key, value string) error 23 | 24 | // Delete deletes a key/value pair from the store 25 | Delete(key string) error 26 | } 27 | 28 | // Config contains the options for a storage client 29 | type Config struct { 30 | Timeout time.Duration 31 | Addr string // The address of the backend store with "host:port" format 32 | Password string // The authority password if necessarily 33 | } 34 | 35 | // The backend store name 36 | const ( 37 | MEMCACHED string = "memcached" 38 | REDIS string = "redis" 39 | ) 40 | 41 | // StoreCreator is a function to create a new instance of Store 42 | type StoreCreator func(Config) (Store, error) 43 | 44 | var creators = make(map[string]StoreCreator) 45 | 46 | // Register makes a store creator available by name. 47 | func Register(name string, creator StoreCreator) { 48 | if creator == nil { 49 | panic("cache: Register adapter is nil") 50 | } 51 | if _, ok := creators[name]; ok { 52 | panic("cache: Register called twice for adapter " + name) 53 | } 54 | creators[name] = creator 55 | } 56 | 57 | // NewStore creates a new store driver by store name and configurations. 58 | func NewStore(name string, c Config) (db Store, err error) { 59 | creator, ok := creators[name] 60 | if !ok { 61 | err = fmt.Errorf("unknown store name [%v] (forgot to import?)", name) 62 | return 63 | } 64 | db, err = creator(c) 65 | if err != nil { 66 | db = nil 67 | } 68 | return db, err 69 | } 70 | -------------------------------------------------------------------------------- /service/proxy/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ ! -f build.sh ]; then 3 | echo 'build.sh must be run within its container folder' 1>&2 4 | exit 1 5 | fi 6 | 7 | ROOT_DIR=`pwd` 8 | APP="poseidon_proxy" 9 | VER="0.1" 10 | 11 | mkdir -p bin logs 12 | mkdir proxy 13 | 14 | go build -o bin/$APP 15 | 16 | cp -r bin/ proxy/ 17 | cp -r conf/ proxy/ 18 | cp serverctl proxy/ 19 | cp -r logs proxy/ 20 | 21 | tar -zcvf $ROOT_DIR/../../dist/$APP-$VER.tar.gz proxy/ 22 | 23 | rm -rf ./proxy 24 | -------------------------------------------------------------------------------- /service/proxy/conf/app.ini: -------------------------------------------------------------------------------- 1 | ######## 基础配置项 ############## 2 | [common] 3 | # 保存进程id的文件路径, 用于进程停止和重启 4 | pid_file = ./app.pid 5 | 6 | # 是否记录类似于nginx的access.log的请求日志 7 | access_log = true 8 | 9 | # http请求监听端口 10 | http_port = 39460 11 | 12 | # http请求读超时和写超时ms 13 | http_read_timeout_ms = 400 14 | http_write_timeout_ms = 400 15 | 16 | # 是否打开pprof 17 | pprof_enable = true 18 | 19 | #monitor_status_file_path = /the/path/to/status.html 20 | monitor_status_file_path = ../conf/status.html 21 | 22 | # debug flag 23 | debug = true 24 | 25 | ############ 业务配置项 ############## 26 | [proxy] 27 | poseidon_search_url = http://127.0.0.1:39504/service/search 28 | 29 | -------------------------------------------------------------------------------- /service/proxy/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | proxy "github.com/Qihoo360/poseidon/service/proxy/module" 5 | "github.com/zieckey/simgo" 6 | "io/ioutil" 7 | "os" 8 | "path" 9 | "strconv" 10 | ) 11 | 12 | var pidfile string = "./pid" 13 | 14 | func main() { 15 | os.Chdir(path.Dir(os.Args[0])) 16 | fw := simgo.DefaultFramework 17 | fw.RegisterModule("proxy_service", proxy.New()) 18 | managePid(true, pidfile) 19 | err := fw.Initialize() 20 | if err != nil { 21 | panic(err.Error()) 22 | } 23 | 24 | fw.Run() 25 | } 26 | 27 | //生成/删除当前进程id文件 28 | func managePid(create bool, pidfile string) { 29 | if pidfile == "" { 30 | panic("pidfile is null, please configure it") 31 | } 32 | if create { 33 | pid := os.Getpid() 34 | pidString := strconv.Itoa(pid) 35 | ioutil.WriteFile(pidfile, []byte(pidString), 0777) 36 | } else { 37 | os.Remove(pidfile) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /service/searcher/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ ! -f build.sh ]; then 3 | echo 'build.sh must be run within its container folder' 1>&2 4 | exit 1 5 | fi 6 | 7 | ROOT_DIR=`pwd` 8 | APP="poseidon_searcher" 9 | VER="0.1" 10 | 11 | mkdir -p bin logs 12 | 13 | go build -o bin/$APP 14 | 15 | mkdir searcher 16 | cp -r bin/ searcher/ 17 | cp -r conf/ searcher/ 18 | cp serverctl searcher/ 19 | cp -r logs searcher/ 20 | 21 | tar -zcvf $ROOT_DIR/../../dist/$APP-$VER.tar.gz searcher/ 22 | 23 | rm -rf ./searcher 24 | -------------------------------------------------------------------------------- /service/searcher/conf/app.ini: -------------------------------------------------------------------------------- 1 | ######## 基础配置项 ############## 2 | [common] 3 | # 保存进程id的文件路径, 用于进程停止和重启 4 | pid_file = ./app.pid 5 | 6 | # 是否记录类似于nginx的access.log的请求日志 7 | access_log = true 8 | 9 | # http请求监听端口 10 | http_port = 39504 11 | 12 | # http请求读超时和写超时ms 13 | http_read_timeout_ms = 400 14 | http_write_timeout_ms = 400 15 | 16 | # 是否打开pprof 17 | pprof_enable = true 18 | 19 | #monitor_status_file_path = /the/path/to/status.html 20 | monitor_status_file_path = ../conf/status.html 21 | 22 | # debug flag 23 | debug = true 24 | 25 | ############ 业务配置项 ############## 26 | [searcher] 27 | indexStored = http://127.0.0.1:39610/service/meta/test/index/get 28 | docStored = http://127.0.0.1:39610/service/meta/test/doc/get 29 | hdfs = 127.0.0.1:39997 30 | -------------------------------------------------------------------------------- /service/searcher/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | searcher "github.com/Qihoo360/poseidon/service/searcher/module" 5 | "github.com/zieckey/simgo" 6 | "io/ioutil" 7 | "os" 8 | "path" 9 | "strconv" 10 | ) 11 | 12 | var pidfile string = "./pid" 13 | 14 | func main() { 15 | os.Chdir(path.Dir(os.Args[0])) 16 | fw := simgo.DefaultFramework 17 | fw.RegisterModule("searcher_service", searcher.New()) 18 | managePid(true, pidfile) 19 | err := fw.Initialize() 20 | if err != nil { 21 | panic(err.Error()) 22 | } 23 | 24 | fw.Run() 25 | } 26 | 27 | //生成/删除当前进程id文件 28 | func managePid(create bool, pidfile string) { 29 | if pidfile == "" { 30 | panic("pidfile is null, please configure it") 31 | } 32 | if create { 33 | pid := os.Getpid() 34 | pidString := strconv.Itoa(pid) 35 | ioutil.WriteFile(pidfile, []byte(pidString), 0777) 36 | } else { 37 | os.Remove(pidfile) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /service/searcher/poseidon/RedisPool.go: -------------------------------------------------------------------------------- 1 | package poseidon 2 | 3 | /** 4 | * 底层redis 连接池 5 | * @author guojun-s@360.cn 6 | * 7 | */ 8 | 9 | import ( 10 | "github.com/garyburd/redigo/redis" 11 | "time" 12 | ) 13 | 14 | const ( 15 | PROTOCOL = "tcp" //connection protocol 16 | ) 17 | 18 | var ( 19 | MaxIdle int = 100 20 | MaxActive int = 1000 21 | IdleTimeout time.Duration = time.Duration(28 * time.Second) 22 | ) 23 | 24 | var RedisPool *redis.Pool 25 | var Rp = &Pool{} 26 | 27 | type Pool struct{} 28 | 29 | func (p *Pool) Init(server string, password string, IdleTimeout time.Duration, MaxIdle, MaxActive int) { 30 | RedisPool = NewPool(server, password, IdleTimeout, MaxIdle, MaxActive) 31 | } 32 | 33 | /** 34 | * Redis Pool 35 | * 36 | * server serverAddress 127.0.0.1:6379 37 | * password password 127.0.0.1:6379:password 38 | * IdleTimeout 超时 39 | * MaxIdle 连接池最大容量 40 | * MaxActive 最大活跃数量 41 | * dbno 选择db127.0.0.1:6379:password:1 42 | * 43 | */ 44 | func NewPool(server string, password string, IdleTimeout time.Duration, MaxIdle, MaxActive int) *redis.Pool { 45 | 46 | return &redis.Pool{ 47 | MaxIdle: MaxIdle, 48 | MaxActive: MaxActive, 49 | IdleTimeout: IdleTimeout, 50 | Dial: func() (redis.Conn, error) { 51 | c, err := redis.Dial(PROTOCOL, server) 52 | if err != nil { 53 | return nil, err 54 | } 55 | 56 | //要求必须校验密码 @TODO 密码为空 57 | if _, err := c.Do("AUTH", password); err != nil { 58 | c.Close() 59 | return nil, err 60 | } 61 | 62 | return c, err 63 | 64 | }, 65 | TestOnBorrow: func(c redis.Conn, t time.Time) error { 66 | _, err := c.Do("PING") 67 | return err 68 | }, 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /service/searcher/poseidon/RedisPool_test.go: -------------------------------------------------------------------------------- 1 | package poseidon 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | var servers []string = []string{ 9 | "127.0.0.1:9999:61c7b9811799779d", 10 | "127.0.0.1:6134:74de338358be6568:1", 11 | } 12 | 13 | // func Test_multipool(t *testing.T) { 14 | // Rp.Init(servers) 15 | // fmt.Println(RedisMultiPool[1].Get().Do("keys", "*")) 16 | // } 17 | // 18 | func Test_Getconn(t *testing.T) { 19 | Rp.Init(servers) 20 | conn := GetConn("hhjj") 21 | 22 | fmt.Println(conn) 23 | } 24 | -------------------------------------------------------------------------------- /service/searcher/poseidon/Symc.go: -------------------------------------------------------------------------------- 1 | package poseidon 2 | 3 | import ( 4 | "bytes" 5 | "encoding/base64" 6 | "io/ioutil" 7 | "log" 8 | "net/http" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | func SymcGet(symcurl string, keys map[string]string) (map[string]string, error) { 14 | var err error 15 | log.Println(symcurl) 16 | buf := bytes.NewBufferString("") 17 | for key, _ := range keys { 18 | buf.WriteString(key) 19 | buf.WriteString("\n") 20 | } 21 | log.Println(symcurl) 22 | req, err := http.NewRequest("POST", symcurl, buf) 23 | client := &http.Client{ 24 | Timeout: time.Duration(60 * time.Second), 25 | } 26 | 27 | result, err := client.Do(req) 28 | if err != nil { 29 | log.Println("symc error", err.Error()) 30 | return nil, err 31 | } 32 | defer result.Body.Close() 33 | 34 | re, err := ioutil.ReadAll(result.Body) 35 | log.Println(string(re)) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | restring := string(re) 41 | m := make(map[string]string, 50) //预分配50 42 | strslice := strings.Split(restring, "\n") 43 | 44 | if strslice != nil { 45 | for _, item := range strslice { 46 | if item == "" { 47 | continue 48 | } 49 | kvslice := strings.Split(item, "\t") 50 | key := kvslice[0] 51 | value, err := base64.StdEncoding.DecodeString(kvslice[1]) 52 | if err != nil { 53 | continue 54 | } 55 | m[key] = string(value) 56 | } 57 | } 58 | log.Println(m) 59 | return m, nil 60 | } 61 | -------------------------------------------------------------------------------- /service/searcher/poseidon/Symc_test.go: -------------------------------------------------------------------------------- 1 | package poseidon 2 | 3 | import ( 4 | "log" 5 | "testing" 6 | ) 7 | 8 | func Test_StmcGet(t *testing.T) { 9 | log.Println("test...") 10 | m := make(map[string]string, 10) 11 | m["160630401156861"] = "" 12 | m["160630401156816"] = "" 13 | m["160630401156858"] = "" 14 | m["160630401156874"] = "" 15 | m["160630401156817"] = "" 16 | 17 | r, err := SymcGet(m) 18 | if r != nil { 19 | panic(err) 20 | } 21 | log.Println(r) 22 | } 23 | -------------------------------------------------------------------------------- /service/searcher/poseidon/memcache_pool.go: -------------------------------------------------------------------------------- 1 | // package main 2 | package poseidon 3 | 4 | import ( 5 | "errors" 6 | "github.com/bradfitz/gomemcache/memcache" 7 | "sync" 8 | "time" 9 | ) 10 | 11 | var gAddrMemcachePool = AddrMemcachePool{} 12 | 13 | type AddrMemcachePool struct { 14 | poolMap map[string]*MemcachePool 15 | sync.Mutex 16 | } 17 | 18 | func (self *AddrMemcachePool) Init() { 19 | self.poolMap = make(map[string]*MemcachePool) 20 | } 21 | 22 | func (self *AddrMemcachePool) Get(addr string) *MemcachePool { 23 | self.Lock() 24 | defer self.Unlock() 25 | if pool, ok := self.poolMap[addr]; ok { 26 | return pool 27 | } 28 | pool := NewPoolInstance(addr, 50) 29 | self.poolMap[addr] = pool 30 | return pool 31 | } 32 | 33 | func init() { 34 | gAddrMemcachePool.Init() 35 | } 36 | 37 | func NewPoolInstance(addr string, limit int) *MemcachePool { 38 | var memcachePool = MemcachePool{addr: addr, limit: limit} 39 | return &memcachePool 40 | } 41 | 42 | type MemcachePool struct { 43 | // TODO: add lcoal cache 44 | pool []*memcache.Client 45 | sync.Mutex 46 | 47 | addr string 48 | allocated int 49 | limit int 50 | } 51 | 52 | func (self *MemcachePool) Alloc() (client *memcache.Client, err error) { 53 | self.Lock() 54 | 55 | if len(self.pool) > 0 { 56 | client := self.pool[0] 57 | self.pool = self.pool[1:] 58 | self.allocated++ 59 | self.Unlock() 60 | return client, nil 61 | } 62 | 63 | if self.allocated >= self.limit { 64 | self.Unlock() 65 | return nil, errors.New("too_many_conns") 66 | } 67 | 68 | self.allocated++ 69 | self.Unlock() 70 | 71 | c := memcache.New(self.addr) 72 | c.Timeout = 300 * time.Millisecond 73 | return c, nil 74 | } 75 | 76 | func (self *MemcachePool) Release(client *memcache.Client, success bool) { 77 | if !success { 78 | // client.DeleteAll() 79 | } 80 | 81 | self.Lock() 82 | defer self.Unlock() 83 | self.allocated-- 84 | 85 | if success { 86 | self.pool = append(self.pool, client) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /service/vendor/github.com/deckarep/golang-set/.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | -------------------------------------------------------------------------------- /service/vendor/github.com/deckarep/golang-set/.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.2 5 | - 1.3 6 | - tip 7 | 8 | script: 9 | - go test -race ./... 10 | - go test -bench=. 11 | 12 | -------------------------------------------------------------------------------- /service/vendor/github.com/deckarep/golang-set/LICENSE: -------------------------------------------------------------------------------- 1 | Open Source Initiative OSI - The MIT License (MIT):Licensing 2 | 3 | The MIT License (MIT) 4 | Copyright (c) 2013 Ralph Caraveo (deckarep@gmail.com) 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy of 7 | this software and associated documentation files (the "Software"), to deal in 8 | the Software without restriction, including without limitation the rights to 9 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 10 | of the Software, and to permit persons to whom the Software is furnished to do 11 | so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. -------------------------------------------------------------------------------- /service/vendor/github.com/garyburd/redigo/internal/commandinfo.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Gary Burd 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package internal 16 | 17 | import ( 18 | "strings" 19 | ) 20 | 21 | const ( 22 | WatchState = 1 << iota 23 | MultiState 24 | SubscribeState 25 | MonitorState 26 | ) 27 | 28 | type CommandInfo struct { 29 | Set, Clear int 30 | } 31 | 32 | var commandInfos = map[string]CommandInfo{ 33 | "WATCH": {Set: WatchState}, 34 | "UNWATCH": {Clear: WatchState}, 35 | "MULTI": {Set: MultiState}, 36 | "EXEC": {Clear: WatchState | MultiState}, 37 | "DISCARD": {Clear: WatchState | MultiState}, 38 | "PSUBSCRIBE": {Set: SubscribeState}, 39 | "SUBSCRIBE": {Set: SubscribeState}, 40 | "MONITOR": {Set: MonitorState}, 41 | } 42 | 43 | func init() { 44 | for n, ci := range commandInfos { 45 | commandInfos[strings.ToLower(n)] = ci 46 | } 47 | } 48 | 49 | func LookupCommandInfo(commandName string) CommandInfo { 50 | if ci, ok := commandInfos[commandName]; ok { 51 | return ci 52 | } 53 | return commandInfos[strings.ToUpper(commandName)] 54 | } 55 | -------------------------------------------------------------------------------- /service/vendor/github.com/garyburd/redigo/redis/redis.go: -------------------------------------------------------------------------------- 1 | // Copyright 2012 Gary Burd 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | // not use this file except in compliance with the License. You may obtain 5 | // a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | // License for the specific language governing permissions and limitations 13 | // under the License. 14 | 15 | package redis 16 | 17 | // Error represents an error returned in a command reply. 18 | type Error string 19 | 20 | func (err Error) Error() string { return string(err) } 21 | 22 | // Conn represents a connection to a Redis server. 23 | type Conn interface { 24 | // Close closes the connection. 25 | Close() error 26 | 27 | // Err returns a non-nil value if the connection is broken. The returned 28 | // value is either the first non-nil value returned from the underlying 29 | // network connection or a protocol parsing error. Applications should 30 | // close broken connections. 31 | Err() error 32 | 33 | // Do sends a command to the server and returns the received reply. 34 | Do(commandName string, args ...interface{}) (reply interface{}, err error) 35 | 36 | // Send writes the command to the client's output buffer. 37 | Send(commandName string, args ...interface{}) error 38 | 39 | // Flush flushes the output buffer to the Redis server. 40 | Flush() error 41 | 42 | // Receive receives a single reply from the Redis server 43 | Receive() (reply interface{}, err error) 44 | } 45 | -------------------------------------------------------------------------------- /service/vendor/github.com/golang/glog/README: -------------------------------------------------------------------------------- 1 | glog 2 | ==== 3 | 4 | Leveled execution logs for Go. 5 | 6 | This is an efficient pure Go implementation of leveled logs in the 7 | manner of the open source C++ package 8 | https://github.com/google/glog 9 | 10 | By binding methods to booleans it is possible to use the log package 11 | without paying the expense of evaluating the arguments to the log. 12 | Through the -vmodule flag, the package also provides fine-grained 13 | control over logging at the file level. 14 | 15 | The comment from glog.go introduces the ideas: 16 | 17 | Package glog implements logging analogous to the Google-internal 18 | C++ INFO/ERROR/V setup. It provides functions Info, Warning, 19 | Error, Fatal, plus formatting variants such as Infof. It 20 | also provides V-style logging controlled by the -v and 21 | -vmodule=file=2 flags. 22 | 23 | Basic examples: 24 | 25 | glog.Info("Prepare to repel boarders") 26 | 27 | glog.Fatalf("Initialization failed: %s", err) 28 | 29 | See the documentation for the V function for an explanation 30 | of these examples: 31 | 32 | if glog.V(2) { 33 | glog.Info("Starting transaction...") 34 | } 35 | 36 | glog.V(2).Infoln("Processed", nItems, "elements") 37 | 38 | 39 | The repository contains an open source version of the log package 40 | used inside Google. The master copy of the source lives inside 41 | Google, not here. The code in this repo is for export only and is not itself 42 | under development. Feature requests will be ignored. 43 | 44 | Send bug reports to golang-nuts@googlegroups.com. 45 | -------------------------------------------------------------------------------- /service/vendor/github.com/golang/protobuf/LICENSE: -------------------------------------------------------------------------------- 1 | Go support for Protocol Buffers - Google's data interchange format 2 | 3 | Copyright 2010 The Go Authors. All rights reserved. 4 | https://github.com/golang/protobuf 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are 8 | met: 9 | 10 | * Redistributions of source code must retain the above copyright 11 | notice, this list of conditions and the following disclaimer. 12 | * Redistributions in binary form must reproduce the above 13 | copyright notice, this list of conditions and the following disclaimer 14 | in the documentation and/or other materials provided with the 15 | distribution. 16 | * Neither the name of Google Inc. nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | -------------------------------------------------------------------------------- /service/vendor/github.com/golang/protobuf/proto/Makefile: -------------------------------------------------------------------------------- 1 | # Go support for Protocol Buffers - Google's data interchange format 2 | # 3 | # Copyright 2010 The Go Authors. All rights reserved. 4 | # https://github.com/golang/protobuf 5 | # 6 | # Redistribution and use in source and binary forms, with or without 7 | # modification, are permitted provided that the following conditions are 8 | # met: 9 | # 10 | # * Redistributions of source code must retain the above copyright 11 | # notice, this list of conditions and the following disclaimer. 12 | # * Redistributions in binary form must reproduce the above 13 | # copyright notice, this list of conditions and the following disclaimer 14 | # in the documentation and/or other materials provided with the 15 | # distribution. 16 | # * Neither the name of Google Inc. nor the names of its 17 | # contributors may be used to endorse or promote products derived from 18 | # this software without specific prior written permission. 19 | # 20 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | install: 33 | go install 34 | 35 | test: install generate-test-pbs 36 | go test 37 | 38 | 39 | generate-test-pbs: 40 | make install 41 | make -C testdata 42 | protoc --go_out=Mtestdata/test.proto=github.com/golang/protobuf/proto/testdata,Mgoogle/protobuf/any.proto=github.com/golang/protobuf/ptypes/any:. proto3_proto/proto3.proto 43 | make 44 | -------------------------------------------------------------------------------- /service/vendor/github.com/gorilla/context/.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | sudo: false 3 | 4 | matrix: 5 | include: 6 | - go: 1.3 7 | - go: 1.4 8 | - go: 1.5 9 | - go: 1.6 10 | - go: tip 11 | allow_failures: 12 | - go: tip 13 | 14 | script: 15 | - go get -t -v ./... 16 | - diff -u <(echo -n) <(gofmt -d .) 17 | - go vet $(go list ./... | grep -v /vendor/) 18 | - go test -v -race ./... 19 | -------------------------------------------------------------------------------- /service/vendor/github.com/gorilla/context/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 Rodrigo Moraes. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /service/vendor/github.com/gorilla/context/README.md: -------------------------------------------------------------------------------- 1 | context 2 | ======= 3 | [![Build Status](https://travis-ci.org/gorilla/context.png?branch=master)](https://travis-ci.org/gorilla/context) 4 | 5 | gorilla/context is a general purpose registry for global request variables. 6 | 7 | Read the full documentation here: http://www.gorillatoolkit.org/pkg/context 8 | -------------------------------------------------------------------------------- /service/vendor/github.com/gorilla/mux/.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | sudo: false 3 | 4 | matrix: 5 | include: 6 | - go: 1.2 7 | - go: 1.3 8 | - go: 1.4 9 | - go: 1.5 10 | - go: 1.6 11 | - go: tip 12 | 13 | script: 14 | - go get -t -v ./... 15 | - diff -u <(echo -n) <(gofmt -d .) 16 | - go tool vet . 17 | - go test -v -race ./... 18 | -------------------------------------------------------------------------------- /service/vendor/github.com/gorilla/mux/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 Rodrigo Moraes. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /service/vendor/github.com/spaolacci/murmur3/.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | -------------------------------------------------------------------------------- /service/vendor/github.com/spaolacci/murmur3/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2013, Sébastien Paolacci. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | * Neither the name of the library nor the 12 | names of its contributors may be used to endorse or promote products 13 | derived from this software without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY 19 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /service/vendor/github.com/spaolacci/murmur3/murmur.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013, Sébastien Paolacci. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | /* 6 | Native (and fast) implementation of Austin Appleby's MurmurHash3. 7 | 8 | Package murmur3 implements Austin Appleby's non-cryptographic MurmurHash3. 9 | 10 | Reference implementation: 11 | http://code.google.com/p/smhasher/wiki/MurmurHash3 12 | 13 | History, characteristics and (legacy) perfs: 14 | https://sites.google.com/site/murmurhash/ 15 | https://sites.google.com/site/murmurhash/statistics 16 | */ 17 | package murmur3 18 | 19 | type bmixer interface { 20 | bmix(p []byte) (tail []byte) 21 | Size() (n int) 22 | reset() 23 | } 24 | 25 | type digest struct { 26 | clen int // Digested input cumulative length. 27 | tail []byte // 0 to Size()-1 bytes view of `buf'. 28 | buf [16]byte // Expected (but not required) to be Size() large. 29 | bmixer 30 | } 31 | 32 | func (d *digest) BlockSize() int { return 1 } 33 | 34 | func (d *digest) Write(p []byte) (n int, err error) { 35 | n = len(p) 36 | d.clen += n 37 | 38 | if len(d.tail) > 0 { 39 | // Stick back pending bytes. 40 | nfree := d.Size() - len(d.tail) // nfree ∈ [1, d.Size()-1]. 41 | if nfree < len(p) { 42 | // One full block can be formed. 43 | block := append(d.tail, p[:nfree]...) 44 | p = p[nfree:] 45 | _ = d.bmix(block) // No tail. 46 | } else { 47 | // Tail's buf is large enough to prevent reallocs. 48 | p = append(d.tail, p...) 49 | } 50 | } 51 | 52 | d.tail = d.bmix(p) 53 | 54 | // Keep own copy of the 0 to Size()-1 pending bytes. 55 | nn := copy(d.buf[:], d.tail) 56 | d.tail = d.buf[:nn] 57 | 58 | return n, nil 59 | } 60 | 61 | func (d *digest) Reset() { 62 | d.clen = 0 63 | d.tail = nil 64 | d.bmixer.reset() 65 | } 66 | -------------------------------------------------------------------------------- /service/vendor/github.com/spaolacci/murmur3/murmur64.go: -------------------------------------------------------------------------------- 1 | package murmur3 2 | 3 | import ( 4 | "hash" 5 | ) 6 | 7 | // Make sure interfaces are correctly implemented. 8 | var ( 9 | _ hash.Hash = new(digest64) 10 | _ hash.Hash64 = new(digest64) 11 | _ bmixer = new(digest64) 12 | ) 13 | 14 | // digest64 is half a digest128. 15 | type digest64 digest128 16 | 17 | func New64() hash.Hash64 { 18 | d := (*digest64)(New128().(*digest128)) 19 | return d 20 | } 21 | 22 | func (d *digest64) Sum(b []byte) []byte { 23 | h1 := d.Sum64() 24 | return append(b, 25 | byte(h1>>56), byte(h1>>48), byte(h1>>40), byte(h1>>32), 26 | byte(h1>>24), byte(h1>>16), byte(h1>>8), byte(h1)) 27 | } 28 | 29 | func (d *digest64) Sum64() uint64 { 30 | h1, _ := (*digest128)(d).Sum128() 31 | return h1 32 | } 33 | 34 | // Sum64 returns the MurmurHash3 sum of data. It is equivalent to the 35 | // following sequence (without the extra burden and the extra allocation): 36 | // hasher := New64() 37 | // hasher.Write(data) 38 | // return hasher.Sum64() 39 | func Sum64(data []byte) uint64 { 40 | d := &digest128{h1: 0, h2: 0} 41 | d.tail = d.bmix(data) 42 | d.clen = len(data) 43 | h1, _ := d.Sum128() 44 | return h1 45 | } 46 | -------------------------------------------------------------------------------- /service/vendor/github.com/zieckey/dbuf/.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | -------------------------------------------------------------------------------- /service/vendor/github.com/zieckey/goini/.gitignore: -------------------------------------------------------------------------------- 1 | *.exe 2 | coverage.out 3 | examples/ini_file_parsing/ini_file_parsing 4 | examples/inherited_ini_parsing/inherited_ini_parsing 5 | examples/memory_ini_data_parsing/memory_ini_data_parsing 6 | -------------------------------------------------------------------------------- /service/vendor/github.com/zieckey/goini/.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - tip 4 | install: 5 | - go get golang.org/x/tools/cmd/cover 6 | - go get github.com/mattn/goveralls 7 | - go get github.com/bmizerany/assert 8 | script: 9 | - go test --bench=".*" -v 10 | - go test -v -covermode=count -coverprofile=coverage.out 11 | - $HOME/gopath/bin/goveralls -coverprofile=coverage.out -service=travis-ci -repotoken $COVERALLS_TOKEN 12 | env: 13 | - secure: "ZhPXb88DDCdQDsrh1p8QPCkRIgwuWaD50TC+Y6+xAa8UFQFJtrkjvUnlaKIcococB3LT7YsJlqSHvo+qNVPkc4KJ+FHRCQUlue3qCPjgXalOCjSGra/Au8MIqpwg/c8Ijon86H4cKYVA5FEc4N3u/4QKV0grAFxCcdZbkUKQAU8=" 14 | notifications: 15 | email: false 16 | -------------------------------------------------------------------------------- /service/vendor/github.com/zieckey/goini/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 zieckey. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /service/vendor/github.com/zieckey/goini/inherited_ini.go: -------------------------------------------------------------------------------- 1 | package goini 2 | 3 | import ( 4 | "path/filepath" 5 | "errors" 6 | "log" 7 | ) 8 | 9 | // Suppress error if they are not otherwise used. 10 | var _ = log.Printf 11 | 12 | const ( 13 | InheritedFrom = "inherited_from" // The key of the INI path which will be inherited from 14 | ) 15 | 16 | // LoadInheritedINI loads an INI file which inherits from another INI 17 | // e.g: 18 | // The common.ini has contents: 19 | // project=common 20 | // ip=192.168.0.1 21 | // 22 | // And the project.ini has contents: 23 | // project=ppp 24 | // combo=ppp 25 | // inherited_from=common.ini 26 | // 27 | // The project.ini has the same configure as below : 28 | // project=ppp 29 | // combo=ppp 30 | // ip=192.168.0.1 31 | // 32 | func LoadInheritedINI(filename string) (*INI, error) { 33 | ini := New() 34 | err := ini.ParseFile(filename) 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | inherited, ok := ini.Get(InheritedFrom) 40 | if !ok { 41 | return ini, nil 42 | } 43 | 44 | inherited = GetPathByRelativePath(filename, inherited) 45 | inheritedINI, err := LoadInheritedINI(inherited) 46 | if err != nil { 47 | return nil, errors.New(err.Error() + " " + inherited) 48 | } 49 | 50 | ini.Merge(inheritedINI, false) 51 | return ini, nil 52 | } 53 | 54 | // Merge merges the data in another INI (from) to this INI (ini), and 55 | // from INI will not be changed 56 | func (ini *INI) Merge(from *INI, override bool) { 57 | for section, kv := range from.sections { 58 | for key, value := range kv { 59 | _, found := ini.SectionGet(section, key) 60 | if override || !found { 61 | ini.SectionSet(section, key, value) 62 | } 63 | } 64 | } 65 | } 66 | 67 | // GetPathByRelativePath gets the real path according to the relative file path 68 | // e.g. : 69 | // relativeFilePath = /home/goini/conf/common.conf 70 | // inheritedPath = app.conf 71 | // 72 | // and then the GetPathByRelativePath(relativeFilePath, inheritedPath) will 73 | // return /home/goini/conf/app.conf 74 | func GetPathByRelativePath(relativeFilePath, inheritedPath string) string { 75 | if filepath.IsAbs(inheritedPath) { 76 | return inheritedPath 77 | } 78 | 79 | dir, _ := filepath.Split(relativeFilePath) 80 | return filepath.Join(dir, inheritedPath) 81 | } 82 | -------------------------------------------------------------------------------- /service/vendor/github.com/zieckey/simgo/.gitignore: -------------------------------------------------------------------------------- 1 | *.pid 2 | *.exe 3 | -------------------------------------------------------------------------------- /service/vendor/github.com/zieckey/simgo/flags.go: -------------------------------------------------------------------------------- 1 | package simgo 2 | 3 | import ( 4 | "flag" 5 | ) 6 | 7 | var ConfPath = flag.String("f", "../conf/app.ini", "The config INI file path") 8 | -------------------------------------------------------------------------------- /service/vendor/github.com/zieckey/simgo/handler.go: -------------------------------------------------------------------------------- 1 | package simgo 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | "reflect" 7 | "strings" 8 | 9 | "github.com/golang/glog" 10 | "github.com/gorilla/mux" 11 | ) 12 | 13 | type HandlerFunc func(w http.ResponseWriter, r *http.Request) 14 | 15 | type Handler struct { 16 | n string // the name of the module 17 | m Module 18 | f HandlerFunc 19 | } 20 | 21 | 22 | // HandleFunc registers a new route with a matcher for the URL path. 23 | // See Route.Path() and Route.HandlerFunc(). 24 | func HandleFunc(path string, f func(http.ResponseWriter, *http.Request), m Module) *mux.Route { 25 | h := &Handler{ 26 | n: getModuleName(m), 27 | m: m, 28 | f: f, 29 | } 30 | 31 | return duxFramework.Router.HandleFunc(path, h.serveHTTP) 32 | } 33 | 34 | func (h *Handler) serveHTTP(w http.ResponseWriter, r *http.Request) { 35 | beginTime := time.Now() 36 | if duxFramework.debug { 37 | glog.Infof("%v url=[%v]", h.n, r.URL.String()) 38 | } 39 | 40 | h.f(w, r) 41 | costMs := float64(time.Since(beginTime).Nanoseconds()) / 1000000.0 42 | if duxFramework.accessLog { 43 | glog.Warningf("%v\t%v\tcost:%v", r.RemoteAddr, r.URL.String(), costMs) 44 | } 45 | } 46 | 47 | // Get the name of m using reflect mechanism 48 | func getModuleName(m Module) string { 49 | n := reflect.ValueOf(m).Elem().String() // n = "" 50 | n = strings.Trim(n, "><") // n = "simgo.MonitorModule Value" 51 | n = strings.Split(n, " ")[0] // n = "simgo.MonitorModule" 52 | return n 53 | } 54 | -------------------------------------------------------------------------------- /service/vendor/github.com/zieckey/simgo/module.go: -------------------------------------------------------------------------------- 1 | package simgo 2 | 3 | type Module interface { 4 | Initialize() error 5 | Uninitialize() error 6 | } 7 | -------------------------------------------------------------------------------- /service/vendor/github.com/zieckey/simgo/module_admin.go: -------------------------------------------------------------------------------- 1 | package simgo 2 | 3 | import ( 4 | "fmt" 5 | _ "net/http/pprof" 6 | "github.com/golang/glog" 7 | "net/http" 8 | ) 9 | 10 | type AdminModule struct { 11 | } 12 | 13 | func (m *AdminModule) Initialize() error { 14 | HandleFunc("/admin/reload", m.Reload, m).Queries("name", "", "path", "") 15 | return nil 16 | } 17 | 18 | func (m *AdminModule) Uninitialize() error { 19 | return nil 20 | } 21 | 22 | func (m *AdminModule) Reload(w http.ResponseWriter, r *http.Request) { 23 | name := r.URL.Query().Get("name") 24 | path := r.URL.Query().Get("path") 25 | 26 | glog.Info("url=[%v] name=[%v] path=[%v]\n", r.URL.String(), name, path) 27 | 28 | if len(name) == 0 { 29 | w.Write([]byte(fmt.Sprint("parameter 'name' ERROR, URI=[%v]", r.URL.String()))) 30 | return 31 | } 32 | 33 | if len(path) == 0 { 34 | w.Write([]byte(fmt.Sprint("parameter 'path' ERROR, URI=[%v]", r.URL.String()))) 35 | return 36 | } 37 | 38 | err := duxFramework.DBufManager.Reload(name, path) 39 | if err == nil { 40 | w.Write([]byte("OK")) 41 | return 42 | } 43 | 44 | w.Write([]byte(fmt.Sprintf("Reload name=%s path=%s failed : %v", name, path, err.Error()))) 45 | } 46 | -------------------------------------------------------------------------------- /service/vendor/github.com/zieckey/simgo/module_monitor.go: -------------------------------------------------------------------------------- 1 | package simgo 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "bufio" 7 | "strings" 8 | "io" 9 | "net/http" 10 | _ "net/http/pprof" 11 | 12 | "github.com/golang/glog" 13 | ) 14 | 15 | const ( 16 | StatusOK = "OK" 17 | QPollerOK = "ok" 18 | MAINTAIN = "MAINTAIN" 19 | FAILED = "FAILED" 20 | ) 21 | 22 | type MonitorModule struct { 23 | } 24 | 25 | func (m *MonitorModule) Initialize() error { 26 | HandleFunc("/status.html", m.Status, m).Methods("GET") 27 | HandleFunc("/qpoller/status.html", m.Status, m).Methods("GET") 28 | return nil 29 | } 30 | 31 | func (m *MonitorModule) Uninitialize() error { 32 | return nil 33 | } 34 | 35 | func (m *MonitorModule) Status(w http.ResponseWriter, req *http.Request) { 36 | file, err := os.Open(duxFramework.statusFilePath) 37 | if err != nil { 38 | fmt.Printf("ERROR open file <%v> failed : %v\n", duxFramework.statusFilePath, err.Error()) 39 | w.Write([]byte(FAILED)) 40 | return 41 | } 42 | defer file.Close() 43 | 44 | r := bufio.NewReader(file) 45 | line, err := r.ReadString('\n') 46 | if err != nil && err != io.EOF { 47 | fmt.Printf("ERROR read the first line from file <%v> failed : %v\n", duxFramework.statusFilePath, err.Error()) 48 | w.Write([]byte(FAILED)) 49 | return 50 | } 51 | 52 | line = strings.TrimSpace(line) 53 | if strings.ToLower(line) == "ok" { 54 | if req.URL.Path == "/status.html" { 55 | w.Write([]byte(StatusOK)) 56 | return 57 | } else if req.URL.Path == "/qpoller/status.html" { 58 | w.Write([]byte(QPollerOK)) 59 | return 60 | } else { 61 | w.Write([]byte(StatusOK)) 62 | return 63 | } 64 | } 65 | 66 | if strings.ToUpper(line) == MAINTAIN { 67 | w.Write([]byte(MAINTAIN)) 68 | return 69 | } 70 | 71 | glog.Errorf("ERROR the first line from file <%v> format wrong: [%v]\n", duxFramework.statusFilePath, line) 72 | w.Write([]byte(FAILED)) 73 | } 74 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -i 3 | ####################################### 4 | #查询测试脚本，由于生成索引分词算法和策略，并不能保证每个测试key都能找出数据，可以从原始日志weibo_data.tar.gz找关键字查询测试，如果关键字是汉字需要urlencode 5 | ###################################### 6 | 7 | 8 | if [ $# -lt 1 ]; 9 | then 10 | echo usage: $0 token 11 | echo eg : $0 good 12 | exit 13 | fi 14 | 15 | D=`date -d '-1 day' +%Y-%m-%d` 16 | 17 | host=127.0.0.1:39460 18 | curl -XPOST "http://$host/service/proxy/mdsearch" -d "{\"query\":{\"page_size\":100,\"page_number\":0,\"days\":[\"$D\"],\"business\":\"test\",\"options\":{\"pv_only\":0,\"filter\":\"\"},\"keywords\":{\"text\":\"$1\"}}}" 19 | --------------------------------------------------------------------------------