├── files ├── config │ └── redis │ │ ├── staging.ini │ │ ├── development.ini │ │ └── production.ini └── data │ └── stopwords_id.json ├── demo ├── README.md ├── stylesheet.css └── index.html ├── .gitignore ├── util ├── util.go ├── convert.go ├── collection.go ├── env.go ├── tokenizer.go ├── convert_test.go ├── env_test.go ├── collection_test.go ├── setopt.go ├── tokenizer_test.go └── setopt_test.go ├── module ├── const.go ├── response.go ├── document.go ├── module.go ├── fetcher.go ├── ranking.go ├── keyword_suggestion.go ├── searching.go └── indexing.go ├── entity ├── rank.go ├── stopwords.go ├── entity.go ├── document.go ├── document_test.go └── entity_test.go ├── router ├── app.go ├── internal.go └── router.go ├── service ├── suggesting.go ├── service.go ├── searching.go └── indexing.go ├── config ├── redis.go └── config.go ├── sdk ├── sdk_test.go └── sdk.go ├── redis ├── redis.go └── redis_test.go ├── README.md ├── main.go ├── elasthink_insomnia_api_documentation.json └── LICENSE /files/config/redis/staging.ini: -------------------------------------------------------------------------------- 1 | [RedisElasthink] 2 | Address=i.love.you:6379 3 | MaxActive=30 4 | MaxIdle=10 5 | Timeout=10 6 | -------------------------------------------------------------------------------- /files/config/redis/development.ini: -------------------------------------------------------------------------------- 1 | [RedisElasthink] 2 | Address=localhost:6379 3 | MaxActive=30 4 | MaxIdle=10 5 | Timeout=10 6 | -------------------------------------------------------------------------------- /files/config/redis/production.ini: -------------------------------------------------------------------------------- 1 | [RedisElasthink] 2 | Address=i.love.you:6379 3 | MaxActive=30 4 | MaxIdle=10 5 | Timeout=10 6 | -------------------------------------------------------------------------------- /demo/README.md: -------------------------------------------------------------------------------- 1 | DEMO 2 | ========= 3 | 4 | This folder contains files for demo UI (example) for searching using Elasthink. 5 | Feel free to use and modify ;) 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | -------------------------------------------------------------------------------- /demo/stylesheet.css: -------------------------------------------------------------------------------- 1 | .container { 2 | padding: 16px; 3 | } 4 | 5 | .listresult { 6 | padding: 16px; 7 | } 8 | 9 | input[type=text], input[type=password] { 10 | width: 100%; 11 | padding: 15px; 12 | margin: 5px 0 22px 0; 13 | display: inline-block; 14 | border: none; 15 | background: #f1f1f1; 16 | } 17 | 18 | input[type=text]:focus, input[type=password]:focus { 19 | background-color: #ddd; 20 | outline: none; 21 | } 22 | 23 | hr { 24 | border: 1px solid #f1f1f1; 25 | margin-bottom: 25px; 26 | } 27 | 28 | .searchbtn { 29 | background-color: #00aeef; 30 | color: white; 31 | padding: 16px 20px; 32 | margin: 8px 0; 33 | border: none; 34 | cursor: pointer; 35 | width: 100%; 36 | opacity: 0.9; 37 | } 38 | 39 | .searchbtn:hover { 40 | opacity:1; 41 | } 42 | -------------------------------------------------------------------------------- /util/util.go: -------------------------------------------------------------------------------- 1 | //Package util is where we place all utility funcs that can be used in every other packages 2 | package util 3 | 4 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 5 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 6 | // 7 | // This file is part of Elasthink 8 | // 9 | // Elasthink is free software: you can redistribute it and/or modify 10 | // it under the terms of the GNU General Public License as published by 11 | // the Free Software Foundation, either version 3 of the License, or 12 | // (at your option) any later version. 13 | // 14 | // Elasthink is distributed in the hope that it will be useful, 15 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | // GNU General Public License for more details. 18 | // 19 | // You should have received a copy of the GNU General Public License 20 | // along with this program. If not, see . 21 | import () 22 | -------------------------------------------------------------------------------- /module/const.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | 21 | const elasthinkInvertedIndexPrefix string = "elasthink:inverted:" 22 | const elasthinkNormalIndexPrefix string = "elasthink:normal:" 23 | -------------------------------------------------------------------------------- /module/response.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import () 21 | 22 | //Response is the universal response struct for all API 23 | type Response struct { 24 | StatusCode int 25 | ErrorMessage string 26 | Data interface{} 27 | } 28 | -------------------------------------------------------------------------------- /entity/rank.go: -------------------------------------------------------------------------------- 1 | package entity 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | 21 | //SearchResultRankData is the core struct that represent search result rank item 22 | type SearchResultRankData struct { 23 | ID int64 `json:"id"` 24 | ShowCount int `json:"showCount"` 25 | Rank int `json:"rank"` 26 | } 27 | -------------------------------------------------------------------------------- /util/convert.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import ( 21 | "strconv" 22 | ) 23 | 24 | //StringToInt64 convert a string into int64 25 | func StringToInt64(s string) int64 { 26 | result, err := strconv.ParseInt(s, 10, 64) 27 | if err != nil { 28 | return int64(0) 29 | } 30 | return result 31 | } 32 | -------------------------------------------------------------------------------- /entity/stopwords.go: -------------------------------------------------------------------------------- 1 | package entity 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import () 21 | 22 | //StopwordData is a struct that represent stopwords data that we have in a file for a specified language 23 | type StopwordData struct { 24 | Words []string `json:"words"` //Words is a collection of stopwords for a specified language 25 | } 26 | -------------------------------------------------------------------------------- /util/collection.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import () 21 | 22 | //SliceStringToInt64 converts a slice of string into a slice of int64 23 | func SliceStringToInt64(ss []string) []int64 { 24 | result := make([]int64, len(ss)) 25 | for i := 0; i < len(ss); i++ { 26 | result[i] = StringToInt64(ss[i]) 27 | } 28 | return result 29 | } 30 | -------------------------------------------------------------------------------- /util/env.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import ( 21 | "strings" 22 | ) 23 | 24 | //GetEnv gets environment from given string 25 | func GetEnv(env string) string { 26 | env = strings.Trim(env, " ") 27 | env = strings.ToLower(env) 28 | switch env { 29 | case "stg", "staging": 30 | return "staging" 31 | case "prod", "production": 32 | return "production" 33 | default: 34 | return "development" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /router/app.go: -------------------------------------------------------------------------------- 1 | package router 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import ( 21 | "net/http" 22 | 23 | "github.com/SurgicalSteel/elasthink/service" 24 | ) 25 | 26 | //RegisterAppHandler register app handlers (external endpoints) 27 | func (rw *RouterWrap) RegisterAppHandler() { 28 | subRouteV1 := rw.Router.PathPrefix("/v1").Subrouter() 29 | 30 | subRouteV1.HandleFunc("/{document_type}/_search", service.HandleSearch).Methods(http.MethodPost) 31 | subRouteV1.HandleFunc("/{document_type}/{prefix}/_suggest", service.HandleKeywordSuggestion).Methods(http.MethodGet) 32 | } 33 | -------------------------------------------------------------------------------- /router/internal.go: -------------------------------------------------------------------------------- 1 | package router 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | 21 | // this to route internal requests from admin page 22 | import ( 23 | "github.com/SurgicalSteel/elasthink/service" 24 | "net/http" 25 | ) 26 | 27 | //RegisterInternalHandler registers internal handlers (internal endpoints) 28 | func (rw *RouterWrap) RegisterInternalHandler() { 29 | subRouteInternalV1 := rw.Router.PathPrefix("/internal/v1").Subrouter() 30 | 31 | subRouteInternalV1.HandleFunc("/index/{document_type}/{document_id}", service.HandleCreateIndex).Methods(http.MethodPost) 32 | subRouteInternalV1.HandleFunc("/index/{document_type}/{document_id}", service.HandleUpdateIndex).Methods(http.MethodPut) 33 | 34 | } 35 | -------------------------------------------------------------------------------- /entity/entity.go: -------------------------------------------------------------------------------- 1 | //Package entity is the package where we store and manage elasthink's core entity 2 | package entity 3 | 4 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 5 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 6 | // 7 | // This file is part of Elasthink 8 | // 9 | // Elasthink is free software: you can redistribute it and/or modify 10 | // it under the terms of the GNU General Public License as published by 11 | // the Free Software Foundation, either version 3 of the License, or 12 | // (at your option) any later version. 13 | // 14 | // Elasthink is distributed in the hope that it will be useful, 15 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | // GNU General Public License for more details. 18 | // 19 | // You should have received a copy of the GNU General Public License 20 | // along with this program. If not, see . 21 | 22 | type entityData struct { 23 | documentTypes map[DocumentType]int 24 | stopwordData StopwordData 25 | } 26 | 27 | var Entity entityData 28 | 29 | func (e *entityData) Initialize(stopwordData StopwordData) { 30 | e.documentTypes = map[DocumentType]int{ 31 | CampaignDocument: 1, 32 | AdvertisementCampaignDocument: 1, 33 | } 34 | e.stopwordData = stopwordData 35 | } 36 | 37 | func (e *entityData) GetDocumentTypes() map[DocumentType]int { 38 | return e.documentTypes 39 | } 40 | 41 | func (e *entityData) GetStopwordData() StopwordData { 42 | return e.stopwordData 43 | } 44 | -------------------------------------------------------------------------------- /module/document.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import ( 21 | "strings" 22 | 23 | "github.com/SurgicalSteel/elasthink/entity" 24 | ) 25 | 26 | func validateDocumentType(docType string, documentTypeMap map[entity.DocumentType]int) error { 27 | docType = strings.ToLower(docType) 28 | documentType := entity.DocumentType(docType) 29 | return documentType.IsValidFromCustomDocumentType(documentTypeMap) 30 | } 31 | 32 | func getDocumentType(docType string, documentTypeMap map[entity.DocumentType]int) entity.DocumentType { 33 | docType = strings.ToLower(docType) 34 | documentType := entity.DocumentType(docType) 35 | err := documentType.IsValidFromCustomDocumentType(documentTypeMap) 36 | if err != nil { 37 | return "" 38 | } 39 | return documentType 40 | 41 | } 42 | -------------------------------------------------------------------------------- /service/suggesting.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | 21 | import ( 22 | "context" 23 | "encoding/json" 24 | "net/http" 25 | 26 | "github.com/SurgicalSteel/elasthink/module" 27 | "github.com/gorilla/mux" 28 | ) 29 | 30 | func HandleKeywordSuggestion(w http.ResponseWriter, r *http.Request) { 31 | ctx := context.Background() 32 | vars := mux.Vars(r) 33 | documentType := vars["document_type"] 34 | prefix := vars["prefix"] 35 | 36 | response := module.SuggestKeywords(ctx, documentType, prefix) 37 | responsePayload := constructResponsePayload(response) 38 | 39 | responsePayloadJSON, err := json.Marshal(responsePayload) 40 | if err != nil { 41 | http.Error(w, err.Error(), http.StatusInternalServerError) 42 | return 43 | } 44 | w.WriteHeader(response.StatusCode) 45 | w.Write(responsePayloadJSON) 46 | } 47 | -------------------------------------------------------------------------------- /service/service.go: -------------------------------------------------------------------------------- 1 | //Package service is where we handle every incoming services that are defined in router package 2 | package service 3 | 4 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 5 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 6 | // 7 | // This file is part of Elasthink 8 | // 9 | // Elasthink is free software: you can redistribute it and/or modify 10 | // it under the terms of the GNU General Public License as published by 11 | // the Free Software Foundation, either version 3 of the License, or 12 | // (at your option) any later version. 13 | // 14 | // Elasthink is distributed in the hope that it will be useful, 15 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | // GNU General Public License for more details. 18 | // 19 | // You should have received a copy of the GNU General Public License 20 | // along with this program. If not, see . 21 | import ( 22 | "github.com/SurgicalSteel/elasthink/module" 23 | "net/http" 24 | ) 25 | 26 | //HandlePing is the handler for a ping endpoint 27 | func HandlePing(w http.ResponseWriter, r *http.Request) { 28 | w.Write([]byte("PONG!")) 29 | } 30 | 31 | //ResponsePayload is the response payload struct for all API 32 | type ResponsePayload struct { 33 | ErrorMessage string `json:"errorMessage"` 34 | Data interface{} `json:"data"` 35 | } 36 | 37 | func constructResponsePayload(rawModuleResponse module.Response) ResponsePayload { 38 | return ResponsePayload{ 39 | ErrorMessage: rawModuleResponse.ErrorMessage, 40 | Data: rawModuleResponse.Data, 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /util/tokenizer.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import ( 21 | "regexp" 22 | "strings" 23 | ) 24 | 25 | //tokenize is a tokenizer function, basically to remove punctuations and (optional) to remove stopwords to a document name or a search term 26 | func Tokenize(s string, isUsingStopwordRemoval bool, stopwordSet map[string]int) map[string]int { 27 | s = strings.ToLower(s) 28 | regex := regexp.MustCompile(`[^a-zA-Z0-9 ]`) 29 | s = regex.ReplaceAllString(s, ` `) 30 | rawWords := strings.Split(s, " ") 31 | 32 | words := make([]string, 0) 33 | 34 | for _, rw := range rawWords { 35 | rw = strings.Trim(rw, " ") 36 | if len(rw) > 0 { 37 | words = append(words, rw) 38 | } 39 | } 40 | 41 | wordsSet := CreateWordSet(words) 42 | 43 | if isUsingStopwordRemoval { 44 | return WordsSetSubtraction(wordsSet, stopwordSet) 45 | } 46 | 47 | return wordsSet 48 | } 49 | -------------------------------------------------------------------------------- /util/convert_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import ( 21 | "fmt" 22 | "github.com/stretchr/testify/assert" 23 | "testing" 24 | ) 25 | 26 | func TestStringToInt64(t *testing.T) { 27 | type tcase struct { 28 | sourceString string 29 | expected int64 30 | } 31 | testCases := make(map[string]tcase) 32 | 33 | testCases["empty string"] = tcase{ 34 | sourceString: "", 35 | expected: int64(0), 36 | } 37 | 38 | testCases["negative number in string"] = tcase{ 39 | sourceString: "-666", 40 | expected: int64(-666), 41 | } 42 | 43 | testCases["positive number in string"] = tcase{ 44 | sourceString: "666", 45 | expected: int64(666), 46 | } 47 | 48 | for ktc, vtc := range testCases { 49 | fmt.Println("doing test on StringToInt64 with test case:", ktc) 50 | actual := StringToInt64(vtc.sourceString) 51 | assert.Equal(t, vtc.expected, actual) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /config/redis.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import ( 21 | "fmt" 22 | 23 | "gopkg.in/gcfg.v1" 24 | ) 25 | 26 | var redisConfig *RedisConfigWrap 27 | 28 | //RedisConfigWrap is A wrapper for reading all redis connections (can be multiple connections). 29 | type RedisConfigWrap struct { 30 | RedisElasthink RedisConfig 31 | } 32 | 33 | //RedisConfig is the basic configuration for a redis connection 34 | type RedisConfig struct { 35 | Address string 36 | MaxActive int 37 | MaxIdle int 38 | Timeout int 39 | } 40 | 41 | func readRedisConfig(path, env string) error { 42 | redisConfig = &RedisConfigWrap{} 43 | fileName := fmt.Sprintf("%s/redis/%s.ini", path, env) 44 | err := gcfg.ReadFileInto(redisConfig, fileName) 45 | return err 46 | } 47 | 48 | //GetRedisConfig gets the redis config that has been initializad 49 | func GetRedisConfig() *RedisConfigWrap { 50 | return redisConfig 51 | } 52 | -------------------------------------------------------------------------------- /module/module.go: -------------------------------------------------------------------------------- 1 | //Package module is the core package of elasthink 2 | package module 3 | 4 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 5 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 6 | // 7 | // This file is part of Elasthink 8 | // 9 | // Elasthink is free software: you can redistribute it and/or modify 10 | // it under the terms of the GNU General Public License as published by 11 | // the Free Software Foundation, either version 3 of the License, or 12 | // (at your option) any later version. 13 | // 14 | // Elasthink is distributed in the hope that it will be useful, 15 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | // GNU General Public License for more details. 18 | // 19 | // You should have received a copy of the GNU General Public License 20 | // along with this program. If not, see . 21 | import ( 22 | "github.com/SurgicalSteel/elasthink/entity" 23 | "github.com/SurgicalSteel/elasthink/redis" 24 | "github.com/SurgicalSteel/elasthink/util" 25 | ) 26 | 27 | //Module is the main struct to represent a core module 28 | type Module struct { 29 | StopwordSet map[string]int 30 | Redis *redis.Redis 31 | IsUsingStopwordRemoval bool 32 | } 33 | 34 | var moduleObj *Module 35 | 36 | //InitModule is a function that initializes a module object and its requirements (dependencies) 37 | func InitModule(stopwordData entity.StopwordData, redisObject *redis.Redis, stopwordRemovalUsage bool) { 38 | moduleObj = new(Module) 39 | moduleObj.StopwordSet = util.CreateWordSet(stopwordData.Words) 40 | moduleObj.Redis = redisObject 41 | moduleObj.IsUsingStopwordRemoval = stopwordRemovalUsage 42 | } 43 | -------------------------------------------------------------------------------- /router/router.go: -------------------------------------------------------------------------------- 1 | //Package router is the package where we define web routes on elasthink 2 | package router 3 | 4 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 5 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 6 | // 7 | // This file is part of Elasthink 8 | // 9 | // Elasthink is free software: you can redistribute it and/or modify 10 | // it under the terms of the GNU General Public License as published by 11 | // the Free Software Foundation, either version 3 of the License, or 12 | // (at your option) any later version. 13 | // 14 | // Elasthink is distributed in the hope that it will be useful, 15 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | // GNU General Public License for more details. 18 | // 19 | // You should have received a copy of the GNU General Public License 20 | // along with this program. If not, see . 21 | import ( 22 | "github.com/SurgicalSteel/elasthink/service" 23 | "github.com/gorilla/mux" 24 | "net/http" 25 | ) 26 | 27 | //RouterWrap is a custom wrapper type for router, you can add more configuration fields here 28 | type RouterWrap struct { 29 | Router *mux.Router 30 | } 31 | 32 | var routeWrap *RouterWrap 33 | 34 | // RegisterHandler is a RouterWrap 'method' to register your API endpoints. 35 | // Usually handler calls services module 36 | func (rw *RouterWrap) RegisterHandler() { 37 | rw.Router.HandleFunc("/ping", service.HandlePing).Methods(http.MethodGet) 38 | } 39 | 40 | // InitializeRoute is a function which returns new RouterWrap which has a mux's router inside 41 | func InitializeRoute() *RouterWrap { 42 | routeWrap = new(RouterWrap) 43 | routeWrap.Router = mux.NewRouter() 44 | return routeWrap 45 | } 46 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | //Package config is where we parse configurations from configuration files (.ini) into configuration objects 2 | package config 3 | 4 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 5 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 6 | // 7 | // This file is part of Elasthink 8 | // 9 | // Elasthink is free software: you can redistribute it and/or modify 10 | // it under the terms of the GNU General Public License as published by 11 | // the Free Software Foundation, either version 3 of the License, or 12 | // (at your option) any later version. 13 | // 14 | // Elasthink is distributed in the hope that it will be useful, 15 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | // GNU General Public License for more details. 18 | // 19 | // You should have received a copy of the GNU General Public License 20 | // along with this program. If not, see . 21 | import ( 22 | "log" 23 | ) 24 | 25 | const configTag string = "[CONFIG]" 26 | 27 | //InitConfig initializes all configuration (database, mq, api calls, redis, etc.) 28 | func InitConfig(path, env string) error { 29 | 30 | // err := readDatabaseConfig(path, env) 31 | // if err != nil { 32 | // log.Println(configTag, "Error on reading DB config. Detail :", err.Error()) 33 | // return err 34 | // } 35 | // 36 | // err = readMQConfig(path, env) 37 | // if err != nil { 38 | // log.Println(configTag, "Error on reading MQ config. Detail :", err.Error()) 39 | // return err 40 | // } 41 | 42 | err := readRedisConfig(path, env) 43 | if err != nil { 44 | log.Println(configTag, "Error on reading Redis cofig. Detail :", err.Error()) 45 | return err 46 | } 47 | 48 | return nil 49 | } 50 | -------------------------------------------------------------------------------- /util/env_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import ( 21 | "fmt" 22 | "github.com/stretchr/testify/assert" 23 | "testing" 24 | ) 25 | 26 | func TestGetEnv(t *testing.T) { 27 | type tcase struct { 28 | sourceEnvString string 29 | expected string 30 | } 31 | testCases := make(map[string]tcase) 32 | 33 | testCases["env staging with uppercase and space"] = tcase{ 34 | sourceEnvString: " STAGING", 35 | expected: "staging", 36 | } 37 | 38 | testCases["env stg with space"] = tcase{ 39 | sourceEnvString: " stg", 40 | expected: "staging", 41 | } 42 | 43 | testCases["empty string"] = tcase{ 44 | sourceEnvString: "", 45 | expected: "development", 46 | } 47 | 48 | testCases["invalid env"] = tcase{ 49 | sourceEnvString: "?B$A!J&I*N{G}A%N+", 50 | expected: "development", 51 | } 52 | 53 | for ktc, vtc := range testCases { 54 | fmt.Println("doing test on GetEnv with test case:", ktc) 55 | actual := GetEnv(vtc.sourceEnvString) 56 | assert.Equal(t, vtc.expected, actual) 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /util/collection_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import ( 21 | "fmt" 22 | "github.com/stretchr/testify/assert" 23 | "testing" 24 | ) 25 | 26 | func TestSliceStringToInt64(t *testing.T) { 27 | type tcase struct { 28 | sourceSlice []string 29 | expected []int64 30 | } 31 | testCases := make(map[string]tcase) 32 | 33 | testCases["empty slice"] = tcase{ 34 | sourceSlice: make([]string, 0), 35 | expected: make([]int64, 0), 36 | } 37 | 38 | testCases["slice of numbers in string"] = tcase{ 39 | sourceSlice: []string{"1", "1", "2", "3", "5", "8", "13"}, 40 | expected: []int64{1, 1, 2, 3, 5, 8, 13}, 41 | } 42 | 43 | testCases["slice of numbers in string and some random characters"] = tcase{ 44 | sourceSlice: []string{"111", "222", "333", "???", "555", "???"}, 45 | expected: []int64{111, 222, 333, 0, 555, 0}, 46 | } 47 | 48 | for ktc, vtc := range testCases { 49 | fmt.Println("doing test on SliceStringToInt64 with test case:", ktc) 50 | actual := SliceStringToInt64(vtc.sourceSlice) 51 | assert.ElementsMatch(t, actual, vtc.expected) 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /service/searching.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import ( 21 | "context" 22 | "encoding/json" 23 | "github.com/SurgicalSteel/elasthink/module" 24 | "github.com/gorilla/mux" 25 | "io/ioutil" 26 | "net/http" 27 | ) 28 | 29 | //HandleSearch handles the search for a document id (from internal & external endpoint) 30 | func HandleSearch(w http.ResponseWriter, r *http.Request) { 31 | ctx := context.Background() 32 | vars := mux.Vars(r) 33 | documentType := vars["document_type"] 34 | 35 | var requestPayload module.SearchRequestPayload 36 | 37 | body, err := ioutil.ReadAll(r.Body) 38 | if err != nil { 39 | http.Error(w, err.Error(), http.StatusInternalServerError) 40 | return 41 | } 42 | 43 | err = json.Unmarshal(body, &requestPayload) 44 | if err != nil { 45 | http.Error(w, err.Error(), http.StatusInternalServerError) 46 | return 47 | } 48 | 49 | response := module.Search(ctx, documentType, requestPayload) 50 | responsePayload := constructResponsePayload(response) 51 | 52 | responsePayloadJSON, err := json.Marshal(responsePayload) 53 | if err != nil { 54 | http.Error(w, err.Error(), http.StatusInternalServerError) 55 | return 56 | } 57 | w.WriteHeader(response.StatusCode) 58 | w.Write(responsePayloadJSON) 59 | } 60 | -------------------------------------------------------------------------------- /sdk/sdk_test.go: -------------------------------------------------------------------------------- 1 | package sdk 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | "time" 7 | 8 | er "github.com/SurgicalSteel/elasthink/redis" 9 | redigo "github.com/gomodule/redigo/redis" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestInitSDK(t *testing.T) { 14 | initializeSpec := InitializeSpec{ 15 | RedisConfig: RedisConfig{ 16 | Address: "a.redis.address", 17 | MaxActive: 30, 18 | MaxIdle: 10, 19 | Timeout: 10, 20 | }, 21 | SdkConfig: SdkConfig{ 22 | IsUsingStopWordsRemoval: true, 23 | StopWordRemovalData: getDummyStopwords(), 24 | AvailableDocumentType: getDummyDocumentType(), 25 | }, 26 | } 27 | 28 | expectedRedis := &er.Redis{ 29 | Pool: &redigo.Pool{ 30 | MaxIdle: initializeSpec.RedisConfig.MaxIdle, 31 | MaxActive: initializeSpec.RedisConfig.MaxActive, 32 | IdleTimeout: time.Duration(initializeSpec.RedisConfig.Timeout) * time.Second, 33 | Dial: func() (redigo.Conn, error) { 34 | return redigo.Dial("tcp", initializeSpec.RedisConfig.Address) 35 | }, 36 | }, 37 | } 38 | 39 | actualElasthinkSDK := Initialize(initializeSpec) 40 | assert.Equal(t, expectedRedis.Pool.MaxActive, actualElasthinkSDK.Redis.Pool.MaxActive) 41 | assert.Equal(t, expectedRedis.Pool.MaxIdle, actualElasthinkSDK.Redis.Pool.MaxIdle) 42 | assert.Equal(t, expectedRedis.Pool.IdleTimeout, actualElasthinkSDK.Redis.Pool.IdleTimeout) 43 | assert.NotNil(t, actualElasthinkSDK.Redis.Pool.Dial) 44 | assert.Equal(t, true, actualElasthinkSDK.isUsingStopWordsRemoval) 45 | 46 | if len(actualElasthinkSDK.availableDocumentType) != len(getDummyDocumentType()) { 47 | t.Fatal("Result 'DocumentTypes' is not same with what we expected.") 48 | } 49 | 50 | equalStopwords := reflect.DeepEqual(getDummyStopwords(), actualElasthinkSDK.stopWordRemovalData) 51 | if !equalStopwords { 52 | t.Fatal("Result 'Stopwords' is not same with what we expected.") 53 | } 54 | } 55 | 56 | //private functions 57 | func getDummyInitializedSDK() ElasthinkSDK { 58 | return ElasthinkSDK{} 59 | } 60 | 61 | func getDummyStopwords() []string { 62 | return []string{"ada", "adalah", "adanya", "adapun", "waktu", "waktunya", "walau", "walaupun", "wong", "yaitu", "yakin", "yakni", "yang"} 63 | } 64 | 65 | func getDummyDocumentType() []string { 66 | return []string{"campaign", "advertisement"} 67 | } 68 | -------------------------------------------------------------------------------- /entity/document.go: -------------------------------------------------------------------------------- 1 | package entity 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import ( 21 | "errors" 22 | ) 23 | 24 | //DocumentType is a type that represent document type 25 | type DocumentType string 26 | 27 | const ( 28 | //These document types below are just for example. 29 | //You can create your own document type and don't forget to modify the module/document.go for document type validation. 30 | 31 | //AdvertisementCampaignDocument is the document type that represent Advertisement Campaign document type 32 | AdvertisementCampaignDocument DocumentType = "advcampaign" 33 | //CampaignDocument is the document type that represent Campaign document type (coupon for promotion) 34 | CampaignDocument DocumentType = "campaign" 35 | ) 36 | 37 | /* 38 | //IsValid checks if the document type is a valid (registered) document type in entity const 39 | func (dt DocumentType) IsValid() error { 40 | switch dt { 41 | case AdvertisementCampaignDocument, CampaignDocument: 42 | return nil 43 | } 44 | return errors.New("Invalid Document Type") 45 | } 46 | */ 47 | //IsValidFromCustomType checks if the document type is a valid (registered) in a documentTypeMap (Custom Document Type) 48 | func (dt DocumentType) IsValidFromCustomDocumentType(documentTypeMap map[DocumentType]int) error { 49 | if _, ok := documentTypeMap[dt]; ok { 50 | return nil 51 | } 52 | return errors.New("Invalid Document Type") 53 | } 54 | -------------------------------------------------------------------------------- /util/setopt.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import () 21 | 22 | //CreateWordSet create a set of words from a slice of string 23 | func CreateWordSet(words []string) map[string]int { 24 | result := make(map[string]int) 25 | 26 | for _, vw := range words { 27 | if _, okr := result[vw]; !okr { 28 | result[vw] = 1 29 | } 30 | } 31 | 32 | return result 33 | } 34 | 35 | //WordsSetUnion do a union from two given words sets 36 | func WordsSetUnion(ma, mb map[string]int) map[string]int { 37 | result := make(map[string]int) 38 | for ka, _ := range ma { 39 | if _, oka := result[ka]; !oka { 40 | result[ka] = 1 41 | } 42 | } 43 | 44 | for kb, _ := range mb { 45 | if _, okb := result[kb]; !okb { 46 | result[kb] = 1 47 | } 48 | } 49 | return result 50 | } 51 | 52 | //WordsSetSubtraction do a substraction between two sets ma - mb 53 | func WordsSetSubtraction(ma, mb map[string]int) map[string]int { 54 | result := ma 55 | for kb, _ := range mb { 56 | if _, okb := result[kb]; okb { 57 | delete(result, kb) 58 | } 59 | } 60 | return result 61 | } 62 | 63 | //WordsSetIntersection find the intersection between two sets (ma & mb) 64 | func WordsSetIntersection(ma, mb map[string]int) map[string]int { 65 | result := make(map[string]int) 66 | for ka, _ := range ma { 67 | if _, okb := mb[ka]; okb { 68 | result[ka] = 1 69 | } 70 | } 71 | return result 72 | } 73 | -------------------------------------------------------------------------------- /entity/document_test.go: -------------------------------------------------------------------------------- 1 | package entity 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | 21 | import ( 22 | "errors" 23 | "fmt" 24 | "github.com/stretchr/testify/assert" 25 | "testing" 26 | ) 27 | 28 | func TestIsValidFromCustomDocumentType(t *testing.T) { 29 | type tcase struct { 30 | documentType DocumentType 31 | documentTypeMap map[DocumentType]int 32 | expectedError error 33 | } 34 | var testDocumentTypeA DocumentType = "type_a" 35 | var testDocumentTypeB DocumentType = "type_b" 36 | var testDocumentTypeC DocumentType = "type_c" 37 | var testDocumentTypeD DocumentType = "type_d" 38 | testCases := make(map[string]tcase) 39 | testCases["Valid Document Type"] = tcase{ 40 | documentType: testDocumentTypeD, 41 | documentTypeMap: map[DocumentType]int{ 42 | testDocumentTypeA: 1, 43 | testDocumentTypeB: 1, 44 | testDocumentTypeC: 1, 45 | testDocumentTypeD: 1, 46 | }, 47 | expectedError: nil, 48 | } 49 | testCases["Invalid Document Type"] = tcase{ 50 | documentType: testDocumentTypeD, 51 | documentTypeMap: map[DocumentType]int{ 52 | testDocumentTypeA: 1, 53 | testDocumentTypeB: 1, 54 | testDocumentTypeC: 1, 55 | }, 56 | expectedError: errors.New("Invalid Document Type"), 57 | } 58 | 59 | for ktc, vtc := range testCases { 60 | fmt.Println("doing test on IsValid of Document Type with test case:", ktc) 61 | actual := vtc.documentType.IsValidFromCustomDocumentType(vtc.documentTypeMap) 62 | assert.Equal(t, vtc.expectedError, actual) 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /module/fetcher.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import ( 21 | "fmt" 22 | "log" 23 | "strings" 24 | 25 | "github.com/SurgicalSteel/elasthink/entity" 26 | "github.com/SurgicalSteel/elasthink/util" 27 | ) 28 | 29 | func fetchWordIndexSets(documentType entity.DocumentType, searchTermSet map[string]int) map[string][]int64 { 30 | result := make(map[string][]int64) 31 | 32 | // set key format --> elasthink:inverted:documentType:word 33 | for k := range searchTermSet { 34 | key := fmt.Sprintf("%s%s:%s", elasthinkInvertedIndexPrefix, documentType, k) 35 | members, err := moduleObj.Redis.SMembers(key) 36 | if err != nil { 37 | log.Println("[MODULE][FETCHER] Failed to get members of key :", key) 38 | continue 39 | } 40 | documentIds := util.SliceStringToInt64(members) 41 | result[k] = documentIds 42 | } 43 | 44 | return result 45 | } 46 | 47 | func fetchKeywords(documentType entity.DocumentType, prefix string) ([]string, error) { 48 | prefixKey := fmt.Sprintf("%s%s:%s", elasthinkInvertedIndexPrefix, documentType, prefix) 49 | rawKeys, err := moduleObj.Redis.KeysPrefix(prefixKey) 50 | if err != nil { 51 | log.Printf("[MODULE][FETCHER] Failed to get keys with prefix :%s Detail :%s\n", prefixKey, err.Error()) 52 | return []string{}, err 53 | } 54 | finalKeywords := make([]string, len(rawKeys)) 55 | trimPrefix := fmt.Sprintf("%s%s:", elasthinkInvertedIndexPrefix, documentType) 56 | for i := 0; i < len(rawKeys); i++ { 57 | rawKey := rawKeys[i] 58 | finalKeywords[i] = strings.TrimPrefix(rawKey, trimPrefix) 59 | } 60 | return finalKeywords, nil 61 | } 62 | -------------------------------------------------------------------------------- /util/tokenizer_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import ( 21 | "fmt" 22 | "reflect" 23 | "testing" 24 | ) 25 | 26 | func TestTokenizer(t *testing.T) { 27 | type tcase struct { 28 | sourceString string 29 | isUsingStopwordRemoval bool 30 | stopwordSet map[string]int 31 | expected map[string]int 32 | } 33 | 34 | testCases := make(map[string]tcase) 35 | 36 | testCases["normal testcase"] = tcase{ 37 | sourceString: "quick brown fox", 38 | isUsingStopwordRemoval: false, 39 | stopwordSet: make(map[string]int), 40 | expected: map[string]int{ 41 | "quick": 1, 42 | "brown": 1, 43 | "fox": 1, 44 | }, 45 | } 46 | 47 | testCases["using stopwords"] = tcase{ 48 | sourceString: "quick brown fox", 49 | isUsingStopwordRemoval: true, 50 | stopwordSet: map[string]int{ 51 | "quick": 1, 52 | "fox": 1, 53 | }, 54 | expected: map[string]int{ 55 | "brown": 1, 56 | }, 57 | } 58 | 59 | testCases["invalid word"] = tcase{ 60 | sourceString: " ==== ==== ", 61 | isUsingStopwordRemoval: false, 62 | stopwordSet: make(map[string]int), 63 | expected: make(map[string]int), 64 | } 65 | 66 | for ktc, vtc := range testCases { 67 | fmt.Println("doing test on Tokenizer with test case:", ktc) 68 | actual := Tokenize(vtc.sourceString, vtc.isUsingStopwordRemoval, vtc.stopwordSet) 69 | isEqual := reflect.DeepEqual(vtc.expected, actual) 70 | if !isEqual { 71 | t.Fatal("Result WordSet is not same with what we expected.") 72 | continue 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /module/ranking.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import ( 21 | "github.com/SurgicalSteel/elasthink/entity" 22 | "sort" 23 | ) 24 | 25 | //RankByShowCount is the additional struct for document ranking purpose based on its ShowCount 26 | type RankByShowCount []entity.SearchResultRankData 27 | 28 | func (r RankByShowCount) Len() int { return len(r) } 29 | func (r RankByShowCount) Less(i, j int) bool { return r[i].ShowCount > r[j].ShowCount } 30 | func (r RankByShowCount) Swap(i, j int) { r[i], r[j] = r[j], r[i] } 31 | 32 | //rankSearchResult ranks search result (document id by its appeareance count). word indexes is a map with word as a key and slice of ids as value. Returns ordered search result rank slice. 33 | func rankSearchResult(wordIndexes map[string][]int64) []entity.SearchResultRankData { 34 | counterMap := make(map[int64]int) 35 | for _, ids := range wordIndexes { 36 | for i := 0; i < len(ids); i++ { 37 | if vcm, ok := counterMap[ids[i]]; ok { 38 | counterMap[ids[i]] = vcm + 1 39 | } else { 40 | counterMap[ids[i]] = 1 41 | } 42 | } 43 | } 44 | 45 | result := make([]entity.SearchResultRankData, len(counterMap)) 46 | 47 | iterator := 0 48 | for kcm, vcm := range counterMap { 49 | result[iterator] = entity.SearchResultRankData{ 50 | ID: kcm, 51 | ShowCount: vcm, 52 | } 53 | iterator++ 54 | } 55 | 56 | //sort by appeareance count (descending) 57 | sort.Sort(RankByShowCount(result)) 58 | 59 | //assign rank to each search result data 60 | for i := 0; i < len(result); i++ { 61 | temp := result[i] 62 | temp.Rank = i + 1 63 | result[i] = temp 64 | } 65 | 66 | return result 67 | } 68 | -------------------------------------------------------------------------------- /module/keyword_suggestion.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | 21 | import ( 22 | "context" 23 | "errors" 24 | "net/http" 25 | "sort" 26 | "strings" 27 | 28 | "github.com/SurgicalSteel/elasthink/entity" 29 | ) 30 | 31 | func validateKeywordSuggestionRequest(documentType, prefix string) error { 32 | if len(strings.Trim(prefix, " ")) == 0 { 33 | return errors.New("Keyword prefix is required to get suggested keywords") 34 | } 35 | 36 | if len(strings.Trim(documentType, " ")) == 0 { 37 | return errors.New("Document Type is required") 38 | } 39 | 40 | err := validateDocumentType(documentType, entity.Entity.GetDocumentTypes()) 41 | if err != nil { 42 | return err 43 | } 44 | 45 | return nil 46 | } 47 | 48 | //KeywordSuggestionResponsePayload is the universal response payload for keyword suggestion API handler 49 | type KeywordSuggestionResponsePayload struct { 50 | SortedKeywords []string `json:"sortedKeywords"` 51 | } 52 | 53 | //SuggestKeywords is the core function for keyword suggestion (by document type and prefix) 54 | func SuggestKeywords(ctx context.Context, documentType, prefix string) Response { 55 | err := validateKeywordSuggestionRequest(documentType, prefix) 56 | if err != nil { 57 | return Response{ 58 | StatusCode: http.StatusBadRequest, 59 | ErrorMessage: err.Error(), 60 | Data: nil, 61 | } 62 | } 63 | prefix = strings.ToLower(prefix) 64 | docType := getDocumentType(documentType, entity.Entity.GetDocumentTypes()) 65 | keywords, err := fetchKeywords(docType, prefix) 66 | if err != nil { 67 | return Response{ 68 | StatusCode: http.StatusInternalServerError, 69 | ErrorMessage: "There's an error when suggesting keywords.", 70 | Data: nil, 71 | } 72 | } 73 | sort.Strings(keywords) 74 | return Response{ 75 | StatusCode: http.StatusOK, 76 | Data: keywords, 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /entity/entity_test.go: -------------------------------------------------------------------------------- 1 | package entity 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | 21 | import ( 22 | "github.com/stretchr/testify/assert" 23 | "testing" 24 | ) 25 | 26 | func TestInitializeEntity(t *testing.T) { 27 | entityDataMock := entityData{} 28 | stopwordDataMock := StopwordData{ 29 | Words: []string{"tidak", "tidakkah", "tidaklah"}, 30 | } 31 | 32 | assert.Nil(t, entityDataMock.documentTypes) 33 | assert.Equal(t, 0, len(entityDataMock.stopwordData.Words)) 34 | 35 | entityDataMock.Initialize(stopwordDataMock) 36 | 37 | assert.NotNil(t, entityDataMock.documentTypes) 38 | assert.Equal(t, stopwordDataMock.Words, entityDataMock.stopwordData.Words) 39 | if len(entityDataMock.documentTypes) == 0 { 40 | t.Error("Expected non empty Document Types, but found empty document types") 41 | } 42 | } 43 | 44 | func TestGetDocumentTypes(t *testing.T) { 45 | entityDataMock := entityData{} 46 | stopwordDataMock := StopwordData{ 47 | Words: []string{"tidak", "tidakkah", "tidaklah"}, 48 | } 49 | 50 | assert.Nil(t, entityDataMock.GetDocumentTypes()) 51 | assert.Equal(t, 0, len(entityDataMock.stopwordData.Words)) 52 | 53 | entityDataMock.Initialize(stopwordDataMock) 54 | 55 | assert.NotNil(t, entityDataMock.GetDocumentTypes()) 56 | 57 | if len(entityDataMock.GetDocumentTypes()) == 0 { 58 | t.Error("Expected non empty Document Types, but found empty document types") 59 | } 60 | } 61 | 62 | func TestGetStopwordData(t *testing.T) { 63 | entityDataMock := entityData{} 64 | stopwordDataMock := StopwordData{ 65 | Words: []string{"tidak", "tidakkah", "tidaklah"}, 66 | } 67 | 68 | assert.Nil(t, entityDataMock.GetDocumentTypes()) 69 | assert.Equal(t, 0, len(entityDataMock.stopwordData.Words)) 70 | entityDataMock.Initialize(stopwordDataMock) 71 | assert.Equal(t, 3, len(entityDataMock.stopwordData.Words)) 72 | 73 | actualStopwordData := entityDataMock.GetStopwordData() 74 | assert.Equal(t, stopwordDataMock, actualStopwordData) 75 | 76 | if len(entityDataMock.GetDocumentTypes()) == 0 { 77 | t.Error("Expected non empty Document Types, but found empty document types") 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /redis/redis.go: -------------------------------------------------------------------------------- 1 | //Package redis is where we place all funcs related to redis operations 2 | package redis 3 | 4 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 5 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 6 | // 7 | // This file is part of Elasthink 8 | // 9 | // Elasthink is free software: you can redistribute it and/or modify 10 | // it under the terms of the GNU General Public License as published by 11 | // the Free Software Foundation, either version 3 of the License, or 12 | // (at your option) any later version. 13 | // 14 | // Elasthink is distributed in the hope that it will be useful, 15 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | // GNU General Public License for more details. 18 | // 19 | // You should have received a copy of the GNU General Public License 20 | // along with this program. If not, see . 21 | import ( 22 | "errors" 23 | "fmt" 24 | "github.com/SurgicalSteel/elasthink/config" 25 | redigo "github.com/gomodule/redigo/redis" 26 | "strings" 27 | "sync" 28 | "time" 29 | ) 30 | 31 | // Redis main struct 32 | type Redis struct { 33 | Pool *redigo.Pool 34 | mutex sync.Mutex 35 | } 36 | 37 | //NetworkTCP is the default network TCP 38 | const NetworkTCP string = "tcp" 39 | 40 | //InitRedis is a func to initialize redis that we are going to use 41 | func InitRedis(redisConfig config.RedisConfigWrap) *Redis { 42 | newRedis := &Redis{ 43 | Pool: &redigo.Pool{ 44 | MaxIdle: redisConfig.RedisElasthink.MaxIdle, 45 | MaxActive: redisConfig.RedisElasthink.MaxActive, 46 | IdleTimeout: time.Duration(redisConfig.RedisElasthink.Timeout) * time.Second, 47 | Dial: func() (redigo.Conn, error) { 48 | return redigo.Dial(NetworkTCP, redisConfig.RedisElasthink.Address) 49 | }, 50 | }, 51 | } 52 | return newRedis 53 | } 54 | 55 | // SAdd add an item into a set 56 | func (r *Redis) SAdd(key string, args []interface{}) (int64, error) { 57 | conn := r.Pool.Get() 58 | defer conn.Close() 59 | 60 | return redigo.Int64(conn.Do("SADD", redigo.Args{key}.AddFlat(args)...)) 61 | } 62 | 63 | // SMembers get members of a set 64 | func (r *Redis) SMembers(key string) ([]string, error) { 65 | conn := r.Pool.Get() 66 | defer conn.Close() 67 | 68 | return redigo.Strings(conn.Do("SMEMBERS", key)) 69 | } 70 | 71 | // SRem remove an item from a set 72 | func (r *Redis) SRem(keyRedis string, members []interface{}) (int64, error) { 73 | conn := r.Pool.Get() 74 | defer conn.Close() 75 | 76 | return redigo.Int64(conn.Do("SREM", redigo.Args{keyRedis}.AddFlat(members)...)) 77 | } 78 | 79 | // KeysPrefix get keys by a defined prefix 80 | func (r *Redis) KeysPrefix(prefix string) ([]string, error) { 81 | prefix = strings.Trim(prefix, " ") 82 | if len(prefix) == 0 { 83 | return make([]string, 0), errors.New("Prefix must be defined!") 84 | } 85 | 86 | conn := r.Pool.Get() 87 | defer conn.Close() 88 | 89 | finalKeyPrefix := fmt.Sprintf("%s*", prefix) 90 | 91 | return redigo.Strings(conn.Do("KEYS", finalKeyPrefix)) 92 | } 93 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | elasthink 2 | ========= 3 | 4 | [![GoDoc](https://godoc.org/github.com/SurgicalSteel/elasthink?status.png)](https://godoc.org/github.com/SurgicalSteel/elasthink) 5 | 6 | An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 7 | 8 | ## Table of Contents 9 | 10 | * [Use Cases](#use-cases) 11 | * [Elasthink SDK](#elasthink-sdk) 12 | * [Installation](#installation) 13 | * [Documentation](#documentation) 14 | * [Dependencies](#dependencies) 15 | * [Reference](#reference) 16 | * [Additional Note](#additional-note) 17 | 18 | ## Use Cases 19 | 1. Create Index for a document (needs `document_type`, `document_id`, and `document_name`) 20 | 2. Update Index of a document (needs `document_type`, `document_id`, and `document_name`) 21 | 3. Search document_id by document name using search term (needs `document_type` and `search_term`) 22 | 4. Keyword Suggestion by prefix (needs `document_type` and `keyword_prefix`) 23 | 24 | ## Elasthink SDK 25 | Coming Soon! 26 | With the SDK, you can run all the core functionality of elasthink from your go service by providing a redis connection and without setting up a dedicated elasthink server. 27 | Currently, the SDK is on preparation for the release. So stay tuned to get the latest update. 28 | 29 | ## Installation 30 | 1. To install elasthink, you need to run `$ go get github.com/SurgicalSteel/elasthink` 31 | 2. Then you need to specify your redis addresses for each environment in `files/config/redis` folder 32 | 3. To start with your own document, you need to modify the document type const in `entity/document.go` and its validation function in `module/document.go` 33 | 4. To build elasthink, run `$ go build` 34 | 5. To view all available flags, run `$ ./elasthink -h` 35 | 6. To run elasthink, run `$ ./elasthink -env={your-environment} -swr={stopword Removal option (true/false)}` and your elasthink web service should run on `localhost:9000` 36 | 37 | 38 | ## Documentation 39 | API documentation (insomnia format) is available in the `elasthink_insomnia_api_documentation.json`. You can open it using [Insomnia REST Client](https://insomnia.rest/) 40 | For code documentation, we use standard godoc as our code documentation tool. To view the code documentation, follow these steps : 41 | 1. Open your terminal, head to this cloned repo (SurgicalSteel/elasthink) 42 | 2. run `godoc -http=:6060` (this will trigger godoc at port 6060) 43 | 3. Open your browser, and hit `http://127.0.0.1:6060/pkg/github.com/SurgicalSteel/elasthink/` 44 | 45 | ## Dependencies 46 | 1. [gorilla/mux](https://github.com/gorilla/mux) 47 | 2. [gomodule/redigo](https://github.com/gomodule/redigo) 48 | 3. [gcfg.v1](https://gopkg.in/gcfg.v1) 49 | 4. [stretchr/testify](https://github.com/stretchr/testify) 50 | 5. [rafaeljusto/redigomock](https://github.com/rafaeljusto/redigomock) 51 | 52 | ## Reference 53 | [E-Book Redis in Action Part 2 Chapter 7](https://redislabs.com/ebook/part-2-core-concepts/chapter-7-search-based-applications/7-1-searching-in-redis/7-1-1-basic-search-theory/) 54 | 55 | ## Additional Note 56 | Currently, elasthink supports stopwords removal option when doing tokenization for document name and search term. 57 | But, for now we only support stopwords removal for bahasa Indonesia (Indonesian). 58 | -------------------------------------------------------------------------------- /module/searching.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import ( 21 | "context" 22 | "errors" 23 | "net/http" 24 | "strings" 25 | 26 | "github.com/SurgicalSteel/elasthink/entity" 27 | "github.com/SurgicalSteel/elasthink/util" 28 | ) 29 | 30 | //SearchRequestPayload is the universal request payload for search handlers 31 | type SearchRequestPayload struct { 32 | SearchTerm string `json:"searchTerm"` 33 | } 34 | 35 | func validateSearchRequestPayload(documentType, searchTerm string) error { 36 | if len(strings.Trim(searchTerm, " ")) == 0 { 37 | return errors.New("Search Term is required") 38 | } 39 | 40 | if len(strings.Trim(documentType, " ")) == 0 { 41 | return errors.New("Document Type is required") 42 | } 43 | 44 | err := validateDocumentType(documentType, entity.Entity.GetDocumentTypes()) 45 | if err != nil { 46 | return err 47 | } 48 | 49 | return nil 50 | } 51 | 52 | //SearchResponsePayload is the universal response payload for search handlers 53 | type SearchResponsePayload struct { 54 | RankedResultList []entity.SearchResultRankData `json:"rankedResultList"` 55 | } 56 | 57 | //Search is the core function of searching a document 58 | func Search(ctx context.Context, documentType string, requestPayload SearchRequestPayload) Response { 59 | err := validateSearchRequestPayload(documentType, requestPayload.SearchTerm) 60 | if err != nil { 61 | return Response{ 62 | StatusCode: http.StatusBadRequest, 63 | ErrorMessage: err.Error(), 64 | Data: nil, 65 | } 66 | } 67 | 68 | searchTermSet := util.Tokenize(requestPayload.SearchTerm, moduleObj.IsUsingStopwordRemoval, moduleObj.StopwordSet) 69 | if len(searchTermSet) == 0 { 70 | return Response{ 71 | StatusCode: http.StatusOK, 72 | ErrorMessage: "", 73 | Data: nil, 74 | } 75 | } 76 | 77 | docType := getDocumentType(documentType, entity.Entity.GetDocumentTypes()) 78 | 79 | wordIndexSets := fetchWordIndexSets(docType, searchTermSet) 80 | 81 | if len(wordIndexSets) == 0 { 82 | return Response{ 83 | StatusCode: http.StatusOK, 84 | ErrorMessage: "", 85 | Data: nil, 86 | } 87 | } 88 | 89 | rankedSearchResult := rankSearchResult(wordIndexSets) 90 | searchResponsePayload := SearchResponsePayload{RankedResultList: rankedSearchResult} 91 | 92 | return Response{ 93 | StatusCode: http.StatusOK, 94 | ErrorMessage: "", 95 | Data: searchResponsePayload, 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /service/indexing.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import ( 21 | "context" 22 | "encoding/json" 23 | "github.com/SurgicalSteel/elasthink/module" 24 | "github.com/SurgicalSteel/elasthink/util" 25 | "github.com/gorilla/mux" 26 | "io/ioutil" 27 | "net/http" 28 | ) 29 | 30 | //HandleCreateIndex handles create index (from internal endpoint) 31 | func HandleCreateIndex(w http.ResponseWriter, r *http.Request) { 32 | ctx := context.Background() 33 | vars := mux.Vars(r) 34 | documentType := vars["document_type"] 35 | documentIDRaw := vars["document_id"] 36 | documentID := util.StringToInt64(documentIDRaw) 37 | 38 | var requestPayload module.CreateIndexRequestPayload 39 | 40 | body, err := ioutil.ReadAll(r.Body) 41 | if err != nil { 42 | http.Error(w, err.Error(), http.StatusInternalServerError) 43 | return 44 | } 45 | 46 | err = json.Unmarshal(body, &requestPayload) 47 | if err != nil { 48 | http.Error(w, err.Error(), http.StatusInternalServerError) 49 | return 50 | } 51 | 52 | response := module.CreateIndex(ctx, documentID, documentType, requestPayload) 53 | responsePayload := constructResponsePayload(response) 54 | 55 | responsePayloadJSON, err := json.Marshal(responsePayload) 56 | if err != nil { 57 | http.Error(w, err.Error(), http.StatusInternalServerError) 58 | return 59 | } 60 | 61 | w.WriteHeader(response.StatusCode) 62 | w.Write(responsePayloadJSON) 63 | } 64 | 65 | //HandleUpdateIndex handles update index (from internal endpoint) 66 | func HandleUpdateIndex(w http.ResponseWriter, r *http.Request) { 67 | ctx := context.Background() 68 | vars := mux.Vars(r) 69 | documentType := vars["document_type"] 70 | documentIDRaw := vars["document_id"] 71 | documentID := util.StringToInt64(documentIDRaw) 72 | 73 | var requestPayload module.UpdateIndexRequestPayload 74 | 75 | body, err := ioutil.ReadAll(r.Body) 76 | if err != nil { 77 | http.Error(w, err.Error(), http.StatusInternalServerError) 78 | return 79 | } 80 | 81 | err = json.Unmarshal(body, &requestPayload) 82 | if err != nil { 83 | http.Error(w, err.Error(), http.StatusInternalServerError) 84 | return 85 | } 86 | 87 | response := module.UpdateIndex(ctx, documentID, documentType, requestPayload) 88 | responsePayload := constructResponsePayload(response) 89 | 90 | responsePayloadJSON, err := json.Marshal(responsePayload) 91 | if err != nil { 92 | http.Error(w, err.Error(), http.StatusInternalServerError) 93 | return 94 | } 95 | 96 | w.WriteHeader(response.StatusCode) 97 | w.Write(responsePayloadJSON) 98 | } 99 | -------------------------------------------------------------------------------- /demo/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Elasthink Demo UI 4 | 5 | 9 | 10 | 63 | 64 | 65 | 66 |

Elasthink Demo UI

67 |
68 | 69 | 70 | 71 | 72 | 76 | 77 | 78 |
79 |
80 | 81 | 82 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | //Package main is where all the magic happens ;) 2 | package main 3 | 4 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 5 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 6 | // 7 | // This file is part of Elasthink 8 | // 9 | // Elasthink is free software: you can redistribute it and/or modify 10 | // it under the terms of the GNU General Public License as published by 11 | // the Free Software Foundation, either version 3 of the License, or 12 | // (at your option) any later version. 13 | // 14 | // Elasthink is distributed in the hope that it will be useful, 15 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | // GNU General Public License for more details. 18 | // 19 | // You should have received a copy of the GNU General Public License 20 | // along with this program. If not, see . 21 | 22 | import ( 23 | "context" 24 | "encoding/json" 25 | "flag" 26 | "github.com/SurgicalSteel/elasthink/config" 27 | "github.com/SurgicalSteel/elasthink/entity" 28 | "github.com/SurgicalSteel/elasthink/module" 29 | "github.com/SurgicalSteel/elasthink/redis" 30 | "github.com/SurgicalSteel/elasthink/router" 31 | "github.com/SurgicalSteel/elasthink/util" 32 | "io/ioutil" 33 | "log" 34 | "net/http" 35 | "os" 36 | "os/signal" 37 | "syscall" 38 | "time" 39 | ) 40 | 41 | const stopwordsFileName string = "files/data/stopwords_id.json" 42 | const configPath string = "files/config" 43 | 44 | func main() { 45 | log.SetOutput(os.Stdout) 46 | environmentFlag := flag.String("env", "development", "specify your environment for running elasthink (development / staging / production)") 47 | stopwordsRemovalUsageFlag := flag.Bool("swr", false, "option to use stopwords removal during create index & update index & searching (default false)") 48 | 49 | flag.Parse() 50 | 51 | environment := util.GetEnv(*environmentFlag) 52 | log.Println("Environment for elasthink:", environment) 53 | 54 | isUsingStopwordsRemoval := *stopwordsRemovalUsageFlag 55 | 56 | //read stop words file 57 | stopwordData, err := readStopwordsFile(stopwordsFileName) 58 | if err != nil { 59 | log.Fatalln(err) 60 | return 61 | } 62 | 63 | //init config 64 | err = config.InitConfig(configPath, environment) 65 | if err != nil { 66 | log.Fatalln(err) 67 | return 68 | } 69 | 70 | //init redis 71 | redisObject := redis.InitRedis(*config.GetRedisConfig()) 72 | 73 | //init entity data 74 | entity.Entity.Initialize(stopwordData) 75 | 76 | //init module 77 | module.InitModule(entity.Entity.GetStopwordData(), redisObject, isUsingStopwordsRemoval) 78 | 79 | routing := router.InitializeRoute() 80 | routing.RegisterHandler() 81 | routing.RegisterAppHandler() 82 | routing.RegisterInternalHandler() 83 | server := &http.Server{ 84 | Addr: "0.0.0.0:9000", 85 | // Good practice to set timeouts to avoid Slowloris attacks. 86 | WriteTimeout: time.Second * 15, 87 | ReadTimeout: time.Second * 15, 88 | IdleTimeout: time.Second * 60, 89 | Handler: routing.Router, 90 | } 91 | log.Println("Starting elasthink in port 9000...") 92 | done := make(chan os.Signal, 1) 93 | signal.Notify(done, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) 94 | 95 | go func() { 96 | if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { 97 | log.Fatalln("Failed to start service! Reason :", err.Error()) 98 | } 99 | }() 100 | log.Println("Server Started") 101 | 102 | <-done 103 | log.Println("Server Stopped") 104 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 105 | 106 | defer func() { 107 | // extra handling here 108 | cancel() 109 | }() 110 | 111 | if err := server.Shutdown(ctx); err != nil { 112 | log.Fatalf("Server Shutdown Failed:%+v\n", err) 113 | } 114 | 115 | log.Println("Server Exited Properly") 116 | log.Println("👋") 117 | } 118 | 119 | func readStopwordsFile(fileName string) (entity.StopwordData, error) { 120 | var stopwordData entity.StopwordData 121 | 122 | rawStopwordsFile, err := os.Open(fileName) 123 | if err != nil { 124 | log.Println("Failed to open stopwords file. Reason :", err.Error()) 125 | return stopwordData, err 126 | } 127 | defer rawStopwordsFile.Close() 128 | 129 | rawStopwordsBody, err := ioutil.ReadAll(rawStopwordsFile) 130 | if err != nil { 131 | log.Println("Failed to read stopwords file. Reason :", err.Error()) 132 | return stopwordData, err 133 | } 134 | 135 | err = json.Unmarshal(rawStopwordsBody, &stopwordData) 136 | if err != nil { 137 | log.Println("Failed to unmarshal raw stopwords file. Reason :", err.Error()) 138 | return stopwordData, err 139 | } 140 | 141 | return stopwordData, nil 142 | } 143 | -------------------------------------------------------------------------------- /redis/redis_test.go: -------------------------------------------------------------------------------- 1 | package redis 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import ( 21 | "errors" 22 | "github.com/SurgicalSteel/elasthink/config" 23 | redigo "github.com/gomodule/redigo/redis" 24 | "github.com/rafaeljusto/redigomock" 25 | "github.com/stretchr/testify/assert" 26 | "testing" 27 | "time" 28 | ) 29 | 30 | func TestInitRedis(t *testing.T) { 31 | redisConfig := config.RedisConfigWrap{ 32 | RedisElasthink: config.RedisConfig{ 33 | Address: "fake.redis.address", 34 | MaxActive: 30, 35 | MaxIdle: 10, 36 | Timeout: 10, 37 | }, 38 | } 39 | 40 | expectedRedis := &Redis{ 41 | Pool: &redigo.Pool{ 42 | MaxIdle: redisConfig.RedisElasthink.MaxIdle, 43 | MaxActive: redisConfig.RedisElasthink.MaxActive, 44 | IdleTimeout: time.Duration(redisConfig.RedisElasthink.Timeout) * time.Second, 45 | Dial: func() (redigo.Conn, error) { 46 | return redigo.Dial("tcp", redisConfig.RedisElasthink.Address) 47 | }, 48 | }, 49 | } 50 | 51 | actualRedis := InitRedis(redisConfig) 52 | 53 | assert.Equal(t, expectedRedis.Pool.MaxActive, actualRedis.Pool.MaxActive) 54 | assert.Equal(t, expectedRedis.Pool.MaxIdle, actualRedis.Pool.MaxIdle) 55 | assert.Equal(t, expectedRedis.Pool.IdleTimeout, actualRedis.Pool.IdleTimeout) 56 | assert.NotNil(t, actualRedis.Pool.Dial) 57 | 58 | } 59 | 60 | func TestSAdd(t *testing.T) { 61 | conn := redigomock.NewConn() 62 | redisMock := &Redis{ 63 | Pool: redigo.NewPool(func() (redigo.Conn, error) { 64 | return conn, nil 65 | }, 10), 66 | } 67 | cmd := conn.Command("SADD", "campaign:ganteng", 666).Expect(int64(1)) 68 | _, err := redisMock.SAdd("campaign:ganteng", []interface{}{666}) 69 | if err != nil { 70 | t.Error("Expected : ok, but found error! err:", err.Error()) 71 | return 72 | } 73 | if conn.Stats(cmd) != 1 { 74 | t.Error("Command SADD is not used!") 75 | return 76 | } 77 | conn.Clear() 78 | } 79 | 80 | func TestSMembers(t *testing.T) { 81 | conn := redigomock.NewConn() 82 | redisMock := &Redis{ 83 | Pool: redigo.NewPool(func() (redigo.Conn, error) { 84 | return conn, nil 85 | }, 10), 86 | } 87 | cmd := conn.Command("SMEMBERS", "campaign:bangun").Expect([]interface{}{"123", "234", "345", "456"}) 88 | _, err := redisMock.SMembers("campaign:bangun") 89 | if err != nil { 90 | t.Error("Expected : ok, but found error! err:", err.Error()) 91 | return 92 | } 93 | if conn.Stats(cmd) != 1 { 94 | t.Error("Command SMEMBERS is not used!") 95 | return 96 | } 97 | conn.Clear() 98 | } 99 | 100 | func TestSRem(t *testing.T) { 101 | conn := redigomock.NewConn() 102 | redisMock := &Redis{ 103 | Pool: redigo.NewPool(func() (redigo.Conn, error) { 104 | return conn, nil 105 | }, 10), 106 | } 107 | cmd := conn.Command("SREM", "campaign:ganteng", 666).Expect(int64(1)) 108 | _, err := redisMock.SRem("campaign:ganteng", []interface{}{666}) 109 | if err != nil { 110 | t.Error("Expected : ok, but found error! err:", err.Error()) 111 | return 112 | } 113 | if conn.Stats(cmd) != 1 { 114 | t.Error("Command SREM is not used!") 115 | return 116 | } 117 | conn.Clear() 118 | } 119 | 120 | func TestKeysPrefix(t *testing.T) { 121 | conn := redigomock.NewConn() 122 | redisMock := &Redis{ 123 | Pool: redigo.NewPool(func() (redigo.Conn, error) { 124 | return conn, nil 125 | }, 10), 126 | } 127 | //test case 1 : normal 128 | cmd := conn.Command("KEYS", "campaign:*").Expect([]interface{}{"campaign:bangun", "campaign:tidur", "campaign:ganteng", "campaign:jalan"}) 129 | _, err := redisMock.KeysPrefix("campaign:") 130 | if err != nil { 131 | t.Error("Expected : ok, but found error! err:", err.Error()) 132 | return 133 | } 134 | if conn.Stats(cmd) != 1 { 135 | t.Error("Command KEYS is not used!") 136 | return 137 | } 138 | 139 | //test case 2 : expect error 140 | keys, err := redisMock.KeysPrefix(" ") 141 | assert.Equal(t, errors.New("Prefix must be defined!"), err) 142 | assert.Equal(t, make([]string, 0), keys) 143 | conn.Clear() 144 | } 145 | -------------------------------------------------------------------------------- /elasthink_insomnia_api_documentation.json: -------------------------------------------------------------------------------- 1 | { 2 | "__export_date": "2020-06-03T09:20:18.333Z", 3 | "__export_format": 4, 4 | "__export_source": "insomnia.desktop.app:v2020.2.0", 5 | "_type": "export", 6 | "resources": [ 7 | { 8 | "_id": "req_b83acdf147f449b0aacafdecc0cf2655", 9 | "_type": "request", 10 | "authentication": {}, 11 | "body": {}, 12 | "created": 1591174663645, 13 | "description": "Endpoint format : /v1/{document_type}/{prefix_keyword}/_suggest", 14 | "headers": [], 15 | "isPrivate": false, 16 | "metaSortKey": -1591174663645, 17 | "method": "GET", 18 | "modified": 1591175817346, 19 | "name": "[App] Keyword Suggestion", 20 | "parameters": [], 21 | "parentId": "wrk_fcb7567f74834b80b79d2ddbb390a1e1", 22 | "settingDisableRenderRequestBody": false, 23 | "settingEncodeUrl": true, 24 | "settingFollowRedirects": "global", 25 | "settingRebuildPath": true, 26 | "settingSendCookies": true, 27 | "settingStoreCookies": true, 28 | "url": "0.0.0.0:9000/v1/campaign/se/_suggest" 29 | }, 30 | { 31 | "_id": "wrk_fcb7567f74834b80b79d2ddbb390a1e1", 32 | "_type": "workspace", 33 | "created": 1577438488442, 34 | "description": "", 35 | "modified": 1577438488442, 36 | "name": "elasthink", 37 | "parentId": null, 38 | "scope": null 39 | }, 40 | { 41 | "_id": "req_63a264ccee134b26981a844e8e407d4c", 42 | "_type": "request", 43 | "authentication": {}, 44 | "body": { 45 | "mimeType": "application/json", 46 | "text": "{\n\t\"oldDocumentName\":\"Ini waktumu berlibur ke Jogja bersama Mantanmu\",\n\t\"newDocumentName\":\"Ini Waktumu Nikmati Istimewanya Jogja\"\n}" 47 | }, 48 | "created": 1577971239866, 49 | "description": "Endpoint format : /internal/v1/index/{document_type}/{document_id}", 50 | "headers": [ 51 | { 52 | "id": "pair_efe96212192043c6a1175a923efe0011", 53 | "name": "Content-Type", 54 | "value": "application/json" 55 | } 56 | ], 57 | "isPrivate": false, 58 | "metaSortKey": -1577971239866, 59 | "method": "PUT", 60 | "modified": 1591174941904, 61 | "name": "[Internal] Update Index", 62 | "parameters": [], 63 | "parentId": "wrk_fcb7567f74834b80b79d2ddbb390a1e1", 64 | "settingDisableRenderRequestBody": false, 65 | "settingEncodeUrl": true, 66 | "settingFollowRedirects": "global", 67 | "settingRebuildPath": true, 68 | "settingSendCookies": true, 69 | "settingStoreCookies": true, 70 | "url": "0.0.0.0:9000/internal/v1/index/campaign/1672" 71 | }, 72 | { 73 | "_id": "req_f178e1b096634691893f4e5a5f482fdf", 74 | "_type": "request", 75 | "authentication": {}, 76 | "body": { 77 | "mimeType": "application/json", 78 | "text": "{\n\t\"searchTerm\":\"diskon belanja bpjs\"\n}" 79 | }, 80 | "created": 1577962747403, 81 | "description": "Endpoint format : /v1/{document_type}/_search", 82 | "headers": [ 83 | { 84 | "id": "pair_b053bdbf72194a1bb80579dea9060d56", 85 | "name": "Content-Type", 86 | "value": "application/json" 87 | } 88 | ], 89 | "isPrivate": false, 90 | "metaSortKey": -1577962747403, 91 | "method": "POST", 92 | "modified": 1591175835112, 93 | "name": "[App] Search Document", 94 | "parameters": [], 95 | "parentId": "wrk_fcb7567f74834b80b79d2ddbb390a1e1", 96 | "settingDisableRenderRequestBody": false, 97 | "settingEncodeUrl": true, 98 | "settingFollowRedirects": "global", 99 | "settingRebuildPath": true, 100 | "settingSendCookies": true, 101 | "settingStoreCookies": true, 102 | "url": "0.0.0.0:9000/v1/campaign/_search" 103 | }, 104 | { 105 | "_id": "req_585c48e4228b4b81b851df8d4beae49c", 106 | "_type": "request", 107 | "authentication": {}, 108 | "body": { 109 | "mimeType": "application/json", 110 | "text": "{\n\t\"documentName\":\"Ini Waktumu liburan hemat bareng keluarga di Bandung\"\t\n}" 111 | }, 112 | "created": 1577958211317, 113 | "description": "Endpoint format : /internal/v1/index/{document_type}/{document_id}", 114 | "headers": [ 115 | { 116 | "id": "pair_ec1fdb2823684c0fb683b7d194cf23a9", 117 | "name": "Content-Type", 118 | "value": "application/json" 119 | } 120 | ], 121 | "isPrivate": false, 122 | "metaSortKey": -1577958211317, 123 | "method": "POST", 124 | "modified": 1591175619162, 125 | "name": "[Internal] Create Index", 126 | "parameters": [], 127 | "parentId": "wrk_fcb7567f74834b80b79d2ddbb390a1e1", 128 | "settingDisableRenderRequestBody": false, 129 | "settingEncodeUrl": true, 130 | "settingFollowRedirects": "global", 131 | "settingRebuildPath": true, 132 | "settingSendCookies": true, 133 | "settingStoreCookies": true, 134 | "url": "0.0.0.0:9000/internal/v1/index/campaign/1672" 135 | }, 136 | { 137 | "_id": "req_0316eb812bf646e2b897269c71f717fa", 138 | "_type": "request", 139 | "authentication": {}, 140 | "body": {}, 141 | "created": 1577438592434, 142 | "description": "", 143 | "headers": [], 144 | "isPrivate": false, 145 | "metaSortKey": -1577438592434, 146 | "method": "GET", 147 | "modified": 1577438609123, 148 | "name": "PING", 149 | "parameters": [], 150 | "parentId": "wrk_fcb7567f74834b80b79d2ddbb390a1e1", 151 | "settingDisableRenderRequestBody": false, 152 | "settingEncodeUrl": true, 153 | "settingFollowRedirects": "global", 154 | "settingRebuildPath": true, 155 | "settingSendCookies": true, 156 | "settingStoreCookies": true, 157 | "url": "0.0.0.0:9000/ping" 158 | }, 159 | { 160 | "_id": "env_820851f722b77f704912f063740c33825b71d707", 161 | "_type": "environment", 162 | "color": null, 163 | "created": 1577438488774, 164 | "data": {}, 165 | "dataPropertyOrder": null, 166 | "isPrivate": false, 167 | "metaSortKey": 1577438488774, 168 | "modified": 1577438488774, 169 | "name": "Base Environment", 170 | "parentId": "wrk_fcb7567f74834b80b79d2ddbb390a1e1" 171 | }, 172 | { 173 | "_id": "jar_820851f722b77f704912f063740c33825b71d707", 174 | "_type": "cookie_jar", 175 | "cookies": [], 176 | "created": 1577438488780, 177 | "modified": 1577438488780, 178 | "name": "Default Jar", 179 | "parentId": "wrk_fcb7567f74834b80b79d2ddbb390a1e1" 180 | }, 181 | { 182 | "_id": "spc_8c5def0b34784e8bb10e24079a7289e9", 183 | "_type": "api_spec", 184 | "contentType": "yaml", 185 | "contents": "", 186 | "created": 1591117375667, 187 | "fileName": "elasthink", 188 | "modified": 1591117375667, 189 | "parentId": "wrk_fcb7567f74834b80b79d2ddbb390a1e1" 190 | } 191 | ] 192 | } -------------------------------------------------------------------------------- /module/indexing.go: -------------------------------------------------------------------------------- 1 | package module 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import ( 21 | "context" 22 | "errors" 23 | "fmt" 24 | "log" 25 | "net/http" 26 | "strings" 27 | 28 | "github.com/SurgicalSteel/elasthink/entity" 29 | "github.com/SurgicalSteel/elasthink/util" 30 | ) 31 | 32 | //CreateIndexRequestPayload is the universal request payload for create index handler 33 | type CreateIndexRequestPayload struct { 34 | DocumentName string `json:"documentName"` 35 | } 36 | 37 | func validateCreateIndexRequestPayload(documentID int64, documentType, documentName string) error { 38 | err := validateDocumentType(documentType, entity.Entity.GetDocumentTypes()) 39 | if err != nil { 40 | return err 41 | } 42 | 43 | if documentID <= 0 { 44 | return errors.New("Invalid Document ID") 45 | } 46 | 47 | if len(strings.Trim(documentName, " ")) == 0 { 48 | return errors.New("Document Name must not be empty") 49 | } 50 | 51 | return nil 52 | } 53 | 54 | //CreateIndex is the core function to create an index of a document 55 | func CreateIndex(ctx context.Context, documentID int64, documentType string, requestPayload CreateIndexRequestPayload) Response { 56 | err := validateCreateIndexRequestPayload(documentID, documentType, requestPayload.DocumentName) 57 | if err != nil { 58 | return Response{ 59 | StatusCode: http.StatusBadRequest, 60 | ErrorMessage: err.Error(), 61 | Data: nil, 62 | } 63 | } 64 | 65 | documentNameSet := util.Tokenize(requestPayload.DocumentName, moduleObj.IsUsingStopwordRemoval, moduleObj.StopwordSet) 66 | 67 | docType := getDocumentType(documentType, entity.Entity.GetDocumentTypes()) 68 | errorExist := false 69 | errorKeys := "" 70 | 71 | for k := range documentNameSet { 72 | key := fmt.Sprintf("%s%s:%s", elasthinkInvertedIndexPrefix, docType, k) 73 | value := make([]interface{}, 1) 74 | value[0] = fmt.Sprintf("%d", documentID) 75 | _, err = moduleObj.Redis.SAdd(key, value) 76 | if err != nil { 77 | errorExist = true 78 | errorKeys = errorKeys + " " + key + "," 79 | log.Println("[MODULE][CREATE INDEX] failed to add index on key :", key, "and document ID:", documentID) 80 | continue 81 | } 82 | } 83 | 84 | if errorExist { 85 | errorKeys = strings.TrimRight(errorKeys, ",") 86 | errorKeys = strings.TrimLeft(errorKeys, " ") 87 | return Response{ 88 | StatusCode: http.StatusInternalServerError, 89 | ErrorMessage: fmt.Sprintf("Error on adding following keys :%s", errorKeys), 90 | Data: nil, 91 | } 92 | } 93 | return Response{ 94 | StatusCode: http.StatusOK, 95 | ErrorMessage: "", 96 | Data: nil, 97 | } 98 | 99 | } 100 | 101 | //UpdateIndexRequestPayload is the universal request payload for update index handler 102 | type UpdateIndexRequestPayload struct { 103 | OldDocumentName string `json:"oldDocumentName"` 104 | NewDocumentName string `json:"newDocumentName"` 105 | } 106 | 107 | func validateUpdateIndexRequestPayload(documentID int64, documentType, oldDocumentName, newDocumentName string) error { 108 | err := validateDocumentType(documentType, entity.Entity.GetDocumentTypes()) 109 | if err != nil { 110 | return err 111 | } 112 | 113 | if documentID <= 0 { 114 | return errors.New("Invalid Document ID") 115 | } 116 | 117 | if len(strings.Trim(oldDocumentName, " ")) == 0 { 118 | return errors.New("Old Document Name must not be empty") 119 | } 120 | 121 | if len(strings.Trim(newDocumentName, " ")) == 0 { 122 | return errors.New("Document Name must not be empty") 123 | } 124 | 125 | return nil 126 | } 127 | 128 | //UpdateIndex is the core function to update the index of a document. This requires old document name and new document name 129 | func UpdateIndex(ctx context.Context, documentID int64, documentType string, requestPayload UpdateIndexRequestPayload) Response { 130 | err := validateUpdateIndexRequestPayload(documentID, documentType, requestPayload.OldDocumentName, requestPayload.NewDocumentName) 131 | if err != nil { 132 | return Response{ 133 | StatusCode: http.StatusBadRequest, 134 | ErrorMessage: err.Error(), 135 | Data: nil, 136 | } 137 | } 138 | 139 | oldDocumentNameSet := util.Tokenize(requestPayload.OldDocumentName, moduleObj.IsUsingStopwordRemoval, moduleObj.StopwordSet) 140 | newDocumentNameSet := util.Tokenize(requestPayload.NewDocumentName, moduleObj.IsUsingStopwordRemoval, moduleObj.StopwordSet) 141 | 142 | docType := getDocumentType(documentType, entity.Entity.GetDocumentTypes()) 143 | 144 | // remove old document indexes 145 | isErrorRemoveExist := false 146 | errorRemoveKeys := "" 147 | 148 | for k := range oldDocumentNameSet { 149 | key := fmt.Sprintf("%s%s:%s", elasthinkInvertedIndexPrefix, docType, k) 150 | value := make([]interface{}, 1) 151 | value[0] = fmt.Sprintf("%d", documentID) 152 | _, err = moduleObj.Redis.SRem(key, value) 153 | if err != nil { 154 | isErrorRemoveExist = true 155 | errorRemoveKeys = errorRemoveKeys + " " + key + "," 156 | log.Println("[MODULE][UPDATE INDEX] failed to remove index on key :", key, "and document ID:", documentID) 157 | continue 158 | } 159 | } 160 | 161 | // add new document indexes 162 | isErrorAddExist := false 163 | errorAddKeys := "" 164 | 165 | for k := range newDocumentNameSet { 166 | key := fmt.Sprintf("%s%s:%s", elasthinkInvertedIndexPrefix, docType, k) 167 | value := make([]interface{}, 1) 168 | value[0] = fmt.Sprintf("%d", documentID) 169 | _, err = moduleObj.Redis.SAdd(key, value) 170 | if err != nil { 171 | isErrorAddExist = true 172 | errorAddKeys = errorAddKeys + " " + key + "," 173 | log.Println("[MODULE][UPDATE INDEX] failed to add index on key :", key, "and document ID:", documentID) 174 | continue 175 | } 176 | } 177 | 178 | if isErrorAddExist || isErrorRemoveExist { 179 | errorRemoveKeys = strings.TrimRight(errorRemoveKeys, ",") 180 | errorRemoveKeys = strings.TrimLeft(errorRemoveKeys, " ") 181 | 182 | errorAddKeys = strings.TrimRight(errorAddKeys, ",") 183 | errorAddKeys = strings.TrimLeft(errorAddKeys, " ") 184 | 185 | errorMessage := fmt.Sprintf("Error on removing following keys: %s and/or Error on adding following keys: %s", errorRemoveKeys, errorAddKeys) 186 | return Response{ 187 | StatusCode: http.StatusInternalServerError, 188 | ErrorMessage: errorMessage, 189 | Data: nil, 190 | } 191 | } 192 | 193 | return Response{ 194 | StatusCode: http.StatusOK, 195 | ErrorMessage: "", 196 | Data: nil, 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /files/data/stopwords_id.json: -------------------------------------------------------------------------------- 1 | { 2 | "words":["ada","adalah","adanya","adapun","agak","agaknya","agar","akan","akankah","akhir","akhiri","akhirnya","aku","akulah","amat","amatlah","anda","andalah","antar","antara","antaranya","apa","apaan","apabila","apakah","apalagi","apatah","artinya","asal","asalkan","atas","atau","ataukah","ataupun","awal","awalnya","bagai","bagaikan","bagaimana","bagaimanakah","bagaimanapun","bagi","bagian","bahkan","bahwa","bahwasanya","baik","bakal","bakalan","balik","banyak","bapak","baru","bawah","beberapa","begini","beginian","beginikah","beginilah","begitu","begitukah","begitulah","begitupun","bekerja","belakang","belakangan","belum","belumlah","benar","benarkah","benarlah","berada","berakhir","berakhirlah","berakhirnya","berapa","berapakah","berapalah","berapapun","berarti","berawal","berbagai","berdatangan","beri","berikan","berikut","berikutnya","berjumlah","berkali-kali","berkata","berkehendak","berkeinginan","berkenaan","berlainan","berlalu","berlangsung","berlebihan","bermacam","bermacam-macam","bermaksud","bermula","bersama","bersama-sama","bersiap","bersiap-siap","bertanya","bertanya-tanya","berturut","berturut-turut","bertutur","berujar","berupa","besar","betul","betulkah","biasa","biasanya","bila","bilakah","bisa","bisakah","boleh","bolehkah","bolehlah","buat","bukan","bukankah","bukanlah","bukannya","bulan","bung","cara","caranya","cukup","cukupkah","cukuplah","cuma","dahulu","dalam","dan","dapat","dari","daripada","datang","dekat","demi","demikian","demikianlah","dengan","depan","di","dia","diakhiri","diakhirinya","dialah","diantara","diantaranya","diberi","diberikan","diberikannya","dibuat","dibuatnya","didapat","didatangkan","digunakan","diibaratkan","diibaratkannya","diingat","diingatkan","diinginkan","dijawab","dijelaskan","dijelaskannya","dikarenakan","dikatakan","dikatakannya","dikerjakan","diketahui","diketahuinya","dikira","dilakukan","dilalui","dilihat","dimaksud","dimaksudkan","dimaksudkannya","dimaksudnya","diminta","dimintai","dimisalkan","dimulai","dimulailah","dimulainya","dimungkinkan","dini","dipastikan","diperbuat","diperbuatnya","dipergunakan","diperkirakan","diperlihatkan","diperlukan","diperlukannya","dipersoalkan","dipertanyakan","dipunyai","diri","dirinya","disampaikan","disebut","disebutkan","disebutkannya","disini","disinilah","ditambahkan","ditandaskan","ditanya","ditanyai","ditanyakan","ditegaskan","ditujukan","ditunjuk","ditunjuki","ditunjukkan","ditunjukkannya","ditunjuknya","dituturkan","dituturkannya","diucapkan","diucapkannya","diungkapkan","dong","dua","dulu","empat","enggak","enggaknya","entah","entahlah","guna","gunakan","hal","hampir","hanya","hanyalah","hari","harus","haruslah","harusnya","hendak","hendaklah","hendaknya","hingga","ia","ialah","ibarat","ibaratkan","ibaratnya","ibu","ikut","ingat","ingat-ingat","ingin","inginkah","inginkan","ini","inikah","inilah","itu","itukah","itulah","jadi","jadilah","jadinya","jangan","jangankan","janganlah","jauh","jawab","jawaban","jawabnya","jelas","jelaskan","jelaslah","jelasnya","jika","jikalau","juga","jumlah","jumlahnya","justru","kala","kalau","kalaulah","kalaupun","kalian","kami","kamilah","kamu","kamulah","kan","kapan","kapankah","kapanpun","karena","karenanya","kasus","kata","katakan","katakanlah","katanya","ke","keadaan","kebetulan","kecil","kedua","keduanya","keinginan","kelamaan","kelihatan","kelihatannya","kelima","keluar","kembali","kemudian","kemungkinan","kemungkinannya","kenapa","kepada","kepadanya","kesampaian","keseluruhan","keseluruhannya","keterlaluan","ketika","khususnya","kini","kinilah","kira","kira-kira","kiranya","kita","kitalah","kok","kurang","lagi","lagian","lah","lain","lainnya","lalu","lama","lamanya","lanjut","lanjutnya","lebih","lewat","lima","luar","macam","maka","makanya","makin","malah","malahan","mampu","mampukah","mana","manakala","manalagi","masa","masalah","masalahnya","masih","masihkah","masing","masing-masing","mau","maupun","melainkan","melakukan","melalui","melihat","melihatnya","memang","memastikan","memberi","memberikan","membuat","memerlukan","memihak","meminta","memintakan","memisalkan","memperbuat","mempergunakan","memperkirakan","memperlihatkan","mempersiapkan","mempersoalkan","mempertanyakan","mempunyai","memulai","memungkinkan","menaiki","menambahkan","menandaskan","menanti","menanti-nanti","menantikan","menanya","menanyai","menanyakan","mendapat","mendapatkan","mendatang","mendatangi","mendatangkan","menegaskan","mengakhiri","mengapa","mengatakan","mengatakannya","mengenai","mengerjakan","mengetahui","menggunakan","menghendaki","mengibaratkan","mengibaratkannya","mengingat","mengingatkan","menginginkan","mengira","mengucapkan","mengucapkannya","mengungkapkan","menjadi","menjawab","menjelaskan","menuju","menunjuk","menunjuki","menunjukkan","menunjuknya","menurut","menuturkan","menyampaikan","menyangkut","menyatakan","menyebutkan","menyeluruh","menyiapkan","merasa","mereka","merekalah","merupakan","meski","meskipun","meyakini","meyakinkan","minta","mirip","misal","misalkan","misalnya","mula","mulai","mulailah","mulanya","mungkin","mungkinkah","nah","naik","namun","nanti","nantinya","nyaris","nyatanya","oleh","olehnya","pada","padahal","padanya","pak","paling","panjang","pantas","para","pasti","pastilah","penting","pentingnya","per","percuma","perlu","perlukah","perlunya","pernah","persoalan","pertama","pertama-tama","pertanyaan","pertanyakan","pihak","pihaknya","pukul","pula","pun","punya","rasa","rasanya","rata","rupanya","saat","saatnya","saja","sajalah","saling","sama","sama-sama","sambil","sampai","sampai-sampai","sampaikan","sana","sangat","sangatlah","satu","saya","sayalah","se","sebab","sebabnya","sebagai","sebagaimana","sebagainya","sebagian","sebaik","sebaik-baiknya","sebaiknya","sebaliknya","sebanyak","sebegini","sebegitu","sebelum","sebelumnya","sebenarnya","seberapa","sebesar","sebetulnya","sebisanya","sebuah","sebut","sebutlah","sebutnya","secara","secukupnya","sedang","sedangkan","sedemikian","sedikit","sedikitnya","seenaknya","segala","segalanya","segera","seharusnya","sehingga","seingat","sejak","sejauh","sejenak","sejumlah","sekadar","sekadarnya","sekali","sekali-kali","sekalian","sekaligus","sekalipun","sekarang","sekecil","seketika","sekiranya","sekitar","sekitarnya","sekurang-kurangnya","sekurangnya","sela","selagi","selain","selaku","selalu","selama","selama-lamanya","selamanya","selanjutnya","seluruh","seluruhnya","semacam","semakin","semampu","semampunya","semasa","semasih","semata","semata-mata","semaunya","sementara","semisal","semisalnya","sempat","semua","semuanya","semula","sendiri","sendirian","sendirinya","seolah","seolah-olah","seorang","sepanjang","sepantasnya","sepantasnyalah","seperlunya","seperti","sepertinya","sepihak","sering","seringnya","serta","serupa","sesaat","sesama","sesampai","sesegera","sesekali","seseorang","sesuatu","sesuatunya","sesudah","sesudahnya","setelah","setempat","setengah","seterusnya","setiap","setiba","setibanya","setidak-tidaknya","setidaknya","setinggi","seusai","sewaktu","siap","siapa","siapakah","siapapun","sini","sinilah","soal","soalnya","suatu","sudah","sudahkah","sudahlah","supaya","tadi","tadinya","tahu","tahun","tak","tambah","tambahnya","tampak","tampaknya","tandas","tandasnya","tanpa","tanya","tanyakan","tanyanya","tapi","tegas","tegasnya","telah","tempat","tengah","tentang","tentu","tentulah","tentunya","tepat","terakhir","terasa","terbanyak","terdahulu","terdapat","terdiri","terhadap","terhadapnya","teringat","teringat-ingat","terjadi","terjadilah","terjadinya","terkira","terlalu","terlebih","terlihat","termasuk","ternyata","tersampaikan","tersebut","tersebutlah","tertentu","tertuju","terus","terutama","tetap","tetapi","tiap","tiba","tiba-tiba","tidak","tidakkah","tidaklah","tiga","tinggi","toh","tunjuk","turut","tutur","tuturnya","ucap","ucapnya","ujar","ujarnya","umum","umumnya","ungkap","ungkapnya","untuk","usah","usai","waduh","wah","wahai","waktu","waktunya","walau","walaupun","wong","yaitu","yakin","yakni","yang"] 3 | } 4 | -------------------------------------------------------------------------------- /util/setopt_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 4 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 5 | // 6 | // This file is part of Elasthink 7 | // 8 | // Elasthink is free software: you can redistribute it and/or modify 9 | // it under the terms of the GNU General Public License as published by 10 | // the Free Software Foundation, either version 3 of the License, or 11 | // (at your option) any later version. 12 | // 13 | // Elasthink is distributed in the hope that it will be useful, 14 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | // GNU General Public License for more details. 17 | // 18 | // You should have received a copy of the GNU General Public License 19 | // along with this program. If not, see . 20 | import ( 21 | "fmt" 22 | "reflect" 23 | "testing" 24 | ) 25 | 26 | func TestCreateWordSet(t *testing.T) { 27 | type tcase struct { 28 | sourceWordSlice []string 29 | expected map[string]int 30 | } 31 | 32 | testCases := make(map[string]tcase) 33 | 34 | testCases["empty word slice"] = tcase{ 35 | sourceWordSlice: make([]string, 0), 36 | expected: make(map[string]int), 37 | } 38 | 39 | testCases["normal word slice"] = tcase{ 40 | sourceWordSlice: []string{"invisible", "means", "can", "not", "be", "seen"}, 41 | expected: map[string]int{ 42 | "invisible": 1, 43 | "means": 1, 44 | "can": 1, 45 | "not": 1, 46 | "be": 1, 47 | "seen": 1, 48 | }, 49 | } 50 | 51 | testCases["repeated normal word slice"] = tcase{ 52 | sourceWordSlice: []string{"this", "one", "and", "that", "one", "are", "yours", "and", "shut", "up"}, 53 | expected: map[string]int{ 54 | "this": 1, 55 | "one": 1, 56 | "and": 1, 57 | "that": 1, 58 | "are": 1, 59 | "yours": 1, 60 | "shut": 1, 61 | "up": 1, 62 | }, 63 | } 64 | 65 | for ktc, vtc := range testCases { 66 | fmt.Println("doing test on CreateWordSet with test case:", ktc) 67 | actual := CreateWordSet(vtc.sourceWordSlice) 68 | isEqual := reflect.DeepEqual(vtc.expected, actual) 69 | if !isEqual { 70 | t.Fatal("Result WordSet is not same with what we expected.") 71 | continue 72 | } 73 | } 74 | } 75 | func TestWordsSetUnion(t *testing.T) { 76 | type tcase struct { 77 | sourceSetA map[string]int 78 | sourceSetB map[string]int 79 | expected map[string]int 80 | } 81 | 82 | testCases := make(map[string]tcase) 83 | 84 | testCases["empty setA and empty setB"] = tcase{ 85 | sourceSetA: make(map[string]int), 86 | sourceSetB: make(map[string]int), 87 | expected: make(map[string]int), 88 | } 89 | 90 | testCases["empty setA and filled setB"] = tcase{ 91 | sourceSetA: make(map[string]int), 92 | sourceSetB: map[string]int{ 93 | "the": 1, 94 | "quick": 1, 95 | "brown": 1, 96 | "fox": 1, 97 | "jumps": 1, 98 | "over": 1, 99 | "lazy": 1, 100 | "dog": 1, 101 | }, 102 | expected: map[string]int{ 103 | "the": 1, 104 | "quick": 1, 105 | "brown": 1, 106 | "fox": 1, 107 | "jumps": 1, 108 | "over": 1, 109 | "lazy": 1, 110 | "dog": 1, 111 | }, 112 | } 113 | 114 | testCases["filled setA and empty setB"] = tcase{ 115 | sourceSetA: map[string]int{ 116 | "the": 1, 117 | "quick": 1, 118 | "brown": 1, 119 | "fox": 1, 120 | "jumps": 1, 121 | "over": 1, 122 | "lazy": 1, 123 | "dog": 1, 124 | }, 125 | sourceSetB: make(map[string]int), 126 | expected: map[string]int{ 127 | "the": 1, 128 | "quick": 1, 129 | "brown": 1, 130 | "fox": 1, 131 | "jumps": 1, 132 | "over": 1, 133 | "lazy": 1, 134 | "dog": 1, 135 | }, 136 | } 137 | 138 | testCases["filled setA and filled setB with no overlapping items"] = tcase{ 139 | sourceSetA: map[string]int{ 140 | "invisible": 1, 141 | "means": 1, 142 | "can": 1, 143 | "not": 1, 144 | "be": 1, 145 | "seen": 1, 146 | }, 147 | sourceSetB: map[string]int{ 148 | "the": 1, 149 | "quick": 1, 150 | "brown": 1, 151 | "fox": 1, 152 | "jumps": 1, 153 | "over": 1, 154 | "lazy": 1, 155 | "dog": 1, 156 | }, 157 | expected: map[string]int{ 158 | "invisible": 1, 159 | "means": 1, 160 | "can": 1, 161 | "not": 1, 162 | "be": 1, 163 | "seen": 1, 164 | "the": 1, 165 | "quick": 1, 166 | "brown": 1, 167 | "fox": 1, 168 | "jumps": 1, 169 | "over": 1, 170 | "lazy": 1, 171 | "dog": 1, 172 | }, 173 | } 174 | 175 | testCases["filled setA and filled setB with overlapping items"] = tcase{ 176 | sourceSetA: map[string]int{ 177 | "it": 1, 178 | "is": 1, 179 | "fun": 1, 180 | "to": 1, 181 | "do": 1, 182 | "the": 1, 183 | "impossible": 1, 184 | }, 185 | sourceSetB: map[string]int{ 186 | "impossible": 1, 187 | "is": 1, 188 | "nothing": 1, 189 | }, 190 | expected: map[string]int{ 191 | "it": 1, 192 | "is": 1, 193 | "fun": 1, 194 | "to": 1, 195 | "do": 1, 196 | "the": 1, 197 | "impossible": 1, 198 | "nothing": 1, 199 | }, 200 | } 201 | 202 | for ktc, vtc := range testCases { 203 | fmt.Println("doing test on WordSetUnion with test case:", ktc) 204 | actual := WordsSetUnion(vtc.sourceSetA, vtc.sourceSetB) 205 | isEqual := reflect.DeepEqual(vtc.expected, actual) 206 | if !isEqual { 207 | t.Fatal("Result WordSet is not same with what we expected.") 208 | continue 209 | } 210 | } 211 | 212 | } 213 | func TestWordsSetSubtraction(t *testing.T) { 214 | type tcase struct { 215 | sourceSetA map[string]int 216 | sourceSetB map[string]int 217 | expected map[string]int 218 | } 219 | 220 | testCases := make(map[string]tcase) 221 | 222 | testCases["empty setA and empty setB"] = tcase{ 223 | sourceSetA: make(map[string]int), 224 | sourceSetB: make(map[string]int), 225 | expected: make(map[string]int), 226 | } 227 | 228 | testCases["empty setA and filled setB"] = tcase{ 229 | sourceSetA: make(map[string]int), 230 | sourceSetB: map[string]int{ 231 | "the": 1, 232 | "quick": 1, 233 | "brown": 1, 234 | "fox": 1, 235 | "jumps": 1, 236 | "over": 1, 237 | "lazy": 1, 238 | "dog": 1, 239 | }, 240 | expected: make(map[string]int), 241 | } 242 | 243 | testCases["filled setA and empty setB"] = tcase{ 244 | sourceSetA: map[string]int{ 245 | "the": 1, 246 | "quick": 1, 247 | "brown": 1, 248 | "fox": 1, 249 | "jumps": 1, 250 | "over": 1, 251 | "lazy": 1, 252 | "dog": 1, 253 | }, 254 | sourceSetB: make(map[string]int), 255 | expected: map[string]int{ 256 | "the": 1, 257 | "quick": 1, 258 | "brown": 1, 259 | "fox": 1, 260 | "jumps": 1, 261 | "over": 1, 262 | "lazy": 1, 263 | "dog": 1, 264 | }, 265 | } 266 | 267 | testCases["filled setA and filled setB with no overlapping items"] = tcase{ 268 | sourceSetA: map[string]int{ 269 | "invisible": 1, 270 | "means": 1, 271 | "can": 1, 272 | "not": 1, 273 | "be": 1, 274 | "seen": 1, 275 | }, 276 | sourceSetB: map[string]int{ 277 | "the": 1, 278 | "quick": 1, 279 | "brown": 1, 280 | "fox": 1, 281 | "jumps": 1, 282 | "over": 1, 283 | "lazy": 1, 284 | "dog": 1, 285 | }, 286 | expected: map[string]int{ 287 | "invisible": 1, 288 | "means": 1, 289 | "can": 1, 290 | "not": 1, 291 | "be": 1, 292 | "seen": 1, 293 | }, 294 | } 295 | 296 | testCases["filled setA and filled setB with overlapping items"] = tcase{ 297 | sourceSetA: map[string]int{ 298 | "it": 1, 299 | "is": 1, 300 | "fun": 1, 301 | "to": 1, 302 | "do": 1, 303 | "the": 1, 304 | "impossible": 1, 305 | }, 306 | sourceSetB: map[string]int{ 307 | "impossible": 1, 308 | "is": 1, 309 | "nothing": 1, 310 | }, 311 | expected: map[string]int{ 312 | "it": 1, 313 | "fun": 1, 314 | "to": 1, 315 | "do": 1, 316 | "the": 1, 317 | }, 318 | } 319 | 320 | for ktc, vtc := range testCases { 321 | fmt.Println("doing test on WordSetSubtraction with test case:", ktc) 322 | actual := WordsSetSubtraction(vtc.sourceSetA, vtc.sourceSetB) 323 | isEqual := reflect.DeepEqual(vtc.expected, actual) 324 | if !isEqual { 325 | t.Fatal("Result WordSet is not same with what we expected.") 326 | continue 327 | } 328 | } 329 | } 330 | func TestWordsSetIntersection(t *testing.T) { 331 | type tcase struct { 332 | sourceSetA map[string]int 333 | sourceSetB map[string]int 334 | expected map[string]int 335 | } 336 | 337 | testCases := make(map[string]tcase) 338 | 339 | testCases["empty setA and empty setB"] = tcase{ 340 | sourceSetA: make(map[string]int), 341 | sourceSetB: make(map[string]int), 342 | expected: make(map[string]int), 343 | } 344 | 345 | testCases["empty setA and filled setB"] = tcase{ 346 | sourceSetA: make(map[string]int), 347 | sourceSetB: map[string]int{ 348 | "the": 1, 349 | "quick": 1, 350 | "brown": 1, 351 | "fox": 1, 352 | "jumps": 1, 353 | "over": 1, 354 | "lazy": 1, 355 | "dog": 1, 356 | }, 357 | expected: make(map[string]int), 358 | } 359 | 360 | testCases["filled setA and empty setB"] = tcase{ 361 | sourceSetA: map[string]int{ 362 | "the": 1, 363 | "quick": 1, 364 | "brown": 1, 365 | "fox": 1, 366 | "jumps": 1, 367 | "over": 1, 368 | "lazy": 1, 369 | "dog": 1, 370 | }, 371 | sourceSetB: make(map[string]int), 372 | expected: make(map[string]int), 373 | } 374 | 375 | testCases["filled setA and filled setB with no overlapping items"] = tcase{ 376 | sourceSetA: map[string]int{ 377 | "invisible": 1, 378 | "means": 1, 379 | "can": 1, 380 | "not": 1, 381 | "be": 1, 382 | "seen": 1, 383 | }, 384 | sourceSetB: map[string]int{ 385 | "the": 1, 386 | "quick": 1, 387 | "brown": 1, 388 | "fox": 1, 389 | "jumps": 1, 390 | "over": 1, 391 | "lazy": 1, 392 | "dog": 1, 393 | }, 394 | expected: make(map[string]int), 395 | } 396 | 397 | testCases["filled setA and filled setB with overlapping items"] = tcase{ 398 | sourceSetA: map[string]int{ 399 | "it": 1, 400 | "is": 1, 401 | "fun": 1, 402 | "to": 1, 403 | "do": 1, 404 | "the": 1, 405 | "impossible": 1, 406 | }, 407 | sourceSetB: map[string]int{ 408 | "impossible": 1, 409 | "is": 1, 410 | "nothing": 1, 411 | }, 412 | expected: map[string]int{ 413 | "impossible": 1, 414 | "is": 1, 415 | }, 416 | } 417 | 418 | for ktc, vtc := range testCases { 419 | fmt.Println("doing test on WordSetIntersection with test case:", ktc) 420 | actual := WordsSetIntersection(vtc.sourceSetA, vtc.sourceSetB) 421 | isEqual := reflect.DeepEqual(vtc.expected, actual) 422 | if !isEqual { 423 | t.Fatal("Result WordSet is not same with what we expected.") 424 | continue 425 | } 426 | } 427 | } 428 | -------------------------------------------------------------------------------- /sdk/sdk.go: -------------------------------------------------------------------------------- 1 | //package sdk provides the elasthink SDK which allows you to run all elasthink's core functionality in your service 2 | package sdk 3 | 4 | // Elasthink, An alternative to elasticsearch engine written in Go for small set of documents that uses inverted index to build the index and utilizes redis to store the indexes. 5 | // Copyright (C) 2020 Yuwono Bangun Nagoro (a.k.a SurgicalSteel) 6 | // 7 | // This file is part of Elasthink 8 | // 9 | // Elasthink is free software: you can redistribute it and/or modify 10 | // it under the terms of the GNU General Public License as published by 11 | // the Free Software Foundation, either version 3 of the License, or 12 | // (at your option) any later version. 13 | // 14 | // Elasthink is distributed in the hope that it will be useful, 15 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | // GNU General Public License for more details. 18 | // 19 | // You should have received a copy of the GNU General Public License 20 | // along with this program. If not, see . 21 | import ( 22 | "errors" 23 | "fmt" 24 | "sort" 25 | "strings" 26 | 27 | "github.com/SurgicalSteel/elasthink/config" 28 | "github.com/SurgicalSteel/elasthink/redis" 29 | "github.com/SurgicalSteel/elasthink/util" 30 | ) 31 | 32 | /// Const 33 | //elasthinkInvertedIndexPrefix is the prefix key for each word set in the inverted index 34 | const elasthinkInvertedIndexPrefix string = "elasthink:inverted:" 35 | 36 | //elasthinkNormalIndexPrefix is the prefix key for each normal index (followed by document type and document id). The value with this key contains the original document name 37 | const elasthinkNormalIndexPrefix string = "elasthink:normal:" 38 | 39 | /// Variables and structures 40 | 41 | // ElasthinkSDK is the main struct of elasthink SDK, initialized using initalize function 42 | type ElasthinkSDK struct { 43 | Redis *redis.Redis 44 | isUsingStopWordsRemoval bool 45 | stopWordRemovalData []string 46 | availableDocumentType map[string]int 47 | } 48 | 49 | // InitializeSpec is the payload to initialize Elasthink SDK 50 | type InitializeSpec struct { 51 | RedisConfig RedisConfig 52 | SdkConfig SdkConfig 53 | } 54 | 55 | // RedisConfig is the basic configuration to initialize a redis connection 56 | // Address is the address of the redis will be used, for localhost with port 6379 (default redis port), just use "localhost:6379" 57 | // MaxActive is the maximum of access to the redis, example value is 30 58 | // MaxIdle is the maximum idle access to the redis, example value is 10 59 | // Timeout is the timeout of accessing redis (in second), example value is 10 60 | type RedisConfig struct { 61 | Address string 62 | MaxActive int 63 | MaxIdle int 64 | Timeout int 65 | } 66 | 67 | // SdkConfig is the configuration to initialize Elasthink SDK 68 | // IsUsingStopWordsRemoval enables Elasthink to remove stop words 69 | // StopWordRemovalData define the stop words 70 | // AvailableDocumentType the document type available, for example "campaign" 71 | type SdkConfig struct { 72 | IsUsingStopWordsRemoval bool 73 | StopWordRemovalData []string 74 | AvailableDocumentType []string 75 | } 76 | 77 | // CreateIndexSpec is the spec of CreateIndex function 78 | // DocumentType is the type of the document 79 | // DocumentName is the name of the document 80 | // DocumentID is the id of the document 81 | type CreateIndexSpec struct { 82 | DocumentType string 83 | DocumentName string 84 | DocumentID int64 85 | } 86 | 87 | // UpdateIndexSpec is the spec of UpdateIndex function 88 | // OldDocumentName is the name of the old document which will be replaced by new document 89 | // NewDocumentName is the name of the new document 90 | // DocumentID is the id of the document 91 | type UpdateIndexSpec struct { 92 | DocumentType string 93 | OldDocumentName string 94 | NewDocumentName string 95 | DocumentID int64 96 | } 97 | 98 | // SearchSpec is the spec of Search function 99 | type SearchSpec struct { 100 | DocumentType string 101 | SearchTerm string 102 | } 103 | 104 | // SearchResultRankData is the search result datum 105 | type SearchResultRankData struct { 106 | ID int64 107 | ShowCount int 108 | Rank int 109 | } 110 | 111 | // GetKeywordSuggestionSpec is the spec of Getting Keyword Suggestion function 112 | type GetKeywordSuggestionSpec struct { 113 | DocumentType string 114 | Prefix string 115 | } 116 | 117 | // SearchResult is the result of Search, it have array of search result datum 118 | type SearchResult struct { 119 | RankedResultList RankByShowCount 120 | } 121 | 122 | //RankByShowCount is the additional struct for document ranking purpose based on its ShowCount 123 | type RankByShowCount []SearchResultRankData 124 | 125 | // Len overrides Len function of RankByShowCount 126 | func (r RankByShowCount) Len() int { return len(r) } 127 | 128 | // Less overrides Less function of RankByShowCount 129 | func (r RankByShowCount) Less(i, j int) bool { 130 | return r[i].ShowCount > r[j].ShowCount 131 | } 132 | 133 | // Swap overrides Swap function of RankByShowCount 134 | func (r RankByShowCount) Swap(i, j int) { 135 | r[i], r[j] = r[j], r[i] 136 | } 137 | 138 | // Initialize is the function that return ElasthinkSDK 139 | func Initialize(initializeSpec InitializeSpec) ElasthinkSDK { 140 | 141 | spec := config.RedisConfigWrap{ 142 | RedisElasthink: config.RedisConfig{ 143 | MaxIdle: initializeSpec.RedisConfig.MaxIdle, 144 | MaxActive: initializeSpec.RedisConfig.MaxActive, 145 | Address: initializeSpec.RedisConfig.Address, 146 | Timeout: initializeSpec.RedisConfig.Timeout, 147 | }, 148 | } 149 | newRedis := redis.InitRedis(spec) 150 | 151 | availableDocumentType := make(map[string]int) 152 | for _, doctype := range initializeSpec.SdkConfig.AvailableDocumentType { 153 | availableDocumentType[doctype] = 1 154 | } 155 | 156 | elasthinkSDK := ElasthinkSDK{ 157 | Redis: newRedis, 158 | isUsingStopWordsRemoval: initializeSpec.SdkConfig.IsUsingStopWordsRemoval, 159 | stopWordRemovalData: initializeSpec.SdkConfig.StopWordRemovalData, 160 | availableDocumentType: availableDocumentType, 161 | } 162 | return elasthinkSDK 163 | } 164 | 165 | // CreateIndex is a function to create new index based on documentType, documentID, and document name 166 | // documentType is the type of the document, to categorize documents. For example: campaign 167 | // documentID, is the ID of document, the key of document. For example: 1 168 | // documentName, is the name of documennt, the value which will be indexed. For example: "we want to eat seafood on a restaurant" 169 | func (es *ElasthinkSDK) CreateIndex(spec CreateIndexSpec) (bool, error) { 170 | documentID := spec.DocumentID 171 | documentType := spec.DocumentType 172 | documentName := spec.DocumentName 173 | 174 | // Validation 175 | err := es.validateCreateIndexSpec(documentID, documentType, documentName) 176 | if err != nil { 177 | return false, err 178 | } 179 | 180 | // Tokenize document name set 181 | stopword := make(map[string]int) 182 | for _, k := range es.stopWordRemovalData { 183 | stopword[k] = 1 184 | } 185 | redis := es.Redis 186 | documentNameSet := util.Tokenize(documentName, es.isUsingStopWordsRemoval, stopword) 187 | 188 | docType := documentType 189 | errorExist := false 190 | errorKeys := "" 191 | 192 | //add index for each tokenized items on documentNameSet 193 | //if there is an error in each indexing process, construct the error keys string (to log which keys affected by the errors) 194 | for k := range documentNameSet { 195 | key := fmt.Sprintf("%s%s:%s", elasthinkInvertedIndexPrefix, docType, k) 196 | value := make([]interface{}, 1) 197 | value[0] = fmt.Sprintf("%d", documentID) 198 | _, err := redis.SAdd(key, value) 199 | if err != nil { 200 | errorExist = true 201 | errorKeys = errorKeys + " " + key + "," 202 | continue 203 | } 204 | } 205 | 206 | if errorExist { 207 | errorKeys = strings.TrimRight(errorKeys, ",") 208 | errorKeys = strings.TrimLeft(errorKeys, " ") 209 | return false, fmt.Errorf("Error on adding following keys :%s", errorKeys) 210 | } 211 | 212 | return true, nil 213 | } 214 | 215 | //UpdateIndex is function to update previously created index 216 | func (es *ElasthinkSDK) UpdateIndex(spec UpdateIndexSpec) (bool, error) { 217 | documentID := spec.DocumentID 218 | documentType := spec.DocumentType 219 | oldDocumentName := spec.OldDocumentName 220 | newDocumentName := spec.NewDocumentName 221 | redis := es.Redis 222 | 223 | // Validate 224 | err := es.validateUpdateIndexSpec(documentID, documentType, spec.OldDocumentName, spec.NewDocumentName) 225 | if err != nil { 226 | return false, err 227 | } 228 | 229 | // Tokenize 230 | stopword := make(map[string]int) 231 | for _, k := range es.stopWordRemovalData { 232 | stopword[k] = 1 233 | } 234 | oldDocumentNameSet := util.Tokenize(oldDocumentName, es.isUsingStopWordsRemoval, stopword) 235 | newDocumentNameSet := util.Tokenize(newDocumentName, es.isUsingStopWordsRemoval, stopword) 236 | 237 | docType := spec.DocumentType 238 | 239 | // remove old document indexes 240 | isErrorRemoveExist := false 241 | errorRemoveKeys := "" 242 | 243 | for k := range oldDocumentNameSet { 244 | key := fmt.Sprintf("%s%s:%s", elasthinkInvertedIndexPrefix, docType, k) 245 | value := make([]interface{}, 1) 246 | value[0] = fmt.Sprintf("%d", documentID) 247 | _, err = redis.SRem(key, value) 248 | if err != nil { 249 | isErrorRemoveExist = true 250 | errorRemoveKeys = errorRemoveKeys + " " + key + "," 251 | continue 252 | } 253 | } 254 | 255 | // add new document indexes 256 | isErrorAddExist := false 257 | errorAddKeys := "" 258 | 259 | for k := range newDocumentNameSet { 260 | key := fmt.Sprintf("%s%s:%s", elasthinkInvertedIndexPrefix, docType, k) 261 | value := make([]interface{}, 1) 262 | value[0] = fmt.Sprintf("%d", documentID) 263 | _, err = redis.SAdd(key, value) 264 | if err != nil { 265 | isErrorAddExist = true 266 | errorAddKeys = errorAddKeys + " " + key + "," 267 | continue 268 | } 269 | } 270 | 271 | if isErrorAddExist || isErrorRemoveExist { 272 | errorRemoveKeys = strings.TrimRight(errorRemoveKeys, ",") 273 | errorRemoveKeys = strings.TrimLeft(errorRemoveKeys, " ") 274 | 275 | errorAddKeys = strings.TrimRight(errorAddKeys, ",") 276 | errorAddKeys = strings.TrimLeft(errorAddKeys, " ") 277 | 278 | errorMessage := fmt.Sprintf("Error on removing following keys: %s and/or Error on adding following keys: %s", errorRemoveKeys, errorAddKeys) 279 | return false, errors.New(errorMessage) 280 | } 281 | 282 | return true, nil 283 | } 284 | 285 | //Search is the core function of searching a document 286 | func (es *ElasthinkSDK) Search(spec SearchSpec) (SearchResult, error) { 287 | searchTerm := spec.SearchTerm 288 | documentType := spec.DocumentType 289 | 290 | ret := SearchResult{RankedResultList: make([]SearchResultRankData, 0)} 291 | 292 | err := es.validateSearchSpec(documentType, searchTerm) 293 | if err != nil { 294 | return ret, err 295 | } 296 | 297 | stopword := make(map[string]int) 298 | for _, k := range es.stopWordRemovalData { 299 | stopword[k] = 1 300 | } 301 | searchTermSet := util.Tokenize(searchTerm, es.isUsingStopWordsRemoval, stopword) 302 | if len(searchTermSet) == 0 { 303 | return ret, nil 304 | } 305 | 306 | wordIndexSets, err := es.fetchWordIndexSets(documentType, searchTermSet) 307 | if len(wordIndexSets) == 0 || err != nil { 308 | return ret, err 309 | } 310 | 311 | rankedSearchResult := rankSearchResult(wordIndexSets) 312 | ret.RankedResultList = rankedSearchResult 313 | 314 | return ret, nil 315 | } 316 | 317 | //GetKeywordSuggestion is the core function to get keyword suggestion from a given keyword prefix and document type 318 | func (es *ElasthinkSDK) GetKeywordSuggestion(spec GetKeywordSuggestionSpec) ([]string, error) { 319 | err := es.validateGetKeywordSuggestionSpec(spec.DocumentType, spec.Prefix) 320 | if err != nil { 321 | return []string{}, err 322 | } 323 | 324 | prefix := strings.ToLower(spec.Prefix) 325 | documentType := spec.DocumentType 326 | 327 | keywords, err := es.fetchKeywords(documentType, prefix) 328 | if err != nil { 329 | return []string{}, err 330 | } 331 | 332 | sort.Strings(keywords) 333 | return keywords, nil 334 | 335 | } 336 | 337 | /// Private Functions 338 | 339 | // validateCreateIndexSpec validates create index spec 340 | func (es *ElasthinkSDK) validateCreateIndexSpec(documentID int64, documentType, documentName string) error { 341 | if documentID <= 0 { 342 | return errors.New("Invalid Document ID") 343 | } 344 | 345 | if len(strings.Trim(documentName, " ")) == 0 { 346 | return errors.New("Document Name must not be empty") 347 | } 348 | 349 | err := es.isValidFromCustomDocumentType(documentType) 350 | return err 351 | } 352 | 353 | // Validate is the document type is valid or not 354 | func (es *ElasthinkSDK) isValidFromCustomDocumentType(documentType string) error { 355 | if _, ok := es.availableDocumentType[documentType]; ok { 356 | return nil 357 | } 358 | return errors.New("Invalid Document Type") 359 | } 360 | 361 | // validateUpdateIndexSpec validate update index spec 362 | func (es *ElasthinkSDK) validateUpdateIndexSpec(documentID int64, documentType, oldDocumentName, newDocumentName string) error { 363 | if documentID <= 0 { 364 | return errors.New("Invalid Document ID") 365 | } 366 | 367 | if len(strings.Trim(oldDocumentName, " ")) == 0 { 368 | return errors.New("Old Document Name must not be empty") 369 | } 370 | 371 | if len(strings.Trim(newDocumentName, " ")) == 0 { 372 | return errors.New("Document Name must not be empty") 373 | } 374 | 375 | err := es.isValidFromCustomDocumentType(documentType) 376 | return err 377 | } 378 | 379 | // validateSearchSpec validate search spec 380 | func (es *ElasthinkSDK) validateSearchSpec(documentType, searchTerm string) error { 381 | if len(strings.Trim(searchTerm, " ")) == 0 { 382 | return errors.New("Search Term is required") 383 | } 384 | 385 | if len(strings.Trim(documentType, " ")) == 0 { 386 | return errors.New("Document Type is required") 387 | } 388 | 389 | err := es.isValidFromCustomDocumentType(documentType) 390 | return err 391 | } 392 | 393 | // validateGetKeywordSuggestionSpec validate search spec 394 | func (es *ElasthinkSDK) validateGetKeywordSuggestionSpec(documentType, prefix string) error { 395 | if len(strings.Trim(prefix, " ")) == 0 { 396 | return errors.New("prefix of a keyword is required") 397 | } 398 | 399 | if len(strings.Trim(documentType, " ")) == 0 { 400 | return errors.New("Document Type is required") 401 | } 402 | 403 | err := es.isValidFromCustomDocumentType(documentType) 404 | return err 405 | } 406 | 407 | // fetchWordIndexSets 408 | func (es *ElasthinkSDK) fetchWordIndexSets(documentType string, searchTermSet map[string]int) (map[string][]int64, error) { 409 | result := make(map[string][]int64) 410 | 411 | errorExist := false 412 | errorKeys := "" 413 | 414 | // set key format --> elasthink:inverted:documentType:word 415 | for k := range searchTermSet { 416 | key := fmt.Sprintf("%s%s:%s", elasthinkInvertedIndexPrefix, documentType, k) 417 | members, err := es.Redis.SMembers(key) 418 | if err != nil { 419 | errorExist = true 420 | errorKeys = errorKeys + " " + key + "," 421 | continue 422 | } 423 | documentIds := util.SliceStringToInt64(members) 424 | result[k] = documentIds 425 | } 426 | 427 | if errorExist { 428 | errorKeys = strings.TrimRight(errorKeys, ",") 429 | errorKeys = strings.TrimLeft(errorKeys, " ") 430 | return make(map[string][]int64), fmt.Errorf("Error on adding following keys :%s", errorKeys) 431 | } 432 | 433 | return result, nil 434 | } 435 | 436 | //rankSearchResult ranks search result (document id by its appeareance count). word indexes is a map with word as a key and slice of ids as value. Returns ordered search result rank slice. 437 | func rankSearchResult(wordIndexes map[string][]int64) []SearchResultRankData { 438 | counterMap := make(map[int64]int) 439 | for _, ids := range wordIndexes { 440 | for i := 0; i < len(ids); i++ { 441 | if vcm, ok := counterMap[ids[i]]; ok { 442 | counterMap[ids[i]] = vcm + 1 443 | } else { 444 | counterMap[ids[i]] = 1 445 | } 446 | } 447 | } 448 | 449 | result := make([]SearchResultRankData, len(counterMap)) 450 | 451 | iterator := 0 452 | for kcm, vcm := range counterMap { 453 | result[iterator] = SearchResultRankData{ 454 | ID: kcm, 455 | ShowCount: vcm, 456 | } 457 | iterator++ 458 | } 459 | 460 | //sort by appeareance count (descending) 461 | sort.Sort(RankByShowCount(result)) 462 | 463 | //assign rank to each search result data 464 | for i := 0; i < len(result); i++ { 465 | temp := result[i] 466 | temp.Rank = i + 1 467 | result[i] = temp 468 | } 469 | 470 | return result 471 | } 472 | 473 | //fetchKeywords to fetch suggested keywords by prefix 474 | func (es *ElasthinkSDK) fetchKeywords(documentType, prefix string) ([]string, error) { 475 | prefixKey := fmt.Sprintf("%s%s:%s", elasthinkInvertedIndexPrefix, documentType, prefix) 476 | rawKeys, err := es.Redis.KeysPrefix(prefixKey) 477 | if err != nil { 478 | return []string{}, err 479 | } 480 | 481 | finalKeywords := make([]string, len(rawKeys)) 482 | trimPrefix := fmt.Sprintf("%s%s:", elasthinkInvertedIndexPrefix, documentType) 483 | 484 | for i := 0; i < len(rawKeys); i++ { 485 | rawKey := rawKeys[i] 486 | finalKeywords[i] = strings.TrimPrefix(rawKey, trimPrefix) 487 | } 488 | 489 | return finalKeywords, nil 490 | } 491 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | --------------------------------------------------------------------------------