├── .circleci └── config.yml ├── .gitignore ├── LICENSE.txt ├── README.md ├── bm ├── README.md ├── bm.go └── bm_test.go ├── doc.go ├── errors.go ├── go.mod ├── go.sum ├── gom ├── README.md ├── export_test.go ├── gom.go └── gom_test.go ├── intergo.go ├── intergo_test.go └── tdm ├── README.md ├── export_test.go ├── tdm.go └── tdm_test.go /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | jobs: 4 | build: 5 | working_directory: ~/intergo 6 | docker: 7 | - image: circleci/golang:1.12.5 8 | steps: 9 | - checkout 10 | - run: go get 11 | - run: go vet 12 | - run: go test -v -race ./... 13 | - run: go test -bench . 14 | - run: go build . 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.gitignore.io/api/go,intellij 2 | 3 | ### Go ### 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | 11 | # Test binary, build with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | ### Go Patch ### 18 | /vendor/ 19 | /Godeps/ 20 | 21 | ### Intellij ### 22 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 23 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 24 | 25 | # User-specific stuff 26 | .idea/**/workspace.xml 27 | .idea/**/tasks.xml 28 | .idea/**/usage.statistics.xml 29 | .idea/**/dictionaries 30 | .idea/**/shelf 31 | 32 | # Sensitive or high-churn files 33 | .idea/**/dataSources/ 34 | .idea/**/dataSources.ids 35 | .idea/**/dataSources.local.xml 36 | .idea/**/sqlDataSources.xml 37 | .idea/**/dynamic.xml 38 | .idea/**/uiDesigner.xml 39 | .idea/**/dbnavigator.xml 40 | 41 | # Gradle 42 | .idea/**/gradle.xml 43 | .idea/**/libraries 44 | 45 | # Gradle and Maven with auto-import 46 | # When using Gradle or Maven with auto-import, you should exclude module files, 47 | # since they will be recreated, and may cause churn. Uncomment if using 48 | # auto-import. 49 | # .idea/modules.xml 50 | # .idea/*.iml 51 | # .idea/modules 52 | 53 | # CMake 54 | cmake-build-*/ 55 | 56 | # Mongo Explorer plugin 57 | .idea/**/mongoSettings.xml 58 | 59 | # File-based project format 60 | *.iws 61 | 62 | # IntelliJ 63 | out/ 64 | 65 | # mpeltonen/sbt-idea plugin 66 | .idea_modules/ 67 | 68 | # JIRA plugin 69 | atlassian-ide-plugin.xml 70 | 71 | # Cursive Clojure plugin 72 | .idea/replstate.xml 73 | 74 | # Crashlytics plugin (for Android Studio and IntelliJ) 75 | com_crashlytics_export_strings.xml 76 | crashlytics.properties 77 | crashlytics-build.properties 78 | fabric.properties 79 | 80 | # Editor-based Rest Client 81 | .idea/httpRequests 82 | 83 | ### Intellij Patch ### 84 | # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 85 | 86 | # *.iml 87 | # modules.xml 88 | # .idea/misc.xml 89 | # *.ipr 90 | 91 | # Sonarlint plugin 92 | .idea/sonarlint 93 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 @mathetake 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # intergo 2 | [![CircleCI](https://circleci.com/gh/mathetake/intergo.svg?style=shield&circle-token=89a8a65229dd121bd61be11222cdc2a0416cef22)](https://circleci.com/gh/mathetake/intergo) 3 | [![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](LICENSE) 4 | [![](https://godoc.org/github.com/mathetake/intergo?status.svg)](http://godoc.org/github.com/mathetake/intergo) 5 | 6 | A package for interleaving / multileaving ranking generation in go 7 | 8 | It is mainly tailored to be used for generating interleaved or multileaved ranking based on the following algorithm 9 | 10 | - Balanced Interleaving/Multileaving (in `github.com/mathetake/itergo/bm` package) 11 | - Greedy Optimized Multileaving (in `github.com/mathetake/intergo/gom` package) 12 | - Team Draft Interleaving/Multileaving (in `github.com/mathetake/itergo/tdm` package) 13 | 14 | __NOTE:__ this package aims only at generating a single combined ranking and does not implement the evaluation functions of the given rankings. 15 | 16 | ## Usage 17 | 18 | Make sure that all of your rankings implement `intergo.Ranking` interface defined in `intergo.go` 19 | 20 | ```go 21 | package intergo 22 | 23 | type ID string 24 | 25 | type Ranking interface { 26 | GetIDByIndex(int) ID 27 | Len() int 28 | } 29 | ``` 30 | 31 | Then choose one of `bm` or `gom` or `tdm` package which corresponds to the algorithm you want to use. 32 | 33 | In each of these packages, there is a type which implements `intergo.Interleaving` interface defined in `intergo.go`, 34 | 35 | ```go 36 | package intergo 37 | 38 | type Result struct { 39 | RankingIndex int 40 | ItemIndex int 41 | } 42 | 43 | type Interleaving interface { 44 | GetInterleavedRanking(num int, rankings ...Ranking) ([]*Result, error) 45 | } 46 | ``` 47 | and you can generate interleaved/multileaved ranking by calling `GetInterleavedRanking`. 48 | 49 | The following is an example using Team Draft MultiLeaving (implemented in `tdm` package) 50 | 51 | ```go 52 | package main 53 | 54 | import ( 55 | "fmt" 56 | "strconv" 57 | 58 | "github.com/mathetake/intergo" 59 | "github.com/mathetake/intergo/tdm" 60 | ) 61 | 62 | type tRanking []int 63 | 64 | func (rk tRanking) GetIDByIndex(i int) intergo.ID { 65 | return intergo.ID(strconv.Itoa(rk[i])) 66 | } 67 | 68 | func (rk tRanking) Len() int { 69 | return len(rk) 70 | } 71 | 72 | // tRanking implements intergo.Ranking interface 73 | var _ intergo.Ranking = tRanking{} 74 | 75 | func main() { 76 | ml := &tdm.TeamDraftMultileaving{} 77 | rankingA := tRanking{1, 2, 3, 4, 5} 78 | rankingB := tRanking{10, 20, 30, 40, 50} 79 | 80 | idxToRk := map[int]tRanking{ 81 | 0: rankingA, 82 | 1: rankingB, 83 | } 84 | 85 | res, _ := ml.GetInterleavedRanking(4, rankingA, rankingB) 86 | iRanking := tRanking{} 87 | for _, it := range res { 88 | iRanking = append(iRanking, idxToRk[it.RankingIndex][it.ItemIndex]) 89 | } 90 | 91 | fmt.Printf("Result: %v\n", iRanking) 92 | } 93 | ``` 94 | 95 | ## References 96 | 97 | 1. Radlinski, Filip, Madhu Kurup, and Thorsten Joachims. "How does clickthrough data reflect retrieval quality?." Proceedings of the 17th ACM conference on Information and knowledge management. ACM, 2008. 98 | 2. Schuth, Anne, et al. "Multileaved comparisons for fast online evaluation." Proceedings of the 23rd ACM International Conference on Conference on Information and Knowledge Management. ACM, 2014. 99 | 3. Manabe, Tomohiro, et al. "A comparative live evaluation of multileaving methods on a commercial cqa search." Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval. ACM, 2017. 100 | 4. Kojiro Iizuka, Takeshi Yoneda, Yoshifumi Seki. "Greedy Optimized Multileaving for Personalization." Proceedings of the 13th International ACM Conference on Recommender Systems. ACM, 2019. 101 | 102 | ## Author 103 | 104 | - [@koiizukag](https://github.com/koiizukag) 105 | - [@mathetake](https://twitter.com/mathetake) 106 | 107 | 108 | ## license 109 | 110 | MIT 111 | -------------------------------------------------------------------------------- /bm/README.md: -------------------------------------------------------------------------------- 1 | ## bm package 2 | 3 | `bm` stands for `Balanced Multileaving` and its algorithm is implemented in this package. 4 | 5 | type `BalancedMultileaving` satisfies `intergo.Inteleaving` interface and you can generate 6 | interleaved/multileaved rankings by calling `GetInterleavedRanking` method. 7 | 8 | ### References 9 | 10 | 1. Joachims, Thorsten. "Unbiased evaluation of retrieval quality using clickthrough data." (2002). 11 | 2. Joachims, Thorsten. "Evaluating Retrieval Performance Using Clickthrough Data." (2003): 79-96. 12 | -------------------------------------------------------------------------------- /bm/bm.go: -------------------------------------------------------------------------------- 1 | package bm 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/mathetake/intergo" 7 | ) 8 | 9 | type BalancedMultileaving struct{} 10 | 11 | var _ intergo.Interleaving = &BalancedMultileaving{} 12 | 13 | func (*BalancedMultileaving) GetInterleavedRanking(num int, rankings ...intergo.Ranking) ([]*intergo.Result, error) { 14 | if num < 1 { 15 | return nil, intergo.ErrNonPositiveSamplingNumParameters 16 | } else if len(rankings) < 1 { 17 | return nil, intergo.ErrInsufficientRankingsParameters 18 | } 19 | 20 | var numR = len(rankings) 21 | res := make([]*intergo.Result, 0, num) 22 | 23 | // sIDs stores item's ID in order to prevent duplication in the generated list. 24 | sIDs := make(map[intergo.ID]struct{}, num) 25 | 26 | // The fact that the index stored in usedUpRks means it is already used up. 27 | usedUpRks := make(map[int]struct{}, numR) 28 | 29 | counter := make(map[int]int, numR) 30 | 31 | for len(res) < num && len(usedUpRks) != numR { 32 | 33 | // chose randomly one ranking from the ones used up yet 34 | var selectedRkIdx = rand.Intn(numR) 35 | if _, ok := usedUpRks[selectedRkIdx]; ok { 36 | continue 37 | } 38 | 39 | // get pointer on the selected ranking 40 | c, _ := counter[selectedRkIdx] 41 | 42 | // get ID of the pointed item 43 | itemID := rankings[selectedRkIdx].GetIDByIndex(c) 44 | 45 | if _, ok := sIDs[itemID]; !ok { 46 | res = append(res, &intergo.Result{ 47 | RankingIndex: selectedRkIdx, 48 | ItemIndex: c, 49 | }) 50 | sIDs[itemID] = struct{}{} 51 | } 52 | 53 | // increment pointer on the selected ranking 54 | counter[selectedRkIdx]++ 55 | 56 | if c, _ := counter[selectedRkIdx]; c >= rankings[selectedRkIdx].Len() { 57 | usedUpRks[selectedRkIdx] = struct{}{} 58 | } 59 | } 60 | return res, nil 61 | } 62 | -------------------------------------------------------------------------------- /bm/bm_test.go: -------------------------------------------------------------------------------- 1 | package bm_test 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "testing" 7 | 8 | "github.com/mathetake/intergo" 9 | "github.com/mathetake/intergo/bm" 10 | "gotest.tools/assert" 11 | ) 12 | 13 | type tRanking []int 14 | 15 | func (rk tRanking) GetIDByIndex(i int) intergo.ID { 16 | return intergo.ID(strconv.Itoa(rk[i])) 17 | } 18 | 19 | func (rk tRanking) Len() int { 20 | return len(rk) 21 | } 22 | 23 | var _ intergo.Ranking = tRanking{} 24 | 25 | func TestBalancedMultileaving(t *testing.T) { 26 | b := &bm.BalancedMultileaving{} 27 | 28 | cases := []struct { 29 | inputRks []intergo.Ranking 30 | num int 31 | expectedPatterns [][]intergo.Result 32 | }{ 33 | { 34 | inputRks: []intergo.Ranking{ 35 | tRanking{1, 2, 3, 4, 5}, 36 | tRanking{10, 20, 30, 40, 50}, 37 | }, 38 | num: 2, 39 | expectedPatterns: [][]intergo.Result{ 40 | { 41 | intergo.Result{RankingIndex: 0, ItemIndex: 0}, 42 | intergo.Result{RankingIndex: 1, ItemIndex: 0}, 43 | }, 44 | { 45 | intergo.Result{RankingIndex: 0, ItemIndex: 0}, 46 | intergo.Result{RankingIndex: 0, ItemIndex: 1}, 47 | }, 48 | { 49 | intergo.Result{RankingIndex: 1, ItemIndex: 0}, 50 | intergo.Result{RankingIndex: 0, ItemIndex: 0}, 51 | }, 52 | { 53 | intergo.Result{RankingIndex: 1, ItemIndex: 0}, 54 | intergo.Result{RankingIndex: 1, ItemIndex: 1}, 55 | }, 56 | }, 57 | }, 58 | { 59 | inputRks: []intergo.Ranking{ 60 | tRanking{1, 2, 3, 4, 5}, 61 | tRanking{1, 20, 30, 40, 50}, 62 | }, 63 | num: 2, 64 | expectedPatterns: [][]intergo.Result{ 65 | { 66 | intergo.Result{RankingIndex: 0, ItemIndex: 0}, 67 | intergo.Result{RankingIndex: 1, ItemIndex: 1}, 68 | }, 69 | { 70 | intergo.Result{RankingIndex: 0, ItemIndex: 0}, 71 | intergo.Result{RankingIndex: 0, ItemIndex: 1}, 72 | }, 73 | { 74 | intergo.Result{RankingIndex: 1, ItemIndex: 0}, 75 | intergo.Result{RankingIndex: 0, ItemIndex: 1}, 76 | }, 77 | { 78 | intergo.Result{RankingIndex: 1, ItemIndex: 0}, 79 | intergo.Result{RankingIndex: 1, ItemIndex: 1}, 80 | }, 81 | }, 82 | }, 83 | { 84 | inputRks: []intergo.Ranking{ 85 | tRanking{1, 2, 3, 4, 5}, 86 | tRanking{1, 1, 30, 40, 50}, 87 | }, 88 | num: 2, 89 | expectedPatterns: [][]intergo.Result{ 90 | { 91 | intergo.Result{RankingIndex: 0, ItemIndex: 0}, 92 | intergo.Result{RankingIndex: 1, ItemIndex: 2}, 93 | }, 94 | { 95 | intergo.Result{RankingIndex: 0, ItemIndex: 0}, 96 | intergo.Result{RankingIndex: 0, ItemIndex: 1}, 97 | }, 98 | { 99 | intergo.Result{RankingIndex: 1, ItemIndex: 0}, 100 | intergo.Result{RankingIndex: 0, ItemIndex: 1}, 101 | }, 102 | { 103 | intergo.Result{RankingIndex: 1, ItemIndex: 0}, 104 | intergo.Result{RankingIndex: 1, ItemIndex: 2}, 105 | }, 106 | }, 107 | }, 108 | } 109 | 110 | for n, tc := range cases { 111 | tcc := tc 112 | t.Run(fmt.Sprintf("%d-th unit test", n), func(t *testing.T) { 113 | actual, _ := b.GetInterleavedRanking(tcc.num, tcc.inputRks...) 114 | t.Log("actual: ", actual) 115 | assert.Equal(t, true, len(actual) <= tcc.num) 116 | 117 | var isExpected bool 118 | for _, expected := range tcc.expectedPatterns { 119 | 120 | var isExpectedPattern = true 121 | for i := 0; i < tcc.num; i++ { 122 | if *actual[i] != expected[i] { 123 | isExpectedPattern = false 124 | } 125 | } 126 | 127 | if isExpectedPattern { 128 | isExpected = true 129 | break 130 | } 131 | } 132 | assert.Equal(t, true, isExpected) 133 | }) 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // In package intergo, some interfaces and types are defined for interleaving/multileaving algorithms implementation. 2 | // Specific algorithms are implemented in its subpackages: 3 | // 4 | // https://github.com/mathetake/intergo/blob/master/bm implements balanced multileaving algorithm. 5 | // 6 | // https://github.com/mathetake/intergo/blob/master/gom implements greedy optimized multileaving algorithm. 7 | // 8 | // https://github.com/mathetake/intergo/blob/master/tdm implements team draft multileaving algorithm. 9 | // 10 | // See https://github.com/mathetake/intergo/blob/master/README.md for more details. 11 | package intergo 12 | -------------------------------------------------------------------------------- /errors.go: -------------------------------------------------------------------------------- 1 | package intergo 2 | 3 | import "github.com/pkg/errors" 4 | 5 | // these errors are intended to be returned by Interleaving.GetInterleavedRanking function. 6 | var ( 7 | // ErrNonPositiveSamplingNumParameters should be returned when given "num" is non-positive integer 8 | ErrNonPositiveSamplingNumParameters = errors.New("`num` parameter should be positive") 9 | // ErrInsufficientRankingsParameters should be returned when given rankings is empty. 10 | ErrInsufficientRankingsParameters = errors.New("the number of provided rankings should be positive") 11 | ) 12 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/mathetake/intergo 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/google/go-cmp v0.2.0 // indirect 7 | github.com/pkg/errors v0.8.1 8 | gotest.tools v2.1.0+incompatible 9 | ) 10 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= 2 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 3 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 4 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 5 | gotest.tools v2.1.0+incompatible h1:5USw7CrJBYKqjg9R7QlA6jzqZKEAtvW82aNmsxxGPxw= 6 | gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= 7 | -------------------------------------------------------------------------------- /gom/README.md: -------------------------------------------------------------------------------- 1 | ## gom package 2 | 3 | `gom` stands for `Greedy Optimized Multileaving` and its algorithm is implemented in this package. 4 | 5 | type `GreedyOptimizedMultiLeaving` satisfies `intergo.Inteleaving` interface and you can generate 6 | interleaved/multileaved rankings by calling `GetInterleavedRanking` method. 7 | 8 | ### References 9 | 10 | 1. Schuth, Anne, et al. "Multileaved comparisons for fast online evaluation." Proceedings of the 23rd ACM International Conference on Conference on Information and Knowledge Management. ACM, 2014. 11 | 2. Manabe, Tomohiro, et al. "A comparative live evaluation of multileaving methods on a commercial cqa search." Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval. ACM, 2017. 12 | 3. Kojiro Iizuka, Takeshi Yoneda, Yoshifumi Seki. "Greedy Optimized Multileaving for Personalization." Proceedings of the 13th International ACM Conference on Recommender Systems. ACM, 2019. 13 | -------------------------------------------------------------------------------- /gom/export_test.go: -------------------------------------------------------------------------------- 1 | package gom 2 | 3 | import ( 4 | "github.com/mathetake/intergo" 5 | ) 6 | 7 | func (o *GreedyOptimizedMultiLeaving) ExportedPrefixConstraintSampling(num int, rks ...intergo.Ranking) []*intergo.Result { 8 | return o.prefixConstraintSampling(num, rks...) 9 | } 10 | 11 | func (o *GreedyOptimizedMultiLeaving) ExportedCalcInsensitivity(rks []intergo.Ranking, cRks [][]*intergo.Result) []float64 { 12 | return o.calcInsensitivities(rks, cRks) 13 | } 14 | -------------------------------------------------------------------------------- /gom/gom.go: -------------------------------------------------------------------------------- 1 | package gom 2 | 3 | import ( 4 | "math" 5 | "math/rand" 6 | "sync" 7 | "time" 8 | 9 | "github.com/mathetake/intergo" 10 | ) 11 | 12 | type GreedyOptimizedMultiLeaving struct { 13 | NumSampling int 14 | CreditLabel int 15 | Alpha float64 16 | } 17 | 18 | const els = 1e-20 19 | 20 | var _ intergo.Interleaving = &GreedyOptimizedMultiLeaving{} 21 | 22 | func init() { 23 | rand.Seed(time.Now().UnixNano()) 24 | } 25 | 26 | // GetInterleavedRanking ... get a Interleaved ranking sampled from a set of interleaved rankings 27 | // generated by `prefixConstraintSampling` method. 28 | // Note that the way of the sampling is different from the original paper [Schuth, Anne, et al.,2014] 29 | // where they solved LP with the unbiased constraint. 30 | // We omit the unbiased constraint and only take `sensitivity` into account. Then we sample a ranking 31 | // according to calculated sensitivities defined by equation (1) in [Manabe, Tomohiro, et al., 2017] 32 | func (o *GreedyOptimizedMultiLeaving) GetInterleavedRanking(num int, rankings ...intergo.Ranking) ([]*intergo.Result, error) { 33 | if num < 1 { 34 | return nil, intergo.ErrNonPositiveSamplingNumParameters 35 | } else if len(rankings) < 1 { 36 | return nil, intergo.ErrInsufficientRankingsParameters 37 | } 38 | 39 | var wg sync.WaitGroup 40 | cRks := make([][]*intergo.Result, o.NumSampling) 41 | wg.Add(o.NumSampling) 42 | for i := 0; i < o.NumSampling; i++ { 43 | go func(i int) { 44 | defer wg.Done() 45 | cRks[i] = o.prefixConstraintSampling(num, rankings...) 46 | }(i) 47 | } 48 | wg.Wait() 49 | 50 | // calc Insensitivity of sampled rankings 51 | ins := o.calcInsensitivities(rankings, cRks) 52 | 53 | // init +inf value 54 | min := math.Inf(0) 55 | var maxIdx int 56 | for i, v := range ins { 57 | if v < min { 58 | maxIdx, min = i, v 59 | } 60 | } 61 | return cRks[maxIdx], nil 62 | } 63 | 64 | func (o *GreedyOptimizedMultiLeaving) GetCredit(rankingIndex int, itemId intergo.ID, idToPlacements []map[intergo.ID]int, creditLabel int, isSameRankingIndex bool) float64 { 65 | switch creditLabel { 66 | case 0: 67 | // credit = 1 / (original rank) 68 | placement, ok := idToPlacements[rankingIndex][itemId] 69 | if ok { 70 | return 1 / float64(placement) 71 | } else { 72 | return 1 / float64(len(idToPlacements[rankingIndex])+1) 73 | } 74 | case 1: 75 | // credit = -(relative rank - 1) 76 | if _, ok := idToPlacements[rankingIndex][itemId]; !ok { 77 | return -float64(len(idToPlacements)) 78 | } 79 | var numLess float64 80 | for i := 0; i < len(idToPlacements); i++ { 81 | if _, ok := idToPlacements[i][itemId]; !ok { 82 | continue 83 | } 84 | if idToPlacements[i][itemId] < idToPlacements[rankingIndex][itemId] { 85 | numLess += 1 86 | } 87 | } 88 | return -numLess 89 | default: 90 | // credit = 1 if output ranking idx equals input ranking idx 91 | // else credit = 0 92 | if isSameRankingIndex { 93 | return 1 94 | } 95 | return 0 96 | } 97 | } 98 | 99 | func (o *GreedyOptimizedMultiLeaving) GetIdToPlacementMap(rks []intergo.Ranking) []map[intergo.ID]int { 100 | var iRkNum = len(rks) 101 | itemIds := make(map[intergo.ID]bool) 102 | idToPlacements := make([]map[intergo.ID]int, iRkNum) 103 | // idToPlacements[ranking idx][item id] -> original ranking placement 104 | for i := 0; i < iRkNum; i++ { 105 | m := make(map[intergo.ID]int, rks[i].Len()) 106 | for j := 0; j < rks[i].Len(); j++ { 107 | itemId := rks[i].GetIDByIndex(j) 108 | m[itemId] = j + 1 109 | itemIds[itemId] = true 110 | } 111 | idToPlacements[i] = m 112 | } 113 | return idToPlacements 114 | } 115 | 116 | func (o *GreedyOptimizedMultiLeaving) CalcInsensitivityAndBias(rks []intergo.Ranking, res []*intergo.Result, creditLabel int, alpha float64) (float64, float64) { 117 | var iRkNum = len(rks) 118 | var insensitivityMean float64 119 | 120 | idToPlacements := o.GetIdToPlacementMap(rks) 121 | insensitivityMap := make([]float64, iRkNum) 122 | biasMap := make([][]float64, iRkNum) 123 | 124 | for i := 0; i < iRkNum; i++ { 125 | biasMap[i] = make([]float64, len(res)) 126 | var bias float64 127 | for j := 0; j < len(res); j++ { 128 | var s = 1 / float64(j+1) 129 | itemId := rks[res[j].RankingIndex].GetIDByIndex(res[j].ItemIndex) 130 | credit := o.GetCredit(i, itemId, idToPlacements, creditLabel, res[j].RankingIndex == i) 131 | ss := s * credit 132 | insensitivityMap[i] += ss 133 | insensitivityMean += ss 134 | bias += credit 135 | biasMap[i][j] = bias 136 | } 137 | } 138 | 139 | var biasSum float64 140 | for r := 0; r < len(res); r++ { 141 | min := math.Inf(1) 142 | max := math.Inf(-1) 143 | for i := 0; i < iRkNum; i++ { 144 | v := math.Abs(biasMap[i][r]) 145 | if min > v { 146 | min = v 147 | } 148 | if max < v { 149 | max = v 150 | } 151 | } 152 | if creditLabel != 0 { 153 | min += 1 154 | max += 1 155 | } 156 | biasSum += 1.0 - math.Abs(min/max) 157 | } 158 | 159 | var fResLen = float64(len(res)) 160 | 161 | insensitivityMean /= float64(iRkNum) 162 | if math.Abs(insensitivityMean) < els { 163 | return math.Inf(1), biasSum / fResLen 164 | } 165 | var insensitivitySum float64 166 | for i := 0; i < iRkNum; i++ { 167 | var in = insensitivityMap[i] - insensitivityMean 168 | insensitivitySum += in * in 169 | } 170 | bias := biasSum / fResLen 171 | return (insensitivitySum + alpha*bias) / (insensitivityMean * insensitivityMean), biasSum / fResLen 172 | } 173 | 174 | func (o *GreedyOptimizedMultiLeaving) calcInsensitivities(rks []intergo.Ranking, cRks [][]*intergo.Result) []float64 { 175 | res := make([]float64, len(cRks)) 176 | 177 | var wg sync.WaitGroup 178 | wg.Add(len(cRks)) 179 | for k := 0; k < len(cRks); k++ { 180 | go func(k int) { 181 | defer wg.Done() 182 | res[k], _ = o.CalcInsensitivityAndBias(rks, cRks[k], o.CreditLabel, o.Alpha) 183 | }(k) 184 | } 185 | wg.Wait() 186 | return res 187 | } 188 | 189 | func (*GreedyOptimizedMultiLeaving) prefixConstraintSampling(num int, rks ...intergo.Ranking) []*intergo.Result { 190 | var numR = len(rks) 191 | res := make([]*intergo.Result, 0, num) 192 | 193 | // sIDs stores item's ID in order to prevent duplication in the generated list. 194 | sIDs := make(map[intergo.ID]struct{}, num) 195 | 196 | // The fact that the index stored in usedUpRks means it is already used up. 197 | usedUpRks := make(map[int]struct{}, numR) 198 | 199 | for len(res) < num && len(usedUpRks) != numR { 200 | 201 | // chose randomly one ranking from the ones used up yet 202 | var selectedRkIdx = rand.Intn(numR) 203 | if _, ok := usedUpRks[selectedRkIdx]; ok { 204 | continue 205 | } 206 | 207 | var rk = rks[selectedRkIdx] 208 | var bef = len(res) 209 | for j := 0; j < rk.Len(); j++ { 210 | if _, ok := sIDs[rk.GetIDByIndex(j)]; !ok { 211 | res = append(res, &intergo.Result{ 212 | RankingIndex: selectedRkIdx, 213 | ItemIndex: j, 214 | }) 215 | sIDs[rk.GetIDByIndex(j)] = struct{}{} 216 | break 217 | } 218 | } 219 | 220 | if len(res) == bef { 221 | usedUpRks[selectedRkIdx] = struct{}{} 222 | } 223 | } 224 | return res 225 | } 226 | -------------------------------------------------------------------------------- /gom/gom_test.go: -------------------------------------------------------------------------------- 1 | package gom_test 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "testing" 7 | 8 | "github.com/mathetake/intergo" 9 | "github.com/mathetake/intergo/gom" 10 | "gotest.tools/assert" 11 | ) 12 | 13 | type tRanking []int 14 | 15 | func (rk tRanking) GetIDByIndex(i int) intergo.ID { 16 | return intergo.ID(strconv.Itoa(rk[i])) 17 | } 18 | 19 | func (rk tRanking) Len() int { 20 | return len(rk) 21 | } 22 | 23 | var _ intergo.Ranking = tRanking{} 24 | 25 | func TestGetInterleavedRanking(t *testing.T) { 26 | o := &gom.GreedyOptimizedMultiLeaving{ 27 | NumSampling: 100, 28 | CreditLabel: 0, 29 | Alpha: 0, 30 | } 31 | 32 | cases := []struct { 33 | num int 34 | inputRankings []intergo.Ranking 35 | expected []*intergo.Result 36 | }{ 37 | { 38 | inputRankings: []intergo.Ranking{ 39 | tRanking{1, 2, 3, 4, 5}, 40 | tRanking{10, 20, 30, 40, 50}, 41 | }, 42 | expected: []*intergo.Result{ 43 | {RankingIndex: 0, ItemIndex: 0}, 44 | {RankingIndex: 1, ItemIndex: 0}, 45 | }, 46 | num: 2, 47 | }, 48 | { 49 | inputRankings: []intergo.Ranking{ 50 | tRanking{1, 2, 3}, 51 | tRanking{10, 20, 30}, 52 | }, 53 | expected: []*intergo.Result{ 54 | {RankingIndex: 0, ItemIndex: 0}, 55 | {RankingIndex: 1, ItemIndex: 0}, 56 | {RankingIndex: 1, ItemIndex: 1}, 57 | }, 58 | num: 10, 59 | }, 60 | { 61 | inputRankings: []intergo.Ranking{ 62 | tRanking{1, 2, 3, 10, 10, 30}, 63 | tRanking{10, 20, 30}, 64 | tRanking{100, 200, 300}, 65 | }, 66 | expected: []*intergo.Result{ 67 | {RankingIndex: 0, ItemIndex: 0}, 68 | {RankingIndex: 1, ItemIndex: 0}, 69 | {RankingIndex: 2, ItemIndex: 0}, 70 | }, 71 | num: 2, 72 | }, 73 | } 74 | 75 | for n, tc := range cases { 76 | tcc := tc 77 | t.Run(fmt.Sprintf("%d-th unit test", n), func(t *testing.T) { 78 | actual, err := o.GetInterleavedRanking(tcc.num, tcc.inputRankings...) 79 | if err != nil { 80 | t.Fatal(err) 81 | } 82 | fmt.Println("actual: ", actual) 83 | }) 84 | } 85 | } 86 | 87 | func TestGetCredit(t *testing.T) { 88 | 89 | cases := []struct { 90 | RankingIndex int 91 | itemId intergo.ID 92 | idToPlacements []map[intergo.ID]int 93 | creditLabel int 94 | isSameRankingIndex bool 95 | expected float64 96 | }{ 97 | { 98 | RankingIndex: 1, 99 | itemId: "item1", 100 | idToPlacements: []map[intergo.ID]int{ 101 | {"item1": 1, "item2": 2, "item3": 3}, 102 | {"item1": 3, "item2": 1, "item3": 2}, 103 | {"item1": 2, "item2": 1, "item3": 3}, 104 | }, 105 | creditLabel: 0, 106 | isSameRankingIndex: false, 107 | expected: 0.3333333333333333, 108 | }, 109 | { 110 | RankingIndex: 1, 111 | itemId: "item1", 112 | idToPlacements: []map[intergo.ID]int{ 113 | {"item1": 1, "item2": 2, "item3": 3}, 114 | {"item1": 3, "item2": 1, "item3": 2}, 115 | {"item1": 2, "item2": 1, "item3": 3}, 116 | }, 117 | creditLabel: 1, 118 | isSameRankingIndex: false, 119 | expected: -2.0, 120 | }, 121 | { 122 | RankingIndex: 0, 123 | itemId: "item2", 124 | idToPlacements: []map[intergo.ID]int{ 125 | {"item1": 1, "item3": 3}, 126 | {"item1": 3, "item2": 1, "item3": 2}, 127 | {"item1": 2, "item2": 1, "item3": 3}, 128 | }, 129 | creditLabel: 1, 130 | isSameRankingIndex: false, 131 | expected: -3.0, 132 | }, 133 | { 134 | RankingIndex: 1, 135 | itemId: "item2", 136 | idToPlacements: []map[intergo.ID]int{ 137 | {"item1": 1, "item3": 3}, 138 | {"item1": 3, "item2": 1, "item3": 2}, 139 | {"item1": 2, "item2": 1, "item3": 3}, 140 | }, 141 | creditLabel: 1, 142 | isSameRankingIndex: false, 143 | expected: 0.0, 144 | }, 145 | { 146 | RankingIndex: 0, 147 | itemId: "item2", 148 | idToPlacements: []map[intergo.ID]int{ 149 | {"item1": 1, "item2": 2, "item3": 3}, 150 | {"item1": 3, "item2": 1, "item3": 2}, 151 | {"item1": 2, "item2": 1, "item3": 3}, 152 | }, 153 | creditLabel: 3, 154 | isSameRankingIndex: false, 155 | expected: 0, 156 | }, 157 | { 158 | RankingIndex: 0, 159 | itemId: "item2", 160 | idToPlacements: []map[intergo.ID]int{ 161 | {"item1": 1, "item2": 2, "item3": 3}, 162 | {"item1": 3, "item2": 1, "item3": 2}, 163 | {"item1": 2, "item2": 1, "item3": 3}, 164 | }, 165 | creditLabel: 3, 166 | isSameRankingIndex: true, 167 | expected: 1, 168 | }, 169 | } 170 | 171 | for n, tc := range cases { 172 | tcc := tc 173 | o := &gom.GreedyOptimizedMultiLeaving{ 174 | CreditLabel: tc.creditLabel, 175 | } 176 | t.Run(fmt.Sprintf("%d-th unit test", n), func(t *testing.T) { 177 | actual := o.GetCredit(tcc.RankingIndex, tcc.itemId, tcc.idToPlacements, tcc.creditLabel, tcc.isSameRankingIndex) 178 | assert.Equal(t, tcc.expected, actual) 179 | }) 180 | } 181 | } 182 | 183 | func TestPrefixConstraintSampling(t *testing.T) { 184 | o := &gom.GreedyOptimizedMultiLeaving{} 185 | 186 | cases := []struct { 187 | inputRks []intergo.Ranking 188 | num int 189 | expectedPatterns [][]intergo.Result 190 | }{ 191 | { 192 | inputRks: []intergo.Ranking{ 193 | tRanking{1, 2, 3, 4, 5}, 194 | tRanking{10, 20, 30, 40, 50}, 195 | }, 196 | num: 2, 197 | expectedPatterns: [][]intergo.Result{ 198 | { 199 | intergo.Result{RankingIndex: 0, ItemIndex: 0}, 200 | intergo.Result{RankingIndex: 1, ItemIndex: 0}, 201 | }, 202 | { 203 | intergo.Result{RankingIndex: 0, ItemIndex: 0}, 204 | intergo.Result{RankingIndex: 0, ItemIndex: 1}, 205 | }, 206 | { 207 | intergo.Result{RankingIndex: 1, ItemIndex: 0}, 208 | intergo.Result{RankingIndex: 0, ItemIndex: 0}, 209 | }, 210 | { 211 | intergo.Result{RankingIndex: 1, ItemIndex: 0}, 212 | intergo.Result{RankingIndex: 1, ItemIndex: 1}, 213 | }, 214 | }, 215 | }, 216 | { 217 | inputRks: []intergo.Ranking{ 218 | tRanking{1, 2, 3, 4, 5}, 219 | tRanking{1, 20, 30, 40, 50}, 220 | }, 221 | num: 2, 222 | expectedPatterns: [][]intergo.Result{ 223 | { 224 | intergo.Result{RankingIndex: 0, ItemIndex: 0}, 225 | intergo.Result{RankingIndex: 1, ItemIndex: 1}, 226 | }, 227 | { 228 | intergo.Result{RankingIndex: 0, ItemIndex: 0}, 229 | intergo.Result{RankingIndex: 0, ItemIndex: 1}, 230 | }, 231 | { 232 | intergo.Result{RankingIndex: 1, ItemIndex: 0}, 233 | intergo.Result{RankingIndex: 0, ItemIndex: 1}, 234 | }, 235 | { 236 | intergo.Result{RankingIndex: 1, ItemIndex: 0}, 237 | intergo.Result{RankingIndex: 1, ItemIndex: 1}, 238 | }, 239 | }, 240 | }, 241 | { 242 | inputRks: []intergo.Ranking{ 243 | tRanking{1, 2, 3, 4, 5}, 244 | tRanking{1, 20, 30, 40, 50}, 245 | }, 246 | num: 3, 247 | expectedPatterns: [][]intergo.Result{ 248 | { 249 | intergo.Result{RankingIndex: 0, ItemIndex: 0}, 250 | intergo.Result{RankingIndex: 1, ItemIndex: 1}, 251 | intergo.Result{RankingIndex: 0, ItemIndex: 1}, 252 | }, 253 | { 254 | intergo.Result{RankingIndex: 0, ItemIndex: 0}, 255 | intergo.Result{RankingIndex: 0, ItemIndex: 1}, 256 | intergo.Result{RankingIndex: 1, ItemIndex: 1}, 257 | }, 258 | { 259 | intergo.Result{RankingIndex: 0, ItemIndex: 0}, 260 | intergo.Result{RankingIndex: 0, ItemIndex: 1}, 261 | intergo.Result{RankingIndex: 0, ItemIndex: 2}, 262 | }, 263 | { 264 | intergo.Result{RankingIndex: 0, ItemIndex: 0}, 265 | intergo.Result{RankingIndex: 1, ItemIndex: 1}, 266 | intergo.Result{RankingIndex: 1, ItemIndex: 2}, 267 | }, 268 | { 269 | intergo.Result{RankingIndex: 1, ItemIndex: 0}, 270 | intergo.Result{RankingIndex: 1, ItemIndex: 1}, 271 | intergo.Result{RankingIndex: 0, ItemIndex: 1}, 272 | }, 273 | { 274 | intergo.Result{RankingIndex: 1, ItemIndex: 0}, 275 | intergo.Result{RankingIndex: 1, ItemIndex: 1}, 276 | intergo.Result{RankingIndex: 1, ItemIndex: 2}, 277 | }, 278 | { 279 | intergo.Result{RankingIndex: 1, ItemIndex: 0}, 280 | intergo.Result{RankingIndex: 0, ItemIndex: 1}, 281 | intergo.Result{RankingIndex: 1, ItemIndex: 1}, 282 | }, 283 | { 284 | intergo.Result{RankingIndex: 1, ItemIndex: 0}, 285 | intergo.Result{RankingIndex: 0, ItemIndex: 1}, 286 | intergo.Result{RankingIndex: 0, ItemIndex: 2}, 287 | }, 288 | }, 289 | }, 290 | } 291 | 292 | for n, tc := range cases { 293 | tcc := tc 294 | t.Run(fmt.Sprintf("%d-th unit test", n), func(t *testing.T) { 295 | actual := o.ExportedPrefixConstraintSampling(tcc.num, tcc.inputRks...) 296 | assert.Equal(t, true, len(actual) <= tcc.num) 297 | 298 | var isExpected bool 299 | for _, expected := range tcc.expectedPatterns { 300 | 301 | var isExpectedPattern = true 302 | for i := 0; i < tcc.num; i++ { 303 | if *actual[i] != expected[i] { 304 | isExpectedPattern = false 305 | } 306 | } 307 | 308 | if isExpectedPattern { 309 | isExpected = true 310 | break 311 | } 312 | } 313 | assert.Equal(t, true, isExpected) 314 | }) 315 | } 316 | } 317 | 318 | func TestCalcInsensitivity(t *testing.T) { 319 | o := &gom.GreedyOptimizedMultiLeaving{Alpha: 0, CreditLabel: 0} 320 | 321 | cases := []struct { 322 | inputRankings []intergo.Ranking 323 | combinedRankings [][]*intergo.Result 324 | expected []float64 325 | threshold float64 326 | }{ 327 | { 328 | inputRankings: []intergo.Ranking{ 329 | tRanking{1, 2, 3, 4, 5}, 330 | tRanking{10, 20, 30, 40, 50}, 331 | }, 332 | combinedRankings: [][]*intergo.Result{ 333 | { 334 | &intergo.Result{RankingIndex: 0, ItemIndex: 0}, 335 | &intergo.Result{RankingIndex: 1, ItemIndex: 0}, 336 | }, 337 | { 338 | &intergo.Result{RankingIndex: 0, ItemIndex: 0}, 339 | &intergo.Result{RankingIndex: 0, ItemIndex: 1}, 340 | }, 341 | }, 342 | expected: []float64{0.1133786848, 0.8888888889}, 343 | threshold: 10e-7, 344 | }, 345 | { 346 | inputRankings: []intergo.Ranking{ 347 | tRanking{1, 2, 3}, 348 | tRanking{10, 20, 30}, 349 | }, 350 | combinedRankings: [][]*intergo.Result{ 351 | { 352 | &intergo.Result{RankingIndex: 0, ItemIndex: 0}, 353 | &intergo.Result{RankingIndex: 1, ItemIndex: 0}, 354 | &intergo.Result{RankingIndex: 1, ItemIndex: 1}, 355 | }, 356 | { 357 | &intergo.Result{RankingIndex: 0, ItemIndex: 0}, 358 | &intergo.Result{RankingIndex: 0, ItemIndex: 1}, 359 | &intergo.Result{RankingIndex: 0, ItemIndex: 2}, 360 | }, 361 | }, 362 | expected: []float64{0.0376778162, 0.4923955480}, 363 | threshold: 10e-8, 364 | }, 365 | { 366 | inputRankings: []intergo.Ranking{ 367 | tRanking{1, 2, 3}, 368 | tRanking{10, 20, 30}, 369 | tRanking{100, 200, 300}, 370 | }, 371 | combinedRankings: [][]*intergo.Result{ 372 | { 373 | &intergo.Result{RankingIndex: 0, ItemIndex: 0}, 374 | &intergo.Result{RankingIndex: 1, ItemIndex: 0}, 375 | &intergo.Result{RankingIndex: 2, ItemIndex: 0}, 376 | }, 377 | { 378 | &intergo.Result{RankingIndex: 0, ItemIndex: 0}, 379 | &intergo.Result{RankingIndex: 0, ItemIndex: 1}, 380 | &intergo.Result{RankingIndex: 2, ItemIndex: 0}, 381 | }, 382 | { 383 | &intergo.Result{RankingIndex: 1, ItemIndex: 0}, 384 | &intergo.Result{RankingIndex: 1, ItemIndex: 1}, 385 | &intergo.Result{RankingIndex: 0, ItemIndex: 0}, 386 | }, 387 | }, 388 | expected: []float64{0.1611570248, 0.5850000000, 0.5850000000}, 389 | threshold: 10e-8, 390 | }, 391 | } 392 | 393 | for n, tc := range cases { 394 | tcc := tc 395 | t.Run(fmt.Sprintf("%d-th unit test", n), func(t *testing.T) { 396 | actual := o.ExportedCalcInsensitivity(tcc.inputRankings, tcc.combinedRankings) 397 | assert.Equal(t, len(tcc.expected), len(actual)) 398 | for i := range tcc.expected { 399 | diff := actual[i] - tcc.expected[i] 400 | if diff < 0 { 401 | diff = -diff 402 | } 403 | if true != (diff < tcc.threshold) { 404 | t.Logf("unexpected difference at %d-th element: actual:%.10f != expected:%.10f", i, actual[i], tcc.expected[i]) 405 | } 406 | assert.Equal(t, true, diff < tcc.threshold) 407 | } 408 | }) 409 | } 410 | } 411 | -------------------------------------------------------------------------------- /intergo.go: -------------------------------------------------------------------------------- 1 | package intergo 2 | 3 | // type ID is used as identifier of items. 4 | // The purpose is to remove item duplication in generated rankings. 5 | type ID string 6 | 7 | // Ranking is the interface which all of target ranking should implement. 8 | type Ranking interface { 9 | // GetIDByIndex allows interleaving/multileaving algorithms to access items' identifier 10 | GetIDByIndex(int) ID 11 | 12 | // Len is used to get the "length" of the ranking 13 | Len() int 14 | } 15 | 16 | // Result is the type of generated ranking's each entity. 17 | type Result struct { 18 | // RankingIndex represents to which ranking the item belongs 19 | RankingIndex int 20 | 21 | // ItemIndex represents the item's index in the ranking declared by RankingIndex 22 | ItemIndex int 23 | } 24 | 25 | // Interleaving is the interface which every interleaving/multileaving algorithm should implement. 26 | type Interleaving interface { 27 | // GetInterleavedRanking is intended to be used for ranking generation. 28 | // 29 | // First argument "num" is the expected length of a resulted ranking. Ideally, 30 | // if the total number of unique items in given rankings is greater than equal "num", 31 | // the resulted length should equal "num". However, it depends on the implementations. 32 | // 33 | // "rankings" should be your rankings from which you want to generate a multileaved ranking. 34 | GetInterleavedRanking(num int, rankings ...Ranking) ([]*Result, error) 35 | } 36 | -------------------------------------------------------------------------------- /intergo_test.go: -------------------------------------------------------------------------------- 1 | // put bench marks on implemented algorithms 2 | package intergo_test 3 | 4 | import ( 5 | "fmt" 6 | "strconv" 7 | "testing" 8 | 9 | "github.com/mathetake/intergo" 10 | "github.com/mathetake/intergo/bm" 11 | "github.com/mathetake/intergo/gom" 12 | "github.com/mathetake/intergo/tdm" 13 | ) 14 | 15 | type tRanking []int 16 | 17 | func (rk tRanking) GetIDByIndex(i int) intergo.ID { 18 | return intergo.ID(strconv.Itoa(rk[i])) 19 | } 20 | 21 | func (rk tRanking) Len() int { 22 | return len(rk) 23 | } 24 | 25 | type fixture struct { 26 | inputRankingItemNum int 27 | interleavedRankingItemNum int 28 | } 29 | 30 | var fixtures = []fixture{ 31 | {inputRankingItemNum: 10, interleavedRankingItemNum: 5}, 32 | {inputRankingItemNum: 200, interleavedRankingItemNum: 50}, 33 | {inputRankingItemNum: 200, interleavedRankingItemNum: 200}, 34 | {inputRankingItemNum: 1000, interleavedRankingItemNum: 200}, 35 | } 36 | 37 | func BenchmarkMultileaving(b *testing.B) { 38 | for _, inputRankingNum := range []int{2, 10, 50, 100} { 39 | for n, fx := range fixtures { 40 | fxx := fx 41 | b.ReportAllocs() 42 | fmt.Println("") 43 | fmt.Printf( 44 | "inputRankingNum: %d, inputRankingItemNum: %d, interleavedRankingItemNum: %d\n", 45 | inputRankingNum, fxx.inputRankingItemNum, fxx.interleavedRankingItemNum, 46 | ) 47 | 48 | b.Run(fmt.Sprintf("[[%d-th bench on Team Draft Multileaving]]", n), func(b *testing.B) { 49 | benchmarkInputNum(fxx, inputRankingNum, &tdm.TeamDraftMultileaving{}, b) 50 | }) 51 | 52 | b.Run(fmt.Sprintf("[[%d-th bench on Balanced Multileaving]]", n), func(b *testing.B) { 53 | benchmarkInputNum(fxx, inputRankingNum, &bm.BalancedMultileaving{}, b) 54 | }) 55 | 56 | for _, samplingSize := range []int{2, 10, 50, 100} { 57 | b.Run(fmt.Sprintf("[[%d-th bench on Greedy Optimized Multileaving with sampling size: %d]]", n, samplingSize), func(b *testing.B) { 58 | benchmarkInputNum(fxx, inputRankingNum, &gom.GreedyOptimizedMultiLeaving{ 59 | NumSampling: samplingSize, CreditLabel: 0, Alpha: 0, 60 | }, b) 61 | }) 62 | } 63 | fmt.Println("") 64 | } 65 | } 66 | } 67 | 68 | func benchmarkInputNum(fx fixture, inputRankingNum int, il intergo.Interleaving, b *testing.B) { 69 | rks := getRankings(fx, inputRankingNum) 70 | b.ResetTimer() 71 | for n := 0; n < b.N; n++ { 72 | il.GetInterleavedRanking(fx.interleavedRankingItemNum, rks...) 73 | } 74 | } 75 | 76 | func getRankings(fx fixture, inputRankingNum int) []intergo.Ranking { 77 | rks := make([]intergo.Ranking, inputRankingNum) 78 | for i := 0; i < inputRankingNum; i++ { 79 | rk := tRanking{} 80 | for j := 0; j < fx.inputRankingItemNum; j++ { 81 | rk = append(rk, i*fx.inputRankingItemNum+j) 82 | } 83 | rks[i] = rk 84 | } 85 | return rks 86 | } 87 | -------------------------------------------------------------------------------- /tdm/README.md: -------------------------------------------------------------------------------- 1 | ## tdm package 2 | 3 | `tdm` stands for `Team Draft Multileaving` and its algorithm is implemented in this package. 4 | 5 | type `TeamDraftMultileaving` satisfies `intergo.Inteleaving` interface and you can generate 6 | interleaved/multileaved rankings by calling `GetInterleavedRanking` method. 7 | 8 | ### References 9 | 10 | 1. Radlinski, Filip, Madhu Kurup, and Thorsten Joachims. "How does clickthrough data reflect retrieval quality?." Proceedings of the 17th ACM conference on Information and knowledge management. ACM, 2008. 11 | 2. Schuth, Anne, et al. "Multileaved comparisons for fast online evaluation." Proceedings of the 23rd ACM International Conference on Conference on Information and Knowledge Management. ACM, 2014. 12 | -------------------------------------------------------------------------------- /tdm/export_test.go: -------------------------------------------------------------------------------- 1 | package tdm 2 | 3 | func ExportedPopRandomIdx(target []int) (int, []int) { 4 | return popRandomIdx(target) 5 | } 6 | -------------------------------------------------------------------------------- /tdm/tdm.go: -------------------------------------------------------------------------------- 1 | package tdm 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/mathetake/intergo" 7 | ) 8 | 9 | type TeamDraftMultileaving struct{} 10 | 11 | var _ intergo.Interleaving = &TeamDraftMultileaving{} 12 | 13 | func (tdm *TeamDraftMultileaving) GetInterleavedRanking(num int, rankings ...intergo.Ranking) ([]*intergo.Result, error) { 14 | if num < 1 { 15 | return nil, intergo.ErrNonPositiveSamplingNumParameters 16 | } else if len(rankings) < 1 { 17 | return nil, intergo.ErrInsufficientRankingsParameters 18 | } 19 | 20 | var numR = len(rankings) 21 | res := make([]*intergo.Result, 0, num) 22 | 23 | // sIDs stores item's ID in order to prevent duplication in the generated list. 24 | sIDs := make(map[intergo.ID]struct{}, num) 25 | 26 | // minRks have rankings' index whose number of selected items is minimum 27 | minRks := make([]int, 0, numR) 28 | 29 | // lastIdx has a last index of the indexed ranking 30 | lastIdx := make(map[int]int, numR) 31 | for i := 0; i < numR; i++ { 32 | minRks = append(minRks, i) 33 | lastIdx[i] = 0 34 | } 35 | 36 | // The fact that the index stored in usedUpRks means it is already used up. 37 | usedUpRks := make(map[int]struct{}, numR) 38 | 39 | for len(res) < num && len(usedUpRks) != numR { 40 | 41 | // chose one ranking from keys of minRks 42 | var selected int 43 | selected, minRks = popRandomIdx(minRks) 44 | var rk = rankings[selected] 45 | 46 | var bef = len(res) 47 | 48 | for j := lastIdx[selected]; j < rk.Len(); j++ { 49 | if _, ok := sIDs[rk.GetIDByIndex(j)]; !ok { 50 | res = append(res, &intergo.Result{ 51 | RankingIndex: selected, 52 | ItemIndex: j, 53 | }) 54 | 55 | sIDs[rk.GetIDByIndex(j)] = struct{}{} 56 | lastIdx[selected] = j 57 | break 58 | } 59 | } 60 | 61 | if len(res) == bef { 62 | usedUpRks[selected] = struct{}{} 63 | } 64 | 65 | if len(minRks) == 0 { 66 | // restore the targets 67 | minRks = make([]int, 0, numR-len(usedUpRks)) 68 | for i := 0; i < numR; i++ { 69 | if _, ok := usedUpRks[i]; !ok { 70 | minRks = append(minRks, i) 71 | } 72 | } 73 | } 74 | } 75 | return res, nil 76 | } 77 | 78 | func popRandomIdx(target []int) (int, []int) { 79 | if len(target) == 1 { 80 | return target[0], []int{} 81 | } 82 | 83 | selectedIdx := rand.Intn(len(target)) 84 | selected := target[selectedIdx] 85 | 86 | popped := make([]int, 0, len(target)-1) 87 | 88 | for i, idx := range target { 89 | if i < selectedIdx { 90 | popped = append(popped, idx) 91 | } else if i == selectedIdx { 92 | continue 93 | } else { 94 | popped = append(popped, idx) 95 | } 96 | } 97 | return selected, popped 98 | } 99 | -------------------------------------------------------------------------------- /tdm/tdm_test.go: -------------------------------------------------------------------------------- 1 | package tdm_test 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "testing" 7 | 8 | "github.com/mathetake/intergo" 9 | "github.com/mathetake/intergo/tdm" 10 | "gotest.tools/assert" 11 | ) 12 | 13 | type tRanking []int 14 | 15 | func (rk tRanking) GetIDByIndex(i int) intergo.ID { 16 | return intergo.ID(strconv.Itoa(rk[i])) 17 | } 18 | 19 | func (rk tRanking) Len() int { 20 | return len(rk) 21 | } 22 | 23 | var _ intergo.Ranking = tRanking{} 24 | 25 | func TestTeamDraftMultileaving(t *testing.T) { 26 | td := &tdm.TeamDraftMultileaving{} 27 | 28 | cases := []struct { 29 | inputRks []intergo.Ranking 30 | num int 31 | expectedPatterns [][]intergo.Result 32 | expErr error 33 | }{ 34 | { 35 | inputRks: []intergo.Ranking{}, 36 | num: 10, 37 | expErr: intergo.ErrInsufficientRankingsParameters, 38 | }, 39 | { 40 | inputRks: []intergo.Ranking{ 41 | tRanking{1, 2, 3, 4, 5}, 42 | tRanking{10, 20, 30, 40, 50}, 43 | }, 44 | num: 0, 45 | expErr: intergo.ErrNonPositiveSamplingNumParameters, 46 | }, 47 | { 48 | expErr: intergo.ErrNonPositiveSamplingNumParameters, 49 | }, 50 | { 51 | inputRks: []intergo.Ranking{ 52 | tRanking{1, 2, 3, 4, 5}, 53 | tRanking{10, 20, 30, 40, 50}, 54 | }, 55 | num: 2, 56 | expectedPatterns: [][]intergo.Result{ 57 | { 58 | intergo.Result{RankingIndex: 0, ItemIndex: 0}, 59 | intergo.Result{RankingIndex: 1, ItemIndex: 0}, 60 | }, 61 | { 62 | intergo.Result{RankingIndex: 1, ItemIndex: 0}, 63 | intergo.Result{RankingIndex: 0, ItemIndex: 0}, 64 | }, 65 | }, 66 | }, 67 | { 68 | inputRks: []intergo.Ranking{ 69 | tRanking{1, 2, 3, 4, 5}, 70 | tRanking{1, 20, 30, 40, 50}, 71 | }, 72 | num: 2, 73 | expectedPatterns: [][]intergo.Result{ 74 | { 75 | intergo.Result{RankingIndex: 0, ItemIndex: 0}, 76 | intergo.Result{RankingIndex: 1, ItemIndex: 1}, 77 | }, 78 | { 79 | intergo.Result{RankingIndex: 1, ItemIndex: 0}, 80 | intergo.Result{RankingIndex: 0, ItemIndex: 1}, 81 | }, 82 | }, 83 | }, 84 | { 85 | inputRks: []intergo.Ranking{ 86 | tRanking{1, 2, 3, 4, 5}, 87 | tRanking{1, 20, 30, 40, 50}, 88 | }, 89 | num: 3, 90 | expectedPatterns: [][]intergo.Result{ 91 | { 92 | intergo.Result{RankingIndex: 0, ItemIndex: 0}, 93 | intergo.Result{RankingIndex: 1, ItemIndex: 1}, 94 | intergo.Result{RankingIndex: 0, ItemIndex: 1}, 95 | }, 96 | { 97 | intergo.Result{RankingIndex: 1, ItemIndex: 0}, 98 | intergo.Result{RankingIndex: 0, ItemIndex: 1}, 99 | intergo.Result{RankingIndex: 1, ItemIndex: 1}, 100 | }, 101 | { 102 | intergo.Result{RankingIndex: 1, ItemIndex: 0}, 103 | intergo.Result{RankingIndex: 0, ItemIndex: 1}, 104 | intergo.Result{RankingIndex: 0, ItemIndex: 2}, 105 | }, 106 | { 107 | intergo.Result{RankingIndex: 0, ItemIndex: 0}, 108 | intergo.Result{RankingIndex: 1, ItemIndex: 1}, 109 | intergo.Result{RankingIndex: 1, ItemIndex: 2}, 110 | }, 111 | }, 112 | }, 113 | } 114 | 115 | for n, tc := range cases { 116 | tc := tc 117 | t.Run(fmt.Sprintf("%d-th unit test", n), func(t *testing.T) { 118 | actual, actualErr := td.GetInterleavedRanking(tc.num, tc.inputRks...) 119 | if tc.expErr != nil { 120 | assert.Equal(t, tc.expErr, actualErr) 121 | return // exit 122 | } else if actualErr != nil { 123 | t.Fatal(actualErr) 124 | } 125 | 126 | assert.Equal(t, true, len(actual) <= tc.num) 127 | 128 | var isExpected bool 129 | for _, expected := range tc.expectedPatterns { 130 | 131 | var isExpectedPattern = true 132 | for i := 0; i < tc.num; i++ { 133 | if *actual[i] != expected[i] { 134 | isExpectedPattern = false 135 | } 136 | } 137 | 138 | if isExpectedPattern { 139 | isExpected = true 140 | break 141 | } 142 | } 143 | assert.Equal(t, true, isExpected) 144 | }) 145 | } 146 | } 147 | 148 | func TestPopRandomIdx(t *testing.T) { 149 | for i, cc := range []struct { 150 | target []int 151 | expLen int 152 | }{ 153 | { 154 | target: []int{1}, 155 | expLen: 0, 156 | }, 157 | { 158 | target: []int{1, 2, 3, 4}, 159 | expLen: 3, 160 | }, 161 | { 162 | target: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, 163 | expLen: 9, 164 | }, 165 | } { 166 | c := cc 167 | t.Run(fmt.Sprintf("%d-th case", i), func(t *testing.T) { 168 | actualS, actualP := tdm.ExportedPopRandomIdx(c.target) 169 | assert.Equal(t, c.expLen, len(actualP)) 170 | 171 | isIncluded := false 172 | for _, actual := range actualP { 173 | if actual == actualS { 174 | isIncluded = true 175 | } 176 | } 177 | 178 | assert.Equal(t, false, isIncluded) 179 | }) 180 | } 181 | } 182 | 183 | func TestTeamDraftMultileaving_RankingRatio(t *testing.T) { 184 | ml := &tdm.TeamDraftMultileaving{} 185 | 186 | for i, cc := range []struct { 187 | itemNum, rankingNum, returnedNum int 188 | threshold float64 189 | }{ 190 | {1e2, 3, 10, 1e-1}, 191 | {1e3, 3, 100, 1e-2}, 192 | {1e4, 3, 1000, 1e-3}, 193 | {1e5, 3, 10000, 1e-4}, 194 | } { 195 | c := cc 196 | t.Run(fmt.Sprintf("%d-th case", i), func(t *testing.T) { 197 | rks := getRankings(c.itemNum, c.rankingNum) 198 | 199 | res, err := ml.GetInterleavedRanking(c.returnedNum, rks...) 200 | if err != nil { 201 | t.Fatalf("GetInterleavedRanking failed: %v", err) 202 | } 203 | 204 | counts := map[int]int{} 205 | for _, it := range res { 206 | counts[it.RankingIndex]++ 207 | } 208 | fmt.Println(counts) 209 | 210 | for _, v := range counts { 211 | diff := float64(v)/float64(c.returnedNum) - float64(1)/float64(c.rankingNum) 212 | 213 | if diff < 0 { 214 | diff *= -1 215 | } 216 | 217 | assert.Equal(t, true, diff < c.threshold) 218 | } 219 | }) 220 | } 221 | } 222 | 223 | func getRankings(itemNum, RankingNum int) []intergo.Ranking { 224 | rks := make([]intergo.Ranking, RankingNum) 225 | for i := 0; i < RankingNum; i++ { 226 | rk := tRanking{} 227 | for j := 0; j < itemNum; j++ { 228 | rk = append(rk, i*itemNum+j) 229 | } 230 | rks[i] = rk 231 | } 232 | return rks 233 | } 234 | --------------------------------------------------------------------------------