├── README.md ├── cmd ├── eval │ └── chibeval.go ├── fit │ └── rldafit.go └── inf │ └── rldainf.go ├── ints └── ints.go ├── lda └── lda.go ├── model ├── data.go ├── model.go └── trainee.go └── umath ├── optimize.go └── umath.go /README.md: -------------------------------------------------------------------------------- 1 | # CompareLDA: A Topic Model for Document Comparison 2 | 3 | Comparative Latent Dirichlet Allocation (CompareLDA) learns predictive topic distributions that comply with the pairwise comparison observations. 4 | 5 | ### Installation 6 | 7 | This is a GoLang project and can be assembled using standard [GoLang](https://golang.org) infrastructure: 8 | * `cmd/eval/chibeval.go` is a Chib-style estimator of the predictive log-likelihood; 9 | * `cmd/fit/rldafit.go` is a CompareLDA trainer; 10 | * `cmd/inf/rldainf.go` is a CompareLDA predictor. 11 | -------------------------------------------------------------------------------- /cmd/eval/chibeval.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "math" 9 | "math/rand" 10 | "os" 11 | "strconv" 12 | "strings" 13 | 14 | "github.com/gonum/floats" 15 | ) 16 | 17 | type lda struct { 18 | alpha float64 19 | phi [][]float64 20 | } 21 | 22 | func parseFloats(line string, isLog bool) []float64 { 23 | sc := bufio.NewScanner(strings.NewReader(line)) 24 | parse := make([]float64, 0, 64) 25 | sc.Split(bufio.ScanWords) 26 | for sc.Scan() { 27 | if val, err := strconv.ParseFloat(sc.Text(), 64); err == nil { 28 | if isLog { 29 | val = math.Exp(val) 30 | } 31 | parse = append(parse, val) 32 | } else { 33 | log.Panic("ERROR: unable to parse floats", err) 34 | } 35 | } 36 | if isLog { 37 | floats.Scale(1/floats.Sum(parse), parse) 38 | } 39 | return parse 40 | } 41 | 42 | func readPhi(fn string, isLog bool) [][]float64 { 43 | f, err := os.Open(fn) 44 | if err != nil { 45 | log.Fatal(err) 46 | } 47 | defer f.Close() 48 | 49 | phi := make([][]float64, 0, 10) 50 | sc := bufio.NewScanner(f) 51 | buf := make([]byte, 0, 64*1024) 52 | sc.Buffer(buf, 1024*1024) 53 | for sc.Scan() { 54 | phi = append(phi, parseFloats(sc.Text(), isLog)) 55 | } 56 | if sc.Err() != nil { 57 | log.Panic("ERROR: unable to parse data file", sc.Err().Error()) 58 | } 59 | return phi 60 | } 61 | 62 | func parseAndFilterInts(s string, v int) []int { 63 | sc := bufio.NewScanner(strings.NewReader(s)) 64 | sc.Split(bufio.ScanWords) 65 | result := make([]int, 0, 10) 66 | for sc.Scan() { 67 | if val, err := strconv.Atoi(sc.Text()); err == nil { 68 | if val < v { 69 | result = append(result, val) 70 | } 71 | } else { 72 | log.Panic("ERROR: unable to parse data file", err) 73 | } 74 | } 75 | return result 76 | } 77 | 78 | func readDocs(fn string, v int) [][]int { 79 | f, err := os.Open(fn) 80 | if err != nil { 81 | log.Fatal(err) 82 | } 83 | defer f.Close() 84 | 85 | docs := make([][]int, 0, 10) 86 | 87 | sc := bufio.NewScanner(f) 88 | for sc.Scan() { 89 | docs = append(docs, parseAndFilterInts(sc.Text(), v)) 90 | } 91 | 92 | return docs 93 | } 94 | 95 | func sample(p []float64) int { 96 | x := rand.Float64() 97 | cda := 0.0 98 | for i, pi := range p { 99 | cda += pi 100 | if x < cda { 101 | return i 102 | } 103 | } 104 | return len(p) - 1 105 | } 106 | 107 | func sampleGibbsForward(model *lda, doc []int, z []int, zn []int) { 108 | n := len(doc) 109 | k := len(zn) 110 | p := make([]float64, k) 111 | for i := 0; i < n; i++ { 112 | zn[z[i]]-- 113 | for j := 0; j < k; j++ { 114 | p[j] = model.phi[j][doc[i]] * (float64(zn[j]) + model.alpha) 115 | } 116 | floats.Scale(1/floats.Sum(p), p) 117 | newZ := sample(p) 118 | zn[newZ]++ 119 | z[i] = newZ 120 | } 121 | } 122 | 123 | func sampleGibbsBackward(model *lda, doc []int, z []int, zn []int) { 124 | n := len(doc) 125 | k := len(zn) 126 | p := make([]float64, k) 127 | for i := n - 1; i > -1; i-- { 128 | zn[z[i]]-- 129 | for j := 0; j < k; j++ { 130 | p[j] = model.phi[j][doc[i]] * (float64(zn[j]) + model.alpha) 131 | } 132 | floats.Scale(1/floats.Sum(p), p) 133 | newZ := sample(p) 134 | zn[newZ]++ 135 | z[i] = newZ 136 | } 137 | } 138 | 139 | func evalPwz(model *lda, doc []int, z []int) float64 { 140 | val := 0.0 141 | for i, w := range doc { 142 | val += math.Log2(model.phi[z[i]][w]) 143 | } 144 | return val 145 | } 146 | 147 | func lgamma(x float64) float64 { 148 | y, _ := math.Lgamma(x) 149 | return y 150 | } 151 | 152 | func evalPz(model *lda, doc []int, zn []int) float64 { 153 | k := float64(len(zn)) 154 | val := lgamma(model.alpha*k) - lgamma(model.alpha*k+float64(len(doc))) 155 | for _, zni := range zn { 156 | val += lgamma(float64(zni)+model.alpha) - lgamma(model.alpha) 157 | } 158 | return val / math.Log(2.0) 159 | } 160 | 161 | func evalTz(model *lda, doc []int, za, zb []int, zn []int) float64 { 162 | k := len(zn) 163 | n := len(doc) 164 | curZn := make([]int, k) 165 | copy(curZn, zn) 166 | 167 | val := 0.0 168 | p := make([]float64, k) 169 | for i := n - 1; i > -1; i-- { 170 | curZn[za[i]]-- 171 | for j := 0; j < k; j++ { 172 | p[j] = model.phi[j][doc[i]] * (float64(curZn[j]) + model.alpha) 173 | } 174 | floats.Scale(1/floats.Sum(p), p) 175 | val += math.Log(p[zb[i]]) 176 | curZn[zb[i]]++ 177 | } 178 | return val 179 | } 180 | 181 | func evalOne(model *lda, doc []int, numSamples int) float64 { 182 | burnIn := 1000 183 | k := len(model.phi) 184 | n := len(doc) 185 | z := make([]int, n) 186 | zn := make([]int, k) 187 | 188 | for i := 0; i < n; i++ { 189 | z[i] = rand.Intn(k) 190 | zn[z[i]]++ 191 | } 192 | 193 | // burn-in first to get a good sample 194 | for a := 0; a < burnIn; a++ { 195 | sampleGibbsForward(model, doc, z, zn) 196 | } 197 | 198 | samples := make([][]int, 0, numSamples) 199 | znIndex := make([][]int, 0, numSamples) 200 | for i := 0; i < numSamples; i++ { 201 | samples = append(samples, make([]int, n)) 202 | znIndex = append(znIndex, make([]int, k)) 203 | } 204 | x := rand.Intn(numSamples) 205 | copy(samples[x], z) 206 | copy(znIndex[x], zn) 207 | sampleGibbsBackward(model, doc, samples[x], znIndex[x]) 208 | for i := x + 1; i < numSamples; i++ { 209 | copy(samples[i], samples[i-1]) 210 | copy(znIndex[i], znIndex[i-1]) 211 | sampleGibbsForward(model, doc, samples[i], znIndex[i]) 212 | } 213 | 214 | for i := x - 1; i > -1; i-- { 215 | copy(samples[i], samples[i+1]) 216 | copy(znIndex[i], znIndex[i+1]) 217 | sampleGibbsBackward(model, doc, samples[i], znIndex[i]) 218 | } 219 | 220 | tzs := make([]float64, 0, numSamples) 221 | for _, sample := range samples { 222 | tzs = append(tzs, evalTz(model, doc, z, sample, zn)) 223 | } 224 | 225 | return evalPwz(model, doc, z) + evalPz(model, doc, zn) + math.Log2(float64(numSamples)) - floats.LogSumExp(tzs)/math.Log(2.0) 226 | } 227 | 228 | func eval(model *lda, docs [][]int, numSamples int) float64 { 229 | sum := 0.0 230 | for i, doc := range docs { 231 | eval := evalOne(model, doc, numSamples) 232 | log.Printf("chibeval(docs[%d]) = %f\n", i, eval) 233 | sum += eval 234 | } 235 | return sum 236 | } 237 | 238 | func smooth(phis [][]float64, smoothing float64) { 239 | for _, phi := range phis { 240 | floats.AddConst(smoothing, phi) 241 | floats.Scale(1.0/floats.Sum(phi), phi) 242 | } 243 | } 244 | 245 | func main() { 246 | var seed int64 247 | var numSamples int 248 | var phiFn string 249 | var alpha float64 250 | var dataFn string 251 | var isLog bool 252 | var smoothing float64 253 | 254 | flag.Int64Var(&seed, "seed", 1, "random seed") 255 | flag.Float64Var(&alpha, "alpha", 1.0, "alpha") 256 | flag.StringVar(&phiFn, "phi", "", "phi definition") 257 | flag.StringVar(&dataFn, "data", "", "held-out data file") 258 | flag.IntVar(&numSamples, "samples", 100, "num of samples") 259 | flag.BoolVar(&isLog, "log", false, "exp transformation required") 260 | flag.Float64Var(&smoothing, "smoothing", 0, "smoothing param") 261 | 262 | flag.Parse() 263 | 264 | rand.Seed(seed) 265 | 266 | phis := readPhi(phiFn, isLog) 267 | smooth(phis, smoothing) 268 | model := &lda{alpha, phis} 269 | docs := readDocs(dataFn, len(model.phi[0])) 270 | eval := eval(model, docs, numSamples) 271 | log.Printf("chibeval(docs[0:%d]) = %.2f\n", len(docs), eval) 272 | fmt.Printf("%.2f\n", eval) 273 | } 274 | -------------------------------------------------------------------------------- /cmd/fit/rldafit.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "log" 6 | "math/rand" 7 | "os" 8 | 9 | "bitbucket.org/sitfoxfly/ranklda/lda" 10 | "bitbucket.org/sitfoxfly/ranklda/model" 11 | ) 12 | 13 | func ensureDir(dir string) { 14 | if _, err := os.Stat(dir); os.IsNotExist(err) { 15 | os.MkdirAll(dir, os.ModePerm) 16 | } 17 | } 18 | 19 | func ensureFile(fn string) { 20 | if fn != "" { 21 | if f, err := os.Create(fn); err == nil { 22 | f.Close() 23 | } else { 24 | log.Panic("ERROR: unable to create model file", fn) 25 | } 26 | } 27 | } 28 | 29 | func ensureCondition(condition bool) { 30 | if !condition { 31 | flag.PrintDefaults() 32 | os.Exit(1) 33 | } 34 | } 35 | 36 | func main() { 37 | init := &model.InitSet{} 38 | settings := &model.OptSettings{} 39 | flag.IntVar(&init.K, "k", 5, "number of topics") 40 | flag.Float64Var(&init.Sigma, "g", 1.0, "Gaussian regularization") 41 | flag.Int64Var(&init.Seed, "s", 1, "random seed") 42 | flag.Float64Var(&init.Alpha, "a", 1e-6, "phi pseudocounts") 43 | flag.Float64Var(&init.Beta, "b", 0.1, "symmetrical beta prior") 44 | 45 | flag.BoolVar(&settings.BetaOpt, "o", false, "optimize betas") 46 | flag.IntVar(&settings.NumIter, "i", 15, "number of iterations") 47 | flag.IntVar(&settings.BurnInIter, "bi", 0, "burn-in iterations") 48 | flag.Float64Var(&settings.InitT, "t", 1.0, "initial temperature") 49 | flag.IntVar(&settings.NumSAIter, "ti", 50, "number of iterations for SA optimization") 50 | flag.Float64Var(&settings.GlobalCRate, "tg", 1, "global cooling rate") 51 | flag.Float64Var(&settings.LocalCRate, "tl", 1, "local cooling rate") 52 | flag.Float64Var(&settings.ComparisonDropRate, "dr", 0.0, "comparison drop rate") 53 | 54 | var seedfn string 55 | var datafn string 56 | var modeldir string 57 | var modelfn string 58 | var ldaOutFn string 59 | 60 | flag.StringVar(&seedfn, "assign", "", "Zs seed initializer") 61 | flag.StringVar(&datafn, "data", "", "data file") 62 | flag.StringVar(&modeldir, "model-dir", "", "model directory") 63 | flag.StringVar(&modelfn, "model", "", "final model") 64 | flag.StringVar(&ldaOutFn, "lda-output", "", "vanilla LDA model") 65 | flag.Parse() 66 | 67 | ensureCondition(datafn != "") 68 | ensureCondition(modelfn != "") 69 | 70 | rand.Seed(init.Seed) 71 | 72 | if modelfn != "" { 73 | ensureFile(modelfn) 74 | } 75 | 76 | if modeldir != "" { 77 | ensureDir(modeldir) 78 | } 79 | 80 | data := model.ReadData(datafn) 81 | 82 | var m *model.Model 83 | if seedfn == "" { 84 | m = model.RandomModel(data, init) 85 | } else { 86 | assignments := lda.ReadLDA(seedfn, data.N) 87 | m = model.AssignedModel(data, init, assignments) 88 | } 89 | 90 | m.Optimize(settings, modeldir) 91 | 92 | if modelfn != "" { 93 | m.Save(modelfn) 94 | } 95 | 96 | if ldaOutFn != "" { 97 | m.SaveLDA(ldaOutFn) 98 | } 99 | 100 | } 101 | -------------------------------------------------------------------------------- /cmd/inf/rldainf.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "math/rand" 7 | 8 | "os" 9 | 10 | "bitbucket.org/sitfoxfly/ranklda/model" 11 | ) 12 | 13 | func save(fn string, z [][]int, scores []float64, perplexity float64) { 14 | f, err := os.Create(fn) 15 | if err != nil { 16 | fmt.Println("Cannot save file!") 17 | os.Exit(1) 18 | } 19 | defer f.Close() 20 | 21 | fmt.Fprintf(f, "%d\n", len(z)) 22 | for _, zi := range z { 23 | for j, zij := range zi { 24 | if j == 0 { 25 | fmt.Fprintf(f, "%d", zij) 26 | } else { 27 | fmt.Fprintf(f, " %d", zij) 28 | } 29 | } 30 | fmt.Fprintf(f, "\n") 31 | } 32 | 33 | for i, score := range scores { 34 | if i == 0 { 35 | fmt.Fprintf(f, "%f", score) 36 | } else { 37 | fmt.Fprintf(f, " %f", score) 38 | } 39 | } 40 | fmt.Fprintf(f, "\n%f\n", perplexity) 41 | } 42 | 43 | func main() { 44 | var seed int64 45 | settings := &model.InferSettings{} 46 | 47 | flag.Int64Var(&seed, "s", 1, "random seed") 48 | 49 | flag.Float64Var(&settings.InitT, "t", 1.0, "initial temperature") 50 | flag.IntVar(&settings.NumSAIter, "ti", 1000, "number of iterations for SA optimization") 51 | flag.Float64Var(&settings.CoolingRate, "tg", 1.0, "global cooling rate") 52 | var modelDataFn string 53 | flag.StringVar(&modelDataFn, "data", "", "model data file") 54 | flag.Parse() 55 | 56 | if flag.NArg() != 3 { 57 | fmt.Println("USAGE: rldainf ") 58 | flag.PrintDefaults() 59 | os.Exit(1) 60 | } 61 | 62 | modelfn := flag.Arg(0) 63 | datafn := flag.Arg(1) 64 | outfn := flag.Arg(2) 65 | 66 | rand.Seed(seed) 67 | 68 | var m *model.Model 69 | if modelDataFn == "" { 70 | m = model.ReadModel(modelfn) 71 | } else { 72 | m = model.ReadModelWithData(modelfn, modelDataFn) 73 | } 74 | data := model.Reduce(model.ReadData(datafn), m) 75 | 76 | z := m.Infer(data, settings) 77 | scores := m.Score(z) 78 | perplexity := m.Perplexity2(data.W) 79 | 80 | save(outfn, z, scores, perplexity) 81 | } 82 | -------------------------------------------------------------------------------- /ints/ints.go: -------------------------------------------------------------------------------- 1 | package ints 2 | 3 | import ( 4 | "github.com/gonum/floats" 5 | ) 6 | 7 | // Pair is a pair of int elements (x, y) 8 | type Pair struct { 9 | X int 10 | Y int 11 | } 12 | 13 | // Sum returns sum of ints 14 | func Sum(x []int) (sum int) { 15 | if x == nil { 16 | panic("invalid argument to Sum") 17 | } 18 | for _, v := range x { 19 | sum += v 20 | } 21 | return 22 | } 23 | 24 | // Max1D returns the maximum int in the array 25 | func Max1D(x []int) (max int) { 26 | if x == nil || len(x) == 0 { 27 | panic("invalid argument to Max") 28 | } 29 | max = x[0] 30 | for _, v := range x[1:] { 31 | if max < v { 32 | max = v 33 | } 34 | } 35 | return 36 | } 37 | 38 | // Max2D returns the maximum int in the 2D array 39 | func Max2D(x [][]int) (max int) { 40 | if x == nil || len(x) == 0 { 41 | panic("invalid argument to MaxMat") 42 | } 43 | max = Max1D(x[0]) 44 | for _, row := range x[1:] { 45 | v := Max1D(row) 46 | if max < v { 47 | max = v 48 | } 49 | } 50 | return 51 | } 52 | 53 | // Count creates the count slice of elemtns in src 54 | func Count(src []int, n int) []int { 55 | if src == nil { 56 | panic("invalid argument to Count") 57 | } 58 | dst := make([]int, n) 59 | for _, x := range src { 60 | dst[x]++ 61 | } 62 | return dst 63 | } 64 | 65 | // Dist converts counts to the density values 66 | func Dist(src []int) []float64 { 67 | if src == nil { 68 | panic("invalid argument to Dist") 69 | } 70 | res := make([]float64, 0, len(src)) 71 | sum := 0.0 72 | for _, x := range src { 73 | fx := float64(x) 74 | res = append(res, fx) 75 | sum += fx 76 | } 77 | floats.Scale(1.0/sum, res) 78 | return res 79 | } 80 | -------------------------------------------------------------------------------- /lda/lda.go: -------------------------------------------------------------------------------- 1 | package lda 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "log" 7 | "os" 8 | "strconv" 9 | "strings" 10 | 11 | "bitbucket.org/sitfoxfly/ranklda/ints" 12 | ) 13 | 14 | func parseLine(s string) []ints.Pair { 15 | sc := bufio.NewScanner(strings.NewReader(s)) 16 | sc.Split(bufio.ScanWords) 17 | sc.Scan() 18 | if n, err := strconv.Atoi(sc.Text()); err == nil { 19 | result := make([]ints.Pair, n) 20 | for i := 0; i < n; i++ { 21 | sc.Scan() 22 | p := ints.Pair{} 23 | fmt.Sscanf(sc.Text(), "%d:%d", &p.X, &p.Y) 24 | result[i] = p 25 | } 26 | return result 27 | } 28 | log.Fatal("ERROR: unable to parse LDA data line") 29 | return nil 30 | } 31 | 32 | func ReadLDA(fn string, n int) [][]ints.Pair { 33 | f, err := os.Open(fn) 34 | if err != nil { 35 | log.Fatal("ERROR: unable to read LDA model") 36 | } 37 | defer f.Close() 38 | result := make([][]ints.Pair, n) 39 | scanner := bufio.NewScanner(f) 40 | for i := 0; i < n; i++ { 41 | scanner.Scan() 42 | result[i] = parseLine(scanner.Text()) 43 | } 44 | return result 45 | } 46 | -------------------------------------------------------------------------------- /model/data.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | 8 | "bufio" 9 | 10 | "strings" 11 | 12 | "strconv" 13 | 14 | "bitbucket.org/sitfoxfly/ranklda/ints" 15 | ) 16 | 17 | type Data struct { 18 | W [][]int 19 | C []ints.Pair 20 | V int 21 | N int 22 | M int 23 | } 24 | 25 | func parseInts(s string) []int { 26 | sc := bufio.NewScanner(strings.NewReader(s)) 27 | sc.Split(bufio.ScanWords) 28 | result := make([]int, 0, 10) 29 | for sc.Scan() { 30 | if val, err := strconv.Atoi(sc.Text()); err == nil { 31 | result = append(result, val) 32 | } else { 33 | log.Fatal("Unable to parse data file", err) 34 | } 35 | } 36 | return result 37 | } 38 | 39 | func parseLine(s string) []ints.Pair { 40 | sc := bufio.NewScanner(strings.NewReader(s)) 41 | sc.Split(bufio.ScanWords) 42 | var result []ints.Pair 43 | for sc.Scan() { 44 | p := ints.Pair{} 45 | fmt.Sscanf(sc.Text(), "%d:%d", &p.X, &p.Y) 46 | result = append(result, p) 47 | } 48 | if sc.Err() != nil { 49 | log.Fatal("Error", sc.Err()) 50 | } 51 | return result 52 | } 53 | 54 | // ReadData reads the data file 55 | func ReadData(fn string) *Data { 56 | f, err := os.Open(fn) 57 | if err != nil { 58 | log.Fatal("Unable to open data file", err) 59 | } 60 | defer f.Close() 61 | sc := bufio.NewScanner(bufio.NewReader(f)) 62 | buf := make([]byte, 64*1024) 63 | sc.Buffer(buf, 1024*1024) 64 | sc.Scan() 65 | var n, m int 66 | fmt.Sscanf(sc.Text(), "%d %d", &n, &m) 67 | 68 | // reading documents 69 | 70 | docs := make([][]int, 0, n) 71 | vocabSize := 0 72 | for i := 0; i < n; i++ { 73 | sc.Scan() 74 | doc := parseLine(sc.Text()) 75 | len := 0 76 | for _, cn := range doc { 77 | if vocabSize < cn.X { 78 | vocabSize = cn.X 79 | } 80 | len += cn.Y 81 | } 82 | w := make([]int, len) 83 | h := 0 84 | for _, cn := range doc { 85 | for j := 0; j < cn.Y; j++ { 86 | w[h] = cn.X 87 | h++ 88 | } 89 | } 90 | docs = append(docs, w) 91 | } 92 | vocabSize++ 93 | 94 | // reading comparisons 95 | 96 | comparisons := make([]ints.Pair, m) 97 | for i := 0; i < m; i++ { 98 | sc.Scan() 99 | fmt.Sscanf(sc.Text(), "%d %d", &comparisons[i].X, &comparisons[i].Y) 100 | } 101 | 102 | // assignment 103 | 104 | return &Data{docs, comparisons, vocabSize, len(docs), len(comparisons)} 105 | } 106 | 107 | func Reduce(data *Data, model *Model) *Data { 108 | data.V = model.data.V 109 | filtered := make([][]int, data.N) 110 | for i, ws := range data.W { 111 | filtered[i] = make([]int, 0, len(ws)) 112 | for _, w := range ws { 113 | if w < model.data.V { 114 | filtered[i] = append(filtered[i], w) 115 | } 116 | } 117 | } 118 | data.W = filtered 119 | return data 120 | } 121 | -------------------------------------------------------------------------------- /model/model.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "log" 7 | "math" 8 | "math/rand" 9 | "os" 10 | "path" 11 | "strconv" 12 | "strings" 13 | 14 | "bitbucket.org/sitfoxfly/ranklda/ints" 15 | "bitbucket.org/sitfoxfly/ranklda/umath" 16 | "github.com/gonum/floats" 17 | ) 18 | 19 | // Model is a structure to represet RankLDA model 20 | type Model struct { 21 | data *Data 22 | k int 23 | alpha float64 24 | beta []float64 25 | logPhi [][]float64 26 | z [][]int 27 | nu []float64 28 | sigma float64 29 | } 30 | 31 | // InferSettings - inference settings 32 | type InferSettings struct { 33 | NumSAIter int 34 | InitT float64 35 | CoolingRate float64 36 | } 37 | 38 | // OptSettings - optimization settings 39 | type OptSettings struct { 40 | NumIter int 41 | NumSAIter int 42 | BetaOpt bool 43 | BurnInIter int 44 | InitT float64 45 | LocalCRate float64 46 | GlobalCRate float64 47 | ComparisonDropRate float64 48 | } 49 | 50 | // InitSet initialized for the random model 51 | type InitSet struct { 52 | Seed int64 53 | K int 54 | Sigma float64 55 | Alpha float64 56 | Beta float64 57 | } 58 | 59 | func ReadModelWithData(fn1, fn2 string) *Model { 60 | m := ReadModel(fn1) 61 | m.data = ReadData(fn2) 62 | return m 63 | } 64 | 65 | func lreadInts(s string) []int { 66 | sc := bufio.NewScanner(strings.NewReader(s)) 67 | sc.Split(bufio.ScanWords) 68 | result := make([]int, 0, 10) 69 | for sc.Scan() { 70 | if w, err := strconv.Atoi(sc.Text()); err == nil { 71 | result = append(result, w) 72 | } else { 73 | panic(err) 74 | } 75 | } 76 | return result 77 | } 78 | 79 | // ReadModel reads RankLDA model from the file 80 | func ReadModel(fn string) *Model { 81 | f, err := os.Open(fn) 82 | if err != nil { 83 | log.Fatal("ERROR: unable to open file", err) 84 | } 85 | defer f.Close() 86 | scanner := bufio.NewScanner(f) 87 | buf := make([]byte, 0, 64*1024) 88 | scanner.Buffer(buf, 1024*1024) 89 | var vocabSize int 90 | model := &Model{} 91 | scanner.Scan() 92 | fmt.Sscanf(scanner.Text(), "%d %d", &model.k, &vocabSize) 93 | model.beta = make([]float64, model.k) 94 | { 95 | scanner.Scan() 96 | lineReader := strings.NewReader(scanner.Text()) 97 | for i := 0; i < model.k; i++ { 98 | fmt.Fscanf(lineReader, "%f", &model.beta[i]) 99 | } 100 | } 101 | scanner.Scan() 102 | fmt.Sscanf(scanner.Text(), "%f", &model.alpha) 103 | model.logPhi = make([][]float64, model.k) 104 | for i := 0; i < model.k; i++ { 105 | scanner.Scan() 106 | lineReader := strings.NewReader(scanner.Text()) 107 | model.logPhi[i] = make([]float64, vocabSize) 108 | for j := 0; j < vocabSize; j++ { 109 | fmt.Fscanf(lineReader, "%f", &model.logPhi[i][j]) 110 | } 111 | } 112 | { 113 | scanner.Scan() 114 | lineReader := strings.NewReader(scanner.Text()) 115 | model.nu = make([]float64, model.k) 116 | for i := 0; i < model.k; i++ { 117 | fmt.Fscanf(lineReader, "%f", &model.nu[i]) 118 | } 119 | } 120 | var n int 121 | scanner.Scan() 122 | fmt.Sscanf(scanner.Text(), "%d", &n) 123 | model.z = make([][]int, n) 124 | for i := 0; i < n; i++ { 125 | scanner.Scan() 126 | model.z[i] = lreadInts(scanner.Text()) 127 | } 128 | return model 129 | } 130 | 131 | // Infer infers topic assigment for the unseen data 132 | func (model *Model) Infer(data *Data, s *InferSettings) [][]int { 133 | n := len(data.W) 134 | z := make([][]int, 0, n) 135 | trainee := model.trainable() 136 | for _, doc := range data.W { 137 | zi := trainee.InferDoc(doc, s) 138 | z = append(z, zi) 139 | } 140 | return z 141 | } 142 | 143 | func (model *Model) scoreDoc(w []int, z []int) float64 { 144 | n := len(w) 145 | zInd := ints.Count(z, model.k) 146 | betaSum := floats.Sum(model.beta) 147 | res := 0.0 148 | 149 | res += umath.Lgamma(betaSum) - umath.Lgamma(float64(n)+betaSum) 150 | for i := 0; i < model.k; i++ { 151 | res += umath.Lgamma(model.beta[i]+float64(zInd[i])) - umath.Lgamma(model.beta[i]) 152 | } 153 | 154 | for i := 0; i < n; i++ { 155 | res += model.logPhi[z[i]][w[i]] 156 | } 157 | 158 | return res 159 | } 160 | 161 | func (model *Model) scoreDoc2(w []int, z []int) float64 { 162 | n := len(w) 163 | //zInd := ints.Count(z, mod.k) 164 | //betaSum := floats.Sum(mod.bs) 165 | res := 0.0 166 | 167 | //res += umath.Lgamma(betaSum) - umath.Lgamma(float64(n)+betaSum) 168 | //for i := 0; i < mod.k; i++ { 169 | // res += umath.Lgamma(mod.bs[i]+float64(zInd[i])) - umath.Lgamma(mod.bs[i]) 170 | //} 171 | 172 | for i := 0; i < n; i++ { 173 | res += model.logPhi[z[i]][w[i]] 174 | } 175 | 176 | return res 177 | } 178 | 179 | func (model *Model) inferDoc(doc []int, s *InferSettings) []int { 180 | n := len(doc) 181 | z := make([]int, n) 182 | for i := 0; i < n; i++ { 183 | z[i] = rand.Intn(model.k) 184 | } 185 | zInd := ints.Count(z, model.k) 186 | T := s.InitT 187 | //fmt.Printf("OBJ = %g\n", mod.scoreDoc(doc, z)) 188 | for iter := 0; iter < s.NumSAIter; iter++ { 189 | for i := 0; i < n; i++ { 190 | curZ := z[i] 191 | newZ := rand.Intn(model.k) 192 | w := doc[i] 193 | diff := math.Log(model.beta[curZ]+float64(zInd[curZ]-1)) - math.Log(model.beta[newZ]+float64(zInd[newZ])) + model.logPhi[curZ][w] - model.logPhi[newZ][w] 194 | prob := rand.Float64() 195 | if diff <= 0.0 || prob < math.Exp(-diff/T) { 196 | z[i] = newZ 197 | zInd[curZ]-- 198 | zInd[newZ]++ 199 | } 200 | } 201 | T *= s.CoolingRate 202 | } 203 | //fmt.Printf("OBJ = %g\n", mod.scoreDoc(doc, z)) 204 | //fmt.Println("=== DONE ===") 205 | return z 206 | } 207 | 208 | // Perplexity computes point estimate perplexity on a set of documents 209 | func (model *Model) Perplexity(docs [][]int, z [][]int) float64 { 210 | logProb := 0.0 211 | normalizer := 0 212 | trainee := model.trainable() 213 | trainee.optimizePhi() 214 | for i, doc := range docs { 215 | logProb += trainee.scoreDoc(doc, z[i]) 216 | normalizer += len(doc) 217 | } 218 | return logProb / float64(normalizer) 219 | } 220 | 221 | func (model *Model) Perplexity2(docs [][]int) float64 { 222 | s := 10 223 | logProb := 0.0 224 | normalizer := 0 225 | trainee := model.trainable() 226 | trainee.optimizePhi() 227 | for _, doc := range docs { 228 | z := make([]int, len(doc)) 229 | cumLogProb := 0.0 230 | for j := 0; j < s; j++ { 231 | for k := 0; k < len(doc); k++ { 232 | z[k] = rand.Intn(model.k) 233 | } 234 | cumLogProb += trainee.scoreDoc2(doc, z) 235 | } 236 | logProb += cumLogProb / float64(s) 237 | normalizer += len(doc) 238 | } 239 | return logProb / float64(normalizer) 240 | } 241 | 242 | // Score computes the doc scores and builds pairwise comparison list 243 | func (model *Model) Score(z [][]int) []float64 { 244 | scores := make([]float64, 0) 245 | for _, zi := range z { 246 | //fmt.Println(mod.nu, ints.Dist(ints.Count(zi, mod.k)), floats.Dot(mod.nu, ints.Dist(ints.Count(zi, mod.k)))) 247 | scores = append(scores, floats.Dot(model.nu, ints.Dist(ints.Count(zi, model.k)))) 248 | } 249 | return scores 250 | } 251 | 252 | // RandomModel builds a randomly initialized model for the data supplied 253 | func RandomModel(data *Data, init *InitSet) *Model { 254 | model := &Model{} 255 | model.k = init.K 256 | model.sigma = init.Sigma 257 | model.alpha = init.Alpha 258 | model.data = data 259 | model.z = make([][]int, len(data.W)) 260 | for i, doc := range data.W { 261 | n := len(doc) 262 | model.z[i] = make([]int, n) 263 | for j := 0; j < n; j++ { 264 | model.z[i][j] = rand.Intn(model.k) 265 | } 266 | } 267 | 268 | model.beta = make([]float64, model.k) 269 | model.nu = make([]float64, model.k) 270 | model.logPhi = make([][]float64, model.k) 271 | for i := 0; i < model.k; i++ { 272 | model.beta[i] = init.Beta 273 | model.nu[i] = rand.NormFloat64() * model.sigma 274 | model.logPhi[i] = make([]float64, data.V) 275 | for j := 0; j < data.V; j++ { 276 | model.logPhi[i][j] = -math.Log(float64(data.V)) 277 | } 278 | } 279 | return model 280 | } 281 | 282 | // AssignedModel builds a model with initialized Zs 283 | func AssignedModel(data *Data, init *InitSet, assignments [][]ints.Pair) *Model { 284 | model := &Model{} 285 | model.k = init.K 286 | model.sigma = init.Sigma 287 | model.alpha = init.Alpha 288 | model.data = data 289 | model.z = make([][]int, len(data.W)) 290 | for i, v := range data.W { 291 | docAssign := assignments[i] 292 | assignMap := make(map[int]int) 293 | for j := 0; j < len(docAssign); j++ { 294 | assignMap[docAssign[j].X] = docAssign[j].Y 295 | } 296 | n := len(v) 297 | model.z[i] = make([]int, n) 298 | for j := 0; j < n; j++ { 299 | model.z[i][j] = assignMap[v[j]] 300 | } 301 | } 302 | 303 | model.beta = make([]float64, model.k) 304 | model.nu = make([]float64, model.k) 305 | model.logPhi = make([][]float64, model.k) 306 | for i := 0; i < model.k; i++ { 307 | model.beta[i] = init.Beta 308 | model.nu[i] = rand.NormFloat64() * model.sigma 309 | model.logPhi[i] = make([]float64, data.V) 310 | for j := 0; j < data.V; j++ { 311 | model.logPhi[i][j] = -math.Log(float64(data.V)) 312 | } 313 | } 314 | return model 315 | } 316 | 317 | // Optimize optimizes the RankLDA model with Variational Approximation Algorithm 318 | func (model *Model) Optimize(s *OptSettings, dir string) { 319 | 320 | var lhLog *os.File 321 | if dir != "" { 322 | if lhLog, err := os.Open(path.Join(dir, "likelihood.txt")); err != nil { 323 | defer lhLog.Close() 324 | } 325 | } 326 | 327 | if s.BurnInIter > 0 { 328 | plainTrainable := model.plainTrainable() 329 | log.Printf("likelihood(topics) = %f\n", plainTrainable.logLikelihoodOfTopics()) 330 | for i := 0; i < s.BurnInIter; i++ { 331 | plainTrainable.optimizeZ(s.NumSAIter, 1.0, s.LocalCRate, s.ComparisonDropRate) 332 | plainTrainable.optimizePhi() 333 | log.Printf("likelihood(topics) = %f\n", plainTrainable.logLikelihoodOfTopics()) 334 | } 335 | } 336 | 337 | trainable := model.trainable() 338 | T := s.InitT 339 | for i := 0; i < s.NumIter; i++ { 340 | log.Printf("starting new iteration: %d (T = %g)\n", i, T) 341 | trainable.optimizeNu() 342 | trainable.optimizeZ(s.NumSAIter, T, s.LocalCRate, s.ComparisonDropRate) 343 | trainable.optimizePhi() 344 | if s.BetaOpt { 345 | trainable.optimizeBeta() 346 | } 347 | likelihood := trainable.logLikelihood() 348 | log.Printf("likelihood = %f\n", likelihood) 349 | if lhLog != nil { 350 | lhLog.WriteString(fmt.Sprintln(likelihood)) 351 | } 352 | if dir != "" { 353 | trainable.Save(path.Join(dir, fmt.Sprintf("%02d-model.txt", i))) 354 | } 355 | T *= s.GlobalCRate 356 | } 357 | trainable.optimizeNu() 358 | } 359 | 360 | func (model *Model) plainTrainable() *trainableModel { 361 | nIndex := make([][]int, model.data.N) 362 | cIndex := make([][]int, model.k) 363 | for i := 0; i < model.k; i++ { 364 | cIndex[i] = make([]int, model.data.V) 365 | } 366 | zIndex := make([]int, model.k) 367 | for i, row := range model.z { 368 | nIndex[i] = make([]int, model.k) 369 | for j, z := range row { 370 | w := model.data.W[i][j] 371 | nIndex[i][z]++ 372 | cIndex[z][w]++ 373 | zIndex[z]++ 374 | } 375 | } 376 | 377 | comparisonIndex := make([][]*coI, model.data.N) 378 | for i := 0; i < model.data.N; i++ { 379 | comparisonIndex[i] = make([]*coI, 0) 380 | } 381 | 382 | return &trainableModel{model, nIndex, cIndex, zIndex, comparisonIndex} 383 | } 384 | 385 | func (model *Model) trainable() *trainableModel { 386 | nIndex := make([][]int, model.data.N) 387 | cIndex := make([][]int, model.k) 388 | for i := 0; i < model.k; i++ { 389 | cIndex[i] = make([]int, model.data.V) 390 | } 391 | zIndex := make([]int, model.k) 392 | for i, row := range model.z { 393 | nIndex[i] = make([]int, model.k) 394 | for j, z := range row { 395 | w := model.data.W[i][j] 396 | nIndex[i][z]++ 397 | cIndex[z][w]++ 398 | zIndex[z]++ 399 | } 400 | } 401 | 402 | comparisonIndex := make([][]*coI, model.data.N) 403 | for i := 0; i < model.data.N; i++ { 404 | comparisonIndex[i] = make([]*coI, 0, 10) 405 | } 406 | for _, comp := range model.data.C { 407 | xLength := len(model.data.W[comp.X]) 408 | yLength := len(model.data.W[comp.Y]) 409 | ref := &coI{comp.X, float64(xLength), -float64(yLength), umath.Anxmany(model.nu, nIndex[comp.X], nIndex[comp.Y], xLength, yLength)} 410 | comparisonIndex[comp.X] = append(comparisonIndex[comp.X], ref) 411 | comparisonIndex[comp.Y] = append(comparisonIndex[comp.Y], ref) 412 | } 413 | 414 | return &trainableModel{model, nIndex, cIndex, zIndex, comparisonIndex} 415 | } 416 | 417 | func (model *Model) Save(fn string) { 418 | f, err := os.Create(fn) 419 | if err != nil { 420 | log.Fatal("ERROR: unable to save model file:", fn) 421 | } 422 | defer f.Close() 423 | 424 | fmt.Fprintf(f, "%d %d\n", model.k, model.data.V) 425 | for _, b := range model.beta { 426 | fmt.Fprintf(f, "%f ", b) 427 | } 428 | fmt.Fprintln(f) 429 | fmt.Fprintf(f, "%f\n", model.alpha) 430 | for _, row := range model.logPhi { 431 | for _, logPhi := range row { 432 | fmt.Fprintf(f, "%f ", logPhi) 433 | } 434 | fmt.Fprintln(f) 435 | } 436 | for _, w := range model.nu { 437 | fmt.Fprintf(f, "%f ", w) 438 | } 439 | fmt.Fprintln(f) 440 | fmt.Fprintf(f, "%d\n", len(model.z)) 441 | for _, row := range model.z { 442 | for _, z := range row { 443 | fmt.Fprintf(f, "%d ", z) 444 | } 445 | fmt.Fprintln(f) 446 | } 447 | } 448 | 449 | func (model *Model) SaveLDA(fn string) { 450 | f, err := os.Create(fn) 451 | if err != nil { 452 | log.Fatal("ERROR: unbale to save LDA model", err) 453 | } 454 | defer f.Close() 455 | 456 | for i := 0; i < model.k; i++ { 457 | for j := 0; j < model.data.V; j++ { 458 | fmt.Fprintf(f, "%.10f ", math.Exp(model.logPhi[i][j])) 459 | } 460 | fmt.Fprintln(f) 461 | } 462 | } 463 | -------------------------------------------------------------------------------- /model/trainee.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "log" 5 | "math" 6 | "math/rand" 7 | 8 | "bitbucket.org/sitfoxfly/ranklda/ints" 9 | "bitbucket.org/sitfoxfly/ranklda/umath" 10 | "github.com/gonum/floats" 11 | "github.com/gonum/matrix/mat64" 12 | "github.com/gonum/optimize" 13 | ) 14 | 15 | type coI struct { 16 | X int 17 | etaX float64 18 | etaY float64 19 | eval float64 20 | } 21 | 22 | type trainableModel struct { 23 | *Model 24 | nIndex [][]int 25 | cIndex [][]int 26 | zIndex []int 27 | comparisonIndex [][]*coI 28 | } 29 | 30 | func (model *trainableModel) logLikelihoodOfTopics() float64 { 31 | k := model.k 32 | betaSum := floats.Sum(model.beta) 33 | alphaSum := model.alpha * float64(model.data.V) 34 | result := 0.0 35 | for i := 0; i < model.data.N; i++ { 36 | n := len(model.data.W[i]) 37 | result += umath.Lgamma(betaSum) - umath.Lgamma(float64(n)+betaSum) 38 | for j := 0; j < k; j++ { 39 | result += umath.Lgamma(model.beta[j]+float64(model.nIndex[i][j])) - umath.Lgamma(model.beta[j]) 40 | } 41 | 42 | //for j := 0; j < m; j++ { 43 | // res += mod.logPhis[mod.zs[i][j]][mod.data.Docs[i][j]] 44 | //} 45 | } 46 | 47 | result -= float64(model.k) * (math.Log(float64(model.data.V)) + umath.Lgamma(model.alpha) - umath.Lgamma(alphaSum)) 48 | for k := 0; k < model.k; k++ { 49 | result -= umath.Lgamma(float64(model.zIndex[k]) + alphaSum) 50 | for w := 0; w < model.data.V; w++ { 51 | result += umath.Lgamma(float64(model.cIndex[k][w]) + model.alpha) 52 | } 53 | } 54 | 55 | return result 56 | } 57 | 58 | func (model *trainableModel) logLikelihood() float64 { 59 | k := model.k 60 | betaSum := floats.Sum(model.beta) 61 | alphaSum := model.alpha * float64(model.data.V) 62 | result := 0.0 63 | for i := 0; i < model.data.N; i++ { 64 | n := len(model.data.W[i]) 65 | result += umath.Lgamma(betaSum) - umath.Lgamma(float64(n)+betaSum) 66 | for j := 0; j < k; j++ { 67 | result += umath.Lgamma(model.beta[j]+float64(model.nIndex[i][j])) - umath.Lgamma(model.beta[j]) 68 | } 69 | 70 | //for j := 0; j < m; j++ { 71 | // res += mod.logPhis[mod.zs[i][j]][mod.data.Docs[i][j]] 72 | //} 73 | } 74 | 75 | result -= float64(model.k) * (math.Log(float64(model.data.V)) + umath.Lgamma(model.alpha) - umath.Lgamma(alphaSum)) 76 | for k := 0; k < model.k; k++ { 77 | result -= umath.Lgamma(float64(model.zIndex[k]) + alphaSum) 78 | for w := 0; w < model.data.V; w++ { 79 | result += umath.Lgamma(float64(model.cIndex[k][w]) + model.alpha) 80 | } 81 | } 82 | 83 | for _, comp := range model.data.C { 84 | xLength := len(model.data.W[comp.X]) 85 | yLength := len(model.data.W[comp.Y]) 86 | result += umath.LogSigmoid(umath.Anxmany(model.nu, model.nIndex[comp.X], model.nIndex[comp.Y], xLength, yLength)) 87 | } 88 | 89 | for _, nui := range model.nu { 90 | result -= 0.5 * nui * nui / model.sigma 91 | } 92 | 93 | return result 94 | } 95 | 96 | func (model *trainableModel) InferDoc(doc []int, s *InferSettings) []int { 97 | n := len(doc) 98 | alphaSum := model.alpha * float64(model.data.V) 99 | z := make([]int, n) 100 | for i := 0; i < n; i++ { 101 | z[i] = rand.Intn(model.k) 102 | } 103 | nIndex := ints.Count(z, model.k) 104 | cIndex := make([][]int, model.k) 105 | for i := 0; i < model.k; i++ { 106 | cIndex[i] = make([]int, model.data.V) 107 | } 108 | for i, w := range doc { 109 | cIndex[z[i]][w]++ 110 | } 111 | T := s.InitT 112 | for iter := 0; iter < s.NumSAIter; iter++ { 113 | for i := 0; i < n; i++ { 114 | curZ := z[i] 115 | newZ := rand.Intn(model.k) 116 | if curZ == newZ { 117 | continue 118 | } 119 | w := doc[i] 120 | diff := math.Log(model.beta[curZ]+float64(nIndex[curZ]-1)) - 121 | math.Log(model.beta[newZ]+float64(nIndex[newZ])) + 122 | math.Log(model.alpha+float64(model.cIndex[curZ][w]+cIndex[curZ][w]-1)) - 123 | math.Log(model.alpha+float64(model.cIndex[newZ][w]+cIndex[newZ][w])) + 124 | math.Log(float64(model.zIndex[newZ]+nIndex[newZ])+alphaSum) - 125 | math.Log(float64(model.zIndex[curZ]+nIndex[curZ]-1)+alphaSum) 126 | prob := rand.Float64() 127 | if diff <= 0.0 || prob < math.Exp(-diff/T) { 128 | z[i] = newZ 129 | nIndex[curZ]-- 130 | nIndex[newZ]++ 131 | cIndex[curZ][w]-- 132 | cIndex[newZ][w]++ 133 | } 134 | } 135 | T *= s.CoolingRate 136 | } 137 | return z 138 | } 139 | 140 | func (model *trainableModel) nuObjEval(nu []float64) float64 { 141 | result := 0.0 142 | for _, comp := range model.data.C { 143 | xLength := len(model.data.W[comp.X]) 144 | yLength := len(model.data.W[comp.Y]) 145 | // re-weighted 146 | result += umath.LogSigmoid(umath.Anxmany(nu, model.nIndex[comp.X], model.nIndex[comp.Y], xLength, yLength)) 147 | } 148 | for _, nui := range nu { 149 | result -= 0.5 * nui * nui / model.sigma 150 | //result -= math.Abs(nui) / model.sigma 151 | } 152 | return -result 153 | } 154 | 155 | func (model *trainableModel) nuObjGrad(grad []float64, nu []float64) { 156 | k := len(nu) 157 | //for i, nui := range nu { 158 | for i, nui := range nu { 159 | grad[i] = -nui / model.sigma 160 | //grad[i] = -1.0 / model.sigma 161 | } 162 | for _, comp := range model.data.C { 163 | xLength := len(model.data.W[comp.X]) 164 | yLength := len(model.data.W[comp.Y]) 165 | // re-weighted 166 | sigmoid := umath.Sigmoid(-umath.Anxmany(nu, model.nIndex[comp.X], model.nIndex[comp.Y], xLength, yLength)) 167 | 168 | for i := 0; i < k; i++ { 169 | grad[i] += (float64(model.nIndex[comp.X][i])/float64(xLength) - float64(model.nIndex[comp.Y][i])/float64(yLength)) * sigmoid 170 | } 171 | } 172 | floats.Scale(-1, grad) 173 | } 174 | 175 | func (model *trainableModel) betaObjEval(bs []float64) float64 { 176 | n := len(model.nIndex) 177 | k := len(bs) 178 | sumLGammaBeta := 0.0 179 | sumBeta := 0.0 180 | for _, b := range bs { 181 | sumLGammaBeta += umath.Lgamma(b) 182 | sumBeta += b 183 | } 184 | res := float64(n) * (umath.Lgamma(sumBeta) - sumLGammaBeta) 185 | for i := 0; i < n; i++ { 186 | for j := 0; j < k; j++ { 187 | res += umath.Lgamma(bs[j] + float64(model.nIndex[i][j])) 188 | } 189 | res -= umath.Lgamma(float64(len(model.data.W[i])) + sumBeta) 190 | } 191 | return -res /*/ float64(n)*/ 192 | } 193 | 194 | func (model *trainableModel) betaObjGrad(grad []float64, bs []float64) { 195 | sumBeta := floats.Sum(bs) 196 | n := len(model.nIndex) 197 | k := len(bs) 198 | for i := 0; i < k; i++ { 199 | grad[i] = 0.0 200 | } 201 | 202 | for i := 0; i < n; i++ { 203 | for j := 0; j < k; j++ { 204 | grad[j] += umath.H(bs[j], model.nIndex[i][j]) - umath.H(sumBeta, len(model.data.W[i])) 205 | } 206 | } 207 | floats.Scale(-1 /*/float64(n)*/, grad) 208 | } 209 | 210 | func (model *trainableModel) betaObjSpecialHess(h []float64, z *float64, bs []float64) { 211 | sumBeta := floats.Sum(bs) 212 | n := len(model.nIndex) 213 | k := len(bs) 214 | 215 | *z = 0 216 | for i := 0; i < k; i++ { 217 | h[i] = 0 218 | } 219 | 220 | for i := 0; i < n; i++ { 221 | ni := len(model.data.W[i]) 222 | *z -= umath.DiH(sumBeta, ni) 223 | for j := 0; j < k; j++ { 224 | h[j] += umath.DiH(bs[j], model.nIndex[i][j]) 225 | } 226 | } 227 | 228 | floats.Scale(-1 /*/float64(n)*/, h) 229 | *z = -(*z) /*/ float64(n)*/ 230 | } 231 | 232 | func (model *trainableModel) betaObjHess(hess mat64.MutableSymmetric, bs []float64) { 233 | sumBeta := floats.Sum(bs) 234 | n := len(model.nIndex) 235 | k := len(bs) 236 | for i := 0; i < k; i++ { 237 | for j := i; j < k; j++ { 238 | hess.SetSym(i, j, 0.0) 239 | } 240 | } 241 | 242 | for i := 0; i < n; i++ { 243 | ni := len(model.data.W[i]) 244 | diHSum := umath.DiH(sumBeta, ni) 245 | for j := 0; j < k; j++ { 246 | hess.SetSym(j, j, hess.At(j, j)+umath.DiH(bs[j], model.nIndex[i][j])-diHSum) 247 | for m := j + 1; m < k; m++ { 248 | hess.SetSym(j, m, hess.At(j, m)-diHSum) 249 | } 250 | } 251 | } 252 | 253 | for i := 0; i < k; i++ { 254 | for j := i; j < k; j++ { 255 | hess.SetSym(i, j, -hess.At(i, j) /*/float64(n)*/) 256 | } 257 | } 258 | } 259 | 260 | func (model *trainableModel) zCurObjEval() float64 { 261 | n := len(model.z) 262 | alphaSum := model.alpha * float64(model.data.V) 263 | result := 0.0 264 | 265 | for i := 0; i < n; i++ { 266 | for j := 0; j < model.k; j++ { 267 | result += umath.Lgamma(model.beta[j] + float64(model.nIndex[i][j])) 268 | } 269 | } 270 | 271 | for i := 0; i < model.k; i++ { 272 | result -= umath.Lgamma(float64(model.zIndex[i]) + alphaSum) 273 | for j := 0; j < model.data.V; j++ { 274 | result += umath.Lgamma(float64(model.cIndex[i][j]) + model.alpha) 275 | } 276 | } 277 | 278 | for _, comp := range model.data.C { 279 | xLength := len(model.data.W[comp.X]) 280 | yLength := len(model.data.W[comp.Y]) 281 | result += umath.LogSigmoid(umath.Anxmany(model.nu, model.nIndex[comp.X], model.nIndex[comp.Y], xLength, yLength)) 282 | } 283 | return result 284 | } 285 | 286 | func (model *trainableModel) zObjEval(z [][]int) float64 { 287 | n := len(z) 288 | res := 0.0 289 | nZ := make([][]int, n) 290 | for i := 0; i < n; i++ { 291 | zi := z[i] 292 | nZ[i] = ints.Count(zi, model.k) 293 | for j := 0; j < model.k; j++ { 294 | res += umath.Lgamma(model.beta[j] + float64(nZ[i][j])) 295 | } 296 | 297 | m := len(z[i]) 298 | for j := 0; j < m; j++ { 299 | zij := zi[j] 300 | res += model.logPhi[zij][model.data.W[i][j]] 301 | } 302 | } 303 | 304 | for _, comp := range model.data.C { 305 | zX := len(model.data.W[comp.X]) 306 | zY := len(model.data.W[comp.Y]) 307 | res += umath.LogSigmoid(umath.Anxmany(model.nu, nZ[comp.X], nZ[comp.Y], zX, zY)) 308 | } 309 | 310 | return res 311 | } 312 | 313 | func (model *trainableModel) optimizeZ(numIter int, initT, cRate, dropRate float64) { 314 | log.Printf("optimizing Obj(z) = %g\n", model.zCurObjEval()) 315 | 316 | alphaSum := model.alpha * float64(model.data.V) 317 | 318 | for i := 0; i < model.data.N; i++ { 319 | n := len(model.data.W[i]) 320 | T := initT 321 | //for k := 0; k < numIter; k++ { 322 | for j := 0; j < n; j++ { 323 | curZ := model.z[i][j] 324 | newZ := rand.Intn(model.k) 325 | if curZ == newZ { 326 | continue 327 | } 328 | w := model.data.W[i][j] 329 | eval := math.Log(model.beta[curZ]+float64(model.nIndex[i][curZ]-1)) - 330 | math.Log(model.beta[newZ]+float64(model.nIndex[i][newZ])) + 331 | math.Log(model.alpha+float64(model.cIndex[curZ][w]-1)) - 332 | math.Log(model.alpha+float64(model.cIndex[newZ][w])) + 333 | math.Log(float64(model.zIndex[newZ])+alphaSum) - 334 | math.Log(float64(model.zIndex[curZ]-1)+alphaSum) 335 | 336 | for _, entry := range model.comparisonIndex[i] { 337 | if rand.Float64() < dropRate { 338 | continue 339 | } 340 | var eta float64 341 | if entry.X == i { 342 | eta = entry.etaX 343 | } else { 344 | eta = entry.etaY 345 | } 346 | delta := float64(model.nu[newZ]-model.nu[curZ]) / eta 347 | eval += umath.LogSigmoid(entry.eval) - umath.LogSigmoid(entry.eval+delta) 348 | } 349 | 350 | prob := rand.Float64() 351 | if eval <= 0.0 || prob < math.Exp(-eval/T) { 352 | model.z[i][j] = newZ 353 | model.nIndex[i][curZ]-- 354 | model.nIndex[i][newZ]++ 355 | model.cIndex[curZ][w]-- 356 | model.cIndex[newZ][w]++ 357 | model.zIndex[curZ]-- 358 | model.zIndex[newZ]++ 359 | for _, entry := range model.comparisonIndex[i] { 360 | var eta float64 361 | if entry.X == i { 362 | eta = entry.etaX 363 | } else { 364 | eta = entry.etaY 365 | } 366 | entry.eval += float64(model.nu[newZ]-model.nu[curZ]) / float64(eta) 367 | } 368 | } 369 | } 370 | T *= cRate 371 | //} 372 | } 373 | log.Printf(" Obj(z) = %g\n", model.zCurObjEval()) 374 | } 375 | 376 | func (model *trainableModel) optimizePhi() { 377 | log.Printf("optimizing Obj(phi) = ?\n") 378 | 379 | for i := 0; i < model.k; i++ { 380 | for j := 0; j < model.data.V; j++ { 381 | model.logPhi[i][j] = model.alpha 382 | } 383 | } 384 | 385 | for i := 0; i < model.data.N; i++ { 386 | n := len(model.data.W[i]) 387 | for j := 0; j < n; j++ { 388 | model.logPhi[model.z[i][j]][model.data.W[i][j]] += 1.0 389 | } 390 | } 391 | 392 | for i := 0; i < model.k; i++ { 393 | z := math.Log(floats.Sum(model.logPhi[i])) 394 | for j := 0; j < model.data.V; j++ { 395 | model.logPhi[i][j] = math.Log(model.logPhi[i][j]) - z 396 | } 397 | } 398 | log.Printf(" Obj(phi) = done\n") 399 | } 400 | 401 | func (model *trainableModel) optimizeBeta() { 402 | x0 := make([]float64, model.k) 403 | for i := 0; i < model.k; i++ { 404 | x0[i] = 1e-6 405 | } 406 | log.Printf("opimizing Obj(beta) = %g\n", model.betaObjEval(x0)) 407 | betaOptProblem := umath.NewtonRaphson{Func: model.betaObjEval, Grad: model.betaObjGrad, SpecialHess: model.betaObjSpecialHess} 408 | x := umath.FindStationaryPoint(betaOptProblem, x0) 409 | copy(model.beta, x) 410 | log.Printf(" Obj(beta) = %g\n", model.betaObjEval(model.beta)) 411 | } 412 | 413 | func (model *trainableModel) optimizeNu() { 414 | nuOptProb := optimize.Problem{Func: model.nuObjEval, Grad: model.nuObjGrad} 415 | log.Printf("optimizing Obj(nu) = %g\n", nuOptProb.Func(model.nu)) 416 | settings := optimize.DefaultSettings() 417 | result, err := optimize.Local(nuOptProb, model.nu, settings, &optimize.GradientDescent{}) 418 | if err != nil { 419 | log.Println("WARNING:", err) 420 | } 421 | copy(model.nu, result.X) 422 | log.Printf(" Obj(nu) = %g\n", nuOptProb.Func(model.nu)) 423 | } 424 | -------------------------------------------------------------------------------- /umath/optimize.go: -------------------------------------------------------------------------------- 1 | package umath 2 | 3 | import "math" 4 | 5 | // NewtonRaphson is an instance of minimization problem with a special Hessian structure 6 | type NewtonRaphson struct { 7 | Func func(x []float64) float64 8 | Grad func(grad []float64, x []float64) 9 | SpecialHess func(h []float64, z *float64, x []float64) 10 | } 11 | 12 | // FindStationaryPoint solves local minimization problem 13 | func FindStationaryPoint(p NewtonRaphson, x0 []float64) []float64 { 14 | k := len(x0) 15 | x := make([]float64, k) 16 | copy(x, x0) 17 | 18 | z := 0.0 19 | h := make([]float64, k) 20 | g := make([]float64, k) 21 | 22 | prevEval := p.Func(x0) 23 | 24 | for iter := 0; iter < 10000; iter++ { 25 | p.Grad(g, x) 26 | p.SpecialHess(h, &z, x) 27 | 28 | c := 0.0 29 | d := 0.0 30 | for i := 0; i < k; i++ { 31 | c += g[i] / h[i] 32 | d += 1.0 / h[i] 33 | } 34 | c /= 1.0/z + d 35 | 36 | for i := 0; i < k; i++ { 37 | x[i] -= (g[i] - c) / h[i] 38 | } 39 | 40 | curEval := p.Func(x) 41 | if math.Abs(prevEval-curEval) < 1e-6 { 42 | break 43 | } 44 | prevEval = curEval 45 | } 46 | 47 | return x 48 | } 49 | -------------------------------------------------------------------------------- /umath/umath.go: -------------------------------------------------------------------------------- 1 | package umath 2 | 3 | import ( 4 | "log" 5 | "math" 6 | "math/rand" 7 | ) 8 | 9 | // Lgamma returns the natural logarithm of Gamma(x). 10 | func Lgamma(x float64) float64 { 11 | result, _ := math.Lgamma(x) 12 | return result 13 | } 14 | 15 | func _Sigmoid(x float64) float64 { 16 | return 0.5*(x/(1.0+math.Abs(x))) + 0.5 17 | } 18 | 19 | func _LogSigmoid(x float64) float64 { 20 | return math.Log(Sigmoid(x)) 21 | } 22 | 23 | // Sigmoid returns 1/(1+Exp(x)) 24 | func Sigmoid(x float64) float64 { 25 | return 1.0 / (1.0 + math.Exp(-x)) 26 | } 27 | 28 | // LogSigmoid returns the natural logarithm of Sigmoid(x). 29 | func LogSigmoid(x float64) float64 { 30 | if x < -10 { 31 | return x - math.Exp(x) 32 | } else if x < 10 { 33 | return -math.Log(1.0 + math.Exp(-x)) 34 | } else { 35 | return -math.Exp(-x) 36 | } 37 | } 38 | 39 | // H returns the difference: DiGamma(x + n) - DiGamma(x) 40 | func H(x float64, n int) float64 { 41 | res := 0.0 42 | for i := 0; i < n; i++ { 43 | res += 1.0 / (x + float64(i)) 44 | } 45 | return res 46 | } 47 | 48 | // DiH returns the derivative with respect to x of H(x) 49 | func DiH(x float64, n int) float64 { 50 | result := 0.0 51 | for i := 0; i < n; i++ { 52 | val := x + float64(i) 53 | result -= 1.0 / (val * val) 54 | } 55 | return result 56 | } 57 | 58 | // Anxmany alpha * (x / nx - y / ny) 59 | func Anxmany(a []float64, x, y []int, nx, ny int) float64 { 60 | length := len(a) 61 | if length != len(x) || length != len(y) { 62 | log.Panic("ERROR: vector length mismatch") 63 | } 64 | result := 0.0 65 | for i := 0; i < length; i++ { 66 | result += a[i] * (float64(x[i])/float64(nx) - float64(y[i])/float64(ny)) 67 | } 68 | return result 69 | } 70 | 71 | func SampleFromLogDist(dist []float64) int { 72 | p := rand.Float64() 73 | cdf := 0.0 74 | for i, logProb := range dist { 75 | cdf += math.Exp(logProb) 76 | if p < cdf { 77 | return i 78 | } 79 | } 80 | return len(dist) - 1 81 | } 82 | --------------------------------------------------------------------------------