├── .gitignore ├── .github └── workflows │ └── ci.yml ├── LICENSE ├── go.mod ├── posthog_importer.go ├── README.md ├── go.sum ├── mixpanel_exporter.go └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | .env -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | tags: 8 | - '**' 9 | 10 | jobs: 11 | release: 12 | if: startsWith(github.ref, 'refs/tags/v') 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v3 16 | with: 17 | fetch-depth: 0 18 | - uses: actions/setup-go@v3 19 | with: 20 | go-version-file: go.mod 21 | cache: true 22 | - uses: goreleaser/goreleaser-action@v2 23 | with: 24 | version: latest 25 | args: release --rm-dist 26 | env: 27 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Stablecog Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/stablecog/sc-mp-to-ph 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/briandowns/spinner v1.22.0 7 | github.com/charmbracelet/log v0.1.2 8 | github.com/fatih/color v1.14.1 9 | github.com/joho/godotenv v1.5.1 10 | github.com/manifoldco/promptui v0.9.0 11 | github.com/posthog/posthog-go v1.4.9 12 | ) 13 | 14 | require ( 15 | github.com/charmbracelet/lipgloss v0.6.0 // indirect 16 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect 17 | github.com/go-logfmt/logfmt v0.6.0 // indirect 18 | github.com/google/uuid v1.6.0 19 | github.com/lucasb-eyer/go-colorful v1.2.0 // indirect 20 | github.com/mattn/go-colorable v0.1.13 // indirect 21 | github.com/mattn/go-isatty v0.0.17 // indirect 22 | github.com/mattn/go-runewidth v0.0.13 // indirect 23 | github.com/muesli/reflow v0.2.1-0.20210115123740-9e1d0d53df68 // indirect 24 | github.com/muesli/termenv v0.11.1-0.20220204035834-5ac8409525e0 // indirect 25 | github.com/rivo/uniseg v0.2.0 // indirect 26 | github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect 27 | golang.org/x/sys v0.3.0 // indirect 28 | golang.org/x/term v0.1.0 // indirect 29 | ) 30 | -------------------------------------------------------------------------------- /posthog_importer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/fatih/color" 7 | "github.com/posthog/posthog-go" 8 | ) 9 | 10 | func PosthogImport(client posthog.Client, data []MixpanelDataLine) error { 11 | for _, line := range data { 12 | if line.Event == "$mp_web_page_view" { 13 | line.Event = "$pageview" 14 | } 15 | // Construct properties 16 | properties := posthog.NewProperties() 17 | for k, v := range line.Properties { 18 | properties.Set(k, v) 19 | } 20 | properties.Set("$geoip_disable", true) 21 | err := client.Enqueue(posthog.Capture{ 22 | DistinctId: line.DistinctID, 23 | Event: line.Event, 24 | Properties: properties, 25 | Timestamp: line.Time, 26 | }) 27 | if err != nil { 28 | color.Red("\nError importing event: %s", line.Event) 29 | return err 30 | } 31 | // Sleep in between to avoid overloading the API 32 | time.Sleep(DELAY_MS * time.Millisecond) 33 | } 34 | return nil 35 | } 36 | 37 | func PosthogImportUsers(client posthog.Client, users []MixpanelUser) error { 38 | for _, user := range users { 39 | // Construct properties 40 | properties := posthog.NewProperties() 41 | ts := time.Now() 42 | for k, v := range user.Properties { 43 | if k == "$last_seen" { 44 | // Parse date 2022-12-29T12:49:16" 45 | t, err := time.Parse("2006-01-02T15:04:05", v.(string)) 46 | if err == nil { 47 | ts = t 48 | } 49 | continue 50 | } 51 | if v != "undefined" { 52 | properties.Set(k, v) 53 | } 54 | } 55 | properties.Set("$geoip_disable", true) 56 | properties.Set("$lib", "mixpanel-importer") 57 | err := client.Enqueue(posthog.Identify{ 58 | Timestamp: ts, 59 | DistinctId: user.DistinctID, 60 | Properties: properties, 61 | }) 62 | if err != nil { 63 | color.Red("\nError importing user: %s", user.DistinctID) 64 | return err 65 | } 66 | // Sleep in between to avoid overloading the API 67 | time.Sleep(DELAY_MS * 5 * time.Millisecond) 68 | } 69 | return nil 70 | } 71 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SC Mixpanel to Posthog Data Migrator 2 | 3 | A tool for easily migrating data from [Mixpanel](https://mixpanel.com/) to [Posthog](https://posthog.com) (self-hosted or cloud) 4 | 5 | ## Disclaimer 6 | 7 | This is **NOT an official tool**. We are not affiliated with Posthog or Mixpanel 8 | 9 | However, if you are looking to migrate from Mixpanel to Posthog like we were - we hope you find this tool useful. 10 | 11 | ## Setup 12 | 13 | ### Mixpanel 14 | 15 | You will need the following from Mixpanel: 16 | 17 | - Service account username and password (with owner privileges) 18 | - The project ID (found in settings -> overview) 19 | 20 | You will be prompted in CLI to input these, or you can set the following env variables: 21 | 22 | ``` 23 | MIXPANEL_USERNAME= 24 | MIXPANEL_PASSWORD= 25 | MIXPANEL_PROJECT_ID= 26 | # Optional override, defaults to https://data.mixpanel.com/api/2.0 27 | MIXPANEL_API_URL 28 | ``` 29 | 30 | You can also put these in `.env` for convenience. 31 | 32 | ### Posthog 33 | 34 | You will need the following from Posthog: 35 | 36 | - Project API key 37 | - Personal API key 38 | - Endpoint URL 39 | 40 | You will be prompted in CLI for these, but can also set them in the environment: 41 | 42 | ``` 43 | POSTHOG_PROJECT_KEY= 44 | POSTHOG_API_KEY= 45 | POSTHOG_ENDPOINT= 46 | ``` 47 | 48 | ## **WARNING** Do not use this without reading this first. 49 | 50 | The mixpanel export API has no pagination, the CLI will prompt you for a date range (required by Mixpanel) 51 | 52 | If you have a very large data set, **do not try to get it all at once** 53 | 54 | Mixpanel could rate limit you, your system could run out of memory and crash. 55 | 56 | It's recommended to do smaller chunks at a time (dates are inclusive, so from_date=2023-03-01 and to_date=2023-03-01 will import 1 days worth of data) 57 | 58 | # Usage 59 | 60 | Download the latest [Release](https://github.com/stablecog/sc-mp-to-ph/releases) for your system. 61 | 62 | ## Recommended flow 63 | 64 | The best way we found to migrate data is to do the following. 65 | 66 | 1. Disable GeoIP app (if enabled) 67 | 2. Import events (see below) 68 | 3. In Mixpanel UI, export all users and columns as CSV 69 | 4. Import the users (see below) 70 | 5. Enable GeoIP app (if enabled) 71 | 72 | ## (Step 1) Import Events 73 | 74 | Just run without any parameters: 75 | 76 | `./mixpanel-to-posthog` or `./mixpanel-to-posthog.exe` if using windows. 77 | 78 | ## (Step 2) Import Users 79 | 80 | The mixpanel Web UI allows exporting users as csv format. Select all columns, all users, get a .csv file. 81 | 82 | Run `./mixpanel-to-posthog -users-csv-file /path/to/users-export.csv` to load all users into mixpanel 83 | 84 | ## Check us out 85 | 86 | [Stablecog](https://stablecog.com/) 87 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 | github.com/briandowns/spinner v1.22.0 h1:fJ/7tyeM2q9ebM57kGfjnUSrgPJBsULk+/s61UpMGrw= 3 | github.com/briandowns/spinner v1.22.0/go.mod h1:rPG4gmXeN3wQV/TsAY4w8lPdIM6RX3yqeBQJSrbXjuE= 4 | github.com/charmbracelet/lipgloss v0.6.0 h1:1StyZB9vBSOyuZxQUcUwGr17JmojPNm87inij9N3wJY= 5 | github.com/charmbracelet/lipgloss v0.6.0/go.mod h1:tHh2wr34xcHjC2HCXIlGSG1jaDF0S0atAUvBMP6Ppuk= 6 | github.com/charmbracelet/log v0.1.2 h1:xmKMxo0T/lcftgggQOhUkS32exku2/ID55FGYbr4nKQ= 7 | github.com/charmbracelet/log v0.1.2/go.mod h1:86XdIdmrubqtL/6u0z+jGFol1bQejBGG/qPSTwGZuQQ= 8 | github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= 9 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= 10 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= 11 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= 12 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= 13 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= 14 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 15 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 16 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 17 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 18 | github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= 19 | github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= 20 | github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= 21 | github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= 22 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 23 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 24 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 25 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 26 | github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= 27 | github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= 28 | github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= 29 | github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= 30 | github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA= 31 | github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg= 32 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 33 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 34 | github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= 35 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 36 | github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= 37 | github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 38 | github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= 39 | github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= 40 | github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 41 | github.com/muesli/reflow v0.2.1-0.20210115123740-9e1d0d53df68 h1:y1p/ycavWjGT9FnmSjdbWUlLGvcxrY0Rw3ATltrxOhk= 42 | github.com/muesli/reflow v0.2.1-0.20210115123740-9e1d0d53df68/go.mod h1:Xk+z4oIWdQqJzsxyjgl3P22oYZnHdZ8FFTHAQQt5BMQ= 43 | github.com/muesli/termenv v0.11.1-0.20220204035834-5ac8409525e0 h1:STjmj0uFfRryL9fzRA/OupNppeAID6QJYPMavTL7jtY= 44 | github.com/muesli/termenv v0.11.1-0.20220204035834-5ac8409525e0/go.mod h1:Bd5NYQ7pd+SrtBSrSNoBBmXlcY8+Xj4BMJgh8qcZrvs= 45 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 46 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 47 | github.com/posthog/posthog-go v0.0.0-20221221115252-24dfed35d71a h1:Ey0XWvrg6u6hyIn1Kd/jCCmL+bMv9El81tvuGBbxZGg= 48 | github.com/posthog/posthog-go v0.0.0-20221221115252-24dfed35d71a/go.mod h1:oa2sAs9tGai3VldabTV0eWejt/O4/OOD7azP8GaikqU= 49 | github.com/posthog/posthog-go v0.0.0-20240115103626-fbd687c18571 h1:ql4li84J/32ExlZ4aacyk076tHO0oqy1TtJRv8JWyO4= 50 | github.com/posthog/posthog-go v0.0.0-20240115103626-fbd687c18571/go.mod h1:migYMxlAqcnQy+3eN8mcL0b2tpKy6R+8Zc0lxwk4dKM= 51 | github.com/posthog/posthog-go v1.4.9 h1:RmbcZQuTKRptnn6egUiV3bLpIE+D3sbE2irY9i2hYro= 52 | github.com/posthog/posthog-go v1.4.9/go.mod h1:uYC2l1Yktc8E+9FAHJ9QZG4vQf/NHJPD800Hsm7DzoM= 53 | github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 54 | github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= 55 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 56 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 57 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 58 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 59 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 60 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 61 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 62 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 63 | github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= 64 | github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 65 | github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= 66 | github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c h1:3lbZUMbMiGUW/LMkfsEABsc5zNT9+b1CvsJx47JzJ8g= 67 | github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c/go.mod h1:UrdRz5enIKZ63MEE3IF9l2/ebyx59GyGgPi+tICQdmM= 68 | golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 69 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 70 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 71 | golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= 72 | golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 73 | golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw= 74 | golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 75 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 76 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 77 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 78 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 79 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 80 | -------------------------------------------------------------------------------- /mixpanel_exporter.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/csv" 6 | "encoding/json" 7 | "fmt" 8 | "io" 9 | "net/http" 10 | "os" 11 | "strings" 12 | "time" 13 | 14 | "github.com/charmbracelet/log" 15 | "github.com/google/uuid" 16 | ) 17 | 18 | type Mixpanel struct { 19 | APIUrl string 20 | Token string 21 | FromDate time.Time 22 | ToDate time.Time 23 | ProjectID string 24 | Client *http.Client 25 | Version string 26 | } 27 | 28 | func basicAuth(username, password string) string { 29 | auth := username + ":" + password 30 | return base64.StdEncoding.EncodeToString([]byte(auth)) 31 | } 32 | 33 | // Create a new mixpanel client 34 | func NewExporter(version string, apiUrl string, user string, password string, projectId string, fromDate time.Time, toDate time.Time) *Mixpanel { 35 | return &Mixpanel{ 36 | Version: version, 37 | APIUrl: apiUrl, 38 | Token: basicAuth(user, password), 39 | FromDate: fromDate, 40 | ToDate: toDate, 41 | ProjectID: projectId, 42 | Client: http.DefaultClient, 43 | } 44 | } 45 | 46 | func (c *Mixpanel) Export() ([]MixpanelDataLine, error) { 47 | // Format times to yyyy-mm-dd 48 | fromDate := c.FromDate.Format("2006-01-02") 49 | toDate := c.ToDate.Format("2006-01-02") 50 | url := c.APIUrl + fmt.Sprintf("/export?from_date=%s&to_date=%s&project_id=%s", fromDate, toDate, c.ProjectID) 51 | 52 | request, err := http.NewRequest("GET", url, nil) 53 | if err != nil { 54 | return nil, err 55 | } 56 | request.Header.Set("Authorization", fmt.Sprintf("Basic %s", c.Token)) 57 | resp, err := c.Client.Do(request) 58 | if err != nil { 59 | return nil, err 60 | } else if resp.StatusCode != 200 { 61 | return nil, fmt.Errorf("status=%s; httpCode=%d Export failed", resp.Status, resp.StatusCode) 62 | } 63 | defer resp.Body.Close() 64 | 65 | // Custom decoder since they have a wonky format 66 | dec := json.NewDecoder(resp.Body) 67 | ret := []MixpanelDataLine{} 68 | 69 | for { 70 | var line MixpanelDataLineRaw 71 | if err := dec.Decode(&line); err != nil { 72 | if err == io.EOF { 73 | break 74 | } 75 | return nil, err 76 | } 77 | 78 | // Format the data 79 | formattedDataLine := MixpanelDataLine{} 80 | 81 | // Some events have internal names in posthog 82 | switch line.Event { 83 | case "Pageview": 84 | formattedDataLine.Event = "$pageview" 85 | default: 86 | formattedDataLine.Event = line.Event 87 | } 88 | 89 | // Parse properties 90 | formattedDataLine.Properties = make(map[string]interface{}) 91 | formattedDataLine.Properties["$lib_version"] = fmt.Sprintf("stablecog/mp-to-ph@%s", c.Version) 92 | 93 | for k, v := range line.Properties { 94 | if k == "distinct_id" { 95 | formattedDataLine.DistinctID = v.(string) 96 | } else if k == "time" { 97 | // Seconds since epoch to time.Time 98 | formattedDataLine.Time = time.Unix(int64(v.(float64)), 0) 99 | } else { 100 | switch k { 101 | case "mp_lib": 102 | formattedDataLine.Properties["$lib"] = fmt.Sprintf("%s-imported", v) 103 | // Do nothing with these 104 | case "$mp_api_endpoint", "$mp_api_timestamp_ms", "mp_processing_time_ms": 105 | default: 106 | formattedDataLine.Properties[k] = v 107 | } 108 | } 109 | } 110 | 111 | if formattedDataLine.DistinctID == "" || formattedDataLine.Time.IsZero() { 112 | log.Info("Skipping event with no distinct_id or time", "event", formattedDataLine.Event) 113 | continue 114 | } 115 | ret = append(ret, formattedDataLine) 116 | } 117 | 118 | return ret, nil 119 | } 120 | 121 | type MixpanelDataLineRaw struct { 122 | Event string `json:"event"` 123 | Properties map[string]interface{} `json:"properties"` 124 | } 125 | 126 | type MixpanelDataLine struct { 127 | Event string `json:"event"` 128 | DistinctID string `json:"distinct_id"` 129 | Time time.Time `json:"time"` 130 | Properties map[string]interface{} `json:"properties"` 131 | } 132 | 133 | func readCsvFile(filePath string) [][]string { 134 | f, err := os.Open(filePath) 135 | if err != nil { 136 | log.Fatal("Unable to read input file "+filePath, err) 137 | } 138 | defer f.Close() 139 | 140 | csvReader := csv.NewReader(f) 141 | records, err := csvReader.ReadAll() 142 | if err != nil { 143 | log.Fatal("Unable to parse file as CSV for "+filePath, err) 144 | } 145 | 146 | return records 147 | } 148 | 149 | type CSVHeaderIndex struct { 150 | index int 151 | name string 152 | } 153 | 154 | // Dont translate these properties 155 | const NOOP = "NOOP" 156 | 157 | func LoadMixpanelUsersFromCSVFile(csvFile string) ([]MixpanelUser, error) { 158 | // read csv from string 159 | records := readCsvFile(csvFile) 160 | headers := records[0] 161 | headerIndex := make(map[int]string) 162 | for i, header := range headers { 163 | if strings.HasPrefix(header, "$") { 164 | switch header { 165 | case "$mp_first_event_time": 166 | headerIndex[i] = "NOOP" 167 | case "$timezone": 168 | headerIndex[i] = "$geoip_time_zone" 169 | case "$region": 170 | headerIndex[i] = "$geoip_subdivision_1_name" 171 | case "$country_code": 172 | headerIndex[i] = "$geoip_country_code" 173 | case "$city": 174 | headerIndex[i] = "$geoip_city_name" 175 | case "$email": 176 | headerIndex[i] = "email" 177 | default: 178 | headerIndex[i] = header 179 | } 180 | } else { 181 | headerIndex[i] = header 182 | } 183 | } 184 | 185 | ret := []MixpanelUser{} 186 | for _, record := range records[1:] { 187 | properties := make(map[string]interface{}) 188 | distinctId := "" 189 | for i, value := range record { 190 | if headerIndex[i] == "$distinct_id" { 191 | distinctId = value 192 | continue 193 | } 194 | if headerIndex[i] != NOOP { 195 | properties[headerIndex[i]] = value 196 | } 197 | } 198 | ret = append(ret, MixpanelUser{ 199 | DistinctID: distinctId, 200 | Properties: properties, 201 | }) 202 | } 203 | 204 | return MergeMixpanelUsers(ret), nil 205 | } 206 | 207 | // We need to do a merge to find valid uuids for emails and replace them 208 | // Some mixpanel IDs are not valid uuidv4 209 | func MergeMixpanelUsers(users []MixpanelUser) []MixpanelUser { 210 | // Find users with invalid uuids 211 | invalidIdEmailMap := make(map[string]string) 212 | 213 | // Loop users 214 | for _, user := range users { 215 | // If this uuid is valid 216 | if _, err := uuid.Parse(user.DistinctID); err == nil { 217 | // Get email 218 | email, ok := user.Properties["email"] 219 | if !ok { 220 | continue 221 | } 222 | email, ok = email.(string) 223 | if !ok { 224 | continue 225 | } 226 | // Map email to valid UUID 227 | invalidIdEmailMap[email.(string)] = user.DistinctID 228 | } 229 | } 230 | 231 | // Replace all IDs for the valid one 232 | for i, user := range users { 233 | // If user has an email 234 | if email, ok := user.Properties["email"]; ok { 235 | // If we have a valid ID for this email 236 | if validId, ok := invalidIdEmailMap[email.(string)]; ok { 237 | if _, err := uuid.Parse(users[i].DistinctID); err != nil { 238 | log.Info("Replacing invalid ID with valid one", "invalid", user.DistinctID, "valid", validId) 239 | users[i].DistinctID = validId 240 | } 241 | } 242 | } 243 | } 244 | return users 245 | } 246 | 247 | type MixpanelUser struct { 248 | DistinctID string 249 | Properties map[string]interface{} 250 | } 251 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "net/url" 7 | "os" 8 | "time" 9 | 10 | "github.com/briandowns/spinner" 11 | "github.com/charmbracelet/log" 12 | "github.com/fatih/color" 13 | "github.com/joho/godotenv" 14 | "github.com/manifoldco/promptui" 15 | "github.com/posthog/posthog-go" 16 | ) 17 | 18 | var version = "dev" 19 | 20 | // Delay between posthog queue events to avoid overloading the API 21 | const DELAY_MS = 1 22 | 23 | func getPosthogClient() posthog.Client { 24 | // ** Get Posthog credentials ** // 25 | if os.Getenv("POSTHOG_API_KEY") == "" || os.Getenv("POSTHOG_ENDPOINT") == "" || os.Getenv("POSTHOG_PROJECT_KEY") == "" { 26 | color.Cyan("\nPosthog Credentials") 27 | color.Cyan("See the README for reference on what these are and how to get them.\n\n") 28 | } 29 | 30 | // If in env, don't ask 31 | var posthogApiKey string 32 | if os.Getenv("POSTHOG_PROJECT_KEY") != "" { 33 | posthogApiKey = os.Getenv("POSTHOG_PROJECT_KEY") 34 | } else { 35 | posthogApiKeyPrompt := promptui.Prompt{ 36 | Label: "Enter Posthog Project API Key", 37 | Mask: '*', 38 | } 39 | pR, _ := posthogApiKeyPrompt.Run() 40 | posthogApiKey = pR 41 | } 42 | 43 | var posthogPersonalApiKey string 44 | if os.Getenv("POSTHOG_API_KEY") != "" { 45 | posthogPersonalApiKey = os.Getenv("POSTHOG_API_KEY") 46 | } else { 47 | posthogApiKeyPrompt := promptui.Prompt{ 48 | Label: "Enter Posthog Personal API Key", 49 | Mask: '*', 50 | } 51 | pR, _ := posthogApiKeyPrompt.Run() 52 | posthogPersonalApiKey = pR 53 | } 54 | 55 | // If in env, don't ask 56 | var posthogEndpoint string 57 | if os.Getenv("POSTHOG_ENDPOINT") != "" { 58 | posthogEndpoint = os.Getenv("POSTHOG_ENDPOINT") 59 | } else { 60 | posthogApiKeyPrompt := promptui.Prompt{ 61 | Label: "Enter Posthog API Endpoint", 62 | Validate: func(input string) error { 63 | _, err := url.Parse(input) 64 | return err 65 | }, 66 | } 67 | pR, _ := posthogApiKeyPrompt.Run() 68 | posthogEndpoint = pR 69 | } 70 | 71 | // Create posthog client 72 | posthogClient, err := posthog.NewWithConfig(posthogApiKey, posthog.Config{ 73 | Endpoint: posthogEndpoint, 74 | PersonalApiKey: posthogPersonalApiKey, 75 | HistoricalMigration: true, 76 | }) 77 | if err != nil { 78 | color.Red("\nEncountered an error while creating Posthog client: %v", err) 79 | os.Exit(1) 80 | } 81 | return posthogClient 82 | } 83 | 84 | func main() { 85 | godotenv.Load(".env") 86 | 87 | fmt.Println("------------------------------------") 88 | color.Green("SC Mixpanel to Posthog Data Migrator") 89 | fmt.Println("------------------------------------") 90 | 91 | // They can optionally just identify users 92 | csvFile := flag.String("users-csv-file", "", "Path to CSV file to import users") 93 | showVersion := flag.Bool("version", false, "Print version and exit") 94 | flag.Parse() 95 | 96 | if *showVersion { 97 | fmt.Printf("\nVersion: %v\n", color.GreenString(version)) 98 | os.Exit(0) 99 | } 100 | 101 | if *csvFile != "" { 102 | // See if file exists 103 | if _, err := os.Stat(*csvFile); os.IsNotExist(err) { 104 | color.Red("CSV %s file does not exist or cannot be read", *csvFile) 105 | os.Exit(1) 106 | } 107 | // Load from MP 108 | users, err := LoadMixpanelUsersFromCSVFile(*csvFile) 109 | if err != nil { 110 | color.Red("Error loading users from CSV file: %v", err) 111 | os.Exit(1) 112 | } 113 | 114 | // Import users 115 | // Calculate duration 116 | totalMs := DELAY_MS * 5 * len(users) 117 | totalDuration := time.Duration(totalMs) * time.Millisecond 118 | color.Cyan("Importing users from %s (This will take approximately %d minutes, the current time is %s)", *csvFile, int(totalDuration.Minutes()), time.Now().Format("15:04:05")) 119 | s := spinner.New(spinner.CharSets[43], 100*time.Millisecond) 120 | s.Start() 121 | posthogClient := getPosthogClient() 122 | defer posthogClient.Close() 123 | 124 | err = PosthogImportUsers(posthogClient, users) 125 | if err != nil { 126 | color.Red("Error importing users: %v", err) 127 | os.Exit(1) 128 | } 129 | s.Stop() 130 | color.Green("Successfully imported %d users", len(users)) 131 | // Block until user presses control C 132 | color.Red("It's recommended to wait several minutes for posthog to process the users.") 133 | color.Red("Once you see all users in posthog console, you can exit this program.") 134 | color.Red("Press control C to exit...") 135 | select {} 136 | } 137 | 138 | // User inputs 139 | 140 | // ** Get mixpanel credentials ** // 141 | 142 | if os.Getenv("MIXPANEL_API_URL") == "" || os.Getenv("MIXPANEL_PROJECT_ID") == "" || os.Getenv("MIXPANEL_USERNAME") == "" || os.Getenv("MIXPANEL_PASSWORD") == "" { 143 | color.Cyan("\nMixpanel Credentials") 144 | color.Cyan("See the README for reference on what these are and how to get them.\n\n") 145 | } 146 | // If in env, don't ask 147 | var apiUrlResult string 148 | if os.Getenv("MIXPANEL_API_URL") != "" { 149 | apiUrlResult = os.Getenv("MIXPANEL_API_URL") 150 | } else { 151 | apiUrlPrompt := promptui.Prompt{ 152 | Label: "Enter Mixpanel API URL (for EU, use the EU-specific URL):", 153 | AllowEdit: false, 154 | Default: "https://data.mixpanel.com/api/2.0", 155 | Validate: func(input string) error { 156 | // Validate URL 157 | _, err := url.ParseRequestURI(input) 158 | return err 159 | }, 160 | } 161 | pR, _ := apiUrlPrompt.Run() 162 | apiUrlResult = pR 163 | } 164 | 165 | // If in env, don't ask 166 | var projectIdResult string 167 | if os.Getenv("MIXPANEL_PROJECT_ID") != "" { 168 | projectIdResult = os.Getenv("MIXPANEL_PROJECT_ID") 169 | } else { 170 | projectIdPrompt := promptui.Prompt{ 171 | Label: "Enter Mixpanel Project ID", 172 | } 173 | pR, _ := projectIdPrompt.Run() 174 | projectIdResult = pR 175 | } 176 | 177 | // If in env, don't ask 178 | var serviceUsernameResult string 179 | if os.Getenv("MIXPANEL_USERNAME") != "" { 180 | serviceUsernameResult = os.Getenv("MIXPANEL_USERNAME") 181 | } else { 182 | serviceUsernamePrompt := promptui.Prompt{ 183 | Label: "Enter Mixpanel Username (Service Account)", 184 | } 185 | pR, _ := serviceUsernamePrompt.Run() 186 | serviceUsernameResult = pR 187 | } 188 | 189 | // If in env, don't ask 190 | var servicePasswordResult string 191 | if os.Getenv("MIXPANEL_PASSWORD") != "" { 192 | servicePasswordResult = os.Getenv("MIXPANEL_PASSWORD") 193 | } else { 194 | servicePasswordPrompt := promptui.Prompt{ 195 | Label: "Enter Mixpanel Password (Service Account)", 196 | Mask: '*', 197 | } 198 | pR, _ := servicePasswordPrompt.Run() 199 | servicePasswordResult = pR 200 | } 201 | 202 | // ** Get Mixpanel date range ** // 203 | 204 | color.Yellow("\nWARNING: If you have a large dataset, consider entering smaller date ranges at a time.") 205 | color.Yellow("You may crash your machine if you try to export too much data at once.\n\n") 206 | 207 | // Prompt for from_date and to_date in the format 2006-01-02 208 | fromDtPrompt := promptui.Prompt{ 209 | Label: "Enter from_date in the format YYYY-MM-DD", 210 | Validate: func(input string) error { 211 | // Validate date is in the format 2006-01-02 212 | _, err := time.Parse("2006-01-02", input) 213 | return err 214 | }, 215 | } 216 | fromDtResult, err := fromDtPrompt.Run() 217 | if err != nil { 218 | log.Fatal(err) 219 | os.Exit(1) 220 | } 221 | to_date := promptui.Prompt{ 222 | Label: "Enter to_date in the format YYYY-MM-DD", 223 | Validate: func(input string) error { 224 | // Validate date is in the format 2006-01-02 225 | _, err := time.Parse("2006-01-02", input) 226 | return err 227 | }, 228 | } 229 | toDtResult, err := to_date.Run() 230 | if err != nil { 231 | log.Fatal(err) 232 | os.Exit(1) 233 | } 234 | 235 | // Parse dates 236 | fromDt, _ := time.Parse("2006-01-02", fromDtResult) 237 | toDt, _ := time.Parse("2006-01-02", toDtResult) 238 | 239 | posthogClient := getPosthogClient() 240 | defer posthogClient.Close() 241 | 242 | // ** Mixpanel Export ** // 243 | 244 | // Create mixpanel exporter 245 | exporter := NewExporter(version, apiUrlResult, serviceUsernameResult, servicePasswordResult, projectIdResult, fromDt, toDt) 246 | color.Blue("Exporting data from Mixpanel (This may take awhile)") 247 | s := spinner.New(spinner.CharSets[43], 100*time.Millisecond) 248 | s.Reverse() 249 | s.Start() 250 | data, err := exporter.Export() 251 | if err != nil { 252 | color.Red("\nEncountered an error while exporting data from Mixpanel: %v", err) 253 | os.Exit(1) 254 | } 255 | s.Stop() 256 | color.Green("Exported %d events from Mixpanel", len(data)) 257 | 258 | // ** Posthog Import ** // 259 | 260 | totalMs := DELAY_MS * len(data) 261 | totalDuration := time.Duration(totalMs) * time.Millisecond 262 | color.Green("\nImporting data into Posthog (This will take approximately %s, the current time is %s)", totalDuration.String(), time.Now().Format("15:04:05")) 263 | s.Reverse() 264 | s.Start() 265 | err = PosthogImport(posthogClient, data) 266 | if err != nil { 267 | color.Red("\nEncountered an error while importing data into Posthog: %v", err) 268 | os.Exit(1) 269 | } 270 | s.Stop() 271 | if err != nil { 272 | color.Red("\nEncountered an error while closing Posthog client: %v", err) 273 | os.Exit(1) 274 | } 275 | 276 | color.Green("Success!") 277 | color.Green("Imported %d events into Posthog", len(data)) 278 | 279 | // Block until user presses control C 280 | color.Red("It's recommended to wait several minutes for posthog to process the events.") 281 | color.Red("Once you see all events in posthog, you can exit this program.") 282 | color.Red("Press control C to exit...") 283 | select {} 284 | } 285 | --------------------------------------------------------------------------------