├── .github └── workflows │ └── go.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── Vagrantfile ├── cmd └── grafana-migrate │ └── main.go ├── docker-compose.yml ├── go.mod ├── go.sum ├── pkg ├── postgresql │ ├── import.go │ └── tablechanges.go └── sqlite │ ├── dump.go │ └── sanitize.go └── test ├── grafana ├── provisioning │ └── datasources │ │ └── prometheus.yaml └── setup.env └── prometheus └── prometheus.yml /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a golang project 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go 3 | 4 | name: Go 5 | 6 | on: 7 | push: 8 | branches: [ "master" ] 9 | tags: 10 | - "v*.*.*" 11 | pull_request: 12 | branches: [ "master" ] 13 | 14 | jobs: 15 | 16 | build: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: actions/checkout@v3 20 | 21 | - name: Set up Go 22 | uses: actions/setup-go@v4 23 | with: 24 | go-version: '1.20' 25 | 26 | - name: Build 27 | run: make build-all 28 | 29 | - name: Release 30 | uses: softprops/action-gh-release@v1 31 | if: startsWith(github.ref, 'refs/tags/') 32 | with: 33 | generate_release_notes: true 34 | files: dist/* 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | cmd/grafana-migrate/grafana.db 2 | graf 3 | .vagrant 4 | bin 5 | .idea 6 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.18 AS builder 2 | WORKDIR /go/src/github.com/wbh1/grafana-sqlite-to-postgres/ 3 | COPY . . 4 | RUN rm -rf /go/src/github.com/wbh1/grafana-sqlite-to-postgres/dist/grafana-migrate_linux* 5 | RUN make 6 | RUN ls /go/src/github.com/wbh1/grafana-sqlite-to-postgres/dist/grafana-migrate_linux*refs/tags/* 7 | 8 | FROM golang:1.18 9 | WORKDIR /root/ 10 | COPY --from=builder /go/src/github.com/wbh1/grafana-sqlite-to-postgres/dist/grafana-migrate_linux*+refs/tags/v* ./grafana-migrate 11 | RUN apt-get update && apt-get install -y \ 12 | sqlite3 \ 13 | && rm -rf /var/lib/apt/lists/* 14 | ENTRYPOINT ["./grafana-migrate"] 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Will Hegedus 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # The binary to build (just the basename). 2 | BIN := grafana-migrate 3 | SRC_DIR := ./cmd/grafana-migrate 4 | 5 | # This version-strategy uses git tags to set the version string 6 | VERSION := $(shell git describe --tags --always --dirty) 7 | OS := $(shell go env GOOS) 8 | ARCH := $(shell go env GOARCH) 9 | TAG := $(shell git tag | tail -1) 10 | PWD := $(shell pwd) 11 | 12 | 13 | build: 14 | go build -o dist/$(BIN)_$(OS)_$(ARCH)-$(VERSION) $(SRC_DIR) 15 | 16 | build-all: 17 | env GOOS=darwin GOARCH=amd64 go build -o dist/$(BIN)_darwin_amd64-$(VERSION) $(SRC_DIR) 18 | env GOOS=linux GOARCH=amd64 go build -o dist/$(BIN)_linux_amd64-$(VERSION) $(SRC_DIR) 19 | env GOOS=windows GOARCH=amd64 go build -o dist/$(BIN)_windows_amd64-$(VERSION).exe $(SRC_DIR) 20 | 21 | # For manually releasing when Drone Cloud is having issues 22 | manual-release: build-all 23 | docker run --rm \ 24 | -e DRONE_BUILD_EVENT=tag \ 25 | -e DRONE_REPO_OWNER=wbh1 \ 26 | -e DRONE_REPO_NAME='grafana-sqlite-to-postgres' \ 27 | -e DRONE_COMMIT_REF="refs/tags/$(TAG)" \ 28 | -e PLUGIN_TITLE="$(TAG)" \ 29 | -e PLUGIN_API_KEY="${GITHUB_TOKEN}" \ 30 | -e PLUGIN_FILES='dist/*' \ 31 | -e PLUGIN_OVERWRITE='true' \ 32 | -e DRONE_REPO_LINK='https://github.com/wbh1/grafana-sqlite-to-postgres' \ 33 | -v "$(PWD):$(PWD)" \ 34 | -w "$(PWD)" \ 35 | plugins/github-release 36 | 37 | clean: 38 | rm dist/* 39 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Grafana SQLite to Postgres Database Migrator 2 | 3 | [![Go Report Card](https://goreportcard.com/badge/github.com/wbh1/grafana-sqlite-to-postgres)](https://goreportcard.com/report/github.com/wbh1/grafana-sqlite-to-postgres) 4 | 5 | # ⚠️ Unmaintained Warning ⚠️ 6 | WARNING: This project is currently maintained at a "best-effort" level. I no longer have the free time to keep up with changes to Grafana's database schema and handle/test edge cases. 7 | 8 | One day, perhaps this can be rewritten to be more flexible, but it is not now. I'll continue to try to accept PRs as needed. This project started out of necessity and a desire to make a difficult move simpler for others. However, it's not something that I've personally had to use for years, so it's difficult to justify continuing to put in effort to this. 9 | 10 | Use the project at your own risk (always back up your database [and test the backup!!]). 11 | ## Background 12 | [My blog post](https://wbhegedus.me/migrating-grafanas-database-from-sqlite-to-postgres/) 13 | 14 | I ran into the issue of Grafana logging users out because the SQLite database was locked and tokens couldn't be looked up. This is discussed in [#10727 on grafana/grafana](https://github.com/grafana/grafana/issues/10727#issuecomment-479378941). The solution is to migrate to a database that isn't the default one of SQLite. 15 | 16 | ## Prerequisites 17 | You **must** already have an existing database in Postgres for Grafana. 18 | 19 | Run `CREATE DATABASE grafana` in `psql` to make the database. Then, start up an instance of Grafana pointed to the new database. Grafana will automagically create all the tables that it will need. You can shut Grafana down once those tables are made. We **need** those tables to exist for the migration to work. 20 | 21 | ## Compatability 22 | Tested on: 23 | 24 | | OS | SQLite Version | Postgres Version | Grafana Version | 25 | | -------------- | -------------- | ---------------- | --------------- | 26 | | MacOS | 3.24.0 | 11.3 | 6.1.0+ | 27 | | CentOS 7/RHEL7 | 3.7.17 | 11.3 | 6.1.0+ | 28 | | Fedora 36 | 3.36.0 | 15.0 | 9.2.0 | 29 | 30 | ## Usage 31 | ``` 32 | usage: Grafana SQLite to Postgres Migrator [] 33 | 34 | A command-line application to migrate Grafana data from SQLite to Postgres. 35 | 36 | Flags: 37 | --help Show context-sensitive help (also try --help-long and --help-man). 38 | --dump=/tmp Directory path where the sqlite dump should be stored. 39 | 40 | Args: 41 | Path to SQLite file being imported. 42 | URL-format database connection string to use in the URL format (postgres://USERNAME:PASSWORD@HOST/DATABASE). 43 | ``` 44 | ### Use as Docker image 45 | 1. Build docker image: `docker build -t grafana-sqlite-to-postgres .` 46 | 2. Run migration: `docker run --rm -ti -v :/grafana.db grafana-sqlite-to-postgres /grafana.db "postgres://:@:5432/?sslmode=disable"` 47 | 48 | ## Example Command 49 | This is the command I used to transfer my Grafana database: 50 | ``` 51 | ./grafana-migrate grafana.db "postgres://postgres:PASSWORDHERE@localhost:5432/grafana?sslmode=disable" 52 | ``` 53 | Notice the `?sslmode=disable` parameter. The [pq](https://github.com/lib/pq) driver has sslmode turned on by default, so you may need to add a parameter to adjust it. You can see all the support connection string parameters [here](https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters). 54 | 55 | ## How it works 56 | 1. Dumps SQLite database to /tmp 57 | 2. Sanitize the dump so it can be imported to Postgres 58 | 3. Import the dump to the Grafana database 59 | 60 | ## Acknowledgments 61 | Inspiration for this program was taken from 62 | - [haron/grafana-migrator](https://github.com/haron/grafana-migrator) 63 | - [This blog post](https://0x63.me/migrating-grafana-from-sqlite-to-postgresql/) 64 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | # All Vagrant configuration is done below. The "2" in Vagrant.configure 5 | # configures the configuration version (we support older styles for 6 | # backwards compatibility). Please don't change it unless you know what 7 | # you're doing. 8 | Vagrant.configure("2") do |config| 9 | # The most common configuration options are documented and commented below. 10 | # For a complete reference, please see the online documentation at 11 | # https://docs.vagrantup.com. 12 | 13 | # Every Vagrant development environment requires a box. You can search for 14 | # boxes at https://vagrantcloud.com/search. 15 | config.vm.box = "centos/7" 16 | 17 | # Disable automatic box update checking. If you disable this, then 18 | # boxes will only be checked for updates when the user runs 19 | # `vagrant box outdated`. This is not recommended. 20 | # config.vm.box_check_update = false 21 | 22 | # Create a forwarded port mapping which allows access to a specific port 23 | # within the machine from a port on the host machine. In the example below, 24 | # accessing "localhost:8080" will access port 80 on the guest machine. 25 | # NOTE: This will enable public access to the opened port 26 | # config.vm.network "forwarded_port", guest: 80, host: 8080 27 | 28 | # Create a forwarded port mapping which allows access to a specific port 29 | # within the machine from a port on the host machine and only allow access 30 | # via 127.0.0.1 to disable public access 31 | config.vm.network "forwarded_port", guest: 3000, host: 3001, host_ip: "127.0.0.1" 32 | 33 | # Create a private network, which allows host-only access to the machine 34 | # using a specific IP. 35 | # config.vm.network "private_network", ip: "192.168.33.10" 36 | 37 | # Create a public network, which generally matched to bridged network. 38 | # Bridged networks make the machine appear as another physical device on 39 | # your network. 40 | # config.vm.network "public_network" 41 | 42 | # Share an additional folder to the guest VM. The first argument is 43 | # the path on the host to the actual folder. The second argument is 44 | # the path on the guest to mount the folder. And the optional third 45 | # argument is a set of non-required options. 46 | # config.vm.synced_folder "../data", "/vagrant_data" 47 | 48 | # Provider-specific configuration so you can fine-tune various 49 | # backing providers for Vagrant. These expose provider-specific options. 50 | # Example for VirtualBox: 51 | # 52 | # config.vm.provider "virtualbox" do |vb| 53 | # # Display the VirtualBox GUI when booting the machine 54 | # vb.gui = true 55 | # 56 | # # Customize the amount of memory on the VM: 57 | # vb.memory = "1024" 58 | # end 59 | # 60 | # View the documentation for the provider you are using for more 61 | # information on available options. 62 | 63 | # Enable provisioning with a shell script. Additional provisioners such as 64 | # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the 65 | # documentation for more information about their specific syntax and use. 66 | config.vm.provision "shell", inline: <<-SHELL 67 | yum check-update 68 | curl -fsSL https://get.docker.com/ | sh 69 | sudo systemctl start docker 70 | sudo docker run --rm --network host --name grafana-postgres -e POSTGRES_PASSWORD=testing -d postgres:alpine 71 | sleep 10 72 | sudo docker exec -it grafana-postgres psql -U postgres postgres -c 'create database grafana;' 73 | 74 | sudo docker run -d --network host --rm --name=grafana -e GF_DATABASE_URL=postgres://postgres:testing@localhost/grafana grafana/grafana:latest 75 | sudo docker stop grafana 76 | 77 | curl -O https://dl.google.com/go/go1.13.6.linux-amd64.tar.gz 78 | tar -C /usr/local -xzvf go1.13.6.linux-amd64.tar.gz 79 | SHELL 80 | end 81 | -------------------------------------------------------------------------------- /cmd/grafana-migrate/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/wbh1/grafana-sqlite-to-postgres/pkg/postgresql" 7 | "github.com/wbh1/grafana-sqlite-to-postgres/pkg/sqlite" 8 | 9 | "github.com/sirupsen/logrus" 10 | "gopkg.in/alecthomas/kingpin.v2" 11 | ) 12 | 13 | var ( 14 | log = logrus.New() 15 | app = kingpin.New("Grafana SQLite to Postgres Migrator", "A command-line application to migrate Grafana data from SQLite to Postgres.") 16 | dump = app.Flag("dump", "Directory path where the sqlite dump should be stored.").Default("/tmp").ExistingDir() 17 | sqlitefile = app.Arg("sqlite-file", "Path to SQLite file being imported.").Required().File() 18 | connstring = app.Arg("postgres-connection-string", "URL-format database connection string to use in the URL format (postgres://USERNAME:PASSWORD@HOST/DATABASE).").Required().String() 19 | debug = app.Flag("debug", "Enable debug level logging").Bool() 20 | ) 21 | 22 | func main() { 23 | 24 | kingpin.MustParse(app.Parse(os.Args[1:])) 25 | log.SetFormatter(&logrus.TextFormatter{ 26 | DisableLevelTruncation: true, 27 | FullTimestamp: true, 28 | }) 29 | 30 | if *debug == true { 31 | log.SetLevel(logrus.DebugLevel) 32 | } 33 | 34 | dumpPath := *dump + "/grafana.sql" 35 | 36 | // Must dereference 37 | f := *sqlitefile 38 | log.Infof("📁 SQLlite file: %v", f.Name()) 39 | log.Infof("📁 Dump directory: %v", *dump) 40 | 41 | // Make sure SQLite exists on machine 42 | if err := sqlite.Exists(); err != nil { 43 | log.Fatalf("❌ %v - is the sqlite3 command line tool installed?", err) 44 | } 45 | log.Infof("✅ sqlite3 command exists") 46 | 47 | // Dump the SQLite database 48 | if err := sqlite.Dump(f.Name(), dumpPath); err != nil { 49 | log.Fatalf("❌ %v - failed to dump database.", err) 50 | } 51 | log.Infof("✅ sqlite3 database dumped to %v", dumpPath) 52 | 53 | // Remove CREATE statements 54 | if err := sqlite.RemoveCreateStatements(dumpPath); err != nil { 55 | log.Fatalf("❌ %v - failed to remove CREATE statements from dump file.", err) 56 | } 57 | log.Infoln("✅ CREATE statements removed from dump file") 58 | 59 | // Sanitize the SQLite dump 60 | if err := sqlite.Sanitize(dumpPath); err != nil { 61 | log.Fatalf("❌ %v - failed to sanitize dump file.", err) 62 | } 63 | log.Infoln("✅ sqlite3 dump sanitized") 64 | 65 | // Don't bother adding anything to the migration_log table. 66 | if err := sqlite.CustomSanitize(dumpPath, `(?msU)[\r\n]+^.*"migration_log.*;$`, nil); err != nil { 67 | log.Fatalf("❌ %v - failed to perform additional sanitizing of the dump file.", err) 68 | } 69 | log.Infoln("✅ migration_log statements removed") 70 | // Fix char conversion (char -> chr) 71 | if err := sqlite.CustomSanitize(dumpPath, `char\(10\)\)`, []byte("chr(10))")); err != nil { 72 | log.Fatalf("❌ %v - failed to perform char (LF) keyword sanitizing of the dump file.", err) 73 | } 74 | if err := sqlite.CustomSanitize(dumpPath, `char\(13\)\)`, []byte("chr(13))")); err != nil { 75 | log.Fatalf("❌ %v - failed to perform char (CR) keyword sanitizing of the dump file.", err) 76 | } 77 | log.Infoln("✅ char keyword transformed") 78 | 79 | // Do HexDecoding 80 | if err := sqlite.HexDecode(dumpPath); err != nil { 81 | log.Fatalf("❌ %v - failed to wrap hex-encoded values in the dump file.", err) 82 | } 83 | log.Infoln("✅ hex-encoded data values wrapped for insertion") 84 | 85 | // Connect to Postgres 86 | db, err := postgresql.New(*connstring, log) 87 | if err != nil { 88 | log.Fatalf("❌ %v - failed to connect to Postgres database.", err) 89 | } 90 | 91 | // Import the now-sanitized dump file into Postgres 92 | if err := db.ImportDump(dumpPath); err != nil { 93 | log.Fatalf("❌ %v - failed to import dump file to Postgres.", err) 94 | } 95 | log.Infoln("✅ Imported dump file to Postgres") 96 | log.Infoln("🎉 All done!") 97 | 98 | } 99 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2.4" 2 | name: grafana-migrator 3 | services: 4 | grafana-sqlite: 5 | networks: 6 | - grafananet 7 | image: grafana/grafana:latest 8 | ports: 9 | - "3001:3000" 10 | volumes: 11 | - ./test/grafana/provisioning:/etc/grafana/provisioning:ro 12 | - grafana_sqlite:/var/lib/grafana 13 | env_file: 14 | - ./test/grafana/setup.env 15 | healthcheck: 16 | test: wget --spider http://localhost:3000/robots.txt 17 | interval: 5s 18 | timeout: 3s 19 | retries: 3 20 | depends_on: 21 | - prometheus 22 | 23 | sqlite-to-postgres: 24 | command: /grafanadb/grafana.db "postgres://postgres:postgres@postgres_grafana:5432/grafana?sslmode=disable" 25 | networks: 26 | - grafananet 27 | image: grafana-sqlite-to-postgres 28 | volumes: 29 | - grafana_sqlite:/grafanadb:ro 30 | build: ./ 31 | depends_on: 32 | postgres: 33 | condition: service_healthy 34 | grafana-sqlite: 35 | condition: service_healthy 36 | 37 | grafana-postgres: 38 | networks: 39 | - grafananet 40 | image: grafana/grafana:latest 41 | ports: 42 | - "3000:3000" 43 | environment: 44 | GF_DATABASE_URL: "postgres://postgres:postgres@postgres/grafana" 45 | env_file: 46 | - ./test/grafana/setup.env 47 | healthcheck: 48 | test: wget --spider http://localhost:3000/robots.txt 49 | interval: 5s 50 | timeout: 3s 51 | retries: 3 52 | depends_on: 53 | postgres: 54 | condition: service_healthy 55 | 56 | prometheus: 57 | user: root 58 | networks: 59 | - grafananet 60 | image: prom/prometheus:latest 61 | volumes: 62 | - ./test/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro 63 | - /var/run/docker.sock:/var/run/docker.sock 64 | ports: 65 | - "9090:9090" 66 | 67 | postgres: 68 | networks: 69 | - grafananet 70 | image: postgres:alpine 71 | ports: 72 | - "5432:5432" 73 | environment: 74 | POSTGRES_DB: grafana 75 | POSTGRES_USER: postgres 76 | POSTGRES_PASSWORD: postgres 77 | healthcheck: 78 | test: ["CMD", "psql", "-U", "postgres", "grafana", "-c", "select 1;"] 79 | interval: 5s 80 | timeout: 3s 81 | retries: 3 82 | networks: 83 | grafananet: 84 | volumes: 85 | grafana_sqlite: 86 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/wbh1/grafana-sqlite-to-postgres 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/lib/pq v1.10.7 7 | github.com/sirupsen/logrus v1.9.0 8 | gopkg.in/alecthomas/kingpin.v2 v2.2.6 9 | ) 10 | 11 | require ( 12 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect 13 | github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect 14 | golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43 // indirect 15 | ) 16 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= 2 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 3 | github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= 4 | github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= 5 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= 9 | github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 10 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 11 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 12 | github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= 13 | github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 14 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 15 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 16 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 17 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 18 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 19 | golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43 h1:OK7RB6t2WQX54srQQYSXMW8dF5C6/8+oA/s5QBmmto4= 20 | golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 21 | gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= 22 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 23 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 24 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 25 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 26 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 27 | -------------------------------------------------------------------------------- /pkg/postgresql/import.go: -------------------------------------------------------------------------------- 1 | package postgresql 2 | 3 | import ( 4 | "bufio" 5 | "database/sql" 6 | "fmt" 7 | "io/ioutil" 8 | "os" 9 | "strings" 10 | 11 | // Postgres driver 12 | _ "github.com/lib/pq" 13 | "github.com/sirupsen/logrus" 14 | ) 15 | 16 | // DB allows for interface methods. 17 | // It just holds a connection pointer. 18 | type DB struct { 19 | conn *sql.DB 20 | log *logrus.Logger 21 | } 22 | 23 | // New returns a Postgres database connection. 24 | func New(connString string, logger *logrus.Logger) (db DB, err error) { 25 | db.log = logger 26 | db.conn, err = sql.Open("postgres", connString) 27 | if err != nil { 28 | return 29 | } 30 | _, err = db.conn.Exec("SELECT 1") 31 | return 32 | } 33 | 34 | // ImportDump imports a SQL dump file. 35 | func (db *DB) ImportDump(dumpFile string) error { 36 | 37 | promptToContinue := func() bool { 38 | reader := bufio.NewReader(os.Stdin) 39 | fmt.Print("You seem to have encountered some errors. Would you still like to continue? [Y/n]: ") 40 | text, _ := reader.ReadString('\n') 41 | switch response := strings.ToLower(text); response { 42 | case "n\n": 43 | return false 44 | default: 45 | return true 46 | } 47 | } 48 | 49 | // Alter tables because of boolean issues 50 | // SQLite has booleans as 1's and 0's 51 | // Postgres is true/false 52 | // We'll convert it after importing the dump. 53 | if errorEncountered := db.prepareTables(); errorEncountered == true { 54 | if promptToContinue() != true { 55 | return fmt.Errorf("%s", "Stopping migration at user's request.") 56 | } 57 | } 58 | 59 | file, err := ioutil.ReadFile(dumpFile) 60 | if err != nil { 61 | return err 62 | } 63 | 64 | sqlStmts := strings.Split(string(file), ";\n") 65 | 66 | for _, stmt := range sqlStmts { 67 | if _, err := db.conn.Exec(stmt); err != nil { 68 | // We can safely ignore "duplicate key value violates unique constraint" errors. 69 | if strings.Contains(err.Error(), "duplicate key") { 70 | continue 71 | } else if strings.Contains(err.Error(), "is of type bytea but expression is of type text") { 72 | // TODO(wbh1): This is absolutely horrible and I am ashamed of this code. Should figure out column types ahead of time. 73 | db.log.Debugf("Failed to import because of type issue (%v). Trying to fix...\n", err.Error()) 74 | stmt = strings.Replace( 75 | strings.Replace(stmt, `,convert_from('\x`, ",decode('", 1), 76 | "'utf-8'", "'hex'", 1) 77 | if _, err := db.conn.Exec(stmt); err != nil { 78 | return fmt.Errorf("%v %v", err.Error(), stmt) 79 | } 80 | } else { 81 | return fmt.Errorf("%v %v", err.Error(), stmt) 82 | } 83 | } 84 | } 85 | 86 | // Fix boolean columns that we converted before. 87 | if errorEncountered := db.decodeBooleanColumns(); errorEncountered == true { 88 | if promptToContinue() != true { 89 | return fmt.Errorf("%s", "Stopping migration at user's request.") 90 | } 91 | } 92 | 93 | // Fix sequences for new items. 94 | if err := db.fixSequences(); err != nil { 95 | return err 96 | } 97 | 98 | return nil 99 | 100 | } 101 | 102 | // Change column types that expect boolean to integer so that we can get the data in. 103 | // We'll decode their values into booleans later. 104 | func (db *DB) prepareTables() (errorEncountered bool) { 105 | for _, table := range TableChanges { 106 | // for each column associated with the table, 107 | // update the column type to be integer so that it's compatible with sqlite's 0/1 bool values 108 | for _, column := range table.Columns { 109 | // If the column has a default value associated with it, drop it. 110 | if column.Default != "" { 111 | stmt := fmt.Sprintf("ALTER TABLE %s ALTER COLUMN %s DROP DEFAULT", table.Table, column.Name) 112 | db.log.Debugln("Executing: ", stmt) 113 | if _, err := db.conn.Exec(stmt); err != nil { 114 | if strings.Contains(err.Error(), "does not exist") { 115 | db.log.Debugf("%s %v %v", "Column/table doesn't exist. This is usually fine to ignore, but here's the info:", err.Error(), stmt) 116 | } else { 117 | db.log.Warnf("%v %v", err.Error(), stmt) 118 | errorEncountered = true 119 | } 120 | } 121 | } 122 | 123 | stmt := fmt.Sprintf("ALTER TABLE %s ALTER COLUMN %s TYPE integer USING %s::integer", table.Table, column.Name, column.Name) 124 | db.log.Debugln("Executing: ", stmt) 125 | if _, err := db.conn.Exec(stmt); err != nil { 126 | if strings.Contains(err.Error(), "does not exist") { 127 | db.log.Debugf("%s %v %v", "Column/table doesn't exist. This is usually fine to ignore, but here's the info:", err.Error(), stmt) 128 | } else { 129 | db.log.Warnf("%v %v", err.Error(), stmt) 130 | errorEncountered = true 131 | } 132 | } 133 | 134 | } 135 | } 136 | 137 | // Delete the org that gets auto-generated the first time Grafana runs. 138 | stmt := "DELETE FROM org WHERE id=1" 139 | db.log.Debugln("Executing: ", stmt) 140 | if _, err := db.conn.Exec(stmt); err != nil { 141 | db.log.Errorf("%v %v", err.Error(), stmt) 142 | errorEncountered = true 143 | } 144 | 145 | return 146 | } 147 | 148 | // Change columns back to boolean type by decoding their current values 149 | func (db *DB) decodeBooleanColumns() bool { 150 | 151 | var errorEncountered bool 152 | 153 | for _, table := range TableChanges { 154 | for _, column := range table.Columns { 155 | stmt := fmt.Sprintf("ALTER TABLE %s ALTER COLUMN %s TYPE boolean USING CASE WHEN %s = 0 THEN FALSE WHEN %s = 1 THEN TRUE ELSE NULL END", table.Table, column.Name, column.Name, column.Name) 156 | db.log.Debugln("Executing: ", stmt) 157 | if _, err := db.conn.Exec(stmt); err != nil { 158 | if strings.Contains(err.Error(), "does not exist") { 159 | db.log.Debugf("%s %v %v", "Column/table doesn't exist. This is usually fine to ignore, but here's the info:", err.Error(), stmt) 160 | } else { 161 | db.log.Warnf("%v %v", err.Error(), stmt) 162 | errorEncountered = true 163 | } 164 | } 165 | 166 | // If the column has a default value associated with it, drop it. 167 | if column.Default != "" { 168 | stmt = fmt.Sprintf("ALTER TABLE %s ALTER COLUMN %s SET DEFAULT %s", table.Table, column.Name, column.Default) 169 | db.log.Debugln("Executing: ", stmt) 170 | if _, err := db.conn.Exec(stmt); err != nil { 171 | if strings.Contains(err.Error(), "does not exist") { 172 | db.log.Debugf("%s %v %v", "Column/table doesn't exist. This is usually fine to ignore, but here's the info:", err.Error(), stmt) 173 | } else { 174 | db.log.Warnf("%v %v", err.Error(), stmt) 175 | errorEncountered = true 176 | } 177 | } 178 | } 179 | 180 | } // end column loop 181 | } // end table loop 182 | 183 | return errorEncountered 184 | } 185 | 186 | // Make sure that sequences are fine on the tables 187 | func (db *DB) fixSequences() error { 188 | 189 | // Query from https://wiki.postgresql.org/wiki/Fixing_Sequences 190 | stmt := `SELECT 'SELECT SETVAL(' || 191 | quote_literal(quote_ident(PGT.schemaname) || '.' || quote_ident(S.relname)) || 192 | ', COALESCE(MAX(' ||quote_ident(C.attname)|| '), 1) ) FROM ' || 193 | quote_ident(PGT.schemaname)|| '.'||quote_ident(T.relname)|| ';' stmt 194 | FROM pg_class AS S, 195 | pg_depend AS D, 196 | pg_class AS T, 197 | pg_attribute AS C, 198 | pg_tables AS PGT 199 | WHERE S.relkind = 'S' 200 | AND S.oid = D.objid 201 | AND D.refobjid = T.oid 202 | AND D.refobjid = C.attrelid 203 | AND D.refobjsubid = C.attnum 204 | AND T.relname = PGT.tablename 205 | ORDER BY S.relname;` 206 | 207 | db.log.Debugln("Running query to generate statements to reset all sequences.") 208 | rows, err := db.conn.Query(stmt) 209 | if err != nil { 210 | return fmt.Errorf("%v %v", err.Error(), stmt) 211 | } 212 | defer rows.Close() 213 | 214 | db.log.Debugln("Running generated queries to reset all sequences.") 215 | for rows.Next() { 216 | var stmt string 217 | if err := rows.Scan(&stmt); err != nil { 218 | return fmt.Errorf("%v %v", "Failed to retrieve sequence reset statement", err) 219 | } 220 | 221 | // Execute the generate statement 222 | if _, err := db.conn.Exec(stmt); err != nil { 223 | return fmt.Errorf("%v %v", err.Error(), stmt) 224 | } 225 | } 226 | 227 | return nil 228 | 229 | } 230 | -------------------------------------------------------------------------------- /pkg/postgresql/tablechanges.go: -------------------------------------------------------------------------------- 1 | package postgresql 2 | 3 | type Column struct { 4 | // Name of the column for the table 5 | Name string 6 | 7 | // If the column gets a default value set, specify it here 8 | Default string 9 | } 10 | 11 | // TableChange documents a table that needs to be changed 12 | // and specificly which Columns need to be changed. 13 | type TableChange struct { 14 | Table string 15 | Columns []Column 16 | } 17 | 18 | var TableChanges = []TableChange{ 19 | { 20 | Table: "alert", 21 | Columns: []Column{ 22 | { 23 | Name: "silenced", 24 | }, 25 | }, 26 | }, 27 | { 28 | Table: "alert_configuration", 29 | Columns: []Column{ 30 | { 31 | Name: "\"default\"", 32 | Default: "false", 33 | }, 34 | }, 35 | }, 36 | { 37 | Table: "alert_configuration_history", 38 | Columns: []Column{ 39 | { 40 | Name: "\"default\"", 41 | Default: "false", 42 | }, 43 | }, 44 | }, 45 | { 46 | Table: "alert_rule", 47 | Columns: []Column{ 48 | { 49 | Name: "is_paused", 50 | Default: "false", 51 | }, 52 | }, 53 | }, 54 | { 55 | Table: "alert_rule_version", 56 | Columns: []Column{ 57 | { 58 | Name: "is_paused", 59 | Default: "false", 60 | }, 61 | }, 62 | }, 63 | { 64 | Table: "alert_notification", 65 | Columns: []Column{ 66 | { 67 | Name: "is_default", 68 | Default: "false", 69 | }, 70 | { 71 | Name: "send_reminder", 72 | Default: "false", 73 | }, 74 | { 75 | Name: "disable_resolve_message", 76 | Default: "false", 77 | }, 78 | }, 79 | }, 80 | { 81 | Table: "dashboard", 82 | Columns: []Column{ 83 | { 84 | Name: "is_folder", 85 | Default: "false", 86 | }, 87 | { 88 | Name: "has_acl", 89 | Default: "false", 90 | }, 91 | { 92 | Name: "is_public", 93 | Default: "false", 94 | }, 95 | }, 96 | }, 97 | { 98 | Table: "dashboard_snapshot", 99 | Columns: []Column{ 100 | { 101 | Name: "external", 102 | }, 103 | }, 104 | }, 105 | { 106 | Table: "data_source", 107 | Columns: []Column{ 108 | { 109 | Name: "basic_auth", 110 | }, 111 | { 112 | Name: "is_default", 113 | }, 114 | { 115 | Name: "read_only", 116 | }, 117 | { 118 | Name: "with_credentials", 119 | Default: "false", 120 | }, 121 | }, 122 | }, 123 | { 124 | Table: "migration_log", 125 | Columns: []Column{ 126 | { 127 | Name: "success", 128 | }, 129 | }, 130 | }, 131 | { 132 | Table: "plugin_setting", 133 | Columns: []Column{ 134 | { 135 | Name: "enabled", 136 | }, 137 | { 138 | Name: "pinned", 139 | }, 140 | }, 141 | }, 142 | { 143 | Table: "team_member", 144 | Columns: []Column{ 145 | { 146 | Name: "external", 147 | }, 148 | }, 149 | }, 150 | { 151 | Table: "temp_user", 152 | Columns: []Column{ 153 | { 154 | Name: "email_sent", 155 | }, 156 | }, 157 | }, 158 | { 159 | Table: "\"user\"", 160 | Columns: []Column{ 161 | { 162 | Name: "is_admin", 163 | }, 164 | { 165 | Name: "email_verified", 166 | }, 167 | { 168 | Name: "is_disabled", 169 | Default: "false", 170 | }, 171 | { 172 | Name: "is_service_account", 173 | Default: "false", 174 | }, 175 | }, 176 | }, 177 | { 178 | Table: "user_auth_token", 179 | Columns: []Column{ 180 | { 181 | Name: "auth_token_seen", 182 | }, 183 | }, 184 | }, 185 | { 186 | Table: "role", 187 | Columns: []Column{ 188 | { 189 | Name: "hidden", 190 | Default: "false", 191 | }, 192 | }, 193 | }, 194 | { 195 | Table: "data_keys", 196 | Columns: []Column{ 197 | { 198 | Name: "active", 199 | }, 200 | }, 201 | }, 202 | { 203 | Table: "api_key", 204 | Columns: []Column{ 205 | { 206 | Name: "is_revoked", 207 | Default: "false", 208 | }, 209 | }, 210 | }, 211 | } 212 | -------------------------------------------------------------------------------- /pkg/sqlite/dump.go: -------------------------------------------------------------------------------- 1 | package sqlite 2 | 3 | import ( 4 | "io" 5 | "os/exec" 6 | ) 7 | 8 | // Exists checks if `sqlite3 --version` returns without errors 9 | func Exists() error { 10 | // Check to make sure sqlite3 exists 11 | cmd := exec.Command("sqlite3", "--version") 12 | return cmd.Run() 13 | } 14 | 15 | // Dump performs a full dump of the SQLite database 16 | func Dump(dbFile string, destination string) error { 17 | 18 | cmd := exec.Command("sqlite3", dbFile) 19 | 20 | // Interact with sqlite3 command line tool by sending data to STDIN 21 | stdin, _ := cmd.StdinPipe() 22 | go func() { 23 | defer stdin.Close() 24 | io.WriteString(stdin, ".output "+destination+"\n") 25 | io.WriteString(stdin, ".dump\n") 26 | io.WriteString(stdin, ".quit\n") 27 | }() 28 | 29 | _, err := cmd.CombinedOutput() 30 | 31 | return err 32 | } 33 | -------------------------------------------------------------------------------- /pkg/sqlite/sanitize.go: -------------------------------------------------------------------------------- 1 | package sqlite 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "regexp" 7 | ) 8 | 9 | // Sanitize cleans up a SQLite dump file to prep it for import into Postgres. 10 | func Sanitize(dumpFile string) error { 11 | // Change ` to " 12 | re := regexp.MustCompile("`") 13 | data, err := ioutil.ReadFile(dumpFile) 14 | if err != nil { 15 | return err 16 | } 17 | sanitized := re.ReplaceAll(data, []byte("\"")) 18 | 19 | // Remove SQLite-specific PRAGMA statements 20 | // and statements that start with BEGIN 21 | // and statements pertaining to the sqlite_sequence table. 22 | re = regexp.MustCompile(`(?m)[\r\n]?^(PRAGMA.*;|BEGIN.*;|.*sqlite_sequence.*;)$`) 23 | sanitized = re.ReplaceAll(sanitized, nil) 24 | 25 | // Ensure there are quotes around table names to avoid using reserved table names like user. 26 | re = regexp.MustCompile(`(?msU)^(INSERT INTO) "?([a-zA-Z0-9_]*)"? (VALUES.*;)$`) 27 | sanitized = re.ReplaceAll(sanitized, []byte(`$1 "$2" $3`)) 28 | 29 | return ioutil.WriteFile(dumpFile, sanitized, 0644) 30 | } 31 | 32 | // CustomSanitize allows you to expand upon the default Sanitize function 33 | // by providing your own regex matcher and replacement to modify data from the dump file. 34 | func CustomSanitize(dumpFile string, regex string, replacement []byte) error { 35 | re := regexp.MustCompile(regex) 36 | data, err := ioutil.ReadFile(dumpFile) 37 | if err != nil { 38 | return err 39 | } 40 | 41 | sanitized := re.ReplaceAll(data, replacement) 42 | 43 | return ioutil.WriteFile(dumpFile, sanitized, 0644) 44 | 45 | } 46 | 47 | // RemoveCreateStatements takes all the CREATE statements out of a dump 48 | // so that no new tables are created. 49 | func RemoveCreateStatements(dumpFile string) error { 50 | re := regexp.MustCompile(`(?msU)[\r\n]+^CREATE.*;$`) 51 | data, err := ioutil.ReadFile(dumpFile) 52 | if err != nil { 53 | return err 54 | } 55 | sanitized := re.ReplaceAll(data, nil) 56 | return ioutil.WriteFile(dumpFile, sanitized, 0644) 57 | } 58 | 59 | // HexDecode takes a file path containing a SQLite dump and 60 | // decodes any hex-encoded data it finds. 61 | func HexDecode(dumpFile string) error { 62 | re := regexp.MustCompile(`(?m)X\'([a-fA-F0-9]+)\'`) 63 | data, err := ioutil.ReadFile(dumpFile) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | // Define a function to wrap encoded hex data in a call to decode hexstring. 69 | decodeHex := func(hexEncoded []byte) []byte { 70 | return []byte(fmt.Sprintf("convert_from('%s%s', 'utf-8')", `\x`, re.FindSubmatch(hexEncoded)[1])) 71 | } 72 | 73 | // Replace regex matches from the dumpFile using the `decodeHex` function defined above. 74 | sanitized := re.ReplaceAllFunc(data, decodeHex) 75 | return ioutil.WriteFile(dumpFile, sanitized, 0644) 76 | } 77 | -------------------------------------------------------------------------------- /test/grafana/provisioning/datasources/prometheus.yaml: -------------------------------------------------------------------------------- 1 | # config file version 2 | apiVersion: 1 3 | 4 | # list of datasources that should be deleted from the database 5 | deleteDatasources: 6 | - name: Prometheus 7 | orgId: 1 8 | 9 | # list of datasources to insert/update depending 10 | # what's available in the database 11 | datasources: 12 | # name of the datasource. Required 13 | - name: Prometheus 14 | # datasource type. Required 15 | type: prometheus 16 | # access mode. proxy or direct (Server or Browser in the UI). Required 17 | access: proxy 18 | # org id. will default to orgId 1 if not specified 19 | orgId: 1 20 | # custom UID which can be used to reference this datasource in other parts of the configuration, if not specified will be generated automatically 21 | uid: prometheus_default 22 | # url 23 | url: http://prometheus:9090 24 | # database user, if used 25 | user: 26 | # database name, if used 27 | database: 28 | # enable/disable basic auth 29 | basicAuth: 30 | # basic auth username 31 | basicAuthUser: 32 | # enable/disable with credentials headers 33 | withCredentials: 34 | # mark as default datasource. Max one per org 35 | isDefault: true 36 | version: 1 37 | # allow users to edit datasources from the UI. 38 | editable: false 39 | -------------------------------------------------------------------------------- /test/grafana/setup.env: -------------------------------------------------------------------------------- 1 | GF_SECURITY_ADMIN_USER=admin 2 | GF_SECURITY_ADMIN_PASSWORD=grafana123 3 | -------------------------------------------------------------------------------- /test/prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | scrape_configs: 2 | - job_name: "docker-containers" 3 | docker_sd_configs: 4 | - host: unix:///var/run/docker.sock # You can also use http/https to connect to the Docker daemon. 5 | --------------------------------------------------------------------------------