├── cli ├── redis │ └── cli.go ├── zookeeper │ └── cli.go ├── config.go ├── cli.go └── file │ └── cli.go ├── .gitignore ├── store ├── config.go ├── client.go ├── file │ └── client.go ├── redis │ └── client.go └── zookeeper │ └── client.go ├── .travis.yml ├── docs ├── files │ ├── example_1.toml │ ├── example_2.toml │ ├── example.tmpl │ ├── filestore.toml │ └── config.toml └── install.sh ├── processor ├── config.go ├── processor.go └── meta.go ├── utils ├── util.go └── kv.go ├── test └── command.go ├── config ├── cli_config.go └── config.go ├── confd.go ├── README.md └── confd_cli.go /cli/redis/cli.go: -------------------------------------------------------------------------------- 1 | package redis 2 | -------------------------------------------------------------------------------- /cli/zookeeper/cli.go: -------------------------------------------------------------------------------- 1 | package zookeeper 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | pkg/ 3 | release 4 | release.tar.gz 5 | confd 6 | confd-cli -------------------------------------------------------------------------------- /cli/config.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | type CliConfig struct { 4 | Store string 5 | ConnectAddr string 6 | } 7 | -------------------------------------------------------------------------------- /store/config.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | type StoreConfig struct { 4 | Store string 5 | ConnectAddr string 6 | } 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.4 4 | 5 | sudo: required 6 | 7 | script: 8 | - go test ./cli 9 | - go test ./store 10 | -------------------------------------------------------------------------------- /docs/files/example_1.toml: -------------------------------------------------------------------------------- 1 | [metaObject] 2 | tmpl = "example.tmpl" 3 | dest = "/tmp/example_1.properties" 4 | keys = [ 5 | "mysql_host", 6 | "mysql_port", 7 | "mysql_db", 8 | "redis_host", 9 | "redis_port", 10 | "redis_db" 11 | ] 12 | 13 | -------------------------------------------------------------------------------- /docs/files/example_2.toml: -------------------------------------------------------------------------------- 1 | [metaObject] 2 | tmpl = "example.tmpl" 3 | dest = "/tmp/example_2.properties" 4 | keys = [ 5 | "mysql_host", 6 | "mysql_port", 7 | "mysql_db", 8 | "redis_host", 9 | "redis_port", 10 | "redis_db" 11 | ] 12 | 13 | -------------------------------------------------------------------------------- /processor/config.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import "github.com/sumory/confd/store" 4 | 5 | type TemplateConfig struct { 6 | ConfDir string 7 | MetaDir string 8 | TemplateDir string 9 | StoreClient store.StoreClient 10 | } 11 | -------------------------------------------------------------------------------- /docs/files/example.tmpl: -------------------------------------------------------------------------------- 1 | #### mysql配置 2 | jdbc.url=jdbc:mysql://{{get "mysql_host"}}:{{get "mysql_port"}}/{{get "mysql_db"}} 3 | 4 | #### redis配置 5 | redis.hostname={{get "redis_host"}} 6 | redis.port={{get "redis_port"}} 7 | redis.db={{get "redis_db"}} 8 | -------------------------------------------------------------------------------- /docs/files/filestore.toml: -------------------------------------------------------------------------------- 1 | [data] 2 | 3 | ### example data start ##### 4 | 5 | mysql_host = "192.168.100.188" 6 | mysql_port = 3307 7 | mysql_db = "test" 8 | 9 | redis_host = "192.168.100.185" 10 | redis_port = 6379 11 | redis_db = 1 12 | 13 | ### example data end ##### 14 | -------------------------------------------------------------------------------- /docs/files/config.toml: -------------------------------------------------------------------------------- 1 | #数据存储方式，默认使用file，即使用filestore.toml里的数据 2 | Store = "file" 3 | #具体存储方式的地址 4 | ConnectAddr = "/data/confd/data/filestore.toml" 5 | #confd所有配置（程序配置、模板文件、meta文件）默认所在的文件夹 6 | ConfDir = "/data/confd" 7 | #非debug模式下，confd多久主动生成一次配置文件，单位为秒 8 | Interval = 600 9 | #是否debug，debug模式下运行confd之后推出，非debug将在后台运行 10 | Debug = true 11 | 12 | #### redis based ############################# 13 | #Store = "redis" 14 | #ConnectAddr = "192.168.100.185:6379" 15 | #ConfDir = "/data/confd" 16 | #Interval = 600 17 | #Debug = false 18 | ############################################## 19 | 20 | #### zookeeper based ######################### 21 | #Store = "zookeeper" 22 | #ConnectAddr = "192.168.100.185:2181" 23 | #ConfDir = "/data/confd" 24 | #Interval = 600 25 | #Debug = false 26 | ############################################## -------------------------------------------------------------------------------- /utils/util.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | ) 7 | 8 | func IsFileExist(fpath string) bool { 9 | if _, err := os.Stat(fpath); os.IsNotExist(err) { 10 | return false 11 | } 12 | return true 13 | } 14 | 15 | func RecursiveFindFiles(root string, pattern string) ([]string, error) { 16 | files := make([]string, 0) 17 | findfile := func(path string, f os.FileInfo, err error) (inner error) { 18 | if err != nil { 19 | return 20 | } 21 | if f.IsDir() { 22 | return 23 | } else if match, innerr := filepath.Match(pattern, f.Name()); innerr == nil && match { 24 | files = append(files, path) 25 | } 26 | return 27 | } 28 | err := filepath.Walk(root, findfile) 29 | if len(files) == 0 { 30 | return files, err 31 | } else { 32 | return files, err 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /store/client.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "errors" 5 | "github.com/sumory/confd/store/file" 6 | "github.com/sumory/confd/store/redis" 7 | "github.com/sumory/confd/store/zookeeper" 8 | ) 9 | 10 | type StoreClient interface { 11 | GetValues(keys []string) (map[string]interface{}, error) 12 | WatchPrefix(prefix string, waitIndex uint64, stopChan chan bool) (uint64, error) 13 | } 14 | 15 | func New(c *StoreConfig) (StoreClient, error) { 16 | if c.Store == "" { 17 | c.Store = "file" //default store 18 | } 19 | 20 | connectAddr := c.ConnectAddr 21 | switch c.Store { 22 | case "zookeeper": 23 | return zookeeper.NewZookeeperClient(connectAddr) 24 | case "redis": 25 | return redis.NewRedisClient(connectAddr) 26 | case "file": 27 | return file.NewFileClient(connectAddr) 28 | } 29 | 30 | return nil, errors.New("Invalid store...") 31 | } 32 | -------------------------------------------------------------------------------- /cli/cli.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "errors" 5 | "github.com/sumory/confd/cli/file" 6 | // "github.com/sumory/confd/cli/redis" 7 | // "github.com/sumory/confd/cli/zookeeper" 8 | ) 9 | 10 | type Cli interface { 11 | GetAll() (map[string]interface{}, error) 12 | GetValues(keys []string) (map[string]interface{}, error) 13 | GetValue(key string) (interface{}, error) 14 | SetValue(key string, value interface{}) error 15 | DeleteKey(key string) error 16 | } 17 | 18 | func New(c *CliConfig) (error, Cli) { 19 | if c.Store == "" { 20 | c.Store = "file" //default cli 21 | } 22 | 23 | connectAddr := c.ConnectAddr 24 | switch c.Store { 25 | // case "zookeeper": 26 | // return zookeeper.NewZookeeperCli(connectAddr) 27 | // case "redis": 28 | // return redis.NewRedisCli(connectAddr) 29 | case "file": 30 | return file.NewFileCli(connectAddr) 31 | } 32 | 33 | return errors.New("Invalid cli..."), nil 34 | } 35 | -------------------------------------------------------------------------------- /store/file/client.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "strings" 5 | 6 | "fmt" 7 | "github.com/BurntSushi/toml" 8 | ) 9 | 10 | type FileStore struct { 11 | Data map[string]interface{} `toml:"data"` 12 | } 13 | 14 | type Client struct { 15 | FileStore *FileStore 16 | } 17 | 18 | func NewFileClient(filePath string) (*Client, error) { 19 | var fileStore *FileStore 20 | _, err := toml.DecodeFile(filePath, &fileStore) 21 | fmt.Printf("%s %v\n", filePath, fileStore) 22 | 23 | if err != nil { 24 | return nil, fmt.Errorf("Cannot process file store %s - %s", filePath, err.Error()) 25 | } 26 | 27 | return &Client{ 28 | FileStore: fileStore, 29 | }, nil 30 | } 31 | 32 | func (c *Client) GetValues(keys []string) (map[string]interface{}, error) { 33 | vars := make(map[string]interface{}) 34 | for _, key := range keys { 35 | key = strings.Trim(key, " ") 36 | value, ok := c.FileStore.Data[key] 37 | if ok { 38 | vars[key] = value 39 | continue 40 | } 41 | } 42 | return vars, nil 43 | } 44 | 45 | func (c *Client) WatchPrefix(prefix string, waitIndex uint64, stopChan chan bool) (uint64, error) { 46 | <-stopChan 47 | return 0, nil 48 | } 49 | -------------------------------------------------------------------------------- /utils/kv.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "errors" 5 | "sync" 6 | ) 7 | 8 | var ErrNotExist = errors.New("key does not exist") 9 | 10 | type KVStore struct { 11 | data map[string]interface{} 12 | FuncMap map[string]interface{} 13 | sync.RWMutex 14 | } 15 | 16 | func NewKVStore() KVStore { 17 | s := KVStore{data: make(map[string]interface{})} 18 | s.FuncMap = map[string]interface{}{ 19 | "exists": s.Exists, 20 | "get": s.Get, 21 | } 22 | return s 23 | } 24 | 25 | func (s KVStore) Del(key string) { 26 | s.Lock() 27 | delete(s.data, key) 28 | s.Unlock() 29 | } 30 | 31 | func (s KVStore) Exists(key string) bool { 32 | _, err := s.Get(key) 33 | if err != nil { 34 | return false 35 | } 36 | return true 37 | } 38 | 39 | func (s KVStore) Get(key string) (interface{}, error) { 40 | s.RLock() 41 | v, ok := s.data[key] 42 | s.RUnlock() 43 | if !ok { 44 | return v, ErrNotExist 45 | } 46 | return v, nil 47 | } 48 | 49 | func (s KVStore) Set(key string, value interface{}) { 50 | s.Lock() 51 | s.data[key] = value 52 | s.Unlock() 53 | } 54 | 55 | func (s KVStore) Clean() { 56 | s.Lock() 57 | for k := range s.data { 58 | delete(s.data, k) 59 | } 60 | s.Unlock() 61 | } 62 | -------------------------------------------------------------------------------- /store/redis/client.go: -------------------------------------------------------------------------------- 1 | package redis 2 | 3 | import ( 4 | "github.com/garyburd/redigo/redis" 5 | "os" 6 | "strings" 7 | "time" 8 | ) 9 | 10 | type Client struct { 11 | client redis.Conn 12 | } 13 | 14 | func NewRedisClient(address string) (*Client, error) { 15 | var err error 16 | var conn redis.Conn 17 | network := "tcp" 18 | if _, err = os.Stat(address); err == nil { 19 | network = "unix" 20 | } 21 | conn, err = redis.DialTimeout(network, address, time.Second, time.Second, time.Second) 22 | if err != nil { 23 | return nil, err 24 | } 25 | return &Client{conn}, nil 26 | 27 | } 28 | 29 | func (c *Client) GetValues(keys []string) (map[string]interface{}, error) { 30 | vars := make(map[string]interface{}) 31 | for _, key := range keys { 32 | key = strings.Trim(key, " ") 33 | value, err := redis.String(c.client.Do("GET", key)) 34 | if err == nil { 35 | vars[key] = value 36 | continue 37 | } 38 | 39 | if err != redis.ErrNil { 40 | return vars, err 41 | } 42 | } 43 | return vars, nil 44 | } 45 | 46 | // WatchPrefix is not yet implemented. 47 | func (c *Client) WatchPrefix(prefix string, waitIndex uint64, stopChan chan bool) (uint64, error) { 48 | <-stopChan 49 | return 0, nil 50 | } 51 | -------------------------------------------------------------------------------- /docs/install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | #confd config dir 4 | DEFAULT_PATH="$1" 5 | 6 | if [ -n "$DEFAULT_PATH" ];then 7 | echo "use defined config dir: "${DEFAULT_PATH} 8 | else 9 | echo "use default config dir: /data/confd" 10 | DEFAULT_PATH=/data/confd 11 | fi 12 | 13 | mkdir -p $DEFAULT_PATH 14 | 15 | #make a backup for the last configurations 16 | CURRENT_TIME=$(date +%Y%m%d-%H%M%S) 17 | tar zcvf $DEFAULT_PATH".${CURRENT_TIME}.tar.gz" $DEFAULT_PATH 18 | rm -rf $DEFAULT_PATH/* 19 | 20 | 21 | echo "Install confd in default path: $DEFAULT_PATH" 22 | 23 | mkdir -p $DEFAULT_PATH/data 24 | mkdir -p $DEFAULT_PATH/meta 25 | mkdir -p $DEFAULT_PATH/templates 26 | 27 | 28 | cp ./files/config.toml $DEFAULT_PATH/data/ 29 | cp ./files/filestore.toml $DEFAULT_PATH/data/ 30 | cp ./files/example_1.toml $DEFAULT_PATH/meta/ 31 | cp ./files/example_2.toml $DEFAULT_PATH/meta/ 32 | cp ./files/example.tmpl $DEFAULT_PATH/templates/ 33 | 34 | cp ./files/confd $DEFAULT_PATH/ 35 | cp ./files/confd-cli $DEFAULT_PATH/ 36 | 37 | echo "Installed confd" 38 | echo "By default, confd uses a file store, you also can choose redis or zookeeper" 39 | echo 'Now you can:' 40 | echo " 1. cd $DEFAULT_PATH" 41 | echo ' 2. use "./confd --confdir='$DEFAULT_PATH'" to see how confd works.' 42 | echo " 3. check folders[data/meta/templates] under $DEFAULT_PATH to learn how to use confd" -------------------------------------------------------------------------------- /test/command.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/codegangsta/cli" 6 | "os" 7 | ) 8 | 9 | func main() { 10 | app := cli.NewApp() 11 | app.Name = "commander" 12 | app.Usage = "test command args!" 13 | 14 | app.Flags = []cli.Flag{ 15 | cli.StringFlag{ 16 | Name: "name", 17 | Value: "sumory", 18 | Usage: "my name", 19 | }, 20 | cli.StringFlag{ 21 | Name: "age", 22 | Value: "18", 23 | Usage: "my age", 24 | }, 25 | } 26 | 27 | app.Commands = []cli.Command{ 28 | { 29 | Name: "set", 30 | Aliases: []string{"s"}, 31 | Usage: "set key/value", 32 | //Flags: app.Flags, 33 | Action: func(c *cli.Context) { 34 | for i, k := range c.GlobalFlagNames() { 35 | fmt.Printf("%d %s:%v\n", i, k, c.GlobalString(k)) 36 | } 37 | println("set k/v: ", c.Args().Get(0), c.Args().Get(1)) 38 | }, 39 | }, 40 | { 41 | Name: "get", 42 | Aliases: []string{"g"}, 43 | Usage: "get value of the given key", 44 | Flags: app.Flags, 45 | Action: func(c *cli.Context) { 46 | fmt.Printf("%+v\n", c.Args()) 47 | println("get key: ", c.Args().Get(0)) 48 | }, 49 | }, 50 | { 51 | Name: "delete", 52 | Aliases: []string{"d"}, 53 | Usage: "delete the given key", 54 | Flags: app.Flags, 55 | Action: func(c *cli.Context) { 56 | fmt.Printf("%+v\n", c.Args()) 57 | println("delete key: ", c.Args().Get(0)) 58 | }, 59 | }, 60 | } 61 | 62 | app.Run(os.Args) 63 | } 64 | -------------------------------------------------------------------------------- /config/cli_config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | 7 | "fmt" 8 | "github.com/BurntSushi/toml" 9 | log "github.com/Sirupsen/logrus" 10 | "github.com/sumory/confd/cli" 11 | ) 12 | 13 | func InitCliConfig(cliStoreType, cliConfDir, cliConfigFile, clieConnectAddr string, cliConfig *cli.CliConfig) error { 14 | if cliConfigFile == "" { 15 | if _, err := os.Stat(defaultConfigFile); !os.IsNotExist(err) { 16 | cliConfigFile = defaultConfigFile 17 | } 18 | } 19 | 20 | // 默认配置 21 | config := Config{ 22 | Store: "file", 23 | ConfDir: "/data/confd", 24 | } 25 | 26 | //从toml文件更新配置 27 | if cliConfigFile == "" { 28 | log.Debug("Skipping confd config file.") 29 | } else { 30 | log.Debug("Loading " + cliConfigFile) 31 | configBytes, err := ioutil.ReadFile(cliConfigFile) 32 | if err != nil { 33 | return err 34 | } 35 | _, err = toml.Decode(string(configBytes), &config) 36 | if err != nil { 37 | return err 38 | } 39 | } 40 | 41 | //从命令行参数覆盖配置 42 | config.Store = cliStoreType 43 | config.ConfDir = cliConfDir 44 | config.ConnectAddr = clieConnectAddr 45 | 46 | //配置connectAddr 47 | if config.ConnectAddr == "" { 48 | switch config.Store { 49 | case "file": 50 | config.ConnectAddr = fmt.Sprintf("%s/data/filestore.toml", config.ConfDir) 51 | case "redis": 52 | config.ConnectAddr = "127.0.0.1:6379" 53 | case "zookeeper": 54 | config.ConnectAddr = "127.0.0.1:2181" 55 | } 56 | } 57 | 58 | //cli设置 59 | cliConfig.Store = config.Store 60 | cliConfig.ConnectAddr = config.ConnectAddr 61 | 62 | return nil 63 | } 64 | -------------------------------------------------------------------------------- /confd.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | log "github.com/Sirupsen/logrus" 7 | "github.com/sumory/confd/config" 8 | "github.com/sumory/confd/processor" 9 | "github.com/sumory/confd/store" 10 | "os" 11 | "os/signal" 12 | "syscall" 13 | ) 14 | 15 | func init() { 16 | log.SetFormatter(&log.TextFormatter{ 17 | ForceColors: false, 18 | DisableColors: true, 19 | TimestampFormat: "2006-01-02 15:04:05.00000", 20 | }) 21 | log.SetLevel(log.DebugLevel) 22 | } 23 | 24 | var configDirFromBuild string = "" 25 | 26 | func main() { 27 | flag.Parse() 28 | 29 | err, myConfig, templateConfig, storeConfig := config.InitConfig(configDirFromBuild) 30 | if err != nil { 31 | log.Error(err.Error()) 32 | } 33 | 34 | log.Info("Starting confd") 35 | 36 | storeClient, err := store.New(storeConfig) 37 | if err != nil { 38 | log.Error(err.Error()) 39 | } 40 | 41 | templateConfig.StoreClient = storeClient 42 | 43 | if myConfig.Debug { 44 | processor.Process(templateConfig) 45 | return 46 | } 47 | 48 | stopChan := make(chan bool) 49 | doneChan := make(chan bool) 50 | errChan := make(chan error, 10) 51 | 52 | processor := processor.NewIntervalProcessor(templateConfig, stopChan, doneChan, errChan, myConfig.Interval) 53 | 54 | go processor.Process() 55 | 56 | signalChan := make(chan os.Signal, 1) 57 | signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) 58 | for { 59 | select { 60 | case err := <-errChan: 61 | log.Error(err.Error()) 62 | case s := <-signalChan: 63 | log.Info(fmt.Sprintf("Captured %v. Exiting...", s)) 64 | close(doneChan) 65 | case <-doneChan: 66 | os.Exit(0) 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /store/zookeeper/client.go: -------------------------------------------------------------------------------- 1 | package zookeeper 2 | 3 | import ( 4 | "strings" 5 | "time" 6 | 7 | zk "github.com/samuel/go-zookeeper/zk" 8 | ) 9 | 10 | type Client struct { 11 | client *zk.Conn 12 | } 13 | 14 | func NewZookeeperClient(connectAddr string) (*Client, error) { 15 | c, _, err := zk.Connect([]string{connectAddr}, time.Second) //*10) 16 | if err != nil { 17 | panic(err) 18 | } 19 | return &Client{c}, nil 20 | } 21 | 22 | func nodeWalk(prefix string, c *Client, vars map[string]interface{}) error { 23 | l, stat, err := c.client.Children(prefix) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | if stat.NumChildren == 0 { 29 | b, _, err := c.client.Get(prefix) 30 | if err != nil { 31 | return err 32 | } 33 | vars[prefix] = string(b) 34 | 35 | } else { 36 | for _, key := range l { 37 | s := prefix + "/" + key 38 | _, stat, err := c.client.Exists(s) 39 | if err != nil { 40 | return err 41 | } 42 | if stat.NumChildren == 0 { 43 | b, _, err := c.client.Get(s) 44 | if err != nil { 45 | return err 46 | } 47 | vars[s] = string(b) 48 | } else { 49 | nodeWalk(s, c, vars) 50 | } 51 | } 52 | } 53 | return nil 54 | } 55 | 56 | func (c *Client) GetValues(keys []string) (map[string]interface{}, error) { 57 | vars := make(map[string]interface{}) 58 | for _, v := range keys { 59 | v = strings.Replace(v, "/*", "", -1) 60 | _, _, err := c.client.Exists(v) 61 | if err != nil { 62 | return vars, err 63 | } 64 | if v == "/" { 65 | v = "" 66 | } 67 | err = nodeWalk(v, c, vars) 68 | if err != nil { 69 | return vars, err 70 | } 71 | } 72 | return vars, nil 73 | } 74 | 75 | func (c *Client) WatchPrefix(prefix string, waitIndex uint64, stopChan chan bool) (uint64, error) { 76 | <-stopChan 77 | return 0, nil 78 | } 79 | -------------------------------------------------------------------------------- /processor/processor.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | log "github.com/Sirupsen/logrus" 5 | "github.com/sumory/confd/utils" 6 | "time" 7 | ) 8 | 9 | type Processor interface { 10 | Process() 11 | } 12 | 13 | type intervalProcessor struct { 14 | config *TemplateConfig 15 | stopChan chan bool 16 | doneChan chan bool 17 | errChan chan error 18 | interval int 19 | } 20 | 21 | func Process(config *TemplateConfig) error { 22 | ts, err := getMetaObjects(config) 23 | if err != nil { 24 | return err 25 | } 26 | return process(ts) 27 | } 28 | 29 | func process(ts []*MetaObject) error { 30 | var lastErr error 31 | for _, t := range ts { 32 | if err := t.process(); err != nil { 33 | log.Error(err.Error()) 34 | lastErr = err 35 | } 36 | } 37 | return lastErr 38 | } 39 | 40 | func NewIntervalProcessor(config *TemplateConfig, stopChan, doneChan chan bool, errChan chan error, interval int) Processor { 41 | return &intervalProcessor{config, stopChan, doneChan, errChan, interval} 42 | } 43 | 44 | func (p *intervalProcessor) Process() { 45 | defer close(p.doneChan) 46 | for { 47 | ts, err := getMetaObjects(p.config) 48 | if err != nil { 49 | log.Fatal(err.Error()) 50 | break 51 | } 52 | process(ts) 53 | 54 | select { 55 | case <-p.stopChan: 56 | break 57 | case <-time.After(time.Duration(p.interval) * time.Second): 58 | log.Infof("Restart process all configurations.") 59 | continue 60 | } 61 | } 62 | } 63 | 64 | func getMetaObjects(config *TemplateConfig) ([]*MetaObject, error) { 65 | var lastError error 66 | metaObjects := make([]*MetaObject, 0) 67 | 68 | paths, err := utils.RecursiveFindFiles(config.MetaDir, "*toml") 69 | if err != nil { 70 | return nil, err 71 | } 72 | for _, p := range paths { 73 | log.Infof("New meta info from path %s", p) 74 | t, err := NewMetaObject(p, config) 75 | if err != nil { 76 | lastError = err 77 | continue 78 | } 79 | metaObjects = append(metaObjects, t) 80 | } 81 | return metaObjects, lastError 82 | } 83 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ### confd 2 | 3 | confd是一个统一配置管理工具，目前仍在开发中，基于本地文件存储的部署方式已经可以用于生产环境中。 4 | 5 | ##### Features 6 | 7 | - [x] 基础解析：配置文件使用toml格式，模板数据分离 8 | - [x] 多种存储支持：local file、redis、zookeeper 9 | - [x] 两种运行模式：Debug或Daemon 10 | - [x] cli工具：修改store，批量更新配置 11 | - [ ] 更友好的交互方式，比如通过web界面 12 | - [ ] 分离server和client，提供客户端cli或API供拉取指定配置 13 | - [ ] 加密支持：store中存储的配置可加密，防止泄露 14 | 15 | ##### Usage 16 | 17 | **安装** 18 | 19 | ``` 20 | #假设安装路径为/data/server/confd 21 | #go get获取依赖的第三方库 22 | 23 | sh build.sh linux /data/server/confd 24 | #执行以上构建脚本后，在docs目录下生成了安装所需的文件 25 | #若build.sh增加了参数$2，则默认加载$2/data/config.toml作为confd运行所需的配置文件 26 | 27 | cd docs 28 | sh install.sh /data/server/confd 29 | cd /data/server/confd 30 | #注意修改${path}/data/config.toml里的ConfDir和ConnectAddr，前缀为/data/server/confd 31 | 32 | #然后使用confd、confd-cli即可 33 | ``` 34 | 35 | **使用** 36 | 37 | ``` 38 | . 39 | ├── confd 40 | ├── confd-cli 41 | ├── data 42 | │   ├── config.toml 43 | │   └── filestore.toml 44 | ├── meta 45 | │   ├── example_1.toml 46 | │   └── example_2.toml 47 | └── templates 48 | └── example.tmpl 49 | ``` 50 | 51 | 52 | **confd**的使用： 53 | 54 | - 首先了解： 55 | - confd的配置可参看[config.toml](./docs/files/config.toml), config.toml里的配置均可通过运行时指定flag来覆盖默认配置。 56 | - confd.toml中指定的配置数据存储为“file”形式，即数据存储在同目录下filestore.toml文件中。 57 | - 根据需要编辑模板，参考[example.tmpl](docs/files/example.tmpl)，模板中使用的变量目前只支持k/v形式。 58 | - 根据需要编辑meta文件，参考[example_1.toml](docs/files/example_1.toml)，meta文件指定了生成最终配置文件时需要的模板文件、数据、最终文件地址等。 59 | - 命令 60 | - `./confd`, 在各个meta文件指定的目的地址生成了需要的配置文件 61 | - `./confd --debug=false`, 默认confd在后台运行，每10分钟重新生成一次全部的配置文件 62 | 63 | **confd-cli**的使用： 64 | 65 | - `confd-cli`是操作confd的命令行程序 66 | - 目前支持的子命令 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 |
./confd-cli getall获取当前所有配置需要的数据
./confd-cli get key1获取key1现在的值
./confd-cli set key1 value1设置key1值为value1
./confd-cli delete key1删除key1
86 | 87 | -------------------------------------------------------------------------------- /cli/file/cli.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "fmt" 7 | "github.com/BurntSushi/toml" 8 | log "github.com/Sirupsen/logrus" 9 | "io/ioutil" 10 | "os" 11 | "path/filepath" 12 | ) 13 | 14 | type FileStore struct { 15 | Path string 16 | Data map[string]interface{} `toml:"data"` 17 | } 18 | 19 | type Cli struct { 20 | FileStore *FileStore 21 | } 22 | 23 | var EmptyErr = errors.New("empty key/value") 24 | 25 | func NewFileCli(filePath string) (error, *Cli) { 26 | var fileStore *FileStore 27 | _, err := toml.DecodeFile(filePath, &fileStore) 28 | log.Infof("%s %v", filePath, fileStore) 29 | 30 | if err != nil { 31 | return fmt.Errorf("Cannot process file store %s - %s", filePath, err.Error()), nil 32 | } 33 | 34 | fileStore.Path = filePath 35 | return nil, &Cli{ 36 | FileStore: fileStore, 37 | } 38 | } 39 | 40 | //重新获取所有value 41 | func (c *Cli) Fetch() error { 42 | _, err := toml.DecodeFile(c.FileStore.Path, &c.FileStore) 43 | //log.Infof("Fetch %s, values: %+v", c.FileStore.Path, *c.FileStore) 44 | if err != nil { 45 | return fmt.Errorf("Cannot process file store %s - %s", c.FileStore.Path, err.Error()) 46 | } 47 | 48 | return nil 49 | } 50 | 51 | func (c *Cli) GetAll() (map[string]interface{}, error) { 52 | return c.FileStore.Data, nil 53 | } 54 | 55 | func (c *Cli) GetValues(keys []string) (map[string]interface{}, error) { 56 | return c.FileStore.Data, nil 57 | } 58 | 59 | func (c *Cli) GetValue(key string) (interface{}, error) { 60 | if value, ok := c.FileStore.Data[key]; ok { 61 | return value, nil 62 | } 63 | return nil, EmptyErr 64 | } 65 | 66 | func (c *Cli) DeleteKey(key string) error { 67 | err := c.Fetch() 68 | if err != nil { 69 | return err 70 | } 71 | 72 | delete(c.FileStore.Data, key) 73 | return c.setValues(c.FileStore.Data) 74 | } 75 | 76 | func (c *Cli) setValues(values map[string]interface{}) error { 77 | 78 | filename := c.FileStore.Path 79 | 80 | // 创建临时文件 81 | temp, err := ioutil.TempFile(filepath.Dir(filename), "."+filepath.Base(filename)) 82 | temp.WriteString("[data]\n") 83 | defer os.Remove(temp.Name()) 84 | defer temp.Close() 85 | 86 | // fmt.Println(filename) 87 | // fmt.Println(temp.Name()) 88 | // fmt.Println(values) 89 | 90 | w := bufio.NewWriter(temp) 91 | toml.NewEncoder(w).Encode(values) 92 | os.Chmod(temp.Name(), 0644) 93 | 94 | err = os.Rename(temp.Name(), filename) 95 | if err != nil { 96 | return err 97 | } 98 | return nil 99 | } 100 | 101 | func (c *Cli) SetValue(key string, value interface{}) error { 102 | err := c.Fetch() 103 | if err != nil { 104 | return err 105 | } 106 | 107 | c.FileStore.Data[key] = value 108 | return c.setValues(c.FileStore.Data) 109 | } 110 | -------------------------------------------------------------------------------- /processor/meta.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path" 9 | "path/filepath" 10 | "strconv" 11 | "text/template" 12 | 13 | "github.com/BurntSushi/toml" 14 | log "github.com/Sirupsen/logrus" 15 | "github.com/sumory/confd/store" 16 | "github.com/sumory/confd/utils" 17 | ) 18 | 19 | type MetaConfig struct { 20 | MetaObject MetaObject `toml:"metaObject"` 21 | } 22 | 23 | type MetaObject struct { 24 | Tmpl string //模板文件，用于生成最后的配置文件 25 | Dest string //最终配置文件的路径 26 | Keys []string //需要使用的key 27 | 28 | FileMode os.FileMode 29 | Mode string 30 | funcMap map[string]interface{} 31 | storeClient store.StoreClient 32 | 33 | kvStore utils.KVStore 34 | } 35 | 36 | var EmptyErr = errors.New("empty meta file") 37 | 38 | func NewMetaObject(path string, config *TemplateConfig) (*MetaObject, error) { 39 | if config.StoreClient == nil { 40 | return nil, errors.New("StoreClient is not found.") 41 | } 42 | 43 | log.Debug("Loading meta info from ", path) 44 | 45 | var tc *MetaConfig 46 | _, err := toml.DecodeFile(path, &tc) 47 | if err != nil { 48 | return nil, fmt.Errorf("Cannot process meta file %s - %s", path, err.Error()) 49 | } 50 | 51 | log.Debug(fmt.Sprintf("metaObject: %+v", tc.MetaObject)) 52 | tr := tc.MetaObject 53 | tr.storeClient = config.StoreClient 54 | tr.kvStore = utils.NewKVStore() 55 | tr.funcMap = tr.kvStore.FuncMap 56 | 57 | if tr.Tmpl == "" { 58 | return nil, EmptyErr 59 | } 60 | tr.Tmpl = filepath.Join(config.TemplateDir, tr.Tmpl) 61 | 62 | return &tr, nil 63 | } 64 | 65 | func (t *MetaObject) process() error { 66 | if err := t.setFileMode(); err != nil { 67 | return err 68 | } 69 | 70 | result, err := t.setVars() 71 | if err != nil { 72 | return err 73 | } 74 | if err = t.createConfigFile(result); err != nil { 75 | return err 76 | } 77 | 78 | return nil 79 | } 80 | 81 | // setVars sets the Vars for template resource. 82 | func (t *MetaObject) setVars() (map[string]interface{}, error) { 83 | var err error 84 | log.Debug("Retrieving keys from store") 85 | result, err := t.storeClient.GetValues(t.Keys) 86 | 87 | if err != nil { 88 | return nil, err 89 | } 90 | fmt.Printf("%+v\n", result) 91 | t.kvStore.Clean() 92 | for k, v := range result { 93 | t.kvStore.Set(k, v) 94 | } 95 | 96 | return result, nil 97 | } 98 | 99 | func (t *MetaObject) createConfigFile(data map[string]interface{}) error { 100 | log.Debug("Using template " + t.Tmpl) 101 | 102 | if !utils.IsFileExist(t.Tmpl) { 103 | return errors.New("Missing meta file: " + t.Tmpl) 104 | } 105 | 106 | log.Debug("Compiling source template " + t.Tmpl) 107 | tmpl, err := template.New(path.Base(t.Tmpl)).Funcs(t.funcMap).ParseFiles(t.Tmpl) 108 | if err != nil { 109 | return fmt.Errorf("Unable to process template file %s, %s", t.Tmpl, err) 110 | } 111 | 112 | // 创建临时文件 113 | temp, err := ioutil.TempFile(filepath.Dir(t.Dest), "."+filepath.Base(t.Dest)) 114 | defer os.Remove(temp.Name()) 115 | defer temp.Close() 116 | 117 | log.Debug("Temp path: ", temp.Name()) 118 | 119 | if err != nil { 120 | return err 121 | } 122 | 123 | if err = tmpl.Execute(temp, nil); err != nil { 124 | temp.Close() 125 | os.Remove(temp.Name()) 126 | return err 127 | } 128 | os.Chmod(temp.Name(), t.FileMode) 129 | 130 | log.Debug("Overwriting target config: ", t.Dest) 131 | err = os.Rename(temp.Name(), t.Dest) 132 | if err != nil { 133 | log.Fatal("Rename ", temp.Name(), " to ", t.Dest, " failed") 134 | return err 135 | } 136 | 137 | log.Info("Target config has been updated: ", t.Dest) 138 | return nil 139 | } 140 | 141 | func (t *MetaObject) setFileMode() error { 142 | if t.Mode == "" { 143 | if !utils.IsFileExist(t.Dest) { 144 | t.FileMode = 0644 145 | } else { 146 | fi, err := os.Stat(t.Dest) 147 | if err != nil { 148 | return err 149 | } 150 | t.FileMode = fi.Mode() 151 | } 152 | } else { 153 | mode, err := strconv.ParseUint(t.Mode, 0, 32) 154 | if err != nil { 155 | return err 156 | } 157 | t.FileMode = os.FileMode(mode) 158 | } 159 | return nil 160 | } 161 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "flag" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | 9 | "fmt" 10 | "github.com/BurntSushi/toml" 11 | log "github.com/Sirupsen/logrus" 12 | "github.com/sumory/confd/processor" 13 | "github.com/sumory/confd/store" 14 | ) 15 | 16 | var ( 17 | configFile = "" 18 | defaultConfigFile = "/data/confd/data/config.toml" 19 | storeType string 20 | connectAddr string 21 | confDir string 22 | interval int 23 | debug bool 24 | ) 25 | 26 | //confd 27 | type Config struct { 28 | Store string `toml:"store"` 29 | ConnectAddr string `toml:"connect_addr"` 30 | 31 | ConfDir string `toml:"confdir"` //confd配置文件、模板文件、meta文件目录 32 | 33 | Interval int `toml:"interval"` 34 | Debug bool 35 | } 36 | 37 | func init() { 38 | flag.StringVar(&storeType, "store-type", "file", "backend store to use") 39 | flag.StringVar(&confDir, "confdir", "/data/confd", "confd conf directory") 40 | flag.StringVar(&configFile, "config-file", "", "the confd config file") 41 | flag.StringVar(&connectAddr, "connect-addr", "", "backend store address") 42 | flag.IntVar(&interval, "interval", 600, "backend polling interval") 43 | flag.BoolVar(&debug, "debug", false, "debug mode") 44 | } 45 | 46 | func InitConfig(configDirFromBuild string) (error, *Config, *processor.TemplateConfig, *store.StoreConfig) { 47 | // 默认配置 48 | config := Config{ 49 | Store: "file", 50 | ConfDir: "/data/confd", 51 | Interval: 600, 52 | } 53 | 54 | if configDirFromBuild != "" { //尝试使用build脚本的变量初始化configFile 55 | fmt.Printf("configDirFromBuild:%s\n", configDirFromBuild) 56 | configFileFromBuild := fmt.Sprintf("%s/data/config.toml", configDirFromBuild) 57 | fmt.Printf("use config file[%s] from build script\n", configFileFromBuild) 58 | if _, err := os.Stat(configFileFromBuild); !os.IsNotExist(err) { 59 | configFile = configFileFromBuild 60 | config.ConfDir = configDirFromBuild 61 | } else { 62 | fmt.Printf("use config file[%s] from build script error, file not exist, skip...\n", configFileFromBuild) 63 | } 64 | } 65 | 66 | if configFile == "" { 67 | if _, err := os.Stat(defaultConfigFile); !os.IsNotExist(err) { 68 | fmt.Printf("use defaultConfigFile[%s]\n", defaultConfigFile) 69 | configFile = defaultConfigFile 70 | } 71 | } 72 | 73 | //从toml文件更新配置 74 | if configFile == "" { 75 | log.Debug("Skipping confd config file.") 76 | } else { 77 | log.Debug("Loading " + configFile) 78 | configBytes, err := ioutil.ReadFile(configFile) 79 | if err != nil { 80 | return err, nil, nil, nil 81 | } 82 | _, err = toml.Decode(string(configBytes), &config) 83 | if err != nil { 84 | return err, nil, nil, nil 85 | } 86 | } 87 | 88 | // 根据命令行参数更新配置 89 | processFlags(&config) 90 | 91 | if config.ConnectAddr == "" { 92 | switch config.Store { 93 | case "file": 94 | config.ConnectAddr = fmt.Sprintf("%s/data/filestore.toml", config.ConfDir) 95 | case "redis": 96 | config.ConnectAddr = "127.0.0.1:6379" 97 | case "zookeeper": 98 | config.ConnectAddr = "127.0.0.1:2181" 99 | } 100 | } 101 | 102 | log.Info("Store set to " + config.Store) 103 | 104 | //后端client设置 105 | storeConfig := store.StoreConfig{ 106 | Store: config.Store, 107 | ConnectAddr: config.ConnectAddr, 108 | } 109 | // 模板设置BT 110 | templateConfig := processor.TemplateConfig{ 111 | ConfDir: config.ConfDir, 112 | MetaDir: filepath.Join(config.ConfDir, "meta"), 113 | TemplateDir: filepath.Join(config.ConfDir, "templates"), 114 | } 115 | 116 | return nil, &config, &templateConfig, &storeConfig 117 | } 118 | 119 | func processFlags(config *Config) { 120 | flag.Visit(func(f *flag.Flag) { 121 | setConfigFromFlag(f, config) 122 | }) 123 | } 124 | 125 | func setConfigFromFlag(f *flag.Flag, config *Config) { 126 | switch f.Name { 127 | case "store-type": 128 | config.Store = storeType 129 | case "confdir": 130 | config.ConfDir = confDir 131 | case "connect-addr": 132 | config.ConnectAddr = connectAddr 133 | case "interval": 134 | config.Interval = interval 135 | case "debug": 136 | config.Debug = debug 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /confd_cli.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | log "github.com/Sirupsen/logrus" 6 | "github.com/codegangsta/cli" 7 | mycli "github.com/sumory/confd/cli" 8 | "github.com/sumory/confd/config" 9 | "os" 10 | ) 11 | 12 | func init() { 13 | log.SetFormatter(&log.TextFormatter{ 14 | ForceColors: false, 15 | DisableColors: true, 16 | TimestampFormat: "2006-01-02 15:04:05.00000", 17 | }) 18 | log.SetLevel(log.DebugLevel) 19 | } 20 | 21 | func parseArgs() { 22 | app := cli.NewApp() 23 | app.Name = "confd cli tool" 24 | app.Usage = "use this tool to operate confd store!" 25 | app.Version = "0.0.1" 26 | 27 | app.Flags = []cli.Flag{ 28 | cli.StringFlag{ 29 | Name: "store-type", 30 | Value: "file", 31 | Usage: "backend store to use", 32 | }, 33 | cli.StringFlag{ 34 | Name: "confdir", 35 | Value: "/data/confd", 36 | Usage: "confd conf directory", 37 | }, 38 | cli.StringFlag{ 39 | Name: "config-file", 40 | Value: "/data/confd/data/config.toml", 41 | Usage: "the confd config file", 42 | }, 43 | cli.StringFlag{ 44 | Name: "connect-addr", 45 | Value: "", 46 | Usage: "backend store address", 47 | }, 48 | } 49 | 50 | app.Commands = []cli.Command{ 51 | { 52 | Name: "getall", 53 | Aliases: []string{"gl"}, 54 | Usage: "get all key/value, e.g. `confd-cli getall`", 55 | Action: func(c *cli.Context) { 56 | getall(c) 57 | }, 58 | }, 59 | { 60 | Name: "set", 61 | Aliases: []string{"s"}, 62 | Usage: "set key/value, e.g. `confd-cli set key1 value1`", 63 | //Flags: app.Flags, 64 | Action: func(c *cli.Context) { 65 | set(c) 66 | }, 67 | }, 68 | { 69 | Name: "get", 70 | Aliases: []string{"g"}, 71 | Usage: "get value of the given key, e.g. `confd-cli get key1`", 72 | Flags: app.Flags, 73 | Action: func(c *cli.Context) { 74 | get(c) 75 | }, 76 | }, 77 | { 78 | Name: "delete", 79 | Aliases: []string{"d"}, 80 | Usage: "delete the given key, e.g. `confd-cli delete key1`", 81 | Flags: app.Flags, 82 | Action: func(c *cli.Context) { 83 | delete(c) 84 | }, 85 | }, 86 | } 87 | 88 | app.Run(os.Args) 89 | } 90 | 91 | func printGlobalOptions(c *cli.Context) { 92 | fmt.Println("\nThe input flag args is:") 93 | 94 | for i, k := range c.GlobalFlagNames() { 95 | fmt.Printf("%d %s:%v\n", i, k, c.GlobalString(k)) 96 | } 97 | } 98 | 99 | func printCliConfig(cliConfig *mycli.CliConfig) { 100 | fmt.Println("\nThe cliConfig is:") 101 | fmt.Printf("%+v\n\n", *cliConfig) 102 | } 103 | 104 | func newMyCli(c *cli.Context) (error, mycli.Cli) { 105 | log.SetLevel(log.WarnLevel) 106 | printGlobalOptions(c) 107 | 108 | var cliConfig *mycli.CliConfig = &mycli.CliConfig{} 109 | config.InitCliConfig(c.GlobalString("store-type"), c.GlobalString("confdir"), c.GlobalString("config-file"), c.GlobalString("connect-addr"), cliConfig) 110 | printCliConfig(cliConfig) 111 | 112 | return mycli.New(cliConfig) 113 | } 114 | 115 | func getall(c *cli.Context) { 116 | err, mycli := newMyCli(c) 117 | if err != nil { 118 | log.Error("New confd cli error: ", err.Error()) 119 | } 120 | 121 | data, err := mycli.GetAll() 122 | if err != nil { 123 | fmt.Println("error when get all key/value") 124 | return 125 | } 126 | 127 | fmt.Println("The stored k/v:") 128 | 129 | for k, v := range data { 130 | fmt.Printf("%s\t %v \n", k, v) 131 | } 132 | } 133 | 134 | func get(c *cli.Context) { 135 | err, mycli := newMyCli(c) 136 | if err != nil { 137 | log.Error("New confd cli error: ", err.Error()) 138 | } 139 | 140 | key := c.Args().Get(0) 141 | data, err := mycli.GetValue(key) 142 | if err != nil { 143 | fmt.Printf("\nerror when get key[%s], error: %v\n", key, err) 144 | return 145 | } 146 | 147 | fmt.Printf("The value of key[%s] is: %v\n", key, data) 148 | 149 | } 150 | 151 | //confd-cli set key value 152 | func set(c *cli.Context) { 153 | err, mycli := newMyCli(c) 154 | if err != nil { 155 | log.Error("New confd cli error: ", err.Error()) 156 | } 157 | 158 | key, value := c.Args().Get(0), c.Args().Get(1) 159 | err = mycli.SetValue(key, value) 160 | if err != nil { 161 | fmt.Printf("set error: %s", err) 162 | } else { 163 | fmt.Println("set ok.") 164 | } 165 | 166 | } 167 | 168 | func delete(c *cli.Context) { 169 | err, mycli := newMyCli(c) 170 | if err != nil { 171 | log.Error("New confd cli error: ", err.Error()) 172 | } 173 | 174 | key := c.Args().Get(0) 175 | err = mycli.DeleteKey(key) 176 | if err != nil { 177 | fmt.Printf("delete key error: %s", err) 178 | } else { 179 | fmt.Println("delete ok.") 180 | } 181 | 182 | } 183 | 184 | func main() { 185 | fmt.Println("Confd cli tool...") 186 | parseArgs() 187 | } 188 | --------------------------------------------------------------------------------