├── .github └── workflows │ ├── golangci-lint.yml │ └── tests.yml ├── .gitignore ├── CODEOWNERS ├── LICENSE ├── README.md ├── bigquery └── executor.go ├── cmd ├── completion.go ├── copy.go ├── isolateDAG.go ├── lookml_gen.go ├── model_completion.go ├── root.go ├── run.go ├── schema_gen.go ├── schema_gen_test.go ├── seed.go ├── show.go ├── showDAG.go ├── test.go ├── test_gen.go ├── version.go └── watch.go ├── compiler ├── ExecutionContext.go ├── GlobalContext.go ├── builtInFunctions.go ├── builtInMacros.go ├── compiler.go └── dbtUtils │ ├── queryMacros.go │ ├── replacements.go │ └── utils.go ├── compilerInterface ├── types.go └── value.go ├── config ├── config.go ├── config_test.go ├── modelConfig.go ├── modelConfig_test.go ├── modelGroups.go ├── seedConfig.go ├── seedConfig_test.go └── target.go ├── fs ├── docs.go ├── file.go ├── filesystem.go ├── graph.go ├── schema.go ├── seed.go └── workpool.go ├── go.mod ├── go.sum ├── jinja ├── ast │ ├── AndCondition.go │ ├── AtomExpressionBlock.go │ ├── Body.go │ ├── BoolValue.go │ ├── BracketGroup.go │ ├── BuiltInTest.go │ ├── CallBlock.go │ ├── DoBlock.go │ ├── EndOfFile.go │ ├── ForLoop.go │ ├── FunctionCall.go │ ├── IfStatement.go │ ├── InOperator.go │ ├── List.go │ ├── LogicalOp.go │ ├── Macro.go │ ├── Map.go │ ├── MathsOp.go │ ├── NoneValue.go │ ├── NotOperator.go │ ├── NullValue.go │ ├── Number.go │ ├── OrCondition.go │ ├── SetCall.go │ ├── StringConcat.go │ ├── TextBlock.go │ ├── UniaryMathsOp.go │ ├── UnsupportedExpressionBlock.go │ ├── Variable.go │ └── types.go ├── lexer │ ├── lexer.go │ └── token.go └── parser.go ├── main.go ├── properties ├── propertiesFile.go ├── propertiesFile_test.go └── test.go ├── schemaTestMacros ├── testNotNullMacro.go └── testUniqueMacro.go ├── tests ├── basic_test.go ├── builtin_jinja_tests_test.go ├── builtin_macro_tests_test.go ├── macro_test.go ├── regression_test.go └── utils_test.go ├── utils ├── Debounce.go ├── ProgressBar.go ├── terminal.go └── version.go └── watcher ├── Event.go └── Watcher.go /.github/workflows/golangci-lint.yml: -------------------------------------------------------------------------------- 1 | name: golangci-lint 2 | on: [push, pull_request] 3 | jobs: 4 | golangci: 5 | strategy: 6 | matrix: 7 | go-version: [1.16.x] 8 | os: [ubuntu-latest, macos-latest] 9 | name: lint 10 | runs-on: ${{ matrix.os }} 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: golangci-lint 14 | uses: golangci/golangci-lint-action@v2 15 | with: 16 | # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. 17 | version: latest 18 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: [push, pull_request] 3 | jobs: 4 | test: 5 | strategy: 6 | matrix: 7 | go-version: [1.16.x] 8 | os: [ubuntu-latest, macos-latest] 9 | runs-on: ${{ matrix.os }} 10 | steps: 11 | - name: Install Go 12 | uses: actions/setup-go@v2 13 | with: 14 | go-version: ${{ matrix.go-version }} 15 | - name: Checkout code 16 | uses: actions/checkout@v2 17 | - name: Lint 18 | run: 19 | go vet ./... 20 | - name: Test 21 | run: | 22 | go test ./... 23 | go test -race ./... 24 | test-cache: 25 | runs-on: ubuntu-latest 26 | steps: 27 | - name: Install Go 28 | uses: actions/setup-go@v2 29 | with: 30 | go-version: 1.16.x 31 | - name: Checkout code 32 | uses: actions/checkout@v2 33 | - uses: actions/cache@v2 34 | with: 35 | # In order: 36 | # * Module download cache 37 | # * Build cache (Linux) 38 | # * Build cache (Mac) 39 | # * Build cache (Windows) 40 | path: | 41 | ~/go/pkg/mod 42 | ~/.cache/go-build 43 | ~/Library/Caches/go-build 44 | %LocalAppData%\go-build 45 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 46 | restore-keys: | 47 | ${{ runner.os }}-go- 48 | - name: Test 49 | run: | 50 | go test ./... 51 | go test -race ./... 52 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /ddbt_dev 2 | /.DS_Store 3 | /.idea -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Lines starting with '#' are comments. 2 | # Each line is a file pattern followed by one or more owners. 3 | 4 | # These owners will be the default owners for everything in the repo. 5 | * @monzo/ddbt-approvers -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Monzo Bank Limited 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # This repo is now archived 2 | 3 | It's left here for historic purposes so as to preserve the code that is within it, however no active development is continuing on `ddbt`. If support/discussion around re-opening the repo want to be had, please come visit us in #data-engineering-ask. 4 | 5 | The original state of the README can be seen below. 6 | 7 | # Dom's Data Build Tool 8 | 9 | [![Build Status](https://github.com/monzo/ddbt/actions/workflows/tests.yml/badge.svg)](https://github.com/monzo/ddbt/actions/workflows/tests.yml) 10 | [![GoDoc](https://godoc.org/github.com/monzo/ddbt?status.svg)](https://godoc.org/github.com/monzo/ddbt) 11 | 12 | This repo represents my attempt to build a fast version of [DBT](https://www.getdbt.com/) which gets very slow on large 13 | projects (3000+ data models). This project attempts to be a direct drop in replacement for DBT at the command line. 14 | 15 | *Warning:* This is experimental and may not work exactly as you expect 16 | 17 | ## Installation 18 | 1. Clone this repo 19 | ```bash 20 | $ git clone git@github.com:monzo/ddbt.git 21 | ``` 22 | 23 | 2. Change directory into cloned repo 24 | ```bash 25 | $ cd ddbt 26 | ``` 27 | 28 | 3. Install (requires go-lang) 29 | ```bash 30 | $ go install 31 | ``` 32 | 33 | 4. Confirm installation 34 | ```bash 35 | $ ddbt --version 36 | ddbt version 0.6.7 37 | ``` 38 | 39 | ## Command Quickstart 40 | - `ddbt run` will compile and execute all your models, or those filtered for, against your data warehouse 41 | - `ddbt test` will run all tests referencing all your models, or those filtered for, in your project against your data warehouse 42 | - `ddbt show my_model` will output the compiled SQL to the terminal 43 | - `ddbt copy my_model` will copy the compiled SQL into your clipboard 44 | - `ddbt show-dag` will output the order of how the models will execute 45 | - `ddbt watch` will get act like `run`, followed by `test`. DDBT will then watch your file system for any changes and automatically rerun those parts of the DAG and affected downstream tests or failing tests. 46 | - `ddbt watch --skip-run` is the same as watch, but will skip the initial run (preventing you having to wait for all the models to run) before running the tests and starting to watch your file system. 47 | - `ddbt completion zsh` will generate a shell completion script zsh (or bash if you pass that as argument). Detailed steps to set up the completion script can be found in `ddbt completion --help` 48 | - `ddbt isolate-dag` will create a temporary directory and symlink in all files needed for the given _model_filter_ such that Fishtown's DBT could be run against it without having to be run against every model in your data warehouse 49 | - `ddbt schema-gen -m my_model` will output a new or updated schema yml file for the model provided in the same directory as the dbt model file. 50 | - `ddbt lookml-gen my_model` will generate lookml view and copy it to your clipboard 51 | 52 | ### Global Arguments 53 | - `--models model_filter` _or_ `-m model_filter`: Instead of running for every model in your project, DDBT will only execute against the requested models. See filters below for what is accepted for `my_model` 54 | - `--threads=n`: force DDBT to run with `n` threads instead of what is defined in your `dbt_project.yml` 55 | - `--target=x` _or_ `-t x`: force DDBT to run against the `x` output defined in your `profile.yml` instead of the default defined in that file. 56 | - `--upstream=y` _or_ `-u y`: For any references to models outside the explicit models specified by run or test, the upstream target used to read that data will be swapped to `y` instead of the output target of `x` 57 | - `--fail-on-not-found=false` _or_ `-f=false`: By default, ddbt will fail if a the specified models don't exist, passing in this argument as false will warn instead of failing 58 | - `--enable-schema-based-tests` _or_ `-s=true`: Schema-based tests are disabled by default for now, but as a way to enable them pass this argument as true 59 | - `--custom-config-path=my/custom/path` _or_ `-c=my/custom/path`: Allows a custom path to be used for the `dbt_project.yml`. This is useful if you want to use a different location than the default one. For example if you're mid-way through migrating commands from an old dbt version to a new version and using two different versions of `dbt_project.yml` at the same time. 60 | 61 | ### Model Filters 62 | When running or testing the project, you may only want to run for a subset of your models. 63 | 64 | Currently DDBT supports the following syntax options: 65 | - `-m my_model`: DDBT will only execute against the model with that name 66 | - `-m +my_model`: DDBT will run against `my_model` and all upstreams referenced by it 67 | - `-m my_model+`: DDBT will run against `my_model` and all downstreams that referenced it 68 | - `-m +my_model+`: DDBT will run against `my_model` and both all upstreams and downstreams. 69 | - `-m tag:tagValue`: DDBT will only execute models which have a tag which is equal to `tagValue` 70 | -------------------------------------------------------------------------------- /cmd/completion.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | var file string 11 | 12 | func init() { 13 | completionCmd.Flags().StringVar(&file, "file", "", "file to which output has to be written") 14 | _ = completionCmd.MarkFlagFilename("file") 15 | 16 | rootCmd.AddCommand(completionCmd) 17 | } 18 | 19 | // completionCmd represents the completion command 20 | var completionCmd = &cobra.Command{ 21 | Use: "completion [bash|zsh]", 22 | Short: "Generate completion script", 23 | Long: `To load completions: 24 | 25 | Bash: 26 | 27 | $ source <(ddbt completion bash) 28 | 29 | # To load completions for each session, execute once: 30 | Linux: 31 | $ ddbt completion bash > /etc/bash_completion.d/ddbt 32 | MacOS: 33 | $ ddbt completion bash > /usr/local/etc/bash_completion.d/ddbt 34 | 35 | Zsh: 36 | 37 | # If shell completion is not already enabled in your environment you will need 38 | # to enable it. You can execute the following once: 39 | 40 | $ echo "autoload -U compinit; compinit" >> ~/.zshrc 41 | 42 | # To load completions for each session, execute once: 43 | $ ddbt completion zsh > "${fpath[1]}/_ddbt" 44 | 45 | # You will need to start a new shell for this setup to take effect.`, 46 | DisableFlagsInUseLine: true, 47 | ValidArgs: []string{"bash", "zsh", "fish", "powershell"}, 48 | Args: cobra.ExactValidArgs(1), 49 | Run: func(cmd *cobra.Command, args []string) { 50 | var err error 51 | switch args[0] { 52 | case "bash": 53 | if file != "" { 54 | err = cmd.Root().GenBashCompletionFile(file) 55 | } else { 56 | err = cmd.Root().GenBashCompletion(os.Stdout) 57 | } 58 | case "zsh": 59 | if file != "" { 60 | err = cmd.Root().GenZshCompletionFile(file) 61 | } else { 62 | err = cmd.Root().GenZshCompletion(os.Stdout) 63 | } 64 | case "fish": 65 | if file != "" { 66 | err = cmd.Root().GenFishCompletionFile(file, true) 67 | } else { 68 | err = cmd.Root().GenFishCompletion(os.Stdout, true) 69 | } 70 | case "powershell": 71 | if file != "" { 72 | err = cmd.Root().GenPowerShellCompletionFile(file) 73 | } else { 74 | err = cmd.Root().GenPowerShellCompletion(os.Stdout) 75 | } 76 | } 77 | if err != nil { 78 | fmt.Fprintf(os.Stderr, "❌ Unable to generate shell completion for %s: %s\n", args[0], err) 79 | } 80 | }, 81 | } 82 | -------------------------------------------------------------------------------- /cmd/copy.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/atotto/clipboard" 8 | "github.com/spf13/cobra" 9 | ) 10 | 11 | func init() { 12 | rootCmd.AddCommand(copyCommand) 13 | } 14 | 15 | var copyCommand = &cobra.Command{ 16 | Use: "copy [model name]", 17 | Short: "Copies the SQL that would be executed for the given model into your clipboard", 18 | Args: cobra.ExactValidArgs(1), 19 | Run: func(cmd *cobra.Command, args []string) { 20 | if err := clipboard.WriteAll(getModelSQL(args[0])); err != nil { 21 | fmt.Printf("❌ Unable to copy query into your clipboard: %s\n", err) 22 | os.Exit(1) 23 | } 24 | 25 | fmt.Printf("📎 Query has been copied into your clipboard\n") 26 | }, 27 | ValidArgsFunction: completeModelFn, 28 | } 29 | -------------------------------------------------------------------------------- /cmd/isolateDAG.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | 10 | "github.com/spf13/cobra" 11 | 12 | "ddbt/config" 13 | "ddbt/fs" 14 | "ddbt/utils" 15 | ) 16 | 17 | func init() { 18 | rootCmd.AddCommand(isolateDAG) 19 | addModelsFlag(isolateDAG) 20 | addFailOnNotFoundFlag(isolateDAG) 21 | } 22 | 23 | var isolateDAG = &cobra.Command{ 24 | Use: "isolate-dag", 25 | Short: "Creates a symlinked copy of the selected models, which can be then passed to Fishtown's DBT", 26 | Run: func(cmd *cobra.Command, args []string) { 27 | fileSystem, _ := compileAllModels() 28 | 29 | graph := buildGraph(fileSystem, ModelFilters) // Build the execution graph for the given command 30 | graph.AddReferencingTests() // And then add any tests which reference that graph 31 | graph.AddEphemeralUpstreams() // Add any ephemeral upstream nodes to the graph 32 | 33 | if err := graph.AddAllUsedMacros(); err != nil { 34 | fmt.Printf("❌ Unable to get all used macros: %s\n", err) 35 | os.Exit(1) 36 | } 37 | 38 | isolateGraph(graph) 39 | }, 40 | } 41 | 42 | func isolateGraph(graph *fs.Graph) { 43 | pb := utils.NewProgressBar("🔪 Isolating DAG", graph.Len()) 44 | defer pb.Stop() 45 | 46 | // Create a temporary directory to stick the isolated models in 47 | isolationDir, err := ioutil.TempDir(os.TempDir(), "isolated-dag-") 48 | if err != nil { 49 | fmt.Printf("❌ Unable to create temporarily directory for DAG isolation: %s\n", err) 50 | os.Exit(1) 51 | } 52 | 53 | // Get the current working directory 54 | cwd, err := os.Getwd() 55 | if err != nil { 56 | fmt.Printf("❌ Unable to get working directory: %s\n", err) 57 | os.Exit(1) 58 | } 59 | 60 | symLink := func(pathInProject string) error { 61 | fullOrgPath := filepath.Join(cwd, pathInProject) 62 | symlinkedPath := filepath.Join(isolationDir, pathInProject) 63 | 64 | // Create the folder in the isolated dir if needed 65 | err := os.MkdirAll(filepath.Dir(symlinkedPath), os.ModePerm) 66 | if err != nil { 67 | return err 68 | } 69 | 70 | // Symlink the file in there 71 | err = os.Symlink(fullOrgPath, symlinkedPath) 72 | if err != nil { 73 | return err 74 | } 75 | 76 | return nil 77 | } 78 | 79 | // Create a blank file which DBT can read 80 | touch := func(pathInProject string) error { 81 | symlinkedPath := filepath.Join(isolationDir, pathInProject) 82 | 83 | // Create the folder in the isolated dir if needed 84 | err := os.MkdirAll(filepath.Dir(symlinkedPath), os.ModePerm) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | // If the file doesn't exist create it with no contents 90 | if _, err := os.Stat(symlinkedPath); os.IsNotExist(err) { 91 | file, err := os.Create(symlinkedPath) 92 | if err != nil { 93 | return err 94 | } 95 | return file.Close() 96 | } 97 | 98 | return nil 99 | } 100 | 101 | // Create a file containing only the config block which DBT can read 102 | stubWithConfig := func(pathInProject string) error { 103 | fullOrgPath := filepath.Join(cwd, pathInProject) 104 | modelBytes, err := ioutil.ReadFile(fullOrgPath) 105 | if err != nil { 106 | fmt.Printf("❌ Unable to to read model: %s\n", err) 107 | return touch(pathInProject) 108 | } 109 | model := string(modelBytes) 110 | configBlockEndIndex := strings.Index(model, "}}") 111 | if configBlockEndIndex == -1 { 112 | fmt.Printf("❌ '%s' has no model config \n", pathInProject) 113 | return touch(pathInProject) 114 | } 115 | configBlock := model[:configBlockEndIndex+2] 116 | 117 | stubPath := filepath.Join(isolationDir, pathInProject) 118 | 119 | // Create the folder in the isolated dir if needed 120 | if err = os.MkdirAll(filepath.Dir(stubPath), os.ModePerm); err != nil { 121 | fmt.Printf("❌ Unable to to create model dir: %s\n", err) 122 | return touch(pathInProject) 123 | } 124 | 125 | // If the file doesn't exist create it with no contents 126 | if _, err := os.Stat(stubPath); os.IsNotExist(err) { 127 | file, err := os.OpenFile(stubPath, os.O_RDWR|os.O_CREATE, 0755) 128 | if err != nil { 129 | fmt.Printf("❌ Unable to open stub file to write: %s\n", err) 130 | return touch(pathInProject) 131 | } 132 | if _, err = file.WriteString(configBlock); err != nil { 133 | fmt.Printf("❌ Unable to write to stub file: %s\n", err) 134 | return touch(pathInProject) 135 | } 136 | return file.Close() 137 | } 138 | 139 | return nil 140 | } 141 | 142 | projectFiles := []string{ 143 | "dbt_project.yml", 144 | "ddbt_config.yml", 145 | "profiles", 146 | "debug", 147 | "docs", 148 | "dbt_modules", 149 | "macros", 150 | // Hack for migration where we're using two version of dbt 151 | "dbt_modules_v0_21_x", 152 | } 153 | 154 | // If we have a model groups file bring that too 155 | if config.GlobalCfg.ModelGroupsFile != "" { 156 | projectFiles = append(projectFiles, config.GlobalCfg.ModelGroupsFile) 157 | } 158 | 159 | for _, file := range projectFiles { 160 | if err := symLink(file); err != nil && !os.IsNotExist(err) { 161 | pb.Stop() 162 | fmt.Printf("❌ Unable to isolate project file `%s`: %s\n", file, err) 163 | os.Exit(1) 164 | } 165 | } 166 | 167 | err = graph.Execute(func(file *fs.File) error { 168 | // Symlink the file from the DAG into the isolated folder 169 | 170 | // Currently ddbt doesn't use dbt materializations. dbt materializations contain macros. 171 | // If one needs to override a macro used in a dbt materialization, isolate-dag will not bring the 172 | // macro into the new isolated environment. Instead, we (as a temporary workaround) copy over the 173 | // whole macros directory and don't symlink individual macros. 174 | if file.Type == fs.MacroFile { 175 | return nil 176 | } 177 | 178 | if err := symLink(file.Path); err != nil { 179 | pb.Stop() 180 | fmt.Printf("❌ Unable to isolate %s `%s`: %s\n", file.Type, file.Name, err) 181 | return err 182 | } 183 | 184 | // Symlink the schema if it exists 185 | schemaFile := strings.TrimSuffix(file.Path, filepath.Ext(file.Path)) + ".yml" 186 | if _, err := os.Stat(schemaFile); file.Schema != nil && err == nil { 187 | if err := symLink(schemaFile); err != nil { 188 | pb.Stop() 189 | fmt.Printf("❌ Unable to isolate schema for %s `%s`: %s\n", file.Type, file.Name, err) 190 | return err 191 | } 192 | } 193 | 194 | // Ensure usptream models are handled 195 | for _, upstream := range file.Upstreams() { 196 | if graph.Contains(upstream) { 197 | continue 198 | } 199 | 200 | switch upstream.Type { 201 | case fs.ModelFile: 202 | // Model's outside of the DAG but referenced by it need to exist for DBT to be able to run on this DAG 203 | // even if we run with the upstream command 204 | if err := stubWithConfig(upstream.Path); err != nil { 205 | pb.Stop() 206 | fmt.Printf("❌ Unable to touch %s `%s`: %s\n", upstream.Type, upstream.Name, err) 207 | return err 208 | } 209 | 210 | default: 211 | // Any other than a model which is being used _should_ already be in the graph 212 | pb.Stop() 213 | fmt.Printf("❌ Unexpected Upstream %s `%s`\n", upstream.Type, upstream.Name) 214 | return err 215 | } 216 | } 217 | 218 | pb.Increment() 219 | return nil 220 | }, config.NumberThreads(), pb) 221 | 222 | if err != nil { 223 | os.Exit(1) 224 | } 225 | 226 | pb.Stop() 227 | 228 | fmt.Printf("Isolation Directory: %s\n", isolationDir) 229 | } 230 | -------------------------------------------------------------------------------- /cmd/lookml_gen.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "ddbt/bigquery" 5 | "ddbt/config" 6 | "errors" 7 | 8 | "fmt" 9 | "os" 10 | 11 | "github.com/atotto/clipboard" 12 | "github.com/spf13/cobra" 13 | ) 14 | 15 | // map bigquery data types to looker data types 16 | var mapBqToLookerDtypes map[string]string = map[string]string{ 17 | "INTEGER": "number", 18 | "FLOAT": "number", 19 | "NUMERIC": "number", 20 | "BOOLEAN": "yesno", 21 | "STRING": "string", 22 | "TIMESTAMP": "time", 23 | "DATETIME": "time", 24 | "DATE": "time", 25 | "TIME": "time", 26 | "BOOL": "yesno", 27 | "ARRAY": "string", 28 | "GEOGRAPHY": "string", 29 | } 30 | 31 | // specify looker timeframes for datetime/date/time variable data types 32 | const timeBlock string = `timeframes: [ 33 | raw, 34 | time, 35 | date, 36 | week, 37 | month, 38 | quarter, 39 | year 40 | ] 41 | ` 42 | 43 | func init() { 44 | rootCmd.AddCommand(lookmlGenCmd) 45 | } 46 | 47 | var lookmlGenCmd = &cobra.Command{ 48 | Use: "lookml-gen [model name]", 49 | Short: "Generates the .view.lkml file for a given model", 50 | Args: cobra.ExactValidArgs(1), 51 | ValidArgsFunction: completeModelFn, 52 | Run: func(cmd *cobra.Command, args []string) { 53 | modelName := args[0] 54 | 55 | // get filesystem, model and target 56 | fileSystem, _ := compileAllModels() 57 | model := fileSystem.Model(modelName) 58 | 59 | target, err := model.GetTarget() 60 | if err != nil { 61 | fmt.Println("Could not get target for schema") 62 | os.Exit(1) 63 | } 64 | fmt.Println("\n🎯 Target for retrieving schema:", target.ProjectID+"."+target.DataSet) 65 | 66 | // generate lookml view 67 | err = generateNewLookmlView(modelName, target) 68 | 69 | if err != nil { 70 | fmt.Println("😒 Something went wrong at lookml view generation: ", err) 71 | os.Exit(1) 72 | } 73 | 74 | }, 75 | } 76 | 77 | func getColumnsForModelWithDtypes(modelName string, target *config.Target) (columns []string, dtypes []string, err error) { 78 | schema, err := bigquery.GetColumnsFromTable(modelName, target) 79 | if err != nil { 80 | fmt.Println("Could not retrieve schema from BigQuery") 81 | os.Exit(1) 82 | } 83 | 84 | // itereate over fields, record field names and data types 85 | for _, fieldSchema := range schema { 86 | columns = append(columns, fieldSchema.Name) 87 | dtypes = append(dtypes, string(fieldSchema.Type)) 88 | } 89 | return columns, dtypes, err 90 | } 91 | 92 | func generateNewLookmlView(modelName string, target *config.Target) error { 93 | bqColumns, bqDtypes, err := getColumnsForModelWithDtypes(modelName, target) 94 | if err != nil { 95 | fmt.Println("Retrieved BigQuery schema but failed to parse it") 96 | os.Exit(1) 97 | } 98 | 99 | // initialise lookml view head 100 | lookmlView := "view: " + modelName + " {\n\n" 101 | lookmlView += "sql_table_name: `" + target.ProjectID + "." + target.DataSet + "." + modelName + "` ;;\n" 102 | 103 | // add dimensions and appropriate blocks for each field 104 | for i := 0; i < len(bqColumns); i++ { 105 | colName := bqColumns[i] 106 | colDtype := mapBqToLookerDtypes[bqDtypes[i]] 107 | if colDtype == "" { 108 | return errors.New("Did not find Looker data type corresponding to BigQuery data type: " + bqDtypes[i]) 109 | } 110 | newBlock := "\n" 111 | 112 | if colDtype == "date_time" || colDtype == "date" || colDtype == "time" { 113 | newBlock += "dimension_group: " + colName + " {\n" 114 | newBlock += "type: " + colDtype + "\n" 115 | newBlock += timeBlock 116 | } else { 117 | newBlock += "dimension: " + colName + " {\n" 118 | newBlock += "type: " + colDtype + "\n" 119 | } 120 | 121 | newBlock += "sql: ${TABLE}." + colName + " ;;\n}\n" 122 | 123 | lookmlView += newBlock 124 | } 125 | 126 | // add closing curly bracket and copy to clipboard 127 | lookmlView += "}" 128 | 129 | err = clipboard.WriteAll(lookmlView) 130 | if err != nil { 131 | fmt.Println("Could not write generated LookML to your clipboard") 132 | os.Exit(1) 133 | } 134 | fmt.Println("\n✅ LookML view for " + modelName + " has been copied to your clipboard!") 135 | 136 | return nil 137 | } 138 | -------------------------------------------------------------------------------- /cmd/model_completion.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "sort" 7 | "strings" 8 | "sync" 9 | 10 | "ddbt/compiler" 11 | "ddbt/config" 12 | "ddbt/fs" 13 | 14 | "github.com/spf13/cobra" 15 | ) 16 | 17 | var ( 18 | readFSOnce sync.Once 19 | cachedFS *fs.FileSystem 20 | ) 21 | 22 | // completeModelFn is a custom valid argument function for cobra.Command. 23 | func completeModelFn(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { 24 | if len(args) > 0 { 25 | // Only complete the first arg. 26 | return nil, cobra.ShellCompDirectiveNoFileComp 27 | } 28 | return matchModel(toComplete), cobra.ShellCompDirectiveNoFileComp 29 | } 30 | 31 | // completeModelFilterFn is a custom valid argument function for cobra.Command. 32 | // It is a variation of completeModelFn, but supports model filters 33 | // (e.g. specifying upstream with + or with tags) 34 | func completeModelFilterFn(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { 35 | if len(args) > 0 { 36 | // Only complete the first arg. 37 | return nil, cobra.ShellCompDirectiveNoFileComp 38 | } 39 | switch { 40 | case strings.HasPrefix(toComplete, "tag:"): // try to match tag:key=value 41 | return matchTag(toComplete[4:]), cobra.ShellCompDirectiveNoFileComp 42 | case strings.HasPrefix(toComplete, "+"): // try to match +model_name 43 | models := matchModel(toComplete[1:]) 44 | suggestion := make([]string, 0, len(models)) 45 | for _, m := range models { 46 | suggestion = append(suggestion, "+"+m) 47 | } 48 | return suggestion, cobra.ShellCompDirectiveNoFileComp 49 | } 50 | // Normal model matching 51 | return matchModel(toComplete), cobra.ShellCompDirectiveNoFileComp 52 | } 53 | 54 | func getFileSystem() *fs.FileSystem { 55 | readFSOnce.Do(func() { 56 | // Read filesystem w/o output as it'll interfere with autocompletion 57 | fileSystem, err := fs.ReadFileSystem(ioutil.Discard) 58 | if err != nil { 59 | cobra.CompError(fmt.Sprintf("❌ Unable to read filesystem: %s\n", err)) 60 | } 61 | cachedFS = fileSystem 62 | }) 63 | return cachedFS 64 | } 65 | 66 | // matchModel returns a list of models with the given prefix. 67 | // If prefix is empty, it returns all models. 68 | func matchModel(prefix string) []string { 69 | fileSys := getFileSystem() 70 | if fileSys != nil { 71 | matched := make([]string, 0, len(fileSys.Models())) 72 | for _, m := range fileSys.Models() { 73 | if strings.HasPrefix(m.Name, prefix) { 74 | // Include as suggestion: 75 | // model_name -- full/path/to/model_name.sql 76 | matched = append(matched, fmt.Sprintf("%s\t%s", m.Name, m.Path)) 77 | } 78 | } 79 | return matched 80 | } 81 | return nil // Nothing matched 82 | } 83 | 84 | // matchModel returns a list of tags with the given prefix. 85 | // If prefix is empty, it returns all tags. 86 | // A tag is in the format of key=value. 87 | func matchTag(prefix string) []string { 88 | fileSys := getFileSystem() 89 | if fileSys != nil { 90 | tags := getAllTags(fileSys) 91 | var matched []string 92 | for tagName, files := range tags { 93 | if strings.HasPrefix(tagName, prefix) { 94 | // Include as suggestion: 95 | // tag:key=value -- Matches N models 96 | matched = append(matched, fmt.Sprintf("tag:%s\tMatches %d models", tagName, len(files))) 97 | } 98 | } 99 | // Sort output so consecutive autocompletes suggestions are stable 100 | sort.Strings(matched) 101 | return matched 102 | } 103 | return nil 104 | } 105 | 106 | // getAllTags reads all the tags on models. 107 | func getAllTags(fileSys *fs.FileSystem) map[string][]string { 108 | if fileSys != nil { 109 | tags := make(map[string][]string) 110 | // This is simplified version of compileAllModels() to read all tags 111 | // from the models. 112 | for _, f := range fileSys.AllFiles() { 113 | if err := compiler.ParseFile(f); err != nil { 114 | cobra.CompError(fmt.Sprintf("❌ Unable to parse file %s: %s\n", f.Path, err)) 115 | continue 116 | } 117 | } 118 | gc, err := compiler.NewGlobalContext(config.GlobalCfg, fileSys) 119 | if err != nil { 120 | cobra.CompError(fmt.Sprintf("❌ Unable to create a global context: %s\n", err)) 121 | } 122 | 123 | for _, f := range append(fileSys.Macros(), fileSys.Models()...) { 124 | if err := compiler.CompileModel(f, gc, false); err != nil { 125 | cobra.CompError(fmt.Sprintf("❌ Unable to compile file %s: %s\n", f.Path, err)) 126 | continue 127 | } 128 | for _, tag := range f.GetTags() { 129 | tags[tag] = append(tags[tag], f.Name) 130 | } 131 | } 132 | return tags 133 | } 134 | return nil 135 | } 136 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path" 7 | "path/filepath" 8 | 9 | "github.com/spf13/cobra" 10 | 11 | "ddbt/bigquery" 12 | "ddbt/compiler" 13 | "ddbt/config" 14 | "ddbt/utils" 15 | ) 16 | 17 | var rootCmd = &cobra.Command{ 18 | Use: "ddbt", 19 | Short: "Dom's Data Build tool is very fast version of DBT", 20 | Long: "DDBT is an experimental drop in replacement for DBT which aims to be much faster at building the DAG for projects with large numbers of models", 21 | Version: utils.DdbtVersion, 22 | PersistentPreRun: func(cmd *cobra.Command, args []string) { 23 | // Do not run init if we're running info commands which aren't actually going to execute operate on a project 24 | if cmd != versionCmd && cmd != completionCmd && cmd.Name() != "help" { 25 | initDDBT() 26 | } 27 | }, 28 | } 29 | 30 | var ( 31 | targetProfile string 32 | upstreamProfile string 33 | threads int 34 | customConfigPath string 35 | ) 36 | 37 | func init() { 38 | rootCmd.PersistentFlags().StringVarP(&targetProfile, "target", "t", "", "Which target profile to use") 39 | rootCmd.PersistentFlags().StringVarP(&upstreamProfile, "upstream", "u", "", "Which target profile to use when reading data outside the current DAG") 40 | rootCmd.PersistentFlags().IntVar(&threads, "threads", 0, "How many threads to execute with") 41 | rootCmd.PersistentFlags().StringVarP(&customConfigPath, "custom-config-path", "c", "", "Pass in a custom config path") 42 | } 43 | 44 | func Execute() { 45 | if err := rootCmd.Execute(); err != nil { 46 | fmt.Println(err) 47 | os.Exit(1) 48 | } 49 | } 50 | 51 | func initDDBT() { 52 | // If you happen to be one folder up from the DBT project, we'll cd in there for you to be nice :) 53 | cdIntoDBTFolder() 54 | 55 | // Read the project config 56 | cfg, err := config.Read(targetProfile, upstreamProfile, threads, customConfigPath, compiler.CompileStringWithCache) 57 | if err != nil { 58 | fmt.Printf("❌ Unable to load config: %s\n", err) 59 | os.Exit(1) 60 | } 61 | 62 | // Init our connection to BigQuery 63 | if err := bigquery.Init(cfg); err != nil { 64 | fmt.Printf("❌ Unable to init BigQuery: %s\n", err) 65 | os.Exit(1) 66 | } 67 | } 68 | 69 | func cdIntoDBTFolder() { 70 | wd, err := os.Getwd() 71 | if err != nil { 72 | panic(err) 73 | } 74 | 75 | if path.Base(wd) != "dbt" { 76 | if stat, err := os.Stat(filepath.Join(wd, "dbt")); !os.IsNotExist(err) && stat.IsDir() { 77 | err = os.Chdir(filepath.Join(wd, "dbt")) 78 | if err != nil { 79 | panic(err) 80 | } 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /cmd/schema_gen_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestAddMissingColumnsToSchema(t *testing.T) { 11 | allColumns := []string{ 12 | "column_a", 13 | "column_b", 14 | "column_c", 15 | "column_d", 16 | "column_e", 17 | } 18 | 19 | originalSchemaModel := generateNewSchemaModel("test_model", allColumns[:3]) 20 | fullSchemaModel := generateNewSchemaModel("test_model", allColumns) 21 | 22 | addMissingColumnsToSchema(originalSchemaModel, allColumns) 23 | assert.Equal(t, originalSchemaModel, fullSchemaModel) 24 | require.Len(t, originalSchemaModel.Columns, len(allColumns), "Wrong number of columns in result") 25 | } 26 | 27 | func TestRemoveOutdatedColumnsFromSchema(t *testing.T) { 28 | allColumns := []string{ 29 | "column_a", 30 | "column_b", 31 | "column_c", 32 | "column_d", 33 | "column_e", 34 | } 35 | updatedColumns := []string{ 36 | "column_a", 37 | "column_d", 38 | } 39 | fullSchemaModel := generateNewSchemaModel("test_model", allColumns) 40 | updatedSchemaModel := generateNewSchemaModel("test_model", updatedColumns) 41 | 42 | removeOutdatedColumnsFromSchema(fullSchemaModel, updatedColumns) 43 | assert.Equal(t, updatedSchemaModel, fullSchemaModel) 44 | require.Len(t, fullSchemaModel.Columns, 2, "Wrong number of columns in result") 45 | } 46 | -------------------------------------------------------------------------------- /cmd/seed.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "ddbt/bigquery" 6 | "ddbt/fs" 7 | "ddbt/utils" 8 | "fmt" 9 | "os" 10 | 11 | "github.com/spf13/cobra" 12 | ) 13 | 14 | func init() { 15 | rootCmd.AddCommand(seedCommand) 16 | } 17 | 18 | var seedCommand = &cobra.Command{ 19 | Use: "seed", 20 | Short: "Load data in the data warehouse with seed files", 21 | Run: func(cmd *cobra.Command, args []string) { 22 | fileSystem, err := fs.ReadFileSystem(os.Stdout) 23 | if err != nil { 24 | fmt.Printf("❌ Unable to read filesystem: %s\n", err) 25 | os.Exit(1) 26 | } 27 | 28 | if err := loadSeeds(fileSystem); err != nil { 29 | fmt.Printf("❌ %s\n", err) 30 | os.Exit(1) 31 | } 32 | }, 33 | } 34 | 35 | func loadSeeds(fileSystem *fs.FileSystem) error { 36 | seeds := fileSystem.Seeds() 37 | 38 | ctx, cancel := context.WithCancel(context.Background()) 39 | defer cancel() 40 | 41 | if err := readSeedColumns(ctx, seeds); err != nil { 42 | return err 43 | } 44 | if err := uploadSeeds(ctx, seeds); err != nil { 45 | return err 46 | } 47 | return nil 48 | } 49 | 50 | func readSeedColumns(ctx context.Context, seeds []*fs.SeedFile) error { 51 | pb := utils.NewProgressBar("🚜 Inferring Seed Schema", len(seeds)) 52 | defer pb.Stop() 53 | 54 | return fs.ProcessSeeds( 55 | seeds, 56 | func(seed *fs.SeedFile) error { 57 | if err := seed.ReadColumns(); err != nil { 58 | return err 59 | } 60 | 61 | pb.Increment() 62 | return nil 63 | }, 64 | nil, 65 | ) 66 | } 67 | 68 | func uploadSeeds(ctx context.Context, seeds []*fs.SeedFile) error { 69 | pb := utils.NewProgressBar("🌱 Uploading Seeds", len(seeds)) 70 | defer pb.Stop() 71 | 72 | return fs.ProcessSeeds( 73 | seeds, 74 | func(seed *fs.SeedFile) error { 75 | if err := bigquery.LoadSeedFile(ctx, seed); err != nil { 76 | return err 77 | } 78 | 79 | pb.Increment() 80 | return nil 81 | }, 82 | nil, 83 | ) 84 | } 85 | -------------------------------------------------------------------------------- /cmd/show.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/spf13/cobra" 8 | 9 | "ddbt/bigquery" 10 | "ddbt/compiler" 11 | ) 12 | 13 | func init() { 14 | rootCmd.AddCommand(showCmd) 15 | } 16 | 17 | var showCmd = &cobra.Command{ 18 | Use: "show [model name]", 19 | Short: "Shows the SQL that would be executed for the given model", 20 | Args: cobra.ExactValidArgs(1), 21 | Run: func(cmd *cobra.Command, args []string) { 22 | fmt.Println(getModelSQL(args[0])) 23 | }, 24 | ValidArgsFunction: completeModelFn, 25 | } 26 | 27 | func getModelSQL(modelName string) string { 28 | fileSystem, gc := compileAllModels() 29 | 30 | model := fileSystem.Model(modelName) 31 | if model == nil { 32 | fmt.Printf("❌ Model %s not found\n", modelName) 33 | os.Exit(1) 34 | } 35 | 36 | if model.IsDynamicSQL() || upstreamProfile != "" { 37 | if err := compiler.CompileModel(model, gc, true); err != nil { 38 | fmt.Printf("❌ Unable to compile dynamic SQL: %s\n", err) 39 | os.Exit(1) 40 | } 41 | } 42 | 43 | return bigquery.BuildQuery(model) 44 | } 45 | -------------------------------------------------------------------------------- /cmd/showDAG.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/spf13/cobra" 8 | 9 | "ddbt/fs" 10 | "ddbt/utils" 11 | ) 12 | 13 | func init() { 14 | rootCmd.AddCommand(showDAG) 15 | addModelsFlag(showDAG) 16 | addFailOnNotFoundFlag(showDAG) 17 | } 18 | 19 | var showDAG = &cobra.Command{ 20 | Use: "show-dag", 21 | Short: "Shows the order in which the DAG would execute", 22 | Run: func(cmd *cobra.Command, args []string) { 23 | fileSystem, _ := compileAllModels() 24 | 25 | // If we've been given a model to run, run it 26 | graph := buildGraph(fileSystem, ModelFilters) 27 | 28 | printGraph(graph) 29 | }, 30 | } 31 | 32 | func printGraph(graph *fs.Graph) { 33 | pb := utils.NewProgressBar("🔖 Writing DAG out", graph.Len()) 34 | defer pb.Stop() 35 | 36 | var builder strings.Builder 37 | 38 | builder.WriteRune('\n') 39 | 40 | _ = graph.Execute( 41 | func(file *fs.File) error { 42 | if file.Type == fs.ModelFile { 43 | builder.WriteString("- ") 44 | builder.WriteString(file.Name) 45 | builder.WriteRune('\n') 46 | } 47 | 48 | pb.Increment() 49 | 50 | return nil 51 | }, 52 | 1, 53 | pb, 54 | ) 55 | 56 | pb.Stop() 57 | 58 | fmt.Println(builder.String()) 59 | } 60 | -------------------------------------------------------------------------------- /cmd/test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "strings" 8 | "sync" 9 | 10 | "github.com/atotto/clipboard" 11 | "github.com/spf13/cobra" 12 | 13 | "ddbt/bigquery" 14 | "ddbt/compiler" 15 | "ddbt/fs" 16 | "ddbt/utils" 17 | ) 18 | 19 | func init() { 20 | rootCmd.AddCommand(testCmd) 21 | addModelsFlag(testCmd) 22 | addFailOnNotFoundFlag(testCmd) 23 | } 24 | 25 | var testCmd = &cobra.Command{ 26 | Use: "test", 27 | Short: "Tests the DAG", 28 | Long: "Will execute any tests which reference models in the target DAG", 29 | Example: "ddbt test -m +my_model", 30 | Run: func(cmd *cobra.Command, args []string) { 31 | fileSystem, globalContext := compileAllModels() 32 | 33 | // If we've been given a model to run, run it 34 | graph := buildGraph(fileSystem, ModelFilters) 35 | 36 | // Add all tests which reference the graph 37 | tests := graph.AddReferencingTests() 38 | 39 | if executeTests(tests, globalContext, graph) { 40 | os.Exit(2) // Exit with a test error 41 | } 42 | }, 43 | } 44 | 45 | func executeTests(tests []*fs.File, globalContext *compiler.GlobalContext, graph *fs.Graph) bool { 46 | pb := utils.NewProgressBar("🔬 Running Tests", len(tests)) 47 | 48 | ctx, cancel := context.WithCancel(context.Background()) 49 | 50 | var m sync.Mutex 51 | widestTestName := 0 52 | type testResult struct { 53 | file *fs.File 54 | name string 55 | rows uint64 56 | err error 57 | query string 58 | } 59 | testResults := make(map[*fs.File]testResult) 60 | 61 | _ = fs.ProcessFiles( 62 | tests, 63 | func(file *fs.File) error { 64 | if file.IsDynamicSQL() { 65 | if err := compiler.CompileModel(file, globalContext, true); err != nil { 66 | pb.Stop() 67 | fmt.Printf("❌ %s\n", err) 68 | cancel() 69 | os.Exit(1) 70 | } 71 | } 72 | 73 | query := bigquery.BuildQuery(file) 74 | 75 | if strings.TrimSpace(query) != "" { 76 | target, err := file.GetTarget() 77 | if err != nil { 78 | pb.Stop() 79 | fmt.Printf("❌ Unable to get target for %s: %s\n", file.Name, err) 80 | cancel() 81 | os.Exit(1) 82 | } 83 | 84 | var rows uint64 85 | 86 | if file.GetConfig("isSchemaTest").BooleanValue { 87 | // schema tests: applied in YAML, returns the number of records that do not pass an assertion — 88 | // when this number is 0, all records pass, therefore, your test passes 89 | var results [][]bigquery.Value 90 | results, _, err = bigquery.GetRows(ctx, query, target) 91 | 92 | if err == nil { 93 | if len(results) != 1 { 94 | err = fmt.Errorf("a schema test should only return 1 row, got %d", len(results)) 95 | } else if len(results[0]) != 1 { 96 | err = fmt.Errorf("a schema test should only return 1 column, got %d", len(results[0])) 97 | } else { 98 | rows, err = bigquery.ValueAsUint64(results[0][0]) 99 | } 100 | } 101 | } else { 102 | // data tests: specific queries that return 0 records 103 | rows, err = bigquery.NumberRows(query, target) 104 | } 105 | 106 | m.Lock() 107 | testResults[file] = testResult{ 108 | file: file, 109 | name: file.Name, 110 | rows: rows, 111 | err: err, 112 | query: query, 113 | } 114 | 115 | if len(file.Name) > widestTestName { 116 | widestTestName = len(file.Name) 117 | } 118 | m.Unlock() 119 | } 120 | 121 | pb.Increment() 122 | 123 | return nil 124 | }, 125 | pb, 126 | ) 127 | 128 | pb.Stop() 129 | 130 | var firstError *testResult 131 | 132 | fmt.Printf("\nTest Results:\n") 133 | for test, results := range testResults { 134 | results := results 135 | 136 | // Force this test to be-rerun in future watch loops 137 | graph.UnmarkFileAsRun(results.file) 138 | 139 | var statusText string 140 | var statusEmoji rune 141 | 142 | switch { 143 | case results.err == context.Canceled: 144 | statusText = "Cancelled" 145 | statusEmoji = '🚧' 146 | 147 | case results.err != nil: 148 | statusText = fmt.Sprintf("Error: %s", results.err) 149 | statusEmoji = '🔴' 150 | 151 | case results.rows > 0: 152 | statusText = fmt.Sprintf("%d Failures", results.rows) 153 | statusEmoji = '❌' 154 | 155 | default: 156 | statusText = "Success" 157 | statusEmoji = '✅' 158 | } 159 | 160 | if firstError == nil && statusEmoji != '✅' { 161 | firstError = &results 162 | } 163 | 164 | fmt.Printf( 165 | " %c %s %s %s\n", 166 | statusEmoji, 167 | test.Name, 168 | strings.Repeat(".", widestTestName-len(test.Name)+3), 169 | statusText, 170 | ) 171 | } 172 | 173 | if firstError != nil { 174 | if err := clipboard.WriteAll(firstError.query); err != nil { 175 | fmt.Printf(" Unable to copy query to clipboard: %s\n", err) 176 | } else { 177 | fmt.Printf("📎 Test Query for %s has been copied into your clipboard\n\n", firstError.name) 178 | } 179 | 180 | return true 181 | } 182 | 183 | return false 184 | } 185 | -------------------------------------------------------------------------------- /cmd/version.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/spf13/cobra" 7 | 8 | "ddbt/utils" 9 | ) 10 | 11 | func init() { 12 | rootCmd.AddCommand(versionCmd) 13 | } 14 | 15 | var versionCmd = &cobra.Command{ 16 | Use: "version", 17 | Short: "Prints the version of DDBT", 18 | Run: func(cmd *cobra.Command, args []string) { 19 | fmt.Println("ddbt version", utils.DdbtVersion) 20 | }, 21 | } 22 | -------------------------------------------------------------------------------- /compiler/ExecutionContext.go: -------------------------------------------------------------------------------- 1 | package compiler 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "sync" 7 | 8 | "ddbt/compilerInterface" 9 | "ddbt/config" 10 | "ddbt/fs" 11 | ) 12 | 13 | type ExecutionContext struct { 14 | file *fs.File 15 | fileSystem *fs.FileSystem 16 | varaiblesMutex sync.RWMutex 17 | variables map[string]*compilerInterface.Value 18 | states []map[string]*compilerInterface.Value //nolint:golint,unused,structcheck 19 | isExecuting bool 20 | 21 | globalContext *GlobalContext 22 | parentContext compilerInterface.ExecutionContext 23 | } 24 | 25 | // Ensure our execution context matches the interface in the AST package 26 | var _ compilerInterface.ExecutionContext = &ExecutionContext{} 27 | 28 | func NewExecutionContext(file *fs.File, fileSystem *fs.FileSystem, isExecuting bool, globalContext *GlobalContext, parent compilerInterface.ExecutionContext) *ExecutionContext { 29 | return &ExecutionContext{ 30 | file: file, 31 | fileSystem: fileSystem, 32 | variables: make(map[string]*compilerInterface.Value), 33 | isExecuting: isExecuting, 34 | globalContext: globalContext, 35 | parentContext: parent, 36 | } 37 | } 38 | 39 | func (e *ExecutionContext) SetVariable(name string, value *compilerInterface.Value) { 40 | e.varaiblesMutex.Lock() 41 | e.variables[name] = value 42 | e.varaiblesMutex.Unlock() 43 | } 44 | 45 | func (e *ExecutionContext) GetVariable(name string) *compilerInterface.Value { 46 | e.varaiblesMutex.RLock() 47 | // Then check the local variable map 48 | variable, found := e.variables[name] 49 | e.varaiblesMutex.RUnlock() 50 | 51 | if !found { 52 | return e.parentContext.GetVariable(name) 53 | } else { 54 | return variable 55 | } 56 | } 57 | 58 | func (e *ExecutionContext) RegisterMacro(name string, ec compilerInterface.ExecutionContext, function compilerInterface.FunctionDef) { 59 | e.parentContext.RegisterMacro(name, ec, function) 60 | } 61 | 62 | func (e *ExecutionContext) ErrorAt(part compilerInterface.AST, error string) error { 63 | if part == nil { 64 | return fmt.Errorf("%s @ unknown", error) 65 | } else { 66 | pos := part.Position() 67 | return fmt.Errorf("%s @ %s:%d:%d", error, pos.File, pos.Row, pos.Column) 68 | } 69 | } 70 | 71 | func (e *ExecutionContext) NilResultFor(part compilerInterface.AST) error { 72 | return e.ErrorAt(part, fmt.Sprintf("%v returned a nil result after execution", reflect.TypeOf(part))) 73 | } 74 | 75 | func (e *ExecutionContext) PushState() compilerInterface.ExecutionContext { 76 | return NewExecutionContext(e.file, e.fileSystem, e.isExecuting, e.globalContext, e) 77 | } 78 | 79 | func (e *ExecutionContext) CopyVariablesInto(ec compilerInterface.ExecutionContext) { 80 | e.parentContext.CopyVariablesInto(ec) 81 | 82 | e.varaiblesMutex.RLock() 83 | defer e.varaiblesMutex.RUnlock() 84 | 85 | for key, value := range e.variables { 86 | ec.SetVariable(key, value) 87 | } 88 | } 89 | 90 | func (e *ExecutionContext) RegisterUpstreamAndGetRef(modelName string, fileType string) (*compilerInterface.Value, error) { 91 | var upstream *fs.File 92 | 93 | switch fs.FileType(fileType) { 94 | case fs.ModelFile: 95 | upstream = e.fileSystem.Model(modelName) 96 | 97 | case fs.MacroFile: 98 | upstream = e.fileSystem.Macro(modelName) 99 | 100 | if upstream == nil { 101 | // For tests 102 | upstream = e.fileSystem.Model(modelName) 103 | } 104 | 105 | default: 106 | return nil, fmt.Errorf("unknown file type: %s", fileType) 107 | } 108 | 109 | if upstream == nil { 110 | return nil, fmt.Errorf("Unable to find model `%s`", modelName) 111 | } 112 | 113 | e.file.RecordDependencyOn(upstream) 114 | 115 | target, err := upstream.GetTarget() 116 | if err != nil { 117 | return nil, err 118 | } 119 | 120 | switch upstream.GetMaterialization() { 121 | case "table", "incremental", "project_sharded_table", "view": 122 | //ToDo: views are being treated as tables until they are properly implemented 123 | 124 | // If "--upstream=target" has been provided and this model is not in the DAG, then we read from the upstream 125 | // target, rather than the target defined in "--target=target" 126 | if target.ReadUpstream != nil && !upstream.IsInDAG() { 127 | return compilerInterface.NewString( 128 | "`" + target.ReadUpstream.ProjectID + "`.`" + target.ReadUpstream.DataSet + "`.`" + modelName + "`", 129 | ), nil 130 | } else { 131 | return compilerInterface.NewString( 132 | "`" + target.ProjectID + "`.`" + target.DataSet + "`.`" + modelName + "`", 133 | ), nil 134 | } 135 | 136 | case "ephemeral": 137 | err := CompileModel(upstream, e.globalContext, e.isExecuting) 138 | if err != nil { 139 | return nil, err 140 | } 141 | 142 | cteName := fmt.Sprintf("__dbt__CTE__%s", upstream.Name) 143 | 144 | e.file.Mutex.Lock() 145 | e.file.EphemeralCTES[cteName] = upstream 146 | e.file.Mutex.Unlock() 147 | 148 | return compilerInterface.NewString(cteName), nil 149 | 150 | default: 151 | return nil, fmt.Errorf("unknown materialized config '%s' in model '%s'", upstream.GetMaterialization(), upstream.Name) 152 | } 153 | } 154 | 155 | func (e *ExecutionContext) FileName() string { 156 | return e.file.Name 157 | } 158 | 159 | func (e *ExecutionContext) GetTarget() (*config.Target, error) { 160 | return e.file.GetTarget() 161 | } 162 | 163 | func (e *ExecutionContext) MarkAsDynamicSQL() (*compilerInterface.Value, error) { 164 | e.file.MaskAsDynamicSQL() 165 | return compilerInterface.NewUndefined(), nil 166 | } 167 | -------------------------------------------------------------------------------- /compiler/GlobalContext.go: -------------------------------------------------------------------------------- 1 | package compiler 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | 7 | "ddbt/compiler/dbtUtils" 8 | "ddbt/compilerInterface" 9 | "ddbt/config" 10 | "ddbt/fs" 11 | ) 12 | 13 | type GlobalContext struct { 14 | fileSystem *fs.FileSystem 15 | 16 | macroMutex sync.RWMutex 17 | macros map[string]*macroDef 18 | 19 | constants map[string]*compilerInterface.Value 20 | } 21 | 22 | type macroDef struct { 23 | ec compilerInterface.ExecutionContext 24 | function compilerInterface.FunctionDef 25 | fileName string 26 | } 27 | 28 | var _ compilerInterface.ExecutionContext = &GlobalContext{} 29 | 30 | func NewGlobalContext(cfg *config.Config, fileSystem *fs.FileSystem) (*GlobalContext, error) { 31 | if err := addBuiltInMacros(fileSystem); err != nil { 32 | return nil, err 33 | } 34 | 35 | return &GlobalContext{ 36 | fileSystem: fileSystem, 37 | macros: make(map[string]*macroDef), 38 | constants: map[string]*compilerInterface.Value{ 39 | "adapter": funcMapAsValue(adapterFunctions), 40 | 41 | "dbt_utils": funcMapAsValue(map[string]compilerInterface.FunctionDef{ 42 | "union_all_tables": dbtUtils.UnionAllTables, 43 | "get_column_values": dbtUtils.GetColumnValues, 44 | "pivot": dbtUtils.Pivot, 45 | "unpivot": dbtUtils.Unpivot, 46 | "group_by": dbtUtils.GroupBy, 47 | }), 48 | 49 | "exceptions": funcMapAsValue(funcMap{ 50 | "raise_compiler_error": func(ec compilerInterface.ExecutionContext, caller compilerInterface.AST, args compilerInterface.Arguments) (*compilerInterface.Value, error) { 51 | err := "error raised" 52 | 53 | if len(args) > 0 { 54 | err = args[0].Value.AsStringValue() 55 | } 56 | 57 | return nil, ec.ErrorAt(caller, err) 58 | }, 59 | 60 | "warn": func(ec compilerInterface.ExecutionContext, caller compilerInterface.AST, args compilerInterface.Arguments) (*compilerInterface.Value, error) { 61 | err := "warning raised" 62 | 63 | if len(args) > 0 { 64 | err = args[0].Value.AsStringValue() 65 | } 66 | 67 | fmt.Printf("\n\nWARN: %s @ %s:%d:%d\n\n", err, caller.Position().File, caller.Position().Row, caller.Position().Column) 68 | 69 | return compilerInterface.NewUndefined(), nil 70 | }, 71 | }), 72 | 73 | // We are always executing (https://docs.getdbt.com/reference/dbt-jinja-functions/execute) 74 | "execute": nil, // Set by the compiler when it creates an execution context 75 | 76 | // https://docs.getdbt.com/reference/dbt-jinja-functions/modules 77 | "modules": compilerInterface.NewMap(map[string]*compilerInterface.Value{ 78 | "datetime": funcMapAsValue(datetimeFunctions), 79 | }), 80 | 81 | // https://docs.getdbt.com/reference/dbt-jinja-functions/project_name 82 | "project_name": compilerInterface.NewString(cfg.Name), 83 | 84 | // https://docs.getdbt.com/reference/dbt-jinja-functions/target 85 | "target": nil, // Set by the compiler when it creates an execution context 86 | }, 87 | }, nil 88 | } 89 | 90 | func (g *GlobalContext) SetVariable(name string, value *compilerInterface.Value) { 91 | panic("Cannot set variable on parentContext context - read only during execution") 92 | } 93 | 94 | func (g *GlobalContext) GetVariable(name string) *compilerInterface.Value { 95 | // Check the built in functions first 96 | builtInFunction := builtInFunctions[name] 97 | 98 | // Then check if a macro has been defined 99 | macro, err := g.GetMacro(name) 100 | if err == nil && macro != nil { 101 | // If macro's rely on each other, they may not be compiled yet and they will seperately 102 | // so we can ignore the error 103 | builtInFunction = macro 104 | } 105 | 106 | // Then check the local variable map 107 | variable, found := g.constants[name] 108 | if !found { 109 | if builtInFunction != nil { 110 | return compilerInterface.NewFunction(builtInFunction) 111 | } else { 112 | return &compilerInterface.Value{IsUndefined: true} 113 | } 114 | } else { 115 | return variable 116 | } 117 | } 118 | 119 | func (g *GlobalContext) ErrorAt(part compilerInterface.AST, error string) error { 120 | panic("ErrorAt not implemented for global context") 121 | } 122 | 123 | func (g *GlobalContext) NilResultFor(part compilerInterface.AST) error { 124 | panic("NilResultFor not implemented for global context") 125 | } 126 | 127 | func (g *GlobalContext) PushState() compilerInterface.ExecutionContext { 128 | panic("PushState not implemented for global context") 129 | } 130 | 131 | func (g *GlobalContext) CopyVariablesInto(_ compilerInterface.ExecutionContext) { 132 | // No-op 133 | } 134 | 135 | func (g *GlobalContext) GetMacro(name string) (compilerInterface.FunctionDef, error) { 136 | g.macroMutex.RLock() 137 | macro, found := g.macros[name] 138 | g.macroMutex.RUnlock() 139 | 140 | // Check if it's compiled and registered 141 | if !found { 142 | // Do we have a macro file which isn't compiled yet? 143 | if file := g.fileSystem.Macro(name); file != nil { 144 | // Compile it 145 | if err := ParseFile(file); err != nil { 146 | return nil, err 147 | } 148 | 149 | if err := CompileModel(file, g, true); err != nil { 150 | return nil, err 151 | } 152 | 153 | // Attempt to re-read the compiled macro 154 | g.macroMutex.RLock() 155 | macro, found = g.macros[name] 156 | g.macroMutex.RUnlock() 157 | 158 | // If it's still not found, then the macro is not registering it self with it's filename 159 | if !found { 160 | return nil, fmt.Errorf("The macro file %s is not registering a macro with the same name!", name) 161 | } 162 | } else { 163 | // No macro exists for this 164 | return nil, nil 165 | } 166 | } 167 | 168 | return func(ec compilerInterface.ExecutionContext, caller compilerInterface.AST, args compilerInterface.Arguments) (*compilerInterface.Value, error) { 169 | if _, err := ec.RegisterUpstreamAndGetRef(macro.fileName, string(fs.MacroFile)); err != nil { 170 | return nil, ec.ErrorAt(caller, err.Error()) 171 | } 172 | 173 | newEC := ec.PushState() 174 | // Note we copy any varaibles defined within the macro's own file in to the context being executed here too 175 | macro.ec.CopyVariablesInto(newEC) 176 | 177 | // We keep the caller and execute context however as these will change from when the macro was registered to when 178 | // it is called 179 | newEC.SetVariable("caller", ec.GetVariable("caller")) 180 | newEC.SetVariable("execute", ec.GetVariable("execute")) 181 | 182 | return macro.function(newEC, caller, args) 183 | }, nil 184 | } 185 | 186 | func (g *GlobalContext) RegisterMacro(name string, ec compilerInterface.ExecutionContext, function compilerInterface.FunctionDef) { 187 | g.macroMutex.Lock() 188 | defer g.macroMutex.Unlock() 189 | 190 | g.macros[name] = ¯oDef{ 191 | ec: ec, 192 | function: function, 193 | fileName: ec.FileName(), 194 | } 195 | } 196 | 197 | func (g *GlobalContext) RegisterUpstreamAndGetRef(name string, fileType string) (*compilerInterface.Value, error) { 198 | panic("RegisterUpstreamAndGetRef not implemented for global context") 199 | } 200 | 201 | func (g *GlobalContext) FileName() string { 202 | panic("FileName not implemented for global context") 203 | } 204 | 205 | func (g *GlobalContext) GetTarget() (*config.Target, error) { 206 | panic("GetTarget not implemented for global context") 207 | } 208 | 209 | func (g *GlobalContext) MarkAsDynamicSQL() (*compilerInterface.Value, error) { 210 | panic("Mark as dynamic SQL not support on the global context") 211 | } 212 | 213 | func (g *GlobalContext) UnregisterMacrosInFile(file *fs.File) { 214 | g.macroMutex.Lock() 215 | defer g.macroMutex.Unlock() 216 | 217 | toDelete := make([]string, 0) 218 | 219 | // In case the macro name is different to the file name 220 | // as one file might contain multiple macro's 221 | for key, macroDef := range g.macros { 222 | if macroDef.fileName == file.Name { 223 | toDelete = append(toDelete, key) 224 | } 225 | } 226 | 227 | for _, key := range toDelete { 228 | delete(g.macros, key) 229 | } 230 | } 231 | -------------------------------------------------------------------------------- /compiler/builtInMacros.go: -------------------------------------------------------------------------------- 1 | package compiler 2 | 3 | import "ddbt/fs" 4 | 5 | // All our built in Macros 6 | const builtInMacros = ` 7 | {# This test checks that the value in column_name is always unique #} 8 | {% macro test_unique(model, column_name) %} 9 | WITH test_data AS ( 10 | SELECT 11 | {{ column_name }} AS value, 12 | COUNT({{ column_name }}) AS count 13 | 14 | FROM {{ model }} 15 | 16 | GROUP BY {{ column_name }} 17 | 18 | HAVING COUNT({{ column_name }}) > 1 19 | ) 20 | 21 | SELECT COUNT(*) as num_errors FROM test_data 22 | {% endmacro %} 23 | 24 | 25 | {# This test that the value is never null in column_name #} 26 | {% macro test_not_null(model, column_name) %} 27 | WITH test_data AS ( 28 | SELECT 29 | {{ column_name }} AS value 30 | 31 | FROM {{ model }} 32 | 33 | WHERE {{ column_name }} IS NULL 34 | ) 35 | 36 | SELECT COUNT(*) as num_errors FROM test_data 37 | {% endmacro %} 38 | 39 | 40 | {# This test checks that the value in column_name is always one of the #} 41 | {% macro test_accepted_values(model, column_name, values) %} 42 | WITH test_data AS ( 43 | SELECT 44 | {{ column_name }} AS value 45 | 46 | FROM {{ model }} 47 | 48 | WHERE {{ column_name }} NOT IN ( 49 | {% for value in values -%} 50 | {% if value is string and kwargs.get('quote', true) %}'{{ value }}'{% else %}{{value}}{% endif %} 51 | {%- if not loop.last %}, {% endif %} 52 | {%- endfor %} 53 | ) 54 | ) 55 | 56 | SELECT COUNT(*) as num_errors FROM test_data 57 | {% endmacro %} 58 | 59 | {% macro test_relationships(model, column_name, to, field) %} 60 | WITH test_data AS ( 61 | SELECT 62 | {{ column_name }} AS value 63 | 64 | FROM {{ model }} AS src 65 | 66 | LEFT JOIN {{ to }} AS dest 67 | ON dest.{{ field }} = src.{{ column_name }} 68 | 69 | WHERE dest.{{ field }} IS NULL AND src.{{ column_name }} IS NOT NULL 70 | ) 71 | 72 | SELECT COUNT(*) as num_errors FROM test_data 73 | {% endmacro %} 74 | ` 75 | 76 | // Adds and compiles in built in macros 77 | func addBuiltInMacros(fileSystem *fs.FileSystem) error { 78 | file, err := fileSystem.AddMacroWithContents("built-in-macros", builtInMacros) 79 | if err != nil { 80 | return err 81 | } 82 | 83 | if err := ParseFile(file); err != nil { 84 | return err 85 | } 86 | 87 | return nil 88 | } 89 | -------------------------------------------------------------------------------- /compiler/compiler.go: -------------------------------------------------------------------------------- 1 | package compiler 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | 8 | "ddbt/compilerInterface" 9 | "ddbt/config" 10 | "ddbt/fs" 11 | "ddbt/jinja" 12 | ) 13 | 14 | func ParseFile(file *fs.File) error { 15 | file.Mutex.Lock() 16 | defer file.Mutex.Unlock() 17 | 18 | if file.SyntaxTree == nil { 19 | syntaxTree, err := jinja.Parse(file) 20 | if err != nil { 21 | return err 22 | } 23 | 24 | file.SyntaxTree = syntaxTree 25 | } 26 | return nil 27 | } 28 | 29 | func CompileModel(file *fs.File, gc *GlobalContext, isExecuting bool) error { 30 | ec := NewExecutionContext(file, gc.fileSystem, isExecuting, gc, gc) 31 | 32 | target, err := file.GetTarget() 33 | if err != nil { 34 | return err 35 | } 36 | 37 | ec.SetVariable("this", compilerInterface.NewMap(map[string]*compilerInterface.Value{ 38 | "schema": compilerInterface.NewString(target.DataSet), 39 | "table": compilerInterface.NewString(file.Name), 40 | "name": compilerInterface.NewString(file.Name), 41 | })) 42 | 43 | ec.SetVariable("target", compilerInterface.NewMap(map[string]*compilerInterface.Value{ 44 | "name": compilerInterface.NewString(config.GlobalCfg.Target.Name), 45 | "schema": compilerInterface.NewString(target.DataSet), 46 | "dataset": compilerInterface.NewString(target.DataSet), 47 | "type": compilerInterface.NewString("bigquery"), 48 | "threads": compilerInterface.NewNumber(float64(target.Threads)), 49 | "project": compilerInterface.NewString(target.ProjectID), 50 | })) 51 | 52 | ec.SetVariable("config", file.ConfigObject()) 53 | ec.SetVariable("execute", compilerInterface.NewBoolean(isExecuting)) 54 | 55 | if file.SyntaxTree == nil { 56 | return fmt.Errorf("file %s has not been parsed before we attempt the compile", file.Name) 57 | } 58 | 59 | finalAST, err := file.SyntaxTree.Execute(ec) 60 | if err != nil { 61 | return err 62 | } 63 | 64 | if finalAST == nil { 65 | return errors.New("no AST returned after execution") 66 | } 67 | 68 | if finalAST.Type() != compilerInterface.StringVal && finalAST.Type() != compilerInterface.ReturnVal { 69 | return errors.New("AST did not return a string") 70 | } 71 | 72 | file.Mutex.Lock() 73 | file.CompiledContents = finalAST.AsStringValue() 74 | 75 | if len(file.EphemeralCTES) > 0 { 76 | var builder strings.Builder 77 | 78 | builder.WriteString("WITH ") 79 | 80 | for name, model := range file.EphemeralCTES { 81 | builder.WriteString(name) 82 | builder.WriteString(" AS (\n\t") 83 | model.Mutex.Lock() 84 | builder.WriteString(strings.Replace(strings.TrimSpace(model.CompiledContents), "\n", "\n\t", -1)) 85 | model.Mutex.Unlock() 86 | builder.WriteString("\n),\n\n") 87 | } 88 | 89 | builder.WriteString("__dbt__main_query AS (\n\t") 90 | builder.WriteString(strings.Replace(strings.TrimSpace(file.CompiledContents), "\n", "\n\t", -1)) 91 | builder.WriteString("\n)\n\nSELECT * FROM __dbt__main_query") 92 | 93 | file.CompiledContents = builder.String() 94 | } 95 | 96 | file.Mutex.Unlock() 97 | 98 | return err 99 | } 100 | 101 | func CompileStringWithCache(s string) (string, error) { 102 | fileSystem, err := fs.InMemoryFileSystem(map[string]string{"models/____": s}) 103 | if err != nil { 104 | return "", err 105 | } 106 | 107 | model := fileSystem.Model("____") 108 | err = ParseFile(model) 109 | if err != nil { 110 | return "", err 111 | } 112 | 113 | gc, err := NewGlobalContext(config.GlobalCfg, fileSystem) 114 | if err != nil { 115 | return "", err 116 | } 117 | 118 | for _, file := range fileSystem.Macros() { 119 | if err := CompileModel(file, gc, false); err != nil { 120 | return "", err 121 | } 122 | } 123 | ec := NewExecutionContext(model, fileSystem, true, gc, gc) 124 | 125 | finalValue, err := model.SyntaxTree.Execute(ec) 126 | if err != nil { 127 | return "", err 128 | } 129 | 130 | return finalValue.AsStringValue(), nil 131 | } 132 | 133 | func isOnlyCompilingSQL(ec compilerInterface.ExecutionContext) bool { 134 | value := ec.GetVariable("execute") 135 | 136 | if value.Type() == compilerInterface.BooleanValue { 137 | return !value.BooleanValue 138 | } else { 139 | return true 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /compiler/dbtUtils/queryMacros.go: -------------------------------------------------------------------------------- 1 | package dbtUtils 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strconv" 7 | "strings" 8 | 9 | "ddbt/bigquery" 10 | "ddbt/compilerInterface" 11 | ) 12 | 13 | // GetColumnValues is a fallback GetColumnValuesWithContext 14 | // with a background context. 15 | func GetColumnValues(ec compilerInterface.ExecutionContext, caller compilerInterface.AST, arguments compilerInterface.Arguments) (*compilerInterface.Value, error) { 16 | return GetColumnValuesWithContext(context.Background(), ec, caller, arguments) 17 | } 18 | 19 | func GetColumnValuesWithContext(ctx context.Context, ec compilerInterface.ExecutionContext, caller compilerInterface.AST, arguments compilerInterface.Arguments) (*compilerInterface.Value, error) { 20 | if IsOnlyCompilingSQL(ec) { 21 | return ec.MarkAsDynamicSQL() 22 | } 23 | 24 | args, err := GetArgs(arguments, Param("table"), Param("column"), Param("max_records")) 25 | if err != nil { 26 | return nil, ec.ErrorAt(caller, fmt.Sprintf("%s", err)) 27 | } 28 | 29 | // Build a query to execute 30 | query := fmt.Sprintf( 31 | "SELECT %s as value FROM %s GROUP BY 1 ORDER BY COUNT(*) DESC", 32 | args[1].AsStringValue(), 33 | args[0].AsStringValue(), 34 | ) 35 | 36 | if !args[2].IsUndefined { 37 | num, err := args[2].AsNumberValue() 38 | if err != nil { 39 | return nil, ec.ErrorAt(caller, fmt.Sprintf("%s", err)) 40 | } 41 | 42 | query += " LIMIT " + strconv.Itoa(int(num)) 43 | } 44 | 45 | target, err := ec.GetTarget() 46 | if err != nil { 47 | return nil, ec.ErrorAt(caller, fmt.Sprintf("%s", err)) 48 | } 49 | 50 | rows, _, err := bigquery.GetRows(ctx, query, target) 51 | if err != nil { 52 | return nil, ec.ErrorAt(caller, fmt.Sprintf("get_column_values query returned an error: %s", err)) 53 | } 54 | 55 | result := make([]*compilerInterface.Value, len(rows)) 56 | 57 | for i, row := range rows { 58 | r, err := compilerInterface.NewValueFromInterface(row[0]) 59 | if err != nil { 60 | return nil, ec.ErrorAt(caller, fmt.Sprintf("get_column_values was unable to parse a value: %s", err)) 61 | } 62 | 63 | result[i] = r 64 | } 65 | 66 | return compilerInterface.NewList(result), nil 67 | } 68 | 69 | func Unpivot(ec compilerInterface.ExecutionContext, caller compilerInterface.AST, arguments compilerInterface.Arguments) (*compilerInterface.Value, error) { 70 | if IsOnlyCompilingSQL(ec) { 71 | return ec.MarkAsDynamicSQL() 72 | } 73 | 74 | args, err := GetArgs(arguments, 75 | ParamWithDefault("table", compilerInterface.NewString("")), 76 | ParamWithDefault("cast_to", compilerInterface.NewString("varchar")), 77 | ParamWithDefault("exclude", compilerInterface.NewList(make([]*compilerInterface.Value, 0))), 78 | ParamWithDefault("remove", compilerInterface.NewList(make([]*compilerInterface.Value, 0))), 79 | ParamWithDefault("field_name", compilerInterface.NewString("field_name")), 80 | ParamWithDefault("value_name", compilerInterface.NewString("value_name")), 81 | ) 82 | if err != nil { 83 | return nil, ec.ErrorAt(caller, fmt.Sprintf("%s", err)) 84 | } 85 | 86 | table := args[0].AsStringValue() 87 | castTo := args[1].AsStringValue() 88 | exclude := args[2].ListValue 89 | remove := listToSet(args[3].ListValue) 90 | fieldName := args[4].AsStringValue() 91 | valueName := args[5].AsStringValue() 92 | 93 | excludeSet := listToSet(exclude) 94 | 95 | target, err := ec.GetTarget() 96 | if err != nil { 97 | return nil, ec.ErrorAt(caller, fmt.Sprintf("%s", err)) 98 | } 99 | 100 | columns, err := bigquery.GetColumnsFromTable(table, target) 101 | if err != nil { 102 | return nil, ec.ErrorAt(caller, fmt.Sprintf("Unable to get the columns for %s: %s", table, err)) 103 | } 104 | 105 | var builder strings.Builder 106 | 107 | includeColumns := make([]string, 0) 108 | for _, col := range columns { 109 | lowered := strings.ToLower(col.Name) 110 | 111 | if _, found := excludeSet[lowered]; found { 112 | continue 113 | } 114 | 115 | if _, found := remove[lowered]; found { 116 | continue 117 | } 118 | 119 | includeColumns = append(includeColumns, col.Name) 120 | } 121 | 122 | for i, col := range includeColumns { 123 | if i > 0 { 124 | builder.WriteString("\n UNION ALL \n") 125 | } 126 | 127 | builder.WriteString("SELECT \n\t") 128 | 129 | for _, excluded := range exclude { 130 | builder.WriteString(excluded.AsStringValue()) 131 | builder.WriteString(",\n\t") 132 | } 133 | 134 | // cast('{{ col.column }}' as {{ dbt_utils.type_string() }}) as {{ field_name }}, 135 | builder.WriteString("CAST('") 136 | builder.WriteString(col) 137 | builder.WriteString("' AS STRING) AS ") 138 | builder.WriteString(fieldName) 139 | builder.WriteString(",\n\t") 140 | 141 | // cast({{ col.column }} as {{ cast_to }}) as {{ value_name }} 142 | builder.WriteString("CAST(") 143 | builder.WriteString(col) 144 | builder.WriteString(" AS ") 145 | builder.WriteString(castTo) 146 | builder.WriteString(") AS ") 147 | builder.WriteString(valueName) 148 | builder.WriteString("\n\t") 149 | 150 | builder.WriteString("FROM ") 151 | builder.WriteString(table) 152 | } 153 | 154 | return compilerInterface.NewString(builder.String()), nil 155 | } 156 | -------------------------------------------------------------------------------- /compiler/dbtUtils/replacements.go: -------------------------------------------------------------------------------- 1 | package dbtUtils 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | 8 | "ddbt/bigquery" 9 | "ddbt/compilerInterface" 10 | ) 11 | 12 | func UnionAllTables(ec compilerInterface.ExecutionContext, caller compilerInterface.AST, arguments compilerInterface.Arguments) (*compilerInterface.Value, error) { 13 | args, err := GetArgs(arguments, Param("tables"), Param("column_names")) 14 | if err != nil { 15 | return nil, ec.ErrorAt(caller, fmt.Sprintf("%s", err)) 16 | } 17 | 18 | if args[0].Type() != compilerInterface.ListVal || args[1].Type() != compilerInterface.ListVal { 19 | return nil, ec.ErrorAt(caller, "expected arguments to union all tables to be two lists") 20 | } 21 | 22 | var builder strings.Builder 23 | builder.WriteRune('(') 24 | for i, table := range args[0].ListValue { 25 | if i > 0 { 26 | builder.WriteString(" UNION ALL \n") 27 | } 28 | 29 | builder.WriteString("\n(SELECT ") 30 | 31 | for j, column := range args[1].ListValue { 32 | if j > 0 { 33 | builder.WriteString(", ") 34 | } 35 | 36 | builder.WriteString(column.AsStringValue()) 37 | } 38 | 39 | builder.WriteString(" FROM ") 40 | builder.WriteString(table.AsStringValue()) 41 | builder.WriteRune(')') 42 | } 43 | builder.WriteRune(')') 44 | return compilerInterface.NewString(builder.String()), nil 45 | } 46 | 47 | func GroupBy(ec compilerInterface.ExecutionContext, caller compilerInterface.AST, arguments compilerInterface.Arguments) (*compilerInterface.Value, error) { 48 | args, err := GetArgs(arguments, ParamWithDefault("n", compilerInterface.NewNumber(0))) 49 | if err != nil { 50 | return nil, ec.ErrorAt(caller, fmt.Sprintf("%s", err)) 51 | } 52 | 53 | var builder strings.Builder 54 | builder.WriteString(" GROUP BY ") 55 | 56 | max := int(args[0].NumberValue) + 1 57 | 58 | for i := 1; i < max; i++ { 59 | if i > 1 { 60 | builder.WriteString(", ") 61 | } 62 | 63 | builder.WriteString(strconv.Itoa(i)) 64 | } 65 | builder.WriteRune(' ') 66 | 67 | return compilerInterface.NewString(builder.String()), nil 68 | } 69 | 70 | func Pivot(ec compilerInterface.ExecutionContext, caller compilerInterface.AST, arguments compilerInterface.Arguments) (*compilerInterface.Value, error) { 71 | args, err := GetArgs(arguments, 72 | ParamWithDefault("column", compilerInterface.NewString("")), 73 | ParamWithDefault("values", compilerInterface.NewList(make([]*compilerInterface.Value, 0))), 74 | ParamWithDefault("alias", compilerInterface.NewBoolean(true)), 75 | ParamWithDefault("agg", compilerInterface.NewString("sum")), 76 | ParamWithDefault("cmp", compilerInterface.NewString("=")), 77 | ParamWithDefault("prefix", compilerInterface.NewString("")), 78 | ParamWithDefault("suffix", compilerInterface.NewString("")), 79 | Param("then_value"), 80 | Param("else_value"), 81 | ParamWithDefault("quote_identifiers", compilerInterface.NewBoolean(true)), 82 | ) 83 | if err != nil { 84 | return nil, ec.ErrorAt(caller, fmt.Sprintf("%s", err)) 85 | } 86 | 87 | var builder strings.Builder 88 | 89 | column, values, alias, agg, cmp, prefix, suffix, then_value, else_value, quote_identifiers := 90 | args[0].AsStringValue(), args[1].ListValue, args[2].BooleanValue, args[3].AsStringValue(), args[4].AsStringValue(), args[5].AsStringValue(), args[6].AsStringValue(), args[7], args[8], args[9].BooleanValue 91 | 92 | if then_value.IsUndefined { 93 | then_value = compilerInterface.NewNumber(1) 94 | } 95 | if else_value.IsUndefined { 96 | else_value = compilerInterface.NewNumber(0) 97 | } 98 | 99 | for i, value := range values { 100 | if i > 0 { 101 | builder.WriteRune(',') 102 | } 103 | 104 | // {{ agg }}( CASE WHEN {{ column }} {{ cmp }} '{{ v }}' THEN {{ then_value }} ELSE {{ else_value }} END ) 105 | builder.WriteString(agg) 106 | builder.WriteString("(\nCASE WHEN ") 107 | builder.WriteString(column) 108 | builder.WriteRune(' ') 109 | builder.WriteString(cmp) 110 | builder.WriteRune('\'') 111 | builder.WriteString(value.AsStringValue()) 112 | builder.WriteString("' THEN ") 113 | builder.WriteString(then_value.AsStringValue()) 114 | builder.WriteString(" ELSE ") 115 | builder.WriteString(else_value.AsStringValue()) 116 | builder.WriteString(" END)") 117 | 118 | if alias { 119 | builder.WriteString(" AS ") 120 | 121 | str := fmt.Sprintf("%s%s%s", prefix, value.AsStringValue(), suffix) 122 | if quote_identifiers { 123 | str = bigquery.Quote(str) 124 | } 125 | 126 | builder.WriteString(str) 127 | } 128 | 129 | } 130 | 131 | return compilerInterface.NewString(builder.String()), nil 132 | } 133 | 134 | func listToSet(list []*compilerInterface.Value) map[string]struct{} { 135 | set := make(map[string]struct{}) 136 | 137 | for _, value := range list { 138 | set[strings.ToLower(value.AsStringValue())] = struct{}{} 139 | } 140 | 141 | return set 142 | } 143 | -------------------------------------------------------------------------------- /compiler/dbtUtils/utils.go: -------------------------------------------------------------------------------- 1 | package dbtUtils 2 | 3 | import ( 4 | "fmt" 5 | 6 | "ddbt/compilerInterface" 7 | ) 8 | 9 | func Param(name string) compilerInterface.Argument { 10 | return ParamWithDefault(name, nil) 11 | } 12 | 13 | func ParamWithDefault(name string, value *compilerInterface.Value) compilerInterface.Argument { 14 | return compilerInterface.Argument{ 15 | Name: name, 16 | Value: value, 17 | } 18 | } 19 | 20 | func GetArgs(arguments compilerInterface.Arguments, params ...compilerInterface.Argument) ([]*compilerInterface.Value, error) { 21 | args := make([]*compilerInterface.Value, len(params)) 22 | 23 | // quick lookup map 24 | namedArgs := make(map[string]*compilerInterface.Value) 25 | for _, arg := range arguments { 26 | if arg.Name != "" { 27 | namedArgs[arg.Name] = arg.Value 28 | } 29 | } 30 | 31 | stillOrdered := true 32 | 33 | // Process all the parameters, checking what args where provided 34 | for i, param := range params { 35 | if value, found := namedArgs[param.Name]; found { 36 | args[i] = value 37 | 38 | stillOrdered = len(arguments) > i && arguments[i].Name == param.Name 39 | } else if len(arguments) <= i || arguments[i].Name != "" { 40 | stillOrdered = false 41 | if param.Value != nil { 42 | args[i] = param.Value 43 | } else { 44 | args[i] = compilerInterface.NewUndefined() 45 | } 46 | } else if !stillOrdered { 47 | return nil, fmt.Errorf("Named arguments have been used out of order, please either used all named arguments or keep them in order. Unable to identify what %s should be.", param.Name) 48 | } else { 49 | args[i] = arguments[i].Value 50 | } 51 | 52 | // Remove a return wrapper 53 | args[i] = args[i].Unwrap() 54 | 55 | // Check types 56 | if param.Value != nil && !args[i].IsUndefined { 57 | if param.Value.Type() != args[i].Type() { 58 | return nil, fmt.Errorf("Paramter %s should be a %s got a %s", param.Name, param.Value.Type(), args[i].Type()) 59 | } 60 | } 61 | } 62 | 63 | return args, nil 64 | } 65 | 66 | func IsOnlyCompilingSQL(ec compilerInterface.ExecutionContext) bool { 67 | value := ec.GetVariable("execute") 68 | 69 | if value.Type() == compilerInterface.BooleanValue { 70 | return !value.BooleanValue 71 | } else { 72 | return true 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /compilerInterface/types.go: -------------------------------------------------------------------------------- 1 | package compilerInterface 2 | 3 | import ( 4 | "ddbt/config" 5 | "ddbt/jinja/lexer" 6 | ) 7 | 8 | type ExecutionContext interface { 9 | SetVariable(name string, value *Value) 10 | GetVariable(name string) *Value 11 | 12 | ErrorAt(part AST, error string) error 13 | NilResultFor(part AST) error 14 | PushState() ExecutionContext 15 | CopyVariablesInto(ec ExecutionContext) 16 | 17 | RegisterMacro(name string, ec ExecutionContext, function FunctionDef) 18 | RegisterUpstreamAndGetRef(name string, fileType string) (*Value, error) 19 | 20 | FileName() string 21 | GetTarget() (*config.Target, error) 22 | MarkAsDynamicSQL() (*Value, error) 23 | } 24 | 25 | type AST interface { 26 | Execute(ec ExecutionContext) (*Value, error) 27 | Position() lexer.Position 28 | String() string 29 | } 30 | 31 | type Argument struct { 32 | Name string // optional 33 | Value *Value 34 | } 35 | 36 | type Arguments []Argument 37 | 38 | func (args Arguments) ToVarArgs() *Value { 39 | varargs := make([]*Value, len(args)) 40 | 41 | for i, value := range args { 42 | varargs[i] = value.Value 43 | } 44 | 45 | return NewList(varargs) 46 | } 47 | 48 | type FunctionDef func(ec ExecutionContext, caller AST, args Arguments) (*Value, error) 49 | -------------------------------------------------------------------------------- /compilerInterface/value.go: -------------------------------------------------------------------------------- 1 | package compilerInterface 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "reflect" 7 | "strconv" 8 | "strings" 9 | 10 | "ddbt/jinja/lexer" 11 | ) 12 | 13 | type ValueType string 14 | 15 | const ( 16 | Dynamic ValueType = "" 17 | Undefined ValueType = "undefined" 18 | NullVal ValueType = "null" 19 | StringVal ValueType = "String" 20 | NumberVal ValueType = "Number" 21 | BooleanValue ValueType = "Boolean" 22 | MapVal ValueType = "Map" 23 | ListVal ValueType = "List" 24 | FunctionalVal ValueType = "Function" 25 | ReturnVal ValueType = "ReturnVal" // marker to shortcut the rest of the file 26 | ) 27 | 28 | type Value struct { 29 | ValueType ValueType 30 | StringValue string 31 | NumberValue float64 32 | BooleanValue bool 33 | MapValue map[string]*Value 34 | ListValue []*Value 35 | Function FunctionDef 36 | IsUndefined bool 37 | IsNull bool 38 | ReturnValue *Value 39 | } 40 | 41 | func NewBoolean(value bool) *Value { 42 | return &Value{ValueType: BooleanValue, BooleanValue: value} 43 | } 44 | 45 | func NewString(value string) *Value { 46 | return &Value{ValueType: StringVal, StringValue: value} 47 | } 48 | 49 | func NewNumber(value float64) *Value { 50 | return &Value{ValueType: NumberVal, NumberValue: value} 51 | } 52 | 53 | func NewMap(data map[string]*Value) *Value { 54 | return &Value{ValueType: MapVal, MapValue: data} 55 | } 56 | 57 | func NewList(data []*Value) *Value { 58 | return &Value{ValueType: ListVal, ListValue: data} 59 | } 60 | 61 | func NewStringList(data []string) *Value { 62 | l := make([]*Value, len(data)) 63 | 64 | for i, s := range data { 65 | l[i] = NewString(s) 66 | } 67 | 68 | return NewList(l) 69 | } 70 | 71 | func NewFunction(f FunctionDef) *Value { 72 | return &Value{ 73 | ValueType: FunctionalVal, 74 | Function: f, 75 | } 76 | } 77 | 78 | func NewUndefined() *Value { 79 | return &Value{ 80 | ValueType: Undefined, 81 | IsUndefined: true, 82 | } 83 | } 84 | 85 | func NewReturnValue(value *Value) *Value { 86 | return &Value{ 87 | ValueType: ReturnVal, 88 | ReturnValue: value, 89 | } 90 | } 91 | 92 | func (v *Value) Type() ValueType { 93 | if v == nil { 94 | return NullVal 95 | } 96 | 97 | if v.ValueType != Dynamic { 98 | return v.ValueType 99 | } 100 | 101 | switch { 102 | case v.IsUndefined: 103 | return Undefined 104 | 105 | case v.IsNull: 106 | return NullVal 107 | 108 | case v.MapValue != nil: 109 | return MapVal 110 | 111 | case v.ListValue != nil: 112 | return ListVal 113 | 114 | case v.NumberValue != 0: 115 | return NumberVal 116 | 117 | case v.StringValue != "": 118 | return StringVal 119 | 120 | case v.Function != nil: 121 | // Note: function call is last so that if a user overrides a function 122 | // with a value, we could still the original function/macro 123 | return FunctionalVal 124 | 125 | default: 126 | // Incase of "" as the value 127 | return StringVal 128 | } 129 | } 130 | 131 | func (v *Value) Properties(isForFunctionCall bool) map[string]*Value { 132 | switch v.Type() { 133 | case MapVal: 134 | return v.MapValue 135 | 136 | case ListVal: 137 | extendFunc := NewFunction(func(_ ExecutionContext, _ AST, args Arguments) (*Value, error) { 138 | for _, arg := range args[0].Value.ListValue { 139 | if arg != v { 140 | v.ListValue = append(v.ListValue, arg) 141 | } 142 | } 143 | return v, nil 144 | }) 145 | 146 | return map[string]*Value{ 147 | "items": NewFunction(func(_ ExecutionContext, _ AST, _ Arguments) (*Value, error) { return v, nil }), 148 | "extend": extendFunc, 149 | "append": extendFunc, 150 | } 151 | 152 | case ReturnVal: 153 | return v.ReturnValue.Properties(isForFunctionCall) 154 | 155 | case StringVal: 156 | if !isForFunctionCall { 157 | return nil 158 | } 159 | 160 | return map[string]*Value{ 161 | "upper": NewFunction(func(_ ExecutionContext, _ AST, _ Arguments) (*Value, error) { 162 | return NewString(strings.ToUpper(v.StringValue)), nil 163 | }), 164 | "lower": NewFunction(func(_ ExecutionContext, _ AST, _ Arguments) (*Value, error) { 165 | return NewString(strings.ToUpper(v.StringValue)), nil 166 | }), 167 | } 168 | 169 | default: 170 | return nil 171 | } 172 | } 173 | 174 | func (v *Value) TruthyValue() bool { 175 | switch v.Type() { 176 | case BooleanValue: 177 | return v.BooleanValue 178 | 179 | case NumberVal: 180 | return v.NumberValue != 0 181 | 182 | case StringVal: 183 | return v.StringValue != "" 184 | 185 | case ListVal: 186 | return len(v.ListValue) > 0 187 | 188 | case MapVal: 189 | return len(v.MapValue) > 0 190 | 191 | case NullVal, Undefined: 192 | return false 193 | 194 | case ReturnVal: 195 | return v.ReturnValue.TruthyValue() 196 | 197 | default: 198 | panic("Unable to truthy " + v.Type()) 199 | } 200 | } 201 | 202 | func (v *Value) AsStringValue() string { 203 | switch v.Type() { 204 | case BooleanValue: 205 | if v.BooleanValue { 206 | return "TRUE" 207 | } else { 208 | return "FALSE" 209 | } 210 | 211 | case NumberVal: 212 | return strconv.FormatFloat(v.NumberValue, 'f', -1, 64) 213 | 214 | case StringVal: 215 | return v.StringValue 216 | 217 | case ListVal: 218 | return fmt.Sprintf("%p", v.ListValue) 219 | 220 | case MapVal: 221 | return fmt.Sprintf("%p", v.MapValue) 222 | 223 | case NullVal, Undefined: 224 | return "" 225 | 226 | case ReturnVal: 227 | return v.ReturnValue.AsStringValue() 228 | 229 | default: 230 | panic("Unable to truthy " + v.Type()) 231 | } 232 | } 233 | 234 | func (v *Value) AsNumberValue() (float64, error) { 235 | switch v.Type() { 236 | case BooleanValue: 237 | if v.BooleanValue { 238 | return 1, nil 239 | } else { 240 | return 0, nil 241 | } 242 | 243 | case NumberVal: 244 | return v.NumberValue, nil 245 | 246 | case StringVal: 247 | return strconv.ParseFloat(v.StringValue, 64) 248 | 249 | case NullVal, Undefined: 250 | return 0, nil 251 | 252 | case ReturnVal: 253 | return v.ReturnValue.AsNumberValue() 254 | 255 | default: 256 | return 0, fmt.Errorf("unable to convert %s to number", v.Type()) 257 | } 258 | } 259 | 260 | func (v *Value) Unwrap() *Value { 261 | if v.ValueType == ReturnVal { 262 | return v.ReturnValue 263 | } else { 264 | return v 265 | } 266 | } 267 | 268 | func (v *Value) Equals(other *Value) bool { 269 | if v.ValueType == ReturnVal { 270 | return v.ReturnValue.Equals(other) 271 | } 272 | 273 | vType := v.Type() 274 | 275 | other = other.Unwrap() 276 | 277 | if vType != other.Type() { 278 | return false 279 | } 280 | 281 | switch vType { 282 | case BooleanValue: 283 | return v.BooleanValue == other.BooleanValue 284 | 285 | case NumberVal: 286 | return v.NumberValue == other.NumberValue 287 | 288 | case StringVal: 289 | return v.StringValue == other.StringValue 290 | 291 | case ListVal: 292 | if len(v.ListValue) != len(other.ListValue) { 293 | return false 294 | } 295 | 296 | for i, value := range v.ListValue { 297 | if !value.Equals(other.ListValue[i]) { 298 | return false 299 | } 300 | } 301 | 302 | return true 303 | 304 | case MapVal: 305 | if len(v.MapValue) != len(other.MapValue) { 306 | return false 307 | } 308 | 309 | for key, value := range v.MapValue { 310 | if !value.Equals(other.MapValue[key]) { 311 | return false 312 | } 313 | } 314 | 315 | return true 316 | 317 | case NullVal, Undefined: 318 | return true 319 | 320 | default: 321 | panic("Unable to compare value types " + vType) 322 | } 323 | } 324 | 325 | func (v *Value) String() string { 326 | return fmt.Sprintf("%s(%s)", v.Type(), v.AsStringValue()) 327 | } 328 | 329 | func ValueFromToken(t *lexer.Token) (*Value, error) { 330 | switch t.Type { 331 | 332 | case lexer.StringToken: 333 | return NewString(t.Value), nil 334 | 335 | case lexer.NumberToken: 336 | f, err := strconv.ParseFloat(t.Value, 64) 337 | if err != nil { 338 | return nil, err 339 | } 340 | return NewNumber(f), nil 341 | 342 | case lexer.TrueToken: 343 | return NewBoolean(true), nil 344 | 345 | case lexer.FalseToken: 346 | return NewBoolean(false), nil 347 | 348 | case lexer.NullToken: 349 | return &Value{ValueType: NullVal, IsNull: true}, nil 350 | 351 | case lexer.NoneToken: 352 | return NewUndefined(), nil 353 | 354 | case lexer.IdentToken: 355 | return nil, errors.New("unable to convert ident to value: " + t.Value) 356 | 357 | default: 358 | return nil, fmt.Errorf("unable to convert %s to value", t.Type) 359 | } 360 | } 361 | 362 | func NewValueFromInterface(value interface{}) (*Value, error) { 363 | switch v := value.(type) { 364 | case string: 365 | return NewString(v), nil 366 | case int: 367 | return NewNumber(float64(v)), nil 368 | case int64: 369 | return NewNumber(float64(v)), nil 370 | case uint: 371 | return NewNumber(float64(v)), nil 372 | case uint64: 373 | return NewNumber(float64(v)), nil 374 | case float32: 375 | return NewNumber(float64(v)), nil 376 | case float64: 377 | return NewNumber(v), nil 378 | case bool: 379 | return NewBoolean(v), nil 380 | 381 | default: 382 | return nil, fmt.Errorf("Unknown value type %v", reflect.TypeOf(value)) 383 | } 384 | } 385 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "strings" 8 | 9 | "gopkg.in/yaml.v2" 10 | ) 11 | 12 | type Config struct { 13 | Name string 14 | Target *Target 15 | 16 | // Custom behaviour which allows us to override the target information on a per folder basis within `/models/` 17 | ModelGroups map[string]*Target 18 | ModelGroupsFile string 19 | 20 | // seedConfig holds the seed (global) configurations 21 | seedConfig map[string]*SeedConfig 22 | } 23 | 24 | func (c *Config) GetTargetFor(path string) *Target { 25 | if c.ModelGroups == nil { 26 | return c.Target 27 | } 28 | 29 | for modelGroup, target := range c.ModelGroups { 30 | if strings.HasPrefix(path, fmt.Sprintf("models%c%s%c", os.PathSeparator, modelGroup, os.PathSeparator)) { 31 | return target 32 | } 33 | 34 | if strings.HasPrefix(path, fmt.Sprintf("tests%c%s%c", os.PathSeparator, modelGroup, os.PathSeparator)) { 35 | return target 36 | } 37 | 38 | if strings.HasPrefix(path, fmt.Sprintf("data%c%s%c", os.PathSeparator, modelGroup, os.PathSeparator)) { 39 | return target 40 | } 41 | } 42 | 43 | return c.Target 44 | } 45 | 46 | var GlobalCfg *Config 47 | 48 | func Read(targetProfile string, upstreamProfile string, threads int, customConfigPath string, strExecutor func(s string) (string, error)) (*Config, error) { 49 | project, err := readDBTProject(customConfigPath) 50 | if err != nil { 51 | return nil, err 52 | } 53 | 54 | appConfig, err := readDDBTConfig() 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | for _, target := range appConfig.ProtectedTargets { 60 | if strings.EqualFold(target, targetProfile) { 61 | return nil, fmt.Errorf("`%s` is a protected target, DDBT will not run against it.", target) 62 | } 63 | } 64 | 65 | profile, err := readProfile(project.Profile) 66 | if err != nil { 67 | return nil, err 68 | } 69 | 70 | if targetProfile == "" { 71 | targetProfile = profile.Target 72 | } 73 | 74 | output, found := profile.Outputs[targetProfile] 75 | if !found { 76 | return nil, fmt.Errorf("Output `%s` of profile `%s` not found", targetProfile, project.Profile) 77 | } 78 | 79 | if threads <= 0 { 80 | threads = output.Threads 81 | } 82 | 83 | GlobalCfg = &Config{ 84 | Name: project.Name, 85 | Target: &Target{ 86 | Name: targetProfile, 87 | ProjectID: output.Project, 88 | DataSet: output.Dataset, 89 | Location: output.Location, 90 | Threads: threads, 91 | 92 | ProjectSubstitutions: make(map[string]map[string]string), 93 | ExecutionProjects: make([]string, 0), 94 | }, 95 | } 96 | 97 | if upstreamProfile != "" { 98 | output, found := profile.Outputs[upstreamProfile] 99 | if !found { 100 | return nil, fmt.Errorf("Output `%s` of profile `%s` not found", upstreamProfile, project.Profile) 101 | } 102 | 103 | GlobalCfg.Target.ReadUpstream = &Target{ 104 | Name: upstreamProfile, 105 | ProjectID: output.Project, 106 | DataSet: output.Dataset, 107 | Location: output.Location, 108 | Threads: threads, 109 | 110 | ProjectSubstitutions: make(map[string]map[string]string), 111 | ExecutionProjects: make([]string, 0), 112 | } 113 | } 114 | 115 | if appConfig.ModelGroupsFile != "" { 116 | modelGroups, err := readModelGroupConfig(appConfig.ModelGroupsFile, targetProfile, upstreamProfile, GlobalCfg.Target) 117 | if err != nil { 118 | return nil, err 119 | } 120 | 121 | GlobalCfg.ModelGroups = modelGroups 122 | GlobalCfg.ModelGroupsFile = appConfig.ModelGroupsFile 123 | } 124 | 125 | if settings, found := project.Models[project.Name]; found { 126 | if err := readGeneralFolderBasedConfig(settings, strExecutor); err != nil { 127 | return nil, err 128 | } 129 | } else { 130 | return nil, fmt.Errorf("no models config found, expected to find `models: %s:` in `dbt_project.yml`", project.Name) 131 | } 132 | 133 | if seedCfg, found := project.Seeds[project.Name]; found { 134 | cfg, err := readSeedCfg(seedCfg) 135 | if err != nil { 136 | // if parsing of seed section of config has failed, don't error 137 | fmt.Fprintf(os.Stderr, "⚠️ Cannot parse seed config: %v\n", err) 138 | } else { 139 | GlobalCfg.seedConfig = cfg 140 | } 141 | } 142 | 143 | return GlobalCfg, nil 144 | } 145 | 146 | func NumberThreads() int { 147 | return GlobalCfg.Target.Threads 148 | } 149 | 150 | type dbtProject struct { 151 | Name string `yaml:"name"` 152 | Profile string `yaml:"profile"` 153 | Models map[string]map[string]interface{} `yaml:"models"` // "Models[project_name][key]value" 154 | Seeds map[string]map[string]interface{} `yaml:"seeds"` // "Seeds[project_name][key]value" 155 | } 156 | 157 | func handleCustomConfigPath(customConfigPath string) (string, error) { 158 | // if a custom path is provided, ensure that a trailing slash is present 159 | if customConfigPath != "" { 160 | customConfigPath = strings.TrimRight(customConfigPath, string(os.PathSeparator)) 161 | customConfigPath = customConfigPath + string(os.PathSeparator) 162 | } 163 | return customConfigPath, nil 164 | } 165 | 166 | func readDBTProject(customConfigPath string) (dbtProject, error) { 167 | project := dbtProject{} 168 | 169 | customConfigPath, err := handleCustomConfigPath(customConfigPath) 170 | if err != nil { 171 | return dbtProject{}, err 172 | } 173 | 174 | bytes, err := ioutil.ReadFile(customConfigPath + "dbt_project.yml") 175 | if err != nil { 176 | return dbtProject{}, err 177 | } 178 | 179 | if err := yaml.Unmarshal(bytes, &project); err != nil { 180 | return dbtProject{}, err 181 | } 182 | 183 | return project, nil 184 | } 185 | 186 | type dbtOutputs struct { 187 | Project string `yaml:"project"` 188 | Dataset string `yaml:"dataset"` 189 | Location string `yaml:"location"` 190 | Threads int `yaml:"threads"` 191 | } 192 | 193 | type dbtProfile struct { 194 | Target string `yaml:"target"` 195 | Outputs map[string]dbtOutputs 196 | } 197 | 198 | func readProfile(profileName string) (dbtProfile, error) { 199 | m := make(map[string]dbtProfile) 200 | 201 | bytes, err := ioutil.ReadFile("profiles.yml") 202 | if err != nil { 203 | return dbtProfile{}, err 204 | } 205 | 206 | if err := yaml.Unmarshal(bytes, &m); err != nil { 207 | return dbtProfile{}, err 208 | } 209 | 210 | p, found := m[profileName] 211 | if !found { 212 | return dbtProfile{}, fmt.Errorf("dbtProfile `%s` was not found in `profiles.yml`", profileName) 213 | } 214 | 215 | return p, nil 216 | } 217 | 218 | type ddbtConfig struct { 219 | ModelGroupsFile string `yaml:"model-groups-config"` 220 | ProtectedTargets []string `yaml:"protected-targets"` // Targets that DDBT is not allowed to execute against 221 | } 222 | 223 | func readDDBTConfig() (ddbtConfig, error) { 224 | c := ddbtConfig{} 225 | 226 | bytes, err := ioutil.ReadFile("ddbt_config.yml") 227 | if os.IsNotExist(err) { 228 | return c, nil 229 | } 230 | if err != nil { 231 | return c, err 232 | } 233 | 234 | if err := yaml.Unmarshal(bytes, &c); err != nil { 235 | return c, err 236 | } 237 | 238 | return c, nil 239 | } 240 | -------------------------------------------------------------------------------- /config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "testing" 4 | 5 | func Test_handleCustomConfigPath(t *testing.T) { 6 | type args struct { 7 | customConfigPath string 8 | } 9 | tests := []struct { 10 | name string 11 | args args 12 | want string 13 | wantErr bool 14 | }{ 15 | // path with trailing slash returns same path 16 | { 17 | name: "path with trailing slash", 18 | args: args{ 19 | customConfigPath: "foo/", 20 | }, 21 | want: "foo/", 22 | wantErr: false, 23 | }, 24 | // path without trailing slash returns path with trailing slash 25 | { 26 | name: "path without trailing slash", 27 | args: args{ 28 | customConfigPath: "foo", 29 | }, 30 | want: "foo/", 31 | wantErr: false, 32 | }, 33 | } 34 | for _, tt := range tests { 35 | t.Run(tt.name, func(t *testing.T) { 36 | got, err := handleCustomConfigPath(tt.args.customConfigPath) 37 | if (err != nil) != tt.wantErr { 38 | t.Errorf("handleCustomConfigPath() error = %v, wantErr %v", err, tt.wantErr) 39 | return 40 | } 41 | if got != tt.want { 42 | t.Errorf("handleCustomConfigPath() = %v, want %v", got, tt.want) 43 | } 44 | }) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /config/modelConfig.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "reflect" 7 | "strings" 8 | ) 9 | 10 | // ModelConfig represents model configurations. 11 | // 12 | // https://docs.getdbt.com/reference/model-configs 13 | type ModelConfig struct { 14 | GeneralConfig 15 | Materialized string 16 | } 17 | 18 | // GeneralConfig represents common 'general' configurations. 19 | // 20 | // https://docs.getdbt.com/reference/model-configs#general-configurations 21 | type GeneralConfig struct { 22 | Enabled bool 23 | Schema string 24 | Tags []string 25 | PreHooks []string 26 | PostHooks []string 27 | PersistDocs struct { 28 | Relation bool 29 | Columns bool 30 | } 31 | FullRefresh bool 32 | } 33 | 34 | var defaultConfig = ModelConfig{ 35 | GeneralConfig: GeneralConfig{ 36 | Enabled: true, 37 | Tags: []string{}, 38 | PreHooks: []string{}, 39 | PostHooks: []string{}, 40 | }, 41 | Materialized: "table", 42 | } 43 | 44 | var folderBasedConfig = make(map[string]ModelConfig) 45 | 46 | func GetFolderConfig(path string) ModelConfig { 47 | matchPath := "" 48 | config := defaultConfig 49 | 50 | for cfgPath := range folderBasedConfig { 51 | if strings.HasPrefix(path, cfgPath) && len(cfgPath) > len(matchPath) { 52 | matchPath = cfgPath 53 | config = folderBasedConfig[cfgPath] 54 | } 55 | } 56 | 57 | newSlice := make([]string, len(config.Tags)) 58 | copy(newSlice, config.Tags) 59 | config.Tags = newSlice 60 | 61 | return config 62 | } 63 | 64 | func readGeneralFolderBasedConfig(m map[string]interface{}, strExecutor func(s string) (string, error)) error { 65 | 66 | if err := readSubFolder("models/", defaultConfig, m, strExecutor); err != nil { 67 | return err 68 | } 69 | 70 | return nil 71 | } 72 | 73 | func readSubFolder(folderName string, config ModelConfig, m map[string]interface{}, strExecutor func(s string) (string, error)) error { 74 | subFolders := make(map[string]map[string]interface{}) 75 | 76 | // Read common 'general' configurations. 77 | remaining, err := readGeneralConfig(&config.GeneralConfig, m, strExecutor) 78 | if err != nil { 79 | return err 80 | } 81 | 82 | for key, value := range remaining { 83 | switch key { 84 | case "materialized": 85 | materialized, err := asStr("materialized", value, strExecutor) 86 | if err != nil { 87 | return err 88 | } 89 | 90 | config.Materialized = materialized 91 | 92 | default: 93 | if v, ok := value.(map[interface{}]interface{}); ok { 94 | strMap := make(map[string]interface{}) 95 | 96 | for k, v := range v { 97 | kStr, ok := k.(string) 98 | if !ok { 99 | return fmt.Errorf("unable to convert key `%v` into a string", key) 100 | } 101 | 102 | strMap[kStr] = v 103 | } 104 | 105 | subFolders[key] = strMap 106 | } else { 107 | return fmt.Errorf("unable to convert `%s` into a map, got; %v", key, reflect.TypeOf(value)) 108 | } 109 | } 110 | } 111 | 112 | for name, value := range subFolders { 113 | if err := readSubFolder(fmt.Sprintf("%s%s%c", folderName, name, os.PathSeparator), config, value, strExecutor); err != nil { 114 | return err 115 | } 116 | } 117 | 118 | folderBasedConfig[folderName] = config 119 | 120 | return nil 121 | } 122 | 123 | func strOrList(name string, value interface{}, strExecutor func(s string) (string, error)) ([]string, error) { 124 | switch v := value.(type) { 125 | case string: 126 | return []string{v}, nil 127 | 128 | case []string: 129 | return v, nil 130 | 131 | case []interface{}: 132 | list := make([]string, len(v)) 133 | 134 | for i, value := range v { 135 | str, err := asStr(name, value, strExecutor) 136 | if err != nil { 137 | return nil, err 138 | } 139 | 140 | list[i] = str 141 | } 142 | 143 | return list, nil 144 | 145 | default: 146 | return nil, fmt.Errorf("Unable to convert into a list of strings for `%s`, got %v", name, reflect.TypeOf(value)) 147 | } 148 | } 149 | 150 | func asStr(name string, value interface{}, strExecutor func(s string) (string, error)) (string, error) { 151 | strValue, ok := value.(string) 152 | if !ok { 153 | return "", fmt.Errorf("Unable to convert `%s` to string, got: %v", name, reflect.TypeOf(value)) 154 | } 155 | 156 | strValue, err := strExecutor(strValue) 157 | if err != nil { 158 | return "", err 159 | } 160 | 161 | return strValue, nil 162 | } 163 | 164 | // readGeneralConfig reads and consumes common 'general configurations' 165 | // config keys and return all the remaining key/value (or error). 166 | // 167 | // https://docs.getdbt.com/reference/model-configs#general-configurations 168 | func readGeneralConfig( 169 | config *GeneralConfig, 170 | m map[string]interface{}, 171 | strExecutor func(s string) (string, error), 172 | ) (map[string]interface{}, error) { 173 | if config == nil { 174 | return nil, fmt.Errorf("General config is not writable") 175 | } 176 | 177 | var remaining map[string]interface{} 178 | for key, value := range m { 179 | switch key { 180 | case "enabled": 181 | if b, ok := value.(bool); ok { 182 | config.Enabled = b 183 | } else { 184 | return nil, fmt.Errorf("Unable to convert `enabled` to boolean, got: %v", reflect.TypeOf(value)) 185 | } 186 | 187 | case "tags": 188 | list, err := strOrList("tags", value, strExecutor) 189 | if err != nil { 190 | return nil, err 191 | } 192 | config.Tags = list 193 | 194 | case "pre_hook": 195 | list, err := strOrList("pre_hook", value, strExecutor) 196 | if err != nil { 197 | return nil, err 198 | } 199 | config.PreHooks = list 200 | 201 | case "post_hook": 202 | list, err := strOrList("post_hook", value, strExecutor) 203 | if err != nil { 204 | return nil, err 205 | } 206 | config.PostHooks = list 207 | 208 | case "database": 209 | _, err := asStr("database", value, strExecutor) 210 | if err != nil { 211 | return nil, err 212 | } 213 | 214 | case "schema": 215 | schema, err := asStr("schema", value, strExecutor) 216 | if err != nil { 217 | return nil, err 218 | } 219 | config.Schema = schema 220 | 221 | case "alias": 222 | 223 | case "persist_docs": 224 | 225 | case "full_refresh": 226 | if b, ok := value.(bool); ok { 227 | config.FullRefresh = b 228 | } else { 229 | return nil, fmt.Errorf("Unable to convert `full_refresh` to boolean, got: %v", reflect.TypeOf(value)) 230 | } 231 | 232 | default: 233 | // For any key not part of general configurations, 234 | // copy to remaining to be processed later. 235 | if remaining == nil { 236 | remaining = make(map[string]interface{}) 237 | } 238 | remaining[key] = value 239 | } 240 | } 241 | 242 | return remaining, nil 243 | } 244 | -------------------------------------------------------------------------------- /config/modelConfig_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | "gopkg.in/yaml.v2" 9 | ) 10 | 11 | func TestModelConfig(t *testing.T) { 12 | dbtProjectYml := ` 13 | name: package_name 14 | version: '1.0' 15 | 16 | profile: ddbt 17 | models: 18 | ddbt: 19 | enabled: true 20 | schema: default_dataset_name 21 | tags: ["tag_one", "tag_two"] 22 | materialized: ephemeral 23 | table_name: 24 | tags: # General Config 25 | - tag_two 26 | - tag_three 27 | materialized: table # Model config 28 | another_table_name: 29 | tags: [] 30 | ` 31 | 32 | var project dbtProject 33 | require.NoError(t, yaml.Unmarshal([]byte(dbtProjectYml), &project)) 34 | err := readGeneralFolderBasedConfig(project.Models["ddbt"], func(s string) (string, error) { return s, nil }) 35 | require.NoError(t, err) 36 | 37 | assert.NotNil(t, folderBasedConfig["models/"]) 38 | assert.Equal(t, []string{"tag_one", "tag_two"}, folderBasedConfig["models/"].Tags) 39 | assert.Equal(t, "ephemeral", folderBasedConfig["models/"].Materialized) 40 | 41 | // Override parent materialized 42 | assert.NotNil(t, folderBasedConfig["models/table_name/"]) 43 | assert.Equal(t, []string{"tag_two", "tag_three"}, folderBasedConfig["models/table_name/"].Tags) 44 | assert.Equal(t, "table", folderBasedConfig["models/table_name/"].Materialized) 45 | 46 | // Inherit materialized from parent 47 | assert.NotNil(t, folderBasedConfig["models/another_table_name/"]) 48 | assert.Equal(t, []string(nil), folderBasedConfig["models/another_table_name"].Tags) 49 | assert.Equal(t, "ephemeral", folderBasedConfig["models/another_table_name/"].Materialized) 50 | } 51 | -------------------------------------------------------------------------------- /config/modelGroups.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "os/user" 9 | 10 | "gopkg.in/yaml.v2" 11 | ) 12 | 13 | // Model Groups are a custom addition to DBT which allow us to specify different configurations depending on the folder 14 | // the model lives in. This allows us to store all our analytics in a single monorepo and run them as part of the same 15 | // DAG 16 | 17 | type modelGroupTarget struct { 18 | Project string `yaml:"project"` 19 | Dataset interface{} `yaml:"dataset"` // This could either be a string, or a map 20 | ExecutionProjects []string `yaml:"execution_projects"` // Which projects to run the queries under 21 | 22 | // Here, project-tag substitutions can be added. Models with matching tags and projects will have the 23 | // destination project swapped. 24 | ProjectSubstitutions map[string]map[string]string `yaml:"project_tag_substitutions"` 25 | } 26 | 27 | type modelGroupConfig struct { 28 | Targets map[string]modelGroupTarget 29 | } 30 | 31 | type modelGroupConfigFile = map[string]modelGroupConfig 32 | 33 | func readModelGroupConfig(fileName string, targetName string, defaultTarget string, baseTarget *Target) (map[string]*Target, error) { 34 | m := make(modelGroupConfigFile) 35 | 36 | bytes, err := ioutil.ReadFile(fileName) 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | if err := yaml.Unmarshal(bytes, &m); err != nil { 42 | return nil, err 43 | } 44 | 45 | rtn := make(map[string]*Target) 46 | 47 | for modelGroup, targets := range m { 48 | targetCfg, found := targets.Targets[targetName] 49 | if !found { 50 | fmt.Printf("⚠️ Model group `%s` does not have a target for `%s`\n", modelGroup, targetName) 51 | continue 52 | } 53 | 54 | target := baseTarget.Copy() 55 | 56 | if err := updateTargetFromModelGroupConfig(target, targetName, targetCfg); err != nil { 57 | return nil, err 58 | } 59 | 60 | if defaultTarget != "" { 61 | defaultTargetCfg, found := targets.Targets[defaultTarget] 62 | if !found { 63 | fmt.Printf("⚠️ Model group `%s` does not have a target for `%s`\n", modelGroup, targetName) 64 | continue 65 | } 66 | 67 | if err := updateTargetFromModelGroupConfig(target.ReadUpstream, defaultTarget, defaultTargetCfg); err != nil { 68 | return nil, err 69 | } 70 | } 71 | 72 | rtn[modelGroup] = target 73 | } 74 | 75 | return rtn, nil 76 | } 77 | 78 | func updateTargetFromModelGroupConfig(target *Target, targetName string, targetCfg modelGroupTarget) error { 79 | if targetCfg.Project != "" { 80 | target.ProjectID = targetCfg.Project 81 | } 82 | 83 | // Data set is either a string, or "from_env" which means it's auto generated using the users username 84 | if targetCfg.Dataset != nil { 85 | switch v := targetCfg.Dataset.(type) { 86 | case string: 87 | target.DataSet = v 88 | 89 | case map[interface{}]interface{}: 90 | if b, ok := v["from_env"].(bool); ok && b { 91 | u, err := user.Current() 92 | if err != nil { 93 | return err 94 | } 95 | dbtUsername := os.Getenv("DBT_USER") 96 | if dbtUsername != "" { 97 | target.DataSet = fmt.Sprintf("dbt_%s_%s", dbtUsername, targetName) 98 | } else { 99 | target.DataSet = fmt.Sprintf("dbt_%s_%s", u.Username, targetName) 100 | } 101 | } else { 102 | return errors.New("expected dataset to be string or { 'from_env': true }") 103 | } 104 | 105 | default: 106 | return errors.New("expected dataset to be string or { 'from_env': true }") 107 | } 108 | } 109 | 110 | if targetCfg.ExecutionProjects != nil { 111 | target.ExecutionProjects = targetCfg.ExecutionProjects 112 | } 113 | 114 | if targetCfg.ProjectSubstitutions != nil { 115 | target.ProjectSubstitutions = targetCfg.ProjectSubstitutions 116 | } 117 | 118 | return nil 119 | } 120 | -------------------------------------------------------------------------------- /config/seedConfig.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | ) 8 | 9 | // SeedConfig represents the seed configuration specified in dbt_project.yml 10 | type SeedConfig struct { 11 | GeneralConfig 12 | QuoteColumns bool `yaml:"quote_columns"` 13 | ColumnTypes map[string]string `yaml:"column_types"` 14 | } 15 | 16 | // GetSeedConfig returns the seed configuration for a given path. 17 | func (c *Config) GetSeedConfig(path string) *SeedConfig { 18 | return c.GetFolderBasedSeedConfig(path) 19 | } 20 | 21 | // GetFolderBasedSeedConfig is a version of GetFolderConfig to 22 | // apply parent seed config hierarchically to child folders. 23 | func (c *Config) GetFolderBasedSeedConfig(path string) *SeedConfig { 24 | configPath := "data" 25 | parentConfig := c.seedConfig[configPath] 26 | config := &SeedConfig{ 27 | GeneralConfig: GeneralConfig{ 28 | Enabled: parentConfig.Enabled, 29 | Schema: parentConfig.Schema, 30 | }, 31 | QuoteColumns: parentConfig.QuoteColumns, 32 | ColumnTypes: parentConfig.ColumnTypes, 33 | } 34 | 35 | parts := strings.Split(strings.TrimSuffix(path, ".csv"), string(os.PathSeparator)) 36 | for _, part := range parts[1:] { 37 | configPath = fmt.Sprintf("%s%c%s", configPath, os.PathSeparator, part) 38 | childConfig := c.seedConfig[configPath] 39 | if childConfig != nil { 40 | if childConfig.ColumnTypes != nil { 41 | config.ColumnTypes = childConfig.ColumnTypes 42 | } 43 | if childConfig.Schema != "" { 44 | config.Schema = childConfig.Schema 45 | } 46 | if childConfig.Enabled != config.Enabled { 47 | config.Enabled = childConfig.Enabled 48 | } 49 | } 50 | } 51 | 52 | return config 53 | } 54 | 55 | func readSeedCfg(seedCfg map[string]interface{}) (map[string]*SeedConfig, error) { 56 | parser := &seedConfigParser{ 57 | SeedConfigs: make(map[string]*SeedConfig), 58 | } 59 | if err := parser.readCfgForDir("data", seedCfg); err != nil { 60 | return nil, err 61 | } 62 | return parser.SeedConfigs, nil 63 | } 64 | 65 | type seedConfigParser struct { 66 | SeedConfigs map[string]*SeedConfig 67 | } 68 | 69 | func (p *seedConfigParser) readCfgForDir(pathPrefix string, seedCfg map[string]interface{}) error { 70 | subFolders := make(map[string]map[string]interface{}) 71 | var cfg SeedConfig 72 | 73 | // Process common general configurations. 74 | remaining, err := readGeneralConfig(&cfg.GeneralConfig, seedCfg, simpleStringExecutor) 75 | if err != nil { 76 | return err 77 | } 78 | 79 | for key, value := range remaining { 80 | switch key { 81 | case "quote_columns": 82 | b, err := asBool(key, value) 83 | if err != nil { 84 | return err 85 | } 86 | cfg.QuoteColumns = b 87 | 88 | case "column_types": 89 | kvm, err := asKeyValueStringMap(key, value) 90 | if err != nil { 91 | return err 92 | } 93 | cfg.ColumnTypes = kvm 94 | 95 | default: 96 | genericMap, ok := value.(map[interface{}]interface{}) 97 | if !ok { 98 | return fmt.Errorf("Unable to convert `%s` to map, got: %T", key, value) 99 | } 100 | 101 | kvm := make(map[string]interface{}) 102 | for k, v := range genericMap { 103 | kStr, ok := k.(string) 104 | if !ok { 105 | return fmt.Errorf("Unable to convert key `%v` to string", k) 106 | } 107 | kvm[kStr] = v 108 | } 109 | 110 | subFolders[key] = kvm 111 | } 112 | } 113 | 114 | // Recurse into sub folders 115 | for name, value := range subFolders { 116 | if err := p.readCfgForDir(fmt.Sprintf("%s%c%s", pathPrefix, os.PathSeparator, name), value); err != nil { 117 | return err 118 | } 119 | } 120 | 121 | p.SeedConfigs[pathPrefix] = &cfg 122 | 123 | return nil 124 | } 125 | 126 | func simpleStringExecutor(s string) (string, error) { 127 | return s, nil 128 | } 129 | 130 | // asKeyValueStringMap converts value to a map[string]string. 131 | func asKeyValueStringMap(key, value interface{}) (map[string]string, error) { 132 | genericMap, ok := value.(map[interface{}]interface{}) 133 | if !ok { 134 | return nil, fmt.Errorf("Unable to convert `%s` to map, got %T", key, value) 135 | } 136 | 137 | kvm := make(map[string]string) 138 | for k, v := range genericMap { 139 | kStr, ok := k.(string) 140 | if !ok { 141 | return nil, fmt.Errorf("Unable to convert key `%s` to string, got: %T", k, k) 142 | } 143 | vStr, ok := v.(string) 144 | if !ok { 145 | return nil, fmt.Errorf("Unable to convert value `%s` to string, got: %T", v, v) 146 | } 147 | kvm[kStr] = vStr 148 | } 149 | return kvm, nil 150 | } 151 | 152 | // asBool converts value to a bool. 153 | func asBool(key string, value interface{}) (bool, error) { 154 | b, ok := value.(bool) 155 | if !ok { 156 | return false, fmt.Errorf("Unable to convert `%s` to boolean, got: %T", key, value) 157 | } 158 | return b, nil 159 | } 160 | -------------------------------------------------------------------------------- /config/seedConfig_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "testing" 5 | 6 | "gopkg.in/yaml.v2" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestSeedConfig(t *testing.T) { 13 | dbtProjectYml := ` 14 | name: package_name 15 | version: '1.0' 16 | 17 | profile: ddbt 18 | seeds: 19 | ddbt: 20 | enabled: true 21 | schema: default_dataset_name 22 | table_name: 23 | column_types: 24 | id: string 25 | amount: numeric 26 | description: string 27 | subfolder_table_name: 28 | column_types: 29 | extra_field: string 30 | ` 31 | 32 | var project dbtProject 33 | require.NoError(t, yaml.Unmarshal([]byte(dbtProjectYml), &project)) 34 | seedCfg, err := readSeedCfg(project.Seeds["ddbt"]) 35 | require.NoError(t, err) 36 | 37 | assert.NotNil(t, seedCfg["data/table_name"]) 38 | assert.Equal(t, map[string]string{"id": "string", "amount": "numeric", "description": "string"}, seedCfg["data/table_name"].ColumnTypes) 39 | 40 | assert.NotNil(t, seedCfg["data/table_name/subfolder_table_name"]) 41 | assert.Equal(t, map[string]string{"extra_field": "string"}, seedCfg["data/table_name/subfolder_table_name"].ColumnTypes) 42 | } 43 | 44 | func TestSeedConfigParsing(t *testing.T) { 45 | // Valid examples from docs.getdbt.com 46 | tcs := [...]struct { 47 | name string 48 | dbtProjectYml string 49 | }{ 50 | { 51 | name: "Apply the schema configuration to all seeds in your project", 52 | dbtProjectYml: ` 53 | seeds: 54 | jaffle_shop: 55 | schema: seed_data 56 | `, 57 | }, 58 | { 59 | name: "Apply the schema configuration to one seed only", 60 | dbtProjectYml: ` 61 | seeds: 62 | jaffle_shop: 63 | marketing: 64 | utm_parameters: 65 | schema: seed_data 66 | `, 67 | }, 68 | { 69 | name: "Example seed configuration", 70 | dbtProjectYml: ` 71 | name: jaffle_shop 72 | 73 | seeds: 74 | jaffle_shop: 75 | enabled: true 76 | schema: seed_data 77 | # This configures data/country_codes.csv 78 | country_codes: 79 | # Override column types 80 | column_types: 81 | country_code: varchar(2) 82 | country_name: varchar(32) 83 | marketing: 84 | schema: marketing # this will take precedence 85 | `, 86 | }, 87 | } 88 | 89 | for _, tc := range tcs { 90 | t.Run(tc.name, func(t *testing.T) { 91 | var project dbtProject 92 | require.NoError(t, yaml.Unmarshal([]byte(tc.dbtProjectYml), &project)) 93 | _, err := readSeedCfg(project.Seeds["jaffle_shop"]) 94 | require.NoError(t, err) 95 | }) 96 | } 97 | 98 | } 99 | -------------------------------------------------------------------------------- /config/target.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "math/rand" 4 | 5 | type Target struct { 6 | Name string 7 | ProjectID string 8 | DataSet string 9 | Location string 10 | Threads int 11 | ProjectSubstitutions map[string]map[string]string 12 | ExecutionProjects []string 13 | 14 | ReadUpstream *Target // If the reference is outside this DAG, this is the target we should read from 15 | } 16 | 17 | func (t *Target) Copy() *Target { 18 | projectSubs := make(map[string]map[string]string) 19 | for tag, sub := range t.ProjectSubstitutions { 20 | projectSubs[tag] = make(map[string]string) 21 | 22 | for sourceProject, targetProject := range sub { 23 | projectSubs[tag][sourceProject] = targetProject 24 | } 25 | } 26 | 27 | executionProjects := make([]string, len(t.ExecutionProjects)) 28 | for i, project := range t.ExecutionProjects { 29 | executionProjects[i] = project 30 | } 31 | 32 | var defaultUpstream *Target 33 | if t.ReadUpstream != nil { 34 | defaultUpstream = t.ReadUpstream.Copy() 35 | } 36 | 37 | return &Target{ 38 | Name: t.Name, 39 | ProjectID: t.ProjectID, 40 | DataSet: t.DataSet, 41 | Location: t.Location, 42 | Threads: t.Threads, 43 | ProjectSubstitutions: projectSubs, 44 | ExecutionProjects: executionProjects, 45 | ReadUpstream: defaultUpstream, 46 | } 47 | } 48 | 49 | func (t *Target) RandExecutionProject() string { 50 | if len(t.ExecutionProjects) == 0 { 51 | return t.ProjectID 52 | } 53 | 54 | i := rand.Intn(len(t.ExecutionProjects)) 55 | return t.ExecutionProjects[i] 56 | } 57 | -------------------------------------------------------------------------------- /fs/docs.go: -------------------------------------------------------------------------------- 1 | package fs 2 | 3 | import ( 4 | "io/ioutil" 5 | "path/filepath" 6 | "strings" 7 | "sync" 8 | ) 9 | 10 | type DocFile struct { 11 | Name string 12 | Path string 13 | Contents string 14 | 15 | mutex sync.Mutex 16 | } 17 | 18 | func newDocFile(path string) *DocFile { 19 | return &DocFile{ 20 | Name: strings.TrimSuffix(filepath.Base(path), ".md"), 21 | Path: path, 22 | Contents: "", 23 | } 24 | } 25 | 26 | func (d *DocFile) GetName() string { 27 | return d.Name 28 | } 29 | 30 | func (d *DocFile) Parse(fs *FileSystem) error { 31 | d.mutex.Lock() 32 | defer d.mutex.Unlock() 33 | 34 | // Read and parse the schema file 35 | bytes, err := ioutil.ReadFile(d.Path) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | d.Contents = string(bytes) 41 | return nil 42 | } 43 | -------------------------------------------------------------------------------- /fs/schema.go: -------------------------------------------------------------------------------- 1 | package fs 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "path/filepath" 7 | "strings" 8 | "sync" 9 | 10 | "ddbt/properties" 11 | ) 12 | 13 | type SchemaFile struct { 14 | Name string 15 | Path string 16 | Properties *properties.File 17 | 18 | mutex sync.Mutex 19 | } 20 | 21 | func newSchemaFile(path string) *SchemaFile { 22 | return &SchemaFile{ 23 | Name: strings.TrimSuffix(filepath.Base(path), ".yml"), 24 | Path: path, 25 | Properties: &properties.File{}, 26 | } 27 | } 28 | 29 | func (s *SchemaFile) GetName() string { 30 | return s.Name 31 | } 32 | 33 | func (s *SchemaFile) Parse(fs *FileSystem) error { 34 | s.mutex.Lock() 35 | defer s.mutex.Unlock() 36 | 37 | // Read and parse the schema file 38 | bytes, err := ioutil.ReadFile(s.Path) 39 | if err != nil { 40 | return err 41 | } 42 | 43 | err = s.Properties.Unmarshal(bytes) 44 | if err != nil { 45 | return err 46 | } 47 | 48 | // Now attach it to the various models it references 49 | for _, modelSchema := range s.Properties.Models { 50 | model := fs.Model(modelSchema.Name) 51 | if model == nil { 52 | model = fs.Test(modelSchema.Name) 53 | } 54 | 55 | if model == nil { 56 | return fmt.Errorf("Unable to apply model schema; model %s not found", modelSchema.Name) 57 | } 58 | 59 | model.Schema = modelSchema 60 | } 61 | 62 | // Add snapshot records 63 | for _, modelSchema := range s.Properties.Snapshots { 64 | model := fs.Model(modelSchema.Name) 65 | if model == nil { 66 | return fmt.Errorf("Unable to apply snapshot schema; %s not found", modelSchema.Name) 67 | } 68 | 69 | model.Schema = modelSchema 70 | } 71 | 72 | return nil 73 | } 74 | -------------------------------------------------------------------------------- /fs/seed.go: -------------------------------------------------------------------------------- 1 | package fs 2 | 3 | import ( 4 | "encoding/csv" 5 | "fmt" 6 | "io" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | 11 | "ddbt/config" 12 | ) 13 | 14 | // SeedFile is a simplified File where we only keep track of its name and path. 15 | type SeedFile struct { 16 | Name string 17 | Path string 18 | Columns []string 19 | ColumnTypes map[string]string 20 | } 21 | 22 | func newSeedFile(path string) *SeedFile { 23 | return &SeedFile{ 24 | Name: strings.TrimSuffix(filepath.Base(path), ".csv"), 25 | Path: path, 26 | } 27 | } 28 | 29 | func (s *SeedFile) GetName() string { 30 | return s.Name 31 | } 32 | 33 | func (s *SeedFile) GetTarget() (*config.Target, error) { 34 | target := config.GlobalCfg.GetTargetFor(s.Path) 35 | seedCfg, err := s.GetConfig() 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | // Override dataset from config 41 | if seedCfg.Schema != "" { 42 | target.DataSet = seedCfg.Schema 43 | } 44 | return target, nil 45 | } 46 | 47 | func (s *SeedFile) GetConfig() (*config.SeedConfig, error) { 48 | configKey := strings.TrimSuffix(s.Path, ".csv") 49 | return config.GlobalCfg.GetSeedConfig(configKey), nil 50 | } 51 | 52 | // ReadColumns reads columns (first row) from CSV file. 53 | func (s *SeedFile) ReadColumns() error { 54 | f, err := os.Open(s.Path) 55 | if err != nil { 56 | return err 57 | } 58 | defer f.Close() 59 | r := csv.NewReader(f) 60 | 61 | headings, err := r.Read() 62 | if err == io.EOF { 63 | return fmt.Errorf("Seed file %s has less than one row", s.Path) 64 | } 65 | s.Columns = headings 66 | 67 | return s.readColumnTypes() 68 | } 69 | 70 | func (s *SeedFile) readColumnTypes() error { 71 | cfg, err := s.GetConfig() 72 | if err != nil { 73 | return err 74 | } 75 | 76 | if cfg.ColumnTypes == nil { 77 | // Not specified (use auto detect) 78 | return nil 79 | } 80 | 81 | for _, column := range s.Columns { 82 | colType, ok := cfg.ColumnTypes[column] 83 | if !ok || colType == "" { 84 | colType = "string" // default to string 85 | } 86 | if s.ColumnTypes == nil { 87 | s.ColumnTypes = make(map[string]string) 88 | } 89 | s.ColumnTypes[column] = colType 90 | } 91 | return nil 92 | } 93 | 94 | func (s *SeedFile) HasSchema() bool { 95 | return s.ColumnTypes != nil 96 | } 97 | -------------------------------------------------------------------------------- /fs/workpool.go: -------------------------------------------------------------------------------- 1 | package fs 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | 7 | "ddbt/config" 8 | "ddbt/utils" 9 | ) 10 | 11 | func ProcessFiles(files []*File, f func(file *File) error, pb *utils.ProgressBar) error { 12 | fList := make([]hasName, len(files)) 13 | for i, file := range files { 14 | fList[i] = file 15 | } 16 | 17 | return processFiles(fList, func(file hasName) error { return f(file.(*File)) }, pb) 18 | } 19 | 20 | func ProcessSchemas(files []*SchemaFile, f func(file *SchemaFile) error, pb *utils.ProgressBar) error { 21 | fList := make([]hasName, len(files)) 22 | for i, file := range files { 23 | fList[i] = file 24 | } 25 | 26 | return processFiles(fList, func(file hasName) error { return f(file.(*SchemaFile)) }, pb) 27 | } 28 | 29 | func ProcessSeeds(seeds []*SeedFile, fn func(s *SeedFile) error, pb *utils.ProgressBar) error { 30 | seedList := make([]hasName, 0, len(seeds)) 31 | for _, seed := range seeds { 32 | seedList = append(seedList, seed) 33 | } 34 | return processFiles(seedList, func(s hasName) error { return fn(s.(*SeedFile)) }, pb) 35 | } 36 | 37 | type hasName interface { 38 | GetName() string 39 | } 40 | 41 | // Process the given file list through `f`. If a progress bar is given, then it will show the stauts line as we go 42 | func processFiles(files []hasName, f func(file hasName) error, pb *utils.ProgressBar) error { 43 | var wait sync.WaitGroup 44 | var errMutex sync.RWMutex 45 | var firstError error 46 | 47 | c := make(chan hasName, len(files)) 48 | 49 | worker := func() { 50 | var statusRow *utils.StatusRow 51 | 52 | if pb != nil { 53 | statusRow = pb.NewStatusRow() 54 | } 55 | 56 | for file := range c { 57 | errMutex.RLock() 58 | if firstError != nil { 59 | // If we're already had an error, skip through the rest of the items 60 | errMutex.RUnlock() 61 | wait.Done() 62 | continue 63 | } 64 | errMutex.RUnlock() 65 | 66 | if statusRow != nil { 67 | statusRow.Update(fmt.Sprintf("Running %s", file.GetName())) 68 | } 69 | 70 | err := f(file) 71 | wait.Done() 72 | 73 | if statusRow != nil { 74 | statusRow.SetIdle() 75 | } 76 | if err != nil { 77 | errMutex.Lock() 78 | if firstError == nil { 79 | firstError = err 80 | } 81 | errMutex.Unlock() 82 | 83 | return 84 | } 85 | 86 | } 87 | } 88 | 89 | for i := 0; i < config.NumberThreads(); i++ { 90 | go worker() 91 | } 92 | 93 | wait.Add(len(files)) 94 | for _, file := range files { 95 | c <- file 96 | } 97 | 98 | wait.Wait() 99 | 100 | return firstError 101 | } 102 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module ddbt 2 | 3 | go 1.16 4 | 5 | require ( 6 | cloud.google.com/go/bigquery v1.10.0 7 | github.com/atotto/clipboard v0.1.2 8 | github.com/fsnotify/fsnotify v1.4.9 9 | github.com/mattn/go-isatty v0.0.14 10 | github.com/spf13/cobra v1.1.1 11 | github.com/stretchr/testify v1.6.1 12 | google.golang.org/api v0.29.0 13 | gopkg.in/yaml.v2 v2.3.0 14 | ) 15 | -------------------------------------------------------------------------------- /jinja/ast/AndCondition.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | 6 | "ddbt/compilerInterface" 7 | "ddbt/jinja/lexer" 8 | ) 9 | 10 | type AndCondition struct { 11 | a AST 12 | b AST 13 | } 14 | 15 | var _ AST = &AndCondition{} 16 | 17 | func NewAndCondition(a, b AST) *AndCondition { 18 | return &AndCondition{ 19 | a: a, 20 | b: b, 21 | } 22 | } 23 | 24 | func (a *AndCondition) Position() lexer.Position { 25 | return a.a.Position() 26 | } 27 | 28 | func (a *AndCondition) Execute(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 29 | // Execute the LHS 30 | result, err := a.a.Execute(ec) 31 | if err != nil { 32 | return nil, err 33 | } 34 | if result == nil { 35 | return nil, ec.NilResultFor(a.a) 36 | } 37 | 38 | // Short circuit 39 | if !result.TruthyValue() { 40 | return compilerInterface.NewBoolean(false), nil 41 | } 42 | 43 | // Execute the RHS 44 | result, err = a.b.Execute(ec) 45 | if err != nil { 46 | return nil, err 47 | } 48 | if result == nil { 49 | return nil, ec.NilResultFor(a.b) 50 | } 51 | 52 | return compilerInterface.NewBoolean(result.TruthyValue()), nil 53 | } 54 | 55 | func (a *AndCondition) String() string { 56 | return fmt.Sprintf("(%s and %s)", a.a.String(), a.b.String()) 57 | } 58 | -------------------------------------------------------------------------------- /jinja/ast/AtomExpressionBlock.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | 6 | "ddbt/compilerInterface" 7 | "ddbt/jinja/lexer" 8 | ) 9 | 10 | // An special marker to tracking if we're at the end of a parse block 11 | type AtomExpressionBlock struct { 12 | token *lexer.Token 13 | } 14 | 15 | var _ AST = &AtomExpressionBlock{} 16 | 17 | func NewAtomExpressionBlock(token *lexer.Token) *AtomExpressionBlock { 18 | return &AtomExpressionBlock{ 19 | token: token, 20 | } 21 | } 22 | 23 | func (a *AtomExpressionBlock) Position() lexer.Position { 24 | return a.token.Start 25 | } 26 | 27 | func (a *AtomExpressionBlock) Execute(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 28 | return nil, nil 29 | } 30 | 31 | func (a *AtomExpressionBlock) String() string { 32 | return fmt.Sprintf("{%% %s %%}", a.token.Value) 33 | } 34 | 35 | func (a *AtomExpressionBlock) Token() *lexer.Token { 36 | return a.token 37 | } 38 | -------------------------------------------------------------------------------- /jinja/ast/Body.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "strings" 7 | 8 | "ddbt/compilerInterface" 9 | "ddbt/jinja/lexer" 10 | ) 11 | 12 | // A block which represents a simple 13 | type Body struct { 14 | position lexer.Position 15 | parts []AST 16 | } 17 | 18 | var _ AST = &Body{} 19 | 20 | func NewBody(token *lexer.Token) *Body { 21 | return &Body{ 22 | position: token.Start, 23 | parts: make([]AST, 0), 24 | } 25 | } 26 | 27 | func (b *Body) Position() lexer.Position { 28 | return b.position 29 | } 30 | 31 | func (b *Body) Execute(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 32 | // A body should compile down to only text blocks 33 | var builder strings.Builder 34 | 35 | for _, part := range b.parts { 36 | result, err := part.Execute(ec) 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | if err := writeValue(ec, part, &builder, result, false); err != nil { 42 | return nil, err 43 | } 44 | 45 | if result.ValueType == compilerInterface.ReturnVal { 46 | return result, nil 47 | } 48 | } 49 | 50 | value := compilerInterface.NewString(builder.String()) 51 | 52 | return value, nil 53 | } 54 | 55 | func (b *Body) String() string { 56 | var builder strings.Builder 57 | 58 | for _, part := range b.parts { 59 | builder.WriteString(part.String()) 60 | } 61 | 62 | return builder.String() 63 | } 64 | 65 | // Append a node to the body 66 | func (b *Body) Append(node AST) { 67 | b.parts = append(b.parts, node) 68 | } 69 | 70 | func (b *Body) AppendBody(node AST) { 71 | b.Append(node) 72 | } 73 | 74 | func writeValue(ec compilerInterface.ExecutionContext, part compilerInterface.AST, builder *strings.Builder, value *compilerInterface.Value, wrapAndEscape bool) error { 75 | if value == nil { 76 | return ec.NilResultFor(part) 77 | } 78 | 79 | // Remove the "ReturnVal" wrapper 80 | value = value.Unwrap() 81 | 82 | t := value.Type() 83 | switch t { 84 | case compilerInterface.StringVal: 85 | if wrapAndEscape { 86 | str := strings.Replace( 87 | strings.Replace( 88 | value.AsStringValue(), 89 | "\\", 90 | "\\\\'", 91 | -1, 92 | ), 93 | "\"", 94 | "\\\"", 95 | -1, 96 | ) 97 | 98 | builder.WriteString("\\\"") 99 | builder.WriteString(str) 100 | builder.WriteString("\\\"") 101 | } else { 102 | builder.WriteString(value.AsStringValue()) 103 | } 104 | 105 | case compilerInterface.NumberVal, compilerInterface.BooleanValue: 106 | builder.WriteString(value.AsStringValue()) 107 | 108 | case compilerInterface.ListVal: 109 | builder.WriteRune('[') 110 | for i, item := range value.ListValue { 111 | if i > 0 { 112 | builder.WriteString(", ") 113 | } 114 | 115 | if err := writeValue(ec, part, builder, item, true); err != nil { 116 | return err 117 | } 118 | } 119 | builder.WriteRune(']') 120 | 121 | case compilerInterface.MapVal: 122 | builder.WriteRune('{') 123 | i := 0 124 | for key, item := range value.MapValue { 125 | if i > 0 { 126 | builder.WriteString(", ") 127 | } 128 | 129 | builder.WriteString("\\\"") 130 | builder.WriteString(key) 131 | builder.WriteString("\\\": ") 132 | 133 | if err := writeValue(ec, part, builder, item, true); err != nil { 134 | return err 135 | } 136 | i++ 137 | } 138 | builder.WriteRune('}') 139 | 140 | case compilerInterface.Undefined, compilerInterface.NullVal: 141 | // no-op as we can consume these without effect 142 | 143 | default: 144 | return ec.ErrorAt( 145 | part, 146 | fmt.Sprintf( 147 | "A %v returned a %s which can not be combined into a body", 148 | reflect.TypeOf(part), 149 | t, 150 | ), 151 | ) 152 | } 153 | 154 | return nil 155 | } 156 | -------------------------------------------------------------------------------- /jinja/ast/BoolValue.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "ddbt/compilerInterface" 5 | "ddbt/jinja/lexer" 6 | ) 7 | 8 | type BoolValue struct { 9 | position lexer.Position 10 | value bool 11 | } 12 | 13 | var _ AST = &BoolValue{} 14 | 15 | func NewBoolValue(token *lexer.Token) *BoolValue { 16 | return &BoolValue{ 17 | position: token.Start, 18 | value: token.Type == lexer.TrueToken, 19 | } 20 | } 21 | 22 | func (b *BoolValue) Position() lexer.Position { 23 | return b.position 24 | } 25 | 26 | func (b *BoolValue) Execute(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 27 | return compilerInterface.NewBoolean(b.value), nil 28 | } 29 | 30 | func (b *BoolValue) String() string { 31 | if b.value { 32 | return "TRUE" 33 | } else { 34 | return "FALSE" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /jinja/ast/BracketGroup.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | 6 | "ddbt/compilerInterface" 7 | "ddbt/jinja/lexer" 8 | ) 9 | 10 | // Tracks where the user wrapped brackets around parts of the tree 11 | // to force operation order 12 | type BracketGroup struct { 13 | value AST 14 | } 15 | 16 | var _ AST = &BracketGroup{} 17 | 18 | func NewBracketGroup(bracketValue AST) *BracketGroup { 19 | return &BracketGroup{ 20 | value: bracketValue, 21 | } 22 | } 23 | 24 | func (bg *BracketGroup) Position() lexer.Position { 25 | return bg.value.Position() 26 | } 27 | 28 | func (bg *BracketGroup) Execute(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 29 | return bg.value.Execute(ec) 30 | } 31 | 32 | func (bg *BracketGroup) String() string { 33 | return fmt.Sprintf("(%s)", bg.value.String()) 34 | } 35 | -------------------------------------------------------------------------------- /jinja/ast/CallBlock.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | 6 | "ddbt/compilerInterface" 7 | "ddbt/jinja/lexer" 8 | ) 9 | 10 | type CallBlock struct { 11 | position lexer.Position 12 | fc *FunctionCall 13 | body *Body 14 | } 15 | 16 | var _ AST = &CallBlock{} 17 | 18 | func NewCallBlock(token *lexer.Token, fc *FunctionCall) *CallBlock { 19 | return &CallBlock{ 20 | position: token.Start, 21 | fc: fc, 22 | body: NewBody(token), 23 | } 24 | } 25 | 26 | func (cb *CallBlock) Position() lexer.Position { 27 | return cb.position 28 | } 29 | 30 | func (cb *CallBlock) Execute(parentEC compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 31 | ec := parentEC.PushState() 32 | 33 | // Set it so the body AST can be executed using the caller function 34 | ec.SetVariable( 35 | "caller", 36 | compilerInterface.NewFunction(func(sec compilerInterface.ExecutionContext, caller compilerInterface.AST, args compilerInterface.Arguments) (*compilerInterface.Value, error) { 37 | return cb.body.Execute(sec) 38 | }), 39 | ) 40 | 41 | // Execute the function call 42 | return cb.fc.Execute(ec) 43 | } 44 | 45 | func (cb *CallBlock) String() string { 46 | return fmt.Sprintf("{%% call %s %%}%s\n{%% endcall %%}", cb.fc.String(), cb.body.String()) 47 | } 48 | 49 | func (cb *CallBlock) AppendBody(node AST) { 50 | cb.body.Append(node) 51 | } 52 | -------------------------------------------------------------------------------- /jinja/ast/DoBlock.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | 6 | "ddbt/compilerInterface" 7 | "ddbt/jinja/lexer" 8 | ) 9 | 10 | type DoBlock struct { 11 | position lexer.Position 12 | run AST 13 | } 14 | 15 | var _ AST = &DoBlock{} 16 | 17 | // A do block executes the code but returns nothing 18 | func NewDoBlock(token *lexer.Token, run AST) *DoBlock { 19 | return &DoBlock{ 20 | position: token.Start, 21 | run: run, 22 | } 23 | } 24 | 25 | func (d *DoBlock) Position() lexer.Position { 26 | return d.position 27 | } 28 | 29 | func (d *DoBlock) Execute(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 30 | _, err := d.run.Execute(ec) 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | return &compilerInterface.Value{IsUndefined: true}, nil 36 | } 37 | 38 | func (d *DoBlock) String() string { 39 | return fmt.Sprintf("{%% do %s %%}", d.run.String()) 40 | } 41 | -------------------------------------------------------------------------------- /jinja/ast/EndOfFile.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "ddbt/compilerInterface" 5 | "ddbt/jinja/lexer" 6 | ) 7 | 8 | // A block which represents a simple 9 | type EndOfFile struct { 10 | position lexer.Position 11 | } 12 | 13 | var _ AST = &EndOfFile{} 14 | 15 | func NewEndOfFile(token *lexer.Token) *EndOfFile { 16 | return &EndOfFile{ 17 | position: token.Start, 18 | } 19 | } 20 | 21 | func (eof *EndOfFile) Position() lexer.Position { 22 | return eof.position 23 | } 24 | 25 | func (eof *EndOfFile) Execute(_ compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 26 | return &compilerInterface.Value{IsUndefined: true}, nil 27 | } 28 | 29 | func (eof *EndOfFile) String() string { 30 | return "" 31 | } 32 | -------------------------------------------------------------------------------- /jinja/ast/ForLoop.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "strings" 7 | 8 | "ddbt/compilerInterface" 9 | "ddbt/jinja/lexer" 10 | ) 11 | 12 | // A block which represents a simple 13 | type ForLoop struct { 14 | position lexer.Position 15 | keyItrName string 16 | valueItrName string 17 | list *Variable 18 | body *Body 19 | } 20 | 21 | type ForLoopParameter struct { 22 | name string //nolint:structcheck,unused 23 | defaultValue *lexer.Token 24 | } 25 | 26 | var _ AST = &ForLoop{} 27 | 28 | func NewForLoop(valueItrToken *lexer.Token, keyItr string, list *Variable) *ForLoop { 29 | return &ForLoop{ 30 | position: valueItrToken.Start, 31 | keyItrName: keyItr, 32 | valueItrName: valueItrToken.Value, 33 | list: list, 34 | body: NewBody(valueItrToken), 35 | } 36 | } 37 | 38 | func (fl *ForLoop) Position() lexer.Position { 39 | return fl.position 40 | } 41 | 42 | func (fl *ForLoop) Execute(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 43 | // Get the list 44 | list, err := fl.list.Execute(ec) 45 | if err != nil { 46 | return nil, err 47 | } 48 | if list == nil { 49 | return nil, ec.NilResultFor(fl.list) 50 | } 51 | 52 | if list.ValueType == compilerInterface.ReturnVal { 53 | list = list.ReturnValue 54 | } 55 | 56 | switch list.Type() { 57 | case compilerInterface.ListVal: 58 | return fl.executeForList(list.ListValue, ec) 59 | 60 | case compilerInterface.MapVal: 61 | return fl.executeForMap(list.MapValue, ec) 62 | 63 | default: 64 | return nil, ec.ErrorAt(fl, fmt.Sprintf("unable to run for each over %s", list.Type())) 65 | } 66 | } 67 | 68 | func (fl *ForLoop) executeForList(list []*compilerInterface.Value, parentEC compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 69 | var builder strings.Builder 70 | 71 | for index, value := range list { 72 | ec := parentEC.PushState() 73 | 74 | // Set the loop variables 75 | ec.SetVariable("loop", &compilerInterface.Value{ 76 | MapValue: map[string]*compilerInterface.Value{ 77 | "index": compilerInterface.NewNumber(float64(index + 1)), // Python loops start at 1!!! 78 | "last": compilerInterface.NewBoolean(index == (len(list) - 1)), 79 | }, 80 | }) 81 | if fl.keyItrName != "" { 82 | ec.SetVariable(fl.keyItrName, compilerInterface.NewNumber(float64(index))) 83 | } 84 | ec.SetVariable(fl.valueItrName, value) 85 | 86 | result, err := fl.body.Execute(ec) 87 | if err != nil { 88 | return nil, err 89 | } 90 | 91 | if err := writeValue(ec, fl.body, &builder, result, false); err != nil { 92 | return nil, err 93 | } 94 | } 95 | 96 | return &compilerInterface.Value{StringValue: builder.String()}, nil 97 | } 98 | 99 | func (fl *ForLoop) executeForMap(list map[string]*compilerInterface.Value, parentEC compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 100 | var builder strings.Builder 101 | 102 | num := 0 103 | 104 | // Sort keys so this loop excutes stably (i.e. the order doesn't change each time) 105 | keys := make([]string, 0, len(list)) 106 | for key := range list { 107 | keys = append(keys, key) 108 | } 109 | sort.Strings(keys) 110 | 111 | for index, key := range keys { 112 | ec := parentEC.PushState() 113 | 114 | // Set the loop variables 115 | ec.SetVariable("loop", &compilerInterface.Value{ 116 | MapValue: map[string]*compilerInterface.Value{ 117 | "last": compilerInterface.NewBoolean(index == (len(list) - 1)), 118 | }, 119 | }) 120 | 121 | if fl.keyItrName != "" { 122 | ec.SetVariable(fl.keyItrName, compilerInterface.NewString(key)) 123 | } 124 | ec.SetVariable(fl.valueItrName, list[key]) 125 | 126 | result, err := fl.body.Execute(ec) 127 | if err != nil { 128 | return nil, err 129 | } 130 | 131 | if err := writeValue(ec, fl.body, &builder, result, false); err != nil { 132 | return nil, err 133 | } 134 | 135 | num++ 136 | } 137 | 138 | return &compilerInterface.Value{StringValue: builder.String()}, nil 139 | } 140 | 141 | func (fl *ForLoop) String() string { 142 | if fl.keyItrName != "" { 143 | return fmt.Sprintf("\n{%% for %s, %s in %s %%}%s{%% endfor %%}", fl.keyItrName, fl.valueItrName, fl.list.String(), fl.body.String()) 144 | } else { 145 | return fmt.Sprintf("\n{%% for %s in %s %%}%s{%% endfor %%}", fl.valueItrName, fl.list.String(), fl.body.String()) 146 | } 147 | } 148 | 149 | func (fl *ForLoop) AppendBody(node AST) { 150 | fl.body.Append(node) 151 | } 152 | -------------------------------------------------------------------------------- /jinja/ast/FunctionCall.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "ddbt/compilerInterface" 8 | "ddbt/jinja/lexer" 9 | ) 10 | 11 | type FunctionCall struct { 12 | position lexer.Position 13 | name string 14 | arguments funcCallArgs 15 | } 16 | 17 | type funcCallArg struct { 18 | name string 19 | arg AST 20 | } 21 | 22 | type funcCallArgs []funcCallArg 23 | 24 | var _ AST = &FunctionCall{} 25 | 26 | func NewFunctionCall(token *lexer.Token, funcName string) *FunctionCall { 27 | return &FunctionCall{ 28 | position: token.Start, 29 | name: funcName, 30 | arguments: make(funcCallArgs, 0), 31 | } 32 | } 33 | 34 | func (fc *FunctionCall) Position() lexer.Position { 35 | return fc.position 36 | } 37 | 38 | func (fc *FunctionCall) Execute(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 39 | args, err := fc.arguments.Execute(ec) 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | function := ec.GetVariable(fc.name) 45 | if function.IsUndefined { 46 | return nil, ec.ErrorAt(fc, fmt.Sprintf("function `%s` not found", fc.name)) 47 | } 48 | 49 | if function.Type() != compilerInterface.FunctionalVal && function.Function == nil { 50 | return nil, ec.ErrorAt(fc, fmt.Sprintf("expected `%s` to be a function, got %s", fc.name, function.Type())) 51 | } 52 | 53 | if function.Function == nil { 54 | return nil, ec.ErrorAt(fc, "function is nil!") 55 | } 56 | 57 | result, err := function.Function(ec.PushState(), fc, args) 58 | if err != nil { 59 | return nil, err 60 | } 61 | 62 | return result, err 63 | } 64 | 65 | func (fc *FunctionCall) String() string { 66 | var builder strings.Builder 67 | 68 | for i, arg := range fc.arguments { 69 | if i > 0 { 70 | builder.WriteString(", ") 71 | } 72 | 73 | if arg.name != "" { 74 | builder.WriteString(arg.name) 75 | builder.WriteRune('=') 76 | } 77 | 78 | builder.WriteString(arg.arg.String()) 79 | } 80 | 81 | return fmt.Sprintf("{{ %s(%s) }}", fc.name, builder.String()) 82 | } 83 | 84 | func (fc *FunctionCall) AddArgument(argName string, node AST) { 85 | fc.arguments = append(fc.arguments, funcCallArg{argName, node}) 86 | } 87 | 88 | func (fca funcCallArgs) Execute(ec compilerInterface.ExecutionContext) (compilerInterface.Arguments, error) { 89 | arguments := make(compilerInterface.Arguments, 0, len(fca)) 90 | 91 | for _, arg := range fca { 92 | result, err := arg.arg.Execute(ec) 93 | if err != nil { 94 | return nil, err 95 | } 96 | 97 | if result == nil { 98 | return nil, ec.NilResultFor(arg.arg) 99 | } 100 | 101 | arguments = append(arguments, compilerInterface.Argument{Name: arg.name, Value: result}) 102 | } 103 | 104 | return arguments, nil 105 | } 106 | -------------------------------------------------------------------------------- /jinja/ast/IfStatement.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | 6 | "ddbt/compilerInterface" 7 | "ddbt/jinja/lexer" 8 | ) 9 | 10 | type IfStatement struct { 11 | condition AST 12 | body *Body 13 | elseBody *Body 14 | asElseIf bool 15 | } 16 | 17 | var _ AST = &IfStatement{} 18 | 19 | func NewIfStatement(token *lexer.Token, condition AST) *IfStatement { 20 | return &IfStatement{ 21 | condition: condition, 22 | body: NewBody(token), 23 | elseBody: NewBody(token), 24 | } 25 | } 26 | 27 | func (is *IfStatement) Position() lexer.Position { 28 | return is.condition.Position() 29 | } 30 | 31 | func (is *IfStatement) Execute(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 32 | conditionResult, err := is.condition.Execute(ec) 33 | if err != nil { 34 | return nil, err 35 | } 36 | if conditionResult == nil { 37 | return nil, ec.NilResultFor(is.condition) 38 | } 39 | 40 | if conditionResult.TruthyValue() { 41 | if is.body != nil { 42 | return is.body.Execute(ec) 43 | } 44 | } else if is.elseBody != nil { 45 | return is.elseBody.Execute(ec) 46 | } 47 | 48 | return compilerInterface.NewUndefined(), nil 49 | } 50 | 51 | func (is *IfStatement) String() string { 52 | if len(is.elseBody.parts) > 0 { 53 | return fmt.Sprintf("{%% if %s %%}%s{%% else %%}%s{%% endif %%}", is.condition.String(), is.body.String(), is.elseBody.String()) 54 | } else { 55 | return fmt.Sprintf("{%% if %s %%}%s{%% endif %%}", is.condition.String(), is.body.String()) 56 | } 57 | } 58 | 59 | func (is *IfStatement) AppendBody(node AST) { 60 | is.body.Append(node) 61 | } 62 | 63 | func (is *IfStatement) AppendElse(node AST) { 64 | is.elseBody.Append(node) 65 | } 66 | 67 | func (is *IfStatement) SetAsElseIf() { 68 | is.asElseIf = true 69 | } 70 | 71 | func (is *IfStatement) IsElseIf() bool { 72 | return is.asElseIf 73 | } 74 | -------------------------------------------------------------------------------- /jinja/ast/InOperator.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | 6 | "ddbt/compilerInterface" 7 | "ddbt/jinja/lexer" 8 | ) 9 | 10 | type InOperator struct { 11 | position lexer.Position 12 | needle AST 13 | haystack AST 14 | } 15 | 16 | var _ AST = &InOperator{} 17 | 18 | func NewInOperator(token *lexer.Token, needle, haystack AST) *InOperator { 19 | return &InOperator{ 20 | position: token.Start, 21 | needle: needle, 22 | haystack: haystack, 23 | } 24 | } 25 | 26 | func (in *InOperator) Position() lexer.Position { 27 | return in.position 28 | } 29 | 30 | func (in *InOperator) Execute(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 31 | needle, err := in.needle.Execute(ec) 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | haystack, err := in.haystack.Execute(ec) 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | // If these are functions results, strip the result wrapper 42 | needle = needle.Unwrap() 43 | haystack = haystack.Unwrap() 44 | 45 | result, err := BuiltInTests["in"](needle, haystack) 46 | if err != nil { 47 | return nil, ec.ErrorAt(in.haystack, err.Error()) 48 | } 49 | 50 | return compilerInterface.NewBoolean(result), nil 51 | } 52 | 53 | func (in *InOperator) String() string { 54 | return fmt.Sprintf("%s in %s", in.needle, in.haystack) 55 | } 56 | -------------------------------------------------------------------------------- /jinja/ast/List.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "ddbt/compilerInterface" 8 | "ddbt/jinja/lexer" 9 | ) 10 | 11 | type List struct { 12 | position lexer.Position 13 | items []AST 14 | } 15 | 16 | var _ AST = &List{} 17 | 18 | func NewList(token *lexer.Token) *List { 19 | return &List{ 20 | position: token.Start, 21 | items: make([]AST, 0), 22 | } 23 | } 24 | 25 | func (l *List) Position() lexer.Position { 26 | return l.position 27 | } 28 | 29 | func (l *List) Execute(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 30 | resultList := make([]*compilerInterface.Value, 0, len(l.items)) 31 | 32 | for _, item := range l.items { 33 | result, err := item.Execute(ec) 34 | 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | if result == nil { 40 | return nil, ec.NilResultFor(item) 41 | } 42 | 43 | resultList = append(resultList, result) 44 | } 45 | 46 | return &compilerInterface.Value{ 47 | ValueType: compilerInterface.ListVal, 48 | ListValue: resultList, 49 | }, nil 50 | } 51 | 52 | func (l *List) String() string { 53 | var builder strings.Builder 54 | 55 | for i, item := range l.items { 56 | if i > 0 { 57 | builder.WriteString(",\n\t\t") 58 | } 59 | 60 | builder.WriteString(item.String()) 61 | } 62 | 63 | return fmt.Sprintf("[\n\t\t%s\n\t\t]", builder.String()) 64 | } 65 | 66 | func (l *List) Append(item AST) { 67 | l.items = append(l.items, item) 68 | } 69 | -------------------------------------------------------------------------------- /jinja/ast/LogicalOp.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | 6 | "ddbt/compilerInterface" 7 | "ddbt/jinja/lexer" 8 | ) 9 | 10 | type LogicalOp struct { 11 | position lexer.Position 12 | op lexer.TokenType 13 | lhs AST 14 | rhs AST 15 | } 16 | 17 | var _ AST = &LogicalOp{} 18 | 19 | func NewLogicalOp(token *lexer.Token, lhs, rhs AST) *LogicalOp { 20 | return &LogicalOp{ 21 | position: token.Start, 22 | op: token.Type, 23 | lhs: lhs, 24 | rhs: rhs, 25 | } 26 | } 27 | 28 | func (op *LogicalOp) Position() lexer.Position { 29 | return op.position 30 | } 31 | 32 | func (op *LogicalOp) Execute(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 33 | lhs, err := op.lhs.Execute(ec) 34 | if err != nil { 35 | return nil, err 36 | } 37 | if lhs == nil { 38 | return nil, ec.NilResultFor(op.lhs) 39 | } 40 | 41 | rhs, err := op.rhs.Execute(ec) 42 | if err != nil { 43 | return nil, err 44 | } 45 | if rhs == nil { 46 | return nil, ec.NilResultFor(op.rhs) 47 | } 48 | 49 | result := false 50 | 51 | switch op.op { 52 | case lexer.IsEqualsToken: 53 | result = lhs.Equals(rhs) 54 | 55 | case lexer.NotEqualsToken: 56 | result = !lhs.Equals(rhs) 57 | 58 | case lexer.LessThanToken, lexer.LessThanEqualsToken, lexer.GreaterThanToken, lexer.GreaterThanEqualsToken: 59 | lhsNum, err := lhs.AsNumberValue() 60 | if err != nil { 61 | return nil, ec.ErrorAt(op.lhs, fmt.Sprintf("%s", err)) 62 | } 63 | 64 | rhsNum, err := rhs.AsNumberValue() 65 | if err != nil { 66 | return nil, ec.ErrorAt(op.rhs, fmt.Sprintf("%s", err)) 67 | } 68 | 69 | switch op.op { 70 | case lexer.LessThanToken: 71 | result = lhsNum < rhsNum 72 | case lexer.LessThanEqualsToken: 73 | result = lhsNum <= rhsNum 74 | case lexer.GreaterThanToken: 75 | result = lhsNum > rhsNum 76 | case lexer.GreaterThanEqualsToken: 77 | result = lhsNum >= rhsNum 78 | } 79 | 80 | default: 81 | return nil, ec.ErrorAt(op, fmt.Sprintf("Unable to process logical operator `%s`", op.op)) 82 | } 83 | 84 | return compilerInterface.NewBoolean(result), nil 85 | } 86 | 87 | func (op *LogicalOp) String() string { 88 | return fmt.Sprintf("(%s %s %s)", op.lhs, op.op, op.rhs) 89 | } 90 | -------------------------------------------------------------------------------- /jinja/ast/Macro.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | 8 | "ddbt/compilerInterface" 9 | "ddbt/jinja/lexer" 10 | ) 11 | 12 | // A block which represents a simple 13 | type Macro struct { 14 | position lexer.Position 15 | name string 16 | body *Body 17 | parameters []macroParameter 18 | numOptionalParams int 19 | } 20 | 21 | type macroParameter struct { 22 | name string 23 | defaultValue AST 24 | } 25 | 26 | var _ AST = &Macro{} 27 | 28 | func NewMacro(token *lexer.Token) *Macro { 29 | return &Macro{ 30 | position: token.Start, 31 | name: token.Value, 32 | body: NewBody(token), 33 | parameters: make([]macroParameter, 0), 34 | } 35 | } 36 | 37 | func (m *Macro) Position() lexer.Position { 38 | return m.position 39 | } 40 | 41 | func (m *Macro) Execute(macroEC compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 42 | macroEC.RegisterMacro( 43 | m.name, 44 | macroEC, 45 | func(ec compilerInterface.ExecutionContext, caller compilerInterface.AST, args compilerInterface.Arguments) (*compilerInterface.Value, error) { 46 | unusedArguments := make(compilerInterface.Arguments, 0) // Unused positional based arguments 47 | kwargs := make(map[string]*compilerInterface.Value) // Unused name based arguments 48 | 49 | // Init both varargs and kwargs to all arguments passed in 50 | for _, arg := range args { 51 | if arg.Name != "" { 52 | kwargs[arg.Name] = arg.Value 53 | } 54 | 55 | unusedArguments = append(unusedArguments, arg) 56 | } 57 | 58 | stillOrdered := true 59 | 60 | // Process all the parameters, checking what args where provided 61 | for i, param := range m.parameters { 62 | if value, found := kwargs[param.name]; found { 63 | delete(kwargs, param.name) // Consume the used keyword arg 64 | ec.SetVariable(param.name, value) 65 | 66 | stillOrdered = len(args) > i && args[i].Name == param.name 67 | 68 | if stillOrdered { 69 | unusedArguments = unusedArguments[1:] // this positional argument has been used 70 | } 71 | } else if len(args) <= i || args[i].Name != "" { 72 | stillOrdered = false 73 | 74 | if param.defaultValue != nil { 75 | value, err := param.defaultValue.Execute(ec) 76 | if err != nil { 77 | return nil, ec.ErrorAt(caller, fmt.Sprintf("Unable to understand default value for %s: %s", param.name, err)) 78 | } 79 | ec.SetVariable(param.name, value) 80 | } else { 81 | ec.SetVariable(param.name, compilerInterface.NewUndefined()) 82 | } 83 | } else if !stillOrdered { 84 | return nil, ec.ErrorAt(caller, fmt.Sprintf("Named arguments have been used out of order, please either used all named arguments or keep them in order. Unable to identify what %s should be.", param.name)) 85 | } else { 86 | ec.SetVariable(param.name, args[i].Value) 87 | unusedArguments = unusedArguments[1:] // this positional argument has been used 88 | } 89 | } 90 | 91 | // Now take all the remaining unnamed arguments, and place them in the varargs 92 | varArgs := make(compilerInterface.Arguments, 0) 93 | for _, arg := range unusedArguments { 94 | if arg.Name == "" { 95 | varArgs = append(varArgs, arg) 96 | } 97 | } 98 | 99 | ec.SetVariable("varargs", varArgs.ToVarArgs()) 100 | ec.SetVariable("kwargs", compilerInterface.NewMap(kwargs)) 101 | 102 | result, err := m.body.Execute(ec) 103 | if err != nil { 104 | return nil, err 105 | } 106 | if result == nil { 107 | return nil, ec.NilResultFor(caller) 108 | } 109 | 110 | return result.Unwrap(), err 111 | }, 112 | ) 113 | 114 | return &compilerInterface.Value{IsUndefined: true}, nil 115 | } 116 | 117 | func (m *Macro) String() string { 118 | var builder strings.Builder 119 | 120 | for i, param := range m.parameters { 121 | if i > 0 { 122 | builder.WriteString(", ") 123 | } 124 | 125 | builder.WriteString(param.name) 126 | 127 | if param.defaultValue != nil { 128 | builder.WriteString(" = ") 129 | builder.WriteString(param.defaultValue.String()) 130 | } 131 | } 132 | 133 | return fmt.Sprintf("\n{%% macro %s(%s) %%}%s{%% endmacro %%}", m.name, builder.String(), m.body.String()) 134 | } 135 | 136 | func (m *Macro) AddParameter(name string, defaultValue AST) error { 137 | if defaultValue != nil { 138 | m.numOptionalParams++ 139 | } else if m.numOptionalParams > 0 { 140 | return errors.New("can not have non-operation parameter after an optional one") 141 | } 142 | 143 | m.parameters = append( 144 | m.parameters, 145 | macroParameter{name, defaultValue}, 146 | ) 147 | 148 | return nil 149 | } 150 | 151 | func (m *Macro) AppendBody(node AST) { 152 | m.body.Append(node) 153 | } 154 | -------------------------------------------------------------------------------- /jinja/ast/Map.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "ddbt/compilerInterface" 5 | "ddbt/jinja/lexer" 6 | ) 7 | 8 | type Map struct { 9 | position lexer.Position 10 | data map[AST]AST 11 | } 12 | 13 | var _ AST = &Map{} 14 | 15 | func NewMap(token *lexer.Token) *Map { 16 | return &Map{ 17 | position: token.Start, 18 | data: make(map[AST]AST), 19 | } 20 | } 21 | 22 | func (m *Map) Position() lexer.Position { 23 | return m.position 24 | } 25 | 26 | func (m *Map) Execute(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 27 | resultMap := make(map[string]*compilerInterface.Value) 28 | 29 | for key, value := range m.data { 30 | key, err := key.Execute(ec) 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | result, err := value.Execute(ec) 36 | if err != nil { 37 | return nil, err 38 | } 39 | if result == nil { 40 | return nil, ec.NilResultFor(value) 41 | } 42 | 43 | resultMap[key.AsStringValue()] = result 44 | } 45 | 46 | return &compilerInterface.Value{ValueType: compilerInterface.MapVal, MapValue: resultMap}, nil 47 | } 48 | 49 | func (m *Map) String() string { 50 | return "" 51 | } 52 | 53 | func (m *Map) Put(key AST, value AST) { 54 | m.data[key] = value 55 | } 56 | -------------------------------------------------------------------------------- /jinja/ast/MathsOp.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | 7 | "ddbt/compilerInterface" 8 | "ddbt/jinja/lexer" 9 | ) 10 | 11 | type MathsOp struct { 12 | token *lexer.Token 13 | lhs AST 14 | rhs AST 15 | } 16 | 17 | var _ AST = &MathsOp{} 18 | 19 | func NewMathsOp(token *lexer.Token, lhs, rhs AST) *MathsOp { 20 | return &MathsOp{ 21 | token: token, 22 | lhs: lhs, 23 | rhs: rhs, 24 | } 25 | } 26 | 27 | func (op *MathsOp) Position() lexer.Position { 28 | return op.token.Start 29 | } 30 | 31 | func (op *MathsOp) Execute(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 32 | lhs, err := op.lhs.Execute(ec) 33 | if err != nil { 34 | return nil, err 35 | } 36 | if lhs == nil { 37 | return nil, ec.NilResultFor(op.lhs) 38 | } 39 | lhsNum, err := lhs.AsNumberValue() 40 | if err != nil { 41 | return nil, ec.ErrorAt(op.lhs, fmt.Sprintf("%s", err)) 42 | } 43 | 44 | rhs, err := op.rhs.Execute(ec) 45 | if err != nil { 46 | return nil, err 47 | } 48 | if rhs == nil { 49 | return nil, ec.NilResultFor(op.rhs) 50 | } 51 | rhsNum, err := rhs.AsNumberValue() 52 | if err != nil { 53 | return nil, ec.ErrorAt(op.rhs, fmt.Sprintf("%s", err)) 54 | } 55 | 56 | var result float64 57 | 58 | switch op.token.Type { 59 | case lexer.PlusToken: 60 | result = lhsNum + rhsNum 61 | 62 | case lexer.MinusToken: 63 | result = lhsNum - rhsNum 64 | 65 | case lexer.MultiplyToken: 66 | result = lhsNum * rhsNum 67 | 68 | case lexer.DivideToken: 69 | result = lhsNum / rhsNum 70 | 71 | case lexer.PowerToken: 72 | result = math.Pow(lhsNum, rhsNum) 73 | 74 | default: 75 | return nil, ec.ErrorAt(op, fmt.Sprintf("Unknown maths operator `%s`", op.token.Type)) 76 | } 77 | 78 | return compilerInterface.NewNumber(result), nil 79 | } 80 | 81 | // The parse is a 1-token look ahead parser, so it will parse 82 | // `2 + 3 * 4 + 5` into `+(2, *(3, +(4, 5)))` when due to operator 83 | // precedence rules it should be `*(+(2, 3), +(4, 5))`. 84 | // 85 | // This function will rewrite the AST tree starting with this MathOps in a left to right manor 86 | // this means this function should NEVER see a MathsOp as it's left hand side! 87 | func (op *MathsOp) ApplyOperatorPrecedenceRules() *MathsOp { 88 | if _, ok := op.lhs.(*MathsOp); ok { 89 | // Invariant failure 90 | panic("got a MathsOp as the lhs during operator precedence reordering: " + op.String()) 91 | } 92 | 93 | if rhs, ok := op.rhs.(*MathsOp); ok { 94 | if rhs.operatorPrecdence() <= op.operatorPrecdence() { 95 | // Example 1: 2 * 3 + 4 96 | // AST = *(2, +(3, 4)) 97 | // Op = *, LHS = 2, RHS = +(3, 4) 98 | // 99 | // Rewrite as: (2 * 3) + 4 100 | // AST = +(*(2, 3), 4) 101 | // Op = +, LHS = *(2, 3), RHS = 4 102 | 103 | // Example 2: 20 / 4 * 5 104 | // AST = /(20, *(4, 5)) 105 | // Op = /, LHS = 20, RHS = *(4, 5) 106 | // 107 | // Rewrite as: (20 / 4) * 5 108 | // Op = *, LHS = /(20, 4), RHS(5) 109 | 110 | now := NewMathsOp( 111 | rhs.token, 112 | NewMathsOp( 113 | op.token, 114 | op.lhs, 115 | rhs.lhs, 116 | ).ApplyOperatorPrecedenceRules(), 117 | rhs.rhs, 118 | ) 119 | 120 | return now 121 | } 122 | } 123 | 124 | return op 125 | } 126 | 127 | func (op *MathsOp) operatorPrecdence() int { 128 | switch op.token.Type { 129 | case lexer.PowerToken: 130 | return 3 131 | case lexer.MultiplyToken, lexer.DivideToken: 132 | return 2 133 | case lexer.PlusToken, lexer.MinusToken: 134 | return 1 135 | default: 136 | panic("unknown op: " + op.token.Type) 137 | } 138 | } 139 | 140 | func (op *MathsOp) String() string { 141 | return fmt.Sprintf("(%s %s %s)", op.lhs, op.token.Type, op.rhs) 142 | } 143 | -------------------------------------------------------------------------------- /jinja/ast/NoneValue.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "ddbt/compilerInterface" 5 | "ddbt/jinja/lexer" 6 | ) 7 | 8 | type NoneValue struct { 9 | position lexer.Position 10 | } 11 | 12 | var _ AST = &NoneValue{} 13 | 14 | func NewNoneValue(token *lexer.Token) *NoneValue { 15 | return &NoneValue{ 16 | position: token.Start, 17 | } 18 | } 19 | 20 | func (n *NoneValue) Position() lexer.Position { 21 | return n.position 22 | } 23 | 24 | func (n *NoneValue) Execute(_ compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 25 | return compilerInterface.NewUndefined(), nil 26 | } 27 | 28 | func (n *NoneValue) String() string { 29 | return "None" 30 | } 31 | -------------------------------------------------------------------------------- /jinja/ast/NotOperator.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | 6 | "ddbt/compilerInterface" 7 | "ddbt/jinja/lexer" 8 | ) 9 | 10 | type NotOperator struct { 11 | token *lexer.Token 12 | subCondition AST 13 | } 14 | 15 | var _ AST = &NotOperator{} 16 | 17 | func NewNotOperator(token *lexer.Token, subCondition AST) *NotOperator { 18 | return &NotOperator{ 19 | token: token, 20 | subCondition: subCondition, 21 | } 22 | } 23 | 24 | func (n *NotOperator) Position() lexer.Position { 25 | return n.token.Start 26 | } 27 | 28 | func (n *NotOperator) Execute(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 29 | result, err := n.subCondition.Execute(ec) 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | return compilerInterface.NewBoolean(!result.TruthyValue()), nil 35 | } 36 | 37 | func (n *NotOperator) String() string { 38 | return fmt.Sprintf("not %s", n.subCondition.String()) 39 | } 40 | 41 | func (n *NotOperator) ApplyOperatorPrecedenceRules() AST { 42 | if and, ok := n.subCondition.(*AndCondition); ok { 43 | and.a = NewNotOperator(n.token, and.a).ApplyOperatorPrecedenceRules() 44 | 45 | return and 46 | } 47 | 48 | if or, ok := n.subCondition.(*OrCondition); ok { 49 | or.a = NewNotOperator(n.token, or.a).ApplyOperatorPrecedenceRules() 50 | 51 | return or 52 | } 53 | 54 | return n 55 | } 56 | -------------------------------------------------------------------------------- /jinja/ast/NullValue.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "ddbt/compilerInterface" 5 | "ddbt/jinja/lexer" 6 | ) 7 | 8 | type NullValue struct { 9 | position lexer.Position 10 | } 11 | 12 | var _ AST = &NullValue{} 13 | 14 | func NewNullValue(token *lexer.Token) *NullValue { 15 | return &NullValue{ 16 | position: token.Start, 17 | } 18 | } 19 | 20 | func (n *NullValue) Position() lexer.Position { 21 | return n.position 22 | } 23 | 24 | func (n *NullValue) Execute(_ compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 25 | return &compilerInterface.Value{ValueType: compilerInterface.NullVal, IsNull: true}, nil 26 | } 27 | 28 | func (n *NullValue) String() string { 29 | return "null" 30 | } 31 | -------------------------------------------------------------------------------- /jinja/ast/Number.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "strconv" 5 | 6 | "ddbt/compilerInterface" 7 | "ddbt/jinja/lexer" 8 | ) 9 | 10 | type Number struct { 11 | position lexer.Position 12 | number float64 13 | } 14 | 15 | var _ AST = &Number{} 16 | 17 | func NewNumber(token *lexer.Token, number float64) *Number { 18 | return &Number{ 19 | position: token.Start, 20 | number: number, 21 | } 22 | } 23 | 24 | func (n *Number) Position() lexer.Position { 25 | return n.position 26 | } 27 | 28 | func (n *Number) Execute(_ compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 29 | return compilerInterface.NewNumber(n.number), nil 30 | } 31 | 32 | func (n *Number) String() string { 33 | return strconv.FormatFloat(n.number, 'f', -1, 64) 34 | } 35 | -------------------------------------------------------------------------------- /jinja/ast/OrCondition.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | 6 | "ddbt/compilerInterface" 7 | "ddbt/jinja/lexer" 8 | ) 9 | 10 | type OrCondition struct { 11 | a AST 12 | b AST 13 | } 14 | 15 | var _ AST = &OrCondition{} 16 | 17 | func NewOrCondition(a, b AST) *OrCondition { 18 | return &OrCondition{ 19 | a: a, 20 | b: b, 21 | } 22 | } 23 | 24 | func (o *OrCondition) Position() lexer.Position { 25 | return o.a.Position() 26 | } 27 | 28 | func (o *OrCondition) Execute(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 29 | // Execute the LHS 30 | result, err := o.a.Execute(ec) 31 | if err != nil { 32 | return nil, err 33 | } 34 | if result == nil { 35 | return nil, ec.NilResultFor(o.a) 36 | } 37 | 38 | // Short circuit 39 | if result.TruthyValue() { 40 | return compilerInterface.NewBoolean(true), nil 41 | } 42 | 43 | // Execute the RHS 44 | result, err = o.b.Execute(ec) 45 | if err != nil { 46 | return nil, err 47 | } 48 | if result == nil { 49 | return nil, ec.NilResultFor(o.b) 50 | } 51 | 52 | return compilerInterface.NewBoolean(result.TruthyValue()), nil 53 | } 54 | 55 | func (o *OrCondition) String() string { 56 | return fmt.Sprintf("(%s or %s)", o.a.String(), o.b.String()) 57 | } 58 | -------------------------------------------------------------------------------- /jinja/ast/SetCall.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | 6 | "ddbt/compilerInterface" 7 | "ddbt/jinja/lexer" 8 | ) 9 | 10 | type SetCall struct { 11 | position lexer.Position 12 | variableToSet string 13 | condition AST 14 | } 15 | 16 | var _ AST = &SetCall{} 17 | 18 | func NewSetCall(ident *lexer.Token, condition AST) *SetCall { 19 | return &SetCall{ 20 | position: ident.Start, 21 | variableToSet: ident.Value, 22 | condition: condition, 23 | } 24 | } 25 | 26 | func (sc *SetCall) Position() lexer.Position { 27 | return sc.position 28 | } 29 | 30 | func (sc *SetCall) Execute(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 31 | result, err := sc.condition.Execute(ec) 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | if result == nil { 37 | return nil, ec.NilResultFor(sc.condition) 38 | } 39 | 40 | ec.SetVariable(sc.variableToSet, result) 41 | 42 | return &compilerInterface.Value{IsUndefined: true}, nil 43 | } 44 | 45 | func (sc *SetCall) String() string { 46 | return fmt.Sprintf("{%% set %s = %s %%}", sc.variableToSet, sc.condition.String()) 47 | } 48 | -------------------------------------------------------------------------------- /jinja/ast/StringConcat.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | 6 | "ddbt/compilerInterface" 7 | "ddbt/jinja/lexer" 8 | ) 9 | 10 | type StringConcat struct { 11 | position lexer.Position 12 | lhs AST 13 | rhs AST 14 | } 15 | 16 | var _ AST = &StringConcat{} 17 | 18 | func NewStringConcat(token *lexer.Token, lhs, rhs AST) *StringConcat { 19 | return &StringConcat{ 20 | position: token.Start, 21 | lhs: lhs, 22 | rhs: rhs, 23 | } 24 | } 25 | 26 | func (sc *StringConcat) Position() lexer.Position { 27 | return sc.position 28 | } 29 | 30 | func (sc *StringConcat) Execute(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 31 | lhs, err := sc.lhs.Execute(ec) 32 | if err != nil { 33 | return nil, err 34 | } 35 | if lhs == nil { 36 | return nil, ec.NilResultFor(sc.lhs) 37 | } 38 | 39 | rhs, err := sc.rhs.Execute(ec) 40 | if err != nil { 41 | return nil, err 42 | } 43 | if rhs == nil { 44 | return nil, ec.NilResultFor(sc.rhs) 45 | } 46 | 47 | return compilerInterface.NewString(lhs.AsStringValue() + rhs.AsStringValue()), nil 48 | } 49 | 50 | func (sc *StringConcat) String() string { 51 | return fmt.Sprintf("%s ~ %s", sc.lhs.String(), sc.rhs.String()) 52 | } 53 | -------------------------------------------------------------------------------- /jinja/ast/TextBlock.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "strings" 5 | "unicode" 6 | 7 | "ddbt/compilerInterface" 8 | "ddbt/jinja/lexer" 9 | ) 10 | 11 | // A block which represents a simple 12 | type TextBlock struct { 13 | position lexer.Position 14 | value string 15 | } 16 | 17 | var _ AST = &TextBlock{} 18 | 19 | func NewTextBlock(token *lexer.Token) *TextBlock { 20 | return &TextBlock{ 21 | position: token.Start, 22 | value: token.Value, 23 | } 24 | } 25 | 26 | func (tb *TextBlock) Position() lexer.Position { 27 | return tb.position 28 | } 29 | 30 | func (tb *TextBlock) Execute(_ compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 31 | return &compilerInterface.Value{StringValue: tb.value}, nil 32 | } 33 | 34 | func (tb *TextBlock) String() string { 35 | return tb.value 36 | } 37 | 38 | func (tb *TextBlock) TrimPrefixWhitespace() string { 39 | tb.value = strings.TrimLeftFunc(tb.value, unicode.IsSpace) 40 | 41 | return tb.value 42 | } 43 | 44 | func (tb *TextBlock) TrimSuffixWhitespace() string { 45 | tb.value = strings.TrimRightFunc(tb.value, unicode.IsSpace) 46 | 47 | return tb.value 48 | } 49 | -------------------------------------------------------------------------------- /jinja/ast/UniaryMathsOp.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | 6 | "ddbt/compilerInterface" 7 | "ddbt/jinja/lexer" 8 | ) 9 | 10 | type UniaryMathsOp struct { 11 | position lexer.Position 12 | op lexer.TokenType 13 | value AST 14 | } 15 | 16 | var _ AST = &UniaryMathsOp{} 17 | 18 | func NewUniaryMathsOp(token *lexer.Token, value AST) *UniaryMathsOp { 19 | return &UniaryMathsOp{ 20 | position: token.Start, 21 | op: token.Type, 22 | value: value, 23 | } 24 | } 25 | 26 | func (op *UniaryMathsOp) Position() lexer.Position { 27 | return op.position 28 | } 29 | 30 | func (op *UniaryMathsOp) Execute(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 31 | value, err := op.value.Execute(ec) 32 | if err != nil { 33 | return nil, err 34 | } 35 | if value == nil { 36 | return nil, ec.NilResultFor(op.value) 37 | } 38 | valueNum, err := value.AsNumberValue() 39 | if err != nil { 40 | return nil, ec.ErrorAt(op.value, fmt.Sprintf("%s", err)) 41 | } 42 | 43 | var result float64 44 | 45 | switch op.op { 46 | case lexer.PlusToken: 47 | result = +valueNum 48 | 49 | case lexer.MinusToken: 50 | result = -valueNum 51 | 52 | default: 53 | return nil, ec.ErrorAt(op, fmt.Sprintf("Unknown maths uniary operator `%s`", op.op)) 54 | } 55 | 56 | return compilerInterface.NewNumber(result), nil 57 | } 58 | 59 | func (op *UniaryMathsOp) String() string { 60 | return fmt.Sprintf("%s%s", op.op, op.value) 61 | } 62 | -------------------------------------------------------------------------------- /jinja/ast/UnsupportedExpressionBlock.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "ddbt/compilerInterface" 5 | "ddbt/jinja/lexer" 6 | ) 7 | 8 | type UnsupportedExpressionBlock struct { 9 | position lexer.Position 10 | } 11 | 12 | var _ AST = &UnsupportedExpressionBlock{} 13 | 14 | func NewUnsupportedExpressionBlock(token *lexer.Token) *UnsupportedExpressionBlock { 15 | return &UnsupportedExpressionBlock{ 16 | position: token.Start, 17 | } 18 | } 19 | 20 | func (b *UnsupportedExpressionBlock) Position() lexer.Position { 21 | return b.position 22 | } 23 | 24 | func (b *UnsupportedExpressionBlock) Execute(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 25 | // These will return nil blocks for now 26 | return compilerInterface.NewUndefined(), nil 27 | } 28 | 29 | func (b *UnsupportedExpressionBlock) String() string { 30 | return "" 31 | } 32 | 33 | func (b *UnsupportedExpressionBlock) AppendBody(node AST) { 34 | // no-op 35 | } 36 | -------------------------------------------------------------------------------- /jinja/ast/Variable.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "ddbt/compilerInterface" 8 | "ddbt/jinja/lexer" 9 | ) 10 | 11 | type variableType = string 12 | 13 | const ( 14 | identVar variableType = "IDENT" 15 | propertyLookupVar variableType = "PROPERTY_LOOKUP" 16 | indexLookupVar variableType = "INDEX_LOOKUP" 17 | funcCallVar variableType = "FUNC_CALL" 18 | ) 19 | 20 | type Variable struct { 21 | token *lexer.Token 22 | varType variableType 23 | subVariable *Variable 24 | 25 | argCall funcCallArgs 26 | lookupKey AST 27 | isTemplateBlock bool 28 | } 29 | 30 | var _ AST = &Variable{} 31 | 32 | func NewVariable(token *lexer.Token) *Variable { 33 | return &Variable{ 34 | token: token, 35 | varType: identVar, 36 | argCall: make(funcCallArgs, 0), 37 | } 38 | } 39 | 40 | func (v *Variable) Position() lexer.Position { 41 | return v.token.Start 42 | } 43 | 44 | func (v *Variable) Execute(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 45 | variable, err := v.resolve(ec, false) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | if variable == nil { 51 | return nil, ec.ErrorAt(v, "nil variable received after resolve") 52 | } else { 53 | return variable, nil 54 | } 55 | } 56 | 57 | func (v *Variable) resolve(ec compilerInterface.ExecutionContext, isForFunctionCall bool) (*compilerInterface.Value, error) { 58 | switch v.varType { 59 | case identVar: 60 | return ec.GetVariable(v.token.Value), nil 61 | 62 | case indexLookupVar: 63 | return v.resolveIndexLookup(ec, isForFunctionCall) 64 | 65 | case propertyLookupVar: 66 | return v.resolvePropertyLookup(ec, isForFunctionCall) 67 | 68 | case funcCallVar: 69 | return v.resolveFunctionCall(ec) 70 | 71 | default: 72 | return nil, ec.ErrorAt(v, fmt.Sprintf("unable to resolve variable type %s: not implemented", v.varType)) 73 | } 74 | } 75 | 76 | func (v *Variable) resolveIndexLookup(ec compilerInterface.ExecutionContext, isForFunctionCall bool) (*compilerInterface.Value, error) { 77 | value, err := v.subVariable.resolve(ec, false) 78 | if err != nil { 79 | return nil, err 80 | } 81 | 82 | lookupKey, err := v.lookupKey.Execute(ec) 83 | if err != nil { 84 | return nil, err 85 | } 86 | if lookupKey == nil { 87 | return nil, ec.NilResultFor(v.lookupKey) 88 | } 89 | 90 | t := value.Type() 91 | switch t { 92 | case compilerInterface.ListVal: 93 | lt := lookupKey.Type() 94 | if lt != compilerInterface.NumberVal && !(lookupKey.Type() == compilerInterface.StringVal && lookupKey.StringValue == "") { 95 | return nil, ec.ErrorAt(v.lookupKey, fmt.Sprintf("Number required to index into a list, got %s", lt)) 96 | } 97 | 98 | index := int(lookupKey.NumberValue) 99 | 100 | if index < 0 { 101 | return nil, ec.ErrorAt(v.lookupKey, fmt.Sprintf("index below 0, got: %d", index)) 102 | } 103 | if index >= len(value.ListValue) { 104 | return nil, ec.ErrorAt(v.lookupKey, fmt.Sprintf("index larger than cap %d, got: %d", len(value.ListValue), index)) 105 | } 106 | 107 | return value.ListValue[index], nil 108 | 109 | case compilerInterface.MapVal: 110 | lt := lookupKey.Type() 111 | if lt != compilerInterface.StringVal || lookupKey.StringValue == "" { 112 | return nil, ec.ErrorAt(v.lookupKey, fmt.Sprintf("String required to index into a map, got %s", lt)) 113 | } 114 | 115 | rtnValue, found := value.MapValue[lookupKey.StringValue] 116 | if !found { 117 | if isForFunctionCall && lookupKey.StringValue == "items" { 118 | // If we're asking for the items of a map, return the map back 119 | return compilerInterface.NewFunction(func(_ compilerInterface.ExecutionContext, _ compilerInterface.AST, _ compilerInterface.Arguments) (*compilerInterface.Value, error) { 120 | return value, nil 121 | }), nil 122 | } else { 123 | return &compilerInterface.Value{IsUndefined: true}, nil 124 | } 125 | } 126 | return rtnValue, nil 127 | 128 | default: 129 | return nil, ec.ErrorAt(v, fmt.Sprintf("unable reference by index in a %s", t)) 130 | } 131 | } 132 | 133 | func (v *Variable) resolvePropertyLookup(ec compilerInterface.ExecutionContext, isForFunctionCall bool) (*compilerInterface.Value, error) { 134 | value, err := v.subVariable.resolve(ec, isForFunctionCall) 135 | if err != nil { 136 | return nil, err 137 | } 138 | 139 | data := value.Properties(isForFunctionCall) 140 | if data == nil { 141 | return nil, ec.ErrorAt(v, fmt.Sprintf("unable reference by property key in a %s", value.Type())) 142 | } 143 | 144 | rtnValue, found := data[v.token.Value] 145 | if !found { 146 | if isForFunctionCall && v.token.Value == "items" { 147 | // If we're asking for the items of a map, return the map back 148 | return compilerInterface.NewFunction(func(_ compilerInterface.ExecutionContext, _ compilerInterface.AST, _ compilerInterface.Arguments) (*compilerInterface.Value, error) { 149 | return value, nil 150 | }), nil 151 | } else if isForFunctionCall && v.token.Value == "get" { 152 | return compilerInterface.NewFunction(func(ec compilerInterface.ExecutionContext, caller compilerInterface.AST, args compilerInterface.Arguments) (*compilerInterface.Value, error) { 153 | defaultValue := compilerInterface.NewUndefined() 154 | 155 | switch len(args) { 156 | case 1: 157 | case 2: 158 | defaultValue = args[1].Value.Unwrap() 159 | default: 160 | return nil, ec.ErrorAt(caller, fmt.Sprintf("expected 1 or 2 arguments, got %d", len(args))) 161 | } 162 | 163 | key := args[0].Value.Unwrap() 164 | keyStr := key.AsStringValue() 165 | if keyStr == "" { 166 | return nil, ec.ErrorAt(caller, "key in the call to get() cannot be blank") 167 | } 168 | 169 | v, found := data[keyStr] 170 | if !found { 171 | return defaultValue, nil 172 | } else { 173 | return v, nil 174 | } 175 | }), nil 176 | } else { 177 | return &compilerInterface.Value{IsUndefined: true}, nil 178 | } 179 | } 180 | 181 | return rtnValue, nil 182 | } 183 | 184 | func (v *Variable) resolveFunctionCall(ec compilerInterface.ExecutionContext) (*compilerInterface.Value, error) { 185 | value, err := v.subVariable.resolve(ec, true) 186 | if err != nil { 187 | return nil, err 188 | } 189 | 190 | if value.Type() != compilerInterface.FunctionalVal && value.Function == nil { 191 | return nil, ec.ErrorAt(v.subVariable, fmt.Sprintf("expected `%s` to be a callable function, got %s", v.subVariable.String(), value.Type())) 192 | } 193 | 194 | arguments, err := v.argCall.Execute(ec) 195 | if err != nil { 196 | return nil, err 197 | } 198 | 199 | result, err := value.Function(ec.PushState(), v, arguments) 200 | if err != nil { 201 | return nil, err 202 | } 203 | 204 | if result == nil { 205 | return nil, ec.NilResultFor(v.subVariable) 206 | } 207 | 208 | return result, nil 209 | } 210 | 211 | func (v *Variable) String() string { 212 | var builder strings.Builder 213 | 214 | if v.isTemplateBlock { 215 | builder.WriteString("{{ ") 216 | } 217 | 218 | switch v.varType { 219 | case identVar: 220 | builder.WriteString(v.token.Value) 221 | 222 | case propertyLookupVar: 223 | builder.WriteString(v.subVariable.String()) 224 | builder.WriteRune('.') 225 | builder.WriteString(v.token.Value) 226 | 227 | case indexLookupVar: 228 | builder.WriteString(v.subVariable.String()) 229 | builder.WriteRune('[') 230 | builder.WriteString(v.lookupKey.String()) 231 | builder.WriteRune(']') 232 | 233 | case funcCallVar: 234 | builder.WriteString(v.subVariable.String()) 235 | builder.WriteRune('(') 236 | 237 | for i, arg := range v.argCall { 238 | if i > 0 { 239 | builder.WriteString(", ") 240 | } 241 | 242 | if arg.name != "" { 243 | builder.WriteString(arg.name) 244 | builder.WriteString("=") 245 | } 246 | 247 | builder.WriteString(arg.arg.String()) 248 | } 249 | 250 | builder.WriteRune(')') 251 | } 252 | 253 | if v.isTemplateBlock { 254 | builder.WriteString(" }}") 255 | } 256 | 257 | return builder.String() 258 | } 259 | func (v *Variable) AddArgument(argName string, node AST) { 260 | v.argCall = append(v.argCall, funcCallArg{argName, node}) 261 | } 262 | 263 | func (v *Variable) IsSimpleIdent(name string) bool { 264 | return v.varType == identVar && v.token.Value == name 265 | } 266 | 267 | func (v *Variable) wrap(wrappedVarType variableType) *Variable { 268 | nv := NewVariable(v.token) 269 | nv.varType = wrappedVarType 270 | nv.subVariable = v 271 | 272 | return nv 273 | } 274 | 275 | func (v *Variable) AsCallable() *Variable { 276 | return v.wrap(funcCallVar) 277 | } 278 | 279 | func (v *Variable) AsIndexLookup(key AST) *Variable { 280 | nv := v.wrap(indexLookupVar) 281 | nv.lookupKey = key 282 | return nv 283 | } 284 | 285 | func (v *Variable) AsPropertyLookup(key *lexer.Token) *Variable { 286 | nv := v.wrap(propertyLookupVar) 287 | nv.token = key 288 | return nv 289 | } 290 | 291 | func (v *Variable) SetIsTemplateblock() { 292 | v.isTemplateBlock = true 293 | } 294 | -------------------------------------------------------------------------------- /jinja/ast/types.go: -------------------------------------------------------------------------------- 1 | package ast 2 | 3 | import ( 4 | "ddbt/compilerInterface" 5 | ) 6 | 7 | type AST compilerInterface.AST 8 | 9 | type BodyHoldingAST interface { 10 | AST 11 | AppendBody(node AST) 12 | } 13 | 14 | type ArgumentHoldingAST interface { 15 | AST 16 | AddArgument(argName string, node AST) 17 | } 18 | -------------------------------------------------------------------------------- /jinja/lexer/token.go: -------------------------------------------------------------------------------- 1 | package lexer 2 | 3 | import "fmt" 4 | 5 | type Position struct { 6 | File string 7 | Column int 8 | Row int 9 | } 10 | 11 | type TokenType string 12 | 13 | const ( 14 | ErrorToken TokenType = "ERR" 15 | EOFToken TokenType = "EOF" 16 | 17 | // Token types in the raw text blocks 18 | TextToken TokenType = "TEXT" 19 | TemplateBlockOpen TokenType = "{{" 20 | TemplateBlockClose TokenType = "}}" 21 | ExpressionBlockOpen TokenType = "{%" 22 | ExpressionBlockOpenTrim TokenType = "{%-" 23 | ExpressionBlockClose TokenType = "%}" 24 | ExpressionBlockCloseTrim TokenType = "-%}" 25 | 26 | // Token types purely within the code blocks 27 | IdentToken TokenType = "IDENT" 28 | LeftParenthesesToken TokenType = "(" 29 | RightParenthesesToken TokenType = ")" 30 | LeftBracketToken TokenType = "[" 31 | RightBracketToken TokenType = "]" 32 | LeftBraceToken TokenType = "{" 33 | RightBraceToken TokenType = "}" 34 | EqualsToken TokenType = "=" 35 | IsEqualsToken TokenType = "==" 36 | NotEqualsToken TokenType = "!=" 37 | LessThanEqualsToken TokenType = "<=" 38 | GreaterThanEqualsToken TokenType = ">=" 39 | LessThanToken TokenType = "<" 40 | GreaterThanToken TokenType = ">" 41 | ColonToken TokenType = ":" 42 | StringToken TokenType = "STRING" 43 | NumberToken TokenType = "NUMBER" 44 | CommaToken TokenType = "," 45 | PeriodToken TokenType = "." 46 | MinusToken TokenType = "-" 47 | PlusToken TokenType = "+" 48 | MultiplyToken TokenType = "*" 49 | PowerToken TokenType = "**" 50 | DivideToken TokenType = "/" 51 | PipeToken TokenType = "|" 52 | TildeToken TokenType = "~" 53 | TrueToken TokenType = "TRUE" 54 | FalseToken TokenType = "FALSE" 55 | NullToken TokenType = "NULL" 56 | NoneToken TokenType = "None" 57 | ) 58 | 59 | type Token struct { 60 | Type TokenType 61 | Value string 62 | 63 | // Position 64 | Start Position 65 | End Position 66 | } 67 | 68 | func (t *Token) DisplayString() string { 69 | if t.Value == "" { 70 | return fmt.Sprintf("Token(`%s`)", t.Type) 71 | } else { 72 | return fmt.Sprintf("Token(`%s`, `%s`)", t.Type, t.Value) 73 | } 74 | } 75 | 76 | func (t *Token) String() string { 77 | if t.Value == "" { 78 | return fmt.Sprintf("Token(`%s`) @ %d:%d", t.Type, t.Start.Row, t.Start.Column) 79 | } else { 80 | return fmt.Sprintf("Token(`%s`, `%s`) @ %d:%d", t.Type, t.Value, t.Start.Row, t.Start.Column) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "ddbt/cmd" 5 | ) 6 | 7 | func main() { 8 | cmd.Execute() 9 | } 10 | -------------------------------------------------------------------------------- /properties/propertiesFile.go: -------------------------------------------------------------------------------- 1 | package properties 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | 8 | "gopkg.in/yaml.v2" 9 | ) 10 | 11 | const FileVersion = 2 12 | 13 | // Represents what can be held within a DBT properties file 14 | type File struct { 15 | Version int `yaml:"version"` // What version of the schema we're on (always 2) 16 | Models Models `yaml:"models,omitempty"` // List of the model schemas defined in this file 17 | Macros Macros `yaml:"macros,omitempty"` // List of the macro schemas defined in this file 18 | Seeds Models `yaml:"seeds,omitempty"` // List of the seed schemas defined in this file (same structure as a model) 19 | Snapshots Models `yaml:"snapshots,omitempty"` // List of the snapshot schemas defined in this file (same structure as a model) 20 | } 21 | 22 | // Unmarshals the file 23 | func (f *File) Unmarshal(bytes []byte) error { 24 | return yaml.Unmarshal(bytes, f) 25 | } 26 | 27 | // Marshals the file 28 | func (f *File) Marshal() ([]byte, error) { 29 | return yaml.Marshal(f) 30 | } 31 | 32 | // Write File struct to yml file 33 | func (f *File) WriteToFile(filePath string) error { 34 | y, err := f.Marshal() 35 | if err != nil { 36 | fmt.Println("Marshal yaml failed") 37 | return err 38 | } 39 | yml := string(y) 40 | 41 | sysfile, err := os.Create(filePath) 42 | if err != nil { 43 | return err 44 | } 45 | defer sysfile.Close() 46 | 47 | _, _ = sysfile.WriteString("# Auto-generated by DDBT at " + time.Now().String() + "\n") 48 | _, err = sysfile.WriteString(yml) 49 | if err != nil { 50 | return err 51 | } 52 | return nil 53 | } 54 | 55 | // Lists all the defined tests in this file 56 | // returns a map with the test name as the key and test file jinja as the value 57 | func (f *File) DefinedTests() (map[string]string, error) { 58 | tests := make(map[string]string) 59 | 60 | for _, model := range f.Models { 61 | if err := model.definedTests(tests); err != nil { 62 | return nil, err 63 | } 64 | } 65 | 66 | for _, model := range f.Seeds { 67 | if err := model.definedTests(tests); err != nil { 68 | return nil, err 69 | } 70 | } 71 | 72 | for _, model := range f.Snapshots { 73 | if err := model.definedTests(tests); err != nil { 74 | return nil, err 75 | } 76 | } 77 | 78 | return tests, nil 79 | } 80 | 81 | // The docs struct defines if a schema shows up on the docs server 82 | type Docs struct { 83 | Show *bool `yaml:"show,omitempty"` // If not set, we default to true (but need to track it's not set for when we write YAML back out) 84 | } 85 | 86 | type Models []*Model 87 | 88 | // A model/seed/snapshot schema 89 | type Model struct { 90 | Name string `yaml:"name"` 91 | Description string `yaml:"description"` 92 | Docs Docs `yaml:"docs,omitempty"` 93 | Meta MetaData `yaml:"meta,omitempty"` 94 | Tests Tests `yaml:"tests,omitempty"` // Model level tests 95 | Columns Columns `yaml:"columns,omitempty"` // Columns 96 | } 97 | 98 | func (m *Model) definedTests(tests map[string]string) error { 99 | // Table level tests 100 | for index, tableTest := range m.Tests { 101 | testName := fmt.Sprintf("%s_%s_%d", tableTest.Name, m.Name, index) 102 | jinja, err := tableTest.toTestJinja(m.Name, "") 103 | if err != nil { 104 | return err 105 | } 106 | 107 | tests[testName] = jinja 108 | } 109 | 110 | // Column level tests 111 | for _, column := range m.Columns { 112 | for index, tableTest := range column.Tests { 113 | testName := fmt.Sprintf("%s_%s__%s_%d", tableTest.Name, m.Name, column.Name, index) 114 | jinja, err := tableTest.toTestJinja(m.Name, column.Name) 115 | if err != nil { 116 | return err 117 | } 118 | 119 | tests[testName] = jinja 120 | } 121 | } 122 | 123 | return nil 124 | } 125 | 126 | type Columns []Column 127 | 128 | // Represents a single column within a Model schema 129 | type Column struct { 130 | Name string `yaml:"name"` 131 | Description string `yaml:"description"` 132 | Meta MetaData `yaml:"meta,omitempty"` 133 | Quote bool `yaml:"quote,omitempty"` 134 | Tests Tests `yaml:"tests"` 135 | Tags []string `yaml:"tags,omitempty,flow"` 136 | } 137 | 138 | type Macros []Macro 139 | 140 | // A macro schema 141 | type Macro struct { 142 | Name string `yaml:"name"` 143 | Description string `yaml:"description,omitempty"` 144 | Docs Docs `yaml:"docs,omitempty"` 145 | Arguments MacroArguments `yaml:"arguments,omitempty"` 146 | } 147 | 148 | type MacroArguments []MacroArgument 149 | 150 | // A single argument on a macro 151 | type MacroArgument struct { 152 | Name string `yaml:"name"` 153 | Type string `yaml:"type"` 154 | Description string `yaml:"description"` 155 | } 156 | 157 | // Metadata that we can store against various parts of the schema 158 | type MetaData yaml.MapSlice 159 | -------------------------------------------------------------------------------- /properties/propertiesFile_test.go: -------------------------------------------------------------------------------- 1 | package properties 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | "gopkg.in/yaml.v2" 9 | ) 10 | 11 | func TestPropertiesParse(t *testing.T) { 12 | yml := `version: 2 13 | models: 14 | - name: model_name 15 | description: random description 16 | docs: 17 | show: true 18 | meta: 19 | test: value 20 | tests: 21 | - unique: 22 | column_name: a || '-' || b 23 | columns: 24 | - name: column_name 25 | description: Test description 26 | tests: 27 | - not_null 28 | tags: ['some:tag', 'tag:2'] 29 | - name: another_column 30 | description: | 31 | This is a multi line description! 32 | quote: true 33 | tests: 34 | - accepted_values: 35 | values: 36 | - foo 37 | - bar 38 | tags: 39 | - a 40 | - b 41 | - c 42 | - relationships: 43 | to: ref('another_model') 44 | field: id 45 | - unique: 46 | severity: warn 47 | - name: another_model 48 | description: "" 49 | macros: 50 | - name: my_macro 51 | description: some macro which does stuff 52 | docs: 53 | show: false 54 | arguments: 55 | - name: test_arg 56 | type: string 57 | description: another description block 58 | - name: second_arg 59 | type: bool 60 | description: | 61 | Test description 2 62 | ` 63 | file := &File{} 64 | require.NoError(t, yaml.Unmarshal([]byte(yml), file)) 65 | 66 | assert.Equal(t, 2, file.Version) 67 | require.Len(t, file.Models, 2, "Invalid number of models") 68 | 69 | assert.Equal(t, "model_name", file.Models[0].Name) 70 | assert.Equal(t, "random description", file.Models[0].Description) 71 | assert.Equal(t, true, *file.Models[0].Docs.Show) 72 | 73 | require.Len(t, file.Models[0].Tests, 1, "Not enough model level tests") 74 | assert.Equal(t, "unique", file.Models[0].Tests[0].Name) 75 | assert.Equal(t, "column_name", file.Models[0].Tests[0].Arguments[0].Name) 76 | assert.Equal(t, "a || '-' || b", file.Models[0].Tests[0].Arguments[0].Value) 77 | assert.Empty(t, file.Models[0].Tests[0].Tags) 78 | assert.Empty(t, file.Models[0].Tests[0].Severity) 79 | 80 | require.Len(t, file.Models[0].Columns, 2, "Not enough columns") 81 | 82 | // Test column 1 83 | assert.Equal(t, "column_name", file.Models[0].Columns[0].Name) 84 | assert.Equal(t, "Test description", file.Models[0].Columns[0].Description) 85 | assert.Equal(t, false, file.Models[0].Columns[0].Quote) 86 | 87 | require.Len(t, file.Models[0].Columns[0].Tests, 1, "Not enough tests on the first column") 88 | require.Equal(t, "not_null", file.Models[0].Columns[0].Tests[0].Name) 89 | assert.Empty(t, file.Models[0].Columns[0].Tests[0].Arguments) 90 | assert.Empty(t, file.Models[0].Columns[0].Tests[0].Tags) 91 | assert.Empty(t, file.Models[0].Columns[0].Tests[0].Severity) 92 | 93 | // Test column 2 94 | assert.Equal(t, "another_column", file.Models[0].Columns[1].Name) 95 | assert.Equal(t, "This is a multi line description!\n", file.Models[0].Columns[1].Description) 96 | assert.Equal(t, true, file.Models[0].Columns[1].Quote) 97 | require.Len(t, file.Models[0].Columns[1].Tests, 3, "Not enough tests on the second column") 98 | 99 | // Test column 2; test 1 100 | require.Equal(t, "accepted_values", file.Models[0].Columns[1].Tests[0].Name) 101 | assert.Equal(t, TestArguments{TestArgument{"values", []interface{}{"foo", "bar"}}}, file.Models[0].Columns[1].Tests[0].Arguments) 102 | assert.Equal(t, []string{"a", "b", "c"}, file.Models[0].Columns[1].Tests[0].Tags) 103 | assert.Empty(t, file.Models[0].Columns[1].Tests[0].Severity) 104 | 105 | // Test column 2; test 2 106 | require.Equal(t, "relationships", file.Models[0].Columns[1].Tests[1].Name) 107 | assert.Equal(t, TestArguments{TestArgument{"to", "ref('another_model')"}, TestArgument{"field", "id"}}, file.Models[0].Columns[1].Tests[1].Arguments) 108 | assert.Empty(t, file.Models[0].Columns[1].Tests[1].Tags) 109 | assert.Empty(t, file.Models[0].Columns[1].Tests[1].Severity) 110 | 111 | // Test column 2; test 3 112 | require.Equal(t, "unique", file.Models[0].Columns[1].Tests[2].Name) 113 | assert.Empty(t, file.Models[0].Columns[1].Tests[2].Arguments) 114 | assert.Empty(t, file.Models[0].Columns[1].Tests[2].Tags) 115 | assert.Equal(t, "warn", file.Models[0].Columns[1].Tests[2].Severity) 116 | 117 | // Test output back 118 | bytes, err := yaml.Marshal(file) 119 | require.NoError(t, err, "Unable to marshal properties file back to YAML") 120 | assert.Equal(t, yml, string(bytes), "Output file didn't match") 121 | } 122 | -------------------------------------------------------------------------------- /properties/test.go: -------------------------------------------------------------------------------- 1 | package properties 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "reflect" 7 | "strings" 8 | 9 | "gopkg.in/yaml.v2" 10 | ) 11 | 12 | type Tests []*Test 13 | 14 | // Represents a test we should perform against a Model or Column 15 | // 16 | // Due to how DBT encodes these within YAML we need to write a custom set 17 | // of marshallers to ensure we can read/write the YAML files correctly 18 | // 19 | // Inside the tests slice the YAML could look like this when there are no other parameters; 20 | // 21 | // - test_name 22 | // 23 | // or like this, if we need to provide other params; 24 | // 25 | // - test_name: 26 | // arg1: something 27 | // tags: ['a', 'b', 'c'] 28 | type Test struct { 29 | Name string // the name of the inbuilt test we want to run such as "not_null" or "unique" 30 | Severity string // "warn" or "error" are the only allowed values 31 | Tags []string 32 | Arguments TestArguments 33 | } 34 | 35 | // The arguments which a test requires 36 | // Note; we use a slice here to preserve ordering if we are mutating an existing schema file 37 | // on the filesystem 38 | type TestArguments []TestArgument 39 | type TestArgument struct { 40 | Name string 41 | Value interface{} 42 | } 43 | 44 | func (o *Test) UnmarshalYAML(unmarshal func(v interface{}) error) error { 45 | var m map[string]yaml.MapSlice 46 | 47 | // Handle the case where we just have a test name 48 | if err := unmarshal(&m); err != nil { 49 | if err2 := unmarshal(&o.Name); err2 == nil { 50 | return nil 51 | } 52 | 53 | return err 54 | } 55 | 56 | // Otherwise we expect a map with a single key - the test name 57 | if len(m) != 1 { 58 | return fmt.Errorf("expected 1 key for test, got %d", len(m)) 59 | } 60 | 61 | // Now read the arguments out in the order they are present 62 | for testName, properties := range m { 63 | o.Name = testName 64 | arguments := make(TestArguments, 0) 65 | 66 | for _, property := range properties { 67 | // pull out the severity or tags key to the top level test object 68 | switch property.Key { 69 | case "severity": 70 | switch v := property.Value.(type) { 71 | case string: 72 | if v != "warn" && v != "error" { 73 | return fmt.Errorf("severity expected to be a `warn` or `error`, got %v", v) 74 | } 75 | 76 | o.Severity = v 77 | default: 78 | return fmt.Errorf("severity expected to be a `warn` or `error`, got %v", reflect.TypeOf(v)) 79 | } 80 | 81 | case "tags": 82 | switch v := property.Value.(type) { 83 | case []interface{}: 84 | tags := make([]string, 0, len(v)) 85 | 86 | for _, tagI := range v { 87 | if tag, ok := tagI.(string); ok { 88 | tags = append(tags, tag) 89 | } else { 90 | return fmt.Errorf("expected tag value to be a string, got %v", reflect.TypeOf(tagI)) 91 | } 92 | } 93 | 94 | o.Tags = tags 95 | 96 | default: 97 | return fmt.Errorf("tags expected to be an array, got %v", reflect.TypeOf(v)) 98 | } 99 | 100 | default: 101 | // otherwise transpose the test arguments to our arguments struct 102 | str, ok := property.Key.(string) 103 | if !ok { 104 | return fmt.Errorf("unable to convert property key to string: %v", property.Key) 105 | } 106 | 107 | arguments = append(arguments, TestArgument{ 108 | Name: str, 109 | Value: property.Value, 110 | }) 111 | } 112 | } 113 | 114 | o.Arguments = arguments 115 | } 116 | 117 | return nil 118 | } 119 | 120 | func (o *Test) MarshalYAML() (interface{}, error) { 121 | if len(o.Arguments) == 0 && len(o.Tags) == 0 && o.Severity == "" { 122 | return o.Name, nil 123 | } 124 | 125 | args := make(yaml.MapSlice, 0) 126 | 127 | // Write the arguments back out to a MapSlice (to preserve ordering) 128 | for _, arg := range o.Arguments { 129 | args = append(args, yaml.MapItem{Key: arg.Name, Value: arg.Value}) 130 | } 131 | 132 | // Append our tags 133 | if len(o.Tags) > 0 { 134 | args = append(args, yaml.MapItem{Key: "tags", Value: o.Tags}) 135 | } 136 | 137 | // Append the Severity 138 | if o.Severity != "" { 139 | args = append(args, yaml.MapItem{Key: "severity", Value: o.Severity}) 140 | } 141 | 142 | return yaml.MapSlice{ 143 | {Key: o.Name, Value: args}, 144 | }, nil 145 | } 146 | 147 | // Converts this test to a Jinja comptible format 148 | func (o *Test) toTestJinja(tableName, columnName string) (string, error) { 149 | var builder strings.Builder 150 | 151 | builder.WriteString("{{ test_") 152 | builder.WriteString(o.Name) 153 | 154 | builder.WriteString("( model=ref('") 155 | builder.WriteString(tableName) 156 | builder.WriteString("')") 157 | 158 | if columnName != "" { 159 | builder.WriteString(", column_name='") 160 | builder.WriteString(columnName) 161 | builder.WriteString("'") 162 | } 163 | 164 | for _, arg := range o.Arguments { 165 | jsonValue, err := json.Marshal(arg.Value) 166 | if err != nil { 167 | return "", fmt.Errorf( 168 | "Unable to convert parameter for test %s on column %s of table %s: %s", 169 | o.Name, 170 | columnName, 171 | tableName, 172 | err.Error(), 173 | ) 174 | } 175 | 176 | builder.WriteString(", ") 177 | builder.WriteString(arg.Name) 178 | builder.WriteRune('=') 179 | builder.Write(jsonValue) 180 | } 181 | 182 | builder.WriteString(") }}") 183 | 184 | return builder.String(), nil 185 | } 186 | -------------------------------------------------------------------------------- /schemaTestMacros/testNotNullMacro.go: -------------------------------------------------------------------------------- 1 | package schemaTestMacros 2 | 3 | import "fmt" 4 | 5 | func TestNotNullMacro(project string, dataset string, model string, column_name string) (string, string) { 6 | return fmt.Sprintf(`select count(*) 7 | from %s.%s.%s where %s is null 8 | `, project, dataset, model, column_name), "not_null" 9 | } 10 | -------------------------------------------------------------------------------- /schemaTestMacros/testUniqueMacro.go: -------------------------------------------------------------------------------- 1 | package schemaTestMacros 2 | 3 | import "fmt" 4 | 5 | func TestUniqueMacro(project string, dataset string, model string, column_name string) (string, string) { 6 | return fmt.Sprintf(`select count(*) 7 | from ( 8 | select 9 | %s 10 | from %s.%s.%s 11 | where %s is not null 12 | group by %s 13 | having count(*) > 1 14 | ) validation_errors 15 | `, column_name, project, dataset, model, column_name, column_name), "unique" 16 | } 17 | -------------------------------------------------------------------------------- /tests/builtin_macro_tests_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | 9 | "ddbt/compiler" 10 | "ddbt/properties" 11 | ) 12 | 13 | const testTableRef = "`unit_test_project`.`unit_test_dataset`.`target_model`" 14 | 15 | // Helper function for testing schema tests 16 | func assertTestSchema(t *testing.T, propertiesYaml string, expectedTestSQL string) { 17 | schema := &properties.File{} 18 | require.NoError(t, schema.Unmarshal([]byte(propertiesYaml)), "Unable to parse schema YAML") 19 | 20 | tests, err := schema.DefinedTests() 21 | require.NoError(t, err) 22 | require.Len(t, tests, 1, "Only expected 1 test to be generated") 23 | 24 | fileSystem, gc, _ := CompileFromRaw(t, "SELECT 1 as column_a") 25 | 26 | for testName, testContents := range tests { 27 | file, err := fileSystem.AddTestWithContents(testName, testContents, true) 28 | require.NoError(t, err, "Unable to add test file") 29 | require.NoError(t, compiler.ParseFile(file), "Unable to parse test file") 30 | require.NoError(t, compiler.CompileModel(file, gc, true), "Unable to compile test file") 31 | 32 | assert.Equal(t, expectedTestSQL, file.CompiledContents, "Compiled test doesn't match") 33 | } 34 | } 35 | 36 | func TestTestUniqueMacro(t *testing.T) { 37 | assertTestSchema(t, 38 | `version: 2 39 | models: 40 | - name: target_model 41 | columns: 42 | - name: column_a 43 | tests: 44 | - unique 45 | - name: column_b 46 | `, 47 | ` 48 | WITH test_data AS ( 49 | SELECT 50 | column_a AS value, 51 | COUNT(column_a) AS count 52 | 53 | FROM `+testTableRef+` 54 | 55 | GROUP BY column_a 56 | 57 | HAVING COUNT(column_a) > 1 58 | ) 59 | 60 | SELECT COUNT(*) as num_errors FROM test_data 61 | `, 62 | ) 63 | } 64 | 65 | func TestTestNotNull(t *testing.T) { 66 | assertTestSchema(t, 67 | `version: 2 68 | models: 69 | - name: target_model 70 | columns: 71 | - name: column_a 72 | tests: 73 | - not_null 74 | - name: column_b 75 | `, 76 | ` 77 | WITH test_data AS ( 78 | SELECT 79 | column_a AS value 80 | 81 | FROM `+testTableRef+` 82 | 83 | WHERE column_a IS NULL 84 | ) 85 | 86 | SELECT COUNT(*) as num_errors FROM test_data 87 | `, 88 | ) 89 | } 90 | 91 | func TestAcceptedValues(t *testing.T) { 92 | assertTestSchema(t, 93 | `version: 2 94 | models: 95 | - name: target_model 96 | columns: 97 | - name: column_a 98 | tests: 99 | - accepted_values: 100 | values: ['foo', 'bar'] 101 | - name: column_b 102 | `, 103 | ` 104 | WITH test_data AS ( 105 | SELECT 106 | column_a AS value 107 | 108 | FROM `+testTableRef+` 109 | 110 | WHERE column_a NOT IN ( 111 | 'foo', 'bar' 112 | ) 113 | ) 114 | 115 | SELECT COUNT(*) as num_errors FROM test_data 116 | `, 117 | ) 118 | 119 | assertTestSchema(t, 120 | `version: 2 121 | models: 122 | - name: target_model 123 | columns: 124 | - name: column_a 125 | tests: 126 | - accepted_values: 127 | values: ['foo', 'bar'] 128 | quote: false 129 | - name: column_b 130 | `, 131 | ` 132 | WITH test_data AS ( 133 | SELECT 134 | column_a AS value 135 | 136 | FROM `+testTableRef+` 137 | 138 | WHERE column_a NOT IN ( 139 | foo, bar 140 | ) 141 | ) 142 | 143 | SELECT COUNT(*) as num_errors FROM test_data 144 | `, 145 | ) 146 | } 147 | 148 | func TestTestRelationships(t *testing.T) { 149 | assertTestSchema(t, 150 | `version: 2 151 | models: 152 | - name: target_model 153 | columns: 154 | - name: column_a 155 | tests: 156 | - relationships: 157 | to: another_table 158 | field: id 159 | - name: column_b 160 | `, 161 | ` 162 | WITH test_data AS ( 163 | SELECT 164 | column_a AS value 165 | 166 | FROM `+testTableRef+` AS src 167 | 168 | LEFT JOIN another_table AS dest 169 | ON dest.id = src.column_a 170 | 171 | WHERE dest.id IS NULL AND src.column_a IS NOT NULL 172 | ) 173 | 174 | SELECT COUNT(*) as num_errors FROM test_data 175 | `, 176 | ) 177 | } 178 | -------------------------------------------------------------------------------- /tests/macro_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import "testing" 4 | 5 | func TestBasicMacro(t *testing.T) { 6 | assertCompileOutput(t, "10\nhello world!\n", 7 | ` 8 | {%- macro concat(a, b) %}{{ a ~ b }}{% endmacro -%} 9 | {{ concat(1, 0) }} 10 | {{ concat(concat("hello", " "), concat("world", "!")) }} 11 | `) 12 | } 13 | 14 | func TestCallerMacro(t *testing.T) { 15 | assertCompileOutput(t, "30\n35\n", 16 | ` 17 | {%- macro add(a) %}{{ a + caller() }}{% endmacro -%} 18 | {% call add(5) %}25{% endcall %} 19 | {% call add(5) %}{{25 + a }}{% endcall %} 20 | `) 21 | } 22 | 23 | func TestReturnInMacro(t *testing.T) { 24 | assertCompileOutput(t, "pass", 25 | ` 26 | {%- macro test(a) %}{{ return(a) }} This should not be returned; {{ caller() }}{% endmacro -%} 27 | {% call test("pass") %}fail{% endcall -%} 28 | `) 29 | } 30 | 31 | func TestMacroDefaults(t *testing.T) { 32 | assertCompileOutput(t, "pass, 1, 2, 3", 33 | ` 34 | {%- macro test(a, b=[1, 2, 3]) -%} 35 | {{ a }} 36 | {%- for value in b -%} 37 | , {{ value }} 38 | {%- endfor -%} 39 | {%- endmacro -%} 40 | {{ test("pass") }}`) 41 | } 42 | 43 | func TestMapSetWithVariableKey(t *testing.T) { 44 | assertCompileOutput(t, "pass", 45 | ` 46 | {%- macro test(a) -%} 47 | {{ a.c }} 48 | {%- endmacro -%} 49 | {%- set b = 'c' -%} 50 | {{ test( {b:'pass'} ) }}`) 51 | } 52 | -------------------------------------------------------------------------------- /tests/regression_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestUDFEscapedQuotes(t *testing.T) { 8 | const udf = "test1 \\'test2\\' \\\\\\ " 9 | assertCompileOutput(t, "test1 'test2' \\\\ test3", "{{ config(udf='''"+udf+"''') }}test3") 10 | } 11 | -------------------------------------------------------------------------------- /tests/utils_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "ddbt/bigquery" 8 | "ddbt/compiler" 9 | "ddbt/compilerInterface" 10 | "ddbt/config" 11 | "ddbt/fs" 12 | "ddbt/jinja" 13 | 14 | "github.com/stretchr/testify/assert" 15 | "github.com/stretchr/testify/require" 16 | ) 17 | 18 | var testVariables = map[string]*compilerInterface.Value{ 19 | "table_name": {StringValue: "BLAH"}, 20 | "number_value": {NumberValue: 1}, 21 | "str_number_value": {StringValue: "2"}, 22 | "map_object": { 23 | MapValue: map[string]*compilerInterface.Value{ 24 | "string": {StringValue: "test"}, 25 | "nested": { 26 | MapValue: map[string]*compilerInterface.Value{ 27 | "number": {NumberValue: 3}, 28 | "string": {StringValue: "FROM"}, 29 | }, 30 | }, 31 | "key": {StringValue: "42"}, 32 | }, 33 | }, 34 | "list_object": { 35 | ListValue: []*compilerInterface.Value{ 36 | {StringValue: "first option is string"}, 37 | {StringValue: "second option a string too!"}, 38 | {StringValue: "third"}, 39 | {MapValue: map[string]*compilerInterface.Value{ 40 | "blah": {ListValue: []*compilerInterface.Value{ 41 | {StringValue: "thingy"}, 42 | }}, 43 | }}, 44 | {ListValue: []*compilerInterface.Value{ 45 | {StringValue: "nested list test"}, 46 | {NumberValue: 3}, 47 | }}, 48 | }, 49 | }, 50 | } 51 | 52 | var debugPrintAST = false 53 | 54 | func CompileFromRaw(t *testing.T, raw string) (*fs.FileSystem, *compiler.GlobalContext, string) { 55 | fileSystem, err := fs.InMemoryFileSystem( 56 | map[string]string{ 57 | "models/target_model.sql": raw, 58 | }, 59 | ) 60 | require.NoError(t, err, "Unable to construct in memory file system") 61 | 62 | for _, file := range fileSystem.AllFiles() { 63 | require.NoError(t, parseFile(file), "Unable to parse %s %s", file.Type, file.Name) 64 | } 65 | 66 | file := fileSystem.Model("target_model") 67 | require.NotNil(t, file, "Unable to extract the target_model from the In memory file system") 68 | require.NotNil(t, file.SyntaxTree, "target_model syntax tree is empty!") 69 | 70 | // Create the execution context 71 | config.GlobalCfg = &config.Config{ 72 | Name: "Unit Test", 73 | Target: &config.Target{ 74 | Name: "unit_test", 75 | ProjectID: "unit_test_project", 76 | DataSet: "unit_test_dataset", 77 | Location: "US", 78 | Threads: 4, 79 | }, 80 | } 81 | gc, err := compiler.NewGlobalContext(config.GlobalCfg, fileSystem) 82 | require.NoError(t, err, "Unable to create global context") 83 | 84 | macros := fileSystem.Macro("built-in-macros") 85 | require.NotNil(t, file, "Built in macros was nil") 86 | require.NoError(t, compiler.CompileModel(macros, gc, false), "Unable to compile built in macros") 87 | 88 | ec := compiler.NewExecutionContext(file, fileSystem, true, gc, gc) 89 | ec.SetVariable("config", file.ConfigObject()) 90 | for key, value := range testVariables { 91 | ec.SetVariable(key, value) 92 | } 93 | 94 | finalAST, err := file.SyntaxTree.Execute(ec) 95 | require.NoError(t, err) 96 | require.NotNil(t, finalAST, "Output AST is nil") 97 | file.CompiledContents = finalAST.AsStringValue() 98 | 99 | return fileSystem, gc, bigquery.BuildQuery(file) 100 | } 101 | 102 | func assertCompileOutput(t *testing.T, expected, input string) { 103 | _, _, contentsOfModel := CompileFromRaw(t, input) 104 | 105 | assert.Equal( 106 | t, 107 | expected, 108 | contentsOfModel, 109 | "Unexpected output from %s", 110 | input, 111 | ) 112 | } 113 | 114 | func parseFile(file *fs.File) error { 115 | syntaxTree, err := jinja.Parse(file) 116 | if err != nil { 117 | return err 118 | } 119 | 120 | if debugPrintAST { 121 | debugPrintAST = false 122 | fmt.Println(syntaxTree.String()) 123 | } 124 | 125 | file.SyntaxTree = syntaxTree 126 | return nil 127 | } -------------------------------------------------------------------------------- /utils/Debounce.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | ) 7 | 8 | type DebouncedFunction func() 9 | 10 | type debounce struct { 11 | m sync.Mutex 12 | timer *time.Timer 13 | } 14 | 15 | func Debounce(f DebouncedFunction, duration time.Duration) DebouncedFunction { 16 | debouncer := &debounce{} 17 | 18 | execute := func() { 19 | debouncer.m.Lock() 20 | defer debouncer.m.Unlock() 21 | 22 | f() 23 | debouncer.timer = nil 24 | } 25 | 26 | return func() { 27 | debouncer.m.Lock() 28 | defer debouncer.m.Unlock() 29 | 30 | if debouncer.timer != nil { 31 | debouncer.timer.Stop() 32 | } 33 | 34 | debouncer.timer = time.AfterFunc(duration, execute) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /utils/ProgressBar.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "github.com/mattn/go-isatty" 6 | "io" 7 | "math" 8 | "os" 9 | "strconv" 10 | "strings" 11 | "sync" 12 | "sync/atomic" 13 | "time" 14 | ) 15 | 16 | const ( 17 | leftEdgeRune = '▕' 18 | rightEdgeRune = '▏' 19 | filledRune = '▇' 20 | blankRune = '-' 21 | 22 | labelColumnWidth = 35 23 | rightColumnWidth = 30 24 | refreshRate = 16 * time.Millisecond // 60 fps! 25 | ) 26 | 27 | type ProgressBar struct { 28 | label string 29 | completedItems uint32 30 | numberItems uint32 31 | output io.Writer 32 | startTime time.Time 33 | lastIncremented time.Time 34 | 35 | started bool 36 | startMutex sync.Mutex 37 | finishTicking chan struct{} 38 | tickingFinished chan struct{} 39 | ticker *time.Ticker 40 | 41 | statusRowMutex sync.Mutex 42 | statusRowsLastRender int 43 | statusRows []*StatusRow 44 | } 45 | 46 | func NewProgressBar(label string, numberItems int) *ProgressBar { 47 | pb := &ProgressBar{ 48 | label: label, 49 | completedItems: 0, 50 | numberItems: uint32(numberItems), 51 | output: os.Stderr, 52 | startTime: time.Now(), 53 | lastIncremented: time.Now(), 54 | 55 | finishTicking: make(chan struct{}), 56 | tickingFinished: make(chan struct{}), 57 | 58 | statusRows: make([]*StatusRow, 0), 59 | } 60 | 61 | pb.Start() 62 | 63 | return pb 64 | } 65 | 66 | func (pb *ProgressBar) Increment() { 67 | atomic.AddUint32(&pb.completedItems, 1) 68 | pb.lastIncremented = time.Now() // don't care about raising sets here, it's only for a rough guess of how fast we are 69 | } 70 | 71 | func (pb *ProgressBar) Width() int { 72 | width, err := TerminalWidth() 73 | 74 | if err != nil { 75 | // default width 76 | width = 400 77 | } 78 | 79 | return width 80 | } 81 | 82 | func (pb *ProgressBar) draw(isFinalDraw bool) { 83 | pb.statusRowMutex.Lock() 84 | defer pb.statusRowMutex.Unlock() 85 | 86 | termWidth := pb.Width() 87 | 88 | var builder strings.Builder 89 | 90 | for i := 0; i < pb.statusRowsLastRender; i++ { 91 | // Clear the line 92 | builder.WriteString("\r\033[K") // reset to the beginning of the line and reset it 93 | 94 | // Move up a line 95 | builder.WriteString("\x1b[1A\x1b[2K") 96 | } 97 | 98 | builder.WriteString("\r\033[K") // reset to the beginning of the line and reset it 99 | builder.WriteString(pb.String(termWidth)) 100 | 101 | if !isFinalDraw { 102 | for _, row := range pb.statusRows { 103 | builder.WriteString(row.String(termWidth)) 104 | } 105 | 106 | pb.statusRowsLastRender = len(pb.statusRows) 107 | } else { 108 | builder.WriteRune('\n') 109 | pb.statusRowsLastRender = 0 110 | } 111 | 112 | _, _ = pb.output.Write([]byte(builder.String())) 113 | } 114 | 115 | func (pb *ProgressBar) Start() { 116 | pb.startMutex.Lock() 117 | 118 | if pb.started { 119 | pb.startMutex.Unlock() 120 | return 121 | } 122 | pb.started = true 123 | pb.startMutex.Unlock() 124 | 125 | // Check if we're running in a TTY, we only want to draw if it is 126 | if isatty.IsTerminal(os.Stdout.Fd()) { 127 | pb.ticker = time.NewTicker(refreshRate) 128 | 129 | // Write the initial process of the bar 130 | _, _ = pb.output.Write([]byte(pb.String(pb.Width()))) 131 | go pb.tick() 132 | } else { 133 | // Just print the label and start a goroutine to finish ticking 134 | _, _ = pb.output.Write([]byte(pb.label + "\n")) 135 | go func() { 136 | <-pb.finishTicking 137 | pb.tickingFinished <- struct{}{} 138 | }() 139 | } 140 | } 141 | 142 | func (pb *ProgressBar) Stop() { 143 | pb.startMutex.Lock() 144 | defer pb.startMutex.Unlock() 145 | 146 | if !pb.started { 147 | return 148 | } 149 | 150 | pb.finishTicking <- struct{}{} 151 | <-pb.tickingFinished 152 | 153 | pb.started = false 154 | } 155 | 156 | func (pb *ProgressBar) tick() { 157 | for { 158 | select { 159 | case <-pb.ticker.C: 160 | // Draw an update in progress 161 | pb.draw(false) 162 | 163 | case <-pb.finishTicking: 164 | pb.ticker.Stop() 165 | 166 | // Draw the final update 167 | pb.draw(true) 168 | 169 | pb.tickingFinished <- struct{}{} 170 | return 171 | } 172 | } 173 | } 174 | 175 | func (pb *ProgressBar) lastUpdateTime() time.Time { 176 | if pb.started { 177 | return time.Now() 178 | } else { 179 | return pb.lastIncremented 180 | } 181 | } 182 | 183 | func (pb *ProgressBar) String(termWidth int) string { 184 | completed := atomic.LoadUint32(&pb.completedItems) // Because this is atomically updated, grab a local reference 185 | percentage := float64(completed) / float64(pb.numberItems) 186 | 187 | if percentage != percentage { 188 | // If we have zero items, then that progress bar is always at 100% 189 | percentage = 1 190 | } 191 | 192 | var builder strings.Builder 193 | 194 | // Draw the right hand edge first, so we know how many columns it will be in size 195 | numItemsStr := strconv.Itoa(int(pb.numberItems)) 196 | compeltedStr := strconv.Itoa(int(completed)) 197 | for i := len(compeltedStr); i < len(numItemsStr); i++ { 198 | builder.WriteRune(' ') 199 | } 200 | builder.WriteString(compeltedStr) 201 | builder.WriteRune('/') 202 | builder.WriteString(numItemsStr) 203 | 204 | // Write the time it's taken 205 | duration := pb.lastUpdateTime().Sub(pb.startTime) 206 | builder.WriteString(fmt.Sprintf( 207 | " [%02.0f:%02d]", 208 | math.Floor(duration.Minutes()), 209 | int64(duration.Seconds())%60, 210 | )) 211 | 212 | // Display our operations per second 213 | builder.WriteString(fmt.Sprintf( 214 | " %6.0f op/s", 215 | (float64(completed)*1e9)/float64(duration.Nanoseconds()), 216 | )) 217 | 218 | rightEdge := builder.String() 219 | 220 | if rightColumnWidth > len(rightEdge) { 221 | rightEdge = strings.Repeat(" ", rightColumnWidth-len(rightEdge)) + rightEdge 222 | } 223 | builder.Reset() 224 | 225 | // Draw the left hand edge 226 | builder.WriteString(pb.label) 227 | 228 | if toFill := labelColumnWidth - builder.Len(); toFill > 0 { 229 | builder.WriteString(strings.Repeat(" ", toFill)) // Create a buffer so that all the labels align 230 | } 231 | 232 | builder.WriteString(fmt.Sprintf("%3.0f%%", percentage*100)) 233 | 234 | // Calculate the Percentage & number of bars to fill 235 | spaceForProgressBar := termWidth - builder.Len() - 2 - len(rightEdge) // (left/right edge runes) 236 | barsToFill := int(math.Round(float64(spaceForProgressBar) * percentage)) 237 | 238 | // Draw the actual progress bar itself 239 | builder.WriteRune(leftEdgeRune) 240 | for i := 0; i < spaceForProgressBar; i++ { 241 | if barsToFill > i { 242 | builder.WriteRune(filledRune) 243 | } else { 244 | builder.WriteRune(blankRune) 245 | } 246 | } 247 | builder.WriteRune(rightEdgeRune) 248 | 249 | // Add the right edge text 250 | builder.WriteString(rightEdge) 251 | 252 | return builder.String() 253 | } 254 | 255 | func (pb *ProgressBar) NewStatusRow() *StatusRow { 256 | pb.statusRowMutex.Lock() 257 | defer pb.statusRowMutex.Unlock() 258 | 259 | sr := &StatusRow{} 260 | sr.SetIdle() 261 | pb.statusRows = append(pb.statusRows, sr) 262 | 263 | return sr 264 | } 265 | 266 | type StatusRow struct { 267 | m sync.Mutex 268 | 269 | message string 270 | changed time.Time 271 | isIdle bool 272 | } 273 | 274 | func (sr *StatusRow) Update(message string) { 275 | sr.m.Lock() 276 | defer sr.m.Unlock() 277 | 278 | sr.message = message 279 | sr.changed = time.Now() 280 | sr.isIdle = false 281 | } 282 | 283 | func (sr *StatusRow) SetIdle() { 284 | sr.m.Lock() 285 | defer sr.m.Unlock() 286 | 287 | sr.isIdle = true 288 | sr.changed = time.Now() 289 | sr.message = "Idle" 290 | } 291 | 292 | func (sr *StatusRow) String(termWidth int) string { 293 | sr.m.Lock() 294 | defer sr.m.Unlock() 295 | 296 | var builder strings.Builder 297 | 298 | builder.WriteString("\n ↳ ") 299 | 300 | if !sr.isIdle { 301 | duration := time.Since(sr.changed) 302 | builder.WriteString(fmt.Sprintf( 303 | "[%02.0f:%02d]", 304 | math.Floor(duration.Minutes()), 305 | int64(duration.Seconds())%60, 306 | )) 307 | } else { 308 | builder.WriteString("[--:--]") 309 | } 310 | builder.WriteString(" ") 311 | 312 | builder.WriteString(sr.message) 313 | 314 | // Overwrite any old characters from the previous render 315 | spaces := termWidth - builder.Len() 316 | if spaces > 0 { 317 | builder.WriteString(strings.Repeat(" ", spaces)) 318 | } 319 | 320 | return builder.String() 321 | } 322 | -------------------------------------------------------------------------------- /utils/terminal.go: -------------------------------------------------------------------------------- 1 | // +build linux darwin freebsd netbsd openbsd solaris dragonfly 2 | 3 | package utils 4 | 5 | import ( 6 | "fmt" 7 | "os" 8 | "syscall" 9 | "unsafe" 10 | ) 11 | 12 | var tty *os.File 13 | 14 | func init() { 15 | var err error 16 | tty, err = os.Open("/dev/tty") 17 | 18 | // If we can't get the tty, get the Stdout instead as that should be attached to the terminal 19 | if err != nil { 20 | tty = os.Stdout 21 | } 22 | } 23 | 24 | type window struct { 25 | Row uint16 26 | Col uint16 27 | Xpixel uint16 28 | Ypixel uint16 29 | } 30 | 31 | // Gets the current terminal width and returns an error if we can't get it 32 | func TerminalWidth() (int, error) { 33 | w := new(window) 34 | returnCode, _, err := syscall.Syscall( 35 | syscall.SYS_IOCTL, 36 | tty.Fd(), 37 | uintptr(syscall.TIOCGWINSZ), 38 | uintptr(unsafe.Pointer(w)), 39 | ) 40 | 41 | if int(returnCode) == -1 { 42 | return 0, err 43 | } 44 | 45 | return int(w.Col), nil 46 | } 47 | 48 | func ClearTerminal() { 49 | // Clear the screen and then move the cursor to column 1, row 1 50 | fmt.Print("\033[2J\033[1;1H") 51 | } 52 | -------------------------------------------------------------------------------- /utils/version.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | const DdbtVersion = "0.6.8" 4 | -------------------------------------------------------------------------------- /watcher/Event.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import "os" 4 | 5 | type EventType = string 6 | 7 | const ( 8 | CREATED EventType = "Created" 9 | MODIFIED EventType = "Modified" 10 | DELETED EventType = "Deleted" 11 | ) 12 | 13 | type Event struct { 14 | EventType EventType 15 | Path string 16 | Info os.FileInfo 17 | } 18 | 19 | type Events struct { 20 | latestEvents map[string]Event // Latest event per file 21 | } 22 | 23 | func newEventBatch() *Events { 24 | return &Events{ 25 | latestEvents: make(map[string]Event), 26 | } 27 | } 28 | 29 | func (e *Events) addEvent(path string, event EventType, info os.FileInfo) { 30 | e.latestEvents[path] = Event{ 31 | EventType: event, 32 | Path: path, 33 | Info: info, 34 | } 35 | } 36 | 37 | func (e *Events) Events() []Event { 38 | events := make([]Event, 0, len(e.latestEvents)) 39 | for _, event := range e.latestEvents { 40 | events = append(events, event) 41 | } 42 | 43 | return events 44 | } 45 | -------------------------------------------------------------------------------- /watcher/Watcher.go: -------------------------------------------------------------------------------- 1 | package watcher 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "sync" 8 | "time" 9 | 10 | "github.com/fsnotify/fsnotify" 11 | 12 | "ddbt/utils" 13 | ) 14 | 15 | type Watcher struct { 16 | mutex sync.Mutex 17 | 18 | watcher *fsnotify.Watcher 19 | directories map[string]struct{} 20 | stop chan struct{} 21 | 22 | EventsReady chan struct{} 23 | 24 | events *Events 25 | notifyListener utils.DebouncedFunction 26 | } 27 | 28 | func NewWatcher() (*Watcher, error) { 29 | fswatcher, err := fsnotify.NewWatcher() 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | w := &Watcher{ 35 | watcher: fswatcher, 36 | directories: make(map[string]struct{}), 37 | stop: make(chan struct{}), 38 | events: nil, 39 | EventsReady: make(chan struct{}), 40 | } 41 | 42 | // We debounce this to give the system time to process mass file updates 43 | w.notifyListener = utils.Debounce(func() { 44 | w.EventsReady <- struct{}{} 45 | }, 50*time.Millisecond) 46 | 47 | go w.listenForChangeEvents() 48 | 49 | return w, nil 50 | } 51 | 52 | func (w *Watcher) RecursivelyWatch(folder string) error { 53 | folder = filepath.Clean(folder) 54 | 55 | w.mutex.Lock() 56 | 57 | // Track the fact we're watching this directory 58 | if _, found := w.directories[folder]; found { 59 | w.mutex.Unlock() 60 | return nil 61 | } 62 | w.directories[folder] = struct{}{} 63 | w.mutex.Unlock() // unlock here to prevent reentrant locks during recursion 64 | 65 | if err := w.watcher.Add(folder); err != nil { 66 | return err 67 | } 68 | 69 | return filepath.Walk(folder, func(path string, info os.FileInfo, err error) error { 70 | if err != nil { 71 | return err 72 | } 73 | 74 | if info.IsDir() { 75 | return w.RecursivelyWatch(path) 76 | } 77 | 78 | return nil 79 | }) 80 | } 81 | 82 | func (w *Watcher) listenForChangeEvents() { 83 | for { 84 | select { 85 | case <-w.stop: 86 | _ = w.watcher.Close() 87 | return 88 | 89 | case event := <-w.watcher.Events: 90 | switch { 91 | case event.Op&fsnotify.Create == fsnotify.Create: 92 | w.handleCreateEvent(event.Name) 93 | case event.Op&fsnotify.Write == fsnotify.Write: 94 | w.handleWriteEvent(event.Name) 95 | case event.Op&fsnotify.Remove == fsnotify.Remove: 96 | w.handleDeleteEvent(event.Name) 97 | } 98 | 99 | case err := <-w.watcher.Errors: 100 | fmt.Println("ERROR", err) 101 | } 102 | } 103 | } 104 | 105 | func (w *Watcher) handleCreateEvent(path string) { 106 | if info, err := os.Stat(path); err != nil { 107 | fmt.Printf("⚠️ Unable to stat %s: %s\n", path, err) 108 | } else if info.IsDir() { 109 | if err := w.RecursivelyWatch(path); err != nil { 110 | fmt.Printf("⚠️ Unable to start watching %s: %s\n", path, err) 111 | } 112 | } else { 113 | w.recordEventInBatch(path, CREATED, info) 114 | } 115 | } 116 | 117 | func (w *Watcher) handleDeleteEvent(path string) { 118 | // If it's a directory we're watching, stop watching it 119 | w.mutex.Lock() 120 | delete(w.directories, path) 121 | w.mutex.Unlock() 122 | 123 | w.recordEventInBatch(path, DELETED, nil) 124 | } 125 | 126 | func (w *Watcher) handleWriteEvent(path string) { 127 | if info, err := os.Stat(path); err != nil { 128 | fmt.Printf("⚠️ Unable to stat %s: %s\n", path, err) 129 | } else if !info.IsDir() { 130 | w.recordEventInBatch(path, MODIFIED, info) 131 | } 132 | } 133 | 134 | func (w *Watcher) recordEventInBatch(path string, event EventType, info os.FileInfo) { 135 | w.mutex.Lock() 136 | defer w.mutex.Unlock() 137 | 138 | if w.events == nil { 139 | w.events = newEventBatch() 140 | w.notifyListener() 141 | } 142 | 143 | w.events.addEvent(path, event, info) 144 | } 145 | 146 | func (w *Watcher) GetEventsBatch() *Events { 147 | w.mutex.Lock() 148 | defer w.mutex.Unlock() 149 | 150 | events := w.events 151 | w.events = nil 152 | 153 | return events 154 | } 155 | 156 | func (w *Watcher) Close() { 157 | w.stop <- struct{}{} 158 | close(w.EventsReady) 159 | close(w.stop) 160 | } 161 | --------------------------------------------------------------------------------