├── .github └── workflows │ └── ci.yml ├── .gitignore ├── CHANGELOG.md ├── CODE-OF-CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.txt ├── NOTICE.txt ├── README.md ├── artifact ├── plot.r ├── process_trace.py └── run_emulation.sh ├── benchmarks ├── README.md ├── build.gradle └── src │ └── main │ └── java │ └── com │ └── vmware │ └── dcm │ ├── BatchingBenchmark.java │ ├── EndToEndBenchmark.java │ ├── H2Bench.java │ ├── OrToolsEncodingBenchmark.java │ ├── OrToolsIndexBenchmark.java │ ├── ScaleModelBenchmark.java │ └── ScaleNodeBenchmark.java ├── build.gradle ├── config ├── checkstyle │ ├── checkstyle.xml │ └── suppressions.xml └── spotbugs │ └── findbugs-exclude.xml ├── dcm ├── build.gradle └── src │ ├── main │ ├── codegen │ │ ├── config.fmpp │ │ └── includes │ │ │ └── parserImpls.ftl │ ├── java │ │ └── com │ │ │ └── vmware │ │ │ └── dcm │ │ │ ├── Conf.java │ │ │ ├── ExtractAccessedTables.java │ │ │ ├── ExtractConstraintInQuery.java │ │ │ ├── ExtractGroupTable.java │ │ │ ├── Model.java │ │ │ ├── ModelException.java │ │ │ ├── RemoveControllablePredicates.java │ │ │ ├── SolverException.java │ │ │ ├── annotations │ │ │ ├── FieldsAreNonnullByDefault.java │ │ │ └── MethodsAreNonnullByDefault.java │ │ │ ├── backend │ │ │ ├── IGeneratedBackend.java │ │ │ ├── ISolverBackend.java │ │ │ ├── RewriteArity.java │ │ │ ├── RewriteContains.java │ │ │ ├── minizinc │ │ │ │ ├── FindStringLiterals.java │ │ │ │ ├── MinizincCodeGenerator.java │ │ │ │ ├── MinizincSolver.java │ │ │ │ ├── MinizincString.java │ │ │ │ ├── RewriteCountFunction.java │ │ │ │ ├── RewriteNullPredicates.java │ │ │ │ └── SplitIntoSingleHeadComprehensions.java │ │ │ ├── ortools │ │ │ │ ├── ContainsFunctionCall.java │ │ │ │ ├── DetermineIndexes.java │ │ │ │ ├── GetColumnIdentifiers.java │ │ │ │ ├── GroupContext.java │ │ │ │ ├── IsConstantSubquery.java │ │ │ │ ├── JavaExpression.java │ │ │ │ ├── JavaType.java │ │ │ │ ├── JavaTypeList.java │ │ │ │ ├── JoinStrategy.java │ │ │ │ ├── Ops.java │ │ │ │ ├── OrToolsSolver.java │ │ │ │ ├── OutputIR.java │ │ │ │ ├── ScalarProductOptimization.java │ │ │ │ ├── StringEncoding.java │ │ │ │ ├── SubQueryContext.java │ │ │ │ ├── TranslationContext.java │ │ │ │ ├── TupleGen.java │ │ │ │ └── TupleMetadata.java │ │ │ └── package-info.java │ │ │ ├── compiler │ │ │ ├── DesugarExists.java │ │ │ ├── FromExtractor.java │ │ │ ├── IRColumn.java │ │ │ ├── IRColumnsFromSelectItems.java │ │ │ ├── IRContext.java │ │ │ ├── IRForeignKey.java │ │ │ ├── IRPrimaryKey.java │ │ │ ├── IRTable.java │ │ │ ├── ModelCompiler.java │ │ │ ├── Program.java │ │ │ ├── SyntaxChecking.java │ │ │ ├── TranslateViewToIR.java │ │ │ ├── UsesAggregateFunctions.java │ │ │ ├── UsesControllableFields.java │ │ │ ├── ir │ │ │ │ ├── BinaryOperatorPredicate.java │ │ │ │ ├── BinaryOperatorPredicateWithAggregate.java │ │ │ │ ├── CheckQualifier.java │ │ │ │ ├── ColumnIdentifier.java │ │ │ │ ├── ComprehensionRewriter.java │ │ │ │ ├── ExistsPredicate.java │ │ │ │ ├── Expr.java │ │ │ │ ├── FunctionCall.java │ │ │ │ ├── GroupByComprehension.java │ │ │ │ ├── GroupByQualifier.java │ │ │ │ ├── Head.java │ │ │ │ ├── IRVisitor.java │ │ │ │ ├── IsNotNullPredicate.java │ │ │ │ ├── IsNullPredicate.java │ │ │ │ ├── JoinPredicate.java │ │ │ │ ├── ListComprehension.java │ │ │ │ ├── Literal.java │ │ │ │ ├── Qualifier.java │ │ │ │ ├── SimpleVisitor.java │ │ │ │ ├── TableRowGenerator.java │ │ │ │ ├── UnaryOperator.java │ │ │ │ ├── VoidType.java │ │ │ │ └── package-info.java │ │ │ └── package-info.java │ │ │ ├── package-info.java │ │ │ └── parser │ │ │ └── SqlCreateConstraint.java │ └── resources │ │ ├── mnz_data.ftl │ │ └── mnz_model.ftl │ └── test │ ├── java │ └── com │ │ └── vmware │ │ └── dcm │ │ ├── ExtractGroupTablesTest.java │ │ ├── ModelTest.java │ │ ├── ParserTest.java │ │ ├── ScalarProductOptimizationTest.java │ │ ├── backend │ │ └── ortools │ │ │ ├── CoreTest.java │ │ │ ├── OpsTests.java │ │ │ ├── OrToolsIntervalsTest.java │ │ │ └── OrToolsTest.java │ │ └── compiler │ │ └── SyntaxCheckTest.java │ └── resources │ └── .gitignore ├── docs ├── arch_detailed.png ├── reference.md └── tutorial.md ├── examples ├── build.gradle └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── vmware │ │ │ └── dcm │ │ │ └── examples │ │ │ └── LoadBalance.java │ └── resources │ │ ├── log4j.properties │ │ └── schema.sql │ └── test │ └── java │ └── com │ └── vmware │ └── dcm │ └── examples │ ├── LoadBalanceTest.java │ ├── QuickStartTest.java │ └── SolverConfigurationTest.java ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── k8s-scheduler ├── README.md ├── build.gradle └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── vmware │ │ │ └── dcm │ │ │ ├── AutoScope.java │ │ │ ├── DBConnectionPool.java │ │ │ ├── DBViews.java │ │ │ ├── DebugUtils.java │ │ │ ├── EmulatedCluster.java │ │ │ ├── EmulatedPodDeployer.java │ │ │ ├── EmulatedPodToNodeBinder.java │ │ │ ├── IPodDeployer.java │ │ │ ├── IPodToNodeBinder.java │ │ │ ├── KubernetesBinder.java │ │ │ ├── KubernetesPodDeployer.java │ │ │ ├── KubernetesStateSync.java │ │ │ ├── NodeResourceEventHandler.java │ │ │ ├── PdbResourceEventHandler.java │ │ │ ├── PodEventsToDatabase.java │ │ │ ├── PodResourceEventHandler.java │ │ │ ├── Policies.java │ │ │ ├── Scheduler.java │ │ │ ├── ScopedModel.java │ │ │ ├── Utils.java │ │ │ └── trace │ │ │ └── TraceReplayer.java │ └── resources │ │ ├── log4j2.xml │ │ ├── pod-only.yml │ │ ├── pod-with-affinity.yml │ │ └── scheduler_tables.sql │ └── test │ ├── java │ └── com │ │ └── vmware │ │ └── dcm │ │ ├── AutoScopeTest.java │ │ ├── ITBase.java │ │ ├── KubernetesStateSyncTest.java │ │ ├── SchedulerIT.java │ │ ├── SchedulerTest.java │ │ ├── ScopeTest.java │ │ ├── TestScenario.java │ │ ├── UnbindTest.java │ │ ├── WorkloadGeneratorIT.java │ │ ├── WorkloadGeneratorTest.java │ │ └── WorkloadReplayTest.java │ └── resources │ ├── app-no-constraints.yml │ ├── cache-example.yml │ ├── kind-test-cluster-configuration.yaml │ ├── no-constraints.yml │ ├── test-data.txt │ └── web-store-example.yml ├── settings.gradle ├── verify_docs.sh └── workload-generator ├── azure-combine.awk └── getAzureTraces.sh /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | tests: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | java: ['16'] 15 | name: Tests (Java ${{ matrix.java }}) 16 | 17 | steps: 18 | - uses: actions/checkout@v2 19 | - name: Set up JDK {{ matrix.java }} 20 | uses: actions/setup-java@v2 21 | with: 22 | java-version: ${{ matrix.java }} 23 | distribution: 'adopt' 24 | 25 | - uses: actions/cache@v2 26 | with: 27 | path: | 28 | ~/.gradle/caches 29 | ~/.gradle/wrapper 30 | key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} 31 | restore-keys: | 32 | ${{ runner.os }}-gradle- 33 | 34 | - name: Install MiniZinc 35 | run: | 36 | wget https://github.com/MiniZinc/MiniZincIDE/releases/download/2.3.2/MiniZincIDE-2.3.2-bundle-linux-x86_64.tgz -O minizinc.tgz 37 | tar -xzvf minizinc.tgz 38 | rm minizinc.tgz 39 | 40 | - name: Grant execute permission for gradlew 41 | run: chmod +x gradlew 42 | 43 | - name: Build with Gradle 44 | run: PATH=$PATH:`pwd`"/MiniZincIDE-2.3.2-bundle-linux/bin" LD_LIBRARY_PATH=$LD_LIBRARY_PATH:`pwd`"/MiniZincIDE-2.3.2-bundle-linux/lib" ./gradlew build test codeCoverageReport -i 45 | 46 | - name: Upload Codecov report 47 | run: bash <(curl -s https://codecov.io/bash) -f build/reports/jacoco/codeCoverageReport/codeCoverageReport.xml 48 | 49 | 50 | integration-tests: 51 | runs-on: ubuntu-latest 52 | strategy: 53 | matrix: 54 | java: ['16'] 55 | name: Integration Tests (Java ${{ matrix.java }}) 56 | 57 | steps: 58 | - uses: actions/checkout@v2 59 | - name: Set up JDK {{ matrix.java }} 60 | uses: actions/setup-java@v2 61 | with: 62 | java-version: ${{ matrix.java }} 63 | distribution: 'adopt' 64 | 65 | - uses: actions/cache@v2 66 | with: 67 | path: | 68 | ~/.gradle/caches 69 | ~/.gradle/wrapper 70 | key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} 71 | restore-keys: | 72 | ${{ runner.os }}-gradle- 73 | 74 | - name: Grant execute permission for gradlew 75 | run: chmod +x gradlew 76 | 77 | - name: Bring up Kind cluster 78 | run: kind create cluster --config k8s-scheduler/src/test/resources/kind-test-cluster-configuration.yaml --name dcm-it 79 | 80 | - name: Build with Gradle 81 | run: ./gradlew integrationTest codeCoverageReport -i 82 | 83 | - name: Upload Codecov report 84 | run: bash <(curl -s https://codecov.io/bash) -f build/reports/jacoco/codeCoverageReport/codeCoverageReport.xml 85 | 86 | 87 | verify-docs: 88 | name: Verify Documentation 89 | runs-on: ubuntu-latest 90 | steps: 91 | - uses: actions/checkout@v2 92 | 93 | - name: Verify documentation 94 | run: bash verify_docs.sh 95 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.mzn 2 | *.csv 3 | build/* 4 | */build/* 5 | .gradle/* 6 | .idea/* 7 | .java-version* 8 | v2-cropped.txt 9 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## Ongoing: release 0.16.0 4 | 5 | ## July 18, 2022: Release 0.15.0 6 | 7 | * Issues fixed: 8 | * Bump calcite to version 1.30.0. 9 | 10 | ## July 18, 2022: Release 0.14.0 11 | * Public API changes: 12 | * ortools-backend: added `setUseCapacityPresenceLiterals(bool)` to configure whether `capacity_constraint` uses optionals (#161) 13 | 14 | * Issues fixed: 15 | * ortools-backend: use indexes for correlated subqueries (#154) 16 | * k8s-schedulers: several performance and schema changes to prepare for IVM integration 17 | 18 | ## October 5, 2021: Release 0.13.0 19 | 20 | * Public API changes: 21 | * ortools-backend: `capacity_constraint` now uses optional intervals and does not interfere with core computations. 22 | * ortools-backend: Bump ortools to 9.1.9490 23 | 24 | * Issues fixed: 25 | * ortools-backend: `capacity_constraint` now uses optional intervals and does not interfere with core computations. 26 | * ortools-backend: have tupleXX.hash()/equals() avoid identity-based hashes 27 | * ortools-backend: add eq(Object[], Object[]) 28 | * build: use gradle toolchains to always build dcm/ sub-project with JDK 11 and other sub-projects with 29 | more recent JDKs. 30 | 31 | ## July 26, 2021: Release 0.12.0 32 | 33 | * Public API changes: 34 | * #123: Breaking changes to constraint syntax. We now declare constraints 35 | using a `CREATE CONSTRAINT` DDL statement instead of the `CREATE VIEW` syntax. 36 | Please see the documentation for more information. 37 | * New aggregate functions `ANY` and `ALL`. 38 | 39 | * Issues fixed: 40 | * #99: dcm: Index usage in ortools backend is sensitive to TableRowGenerator ordering in IR 41 | * #117: dcm: Check for supported subset of SQL syntax 42 | * #119: k8s-scheduler: Update Kubernetes client version to 5.5.0 43 | * #121: build: test for both Java 11 and 16 44 | * #123, #124: dcm, build: migrate to the Apache Calcite parser 45 | * Numerous improvements in the compiler 46 | 47 | ## June 30, 2021: Release 0.11.0 48 | * Public API changes: 49 | * #114: dcm: simplify Model and ISolverBackend APIs 50 | * #96: Use consistent syntax for hard and soft constraints 51 | 52 | * Issues fixed: 53 | * #95: k8s-scheduler,benchmarks: eliminate stray running threads 54 | * #96: dcm: Use consistent syntax for hard and soft constraints 55 | * #101,#102: ortools-backend: improve intermediate view type inferrence 56 | * #107: ScaleNodeBenchmark to measure solver's latency 57 | * #109: dcm: make sure string literals in generated code preserve the casing from the DB 58 | * #110,#113: Don't search for an UNSAT core unless solver status is INFEAS…IBLE 59 | * Overhaul of internal APIs from Model -> Backend 60 | 61 | ## June 2, 2021: Release 0.10.0 62 | * Public API changes: 63 | * Upgrade to Google OrTools 9.0.9048 64 | * #93: UNSAT core interface 65 | 66 | * Issues fixed: 67 | * #93: Add UNSAT core interface 68 | * #94: re-organize compiler code as a pipeline of passes against a `Program` representation 69 | 70 | ## Apr 27, 2021: Release 0.9.0 71 | * Public API changes: 72 | * With #91, no longer requires Jcenter repository 73 | * Upgrade to Google OrTools 8.2.9025 74 | 75 | * Issues fixed: 76 | * #91: Migrate to official ortools Maven dependency 77 | * #89: ortools: using sharper types in backend code generator 78 | * #45: Reduce presolve times enhancement 79 | * Several performance improvements 80 | 81 | ## Mar 24, 2021: Release 0.8.0 82 | 83 | * Public API changes: 84 | * Objective functions no longer need to be scalar expressions, but 85 | can be any column expression. Each value in the column will be 86 | treated as an objective function. (#81, #82) 87 | * Issues fixed: #41, 81, #82, #85, #86 88 | 89 | 90 | ## Feb 8, 2021: Release 0.7.1 91 | 92 | * API improvements 93 | * A fetcher API for supplying input data to tables: https://github.com/vmware/declarative-cluster-management/commit/bfdbb7951aa4944e770fef5e5a16318ad12778e8 94 | * Reworks exceptions thrown by the compiler and solver (#73, #74). 95 | * Bug fixes: 96 | * dcm: `all_equal()` now correctly works against variable columns 97 | * dcm: fixes capacity constraint bugs related to empty domains, working with bigint/long columns, overflows, and divide by zeroes (#73, #75) 98 | * k8s-scheduler: use pod_info.uid to uniquely identify pods because pod_info.name is not unique across namespaces (#72) 99 | 100 | 101 | ## Nov 10, 2020: Release 0.6.0 102 | 103 | * or-tools backend and public API changes: 104 | * Views with check clauses can now also have a where clause 105 | 106 | 107 | ## Oct 30, 2020: Release 0.5.0 108 | 109 | * ortools-backend: 110 | * Add all_different() constraint 111 | 112 | 113 | ## Oct 30, 2020: Release 0.4.0 114 | 115 | * API changes: 116 | * model.solve() no longer returns tables without variables 117 | * removed solveModelAndReflectTableChanges() testing API 118 | * Improvements to the or-tools backend (evaluating constant sub-queries only once and better support for nested queries). 119 | 120 | 121 | ## Sep 23, 2020: Release 0.3.0 122 | ## Sep 21, 2020: Release 0.2.0 123 | 124 | -------------------------------------------------------------------------------- /CODE-OF-CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in the 7 | declarative-cluster-management project and our community a harassment-free 8 | experience for everyone, regardless of age, body size, disability, ethnicity, 9 | sex characteristics, gender identity and expression, level of experience, 10 | education, socio-economic status, nationality, personal appearance, race, 11 | religion, or sexual identity and orientation. 12 | 13 | ## Our Standards 14 | 15 | Examples of behavior that contributes to creating a positive environment 16 | include: 17 | 18 | * Using welcoming and inclusive language 19 | * Being respectful of differing viewpoints and experiences 20 | * Gracefully accepting constructive criticism 21 | * Focusing on what is best for the community 22 | * Showing empathy towards other community members 23 | 24 | Examples of unacceptable behavior by participants include: 25 | 26 | * The use of sexualized language or imagery and unwelcome sexual attention or 27 | advances 28 | * Trolling, insulting/derogatory comments, and personal or political attacks 29 | * Public or private harassment 30 | * Publishing others' private information, such as a physical or electronic 31 | address, without explicit permission 32 | * Other conduct which could reasonably be considered inappropriate in a 33 | professional setting 34 | 35 | ## Our Responsibilities 36 | 37 | Project maintainers are responsible for clarifying the standards of acceptable 38 | behavior and are expected to take appropriate and fair corrective action in 39 | response to any instances of unacceptable behavior. 40 | 41 | Project maintainers have the right and responsibility to remove, edit, or 42 | reject comments, commits, code, wiki edits, issues, and other contributions 43 | that are not aligned to this Code of Conduct, or to ban temporarily or 44 | permanently any contributor for other behaviors that they deem inappropriate, 45 | threatening, offensive, or harmful. 46 | 47 | ## Scope 48 | 49 | This Code of Conduct applies both within project spaces and in public spaces 50 | when an individual is representing the project or its community. Examples of 51 | representing a project or community include using an official project e-mail 52 | address, posting via an official social media account, or acting as an appointed 53 | representative at an online or offline event. Representation of a project may be 54 | further defined and clarified by project maintainers. 55 | 56 | ## Enforcement 57 | 58 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 59 | reported by contacting the project team at oss-coc@vmware.com. All 60 | complaints will be reviewed and investigated and will result in a response that 61 | is deemed necessary and appropriate to the circumstances. The project team is 62 | obligated to maintain confidentiality with regard to the reporter of an incident. 63 | Further details of specific enforcement policies may be posted separately. 64 | 65 | Project maintainers who do not follow or enforce the Code of Conduct in good 66 | faith may face temporary or permanent repercussions as determined by other 67 | members of the project's leadership. 68 | 69 | ## Attribution 70 | 71 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 72 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 73 | 74 | [homepage]: https://www.contributor-covenant.org 75 | 76 | For answers to common questions about this code of conduct, see 77 | https://www.contributor-covenant.org/faq 78 | 79 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Contributing to declarative-cluster-management 4 | 5 | The declarative-cluster-management project team welcomes contributions from the community. Before you start working with declarative-cluster-management, please 6 | read our [Developer Certificate of Origin](https://cla.vmware.com/dco). All contributions to this repository must be 7 | signed as described on that page. Your signature certifies that you wrote the patch or have the right to pass it on 8 | as an open-source patch. 9 | 10 | ## Contribution Flow 11 | 12 | This is a rough outline of what a contributor's workflow looks like: 13 | 14 | - Create a topic branch from where you want to base your work 15 | - Make commits of logical units 16 | - Make sure your commit messages are in the proper format (see below) 17 | - Push your changes to a topic branch in your fork of the repository 18 | - Submit a pull request 19 | 20 | Example: 21 | 22 | ``` shell 23 | git remote add upstream https://github.com/vmware/declarative-cluster-management.git 24 | git checkout -b my-new-feature master 25 | git commit -a 26 | git push origin my-new-feature 27 | ``` 28 | 29 | ### Staying In Sync With Upstream 30 | 31 | When your branch gets out of sync with the vmware/master branch, use the following to update: 32 | 33 | ``` shell 34 | git checkout my-new-feature 35 | git fetch -a 36 | git pull --rebase upstream master 37 | git push --force-with-lease origin my-new-feature 38 | ``` 39 | 40 | ### Updating pull requests 41 | 42 | If your PR fails to pass CI or needs changes based on code review, you'll most likely want to squash these changes into 43 | existing commits. 44 | 45 | If your pull request contains a single commit or your changes are related to the most recent commit, you can simply 46 | amend the commit. 47 | 48 | ``` shell 49 | git add . 50 | git commit --amend 51 | git push --force-with-lease origin my-new-feature 52 | ``` 53 | 54 | If you need to squash changes into an earlier commit, you can use: 55 | 56 | ``` shell 57 | git add . 58 | git commit --fixup 59 | git rebase -i --autosquash master 60 | git push --force-with-lease origin my-new-feature 61 | ``` 62 | 63 | Be sure to add a comment to the PR indicating your new changes are ready to review, as GitHub does not generate a 64 | notification when you git push. 65 | 66 | ### Code Style 67 | 68 | ### Formatting Commit Messages 69 | 70 | We follow the conventions on [How to Write a Git Commit Message](http://chris.beams.io/posts/git-commit/). 71 | 72 | Be sure to include any related GitHub issue references in the commit message. See 73 | [GFM syntax](https://guides.github.com/features/mastering-markdown/#GitHub-flavored-markdown) for referencing issues 74 | and commits. 75 | 76 | ## Reporting Bugs and Creating Issues 77 | 78 | When opening a new issue, try to roughly follow the commit message format conventions above. 79 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Declarative Cluster Management 2 | 3 | Copyright (c) 2019, 2020 VMware, Inc. All rights reserved 4 | 5 | The BSD-2 license (the "License") set forth below applies to all parts of the 6 | Declarative Cluster Management project. You may not use this file except in 7 | compliance with the License. 8 | 9 | BSD-2 License 10 | 11 | Redistribution and use in source and binary forms, with or without 12 | modification, are permitted provided that the following conditions are met: 13 | 14 | Redistributions of source code must retain the above copyright notice, this 15 | list of conditions and the following disclaimer. 16 | 17 | Redistributions in binary form must reproduce the above copyright notice, this 18 | list of conditions and the following disclaimer in the documentation and/or 19 | other materials provided with the distribution. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 22 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 23 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 25 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | Declarative Cluster Management 2 | Copyright (c) 2019 VMware, Inc. All Rights Reserved. 3 | 4 | This product is licensed to you under the BSD-2 license (the "License"). You may not use this product except in compliance with the BSD-2 License. 5 | 6 | This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file. 7 | 8 | -------------------------------------------------------------------------------- /artifact/run_emulation.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | SCRIPT_DIR=$(dirname $(readlink -f $0)) 4 | GIT_REV=`git rev-parse HEAD` 5 | IT_BRANCH=`git rev-parse --abbrev-ref HEAD` 6 | 7 | # 1st optional argument 8 | mode=${1:-short} 9 | if [ "$mode" = "full" ]; then 10 | cutOff=10000 11 | exp1Nodes=10000 12 | exp2NodesList=(500 5000 10000) 13 | else 14 | cutOff=5000 15 | exp1Nodes=500 16 | exp2NodesList=(100 500 1000) 17 | fi 18 | # 2nd optional argument 19 | startTimeCutOff=${2:-$cutOff} 20 | 21 | traceFile="v2-cropped.txt" 22 | TRACE_DIR=trace-`date +%s` 23 | mkdir -p $TRACE_DIR/$GIT_REV 24 | 25 | 26 | # plot for varying affinity 27 | for scheduler in "dcm-scheduler" "dcm-scheduler-scoped"; do 28 | for affinityProportion in 0 50 100; do 29 | cd $SCRIPT_DIR/.. 30 | if [ $scheduler == "dcm-scheduler-scoped" ]; then 31 | ./gradlew runBenchmark --args="-n ${exp1Nodes} -f ${traceFile} -c 100 -m 200 -t 100 -s ${startTimeCutOff} -p ${affinityProportion} -S" &> /tmp/out 32 | else 33 | ./gradlew runBenchmark --args="-n ${exp1Nodes} -f ${traceFile} -c 100 -m 200 -t 100 -s ${startTimeCutOff} -p ${affinityProportion}" &> /tmp/out 34 | fi 35 | cd $SCRIPT_DIR 36 | 37 | expId=`date +%s` 38 | mkdir -p $TRACE_DIR/$GIT_REV/$expId 39 | cp /tmp/out $TRACE_DIR/$GIT_REV/$expId/workload_output 40 | cp /tmp/out $TRACE_DIR/$GIT_REV/$expId/dcm_scheduler_trace 41 | 42 | echo "workload,schedulerName,solver,kubeconfig,dcmGitBranch,dcmGitCommitId,numNodes,startTimeCutOff,percentageOfNodesToScoreValue,timeScaleDown,affinityProportion" > $TRACE_DIR/$GIT_REV/$expId/metadata 43 | echo "$traceFile,$scheduler,ORTOOLS,local,$GIT_BRANCH,$GIT_REV,$exp1Nodes,$startTimeCutOff,0,100,$affinityProportion" >> $TRACE_DIR/$GIT_REV/$expId/metadata 44 | done 45 | done 46 | 47 | # Process the above trace (creates a plots/ folder) 48 | python3 process_trace.py $TRACE_DIR dataF.db 49 | Rscript plot.r dataF.db $mode 50 | 51 | 52 | TRACE_DIR=trace-`date +%s` 53 | mkdir -p $TRACE_DIR/$GIT_REV 54 | 55 | for scheduler in "dcm-scheduler" "dcm-scheduler-scoped"; do 56 | for exp2Nodes in ${exp2NodesList[@]}; do 57 | affinityProportion=100 58 | 59 | cd $SCRIPT_DIR/.. 60 | if [ $scheduler == "dcm-scheduler-scoped" ]; then 61 | ./gradlew runBenchmark --args="-n ${exp2Nodes} -f ${traceFile} -c 100 -m 200 -t 100 -s ${startTimeCutOff} -p ${affinityProportion} -S" &> /tmp/out 62 | else 63 | ./gradlew runBenchmark --args="-n ${exp2Nodes} -f ${traceFile} -c 100 -m 200 -t 100 -s ${startTimeCutOff} -p ${affinityProportion}" &> /tmp/out 64 | fi 65 | cd $SCRIPT_DIR 66 | 67 | expId=`date +%s` 68 | mkdir -p $TRACE_DIR/$GIT_REV/$expId 69 | cp /tmp/out $TRACE_DIR/$GIT_REV/$expId/workload_output 70 | cp /tmp/out $TRACE_DIR/$GIT_REV/$expId/dcm_scheduler_trace 71 | 72 | echo "workload,schedulerName,solver,kubeconfig,dcmGitBranch,dcmGitCommitId,numNodes,startTimeCutOff,percentageOfNodesToScoreValue,timeScaleDown,affinityProportion" > $TRACE_DIR/$GIT_REV/$expId/metadata 73 | echo "$traceFile,$scheduler,ORTOOLS,local,$GIT_BRANCH,$GIT_REV,$exp2Nodes,$startTimeCutOff,0,100,$affinityProportion" >> $TRACE_DIR/$GIT_REV/$expId/metadata 74 | done 75 | done 76 | 77 | # Process the above trace (creates a plots/ folder) 78 | python3 process_trace.py $TRACE_DIR dataN.db 79 | Rscript plot.r dataN.db $mode 80 | -------------------------------------------------------------------------------- /benchmarks/README.md: -------------------------------------------------------------------------------- 1 | # Running benchmarks 2 | 3 | This sub-project hosts a suite of micro-benchmarks that we use to inform 4 | our design choices in DCM. 5 | 6 | To run these benchmarks, execute the following commands from the project 7 | root folder: 8 | 9 | ``` 10 | $: mvn package 11 | $: java -jar benchmarks/target/benchmark.jar 12 | ``` -------------------------------------------------------------------------------- /benchmarks/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | */ 4 | 5 | plugins { 6 | id 'com.github.johnrengelman.shadow' version "${shadowJarPluginVersion}" 7 | } 8 | 9 | dependencies { 10 | implementation project(':dcm') 11 | implementation project(':k8s-scheduler') 12 | implementation "io.fabric8:kubernetes-client:${fabricK8sClientVersion}" 13 | implementation "org.openjdk.jmh:jmh-core:${jmhVersion}" 14 | annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:${jmhVersion}" 15 | } 16 | 17 | description = 'DCM benchmarks' 18 | 19 | jar { 20 | manifest { 21 | attributes 'Main-Class': 'org.openjdk.jmh.Main' 22 | } 23 | } 24 | 25 | build.finalizedBy shadowJar 26 | -------------------------------------------------------------------------------- /benchmarks/src/main/java/com/vmware/dcm/OrToolsIndexBenchmark.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm; 8 | 9 | import com.google.errorprone.annotations.CanIgnoreReturnValue; 10 | import com.vmware.dcm.backend.ortools.OrToolsSolver; 11 | import org.jooq.DSLContext; 12 | import org.jooq.SQLDialect; 13 | import org.openjdk.jmh.annotations.Benchmark; 14 | import org.openjdk.jmh.annotations.BenchmarkMode; 15 | import org.openjdk.jmh.annotations.Fork; 16 | import org.openjdk.jmh.annotations.Level; 17 | import org.openjdk.jmh.annotations.Measurement; 18 | import org.openjdk.jmh.annotations.Mode; 19 | import org.openjdk.jmh.annotations.OutputTimeUnit; 20 | import org.openjdk.jmh.annotations.Param; 21 | import org.openjdk.jmh.annotations.Scope; 22 | import org.openjdk.jmh.annotations.Setup; 23 | import org.openjdk.jmh.annotations.State; 24 | import org.openjdk.jmh.annotations.Warmup; 25 | 26 | import javax.annotation.Nullable; 27 | import java.sql.Connection; 28 | import java.sql.DriverManager; 29 | import java.sql.SQLException; 30 | import java.util.List; 31 | import java.util.Properties; 32 | import java.util.concurrent.ThreadLocalRandom; 33 | import java.util.concurrent.TimeUnit; 34 | 35 | import static org.jooq.impl.DSL.using; 36 | 37 | @Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) 38 | @Measurement(iterations = 20, time = 1, timeUnit = TimeUnit.SECONDS) 39 | @Fork(1) 40 | @BenchmarkMode(Mode.AverageTime) 41 | @OutputTimeUnit(TimeUnit.NANOSECONDS) 42 | @State(Scope.Benchmark) 43 | 44 | public class OrToolsIndexBenchmark { 45 | @State(Scope.Benchmark) 46 | public static class BenchmarkState { 47 | @Nullable Model model = null; 48 | 49 | @Param({"true", "false"}) 50 | static boolean useIndex; 51 | 52 | @Setup(Level.Iteration) 53 | public void setUp() { 54 | // create database 55 | final DSLContext conn = setup(); 56 | 57 | conn.execute("create table t1(c1 integer, c2 integer, primary key (c1))"); 58 | conn.execute("create table t2(c1 integer, controllable__c2 integer, primary key (c1))"); 59 | 60 | final List views = List.of( 61 | "CREATE CONSTRAINT constraint_with_join AS " + 62 | "SELECT * FROM t1 " + 63 | "JOIN t2 on t1.c1 = t2.c1 " + 64 | "check c2 = controllable__c2" 65 | ); 66 | 67 | 68 | for (int i = 0; i < 1000; i++) { 69 | conn.execute(String.format("insert into t1 values(%s, %s)", i, 70 | 50 + ThreadLocalRandom.current().nextInt(1, 100))); 71 | } 72 | for (int i = 0; i < 5000; i++) { 73 | conn.execute(String.format("insert into t2 values(%s, null)", i)); 74 | } 75 | 76 | final OrToolsSolver solver = new OrToolsSolver.Builder() 77 | .setUseIndicesForEqualityBasedJoins(useIndex) 78 | .build(); 79 | model = Model.build(conn, solver, views); 80 | } 81 | 82 | @CanIgnoreReturnValue 83 | private Connection getConnection(final String url, final Properties properties) throws SQLException { 84 | return DriverManager.getConnection(url, properties); 85 | } 86 | 87 | private DSLContext setup() { 88 | final Properties properties = new Properties(); 89 | properties.setProperty("foreign_keys", "true"); 90 | try { 91 | // Create a fresh database 92 | final String connectionURL = "jdbc:h2:mem:;create=true"; 93 | final Connection conn = getConnection(connectionURL, properties); 94 | final DSLContext using = using(conn, SQLDialect.H2); 95 | using.execute("create schema curr"); 96 | using.execute("set schema curr"); 97 | return using; 98 | } catch (final SQLException e) { 99 | throw new RuntimeException(e); 100 | } 101 | } 102 | } 103 | 104 | @Benchmark 105 | public void runSolver(final BenchmarkState state) { 106 | assert state.model != null; 107 | state.model.solve("T2"); 108 | } 109 | } -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | */ 4 | 5 | plugins { 6 | id "com.github.spotbugs" version "${spotbugsPluginVersion}" 7 | id "net.ltgt.errorprone" version "${errorPronePluginVersion}" 8 | id "checkstyle" 9 | id "jacoco" 10 | id "com.github.kt3k.coveralls" version "${coverallsPluginVersion}" 11 | } 12 | 13 | allprojects { 14 | group = "${dcmGroupId}" 15 | version = "${dcmVersion}" 16 | } 17 | 18 | subprojects { 19 | apply plugin: 'java' 20 | apply plugin: 'maven-publish' 21 | apply plugin: "com.github.spotbugs" 22 | apply plugin: "checkstyle" 23 | apply plugin: "net.ltgt.errorprone" 24 | apply plugin: "jacoco" 25 | 26 | repositories { 27 | mavenLocal() 28 | mavenCentral() 29 | } 30 | 31 | dependencies { 32 | implementation ("org.apache.logging.log4j:log4j-api") { 33 | version { 34 | strictly("[${log4jVersion}, 3[") 35 | prefer("${log4jVersion}") 36 | } 37 | } 38 | implementation ("org.apache.logging.log4j:log4j-core") { 39 | version { 40 | strictly("[${log4jVersion}, 3[") 41 | prefer("${log4jVersion}") 42 | } 43 | } 44 | implementation ("org.apache.logging.log4j:log4j-slf4j-impl") { 45 | version { 46 | strictly("[${log4jVersion}, 3[") 47 | prefer("${log4jVersion}") 48 | } 49 | } 50 | implementation "org.jooq:jooq:${jooqVersion}" 51 | implementation "org.jooq:jooq-meta:${jooqVersion}" 52 | implementation "com.google.guava:guava:${guavaVersion}" 53 | testImplementation "org.junit.jupiter:junit-jupiter:${junitJupiterVersion}" 54 | testImplementation "org.junit-pioneer:junit-pioneer:${junitPioneerVersion}" 55 | spotbugs "com.github.spotbugs:spotbugs:${spotbugsVersion}" 56 | compileOnly "com.github.spotbugs:spotbugs-annotations:${spotbugsVersion}" 57 | errorprone "com.google.errorprone:error_prone_core:${errorproneVersion}" 58 | } 59 | 60 | spotbugs { 61 | effort = "Max" 62 | excludeFilter = file("../config/spotbugs/findbugs-exclude.xml") 63 | } 64 | 65 | checkstyle { 66 | toolVersion = "${checkstyleVersion}" 67 | } 68 | 69 | tasks.withType(JavaCompile).configureEach { 70 | options.errorprone.disableWarningsInGeneratedCode = true 71 | options.errorprone.excludedPaths = "(.*/com/vmware/dcm/generated/.*|.*/javacc/com/vmware/dcm/parser/.*" + 72 | "|.*/com/vmware/dcm/k8s/generated/.*)" 73 | } 74 | 75 | tasks.withType(JavaCompile) { 76 | options.encoding = 'UTF-8' 77 | } 78 | 79 | test { 80 | useJUnitPlatform() 81 | finalizedBy jacocoTestReport // report is always generated after tests run 82 | testLogging { 83 | exceptionFormat = 'full' 84 | } 85 | } 86 | 87 | // Example to configure HTML report 88 | spotbugsMain { 89 | reports { 90 | html { 91 | enabled = true 92 | destination = file("$buildDir/reports/spotbugs/spotbugs.html") 93 | stylesheet = 'fancy-hist.xsl' 94 | } 95 | } 96 | } 97 | 98 | jacocoTestReport { 99 | reports { 100 | xml.enabled = true // coveralls plugin depends on xml format report 101 | html.enabled = true 102 | } 103 | 104 | afterEvaluate { 105 | classDirectories.setFrom(files(classDirectories.files.collect { 106 | fileTree(dir: it, exclude: '**/generated/**') 107 | })) 108 | } 109 | } 110 | } 111 | 112 | 113 | 114 | task codeCoverageReport(type: JacocoReport) { 115 | repositories { 116 | mavenLocal() 117 | mavenCentral() 118 | } 119 | 120 | // Gather execution data from all subprojects 121 | executionData fileTree(project.rootDir.absolutePath).include("**/build/jacoco/*.exec") 122 | 123 | // Add all relevant sourcesets from the subprojects 124 | subprojects.each { 125 | sourceSets it.sourceSets.main 126 | } 127 | 128 | reports { 129 | xml.enabled true 130 | html.enabled true 131 | csv.enabled false 132 | } 133 | 134 | afterEvaluate { 135 | classDirectories.setFrom(files(classDirectories.files.collect { 136 | fileTree(dir: it, exclude: ['**/generated/**', '**/**Benchmark**']) 137 | })) 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /config/checkstyle/suppressions.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 8 | 9 | 12 | 13 | 14 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /config/spotbugs/findbugs-exclude.xml: -------------------------------------------------------------------------------- 1 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /dcm/src/main/codegen/config.fmpp: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2018-2021 VMware, Inc. All Rights Reserved. 3 | # SPDX-License-Identifier: BSD-2 4 | # 5 | 6 | data: { 7 | parser: { 8 | package: "com.vmware.dcm.generated.parser", 9 | class: "DcmSqlParserImpl", 10 | 11 | imports: [ 12 | "org.apache.calcite.sql.SqlCreate" 13 | "com.vmware.dcm.parser.SqlCreateConstraint" 14 | ] 15 | 16 | keywords: [ 17 | "MAXIMIZE" 18 | ] 19 | 20 | nonReservedKeywordsToAdd: [ 21 | "ALL" 22 | "ANY" 23 | "CONTAINS" 24 | "MATCHES" 25 | ] 26 | 27 | createStatementParserMethods: [ 28 | "SqlCreateConstraint" 29 | ] 30 | 31 | implementationFiles: [ 32 | "parserImpls.ftl" 33 | ] 34 | } 35 | } 36 | 37 | freemarkerLinks: { 38 | includes: includes/ 39 | } 40 | -------------------------------------------------------------------------------- /dcm/src/main/codegen/includes/parserImpls.ftl: -------------------------------------------------------------------------------- 1 | <#-- 2 | // Copyright 2018-2021 VMware, Inc. All Rights Reserved. 3 | // SPDX-License-Identifier: BSD-2 4 | --> 5 | 6 | SqlCreate SqlCreateConstraint(Span s, boolean replace): { 7 | SqlIdentifier constraintName; 8 | SqlNode query; 9 | String type; 10 | SqlNode constraint; 11 | } 12 | { 13 | 14 | constraintName = CompoundIdentifier() 15 | 16 | query = OrderedQueryOrExpr(ExprContext.ACCEPT_QUERY) 17 | ( 18 | ( 19 | {type = "HARD_CONSTRAINT";} 20 | | 21 | {type = "OBJECTIVE";} 22 | | 23 | {return new SqlCreateConstraint(s.pos(), constraintName, query, "INTERMEDIATE_VIEW", null);} 24 | ) 25 | constraint = OrderedQueryOrExpr(ExprContext.ACCEPT_ALL) 26 | { 27 | return new SqlCreateConstraint(s.pos(), constraintName, query, type, constraint); 28 | } 29 | ) 30 | } -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/Conf.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm; 8 | 9 | import javax.annotation.Nullable; 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | 13 | public class Conf { 14 | 15 | private final Map params = new HashMap<>(); 16 | 17 | @Nullable 18 | public String getProperty(final String key) { 19 | return params.get(key); 20 | } 21 | 22 | public void setProperty(final String key, final String value) { 23 | params.put(key, value); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/ExtractAccessedTables.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm; 8 | 9 | import org.apache.calcite.sql.SqlBasicCall; 10 | import org.apache.calcite.sql.SqlCall; 11 | import org.apache.calcite.sql.SqlIdentifier; 12 | import org.apache.calcite.sql.SqlJoin; 13 | import org.apache.calcite.sql.SqlNode; 14 | import org.apache.calcite.sql.SqlSelect; 15 | import org.apache.calcite.sql.util.SqlBasicVisitor; 16 | 17 | import java.util.List; 18 | import java.util.Set; 19 | 20 | /** 21 | * Collects table names for every table referenced within a query. 22 | */ 23 | public class ExtractAccessedTables extends SqlBasicVisitor { 24 | final Set tableNames; 25 | 26 | ExtractAccessedTables(final Set tableNames) { 27 | this.tableNames = tableNames; 28 | } 29 | 30 | @Override 31 | public Void visit(final SqlCall call) { 32 | if (call instanceof SqlSelect) { 33 | final SqlSelect select = (SqlSelect) call; 34 | final SqlNode node = select.getFrom(); 35 | assert node != null; 36 | visitInner(node); 37 | } 38 | return super.visit(call); 39 | } 40 | 41 | private void visitInner(final SqlNode node) { 42 | switch (node.getKind()) { 43 | case IDENTIFIER: 44 | tableNames.add(((SqlIdentifier) node).getSimple()); 45 | break; 46 | case AS: 47 | final List operands = ((SqlBasicCall) node).getOperandList(); 48 | assert operands != null; 49 | final SqlIdentifier relation = (SqlIdentifier) operands.get(0); 50 | assert relation != null; 51 | tableNames.add(relation.getSimple()); 52 | break; 53 | case JOIN: 54 | final SqlJoin join = (SqlJoin) node; 55 | visitInner(join.getLeft()); 56 | visitInner(join.getRight()); 57 | break; 58 | default: 59 | break; 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/ExtractConstraintInQuery.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm; 8 | 9 | import org.apache.calcite.sql.SqlBasicCall; 10 | import org.apache.calcite.sql.SqlCall; 11 | import org.apache.calcite.sql.SqlNode; 12 | import org.apache.calcite.sql.SqlNodeList; 13 | import org.apache.calcite.sql.SqlKind; 14 | import org.apache.calcite.sql.SqlOperator; 15 | import org.apache.calcite.sql.SqlSelect; 16 | import org.apache.calcite.sql.SqlIdentifier; 17 | import org.apache.calcite.sql.util.SqlBasicVisitor; 18 | 19 | import java.util.HashMap; 20 | import java.util.List; 21 | import java.util.Map; 22 | 23 | 24 | public class ExtractConstraintInQuery extends SqlBasicVisitor { 25 | final Map> selectClause; 26 | 27 | ExtractConstraintInQuery(final Map> selectClause) { 28 | this.selectClause = selectClause; 29 | } 30 | 31 | @Override 32 | public Void visit(final SqlCall call) { 33 | final SqlOperator op = call.getOperator(); 34 | final List operands = call.getOperandList(); 35 | // controllable__var IN (SELECT subquery) 36 | //if ((op.getKind() == SqlKind.IN || op.getKind() == SqlKind.NOT_IN) && 37 | if ((op.getKind() == SqlKind.IN) && 38 | operands.size() == 2 && isVariable(operands.get(0)) && operands.get(1) instanceof SqlSelect) { 39 | final SqlSelect select = (SqlSelect) operands.get(1); 40 | final String var = getVarName((SqlIdentifier) operands.get(0)); 41 | if (!selectClause.containsKey(var)) { 42 | final Map clause = new HashMap<>(); 43 | selectClause.put(var, clause); 44 | } 45 | parseSelect(var, select); 46 | } 47 | return super.visit(call); 48 | } 49 | 50 | private boolean isVariable(final SqlNode node) { 51 | if (!(node instanceof SqlIdentifier)) { 52 | return false; 53 | } 54 | final SqlIdentifier id = (SqlIdentifier) node; 55 | if (id.isSimple()) { 56 | return id.getSimple().toLowerCase().startsWith("controllable__"); 57 | } else if (id.names.size() == 2) { 58 | return id.names.get(1).toLowerCase().startsWith("controllable__"); 59 | } 60 | return false; 61 | } 62 | 63 | private String getVarName(final SqlIdentifier id) { 64 | if (id.isSimple()) { 65 | return id.getSimple().toLowerCase().substring(14); 66 | } else { 67 | return id.names.get(1).toLowerCase().substring(14); 68 | } 69 | } 70 | 71 | private String getTableName(final SqlNode from) { 72 | final SqlIdentifier id; 73 | if (from.getKind() == SqlKind.AS) { 74 | id = ((SqlBasicCall) from).operand(0); 75 | } else { 76 | id = ((SqlIdentifier) from); 77 | } 78 | return id.getSimple(); 79 | } 80 | 81 | private void parseSelect(final String var, final SqlSelect select) { 82 | final SqlNode from = select.getFrom(); 83 | final SqlNodeList names = select.getSelectList(); 84 | assert from != null; 85 | assert names.size() == 1; 86 | 87 | // Ignore SELF JOIN cases 88 | // e.g., controllable__var IN (SELECT controllable__var ...) 89 | if (isVariable(names.get(0))) { 90 | return; 91 | } 92 | 93 | // Extract table and field name 94 | if (from.getKind() == SqlKind.IDENTIFIER || from.getKind() == SqlKind.AS) { 95 | final String tableName = getTableName(from); 96 | final SqlIdentifier name = (SqlIdentifier) names.get(0); 97 | String fieldName = ""; 98 | if (name.isSimple()) { 99 | // SELECT name FROM table 100 | fieldName = name.getSimple(); 101 | } else if (name.names.size() == 2 && name.names.get(0).equals(tableName)) { 102 | // SELECT table.name FROM table 103 | fieldName = name.names.get(1); 104 | } 105 | selectClause.get(var).put(tableName, fieldName); 106 | } 107 | } 108 | 109 | 110 | } 111 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/ExtractGroupTable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm; 8 | 9 | import com.facebook.presto.sql.QueryUtil; 10 | import com.facebook.presto.sql.tree.CreateView; 11 | import com.facebook.presto.sql.tree.Expression; 12 | import com.facebook.presto.sql.tree.GroupBy; 13 | import com.facebook.presto.sql.tree.GroupingElement; 14 | import com.facebook.presto.sql.tree.Identifier; 15 | import com.facebook.presto.sql.tree.Join; 16 | import com.facebook.presto.sql.tree.JoinCriteria; 17 | import com.facebook.presto.sql.tree.JoinOn; 18 | import com.facebook.presto.sql.tree.QualifiedName; 19 | import com.facebook.presto.sql.tree.Query; 20 | import com.facebook.presto.sql.tree.QuerySpecification; 21 | import com.facebook.presto.sql.tree.Relation; 22 | import com.facebook.presto.sql.tree.SelectItem; 23 | import com.facebook.presto.sql.tree.SimpleGroupBy; 24 | import com.facebook.presto.sql.tree.SingleColumn; 25 | 26 | import java.util.ArrayList; 27 | import java.util.List; 28 | import java.util.Optional; 29 | 30 | /** 31 | * Given a view with a group by, extract a view that lists the table of unique groups. 32 | * 33 | * Example: 34 | * Input: select * from T where P1 group by c1, c2, c3 having P2; 35 | * Output: select c1, c2, c3 from T where P1 group by c1, c2, c3; 36 | */ 37 | class ExtractGroupTable { 38 | private static final String GROUP_TABLE_PREFIX = "GROUP_TABLE__"; 39 | 40 | Optional process(final String viewName, final Query query) { 41 | final QuerySpecification queryBody = (QuerySpecification) query.getQueryBody(); 42 | final List selectList = new ArrayList<>(); 43 | if (queryBody.getGroupBy().isPresent()) { 44 | final GroupBy groupBy = queryBody.getGroupBy().get(); 45 | for (final GroupingElement e: groupBy.getGroupingElements()) { 46 | assert e instanceof SimpleGroupBy; 47 | final SimpleGroupBy simpleGroupBy = (SimpleGroupBy) e; 48 | assert simpleGroupBy.getExpressions().size() == 1; 49 | final Expression column = simpleGroupBy.getExpressions().get(0); 50 | final SelectItem item = new SingleColumn(column, 51 | new Identifier(column.toString() 52 | .replace(".", "_"))); 53 | selectList.add(item); 54 | } 55 | } 56 | else { 57 | return Optional.empty(); 58 | } 59 | assert queryBody.getFrom().isPresent(); 60 | // Next, we make sure that join criteria does not have controllables in them 61 | final Relation relation = relelationWithControllablesRemoved(queryBody.getFrom().get()); 62 | final Query newQuery = QueryUtil.simpleQuery(QueryUtil.selectList(selectList.toArray(new SelectItem[0])), 63 | relation, 64 | queryBody.getWhere(), 65 | queryBody.getGroupBy(), 66 | Optional.empty(), 67 | Optional.empty(), 68 | Optional.empty()); 69 | return Optional.of(new CreateView(QualifiedName.of(GROUP_TABLE_PREFIX + viewName.toUpperCase()), newQuery, 70 | false)); 71 | } 72 | 73 | private Relation relelationWithControllablesRemoved(final Relation relation) { 74 | if (relation instanceof Join) { 75 | final Join join = (Join) relation; 76 | if (join.getCriteria().isPresent()) { 77 | final RemoveControllablePredicates removeControllablePredicates = new RemoveControllablePredicates(); 78 | final Expression joinOnExpression = ((JoinOn) join.getCriteria().get()).getExpression(); 79 | final Optional newJoinCriteria = removeControllablePredicates.process(joinOnExpression) 80 | .map(JoinOn::new); 81 | // If we end up with no join condition at all, then it is equivalent to using an implicit join 82 | // (e.g., "from T1, T2") 83 | final Join.Type joinType = newJoinCriteria.isPresent() ? 84 | join.getType() : 85 | Join.Type.IMPLICIT; 86 | return new Join(joinType, join.getLeft(), join.getRight(), newJoinCriteria); 87 | } 88 | } 89 | return relation; 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/ModelException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm; 8 | 9 | public class ModelException extends RuntimeException { 10 | public ModelException(final String message, final Throwable cause) { 11 | super(message, cause); 12 | } 13 | 14 | public ModelException(final Exception e) { 15 | super(e); 16 | } 17 | 18 | public ModelException(final String message) { 19 | super(message); 20 | } 21 | } -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/RemoveControllablePredicates.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm; 8 | 9 | import com.facebook.presto.sql.tree.ComparisonExpression; 10 | import com.facebook.presto.sql.tree.DefaultTraversalVisitor; 11 | import com.facebook.presto.sql.tree.DereferenceExpression; 12 | import com.facebook.presto.sql.tree.Expression; 13 | import com.facebook.presto.sql.tree.Identifier; 14 | import com.facebook.presto.sql.tree.LogicalBinaryExpression; 15 | 16 | import java.util.Optional; 17 | 18 | /** 19 | * Remove parts of an expression that involve controllable variables. 20 | */ 21 | class RemoveControllablePredicates extends DefaultTraversalVisitor, Void> { 22 | 23 | @Override 24 | protected Optional visitLogicalBinaryExpression(final LogicalBinaryExpression node, 25 | final Void context) { 26 | final Optional left = this.process(node.getLeft()); 27 | final Optional right = this.process(node.getRight()); 28 | switch (node.getOperator()) { 29 | case AND: 30 | if (left.isEmpty() && right.isEmpty()) { 31 | return Optional.empty(); 32 | } 33 | else if (left.isEmpty()) { 34 | return right; 35 | } 36 | else if (right.isEmpty()) { 37 | return left; 38 | } 39 | else { 40 | return Optional.of(new LogicalBinaryExpression(node.getOperator(), left.get(), right.get())); 41 | } 42 | case OR: 43 | return (left.isEmpty() || right.isEmpty()) 44 | ? Optional.empty() 45 | : Optional.of(new LogicalBinaryExpression(node.getOperator(), left.get(), right.get())); 46 | default: 47 | throw new RuntimeException("Should not happen"); 48 | } 49 | } 50 | 51 | @Override 52 | protected Optional visitComparisonExpression(final ComparisonExpression node, final Void context) { 53 | final Optional left = this.process(node.getLeft()); 54 | final Optional right = this.process(node.getRight()); 55 | return (left.isEmpty() || right.isEmpty()) 56 | ? Optional.empty() 57 | : Optional.of(new ComparisonExpression(node.getOperator(), left.get(), right.get())); 58 | } 59 | 60 | @Override 61 | protected Optional visitDereferenceExpression(final DereferenceExpression node, final Void context) { 62 | return node.getField().getValue().startsWith("controllable__") ? Optional.empty() : Optional.of(node); 63 | } 64 | 65 | @Override 66 | protected Optional visitIdentifier(final Identifier node, final Void context) { 67 | return node.getValue().startsWith("controllable__") ? Optional.empty() : Optional.of(node); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/SolverException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2021 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm; 7 | 8 | import java.util.Collections; 9 | import java.util.HashSet; 10 | import java.util.List; 11 | import java.util.Set; 12 | 13 | /** 14 | * An exception thrown when invoking the solver. Typically used to convey infeasibility or some other error. 15 | * 16 | * Optionally returns an UNSAT core, if the solver supports it. 17 | */ 18 | public class SolverException extends RuntimeException { 19 | private final String reason; 20 | private final List core; 21 | 22 | public SolverException(final String reason) { 23 | super(reason); 24 | this.reason = reason; 25 | this.core = Collections.emptyList(); 26 | } 27 | 28 | public SolverException(final String reason, final List core) { 29 | super(reason + " " + new HashSet<>(core)); 30 | this.reason = reason; 31 | this.core = core; 32 | } 33 | 34 | public String reason() { 35 | return reason; 36 | } 37 | 38 | public Set core() { 39 | return new HashSet<>(core); 40 | } 41 | } -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/annotations/FieldsAreNonnullByDefault.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.annotations; 8 | 9 | import java.lang.annotation.Documented; 10 | import java.lang.annotation.ElementType; 11 | import java.lang.annotation.Retention; 12 | import java.lang.annotation.RetentionPolicy; 13 | 14 | import javax.annotation.Nonnull; 15 | import javax.annotation.meta.TypeQualifierDefault; 16 | 17 | /** 18 | * Applies the {@link Nonnull} annotation to every field unless overridden. 19 | */ 20 | @Documented 21 | @Nonnull 22 | @TypeQualifierDefault(ElementType.FIELD) 23 | @Retention(RetentionPolicy.RUNTIME) 24 | public @interface FieldsAreNonnullByDefault { 25 | } -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/annotations/MethodsAreNonnullByDefault.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.annotations; 8 | 9 | import java.lang.annotation.Documented; 10 | import java.lang.annotation.ElementType; 11 | import java.lang.annotation.Retention; 12 | import java.lang.annotation.RetentionPolicy; 13 | 14 | import javax.annotation.Nonnull; 15 | import javax.annotation.meta.TypeQualifierDefault; 16 | 17 | /** 18 | * Applies the {@link Nonnull} annotation to every method unless overridden. 19 | */ 20 | @Documented 21 | @Nonnull 22 | @TypeQualifierDefault(ElementType.METHOD) 23 | @Retention(RetentionPolicy.RUNTIME) 24 | public @interface MethodsAreNonnullByDefault { 25 | } -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/backend/IGeneratedBackend.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.backend; 8 | 9 | import org.jooq.Record; 10 | import org.jooq.Result; 11 | 12 | import java.util.Map; 13 | 14 | public interface IGeneratedBackend { 15 | Map> solve(final Map> data); 16 | } 17 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/backend/ISolverBackend.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.backend; 8 | 9 | import com.vmware.dcm.compiler.IRContext; 10 | import com.vmware.dcm.compiler.Program; 11 | import com.vmware.dcm.compiler.ir.ListComprehension; 12 | import org.jooq.Record; 13 | import org.jooq.Result; 14 | 15 | import java.util.List; 16 | import java.util.Map; 17 | 18 | public interface ISolverBackend { 19 | Map> runSolver(final Map> inputRecords); 20 | 21 | List generateModelCode(final IRContext context, final Program irProgram); 22 | 23 | boolean needsGroupTables(); 24 | } 25 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/backend/RewriteContains.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.backend; 8 | 9 | import com.vmware.dcm.compiler.ir.FunctionCall; 10 | import com.vmware.dcm.compiler.ir.VoidType; 11 | import com.vmware.dcm.compiler.ir.BinaryOperatorPredicate; 12 | import com.vmware.dcm.compiler.ir.ComprehensionRewriter; 13 | import com.vmware.dcm.compiler.ir.Expr; 14 | import com.vmware.dcm.compiler.ir.GroupByComprehension; 15 | import com.vmware.dcm.compiler.ir.ListComprehension; 16 | import org.slf4j.Logger; 17 | import org.slf4j.LoggerFactory; 18 | 19 | /** 20 | * Rewrites the CONTAINS(a, b) function call into the binary operation (A CONTAINS B) 21 | */ 22 | public class RewriteContains extends ComprehensionRewriter { 23 | private static final Logger LOG = LoggerFactory.getLogger(RewriteContains.class); 24 | 25 | public static ListComprehension apply(final ListComprehension comprehension) { 26 | LOG.trace("Invoking RewriteContains on {}", comprehension); 27 | final RewriteContains rewriter = new RewriteContains(); 28 | final Expr result = rewriter.visit(comprehension); 29 | return comprehension instanceof GroupByComprehension ? 30 | (GroupByComprehension) result : (ListComprehension) result; 31 | } 32 | 33 | @Override 34 | protected Expr visitFunctionCall(final FunctionCall node, final VoidType context) { 35 | if (node.getFunction().equals(FunctionCall.Function.CONTAINS)) { 36 | return new BinaryOperatorPredicate(BinaryOperatorPredicate.Operator.CONTAINS, 37 | node.getArgument().get(0), node.getArgument().get(1)); 38 | } 39 | return super.visitFunctionCall(node, context); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/backend/minizinc/FindStringLiterals.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.backend.minizinc; 8 | 9 | import com.vmware.dcm.compiler.ir.VoidType; 10 | import com.vmware.dcm.compiler.ir.Literal; 11 | import com.vmware.dcm.compiler.ir.SimpleVisitor; 12 | 13 | import java.util.HashSet; 14 | import java.util.Set; 15 | 16 | public class FindStringLiterals extends SimpleVisitor { 17 | private final Set stringLiterals = new HashSet<>(); 18 | 19 | @Override 20 | protected VoidType visitLiteral(final Literal node, final VoidType context) { 21 | if (node.getValue() instanceof String) { 22 | final String s = node.getValue().toString(); 23 | if (s.startsWith("'") && s.endsWith("'")) { 24 | stringLiterals.add(node.getValue().toString()); 25 | } 26 | } 27 | return super.visitLiteral(node, context); 28 | } 29 | 30 | Set getStringLiterals() { 31 | return stringLiterals; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/backend/minizinc/MinizincString.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.backend.minizinc; 8 | 9 | import com.vmware.dcm.compiler.IRColumn; 10 | import com.vmware.dcm.compiler.IRTable; 11 | import com.vmware.dcm.compiler.ir.ColumnIdentifier; 12 | import com.vmware.dcm.compiler.ir.Expr; 13 | import com.vmware.dcm.compiler.ir.Literal; 14 | 15 | import java.util.Locale; 16 | 17 | /** 18 | * Utility class that converts SQL types (table names, iterator names, literals, column names etc.) 19 | * into the corresponding strings used in Minizinc 20 | */ 21 | class MinizincString { 22 | static final String MNZ_AND = " /\\ "; 23 | static final char MNZ_OUTPUT_CSV_DELIMITER = ','; 24 | static final String MNZ_OUTPUT_TABLENAME_TAG = "\n!!"; 25 | private static final String NUM_ROWS_NAME = "NUM_ROWS"; 26 | 27 | static String tableNumRowsName(final IRTable table) { 28 | return tableNumRowsName(table.getName()); 29 | } 30 | 31 | static String tableNumRowsName(final String tableName) { 32 | return String.format("%s__%s", tableName, NUM_ROWS_NAME); 33 | } 34 | 35 | /** 36 | * @return Returns this table qualified name 37 | */ 38 | static String qualifiedName(final IRColumn field) { 39 | return String.format("%s__%s", field.getIRTable().getName(), field.getName()); 40 | } 41 | 42 | static String headItemVariableName(final Expr expr) { 43 | if (expr.getAlias().isPresent()) { 44 | return expr.getAlias().get().toUpperCase(Locale.US); 45 | } 46 | else if (expr instanceof ColumnIdentifier) { 47 | final IRColumn field = ((ColumnIdentifier) expr).getField(); 48 | return String.format("%s__%s", field.getIRTable().getAliasedName(), field.getName()); 49 | } else { 50 | throw new RuntimeException("Expr of type: " + expr + " does not have an alias"); 51 | } 52 | } 53 | 54 | static String columnNameWithIteration(final ColumnIdentifier node, final String iteratorVariable) { 55 | return String.format("%s[%s]", MinizincString.qualifiedName(node.getField()), iteratorVariable); 56 | } 57 | 58 | static String columnNameWithIteration(final ColumnIdentifier node) { 59 | return columnNameWithIteration(node, node.getTableName().toUpperCase(Locale.US) + "__ITER"); 60 | } 61 | 62 | static String groupColumnNameWithIteration(final String viewName, final ColumnIdentifier node) { 63 | return String.format("GROUP_TABLE__%s__%s%s[%s]", viewName.toUpperCase(Locale.getDefault()), 64 | node.fromDereferencedAccess() ? node.getTableName() + "_" : "", 65 | node.getField().getName(), "GROUP__KEY"); 66 | } 67 | 68 | static String literal(final Expr literal) { 69 | if (literal instanceof ColumnIdentifier) { 70 | return MinizincString.columnNameWithIteration((ColumnIdentifier) literal); 71 | } else if (literal instanceof Literal) { 72 | return ((Literal) literal).getValue().toString(); 73 | } 74 | throw new RuntimeException("Unknown literal type " + literal); 75 | } 76 | 77 | 78 | /** 79 | * Used in the mnz_data.ftl and mnz_model.ftl template files 80 | * 81 | * @return Returns the MiniZinc name for the type. Strings type for MiniZinc is always an int 82 | */ 83 | static String typeName(final IRColumn.FieldType type) { 84 | if (type.equals(IRColumn.FieldType.STRING)) { 85 | return "STRING_LITERALS"; 86 | } 87 | if (type.equals(IRColumn.FieldType.LONG)) { 88 | return "int"; // Minizinc int 89 | } 90 | return type.name().toLowerCase(Locale.US); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/backend/minizinc/RewriteCountFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.backend.minizinc; 8 | 9 | import com.vmware.dcm.compiler.ir.ColumnIdentifier; 10 | import com.vmware.dcm.compiler.ir.ComprehensionRewriter; 11 | import com.vmware.dcm.compiler.ir.Expr; 12 | import com.vmware.dcm.compiler.ir.FunctionCall; 13 | import com.vmware.dcm.compiler.ir.ListComprehension; 14 | import com.vmware.dcm.compiler.ir.Literal; 15 | import com.vmware.dcm.compiler.ir.VoidType; 16 | 17 | /** 18 | * Rewrite instances of count([i | qualifiers..]) to sum([1 | qualifiers...]). 19 | */ 20 | public class RewriteCountFunction extends ComprehensionRewriter { 21 | 22 | @Override 23 | protected Expr visitFunctionCall(final FunctionCall function, final VoidType context) { 24 | if (function.getFunction().equals(FunctionCall.Function.COUNT)) { 25 | if (!(function.getArgument().get(0) instanceof ColumnIdentifier)) { 26 | throw new IllegalStateException("RewriteCountFunction is only safe to use on column identifiers"); 27 | } 28 | final FunctionCall newFunction = new FunctionCall(FunctionCall.Function.SUM, 29 | new Literal<>(1L, Long.class)); 30 | function.getAlias().ifPresent(newFunction::setAlias); 31 | return newFunction; 32 | } 33 | return super.visitFunctionCall(function, context); 34 | } 35 | 36 | public static ListComprehension apply(final ListComprehension comprehension) { 37 | final RewriteCountFunction rewriter = new RewriteCountFunction(); 38 | return (ListComprehension) rewriter.visit(comprehension); 39 | } 40 | } -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/backend/minizinc/RewriteNullPredicates.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.backend.minizinc; 8 | 9 | import com.vmware.dcm.compiler.ir.IsNotNullPredicate; 10 | import com.vmware.dcm.compiler.ir.VoidType; 11 | import com.vmware.dcm.compiler.ir.BinaryOperatorPredicate; 12 | import com.vmware.dcm.compiler.ir.ComprehensionRewriter; 13 | import com.vmware.dcm.compiler.ir.Expr; 14 | import com.vmware.dcm.compiler.ir.GroupByComprehension; 15 | import com.vmware.dcm.compiler.ir.IsNullPredicate; 16 | import com.vmware.dcm.compiler.ir.ListComprehension; 17 | import com.vmware.dcm.compiler.ir.Literal; 18 | import org.slf4j.Logger; 19 | import org.slf4j.LoggerFactory; 20 | 21 | /** 22 | * Only used by the Minizinc backend to rewrite IsNull/IsNotNull(expr) predicates into expr =/!= 'null' 23 | */ 24 | public class RewriteNullPredicates extends ComprehensionRewriter { 25 | private static final Logger LOG = LoggerFactory.getLogger(RewriteNullPredicates.class); 26 | 27 | @Override 28 | protected Expr visitIsNullPredicate(final IsNullPredicate node, final VoidType context) { 29 | return new BinaryOperatorPredicate(BinaryOperatorPredicate.Operator.EQUAL, node, 30 | new Literal<>("'null'", String.class)); 31 | } 32 | 33 | @Override 34 | protected Expr visitIsNotNullPredicate(final IsNotNullPredicate node, final VoidType context) { 35 | return new BinaryOperatorPredicate(BinaryOperatorPredicate.Operator.NOT_EQUAL, node, 36 | new Literal<>("'null'", String.class)); 37 | } 38 | 39 | static ListComprehension apply(final ListComprehension comprehension) { 40 | final RewriteNullPredicates rewriter = new RewriteNullPredicates(); 41 | final Expr result = rewriter.visit(comprehension); 42 | LOG.trace("Rewrote {} into {}", comprehension, result); 43 | return comprehension instanceof GroupByComprehension ? 44 | (GroupByComprehension) result : (ListComprehension) result; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/backend/minizinc/SplitIntoSingleHeadComprehensions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.backend.minizinc; 8 | 9 | import com.vmware.dcm.compiler.ir.GroupByComprehension; 10 | import com.vmware.dcm.compiler.ir.Head; 11 | import com.vmware.dcm.compiler.ir.ListComprehension; 12 | 13 | import java.util.Collections; 14 | import java.util.List; 15 | import java.util.stream.Collectors; 16 | 17 | /** 18 | * Splits a comprehension with multiple select items into multiple comprehensions, each with a single select item. 19 | */ 20 | class SplitIntoSingleHeadComprehensions { 21 | 22 | static List apply(final ListComprehension input) { 23 | if (input instanceof GroupByComprehension) { 24 | final GroupByComprehension groupByComprehension = (GroupByComprehension) input; 25 | final ListComprehension innerComprehension = groupByComprehension.getComprehension(); 26 | return innerComprehension.getHead() 27 | .getSelectExprs() 28 | .stream() 29 | .map(e -> { 30 | final ListComprehension mc = 31 | new ListComprehension(new Head(Collections.singletonList(e)), 32 | innerComprehension.getQualifiers()); 33 | return new GroupByComprehension(mc, groupByComprehension.getGroupByQualifier()); 34 | }) 35 | .collect(Collectors.toList()); 36 | } 37 | else { 38 | return input.getHead() 39 | .getSelectExprs() 40 | .stream() 41 | .map(e -> new ListComprehension(new Head(Collections.singletonList(e)), 42 | input.getQualifiers())) 43 | .collect(Collectors.toList()); 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/backend/ortools/ContainsFunctionCall.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm.backend.ortools; 7 | 8 | import com.vmware.dcm.compiler.ir.FunctionCall; 9 | import com.vmware.dcm.compiler.ir.SimpleVisitor; 10 | import com.vmware.dcm.compiler.ir.VoidType; 11 | 12 | class ContainsFunctionCall extends SimpleVisitor { 13 | boolean found = false; 14 | 15 | @Override 16 | protected VoidType visitFunctionCall(final FunctionCall node, final VoidType context) { 17 | found = true; 18 | return super.visitFunctionCall(node, context); 19 | } 20 | 21 | boolean getFound() { 22 | return found; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/backend/ortools/GetColumnIdentifiers.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.backend.ortools; 8 | 9 | import com.vmware.dcm.compiler.ir.ColumnIdentifier; 10 | import com.vmware.dcm.compiler.ir.Expr; 11 | import com.vmware.dcm.compiler.ir.GroupByComprehension; 12 | import com.vmware.dcm.compiler.ir.ListComprehension; 13 | import com.vmware.dcm.compiler.ir.SimpleVisitor; 14 | import com.vmware.dcm.compiler.ir.VoidType; 15 | 16 | import java.util.LinkedHashSet; 17 | 18 | 19 | /** 20 | * A visitor that returns the set of accessed columns within a comprehension's scope, without entering 21 | * sub-queries. 22 | */ 23 | class GetColumnIdentifiers extends SimpleVisitor { 24 | private final LinkedHashSet columnIdentifiers = new LinkedHashSet<>(); 25 | private final boolean visitInnerComprehensions; 26 | 27 | GetColumnIdentifiers(final boolean visitInnerComprehensions) { 28 | this.visitInnerComprehensions = visitInnerComprehensions; 29 | } 30 | 31 | @Override 32 | protected VoidType visitColumnIdentifier(final ColumnIdentifier node, final VoidType context) { 33 | columnIdentifiers.add(node); 34 | return defaultReturn(); 35 | } 36 | 37 | @Override 38 | protected VoidType visitListComprehension(final ListComprehension node, final VoidType context) { 39 | if (visitInnerComprehensions) { 40 | super.visitListComprehension(node, context); 41 | } 42 | return defaultReturn(); 43 | } 44 | 45 | @Override 46 | protected VoidType visitGroupByComprehension(final GroupByComprehension node, final VoidType context) { 47 | if (visitInnerComprehensions) { 48 | super.visitGroupByComprehension(node, context); 49 | } 50 | return defaultReturn(); 51 | } 52 | 53 | LinkedHashSet getColumnIdentifiers() { 54 | return columnIdentifiers; 55 | } 56 | } -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/backend/ortools/GroupContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm.backend.ortools; 7 | 8 | import com.vmware.dcm.compiler.ir.GroupByQualifier; 9 | 10 | import static com.vmware.dcm.backend.ortools.OrToolsSolver.camelCase; 11 | 12 | class GroupContext { 13 | private final GroupByQualifier qualifier; 14 | private final String tempTableName; 15 | private final String groupViewName; 16 | 17 | GroupContext(final GroupByQualifier qualifier, final String tempTableName, final String groupViewName) { 18 | this.qualifier = qualifier; 19 | this.tempTableName = tempTableName; 20 | this.groupViewName = groupViewName; 21 | } 22 | 23 | public GroupByQualifier getQualifier() { 24 | return qualifier; 25 | } 26 | 27 | public String getTempTableName() { 28 | return tempTableName; 29 | } 30 | 31 | public String getGroupViewName() { 32 | return groupViewName; 33 | } 34 | 35 | public String getGroupName() { 36 | return camelCase(groupViewName) + "Group"; 37 | } 38 | 39 | public String getGroupDataName() { 40 | return camelCase(groupViewName) + "Data"; 41 | } 42 | 43 | public String getGroupDataTupleName() { 44 | return getGroupDataName() + "Tuple"; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/backend/ortools/IsConstantSubquery.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm.backend.ortools; 7 | 8 | import com.vmware.dcm.compiler.ir.ColumnIdentifier; 9 | import com.vmware.dcm.compiler.ir.GroupByComprehension; 10 | import com.vmware.dcm.compiler.ir.ListComprehension; 11 | import com.vmware.dcm.compiler.ir.TableRowGenerator; 12 | 13 | import java.util.List; 14 | import java.util.Set; 15 | import java.util.stream.Collectors; 16 | 17 | /* 18 | * Evaluates whether a sub-query (and its inner sub-queries etc.) can be treated as a constant expression. 19 | */ 20 | public class IsConstantSubquery { 21 | 22 | static boolean apply(final ListComprehension expr) { 23 | final GetColumnIdentifiers visitor = new GetColumnIdentifiers(true); 24 | if (expr instanceof GroupByComprehension) { 25 | final ListComprehension comprehension = ((GroupByComprehension) expr).getComprehension(); 26 | comprehension.getHead().getSelectExprs().forEach(visitor::visit); 27 | comprehension.getQualifiers().forEach(visitor::visit); 28 | } else { 29 | expr.getHead().getSelectExprs().forEach(visitor::visit); 30 | expr.getQualifiers().forEach(visitor::visit); 31 | } 32 | final List columnIdentifiers = visitor.getColumnIdentifiers() 33 | .stream() 34 | .filter(e -> e instanceof ColumnIdentifier) 35 | .map(e -> (ColumnIdentifier) e) 36 | .collect(Collectors.toList()); 37 | 38 | final Set accessedTables = expr.getQualifiers() 39 | .stream().filter(q -> q instanceof TableRowGenerator) 40 | .map(e -> ((TableRowGenerator ) e).getTable().getAliasedName()) 41 | .collect(Collectors.toSet()); 42 | return columnIdentifiers.stream().allMatch( 43 | ci -> !ci.getField().isControllable() && accessedTables.contains(ci.getTableName()) 44 | ); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/backend/ortools/JavaExpression.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2021 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm.backend.ortools; 7 | 8 | class JavaExpression { 9 | private final String exprStr; 10 | private final JavaType type; 11 | 12 | JavaExpression(final String varName, final JavaType type) { 13 | this.exprStr = varName; 14 | this.type = type; 15 | } 16 | 17 | /** 18 | * Returns a Java expression (which might be a variable name or a function call) 19 | * 20 | * @return A String representing a variable name or a function call 21 | */ 22 | public String asString() { 23 | return exprStr; 24 | } 25 | 26 | public JavaType type() { 27 | return type; 28 | } 29 | 30 | @Override 31 | public String toString() { 32 | return "JavaIdentifier{" + 33 | "exprStr='" + exprStr + '\'' + 34 | ", type=" + type + 35 | '}'; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/backend/ortools/JavaType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2021 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm.backend.ortools; 7 | 8 | import edu.umd.cs.findbugs.annotations.Nullable; 9 | 10 | import java.util.Optional; 11 | 12 | enum JavaType { 13 | IntVar("IntVar"), 14 | BoolVar("BoolVar"), 15 | String("String"), 16 | Boolean("Boolean"), 17 | Long("Long"), 18 | Integer("Integer"), 19 | Float("Float"), 20 | ObjectArray("Object[]"), 21 | ListOfIntVar("List", IntVar), 22 | ListOfBoolVar("List", BoolVar), 23 | ListOfInteger("List", Integer), 24 | ListOfLong("List", Long), 25 | ListOfBool("List", Boolean), 26 | ListOfString("List", String), 27 | ListOfObjectArray("List", ObjectArray); 28 | 29 | private final String typeString; 30 | 31 | @Nullable private final JavaType innerType; 32 | 33 | JavaType(final String typeString) { 34 | this.typeString = typeString; 35 | this.innerType = null; 36 | } 37 | 38 | JavaType(final String typeString, final JavaType type) { 39 | this.typeString = typeString; 40 | this.innerType = type; 41 | } 42 | 43 | public String typeString() { 44 | if (this.innerType != null) { 45 | return java.lang.String.format("List<%s>", innerType.typeString()); 46 | } 47 | return typeString; 48 | } 49 | 50 | public static JavaType listType(final JavaType innerType) { 51 | switch (innerType) { 52 | case IntVar: 53 | return ListOfIntVar; 54 | case BoolVar: 55 | return ListOfBoolVar; 56 | case Integer: 57 | return ListOfInteger; 58 | case Long: 59 | return ListOfLong; 60 | case String: 61 | return ListOfString; 62 | case Boolean: 63 | return ListOfBool; 64 | case ObjectArray: 65 | return ListOfObjectArray; 66 | default: 67 | throw new IllegalArgumentException(innerType.toString()); 68 | } 69 | } 70 | 71 | public static boolean isVar(final JavaType type) { 72 | return type == JavaType.IntVar || type == JavaType.BoolVar; 73 | } 74 | 75 | public Optional innerType() { 76 | return Optional.ofNullable(innerType); 77 | } 78 | } -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/backend/ortools/JavaTypeList.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2021 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm.backend.ortools; 7 | 8 | import com.google.common.base.Preconditions; 9 | 10 | import java.util.List; 11 | import java.util.stream.Collectors; 12 | 13 | class JavaTypeList { 14 | private final List typeList; 15 | 16 | JavaTypeList(final List typeList) { 17 | Preconditions.checkArgument(!typeList.isEmpty(), "Empty type list detected"); 18 | this.typeList = typeList; 19 | } 20 | 21 | @Override 22 | public String toString() { 23 | return typeList.stream().map(JavaType::typeString).collect(Collectors.joining(", ")); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/backend/ortools/JoinStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm.backend.ortools; 7 | 8 | import com.vmware.dcm.compiler.ir.FunctionCall; 9 | import com.vmware.dcm.compiler.ir.VoidType; 10 | import com.vmware.dcm.compiler.ir.Expr; 11 | import com.vmware.dcm.compiler.ir.SimpleVisitor; 12 | 13 | import java.util.ArrayList; 14 | import java.util.List; 15 | 16 | class JoinStrategy extends SimpleVisitor { 17 | final List nodes = new ArrayList<>(); 18 | 19 | @Override 20 | protected VoidType visitFunctionCall(final FunctionCall node, final VoidType context) { 21 | if (node.getFunction().equals(FunctionCall.Function.CAPACITY_CONSTRAINT)) { 22 | nodes.add(node); 23 | } 24 | return super.visitFunctionCall(node, context); 25 | } 26 | 27 | public static Type apply(final Expr expr) { 28 | final JoinStrategy visitor = new JoinStrategy(); 29 | visitor.visit(expr); 30 | return visitor.nodes.isEmpty() ? Type.MATERIALIZED : Type.UNMATERIALIZED; 31 | } 32 | 33 | enum Type { 34 | MATERIALIZED, 35 | UNMATERIALIZED 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/backend/ortools/ScalarProductOptimization.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2021 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm.backend.ortools; 7 | 8 | import com.vmware.dcm.compiler.ir.BinaryOperatorPredicate; 9 | import com.vmware.dcm.compiler.ir.ComprehensionRewriter; 10 | import com.vmware.dcm.compiler.ir.Expr; 11 | import com.vmware.dcm.compiler.ir.FunctionCall; 12 | import com.vmware.dcm.compiler.ir.GroupByComprehension; 13 | import com.vmware.dcm.compiler.ir.ListComprehension; 14 | import com.vmware.dcm.compiler.ir.VoidType; 15 | import org.slf4j.Logger; 16 | import org.slf4j.LoggerFactory; 17 | 18 | import java.util.List; 19 | 20 | 21 | /** 22 | * A sum of an expression of the form (X * Y), where X is a variable and Y is a constant, is best 23 | * expressed as a scalar product which takes a list of variables, and a corresponding list of coefficients. 24 | * This is common in bin-packing problems where a list of variables corresponding to whether a task is 25 | * assigned to a bin, is weighted by the demands of each task, and the resulting weighted sum of variables 26 | * is used in capacity constraints. 27 | * 28 | * This optimization saves the CP-SAT solver a lot of effort in the pre-solving and solving phases, 29 | * leading to significant performance improvements. 30 | * 31 | * If we cannot perform this optimization, we revert to computing a sum just like any other function. 32 | * 33 | */ 34 | class ScalarProductOptimization extends ComprehensionRewriter { 35 | private static final Logger LOG = LoggerFactory.getLogger(ScalarProductOptimization.class); 36 | private final TupleMetadata metadata; 37 | 38 | ScalarProductOptimization(final TupleMetadata metadata) { 39 | this.metadata = metadata; 40 | } 41 | 42 | static ListComprehension apply(final ListComprehension comprehension, final TupleMetadata metadata) { 43 | LOG.trace("Invoking ScalarProductOptimization on {}", comprehension); 44 | final ScalarProductOptimization rewriter = new ScalarProductOptimization(metadata); 45 | final Expr result = rewriter.visit(comprehension); 46 | return comprehension instanceof GroupByComprehension ? 47 | (GroupByComprehension) result : (ListComprehension) result; 48 | } 49 | 50 | @Override 51 | protected Expr visitFunctionCall(final FunctionCall node, final VoidType context) { 52 | if ((node.getFunction() == FunctionCall.Function.SUM 53 | || node.getFunction() == FunctionCall.Function.COUNT)) { 54 | final Expr argument = node.getArgument().get(0); 55 | if (argument instanceof BinaryOperatorPredicate) { 56 | final BinaryOperatorPredicate operation = ((BinaryOperatorPredicate) argument); 57 | final BinaryOperatorPredicate.Operator op = operation.getOperator(); 58 | final Expr left = operation.getLeft(); 59 | final Expr right = operation.getRight(); 60 | final JavaType leftType = metadata.inferType(left); 61 | final JavaType rightType = metadata.inferType(right); 62 | // TODO: The multiply may not necessarily be the top level operation. 63 | if (op.equals(BinaryOperatorPredicate.Operator.MULTIPLY)) { 64 | if (JavaType.isVar(leftType) && !JavaType.isVar(rightType)) { 65 | return new FunctionCall(FunctionCall.Function.SCALAR_PRODUCT, List.of(left, right), 66 | node.getAlias()); 67 | } 68 | if (JavaType.isVar(rightType) && !JavaType.isVar(leftType)) { 69 | return new FunctionCall(FunctionCall.Function.SCALAR_PRODUCT, List.of(right, left), 70 | node.getAlias()); 71 | } 72 | } 73 | } 74 | } 75 | return super.visitFunctionCall(node, context); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/backend/ortools/StringEncoding.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.backend.ortools; 8 | 9 | import java.util.HashMap; 10 | 11 | public class StringEncoding { 12 | private static final int DEFAULT_CAPACITY = 10000; 13 | private long counter = 0; 14 | private final HashMap strToLongMap = new HashMap<>(DEFAULT_CAPACITY); 15 | private final HashMap longToStrMap = new HashMap<>(DEFAULT_CAPACITY); 16 | 17 | public long toLong(final String str) { 18 | return strToLongMap.computeIfAbsent(str, this::update); 19 | } 20 | 21 | // Pass through 22 | public long toLong(final long i) { 23 | return i; 24 | } 25 | 26 | public String toStr(final long i) { 27 | return longToStrMap.get(i); 28 | } 29 | 30 | final long update(final String str) { 31 | counter += 1; 32 | longToStrMap.put(counter, str); 33 | return counter; 34 | } 35 | } -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/backend/ortools/SubQueryContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm.backend.ortools; 7 | 8 | class SubQueryContext { 9 | private final String subQueryName; 10 | 11 | SubQueryContext(final String subQueryName) { 12 | this.subQueryName = subQueryName; 13 | } 14 | 15 | public String getSubQueryName() { 16 | return subQueryName; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/backend/ortools/TupleGen.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.backend.ortools; 8 | 9 | import com.squareup.javapoet.MethodSpec; 10 | import com.squareup.javapoet.TypeSpec; 11 | import com.squareup.javapoet.TypeVariableName; 12 | 13 | import javax.lang.model.element.Modifier; 14 | import java.util.Collection; 15 | import java.util.Map; 16 | import java.util.concurrent.ConcurrentHashMap; 17 | import java.util.stream.Collectors; 18 | import java.util.stream.IntStream; 19 | 20 | class TupleGen { 21 | private final Map tuples = new ConcurrentHashMap<>(); 22 | 23 | Collection getAllTupleTypes() { 24 | return tuples.values(); 25 | } 26 | 27 | TypeSpec getTupleType(final int numRecords) { 28 | return tuples.computeIfAbsent(numRecords, this::tupleGen); 29 | } 30 | 31 | /** 32 | * Create a tuple type with 'numFields' entries. Results in a generic "plain old java object" 33 | * with a getter per field. 34 | */ 35 | TypeSpec tupleGen(final int numFields) { 36 | final TypeSpec.Builder classBuilder = TypeSpec.classBuilder("Tuple" + numFields) 37 | .addModifiers(Modifier.FINAL, Modifier.PRIVATE, Modifier.STATIC); 38 | final MethodSpec.Builder constructor = MethodSpec.constructorBuilder() 39 | .addModifiers(Modifier.PRIVATE); 40 | for (int i = 0; i < numFields; i++) { 41 | final TypeVariableName type = TypeVariableName.get("T" + i); 42 | // Add parameter to constructor 43 | constructor.addParameter(type, "t" + i, Modifier.FINAL) 44 | .addStatement("this.$1L = $1L", "t" + i); // assign parameters to fields 45 | 46 | // Create getter 47 | final MethodSpec getter = MethodSpec.methodBuilder("value" + i) 48 | .returns(type) 49 | .addStatement("return t" + i) 50 | .build(); 51 | 52 | // Add field and getter to class 53 | classBuilder.addTypeVariable(type) 54 | .addField(type, "t" + i, Modifier.PRIVATE, Modifier.FINAL) 55 | .addMethod(getter); 56 | } 57 | 58 | final String commaSeparatedParameters = IntStream.range(0, numFields) 59 | .mapToObj(i -> "t" + i) 60 | .collect(Collectors.joining(", ")); 61 | 62 | final String formatString = IntStream.range(0, numFields) 63 | .mapToObj(i -> "%s") 64 | .collect(Collectors.joining(",")); 65 | 66 | final MethodSpec toStringMethod = MethodSpec.methodBuilder("toString") 67 | .addAnnotation(Override.class) 68 | .addModifiers(Modifier.PUBLIC) 69 | .returns(String.class) 70 | .addStatement("return String.format(($S), $L)", formatString, commaSeparatedParameters) 71 | .build(); 72 | final MethodSpec hashCodeMethod = MethodSpec.methodBuilder("hashCode") 73 | .addAnnotation(Override.class) 74 | .addModifiers(Modifier.PUBLIC) 75 | .returns(int.class) 76 | .addStatement("return $T.hash($L)", Ops.class, commaSeparatedParameters) 77 | .build(); 78 | final MethodSpec.Builder equals = MethodSpec.methodBuilder("equals") 79 | .addAnnotation(Override.class) 80 | .addModifiers(Modifier.PUBLIC) 81 | .returns(boolean.class) 82 | .addParameter(Object.class, "other", Modifier.FINAL) 83 | .beginControlFlow("if (other == this)") 84 | .addStatement("return true") 85 | .endControlFlow() 86 | .beginControlFlow("if (!(other instanceof Tuple$L))", numFields) 87 | .addStatement("return false") 88 | .endControlFlow() 89 | .addStatement("final Tuple$1L that = (Tuple$1L) other", numFields) 90 | .addCode("return "); 91 | 92 | final String returnValue = IntStream.range(0, numFields) 93 | .mapToObj(i -> String.format("Ops.equals(this.value%s(), that.value%s())", i, i)) 94 | .collect(Collectors.joining(" && ")); 95 | equals.addCode("$L;\n", returnValue); 96 | final MethodSpec equalsMethod = equals.build(); 97 | return classBuilder.addMethod(constructor.build()) 98 | .addMethod(toStringMethod) 99 | .addMethod(hashCodeMethod) 100 | .addMethod(equalsMethod) 101 | .build(); 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/backend/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | /** 8 | * This informs FindBugs to mark parameters, fields and methods as NonNull. 9 | */ 10 | 11 | @ParametersAreNonnullByDefault 12 | @FieldsAreNonnullByDefault 13 | @MethodsAreNonnullByDefault 14 | 15 | package com.vmware.dcm.backend; 16 | 17 | import com.vmware.dcm.annotations.FieldsAreNonnullByDefault; 18 | import com.vmware.dcm.annotations.MethodsAreNonnullByDefault; 19 | 20 | import javax.annotation.ParametersAreNonnullByDefault; 21 | 22 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/DesugarExists.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2021 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm.compiler; 7 | 8 | import com.vmware.dcm.compiler.ir.BinaryOperatorPredicate; 9 | import com.vmware.dcm.compiler.ir.ComprehensionRewriter; 10 | import com.vmware.dcm.compiler.ir.ExistsPredicate; 11 | import com.vmware.dcm.compiler.ir.Expr; 12 | import com.vmware.dcm.compiler.ir.ListComprehension; 13 | import com.vmware.dcm.compiler.ir.Literal; 14 | import com.vmware.dcm.compiler.ir.VoidType; 15 | 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | 19 | /* 20 | * Expands an exists() predicate into "exists() = true" 21 | */ 22 | class DesugarExists extends ComprehensionRewriter { 23 | private final List stack = new ArrayList<>(); 24 | 25 | static ListComprehension apply(final ListComprehension view) { 26 | final DesugarExists inner = new DesugarExists(); 27 | return (ListComprehension) inner.visit(view); 28 | } 29 | 30 | @Override 31 | public Expr visit(final Expr expr, final VoidType context) { 32 | stack.add(expr); 33 | final Expr result = super.visit(expr, context); 34 | stack.remove(stack.size() - 1); 35 | return result; 36 | } 37 | 38 | @Override 39 | protected Expr visitExistsPredicate(final ExistsPredicate node, final VoidType context) { 40 | // The node being visited is at the top of the stack. Peek one level behind it for the ancestor. 41 | final Expr ancestor = stack.get(stack.size() - 2); 42 | if (ancestor instanceof BinaryOperatorPredicate) { 43 | final BinaryOperatorPredicate.Operator op = ((BinaryOperatorPredicate) ancestor).getOperator(); 44 | switch (op) { 45 | case EQUAL: 46 | case NOT_EQUAL: 47 | return super.visitExistsPredicate(node, context); 48 | default: 49 | break; 50 | } 51 | } 52 | return new BinaryOperatorPredicate(BinaryOperatorPredicate.Operator.EQUAL, node, 53 | new Literal<>(true, Boolean.class)); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/FromExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler; 8 | 9 | import com.google.common.collect.ImmutableList; 10 | import com.google.common.collect.ImmutableSet; 11 | import org.apache.calcite.sql.SqlBasicCall; 12 | import org.apache.calcite.sql.SqlCall; 13 | import org.apache.calcite.sql.SqlIdentifier; 14 | import org.apache.calcite.sql.SqlJoin; 15 | import org.apache.calcite.sql.SqlNode; 16 | import org.apache.calcite.sql.SqlSelect; 17 | import org.apache.calcite.sql.util.SqlBasicVisitor; 18 | import org.jooq.Record; 19 | 20 | import java.util.ArrayList; 21 | import java.util.LinkedHashSet; 22 | import java.util.List; 23 | import java.util.Map; 24 | import java.util.Set; 25 | 26 | /** 27 | * Extracts IRTables and filtering conditions from a FROM clause 28 | */ 29 | class FromExtractor extends SqlBasicVisitor { 30 | // Using a LinkedHashSet is necessary to have a deterministic order of tables in the IR 31 | private final Set tables = new LinkedHashSet<>(); 32 | private final List joinConditions = new ArrayList<>(); 33 | private final IRContext irContext; 34 | private boolean seen = false; 35 | 36 | FromExtractor(final IRContext irContext) { 37 | this.irContext = irContext; 38 | } 39 | 40 | @Override 41 | public Void visit(final SqlCall call) { 42 | if (!seen && call instanceof SqlSelect) { 43 | final SqlSelect select = (SqlSelect) call; 44 | final SqlNode from = select.getFrom(); 45 | assert from != null; 46 | visitRelation(from); 47 | seen = true; 48 | } 49 | return super.visit(call); 50 | } 51 | 52 | private void visitRelation(final SqlNode node) { 53 | switch (node.getKind()) { 54 | case IDENTIFIER: 55 | tables.add(toIrTable(node)); 56 | break; 57 | case AS: 58 | final List operands = ((SqlBasicCall) node).getOperandList(); 59 | assert operands != null; 60 | final SqlIdentifier relation = (SqlIdentifier) operands.get(0); 61 | final SqlIdentifier alias = (SqlIdentifier) operands.get(1); 62 | assert relation != null; 63 | assert alias != null; 64 | final IRTable irTable = toIrTable(relation); 65 | // TODO: This duplicates code from Model in creating IRTable and IRColumn instances. 66 | final org.jooq.Table table = irTable.isViewTable() ? null : irTable.getTable(); 67 | final IRTable tableAlias = new IRTable(table, irTable.getName(), alias.getSimple()); 68 | 69 | // parse all fields 70 | for (final Map.Entry entry: irTable.getIRColumns().entrySet()) { 71 | final IRColumn irColumn = entry.getValue(); 72 | final IRColumn aliasIRColumn = irTable.isViewTable() 73 | ? new IRColumn(tableAlias, null, irColumn.getType(), 74 | irColumn.getName()) 75 | : new IRColumn(tableAlias, irColumn.getJooqField()); 76 | tableAlias.addField(aliasIRColumn); 77 | } 78 | 79 | // After adding all the IRFields to the table, we parse the table UniqueKey 80 | // and link the correspondent IRFields as fields that compose the IRTable primary key 81 | if (!irTable.isViewTable()) { 82 | final IRPrimaryKey pk = new IRPrimaryKey(tableAlias, table.getPrimaryKey()); 83 | tableAlias.setPrimaryKey(pk); 84 | } 85 | tables.add(tableAlias); 86 | irContext.addAliasedOrViewTable(tableAlias); 87 | break; 88 | case JOIN: 89 | final SqlJoin join = (SqlJoin) node; 90 | visitRelation(join.getLeft()); 91 | visitRelation(join.getRight()); 92 | joinConditions.add(join.getCondition()); 93 | break; 94 | default: 95 | break; 96 | } 97 | } 98 | 99 | private IRTable toIrTable(final SqlNode identifier) { 100 | return irContext.getTable(((SqlIdentifier) identifier).getSimple()); 101 | } 102 | 103 | Set getTables() { 104 | return ImmutableSet.copyOf(tables); 105 | } 106 | 107 | List getJoinConditions() { 108 | return ImmutableList.copyOf(joinConditions); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/IRColumn.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler; 8 | 9 | import org.jooq.Field; 10 | 11 | import javax.annotation.Nullable; 12 | import java.sql.Types; 13 | import java.util.Locale; 14 | import java.util.Objects; 15 | 16 | 17 | /** 18 | * Represents a jooqField within an SQL table 19 | */ 20 | public class IRColumn { 21 | private static final String VARIABLE_PREFIX = "CONTROLLABLE__"; 22 | @Nullable private final Field jooqField; 23 | @Nullable private final FieldType type; 24 | @Nullable private final IRTable irTable; 25 | private final String name; 26 | 27 | /** 28 | * For now, we coerce SQL types to float, int, string, boolean and arrays. 29 | */ 30 | public enum FieldType { 31 | FLOAT, LONG, INT, STRING, BOOL, ARRAY; 32 | 33 | /** 34 | * Returns the coerced type of an SQL field 35 | * 36 | * @param f SQL table jooqField 37 | * @return the type of the SQL JooqField 38 | */ 39 | public static FieldType fromField(final Field f) { 40 | switch (f.getDataType().getSQLType()) { 41 | case Types.BIGINT: 42 | return FieldType.LONG; 43 | case Types.INTEGER: 44 | case Types.SMALLINT: 45 | case Types.TINYINT: 46 | return FieldType.INT; 47 | case Types.DECIMAL: 48 | case Types.DOUBLE: 49 | case Types.FLOAT: 50 | case Types.NUMERIC: 51 | case Types.REAL: 52 | return FieldType.FLOAT; 53 | case Types.BIT: 54 | case Types.BOOLEAN: 55 | return FieldType.BOOL; 56 | case Types.CHAR: 57 | case Types.VARCHAR: 58 | return FieldType.STRING; 59 | case Types.OTHER: 60 | return FieldType.ARRAY; 61 | default: 62 | throw new IllegalArgumentException("Unknown type jooqField: " + f.getDataType().getSQLType()); 63 | } 64 | } 65 | } 66 | 67 | public IRColumn(final IRTable irTable, final Field jooqField) { 68 | this(irTable, jooqField, FieldType.fromField(jooqField), jooqField.getName()); 69 | } 70 | 71 | /** 72 | * Builds a IRColumn from a SQL jooqField parsing its type and tags 73 | * @param irTable the IRTable that this column belongs to 74 | * @param jooqField the Field that this column corresponds to 75 | * @param fieldType the FieldType of this column 76 | * @param fieldNameInitial the initial name for this column 77 | */ 78 | public IRColumn(@Nullable final IRTable irTable, @Nullable final Field jooqField, 79 | final FieldType fieldType, final String fieldNameInitial) { 80 | this.irTable = irTable; 81 | this.jooqField = jooqField; 82 | this.name = fieldNameInitial.toUpperCase(Locale.US); 83 | this.type = fieldType; 84 | } 85 | 86 | /** 87 | * @return Returns true if a jooqField is a variable column 88 | */ 89 | public boolean isControllable() { 90 | return name.startsWith(VARIABLE_PREFIX); 91 | } 92 | 93 | public boolean isString() { 94 | return Objects.requireNonNull(type).equals(FieldType.STRING); 95 | } 96 | 97 | /** 98 | * Returns the IRTable corresponding to this IRColumn 99 | * @return the IRTable corresponding to this IRColumn 100 | */ 101 | public IRTable getIRTable() { 102 | return Objects.requireNonNull(irTable); 103 | } 104 | 105 | /** 106 | * Returns the Jooq Field that backs this IRColumn 107 | * @return the Jooq Field that backs this IRColumn 108 | */ 109 | public Field getJooqField() { 110 | return Objects.requireNonNull(jooqField); 111 | } 112 | 113 | /** 114 | * @return the column name 115 | */ 116 | public String getName() { 117 | return name; 118 | } 119 | 120 | /** 121 | * @return the column's FieldType 122 | */ 123 | public FieldType getType() { 124 | return Objects.requireNonNull(type); 125 | } 126 | 127 | @Override 128 | public synchronized String toString() { 129 | return "IRColumn{" + 130 | "jooqField=" + this.jooqField + 131 | ", name='" + this.name + '\'' + 132 | ", type=" + this.type + 133 | '}'; 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/IRColumnsFromSelectItems.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2021 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm.compiler; 7 | 8 | import org.apache.calcite.sql.SqlCall; 9 | import org.apache.calcite.sql.SqlIdentifier; 10 | import org.apache.calcite.sql.SqlKind; 11 | import org.apache.calcite.sql.util.SqlBasicVisitor; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import java.util.Set; 16 | 17 | /** 18 | * Identifies and adds IRColumns to a given IRTable (viewTable) by scanning the select items in a particular view. 19 | */ 20 | class IRColumnsFromSelectItems extends SqlBasicVisitor { 21 | private static final Logger LOG = LoggerFactory.getLogger(IRColumnsFromSelectItems.class); 22 | private final IRContext irContext; 23 | private final IRTable viewTable; 24 | private final Set tablesReferencedInView; 25 | 26 | IRColumnsFromSelectItems(final IRContext irContext, final IRTable viewTable, 27 | final Set tablesReferencedInView) { 28 | this.irContext = irContext; 29 | this.viewTable = viewTable; 30 | this.tablesReferencedInView = tablesReferencedInView; 31 | } 32 | 33 | @Override 34 | public Void visit(final SqlCall call) { 35 | if (call.getKind() == SqlKind.AS) { 36 | final String alias = ((SqlIdentifier) call.operand(1)).getSimple(); 37 | LOG.warn("Guessing FieldType for column {} in non-constraint view {} to be INT", 38 | alias, viewTable.getName()); 39 | final IRColumn.FieldType intType = IRColumn.FieldType.INT; 40 | final IRColumn newColumn = new IRColumn(viewTable, null, intType, alias); 41 | viewTable.addField(newColumn); 42 | return null; 43 | } 44 | throw new IllegalArgumentException(call.toString()); 45 | } 46 | 47 | @Override 48 | public Void visit(final SqlIdentifier id) { 49 | if (id.isStar()) { 50 | tablesReferencedInView.forEach( 51 | table -> table.getIRColumns().forEach((fieldName, irColumn) -> viewTable.addField(irColumn)) 52 | ); 53 | } else if (id.isSimple()) { 54 | final IRColumn columnIfUnique = irContext.getColumnIfUnique(id.getSimple(), tablesReferencedInView); 55 | final IRColumn newColumn = new IRColumn(viewTable, null, columnIfUnique.getType(), 56 | columnIfUnique.getName()); 57 | viewTable.addField(newColumn); 58 | } else { 59 | final IRColumn irColumn = TranslateViewToIR.getIRColumnFromDereferencedExpression(id, irContext); 60 | final IRColumn newColumn = new IRColumn(viewTable, null, irColumn.getType(), 61 | irColumn.getName()); 62 | viewTable.addField(newColumn); 63 | } 64 | return super.visit(id); 65 | } 66 | } -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/IRForeignKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler; 8 | 9 | import org.jooq.ForeignKey; 10 | import org.jooq.Record; 11 | import org.jooq.TableField; 12 | 13 | import java.util.HashMap; 14 | import java.util.List; 15 | import java.util.Map; 16 | 17 | public class IRForeignKey { 18 | 19 | private final IRTable childTable; 20 | private final Map fields; 21 | 22 | /** 23 | * Represents an SQL Foreign Key relationship in the IR, and captures the relationship between the fields from 24 | * the child table, with the fields from the parent table. The table containing the foreign key is called the 25 | * child table, and the table containing the candidate key is called the referenced or parent table. 26 | * 27 | * Has to be public so mnz_data.ftl and mnz_model.ftl template files can find the corresponding methods 28 | * 29 | * @param childTable the child table in the FK relationship 30 | * @param parentTable the parent table in the FK relationship 31 | * @param fk SQL foreign key field 32 | */ 33 | IRForeignKey(final IRTable childTable, final IRTable parentTable, final ForeignKey fk) { 34 | this.childTable = childTable; 35 | this.fields = new HashMap<>(fk.getFields().size()); 36 | 37 | // gets the main table fields and the referenced table fields 38 | final List> childFields = fk.getFields(); 39 | final List> parentFields = fk.getKey().getFields(); 40 | 41 | // assuming both child and parent fields have the same size 42 | // SQL shouldn't allow this anyway 43 | // XXX: verify assumption 44 | for (int i = 0; i < childFields.size(); i++) { 45 | final IRColumn childField = childTable.getField(childFields.get(i)); 46 | final IRColumn parentField = parentTable.getField(parentFields.get(i)); 47 | // map fields from child table to parent table 48 | this.fields.put(childField, parentField); 49 | } 50 | } 51 | 52 | /** 53 | * Returns true if one of the child table fields is a controllable, false otherwise 54 | */ 55 | private boolean hasControllableField() { 56 | return fields.keySet().stream().anyMatch(IRColumn::isControllable); 57 | } 58 | 59 | /** 60 | * Used in the mnz_data.ftl and mnz_model.ftl template files 61 | * @return returns the child table in this FK relation 62 | */ 63 | public IRTable getChildTable() { 64 | return childTable; 65 | } 66 | 67 | /** 68 | * Used in the mnz_data.ftl and mnz_model.ftl template files 69 | * @return returns the fields represented in this FK relation 70 | */ 71 | public Map getFields() { 72 | return fields; 73 | } 74 | 75 | /** 76 | * Returns true if this foreign key is defined on a variable column, false otherwise 77 | * @return true if this foreign key is defined on a variable column, false otherwise 78 | */ 79 | public boolean hasConstraint() { 80 | return !this.fields.isEmpty() && this.hasControllableField(); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/IRPrimaryKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler; 8 | 9 | 10 | import org.jooq.Field; 11 | import org.jooq.Record; 12 | import org.jooq.UniqueKey; 13 | 14 | import javax.annotation.Nullable; 15 | import java.util.ArrayList; 16 | import java.util.List; 17 | 18 | public class IRPrimaryKey { 19 | private final List primaryKeyFields; 20 | private final IRTable irTable; 21 | 22 | public IRPrimaryKey(final IRTable irTable, @Nullable final UniqueKey pk) { 23 | this.irTable = irTable; 24 | this.primaryKeyFields = new ArrayList<>(); 25 | // if no pk just skips - this can happen on views e.g. 26 | if (pk != null) { 27 | for (final Field pkField : pk.getFields()) { 28 | this.primaryKeyFields.add(irTable.getField(pkField)); 29 | } 30 | } 31 | } 32 | 33 | /** 34 | * Returns all the fields that compose this primary key 35 | * @return all the fields that compose this primary key 36 | */ 37 | public List getPrimaryKeyFields() { 38 | return primaryKeyFields; 39 | } 40 | 41 | /** 42 | * Returns true if this primaryKey has a controllable field 43 | * @return true if this primaryKey has a controllable field 44 | */ 45 | public boolean hasControllableColumn() { 46 | return primaryKeyFields.stream().anyMatch(IRColumn::isControllable); 47 | } 48 | 49 | /** 50 | * Returns the irTable for which this primary key is configured 51 | * @return the irTable for which this primary key is configured 52 | */ 53 | public IRTable getIRTable() { 54 | return irTable; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/SyntaxChecking.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2021 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm.compiler; 7 | 8 | import com.vmware.dcm.ModelException; 9 | import com.vmware.dcm.compiler.ir.FunctionCall; 10 | import com.vmware.dcm.parser.SqlCreateConstraint; 11 | import org.apache.calcite.sql.JoinType; 12 | import org.apache.calcite.sql.SqlCall; 13 | import org.apache.calcite.sql.SqlIdentifier; 14 | import org.apache.calcite.sql.SqlJoin; 15 | import org.apache.calcite.sql.SqlLiteral; 16 | import org.apache.calcite.sql.util.SqlBasicVisitor; 17 | 18 | /* 19 | * Checks if the parsed AST only uses the supported subset of SQL to specify policies 20 | */ 21 | public class SyntaxChecking extends SqlBasicVisitor { 22 | 23 | @Override 24 | public Boolean visit(final SqlCall call) { 25 | if (isAllowedKind(call)) { 26 | return super.visit(call); 27 | } 28 | throw new ModelException("Unexpected AST type " + call); 29 | } 30 | 31 | @Override 32 | public Boolean visit(final SqlIdentifier id) { 33 | if (id.isStar() || id.isSimple() || id.names.size() == 2) { 34 | return super.visit(id); 35 | } 36 | throw new ModelException("Unexpected AST type " + id); 37 | } 38 | 39 | @Override 40 | public Boolean visit(final SqlLiteral literal) { 41 | return true; 42 | } 43 | 44 | private boolean isAllowedKind(final SqlCall call) { 45 | switch (call.getOperator().getKind()) { 46 | case OTHER_FUNCTION: 47 | try { 48 | FunctionCall.Function.valueOf(call.getOperator().getName().toUpperCase()); 49 | return true; 50 | } catch (final IllegalArgumentException e) { 51 | return false; 52 | } 53 | case JOIN: 54 | final SqlJoin join = (SqlJoin) call; 55 | if (join.getJoinType() == JoinType.INNER) { 56 | return true; 57 | } else { 58 | // We throw an exception here because SqlJoin.toString() is buggy 59 | // https://issues.apache.org/jira/browse/CALCITE-4401 60 | throw new ModelException("Unexpected AST type " + join.getLeft() + " " + join.getJoinType() 61 | + " JOIN " + join.getRight() + " ON (" + join.getCondition() + ")"); 62 | } 63 | case SELECT: 64 | case IN: 65 | case NOT_IN: 66 | case EXISTS: 67 | case NOT: 68 | case PLUS: 69 | case MINUS: 70 | case TIMES: 71 | case DIVIDE: 72 | case MOD: 73 | case EQUALS: 74 | case LESS_THAN: 75 | case GREATER_THAN: 76 | case LESS_THAN_OR_EQUAL: 77 | case GREATER_THAN_OR_EQUAL: 78 | case NOT_EQUALS: 79 | case AND: 80 | case OR: 81 | case MINUS_PREFIX: 82 | case PLUS_PREFIX: 83 | case IS_NULL: 84 | case IS_NOT_NULL: 85 | case AS: 86 | return true; 87 | default: 88 | return false; 89 | } 90 | } 91 | 92 | public static void apply(final SqlCreateConstraint ddl) { 93 | final SyntaxChecking validQuery = new SyntaxChecking(); 94 | ddl.getQuery().accept(validQuery); 95 | ddl.getConstraint().ifPresent(e -> e.accept(validQuery)); 96 | } 97 | } -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/UsesAggregateFunctions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm.compiler; 7 | 8 | import com.vmware.dcm.compiler.ir.FunctionCall; 9 | import org.apache.calcite.sql.SqlCall; 10 | import org.apache.calcite.sql.SqlSelect; 11 | import org.apache.calcite.sql.util.SqlBasicVisitor; 12 | 13 | import static org.apache.calcite.sql.SqlKind.OTHER_FUNCTION; 14 | 15 | public class UsesAggregateFunctions extends SqlBasicVisitor { 16 | private boolean found = false; 17 | 18 | @Override 19 | public Void visit(final SqlCall call) { 20 | if (call.getKind() == OTHER_FUNCTION) { 21 | final FunctionCall.Function function = 22 | FunctionCall.Function.valueOf(call.getOperator().getName().toUpperCase()); 23 | switch (function) { 24 | case COUNT: 25 | case SUM: 26 | case MIN: 27 | case MAX: 28 | case ANY: 29 | case ALL: 30 | case ALL_DIFFERENT: 31 | case ALL_EQUAL: 32 | case INCREASING: 33 | case CAPACITY_CONSTRAINT: 34 | found = true; 35 | break; 36 | default: 37 | found = false; 38 | } 39 | } 40 | if (call instanceof SqlSelect) { 41 | return null; 42 | } 43 | return super.visit(call); 44 | } 45 | 46 | public boolean isFound() { 47 | return found; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/UsesControllableFields.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler; 8 | 9 | import com.vmware.dcm.compiler.ir.ColumnIdentifier; 10 | import com.vmware.dcm.compiler.ir.Expr; 11 | import com.vmware.dcm.compiler.ir.SimpleVisitor; 12 | import com.vmware.dcm.compiler.ir.VoidType; 13 | 14 | /** 15 | * If a query does not have any variables in it (say, in a predicate or a join key), then they return arrays 16 | * of type int. If they do access variables, then they're of type "var opt" int. 17 | */ 18 | public class UsesControllableFields extends SimpleVisitor { 19 | private boolean usesControllable = false; 20 | 21 | private UsesControllableFields() { 22 | // Don't allow instantiation 23 | } 24 | 25 | @Override 26 | protected VoidType visitColumnIdentifier(final ColumnIdentifier node, final VoidType context) { 27 | if (node.getField().isControllable()) { 28 | usesControllable = true; 29 | } 30 | return super.visitColumnIdentifier(node, context); 31 | } 32 | 33 | public boolean usesControllableFields() { 34 | return usesControllable; 35 | } 36 | 37 | public static boolean apply(final Expr expr) { 38 | final UsesControllableFields visitor = new UsesControllableFields(); 39 | visitor.visit(expr); 40 | return visitor.usesControllableFields(); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/BinaryOperatorPredicate.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler.ir; 8 | 9 | public class BinaryOperatorPredicate extends Qualifier { 10 | private final Operator operator; 11 | private final Expr left; 12 | private final Expr right; 13 | 14 | public BinaryOperatorPredicate(final Operator operator, final Expr left, final Expr right) { 15 | this.operator = operator; 16 | this.left = left; 17 | this.right = right; 18 | } 19 | 20 | @Override 21 | public String toString() { 22 | return left + " " + operator + " " + right; 23 | } 24 | 25 | @Override 26 | T acceptVisitor(final IRVisitor visitor, final C context) { 27 | return visitor.visitBinaryOperatorPredicate(this, context); 28 | } 29 | 30 | public Expr getLeft() { 31 | return left; 32 | } 33 | 34 | public Operator getOperator() { 35 | return operator; 36 | } 37 | 38 | public Expr getRight() { 39 | return right; 40 | } 41 | 42 | public enum Operator { 43 | ADD, 44 | SUBTRACT, 45 | MULTIPLY, 46 | DIVIDE, 47 | MODULUS, 48 | EQUAL, 49 | NOT_EQUAL, 50 | AND, 51 | OR, 52 | LESS_THAN, 53 | LESS_THAN_OR_EQUAL, 54 | GREATER_THAN, 55 | GREATER_THAN_OR_EQUAL, 56 | IN, 57 | NOT_IN, 58 | CONTAINS 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/BinaryOperatorPredicateWithAggregate.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler.ir; 8 | 9 | public class BinaryOperatorPredicateWithAggregate extends BinaryOperatorPredicate { 10 | public BinaryOperatorPredicateWithAggregate(final Operator operator, final Expr left, final Expr right) { 11 | super(operator, left, right); 12 | } 13 | } -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/CheckQualifier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm.compiler.ir; 7 | 8 | public class CheckQualifier extends Qualifier { 9 | 10 | private final Expr expr; 11 | 12 | public CheckQualifier(final Expr expr) { 13 | this.expr = expr; 14 | } 15 | 16 | public Expr getExpr() { 17 | return expr; 18 | } 19 | 20 | @Override 21 | T acceptVisitor(final IRVisitor visitor, final C context) { 22 | return visitor.visitCheckExpression(this, context); 23 | } 24 | 25 | @Override 26 | public String toString() { 27 | return "CheckQualifier{" + 28 | "expr=" + expr + 29 | '}'; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/ColumnIdentifier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler.ir; 8 | 9 | import com.vmware.dcm.compiler.IRColumn; 10 | 11 | public class ColumnIdentifier extends Expr { 12 | private final String tableName; 13 | private final IRColumn field; 14 | // True if this ColumnIdentifier was accessed using a dereference (table.fieldName) in the source SQL view 15 | private final boolean fromDereferencedAccess; 16 | 17 | public ColumnIdentifier(final String table, final IRColumn field, final boolean fromDereferencedAccess) { 18 | this.tableName = table; 19 | this.field = field; 20 | this.fromDereferencedAccess = fromDereferencedAccess; 21 | } 22 | 23 | @Override 24 | public String toString() { 25 | return String.format("%s[<%s>]", field.getName(), tableName); 26 | } 27 | 28 | @Override 29 | T acceptVisitor(final IRVisitor visitor, final C context) { 30 | return visitor.visitColumnIdentifier(this, context); 31 | } 32 | 33 | public IRColumn getField() { 34 | return field; 35 | } 36 | 37 | public String getTableName() { 38 | return field.getIRTable().getAliasedName(); 39 | } 40 | 41 | /* 42 | * True if this column was referenced in the original view using a dereference. False otherwise. 43 | * XXX: Is only consumed by the MiniZinc backend, and is likely not required 44 | */ 45 | public boolean fromDereferencedAccess() { 46 | return fromDereferencedAccess; 47 | } 48 | } -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/ExistsPredicate.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler.ir; 8 | 9 | public class ExistsPredicate extends Expr { 10 | private final Expr argument; 11 | 12 | public ExistsPredicate(final Expr argument) { 13 | this.argument = argument; 14 | } 15 | 16 | public Expr getArgument() { 17 | return argument; 18 | } 19 | 20 | @Override 21 | public String toString() { 22 | return "Exists{" + 23 | "argument=" + argument + 24 | '}'; 25 | } 26 | 27 | @Override 28 | T acceptVisitor(final IRVisitor visitor, final C context) { 29 | return visitor.visitExistsPredicate(this, context); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/Expr.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler.ir; 8 | 9 | import java.util.Optional; 10 | 11 | public abstract class Expr { 12 | private Optional alias = Optional.empty(); 13 | 14 | abstract T acceptVisitor(final IRVisitor visitor, final C context); 15 | 16 | public void setAlias(final String alias) { 17 | this.alias = Optional.of(alias); 18 | } 19 | 20 | public Optional getAlias() { 21 | return alias; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/FunctionCall.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler.ir; 8 | 9 | import java.util.List; 10 | import java.util.Optional; 11 | 12 | public class FunctionCall extends Expr { 13 | private final Function function; 14 | private final List argument; 15 | 16 | public FunctionCall(final Function function, final Expr argument) { 17 | this.function = function; 18 | this.argument = List.of(argument); 19 | } 20 | 21 | public FunctionCall(final Function function, final List argument) { 22 | this.function = function; 23 | this.argument = argument; 24 | } 25 | 26 | public FunctionCall(final Function function, final List argument, final Optional alias) { 27 | this.function = function; 28 | this.argument = argument; 29 | alias.ifPresent(this::setAlias); 30 | } 31 | 32 | public FunctionCall(final Function function, final Expr argument, final String alias) { 33 | this.function = function; 34 | this.argument = List.of(argument); 35 | setAlias(alias); 36 | } 37 | 38 | public List getArgument() { 39 | return argument; 40 | } 41 | 42 | public Function getFunction() { 43 | return function; 44 | } 45 | 46 | @Override 47 | public String toString() { 48 | return "FunctionCall{" + 49 | "functionName='" + function + '\'' + 50 | ", argument=" + argument + 51 | '}'; 52 | } 53 | 54 | @Override 55 | T acceptVisitor(final IRVisitor visitor, final C context) { 56 | return visitor.visitFunctionCall(this, context); 57 | } 58 | 59 | public enum Function { 60 | COUNT, 61 | SUM, 62 | MIN, 63 | MAX, 64 | ANY, 65 | ALL, 66 | ALL_DIFFERENT, 67 | ALL_EQUAL, 68 | INCREASING, 69 | CAPACITY_CONSTRAINT, 70 | CONTAINS, 71 | SCALAR_PRODUCT 72 | } 73 | } -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/GroupByComprehension.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler.ir; 8 | 9 | public final class GroupByComprehension extends ListComprehension { 10 | private final ListComprehension comprehension; 11 | private final GroupByQualifier groupByQualifier; 12 | 13 | public GroupByComprehension(final ListComprehension comprehension, final GroupByQualifier qualifier) { 14 | this.groupByQualifier = qualifier; 15 | this.comprehension = comprehension.withQualifier(qualifier); 16 | } 17 | 18 | @Override 19 | public String toString() { 20 | return String.format("[ [i | i in (%s) by group] | group in %s]", 21 | comprehension, groupByQualifier); 22 | } 23 | 24 | public ListComprehension getComprehension() { 25 | return comprehension; 26 | } 27 | 28 | public GroupByQualifier getGroupByQualifier() { 29 | return groupByQualifier; 30 | } 31 | 32 | @Override 33 | T acceptVisitor(final IRVisitor visitor, final C context) { 34 | return visitor.visitGroupByComprehension(this, context); 35 | } 36 | } -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/GroupByQualifier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler.ir; 8 | 9 | import java.util.List; 10 | import java.util.stream.Collectors; 11 | 12 | public final class GroupByQualifier extends Qualifier { 13 | private final List exprs; 14 | 15 | public GroupByQualifier(final List exprs) { 16 | this.exprs = exprs; 17 | } 18 | 19 | @Override 20 | public String toString() { 21 | return "GroupByQualifier{" + 22 | "groupBy=" + exprs + 23 | '}'; 24 | } 25 | 26 | @Override 27 | T acceptVisitor(final IRVisitor visitor, final C context) { 28 | return visitor.visitGroupByQualifier(this, context); 29 | } 30 | 31 | public List getGroupByExprs() { 32 | return exprs; 33 | } 34 | 35 | public List getGroupByColumnIdentifiers() { 36 | return exprs.stream().filter(e -> e instanceof ColumnIdentifier) 37 | .map(e -> (ColumnIdentifier) e) 38 | .collect(Collectors.toList()); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/Head.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler.ir; 8 | 9 | import java.util.List; 10 | 11 | public final class Head extends Expr { 12 | private final List selectExprs; 13 | 14 | public Head(final List selectExprs) { 15 | this.selectExprs = selectExprs; 16 | } 17 | 18 | @Override 19 | public String toString() { 20 | return "Head{" + 21 | "selectExprs=" + selectExprs + 22 | '}'; 23 | } 24 | 25 | @Override 26 | T acceptVisitor(final IRVisitor visitor, final C context) { 27 | return visitor.visitHead(this, context); 28 | } 29 | 30 | public List getSelectExprs() { 31 | return selectExprs; 32 | } 33 | } -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/IRVisitor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler.ir; 8 | 9 | public class IRVisitor { 10 | 11 | public T visit(final Expr expr, final C context) { 12 | return expr.acceptVisitor(this, context); 13 | } 14 | 15 | protected T visitHead(final Head node, final C context) { 16 | return defaultReturn(); 17 | } 18 | 19 | protected T visitTableRowGenerator(final TableRowGenerator node, final C context) { 20 | return defaultReturn(); 21 | } 22 | 23 | protected T visitCheckExpression(final CheckQualifier node, final C context) { 24 | node.getExpr().acceptVisitor(this, context); 25 | return defaultReturn(); 26 | } 27 | 28 | protected T visitListComprehension(final ListComprehension node, final C context) { 29 | node.getHead().acceptVisitor(this, context); 30 | for (final Qualifier qualifier: node.getQualifiers()) { 31 | qualifier.acceptVisitor(this, context); 32 | } 33 | return defaultReturn(); 34 | } 35 | 36 | protected T visitBinaryOperatorPredicate(final BinaryOperatorPredicate node, final C context) { 37 | node.getLeft().acceptVisitor(this, context); 38 | node.getRight().acceptVisitor(this, context); 39 | return defaultReturn(); 40 | } 41 | 42 | protected T visitGroupByComprehension(final GroupByComprehension node, final C context) { 43 | node.getComprehension().acceptVisitor(this, context); 44 | node.getGroupByQualifier().acceptVisitor(this, context); 45 | return defaultReturn(); 46 | } 47 | 48 | protected T visitGroupByQualifier(final GroupByQualifier node, final C context) { 49 | return defaultReturn(); 50 | } 51 | 52 | protected T visitLiteral(final Literal node, final C context) { 53 | return defaultReturn(); 54 | } 55 | 56 | protected T visitFunctionCall(final FunctionCall node, final C context) { 57 | for (final Expr expr: node.getArgument()) { 58 | expr.acceptVisitor(this, context); 59 | } 60 | return defaultReturn(); 61 | } 62 | 63 | protected T visitUnaryOperator(final UnaryOperator node, final C context) { 64 | node.getArgument().acceptVisitor(this, context); 65 | return defaultReturn(); 66 | } 67 | 68 | protected T visitQualifier(final Qualifier node, final C context) { 69 | return defaultReturn(); 70 | } 71 | 72 | protected T visitColumnIdentifier(final ColumnIdentifier node, final C context) { 73 | return defaultReturn(); 74 | } 75 | 76 | protected T visitExistsPredicate(final ExistsPredicate node, final C context) { 77 | node.getArgument().acceptVisitor(this, context); 78 | return defaultReturn(); 79 | } 80 | 81 | protected T visitIsNullPredicate(final IsNullPredicate node, final C context) { 82 | node.getArgument().acceptVisitor(this, context); 83 | return defaultReturn(); 84 | } 85 | 86 | protected T visitIsNotNullPredicate(final IsNotNullPredicate node, final C context) { 87 | node.getArgument().acceptVisitor(this, context); 88 | return defaultReturn(); 89 | } 90 | 91 | protected T defaultReturn() { 92 | throw new UnsupportedOperationException(); 93 | } 94 | } -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/IsNotNullPredicate.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler.ir; 8 | 9 | public class IsNotNullPredicate extends Expr { 10 | private final Expr argument; 11 | 12 | public IsNotNullPredicate(final Expr argument) { 13 | this.argument = argument; 14 | } 15 | 16 | public Expr getArgument() { 17 | return argument; 18 | } 19 | 20 | @Override 21 | public String toString() { 22 | return "IsNotNull{" + 23 | "argument=" + argument + 24 | '}'; 25 | } 26 | 27 | @Override 28 | T acceptVisitor(final IRVisitor visitor, final C context) { 29 | return visitor.visitIsNotNullPredicate(this, context); 30 | } 31 | } -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/IsNullPredicate.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler.ir; 8 | 9 | public class IsNullPredicate extends Expr { 10 | private final Expr argument; 11 | 12 | public IsNullPredicate(final Expr argument) { 13 | this.argument = argument; 14 | } 15 | 16 | public Expr getArgument() { 17 | return argument; 18 | } 19 | 20 | @Override 21 | public String toString() { 22 | return "IsNull{" + 23 | "argument=" + argument + 24 | '}'; 25 | } 26 | 27 | @Override 28 | T acceptVisitor(final IRVisitor visitor, final C context) { 29 | return visitor.visitIsNullPredicate(this, context); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/JoinPredicate.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler.ir; 8 | 9 | public class JoinPredicate extends BinaryOperatorPredicate { 10 | public JoinPredicate(final BinaryOperatorPredicate predicate) { 11 | super(predicate.getOperator(), predicate.getLeft(), predicate.getRight()); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/ListComprehension.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler.ir; 8 | 9 | import javax.annotation.Nullable; 10 | import java.util.ArrayList; 11 | import java.util.Collections; 12 | import java.util.List; 13 | import java.util.Objects; 14 | 15 | public class ListComprehension extends Expr { 16 | @Nullable private final Head head; 17 | private final List qualifiers; 18 | 19 | public ListComprehension(final Head head) { 20 | this.head = head; 21 | this.qualifiers = new ArrayList<>(); 22 | } 23 | 24 | public ListComprehension(final Head head, final List qualifiers) { 25 | this.head = head; 26 | this.qualifiers = qualifiers; 27 | } 28 | 29 | public ListComprehension() { 30 | this.head = null; 31 | this.qualifiers = Collections.emptyList(); 32 | } 33 | 34 | public ListComprehension withQualifier(final Qualifier qualifier) { 35 | final List newQualifiers = new ArrayList<>(qualifiers); 36 | newQualifiers.add(qualifier); 37 | return new ListComprehension(Objects.requireNonNull(head), newQualifiers); 38 | } 39 | 40 | public Head getHead() { 41 | return Objects.requireNonNull(head); 42 | } 43 | 44 | public List getQualifiers() { 45 | return qualifiers; 46 | } 47 | 48 | @Override 49 | public String toString() { 50 | return String.format("[%s | %s]", head, qualifiers); 51 | } 52 | 53 | @Override 54 | T acceptVisitor(final IRVisitor visitor, final C context) { 55 | return visitor.visitListComprehension(this, context); 56 | } 57 | } -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/Literal.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler.ir; 8 | 9 | public class Literal extends Expr { 10 | private final T value; 11 | 12 | public Literal(final T value, final Class type) { 13 | this.value = value; 14 | } 15 | 16 | public T getValue() { 17 | return value; 18 | } 19 | 20 | @Override 21 | public String toString() { 22 | return "Literal{" + 23 | "value='" + value + '\'' + 24 | '}'; 25 | } 26 | 27 | @Override 28 | T1 acceptVisitor(final IRVisitor visitor, final C context) { 29 | return visitor.visitLiteral(this, context); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/Qualifier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler.ir; 8 | 9 | public abstract class Qualifier extends Expr { 10 | } 11 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/SimpleVisitor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm.compiler.ir; 7 | 8 | public class SimpleVisitor extends IRVisitor { 9 | 10 | @Override 11 | public VoidType visit(final Expr expr, final VoidType context) { 12 | return super.visit(expr, context); 13 | } 14 | 15 | public VoidType visit(final Expr expr) { 16 | return visit(expr, defaultReturn()); 17 | } 18 | 19 | @Override 20 | protected final VoidType defaultReturn() { 21 | return VoidType.getAbsent(); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/TableRowGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler.ir; 8 | 9 | import com.vmware.dcm.compiler.IRColumn; 10 | import com.vmware.dcm.compiler.IRPrimaryKey; 11 | import com.vmware.dcm.compiler.IRTable; 12 | 13 | import java.util.Optional; 14 | 15 | public final class TableRowGenerator extends Qualifier { 16 | private final IRTable table; 17 | 18 | public TableRowGenerator(final IRTable table) { 19 | this.table = table; 20 | } 21 | 22 | public IRTable getTable() { 23 | return table; 24 | } 25 | 26 | public Optional getUniquePrimaryKeyColumn() { 27 | return table.getPrimaryKey().map(IRPrimaryKey::getPrimaryKeyFields) 28 | .filter(l -> l.size() == 1) 29 | .map(l -> l.get(0)); 30 | } 31 | 32 | @Override 33 | public String toString() { 34 | return "TableRowGenerator{" + 35 | "table=" + table.getName() + 36 | '}'; 37 | } 38 | 39 | @Override 40 | T acceptVisitor(final IRVisitor visitor, final C context) { 41 | return visitor.visitTableRowGenerator(this, context); 42 | } 43 | } -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/UnaryOperator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.compiler.ir; 8 | 9 | public class UnaryOperator extends Expr { 10 | private final Operator operator; 11 | private final Expr argument; 12 | 13 | public UnaryOperator(final Operator operator, final Expr argument) { 14 | this.operator = operator; 15 | this.argument = argument; 16 | } 17 | 18 | public Expr getArgument() { 19 | return argument; 20 | } 21 | 22 | public Operator getOperator() { 23 | return operator; 24 | } 25 | 26 | @Override 27 | public String toString() { 28 | return "UnaryOperator{" + 29 | "operator='" + operator + '\'' + 30 | ", argument=" + argument + 31 | '}'; 32 | } 33 | 34 | @Override 35 | T acceptVisitor(final IRVisitor visitor, final C context) { 36 | return visitor.visitUnaryOperator(this, context); 37 | } 38 | 39 | public enum Operator { 40 | NOT, 41 | PLUS, 42 | MINUS 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/VoidType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm.compiler.ir; 7 | 8 | public final class VoidType { 9 | private static final VoidType ABSENT = new VoidType(); 10 | 11 | private VoidType() { 12 | // This is empty to prevent initialization 13 | } 14 | 15 | public static VoidType getAbsent() { 16 | return ABSENT; 17 | } 18 | } -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/ir/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | /** 8 | * This informs FindBugs to mark parameters, fields and methods as NonNull. 9 | */ 10 | 11 | @ParametersAreNonnullByDefault 12 | @FieldsAreNonnullByDefault 13 | @MethodsAreNonnullByDefault 14 | 15 | package com.vmware.dcm.compiler.ir; 16 | 17 | import com.vmware.dcm.annotations.FieldsAreNonnullByDefault; 18 | import com.vmware.dcm.annotations.MethodsAreNonnullByDefault; 19 | 20 | import javax.annotation.ParametersAreNonnullByDefault; 21 | 22 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/compiler/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | /** 8 | * This informs FindBugs to mark parameters, fields and methods as NonNull. 9 | */ 10 | 11 | @ParametersAreNonnullByDefault 12 | @FieldsAreNonnullByDefault 13 | @MethodsAreNonnullByDefault 14 | 15 | package com.vmware.dcm.compiler; 16 | 17 | import com.vmware.dcm.annotations.FieldsAreNonnullByDefault; 18 | import com.vmware.dcm.annotations.MethodsAreNonnullByDefault; 19 | 20 | import javax.annotation.ParametersAreNonnullByDefault; 21 | 22 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | /** 8 | * This informs FindBugs to mark parameters, fields and methods as NonNull. 9 | */ 10 | 11 | @ParametersAreNonnullByDefault 12 | @FieldsAreNonnullByDefault 13 | @MethodsAreNonnullByDefault 14 | 15 | package com.vmware.dcm; 16 | 17 | import com.vmware.dcm.annotations.FieldsAreNonnullByDefault; 18 | import com.vmware.dcm.annotations.MethodsAreNonnullByDefault; 19 | 20 | import javax.annotation.ParametersAreNonnullByDefault; 21 | 22 | -------------------------------------------------------------------------------- /dcm/src/main/java/com/vmware/dcm/parser/SqlCreateConstraint.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2021 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm.parser; 7 | 8 | import com.google.common.collect.ImmutableList; 9 | import org.apache.calcite.sql.SqlCreate; 10 | import org.apache.calcite.sql.SqlIdentifier; 11 | import org.apache.calcite.sql.SqlKind; 12 | import org.apache.calcite.sql.SqlNode; 13 | import org.apache.calcite.sql.SqlOperator; 14 | import org.apache.calcite.sql.SqlSelect; 15 | import org.apache.calcite.sql.SqlSpecialOperator; 16 | import org.apache.calcite.sql.SqlWriter; 17 | import org.apache.calcite.sql.parser.SqlParserPos; 18 | 19 | import java.util.List; 20 | import java.util.Optional; 21 | 22 | public class SqlCreateConstraint extends SqlCreate { 23 | public static final SqlOperator OPERATOR = 24 | new SqlSpecialOperator("CREATE CONSTRAINT", SqlKind.OTHER_DDL); 25 | private final SqlIdentifier name; 26 | private final SqlSelect query; 27 | private final SqlNode constraint; 28 | private final Type type; 29 | 30 | /** Creates a SqlCreateConstraint. */ 31 | public SqlCreateConstraint(final SqlParserPos pos, final SqlIdentifier name, final SqlNode query, 32 | final String type, final SqlNode constraint) { 33 | super(OPERATOR, pos, false, false); 34 | this.name = name; 35 | this.query = (SqlSelect) query; 36 | this.constraint = constraint; 37 | this.type = Type.valueOf(type); 38 | } 39 | 40 | @Override 41 | public SqlOperator getOperator() { 42 | return OPERATOR; 43 | } 44 | 45 | public SqlIdentifier getName() { 46 | return name; 47 | } 48 | 49 | public SqlSelect getQuery() { 50 | return query; 51 | } 52 | 53 | public Optional getConstraint() { 54 | return Optional.ofNullable(constraint); 55 | } 56 | 57 | public Type getType() { 58 | return type; 59 | } 60 | 61 | @Override 62 | public List getOperandList() { 63 | return constraint == null ? ImmutableList.of(name, query) : ImmutableList.of(name, query, constraint); 64 | } 65 | 66 | @Override 67 | public String toString() { 68 | return "SqlCreateConstraint{" + 69 | "name=" + name + 70 | '}'; 71 | } 72 | 73 | @Override 74 | public void unparse(final SqlWriter writer, final int leftPrec, final int rightPrec) { 75 | writer.keyword("CREATE"); 76 | writer.keyword("CONSTRAINT"); 77 | getName().unparse(writer, leftPrec, rightPrec); 78 | getQuery().unparse(writer, leftPrec, rightPrec); 79 | getConstraint().ifPresent(e -> e.unparse(writer, leftPrec, rightPrec)); 80 | super.unparse(writer, leftPrec, rightPrec); 81 | } 82 | 83 | public enum Type { 84 | HARD_CONSTRAINT, 85 | OBJECTIVE, 86 | INTERMEDIATE_VIEW 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /dcm/src/main/resources/mnz_data.ftl: -------------------------------------------------------------------------------- 1 | %%%%%%%%%%%%%%% 2 | % TABLES % 3 | %%%%%%%%%%%%%%% 4 | STRING_LITERALS = {${string_literals?join(", ")}}; 5 | 6 | <#list input_parameters as input> 7 | ${input} 8 | -------------------------------------------------------------------------------- /dcm/src/main/resources/mnz_model.ftl: -------------------------------------------------------------------------------- 1 | include "globals.mzn"; 2 | 3 | %%%%%%%%%%%%%%% 4 | % HELPERS % 5 | %%%%%%%%%%%%%%% 6 | 7 | <#-- 8 | Generic comprehension that given a string and an array of strings, it returns the index of string in that array 9 | This is used by the convenience INDEX functions from the string-to-int mapping 10 | --> 11 | % returns the index of a string array 12 | % https://stackoverflow.com/a/44846993 13 | int: index_of(string: str, array[int] of string: strarr) = 14 | sum([ if str = strarr[i] then i else 0 endif | i in index_set(strarr) ]); 15 | 16 | % returns the index of a value in an array 17 | var int: index_of(var int: v, array[int] of int: arr) = 18 | sum([ if v = arr[i] then i else 0 endif | i in index_set(arr) ]); 19 | 20 | predicate member(array[int] of var opt int: x, var int: y) = exists(i in index_set(x)) ( x[i] == y ); 21 | 22 | predicate all_equal(array[int] of var opt int: x) = forall(i, j in index_set(x) where i < j) ( x[i] = x[j] ); 23 | 24 | predicate exists_custom(array[int] of var opt int: col1) = sum([1 | i in col1]) > 0; 25 | predicate exists_custom(array[int] of var int: col1) = length(col1) > 0; 26 | enum STRING_LITERALS; 27 | 28 | %%%%%%%%%%%%%%% 29 | % TABLES % 30 | %%%%%%%%%%%%%%% 31 | 32 | <#list arrayDeclarations as declaration> 33 | ${declaration} 34 | 35 | 36 | 37 | 38 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%% 39 | % NON-CONSTRAINT VIEWS % 40 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%% 41 | 42 | <#list nonConstraintViewCode as declaration> 43 | ${declaration} 44 | 45 | 46 | 47 | %%%%%%%%%%%%%%% 48 | % CONSTRAINTS % 49 | %%%%%%%%%%%%%%% 50 | 51 | <#list constraintViewCode as declaration> 52 | ${declaration} 53 | 54 | 55 | 56 | %%%%%%%%%%%%%%% 57 | % OBJECTIVES % 58 | %%%%%%%%%%%%%%% 59 | 60 | <#list objectiveFunctionsCode as declaration> 61 | ${declaration} 62 | -------------------------------------------------------------------------------- /dcm/src/test/java/com/vmware/dcm/ParserTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2021 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm; 7 | 8 | import com.vmware.dcm.generated.parser.DcmSqlParserImpl; 9 | import com.vmware.dcm.parser.SqlCreateConstraint; 10 | import org.apache.calcite.sql.SqlNode; 11 | import org.apache.calcite.sql.parser.SqlParseException; 12 | import org.apache.calcite.sql.parser.SqlParser; 13 | import org.junit.jupiter.api.Test; 14 | 15 | import static org.apache.calcite.sql.SqlKind.EQUALS; 16 | import static org.apache.calcite.sql.SqlKind.OTHER_DDL; 17 | import static org.apache.calcite.sql.SqlKind.SELECT; 18 | import static org.junit.jupiter.api.Assertions.assertEquals; 19 | 20 | public class ParserTest { 21 | 22 | @Test 23 | public void testMaximize() throws SqlParseException { 24 | final SqlParser.Config config = SqlParser.config().withParserFactory(DcmSqlParserImpl.FACTORY); 25 | final SqlParser parser = 26 | SqlParser.create("create constraint xyz as select * from t1 maximize t1.c1 = 10", config); 27 | final SqlNode sqlNode = parser.parseStmt(); 28 | final SqlCreateConstraint constraint = (SqlCreateConstraint) sqlNode; 29 | assertEquals("XYZ", constraint.getName().getSimple()); 30 | assertEquals(OTHER_DDL, constraint.getOperator().getKind()); 31 | assertEquals(SELECT, constraint.getQuery().getKind()); 32 | assertEquals(EQUALS, constraint.getConstraint().get().getKind()); 33 | assertEquals(SqlCreateConstraint.Type.OBJECTIVE, constraint.getType()); 34 | } 35 | 36 | @Test 37 | public void testCheck() throws SqlParseException { 38 | final SqlParser.Config config = SqlParser.config().withParserFactory(DcmSqlParserImpl.FACTORY); 39 | final SqlParser parser = 40 | SqlParser.create("create constraint xyz as select * from t1 check t1.c1 = 10", config); 41 | final SqlNode sqlNode = parser.parseStmt(); 42 | final SqlCreateConstraint constraint = (SqlCreateConstraint) sqlNode; 43 | assertEquals("XYZ", constraint.getName().getSimple()); 44 | assertEquals(OTHER_DDL, constraint.getOperator().getKind()); 45 | assertEquals(SELECT, constraint.getQuery().getKind()); 46 | assertEquals(EQUALS, constraint.getConstraint().get().getKind()); 47 | assertEquals(SqlCreateConstraint.Type.HARD_CONSTRAINT, constraint.getType()); 48 | } 49 | 50 | @Test 51 | public void testIntermediateView() throws SqlParseException { 52 | final SqlParser.Config config = SqlParser.config().withParserFactory(DcmSqlParserImpl.FACTORY); 53 | final SqlParser parser = 54 | SqlParser.create("create constraint xyz as select * from t1", config); 55 | final SqlNode sqlNode = parser.parseStmt(); 56 | final SqlCreateConstraint constraint = (SqlCreateConstraint) sqlNode; 57 | assertEquals("XYZ", constraint.getName().getSimple()); 58 | assertEquals(OTHER_DDL, constraint.getOperator().getKind()); 59 | assertEquals(SELECT, constraint.getQuery().getKind()); 60 | assertEquals(SqlCreateConstraint.Type.INTERMEDIATE_VIEW, constraint.getType()); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /dcm/src/test/java/com/vmware/dcm/ScalarProductOptimizationTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2021 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm; 7 | 8 | import com.vmware.dcm.backend.ortools.OrToolsSolver; 9 | import org.jooq.DSLContext; 10 | import org.jooq.Record; 11 | import org.jooq.Result; 12 | import org.jooq.impl.DSL; 13 | import org.junit.jupiter.params.ParameterizedTest; 14 | import org.junit.jupiter.params.provider.Arguments; 15 | import org.junit.jupiter.params.provider.MethodSource; 16 | 17 | import java.util.List; 18 | import java.util.stream.Stream; 19 | 20 | import static org.junit.jupiter.api.Assertions.assertEquals; 21 | import static org.junit.jupiter.api.Assertions.assertTrue; 22 | 23 | public class ScalarProductOptimizationTest { 24 | 25 | @ParameterizedTest 26 | @MethodSource("clauses") 27 | public void leftVarRightConst(final String checkClause, final int total) { 28 | final DSLContext conn = DSL.using("jdbc:h2:mem:"); 29 | conn.execute("CREATE TABLE t1 (c1 integer, controllable__c2 integer)"); 30 | conn.execute("insert into t1 values(1, 1)"); 31 | conn.execute("insert into t1 values(1, 1)"); 32 | conn.execute("insert into t1 values(1, 1)"); 33 | 34 | final List constraints = List.of("CREATE CONSTRAINT c1 AS " + 35 | "SELECT * FROM t1 " + 36 | "CHECK " + checkClause); 37 | final OrToolsSolver solver = new OrToolsSolver.Builder().setTryScalarProductEncoding(true).build(); 38 | final Model model = Model.build(conn, solver, constraints); 39 | final Result result = model.solve("T1"); 40 | final int actualTotal = result.stream().map(r -> r.get("C1", Integer.class) 41 | * r.get("CONTROLLABLE__C2", Integer.class)) 42 | .reduce(Integer::sum).get(); 43 | assertEquals(total, actualTotal); 44 | assertTrue(model.compilationOutput().stream().anyMatch(e -> e.contains("o.scalProd"))); 45 | } 46 | 47 | private static Stream clauses() { 48 | return Stream.of(Arguments.of("sum(controllable__c2 * c1) = 10", 10), 49 | Arguments.of("sum(c1 * controllable__c2) = 10", 10), 50 | Arguments.of("sum(c1 * controllable__c2 * 5) = 15", 3), 51 | Arguments.of("sum(5 * c1 * controllable__c2) = 15", 3)); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /dcm/src/test/java/com/vmware/dcm/compiler/SyntaxCheckTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2021 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm.compiler; 7 | 8 | import com.vmware.dcm.Model; 9 | import com.vmware.dcm.ModelException; 10 | import org.jooq.DSLContext; 11 | import org.jooq.impl.DSL; 12 | import org.junit.jupiter.api.Test; 13 | 14 | import java.util.List; 15 | 16 | import static org.junit.jupiter.api.Assertions.assertTrue; 17 | import static org.junit.jupiter.api.Assertions.fail; 18 | 19 | public class SyntaxCheckTest { 20 | 21 | @Test 22 | public void testUnsupportedCheckSyntax() { 23 | final DSLContext conn = DSL.using("jdbc:h2:mem:"); 24 | conn.execute("create table t1(c1 varchar(100), controllable__dummy integer)"); 25 | final List views = List.of("CREATE CONSTRAINT constraint_with_like AS " + 26 | "SELECT * FROM t1 " + 27 | "CHECK c1 LIKE 'node'"); 28 | try { 29 | Model.build(conn, views); 30 | fail(); 31 | } catch (final ModelException err) { 32 | assertTrue(err.getMessage().contains("Unexpected AST type `C1` LIKE 'node'")); 33 | } 34 | } 35 | 36 | @Test 37 | public void testUnsupportedJoinSyntax() { 38 | final DSLContext conn = DSL.using("jdbc:h2:mem:"); 39 | conn.execute("create table t1(c1 varchar(100), controllable__dummy integer)"); 40 | conn.execute("create table t2(c2 varchar(100))"); 41 | 42 | final List views = List.of("CREATE CONSTRAINT constraint_with_left_join AS\n" + 43 | "SELECT * FROM t1 LEFT JOIN t2 on c1 = c2 " + 44 | "CHECK c1 = 'node'"); 45 | try { 46 | Model.build(conn, views); 47 | fail(); 48 | } catch (final ModelException err) { 49 | assertTrue(err.getMessage().contains("Unexpected AST type T1 LEFT JOIN T2 ON (`C1` = `C2`)")); 50 | } 51 | } 52 | 53 | @Test 54 | public void testUnsupportedAggregate() { 55 | final DSLContext conn = DSL.using("jdbc:h2:mem:"); 56 | conn.execute("create table t1(c1 varchar(100), controllable__dummy integer)"); 57 | final List views = List.of("CREATE CONSTRAINT constraint_with_like AS " + 58 | "SELECT * FROM t1 " + 59 | "CHECK XYZ(c1) = true"); 60 | try { 61 | Model.build(conn, views); 62 | fail(); 63 | } catch (final ModelException err) { 64 | assertTrue(err.getMessage().contains("Unexpected AST type `XYZ`(`C1`)")); 65 | } 66 | } 67 | 68 | @Test 69 | public void testDuplicateViewName() { 70 | final DSLContext conn = DSL.using("jdbc:h2:mem:"); 71 | conn.execute("create table t1(c1 varchar(100), controllable__dummy integer)"); 72 | final List views = List.of("CREATE CONSTRAINT some_constraint AS " + 73 | "SELECT * FROM t1 " + 74 | "CHECK controllable__dummy = 1", 75 | "CREATE CONSTRAINT some_constraint AS " + 76 | "SELECT * FROM t1 " + 77 | "CHECK controllable__dummy = 2"); 78 | try { 79 | Model.build(conn, views); 80 | fail(); 81 | } catch (final ModelException err) { 82 | assertTrue(err.getMessage().contains("Duplicate name SOME_CONSTRAINT")); 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /dcm/src/test/resources/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware-archive/declarative-cluster-management/113e4656697de57d5665e44b5ab3c10bd38c0712/dcm/src/test/resources/.gitignore -------------------------------------------------------------------------------- /docs/arch_detailed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware-archive/declarative-cluster-management/113e4656697de57d5665e44b5ab3c10bd38c0712/docs/arch_detailed.png -------------------------------------------------------------------------------- /examples/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | */ 4 | 5 | dependencies { 6 | implementation project(':dcm') 7 | testImplementation "com.h2database:h2:${h2Version}" 8 | } 9 | 10 | description = 'examples' 11 | -------------------------------------------------------------------------------- /examples/src/main/java/com/vmware/dcm/examples/LoadBalance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm.examples; 8 | 9 | import com.google.common.base.Splitter; 10 | import com.vmware.dcm.Model; 11 | import org.jooq.DSLContext; 12 | import org.jooq.Record; 13 | import org.jooq.Result; 14 | 15 | import java.io.BufferedReader; 16 | import java.io.IOException; 17 | import java.io.InputStream; 18 | import java.io.InputStreamReader; 19 | import java.nio.charset.StandardCharsets; 20 | import java.util.List; 21 | import java.util.stream.Collectors; 22 | 23 | import static org.jooq.impl.DSL.table; 24 | import static org.jooq.impl.DSL.using; 25 | 26 | 27 | /** 28 | * A simple class to highlight DCM's capabilities. Loading this class creates a database schema 29 | * that according to resources/schema.sql. It creates two tables, one for physical machines, and 30 | * one for virtual machines. A variable column in the virtual machines table tracks the assignment 31 | * of each virtual machine to a physical machine. 32 | * 33 | * The class is currently driven by tests written in LoadBalanceTest. 34 | */ 35 | class LoadBalance { 36 | private static final String PHYSICAL_MACHINES_TABLE = "PHYSICAL_MACHINE"; 37 | private static final String VIRTUAL_MACHINES_TABLE = "VIRTUAL_MACHINE"; 38 | private final DSLContext conn; 39 | private final Model model; 40 | 41 | LoadBalance(final List constraints) { 42 | conn = setup(); 43 | model = Model.build(conn, constraints); 44 | } 45 | 46 | /** 47 | * Add a physical machine to the inventory. 48 | * 49 | * @param nodeName machine name 50 | * @param cpuCapacity CPU capacity 51 | * @param memoryCapacity Memory capacity 52 | */ 53 | void addPhysicalMachine(final String nodeName, final int cpuCapacity, final int memoryCapacity) { 54 | conn.insertInto(table(PHYSICAL_MACHINES_TABLE)) 55 | .values(nodeName, cpuCapacity, memoryCapacity).execute(); 56 | } 57 | 58 | 59 | /** 60 | * Add a virtual machine to the inventory. 61 | * 62 | * @param vmName name 63 | * @param cpu CPU demand 64 | * @param memory Memory demand 65 | */ 66 | void addVirtualMachine(final String vmName, final int cpu, final int memory) { 67 | conn.insertInto(table(VIRTUAL_MACHINES_TABLE)) 68 | .values(vmName, cpu, memory, null).execute(); 69 | System.out.println(conn.selectFrom(table(PHYSICAL_MACHINES_TABLE)).fetch()); 70 | System.out.println(conn.selectFrom(table(VIRTUAL_MACHINES_TABLE)).fetch()); 71 | } 72 | 73 | /** 74 | * Invoke to solve for a given database (physical and virtual machine tables). 75 | * 76 | * @return The new virtual machine table after the solver identifies a new placement. 77 | */ 78 | Result run() { 79 | // Pull the latest state from the DB, run the solver and return the virtual machines table with 80 | // solver-identified values for the controllable__physical_machines column 81 | return model.solve(VIRTUAL_MACHINES_TABLE); 82 | } 83 | 84 | 85 | /* 86 | * Sets up an in-memory database. 87 | */ 88 | private DSLContext setup() { 89 | try { 90 | final DSLContext using = using("jdbc:h2:mem:"); 91 | final InputStream resourceAsStream = this.getClass().getResourceAsStream("/schema.sql"); 92 | final BufferedReader reader = 93 | new BufferedReader(new InputStreamReader(resourceAsStream, StandardCharsets.UTF_8)); 94 | final String schemaAsString = reader 95 | .lines() 96 | .filter(line -> !line.startsWith("--")) // remove SQL comments 97 | .collect(Collectors.joining("\n")); 98 | final List semiColonSeparated = Splitter.on(";") 99 | .trimResults() 100 | .omitEmptyStrings() 101 | .splitToList(schemaAsString); 102 | reader.close(); 103 | semiColonSeparated.forEach(using::execute); 104 | return using; 105 | } catch (final IOException e) { 106 | throw new RuntimeException(e); 107 | } 108 | } 109 | } -------------------------------------------------------------------------------- /examples/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | # 4 | # SPDX-License-Identifier: BSD-2 5 | # 6 | 7 | log4j.rootLogger=INFO, STDOUT 8 | log4j.logger.org.jooq.Constants=OFF 9 | log4j.appender.STDOUT=org.apache.log4j.ConsoleAppender 10 | log4j.appender.STDOUT.layout=org.apache.log4j.PatternLayout 11 | log4j.appender.STDOUT.layout.ConversionPattern=%5p %d{dd-MMM-yyyy-HH:mm:ss,SSS} [%t] (%F:%L) - %m%n -------------------------------------------------------------------------------- /examples/src/main/resources/schema.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- SQL schema for load balancing example 3 | -- 4 | 5 | create table physical_machine ( 6 | name varchar(30) primary key, 7 | cpu_capacity integer, 8 | memory_capacity integer 9 | ); 10 | 11 | -- controllable__physical_machine represents a variable that the solver will assign values to 12 | create table virtual_machine ( 13 | name varchar(30) primary key not null, 14 | cpu integer not null, 15 | memory integer not null, 16 | controllable__physical_machine varchar(30), 17 | foreign key (controllable__physical_machine) references physical_machine(name) 18 | ); 19 | 20 | -- Constraints can refer to views computed in the database as well 21 | create view vm_subset as 22 | select * from virtual_machine where name ='vm1' or name = 'vm2'; -------------------------------------------------------------------------------- /examples/src/test/java/com/vmware/dcm/examples/QuickStartTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018-2020 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm.examples; 7 | 8 | import com.vmware.dcm.Model; 9 | import org.jooq.DSLContext; 10 | import org.jooq.impl.DSL; 11 | import org.junit.jupiter.api.Test; 12 | 13 | import java.util.List; 14 | 15 | import static org.junit.jupiter.api.Assertions.assertEquals; 16 | import static org.junit.jupiter.api.Assertions.assertTrue; 17 | 18 | public class QuickStartTest { 19 | 20 | @Test 21 | public void quickStart() { 22 | // Create an in-memory database and get a JOOQ connection to it 23 | final DSLContext conn = DSL.using("jdbc:h2:mem:"); 24 | 25 | // A table representing some machines 26 | conn.execute("create table machines(id integer)"); 27 | 28 | // A table representing tasks, that need to be assigned to machines by DCM. 29 | // To do so, create a variable column (prefixed by controllable__). 30 | conn.execute("create table tasks(task_id integer, controllable__worker_id integer, " + 31 | "foreign key (controllable__worker_id) references machines(id))"); 32 | 33 | // Add four machines 34 | conn.execute("insert into machines values(1)"); 35 | conn.execute("insert into machines values(3)"); 36 | conn.execute("insert into machines values(5)"); 37 | conn.execute("insert into machines values(8)"); 38 | 39 | // Add two tasks 40 | conn.execute("insert into tasks values(1, null)"); 41 | conn.execute("insert into tasks values(2, null)"); 42 | 43 | // Time to specify a constraint! Just for fun, let's assign tasks to machines such that 44 | // the machine IDs sum up to 6. 45 | final String constraint = "create constraint example_constraint as " + 46 | "select * from tasks check sum(controllable__worker_id) = 6"; 47 | 48 | // Create a DCM model using the database connection and the above constraint 49 | final Model model = Model.build(conn, List.of(constraint)); 50 | 51 | // Solve and return the tasks table. The controllable__worker_id column will either be [1, 5] or [5, 1] 52 | final List column = model.solve("TASKS") 53 | .map(e -> e.get("CONTROLLABLE__WORKER_ID", Integer.class)); 54 | assertEquals(2, column.size()); 55 | assertTrue(column.contains(1)); 56 | assertTrue(column.contains(5)); 57 | } 58 | } -------------------------------------------------------------------------------- /examples/src/test/java/com/vmware/dcm/examples/SolverConfigurationTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018-2021 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm.examples; 7 | 8 | import com.vmware.dcm.Model; 9 | import com.vmware.dcm.backend.ortools.OrToolsSolver; 10 | import org.jooq.DSLContext; 11 | import org.junit.jupiter.api.Test; 12 | 13 | import java.util.List; 14 | 15 | import static org.jooq.impl.DSL.using; 16 | import static org.junit.jupiter.api.Assertions.assertNotNull; 17 | 18 | public class SolverConfigurationTest { 19 | @Test 20 | public void backendInitializationExample() { 21 | final DSLContext conn = using("jdbc:h2:mem:"); 22 | conn.execute("CREATE TABLE t1(controllable__c1 integer)"); 23 | conn.execute("insert into t1 values(null)"); 24 | final OrToolsSolver orToolsSolver = new OrToolsSolver.Builder() 25 | .setNumThreads(1) 26 | .setPrintDiagnostics(true) 27 | .setMaxTimeInSeconds(5).build(); 28 | final Model build = Model.build(conn, orToolsSolver, 29 | List.of("CREATE CONSTRAINT C1 AS SELECT * FROM t1 CHECK controllable__c1 = 10")); 30 | assertNotNull(build.solve("T1")); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | dcmLibrarySourceLevel = 11 2 | dcmGroupId = com.vmware.dcm 3 | dcmArtifactId = dcm 4 | dcmVersion = 0.16.0-SNAPSHOT 5 | 6 | ## Dependency versions 7 | 8 | # Common libraries 9 | log4jVersion = 2.17.1 10 | jooqVersion = 3.13.1 11 | guavaVersion = 29.0-jre 12 | junitJupiterVersion = 5.7.2 13 | junitPioneerVersion = 1.4.2 14 | errorPronePluginVersion = 2.0.2 15 | spotbugsPluginVersion = 4.7.2 16 | coverallsPluginVersion = 2.10.1 17 | spotbugsVersion = 4.4.1 18 | errorproneVersion = 2.7.0 19 | checkstyleVersion = 8.36.1 20 | 21 | # DCM sub-project 22 | calciteCoreVersion = 1.30.0 23 | fmppMavenPluginVersion = 1.0 24 | javaCCVersion = 4.0 25 | pitestPluginVersion = 1.6.0 26 | caCoglincJavaccPluginVersion = 2.4.0 27 | freemarkerVersion = 2.3.31 28 | commonsIoVersion = 2.4 29 | prestoParserVersion = 0.238.2 30 | javapoetVersion = 1.11.1 31 | orToolsVersion = 9.4.1874 32 | protobufVersion = 3.15.0 33 | h2Version = 1.4.200 34 | 35 | # k8s-scheduler sub-project 36 | jooqPluginVersion = 4.2 37 | gitPropertiesVersion = 2.3.1 38 | hikariCPVersion = 3.4.2 39 | fabricK8sClientVersion = 5.5.0 40 | metricsVersion = 4.1.0-rc3 41 | commonsCliVersion = 1.4 42 | 43 | # benchmark sub-project 44 | shadowJarPluginVersion = 6.0.0 45 | jmhVersion = 1.21 46 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vmware-archive/declarative-cluster-management/113e4656697de57d5665e44b5ab3c10bd38c0712/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /k8s-scheduler/README.md: -------------------------------------------------------------------------------- 1 | An experimental Kubernetes scheduler implemented using DCM. 2 | -------------------------------------------------------------------------------- /k8s-scheduler/build.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | */ 4 | 5 | plugins { 6 | id 'nu.studer.jooq' version "${jooqPluginVersion}" 7 | id "com.gorylenko.gradle-git-properties" version "${gitPropertiesVersion}" 8 | } 9 | 10 | 11 | apply plugin: 'java' 12 | apply plugin: 'nu.studer.jooq' 13 | apply plugin: "application" 14 | 15 | dependencies { 16 | implementation ("org.apache.calcite:calcite-core:${calciteCoreVersion}") { 17 | exclude group: "org.apache.httpcomponents" 18 | exclude group: "org.apache.logging" 19 | exclude group: "org.yaml" 20 | exclude group: "com.fasterxml.jackson.core" 21 | exclude group: "net.minidev" 22 | exclude group: "com.yahoo.datasketches" 23 | exclude group: "org.codehaus.janino" 24 | exclude group: "com.google.guava" 25 | exclude group: "com.google.protobuf" 26 | } 27 | implementation project(':dcm') 28 | implementation "com.zaxxer:HikariCP:${hikariCPVersion}" 29 | implementation "io.fabric8:kubernetes-client:${fabricK8sClientVersion}" 30 | implementation "io.fabric8:kubernetes-server-mock:${fabricK8sClientVersion}" 31 | implementation "io.dropwizard.metrics:metrics-core:${metricsVersion}" 32 | implementation "commons-cli:commons-cli:${commonsCliVersion}" 33 | implementation "commons-io:commons-io:${commonsIoVersion}" 34 | implementation "com.h2database:h2:${h2Version}" 35 | jooqRuntime "org.jooq:jooq-meta-extensions:${jooqVersion}" 36 | } 37 | 38 | gitProperties { 39 | gitPropertiesName = "git.properties" 40 | } 41 | 42 | java { 43 | toolchain { 44 | languageVersion = JavaLanguageVersion.of(16) 45 | } 46 | } 47 | 48 | jooq { 49 | generateSchemaSourceOnCompilation = true 50 | sample(sourceSets.main) { 51 | generator { 52 | database { 53 | name = "org.jooq.meta.extensions.ddl.DDLDatabase" 54 | properties { 55 | property { 56 | key = 'scripts' 57 | value = 'src/main/resources/scheduler_tables.sql' 58 | } 59 | } 60 | } 61 | target { 62 | packageName = 'com.vmware.dcm.k8s.generated' 63 | directory = 'build/generated-sources/jooq' 64 | } 65 | } 66 | } 67 | } 68 | 69 | def test = tasks.named("test") { 70 | useJUnitPlatform { 71 | excludeTags "integration-test", "experiment" 72 | } 73 | } 74 | 75 | def integrationTest = tasks.register("integrationTest2", Test) { 76 | useJUnitPlatform { 77 | includeTags "integration-test" 78 | } 79 | shouldRunAfter test 80 | } 81 | 82 | def experiment = tasks.register("experiment", Test) { 83 | logging.captureStandardOutput LogLevel.INFO 84 | systemProperty "schedulerName", System.getProperty("schedulerName") 85 | systemProperty "cpuScaleDown", System.getProperty("cpuScaleDown") 86 | systemProperty "memScaleDown", System.getProperty("memScaleDown") 87 | systemProperty "timeScaleDown", System.getProperty("timeScaleDown") 88 | systemProperty "startTimeCutOff", System.getProperty("startTimeCutOff") 89 | systemProperty "affinityProportion", System.getProperty("affinityProportion") 90 | useJUnitPlatform { 91 | includeTags "experiment" 92 | } 93 | shouldRunAfter test 94 | } 95 | 96 | 97 | application { 98 | mainClass = 'com.vmware.dcm.Scheduler' 99 | } 100 | 101 | task(runBenchmark, dependsOn: 'classes', type: JavaExec) { 102 | main = 'com.vmware.dcm.EmulatedCluster' 103 | classpath = sourceSets.main.runtimeClasspath 104 | } -------------------------------------------------------------------------------- /k8s-scheduler/src/main/java/com/vmware/dcm/DBConnectionPool.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm; 8 | 9 | import com.google.common.annotations.VisibleForTesting; 10 | import com.zaxxer.hikari.HikariConfig; 11 | import com.zaxxer.hikari.HikariDataSource; 12 | import org.jooq.DSLContext; 13 | import org.jooq.SQLDialect; 14 | import org.jooq.conf.Settings; 15 | 16 | import javax.sql.DataSource; 17 | import java.util.UUID; 18 | 19 | import static org.jooq.impl.DSL.using; 20 | 21 | class DBConnectionPool { 22 | private static final Settings JOOQ_SETTING = new Settings().withExecuteLogging(false); 23 | private final String databaseName; 24 | private final DataSource ds; 25 | 26 | DBConnectionPool() { 27 | this.databaseName = UUID.randomUUID().toString(); 28 | final HikariConfig config = new HikariConfig(); 29 | config.setJdbcUrl(String.format("jdbc:h2:mem:%s;LOG=0;UNDO_LOG=0", databaseName)); 30 | config.addDataSourceProperty("foreign_keys", "true"); 31 | config.addDataSourceProperty("cachePrepStmts", "true"); 32 | config.addDataSourceProperty("prepStmtCacheSize", "250"); 33 | config.addDataSourceProperty("prepStmtCacheSqlLimit", "2048"); 34 | config.addDataSourceProperty("maximumPoolSize", "20"); 35 | this.ds = new HikariDataSource(config); 36 | DBViews.getSchema().forEach(getConnectionToDb()::execute); 37 | } 38 | 39 | /** 40 | * Sets up a private, in-memory database. 41 | */ 42 | DSLContext getConnectionToDb() { 43 | return using(ds, SQLDialect.H2, JOOQ_SETTING); 44 | } 45 | 46 | /** 47 | * Used only for refreshing the DB state between tests 48 | */ 49 | @VisibleForTesting 50 | void refresh() { 51 | getConnectionToDb().execute("drop all objects"); 52 | DBViews.getSchema().forEach(getConnectionToDb()::execute); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /k8s-scheduler/src/main/java/com/vmware/dcm/DebugUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm; 8 | 9 | import com.vmware.dcm.k8s.generated.Tables; 10 | import org.apache.commons.io.FileUtils; 11 | import org.jooq.DSLContext; 12 | import org.jooq.impl.TableImpl; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | import java.io.File; 17 | import java.io.IOException; 18 | import java.nio.charset.StandardCharsets; 19 | import java.util.List; 20 | import java.util.UUID; 21 | 22 | 23 | /** 24 | * Utility methods to dump and reload the state of a database. We can use this to create test cases and reproduce bugs. 25 | */ 26 | class DebugUtils { 27 | private static final Logger LOG = LoggerFactory.getLogger(DebugUtils.class); 28 | private static final List> TABLES = List.of(Tables.NODE_INFO, 29 | Tables.POD_INFO, 30 | Tables.POD_PORTS_REQUEST, 31 | Tables.POD_NODE_SELECTOR_LABELS, 32 | Tables.POD_AFFINITY_MATCH_EXPRESSIONS, 33 | Tables.POD_ANTI_AFFINITY_MATCH_EXPRESSIONS, 34 | Tables.POD_LABELS, 35 | Tables.NODE_LABELS, 36 | Tables.NODE_TAINTS, 37 | Tables.POD_TOLERATIONS, 38 | Tables.NODE_IMAGES, 39 | Tables.POD_IMAGES, 40 | Tables.MATCH_EXPRESSIONS, 41 | Tables.POD_RESOURCE_DEMANDS, 42 | Tables.NODE_RESOURCES, 43 | Tables.POD_TOPOLOGY_SPREAD_CONSTRAINTS); 44 | 45 | static void dbDump(final DSLContext conn, final UUID uuid) { 46 | LOG.error("Creating DB dump with UUID: {}", uuid); 47 | for (final TableImpl table: TABLES) { 48 | try { 49 | FileUtils.writeStringToFile(new File("/tmp/" + uuid, table.getName() + ".json"), 50 | conn.selectFrom(table).fetch().formatJSON(), 51 | StandardCharsets.UTF_8); 52 | } catch (final IOException e) { 53 | LOG.error("Could not create db-dump for table {} because of exception:", table.getName(), e); 54 | } 55 | } 56 | } 57 | 58 | static void dbLoad(final DSLContext conn, final UUID uuid) { 59 | for (final TableImpl table: TABLES) { 60 | try { 61 | final String csv = FileUtils.readFileToString( 62 | new File("/tmp/" + uuid, table.getName() + ".json"), 63 | StandardCharsets.UTF_8); 64 | conn.loadInto(table).onDuplicateKeyError() 65 | .onErrorAbort() 66 | .commitAll().loadJSON(csv).fields(table.fields()) 67 | .execute(); 68 | } catch (final IOException e) { 69 | LOG.error("Could not recover db-dump for table {} because of exception:", table.getName(), e); 70 | } 71 | } 72 | } 73 | } -------------------------------------------------------------------------------- /k8s-scheduler/src/main/java/com/vmware/dcm/EmulatedPodDeployer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm; 8 | 9 | import com.google.common.base.Preconditions; 10 | import io.fabric8.kubernetes.api.model.OwnerReference; 11 | import io.fabric8.kubernetes.api.model.Pod; 12 | import io.fabric8.kubernetes.api.model.PodSpec; 13 | import io.fabric8.kubernetes.api.model.PodStatus; 14 | import org.slf4j.Logger; 15 | import org.slf4j.LoggerFactory; 16 | 17 | import java.util.ArrayList; 18 | import java.util.List; 19 | import java.util.Map; 20 | import java.util.UUID; 21 | import java.util.concurrent.ConcurrentHashMap; 22 | 23 | 24 | /** 25 | * Creates pods that correspond to a deployment without actually deploying them to a real cluster. Allows us 26 | * to replay traces locally. 27 | */ 28 | public class EmulatedPodDeployer implements IPodDeployer { 29 | private static final Logger LOG = LoggerFactory.getLogger(EmulatedPodDeployer.class); 30 | private final PodResourceEventHandler resourceEventHandler; 31 | private final String namespace; 32 | private final Map> pods = new ConcurrentHashMap<>(); 33 | 34 | EmulatedPodDeployer(final PodResourceEventHandler podResourceEventHandler, final String namespace) { 35 | this.resourceEventHandler = podResourceEventHandler; 36 | this.namespace = namespace; 37 | } 38 | 39 | @Override 40 | public Runnable startDeployment(final List deployment) { 41 | return new StartDeployment(deployment); 42 | } 43 | 44 | @Override 45 | public Runnable endDeployment(final List deployment) { 46 | return new EndDeployment(deployment); 47 | } 48 | 49 | private class StartDeployment implements Runnable { 50 | final List deployment; 51 | 52 | StartDeployment(final List dep) { 53 | this.deployment = dep; 54 | } 55 | 56 | @Override 57 | public void run() { 58 | final Pod firstPodInDeployment = deployment.get(0); 59 | final String deploymentName = firstPodInDeployment.getMetadata().getName(); 60 | LOG.info("Creating deployment (name:{}, schedulerName:{}, replicas:{}) at {}", 61 | deploymentName, firstPodInDeployment.getSpec().getSchedulerName(), 62 | deployment.size(), System.currentTimeMillis()); 63 | 64 | for (final Pod pod: deployment) { 65 | pod.getMetadata().setCreationTimestamp("" + System.currentTimeMillis()); 66 | pod.getMetadata().setNamespace(namespace); 67 | pod.getMetadata().setResourceVersion("101"); 68 | pod.getMetadata().setUid(UUID.randomUUID().toString()); 69 | final OwnerReference reference = new OwnerReference(); 70 | reference.setName(deploymentName); 71 | pod.getMetadata().setOwnerReferences(List.of(reference)); 72 | final PodSpec spec = pod.getSpec(); 73 | final PodStatus status = new PodStatus(); 74 | status.setPhase("Pending"); 75 | pod.setSpec(spec); 76 | pod.setStatus(status); 77 | pods.computeIfAbsent(deploymentName, (k) -> new ArrayList<>()).add(pod); 78 | resourceEventHandler.onAdd(pod); 79 | } 80 | } 81 | } 82 | 83 | private class EndDeployment implements Runnable { 84 | final List deployment; 85 | 86 | EndDeployment(final List dep) { 87 | this.deployment = dep; 88 | } 89 | 90 | @Override 91 | public void run() { 92 | final Pod firstPodOfDeployment = deployment.get(0); 93 | LOG.info("Terminating deployment (name:{}, schedulerName:{}, replicas:{}) at {}", 94 | firstPodOfDeployment.getMetadata().getName(), firstPodOfDeployment.getSpec().getSchedulerName(), 95 | deployment.size(), System.currentTimeMillis()); 96 | final List podsList = pods.get(firstPodOfDeployment.getMetadata().getName()); 97 | Preconditions.checkNotNull(podsList); 98 | for (final Pod pod: podsList) { 99 | resourceEventHandler.onDeleteSync(pod, false); 100 | } 101 | } 102 | } 103 | } -------------------------------------------------------------------------------- /k8s-scheduler/src/main/java/com/vmware/dcm/IPodDeployer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm; 8 | 9 | import io.fabric8.kubernetes.api.model.Pod; 10 | 11 | import java.util.List; 12 | 13 | /** 14 | * An interface used by WorkloadGeneratorIT.runTrace() to create and delete pod deployments. It allows 15 | * us to replay traces locally or against an actual Kubernetes cluster 16 | */ 17 | public interface IPodDeployer { 18 | 19 | Runnable startDeployment(final List deployment); 20 | 21 | Runnable endDeployment(final List deployment); 22 | } 23 | -------------------------------------------------------------------------------- /k8s-scheduler/src/main/java/com/vmware/dcm/IPodToNodeBinder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm; 8 | 9 | 10 | import org.jooq.Record; 11 | 12 | import java.util.List; 13 | 14 | /** 15 | * An interface used by the scheduler to bind pods to nodes, either in a real cluster or in an emulated environment 16 | */ 17 | public interface IPodToNodeBinder { 18 | 19 | void bindManyAsnc(final List records); 20 | 21 | void unbindManyAsnc(final List records); 22 | 23 | void notifyFail(final Record record); 24 | } 25 | -------------------------------------------------------------------------------- /k8s-scheduler/src/main/java/com/vmware/dcm/KubernetesPodDeployer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm; 8 | 9 | import io.fabric8.kubernetes.api.model.Pod; 10 | import io.fabric8.kubernetes.client.KubernetesClient; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import java.util.List; 15 | 16 | 17 | /** 18 | * Deploys and deletes pods against an actual Kubernetes cluster 19 | */ 20 | public class KubernetesPodDeployer implements IPodDeployer { 21 | private static final Logger LOG = LoggerFactory.getLogger(KubernetesPodDeployer.class); 22 | 23 | private final KubernetesClient fabricClient; 24 | private final String namespace; 25 | 26 | public KubernetesPodDeployer(final KubernetesClient fabricClient, final String namespace) { 27 | this.fabricClient = fabricClient; 28 | this.namespace = namespace; 29 | } 30 | 31 | @Override 32 | public Runnable startDeployment(final List deployment) { 33 | return new StartDeployment(deployment); 34 | } 35 | 36 | @Override 37 | public Runnable endDeployment(final List deployment) { 38 | return new EndDeployment(deployment); 39 | } 40 | 41 | private class StartDeployment implements Runnable { 42 | final List deployment; 43 | 44 | StartDeployment(final List dep) { 45 | this.deployment = dep; 46 | } 47 | 48 | @Override 49 | public void run() { 50 | final Pod firstPodInDeployment = deployment.get(0); 51 | LOG.info("Creating deployment (name:{}, schedulerName:{}, replicas:{}) with masterUrl {} at {}", 52 | firstPodInDeployment.getMetadata().getName(), firstPodInDeployment.getSpec().getSchedulerName(), 53 | deployment.size(), fabricClient.getConfiguration().getMasterUrl(), 54 | System.currentTimeMillis()); 55 | deployment.forEach(p -> fabricClient.pods().inNamespace(namespace).create(p)); 56 | } 57 | } 58 | 59 | private class EndDeployment implements Runnable { 60 | final List deployment; 61 | 62 | EndDeployment(final List dep) { 63 | this.deployment = dep; 64 | } 65 | 66 | @Override 67 | public void run() { 68 | final Pod firstPodInDeployment = deployment.get(0); 69 | LOG.info("Terminating deployment (name:{}, schedulerName:{}) with masterUrl {} at {}", 70 | firstPodInDeployment.getMetadata().getName(), firstPodInDeployment.getSpec().getSchedulerName(), 71 | fabricClient.getConfiguration().getMasterUrl(), System.currentTimeMillis()); 72 | fabricClient.pods().inNamespace(namespace).delete(deployment); 73 | } 74 | } 75 | } -------------------------------------------------------------------------------- /k8s-scheduler/src/main/java/com/vmware/dcm/KubernetesStateSync.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | 8 | package com.vmware.dcm; 9 | 10 | import com.google.common.util.concurrent.ThreadFactoryBuilder; 11 | import io.fabric8.kubernetes.api.model.Node; 12 | import io.fabric8.kubernetes.api.model.Pod; 13 | import io.fabric8.kubernetes.api.model.policy.v1.PodDisruptionBudget; 14 | import io.fabric8.kubernetes.client.KubernetesClient; 15 | import io.fabric8.kubernetes.client.informers.SharedIndexInformer; 16 | import io.fabric8.kubernetes.client.informers.SharedInformerFactory; 17 | import org.slf4j.Logger; 18 | import org.slf4j.LoggerFactory; 19 | 20 | import java.util.concurrent.ExecutionException; 21 | import java.util.concurrent.ExecutorService; 22 | import java.util.concurrent.Executors; 23 | import java.util.concurrent.ThreadFactory; 24 | import java.util.function.Consumer; 25 | 26 | class KubernetesStateSync { 27 | private static final Logger LOG = LoggerFactory.getLogger(KubernetesStateSync.class); 28 | private final SharedInformerFactory sharedInformerFactory; 29 | private final ThreadFactory namedThreadFactory = 30 | new ThreadFactoryBuilder().setNameFormat("flowable-thread-%d").build(); 31 | private final ExecutorService service = Executors.newFixedThreadPool(10, namedThreadFactory); 32 | 33 | KubernetesStateSync(final KubernetesClient client) { 34 | this.sharedInformerFactory = client.informers(); 35 | } 36 | 37 | void setupInformersAndPodEventStream(final DBConnectionPool dbConnectionPool, 38 | final Consumer podEventNotification) { 39 | final SharedIndexInformer nodeSharedIndexInformer = sharedInformerFactory 40 | .sharedIndexInformerFor(Node.class, 30000); 41 | nodeSharedIndexInformer.addEventHandler(new NodeResourceEventHandler(dbConnectionPool, service)); 42 | 43 | // Pod informer 44 | final SharedIndexInformer podInformer = sharedInformerFactory 45 | .sharedIndexInformerFor(Pod.class, 30000); 46 | podInformer.addEventHandler(new PodResourceEventHandler(podEventNotification, service)); 47 | 48 | // Pod disruption budget listener 49 | final SharedIndexInformer pdbInformer = sharedInformerFactory 50 | .sharedIndexInformerFor(PodDisruptionBudget.class, 30000); 51 | pdbInformer.addEventHandler(new PdbResourceEventHandler(dbConnectionPool)); 52 | 53 | LOG.info("Instantiated node and pod informers. Starting them all now."); 54 | } 55 | 56 | void startProcessingEvents() { 57 | try { 58 | sharedInformerFactory.startAllRegisteredInformers().get(); 59 | } catch (final InterruptedException | ExecutionException e) { 60 | throw new RuntimeException(e); 61 | } 62 | } 63 | 64 | void shutdown() { 65 | sharedInformerFactory.stopAllRegisteredInformers(); 66 | } 67 | } -------------------------------------------------------------------------------- /k8s-scheduler/src/main/java/com/vmware/dcm/PdbResourceEventHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018-2021 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm; 7 | 8 | 9 | import com.vmware.dcm.k8s.generated.Tables; 10 | import io.fabric8.kubernetes.api.model.policy.v1.PodDisruptionBudget; 11 | import io.fabric8.kubernetes.client.informers.ResourceEventHandler; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | /** 16 | * Listens to PodDisruptionBudget resources and updates the database accordingly 17 | */ 18 | public class PdbResourceEventHandler implements ResourceEventHandler { 19 | private static final Logger LOG = LoggerFactory.getLogger(PdbResourceEventHandler.class); 20 | private final DBConnectionPool dbConnectionPool; 21 | 22 | PdbResourceEventHandler(final DBConnectionPool dbConnectionPool) { 23 | this.dbConnectionPool = dbConnectionPool; 24 | } 25 | 26 | @Override 27 | public void onAdd(final PodDisruptionBudget pdb) { 28 | LOG.info("Adding PodDisruptionBudget {}", pdb); 29 | pdb.getSpec().getSelector().getMatchLabels().forEach( 30 | (k, v) -> 31 | dbConnectionPool.getConnectionToDb().insertInto(Tables.PDB_MATCH_EXPRESSIONS) 32 | .values(pdb.getMetadata().getName(), 33 | pdb.getSpec().getSelector(), 34 | pdb.getSpec().getMinAvailable(), 35 | pdb.getSpec().getMaxUnavailable(), 36 | pdb.getStatus().getDisruptionsAllowed()) 37 | ); 38 | LOG.info("Added PodDisruptionBudget {}", pdb); 39 | } 40 | 41 | @Override 42 | public void onUpdate(final PodDisruptionBudget pdb, final PodDisruptionBudget t1) { 43 | 44 | } 45 | 46 | @Override 47 | public void onDelete(final PodDisruptionBudget pdb, final boolean b) { 48 | LOG.info("Deleting PodDisruptionBudget {}", pdb); 49 | pdb.getSpec().getSelector().getMatchLabels().forEach( 50 | (k, v) -> 51 | dbConnectionPool.getConnectionToDb().deleteFrom(Tables.PDB_MATCH_EXPRESSIONS) 52 | .where(Tables.PDB_MATCH_EXPRESSIONS.PDB_NAME.eq(pdb.getMetadata().getName())) 53 | ); 54 | LOG.info("Deleted PodDisruptionBudget {}", pdb); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /k8s-scheduler/src/main/java/com/vmware/dcm/PodResourceEventHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm; 8 | 9 | import io.fabric8.kubernetes.api.model.Pod; 10 | import io.fabric8.kubernetes.client.informers.ResourceEventHandler; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import javax.annotation.Nullable; 15 | import java.util.concurrent.ExecutorService; 16 | import java.util.concurrent.Executors; 17 | import java.util.concurrent.TimeUnit; 18 | import java.util.function.Consumer; 19 | 20 | 21 | /** 22 | * Subscribes to Kubernetes pod events and streams them to a flowable. Notably, it does not write 23 | * it to the database unlike the NodeResourceEventHandler. We do this to have tighter control over 24 | * batching these writes to the database. 25 | */ 26 | 27 | record PodEvent(Action action, Pod pod, @Nullable Pod oldPod) { 28 | 29 | PodEvent(final Action action, final Pod pod) { 30 | this(action, pod, null); 31 | } 32 | 33 | enum Action { 34 | ADDED, 35 | UPDATED, 36 | DELETED 37 | } 38 | } 39 | 40 | class PodResourceEventHandler implements ResourceEventHandler { 41 | private static final Logger LOG = LoggerFactory.getLogger(PodResourceEventHandler.class); 42 | private final Consumer podEventNotification; 43 | private final ExecutorService service; 44 | 45 | PodResourceEventHandler(final Consumer podEventNotification) { 46 | this.podEventNotification = podEventNotification; 47 | this.service = Executors.newFixedThreadPool(1); 48 | } 49 | 50 | PodResourceEventHandler(final Consumer podEventNotification, final ExecutorService service) { 51 | this.podEventNotification = podEventNotification; 52 | this.service = service; 53 | } 54 | 55 | void shutdown() throws InterruptedException { 56 | service.shutdownNow(); 57 | service.awaitTermination(100, TimeUnit.SECONDS); 58 | } 59 | 60 | public void onAddSync(final Pod pod) { 61 | LOG.trace("{} (uid: {}) pod add received", pod.getMetadata().getName(), pod.getMetadata().getUid()); 62 | podEventNotification.accept(new PodEvent(PodEvent.Action.ADDED, pod)); // might be better to add pods in a batch 63 | 64 | } 65 | 66 | public void onUpdateSync(final Pod oldPod, final Pod newPod) { 67 | final String oldPodScheduler = oldPod.getSpec().getSchedulerName(); 68 | final String newPodScheduler = newPod.getSpec().getSchedulerName(); 69 | assert oldPodScheduler.equals(newPodScheduler); 70 | LOG.trace("{} => {} (uid: {}) pod update received", oldPod.getMetadata().getName(), 71 | newPod.getMetadata().getName(), newPod.getMetadata().getUid()); 72 | podEventNotification.accept(new PodEvent(PodEvent.Action.UPDATED, newPod, oldPod)); 73 | } 74 | 75 | public void onDeleteSync(final Pod pod, final boolean deletedFinalStateUnknown) { 76 | final long now = System.nanoTime(); 77 | LOG.trace("{} (uid: {}) pod deleted ({}) in {}ns!", pod.getMetadata().getName(), pod.getMetadata().getUid(), 78 | deletedFinalStateUnknown, (System.nanoTime() - now)); 79 | podEventNotification.accept(new PodEvent(PodEvent.Action.DELETED, pod)); 80 | } 81 | 82 | @Override 83 | public void onAdd(final Pod pod) { 84 | service.execute(() -> onAddSync(pod)); 85 | } 86 | 87 | @Override 88 | public void onUpdate(final Pod oldPod, final Pod newPod) { 89 | service.execute(() -> onUpdateSync(oldPod, newPod)); 90 | } 91 | 92 | @Override 93 | public void onDelete(final Pod pod, final boolean deletedFinalStateUnknown) { 94 | service.execute(() -> onDeleteSync(pod, deletedFinalStateUnknown)); 95 | } 96 | } -------------------------------------------------------------------------------- /k8s-scheduler/src/main/java/com/vmware/dcm/Utils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm; 8 | 9 | import io.fabric8.kubernetes.api.model.Quantity; 10 | 11 | class Utils { 12 | 13 | static long convertUnit(final Quantity quantity, final String resourceName) { 14 | if (resourceName.equals("cpu")) { 15 | final String unit = quantity.getFormat(); 16 | if (unit.equals("")) { 17 | // Convert to milli-cpu 18 | return (long) (Double.parseDouble(quantity.getAmount()) * 1000); 19 | } else if (unit.equals("m")) { 20 | return (long) (Double.parseDouble(quantity.getAmount())); 21 | } 22 | throw new IllegalArgumentException(quantity + " for resource type: " + resourceName); 23 | } else { 24 | // Values are guaranteed to be under 2^63 - 1, so this is safe 25 | return Quantity.getAmountInBytes(quantity).longValue(); 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /k8s-scheduler/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /k8s-scheduler/src/main/resources/pod-only.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: cache 5 | labels: 6 | app: store 7 | spec: 8 | schedulerName: default-scheduler 9 | containers: 10 | - name: cache 11 | image: registry.k8s.io/pause 12 | imagePullPolicy: IfNotPresent 13 | -------------------------------------------------------------------------------- /k8s-scheduler/src/main/resources/pod-with-affinity.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: cache 5 | labels: 6 | app: store 7 | spec: 8 | schedulerName: default-scheduler 9 | containers: 10 | - name: cache 11 | image: registry.k8s.io/pause 12 | imagePullPolicy: IfNotPresent 13 | affinity: 14 | podAntiAffinity: 15 | requiredDuringSchedulingIgnoredDuringExecution: 16 | - labelSelector: 17 | matchExpressions: 18 | - key: app 19 | operator: In 20 | values: 21 | - store 22 | topologyKey: kubernetes.io/hostname 23 | -------------------------------------------------------------------------------- /k8s-scheduler/src/test/java/com/vmware/dcm/AutoScopeTest.java: -------------------------------------------------------------------------------- 1 | package com.vmware.dcm; 2 | 3 | import com.vmware.dcm.compiler.IRContext; 4 | import org.jooq.DSLContext; 5 | import org.junit.jupiter.api.Test; 6 | 7 | 8 | import java.util.List; 9 | import java.util.Map; 10 | 11 | import static org.junit.jupiter.api.Assertions.assertEquals; 12 | 13 | 14 | public class AutoScopeTest { 15 | /* 16 | * Test if Scope only keeps the least loaded nodes when no constraints are present. 17 | */ 18 | @Test 19 | public void testSetup() { 20 | final DBConnectionPool dbConnectionPool = new DBConnectionPool(); 21 | final DSLContext conn = dbConnectionPool.getConnectionToDb(); 22 | final List constraints = Policies.getInitialPlacementPolicies(); 23 | final Model model = Model.build(conn, constraints); 24 | final IRContext irContext = model.getIrContext(); 25 | final Map augmViews = AutoScope.augmentedViews(constraints, irContext, 20); 26 | assertEquals(2, augmViews.size()); 27 | 28 | final List statements = AutoScope.getViewStatements(augmViews); 29 | statements.forEach(conn::execute); 30 | // finish without error 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /k8s-scheduler/src/test/java/com/vmware/dcm/ITBase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * 4 | * SPDX-License-Identifier: BSD-2 5 | */ 6 | 7 | package com.vmware.dcm; 8 | 9 | import io.fabric8.kubernetes.api.model.Namespace; 10 | import io.fabric8.kubernetes.api.model.NamespaceBuilder; 11 | import io.fabric8.kubernetes.api.model.Pod; 12 | import io.fabric8.kubernetes.api.model.PodList; 13 | import io.fabric8.kubernetes.api.model.apps.Deployment; 14 | import io.fabric8.kubernetes.api.model.apps.DeploymentList; 15 | import io.fabric8.kubernetes.client.DefaultKubernetesClient; 16 | import io.fabric8.kubernetes.client.Watcher; 17 | import io.fabric8.kubernetes.client.WatcherException; 18 | import org.junit.jupiter.api.BeforeAll; 19 | import org.junit.jupiter.api.BeforeEach; 20 | import org.junit.jupiter.api.Timeout; 21 | 22 | import java.io.File; 23 | import java.net.URL; 24 | import java.util.concurrent.CountDownLatch; 25 | import java.util.function.Predicate; 26 | 27 | import static org.junit.jupiter.api.Assertions.assertNotNull; 28 | 29 | public class ITBase { 30 | static final String TEST_NAMESPACE = "default"; // make sure that pod-specs use this namespace 31 | 32 | static DefaultKubernetesClient fabricClient; 33 | 34 | @BeforeAll 35 | public static void setupConnectionAndCreateNamespace() { 36 | fabricClient = new DefaultKubernetesClient(); 37 | 38 | // Create new namespace if required 39 | final Namespace namespace = new NamespaceBuilder().withNewMetadata().withName(TEST_NAMESPACE).endMetadata() 40 | .build(); 41 | fabricClient.namespaces().createOrReplace(namespace); 42 | } 43 | 44 | @BeforeEach 45 | @Timeout(300 /* seconds */) 46 | public void deleteAllRunningPods() throws Exception { 47 | deleteAllRunningPods(fabricClient); 48 | } 49 | 50 | public void deleteAllRunningPods(final DefaultKubernetesClient client) throws Exception { 51 | final DeploymentList deployments = client.apps().deployments().inNamespace(TEST_NAMESPACE) 52 | .list(); 53 | client.resourceList(deployments).inNamespace(TEST_NAMESPACE).withGracePeriod(0).delete(); 54 | final PodList pods = client.pods().inNamespace(TEST_NAMESPACE) 55 | .list(); 56 | client.resourceList(pods).inNamespace(TEST_NAMESPACE).withGracePeriod(0).delete(); 57 | waitUntil(client, (n) -> hasDrained(client)); 58 | } 59 | 60 | Deployment launchDeploymentFromFile(final String resourceName, final String schedulerName) { 61 | final URL url = getClass().getClassLoader().getResource(resourceName); 62 | assertNotNull(url); 63 | final File file = new File(url.getFile()); 64 | final Deployment deployment = fabricClient.apps().deployments().load(file).get(); 65 | deployment.getSpec().getTemplate().getSpec().setSchedulerName(schedulerName); 66 | fabricClient.apps().deployments().inNamespace(TEST_NAMESPACE) 67 | .create(deployment); 68 | return deployment; 69 | } 70 | 71 | boolean hasNRunningPods(final int numPods) { 72 | return fabricClient.pods().inNamespace(TEST_NAMESPACE).list().getItems() 73 | .stream().filter(e -> e.getStatus().getPhase().equals("Running")) 74 | .count() == numPods; 75 | } 76 | 77 | void waitUntil(final DefaultKubernetesClient client, final Predicate condition) throws Exception { 78 | if (condition.test(0)) { 79 | return; 80 | } 81 | final CountDownLatch latch = new CountDownLatch(1); 82 | client.pods().inNamespace(TEST_NAMESPACE).watch(new PodConditionWatcher(condition, latch)); 83 | latch.await(); 84 | } 85 | 86 | private boolean hasDrained(final DefaultKubernetesClient client) { 87 | return client.pods().inNamespace(TEST_NAMESPACE).list().getItems().size() == 0; 88 | } 89 | 90 | private static class PodConditionWatcher implements Watcher { 91 | 92 | private final Predicate condition; 93 | private final CountDownLatch latch; 94 | 95 | public PodConditionWatcher(final Predicate condition, final CountDownLatch latch) { 96 | this.condition = condition; 97 | this.latch = latch; 98 | } 99 | 100 | @Override 101 | public void eventReceived(final Watcher.Action action, final Pod pod) { 102 | if (condition.test(0)) { 103 | latch.countDown(); 104 | } 105 | } 106 | 107 | @Override 108 | public void onClose(final WatcherException cause) { 109 | } 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /k8s-scheduler/src/test/java/com/vmware/dcm/UnbindTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2018-2021 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm; 7 | 8 | import io.fabric8.kubernetes.client.server.mock.KubernetesServer; 9 | import org.junit.jupiter.api.AfterEach; 10 | 11 | import javax.annotation.Nullable; 12 | 13 | import static org.junit.jupiter.api.Assertions.assertNotNull; 14 | 15 | public class UnbindTest { 16 | @Nullable 17 | private KubernetesServer server; 18 | 19 | @AfterEach 20 | void tearDown() { 21 | assertNotNull(server); 22 | server.after(); 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /k8s-scheduler/src/test/java/com/vmware/dcm/WorkloadGeneratorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm; 7 | 8 | import com.vmware.dcm.trace.TraceReplayer; 9 | import io.fabric8.kubernetes.api.model.Pod; 10 | import io.fabric8.kubernetes.client.NamespacedKubernetesClient; 11 | import io.fabric8.kubernetes.client.server.mock.KubernetesServer; 12 | import org.junit.jupiter.api.Test; 13 | 14 | import java.util.List; 15 | import java.util.stream.Collectors; 16 | import java.util.stream.IntStream; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertEquals; 19 | import static org.junit.jupiter.api.Assertions.assertNull; 20 | 21 | public class WorkloadGeneratorTest { 22 | 23 | @Test 24 | public void testKubernetesDeployer() { 25 | final KubernetesServer server = new KubernetesServer(false, true); 26 | server.before(); 27 | final NamespacedKubernetesClient client = server.getClient(); 28 | final IPodDeployer deployer = new KubernetesPodDeployer(server.getClient(), "default"); 29 | final Pod pod1 = SchedulerTest.newPod("pod1"); 30 | final Pod pod2 = SchedulerTest.newPod("pod2"); 31 | 32 | // Launch pods via deployer and verify whether they're created 33 | deployer.startDeployment(List.of(pod1, pod2)).run(); 34 | final Pod pod1Result = client.pods().inNamespace("default").withName(pod1.getMetadata().getName()).get(); 35 | assertEquals(pod1Result.getMetadata().getName(), pod1.getMetadata().getName()); 36 | final Pod pod2Result = client.pods().inNamespace("default").withName(pod2.getMetadata().getName()).get(); 37 | assertEquals(pod2Result.getMetadata().getName(), pod2.getMetadata().getName()); 38 | 39 | // Delete pods via deployer and verify whether they're created 40 | deployer.endDeployment(List.of(pod1, pod2)).run(); 41 | assertNull(client.pods().inNamespace("default").withName(pod1.getMetadata().getName()).get()); 42 | assertNull(client.pods().inNamespace("default").withName(pod2.getMetadata().getName()).get()); 43 | } 44 | 45 | @Test 46 | public void testSmallTrace() throws Exception { 47 | final KubernetesServer server = new KubernetesServer(false, true); 48 | server.before(); 49 | final NamespacedKubernetesClient client = server.getClient(); 50 | final IPodDeployer deployer = new KubernetesPodDeployer(client, "default"); 51 | final TraceReplayer traceReplayer = new TraceReplayer(); 52 | traceReplayer.runTrace(client, "test-data.txt", deployer, "default-scheduler", 53 | 400, 1, 1, 100, 1000, 100, 2); 54 | assertEquals(8, server.getMockServer().getRequestCount()); 55 | final List events = IntStream.range(0, 8) 56 | .mapToObj(e -> { 57 | try { 58 | return server.getMockServer().takeRequest().getMethod(); 59 | } catch (final InterruptedException interruptedException) { 60 | return null; 61 | } 62 | }).collect(Collectors.toList()); 63 | for (int i = 0; i < 4; i++) { 64 | assertEquals(events.get(i), "POST"); 65 | } 66 | for (int i = 4; i < 8; i++) { 67 | assertEquals(events.get(i), "DELETE"); 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /k8s-scheduler/src/test/java/com/vmware/dcm/WorkloadReplayTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018-2020 VMware, Inc. All Rights Reserved. 3 | * SPDX-License-Identifier: BSD-2 4 | */ 5 | 6 | package com.vmware.dcm; 7 | 8 | import org.junit.jupiter.api.Test; 9 | 10 | public class WorkloadReplayTest { 11 | 12 | @Test 13 | public void runTest() throws Exception { 14 | final String[] args = 15 | {"-n", "500", "-f", "test-data.txt", "-c", "100", "-m", "200", "-t", "100", "-s", "1000"}; 16 | EmulatedCluster.runWorkload(args); 17 | } 18 | 19 | @Test 20 | public void runTestScope() throws Exception { 21 | final String[] args = 22 | {"-n", "500", "-f", "test-data.txt", "-c", "100", "-m", "200", "-t", "100", "-s", "1000", "-S"}; 23 | EmulatedCluster.runWorkload(args); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /k8s-scheduler/src/test/resources/app-no-constraints.yml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: cache 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: store 9 | replicas: 3 10 | template: 11 | metadata: 12 | labels: 13 | app: store 14 | spec: 15 | schedulerName: default-scheduler 16 | containers: 17 | - name: cache 18 | image: registry.k8s.io/pause 19 | imagePullPolicy: IfNotPresent 20 | -------------------------------------------------------------------------------- /k8s-scheduler/src/test/resources/cache-example.yml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: cache 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: store 9 | replicas: 3 10 | template: 11 | metadata: 12 | labels: 13 | app: store 14 | spec: 15 | schedulerName: default-scheduler 16 | affinity: 17 | podAntiAffinity: 18 | requiredDuringSchedulingIgnoredDuringExecution: 19 | - labelSelector: 20 | matchExpressions: 21 | - key: app 22 | operator: In 23 | values: 24 | - store 25 | topologyKey: "kubernetes.io/hostname" 26 | containers: 27 | - name: cache 28 | image: registry.k8s.io/pause 29 | imagePullPolicy: IfNotPresent 30 | -------------------------------------------------------------------------------- /k8s-scheduler/src/test/resources/kind-test-cluster-configuration.yaml: -------------------------------------------------------------------------------- 1 | kind: Cluster 2 | apiVersion: kind.x-k8s.io/v1alpha4 3 | nodes: 4 | - role: control-plane 5 | - role: worker 6 | - role: worker 7 | - role: worker 8 | -------------------------------------------------------------------------------- /k8s-scheduler/src/test/resources/no-constraints.yml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: nginx-deployment 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: nginx 9 | replicas: 3 10 | template: 11 | metadata: 12 | labels: 13 | app: nginx 14 | spec: 15 | schedulerName: dcm-scheduler 16 | containers: 17 | - name: nginx 18 | image: nginx:1.7.9 19 | imaePullPolicy: IfNotPresent 20 | ports: 21 | - containerPort: 80 -------------------------------------------------------------------------------- /k8s-scheduler/src/test/resources/test-data.txt: -------------------------------------------------------------------------------- 1 | 1040 12378 0 120 8 14 1 2 | 1163 18214 60 300 1 1.75 1 3 | 2709 18693 120 240 4 7 2 4 | -------------------------------------------------------------------------------- /k8s-scheduler/src/test/resources/web-store-example.yml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: web-server 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: web-store 9 | replicas: 3 10 | template: 11 | metadata: 12 | labels: 13 | app: web-store 14 | spec: 15 | schedulerName: default-scheduler 16 | affinity: 17 | podAntiAffinity: 18 | requiredDuringSchedulingIgnoredDuringExecution: 19 | - labelSelector: 20 | matchExpressions: 21 | - key: app 22 | operator: In 23 | values: 24 | - web-store 25 | topologyKey: "kubernetes.io/hostname" 26 | podAffinity: 27 | requiredDuringSchedulingIgnoredDuringExecution: 28 | - labelSelector: 29 | matchExpressions: 30 | - key: app 31 | operator: In 32 | values: 33 | - store 34 | topologyKey: "kubernetes.io/hostname" 35 | containers: 36 | - name: web-app 37 | image: registry.k8s.io/pause 38 | imagePullPolicy: IfNotPresent 39 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | */ 4 | 5 | rootProject.name = 'dcm-project' 6 | include(':dcm') 7 | include(':examples') 8 | include(':k8s-scheduler') 9 | include(':benchmarks') 10 | -------------------------------------------------------------------------------- /verify_docs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | npx embedme README.md --verify 6 | npx embedme docs/tutorial.md --verify 7 | npx embedme docs/reference.md --verify 8 | -------------------------------------------------------------------------------- /workload-generator/azure-combine.awk: -------------------------------------------------------------------------------- 1 | BEGIN { 2 | count = 0; 3 | count2 = 0; 4 | count3 = 0; 5 | } 6 | 7 | FNR == 1 { 8 | ++fIndex 9 | } 10 | 11 | fIndex == 1 { 12 | if (!($1 in dep)) { 13 | dep[$1] = count; 14 | count += 1; 15 | } 16 | next 17 | } 18 | 19 | fIndex == 2 { 20 | if (! ($1 in subsc)) { 21 | subsc[$1] = count2; 22 | count2 += 1; 23 | } 24 | next 25 | } 26 | 27 | fIndex == 3 { 28 | if (! ($1 in vms)) { 29 | vms[$1] = count3; 30 | count3 += 1; 31 | } 32 | print vms[$1], subsc[$2], dep[$3], $4, $5, $10, $11, $6 33 | } 34 | -------------------------------------------------------------------------------- /workload-generator/getAzureTraces.sh: -------------------------------------------------------------------------------- 1 | wget -O v1-deployment.csv.gz https://azurecloudpublicdataset.blob.core.windows.net/azurepublicdataset/trace_data/deployment/deployment.csv.gz 2 | wget -O v1-subscriptions.csv.gz https://azurecloudpublicdataset.blob.core.windows.net/azurepublicdataset/trace_data/subscriptions/subscriptions.csv.gz 3 | wget -O v1-vmtable.csv.gz https://azurecloudpublicdataset.blob.core.windows.net/azurepublicdataset/trace_data/vmtable/vmtable.csv.gz 4 | gunzip v1-deployment.csv.gz 5 | gunzip v1-subscriptions.csv.gz 6 | gunzip v1-vmtable.csv.gz 7 | 8 | wget -O v2-deployment.csv.gz https://azurecloudpublicdataset2.blob.core.windows.net/azurepublicdatasetv2/trace_data/deployments/deployments.csv.gz 9 | wget -O v2-subscriptions.csv.gz https://azurecloudpublicdataset2.blob.core.windows.net/azurepublicdatasetv2/trace_data/subscriptions/subscriptions.csv.gz 10 | wget -O v2-vmtable.csv.gz https://azurecloudpublicdataset2.blob.core.windows.net/azurepublicdatasetv2/trace_data/vmtable/vmtable.csv.gz 11 | gunzip v2-deployment.csv.gz 12 | gunzip v2-subscriptions.csv.gz 13 | gunzip v2-vmtable.csv.gz 14 | 15 | for ver in v1 v2; do 16 | awk -F "," -f azure-combine.awk $ver-deployment.csv $ver-subscriptions.csv $ver-vmtable.csv > $ver-temp-1.txt 17 | sed -e "s/^M//g" $ver-temp-1.txt > $ver-temp-2.txt 18 | sort -nk2,2 -nk3,3 -nk4,4 -nk5,5 $ver-temp-2.txt > $ver-temp-3.txt 19 | cat $ver-temp-3.txt | awk '{if($4 != $5) a[$2," ",$3," ",$4," ",$5," ",$6," ",$7]+=1;}END{for (key in a) print key, a[key]}' > $ver-temp-4.txt 20 | sed -e "s///g" $ver-temp-4.txt > $ver-temp-5.txt 21 | sort -nk3,3 -nk1,1 -nk2,2 $ver-temp-5.txt > $ver-temp-6.txt 22 | sed -e "s///g" $ver-temp-6.txt > $ver-data.txt 23 | rm *-temp-* 24 | done 25 | 26 | cat v1-data.txt | awk '{if(NR>22390) print $0}' > v1-cropped.txt 27 | cat v2-data.txt | awk '{if(NR>27326) print $0}' > v2-cropped.txt 28 | cp {v1-data.txt,v2-data.txt,v1-cropped.txt,v2-cropped.txt} ../k8s-scheduler/src/main/resources/ 29 | --------------------------------------------------------------------------------