├── .circleci └── config.yml ├── .gitignore ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── DESIGN.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── SECURITY.md ├── backend └── backend.go ├── cache ├── cache_test.go ├── coordinate.go ├── coordinate_test.go ├── lru.go ├── lru_test.go ├── namespace.go ├── work_spec.go ├── work_unit.go └── worker.go ├── cborrpc ├── README.md ├── cborrpc.go ├── cborrpc_test.go └── cleanup.go ├── cmd ├── coordbench │ └── main.go ├── coordinated │ ├── cborrpc.go │ ├── http.go │ ├── main.go │ └── metrics.go ├── cptest │ └── main.go └── demoworker │ └── demoworker.go ├── coordinate ├── coordinate.go ├── coordinatetest │ ├── attempt.go │ ├── benchmarks.notgo │ ├── coordinatetest.go │ ├── helpers.go │ ├── namespace.go │ ├── performance.go │ ├── work_spec.go │ ├── work_unit.go │ └── worker.go ├── errors.go ├── helpers.go ├── helpers_test.go ├── marshal.go ├── marshal_test.go ├── scheduler.go ├── scheduler_test.go └── stats.go ├── doc ├── chaining.md ├── changes.md ├── errgone.md ├── index.md ├── model.md ├── python.md ├── runtime.md ├── work_specs.md └── worker.md ├── go.mod ├── go.sum ├── jobserver ├── constants.go ├── interface.go ├── locks.go ├── locks_test.go ├── python_test.go ├── specs.go ├── specs_test.go ├── units.go ├── utils.go ├── work.go ├── work_test.go └── workers.go ├── memory ├── attempt.go ├── available_units.go ├── available_units_test.go ├── coordinate.go ├── coordinate_test.go ├── namespace.go ├── work_spec.go ├── work_unit.go └── worker.go ├── postgres ├── README.md ├── attempt.go ├── constants.go ├── coordinate.go ├── coordinate_test.go ├── expiry.go ├── helpers.go ├── migration.go ├── migrations.go ├── migrations │ ├── 20150927-core.sql │ ├── 20151002-mingb.sql │ ├── 20151006-work-unit-type.sql │ ├── 20151013-index.sql │ ├── 20151014-index.sql │ ├── 20151019-worker-mode.sql │ ├── 20151028-index.sql │ ├── 20151216-work-spec-runtime.sql │ ├── 20160104-not-before.sql │ ├── 20160125-index.sql │ ├── 20160217-attempt-spec.sql │ ├── 20160328-index.sql │ ├── 20160329-index.sql │ ├── 20170316-index.sql │ └── 20170523-work-unit-max-retries.sql ├── namespace.go ├── sql.go ├── sql_test.go ├── stats.go ├── work_spec.go ├── work_unit.go └── worker.go ├── restclient ├── README.md ├── attempt.go ├── coordinate.go ├── coordinate_test.go ├── namespace.go ├── rest.go ├── work_spec.go ├── work_unit.go └── worker.go ├── restdata ├── errors.go ├── marshal.go ├── marshal_test.go ├── restdata.go ├── url.go └── url_test.go ├── restserver ├── attempt.go ├── context.go ├── doc.go ├── helpers.go ├── namespace.go ├── rest.go ├── rest_test.go ├── server.go ├── work_spec.go ├── work_unit.go └── worker.go └── worker ├── worker.go └── worker_test.go /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build: 4 | docker: 5 | - image: cimg/go:1.19 6 | - image: postgres:12 7 | environment: 8 | POSTGRES_PASSWORD: citest 9 | environment: 10 | TEST_RESULTS: /tmp/test-results 11 | GO111MODULE: 'on' 12 | steps: 13 | - checkout 14 | 15 | - run: git describe HEAD 16 | 17 | - type: setup_remote_docker 18 | reusable: true 19 | 20 | - restore_cache: # restores saved cache if no changes are detected since last run 21 | keys: 22 | - v1-pkg-cache 23 | 24 | - run: 25 | name: Install tools 26 | environment: 27 | GO111MODULE: 'off' 28 | command: | 29 | go get golang.org/x/lint/golint 30 | go get github.com/jstemmer/go-junit-report 31 | 32 | - run: 33 | name: Install dependencies 34 | command: | 35 | # Install source dependencies. 36 | go mod tidy -v 37 | 38 | - run: 39 | name: Ensure dependency specification is up to date. 40 | # "go mod tidy" should not cause updates to the config. 41 | command: git diff --name-only --exit-code 42 | 43 | - save_cache: # Store cache in the /go/pkg directory 44 | key: v1-pkg-cache 45 | paths: 46 | - /go/pkg 47 | 48 | - run: 49 | name: Install Go packages 50 | command: go install -v ./... 51 | 52 | - run: 53 | name: Build docker image 54 | command: | 55 | make docker BUILD="$CIRCLE_BUILD_NUM" 56 | docker images 57 | 58 | - run: 59 | name: Run linters 60 | # TODO: Switch golint to ./... - https://github.com/golang/lint/issues/320 61 | command: | 62 | go list ./... | grep -v /vendor/ | xargs -L1 golint 63 | go vet -x ./... 64 | 65 | - run: 66 | name: Run basic tests 67 | command: | 68 | mkdir -p "${TEST_RESULTS}" 69 | trap "go-junit-report <${TEST_RESULTS}/go-test.out > ${TEST_RESULTS}/go-test-report.xml" EXIT 70 | make test | tee ${TEST_RESULTS}/go-test.out 71 | environment: 72 | PGHOST: 127.0.0.1 73 | PGUSER: postgres 74 | PGSSLMODE: disable 75 | PGPASSWORD: citest 76 | 77 | workflows: 78 | version: 2 79 | build: 80 | jobs: 81 | - build: 82 | filters: 83 | tags: 84 | only: /^[0-9]+[.][0-9]+[.][0-9]+.*/ 85 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.test 3 | vendor/ 4 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Comment line immediately above ownership line is reserved for related gus information. Please be careful while editing. 2 | #ECCN:Open Source 3 | #GUSINFO:Open Source,Open Source Workflow -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guide For go-coordinate 2 | 3 | This page lists the operational governance model of this project, as well as the recommendations and requirements for how to best contribute to go-coordinate. We strive to obey these as best as possible. As always, thanks for contributing – we hope these guidelines make it easier and shed some light on our approach and processes. 4 | 5 | # Governance Model 6 | 7 | ## Published but not supported 8 | 9 | The intent and goal of open sourcing this project is because it may contain useful or interesting code/concepts that we wish to share with the larger open source community. Although occasional work may be done on it, we will not be looking for or soliciting contributions. 10 | 11 | # Issues, requests & ideas 12 | 13 | Use GitHub Issues page to submit issues, enhancement requests and discuss ideas. 14 | 15 | ### Bug Reports and Fixes 16 | - If you find a bug, please search for it in the [Issues](https://github.com/swiftlobste/go-coordinate/issues), and if it isn't already tracked, 17 | [create a new issue](https://github.com/swiftlobste/go-coordinate/issues/new). Fill out the "Bug Report" section of the issue template. Even if an Issue is closed, feel free to comment and add details, it will still 18 | be reviewed. 19 | - Issues that have already been identified as a bug (note: able to reproduce) will be labelled `bug`. 20 | - If you'd like to submit a fix for a bug, [send a Pull Request](#creating_a_pull_request) and mention the Issue number. 21 | - Include tests that isolate the bug and verifies that it was fixed. 22 | 23 | ### New Features 24 | - If you'd like to add new functionality to this project, describe the problem you want to solve in a [new Issue](https://github.com/swiftlobste/go-coordinate/issues/new). 25 | - Issues that have been identified as a feature request will be labelled `enhancement`. 26 | - If you'd like to implement the new feature, please wait for feedback from the project 27 | maintainers before spending too much time writing the code. In some cases, `enhancement`s may 28 | not align well with the project objectives at the time. 29 | 30 | ### Tests, Documentation, Miscellaneous 31 | - If you'd like to improve the tests, you want to make the documentation clearer, you have an 32 | alternative implementation of something that may have advantages over the way its currently 33 | done, or you have any other change, we would be happy to hear about it! 34 | - If its a trivial change, go ahead and [send a Pull Request](#creating_a_pull_request) with the changes you have in mind. 35 | - If not, [open an Issue](https://github.com/swiftlobste/go-coordinate/issues/new) to discuss the idea first. 36 | 37 | If you're new to our project and looking for some way to make your first contribution, look for 38 | Issues labelled `good first contribution`. 39 | 40 | # Contribution Checklist 41 | 42 | - [x] Clean, simple, well styled code 43 | - [x] Commits should be atomic and messages must be descriptive. Related issues should be mentioned by Issue number. 44 | - [x] Comments 45 | - Module-level & function-level comments. 46 | - Comments on complex blocks of code or algorithms (include references to sources). 47 | - [x] Tests 48 | - The test suite, if provided, must be complete and pass 49 | - Increase code coverage, not versa. 50 | - Use any of our testkits that contains a bunch of testing facilities you would need. For example: `import com.salesforce.op.test._` and borrow inspiration from existing tests. 51 | - [x] Dependencies 52 | - Minimize number of dependencies. 53 | - Prefer Apache 2.0, BSD3, MIT, ISC and MPL licenses. 54 | - [x] Reviews 55 | - Changes must be approved via peer code review 56 | 57 | # Creating a Pull Request 58 | 59 | 1. **Ensure the bug/feature was not already reported** by searching on GitHub under Issues. If none exists, create a new issue so that other contributors can keep track of what you are trying to add/fix and offer suggestions (or let you know if there is already an effort in progress). 60 | 3. **Clone** the forked repo to your machine. 61 | 4. **Create** a new branch to contain your work (e.g. `git br fix-issue-11`) 62 | 4. **Commit** changes to your own branch. 63 | 5. **Push** your work back up to your fork. (e.g. `git push fix-issue-11`) 64 | 6. **Submit** a Pull Request against the `main` branch and refer to the issue(s) you are fixing. Try not to pollute your pull request with unintended changes. Keep it simple and small. 65 | 7. **Sign** the Salesforce CLA (you will be prompted to do so when submitting the Pull Request) 66 | 67 | > **NOTE**: Be sure to [sync your fork](https://help.github.com/articles/syncing-a-fork/) before making a pull request. 68 | 69 | # Contributor License Agreement ("CLA") 70 | In order to accept your pull request, we need you to submit a CLA. You only need 71 | to do this once to work on any of Salesforce's open source projects. 72 | 73 | Complete your CLA here: 74 | 75 | # Issues 76 | We use GitHub issues to track public bugs. Please ensure your description is 77 | clear and has sufficient instructions to be able to reproduce the issue. 78 | 79 | # Code of Conduct 80 | Please follow our [Code of Conduct](CODE_OF_CONDUCT.md). 81 | 82 | # License 83 | By contributing your code, you agree to license your contribution under the terms of our project [LICENSE](LICENSE.txt) and to sign the [Salesforce CLA](https://cla.salesforce.com/sign-cla) 84 | -------------------------------------------------------------------------------- /DESIGN.md: -------------------------------------------------------------------------------- 1 | (Re)Design of Coordinated 2 | ========================= 3 | 4 | Diffeo `coordinated` was a redesign of 5 | [rejester](http://github.com/diffeo/rejester), and the merged 6 | [Coordinate](http://github.com/diffeo/coordinate) follows the 7 | `coordinated` design and code base. 8 | 9 | Principles 10 | ---------- 11 | 12 | **Distributed and stateless.** The Coordinate daemon should store all 13 | of its state somewhere persistent and remote, like a central external 14 | database. While limited caching may be appropriate, state should be 15 | in the database first. This allows multiple Coordinate daemons to be 16 | running in a cluster, which provides some degree of redundancy and 17 | helps migrations. If code relies on the database to provide 18 | concurrency protection, then this minimizes the amount of troublesome 19 | code within the daemon itself to manage intra-server concurrency. 20 | 21 | **Abstract API.** There are at least three useful implementations of 22 | the Python Coordinate `TaskMaster` object: the rejester version, the 23 | standard Coordinate version, and a unit-test version that hard-wires 24 | the Coordinate client to a Coordinate job server object. These should 25 | share a common API. 26 | 27 | Object Model 28 | ------------ 29 | 30 | **Namespace.** The Coordinate server tracks any number of namespaces. 31 | A namespace is an unordered collection of work specs. In the future 32 | it may gain an access control policy or other system-wide data. 33 | 34 | **Work spec.** A work spec is a definition of what to do for a set of 35 | units. It might define a specific Python function to call, for 36 | instance, where a work unit gives a set of parameters. A work spec 37 | belongs to a single namespace. There are typically at most dozens of 38 | work specs in a namespace. The definition of a work spec is described 39 | as a JSON object with a number of well-known keys, such as `name`. 40 | 41 | **Work unit.** A work unit is a single thing to do for a given work 42 | spec. It might correspond to a single input document in the system or 43 | other job. If it is convenient to batch together smaller units of 44 | work, a single Coordinate work unit might be tuned to take about one 45 | minute to execute. A work unit has a key and a data JSON object, 46 | along with an overall status. 47 | 48 | **Worker.** A worker is a process that executes work units. A worker 49 | may also be a parent worker that is responsible for child worker 50 | processes. A worker chooses a unique ID for itself, and should 51 | periodically report its status and system environment to the 52 | Coordinate server. 53 | 54 | **Attempt.** An attempt is a record that a specific worker is working 55 | on a specific work unit. A work unit has at most one active attempt. 56 | Workers also have lists of active attempts, but they are not strictly 57 | limited to doing one at a time. The attempt includes a new data 58 | dictionary for the work unit, which may include any generated outputs 59 | or failure information. An attempt must be completed by some deadline 60 | (which the worker may extend) or else it ceases to be the active 61 | attempt. 62 | 63 | Changes from Python Coordinate 64 | ------------------------------ 65 | 66 | **Reintroduction of namespaces.** Rejester had the concept of a 67 | "namespace", allowing there to be multiple sets of work specs on a 68 | single (Redis) server. While this was most useful for testing, it 69 | also supported a shared server if users and tests could come up with 70 | reasonably unique namespace names. We add back in the namespace 71 | concept. 72 | 73 | **Attempt tracking.** Both rejester and Coordinate had a basic work 74 | flow for work units: they would move from "available" to "pending", 75 | then either "finished" or "failed". If the assigned worker failed to 76 | complete the work unit promptly, it could silently move back from 77 | "pending" to "available". We add the concept of an "attempt" that 78 | keeps track of who is doing work and what the result of that attempt 79 | was. 80 | 81 | **Work unit data lifespan.** In Python Coordinate (and rejester) 82 | changing a work unit's data changes it globally, even if the work unit 83 | does not complete successfully. This implementation attaches the 84 | changed data to an attempt. In practice this is only a significant 85 | change if a work unit fails and is retried. 86 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Dockerfile for building coordinated into a container. 2 | # setup.sh will prepare prerequisites in the current directory. 3 | 4 | # Build image 5 | FROM golang:1.19 AS builder 6 | 7 | # Outside GOPATH to use go modules 8 | WORKDIR /src 9 | 10 | # Fetch dependencies first, less susceptible to change on every build 11 | COPY ./go.mod ./go.sum ./ 12 | RUN go mod download 13 | 14 | # Copy in code 15 | COPY ./ ./ 16 | 17 | RUN CGO_ENABLED=0 go build -v -o /coordinated ./cmd/coordinated 18 | 19 | # Application image 20 | FROM scratch 21 | 22 | ARG VERSION 23 | ARG BUILD 24 | ARG NOW 25 | 26 | COPY --from=builder /coordinated /coordinated 27 | 28 | # CBOR-RPC interface 29 | EXPOSE 5932 30 | # HTTP REST interface 31 | EXPOSE 5980 32 | 33 | ENTRYPOINT ["/coordinated"] 34 | 35 | LABEL name="coordinated" \ 36 | version="$VERSION" \ 37 | build="$BUILD" \ 38 | architecture="x86_64" \ 39 | build_date="$NOW" \ 40 | vendor="Diffeo, Inc." \ 41 | maintainer="Diffeo Support " \ 42 | url="https://github.com/swiftlobste/go-coordinate" \ 43 | summary="Coordinate job queue daemon" \ 44 | description="Coordinate job queue daemon" \ 45 | vcs-type="git" \ 46 | vcs-url="https://github.com/swiftlobste/go-coordinate" \ 47 | vcs-ref="$VERSION" \ 48 | distribution-scope="public" 49 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015-2017 Diffeo, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | VERSION := $(shell git describe HEAD) 3 | BRANCH := $(shell git rev-parse --abbrev-ref HEAD | tr / -) 4 | BUILD := 0 5 | 6 | DOCKER_REPO := diffeo/coordinated 7 | DOCKER_IMG := $(DOCKER_REPO):$(VERSION) 8 | 9 | .PHONY: test docker docker-push-branch docker-push-latest 10 | 11 | test: 12 | go test -race -v ./... 13 | 14 | docker: 15 | docker build \ 16 | --build-arg VERSION=$(VERSION) \ 17 | --build-arg BUILD=$(BUILD) \ 18 | --build-arg NOW=$(shell TZ=UTC date +%Y-%m-%dT%H:%M:%SZ) \ 19 | -t $(DOCKER_IMG) \ 20 | . 21 | 22 | docker-push-branch: 23 | # Only intended for CI 24 | [ ! -z "$$CI" ] 25 | # Push a "latest" tag to our repository 26 | docker tag $(DOCKER_IMG) $(DOCKER_REPO):$(BRANCH) 27 | docker push $(DOCKER_REPO):$(BRANCH) 28 | 29 | docker-push-latest: 30 | # Only intended for CI 31 | [ ! -z "$$CI" ] 32 | # Push image to our repository 33 | docker push $(DOCKER_IMG) 34 | # Push a "latest" tag to our repository 35 | docker tag $(DOCKER_IMG) $(DOCKER_REPO):latest 36 | docker push $(DOCKER_REPO):latest 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Go Coordinate Daemon 2 | ==================== 3 | 4 | [![CircleCI](https://circleci.com/gh/diffeo/go-coordinate.svg?style=svg)](https://circleci.com/gh/diffeo/go-coordinate) 5 | [![Docker Hub Repository](https://img.shields.io/docker/pulls/diffeo/coordinated.svg "Docker Hub Repository")](https://hub.docker.com/r/diffeo/coordinated/) 6 | 7 | This package provides a reimplementation of the Diffeo Coordinate 8 | (https://github.com/diffeo/coordinate) daemon. It is fully compatible 9 | with existing Python Coordinate code, and provides a REST interface 10 | for Go and other languages. 11 | 12 | * [Documentation index](doc/index.md) 13 | * [Change history](doc/changes.md) 14 | 15 | Overview 16 | -------- 17 | 18 | Coordinate is a job queue system. It is designed for repetitive tasks 19 | with large numbers of inputs, where the inputs and outputs will be 20 | stored externally and do not need to be passed directly through the 21 | system, and where no particular action needs to be taken when a job 22 | finishes. 23 | 24 | Coordinate-based applications can define _work specs_, JSON or YAML 25 | dictionary objects that define specific work to do. A typical work 26 | spec would name a specific Python function to call with a YAML 27 | configuration dictionary, and the Python Coordinate package contains a 28 | worker process that can run these work specs. Each work spec has a 29 | list of _work units_, individual tasks to perform, where each work 30 | unit has a name or key and an additional data dictionary. In typical 31 | operation a work unit key is a filename or database key and the data 32 | is used only to record outputs. Read more about the 33 | [data model](doc/model.md). 34 | 35 | The general expectation is that there will be, at most, dozens of work 36 | specs, but each work spec could have millions of work units. It is 37 | definitely expected that many worker processes will connect to a 38 | single Coordinate daemon, and past data loads have involved 800 or 39 | more workers talking to one server. 40 | 41 | Installation 42 | ------------ 43 | 44 | From source: 45 | 46 | go get github.com/swiftlobste/go-coordinate/cmd/coordinated 47 | 48 | Usage 49 | ----- 50 | 51 | Run the `coordinated` binary. With default options, it will use 52 | in-memory storage and start a network server listening on ports 5932 53 | and 5980. Port 5980 provides the REST interface. 54 | 55 | Go code should use the `backend` package to provide a command-line 56 | flag to get a backend object, which will implement the generic 57 | interface in the `coordinate` package. Test code can directly create 58 | a `memory` backend. Most applications will expect to use the 59 | `restclient` backend to communicate with a central Coordinate daemon. 60 | 61 | 5932 is the default TCP port for the Python Coordinate daemon, and 62 | application configurations that normally point at that daemon should 63 | work against this one as well. Read more about 64 | [Python compatibility](doc/python.md). 65 | 66 | ```sh 67 | pip install coordinate 68 | go get github.com/swiftlobste/go-coordinate/cmd/coordinated 69 | $GOPATH/bin/coordinated & 70 | cat >config.yaml < lru.size { 115 | head := lru.evictList.Front() 116 | item := head.Value.(named) 117 | delete(lru.index, item.Name()) 118 | lru.evictList.Remove(head) 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /cache/lru_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package cache 5 | 6 | import ( 7 | "github.com/stretchr/testify/assert" 8 | "testing" 9 | ) 10 | 11 | type AName struct { 12 | IAm string 13 | } 14 | 15 | func (a AName) Name() string { 16 | return a.IAm 17 | } 18 | 19 | func Make(name string) (named, error) { 20 | return AName{IAm: name}, nil 21 | } 22 | 23 | func DoNotMake(name string) (named, error) { 24 | return nil, assert.AnError 25 | } 26 | 27 | type LRUAssertions struct { 28 | *assert.Assertions 29 | LRU *lru 30 | } 31 | 32 | func NewLRUAssertions(t assert.TestingT, size int) *LRUAssertions { 33 | return &LRUAssertions{ 34 | assert.New(t), 35 | newLRU(size), 36 | } 37 | } 38 | 39 | // PutName adds an item with name to the cache. 40 | func (a *LRUAssertions) PutName(name string) { 41 | item := AName{IAm: name} 42 | a.LRU.Put(item) 43 | } 44 | 45 | // GetName fetches an item with name from the cache; if not present, it 46 | // is added. 47 | func (a *LRUAssertions) GetName(name string) { 48 | item, err := a.LRU.Get(name, Make) 49 | if a.NoError(err) && a.IsType(AName{}, item) { 50 | aName := item.(AName) 51 | a.Equal(aName.Name(), name) 52 | } 53 | } 54 | 55 | // GetPresent fetches an item with name from the cache; if not present, 56 | // it should produce an assertion error. 57 | func (a *LRUAssertions) GetPresent(name string) { 58 | item, err := a.LRU.Get(name, DoNotMake) 59 | if a.NoError(err) && a.IsType(AName{}, item) { 60 | aName := item.(AName) 61 | a.Equal(aName.Name(), name) 62 | } 63 | } 64 | 65 | // GetError tries to fetch an item from the cache, but it should not 66 | // exist, and the resulting error will be caught. 67 | func (a *LRUAssertions) GetError(name string) { 68 | _, err := a.LRU.Get(name, DoNotMake) 69 | a.Error(err) 70 | } 71 | 72 | // LRUHas asserts that an item with name is in the cache. 73 | func (a *LRUAssertions) LRUHas(name string) { 74 | item := a.LRU.Peek(name) 75 | if a.NotNil(item) { 76 | a.Equal(name, item.Name()) 77 | } 78 | } 79 | 80 | // LRUDoesNotHave asserts that no item with name is in the cache. 81 | func (a *LRUAssertions) LRUDoesNotHave(name string) { 82 | item := a.LRU.Peek(name) 83 | a.Nil(item) 84 | } 85 | 86 | // TestLRUSimple tests minimal object presence. 87 | func TestLRUSimple(t *testing.T) { 88 | a := NewLRUAssertions(t, 2) 89 | a.PutName("Sam") 90 | 91 | a.LRUHas("Sam") 92 | a.LRUDoesNotHave("Horton") 93 | } 94 | 95 | // TestLRUAutoInsert tests lru.Get() adding absent items. 96 | func TestLRUAutoInsert(t *testing.T) { 97 | a := NewLRUAssertions(t, 2) 98 | 99 | // Get (and insert) two names 100 | a.GetName("Marvin") 101 | a.GetName("Horton") 102 | 103 | // At this point "Marvin" and "Horton" should both be present 104 | a.LRUHas("Marvin") 105 | a.LRUHas("Horton") 106 | 107 | // Now add one more name; since it is a third one, the oldest 108 | // (Marvin) should be evicted 109 | a.GetName("Sam") 110 | a.LRUDoesNotHave("Marvin") 111 | a.LRUHas("Horton") 112 | a.LRUHas("Sam") 113 | } 114 | 115 | func TestLRUInsertError(t *testing.T) { 116 | a := NewLRUAssertions(t, 2) 117 | 118 | // As before 119 | a.GetName("Marvin") 120 | a.GetName("Horton") 121 | a.LRUHas("Marvin") 122 | a.LRUHas("Horton") 123 | 124 | // Now try to add "Sam", but the add function will return an error 125 | a.GetError("Sam") 126 | // Since no item was added, nothing will be evicted 127 | a.LRUHas("Marvin") 128 | a.LRUHas("Horton") 129 | a.LRUDoesNotHave("Sam") 130 | 131 | // We can call the erroring version of Get() but since the item 132 | // is present it will not fail 133 | a.GetPresent("Marvin") 134 | a.GetPresent("Horton") 135 | } 136 | 137 | // TestLRUOrder tests that getting an item causes it to not get evicted. 138 | func TestLRUOrder(t *testing.T) { 139 | a := NewLRUAssertions(t, 2) 140 | 141 | a.GetName("Marvin") 142 | a.GetName("Horton") 143 | a.LRUHas("Marvin") 144 | a.LRUHas("Horton") 145 | 146 | // Do an *additional* get for Marvin, so he is more-recently-used 147 | a.GetName("Marvin") 148 | 149 | // Now when we add Sam, Horton gets pushed out 150 | a.GetName("Sam") 151 | a.LRUHas("Marvin") 152 | a.LRUDoesNotHave("Horton") 153 | a.LRUHas("Sam") 154 | } 155 | 156 | // TestLRURemoval does simple tests on the Remove call. 157 | func TestLRURemoval(t *testing.T) { 158 | a := NewLRUAssertions(t, 2) 159 | 160 | // Obvious thing #1: 161 | a.GetName("Marvin") 162 | a.LRUHas("Marvin") 163 | a.LRU.Remove("Marvin") 164 | a.LRUDoesNotHave("Marvin") 165 | 166 | // Obvious thing #2: 167 | a.LRU.Remove("Sam") 168 | a.LRUDoesNotHave("Sam") 169 | 170 | // Also if we remove a more-recent thing, the 171 | // older-but-present thing shouldn't get evicted 172 | a.GetName("Marvin") 173 | a.GetName("Horton") 174 | a.LRU.Remove("Horton") 175 | a.GetName("Sam") 176 | a.LRUHas("Marvin") 177 | a.LRUDoesNotHave("Horton") 178 | a.LRUHas("Sam") 179 | } 180 | -------------------------------------------------------------------------------- /cache/work_unit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package cache 5 | 6 | import ( 7 | "github.com/swiftlobste/go-coordinate/coordinate" 8 | ) 9 | 10 | type workUnit struct { 11 | workUnit coordinate.WorkUnit 12 | workSpec *workSpec 13 | } 14 | 15 | func newWorkUnit(upstream coordinate.WorkUnit, workSpec *workSpec) *workUnit { 16 | return &workUnit{ 17 | workUnit: upstream, 18 | workSpec: workSpec, 19 | } 20 | } 21 | 22 | // refresh re-fetches the upstream object if possible. This should be 23 | // called when code strongly expects the cached object is invalid, 24 | // for instance because a method has returned ErrGone. 25 | // 26 | // On success, unit.workUnit points at a newly fetched valid object, 27 | // this object remains the cached work unit for its name in the work 28 | // spec's cache, and returns nil. On error returns the error from 29 | // trying to fetch the work unit. 30 | func (unit *workUnit) refresh() error { 31 | name := unit.workUnit.Name() 32 | var newUnit coordinate.WorkUnit 33 | err := unit.workSpec.withWorkSpec(func(workSpec coordinate.WorkSpec) (err error) { 34 | newUnit, err = workSpec.WorkUnit(name) 35 | return 36 | }) 37 | if err == nil { 38 | unit.workUnit = newUnit 39 | return nil 40 | } 41 | unit.workSpec.invalidateWorkUnit(name) 42 | return err 43 | } 44 | 45 | // withWorkUnit calls some function with the current upstream work 46 | // unit. If that operation returns ErrGone, tries refreshing this 47 | // object then trying again; it may also refresh the work spec and 48 | // namespace. Note that if there is an error doing the refresh, that 49 | // error is discarded, and the original ErrGone is returned (which 50 | // will be more meaningful to the caller). 51 | func (unit *workUnit) withWorkUnit(f func(coordinate.WorkUnit) error) error { 52 | for { 53 | err := f(unit.workUnit) 54 | if err != coordinate.ErrGone { 55 | return err 56 | } 57 | err = unit.refresh() 58 | if err != nil { 59 | return coordinate.ErrGone 60 | } 61 | } 62 | } 63 | 64 | func (unit *workUnit) Name() string { 65 | return unit.workUnit.Name() 66 | } 67 | 68 | func (unit *workUnit) Data() (data map[string]interface{}, err error) { 69 | err = unit.withWorkUnit(func(workUnit coordinate.WorkUnit) (err error) { 70 | data, err = workUnit.Data() 71 | return 72 | }) 73 | return 74 | } 75 | 76 | func (unit *workUnit) WorkSpec() coordinate.WorkSpec { 77 | return unit.workSpec 78 | } 79 | 80 | func (unit *workUnit) Status() (status coordinate.WorkUnitStatus, err error) { 81 | err = unit.withWorkUnit(func(workUnit coordinate.WorkUnit) (err error) { 82 | status, err = workUnit.Status() 83 | return 84 | }) 85 | return 86 | } 87 | 88 | func (unit *workUnit) Meta() (meta coordinate.WorkUnitMeta, err error) { 89 | err = unit.withWorkUnit(func(workUnit coordinate.WorkUnit) (err error) { 90 | meta, err = workUnit.Meta() 91 | return 92 | }) 93 | return 94 | } 95 | 96 | func (unit *workUnit) SetMeta(meta coordinate.WorkUnitMeta) error { 97 | return unit.withWorkUnit(func(workUnit coordinate.WorkUnit) error { 98 | return workUnit.SetMeta(meta) 99 | }) 100 | } 101 | 102 | func (unit *workUnit) Priority() (priority float64, err error) { 103 | err = unit.withWorkUnit(func(workUnit coordinate.WorkUnit) (err error) { 104 | priority, err = workUnit.Priority() 105 | return 106 | }) 107 | return 108 | } 109 | 110 | func (unit *workUnit) SetPriority(priority float64) error { 111 | return unit.withWorkUnit(func(workUnit coordinate.WorkUnit) error { 112 | return workUnit.SetPriority(priority) 113 | }) 114 | } 115 | 116 | func (unit *workUnit) ActiveAttempt() (attempt coordinate.Attempt, err error) { 117 | err = unit.withWorkUnit(func(workUnit coordinate.WorkUnit) (err error) { 118 | attempt, err = workUnit.ActiveAttempt() 119 | return 120 | }) 121 | return 122 | } 123 | 124 | func (unit *workUnit) ClearActiveAttempt() error { 125 | return unit.withWorkUnit(func(workUnit coordinate.WorkUnit) error { 126 | return workUnit.ClearActiveAttempt() 127 | }) 128 | } 129 | 130 | func (unit *workUnit) Attempts() (attempts []coordinate.Attempt, err error) { 131 | err = unit.withWorkUnit(func(workUnit coordinate.WorkUnit) (err error) { 132 | attempts, err = workUnit.Attempts() 133 | return 134 | }) 135 | return 136 | } 137 | 138 | func (unit *workUnit) NumAttempts() (int, error) { 139 | n := 0 140 | var err error 141 | unit.withWorkUnit(func(workUnit coordinate.WorkUnit) (err error) { 142 | n, err = workUnit.NumAttempts() 143 | return err 144 | }) 145 | return n, err 146 | } 147 | -------------------------------------------------------------------------------- /cborrpc/README.md: -------------------------------------------------------------------------------- 1 | Diffeo CBOR-RPC wire format 2 | =========================== 3 | 4 | The Diffeo Coordinate daemon uses a custom wire protocol based on the 5 | [CBOR](http://cbor.io/) data encoding. CBOR's data model is very 6 | similar to JSON's, but with a couple of extensions, including separate 7 | "text" and "byte string" types and tag annotations. 8 | 9 | In all cases the Python Coordinate client code sends and receives 10 | strings as UTF-8-encoded byte strings except as noted. (This may be 11 | an artifact of running it on Python 2, where the default string type 12 | is ASCII-encoded byte string.) It will accept Unicode strings over 13 | the wire but will always send back byte strings. 14 | 15 | Request and response messages in all cases are CBOR-encoded, and the 16 | result sent as a CBOR-in-CBOR byte string with tag 24. The system uses 17 | the following tags: 18 | 19 | * 24 (standard): tags a byte string as holding encoded CBOR 20 | * 37 (standard): tags a length-16 byte string as holding a UUID 21 | * 128 (non-standard): tags a list as holding a Python sequence 22 | 23 | The client and server communicate over a persistent connection. 24 | Requests include a correlation ID. The client may "pipeline" requests 25 | by sending further requests before receiving responses. Only the 26 | client expects to close a connection in the Python code, and it 27 | typically will only do so after all outstanding requests have gotten 28 | responses. 29 | 30 | Requests are mappings where the keys are ASCII byte strings. These 31 | have keys: 32 | 33 | * `id`: correlation ID, typically sequential per connection 34 | * `method`: ASCII byte string, name of RPC method to invoke 35 | * `params`: list of anything, parameters to the method 36 | 37 | Responses are also mappings where the keys are ASCII byte strings. 38 | These have keys: 39 | 40 | * `id`: correlation ID, matches the ID of the request 41 | * `response`: on success, any object, response to the method 42 | * `error`: on failure, a mapping with a single key `message` holding the 43 | error message 44 | 45 | Examples 46 | -------- 47 | 48 | Consider the RPC call `list_work_specs({})`. This would be encoded as: 49 | 50 | { 51 | "id": 1, 52 | "method": "list_work_specs", 53 | "params": [{}] 54 | } 55 | 56 | and sent over the wire as: 57 | 58 | D8 18 Tag 24, CBOR-encoded string follows 59 | 58 25 Byte string of length 37 60 | A3 Map of three pairs 61 | 42 69 64 Byte string "id" 62 | 01 Positive integer 1 63 | 46 6D 65 74 68 6F 64 Byte string "method" 64 | 4F 6C 69 73 74 5F 77 6F 72 6B 5F 73 70 65 63 73 65 | Byte string "list_work_specs" 66 | 46 70 61 72 61 6D 73 Byte string "params" 67 | 81 Array of length 1 68 | A0 Map of 0 pairs 69 | -------------------------------------------------------------------------------- /cborrpc/cleanup.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package cborrpc 5 | 6 | import ( 7 | "errors" 8 | "github.com/mitchellh/mapstructure" 9 | "reflect" 10 | ) 11 | 12 | // CreateParamList tries to match a CBOR-RPC parameter list to a specific 13 | // callable's parameter list. funcv is the reflected method to eventually 14 | // call, and params is the list of parameters from the CBOR-RPC request. 15 | // On success, the return value is a list of parameter values that can be 16 | // passed to funcv.Call(). 17 | func CreateParamList(funcv reflect.Value, params []interface{}) ([]reflect.Value, error) { 18 | funct := funcv.Type() 19 | numParams := funct.NumIn() 20 | if len(params) != numParams { 21 | return nil, errors.New("wrong number of parameters") 22 | } 23 | results := make([]reflect.Value, numParams) 24 | for i := 0; i < numParams; i++ { 25 | paramType := funct.In(i) 26 | paramValue := reflect.New(paramType) 27 | param := paramValue.Interface() 28 | config := mapstructure.DecoderConfig{ 29 | DecodeHook: DecodeBytesAsString, 30 | Result: param, 31 | } 32 | decoder, err := mapstructure.NewDecoder(&config) 33 | if err != nil { 34 | return nil, err 35 | } 36 | err = decoder.Decode(params[i]) 37 | if err != nil { 38 | return nil, err 39 | } 40 | results[i] = paramValue.Elem() 41 | } 42 | return results, nil 43 | } 44 | 45 | // DecodeBytesAsString is a mapstructure decode hook that accepts a 46 | // byte slice where a string is expected. 47 | func DecodeBytesAsString(from, to reflect.Type, data interface{}) (interface{}, error) { 48 | if to.Kind() == reflect.String && from.Kind() == reflect.Slice && from.Elem().Kind() == reflect.Uint8 { 49 | return string(data.([]uint8)), nil 50 | } 51 | return data, nil 52 | } 53 | 54 | // Detuplify removes a tuple wrapper. If obj is a tuple, returns 55 | // the contained slice. If obj is a slice, returns it. Otherwise 56 | // returns failure. 57 | func Detuplify(obj interface{}) ([]interface{}, bool) { 58 | switch t := obj.(type) { 59 | case PythonTuple: 60 | return t.Items, true 61 | case []interface{}: 62 | return t, true 63 | default: 64 | return nil, false 65 | } 66 | } 67 | 68 | // SloppyDetuplify turns any object into a slice. If it is already a 69 | // PythonTuple or a slice, returns the slice as Detuplify; otherwise 70 | // packages up obj into a new slice. This never fails. 71 | func SloppyDetuplify(obj interface{}) []interface{} { 72 | if slice, ok := Detuplify(obj); ok { 73 | return slice 74 | } 75 | return []interface{}{obj} 76 | } 77 | 78 | // Destringify tries to turn any object into a string. If it is a 79 | // string or byte slice, returns the string and true; otherwise returns 80 | // empty string and false. 81 | func Destringify(obj interface{}) (string, bool) { 82 | switch s := obj.(type) { 83 | case string: 84 | return s, true 85 | case []byte: 86 | return string(s), true 87 | default: 88 | return "", false 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /cmd/coordbench/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016-2017 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | // Package coordbench provides a load-generation tool for Coordinate. 5 | package main 6 | 7 | import ( 8 | "github.com/swiftlobste/go-coordinate/backend" 9 | "github.com/swiftlobste/go-coordinate/coordinate" 10 | "github.com/satori/go.uuid" 11 | "github.com/urfave/cli" 12 | "runtime" 13 | "sync" 14 | "time" 15 | ) 16 | 17 | type benchWork struct { 18 | Coordinate coordinate.Coordinate 19 | Namespace coordinate.Namespace 20 | WorkSpec coordinate.WorkSpec 21 | Concurrency int 22 | } 23 | 24 | func (bench *benchWork) Run(runner func()) { 25 | wg := sync.WaitGroup{} 26 | wg.Add(bench.Concurrency) 27 | for i := 0; i < bench.Concurrency; i++ { 28 | go func() { 29 | defer wg.Done() 30 | runner() 31 | }() 32 | } 33 | wg.Wait() 34 | } 35 | 36 | var bench benchWork 37 | 38 | var addUnits = cli.Command{ 39 | Name: "add", 40 | Usage: "create many work units", 41 | Flags: []cli.Flag{ 42 | cli.IntFlag{ 43 | Name: "count", 44 | Value: 100, 45 | Usage: "number of work units to create", 46 | }, 47 | }, 48 | Action: func(c *cli.Context) { 49 | count := c.Int("count") 50 | numbers := make(chan int) 51 | go func() { 52 | for i := 1; i <= count; i++ { 53 | numbers <- i 54 | } 55 | close(numbers) 56 | }() 57 | bench.Run(func() { 58 | for <-numbers != 0 { 59 | name := uuid.NewV4().String() 60 | bench.WorkSpec.AddWorkUnit(name, map[string]interface{}{}, coordinate.WorkUnitMeta{}) 61 | } 62 | }) 63 | }, 64 | } 65 | 66 | var doWork = cli.Command{ 67 | Name: "do", 68 | Usage: "do work as long as there is more", 69 | Flags: []cli.Flag{ 70 | cli.IntFlag{ 71 | Name: "batch", 72 | Value: 100, 73 | Usage: "request this many attempts in one batch", 74 | }, 75 | cli.DurationFlag{ 76 | Name: "delay", 77 | Value: 0, 78 | Usage: "wait this long per work unit before completion", 79 | }, 80 | }, 81 | Action: func(c *cli.Context) { 82 | batch := c.Int("batch") 83 | delay := c.Duration("delay") 84 | name := uuid.NewV4().String() 85 | parent, err := bench.Namespace.Worker(name) 86 | if err != nil { 87 | return 88 | } 89 | bench.Run(func() { 90 | name := uuid.NewV4().String() 91 | worker, err := bench.Namespace.Worker(name) 92 | if err != nil { 93 | return 94 | } 95 | err = worker.SetParent(parent) 96 | if err != nil { 97 | return 98 | } 99 | 100 | for { 101 | attempts, err := worker.RequestAttempts(coordinate.AttemptRequest{NumberOfWorkUnits: batch}) 102 | if err != nil || len(attempts) == 0 { 103 | break 104 | } 105 | for _, attempt := range attempts { 106 | time.Sleep(delay) 107 | _ = attempt.Finish(nil) 108 | } 109 | } 110 | _ = worker.Deactivate() 111 | }) 112 | }, 113 | } 114 | 115 | var clear = cli.Command{ 116 | Name: "clear", 117 | Usage: "delete all of the work units", 118 | Action: func(c *cli.Context) { 119 | bench.WorkSpec.DeleteWorkUnits(coordinate.WorkUnitQuery{}) 120 | }, 121 | } 122 | 123 | func main() { 124 | backend := backend.Backend{Implementation: "memory"} 125 | app := cli.NewApp() 126 | app.Usage = "benchmark the Coordinate job queue system" 127 | app.Flags = []cli.Flag{ 128 | cli.GenericFlag{ 129 | Name: "backend", 130 | Value: &backend, 131 | Usage: "impl:[address] of Coordinate backend", 132 | }, 133 | cli.StringFlag{ 134 | Name: "namespace", 135 | Usage: "Coordinate namespace name", 136 | }, 137 | cli.IntFlag{ 138 | Name: "concurrency", 139 | Value: runtime.NumCPU(), 140 | Usage: "run this many jobs in parallel", 141 | }, 142 | } 143 | app.Commands = []cli.Command{ 144 | addUnits, 145 | doWork, 146 | clear, 147 | } 148 | app.Before = func(c *cli.Context) (err error) { 149 | bench.Coordinate, err = backend.Coordinate() 150 | if err != nil { 151 | return 152 | } 153 | 154 | bench.Namespace, err = bench.Coordinate.Namespace(c.String("namespace")) 155 | if err != nil { 156 | return 157 | } 158 | 159 | bench.WorkSpec, err = bench.Namespace.SetWorkSpec(map[string]interface{}{ 160 | "name": "spec", 161 | }) 162 | if err != nil { 163 | return 164 | } 165 | 166 | bench.Concurrency = c.Int("concurrency") 167 | 168 | return 169 | } 170 | app.RunAndExitOnError() 171 | } 172 | -------------------------------------------------------------------------------- /cmd/coordinated/http.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package main 5 | 6 | import ( 7 | "net/http" 8 | "os" 9 | 10 | "github.com/swiftlobste/go-coordinate/coordinate" 11 | "github.com/swiftlobste/go-coordinate/restserver" 12 | "github.com/google/go-cloud/requestlog" 13 | "github.com/gorilla/mux" 14 | "github.com/prometheus/client_golang/prometheus/promhttp" 15 | "github.com/sirupsen/logrus" 16 | "github.com/urfave/negroni" 17 | ) 18 | 19 | // HTTP serves HTTP coordinated connections. 20 | type HTTP struct { 21 | coord coordinate.Coordinate 22 | laddr string 23 | } 24 | 25 | // Serve runs an HTTP server on the specified local address. This serves 26 | // connections forever, and probably wants to be run in a goroutine. Panics on 27 | // any error in the initial setup or in accepting connections. 28 | func (h *HTTP) Serve(logRequests bool, logFormat string, logger *logrus.Logger) { 29 | r := mux.NewRouter() 30 | r.PathPrefix("/").Subrouter() 31 | restserver.PopulateRouter(r, h.coord) 32 | r.Handle("/metrics", promhttp.Handler()) 33 | 34 | n := negroni.New() 35 | n.Use(negroni.NewRecovery()) 36 | 37 | // Wrap the root handler in a logger if desired. 38 | var handler http.Handler = r 39 | if logRequests { 40 | handler = logWrapper(logFormat, logger, handler) 41 | } 42 | n.UseHandler(handler) 43 | 44 | http.ListenAndServe(h.laddr, n) 45 | } 46 | 47 | // logWrapper creates a wrapping logger for the given handler. It is setup this 48 | // way rather than conforming to the negroni paradigm because the API fo the 49 | // requestlog package, which this uses, is not directly compatible. 50 | func logWrapper(logFormat string, logger *logrus.Logger, inner http.Handler) http.Handler { 51 | var reqLog requestlog.Logger 52 | // See the following documentation for more information on formats: 53 | // https://godoc.org/github.com/google/go-cloud/requestlog 54 | switch logFormat { 55 | case "ncsa": 56 | // Combined Log Format, as used by Apache. 57 | reqLog = requestlog.NewNCSALogger(os.Stdout, func(err error) { 58 | logger.WithError(err).Error("error writing NCSA log") 59 | }) 60 | 61 | case "stackdriver": 62 | // As expected by Stackdriver Logging. 63 | reqLog = requestlog.NewStackdriverLogger(os.Stdout, func(err error) { 64 | logger.WithError(err).Error("error writing Stackdriver log") 65 | }) 66 | 67 | default: 68 | logger.WithField("format", logFormat).Fatal("unrecognized log format") 69 | } 70 | 71 | return requestlog.NewHandler(reqLog, inner) 72 | } 73 | -------------------------------------------------------------------------------- /cmd/coordinated/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2017 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | // Package coordinated provides a wire-compatible reimplementation of 5 | // the Diffeo Coordinate daemon. This is intended to be fully 6 | // compatible with the existing Coordinate toolset at 7 | // https://github.com/diffeo/coordinate. This is purely a server-side 8 | // daemon; it does not include application code or a worker 9 | // implementation. 10 | package main 11 | 12 | import ( 13 | "context" 14 | "flag" 15 | "io/ioutil" 16 | "os" 17 | "time" 18 | 19 | "github.com/swiftlobste/go-coordinate/backend" 20 | "github.com/swiftlobste/go-coordinate/cache" 21 | "github.com/sirupsen/logrus" 22 | "gopkg.in/yaml.v2" 23 | ) 24 | 25 | func main() { 26 | var err error 27 | 28 | cborRPCBind := flag.String("cborrpc", ":5932", 29 | "[ip]:port for CBOR-RPC interface") 30 | httpBind := flag.String("http", ":5980", 31 | "[ip]:port for HTTP REST interface") 32 | backend := backend.Backend{Implementation: "memory", Address: ""} 33 | flag.Var(&backend, "backend", "impl[:address] of the storage backend") 34 | config := flag.String("config", "", "global configuration YAML file") 35 | logRequests := flag.Bool("log-requests", false, "log all requests") 36 | logMetrics := flag.Bool("log-metrics", false, "log metrics") 37 | logFormat := flag.String("log-format", "ncsa", "request log format [ncsa stackdriver]") 38 | metricPeriod := flag.String("metric-period", "2m", "time period between each metric update") 39 | flag.Parse() 40 | 41 | var gConfig map[string]interface{} 42 | if *config != "" { 43 | gConfig, err = loadConfigYaml(*config) 44 | if err != nil { 45 | logrus.WithFields(logrus.Fields{ 46 | "err": err, 47 | }).Fatal("Could not load YAML configuration") 48 | return 49 | } 50 | } 51 | 52 | coordinate, err := backend.Coordinate() 53 | if err != nil { 54 | logrus.WithFields(logrus.Fields{ 55 | "err": err, 56 | }).Fatal("Could not create Coordinate backend") 57 | return 58 | } 59 | coordinate = cache.New(coordinate) 60 | 61 | logrus.SetLevel(logrus.DebugLevel) 62 | logrus.SetOutput(ioutil.Discard) // default unless log flags are passed 63 | 64 | reqLogger := logrus.StandardLogger() 65 | if *logRequests { 66 | reqLogger.Out = os.Stderr 67 | } 68 | 69 | metricsLogger := logrus.StandardLogger() 70 | if *logMetrics { 71 | metricsLogger.Out = os.Stderr 72 | } 73 | 74 | period, err := time.ParseDuration(*metricPeriod) 75 | if err != nil { 76 | return 77 | } 78 | 79 | go ServeCBORRPC(coordinate, gConfig, "tcp", *cborRPCBind, reqLogger) 80 | http := HTTP{ 81 | coord: coordinate, 82 | laddr: *httpBind, 83 | } 84 | go http.Serve(*logRequests, *logFormat, reqLogger) 85 | go Observe(context.Background(), coordinate, period, metricsLogger) 86 | 87 | select {} 88 | } 89 | 90 | func loadConfigYaml(filename string) (map[string]interface{}, error) { 91 | var result map[string]interface{} 92 | var err error 93 | var bytes []byte 94 | bytes, err = ioutil.ReadFile(filename) 95 | if err == nil { 96 | err = yaml.Unmarshal(bytes, &result) 97 | } 98 | return result, err 99 | } 100 | -------------------------------------------------------------------------------- /cmd/coordinated/metrics.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2017 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package main 5 | 6 | import ( 7 | "context" 8 | "math" 9 | "time" 10 | 11 | "github.com/swiftlobste/go-coordinate/coordinate" 12 | "github.com/prometheus/client_golang/prometheus" 13 | "github.com/sirupsen/logrus" 14 | ) 15 | 16 | var ( 17 | workUnitsNumber = prometheus.NewHistogram( 18 | prometheus.HistogramOpts{ 19 | Namespace: "coordinate", 20 | Name: "summary_seconds", 21 | Help: "Seconds required to gather coordinate summary", 22 | Buckets: prometheus.ExponentialBuckets(math.Pow(2, -5), 2, 12), 23 | }) 24 | 25 | summarySeconds = prometheus.NewGaugeVec( 26 | prometheus.GaugeOpts{ 27 | Namespace: "coordinate", 28 | Name: "work_units", 29 | Help: "Number of coordinate work specs", 30 | }, 31 | []string{ 32 | "namespace", 33 | "work_spec", 34 | "status", 35 | }) 36 | ) 37 | 38 | func init() { 39 | prometheus.MustRegister(summarySeconds) 40 | prometheus.MustRegister(workUnitsNumber) 41 | } 42 | 43 | // Observe repeatedly calls Summarize() on coordinate in an infinite loop, and 44 | // observes each SummaryRecord's fields on a prometheus GaugeVec, and the 45 | // resultant time duration on a prometheus Histogram. 46 | func Observe( 47 | ctx context.Context, 48 | coord coordinate.Coordinate, 49 | period time.Duration, 50 | log *logrus.Logger, 51 | ) { 52 | for { 53 | select { 54 | case <-ctx.Done(): 55 | return 56 | case <-time.After(period): 57 | t0 := time.Now() 58 | summary, err := coord.Summarize() 59 | if err != nil { 60 | log.Error(err) 61 | break 62 | } 63 | workUnitsNumber.Observe(time.Since(t0).Seconds()) 64 | for _, record := range summary { 65 | status, err := record.Status.MarshalText() 66 | if err != nil { 67 | log.Error(err) 68 | break 69 | } 70 | summarySeconds.With(prometheus.Labels{ 71 | "namespace": record.Namespace, 72 | "work_spec": record.WorkSpec, 73 | "status": string(status), 74 | }).Set(float64(record.Count)) 75 | } 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /cmd/demoworker/demoworker.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | // Package demoworker provides a complete demonstration Coordinate 5 | // application. This defines two work specs: "generator" runs once 6 | // per 5 seconds, and creates several work units in the "runner" work 7 | // spec, which just prints out the work unit keys. 8 | package main 9 | 10 | import ( 11 | "context" 12 | "flag" 13 | "fmt" 14 | "strings" 15 | 16 | "github.com/swiftlobste/go-coordinate/backend" 17 | "github.com/swiftlobste/go-coordinate/coordinate" 18 | "github.com/swiftlobste/go-coordinate/worker" 19 | "github.com/mitchellh/mapstructure" 20 | ) 21 | 22 | func main() { 23 | backend := backend.Backend{Implementation: "memory", Address: ""} 24 | flag.Var(&backend, "backend", "impl[:address] of the storage backend") 25 | bootstrap := flag.Bool("bootstrap", true, "Create initial work specs") 26 | nsName := flag.String("namespace", "", "Coordinate namespace name") 27 | flag.Parse() 28 | 29 | coordinateRoot, err := backend.Coordinate() 30 | if err != nil { 31 | panic(err) 32 | } 33 | 34 | namespace, err := coordinateRoot.Namespace(*nsName) 35 | if err != nil { 36 | panic(err) 37 | } 38 | 39 | if *bootstrap { 40 | err = createWorkSpecs(namespace) 41 | if err != nil { 42 | panic(err) 43 | } 44 | } 45 | 46 | tasks := map[string]func(context.Context, []coordinate.Attempt){ 47 | "generator": runGenerator, 48 | "runner": runRunner, 49 | } 50 | 51 | worker := worker.Worker{ 52 | Namespace: namespace, 53 | Tasks: tasks, 54 | } 55 | worker.Run(context.Background()) 56 | } 57 | 58 | func createWorkSpecs(namespace coordinate.Namespace) error { 59 | var err error 60 | _, err = namespace.SetWorkSpec(map[string]interface{}{ 61 | "name": "generator", 62 | "runtime": "go", 63 | "task": "generator", 64 | "continuous": true, 65 | "interval": 5, 66 | "then": "runner", 67 | }) 68 | if err != nil { 69 | return err 70 | } 71 | 72 | _, err = namespace.SetWorkSpec(map[string]interface{}{ 73 | "name": "runner", 74 | "runtime": "go", 75 | "task": "runner", 76 | "max_getwork": 10, 77 | }) 78 | if err != nil { 79 | return err 80 | } 81 | 82 | return nil 83 | } 84 | 85 | func runGenerator(ctx context.Context, attempts []coordinate.Attempt) { 86 | for _, attempt := range attempts { 87 | // Trying to stop? 88 | select { 89 | case <-ctx.Done(): 90 | _ = attempt.Fail(nil) 91 | continue 92 | default: 93 | } 94 | 95 | // Generate several more work units 96 | kvps := make([]interface{}, 100) 97 | for n := range kvps { 98 | name := fmt.Sprintf("%s_%03d", attempt.WorkUnit().Name(), n) 99 | data := map[string]interface{}{ 100 | "s": attempt.WorkUnit().Name(), 101 | "n": n, 102 | } 103 | kvps[n] = []interface{}{name, data} 104 | } 105 | _ = attempt.Finish(map[string]interface{}{ 106 | "output": kvps, 107 | }) 108 | } 109 | } 110 | 111 | func runRunner(ctx context.Context, attempts []coordinate.Attempt) { 112 | dead := make(map[int]struct{}) 113 | 114 | // We'll check the "done" flag in a couple of places; this is 115 | // probably, in practice, excessive for the amount of work 116 | // happening here, but is still good practice 117 | select { 118 | case <-ctx.Done(): 119 | for i, attempt := range attempts { 120 | if _, isDead := dead[i]; !isDead { 121 | _ = attempt.Fail(nil) 122 | } 123 | } 124 | return 125 | default: 126 | } 127 | 128 | var err error 129 | found := make(map[string][]int) 130 | for i, attempt := range attempts { 131 | var data map[string]interface{} 132 | data, err = attempt.WorkUnit().Data() 133 | var unit struct { 134 | S string 135 | N int 136 | } 137 | if err == nil { 138 | err = mapstructure.Decode(data, &unit) 139 | } 140 | if err == nil { 141 | found[unit.S] = append(found[unit.S], int(unit.N)) 142 | } 143 | if err != nil { 144 | fmt.Printf(" %v: %v\n", attempt.WorkUnit().Name(), err.Error()) 145 | _ = attempt.Fail(map[string]interface{}{ 146 | "traceback": err.Error(), 147 | }) 148 | dead[i] = struct{}{} 149 | err = nil 150 | } 151 | } 152 | 153 | select { 154 | case <-ctx.Done(): 155 | for i, attempt := range attempts { 156 | if _, isDead := dead[i]; !isDead { 157 | _ = attempt.Fail(nil) 158 | } 159 | } 160 | return 161 | default: 162 | } 163 | 164 | var lines []string 165 | lines = append(lines, "Runner found") 166 | for s, is := range found { 167 | lines = append(lines, fmt.Sprintf(" %v -> %v", s, is)) 168 | } 169 | if len(dead) > 0 { 170 | lines = append(lines, fmt.Sprintf(" rejected %d work units", len(dead))) 171 | } 172 | fmt.Printf("%s\n", strings.Join(lines, "\n")) 173 | 174 | select { 175 | case <-ctx.Done(): 176 | for i, attempt := range attempts { 177 | if _, isDead := dead[i]; !isDead { 178 | _ = attempt.Fail(nil) 179 | } 180 | } 181 | return 182 | default: 183 | } 184 | 185 | for i, attempt := range attempts { 186 | if _, isDead := dead[i]; !isDead { 187 | _ = attempt.Finish(nil) 188 | } 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /coordinate/coordinatetest/benchmarks.notgo: -------------------------------------------------------------------------------- 1 | // -*- go -*- 2 | 3 | // Single-backend performance benchmarks. 4 | // 5 | // These used to be in performance.go; they do not fit in the Suite 6 | // setup (Suite _only_ seems to support tests), and in any case running 7 | // benchmarks via the dedicated coordbench tool is probably better. 8 | // 9 | // These tests were extracted from performance.go. This code 10 | // currently does not compile. 11 | // 12 | // Copyright 2015-2017 Diffeo, Inc. 13 | // This software is released under an MIT/X11 open source license. 14 | 15 | package coordinatetest 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | // BenchmarkWorkUnitCreation times simply creating a significant 22 | // number of work units in a single work spec. 23 | func BenchmarkWorkUnitCreation(b *testing.B) { 24 | namespace, err := Coordinate.Namespace("BenchmarkWorkUnitCreation") 25 | if err != nil { 26 | b.Fatalf("error creating namespace: %+v", err) 27 | } 28 | defer namespace.Destroy() 29 | 30 | spec, err := namespace.SetWorkSpec(map[string]interface{}{ 31 | "name": "spec", 32 | }) 33 | if err != nil { 34 | b.Fatalf("error creating work spec: %+v", err) 35 | } 36 | 37 | counter := make(chan int) 38 | stopCounter := make(chan struct{}) 39 | go count(counter, stopCounter) 40 | 41 | b.RunParallel(func(pb *testing.PB) { 42 | for pb.Next() { 43 | i := <-counter 44 | spec.AddWorkUnit(fmt.Sprintf("u%v", i), map[string]interface{}{}, coordinate.WorkUnitMeta{}) 45 | } 46 | }) 47 | close(stopCounter) 48 | } 49 | 50 | // BenchmarkWorkUnitExecution benchmarks retrieving and completing work 51 | // units. 52 | func BenchmarkWorkUnitExecution(b *testing.B) { 53 | namespace, err := Coordinate.Namespace("BenchmarkWorkUnitExecution") 54 | if err != nil { 55 | b.Fatalf("error creating namespace: %+v", err) 56 | } 57 | defer namespace.Destroy() 58 | 59 | // Create the work spec 60 | spec, err := namespace.SetWorkSpec(map[string]interface{}{ 61 | "name": "spec", 62 | }) 63 | if err != nil { 64 | b.Fatalf("error creating work spec: %+v", err) 65 | } 66 | createWorkUnits(spec, b.N, b) 67 | 68 | // Do some work 69 | b.RunParallel(func(pb *testing.PB) { 70 | worker := createWorker(namespace) 71 | for pb.Next() { 72 | attempts, err := worker.RequestAttempts(coordinate.AttemptRequest{}) 73 | if err != nil { 74 | panic(err) 75 | } 76 | for _, attempt := range attempts { 77 | err = attempt.Finish(nil) 78 | if err != nil { 79 | panic(err) 80 | } 81 | } 82 | } 83 | }) 84 | } 85 | 86 | // BenchmarkMultiAttempts times executing work with multiple attempts 87 | // coming back from one attempt. 88 | func BenchmarkMultiAttempts(b *testing.B) { 89 | namespace, err := Coordinate.Namespace("BenchmarkMultiAttempts") 90 | if err != nil { 91 | b.Fatalf("error creating namespace: %+v", err) 92 | } 93 | defer namespace.Destroy() 94 | 95 | // Create the work spec 96 | spec, err := namespace.SetWorkSpec(map[string]interface{}{ 97 | "name": "spec", 98 | }) 99 | if err != nil { 100 | b.Fatalf("error creating work spec: %+v", err) 101 | } 102 | createWorkUnits(spec, b.N, b) 103 | 104 | b.RunParallel(func(pb *testing.PB) { 105 | worker := createWorker(namespace) 106 | for pb.Next() { 107 | attempts, err := worker.RequestAttempts(coordinate.AttemptRequest{ 108 | NumberOfWorkUnits: 20, 109 | }) 110 | if err != nil { 111 | panic(err) 112 | } 113 | // We are required to drain pb.Next() so keep 114 | // going even if we run out of work...just finish 115 | // whatever attempts we are given 116 | for _, attempt := range attempts { 117 | err = attempt.Finish(nil) 118 | if err != nil { 119 | panic(err) 120 | } 121 | } 122 | } 123 | }) 124 | } 125 | 126 | // BenchmarkUnitOutput times work unit execution, where a first work spec 127 | // creates work units in a second. 128 | func BenchmarkUnitOutput(b *testing.B) { 129 | namespace, err := Coordinate.Namespace("BenchmarkUnitOutput") 130 | if err != nil { 131 | b.Fatalf("error creating namespace: %+v", err) 132 | } 133 | defer namespace.Destroy() 134 | 135 | // Create the work specs 136 | one, err := namespace.SetWorkSpec(map[string]interface{}{ 137 | "name": "one", 138 | "then": "two", 139 | }) 140 | if err != nil { 141 | b.Fatalf("error creating work spec: %+v", err) 142 | } 143 | _, err = namespace.SetWorkSpec(map[string]interface{}{ 144 | "name": "two", 145 | }) 146 | if err != nil { 147 | b.Fatalf("error creating work spec: %+v", err) 148 | } 149 | 150 | createWorkUnits(one, b.N, b) 151 | 152 | b.RunParallel(func(pb *testing.PB) { 153 | worker := createWorker(namespace) 154 | for pb.Next() { 155 | attempts, err := worker.RequestAttempts(coordinate.AttemptRequest{}) 156 | if err != nil { 157 | panic(err) 158 | } 159 | for _, attempt := range attempts { 160 | unit := attempt.WorkUnit() 161 | err = attempt.Finish(map[string]interface{}{ 162 | "output": []string{unit.Name()}, 163 | }) 164 | if err != nil { 165 | panic(err) 166 | } 167 | } 168 | } 169 | }) 170 | } 171 | -------------------------------------------------------------------------------- /coordinate/coordinatetest/coordinatetest.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2017 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | // Package coordinatetest provides generic functional tests for the 5 | // Coordinate interface. A typical backend test module needs to wrap 6 | // Suite to create its backend: 7 | // 8 | // package mybackend 9 | // 10 | // import ( 11 | // "testing" 12 | // "github.com/swiftlobste/go-coordinate/coordinate/coordinatetest" 13 | // "github.com/stretchr/testify/suite" 14 | // ) 15 | // 16 | // // Suite is the per-backend generic test suite. 17 | // type Suite struct{ 18 | // coordinatetest.Suite 19 | // } 20 | // 21 | // // SetupSuite does global setup for the test suite. 22 | // func (s *Suite) SetupSuite() { 23 | // s.Suite.SetupSuite() 24 | // s.Coordinate = NewWithClock(s.Clock) 25 | // } 26 | // 27 | // // TestCoordinate runs the Coordinate generic tests. 28 | // func TestCoordinate(t *testing.T) { 29 | // suite.Run(t, &Suite{}) 30 | // } 31 | package coordinatetest 32 | 33 | import ( 34 | "github.com/benbjohnson/clock" 35 | "github.com/swiftlobste/go-coordinate/coordinate" 36 | "github.com/stretchr/testify/suite" 37 | ) 38 | 39 | // Suite is the generic Coordinate backend test suite. 40 | type Suite struct { 41 | suite.Suite 42 | 43 | // Clock contains the alternate time source to be used in tests. It 44 | // is pre-initialized to a mock clock. 45 | Clock *clock.Mock 46 | 47 | // Coordinate contains the top-level interface to the backend under 48 | // test. It is set by importing packages. 49 | Coordinate coordinate.Coordinate 50 | } 51 | 52 | // SetupSuite does one-time initialization for the test suite. 53 | func (s *Suite) SetupSuite() { 54 | s.Clock = clock.NewMock() 55 | } 56 | -------------------------------------------------------------------------------- /coordinate/coordinatetest/performance.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2017 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package coordinatetest 5 | 6 | import ( 7 | "fmt" 8 | "github.com/swiftlobste/go-coordinate/coordinate" 9 | "math/rand" 10 | "strings" 11 | "sync" 12 | ) 13 | 14 | // ------------------------------------------------------------------------ 15 | // Concurrent test execution helpers: 16 | 17 | // pooled calls an execution function workerCount times in separate 18 | // goroutines and waits for them to finish. The worker function is 19 | // responsible for doing its own work and exiting when done. Returns 20 | // a slice of panic objects, or nil if all were successful. 21 | func pooled(f func()) []interface{} { 22 | wait := sync.WaitGroup{} 23 | count := 8 24 | wait.Add(count) 25 | errors := make(chan interface{}, count) 26 | for seq := 0; seq < count; seq++ { 27 | go func() { 28 | defer func() { 29 | if err := recover(); err != nil { 30 | errors <- err 31 | } 32 | wait.Done() 33 | }() 34 | f() 35 | }() 36 | } 37 | wait.Wait() 38 | close(errors) 39 | var result []interface{} 40 | for err := range errors { 41 | result = append(result, err) 42 | } 43 | return result 44 | } 45 | 46 | // count generates a stream of integers to a channel, until told to stop. 47 | func count(val chan<- int, stop <-chan struct{}) { 48 | for i := 0; ; i++ { 49 | select { 50 | case val <- i: 51 | case <-stop: 52 | close(val) 53 | return 54 | } 55 | } 56 | } 57 | 58 | // ------------------------------------------------------------------------ 59 | // Coordinate setup helpers: 60 | func (s *Suite) createWorkUnits(spec coordinate.WorkSpec, n int) { 61 | for i := 0; i < n; i++ { 62 | _, err := spec.AddWorkUnit(fmt.Sprintf("u%v", i), map[string]interface{}{}, coordinate.WorkUnitMeta{}) 63 | s.NoError(err) 64 | } 65 | } 66 | 67 | // createWorker creates a worker with a random name. If there is a 68 | // failure creating the worker, panics. 69 | func createWorker(namespace coordinate.Namespace) coordinate.Worker { 70 | // Construct a random worker name: 71 | workerName := strings.Map(func(rune) rune { 72 | return rune('A' + rand.Intn(26)) 73 | }, "12345678") 74 | worker, err := namespace.Worker(workerName) 75 | if err != nil { 76 | panic(err) 77 | } 78 | return worker 79 | } 80 | 81 | // ------------------------------------------------------------------------ 82 | // Concurrent execution tests: 83 | 84 | // TestConcurrentExecution creates 100 work units and runs them 85 | // concurrently, testing that each gets executed only once. 86 | func (s *Suite) TestConcurrentExecution() { 87 | sts := SimpleTestSetup{ 88 | NamespaceName: "TestConcurrentExecution", 89 | WorkSpecName: "spec", 90 | } 91 | sts.SetUp(s) 92 | defer sts.TearDown(s) 93 | 94 | numUnits := 100 95 | s.createWorkUnits(sts.WorkSpec, numUnits) 96 | results := make(chan map[string]string, 8) 97 | panics := pooled(func() { 98 | worker := createWorker(sts.Namespace) 99 | me := worker.Name() 100 | done := make(map[string]string) 101 | for { 102 | attempts, err := worker.RequestAttempts(coordinate.AttemptRequest{}) 103 | if !s.NoError(err) { 104 | return 105 | } 106 | if len(attempts) == 0 { 107 | results <- done 108 | return 109 | } 110 | for _, attempt := range attempts { 111 | done[attempt.WorkUnit().Name()] = me 112 | err = attempt.Finish(nil) 113 | if !s.NoError(err) { 114 | return 115 | } 116 | } 117 | } 118 | }) 119 | s.Empty(panics) 120 | 121 | close(results) 122 | allResults := make(map[string]string) 123 | for result := range results { 124 | for name, seq := range result { 125 | if other, dup := allResults[name]; dup { 126 | s.Fail("duplicate work unit", 127 | "work unit %v done by both %v and %v", name, other, seq) 128 | } else { 129 | allResults[name] = seq 130 | } 131 | } 132 | } 133 | for i := 0; i < numUnits; i++ { 134 | name := fmt.Sprintf("u%v", i) 135 | s.Contains(allResults, name, 136 | "work unit %v not done by anybody", name) 137 | } 138 | } 139 | 140 | // TestAddSameUnit creates the same work unit many times in parallel 141 | // and checks for errors. 142 | func (s *Suite) TestAddSameUnit() { 143 | sts := SimpleTestSetup{ 144 | NamespaceName: "TestAddSameUnit", 145 | WorkSpecName: "spec", 146 | } 147 | sts.SetUp(s) 148 | defer sts.TearDown(s) 149 | 150 | numUnits := 1000 151 | panics := pooled(func() { 152 | for i := 0; i < numUnits; i++ { 153 | unit := fmt.Sprintf("unit%03d", i) 154 | _, err := sts.WorkSpec.AddWorkUnit(unit, map[string]interface{}{}, coordinate.WorkUnitMeta{}) 155 | s.NoError(err) 156 | } 157 | }) 158 | s.Empty(panics) 159 | } 160 | -------------------------------------------------------------------------------- /coordinate/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package coordinate 5 | 6 | import ( 7 | "errors" 8 | "fmt" 9 | ) 10 | 11 | // ErrNoWorkSpecName is returned as an error from functions that 12 | // create a work spec from a map, but cannot find "name" in the map. 13 | var ErrNoWorkSpecName = errors.New("No 'name' key in work spec") 14 | 15 | // ErrBadWorkSpecName is returned as an error from functions that 16 | // create a work spec from a map, but find a "name" key that is not a 17 | // string. 18 | var ErrBadWorkSpecName = errors.New("Work spec 'name' must be a string") 19 | 20 | // ErrChangedName is returned from WorkSpec.SetData() if it tries to 21 | // change the name of the work spec. 22 | var ErrChangedName = errors.New("Cannot change work spec 'name'") 23 | 24 | // ErrLostLease is returned as an error from Attempt.Renew() if this 25 | // is no longer the active attempt. 26 | var ErrLostLease = errors.New("No longer the active attempt") 27 | 28 | // ErrNotPending is returned as an error from Attempt methods that try 29 | // to change an Attempt's status if the status is not Pending. 30 | var ErrNotPending = errors.New("Attempt is not pending") 31 | 32 | // ErrCannotBecomeContinuous is returned as an error from 33 | // WorkSpec.SetMeta() if the work spec was not defined with the 34 | // "continuous" flag set. 35 | var ErrCannotBecomeContinuous = errors.New("Cannot set work spec to continuous") 36 | 37 | // ErrWrongBackend is returned from functions that take two different 38 | // coordinate objects and combine them if the two objects come from 39 | // different backends. This is impossible in ordinary usage. 40 | var ErrWrongBackend = errors.New("Cannot combine coordinate objects from different backends") 41 | 42 | // ErrNoWork is returned from scheduler calls when there is no work to 43 | // do. 44 | var ErrNoWork = errors.New("No work to do") 45 | 46 | // ErrWorkUnitNotList is returned from ExtractAddWorkUnitItem if a 47 | // work unit as specified in a work unit's "output" field is not a 48 | // list. 49 | var ErrWorkUnitNotList = errors.New("work unit not a list") 50 | 51 | // ErrWorkUnitTooShort is returned from ExtractAddWorkUnitItem if a 52 | // work unit as specified in a work unit's "output" field has fewer 53 | // than 2 items in its list. 54 | var ErrWorkUnitTooShort = errors.New("too few parameters to work unit") 55 | 56 | // ErrBadPriority is returned from ExtractAddWorkUnitItem if a 57 | // metadata dictionary is supplied and it has a "priority" key but 58 | // that is not a number. 59 | var ErrBadPriority = errors.New("priority must be a number") 60 | 61 | // ErrGone is returned from various points in the API if the object is 62 | // determined to not exist, for instance because another caller in a 63 | // shared database has deleted it. It makes no commitment as to which 64 | // object has been deleted; a work unit operation can return ErrGone if 65 | // its entire work spec is gone. 66 | var ErrGone = errors.New("Object no longer exists") 67 | 68 | // ErrNoSuchWorkSpec is returned by Namespace.WorkSpec() and similar 69 | // functions that want to look up a work spec, but cannot find it. 70 | type ErrNoSuchWorkSpec struct { 71 | Name string 72 | } 73 | 74 | func (err ErrNoSuchWorkSpec) Error() string { 75 | return fmt.Sprintf("No such work spec %v", err.Name) 76 | } 77 | 78 | // ErrNoSuchWorkUnit is returned by WorkSpec.WorkUnit() and similar 79 | // functions that want to look up a work unit by name, but cannot find 80 | // it. 81 | type ErrNoSuchWorkUnit struct { 82 | Name string 83 | } 84 | 85 | func (err ErrNoSuchWorkUnit) Error() string { 86 | return fmt.Sprintf("No such work unit %q", err.Name) 87 | } 88 | -------------------------------------------------------------------------------- /coordinate/helpers_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package coordinate 5 | 6 | import ( 7 | "github.com/stretchr/testify/assert" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | // now is a reference datestamp for tests. 13 | var now = time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC) 14 | 15 | func TestOutputStrings(t *testing.T) { 16 | items := ExtractWorkUnitOutput([]interface{}{"first", "second"}, now) 17 | assert.Equal(t, map[string]AddWorkUnitItem{ 18 | "first": AddWorkUnitItem{ 19 | Key: "first", 20 | Data: map[string]interface{}{}, 21 | }, 22 | "second": AddWorkUnitItem{ 23 | Key: "second", 24 | Data: map[string]interface{}{}, 25 | }, 26 | }, items) 27 | } 28 | 29 | func TestOutputMap(t *testing.T) { 30 | items := ExtractWorkUnitOutput(map[string]interface{}{ 31 | "first": map[string]interface{}{}, 32 | "second": map[string]interface{}{"k": "v"}, 33 | }, now) 34 | assert.Equal(t, map[string]AddWorkUnitItem{ 35 | "first": AddWorkUnitItem{ 36 | Key: "first", 37 | Data: map[string]interface{}{}, 38 | }, 39 | "second": AddWorkUnitItem{ 40 | Key: "second", 41 | Data: map[string]interface{}{"k": "v"}, 42 | }, 43 | }, items) 44 | } 45 | 46 | func TestOutputLists(t *testing.T) { 47 | items := ExtractWorkUnitOutput([]interface{}{ 48 | []interface{}{"a"}, 49 | []interface{}{"b", map[string]interface{}{"k": "v"}}, 50 | []interface{}{"c", map[string]interface{}{}, map[string]interface{}{"priority": 10}}, 51 | []interface{}{"d", map[string]interface{}{}, map[string]interface{}{"delay": 90}}, 52 | []interface{}{"e", map[string]interface{}{}, map[string]interface{}{}, 20.0}, 53 | }, now) 54 | then := now.Add(90 * time.Second) 55 | assert.Equal(t, map[string]AddWorkUnitItem{ 56 | "b": AddWorkUnitItem{ 57 | Key: "b", 58 | Data: map[string]interface{}{"k": "v"}, 59 | }, 60 | "c": AddWorkUnitItem{ 61 | Key: "c", 62 | Data: map[string]interface{}{}, 63 | Meta: WorkUnitMeta{Priority: 10}, 64 | }, 65 | "d": AddWorkUnitItem{ 66 | Key: "d", 67 | Data: map[string]interface{}{}, 68 | Meta: WorkUnitMeta{NotBefore: then}, 69 | }, 70 | "e": AddWorkUnitItem{ 71 | Key: "e", 72 | Data: map[string]interface{}{}, 73 | Meta: WorkUnitMeta{Priority: 20}, 74 | }, 75 | }, items) 76 | } 77 | -------------------------------------------------------------------------------- /coordinate/marshal.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package coordinate 5 | 6 | import ( 7 | "fmt" 8 | ) 9 | 10 | // MarshalText returns a string representing a work unit status. 11 | func (status WorkUnitStatus) MarshalText() ([]byte, error) { 12 | switch status { 13 | case AnyStatus: 14 | return []byte("any"), nil 15 | case AvailableUnit: 16 | return []byte("available"), nil 17 | case PendingUnit: 18 | return []byte("pending"), nil 19 | case FinishedUnit: 20 | return []byte("finished"), nil 21 | case FailedUnit: 22 | return []byte("failed"), nil 23 | case DelayedUnit: 24 | return []byte("delayed"), nil 25 | default: 26 | return nil, fmt.Errorf("invalid status (marshal, %+v)", status) 27 | } 28 | } 29 | 30 | // UnmarshalText populates a work unit status from a string. 31 | func (status *WorkUnitStatus) UnmarshalText(text []byte) error { 32 | switch string(text) { 33 | case "any": 34 | *status = AnyStatus 35 | case "available": 36 | *status = AvailableUnit 37 | case "pending": 38 | *status = PendingUnit 39 | case "finished": 40 | *status = FinishedUnit 41 | case "failed": 42 | *status = FailedUnit 43 | case "delayed": 44 | *status = DelayedUnit 45 | default: 46 | return fmt.Errorf("invalid status (unmarshal, %+v)", string(text)) 47 | } 48 | return nil 49 | } 50 | 51 | // MarshalText returns a string representing an attempt status. 52 | func (status AttemptStatus) MarshalText() ([]byte, error) { 53 | switch status { 54 | case Pending: 55 | return []byte("pending"), nil 56 | case Expired: 57 | return []byte("expired"), nil 58 | case Finished: 59 | return []byte("finished"), nil 60 | case Failed: 61 | return []byte("failed"), nil 62 | case Retryable: 63 | return []byte("retryable"), nil 64 | default: 65 | return nil, fmt.Errorf("invalid status (marshal, %+v)", status) 66 | } 67 | } 68 | 69 | // UnmarshalText populates an attempt status from a string. 70 | func (status *AttemptStatus) UnmarshalText(text []byte) error { 71 | switch string(text) { 72 | case "pending": 73 | *status = Pending 74 | case "expired": 75 | *status = Expired 76 | case "finished": 77 | *status = Finished 78 | case "failed": 79 | *status = Failed 80 | case "retryable": 81 | *status = Retryable 82 | default: 83 | return fmt.Errorf("invalid status (unmarshal, %+v)", string(text)) 84 | } 85 | return nil 86 | } 87 | -------------------------------------------------------------------------------- /coordinate/marshal_test.go: -------------------------------------------------------------------------------- 1 | // Unit tests for marshal.go. 2 | // 3 | // Copyright 2017 Diffeo, Inc. 4 | // This software is released under an MIT/X11 open source license. 5 | 6 | package coordinate_test 7 | 8 | import ( 9 | "encoding/json" 10 | "testing" 11 | 12 | "github.com/stretchr/testify/assert" 13 | 14 | "github.com/swiftlobste/go-coordinate/coordinate" 15 | ) 16 | 17 | type WorkUnitStatusMatrix struct { 18 | Status coordinate.WorkUnitStatus 19 | JSON string 20 | EncodeError string 21 | DecodeError string 22 | } 23 | 24 | var workUnitStatuses = []WorkUnitStatusMatrix{ 25 | {coordinate.AvailableUnit, "available", "", ""}, 26 | {coordinate.PendingUnit, "pending", "", ""}, 27 | {coordinate.FinishedUnit, "finished", "", ""}, 28 | {coordinate.FailedUnit, "failed", "", ""}, 29 | {coordinate.DelayedUnit, "delayed", "", ""}, 30 | {coordinate.WorkUnitStatus(17), "seventeen", 31 | "invalid status (marshal, 17)", 32 | "invalid status (unmarshal, seventeen)"}, 33 | } 34 | 35 | func TestWorkUnitToJSON(t *testing.T) { 36 | for _, w := range workUnitStatuses { 37 | t.Run(w.JSON, func(tt *testing.T) { 38 | actual, err := json.Marshal(w.Status) 39 | if w.EncodeError == "" { 40 | if assert.NoError(tt, err) { 41 | assert.Equal(tt, "\""+w.JSON+"\"", 42 | string(actual)) 43 | } 44 | } else { 45 | assert.EqualError(tt, err, 46 | "json: error calling MarshalText for type coordinate.WorkUnitStatus: "+w.EncodeError) 47 | } 48 | }) 49 | } 50 | } 51 | 52 | func TestWorkUnitToText(t *testing.T) { 53 | for _, w := range workUnitStatuses { 54 | t.Run(w.JSON, func(tt *testing.T) { 55 | actual, err := w.Status.MarshalText() 56 | if w.EncodeError == "" { 57 | if assert.NoError(tt, err) { 58 | assert.Equal(tt, w.JSON, 59 | string(actual)) 60 | } 61 | } else { 62 | assert.EqualError(tt, err, w.EncodeError) 63 | } 64 | }) 65 | } 66 | } 67 | 68 | func TestWorkUnitFromJSON(t *testing.T) { 69 | for _, w := range workUnitStatuses { 70 | t.Run(w.JSON, func(tt *testing.T) { 71 | var actual coordinate.WorkUnitStatus 72 | input := []byte("\"" + w.JSON + "\"") 73 | err := json.Unmarshal(input, &actual) 74 | if w.DecodeError == "" { 75 | if assert.NoError(tt, err) { 76 | assert.Equal(tt, actual, w.Status) 77 | } 78 | } else { 79 | assert.EqualError(tt, err, w.DecodeError) 80 | } 81 | }) 82 | } 83 | } 84 | 85 | func TestWorkUnitFromText(t *testing.T) { 86 | for _, w := range workUnitStatuses { 87 | t.Run(w.JSON, func(tt *testing.T) { 88 | var actual coordinate.WorkUnitStatus 89 | input := []byte(w.JSON) 90 | err := actual.UnmarshalText(input) 91 | if w.DecodeError == "" { 92 | if assert.NoError(tt, err) { 93 | assert.Equal(tt, actual, w.Status) 94 | } 95 | } else { 96 | assert.EqualError(tt, err, w.DecodeError) 97 | } 98 | }) 99 | } 100 | } 101 | 102 | type AttemptStatusMatrix struct { 103 | Status coordinate.AttemptStatus 104 | JSON string 105 | EncodeError string 106 | DecodeError string 107 | } 108 | 109 | var attemptStatuses = []AttemptStatusMatrix{ 110 | {coordinate.Pending, "pending", "", ""}, 111 | {coordinate.Finished, "finished", "", ""}, 112 | {coordinate.Failed, "failed", "", ""}, 113 | {coordinate.Expired, "expired", "", ""}, 114 | {coordinate.Retryable, "retryable", "", ""}, 115 | {coordinate.AttemptStatus(17), "seventeen", 116 | "invalid status (marshal, 17)", 117 | "invalid status (unmarshal, seventeen)"}, 118 | } 119 | 120 | func TestAttemptToJSON(t *testing.T) { 121 | for _, w := range attemptStatuses { 122 | t.Run(w.JSON, func(tt *testing.T) { 123 | actual, err := json.Marshal(w.Status) 124 | if w.EncodeError == "" { 125 | if assert.NoError(tt, err) { 126 | assert.Equal(tt, "\""+w.JSON+"\"", 127 | string(actual)) 128 | } 129 | } else { 130 | assert.EqualError(tt, err, 131 | "json: error calling MarshalText for type coordinate.AttemptStatus: "+w.EncodeError) 132 | } 133 | }) 134 | } 135 | } 136 | 137 | func TestAttemptToText(t *testing.T) { 138 | for _, w := range attemptStatuses { 139 | t.Run(w.JSON, func(tt *testing.T) { 140 | actual, err := w.Status.MarshalText() 141 | if w.EncodeError == "" { 142 | if assert.NoError(tt, err) { 143 | assert.Equal(tt, w.JSON, 144 | string(actual)) 145 | } 146 | } else { 147 | assert.EqualError(tt, err, w.EncodeError) 148 | } 149 | }) 150 | } 151 | } 152 | 153 | func TestAttemptFromJSON(t *testing.T) { 154 | for _, w := range attemptStatuses { 155 | t.Run(w.JSON, func(tt *testing.T) { 156 | var actual coordinate.AttemptStatus 157 | input := []byte("\"" + w.JSON + "\"") 158 | err := json.Unmarshal(input, &actual) 159 | if w.DecodeError == "" { 160 | if assert.NoError(tt, err) { 161 | assert.Equal(tt, actual, w.Status) 162 | } 163 | } else { 164 | assert.EqualError(tt, err, w.DecodeError) 165 | } 166 | }) 167 | } 168 | } 169 | 170 | func TestAttemptFromText(t *testing.T) { 171 | for _, w := range attemptStatuses { 172 | t.Run(w.JSON, func(tt *testing.T) { 173 | var actual coordinate.AttemptStatus 174 | input := []byte(w.JSON) 175 | err := actual.UnmarshalText(input) 176 | if w.DecodeError == "" { 177 | if assert.NoError(tt, err) { 178 | assert.Equal(tt, actual, w.Status) 179 | } 180 | } else { 181 | assert.EqualError(tt, err, w.DecodeError) 182 | } 183 | }) 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /coordinate/stats.go: -------------------------------------------------------------------------------- 1 | // Statistics for Coordinate objects. 2 | // 3 | // Copyright 2017 Diffeo, Inc. 4 | // This software is released under an MIT/X11 open source license. 5 | 6 | package coordinate 7 | 8 | import ( 9 | "sort" 10 | ) 11 | 12 | // SummaryRecord is a single piece of summary data, recording how 13 | // many work units were in some status in some work spec. 14 | type SummaryRecord struct { 15 | Namespace string 16 | WorkSpec string 17 | Status WorkUnitStatus 18 | Count int 19 | } 20 | 21 | // Summary is a summary of work unit statuses for some part of 22 | // the Coordinate system. The records are in no particular order. 23 | // The records should not contain records with zero count. 24 | type Summary []SummaryRecord 25 | 26 | // Sort sorts the records of a summary in place. 27 | func (s Summary) Sort() { 28 | less := func(i, j int) bool { 29 | if s[i].Namespace < s[j].Namespace { 30 | return true 31 | } 32 | if s[i].Namespace > s[j].Namespace { 33 | return false 34 | } 35 | if s[i].WorkSpec < s[j].WorkSpec { 36 | return true 37 | } 38 | if s[i].WorkSpec > s[j].WorkSpec { 39 | return false 40 | } 41 | return s[i].Status < s[j].Status 42 | } 43 | sort.Slice(s, less) 44 | } 45 | 46 | // Summarizable describes Coordinate objects that can be summarized. 47 | // The summary is not required to have exact counts of work units; 48 | // counts may be rounded, delayed, not account for recently-expired 49 | // work units, and so on. 50 | type Summarizable interface { 51 | Summarize() (Summary, error) 52 | } 53 | -------------------------------------------------------------------------------- /doc/chaining.md: -------------------------------------------------------------------------------- 1 | Work Unit Chaining 2 | ================== 3 | 4 | One work spec can indicate that it generates work units for another 5 | work spec, using the `then` key in its work spec data. This maps to 6 | the `NextWorkSpecName` field in the `coordinate.WorkSpecMeta` 7 | structure. 8 | 9 | ```json 10 | { 11 | "name": "one", 12 | "then": "two" 13 | } 14 | ``` 15 | 16 | ```json 17 | { 18 | "name": "two" 19 | } 20 | ``` 21 | 22 | When work units for the first work spec complete successfully, the 23 | engine looks for a special key `output` in the final effective work 24 | unit data, taking into account any changes made in the course of the 25 | active attempt. `output` may be: 26 | 27 | * An object, mapping work unit name to work unit data 28 | * A list of strings, which are work unit names with empty data 29 | * A list of lists, where each list contains at least two items; items 30 | are, in order, the work unit name, the work unit data, a work unit 31 | metadata object, and a work unit priority value 32 | 33 | If the Python CBOR-based interface is used, any of these lists can be 34 | tuples, and the work unit names may be either byte strings or 35 | character strings. 36 | 37 | In the list-of-lists form, the metadata object may contain keys: 38 | 39 | * `priority`: specifies the priority of the created work unit; if a 40 | fourth parameter is included in the list and is not `null`, that 41 | priority parameter takes precedence over this setting 42 | * `delay`: specifies a minimum time to wait before executing the 43 | created work unit, in seconds 44 | 45 | Other keys are ignored. 46 | 47 | Some examples: 48 | 49 | ```json 50 | { 51 | "output": { 52 | "unit": {"key": "value"} 53 | } 54 | } 55 | ``` 56 | 57 | ```json 58 | { 59 | "output": [ 60 | "one", "two", "three" 61 | ] 62 | } 63 | ``` 64 | 65 | ```json 66 | { 67 | "output": [ 68 | ["first", {}, {"priority": 20}], 69 | ["second", {}, {}, 10], 70 | ["delayed", {}, {"delay": 90}], 71 | "third" 72 | ] 73 | } 74 | ``` 75 | 76 | The last example creates four work units. They will execute in order 77 | "first" (priority 20), "second" (priority 10), and then "third" 78 | (priority 0); "delayed" will not execute before 90 seconds have 79 | passed, but if "third" has not executed by then, "delayed" will 80 | probably execute before it (both have the same priority of 0 and 81 | "delayed" is alphabetically first). 82 | -------------------------------------------------------------------------------- /doc/errgone.md: -------------------------------------------------------------------------------- 1 | Concurrency and Deletion 2 | ======================== 3 | 4 | Imagine code that gets a `coordinate.WorkUnit` object and is preparing 5 | to do some work with it. Meanwhile, another process blindly deletes 6 | all of the work units. When happens to the first process, that has a 7 | work unit object, and then tries to use it? What if the other process 8 | kindly recreates a new work unit with the same name in the meantime? 9 | 10 | Prior to Coordinate 0.3.0, this story was extremely backend-dependent. 11 | The `memory` backend would calmly allow you to use the stale work unit 12 | as though nothing changed, and the new work unit with a different name 13 | would be a different object. The `postgres` backend in many cases 14 | would pass through `sql.ErrNoRows`. The `restclient` backend always 15 | does name-based lookup, and would pass through an HTTP 404 error if 16 | the unit was gone and silently use the new work unit if it was 17 | recreated. 18 | 19 | Coordinate 0.3.0 adds a new error, `coordinate.ErrGone`, though its 20 | use is still not totally consistent. `memory` will return `ErrGone` 21 | from all operations on a deleted work unit, work spec, or namespace, 22 | or a descendant of one of those. `postgres` will return `ErrGone` 23 | when it can definitively determine that this is the right answer, 24 | which usually means that functions that return lists or maps of things 25 | will return empty lists if an object has been deleted but functions 26 | that get or set a single property will return `ErrGone`. `restclient` 27 | and `restserver` treat `ErrGone` as a standard error, but will still 28 | do name-based lookup, and so could return more specific errors like 29 | `ErrNoSuchWorkUnit`, and if a work unit is recreated, the object in 30 | the calling process will continue to silently refer to the new record. 31 | 32 | 0.3.0's behavior is motivated by the need, for performance reasons, to 33 | cache Coordinate objects: if `restserver` always does name-based 34 | lookups and HTTP requests are stateless, and it is PostgreSQL-backed, 35 | then every request needs to find a namespace object, a work spec 36 | object, a work unit object, and an attempt object, and these are 37 | frequently objects that a caller has already been using. `ErrGone` 38 | gives a backend-neutral way to tell the cache layer that an object no 39 | longer exists and it should be removed, and even if it's not used 40 | totally consistently, having it is important (and having it is far 41 | better than passing through `sql.ErrNoRows`). 42 | 43 | Really truly solving this issue involves adding a notion of object 44 | identity to Coordinate. There are two parts to this: every object 45 | gains an `ID() int` method that returns a globally unique (per type) 46 | integer identifier, and the top-level `Coordinate` interface has a 47 | family of `WorkSpecWithID(int) (WorkSpec, error)` and similar 48 | functions. The implications of this are all in the REST API: instead 49 | of doing name-based lookup, it does ID-based lookup, and URLs change 50 | from `/namespace/-/work_spec/foo/work_unit/bar` to just 51 | `/work_unit/17`. This solves the problem of recreating an object with 52 | the same name, makes URLs much shorter, interfaces better with REST 53 | libraries, and makes it possible to directly address attempts rather 54 | than indirectly referencing them by their attributes. Currently the 55 | only two "concrete" backends in Coordinate, `memory` and `postgres`, 56 | could easily add this; are there other obvious backends where a 57 | globally unique integer ID is hard to construct, maybe because they 58 | speak only in terms of UUIDs? 59 | 60 | Having the globally unique ID and well-defined semantics for `ErrGone` 61 | provides one more possibility. Say `WorkSpecWithID` does not return 62 | an error, but the object it returns also does not have useful links to 63 | its parent or a name. In the `postgres` case, it creates a `workSpec` 64 | object with a caller-provided ID but no other data, and there is no 65 | lookup at the time it is created. Now operations on it can return 66 | `ErrGone` in the unusual cases where that happens, without having to 67 | do an extra database operation to fetch it where you don't need it. 68 | The benefits of this may be limited (you frequently need to know a 69 | work unit's work spec or an attempt's work unit). 70 | -------------------------------------------------------------------------------- /doc/index.md: -------------------------------------------------------------------------------- 1 | Coordinate Documentation 2 | ======================== 3 | 4 | Basics 5 | ------ 6 | 7 | * [Data Model](model.md) 8 | * [Special Work Spec Keys](work_specs.md) 9 | * [Change History](changes.md) 10 | 11 | Features 12 | -------- 13 | 14 | * [Python Compatibility](python.md) 15 | * [Work Unit Runtime Selection](runtime.md) 16 | * [Work Unit Chaining](chaining.md) 17 | * [Workers in Go](worker.md) 18 | * [Concurrency and Deletion](errgone.md) 19 | -------------------------------------------------------------------------------- /doc/model.md: -------------------------------------------------------------------------------- 1 | Data Model 2 | ========== 3 | 4 | The core purpose of Coordinate is to store and track a sequenc of 5 | jobs, or _work units_. These work units are grouped together by 6 | related tasks, or _work specs_; for instance, you could define a work 7 | spec that extracted text from a PDF file, and then define work units 8 | that were individual files to process. 9 | 10 | Namespaces 11 | ---------- 12 | 13 | The Coordinate system manages a set of logically separate namespaces. 14 | Each namespace is defined by its name. No state is shared between 15 | namespaces. A namespace has any number of work specs and any number 16 | of workers. 17 | 18 | Work specs 19 | ---------- 20 | 21 | A work spec defines a family of related tasks, generally running 22 | near-identical code over a number of different inputs. The 23 | Python-based worker embeds a Python module and function name in the 24 | work spec data, for instance, and calls this function every time a 25 | work unit is executed. A work unit is defined by a JSON object 26 | (Python dictionary, Go `map[string]interface{}`) which is required to 27 | have a key `name` with a string value. 28 | 29 | There are several [special work spec keys](work_specs.md). Work specs 30 | can also be controlled in several ways via a metadata object; for 31 | instance a work spec can be paused so that no new work will be 32 | returned from it. 33 | 34 | Work units 35 | ---------- 36 | 37 | A work unit defines a single task to perform within the context of a 38 | single work spec; for instance a single file or database record. A 39 | work unit is defined by its name (in some contexts also called a key) 40 | and an arbitrary data dictionary. Individual work units have their 41 | own metadata objects, though with many fewer options than the work 42 | spec metadata. 43 | 44 | Workers 45 | ------- 46 | 47 | A worker is a process (in the Unix sense) that executes work units. A 48 | worker is defined by its name. For diagnostic purposes, workers 49 | should periodically "check in" and upload an environment dictionary. 50 | Workers are arranged hierarchically to support a setup where one 51 | parent worker manages a family of child worker processes, one per 52 | core. 53 | 54 | Attempts 55 | -------- 56 | 57 | An attempt is a record that a specific worker is attempting (or has 58 | attempted) a specific work unit. The attempt has a status, which may 59 | be "pending" or may record a completed (or incomplete-but-terminated) 60 | attempt. Attempts also record an updated data dictionary for their 61 | work unit. 62 | 63 | A work unit has at most one _active_ attempt. If a work unit does 64 | have an active attempt, its status should be "pending", "finished", or 65 | "failed", but not "expired" or "retryable". Work units also can 66 | retrieve all of their past attempts. 67 | 68 | Workers similarly keep lists of active and past attempts. If an 69 | attempt is on a worker's active list, that means the worker is 70 | spending cycles on it, though if that attempt is no longer the active 71 | attempt for its work unit it may be a waste of computation. 72 | 73 | Data Objects 74 | ------------ 75 | 76 | Work specs, work units, and workers all have data objects, and 77 | attempts keep updated data objects for their work unit. These are 78 | generally treated as JSON objects, and usually have a Go type of 79 | `map[string]interface{}`. Treatment as JSON means that in some cases 80 | exact data types may not be preserved: map types may be converted to 81 | `map[string]interface{}`, array types to `[]interface{}`, and numeric 82 | types to `float64`. 83 | 84 | Additional type information stored within the system also preserves 85 | values of type `uuid.UUID` from `github.com/satori/go.uuid`, and a 86 | `PythonTuple` type native to this package. If Python code uploads 87 | data using the `uuid.UUID` type from the standard library or native 88 | Python tuples, these will be preserved. 89 | -------------------------------------------------------------------------------- /doc/python.md: -------------------------------------------------------------------------------- 1 | Python Coordinate 2 | ================ 3 | 4 | Coordinate's previous life was as a 5 | [pure Python package](https://github.com/diffeo/coordinate). This 6 | package aims to maintain wire compatibility with the Python 7 | `coordinate` client class; it cannot establish outbound connections to 8 | the Python `coordinated` daemon. 9 | 10 | Configuration 11 | ------------- 12 | 13 | Run the `coordinated` binary from this package. In your Python YAML 14 | configuration, set a pointer to this server: 15 | 16 | ```yaml 17 | coordinate: 18 | addresses: ['localhost:5932'] 19 | ``` 20 | 21 | If the daemon is running on a different host, or you specified a 22 | different port using the `-cborrpc` command-line option, change this 23 | setting accordingly. 24 | 25 | If your application depends on getting a system-global configuration 26 | back from coordinated, start the Go daemon with a `-config` 27 | command-line option pointing at a YAML file. This file will be passed 28 | back without interpretation beyond parsing to clients that request it. 29 | 30 | Running Python tests 31 | -------------------- 32 | 33 | To run the Python tests against this daemon, edit 34 | `coordinate/tests/test_job_client.py`, rename the existing 35 | `task_master` fixture, and add instead 36 | 37 | ```python 38 | @pytest.yield_fixture 39 | def task_master(): 40 | tm = TaskMaster({'address': '127.0.0.1:5932'}) 41 | yield tm 42 | tm.clear() 43 | ``` 44 | 45 | `test_job_client.py`, `test_job_flow.py`, and `test_task_master.py` 46 | all use this fixture and will run against the Go `coordinated` server. 47 | Many of these tests have been extracted into Go tests in the 48 | "jobserver" package. 49 | 50 | Behavioral differences 51 | ---------------------- 52 | 53 | ### Scheduling ### 54 | 55 | The work spec scheduler is much simpler than in the Python 56 | coordinated. See the "Scheduling" section in the 57 | [extended work spec discussion](work_specs.md) for details. 58 | 59 | The Go scheduler only considers work specs' `priority` and `weight` 60 | fields. Work specs' successors, as identified by the `then` field, do 61 | not factor into the scheduler, and the `then_preempts` field is 62 | ignored. The Python scheduler would try to give successor work specs 63 | priority over predecessors (unless `then_preempts: false` was in the 64 | work spec data), and would deterministically pick a work unit based on 65 | weights. This could cause low-weight work specs to never get work if 66 | there were relatively fewer workers. 67 | 68 | For example, given work specs: 69 | 70 | ```yaml 71 | flows: 72 | a: 73 | weight: 1000 74 | then: b 75 | b: 76 | weight: 1 77 | ``` 78 | 79 | Add two work units to "a", and have an implementation that copies 80 | those work units to the work unit data `output` field. 81 | 82 | ```python 83 | def run_function(work_unit): 84 | work_unit.data["output"] = {work_unit.key: {}} 85 | ``` 86 | 87 | The Python coordinated will run a work unit from "a", producing one in 88 | "b"; then its rule that later work specs take precedence applies, and 89 | it will run the work unit in "b"; then "a", then "b". This 90 | implementation does not have that precedence rule, and so the second 91 | request for work will (very probably) get the second work unit from 92 | "a" in accordance with the specified weights. 93 | 94 | If you need a later work spec to preempt an earlier one, set a 95 | `priority` key on the later work spec. 96 | 97 | Enhancements 98 | ------------ 99 | 100 | Python programs that are aware they are talking to this implementation 101 | of the Coordinate daemon can take advantage of some enhancements in 102 | it. 103 | 104 | ### Delayed work units ### 105 | 106 | When a work unit is created, calling code can request that it not 107 | execute for some length of time: 108 | 109 | ```python 110 | task_master.add_work_units('spec', [ 111 | ('unit', {'key': 'value'}, {'delay': 90}) 112 | ]) 113 | ``` 114 | 115 | If `add_work_units()` is passed a list of tuples, each tuple contains 116 | the work unit name, data dictionary, and (optionally) a metadata 117 | dictionary. Native Python coordinated already supports a key 118 | `priority` to set the work unit priority at creation time; Go 119 | coordinated adds the `delay` key giving an initial delay in seconds. 120 | 121 | Delays for work units created using the `output` key for 122 | [chained work specs](chaining.md) also work as described. 123 | 124 | In both cases, running this code against Python coordinated will 125 | ignore the `delay` key, and the added work unit(s) will run 126 | immediately. 127 | 128 | Other notes 129 | ----------- 130 | 131 | Most Python Coordinate applications should run successfully against 132 | this server. This server has been tested against both the Python 133 | Coordinate provided unit tests and some real-world data. 134 | 135 | Work spec names must be valid Unicode strings. Work spec definitions 136 | and work unit data must be Unicode-string-keyed maps. Work unit keys, 137 | however, can be arbitrary byte strings. Python (especially Python 2) 138 | is sloppy about byte vs. character strings and it is easy to inject 139 | the wrong type; if you do create a work spec with a non-UTF-8 byte 140 | string name, the server will eventually return it as an invalid 141 | Unicode-tagged string. Data such as work spec names can be submitted 142 | as byte strings but may be returned as Unicode strings. 143 | -------------------------------------------------------------------------------- /doc/runtime.md: -------------------------------------------------------------------------------- 1 | Runtimes 2 | ======== 3 | 4 | This version of coordinated supports the concept of workers running 5 | multiple language runtimes. The Python worker in the 6 | [coordinate package](https://github.com/diffeo/coordinate) can't run 7 | Go code, for instance. This feature was added in Go Coordinate 0.2.0. 8 | 9 | Work Spec 10 | --------- 11 | 12 | If a work spec contains a key `runtime`, it is the name of a language 13 | runtime that is required to run that work spec. This is generally a 14 | short description such as `python_2`, `go`, or `java_1.7`. In the Go 15 | API, the runtime can be retrieved from the immutable 16 | `WorkSpecMeta.Runtime` field. 17 | 18 | For backwards compatibility, an empty runtime string should generally 19 | be interpreted as equivalent to `python_2`. 20 | 21 | Attempt Requests 22 | ---------------- 23 | 24 | `AttemptRequest.Runtimes` is a list of strings that are runtimes this 25 | worker is capable of handling. Work specs with runtimes that do not 26 | exactly match one of these strings are ignored. If the runtime list 27 | is empty, any runtime is considered acceptable. 28 | 29 | A new Go-based worker could call 30 | 31 | ```go 32 | attempts := worker.RequestAttempts(coordinate.AttemptRequest{ 33 | NumberOfWorkUnits: 20, 34 | Runtimes: []string{"go"}, 35 | }) 36 | ``` 37 | 38 | The Python-compatible interface passes a runtime list containing of a 39 | single empty string. 40 | 41 | Example 42 | ------- 43 | 44 | For a mixed Python/Go system, create a YAML file containing: 45 | 46 | ```yaml 47 | flows: 48 | a_python_spec: 49 | module: python.module 50 | run_function: coordinate_run 51 | 52 | a_go_spec: 53 | runtime: go 54 | task: task_name 55 | ``` 56 | 57 | Using the `coordinate` tool from the Python coordinate package, run 58 | `coordinate flow flow.yaml`, pointing at the file above. 59 | 60 | The Python-based `coordinate_worker` will only retrieve and run work 61 | units from `a_python_spec`. A new Go-based worker, using the 62 | `RequestAttempts` call shown above, will only retrieve and run work 63 | units from `a_go_spec`. 64 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/swiftlobste/go-coordinate 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/benbjohnson/clock v0.0.0-20161215174838-7dc76406b6d3 7 | github.com/google/go-cloud v0.2.0 8 | github.com/gorilla/mux v1.7.3 9 | github.com/jtacoma/uritemplates v1.0.0 10 | github.com/lib/pq v0.0.0-20170313200423-472a0745531a 11 | github.com/mitchellh/mapstructure v1.1.2 12 | github.com/prometheus/client_golang v1.11.1 13 | github.com/rubenv/sql-migrate v0.0.0-20170314191533-a3e296353799 14 | github.com/satori/go.uuid v1.0.0 15 | github.com/sirupsen/logrus v1.6.0 16 | github.com/stretchr/testify v1.4.0 17 | github.com/ugorji/go v0.0.0-20170312112114-708a42d24682 18 | github.com/urfave/cli v1.22.1 19 | github.com/urfave/negroni v1.0.0 20 | gopkg.in/yaml.v2 v2.4.0 21 | ) 22 | 23 | require ( 24 | github.com/beorn7/perks v1.0.1 // indirect 25 | github.com/cespare/xxhash/v2 v2.1.1 // indirect 26 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect 27 | github.com/davecgh/go-spew v1.1.1 // indirect 28 | github.com/golang/protobuf v1.4.3 // indirect 29 | github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect 30 | github.com/mattn/go-sqlite3 v1.10.0 // indirect 31 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 32 | github.com/pmezard/go-difflib v1.0.0 // indirect 33 | github.com/prometheus/client_model v0.2.0 // indirect 34 | github.com/prometheus/common v0.26.0 // indirect 35 | github.com/prometheus/procfs v0.6.0 // indirect 36 | github.com/russross/blackfriday/v2 v2.0.1 // indirect 37 | github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect 38 | github.com/ziutek/mymysql v1.5.4 // indirect 39 | golang.org/x/sys v0.1.0 // indirect 40 | google.golang.org/protobuf v1.26.0-rc.1 // indirect 41 | gopkg.in/gorp.v1 v1.7.1 // indirect 42 | ) 43 | -------------------------------------------------------------------------------- /jobserver/constants.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package jobserver 5 | 6 | // WorkUnitStatus is one of the possible work unit states supported by the 7 | // Python Coordinate server. 8 | type WorkUnitStatus int 9 | 10 | const ( 11 | // Available work units can be returned by the get_work call. 12 | Available WorkUnitStatus = 1 13 | 14 | // Blocked work units cannot run until some other work units 15 | // complete. This was supported in rejester but not in Python 16 | // Coordinate. 17 | // Blocked WorkUnitStatus = 2 18 | 19 | // Pending work units have been returned by the get_work call, 20 | // and have not yet been completed. 21 | Pending WorkUnitStatus = 3 22 | 23 | // Finished work units have completed successfully. 24 | Finished WorkUnitStatus = 4 25 | 26 | // Failed work units have completed unsuccessfully. 27 | Failed WorkUnitStatus = 5 28 | ) 29 | -------------------------------------------------------------------------------- /jobserver/interface.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | // Package jobserver provides a CBOR-RPC interface compatible with 5 | // the Python coordinate module. This defines what is served from 6 | // github.com/swiftlobste/go-coordinate/cmd/coordinated, and probably should 7 | // be merged in some form with that package. 8 | // 9 | // The Python coordinated operates with an extremely irregular 10 | // RPC-like interface. Many methods, but not all, take a dictionary 11 | // of additional options. Many methods, but not all, return an 12 | // in-band string error message, plus the underlying RPC layer allows 13 | // an exception string to be returned. Some methods specifically 14 | // require a Python tuple return, even though the only way to achieve 15 | // this across the wire is through an extension tag in CBOR. 16 | // 17 | // As such, JobServer provides an interface that can be made compatible 18 | // with the Python coordinate library, but it is unlikely to be useful 19 | // to native Go code or other client interfaces. 20 | package jobserver 21 | 22 | import ( 23 | "github.com/benbjohnson/clock" 24 | "github.com/swiftlobste/go-coordinate/coordinate" 25 | "sync" 26 | ) 27 | 28 | // JobServer is a network-accessible interface to Coordinate. Its 29 | // methods are the Python coordinated RPC methods, with more normalized 30 | // parameters and Go-style CamelCase names. 31 | type JobServer struct { 32 | // Namespace is the Coordinate Namespace interface this works 33 | // against. 34 | Namespace coordinate.Namespace 35 | 36 | // GlobalConfig is the configuration that is returned by the 37 | // GetConfig RPC call. 38 | GlobalConfig map[string]interface{} 39 | 40 | // Clock is the system time source. This should agree with the 41 | // time source for the Coordinate backend, if it was created with 42 | // an alternate time source. 43 | Clock clock.Clock 44 | 45 | // locks is the root of the tree for the hierarchical lock 46 | // subsystem. 47 | locks lockNode 48 | 49 | // lockLock is a global mutex over the locks tree. 50 | lockLock sync.Mutex 51 | } 52 | -------------------------------------------------------------------------------- /jobserver/locks_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package jobserver_test 5 | 6 | import ( 7 | "github.com/swiftlobste/go-coordinate/cborrpc" 8 | "github.com/stretchr/testify/assert" 9 | "testing" 10 | ) 11 | 12 | func TestBasic(t *testing.T) { 13 | j := setUpTest(t, "TestBasic") 14 | defer tearDownTest(t, j) 15 | 16 | foo := cborrpc.PythonTuple{Items: []interface{}{"foo"}} 17 | barbaz := cborrpc.PythonTuple{Items: []interface{}{"bar", "baz"}} 18 | bar := cborrpc.PythonTuple{Items: []interface{}{"bar"}} 19 | 20 | ok, msg, err := j.Lock("id", 0, []interface{}{foo, barbaz}) 21 | if assert.NoError(t, err) { 22 | assert.True(t, ok) 23 | assert.Empty(t, msg) 24 | } 25 | 26 | lockid, err := j.Readlock([]interface{}{foo}) 27 | if assert.NoError(t, err) { 28 | assert.Equal(t, []interface{}{"id"}, lockid) 29 | } 30 | 31 | lockid, err = j.Readlock([]interface{}{bar}) 32 | if assert.NoError(t, err) { 33 | assert.Equal(t, []interface{}{nil}, lockid) 34 | } 35 | 36 | lockid, err = j.Readlock([]interface{}{barbaz, foo}) 37 | if assert.NoError(t, err) { 38 | assert.Equal(t, []interface{}{"id", "id"}, lockid) 39 | } 40 | 41 | ok, msg, err = j.Unlock("id", []interface{}{foo, barbaz}) 42 | if assert.NoError(t, err) { 43 | assert.True(t, ok) 44 | assert.Empty(t, msg) 45 | } 46 | 47 | lockid, err = j.Readlock([]interface{}{foo, bar, barbaz}) 48 | if assert.NoError(t, err) { 49 | assert.Equal(t, []interface{}{nil, nil, nil}, lockid) 50 | } 51 | } 52 | 53 | func TestConflict(t *testing.T) { 54 | j := setUpTest(t, "TestConflict") 55 | defer tearDownTest(t, j) 56 | 57 | ok, msg, err := j.Lock("id", 0, 58 | []interface{}{[]interface{}{"foo"}}) 59 | if assert.NoError(t, err) { 60 | assert.True(t, ok) 61 | assert.Empty(t, msg) 62 | } 63 | 64 | // should not be able to lock foo.bar when foo is held 65 | ok, msg, err = j.Lock("id", 0, 66 | []interface{}{[]interface{}{"foo", "bar"}}) 67 | if assert.NoError(t, err) { 68 | assert.False(t, ok) 69 | assert.NotEmpty(t, msg) 70 | } 71 | } 72 | 73 | func TestConflict2(t *testing.T) { 74 | j := setUpTest(t, "TestConflict2") 75 | defer tearDownTest(t, j) 76 | 77 | ok, msg, err := j.Lock("id", 0, 78 | []interface{}{[]interface{}{"foo", "bar"}}) 79 | if assert.NoError(t, err) { 80 | assert.True(t, ok) 81 | assert.Empty(t, msg) 82 | } 83 | 84 | // should not be able to lock foo when foo.bar is held 85 | ok, msg, err = j.Lock("id", 0, 86 | []interface{}{[]interface{}{"foo"}}) 87 | if assert.NoError(t, err) { 88 | assert.False(t, ok) 89 | assert.NotEmpty(t, msg) 90 | } 91 | } 92 | 93 | func TestLocksome(t *testing.T) { 94 | j := setUpTest(t, "TestLocksome") 95 | defer tearDownTest(t, j) 96 | 97 | ok, msg, err := j.Lock("id", 0, 98 | []interface{}{[]interface{}{"foo"}}) 99 | if assert.NoError(t, err) { 100 | assert.True(t, ok) 101 | assert.Empty(t, msg) 102 | } 103 | 104 | locked, msg, err := j.Locksome("id", 0, []interface{}{ 105 | []interface{}{"foo"}, 106 | []interface{}{"bar"}, 107 | []interface{}{"baz"}, 108 | }) 109 | if assert.NoError(t, err) { 110 | assert.Empty(t, msg) 111 | assert.Equal(t, [][]interface{}{ 112 | nil, 113 | []interface{}{"bar"}, 114 | []interface{}{"baz"}, 115 | }, locked) 116 | } 117 | } 118 | 119 | func TestUnlockSanity(t *testing.T) { 120 | j := setUpTest(t, "TestUnlockSanity") 121 | defer tearDownTest(t, j) 122 | 123 | keys := []interface{}{[]interface{}{"foo"}, []interface{}{"bar"}} 124 | 125 | ok, msg, err := j.Lock("id", 0, keys) 126 | if assert.NoError(t, err) { 127 | assert.True(t, ok) 128 | assert.Empty(t, msg) 129 | } 130 | 131 | // Should not be able to unlock something a different locker locked 132 | ok, msg, err = j.Unlock("id2", keys) 133 | if assert.NoError(t, err) { 134 | assert.False(t, ok) 135 | assert.NotEmpty(t, msg) 136 | } 137 | 138 | // Should be able to read original lock 139 | lockid, err := j.Readlock(keys) 140 | if assert.NoError(t, err) { 141 | assert.Equal(t, []interface{}{"id", "id"}, lockid) 142 | } 143 | 144 | ok, msg, err = j.Unlock("id", keys) 145 | if assert.NoError(t, err) { 146 | assert.True(t, ok) 147 | assert.Empty(t, msg) 148 | } 149 | } 150 | 151 | func TestReadlockNotNil(t *testing.T) { 152 | j := setUpTest(t, "TestReadlockNotNil") 153 | defer tearDownTest(t, j) 154 | 155 | locks, err := j.Readlock([]interface{}{}) 156 | if assert.NoError(t, err) { 157 | assert.NotNil(t, locks) 158 | assert.Len(t, locks, 0) 159 | } 160 | } 161 | 162 | func TestUnlockNotLocked(t *testing.T) { 163 | j := setUpTest(t, "TestUnlockNotLocked") 164 | defer tearDownTest(t, j) 165 | 166 | keys := []interface{}{[]interface{}{"foo"}, []interface{}{"bar"}} 167 | ok, _, err := j.Unlock("id", keys) 168 | if assert.NoError(t, err) { 169 | assert.False(t, ok) 170 | } 171 | } 172 | -------------------------------------------------------------------------------- /jobserver/specs_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package jobserver_test 5 | 6 | // This file has miscellaneous work spec tests. 7 | 8 | import ( 9 | "github.com/stretchr/testify/assert" 10 | "testing" 11 | ) 12 | 13 | func TestSpecByteification(t *testing.T) { 14 | j := setUpTest(t, "TestSpecByteification") 15 | defer tearDownTest(t, j) 16 | workSpecName := setWorkSpec(t, j, WorkSpecData) 17 | 18 | data, err := j.GetWorkSpec(workSpecName) 19 | if assert.NoError(t, err) { 20 | assert.Equal(t, map[string]interface{}{ 21 | "name": "test_job_client", 22 | "min_gb": 1, 23 | "module": []byte("coordinate.tests.test_job_client"), 24 | "run_function": []byte("run_function"), 25 | }, data) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /jobserver/utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package jobserver 5 | 6 | import ( 7 | "errors" 8 | "github.com/swiftlobste/go-coordinate/cborrpc" 9 | "github.com/swiftlobste/go-coordinate/coordinate" 10 | "github.com/mitchellh/mapstructure" 11 | ) 12 | 13 | // decode is a helper that uses the mapstructure library to decode a 14 | // string-keyed map into a structure. 15 | func decode(result interface{}, options map[string]interface{}) error { 16 | config := mapstructure.DecoderConfig{ 17 | DecodeHook: cborrpc.DecodeBytesAsString, 18 | Result: result, 19 | } 20 | decoder, err := mapstructure.NewDecoder(&config) 21 | if err == nil { 22 | err = decoder.Decode(options) 23 | } 24 | return err 25 | } 26 | 27 | // translateWorkUnitStatus converts a CBOR-RPC work unit status to a 28 | // Go coordinate.WorkUnitStatus. 0 is not used in the CBOR-RPC API, 29 | // and it is an easy default, so we use it to mean "unspecified" which 30 | // usually translates to coordinate.AnyStatus. Returns an error only 31 | // if the status value is undefined (which includes the unused Python 32 | // BLOCKED). 33 | func translateWorkUnitStatus(status WorkUnitStatus) (coordinate.WorkUnitStatus, error) { 34 | switch status { 35 | case 0: 36 | return coordinate.AnyStatus, nil 37 | case Available: 38 | return coordinate.AvailableUnit, nil 39 | case Pending: 40 | return coordinate.PendingUnit, nil 41 | case Finished: 42 | return coordinate.FinishedUnit, nil 43 | case Failed: 44 | return coordinate.FailedUnit, nil 45 | default: 46 | return coordinate.AnyStatus, errors.New("invalid work unit status") 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /jobserver/work_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package jobserver_test 5 | 6 | // This file has miscellaneous work attempt tests. 7 | 8 | import ( 9 | "github.com/swiftlobste/go-coordinate/cborrpc" 10 | "github.com/swiftlobste/go-coordinate/jobserver" 11 | "github.com/stretchr/testify/assert" 12 | "testing" 13 | "time" 14 | ) 15 | 16 | // TestUpdateAvailable tries to transition a work unit from "available" 17 | // to "failed" state. 18 | func TestUpdateAvailable(t *testing.T) { 19 | j := setUpTest(t, "TestUpdateAvailable") 20 | defer tearDownTest(t, j) 21 | 22 | workSpecName := setWorkSpec(t, j, WorkSpecData) 23 | addWorkUnit(t, j, workSpecName, "unit", map[string]interface{}{}) 24 | 25 | ok, msg, err := j.UpdateWorkUnit(workSpecName, "unit", map[string]interface{}{ 26 | "status": jobserver.Failed, 27 | "worker_id": "child", 28 | }) 29 | if assert.NoError(t, err) { 30 | assert.True(t, ok) 31 | assert.Empty(t, msg) 32 | } 33 | 34 | checkWorkUnitStatus(t, j, workSpecName, "unit", jobserver.Failed) 35 | } 36 | 37 | // TestUpdateAvailableFull verifies a specific race condition that can 38 | // happen in the Python worker. Say the parent asks coordinated for a 39 | // list of its childrens' pending work units. Even though it tries to 40 | // kill them off 15 seconds before they expire, on a bad day 41 | // coordinated will still manage to hit the expiry first, so the work 42 | // unit transitions back to "available". 43 | // 44 | // This test validates this specific sequence of things. 45 | func TestUpdateAvailableFull(t *testing.T) { 46 | j := setUpTest(t, "TestUpdateAvailableFull") 47 | defer tearDownTest(t, j) 48 | 49 | empty := map[string]interface{}{} 50 | workSpecName := setWorkSpec(t, j, WorkSpecData) 51 | addWorkUnit(t, j, workSpecName, "unit", empty) 52 | 53 | ok, msg, err := j.WorkerHeartbeat("parent", "RUN", 900, empty, "") 54 | if assert.NoError(t, err) { 55 | assert.True(t, ok) 56 | } 57 | 58 | ok, msg, err = j.WorkerHeartbeat("child", "RUN", 900, empty, "parent") 59 | if assert.NoError(t, err) { 60 | assert.True(t, ok) 61 | } 62 | 63 | work, msg, err := j.GetWork("child", map[string]interface{}{"available_gb": 1}) 64 | if assert.NoError(t, err) { 65 | assert.Empty(t, msg) 66 | if assert.NotNil(t, work) && assert.IsType(t, cborrpc.PythonTuple{}, work) { 67 | tuple := work.(cborrpc.PythonTuple) 68 | if assert.Len(t, tuple.Items, 3) { 69 | assert.Equal(t, workSpecName, tuple.Items[0]) 70 | assert.Equal(t, []byte("unit"), tuple.Items[1]) 71 | } 72 | } 73 | } 74 | checkWorkUnitStatus(t, j, workSpecName, "unit", jobserver.Pending) 75 | 76 | // Force the work unit back to "available" to simulate expiry 77 | ok, msg, err = j.UpdateWorkUnit(workSpecName, "unit", map[string]interface{}{ 78 | "status": jobserver.Available, 79 | "worker_id": "child", 80 | }) 81 | if assert.NoError(t, err) { 82 | assert.True(t, ok) 83 | assert.Empty(t, msg) 84 | } 85 | checkWorkUnitStatus(t, j, workSpecName, "unit", jobserver.Available) 86 | 87 | // Now kill it from the parent 88 | ok, msg, err = j.UpdateWorkUnit(workSpecName, "unit", map[string]interface{}{ 89 | "status": jobserver.Failed, 90 | "worker_id": "parent", 91 | }) 92 | if assert.NoError(t, err) { 93 | assert.True(t, ok) 94 | assert.Empty(t, msg) 95 | } 96 | checkWorkUnitStatus(t, j, workSpecName, "unit", jobserver.Failed) 97 | } 98 | 99 | // TestDelayedUnit creates a work unit to run in the future. 100 | func TestDelayedUnit(t *testing.T) { 101 | j := setUpTest(t, "TestDelayedUnit") 102 | defer tearDownTest(t, j) 103 | 104 | empty := map[string]interface{}{} 105 | workSpecName := setWorkSpec(t, j, WorkSpecData) 106 | 107 | ok, msg, err := j.AddWorkUnits(workSpecName, []interface{}{ 108 | cborrpc.PythonTuple{Items: []interface{}{ 109 | "unit", 110 | empty, 111 | map[string]interface{}{"delay": 90}, 112 | }}, 113 | }) 114 | if assert.NoError(t, err) { 115 | assert.True(t, ok) 116 | assert.Empty(t, msg) 117 | } 118 | 119 | // Even though it is delayed, we should report it as available 120 | checkWorkUnitStatus(t, j, workSpecName, "unit", jobserver.Available) 121 | 122 | // Get-work should return nothing 123 | doNoWork(t, j) 124 | 125 | // If we wait 60 seconds (out of 90) we should still get nothing 126 | Clock.Add(60 * time.Second) 127 | checkWorkUnitStatus(t, j, workSpecName, "unit", jobserver.Available) 128 | doNoWork(t, j) 129 | 130 | // If we wait another 60 seconds we should be able to do it 131 | Clock.Add(60 * time.Second) 132 | checkWorkUnitStatus(t, j, workSpecName, "unit", jobserver.Available) 133 | doOneWork(t, j, workSpecName, "unit") 134 | } 135 | -------------------------------------------------------------------------------- /memory/available_units.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package memory 5 | 6 | import ( 7 | "container/heap" 8 | ) 9 | 10 | // availableUnits is a priority queue of work units. 11 | type availableUnits []*workUnit 12 | 13 | // Add a work unit to this queue in the appropriate spot. 14 | func (q *availableUnits) Add(unit *workUnit) { 15 | heap.Push(q, unit) 16 | } 17 | 18 | // Next gets the next available unit, with the highest priority and lowest name. 19 | func (q *availableUnits) Next() *workUnit { 20 | return heap.Pop(q).(*workUnit) 21 | } 22 | 23 | // Remove a specific work unit. 24 | func (q *availableUnits) Remove(unit *workUnit) { 25 | if unit.availableIndex > 0 { 26 | heap.Remove(q, unit.availableIndex-1) 27 | } 28 | } 29 | 30 | // Reprioritize a specific work unit (when its priority changes). 31 | func (q *availableUnits) Reprioritize(unit *workUnit) { 32 | if unit.availableIndex > 0 { 33 | heap.Fix(q, unit.availableIndex-1) 34 | } 35 | } 36 | 37 | // sort.Interface 38 | 39 | func (q availableUnits) Len() int { 40 | return len(q) 41 | } 42 | 43 | // isUnitHigherPriority returns true if a is more important than b. 44 | func isUnitHigherPriority(a, b *workUnit) bool { 45 | if a.meta.Priority > b.meta.Priority { 46 | return true 47 | } 48 | if a.meta.Priority < b.meta.Priority { 49 | return false 50 | } 51 | return a.name < b.name 52 | } 53 | 54 | func (q availableUnits) Less(i, j int) bool { 55 | // Remember, position 0 is highest priority. Sorting says 56 | // that if q.Units[i] < q.Units[j], then i should be before j. 57 | // This means the highest-priority thing sorts least...or, 58 | // Less(i, j) is true iff q.Units[i] is higher priority than 59 | // q.Units[j]. 60 | return isUnitHigherPriority(q[i], q[j]) 61 | } 62 | 63 | func (q availableUnits) Swap(i, j int) { 64 | q[i], q[j] = q[j], q[i] 65 | q[i].availableIndex = i + 1 66 | q[j].availableIndex = j + 1 67 | } 68 | 69 | // collections/heap.Interface 70 | 71 | func (q *availableUnits) Push(x interface{}) { 72 | unit := x.(*workUnit) 73 | unit.availableIndex = len(*q) + 1 74 | *q = append(*q, unit) 75 | } 76 | 77 | func (q *availableUnits) Pop() interface{} { 78 | if len(*q) == 0 { 79 | return nil 80 | } 81 | unit := (*q)[len(*q)-1] 82 | *q = (*q)[:len(*q)-1] 83 | unit.availableIndex = 0 84 | return unit 85 | } 86 | -------------------------------------------------------------------------------- /memory/available_units_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package memory 5 | 6 | import ( 7 | "github.com/swiftlobste/go-coordinate/coordinate" 8 | "github.com/stretchr/testify/assert" 9 | "testing" 10 | ) 11 | 12 | // push adds any number of work units to the priority queue, in the order 13 | // of their parameters. 14 | func push(q *availableUnits, units ...*workUnit) { 15 | for _, unit := range units { 16 | q.Add(unit) 17 | } 18 | } 19 | 20 | // popSpecific pulls a single unit out of the priority queue and asserts 21 | // that it is exactly u. 22 | func popSpecific(t *testing.T, q *availableUnits, u *workUnit) { 23 | if assert.NotZero(t, q.Len()) { 24 | out := q.Next() 25 | assert.Equal(t, u, out) 26 | } 27 | } 28 | 29 | // checkEmpty asserts that the priority queue is empty. 30 | func checkEmpty(t *testing.T, q *availableUnits) { 31 | assert.Zero(t, q.Len()) 32 | } 33 | 34 | // popAll pops units off the priority queue one at a time, comparing 35 | // them to each of the parameters in turn, and asserts that the queue 36 | // is empty at the end. 37 | func popAll(t *testing.T, q *availableUnits, units ...*workUnit) { 38 | for _, unit := range units { 39 | popSpecific(t, q, unit) 40 | } 41 | checkEmpty(t, q) 42 | } 43 | 44 | func TestQueueOfOne(t *testing.T) { 45 | q := new(availableUnits) 46 | unit := &workUnit{name: "unit"} 47 | q.Add(unit) 48 | popAll(t, q, unit) 49 | } 50 | 51 | func TestQueueOfTwoInOrder(t *testing.T) { 52 | q := new(availableUnits) 53 | first := &workUnit{name: "first"} 54 | second := &workUnit{name: "second"} 55 | push(q, first, second) 56 | popAll(t, q, first, second) 57 | } 58 | 59 | func TestQueueOfTwoInWrongOrder(t *testing.T) { 60 | q := new(availableUnits) 61 | first := &workUnit{name: "first"} 62 | second := &workUnit{name: "second"} 63 | push(q, second, first) 64 | popAll(t, q, first, second) 65 | } 66 | 67 | func TestQueueOfThreeWithPriorities(t *testing.T) { 68 | q := new(availableUnits) 69 | first := &workUnit{name: "z", meta: coordinate.WorkUnitMeta{Priority: 100}} 70 | second := &workUnit{name: "a"} 71 | third := &workUnit{name: "m"} 72 | push(q, second, third, first) 73 | popAll(t, q, first, second, third) 74 | } 75 | 76 | func TestDeleteJustOne(t *testing.T) { 77 | q := new(availableUnits) 78 | unit := &workUnit{name: "unit"} 79 | q.Add(unit) 80 | q.Remove(unit) 81 | popAll(t, q) 82 | } 83 | 84 | func TestDeleteFirstOfThree(t *testing.T) { 85 | q := new(availableUnits) 86 | first := &workUnit{name: "a"} 87 | second := &workUnit{name: "b"} 88 | third := &workUnit{name: "c"} 89 | push(q, first, second, third) 90 | q.Remove(first) 91 | popAll(t, q, second, third) 92 | } 93 | 94 | func TestDeleteOther(t *testing.T) { 95 | q := new(availableUnits) 96 | first := &workUnit{name: "a"} 97 | second := &workUnit{name: "b"} 98 | q.Add(first) 99 | q.Remove(second) 100 | popAll(t, q, first) 101 | } 102 | 103 | func TestReprioritizeFirst(t *testing.T) { 104 | q := new(availableUnits) 105 | first := &workUnit{name: "a"} 106 | second := &workUnit{name: "b"} 107 | third := &workUnit{name: "c"} 108 | push(q, first, second, third) 109 | first.meta.Priority = -1 110 | q.Reprioritize(first) 111 | popAll(t, q, second, third, first) 112 | } 113 | 114 | func TestReprioritizeMiddle(t *testing.T) { 115 | q := new(availableUnits) 116 | first := &workUnit{name: "a"} 117 | second := &workUnit{name: "b"} 118 | third := &workUnit{name: "c"} 119 | push(q, first, second, third) 120 | second.meta.Priority = 100 121 | q.Reprioritize(second) 122 | popAll(t, q, second, first, third) 123 | } 124 | -------------------------------------------------------------------------------- /memory/coordinate.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | // Package memory provides an in-process, in-memory implementation of 5 | // Coordinate. There is no persistence on this job queue, nor is 6 | // there any automatic sharing. The entire system is behind a single 7 | // global semaphore to protect against concurrent updates; in some 8 | // cases this can limit performance in the name of correctness. 9 | // 10 | // This is mostly intended as a simple reference implementation of 11 | // Coordinate that can be used for testing, including in-process 12 | // testing of higher-level components. It is generally tuned for 13 | // correctness, not performance or scalability. 14 | package memory 15 | 16 | import ( 17 | "github.com/benbjohnson/clock" 18 | "github.com/swiftlobste/go-coordinate/coordinate" 19 | "sync" 20 | ) 21 | 22 | // This is the only external entry point to this package: 23 | 24 | // New creates a new Coordinate interface that operates purely in 25 | // memory. 26 | func New() coordinate.Coordinate { 27 | clk := clock.New() 28 | return NewWithClock(clk) 29 | } 30 | 31 | // NewWithClock returns a new in-memory Coordinate interface, with an 32 | // explicitly specified time source. This is intended for use in 33 | // tests. 34 | func NewWithClock(clk clock.Clock) coordinate.Coordinate { 35 | c := new(memCoordinate) 36 | c.namespaces = make(map[string]*namespace) 37 | c.clock = clk 38 | return c 39 | } 40 | 41 | // coordinable is a common interface for objects that need to take the 42 | // global lock on the Coordinate state. 43 | type coordinable interface { 44 | // Coordinate returns a pointer to the coordinate object 45 | // at the root of this object tree. 46 | Coordinate() *memCoordinate 47 | } 48 | 49 | // globalLock locks the coordinate object at the root of the object 50 | // tree. Pair this with globalUnlock, as 51 | // 52 | // globalLock(self) 53 | // defer globalUnlock(self) 54 | func globalLock(c coordinable) { 55 | c.Coordinate().sem.Lock() 56 | } 57 | 58 | // globalUnlock unlocks the coordinate object at the root of the 59 | // object tree. 60 | func globalUnlock(c coordinable) { 61 | c.Coordinate().sem.Unlock() 62 | } 63 | 64 | // Coordinate wrapper type: 65 | 66 | type memCoordinate struct { 67 | namespaces map[string]*namespace 68 | sem sync.Mutex 69 | clock clock.Clock 70 | } 71 | 72 | func (c *memCoordinate) Namespace(namespace string) (coordinate.Namespace, error) { 73 | globalLock(c) 74 | defer globalUnlock(c) 75 | 76 | ns := c.namespaces[namespace] 77 | if ns == nil { 78 | ns = newNamespace(c, namespace) 79 | c.namespaces[namespace] = ns 80 | } 81 | return ns, nil 82 | } 83 | 84 | func (c *memCoordinate) Namespaces() (map[string]coordinate.Namespace, error) { 85 | globalLock(c) 86 | defer globalUnlock(c) 87 | 88 | result := make(map[string]coordinate.Namespace) 89 | for name, namespace := range c.namespaces { 90 | result[name] = namespace 91 | } 92 | return result, nil 93 | } 94 | 95 | func (c *memCoordinate) Summarize() (coordinate.Summary, error) { 96 | globalLock(c) 97 | defer globalUnlock(c) 98 | 99 | var result coordinate.Summary 100 | for _, ns := range c.namespaces { 101 | result = append(result, ns.summarize()...) 102 | } 103 | return result, nil 104 | } 105 | 106 | func (c *memCoordinate) Coordinate() *memCoordinate { 107 | return c 108 | } 109 | -------------------------------------------------------------------------------- /memory/coordinate_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2017 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package memory_test 5 | 6 | import ( 7 | "github.com/swiftlobste/go-coordinate/coordinate/coordinatetest" 8 | "github.com/swiftlobste/go-coordinate/memory" 9 | "github.com/stretchr/testify/suite" 10 | "testing" 11 | ) 12 | 13 | // Suite runs the generic Coordinate tests with a memory backend. 14 | type Suite struct { 15 | coordinatetest.Suite 16 | } 17 | 18 | // SetupSuite does one-time test setup, creating the memory backend. 19 | func (s *Suite) SetupSuite() { 20 | s.Suite.SetupSuite() 21 | s.Coordinate = memory.NewWithClock(s.Clock) 22 | } 23 | 24 | // TestCoordinate runs the generic Coordinate tests with a memory backend. 25 | func TestCoordinate(t *testing.T) { 26 | suite.Run(t, &Suite{}) 27 | } 28 | -------------------------------------------------------------------------------- /memory/namespace.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package memory 5 | 6 | import ( 7 | "github.com/swiftlobste/go-coordinate/coordinate" 8 | ) 9 | 10 | // namespace is a container type for a coordinate.Namespace. 11 | type namespace struct { 12 | name string 13 | coordinate *memCoordinate 14 | workSpecs map[string]*workSpec 15 | workers map[string]*worker 16 | deleted bool 17 | } 18 | 19 | func newNamespace(coordinate *memCoordinate, name string) *namespace { 20 | return &namespace{ 21 | name: name, 22 | coordinate: coordinate, 23 | workSpecs: make(map[string]*workSpec), 24 | workers: make(map[string]*worker), 25 | } 26 | } 27 | 28 | // coordinate.Namespace interface: 29 | 30 | func (ns *namespace) Name() string { 31 | return ns.name 32 | } 33 | 34 | func (ns *namespace) Destroy() error { 35 | globalLock(ns) 36 | defer globalUnlock(ns) 37 | 38 | delete(ns.coordinate.namespaces, ns.name) 39 | ns.deleted = true 40 | return nil 41 | } 42 | 43 | func (ns *namespace) do(f func() error) error { 44 | globalLock(ns) 45 | defer globalUnlock(ns) 46 | 47 | if ns.deleted { 48 | return coordinate.ErrGone 49 | } 50 | 51 | return f() 52 | } 53 | 54 | func (ns *namespace) SetWorkSpec(data map[string]interface{}) (spec coordinate.WorkSpec, err error) { 55 | err = ns.do(func() error { 56 | nameI := data["name"] 57 | if nameI == nil { 58 | return coordinate.ErrNoWorkSpecName 59 | } 60 | name, ok := nameI.(string) 61 | if !ok { 62 | return coordinate.ErrBadWorkSpecName 63 | } 64 | theSpec := ns.workSpecs[name] 65 | if theSpec == nil { 66 | theSpec = newWorkSpec(ns, name) 67 | ns.workSpecs[name] = theSpec 68 | } 69 | spec = theSpec 70 | return theSpec.setData(data) 71 | }) 72 | return 73 | } 74 | 75 | func (ns *namespace) WorkSpec(name string) (spec coordinate.WorkSpec, err error) { 76 | err = ns.do(func() error { 77 | var present bool 78 | spec, present = ns.workSpecs[name] 79 | if !present { 80 | return coordinate.ErrNoSuchWorkSpec{Name: name} 81 | } 82 | return nil 83 | }) 84 | return 85 | } 86 | 87 | func (ns *namespace) DestroyWorkSpec(name string) error { 88 | return ns.do(func() error { 89 | spec, present := ns.workSpecs[name] 90 | if !present { 91 | return coordinate.ErrNoSuchWorkSpec{Name: name} 92 | } 93 | spec.deleted = true 94 | delete(ns.workSpecs, name) 95 | return nil 96 | }) 97 | } 98 | 99 | func (ns *namespace) WorkSpecNames() (names []string, err error) { 100 | err = ns.do(func() error { 101 | names = make([]string, 0, len(ns.workSpecs)) 102 | for name := range ns.workSpecs { 103 | names = append(names, name) 104 | } 105 | return nil 106 | }) 107 | return 108 | } 109 | 110 | // allMetas retrieves the metadata for all work specs. This cannot 111 | // fail. It expects to run within the global lock. 112 | func (ns *namespace) allMetas(withCounts bool) (map[string]*workSpec, map[string]*coordinate.WorkSpecMeta) { 113 | metas := make(map[string]*coordinate.WorkSpecMeta) 114 | for name, spec := range ns.workSpecs { 115 | meta := spec.getMeta(withCounts) 116 | metas[name] = &meta 117 | } 118 | return ns.workSpecs, metas 119 | } 120 | 121 | func (ns *namespace) Worker(name string) (worker coordinate.Worker, err error) { 122 | err = ns.do(func() error { 123 | var present bool 124 | worker, present = ns.workers[name] 125 | if !present { 126 | ns.workers[name] = newWorker(ns, name) 127 | worker = ns.workers[name] 128 | } 129 | return nil 130 | }) 131 | return 132 | } 133 | 134 | func (ns *namespace) Workers() (workers map[string]coordinate.Worker, err error) { 135 | // subject to change, see comments in coordinate.go 136 | err = ns.do(func() error { 137 | workers = make(map[string]coordinate.Worker) 138 | for name, worker := range ns.workers { 139 | workers[name] = worker 140 | } 141 | return nil 142 | }) 143 | return 144 | } 145 | 146 | // coordinate.Summarizable interface: 147 | 148 | func (ns *namespace) Summarize() (result coordinate.Summary, err error) { 149 | err = ns.do(func() error { 150 | result = ns.summarize() 151 | return nil 152 | }) 153 | return 154 | } 155 | 156 | func (ns *namespace) summarize() coordinate.Summary { 157 | var result coordinate.Summary 158 | for _, spec := range ns.workSpecs { 159 | result = append(result, spec.summarize()...) 160 | } 161 | return result 162 | } 163 | 164 | // memory.coordinable interface: 165 | 166 | func (ns *namespace) Coordinate() *memCoordinate { 167 | return ns.coordinate 168 | } 169 | -------------------------------------------------------------------------------- /memory/work_unit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package memory 5 | 6 | import ( 7 | "github.com/swiftlobste/go-coordinate/coordinate" 8 | ) 9 | 10 | type workUnit struct { 11 | name string 12 | data map[string]interface{} 13 | meta coordinate.WorkUnitMeta 14 | activeAttempt *attempt 15 | attempts []*attempt 16 | workSpec *workSpec 17 | availableIndex int 18 | deleted bool 19 | } 20 | 21 | // coordinate.WorkUnit interface: 22 | 23 | func (unit *workUnit) Name() string { 24 | return unit.name 25 | } 26 | 27 | func (unit *workUnit) Data() (data map[string]interface{}, err error) { 28 | err = unit.do(func() error { 29 | data = unit.data 30 | if unit.activeAttempt != nil && unit.activeAttempt.data != nil { 31 | data = unit.activeAttempt.data 32 | } 33 | return nil 34 | }) 35 | return 36 | } 37 | 38 | func (unit *workUnit) WorkSpec() coordinate.WorkSpec { 39 | return unit.workSpec 40 | } 41 | 42 | func (unit *workUnit) do(f func() error) error { 43 | globalLock(unit) 44 | defer globalUnlock(unit) 45 | if unit.deleted || unit.workSpec.deleted || unit.workSpec.namespace.deleted { 46 | return coordinate.ErrGone 47 | } 48 | return f() 49 | } 50 | 51 | func (unit *workUnit) Status() (status coordinate.WorkUnitStatus, err error) { 52 | err = unit.do(func() error { 53 | unit.workSpec.expireUnits() 54 | status = unit.status() 55 | return nil 56 | }) 57 | return 58 | } 59 | 60 | // status is an internal helper that converts a single unit's attempt 61 | // status to a work unit status. It assumes the global lock (and that 62 | // the active attempt will not change under it). It assumes that, if 63 | // expiry is necessary, it has already been run. 64 | func (unit *workUnit) status() coordinate.WorkUnitStatus { 65 | if unit.activeAttempt == nil { 66 | now := unit.Coordinate().clock.Now() 67 | switch { 68 | case now.Before(unit.meta.NotBefore): 69 | return coordinate.DelayedUnit 70 | default: 71 | return coordinate.AvailableUnit 72 | } 73 | } 74 | switch unit.activeAttempt.status { 75 | case coordinate.Pending: 76 | return coordinate.PendingUnit 77 | case coordinate.Expired: 78 | return coordinate.AvailableUnit 79 | case coordinate.Finished: 80 | return coordinate.FinishedUnit 81 | case coordinate.Failed: 82 | return coordinate.FailedUnit 83 | case coordinate.Retryable: 84 | return coordinate.AvailableUnit 85 | default: 86 | panic("invalid attempt status") 87 | } 88 | } 89 | 90 | func (unit *workUnit) Meta() (meta coordinate.WorkUnitMeta, err error) { 91 | err = unit.do(func() error { 92 | meta = unit.meta 93 | return nil 94 | }) 95 | return 96 | } 97 | 98 | func (unit *workUnit) SetMeta(meta coordinate.WorkUnitMeta) error { 99 | return unit.do(func() error { 100 | unit.meta = meta 101 | unit.workSpec.available.Reprioritize(unit) 102 | return nil 103 | }) 104 | } 105 | 106 | func (unit *workUnit) Priority() (float64, error) { 107 | meta, err := unit.Meta() // does the lock itself 108 | return meta.Priority, err 109 | } 110 | 111 | func (unit *workUnit) SetPriority(priority float64) error { 112 | return unit.do(func() error { 113 | unit.meta.Priority = priority 114 | unit.workSpec.available.Reprioritize(unit) 115 | return nil 116 | }) 117 | } 118 | 119 | func (unit *workUnit) ActiveAttempt() (attempt coordinate.Attempt, err error) { 120 | err = unit.do(func() error { 121 | unit.workSpec.expireUnits() 122 | // Since this returns an interface type, if we just 123 | // return unit.activeAttempt, we will get back a nil 124 | // with a concrete type which is not equal to nil with 125 | // interface type. Go Go go! 126 | if unit.activeAttempt != nil { 127 | attempt = unit.activeAttempt 128 | } 129 | return nil 130 | }) 131 | return 132 | } 133 | 134 | // resetAttempt clears the active attempt for a unit and returns it 135 | // to its work spec's available list. Assumes the global lock. 136 | func (unit *workUnit) resetAttempt() { 137 | if unit.activeAttempt != nil { 138 | unit.activeAttempt = nil 139 | unit.workSpec.available.Add(unit) 140 | } 141 | } 142 | 143 | func (unit *workUnit) ClearActiveAttempt() error { 144 | return unit.do(func() error { 145 | unit.resetAttempt() 146 | return nil 147 | }) 148 | } 149 | 150 | func (unit *workUnit) NumAttempts() (int, error) { 151 | num := 0 152 | unit.do(func() error { 153 | num = len(unit.attempts) 154 | return nil 155 | }) 156 | return num, nil 157 | } 158 | 159 | func (unit *workUnit) Attempts() (attempts []coordinate.Attempt, err error) { 160 | err = unit.do(func() error { 161 | attempts = make([]coordinate.Attempt, len(unit.attempts)) 162 | for i, attempt := range unit.attempts { 163 | attempts[i] = attempt 164 | } 165 | return nil 166 | }) 167 | return 168 | } 169 | 170 | // memory.coordinable interface: 171 | 172 | func (unit *workUnit) Coordinate() *memCoordinate { 173 | return unit.workSpec.namespace.coordinate 174 | } 175 | -------------------------------------------------------------------------------- /postgres/coordinate.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package postgres 5 | 6 | import ( 7 | "database/sql" 8 | "encoding/gob" 9 | "github.com/benbjohnson/clock" 10 | "github.com/swiftlobste/go-coordinate/cborrpc" 11 | "github.com/swiftlobste/go-coordinate/coordinate" 12 | "github.com/satori/go.uuid" 13 | "strings" 14 | ) 15 | 16 | type pgCoordinate struct { 17 | db *sql.DB 18 | clock clock.Clock 19 | Expiry expiry 20 | } 21 | 22 | // New creates a new coordinate.Coordinate connection object using 23 | // the provided PostgreSQL connection string. The connection string 24 | // may be an expanded PostgreSQL string, a "postgres:" URL, or a URL 25 | // without a scheme. These are all equivalent: 26 | // 27 | // "host=localhost user=postgres password=postgres dbname=postgres" 28 | // "postgres://postgres:postgres@localhost/postgres" 29 | // "//postgres:postgres@localhost/postgres" 30 | // 31 | // See http://godoc.org/github.com/lib/pq for more details. If 32 | // parameters are missing from this string (or if you pass an empty 33 | // string) they can be filled in from environment variables as well; 34 | // see 35 | // http://www.postgresql.org/docs/current/static/libpq-envars.html. 36 | // 37 | // The returned Coordinate object carries around a connection pool 38 | // with it. It can (and should) be shared across the application. 39 | // This New() function should be called sparingly, ideally exactly once. 40 | func New(connectionString string) (coordinate.Coordinate, error) { 41 | clk := clock.New() 42 | return NewWithClock(connectionString, clk) 43 | } 44 | 45 | // NewWithClock creates a new coordinate.Coordinate connection object, 46 | // using an explicit time source. See New() for further details. 47 | // Most application code should call New(), and use the default (real) 48 | // time source; this entry point is intended for tests that need to 49 | // inject a mock time source. 50 | func NewWithClock(connectionString string, clk clock.Clock) (coordinate.Coordinate, error) { 51 | // If the connection string is a destructured URL, turn it 52 | // back into a proper URL 53 | if len(connectionString) >= 2 && connectionString[0] == '/' && connectionString[1] == '/' { 54 | connectionString = "postgres:" + connectionString 55 | } 56 | 57 | // Add some custom parameters. 58 | // 59 | // We'd love to make the transaction isolation level 60 | // SERIALIZABLE, and the documentation suggests that it solves 61 | // all our concurrency problems. In practice, at least on 62 | // PostgreSQL 9.3, there are issues with returning duplicate 63 | // attempts...even though that's a sequence 64 | // 65 | // SELECT ... FROM work_units WHERE active_attempt_id IS NULL 66 | // UPDATE work_units SET active_attempt_id=$1 67 | // 68 | // with an obvious conflict? 69 | if strings.Contains(connectionString, "://") { 70 | if strings.Contains(connectionString, "?") { 71 | connectionString += "&" 72 | } else { 73 | connectionString += "?" 74 | } 75 | connectionString += "default_transaction_isolation=repeatable%20read" 76 | } else { 77 | if len(connectionString) > 0 { 78 | connectionString += " " 79 | } 80 | connectionString += "default_transaction_isolation='repeatable read'" 81 | } 82 | 83 | db, err := sql.Open("postgres", connectionString) 84 | if err != nil { 85 | return nil, err 86 | } 87 | // TODO(dmaze): shouldn't unconditionally do this force-upgrade here 88 | err = Upgrade(db) 89 | if err != nil { 90 | return nil, err 91 | } 92 | // Make sure the gob library understands our data maps 93 | gob.Register(map[string]interface{}{}) 94 | gob.Register(map[interface{}]interface{}{}) 95 | gob.Register([]interface{}{}) 96 | gob.Register(cborrpc.PythonTuple{}) 97 | gob.Register(uuid.UUID{}) 98 | 99 | c := pgCoordinate{ 100 | db: db, 101 | clock: clk, 102 | } 103 | c.Expiry.Init() 104 | 105 | return &c, nil 106 | } 107 | 108 | func (c *pgCoordinate) Coordinate() *pgCoordinate { 109 | return c 110 | } 111 | 112 | // coordinable describes the class of structures that can reach back to 113 | // the root pgCoordinate object. 114 | type coordinable interface { 115 | // Coordinate returns the object at the root of the object tree. 116 | Coordinate() *pgCoordinate 117 | } 118 | -------------------------------------------------------------------------------- /postgres/coordinate_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2017 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package postgres_test 5 | 6 | import ( 7 | "github.com/swiftlobste/go-coordinate/coordinate/coordinatetest" 8 | "github.com/swiftlobste/go-coordinate/postgres" 9 | "github.com/stretchr/testify/suite" 10 | "testing" 11 | ) 12 | 13 | // Suite runs the generic Coordinate tests with a PostgreSQL backend. 14 | type Suite struct { 15 | coordinatetest.Suite 16 | } 17 | 18 | // SetupSuite does one-time test setup, creating the PostgreSQL backend. 19 | func (s *Suite) SetupSuite() { 20 | s.Suite.SetupSuite() 21 | c, err := postgres.NewWithClock("", s.Clock) 22 | if err != nil { 23 | panic(err) 24 | } 25 | s.Coordinate = c 26 | } 27 | 28 | // TestCoordinate runs the generic Coordinate tests with a PostgreSQL 29 | // backend. 30 | func TestCoordinate(t *testing.T) { 31 | suite.Run(t, &Suite{}) 32 | } 33 | -------------------------------------------------------------------------------- /postgres/expiry.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package postgres 5 | 6 | import ( 7 | "database/sql" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | // expiry manages the semi-global expiration process. In particular 13 | // it ensures that not more than one instance of expiration is running 14 | // at a time. 15 | type expiry struct { 16 | Cond *sync.Cond 17 | Running bool 18 | } 19 | 20 | // Init initializes an expiry object. 21 | func (exp *expiry) Init() { 22 | exp.Cond = sync.NewCond(&sync.Mutex{}) 23 | } 24 | 25 | // Do runs expiry. When it returns, an instance of expiry has run to 26 | // completion. It may not actually perform expiry itself, instead 27 | // blocking on some other goroutine to finish the job. 28 | func (exp *expiry) Do(c coordinable) { 29 | // This lock protects Running and also is involved in the 30 | // condition variable 31 | exp.Cond.L.Lock() 32 | if exp.Running { 33 | // Note that really our only goal here is to ensure 34 | // that expiry runs once, so while the sync.Cond 35 | // documentation suggests running in a loop to make 36 | // sure the condition really is satisfied, if we ever 37 | // get signaled then our condition has been met. 38 | exp.Cond.Wait() 39 | } else { 40 | exp.Running = true 41 | exp.Cond.L.Unlock() 42 | // Unlock before actually running expiry so that other 43 | // goroutines can run; they will block in the section 44 | // above 45 | 46 | _ = withTx(c, false, func(tx *sql.Tx) error { 47 | return expireAttempts(c, tx) 48 | }) 49 | 50 | exp.Cond.L.Lock() 51 | exp.Running = false 52 | exp.Cond.Broadcast() 53 | } 54 | exp.Cond.L.Unlock() 55 | } 56 | 57 | // expireAttempts finds all attempts whose expiration time has passed 58 | // and expires them. It runs on all attempts for all work units in all 59 | // work specs in all namespaces (which simplifies the query). Expired 60 | // attempts' statuses become "expired", and those attempts cease to be 61 | // the active attempt for their corresponding work unit. 62 | // 63 | // In general this should be called in its own transaction and its error 64 | // return ignored: 65 | // 66 | // _ = withTx(self, false, func(tx *sql.Tx) error { 67 | // return expireAttempts(self, tx) 68 | // }) 69 | // 70 | // Expiry is generally secondary to whatever actual work is going on. 71 | // If a result is different because of expiry, pretend the relevant 72 | // call was made a second earlier or later. If this fails, then 73 | // either there is a concurrency issue (and since the query is 74 | // system-global, the other expirer will clean up for us) or there is 75 | // an operational error (and the caller will fail afterwards). 76 | func expireAttempts(c coordinable, tx *sql.Tx) error { 77 | // There are several places this is called with much smaller 78 | // scope. For instance, Attempt.Status() needs to invoke 79 | // expiry but only actually cares about this very specific 80 | // attempt. If there are multiple namespaces, 81 | // Worker.RequestAttempts() only cares about this namespace 82 | // (though it will run on all work specs). It may help system 83 | // performance to try to run this with narrower scope. 84 | // 85 | // This is probably also an excellent candidate for a stored 86 | // procedure. 87 | var ( 88 | now time.Time 89 | cte, query string 90 | count int64 91 | result sql.Result 92 | err error 93 | ) 94 | 95 | now = c.Coordinate().clock.Now() 96 | 97 | // Remove expiring attempts from their work unit 98 | qp := queryParams{} 99 | cte = buildSelect([]string{ 100 | attemptID, 101 | }, []string{ 102 | attemptTable, 103 | }, []string{ 104 | attemptIsPending, 105 | attemptIsExpired(&qp, now), 106 | }) 107 | query = buildUpdate(workUnitTable, 108 | []string{"active_attempt_id=NULL"}, 109 | []string{"active_attempt_id IN (" + cte + ")"}) 110 | result, err = tx.Exec(query, qp...) 111 | if err != nil { 112 | return err 113 | } 114 | 115 | // If this marked nothing as expired, we're done 116 | count, err = result.RowsAffected() 117 | if err != nil { 118 | return err 119 | } 120 | if count == 0 { 121 | return nil 122 | } 123 | 124 | // Mark attempts as expired 125 | qp = queryParams{} 126 | // A slightly exotic setup, since we want to reuse the "now" 127 | // param 128 | dollarsNow := qp.Param(now) 129 | fields := fieldList{} 130 | fields.AddDirect("expiration_time", dollarsNow) 131 | fields.AddDirect("status", "'expired'") 132 | query = buildUpdate(attemptTable, fields.UpdateChanges(), []string{ 133 | attemptIsPending, 134 | attemptExpirationTime + "<" + dollarsNow, 135 | }) 136 | _, err = tx.Exec(query, qp...) 137 | return err 138 | } 139 | -------------------------------------------------------------------------------- /postgres/helpers.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package postgres 5 | 6 | import ( 7 | "github.com/swiftlobste/go-coordinate/cborrpc" 8 | "github.com/ugorji/go/codec" 9 | ) 10 | 11 | // dictionary <-> binary encoders 12 | 13 | func mapToBytes(in map[string]interface{}) (out []byte, err error) { 14 | cbor := new(codec.CborHandle) 15 | err = cborrpc.SetExts(cbor) 16 | if err != nil { 17 | return 18 | } 19 | encoder := codec.NewEncoderBytes(&out, cbor) 20 | err = encoder.Encode(in) 21 | return 22 | } 23 | 24 | func bytesToMap(in []byte) (out map[string]interface{}, err error) { 25 | cbor := new(codec.CborHandle) 26 | err = cborrpc.SetExts(cbor) 27 | if err != nil { 28 | return 29 | } 30 | decoder := codec.NewDecoderBytes(in, cbor) 31 | err = decoder.Decode(&out) 32 | return 33 | } 34 | -------------------------------------------------------------------------------- /postgres/migration.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package postgres 5 | 6 | import ( 7 | "database/sql" 8 | "github.com/rubenv/sql-migrate" 9 | ) 10 | 11 | // This file maintains the database migration code. See 12 | // https://github.com/rubenv/sql-migrate for details of what goes in 13 | // here. This runs "outside" the normal coordinate flow, either at 14 | // initial startup or from an external tool. 15 | 16 | //go:generate go-bindata -pkg postgres -o migrations.go migrations/ 17 | 18 | var migrationSource = &migrate.AssetMigrationSource{ 19 | Asset: Asset, 20 | AssetDir: AssetDir, 21 | Dir: "migrations", 22 | } 23 | 24 | // Upgrade upgrades a database to the latest database schema version. 25 | func Upgrade(db *sql.DB) error { 26 | _, err := migrate.Exec(db, "postgres", migrationSource, migrate.Up) 27 | return err 28 | } 29 | 30 | // Drop clears a database by running all of the migrations in reverse, 31 | // ultimately resulting in dropping all of the tables. 32 | func Drop(db *sql.DB) error { 33 | _, err := migrate.Exec(db, "postgres", migrationSource, migrate.Down) 34 | return err 35 | } 36 | -------------------------------------------------------------------------------- /postgres/migrations/20150927-core.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres -*- 2 | -- +migrate Up 3 | CREATE TABLE namespace( 4 | id SERIAL PRIMARY KEY, 5 | name VARCHAR UNIQUE NOT NULL 6 | ); 7 | 8 | CREATE TABLE work_spec( 9 | id SERIAL PRIMARY KEY, 10 | namespace_id INTEGER NOT NULL 11 | REFERENCES namespace(id) ON DELETE CASCADE, 12 | name VARCHAR NOT NULL, 13 | data BYTEA NOT NULL, 14 | priority INTEGER NOT NULL, 15 | weight INTEGER NOT NULL, 16 | paused BOOLEAN NOT NULL, 17 | continuous BOOLEAN NOT NULL, 18 | can_be_continuous BOOLEAN NOT NULL, 19 | interval INTERVAL NOT NULL, 20 | next_continuous TIMESTAMP WITH TIME ZONE, 21 | max_running INTEGER NOT NULL, 22 | max_attempts_returned INTEGER NOT NULL, 23 | next_work_spec_name VARCHAR NOT NULL, 24 | next_work_spec_preempts BOOLEAN NOT NULL, 25 | CONSTRAINT work_spec_unique_name UNIQUE(namespace_id, name) 26 | ); 27 | 28 | CREATE TABLE work_unit( 29 | id SERIAL PRIMARY KEY, 30 | work_spec_id INTEGER NOT NULL 31 | REFERENCES work_spec(id) ON DELETE CASCADE, 32 | name VARCHAR NOT NULL, 33 | data BYTEA NOT NULL, 34 | priority DOUBLE PRECISION NOT NULL, 35 | active_attempt_id INTEGER, 36 | CONSTRAINT work_unit_unique_name UNIQUE(work_spec_id, name) 37 | ); 38 | 39 | CREATE TABLE worker( 40 | id SERIAL PRIMARY KEY, 41 | namespace_id INTEGER NOT NULL 42 | REFERENCES namespace(id) ON DELETE CASCADE, 43 | name VARCHAR NOT NULL, 44 | parent INTEGER REFERENCES worker(id) ON DELETE SET NULL, 45 | active BOOLEAN NOT NULL, 46 | mode INTEGER NOT NULL, 47 | data BYTEA NOT NULL, 48 | expiration TIMESTAMP WITH TIME ZONE NOT NULL, 49 | last_update TIMESTAMP WITH TIME ZONE NOT NULL, 50 | CONSTRAINT worker_unique_name UNIQUE(namespace_id, name) 51 | ); 52 | 53 | CREATE TYPE attempt_status AS ENUM('pending', 'expired', 'finished', 54 | 'failed', 'retryable'); 55 | 56 | CREATE TABLE attempt( 57 | id SERIAL PRIMARY KEY, 58 | work_unit_id INTEGER NOT NULL 59 | REFERENCES work_unit(id) ON DELETE CASCADE, 60 | worker_id INTEGER NOT NULL 61 | REFERENCES worker(id) ON DELETE CASCADE, 62 | status attempt_status NOT NULL DEFAULT 'pending', 63 | data BYTEA, 64 | start_time TIMESTAMP WITH TIME ZONE NOT NULL, 65 | end_time TIMESTAMP WITH TIME ZONE, 66 | expiration_time TIMESTAMP WITH TIME ZONE NOT NULL, 67 | active BOOLEAN NOT NULL DEFAULT TRUE 68 | ); 69 | 70 | ALTER TABLE work_unit 71 | ADD CONSTRAINT work_unit_active_attempt_valid 72 | FOREIGN KEY (active_attempt_id) REFERENCES attempt(id) 73 | ON DELETE SET NULL; 74 | 75 | -- +migrate Down 76 | ALTER TABLE work_unit DROP CONSTRAINT work_unit_active_attempt_valid; 77 | DROP TABLE attempt; 78 | DROP TYPE attempt_status; 79 | DROP TABLE worker; 80 | DROP TABLE work_unit; 81 | DROP TABLE work_spec; 82 | DROP TABLE namespace; 83 | -------------------------------------------------------------------------------- /postgres/migrations/20151002-mingb.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres -*- 2 | -- +migrate Up 3 | ALTER TABLE work_spec ADD COLUMN min_memory_gb DOUBLE PRECISION NOT NULL DEFAULT 0; 4 | 5 | -- +migrate Down 6 | ALTER TABLE work_spec DROP COLUMN min_memory_gb; 7 | -------------------------------------------------------------------------------- /postgres/migrations/20151006-work-unit-type.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres -*- 2 | -- This changes the work_unit.name type from VARCHAR to BYTEA. Any existing 3 | -- work units are assumed to be UTF-8 encoded. 4 | 5 | -- +migrate Up 6 | ALTER TABLE work_unit ALTER COLUMN name SET DATA TYPE BYTEA 7 | USING convert_to(name, 'UTF8'); 8 | 9 | -- +migrate Down 10 | ALTER TABLE work_unit ALTER COLUMN name SET DATA TYPE VARCHAR 11 | USING convert_from(name, 'UTF8'); 12 | -------------------------------------------------------------------------------- /postgres/migrations/20151013-index.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres -*- 2 | -- +migrate Up 3 | CREATE INDEX work_unit_ordering ON work_unit(priority DESC, name ASC); 4 | 5 | -- +migrate Down 6 | DROP INDEX work_unit_ordering; 7 | -------------------------------------------------------------------------------- /postgres/migrations/20151014-index.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres -*- 2 | -- +migrate Up 3 | CREATE INDEX work_unit_spec ON work_unit(work_spec_id); 4 | CREATE INDEX work_unit_spec_attempt ON work_unit(work_spec_id, active_attempt_id); 5 | 6 | -- +migrate Down 7 | DROP INDEX work_unit_spec_attempt; 8 | DROP INDEX work_unit_spec; 9 | -------------------------------------------------------------------------------- /postgres/migrations/20151019-worker-mode.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres -*- 2 | -- 3 | -- This changes the worker mode from int to string. Existing worker modes 4 | -- are discarded. 5 | -- 6 | -- +migrate Up 7 | ALTER TABLE worker ALTER COLUMN mode SET DATA TYPE VARCHAR USING ''; 8 | 9 | -- +migrate Down 10 | ALTER TABLE worker ALTER COLUMN mode SET DATA TYPE INTEGER USING 0; 11 | -------------------------------------------------------------------------------- /postgres/migrations/20151028-index.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres -*- 2 | -- +migrate Up 3 | CREATE INDEX attempt_status_expiration ON attempt(status, expiration_time); 4 | CREATE INDEX attempt_worker ON attempt(worker_id); 5 | CREATE INDEX work_unit_attempt ON work_unit(active_attempt_id); 6 | 7 | -- +migrate Down 8 | DROP INDEX work_unit_attempt; 9 | DROP INDEX attempt_worker; 10 | DROP INDEX attempt_status_expiration; 11 | -------------------------------------------------------------------------------- /postgres/migrations/20151216-work-spec-runtime.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres -*- 2 | -- 3 | -- This adds a "runtime" column to the work spec table. 4 | -- 5 | -- +migrate Up 6 | ALTER TABLE work_spec ADD COLUMN runtime VARCHAR NOT NULL DEFAULT ''; 7 | 8 | -- +migrate Down 9 | ALTER TABLE work_spec DROP COLUMN runtime; 10 | -------------------------------------------------------------------------------- /postgres/migrations/20160104-not-before.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres -*- 2 | -- 3 | -- This adds a "not_before" column to the work unit table. 4 | -- 5 | -- +migrate Up 6 | ALTER TABLE work_unit ADD COLUMN not_before TIMESTAMP WITH TIME ZONE; 7 | 8 | -- +migrate Down 9 | ALTER TABLE work_unit DROP COLUMN not_before; 10 | -------------------------------------------------------------------------------- /postgres/migrations/20160125-index.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres -*- 2 | -- 3 | -- This adds an index to efficiently find attempts by their work 4 | -- units. In particular, this fixes a problem where deleting large 5 | -- numbers of finished work units is slow; experimentally it leads to 6 | -- O(n^2) behavior on the single SQL DELETE command. 7 | -- 8 | -- +migrate Up 9 | CREATE INDEX attempt_work_unit ON attempt(work_unit_id); 10 | 11 | -- +migrate Down 12 | DROP INDEX attempt_work_unit; 13 | -------------------------------------------------------------------------------- /postgres/migrations/20160217-attempt-spec.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres -*- 2 | -- 3 | -- This adds a work_spec column to the attempt table, so that we can 4 | -- efficiently find work specs with pending work. 5 | 6 | -- +migrate Up 7 | ALTER TABLE attempt 8 | ADD COLUMN work_spec_id INTEGER 9 | REFERENCES work_spec(id) ON DELETE CASCADE; 10 | 11 | UPDATE attempt 12 | SET work_spec_id=work_unit.work_spec_id 13 | FROM work_unit 14 | WHERE work_unit.id=attempt.work_unit_id; 15 | 16 | ALTER TABLE attempt 17 | ALTER COLUMN work_spec_id SET NOT NULL; 18 | 19 | -- +migrate Down 20 | ALTER TABLE attempt 21 | DROP COLUMN work_spec_id; 22 | -------------------------------------------------------------------------------- /postgres/migrations/20160328-index.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres -*- 2 | -- 3 | -- This updates the index that finds work units with the highest priority 4 | -- to skip over work units with active attempts, that can't be done. It 5 | -- also drops a redundant index of work specs and active attempt IDs; if 6 | -- PostgreSQL needs both things it can ask both single-field indexes. 7 | -- 8 | -- +migrate Up 9 | DROP INDEX work_unit_ordering; 10 | CREATE INDEX work_unit_ordering ON work_unit(priority DESC, name ASC) 11 | WHERE active_attempt_id IS NULL; 12 | DROP INDEX work_unit_spec_attempt; 13 | 14 | -- +migrate Down 15 | DROP INDEX work_unit_ordering; 16 | CREATE INDEX work_unit_ordering ON work_unit(priority DESC, name ASC); 17 | CREATE INDEX work_unit_spec_attempt ON work_unit(work_spec_id, active_attempt_id); 18 | -------------------------------------------------------------------------------- /postgres/migrations/20160329-index.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres -*- 2 | -- 3 | -- This adds another restricted index to help find work specs with 4 | -- available work, when many of the work units have been attempted 5 | -- already. 6 | -- 7 | -- +migrate Up 8 | CREATE INDEX work_unit_spec_available ON work_unit(work_spec_id) 9 | WHERE active_attempt_id IS NULL; 10 | 11 | -- +migrate Down 12 | DROP INDEX work_unit_spec_available; 13 | -------------------------------------------------------------------------------- /postgres/migrations/20170316-index.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres -*- 2 | -- 3 | -- Adds index to worker.parent so a common lookup is optimized 4 | -- 5 | -- +migrate Up 6 | 7 | CREATE INDEX worker_parent_idx on worker(parent); 8 | 9 | -- +migrate Down 10 | DROP INDEX worker_parent_idx; 11 | -------------------------------------------------------------------------------- /postgres/migrations/20170523-work-unit-max-retries.sql: -------------------------------------------------------------------------------- 1 | -- -*- mode: sql; sql-product: postgres -*- 2 | -- 3 | -- Adds a max_retries field to work_spec. 4 | -- 5 | -- +migrate Up 6 | ALTER TABLE work_spec ADD COLUMN max_retries INTEGER NOT NULL DEFAULT 0; 7 | 8 | -- +migrate Down 9 | ALTER TABLE work_spec DROP COLUMN max_retries; 10 | -------------------------------------------------------------------------------- /postgres/namespace.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package postgres 5 | 6 | import ( 7 | "database/sql" 8 | "github.com/swiftlobste/go-coordinate/coordinate" 9 | ) 10 | 11 | type namespace struct { 12 | coordinate *pgCoordinate 13 | id int 14 | name string 15 | } 16 | 17 | // coordinate.Coordinate.Namespace() "constructor": 18 | 19 | func (c *pgCoordinate) Namespace(name string) (coordinate.Namespace, error) { 20 | ns := namespace{ 21 | coordinate: c, 22 | name: name, 23 | } 24 | err := withTx(c, false, func(tx *sql.Tx) error { 25 | row := tx.QueryRow("SELECT id FROM namespace WHERE name=$1", name) 26 | err := row.Scan(&ns.id) 27 | if err == sql.ErrNoRows { 28 | // Create the namespace 29 | row = tx.QueryRow("INSERT INTO namespace(name) VALUES ($1) RETURNING id", name) 30 | err = row.Scan(&ns.id) 31 | } 32 | return err 33 | }) 34 | if err != nil { 35 | return nil, err 36 | } 37 | return &ns, nil 38 | } 39 | 40 | func (c *pgCoordinate) Namespaces() (map[string]coordinate.Namespace, error) { 41 | result := make(map[string]coordinate.Namespace) 42 | params := queryParams{} 43 | query := buildSelect([]string{ 44 | namespaceName, 45 | namespaceID, 46 | }, []string{ 47 | namespaceTable, 48 | }, []string{}) 49 | err := queryAndScan(c, query, params, func(rows *sql.Rows) error { 50 | ns := namespace{coordinate: c} 51 | err := rows.Scan(&ns.name, &ns.id) 52 | if err != nil { 53 | return err 54 | } 55 | result[ns.name] = &ns 56 | return nil 57 | }) 58 | if err != nil { 59 | return nil, err 60 | } 61 | return result, nil 62 | } 63 | 64 | // coordinate.Namespace interface: 65 | 66 | func (ns *namespace) Name() string { 67 | return ns.name 68 | } 69 | 70 | func (ns *namespace) Destroy() error { 71 | params := queryParams{} 72 | query := "DELETE FROM NAMESPACE WHERE id=" + params.Param(ns.id) 73 | return execInTx(ns, query, params, false) 74 | } 75 | 76 | // coordinable interface: 77 | 78 | func (ns *namespace) Coordinate() *pgCoordinate { 79 | return ns.coordinate 80 | } 81 | -------------------------------------------------------------------------------- /postgres/sql_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package postgres 5 | 6 | import ( 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | type duration struct { 14 | Duration time.Duration 15 | Written string 16 | Read string 17 | } 18 | 19 | var someTimes = []duration{ 20 | duration{time.Duration(1 * time.Second), "0 0:0:1.000000", "00:00:01"}, 21 | duration{time.Duration(1 * time.Minute), "0 0:1:0.000000", "00:01:00"}, 22 | duration{time.Duration(1 * time.Hour), "0 1:0:0.000000", "01:00:00"}, 23 | duration{time.Duration(24 * time.Hour), "1 0:0:0.000000", "1 day"}, 24 | duration{time.Duration(25 * time.Hour), "1 1:0:0.000000", "1 day 01:00:00"}, 25 | duration{time.Duration(49 * time.Hour), "2 1:0:0.000000", "2 days 01:00:00"}, 26 | } 27 | 28 | func TestDurationToSQL(t *testing.T) { 29 | for _, d := range someTimes { 30 | actual := string(durationToSQL(d.Duration)) 31 | assert.Equal(t, d.Written, actual) 32 | } 33 | } 34 | 35 | func TestSQLToDuration(t *testing.T) { 36 | for _, d := range someTimes { 37 | actual, err := sqlToDuration(d.Read) 38 | if assert.NoError(t, err, d.Read) { 39 | assert.Equal(t, d.Duration, actual, d.Read) 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /postgres/stats.go: -------------------------------------------------------------------------------- 1 | // Statistics generation for everything that needs it. 2 | // 3 | // Copyright 2017 Diffeo, Inc. 4 | // This software is released under an MIT/X11 open source license. 5 | 6 | package postgres 7 | 8 | import ( 9 | "database/sql" 10 | 11 | "github.com/swiftlobste/go-coordinate/coordinate" 12 | ) 13 | 14 | // summarize computes summary statistics for things. This runs a single 15 | // SQL query over namespaces, work specs, work units, and attempts; 16 | // it selects all active attempts everywhere, further limited by 17 | // whatever is passed as restrictions. 18 | func summarize( 19 | c coordinable, 20 | params queryParams, 21 | restrictions []string, 22 | ) (coordinate.Summary, error) { 23 | var result coordinate.Summary 24 | outputs := []string{ 25 | namespaceName, 26 | workSpecName, 27 | attemptStatus, 28 | workUnitTooSoon(¶ms, c.Coordinate().clock.Now()) + " delayed", 29 | "COUNT(*)", 30 | } 31 | tables := []string{ 32 | namespaceTable, 33 | workSpecTable, 34 | workUnitAttemptJoin, 35 | } 36 | conditions := []string{ 37 | workSpecInThisNamespace, 38 | workUnitInThisSpec, 39 | } 40 | conditions = append(conditions, restrictions...) 41 | query := buildSelect(outputs, tables, conditions) 42 | query += (" GROUP BY " + namespaceName + ", " + workSpecName + ", " + 43 | attemptStatus + ", delayed") 44 | err := queryAndScan(c, query, params, func(rows *sql.Rows) error { 45 | var record coordinate.SummaryRecord 46 | var status sql.NullString 47 | var delayed bool 48 | err := rows.Scan(&record.Namespace, &record.WorkSpec, &status, 49 | &delayed, &record.Count) 50 | if err != nil { 51 | return err 52 | } 53 | if !status.Valid { 54 | if delayed { 55 | record.Status = coordinate.DelayedUnit 56 | } else { 57 | record.Status = coordinate.AvailableUnit 58 | } 59 | } else { 60 | switch status.String { 61 | case "expired": 62 | record.Status = coordinate.AvailableUnit 63 | case "retryable": 64 | record.Status = coordinate.AvailableUnit 65 | case "pending": 66 | record.Status = coordinate.PendingUnit 67 | case "finished": 68 | record.Status = coordinate.FinishedUnit 69 | case "failed": 70 | record.Status = coordinate.FailedUnit 71 | } 72 | } 73 | result = append(result, record) 74 | return nil 75 | }) 76 | if err != nil { 77 | return coordinate.Summary{}, err 78 | } 79 | return result, nil 80 | } 81 | 82 | func (c *pgCoordinate) Summarize() (coordinate.Summary, error) { 83 | return summarize(c, nil, nil) 84 | } 85 | 86 | func (ns *namespace) Summarize() (coordinate.Summary, error) { 87 | var params queryParams 88 | restrictions := []string{ 89 | isNamespace(¶ms, ns.id), 90 | } 91 | return summarize(ns, params, restrictions) 92 | } 93 | 94 | func (spec *workSpec) Summarize() (coordinate.Summary, error) { 95 | var params queryParams 96 | restrictions := []string{ 97 | isWorkSpec(¶ms, spec.id), 98 | } 99 | return summarize(spec, params, restrictions) 100 | } 101 | -------------------------------------------------------------------------------- /restclient/README.md: -------------------------------------------------------------------------------- 1 | Coordinate REST Client 2 | ====================== 3 | 4 | This package defines a REST client for the Coordinate system. It matches 5 | the REST server in the [../restserver](restserver) package. 6 | 7 | Usage 8 | ----- 9 | 10 | Run a [../cmd/coordinated](coordinated) process with an HTTP server, e.g. 11 | 12 | ```sh 13 | go install github.com/swiftlobste/go-coordinate/cmd/coordinated 14 | coordinated -backend memory -http :5980 15 | ``` 16 | 17 | Then run your client code pointing at that service. To run a 18 | coordinated proxy, for instance, 19 | 20 | ```sh 21 | coordinated -backend http://localhost:5980/ -http :5981 22 | ``` 23 | 24 | Limitations 25 | ----------- 26 | 27 | In most cases names of objects are embedded directly into URLs. If 28 | they are not printable, the URL scheme can pass base64-encoded keys as 29 | well; see the [../restserver/doc.go](REST API documentation) for 30 | details. 31 | 32 | There is no single unique identifier for attempts. The URL scheme 33 | assumes that at most one attempt will be created for a specific work 34 | unit, for a specific worker, in a single millisecond. This is not a 35 | hard guarantee but it would be unusual for it to break: it either 36 | implies duplicate attempts are being issued, or attempts are being 37 | forced for specific work units, or an attempt is requested, marked 38 | retryable, and requested again all within 1 ms. 39 | -------------------------------------------------------------------------------- /restclient/attempt.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package restclient 5 | 6 | import ( 7 | "github.com/swiftlobste/go-coordinate/coordinate" 8 | "github.com/swiftlobste/go-coordinate/restdata" 9 | "net/url" 10 | "time" 11 | ) 12 | 13 | type attempt struct { 14 | resource 15 | Representation restdata.Attempt 16 | workUnit *workUnit 17 | worker *worker 18 | } 19 | 20 | func attemptFromURL(parent *resource, path string, workUnit *workUnit, worker *worker) (a *attempt, err error) { 21 | a = &attempt{} 22 | a.URL, err = parent.Template(path, map[string]interface{}{}) 23 | if err == nil { 24 | err = a.Refresh() 25 | } 26 | if err == nil { 27 | err = a.fillReferences(workUnit, worker) 28 | } 29 | if err == nil { 30 | return a, nil 31 | } 32 | return nil, err 33 | } 34 | 35 | func (a *attempt) fillReferences(workUnit *workUnit, worker *worker) error { 36 | var err error 37 | var url *url.URL 38 | 39 | if err == nil { 40 | url, err = a.Template(a.Representation.WorkUnitURL, map[string]interface{}{}) 41 | } 42 | if err == nil && workUnit != nil && workUnit.URL.String() == url.String() { 43 | a.workUnit = workUnit 44 | } 45 | if err == nil && a.workUnit == nil { 46 | a.workUnit, err = workUnitFromURL(&a.resource, a.Representation.WorkUnitURL, nil) 47 | } 48 | 49 | if err == nil { 50 | url, err = a.Template(a.Representation.WorkerURL, map[string]interface{}{}) 51 | } 52 | if err == nil && worker != nil && worker.URL.String() == url.String() { 53 | a.worker = worker 54 | } 55 | if err == nil && a.worker == nil { 56 | a.worker, err = workerFromURL(&a.resource, a.Representation.WorkerURL) 57 | } 58 | 59 | return err 60 | } 61 | 62 | func (a *attempt) Refresh() error { 63 | a.Representation = restdata.Attempt{} 64 | return a.Get(&a.Representation) 65 | } 66 | 67 | func (a *attempt) WorkUnit() coordinate.WorkUnit { 68 | return a.workUnit 69 | } 70 | 71 | func (a *attempt) Worker() coordinate.Worker { 72 | return a.worker 73 | } 74 | 75 | func (a *attempt) Status() (coordinate.AttemptStatus, error) { 76 | err := a.Refresh() 77 | if err == nil { 78 | return a.Representation.Status, nil 79 | } 80 | return 0, err 81 | } 82 | 83 | func (a *attempt) Data() (map[string]interface{}, error) { 84 | err := a.Refresh() 85 | if err == nil { 86 | return a.Representation.Data, nil 87 | } 88 | return nil, err 89 | } 90 | 91 | func (a *attempt) StartTime() (time.Time, error) { 92 | return a.Representation.StartTime, nil 93 | } 94 | 95 | func (a *attempt) EndTime() (time.Time, error) { 96 | err := a.Refresh() 97 | if err == nil { 98 | return a.Representation.EndTime, nil 99 | } 100 | return time.Time{}, err 101 | } 102 | 103 | func (a *attempt) ExpirationTime() (time.Time, error) { 104 | err := a.Refresh() 105 | if err == nil { 106 | return a.Representation.ExpirationTime, nil 107 | } 108 | return time.Time{}, err 109 | } 110 | 111 | func (a *attempt) Renew(extendDuration time.Duration, data map[string]interface{}) error { 112 | repr := restdata.AttemptCompletion{ 113 | ExtendDuration: extendDuration, 114 | Data: data, 115 | } 116 | return a.PostTo(a.Representation.RenewURL, map[string]interface{}{}, repr, nil) 117 | } 118 | 119 | func (a *attempt) Expire(data map[string]interface{}) error { 120 | repr := restdata.AttemptCompletion{Data: data} 121 | return a.PostTo(a.Representation.ExpireURL, map[string]interface{}{}, repr, nil) 122 | } 123 | 124 | func (a *attempt) Finish(data map[string]interface{}) error { 125 | repr := restdata.AttemptCompletion{Data: data} 126 | return a.PostTo(a.Representation.FinishURL, map[string]interface{}{}, repr, nil) 127 | } 128 | 129 | func (a *attempt) Fail(data map[string]interface{}) error { 130 | repr := restdata.AttemptCompletion{Data: data} 131 | return a.PostTo(a.Representation.FailURL, map[string]interface{}{}, repr, nil) 132 | } 133 | 134 | func (a *attempt) Retry(data map[string]interface{}, delay time.Duration) error { 135 | repr := restdata.AttemptCompletion{Data: data, Delay: delay} 136 | return a.PostTo(a.Representation.RetryURL, map[string]interface{}{}, repr, nil) 137 | } 138 | -------------------------------------------------------------------------------- /restclient/coordinate.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | // Package restclient provides a Coordinate-compatible HTTP REST client 5 | // that talks to the matching server in the "restserver" package. 6 | // 7 | // The server in github.com/swiftlobste/go-coordinate/cmd/coordinated can 8 | // run a compatible REST server. Call New() with the base URL of that 9 | // service; for instance, 10 | // 11 | // c, err := restclient.New("http://localhost:5980/") 12 | package restclient 13 | 14 | import ( 15 | "github.com/swiftlobste/go-coordinate/coordinate" 16 | "github.com/swiftlobste/go-coordinate/restdata" 17 | "net/url" 18 | ) 19 | 20 | // New creates a new Coordinate interface that speaks to an external 21 | // REST server. 22 | func New(baseURL string) (coordinate.Coordinate, error) { 23 | var ( 24 | err error 25 | parsedURL *url.URL 26 | c *restCoordinate 27 | ) 28 | parsedURL, err = url.Parse(baseURL) 29 | if err == nil { 30 | c = &restCoordinate{ 31 | resource: resource{URL: parsedURL}, 32 | } 33 | err = c.Refresh() 34 | } 35 | 36 | if err != nil { 37 | return nil, err 38 | } 39 | return c, nil 40 | } 41 | 42 | type restCoordinate struct { 43 | resource 44 | Representation restdata.RootData 45 | } 46 | 47 | func (c *restCoordinate) Refresh() error { 48 | c.Representation = restdata.RootData{} 49 | return c.Get(&c.Representation) 50 | } 51 | 52 | func (c *restCoordinate) Namespace(name string) (coordinate.Namespace, error) { 53 | var err error 54 | ns := &namespace{} 55 | ns.URL, err = c.Template(c.Representation.NamespaceURL, map[string]interface{}{"namespace": name}) 56 | if err == nil { 57 | err = ns.Refresh() 58 | } 59 | if err == nil { 60 | return ns, nil 61 | } 62 | return nil, err 63 | } 64 | 65 | func (c *restCoordinate) Namespaces() (map[string]coordinate.Namespace, error) { 66 | resp := restdata.NamespaceList{} 67 | err := c.GetFrom(c.Representation.NamespacesURL, map[string]interface{}{}, &resp) 68 | if err != nil { 69 | return nil, err 70 | } 71 | result := make(map[string]coordinate.Namespace) 72 | for _, nsR := range resp.Namespaces { 73 | ns := namespace{} 74 | ns.URL, err = c.URL.Parse(nsR.URL) 75 | if err != nil { 76 | return nil, err 77 | } 78 | err = ns.Refresh() 79 | if err != nil { 80 | return nil, err 81 | } 82 | result[ns.Name()] = &ns 83 | } 84 | return result, nil 85 | } 86 | 87 | func (c *restCoordinate) Summarize() (coordinate.Summary, error) { 88 | var summary coordinate.Summary 89 | err := c.GetFrom(c.Representation.SummaryURL, nil, &summary) 90 | return summary, err 91 | } 92 | -------------------------------------------------------------------------------- /restclient/coordinate_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2017 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package restclient_test 5 | 6 | import ( 7 | "github.com/swiftlobste/go-coordinate/coordinate/coordinatetest" 8 | "github.com/swiftlobste/go-coordinate/memory" 9 | "github.com/swiftlobste/go-coordinate/restclient" 10 | "github.com/swiftlobste/go-coordinate/restserver" 11 | "github.com/stretchr/testify/suite" 12 | "net/http/httptest" 13 | "testing" 14 | ) 15 | 16 | // Suite runs the generic Coordinate tests with a REST backend. 17 | type Suite struct { 18 | coordinatetest.Suite 19 | } 20 | 21 | // SetupSuite does one-time test setup, creating the REST backend. 22 | func (s *Suite) SetupSuite() { 23 | s.Suite.SetupSuite() 24 | 25 | // This sets up an object stack where the REST client code talks 26 | // to the REST server code, which points at an in-memory backend. 27 | memBackend := memory.NewWithClock(s.Clock) 28 | router := restserver.NewRouter(memBackend) 29 | server := httptest.NewServer(router) 30 | backend, err := restclient.New(server.URL) 31 | if err != nil { 32 | panic(err) 33 | } 34 | s.Coordinate = backend 35 | } 36 | 37 | // TestCoordinate runs the generic Coordinate tests with a memory backend. 38 | func TestCoordinate(t *testing.T) { 39 | suite.Run(t, &Suite{}) 40 | } 41 | 42 | func TestEmptyURL(t *testing.T) { 43 | _, err := restclient.New("") 44 | if err == nil { 45 | t.Fatal("Expected error when given empty URL.") 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /restclient/namespace.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package restclient 5 | 6 | import ( 7 | "errors" 8 | "github.com/swiftlobste/go-coordinate/coordinate" 9 | "github.com/swiftlobste/go-coordinate/restdata" 10 | ) 11 | 12 | type namespace struct { 13 | resource 14 | Representation restdata.Namespace 15 | } 16 | 17 | func (ns *namespace) Refresh() error { 18 | ns.Representation = restdata.Namespace{} 19 | return ns.Get(&ns.Representation) 20 | } 21 | 22 | func (ns *namespace) Name() string { 23 | return ns.Representation.Name 24 | } 25 | 26 | func (ns *namespace) Destroy() error { 27 | return ns.Delete() 28 | } 29 | 30 | func (ns *namespace) makeWorkSpec(name string) (spec *workSpec, err error) { 31 | spec = &workSpec{} 32 | spec.URL, err = ns.Template(ns.Representation.WorkSpecURL, map[string]interface{}{"spec": name}) 33 | if err == nil { 34 | err = spec.Refresh() 35 | } 36 | return 37 | } 38 | 39 | func (ns *namespace) SetWorkSpec(data map[string]interface{}) (coordinate.WorkSpec, error) { 40 | var ( 41 | err error 42 | reqdata restdata.WorkSpec 43 | respdata restdata.WorkSpecShort 44 | spec *workSpec 45 | ) 46 | reqdata = restdata.WorkSpec{Data: data} 47 | spec = &workSpec{} 48 | if err == nil { 49 | err = ns.PostTo(ns.Representation.WorkSpecsURL, map[string]interface{}{}, reqdata, &respdata) 50 | } 51 | if err == nil { 52 | spec.URL, err = ns.Template(respdata.URL, map[string]interface{}{}) 53 | } 54 | if err == nil { 55 | err = spec.Refresh() 56 | } 57 | if err == nil { 58 | return spec, nil 59 | } 60 | return nil, err 61 | } 62 | 63 | func (ns *namespace) WorkSpec(name string) (coordinate.WorkSpec, error) { 64 | spec, err := ns.makeWorkSpec(name) 65 | if err == nil { 66 | return spec, nil 67 | } 68 | if http, isHTTP := err.(ErrorHTTP); isHTTP { 69 | if http.Response.StatusCode == 404 { 70 | err = coordinate.ErrNoSuchWorkSpec{Name: name} 71 | } 72 | } 73 | return nil, err 74 | } 75 | 76 | func (ns *namespace) DestroyWorkSpec(name string) error { 77 | spec, err := ns.makeWorkSpec(name) 78 | if err == nil { 79 | err = spec.Delete() 80 | } 81 | return err 82 | } 83 | 84 | func (ns *namespace) WorkSpecNames() ([]string, error) { 85 | repr := restdata.WorkSpecList{} 86 | err := ns.GetFrom(ns.Representation.WorkSpecsURL, map[string]interface{}{}, &repr) 87 | if err != nil { 88 | return nil, err 89 | } 90 | result := make([]string, len(repr.WorkSpecs)) 91 | for i, spec := range repr.WorkSpecs { 92 | result[i] = spec.Name 93 | } 94 | return result, nil 95 | } 96 | 97 | func (ns *namespace) Worker(name string) (coordinate.Worker, error) { 98 | var w worker 99 | var err error 100 | w.URL, err = ns.Template(ns.Representation.WorkerURL, map[string]interface{}{"worker": name}) 101 | if err == nil { 102 | err = w.Refresh() 103 | } 104 | return &w, err 105 | } 106 | 107 | func (ns *namespace) Workers() (map[string]coordinate.Worker, error) { 108 | return nil, errors.New("not implemented") 109 | } 110 | 111 | func (ns *namespace) Summarize() (coordinate.Summary, error) { 112 | var summary coordinate.Summary 113 | err := ns.GetFrom(ns.Representation.SummaryURL, nil, &summary) 114 | return summary, err 115 | } 116 | -------------------------------------------------------------------------------- /restclient/work_unit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015-2016 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package restclient 5 | 6 | import ( 7 | "errors" 8 | "github.com/swiftlobste/go-coordinate/coordinate" 9 | "github.com/swiftlobste/go-coordinate/restdata" 10 | ) 11 | 12 | type workUnit struct { 13 | resource 14 | Representation restdata.WorkUnit 15 | workSpec *workSpec 16 | } 17 | 18 | func workUnitFromURL(parent *resource, path string, spec *workSpec) (*workUnit, error) { 19 | unit := workUnit{ 20 | workSpec: spec, 21 | } 22 | var err error 23 | unit.URL, err = parent.Template(path, map[string]interface{}{}) 24 | if err == nil { 25 | err = unit.Refresh() 26 | } 27 | if err == nil && unit.workSpec == nil { 28 | unit.workSpec, err = workSpecFromURL(&unit.resource, unit.Representation.WorkSpecURL) 29 | } 30 | return &unit, err 31 | } 32 | 33 | func (unit *workUnit) Refresh() error { 34 | unit.Representation = restdata.WorkUnit{} 35 | return unit.Get(&unit.Representation) 36 | } 37 | 38 | func (unit *workUnit) Name() string { 39 | return unit.Representation.Name 40 | } 41 | 42 | func (unit *workUnit) Data() (map[string]interface{}, error) { 43 | err := unit.Refresh() 44 | if err == nil { 45 | return unit.Representation.Data, nil 46 | } 47 | return nil, err 48 | } 49 | 50 | func (unit *workUnit) WorkSpec() coordinate.WorkSpec { 51 | return unit.workSpec 52 | } 53 | 54 | func (unit *workUnit) Status() (coordinate.WorkUnitStatus, error) { 55 | err := unit.Refresh() 56 | if err == nil { 57 | return unit.Representation.Status, nil 58 | } 59 | return 0, err 60 | } 61 | 62 | func (unit *workUnit) Meta() (meta coordinate.WorkUnitMeta, err error) { 63 | err = unit.Refresh() 64 | if err == nil && unit.Representation.Meta == nil { 65 | err = errors.New("Invalid work unit response") 66 | } 67 | if err == nil { 68 | meta = *unit.Representation.Meta 69 | } 70 | return 71 | } 72 | 73 | func (unit *workUnit) SetMeta(meta coordinate.WorkUnitMeta) error { 74 | repr := restdata.WorkUnit{} 75 | repr.Meta = &meta 76 | return unit.Put(repr, nil) 77 | } 78 | 79 | func (unit *workUnit) Priority() (float64, error) { 80 | meta, err := unit.Meta() 81 | return meta.Priority, err 82 | } 83 | 84 | func (unit *workUnit) SetPriority(p float64) error { 85 | // This is a roundabout way to do this; but it is the only 86 | // entry point to change *only* the priority 87 | return unit.workSpec.SetWorkUnitPriorities(coordinate.WorkUnitQuery{ 88 | Names: []string{unit.Representation.Name}, 89 | }, p) 90 | } 91 | 92 | func (unit *workUnit) ActiveAttempt() (coordinate.Attempt, error) { 93 | err := unit.Refresh() 94 | if err == nil { 95 | aaURL := unit.Representation.ActiveAttemptURL 96 | if aaURL == "" { 97 | return nil, nil 98 | } 99 | return attemptFromURL(&unit.resource, aaURL, unit, nil) 100 | } 101 | return nil, err 102 | } 103 | 104 | func (unit *workUnit) ClearActiveAttempt() error { 105 | repr := restdata.WorkUnit{} 106 | repr.ActiveAttemptURL = "-" 107 | return unit.Put(repr, nil) 108 | } 109 | 110 | func (unit *workUnit) Attempts() ([]coordinate.Attempt, error) { 111 | // See also commentary in worker.go returnAttempts(). 112 | // Note that at least most work units have very few attempts, 113 | // and that every attempt should be for this work unit. 114 | var repr restdata.AttemptList 115 | err := unit.GetFrom(unit.Representation.AttemptsURL, map[string]interface{}{}, &repr) 116 | if err != nil { 117 | return nil, err 118 | } 119 | attempts := make([]coordinate.Attempt, len(repr.Attempts)) 120 | for i, attempt := range repr.Attempts { 121 | var aUnit *workUnit 122 | if attempt.WorkUnitURL == unit.Representation.URL { 123 | aUnit = unit 124 | } 125 | attempts[i], err = attemptFromURL(&unit.resource, attempt.URL, aUnit, nil) 126 | if err != nil { 127 | return nil, err 128 | } 129 | } 130 | return attempts, nil 131 | } 132 | 133 | func (unit *workUnit) NumAttempts() (int, error) { 134 | var repr restdata.AttemptList 135 | err := unit.GetFrom(unit.Representation.AttemptsURL, map[string]interface{}{}, &repr) 136 | if err != nil { 137 | return 0, err 138 | } 139 | return len(repr.Attempts), nil 140 | } 141 | -------------------------------------------------------------------------------- /restdata/marshal_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package restdata 5 | 6 | import ( 7 | "github.com/swiftlobste/go-coordinate/cborrpc" 8 | "reflect" 9 | "testing" 10 | ) 11 | 12 | func TestDataDictMarshal(t *testing.T) { 13 | tests := []struct { 14 | Object DataDict 15 | JSON string 16 | }{ 17 | { 18 | Object: DataDict{}, 19 | JSON: "{}", 20 | }, 21 | { 22 | Object: DataDict{ 23 | "key": "value", 24 | }, 25 | JSON: "{\"key\":\"value\"}", 26 | }, 27 | { 28 | Object: DataDict{ 29 | "key": cborrpc.PythonTuple{Items: []interface{}{}}, 30 | }, 31 | // The encoded CBOR is 32 | // A1 101 00001 map of 1 item 33 | // 63 6B 65 79 011 00011 string len 3 "key" 34 | // D8 80 110 11000 tuple tag 128 35 | // 80 100 00000 list of 0 items 36 | JSON: "\"oWNrZXnYgIA=\"", 37 | }, 38 | } 39 | for _, test := range tests { 40 | json, err := test.Object.MarshalJSON() 41 | if err != nil { 42 | t.Errorf("MarshalJSON(%+v) => error %+v", 43 | test.Object, err) 44 | } else if string(json) != test.JSON { 45 | t.Errorf("MarshalJSON(%+v) => %v, want %v", 46 | test.Object, string(json), test.JSON) 47 | } 48 | 49 | var obj DataDict 50 | err = (&obj).UnmarshalJSON([]byte(test.JSON)) 51 | if err != nil { 52 | t.Errorf("UnmarshalJSON(%v) => error %+v", 53 | test.JSON, err) 54 | } else if !reflect.DeepEqual(obj, test.Object) { 55 | t.Errorf("UnmarshalJSON(%v) => %+v, want %+v", 56 | test.JSON, obj, test.Object) 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /restdata/url.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package restdata 5 | 6 | import ( 7 | "encoding/base64" 8 | ) 9 | 10 | // MaybeEncodeName examines a name, and if it cannot be directly 11 | // inserted into a URL as-is, base64 encodes it. More specifically, 12 | // the encoded name begins with - and uses the URL-safe base64 13 | // alphabet with no padding. 14 | func MaybeEncodeName(name string) string { 15 | // We must encode empty name, name starting with "-" (because 16 | // it is otherwise ambiguous), and name that includes anything 17 | // that's not URL-safe. 18 | safe := true 19 | if len(name) == 0 { 20 | safe = false 21 | } else if name[0] == '-' { 22 | safe = false 23 | } else { 24 | for _, c := range name { 25 | switch { 26 | // These characters are "unreserved" 27 | // in RFC 3986 section 2.3: 28 | case c == '-', c == '.', c == '_', c == ':', 29 | (c >= 'a' && c <= 'z'), 30 | (c >= 'A' && c <= 'Z'), 31 | (c >= '0' && c <= '9'): 32 | continue 33 | default: 34 | safe = false 35 | break 36 | } 37 | } 38 | } 39 | if safe { 40 | return name 41 | } 42 | return "-" + base64.RawURLEncoding.EncodeToString([]byte(name)) 43 | } 44 | 45 | // MaybeDecodeName examines a name, and if it appears to be base64 46 | // encoded, decodes it. base64 encoded strings begin with an - sign. 47 | // This function is the dual of MaybeEncodeName(). Returns an error 48 | // if the string begins with - and the remainder of the string isn't 49 | // actually base64 encoded. 50 | func MaybeDecodeName(name string) (string, error) { 51 | if len(name) == 0 || name[0] != '-' { 52 | // Not base64 encoded, so return as is 53 | return name, nil 54 | } 55 | bytes, err := base64.RawURLEncoding.DecodeString(name[1:]) 56 | if err != nil { 57 | return "", err 58 | } 59 | return string(bytes), nil 60 | } 61 | -------------------------------------------------------------------------------- /restdata/url_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package restdata 5 | 6 | import ( 7 | "testing" 8 | ) 9 | 10 | func TestEncodeDecode(t *testing.T) { 11 | tests := []struct{ plain, encoded string }{ 12 | {"foo", "foo"}, 13 | {"", "-"}, 14 | {"-", "-LQ"}, 15 | {"\u0000", "-AA"}, 16 | } 17 | for _, test := range tests { 18 | enc := MaybeEncodeName(test.plain) 19 | if enc != test.encoded { 20 | t.Errorf("MaybeEncodeName(%q) => %q, want %q", 21 | test.plain, enc, test.encoded) 22 | } 23 | 24 | dec, err := MaybeDecodeName(test.encoded) 25 | if err != nil { 26 | t.Errorf("MaybeDecodeName(%q) => error %v", 27 | test.encoded, err) 28 | } else if dec != test.plain { 29 | t.Errorf("MaybeDecodeName(%q) => %q, want %q", 30 | test.encoded, dec, test.plain) 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /restserver/context.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package restserver 5 | 6 | import ( 7 | "errors" 8 | "github.com/swiftlobste/go-coordinate/coordinate" 9 | "github.com/swiftlobste/go-coordinate/restdata" 10 | "github.com/gorilla/mux" 11 | "net/http" 12 | "net/url" 13 | "strconv" 14 | "strings" 15 | "time" 16 | ) 17 | 18 | // errUnmarshal is returned if the put/post contract is violated and 19 | // a handler function is passed the wrong type. 20 | var errUnmarshal = restdata.ErrBadRequest{ 21 | Err: errors.New("Invalid input format"), 22 | } 23 | 24 | // context holds all of the information and objects that can be extracted 25 | // from URL parameters. 26 | type context struct { 27 | Namespace coordinate.Namespace 28 | WorkSpec coordinate.WorkSpec 29 | WorkUnit coordinate.WorkUnit 30 | Attempt coordinate.Attempt 31 | Worker coordinate.Worker 32 | QueryParams url.Values 33 | } 34 | 35 | func (api *restAPI) Context(req *http.Request) (ctx *context, err error) { 36 | ctx = &context{} 37 | ctx.QueryParams = req.URL.Query() 38 | vars := mux.Vars(req) 39 | 40 | var present bool 41 | var namespace, spec, unit, worker, start string 42 | 43 | if namespace, present = vars["namespace"]; present && err == nil { 44 | namespace, err = restdata.MaybeDecodeName(namespace) 45 | if err == nil { 46 | ctx.Namespace, err = api.Coordinate.Namespace(namespace) 47 | } 48 | } 49 | 50 | if spec, present = vars["spec"]; present && err == nil && ctx.Namespace != nil { 51 | spec, err = restdata.MaybeDecodeName(spec) 52 | if err == nil { 53 | ctx.WorkSpec, err = ctx.Namespace.WorkSpec(spec) 54 | } 55 | if _, missing := err.(coordinate.ErrNoSuchWorkSpec); missing { 56 | err = restdata.ErrNotFound{Err: err} 57 | } 58 | } 59 | 60 | if unit, present = vars["unit"]; present && err == nil && ctx.WorkSpec != nil { 61 | unit, err = restdata.MaybeDecodeName(unit) 62 | if err == nil { 63 | ctx.WorkUnit, err = ctx.WorkSpec.WorkUnit(unit) 64 | } 65 | // In all cases, if there is a work unit key in the URL 66 | // and that names an absent work unit, it's a missing 67 | // URL and we should return 404 68 | if err == nil && ctx.WorkUnit == nil { 69 | err = restdata.ErrNotFound{Err: coordinate.ErrNoSuchWorkUnit{Name: unit}} 70 | } 71 | } 72 | 73 | if worker, present = vars["worker"]; present && err == nil && ctx.Namespace != nil { 74 | worker, err = restdata.MaybeDecodeName(worker) 75 | if err == nil { 76 | ctx.Worker, err = ctx.Namespace.Worker(worker) 77 | } 78 | } 79 | 80 | if start, present = vars["start"]; present && err == nil { 81 | start, err = restdata.MaybeDecodeName(start) 82 | } 83 | 84 | if err == nil && ctx.WorkUnit != nil && ctx.Worker != nil && start != "" { 85 | // This is enough information to try to find a worker. 86 | // Guess that, of these things, the work unit will have 87 | // the fewest attempts, and scanning them in linear time 88 | // is sane. We won't be able to exactly match times 89 | // but we can check that their serializations match. 90 | // (Even this isn't foolproof because of time zones.) 91 | var attempts []coordinate.Attempt 92 | attempts, err = ctx.WorkUnit.Attempts() 93 | if err == nil { 94 | for _, attempt := range attempts { 95 | var startTime time.Time 96 | startTime, err = attempt.StartTime() 97 | if err != nil { 98 | break 99 | } 100 | myStart := startTime.Format(time.RFC3339) 101 | if attempt.Worker().Name() == ctx.Worker.Name() && myStart == start { 102 | ctx.Attempt = attempt 103 | break 104 | } 105 | } 106 | } 107 | // If we had all of these things, we clearly were 108 | // expecting to find an attempt, so fail if we didn't. 109 | if err == nil && ctx.Attempt == nil { 110 | err = restdata.ErrNotFound{Err: errors.New("no such attempt")} 111 | } 112 | } 113 | 114 | return 115 | } 116 | 117 | // BoolParam looks at ctx.QueryParams for a parameter named name. If 118 | // it has a normally-truthy value (1, on, false, no, ...) then return 119 | // that value. Otherwise (empty string, foo, ...) return def. 120 | func (ctx *context) BoolParam(name string, def bool) bool { 121 | switch strings.ToLower(ctx.QueryParams.Get(name)) { 122 | case "0", "f", "n", "false", "off", "no": 123 | return false 124 | case "1", "t", "y", "true", "on", "yes": 125 | return true 126 | default: 127 | return def 128 | } 129 | } 130 | 131 | // Build a work unit query from query parameters. This can fail (if 132 | // invalid statuses are named, if a non-integer limit is provided) 133 | // so it should only be called if a specific route wants it. 134 | func (ctx *context) WorkUnitQuery() (q coordinate.WorkUnitQuery, err error) { 135 | q.Names = ctx.QueryParams["name"] 136 | if len(ctx.QueryParams["status"]) > 0 { 137 | q.Statuses = make([]coordinate.WorkUnitStatus, len(ctx.QueryParams["status"])) 138 | for i, status := range ctx.QueryParams["status"] { 139 | err = q.Statuses[i].UnmarshalText([]byte(status)) 140 | if err != nil { 141 | return 142 | } 143 | } 144 | } 145 | q.PreviousName = ctx.QueryParams.Get("previous") 146 | limit := ctx.QueryParams.Get("limit") 147 | if limit != "" { 148 | q.Limit, err = strconv.Atoi(limit) 149 | } 150 | return 151 | } 152 | -------------------------------------------------------------------------------- /restserver/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | // Package restserver publishes a Coordinate interface as a REST service. 5 | // The restclient package is a matching client. 6 | // 7 | // The complete REST API is defined in the restdata package. In 8 | // particular, note that the URLs described here are not actually part 9 | // of the API. 10 | // 11 | // HTTP Considerations 12 | // 13 | // HTTP GET requests will default to returning an HTML representation of 14 | // the resource. Clients should use the standard HTTP Accept: header to 15 | // request a different format. See "MIME Types" below. 16 | // 17 | // This interface does not (currently) support HTTP caching or 18 | // authentication headers. 19 | // 20 | // Code will generally follow conventions for the Github API as an 21 | // established example; see https://developer.github.com/v3/ for 22 | // details. 23 | // 24 | // MIME Types 25 | // 26 | // This interface understands MIME types as follows: 27 | // 28 | // application/vnd.diffeo.coordinate.v1+json 29 | // 30 | // JSON representation of version 1 of this interface. 31 | // 32 | // application/vnd.diffeo.coordinate+json 33 | // application/json 34 | // text/json 35 | // 36 | // JSON representation of latest version of this interface. 37 | // 38 | // URL Scheme 39 | // 40 | // In most cases, Coordinate objects follow their natural hierarchy 41 | // and are addressed by name. For instance, the work spec "bar" in 42 | // namespace "foo" would have a resource URL of 43 | // /namespace/foo/work_spec/bar. If the name is not URL-safe 44 | // printable ASCII, it must be base64 encoded using the URL-safe 45 | // alphabet (RFC 4648 section 5), with no padding, and adding an 46 | // additional - at the front of the name: 47 | // /namespace/-Zm9v/work_spec/-YmFy is the same resource as the 48 | // preceding one. Correspondingly, a single - means "empty", and a 49 | // name that begins with - must be URL-encoded. The work spec "-" in 50 | // the empty namespace has a URL of /namespace/-/work_spec/-LQ. 51 | // 52 | // The following URLs are defined: 53 | // 54 | // / 55 | // /namespace 56 | // /namespace/{namespace} 57 | // /namespace/{namespace}/work_spec 58 | // /namespace/{namespace}/work_spec/{spec} 59 | // /namespace/{namespace}/work_spec/{spec}/counts 60 | // /namespace/{namespace}/work_spec/{spec}/change 61 | // /namespace/{namespace}/work_spec/{spec}/adjust 62 | // /namespace/{namespace}/work_spec/{spec}/meta 63 | // /namespace/{namespace}/work_spec/{spec}/work_unit 64 | // /namespace/{namespace}/work_spec/{spec}/work_unit/{unit} 65 | // .../attempts 66 | // .../attempt/{worker}/{start_time} 67 | // .../attempt/{worker}/{start_time}/renew 68 | // .../attempt/{worker}/{start_time}/expire 69 | // .../attempt/{worker}/{start_time}/finish 70 | // .../attempt/{worker}/{start_time}/fail 71 | // .../attempt/{worker}/{start_time}/retry 72 | // /namespace/{namespace}/worker 73 | // /namespace/{namespace}/worker/{worker} 74 | // /namespace/{namespace}/worker/{worker}/request_attempts 75 | // /namespace/{namespace}/worker/{worker}/make_attempt 76 | // /namespace/{namespace}/worker/{worker}/active_attempts 77 | // /namespace/{namespace}/worker/{worker}/all_attempts 78 | // /namespace/{namespace}/worker/{worker}/child_attempts 79 | package restserver 80 | -------------------------------------------------------------------------------- /restserver/helpers.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package restserver 5 | 6 | // This file contains various HTTP-related helpers. I sort of suspect 7 | // most of them belong in some sort of standard library I haven't 8 | // immediately found. 9 | 10 | import ( 11 | "fmt" 12 | "github.com/swiftlobste/go-coordinate/restdata" 13 | "github.com/gorilla/mux" 14 | "net/url" 15 | "strings" 16 | ) 17 | 18 | type urlBuilder struct { 19 | Router *mux.Router 20 | Params []string 21 | Error error 22 | } 23 | 24 | func buildURLs(router *mux.Router, params ...string) *urlBuilder { 25 | // Encode all of the values in params 26 | for i, value := range params { 27 | if i%2 == 1 { 28 | params[i] = restdata.MaybeEncodeName(value) 29 | } 30 | } 31 | return &urlBuilder{Router: router, Params: params} 32 | } 33 | 34 | func (u *urlBuilder) Route(route string) *mux.Route { 35 | if u.Error != nil { 36 | return nil 37 | } 38 | r := u.Router.Get(route) 39 | if r == nil { 40 | u.Error = fmt.Errorf("No such route %q", route) 41 | } 42 | return r 43 | } 44 | 45 | func (u *urlBuilder) URL(out *string, route string) *urlBuilder { 46 | var r *mux.Route 47 | var url *url.URL 48 | if u.Error == nil { 49 | r = u.Route(route) 50 | } 51 | if u.Error == nil { 52 | url, u.Error = r.URL(u.Params...) 53 | } 54 | if u.Error == nil { 55 | *out = url.String() 56 | } 57 | return u 58 | } 59 | 60 | func (u *urlBuilder) Template(out *string, route, param string) *urlBuilder { 61 | var r *mux.Route 62 | var url *url.URL 63 | if u.Error == nil { 64 | r = u.Route(route) 65 | } 66 | if u.Error == nil { 67 | params := append([]string{param, "---"}, u.Params...) 68 | url, u.Error = r.URL(params...) 69 | } 70 | if u.Error == nil { 71 | *out = strings.Replace(url.String(), "---", "{"+param+"}", 1) 72 | } 73 | return u 74 | } 75 | -------------------------------------------------------------------------------- /restserver/namespace.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package restserver 5 | 6 | import ( 7 | "github.com/swiftlobste/go-coordinate/coordinate" 8 | "github.com/swiftlobste/go-coordinate/restdata" 9 | "github.com/gorilla/mux" 10 | ) 11 | 12 | func (api *restAPI) fillNamespaceShort(namespace coordinate.Namespace, summary *restdata.NamespaceShort) error { 13 | summary.Name = namespace.Name() 14 | return buildURLs(api.Router, "namespace", summary.Name). 15 | URL(&summary.URL, "namespace"). 16 | Error 17 | } 18 | 19 | func (api *restAPI) fillNamespace(namespace coordinate.Namespace, result *restdata.Namespace) error { 20 | err := api.fillNamespaceShort(namespace, &result.NamespaceShort) 21 | if err == nil { 22 | err = buildURLs(api.Router, "namespace", result.Name). 23 | URL(&result.SummaryURL, "namespaceSummary"). 24 | URL(&result.WorkSpecsURL, "workSpecs"). 25 | Template(&result.WorkSpecURL, "workSpec", "spec"). 26 | URL(&result.WorkersURL, "workers"). 27 | Template(&result.WorkerURL, "worker", "worker"). 28 | Error 29 | } 30 | return err 31 | } 32 | 33 | // NamespaceList gets a list of all namespaces known in the system. 34 | func (api *restAPI) NamespaceList(ctx *context) (interface{}, error) { 35 | namespaces, err := api.Coordinate.Namespaces() 36 | if err != nil { 37 | return nil, err 38 | } 39 | result := restdata.NamespaceList{} 40 | for _, ns := range namespaces { 41 | summary := restdata.NamespaceShort{} 42 | err = api.fillNamespaceShort(ns, &summary) 43 | if err != nil { 44 | return nil, err 45 | } 46 | result.Namespaces = append(result.Namespaces, summary) 47 | } 48 | return result, nil 49 | } 50 | 51 | // NamespacePost creates a new namespace, or retrieves a pointer to 52 | // an existing one. 53 | func (api *restAPI) NamespacePost(ctx *context, in interface{}) (interface{}, error) { 54 | req, valid := in.(restdata.NamespaceShort) 55 | if !valid { 56 | return nil, errUnmarshal 57 | } 58 | ns, err := api.Coordinate.Namespace(req.Name) 59 | if err != nil { 60 | return nil, err 61 | } 62 | // We will return "created", where the content is the full 63 | // namespace data 64 | result := restdata.Namespace{} 65 | err = api.fillNamespace(ns, &result) 66 | if err != nil { 67 | return nil, err 68 | } 69 | return responseCreated{ 70 | Location: result.URL, 71 | Body: result, 72 | }, nil 73 | } 74 | 75 | // NamespaceGet retrieves an existing namespace, or creates a new one. 76 | func (api *restAPI) NamespaceGet(ctx *context) (interface{}, error) { 77 | // If we've gotten here, we're just returning ctx.Namespace 78 | result := restdata.Namespace{} 79 | err := api.fillNamespace(ctx.Namespace, &result) 80 | if err != nil { 81 | return nil, err 82 | } 83 | return result, nil 84 | } 85 | 86 | // NamespaceDelete destroys an existing namespace. 87 | func (api *restAPI) NamespaceDelete(ctx *context) (interface{}, error) { 88 | err := ctx.Namespace.Destroy() 89 | return nil, err 90 | } 91 | 92 | // NamespaceSummaryGet produces a summary for a namespace. 93 | func (api *restAPI) NamespaceSummaryGet(ctx *context) (interface{}, error) { 94 | return ctx.Namespace.Summarize() 95 | } 96 | 97 | // PopulateNamespace adds namespace-specific routes to a router. 98 | // r should be rooted at the root of the Coordinate URL tree, e.g. "/". 99 | func (api *restAPI) PopulateNamespace(r *mux.Router) { 100 | r.Path("/namespace").Name("namespaces").Handler(&resourceHandler{ 101 | Representation: restdata.NamespaceShort{}, 102 | Context: api.Context, 103 | Get: api.NamespaceList, 104 | Post: api.NamespacePost, 105 | }) 106 | r.Path("/namespace/{namespace}").Name("namespace").Handler(&resourceHandler{ 107 | Representation: restdata.Namespace{}, 108 | Context: api.Context, 109 | Get: api.NamespaceGet, 110 | Delete: api.NamespaceDelete, 111 | }) 112 | r.Path("/namespace/{namespace}/summary").Name("namespaceSummary").Handler(&resourceHandler{ 113 | Representation: coordinate.Summary{}, 114 | Context: api.Context, 115 | Get: api.NamespaceSummaryGet, 116 | }) 117 | sr := r.PathPrefix("/namespace/{namespace}").Subrouter() 118 | api.PopulateWorkSpec(sr) 119 | api.PopulateWorker(sr) 120 | } 121 | -------------------------------------------------------------------------------- /restserver/rest_test.go: -------------------------------------------------------------------------------- 1 | // Regression tests for rest.go. 2 | // 3 | // Main tests are really by running the end-to-end path, using the 4 | // coordinatetest tests driven from restclient. This only contains 5 | // special-case bug tests. 6 | // 7 | // Copyright 2016 Diffeo, Inc. 8 | // This software is released under an MIT/X11 open source license. 9 | 10 | package restserver 11 | 12 | import ( 13 | "errors" 14 | "github.com/swiftlobste/go-coordinate/memory" 15 | "github.com/stretchr/testify/assert" 16 | "net/http" 17 | "net/url" 18 | "testing" 19 | ) 20 | 21 | type failResponseWriter struct { 22 | Headers http.Header 23 | StatusCode int 24 | } 25 | 26 | func (rw *failResponseWriter) Header() http.Header { 27 | if rw.Headers == nil { 28 | rw.Headers = make(http.Header) 29 | } 30 | return rw.Headers 31 | } 32 | 33 | func (rw *failResponseWriter) Write([]byte) (int, error) { 34 | return 0, errors.New("foo") 35 | } 36 | 37 | func (rw *failResponseWriter) WriteHeader(code int) { 38 | rw.StatusCode = code 39 | } 40 | 41 | // TestDoubleFault checks that, if there is an error serializing a JSON 42 | // response, it doesn't actually panic the process. 43 | func TestDoubleFault(t *testing.T) { 44 | backend := memory.New() 45 | namespace, err := backend.Namespace("") 46 | if !assert.NoError(t, err) { 47 | return 48 | } 49 | _, err = namespace.SetWorkSpec(map[string]interface{}{ 50 | "name": "spec", 51 | }) 52 | if !assert.NoError(t, err) { 53 | return 54 | } 55 | 56 | router := NewRouter(backend) 57 | req := &http.Request{ 58 | Method: http.MethodGet, 59 | URL: &url.URL{ 60 | Path: "/namespace/-/work_spec/spec", 61 | }, 62 | Proto: "HTTP/1.1", 63 | ProtoMajor: 1, 64 | ProtoMinor: 1, 65 | Header: http.Header{}, 66 | Close: true, 67 | Host: "localhost", 68 | } 69 | resp := &failResponseWriter{} 70 | router.ServeHTTP(resp, req) 71 | assert.Equal(t, http.StatusOK, resp.StatusCode) 72 | } 73 | -------------------------------------------------------------------------------- /restserver/server.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Diffeo, Inc. 2 | // This software is released under an MIT/X11 open source license. 3 | 4 | package restserver 5 | 6 | import ( 7 | "github.com/swiftlobste/go-coordinate/coordinate" 8 | "github.com/swiftlobste/go-coordinate/restdata" 9 | "github.com/gorilla/mux" 10 | "net/http" 11 | ) 12 | 13 | // NewRouter creates a new HTTP handler that processes all Coordinate 14 | // requests. All Coordinate resources are under the URL path root, 15 | // e.g. /v1/namespace/foo. For more control over this setup, create 16 | // a mux.Router and call PopulateRouter instead. 17 | func NewRouter(c coordinate.Coordinate) http.Handler { 18 | r := mux.NewRouter() 19 | PopulateRouter(r, c) 20 | return r 21 | } 22 | 23 | // PopulateRouter adds Coordinate routes to an existing 24 | // github.com/gorilla/mux router object. This can be used, for 25 | // instance, to place the Coordinate interface under a subpath: 26 | // 27 | // import "github.com/swiftlobste/go-coordinate/memory" 28 | // import "github.com/gorilla/mux" 29 | // r := mux.Router() 30 | // s := r.PathPrefix("/coordinate").Subrouter() 31 | // c := memory.New() 32 | // PopulateRouter(s, c) 33 | func PopulateRouter(r *mux.Router, c coordinate.Coordinate) { 34 | api := &restAPI{Coordinate: c, Router: r} 35 | api.PopulateRouter(r) 36 | } 37 | 38 | // restAPI holds the persistent state for the Coordinate REST API. 39 | type restAPI struct { 40 | Coordinate coordinate.Coordinate 41 | Router *mux.Router 42 | } 43 | 44 | // PopulateRouter adds all Coordinate URL paths to a router. 45 | func (api *restAPI) PopulateRouter(r *mux.Router) { 46 | api.PopulateNamespace(r) 47 | r.Path("/").Name("root").Handler(&resourceHandler{ 48 | Representation: restdata.RootData{}, 49 | Context: api.Context, 50 | Get: api.RootDocument, 51 | }) 52 | r.Path("/summary").Name("rootSummary").Handler(&resourceHandler{ 53 | Representation: coordinate.Summary{}, 54 | Context: api.Context, 55 | Get: api.RootSummary, 56 | }) 57 | } 58 | 59 | func (api *restAPI) RootDocument(ctx *context) (interface{}, error) { 60 | resp := restdata.RootData{} 61 | err := buildURLs(api.Router). 62 | URL(&resp.SummaryURL, "rootSummary"). 63 | URL(&resp.NamespacesURL, "namespaces"). 64 | Template(&resp.NamespaceURL, "namespace", "namespace"). 65 | Error 66 | return resp, err 67 | } 68 | 69 | func (api *restAPI) RootSummary(ctx *context) (interface{}, error) { 70 | return api.Coordinate.Summarize() 71 | } 72 | --------------------------------------------------------------------------------