├── example
├── main.css
├── foo
│ ├── bar.txt
│ └── bar
│ │ └── baz
│ │ └── xxx.txt
├── env.js
├── main.js
├── index.html
├── item.js
└── items.js
├── .envrc
├── API.md
├── .vscode
└── settings.json
├── .prettierrc.yaml
├── ent-web.toml
├── format
├── .gitignore
├── default.nix
├── shell.nix
├── proto
├── dag_pb.proto
├── ent_server_config.proto
├── ent_server_api.proto
├── dag_pb.pb.go
├── ent_server_api_grpc.pb.go
└── ent_server_config.pb.go
├── datastore
├── in_memory.go
├── data_store.go
├── memcache.go
├── file.go
└── cloud.go
├── ent-server.toml
├── cmd
├── ent
│ ├── main.go
│ ├── remote
│ │ └── remote.go
│ ├── config
│ │ └── config.go
│ └── cmd
│ │ ├── root.go
│ │ ├── keygen.go
│ │ ├── put.go
│ │ ├── get.go
│ │ └── digest.go
├── ent-web
│ ├── config.go
│ └── main.go
├── ent-server
│ ├── config.go
│ ├── store.go
│ ├── raw.go
│ ├── bigquery.go
│ ├── main.go
│ └── grpc.go
└── indexer
│ └── main.go
├── .gcloudignore
├── justfile
├── nodeservice
├── node_service.go
├── sequence.go
├── index_client.go
└── remote.go
├── index
├── index_test.go
└── index.go
├── docs
├── contributing.md
└── code-of-conduct.md
├── api
└── api.go
├── objectstore
└── store.go
├── flake.nix
├── utils
├── lib_test.go
├── node_test.go
├── lib.go
└── node.go
├── log
└── log.go
├── flake.lock
├── schema
├── schema_test.go
└── schema.go
├── go.mod
├── README.md
└── LICENSE
/example/main.css:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/.envrc:
--------------------------------------------------------------------------------
1 | use flake
2 |
--------------------------------------------------------------------------------
/example/foo/bar.txt:
--------------------------------------------------------------------------------
1 | test
2 |
--------------------------------------------------------------------------------
/example/foo/bar/baz/xxx.txt:
--------------------------------------------------------------------------------
1 | Test
2 |
--------------------------------------------------------------------------------
/API.md:
--------------------------------------------------------------------------------
1 | ```
2 | openssl rand -base64 64 | tr -d '\n'
3 | ```
4 |
--------------------------------------------------------------------------------
/.vscode/settings.json:
--------------------------------------------------------------------------------
1 | {
2 | "nixEnvSelector.nixFile": "${workspaceRoot}/shell.nix"
3 | }
4 |
--------------------------------------------------------------------------------
/.prettierrc.yaml:
--------------------------------------------------------------------------------
1 | # See https://prettier.io/docs/en/options.html
2 |
3 | printWidth: 80
4 | proseWrap: always
5 | singleQuote: true
6 |
--------------------------------------------------------------------------------
/ent-web.toml:
--------------------------------------------------------------------------------
1 | listenAddress = ":27334"
2 | domainName = "localhost:27334"
3 |
4 | [[remotes]]
5 | name = 'ent-store'
6 | url = 'https://ent-server-62sa4xcfia-ew.a.run.app'
7 |
--------------------------------------------------------------------------------
/format:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | set -o errexit
4 | set -o nounset
5 | set -o xtrace
6 | set -o pipefail
7 |
8 | ./tools/node ./tools/prettier/bin-prettier.js --write README.md
9 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Binaries for programs and plugins
2 | *.exe
3 | *.exe~
4 | *.dll
5 | *.so
6 | *.dylib
7 |
8 | # Test binary, built with `go test -c`
9 | *.test
10 |
11 | # Output of the go coverage tool, specifically when used with LiteIDE
12 | *.out
13 |
14 | # Dependency directories (remove the comment below to include it)
15 | # vendor/
16 |
17 | /data
18 | /tools
19 |
20 | /.git
21 |
22 | /credentials.json
23 |
24 | /.direnv
25 | /result
26 |
27 | /bin
28 |
--------------------------------------------------------------------------------
/default.nix:
--------------------------------------------------------------------------------
1 | { pkgs ? (
2 | let
3 | inherit (builtins) fetchTree fromJSON readFile;
4 | inherit ((fromJSON (readFile ./flake.lock)).nodes) nixpkgs gomod2nix;
5 | in
6 | import (fetchTree nixpkgs.locked) {
7 | overlays = [
8 | (import "${fetchTree gomod2nix.locked}/overlay.nix")
9 | ];
10 | }
11 | )
12 | }:
13 |
14 | pkgs.buildGoApplication {
15 | pname = "myapp";
16 | version = "0.1";
17 | pwd = ./.;
18 | src = ./.;
19 | modules = ./gomod2nix.toml;
20 | }
21 |
--------------------------------------------------------------------------------
/shell.nix:
--------------------------------------------------------------------------------
1 | { pkgs ? (
2 | let
3 | inherit (builtins) fetchTree fromJSON readFile;
4 | inherit ((fromJSON (readFile ./flake.lock)).nodes) nixpkgs gomod2nix;
5 | in
6 | import (fetchTree nixpkgs.locked) {
7 | overlays = [
8 | (import "${fetchTree gomod2nix.locked}/overlay.nix")
9 | ];
10 | }
11 | )
12 | }:
13 |
14 | let
15 | goEnv = pkgs.mkGoEnv { pwd = ./.; };
16 | in
17 | pkgs.mkShell {
18 | packages = [
19 | goEnv
20 | pkgs.gomod2nix
21 | ];
22 | }
23 |
--------------------------------------------------------------------------------
/proto/dag_pb.proto:
--------------------------------------------------------------------------------
1 | syntax = "proto3";
2 |
3 | package ent;
4 |
5 | option go_package = "github.com/ent";
6 |
7 | // https://ipld.io/specs/codecs/dag-pb/spec/#serial-format
8 |
9 | message PBLink {
10 | // binary CID (with no multibase prefix) of the target object
11 | optional bytes Hash = 1;
12 |
13 | // UTF-8 string name
14 | optional string Name = 2;
15 |
16 | // cumulative size of target object
17 | optional uint64 Tsize = 3;
18 | }
19 |
20 | message PBNode {
21 | // refs to other objects
22 | repeated PBLink Links = 2;
23 |
24 | // opaque user data
25 | optional bytes Data = 1;
26 | }
27 |
--------------------------------------------------------------------------------
/datastore/in_memory.go:
--------------------------------------------------------------------------------
1 | package datastore
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | )
7 |
8 | type InMemory struct {
9 | Inner map[string][]byte
10 | }
11 |
12 | func (s InMemory) Get(ctx context.Context, name string) ([]byte, error) {
13 | b, ok := s.Inner[name]
14 | if ok {
15 | return b, nil
16 | } else {
17 | return nil, fmt.Errorf("not found")
18 | }
19 | }
20 |
21 | func (s InMemory) Put(ctx context.Context, name string, value []byte) error {
22 | s.Inner[name] = value
23 | return nil
24 | }
25 |
26 | func (s InMemory) Has(ctx context.Context, name string) (bool, error) {
27 | _, ok := s.Inner[name]
28 | return ok, nil
29 | }
30 |
--------------------------------------------------------------------------------
/ent-server.toml:
--------------------------------------------------------------------------------
1 | listenAddress = ":27333"
2 | domainName = "localhost:27333"
3 |
4 | bigqueryEnabled = false
5 | bigqueryDataset = "access_logs"
6 |
7 | redisEnabled = false
8 | redisEndpoint = "10.51.202.235:6379"
9 |
10 | cloudStorageEnabled = true
11 | cloudStorageBucket = "ent-objects-1"
12 |
13 | projectID = "oak-ci"
14 |
15 | ginMode = "debug"
16 | logLevel = "debug"
17 |
18 | [[users]]
19 | ID = 1
20 | Name = "user"
21 | APIKey = "oAf1MTownkbSkMuCQVUIrNbS4GxKlBsDAQVKEihfajMoPiWreyhmtmVRVPVGOmJUJJAsfrYVtfVqBpGbX3BaZw=="
22 | CanRead = true
23 | CanWrite = true
24 |
25 | [[users]]
26 | ID = 2
27 | Name = "public"
28 | APIKey = ""
29 | CanRead = true
30 | CanWrite = false
31 |
--------------------------------------------------------------------------------
/example/env.js:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2021 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | const env = {
17 | numberOfArticles: 7,
18 | colour: "blue",
19 | };
20 |
--------------------------------------------------------------------------------
/cmd/ent/main.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2021 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package main
17 |
18 | import (
19 | "github.com/google/ent/cmd/ent/cmd"
20 | )
21 |
22 | func main() {
23 | cmd.Execute()
24 | }
25 |
--------------------------------------------------------------------------------
/.gcloudignore:
--------------------------------------------------------------------------------
1 | # This file specifies files that are *not* uploaded to Google Cloud Platform
2 | # using gcloud. It follows the same syntax as .gitignore, with the addition of
3 | # "#!include" directives (which insert the entries of the given .gitignore-style
4 | # file at that point).
5 | #
6 | # For more information, run:
7 | # $ gcloud topic gcloudignore
8 | #
9 | .gcloudignore
10 | # If you would like to upload your .git directory, .gitignore file or files
11 | # from your .gitignore file, remove the corresponding line
12 | # below:
13 | .git
14 | .gitignore
15 |
16 | # Binaries for programs and plugins
17 | *.exe
18 | *.exe~
19 | *.dll
20 | *.so
21 | *.dylib
22 | # Test binary, build with `go test -c`
23 | *.test
24 | # Output of the go coverage tool, specifically when used with LiteIDE
25 | *.out
26 |
27 | #!include:.gitignore
28 |
--------------------------------------------------------------------------------
/example/main.js:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2021 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | const e = React.createElement;
17 |
18 | function start() {
19 | ReactDOM.render(
20 | e(Items, null, null),
21 | document.getElementById("articles")
22 | );
23 | }
24 |
--------------------------------------------------------------------------------
/justfile:
--------------------------------------------------------------------------------
1 | export CGO_ENABLED := "0"
2 |
3 | build-server: build-proto
4 | go build -o ./bin/ent-server github.com/google/ent/cmd/ent-server
5 |
6 | run-server: build-server
7 | ./bin/ent-server -config=ent-server.toml
8 |
9 | build-api:
10 | go build -o ./bin/ent-api github.com/google/ent/cmd/ent-api
11 |
12 | run-api: build-api
13 | ./bin/ent-api -config=ent-api.toml
14 |
15 | build-web:
16 | go build -o ./bin/ent-web github.com/google/ent/cmd/ent-web
17 |
18 | run-web: build-web
19 | ./bin/ent-web -config=ent-web.toml
20 |
21 | build-proto:
22 | protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative proto/*.proto
23 |
24 | install-cli:
25 | go build -o ./bin/ent ./cmd/ent
26 |
27 | test-cli: install-cli
28 | ./bin/ent get --digest=sha2-256:fba3120c26c61a92490736f9afa329faed0117bdb6e579011039dc4e34071507 --out=./.test
29 |
--------------------------------------------------------------------------------
/proto/ent_server_config.proto:
--------------------------------------------------------------------------------
1 | syntax = "proto3";
2 |
3 | package ent.server.config;
4 |
5 | option go_package = "github.com/ent";
6 |
7 | message Config {
8 | string project_id = 1;
9 | string listen_address = 2;
10 | string domain_name = 3;
11 | Redis redis = 4;
12 | BigQuery big_query = 5;
13 | CloudStorage cloud_storage = 6;
14 | string gin_mode = 7;
15 | string log_level = 8;
16 | repeated Remote remote = 9;
17 | repeated User user = 10;
18 | }
19 |
20 | message Redis {
21 | string endpoint = 1;
22 | }
23 |
24 | message BigQuery {
25 | string dataset = 1;
26 | }
27 |
28 | message CloudStorage {
29 | string bucket = 1;
30 | }
31 |
32 | message Remote {
33 | string name = 1;
34 | }
35 |
36 | message User {
37 | int64 id = 1;
38 | string name = 2;
39 | string api_key = 3;
40 | bool can_read = 4;
41 | bool can_write = 5;
42 | }
43 |
--------------------------------------------------------------------------------
/cmd/ent-web/config.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2022 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package main
17 |
18 | type Config struct {
19 | ProjectID string
20 |
21 | ListenAddress string
22 |
23 | DomainName string
24 |
25 | Remotes []Remote
26 | }
27 |
28 | type Remote struct {
29 | Name string
30 | URL string
31 | APIKey string
32 | }
33 |
--------------------------------------------------------------------------------
/example/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | Hacker News
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 | Hacker News
20 |
21 |
22 |
23 |
24 |
25 |
--------------------------------------------------------------------------------
/nodeservice/node_service.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2021 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package nodeservice
17 |
18 | import (
19 | "context"
20 |
21 | "github.com/google/ent/utils"
22 | )
23 |
24 | type ObjectGetter interface {
25 | Get(ctx context.Context, h utils.Digest) ([]byte, error)
26 | // TODO: GetMetadata
27 | Has(ctx context.Context, h utils.Digest) (bool, error)
28 | }
29 |
30 | type ObjectStore interface {
31 | ObjectGetter
32 | Put(ctx context.Context, b []byte) (utils.Digest, error)
33 | }
34 |
--------------------------------------------------------------------------------
/datastore/data_store.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2021 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package datastore
17 |
18 | import (
19 | "context"
20 | )
21 |
22 | // DataStore is an interface defining low-level operations for handling unstructured key/value
23 | // pairs. At this level, there is no concept of digest or any structure of the values.
24 | type DataStore interface {
25 | Get(ctx context.Context, name string) ([]byte, error)
26 | Put(ctx context.Context, name string, value []byte) error
27 | // TODO: return size
28 | Has(ctx context.Context, name string) (bool, error)
29 | }
30 |
--------------------------------------------------------------------------------
/index/index_test.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2021 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package index
17 |
18 | import (
19 | "testing"
20 |
21 | "github.com/go-playground/assert/v2"
22 | "github.com/google/ent/utils"
23 | )
24 |
25 | func TestDigestToPath(t *testing.T) {
26 | digest, err := utils.ParseDigest("sha256:366ac3bdad37d1bdc0ca87e2ea60111872e2c8d7aac8a18f2588d791056e658f")
27 | if err != nil {
28 | t.Fatalf("unexpected error: %v", err)
29 | }
30 | assert.Equal(t, "sha256/36/6a/c3/bd/ad/37/d1/bd/c0/ca/87/e2/ea/60/11/18/72/e2/c8/d7/aa/c8/a1/8f/25/88/d7/91/05/6e/65/8f", DigestToPath(digest))
31 | }
32 |
--------------------------------------------------------------------------------
/docs/contributing.md:
--------------------------------------------------------------------------------
1 | # How to Contribute
2 |
3 | We'd love to accept your patches and contributions to this project. There are
4 | just a few small guidelines you need to follow.
5 |
6 | ## Contributor License Agreement
7 |
8 | Contributions to this project must be accompanied by a Contributor License
9 | Agreement. You (or your employer) retain the copyright to your contribution;
10 | this simply gives us permission to use and redistribute your contributions as
11 | part of the project. Head over to to see
12 | your current agreements on file or to sign a new one.
13 |
14 | You generally only need to submit a CLA once, so if you've already submitted one
15 | (even if it was for a different project), you probably don't need to do it
16 | again.
17 |
18 | ## Code Reviews
19 |
20 | All submissions, including submissions by project members, require review. We
21 | use GitHub pull requests for this purpose. Consult
22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
23 | information on using pull requests.
24 |
25 | ## Community Guidelines
26 |
27 | This project follows [Google's Open Source Community
28 | Guidelines](https://opensource.google/conduct/).
29 |
--------------------------------------------------------------------------------
/api/api.go:
--------------------------------------------------------------------------------
1 | package api
2 |
3 | const GET_ENTRY_METHOD_ID = "z07773083324874402207"
4 | const SNAPSHOT_METHOD_ID = "z14764046203967555820"
5 |
6 | type GetEntryRequest struct {
7 | // Only one digest must be provided.
8 | Digests HexDigests `json:"z17760754216472891664"`
9 | }
10 |
11 | type GetEntryResponse struct {
12 | Metadata ObjectMetadata `json:"z11701617426848460867"`
13 | Mirrors []Mirror `json:"z02797662040442636406"`
14 | }
15 |
16 | type Mirror struct {
17 | URL string `json:"z15006311556098510585"`
18 | CORS string `json:"z03067985653251929561"`
19 | }
20 |
21 | type ObjectMetadata struct {
22 | Digests HexDigests `json:"z00760714168124038847"`
23 | LengthBytes int64 `json:"z05966774115567221820"`
24 | ContentType string `json:"z12467592263966562957"`
25 | }
26 |
27 | // Hex encoded strings.
28 | type HexDigests struct {
29 | Sha2_256 string `json:"sha2-256"`
30 | Sha2_512 string `json:"sha2-512"`
31 |
32 | Sha3_256 string `json:"sha3-256"`
33 | Sha3_384 string `json:"sha3-384"`
34 | Sha3_512 string `json:"sha3-512"`
35 | }
36 |
37 | type SnapshotRequest struct {
38 | URL string `json:"z06362159109170774591"`
39 | }
40 |
41 | type SnapshotResponse struct {
42 | Metadata ObjectMetadata `json:"z11701617426848460867"`
43 | }
44 |
--------------------------------------------------------------------------------
/cmd/ent-server/config.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2021 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package main
17 |
18 | type Config struct {
19 | ProjectID string
20 |
21 | ListenAddress string
22 |
23 | DomainName string
24 |
25 | RedisEnabled bool
26 | RedisEndpoint string
27 |
28 | BigqueryEnabled bool
29 | BigqueryDataset string
30 |
31 | CloudStorageEnabled bool
32 | CloudStorageBucket string
33 |
34 | GinMode string
35 | LogLevel string
36 |
37 | Remotes []Remote
38 |
39 | Users []User
40 | }
41 |
42 | type Remote struct {
43 | Name string
44 | }
45 |
46 | type User struct {
47 | ID int64
48 | Name string
49 | APIKey string
50 | CanRead bool
51 | CanWrite bool
52 | }
53 |
--------------------------------------------------------------------------------
/example/item.js:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2021 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | class Item extends React.Component {
17 | constructor(props) {
18 | super(props);
19 | this.state = {
20 | item: props.item,
21 | };
22 | }
23 |
24 | render() {
25 | return e('div', { className: 'border py-3 my-3 bg-' + env.colour + '-200 hover:bg-' + env.colour + '-300' },
26 | e('span', { className: 'px-5 w-24 inline-block' }, '[' + this.state.item.score + ']'),
27 | e('a', { href: this.state.item.url, className: 'underline' }, this.state.item.title),
28 | e('span', { className: "p-2" }, 'by'),
29 | e('span', null, this.state.item.by),
30 | );
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/datastore/memcache.go:
--------------------------------------------------------------------------------
1 | package datastore
2 |
3 | import (
4 | "context"
5 |
6 | "github.com/go-redis/redis/v8"
7 | "github.com/google/ent/log"
8 | )
9 |
10 | type Memcache struct {
11 | Inner DataStore
12 | RDB *redis.Client
13 | }
14 |
15 | func (s Memcache) Get(ctx context.Context, name string) ([]byte, error) {
16 | cmd := s.RDB.Get(ctx, name)
17 | item, err := cmd.Bytes()
18 | if err != nil {
19 | if err != redis.Nil {
20 | log.Errorf(ctx, "error getting %q from memcache: %v", name, err)
21 | }
22 | b, err := s.Inner.Get(ctx, name)
23 | if err != nil {
24 | return nil, err
25 | }
26 | go s.TrySet(ctx, name, b)
27 | return b, nil
28 | }
29 | log.Infof(ctx, "got %q from memcache", name)
30 | return item, nil
31 | }
32 |
33 | func (s Memcache) Put(ctx context.Context, name string, value []byte) error {
34 | err := s.Inner.Put(ctx, name, value)
35 | if err != nil {
36 | return err
37 | }
38 | go s.TrySet(ctx, name, value)
39 | return nil
40 | }
41 |
42 | func (s Memcache) Has(ctx context.Context, name string) (bool, error) {
43 | return s.Inner.Has(ctx, name)
44 | }
45 |
46 | func (s Memcache) TrySet(ctx context.Context, name string, value []byte) {
47 | err := s.RDB.Set(ctx, name, value, 0)
48 | if err != nil {
49 | log.Errorf(ctx, "error adding %q to memcache: %v", name, err)
50 | } else {
51 | log.Infof(ctx, "added %q to memcache", name)
52 | }
53 | }
54 |
--------------------------------------------------------------------------------
/cmd/ent-server/store.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "context"
5 | "time"
6 |
7 | "cloud.google.com/go/firestore"
8 | )
9 |
10 | type Store struct {
11 | c *firestore.Client
12 | }
13 |
14 | func NewStore(c *firestore.Client) *Store {
15 | return &Store{c: c}
16 | }
17 |
18 | const (
19 | MapEntryCollection = "map_entry"
20 | )
21 |
22 | type Digest struct {
23 | Code int64 `firestore:"0"`
24 | Digest []byte `firestore:"1"`
25 | }
26 |
27 | type MapEntry struct {
28 | PublicKey []byte `firestore:"0"`
29 |
30 | Label string `firestore:"1"`
31 | Target Digest `firestore:"2"`
32 |
33 | EntrySignature []byte `firestore:"3"`
34 |
35 | CreationTime time.Time `firestore:"4"`
36 | ClientIPAddress string `firestore:"5"`
37 | RequestBytes []byte `firestore:"6"`
38 | }
39 |
40 | func (s *Store) GetMapEntry(ctx context.Context, publicKey []byte, label string) (*MapEntry, error) {
41 | docs := s.c.Collection(MapEntryCollection).Query.Where("0", "==", publicKey).Where("1", "==", label).Documents(ctx)
42 | doc, err := docs.Next()
43 | if err != nil { // Not found.
44 | return nil, nil
45 | }
46 | e := MapEntry{}
47 | if err := doc.DataTo(&e); err != nil {
48 | return nil, err
49 | }
50 | return &e, nil
51 | }
52 |
53 | func (s *Store) SetMapEntry(ctx context.Context, e *MapEntry) error {
54 | _, _, err := s.c.Collection(MapEntryCollection).Add(ctx, e)
55 | return err
56 | }
57 |
--------------------------------------------------------------------------------
/index/index.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2021 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package index
17 |
18 | import (
19 | "strings"
20 |
21 | "github.com/google/ent/utils"
22 | )
23 |
24 | const (
25 | EntryFilename = "entry.json"
26 | )
27 |
28 | // Vaguely similar to https://github.com/opencontainers/image-spec/blob/main/descriptor.md
29 | type IndexEntry struct {
30 | MediaType string `json:"mediaType"`
31 | Digest string `json:"digest"`
32 | Size int `json:"size"`
33 | URLS []string `json:"urls"`
34 | }
35 |
36 | // Split the digest into its prefix, and then two character chunks, separated by slashes, so that
37 | // each directory contains at most 255 entries.
38 | func DigestToPath(digest utils.Digest) string {
39 | s := strings.Split(utils.DigestToHumanString(digest), ":")
40 | out := s[0]
41 | for i := 0; i < len(s[1])/2; i++ {
42 | out += "/" + s[1][i*2:(i+1)*2]
43 | }
44 | return out
45 | }
46 |
--------------------------------------------------------------------------------
/datastore/file.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2021 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package datastore
17 |
18 | import (
19 | "context"
20 | "io/ioutil"
21 | "os"
22 | "path"
23 | )
24 |
25 | // File is an implementation of DataStore using the local file system, rooted at the
26 | // specified directory.
27 | type File struct {
28 | DirName string
29 | }
30 |
31 | func (s File) Get(ctx context.Context, name string) ([]byte, error) {
32 | return ioutil.ReadFile(path.Join(s.DirName, name))
33 | }
34 |
35 | func (s File) Put(ctx context.Context, name string, value []byte) error {
36 | return ioutil.WriteFile(path.Join(s.DirName, name), value, 0644)
37 | }
38 |
39 | func (s File) Has(ctx context.Context, name string) (bool, error) {
40 | _, err := os.Stat(path.Join(s.DirName, name))
41 | if err != nil {
42 | if os.IsNotExist(err) {
43 | return false, nil
44 | } else {
45 | return false, err
46 | }
47 | }
48 | return true, nil
49 | }
50 |
--------------------------------------------------------------------------------
/cmd/ent/remote/remote.go:
--------------------------------------------------------------------------------
1 | package remote
2 |
3 | import (
4 | "fmt"
5 | "log"
6 | "net/url"
7 |
8 | "github.com/google/ent/cmd/ent/config"
9 | "github.com/google/ent/nodeservice"
10 | pb "github.com/google/ent/proto"
11 | "google.golang.org/grpc"
12 | "google.golang.org/grpc/credentials"
13 | )
14 |
15 | func GetRemote(c config.Config, remoteName string) (config.Remote, error) {
16 | for _, remote := range c.Remotes {
17 | if remote.Name == remoteName {
18 | return remote, nil
19 | }
20 | }
21 | return config.Remote{}, fmt.Errorf("remote %q not found", remoteName)
22 | }
23 |
24 | func GetObjectStore(remote config.Remote) *nodeservice.Remote {
25 | if remote.Write {
26 | parsedURL, err := url.Parse(remote.URL)
27 | if err != nil {
28 | log.Fatalf("failed to parse url: %v", err)
29 | }
30 |
31 | o := []grpc.DialOption{}
32 | if parsedURL.Scheme == "http" {
33 | o = append(o, grpc.WithInsecure())
34 | } else {
35 | o = append(o, grpc.WithTransportCredentials(credentials.NewTLS(nil)))
36 | }
37 | port := parsedURL.Port()
38 | if port == "" {
39 | if parsedURL.Scheme == "http" {
40 | port = "80"
41 | } else {
42 | port = "443"
43 | }
44 | }
45 | cc, err := grpc.Dial(parsedURL.Hostname()+":"+port, o...)
46 | if err != nil {
47 | log.Fatalf("failed to dial: %v", err)
48 | }
49 | client := pb.NewEntClient(cc)
50 | return &nodeservice.Remote{
51 | APIURL: remote.URL,
52 | APIKey: remote.APIKey,
53 | GRPC: client,
54 | }
55 | } else {
56 | return nil
57 | }
58 | }
59 |
--------------------------------------------------------------------------------
/objectstore/store.go:
--------------------------------------------------------------------------------
1 | package objectstore
2 |
3 | import (
4 | "bytes"
5 | "context"
6 | "fmt"
7 |
8 | "github.com/google/ent/datastore"
9 | "github.com/google/ent/log"
10 | "github.com/google/ent/utils"
11 | "github.com/multiformats/go-multihash"
12 | )
13 |
14 | type Store struct {
15 | Inner datastore.DataStore
16 | }
17 |
18 | func (s Store) Get(ctx context.Context, digest utils.Digest) ([]byte, error) {
19 | b, err := s.Inner.Get(ctx, digest.String())
20 | if err != nil {
21 | decodedDigest, err := multihash.Decode(digest)
22 | if err == nil && decodedDigest.Code == multihash.SHA2_256 {
23 | oldDigest := utils.DigestToHumanString(digest)
24 | log.Infof(ctx, "decoded digest: %v", decodedDigest)
25 | log.Infof(ctx, "old digest: %v", oldDigest)
26 | b, err = s.Inner.Get(ctx, oldDigest)
27 | if err != nil {
28 | return nil, err
29 | }
30 | } else {
31 | return nil, err
32 | }
33 | }
34 | actualDigest := utils.ComputeDigest(b)
35 | if !bytes.Equal(actualDigest, digest) {
36 | return nil, fmt.Errorf("mismatching digest: wanted:%q got:%q", digest.String(), actualDigest.String())
37 | }
38 | return b, nil
39 | }
40 |
41 | func (s Store) Put(ctx context.Context, b []byte) (utils.Digest, error) {
42 | digest := utils.ComputeDigest(b)
43 | err := s.Inner.Put(ctx, digest.String(), b)
44 | if err != nil {
45 | // Return digest anyways, useful for logging errors.
46 | return digest, err
47 | }
48 | return digest, nil
49 | }
50 |
51 | func (s Store) Has(ctx context.Context, digest utils.Digest) (bool, error) {
52 | return s.Inner.Has(ctx, digest.String())
53 | }
54 |
--------------------------------------------------------------------------------
/flake.nix:
--------------------------------------------------------------------------------
1 | {
2 | description = "ent-plus";
3 | inputs = {
4 | nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.11";
5 | flake-utils.url = "github:numtide/flake-utils";
6 | gomod2nix.url = "github:nix-community/gomod2nix";
7 | };
8 | outputs = { self, nixpkgs, flake-utils, gomod2nix } :
9 | (flake-utils.lib.eachDefaultSystem
10 | (system:
11 | let
12 | pkgs = import nixpkgs {
13 | inherit system;
14 | overlays = [
15 | gomod2nix.overlays.default
16 | ];
17 | };
18 | ent-server = pkgs.buildGoApplication {
19 | pname = "ent-server";
20 | version = "0.1.0";
21 | pwd = ./.;
22 | src = ./cmd/ent-server;
23 | modules = ./gomod2nix.toml;
24 | };
25 | ent-server-docker = pkgs.dockerTools.buildImage {
26 | name = "ent-server";
27 | config = { Cmd = [ "${ent-server}/bin/rust_nix_blog" ]; };
28 | };
29 | in {
30 | packages = {
31 | ent-server = ent-server;
32 | ent-server-docker = ent-server-docker;
33 | };
34 | defaultPackage = ent-server;
35 | devShell =
36 | pkgs.mkShell {
37 | packages = [
38 | pkgs.gomod2nix
39 | ];
40 | buildInputs = [
41 | pkgs.go
42 | pkgs.gopls
43 | pkgs.just
44 | pkgs.protobuf
45 | pkgs.protoc-gen-go
46 | pkgs.protoc-gen-go-grpc
47 | ];
48 | };
49 | }));
50 | }
51 |
--------------------------------------------------------------------------------
/proto/ent_server_api.proto:
--------------------------------------------------------------------------------
1 | syntax = "proto3";
2 |
3 | package ent.server.api;
4 |
5 | option go_package = "github.com/ent";
6 |
7 | message Digest {
8 | uint64 code = 1;
9 | bytes digest = 2;
10 | }
11 |
12 | message GetEntryRequest {
13 | Digest digest = 1;
14 | }
15 |
16 | message Chunk {
17 | uint64 offset = 1;
18 | bytes data = 2;
19 | }
20 |
21 | message GetEntryResponse {
22 | oneof entry {
23 | EntryMetadata metadata = 1;
24 | Chunk chunk = 2;
25 | }
26 | }
27 |
28 | message GetEntryMetadataRequest {
29 | Digest digest = 1;
30 | }
31 |
32 | message GetEntryMetadataResponse {
33 | EntryMetadata metadata = 1;
34 | }
35 |
36 | message PutEntryRequest {
37 | Chunk chunk = 1;
38 | }
39 |
40 | message PutEntryResponse {
41 | EntryMetadata metadata = 1;
42 | }
43 |
44 | message EntryMetadata {
45 | repeated Digest digests = 1;
46 | uint64 size = 2;
47 | }
48 |
49 | message GetTagRequest{
50 | bytes public_key = 1;
51 | // TODO: Prefix search.
52 | string label = 2;
53 | }
54 |
55 | message GetTagResponse{
56 | SignedTag signed_tag = 1;
57 | }
58 |
59 | message Tag {
60 | string label = 1;
61 | Digest target = 2;
62 | }
63 |
64 | message SignedTag {
65 | Tag tag = 1;
66 | bytes tag_signature = 2;
67 | bytes public_key = 3;
68 | }
69 |
70 | message SetTagRequest {
71 | SignedTag signed_tag = 1;
72 | }
73 |
74 | message SetTagResponse {
75 | }
76 |
77 | service Ent {
78 | rpc GetTag(GetTagRequest) returns (GetTagResponse) {}
79 | rpc SetTag(SetTagRequest) returns (SetTagResponse) {}
80 |
81 | rpc GetEntry(GetEntryRequest) returns (stream GetEntryResponse) {}
82 | rpc GetEntryMetadata(GetEntryMetadataRequest) returns (GetEntryMetadataResponse) {}
83 | rpc PutEntry(stream PutEntryRequest) returns (PutEntryResponse) {}
84 | }
85 |
--------------------------------------------------------------------------------
/utils/lib_test.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2022 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package utils
17 |
18 | import (
19 | "bytes"
20 | "testing"
21 | )
22 |
23 | func TestParseDigest(t *testing.T) {
24 | digest0, err := ParseDigest("sha256:1f209f17903dc0310f9a0fe337d3a893193f20b4171895a74d0200d6019dedd6")
25 | if err != nil {
26 | t.Fatal(err)
27 | }
28 | digest1, err := ParseDigest("12201f209f17903dc0310f9a0fe337d3a893193f20b4171895a74d0200d6019dedd6")
29 | if err != nil {
30 | t.Fatal(err)
31 | }
32 | if !bytes.Equal(digest0, digest1) {
33 | t.Fatalf("digests should be equal")
34 | }
35 | digestString := digest0.String()
36 | if digestString != "12201f209f17903dc0310f9a0fe337d3a893193f20b4171895a74d0200d6019dedd6" {
37 | t.Fatalf("digest string should be equal")
38 | }
39 | }
40 |
41 | func TestDigestToArray(t *testing.T) {
42 | digest, err := ParseDigest("12201f209f17903dc0310f9a0fe337d3a893193f20b4171895a74d0200d6019dedd6")
43 | if err != nil {
44 | t.Fatal(err)
45 | }
46 | digestArray := [64]byte(DigestToArray(digest))
47 | expectedDigestArray := [64]byte{
48 | 0x12, 0x20,
49 | 0x1f, 0x20, 0x9f, 0x17, 0x90, 0x3d, 0xc0, 0x31, 0x0f, 0x9a, 0x0f, 0xe3, 0x37, 0xd3, 0xa8, 0x93,
50 | 0x19, 0x3f, 0x20, 0xb4, 0x17, 0x18, 0x95, 0xa7, 0x4d, 0x02, 0x00, 0xd6, 0x01, 0x9d, 0xed, 0xd6,
51 | }
52 | if digestArray != expectedDigestArray {
53 | t.Fatalf("digest array should be equal:\n%x\n%x", digestArray, expectedDigestArray)
54 | }
55 | }
56 |
--------------------------------------------------------------------------------
/cmd/ent/config/config.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2022 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package config
17 |
18 | import (
19 | "io/ioutil"
20 | "log"
21 | "os"
22 | "path/filepath"
23 |
24 | "github.com/BurntSushi/toml"
25 | )
26 |
27 | const (
28 | indexBasePath = "https://raw.githubusercontent.com/tiziano88/ent-index/main"
29 | staticSpace = "https://api.static.space/v1"
30 | localhost = "http://localhost:8081/v1"
31 | )
32 |
33 | type Config struct {
34 | Remotes []Remote
35 | SecretKey string `toml:"secret_key"`
36 | }
37 |
38 | // TODO: auth
39 |
40 | type Remote struct {
41 | Name string
42 | URL string
43 | Index bool
44 | APIKey string `toml:"api_key"`
45 | Write bool
46 | ReadGroup uint
47 | }
48 |
49 | func ReadConfig() Config {
50 | s, err := os.UserConfigDir()
51 | if err != nil {
52 | log.Fatalf("could not load config dir: %v", err)
53 | }
54 | s = filepath.Join(s, "ent.toml")
55 | f, err := ioutil.ReadFile(s)
56 | if err != nil {
57 | log.Printf("could not read config: %v", err)
58 | return defaultConfig()
59 | }
60 | config := Config{}
61 | err = toml.Unmarshal(f, &config)
62 | if err != nil {
63 | log.Fatalf("could not parse config: %v", err)
64 | }
65 | return config
66 | }
67 |
68 | func defaultConfig() Config {
69 | return Config{
70 | Remotes: []Remote{
71 | {
72 | Name: "default",
73 | URL: staticSpace,
74 | Index: true,
75 | },
76 | },
77 | }
78 | }
79 |
--------------------------------------------------------------------------------
/cmd/ent/cmd/root.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2021 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package cmd
17 |
18 | import (
19 | "context"
20 | "os"
21 |
22 | "github.com/google/ent/cmd/ent/config"
23 | "github.com/google/ent/log"
24 | "github.com/google/ent/nodeservice"
25 | "github.com/spf13/cobra"
26 | )
27 |
28 | func getMultiplexObjectGetter(c config.Config) nodeservice.ObjectGetter {
29 | inner := make([]nodeservice.Inner, 0)
30 | for _, remote := range c.Remotes {
31 | inner = append(inner, nodeservice.Inner{
32 | Name: remote.Name,
33 | ObjectGetter: getObjectGetter(remote)})
34 | }
35 | return nodeservice.Sequence{
36 | Inner: inner,
37 | }
38 | }
39 |
40 | func getObjectGetter(remote config.Remote) nodeservice.ObjectGetter {
41 | if remote.Index {
42 | return nodeservice.IndexClient{
43 | BaseURL: remote.URL,
44 | }
45 | } else {
46 | return nodeservice.Remote{
47 | APIURL: remote.URL,
48 | APIKey: remote.APIKey,
49 | }
50 | }
51 | }
52 |
53 | var rootCmd = &cobra.Command{
54 | Use: "ent",
55 | }
56 |
57 | func Execute() {
58 | ctx := context.Background()
59 | if err := rootCmd.Execute(); err != nil {
60 | log.Criticalf(ctx, "execute command: %v", err)
61 | os.Exit(1)
62 | }
63 | }
64 |
65 | func init() {
66 | rootCmd.AddCommand(digestCmd)
67 | rootCmd.AddCommand(getCmd)
68 | rootCmd.AddCommand(putCmd)
69 | rootCmd.AddCommand(keygenCmd)
70 | }
71 |
72 | func GetObjectGetter() nodeservice.ObjectGetter {
73 | config := config.ReadConfig()
74 | return getMultiplexObjectGetter(config)
75 | }
76 |
--------------------------------------------------------------------------------
/cmd/ent/cmd/keygen.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2023 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package cmd
17 |
18 | import (
19 | "context"
20 | "crypto/ecdsa"
21 | "crypto/elliptic"
22 | "crypto/rand"
23 | "crypto/x509"
24 | "encoding/base64"
25 | "os"
26 |
27 | "github.com/google/ent/log"
28 | "github.com/spf13/cobra"
29 | )
30 |
31 | var keygenCmd = &cobra.Command{
32 | Use: "keygen",
33 | Run: func(cmd *cobra.Command, args []string) {
34 | ctx := context.Background()
35 | k, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
36 | if err != nil {
37 | log.Criticalf(ctx, "generate key pair: %v", err)
38 | os.Exit(1)
39 | }
40 | sk, err := x509.MarshalECPrivateKey(k)
41 | if err != nil {
42 | log.Criticalf(ctx, "marshal private key: %v", err)
43 | os.Exit(1)
44 | }
45 | sks := base64.URLEncoding.EncodeToString(sk)
46 | log.Infof(ctx, "Secret key: %q", sks)
47 |
48 | pk, err := x509.MarshalPKIXPublicKey(&k.PublicKey)
49 | if err != nil {
50 | log.Criticalf(ctx, "marshal public key: %v", err)
51 | os.Exit(1)
52 | }
53 | pks := base64.URLEncoding.EncodeToString(pk)
54 | log.Infof(ctx, "Public key: %q", pks)
55 |
56 | text := "hello world"
57 | sig, err := ecdsa.SignASN1(rand.Reader, k, []byte(text))
58 | if err != nil {
59 | log.Criticalf(ctx, "sign ASN1: %v", err)
60 | os.Exit(1)
61 | }
62 | sigs := base64.URLEncoding.EncodeToString(sig)
63 | log.Infof(ctx, "Signature: %q", sigs)
64 |
65 | // verify
66 | ok := ecdsa.VerifyASN1(&k.PublicKey, []byte(text), sig)
67 | log.Infof(ctx, "Verify: %v", ok)
68 | },
69 | }
70 |
71 | func init() {
72 | keygenCmd.PersistentFlags().StringVar(&remoteFlag, "remote", "", "remote")
73 | }
74 |
--------------------------------------------------------------------------------
/utils/node_test.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2022 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package utils
17 |
18 | import (
19 | "reflect"
20 | "testing"
21 | )
22 |
23 | func TestParseSelector(t *testing.T) {
24 | tests := []struct {
25 | name string
26 | s string
27 | want Selector
28 | err bool
29 | }{
30 | {
31 | name: "empty",
32 | s: "",
33 | want: Selector(0),
34 | },
35 | {
36 | name: "invalid",
37 | s: "0[1",
38 | want: Selector(0),
39 | },
40 | }
41 | for _, tt := range tests {
42 | t.Run(tt.name, func(t *testing.T) {
43 | if got, err := ParseSelector(tt.s); !reflect.DeepEqual(got, tt.want) {
44 | if (err != nil) != tt.err {
45 | t.Errorf("ParseSelector() error = %v, wantErr %v", err, tt.err)
46 | return
47 | }
48 | t.Errorf("ParseSelector() = %v, want %v", got, tt.want)
49 | }
50 | })
51 | }
52 | }
53 |
54 | func TestParsePath(t *testing.T) {
55 | tests := []struct {
56 | name string
57 | s string
58 | want Path
59 | }{
60 | {
61 | name: "empty",
62 | s: "",
63 | want: []Selector{},
64 | },
65 | {
66 | name: "invalid",
67 | s: "0[1",
68 | want: nil,
69 | },
70 | {
71 | name: "valid",
72 | s: "0",
73 | want: []Selector{0},
74 | },
75 | {
76 | name: "valid",
77 | s: "2",
78 | want: []Selector{2},
79 | },
80 | {
81 | name: "valid",
82 | s: "0/1/2",
83 | want: []Selector{0, 1, 2},
84 | },
85 | }
86 | for _, tt := range tests {
87 | t.Run(tt.name, func(t *testing.T) {
88 | if got, _ := ParsePath(tt.s); !reflect.DeepEqual(got, tt.want) {
89 | t.Errorf("ParsePath() = %v, want %v", got, tt.want)
90 | }
91 | })
92 | }
93 | }
94 |
--------------------------------------------------------------------------------
/nodeservice/sequence.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2021 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package nodeservice
17 |
18 | import (
19 | "context"
20 | "fmt"
21 | "time"
22 |
23 | "github.com/google/ent/log"
24 | "github.com/google/ent/utils"
25 | )
26 |
27 | type Sequence struct {
28 | Inner []Inner
29 | }
30 |
31 | type Inner struct {
32 | Name string
33 | ObjectGetter ObjectGetter
34 | }
35 |
36 | func NewRemote(name string, url string, apiKey string) Inner {
37 | return Inner{
38 | Name: name,
39 | ObjectGetter: Remote{
40 | APIURL: url,
41 | APIKey: apiKey,
42 | },
43 | }
44 | }
45 |
46 | func (s Sequence) Get(ctx context.Context, digest utils.Digest) ([]byte, error) {
47 | for _, ss := range s.Inner {
48 | start := time.Now()
49 | b, err := ss.ObjectGetter.Get(ctx, digest)
50 | if err == ErrNotFound {
51 | log.Infof(ctx, "object %s not found in %s", digest, ss.Name)
52 | continue
53 | } else if err != nil {
54 | log.Errorf(ctx, "error fetching (get %q) from remote %q: %v", digest, ss.Name, err)
55 | continue
56 | }
57 | end := time.Now()
58 | elapsed := end.Sub(start)
59 | log.Infof(ctx, "fetched %q from remote %q in %v", digest, ss.Name, elapsed)
60 | return b, nil
61 | }
62 | return nil, ErrNotFound
63 | }
64 |
65 | func (s Sequence) Has(ctx context.Context, digest utils.Digest) (bool, error) {
66 | for _, ss := range s.Inner {
67 | b, err := ss.ObjectGetter.Has(ctx, digest)
68 | if err != nil {
69 | log.Errorf(ctx, "error fetching (has %q) from remote %q: %v", digest, ss.Name, err)
70 | continue
71 | }
72 | if b {
73 | return b, nil
74 | }
75 | }
76 | return false, nil
77 | }
78 |
79 | func (s Sequence) Put(ctx context.Context, b []byte) (utils.Digest, error) {
80 | // return s.Inner[0].Put(ctx, b)
81 | return utils.Digest{}, fmt.Errorf("not implemented")
82 | }
83 |
--------------------------------------------------------------------------------
/datastore/cloud.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2021 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package datastore
17 |
18 | import (
19 | "context"
20 | "fmt"
21 | "io/ioutil"
22 |
23 | "cloud.google.com/go/storage"
24 | "github.com/google/ent/log"
25 | )
26 |
27 | // Cloud is an implementation of DataStore using a Google Cloud Storage bucket.
28 | type Cloud struct {
29 | Client *storage.Client
30 | BucketName string
31 | }
32 |
33 | func (s Cloud) Get(ctx context.Context, name string) ([]byte, error) {
34 | rc, err := s.Client.Bucket(s.BucketName).Object(name).NewReader(ctx)
35 | if err != nil {
36 | return nil, err
37 | }
38 | body, err := ioutil.ReadAll(rc)
39 | if err != nil {
40 | return nil, fmt.Errorf("error reading from cloud storage: %v", err)
41 | }
42 | err = rc.Close()
43 | if err != nil {
44 | return nil, fmt.Errorf("error closing reader from cloud storage: %v", err)
45 | }
46 | return body, nil
47 | }
48 |
49 | func (s Cloud) Put(ctx context.Context, name string, value []byte) error {
50 | o := s.Client.Bucket(s.BucketName).Object(name)
51 | attr, err := o.Attrs(ctx)
52 | if err == storage.ErrObjectNotExist {
53 | wc := o.NewWriter(ctx)
54 | _, err := wc.Write(value)
55 | if err != nil {
56 | return fmt.Errorf("error writing to cloud storage: %v", err)
57 | }
58 | err = wc.Close()
59 | if err != nil {
60 | return fmt.Errorf("error closing writer to cloud storage: %v", err)
61 | }
62 | return nil
63 | } else if err != nil {
64 | return fmt.Errorf("error getting attrs from cloud storage: %v", err)
65 | }
66 | log.Infof(ctx, "object %q already exists in cloud storage: %+v", name, attr)
67 | return nil
68 | }
69 |
70 | func (s Cloud) Has(ctx context.Context, name string) (bool, error) {
71 | _, err := s.Client.Bucket(s.BucketName).Object(name).Attrs(ctx)
72 | if err != nil {
73 | if err == storage.ErrObjectNotExist {
74 | return false, nil
75 | } else {
76 | return false, err
77 | }
78 | }
79 | return true, nil
80 | }
81 |
--------------------------------------------------------------------------------
/example/items.js:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2021 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | class Items extends React.Component {
17 | constructor() {
18 | super();
19 | this.state = {
20 | items: [
21 | { id: 0, by: "user", score: 126, title: "Pokegb: A gameboy emulator that only plays Pokémon Blue, in 68 lines of C++", url: "https://binji.github.io/posts/pokegb/" },
22 | { id: 1, by: "user", score: 126, title: "Pokegb: A gameboy emulator that only plays Pokémon Blue, in 68 lines of C++", url: "https://binji.github.io/posts/pokegb/" },
23 | { id: 2, by: "user", score: 126, title: "Pokegb: A gameboy emulator that only plays Pokémon Blue, in 68 lines of C++", url: "https://binji.github.io/posts/pokegb/" },
24 | { id: 3, by: "user", score: 126, title: "Pokegb: A gameboy emulator that only plays Pokémon Blue, in 68 lines of C++", url: "https://binji.github.io/posts/pokegb/" },
25 | { id: 4, by: "user", score: 126, title: "Pokegb: A gameboy emulator that only plays Pokémon Blue, in 68 lines of C++", url: "https://binji.github.io/posts/pokegb/" },
26 | ]
27 | };
28 | }
29 |
30 | render() {
31 | return e('div', null,
32 | this.state.items.map((item) => e(Item, { item: item, key: item.id }, null)));
33 | }
34 |
35 | componentDidMount() {
36 | this.go();
37 | }
38 |
39 | async go() {
40 | const topStories = await (await fetch("https://hacker-news.firebaseio.com/v0/topstories.json")).json()
41 | console.log(topStories);
42 | var items = [];
43 | for (var i = 0; i < env.numberOfArticles; i++) {
44 | const itemId = topStories[i];
45 | const item = await (await fetch("https://hacker-news.firebaseio.com/v0/item/" + itemId + ".json")).json();
46 | items.push(item);
47 | }
48 | console.log(items);
49 | this.setState({
50 | items: items
51 | })
52 | }
53 | }
54 |
--------------------------------------------------------------------------------
/log/log.go:
--------------------------------------------------------------------------------
1 | package log
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | "log"
7 |
8 | "cloud.google.com/go/logging"
9 | "github.com/fatih/color"
10 | "github.com/gin-gonic/gin"
11 | )
12 |
13 | type Color func(format string, a ...interface{}) string
14 |
15 | var (
16 | parentLogger *logging.Logger
17 | childLogger *logging.Logger
18 |
19 | severityColor = map[logging.Severity]Color{
20 | logging.Debug: color.BlueString,
21 | logging.Info: color.GreenString,
22 | logging.Warning: color.YellowString,
23 | logging.Error: color.RedString,
24 | logging.Critical: color.MagentaString,
25 | }
26 | )
27 |
28 | func InitLog(projectID string) {
29 | if projectID != "" {
30 | client, err := logging.NewClient(context.Background(), "projects/"+projectID)
31 | if err != nil {
32 | panic(err)
33 | }
34 |
35 | err = client.Ping(context.Background())
36 | if err != nil {
37 | log.Printf("Failed to ping log client: %v", err)
38 | return
39 | }
40 | log.Printf("Successfully created log client: %v", client)
41 |
42 | parentLogger = client.Logger("request_log")
43 | childLogger = client.Logger("request_log_entries")
44 | }
45 | }
46 |
47 | func Log(ctx context.Context, entry logging.Entry) {
48 | // TODO: gRPC
49 | if gc, ok := ctx.(*gin.Context); ok {
50 | entry.HTTPRequest = &logging.HTTPRequest{
51 | Request: gc.Request,
52 | }
53 | }
54 |
55 | // Always log to stderr.
56 | color := severityColor[entry.Severity]
57 | log.Printf("[%s] %v", color("%-7s", entry.Severity), entry.Payload)
58 |
59 | if parentLogger != nil {
60 | parentLogger.Log(entry)
61 | }
62 | }
63 |
64 | func Debugf(ctx context.Context, format string, args ...interface{}) {
65 | Log(ctx, logging.Entry{
66 | Payload: fmt.Sprintf(format, args...),
67 | Severity: logging.Debug,
68 | })
69 | }
70 |
71 | func Infof(ctx context.Context, format string, args ...interface{}) {
72 | Log(ctx, logging.Entry{
73 | Payload: fmt.Sprintf(format, args...),
74 | Severity: logging.Info,
75 | })
76 | }
77 |
78 | func Warningf(ctx context.Context, format string, args ...interface{}) {
79 | Log(ctx, logging.Entry{
80 | Payload: fmt.Sprintf(format, args...),
81 | Severity: logging.Warning,
82 | })
83 | }
84 |
85 | func Errorf(ctx context.Context, format string, args ...interface{}) {
86 | Log(ctx, logging.Entry{
87 | Payload: fmt.Sprintf(format, args...),
88 | Severity: logging.Error,
89 | })
90 | }
91 |
92 | func Criticalf(ctx context.Context, format string, args ...interface{}) {
93 | Log(ctx, logging.Entry{
94 | Payload: fmt.Sprintf(format, args...),
95 | Severity: logging.Critical,
96 | })
97 | }
98 |
--------------------------------------------------------------------------------
/nodeservice/index_client.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2022 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package nodeservice
17 |
18 | import (
19 | "bytes"
20 | "context"
21 | "encoding/json"
22 | "fmt"
23 | "io/ioutil"
24 | "log"
25 | "net/http"
26 | "path"
27 |
28 | "github.com/google/ent/index"
29 | "github.com/google/ent/utils"
30 | )
31 |
32 | type IndexClient struct {
33 | BaseURL string
34 | }
35 |
36 | func (c IndexClient) Get(ctx context.Context, digest utils.Digest) ([]byte, error) {
37 | log.Printf("%s\n", string(digest))
38 | u := c.BaseURL + "/" + path.Join(index.DigestToPath(digest), index.EntryFilename)
39 | log.Printf("fetching entry from %s\n", u)
40 | entryRes, err := http.Get(u)
41 | if err != nil {
42 | return nil, fmt.Errorf("could not fetch index entry: %w", err)
43 | }
44 | if entryRes.StatusCode != http.StatusOK {
45 | return nil, fmt.Errorf("could not fetch index entry: %s", entryRes.Status)
46 | }
47 | entry := index.IndexEntry{}
48 | err = json.NewDecoder(entryRes.Body).Decode(&entry)
49 | if err != nil {
50 | return nil, fmt.Errorf("could not parse index entry as JSON: %w", err)
51 | }
52 | log.Printf("parsed entry: %+v", entry)
53 | return DownloadFromURL(digest, entry.URLS[0])
54 | }
55 |
56 | func (c IndexClient) Has(ctx context.Context, h utils.Digest) (bool, error) {
57 | return false, nil
58 | }
59 |
60 | func DownloadFromURL(digest utils.Digest, url string) ([]byte, error) {
61 | targetRes, err := http.Get(url)
62 | if err != nil {
63 | return nil, fmt.Errorf("could not fetch target: %v", err)
64 | }
65 | if targetRes.StatusCode != http.StatusOK {
66 | return nil, fmt.Errorf("could not fetch target: %s", targetRes.Status)
67 | }
68 | target, err := ioutil.ReadAll(targetRes.Body)
69 | if err != nil {
70 | return nil, fmt.Errorf("could not download target: %v", err)
71 | }
72 | targetDigest := utils.ComputeDigest(target)
73 | if !bytes.Equal(targetDigest, digest) {
74 | return nil, fmt.Errorf("digest mismatch, wanted: %q, got %q", digest, targetDigest)
75 | }
76 | return target, nil
77 | }
78 |
--------------------------------------------------------------------------------
/nodeservice/remote.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2021 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package nodeservice
17 |
18 | import (
19 | "context"
20 | "fmt"
21 | "io"
22 |
23 | "github.com/google/ent/log"
24 | pb "github.com/google/ent/proto"
25 | "github.com/google/ent/utils"
26 | "github.com/schollz/progressbar/v3"
27 | "google.golang.org/grpc"
28 | "google.golang.org/grpc/codes"
29 | "google.golang.org/grpc/metadata"
30 | )
31 |
32 | type Remote struct {
33 | APIURL string
34 | APIKey string
35 | GRPC pb.EntClient
36 | }
37 |
38 | const (
39 | APIKeyHeader = "x-api-key"
40 | chunkSize = 1024 * 1024
41 | )
42 |
43 | var (
44 | ErrNotFound = fmt.Errorf("not found")
45 | )
46 |
47 | func (s Remote) Get(ctx context.Context, digest utils.Digest) ([]byte, error) {
48 | md := metadata.New(nil)
49 | md.Set(APIKeyHeader, s.APIKey)
50 | ctx = metadata.NewOutgoingContext(ctx, md)
51 |
52 | req := pb.GetEntryRequest{
53 | Digest: utils.DigestToProto(digest),
54 | }
55 | c, err := s.GRPC.GetEntry(ctx, &req)
56 | if err != nil {
57 | return nil, err
58 | }
59 |
60 | res, err := c.Recv()
61 | if err != nil {
62 | return nil, err
63 | }
64 |
65 | return res.GetChunk().Data, nil
66 | }
67 |
68 | func (s Remote) Put(ctx context.Context, size uint64, r io.Reader) (utils.Digest, error) {
69 | md := metadata.New(nil)
70 | md.Set(APIKeyHeader, s.APIKey)
71 | ctx = metadata.NewOutgoingContext(ctx, md)
72 |
73 | c, err := s.GRPC.PutEntry(ctx)
74 | if err != nil {
75 | return nil, err
76 | }
77 |
78 | chunk := make([]byte, chunkSize)
79 | bar := progressbar.DefaultBytes(int64(size))
80 | for {
81 | n, err := r.Read(chunk)
82 | if err == io.EOF {
83 | break
84 | } else if err != nil {
85 | return nil, err
86 | }
87 | req := pb.PutEntryRequest{
88 | Chunk: &pb.Chunk{
89 | Data: chunk[:n],
90 | },
91 | }
92 | err = c.Send(&req)
93 | if err != nil {
94 | return nil, err
95 | }
96 | bar.Add(n)
97 | }
98 | bar.Finish()
99 | log.Debugf(ctx, "done sending chunks")
100 |
101 | res, err := c.CloseAndRecv()
102 | if err != nil {
103 | return nil, err
104 | }
105 | if len(res.GetMetadata().GetDigests()) != 1 {
106 | return utils.Digest{}, fmt.Errorf("expected 1 digest, got %d", len(res.GetMetadata().GetDigests()))
107 | }
108 | log.Infof(ctx, "put entry: %v", res)
109 |
110 | digest := utils.DigestFromProto(res.Metadata.Digests[0])
111 |
112 | return digest, nil
113 | }
114 |
115 | func (s Remote) Has(ctx context.Context, digest utils.Digest) (bool, error) {
116 | log.Debugf(ctx, "checking existence of %s", utils.DigestForLog(digest))
117 | md := metadata.New(nil)
118 | md.Set(APIKeyHeader, s.APIKey)
119 | ctx = metadata.NewOutgoingContext(ctx, md)
120 |
121 | req := pb.GetEntryMetadataRequest{
122 | Digest: utils.DigestToProto(digest),
123 | }
124 | res, err := s.GRPC.GetEntryMetadata(ctx, &req)
125 | if grpc.Code(err) == codes.NotFound {
126 | log.Debugf(ctx, "entry not found: %s", err)
127 | return false, nil
128 | } else if err != nil {
129 | log.Debugf(ctx, "error checking entry existence: %s", err)
130 | return false, err
131 | }
132 |
133 | ok := len(res.GetMetadata().GetDigests()) > 0
134 | log.Debugf(ctx, "entry found: %v", res)
135 |
136 | return ok, nil
137 | }
138 |
--------------------------------------------------------------------------------
/flake.lock:
--------------------------------------------------------------------------------
1 | {
2 | "nodes": {
3 | "flake-utils": {
4 | "inputs": {
5 | "systems": "systems"
6 | },
7 | "locked": {
8 | "lastModified": 1731533236,
9 | "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
10 | "owner": "numtide",
11 | "repo": "flake-utils",
12 | "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
13 | "type": "github"
14 | },
15 | "original": {
16 | "owner": "numtide",
17 | "repo": "flake-utils",
18 | "type": "github"
19 | }
20 | },
21 | "flake-utils_2": {
22 | "inputs": {
23 | "systems": "systems_2"
24 | },
25 | "locked": {
26 | "lastModified": 1694529238,
27 | "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=",
28 | "owner": "numtide",
29 | "repo": "flake-utils",
30 | "rev": "ff7b65b44d01cf9ba6a71320833626af21126384",
31 | "type": "github"
32 | },
33 | "original": {
34 | "owner": "numtide",
35 | "repo": "flake-utils",
36 | "type": "github"
37 | }
38 | },
39 | "gomod2nix": {
40 | "inputs": {
41 | "flake-utils": "flake-utils_2",
42 | "nixpkgs": "nixpkgs"
43 | },
44 | "locked": {
45 | "lastModified": 1745875161,
46 | "narHash": "sha256-0YkWCS13jpoo3+sX/3kcgdxBNt1VZTmvF+FhZb4rFKI=",
47 | "owner": "nix-community",
48 | "repo": "gomod2nix",
49 | "rev": "2cbd7fdd6eeab65c494cc426e18f4e4d2a5e35c0",
50 | "type": "github"
51 | },
52 | "original": {
53 | "owner": "nix-community",
54 | "repo": "gomod2nix",
55 | "type": "github"
56 | }
57 | },
58 | "nixpkgs": {
59 | "locked": {
60 | "lastModified": 1658285632,
61 | "narHash": "sha256-zRS5S/hoeDGUbO+L95wXG9vJNwsSYcl93XiD0HQBXLk=",
62 | "owner": "NixOS",
63 | "repo": "nixpkgs",
64 | "rev": "5342fc6fb59d0595d26883c3cadff16ce58e44f3",
65 | "type": "github"
66 | },
67 | "original": {
68 | "owner": "NixOS",
69 | "ref": "master",
70 | "repo": "nixpkgs",
71 | "type": "github"
72 | }
73 | },
74 | "nixpkgs_2": {
75 | "locked": {
76 | "lastModified": 1688392541,
77 | "narHash": "sha256-lHrKvEkCPTUO+7tPfjIcb7Trk6k31rz18vkyqmkeJfY=",
78 | "owner": "NixOS",
79 | "repo": "nixpkgs",
80 | "rev": "ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b",
81 | "type": "github"
82 | },
83 | "original": {
84 | "owner": "NixOS",
85 | "ref": "nixos-22.11",
86 | "repo": "nixpkgs",
87 | "type": "github"
88 | }
89 | },
90 | "root": {
91 | "inputs": {
92 | "flake-utils": "flake-utils",
93 | "gomod2nix": "gomod2nix",
94 | "nixpkgs": "nixpkgs_2"
95 | }
96 | },
97 | "systems": {
98 | "locked": {
99 | "lastModified": 1681028828,
100 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
101 | "owner": "nix-systems",
102 | "repo": "default",
103 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
104 | "type": "github"
105 | },
106 | "original": {
107 | "owner": "nix-systems",
108 | "repo": "default",
109 | "type": "github"
110 | }
111 | },
112 | "systems_2": {
113 | "locked": {
114 | "lastModified": 1681028828,
115 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
116 | "owner": "nix-systems",
117 | "repo": "default",
118 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
119 | "type": "github"
120 | },
121 | "original": {
122 | "owner": "nix-systems",
123 | "repo": "default",
124 | "type": "github"
125 | }
126 | }
127 | },
128 | "root": "root",
129 | "version": 7
130 | }
131 |
--------------------------------------------------------------------------------
/cmd/ent-server/raw.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2021 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package main
17 |
18 | import (
19 | "fmt"
20 | "io/ioutil"
21 | "net/http"
22 |
23 | "github.com/gin-gonic/gin"
24 | "github.com/google/ent/log"
25 | "github.com/google/ent/utils"
26 | )
27 |
28 | func rawGetHandler(c *gin.Context) {
29 | ctx := c
30 | log.Infof(ctx, "rawGetHandler")
31 |
32 | accessItem := &LogItemGet{
33 | LogItem: BaseLogItem(c),
34 | Source: SourceRaw,
35 | }
36 | defer LogGet(ctx, accessItem)
37 |
38 | apiKey := getAPIKey(c)
39 | log.Debugf(ctx, "apiKey: %q", redact(apiKey))
40 | user := apiKeyToUser[apiKey]
41 | if user == nil {
42 | log.Warningf(ctx, "invalid API key: %q", apiKey)
43 | c.AbortWithStatus(http.StatusForbidden)
44 | return
45 | }
46 | log.Debugf(ctx, "user: %q %d", user.Name, user.ID)
47 | log.Debugf(ctx, "perms: read:%v write:%v", user.CanRead, user.CanWrite)
48 | if !user.CanRead {
49 | log.Warningf(ctx, "user %d does not have read permission", user.ID)
50 | c.AbortWithStatus(http.StatusForbidden)
51 | return
52 | }
53 | accessItem.UserID = user.ID
54 |
55 | rawDigest := c.Param("digest")
56 | log.Infof(ctx, "rawDigest: %q", rawDigest)
57 |
58 | digest, err := utils.ParseDigest(rawDigest)
59 | if err != nil {
60 | log.Warningf(ctx, "could not parse digest: %s", err)
61 | c.AbortWithStatus(http.StatusNotFound)
62 | return
63 | }
64 | log.Infof(ctx, "digest: %s", utils.DigestForLog(digest))
65 |
66 | accessItem.Digest = append(accessItem.Digest, string(digest))
67 |
68 | target := digest
69 |
70 | nodeRaw, err := blobStore.Get(ctx, target)
71 | if err != nil {
72 | log.Warningf(ctx, "could not get blob %s: %s", target, err)
73 | accessItem.NotFound = append(accessItem.NotFound, string(digest))
74 | c.AbortWithStatus(http.StatusNotFound)
75 | return
76 | }
77 | accessItem.Found = append(accessItem.Found, string(digest))
78 |
79 | c.Data(http.StatusOK, "text/plain; charset=utf-8", nodeRaw)
80 | }
81 |
82 | func rawPutHandler(c *gin.Context) {
83 | ctx := c
84 |
85 | accessItem := &LogItemPut{
86 | LogItem: BaseLogItem(c),
87 | Source: SourceRaw,
88 | }
89 | defer LogPut(ctx, accessItem)
90 |
91 | apiKey := getAPIKey(c)
92 | user := apiKeyToUser[apiKey]
93 | if user == nil {
94 | log.Warningf(ctx, "invalid API key: %q", apiKey)
95 | c.AbortWithStatus(http.StatusForbidden)
96 | return
97 | }
98 | if !user.CanWrite {
99 | log.Warningf(ctx, "user %d does not have write permission", user.ID)
100 | c.AbortWithStatus(http.StatusForbidden)
101 | return
102 | }
103 | accessItem.UserID = user.ID
104 |
105 | nodeRaw, err := ioutil.ReadAll(c.Request.Body)
106 | if err != nil {
107 | log.Warningf(ctx, "could not read node: %s", err)
108 | c.AbortWithStatus(http.StatusInternalServerError)
109 | return
110 | }
111 |
112 | h, err := blobStore.Put(ctx, nodeRaw)
113 | accessItem.Digest = append(accessItem.Digest, string(h))
114 | if err != nil {
115 | log.Errorf(ctx, "could not put blob: %s", err)
116 | accessItem.NotCreated = append(accessItem.NotCreated, string(h))
117 | c.AbortWithStatus(http.StatusInternalServerError)
118 | return
119 | }
120 | accessItem.Created = append(accessItem.Created, string(h))
121 |
122 | location := fmt.Sprintf("/raw/%s", h)
123 | log.Infof(ctx, "new object location: %q", location)
124 |
125 | c.Header("Location", location)
126 | // https://stackoverflow.com/questions/797834/should-a-restful-put-operation-return-something
127 | c.Status(http.StatusCreated)
128 | }
129 |
--------------------------------------------------------------------------------
/cmd/ent-server/bigquery.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2022 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package main
17 |
18 | import (
19 | "context"
20 | "io/ioutil"
21 | "time"
22 |
23 | "cloud.google.com/go/bigquery"
24 | "github.com/google/ent/log"
25 | "google.golang.org/api/option"
26 | )
27 |
28 | type LogItem struct {
29 | Timestamp time.Time
30 | IP string
31 | UserAgent string
32 | RequestMethod string
33 | RequestURI string
34 | }
35 |
36 | type LogItemGet struct {
37 | LogItem
38 | UserID int64
39 | Source string
40 | Digest []string
41 | Found []string
42 | NotFound []string
43 | }
44 |
45 | type LogItemPut struct {
46 | LogItem
47 | UserID int64
48 | Source string
49 | Digest []string
50 | Created []string
51 | NotCreated []string
52 | }
53 |
54 | const (
55 | SourceAPI = "api"
56 | SourceRaw = "raw"
57 | SourceWeb = "web"
58 |
59 | logsGetTable = "logs_get"
60 | logsPutTable = "logs_put"
61 | )
62 |
63 | var bigqueryDataset *bigquery.Dataset
64 |
65 | func InitBigquery(ctx context.Context, projectID string, dataset string) {
66 | opts := []option.ClientOption{}
67 | c, _ := ioutil.ReadFile("./credentials.json")
68 | if len(c) > 0 {
69 | log.Infof(ctx, "using credentials file")
70 | opts = append(opts, option.WithCredentialsJSON(c))
71 | } else {
72 | log.Infof(ctx, "using application default credentials")
73 | }
74 | bigqueryClient, err := bigquery.NewClient(ctx, projectID, opts...)
75 | if err != nil {
76 | log.Errorf(ctx, "could not create bigquery client: %v", err)
77 | }
78 | bigqueryDataset = bigqueryClient.Dataset(dataset)
79 |
80 | ensureTable(ctx, logsGetTable, LogItemGet{})
81 | ensureTable(ctx, logsPutTable, LogItemPut{})
82 | }
83 |
84 | func ensureTable(ctx context.Context, name string, st interface{}) {
85 | table := bigqueryDataset.Table(name)
86 |
87 | tableSchema, err := bigquery.InferSchema(st)
88 | if err != nil {
89 | log.Errorf(ctx, "could not infer schema: %v", err)
90 | return
91 | }
92 | tableSchema = tableSchema.Relax()
93 |
94 | tableMetadata, err := table.Metadata(ctx)
95 | if err != nil {
96 | log.Errorf(ctx, "could not get table metadata: %v", err)
97 | err = table.Create(ctx, &bigquery.TableMetadata{
98 | Name: name,
99 | Schema: tableSchema,
100 | })
101 | if err != nil {
102 | log.Errorf(ctx, "could not create table %q: %v", name, err)
103 | return
104 | }
105 | log.Infof(ctx, "created table %q", name)
106 | } else {
107 | log.Infof(ctx, "table %q already exists: %+v", name, tableMetadata)
108 | if len(tableMetadata.Schema) == len(tableSchema) {
109 | log.Infof(ctx, "table schema is up to date")
110 | return
111 | }
112 | log.Infof(ctx, "table schema differs; trying to update table %q", name)
113 | newMetadata, err := table.Update(ctx, bigquery.TableMetadataToUpdate{
114 | Schema: tableSchema,
115 | }, "")
116 | if err != nil {
117 | log.Errorf(ctx, "could not update table %q: %v", name, err)
118 | return
119 | }
120 | log.Infof(ctx, "updated table %q: %+v", name, newMetadata)
121 | }
122 | }
123 |
124 | func LogGet(ctx context.Context, v *LogItemGet) {
125 | logAccess(ctx, "logs_get", v)
126 | }
127 |
128 | func LogPut(ctx context.Context, v *LogItemPut) {
129 | logAccess(ctx, "logs_put", v)
130 | }
131 |
132 | func logAccess(ctx context.Context, table string, v interface{}) {
133 | if bigqueryDataset == nil {
134 | return
135 | }
136 | t := bigqueryDataset.Table(table)
137 | log.Debugf(ctx, "logging access: %+v", v)
138 | err := t.Inserter().Put(ctx, v)
139 | if err != nil {
140 | log.Errorf(ctx, "could not insert into bigquery: %v", err)
141 | return
142 | }
143 | log.Debugf(ctx, "logged access: %+v", v)
144 | }
145 |
--------------------------------------------------------------------------------
/utils/lib.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2021 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package utils
17 |
18 | import (
19 | "encoding/hex"
20 | "fmt"
21 | "strings"
22 |
23 | "github.com/google/ent/api"
24 | pb "github.com/google/ent/proto"
25 | "github.com/ipfs/go-cid"
26 | "github.com/multiformats/go-multihash"
27 | )
28 |
29 | type Digest = multihash.Multihash
30 | type DigestArray [64]byte
31 |
32 | const hash = multihash.SHA2_256
33 |
34 | func ParseDigest(s string) (Digest, error) {
35 | digest, err := multihash.FromHexString(s)
36 | if err == nil {
37 | return Digest(digest), nil
38 | } else {
39 | digest, err := multihash.FromB58String(s)
40 | if err == nil {
41 | return Digest(digest), nil
42 | } else {
43 | parts := strings.Split(s, ":")
44 | if len(parts) == 2 {
45 | code, ok := multihash.Names[strings.ToLower(parts[0])]
46 | if !ok {
47 | return nil, fmt.Errorf("invalid digest code: %q", parts[0])
48 | }
49 | ss, err := hex.DecodeString(parts[1])
50 | if err != nil {
51 | return nil, err
52 | }
53 | digest, err = multihash.Encode(ss, code)
54 | if err != nil {
55 | return nil, err
56 | }
57 | return Digest(digest), err
58 | } else {
59 | return nil, fmt.Errorf("invalid digest: %v", err)
60 | }
61 | }
62 | }
63 | }
64 |
65 | func DigestToHumanString(d Digest) string {
66 | m, err := multihash.Decode(d)
67 | if err != nil {
68 | panic(err)
69 | }
70 | codeString := multihash.Codes[m.Code]
71 | return fmt.Sprintf(codeString + ":" + hex.EncodeToString(m.Digest))
72 | }
73 |
74 | func DigestToProto(d Digest) *pb.Digest {
75 | m, err := multihash.Decode(d)
76 | if err != nil {
77 | panic(err)
78 | }
79 | return &pb.Digest{
80 | Code: m.Code,
81 | Digest: m.Digest,
82 | }
83 | }
84 |
85 | func DigestFromProto(d *pb.Digest) Digest {
86 | b, err := multihash.Encode(d.Digest, d.Code)
87 | if err != nil {
88 | panic(err)
89 | }
90 | return Digest(b)
91 | }
92 |
93 | func ComputeDigest(b []byte) Digest {
94 | d, err := multihash.Sum(b, hash, -1)
95 | if err != nil {
96 | panic(err)
97 | }
98 | return Digest(d)
99 | }
100 |
101 | type NodeID struct {
102 | Root cid.Cid
103 | Path Path
104 | }
105 |
106 | // Convert to a fixed size array. Only to be used for in-memory caching.
107 | func DigestToArray(digest multihash.Multihash) DigestArray {
108 | var a DigestArray
109 | if len(digest) > len(a) {
110 | panic(fmt.Sprintf("invalid digest length; got %d, want > %d", len(digest), len(a)))
111 | }
112 | copy(a[:], digest[:])
113 | return a
114 | }
115 |
116 | func FormatDigest(digest Digest, format string) string {
117 | switch format {
118 | case "human":
119 | return DigestToHumanString(digest)
120 | case "hex":
121 | return digest.HexString()
122 | case "b58":
123 | return digest.B58String()
124 | default:
125 | return "-"
126 | }
127 | }
128 |
129 | func DigestForLog(digest Digest) string {
130 | return digest.B58String()
131 | }
132 |
133 | func DigestToApi(digest Digest) api.HexDigests {
134 | mh, err := multihash.Decode(digest)
135 | if err != nil {
136 | panic(err)
137 | }
138 | switch mh.Code {
139 | case multihash.SHA2_256:
140 | return api.HexDigests{Sha2_256: hex.EncodeToString(mh.Digest)}
141 | case multihash.SHA2_512:
142 | return api.HexDigests{Sha2_512: hex.EncodeToString(mh.Digest)}
143 | case multihash.SHA3_256:
144 | return api.HexDigests{Sha3_256: hex.EncodeToString(mh.Digest)}
145 | case multihash.SHA3_384:
146 | return api.HexDigests{Sha3_384: hex.EncodeToString(mh.Digest)}
147 | case multihash.SHA3_512:
148 | return api.HexDigests{Sha3_512: hex.EncodeToString(mh.Digest)}
149 | default:
150 | panic(fmt.Sprintf("unsupported hash code: %v", mh.Code))
151 | }
152 | }
153 |
--------------------------------------------------------------------------------
/cmd/ent/cmd/put.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2022 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package cmd
17 |
18 | import (
19 | "bytes"
20 | "context"
21 | "fmt"
22 | "io/ioutil"
23 | "os"
24 |
25 | "github.com/fatih/color"
26 | "github.com/google/ent/cmd/ent/config"
27 | "github.com/google/ent/cmd/ent/remote"
28 | "github.com/google/ent/log"
29 | "github.com/google/ent/nodeservice"
30 | "github.com/google/ent/utils"
31 | "github.com/ipfs/go-cid"
32 | "github.com/multiformats/go-multihash"
33 | "github.com/spf13/cobra"
34 | "github.com/tonistiigi/units"
35 | )
36 |
37 | var (
38 | remoteFlag string
39 | digestFormatFlag string
40 | porcelainFlag bool
41 | )
42 |
43 | var putCmd = &cobra.Command{
44 | Use: "put [filename]",
45 | Args: cobra.MaximumNArgs(1),
46 | Run: func(cmd *cobra.Command, args []string) {
47 | filename := ""
48 | if len(args) > 0 {
49 | filename = args[0]
50 | }
51 | ctx := context.Background()
52 | if filename == "" {
53 | err := putStdin()
54 | if err != nil {
55 | log.Criticalf(ctx, "could not read from stdin: %v", err)
56 | os.Exit(1)
57 | }
58 | } else {
59 | _, err := traverseFileOrDir(filename, put)
60 | if err != nil {
61 | log.Criticalf(ctx, "could not traverse file: %v", err)
62 | os.Exit(1)
63 | }
64 | }
65 |
66 | },
67 | }
68 |
69 | func putStdin() error {
70 | data, err := ioutil.ReadAll(os.Stdin)
71 | if err != nil {
72 | return fmt.Errorf("could not read stdin: %v", err)
73 | }
74 | digest := utils.ComputeDigest(data)
75 | link := cid.NewCidV1(utils.TypeRaw, multihash.Multihash(digest))
76 | return put(data, link, "-")
77 | }
78 |
79 | // func putFile(filename string) error {
80 | // data, err := os.ReadFile(filename)
81 | // if err != nil {
82 | // return fmt.Errorf("could not read file %q: %v", filename, err)
83 | // }
84 | // return putData(data)
85 | // }
86 |
87 | func put(b []byte, link cid.Cid, name string) error {
88 | ctx := context.Background()
89 | config := config.ReadConfig()
90 | r := config.Remotes[0]
91 | if remoteFlag != "" {
92 | var err error
93 | r, err = remote.GetRemote(config, remoteFlag)
94 | if err != nil {
95 | return fmt.Errorf("could not use remote: %v", err)
96 | }
97 | }
98 | nodeService := remote.GetObjectStore(r)
99 | size := len(b)
100 |
101 | switch link.Type() {
102 | case utils.TypeRaw:
103 | digest := utils.Digest(link.Hash())
104 | digestString := utils.FormatDigest(digest, digestFormatFlag)
105 | marker := color.GreenString("-")
106 | if exists(nodeService, digest) {
107 | marker = color.GreenString("✓")
108 | } else {
109 | log.Infof(ctx, "putting object %q", digestString)
110 | _, err := nodeService.Put(ctx, uint64(size), bytes.NewReader(b))
111 | if err != nil {
112 | log.Errorf(ctx, "could not put object: %v", err)
113 | return fmt.Errorf("could not put object: %v", err)
114 | }
115 | marker = color.BlueString("↑")
116 | }
117 | if porcelainFlag {
118 | fmt.Printf("%s\n", digestString)
119 | } else {
120 | fmt.Printf("%s [%s %s] %s %.0f\n", color.YellowString(digestString), marker, r.Name, name, units.Bytes(size))
121 | }
122 | return nil
123 | case utils.TypeDAG:
124 | return nil
125 | default:
126 | return fmt.Errorf("unknown type: %v", link.Type())
127 | }
128 | }
129 |
130 | func exists(nodeService nodeservice.ObjectGetter, digest utils.Digest) bool {
131 | ctx := context.Background()
132 | ok, err := nodeService.Has(ctx, digest)
133 | if err != nil {
134 | log.Errorf(ctx, "could not check existence of %q: %v", digest, err)
135 | return false
136 | }
137 | return ok
138 | }
139 |
140 | func init() {
141 | putCmd.PersistentFlags().StringVar(&remoteFlag, "remote", "", "remote")
142 | putCmd.PersistentFlags().StringVar(&digestFormatFlag, "digest-format", "b58", "format [human, hex, b58]")
143 | putCmd.PersistentFlags().BoolVar(&porcelainFlag, "porcelain", false, "porcelain output (parseable by machines)")
144 | }
145 |
--------------------------------------------------------------------------------
/docs/code-of-conduct.md:
--------------------------------------------------------------------------------
1 | # Code of Conduct
2 |
3 | ## Our Pledge
4 |
5 | In the interest of fostering an open and welcoming environment, we as
6 | contributors and maintainers pledge to making participation in our project and
7 | our community a harassment-free experience for everyone, regardless of age, body
8 | size, disability, ethnicity, gender identity and expression, level of
9 | experience, education, socio-economic status, nationality, personal appearance,
10 | race, religion, or sexual identity and orientation.
11 |
12 | ## Our Standards
13 |
14 | Examples of behavior that contributes to creating a positive environment
15 | include:
16 |
17 | * Using welcoming and inclusive language
18 | * Being respectful of differing viewpoints and experiences
19 | * Gracefully accepting constructive criticism
20 | * Focusing on what is best for the community
21 | * Showing empathy towards other community members
22 |
23 | Examples of unacceptable behavior by participants include:
24 |
25 | * The use of sexualized language or imagery and unwelcome sexual attention or
26 | advances
27 | * Trolling, insulting/derogatory comments, and personal or political attacks
28 | * Public or private harassment
29 | * Publishing others' private information, such as a physical or electronic
30 | address, without explicit permission
31 | * Other conduct which could reasonably be considered inappropriate in a
32 | professional setting
33 |
34 | ## Our Responsibilities
35 |
36 | Project maintainers are responsible for clarifying the standards of acceptable
37 | behavior and are expected to take appropriate and fair corrective action in
38 | response to any instances of unacceptable behavior.
39 |
40 | Project maintainers have the right and responsibility to remove, edit, or reject
41 | comments, commits, code, wiki edits, issues, and other contributions that are
42 | not aligned to this Code of Conduct, or to ban temporarily or permanently any
43 | contributor for other behaviors that they deem inappropriate, threatening,
44 | offensive, or harmful.
45 |
46 | ## Scope
47 |
48 | This Code of Conduct applies both within project spaces and in public spaces
49 | when an individual is representing the project or its community. Examples of
50 | representing a project or community include using an official project e-mail
51 | address, posting via an official social media account, or acting as an appointed
52 | representative at an online or offline event. Representation of a project may be
53 | further defined and clarified by project maintainers.
54 |
55 | This Code of Conduct also applies outside the project spaces when the Project
56 | Steward has a reasonable belief that an individual's behavior may have a
57 | negative impact on the project or its community.
58 |
59 | ## Conflict Resolution
60 |
61 | We do not believe that all conflict is bad; healthy debate and disagreement
62 | often yield positive results. However, it is never okay to be disrespectful or
63 | to engage in behavior that violates the project’s code of conduct.
64 |
65 | If you see someone violating the code of conduct, you are encouraged to address
66 | the behavior directly with those involved. Many issues can be resolved quickly
67 | and easily, and this gives people more control over the outcome of their
68 | dispute. If you are unable to resolve the matter for any reason, or if the
69 | behavior is threatening or harassing, report it. We are dedicated to providing
70 | an environment where participants feel welcome and safe.
71 |
72 | Reports should be directed to *[PROJECT STEWARD NAME(s) AND EMAIL(s)]*, the
73 | Project Steward(s) for *[PROJECT NAME]*. It is the Project Steward’s duty to
74 | receive and address reported violations of the code of conduct. They will then
75 | work with a committee consisting of representatives from the Open Source
76 | Programs Office and the Google Open Source Strategy team. If for any reason you
77 | are uncomfortable reaching out to the Project Steward, please email
78 | opensource@google.com.
79 |
80 | We will investigate every complaint, but you may not receive a direct response.
81 | We will use our discretion in determining when and how to follow up on reported
82 | incidents, which may range from not taking action to permanent expulsion from
83 | the project and project-sponsored spaces. We will notify the accused of the
84 | report and provide them an opportunity to discuss it before any action is taken.
85 | The identity of the reporter will be omitted from the details of the report
86 | supplied to the accused. In potentially harmful situations, such as ongoing
87 | harassment or threats to anyone's safety, we may take action without notice.
88 |
89 | ## Attribution
90 |
91 | This Code of Conduct is adapted from the Contributor Covenant, version 1.4,
92 | available at
93 | https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
94 |
--------------------------------------------------------------------------------
/schema/schema_test.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2022 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package schema
17 |
18 | import (
19 | "context"
20 | "reflect"
21 | "testing"
22 |
23 | "github.com/google/ent/datastore"
24 | "github.com/google/ent/objectstore"
25 | "github.com/google/ent/utils"
26 | "github.com/ipfs/go-cid"
27 | "github.com/multiformats/go-multihash"
28 | )
29 |
30 | func TestGetString(t *testing.T) {
31 | o := objectstore.Store{
32 | Inner: datastore.InMemory{
33 | Inner: make(map[string][]byte),
34 | },
35 | }
36 |
37 | stringFieldDigest, err := o.Put(context.Background(), []byte("hello"))
38 | if err != nil {
39 | t.Fatalf("failed to put string: %v", err)
40 | }
41 |
42 | s, err := GetString(o, stringFieldDigest)
43 | if err != nil {
44 | t.Fatalf("unexpected error: %v", err)
45 | }
46 | if s != "hello" {
47 | t.Fatalf("unexpected string: %v", s)
48 | }
49 | }
50 |
51 | func TestGetUint32(t *testing.T) {
52 | o := objectstore.Store{
53 | Inner: datastore.InMemory{
54 | Inner: make(map[string][]byte),
55 | },
56 | }
57 |
58 | uintFieldDigest, err := o.Put(context.Background(), []byte{0x00, 0x00, 0x07, 0xC4})
59 | if err != nil {
60 | t.Fatalf("failed to put uint32: %v", err)
61 | }
62 |
63 | s, err := GetUint32(o, uintFieldDigest)
64 | if err != nil {
65 | t.Fatalf("unexpected error: %v", err)
66 | }
67 | if s != 1988 {
68 | t.Fatalf("unexpected value: %v", s)
69 | }
70 | }
71 |
72 | func TestGetStruct(t *testing.T) {
73 | o := objectstore.Store{
74 | Inner: datastore.InMemory{
75 | Inner: make(map[string][]byte),
76 | },
77 | }
78 |
79 | stringFieldDigest, err := o.Put(context.Background(), []byte("hello"))
80 | if err != nil {
81 | t.Fatalf("failed to put string: %v", err)
82 | }
83 | uintFieldDigest, err := o.Put(context.Background(), []byte{0x00, 0x00, 0x07, 0xC4})
84 | if err != nil {
85 | t.Fatalf("failed to put string: %v", err)
86 | }
87 | node := &utils.DAGNode{
88 | Links: []cid.Cid{
89 | cid.NewCidV1(cid.Raw, multihash.Multihash(uintFieldDigest)),
90 | cid.NewCidV1(cid.Raw, multihash.Multihash(stringFieldDigest)),
91 | cid.NewCidV1(cid.Raw, multihash.Multihash(uintFieldDigest)),
92 | cid.NewCidV1(cid.Raw, multihash.Multihash(uintFieldDigest)),
93 | },
94 | }
95 | nodeBytes, err := utils.SerializeDAGNode(node)
96 | if err != nil {
97 | t.Fatalf("failed to serialize node: %v", err)
98 | }
99 | nodeDigest, err := o.Put(context.Background(), nodeBytes)
100 | if err != nil {
101 | t.Fatalf("failed to put string: %v", err)
102 | }
103 |
104 | v := &Field{}
105 |
106 | err = GetStruct(o, nodeDigest, v)
107 | if err != nil {
108 | t.Fatalf("unexpected error: %v", err)
109 | }
110 |
111 | // if v.FieldID != 1988 {
112 | // t.Fatalf("unexpected value: %v", v.FieldID)
113 | // }
114 | // if v.Name != "hello" {
115 | // t.Fatalf("unexpected value: %v", v.Name)
116 | // }
117 | }
118 |
119 | type T struct {
120 | A uint32 `ent:"0"`
121 | B string `ent:"1"`
122 | C []uint32 `ent:"2"`
123 | D []string `ent:"3"`
124 | E U `ent:"4"`
125 | F []U `ent:"5"`
126 | G uint64 `ent:"6"`
127 | }
128 |
129 | type U struct {
130 | A uint32 `ent:"0"`
131 | B string `ent:"1"`
132 | }
133 |
134 | func TestRoundTrip(t *testing.T) {
135 | o := objectstore.Store{
136 | Inner: datastore.InMemory{
137 | Inner: make(map[string][]byte),
138 | },
139 | }
140 |
141 | vv := []*T{
142 | {},
143 | {
144 | A: 123,
145 | B: "hello",
146 | C: []uint32{0, 1, 2},
147 | D: []string{"a", "b", "c"},
148 | E: U{
149 | A: 456,
150 | B: "world",
151 | },
152 | F: []U{
153 | {
154 | A: 78,
155 | B: "mondo",
156 | },
157 | {
158 | A: 90,
159 | B: "monde",
160 | },
161 | },
162 | G: 123456789,
163 | },
164 | }
165 |
166 | for _, v := range vv {
167 | h, err := PutStruct(o, v)
168 | if err != nil {
169 | t.Fatalf("unexpected error: %v", err)
170 | }
171 |
172 | v1 := &T{}
173 |
174 | err = GetStruct(o, h, v1)
175 | if err != nil {
176 | t.Fatalf("unexpected error: %v", err)
177 | }
178 |
179 | if !reflect.DeepEqual(v, v1) {
180 | t.Fatalf("unexpected value: %v, expected: %v", v1, v)
181 | }
182 | }
183 | }
184 |
--------------------------------------------------------------------------------
/cmd/ent/cmd/get.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2022 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package cmd
17 |
18 | import (
19 | "bytes"
20 | "context"
21 | "encoding/json"
22 | "fmt"
23 | "io"
24 | "net/http"
25 | "os"
26 |
27 | "github.com/google/ent/api"
28 | "github.com/google/ent/cmd/ent/config"
29 | "github.com/google/ent/log"
30 | "github.com/google/ent/utils"
31 | "github.com/multiformats/go-multihash"
32 | "github.com/spf13/cobra"
33 | )
34 |
35 | var (
36 | urlFlag string
37 | outFlag string
38 | digestFlag string
39 | )
40 |
41 | var getCmd = &cobra.Command{
42 | Use: "get",
43 | Args: cobra.ExactArgs(0),
44 | Run: func(cmd *cobra.Command, args []string) {
45 | ctx := context.Background()
46 | expectedDigest, err := utils.ParseDigest(digestFlag)
47 | if err != nil {
48 | log.Criticalf(ctx, "parse digest: %v", err)
49 | os.Exit(1)
50 | }
51 |
52 | parsedDigest, err := multihash.Decode(expectedDigest)
53 | if err != nil {
54 | log.Criticalf(ctx, "decode digest: %v", err)
55 | os.Exit(1)
56 | }
57 | fmt.Printf("parsed digest: %+v\n", parsedDigest)
58 |
59 | // Make API request to get entry metadata and mirrors
60 | resp, err := getEntry(ctx, expectedDigest)
61 | if err != nil {
62 | log.Criticalf(ctx, "get entry request failed: %v", err)
63 | os.Exit(1)
64 | }
65 | log.Debugf(ctx, "got entry response: metadata=%+v mirrors=%+v", resp.Metadata, resp.Mirrors)
66 | fmt.Printf("size: %v\n", resp.Metadata.LengthBytes)
67 | for _, mirror := range resp.Mirrors {
68 | log.Debugf(ctx, "mirror: %v", mirror)
69 | object, err := http.Get(mirror.URL)
70 | if err != nil {
71 | log.Criticalf(ctx, "get mirror: %v", err)
72 | os.Exit(1)
73 | }
74 | defer object.Body.Close()
75 |
76 | // Read the full response body
77 | body, err := io.ReadAll(object.Body)
78 | if err != nil {
79 | log.Criticalf(ctx, "read mirror body: %v", err)
80 | os.Exit(1)
81 | }
82 |
83 | actualDigest, err := multihash.Sum(body, parsedDigest.Code, -1)
84 | if err != nil {
85 | log.Criticalf(ctx, "compute digest: %v", err)
86 | os.Exit(1)
87 | }
88 |
89 | log.Debugf(ctx, "expected digest: %v", utils.DigestToHumanString(expectedDigest))
90 | log.Debugf(ctx, "actual digest: %v", utils.DigestToHumanString(actualDigest))
91 |
92 | // Verify the digest matches
93 | if !bytes.Equal(actualDigest, expectedDigest) {
94 | log.Criticalf(ctx, "digest mismatch: got %v, want %v", actualDigest, expectedDigest)
95 | continue
96 | }
97 | log.Debugf(ctx, "digest matches")
98 |
99 | if outFlag != "" {
100 | os.WriteFile(outFlag, body, 0644)
101 | } else {
102 | log.Infof(ctx, "no output file specified. use --out to write to a file")
103 | }
104 |
105 | os.Exit(0)
106 | }
107 | os.Exit(0)
108 | },
109 | }
110 |
111 | func getEntry(ctx context.Context, digest utils.Digest) (*api.GetEntryResponse, error) {
112 | req := api.GetEntryRequest{
113 | Digests: utils.DigestToApi(digest),
114 | }
115 | log.Debugf(ctx, "sending request: %+v", req)
116 | var resp api.GetEntryResponse
117 | config := config.ReadConfig()
118 | client := &http.Client{}
119 | jsonData, err := json.Marshal(req)
120 | if err != nil {
121 | return nil, err
122 | }
123 |
124 | httpReq, err := http.NewRequestWithContext(
125 | ctx,
126 | http.MethodPost,
127 | config.Remotes[0].URL+"/"+api.GET_ENTRY_METHOD_ID,
128 | bytes.NewBuffer(jsonData),
129 | )
130 | if err != nil {
131 | return nil, err
132 | }
133 | httpReq.Header.Set("Content-Type", "application/json")
134 |
135 | httpResp, err := client.Do(httpReq)
136 | if err != nil {
137 | return nil, err
138 | }
139 | defer httpResp.Body.Close()
140 |
141 | if httpResp.StatusCode != http.StatusOK {
142 | return nil, fmt.Errorf("server returned status %v", httpResp.Status)
143 | }
144 |
145 | err = json.NewDecoder(httpResp.Body).Decode(&resp)
146 | if err != nil {
147 | return nil, err
148 | }
149 |
150 | return &api.GetEntryResponse{
151 | Metadata: resp.Metadata,
152 | Mirrors: resp.Mirrors,
153 | }, nil
154 | }
155 |
156 | func init() {
157 | getCmd.PersistentFlags().StringVar(&digestFlag, "digest", "", "digest of the object to fetch")
158 | getCmd.PersistentFlags().StringVar(&urlFlag, "url", "", "optional URL of the object to fetch")
159 | getCmd.PersistentFlags().StringVar(&outFlag, "out", "", "optional output file")
160 | }
161 |
--------------------------------------------------------------------------------
/cmd/ent/cmd/digest.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2022 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package cmd
17 |
18 | import (
19 | "context"
20 | "fmt"
21 | "io/ioutil"
22 | "os"
23 |
24 | "github.com/fatih/color"
25 | "github.com/google/ent/log"
26 | "github.com/google/ent/utils"
27 | "github.com/ipfs/go-cid"
28 | mbase "github.com/multiformats/go-multibase"
29 | "github.com/multiformats/go-multihash"
30 | "github.com/spf13/cobra"
31 | )
32 |
33 | var digestCmd = &cobra.Command{
34 | Use: "digest [filename]",
35 | Args: cobra.MaximumNArgs(1),
36 | Run: func(cmd *cobra.Command, args []string) {
37 | ctx := context.Background()
38 | filename := ""
39 | if len(args) > 0 {
40 | filename = args[0]
41 | }
42 | if filename == "" {
43 | _, err := digestStdin()
44 | if err != nil {
45 | log.Criticalf(ctx, "computing digest of stdin: %v", err)
46 | os.Exit(1)
47 | }
48 | } else {
49 | _, err := traverseFileOrDir(filename, print)
50 | if err != nil {
51 | log.Criticalf(ctx, "traversing file or dir: %v", err)
52 | os.Exit(1)
53 | }
54 | }
55 | },
56 | }
57 |
58 | type traverseF func([]byte, cid.Cid, string) error
59 |
60 | func print(bytes []byte, link cid.Cid, name string) error {
61 | fmt.Printf("%s", formatLink(link, name))
62 | return nil
63 | }
64 |
65 | func digestStdin() (utils.Digest, error) {
66 | data, err := ioutil.ReadAll(os.Stdin)
67 | if err != nil {
68 | return utils.Digest{}, fmt.Errorf("could not read stdin: %v", err)
69 | }
70 | return digestData(data)
71 | }
72 |
73 | func traverseFileOrDir(filename string, f traverseF) (utils.Digest, error) {
74 | info, err := os.Stat(filename)
75 | if err != nil {
76 | return utils.Digest{}, fmt.Errorf("could not stat %s: %v", filename, err)
77 | }
78 | if info.IsDir() {
79 | return traverseDir(filename, f)
80 | } else {
81 | return traverseFile(filename, f)
82 | }
83 | }
84 |
85 | func traverseDir(dirname string, f traverseF) (utils.Digest, error) {
86 | ctx := context.Background()
87 | files, err := ioutil.ReadDir(dirname)
88 | if err != nil {
89 | return utils.Digest{}, fmt.Errorf("could not read directory %s: %v", dirname, err)
90 | }
91 | links := make([]cid.Cid, 0, len(files))
92 | data := ""
93 | for _, file := range files {
94 | filename := dirname + "/" + file.Name()
95 | info, err := os.Stat(filename)
96 | if err != nil {
97 | return utils.Digest{}, fmt.Errorf("could not stat %q: %v", filename, err)
98 | }
99 | if info.IsDir() {
100 | digest, err := traverseDir(filename, f)
101 | if err != nil {
102 | return utils.Digest{}, err
103 | }
104 | links = append(links, cid.NewCidV1(utils.TypeDAG, multihash.Multihash(digest)))
105 | data += file.Name() + "\n"
106 | } else {
107 | digest, err := traverseFile(filename, f)
108 | if err != nil {
109 | return utils.Digest{}, err
110 | }
111 | links = append(links, cid.NewCidV1(utils.TypeRaw, multihash.Multihash(digest)))
112 | data += file.Name() + "\n"
113 | }
114 | }
115 | dagNode := utils.DAGNode{
116 | Links: links,
117 | Bytes: []byte(data),
118 | }
119 | log.Infof(ctx, "DAG node: %v\n", dagNode)
120 | serialized, err := utils.SerializeDAGNode(&dagNode)
121 | if err != nil {
122 | return utils.Digest{}, err
123 | }
124 | digest := utils.ComputeDigest(serialized)
125 | link := cid.NewCidV1(utils.TypeDAG, multihash.Multihash(digest))
126 | err = f(serialized, link, dirname+"/")
127 | if err != nil {
128 | log.Infof(ctx, "could not traverse directory %q: %v", dirname, err)
129 | }
130 | return digest, nil
131 | }
132 |
133 | func traverseFile(filename string, f traverseF) (utils.Digest, error) {
134 | data, err := os.ReadFile(filename)
135 | if err != nil {
136 | return utils.Digest{}, fmt.Errorf("could not read file %q: %v", filename, err)
137 | }
138 | link := cid.NewCidV1(utils.TypeRaw, multihash.Multihash(utils.ComputeDigest(data)))
139 | f(data, link, filename)
140 | return digestData(data)
141 | }
142 |
143 | func digestData(data []byte) (utils.Digest, error) {
144 | return utils.ComputeDigest(data), nil
145 | }
146 |
147 | func formatDigest(digest utils.Digest, name string) string {
148 | digestString := utils.FormatDigest(digest, digestFormatFlag)
149 | return fmt.Sprintf("%s %s\n", color.YellowString(digestString), name)
150 | }
151 |
152 | func formatLink(link cid.Cid, name string) string {
153 | linkString, err := link.StringOfBase(mbase.Base32)
154 | if err != nil {
155 | // Should never happen.
156 | panic(err)
157 | }
158 | return fmt.Sprintf("%s %s\n", color.YellowString(linkString), name)
159 | }
160 |
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/google/ent
2 |
3 | go 1.19
4 |
5 | require (
6 | cloud.google.com/go/bigquery v1.51.2
7 | cloud.google.com/go/firestore v1.10.0
8 | cloud.google.com/go/logging v1.7.0
9 | cloud.google.com/go/storage v1.30.1
10 | github.com/BurntSushi/toml v1.3.2
11 | github.com/fatih/color v1.15.0
12 | github.com/gin-contrib/cors v1.4.0
13 | github.com/gin-gonic/gin v1.9.1
14 | github.com/go-playground/assert/v2 v2.2.0
15 | github.com/go-redis/redis/v8 v8.11.5
16 | github.com/golang/protobuf v1.5.3
17 | github.com/ipfs/go-cid v0.4.1
18 | github.com/multiformats/go-multihash v0.2.3
19 | github.com/multiformats/go-varint v0.0.7
20 | github.com/spf13/cobra v1.7.0
21 | golang.org/x/net v0.11.0
22 | google.golang.org/api v0.127.0
23 | google.golang.org/protobuf v1.30.0
24 | )
25 |
26 | require (
27 | cloud.google.com/go v0.110.2 // indirect
28 | cloud.google.com/go/compute v1.20.0 // indirect
29 | cloud.google.com/go/compute/metadata v0.2.3 // indirect
30 | cloud.google.com/go/iam v1.1.0 // indirect
31 | cloud.google.com/go/longrunning v0.5.0 // indirect
32 | github.com/andybalholm/brotli v1.0.5 // indirect
33 | github.com/apache/arrow/go/v12 v12.0.1 // indirect
34 | github.com/apache/thrift v0.18.1 // indirect
35 | github.com/bytedance/sonic v1.9.1 // indirect
36 | github.com/cespare/xxhash/v2 v2.2.0 // indirect
37 | github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
38 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
39 | github.com/fsnotify/fsnotify v1.6.0 // indirect
40 | github.com/gabriel-vasile/mimetype v1.4.2 // indirect
41 | github.com/gin-contrib/sse v0.1.0 // indirect
42 | github.com/go-playground/locales v0.14.1 // indirect
43 | github.com/go-playground/universal-translator v0.18.1 // indirect
44 | github.com/go-playground/validator/v10 v10.14.1 // indirect
45 | github.com/goccy/go-json v0.10.2 // indirect
46 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
47 | github.com/golang/snappy v0.0.4 // indirect
48 | github.com/google/flatbuffers v23.5.26+incompatible // indirect
49 | github.com/google/go-cmp v0.5.9 // indirect
50 | github.com/google/s2a-go v0.1.4 // indirect
51 | github.com/google/uuid v1.3.0 // indirect
52 | github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect
53 | github.com/googleapis/gax-go/v2 v2.11.0 // indirect
54 | github.com/inconshreveable/mousetrap v1.1.0 // indirect
55 | github.com/json-iterator/go v1.1.12 // indirect
56 | github.com/klauspost/asmfmt v1.3.2 // indirect
57 | github.com/klauspost/compress v1.16.6 // indirect
58 | github.com/klauspost/cpuid/v2 v2.2.5 // indirect
59 | github.com/kr/pretty v0.3.1 // indirect
60 | github.com/leodido/go-urn v1.2.4 // indirect
61 | github.com/mattn/go-colorable v0.1.13 // indirect
62 | github.com/mattn/go-isatty v0.0.19 // indirect
63 | github.com/mattn/go-runewidth v0.0.15 // indirect
64 | github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
65 | github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
66 | github.com/minio/sha256-simd v1.0.1 // indirect
67 | github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
68 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
69 | github.com/modern-go/reflect2 v1.0.2 // indirect
70 | github.com/mr-tron/base58 v1.2.0 // indirect
71 | github.com/multiformats/go-base32 v0.1.0 // indirect
72 | github.com/multiformats/go-base36 v0.2.0 // indirect
73 | github.com/multiformats/go-multibase v0.2.0 // indirect
74 | github.com/onsi/gomega v1.27.4 // indirect
75 | github.com/pelletier/go-toml/v2 v2.0.8 // indirect
76 | github.com/pierrec/lz4/v4 v4.1.18 // indirect
77 | github.com/rivo/uniseg v0.4.4 // indirect
78 | github.com/rogpeppe/go-internal v1.10.0 // indirect
79 | github.com/schollz/progressbar/v3 v3.13.1 // indirect
80 | github.com/spaolacci/murmur3 v1.1.0 // indirect
81 | github.com/spf13/pflag v1.0.5 // indirect
82 | github.com/stretchr/testify v1.8.4 // indirect
83 | github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea // indirect
84 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
85 | github.com/ugorji/go/codec v1.2.11 // indirect
86 | github.com/zeebo/xxh3 v1.0.2 // indirect
87 | go.opencensus.io v0.24.0 // indirect
88 | golang.org/x/arch v0.3.0 // indirect
89 | golang.org/x/crypto v0.10.0 // indirect
90 | golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect
91 | golang.org/x/mod v0.11.0 // indirect
92 | golang.org/x/oauth2 v0.9.0 // indirect
93 | golang.org/x/sync v0.3.0 // indirect
94 | golang.org/x/sys v0.11.0 // indirect
95 | golang.org/x/term v0.11.0 // indirect
96 | golang.org/x/text v0.10.0 // indirect
97 | golang.org/x/time v0.3.0 // indirect
98 | golang.org/x/tools v0.9.3 // indirect
99 | golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
100 | google.golang.org/appengine v1.6.7 // indirect
101 | google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
102 | google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
103 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
104 | google.golang.org/grpc v1.55.0 // indirect
105 | gopkg.in/yaml.v3 v3.0.1 // indirect
106 | lukechampine.com/blake3 v1.2.1 // indirect
107 | )
108 |
--------------------------------------------------------------------------------
/cmd/indexer/main.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2021 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package main
17 |
18 | import (
19 | "context"
20 | "encoding/json"
21 | "fmt"
22 | "io/ioutil"
23 | "log"
24 | "net/http"
25 | "net/url"
26 | "os"
27 | "path/filepath"
28 | "sort"
29 | "sync"
30 |
31 | "cloud.google.com/go/firestore"
32 | "github.com/google/ent/index"
33 | "github.com/google/ent/utils"
34 | "github.com/spf13/cobra"
35 | "google.golang.org/api/iterator"
36 | "google.golang.org/api/option"
37 | )
38 |
39 | var (
40 | indexFlag string
41 |
42 | firebaseCredentials string
43 | firebaseProject string
44 | concurrency int
45 |
46 | urlFlag string
47 | )
48 |
49 | type URL struct {
50 | URL string
51 | }
52 |
53 | func server(cmd *cobra.Command, args []string) {
54 | ctx := context.Background()
55 | client, err := firestore.NewClient(ctx, firebaseProject, option.WithCredentialsFile(firebaseCredentials))
56 | if err != nil {
57 | log.Fatalf("could not create client: %v", err)
58 | }
59 | if indexFlag == "" {
60 | log.Fatal("index flag is required")
61 | }
62 | iter := client.Collection("urls").Documents(ctx)
63 | defer iter.Stop()
64 | wg := sync.WaitGroup{}
65 | tokens := make(chan struct{}, concurrency)
66 | for {
67 | doc, err := iter.Next()
68 | if err == iterator.Done {
69 | break
70 | }
71 | if err != nil {
72 | log.Fatalf("error iterating over URLs: %v", err)
73 | }
74 | url := doc.Data()["url"].(string)
75 | if url == "" {
76 | continue
77 | }
78 | wg.Add(1)
79 | tokens <- struct{}{}
80 | go func() {
81 | defer func() {
82 | wg.Done()
83 | <-tokens
84 | }()
85 | _, err := fetch(url)
86 | if err != nil {
87 | log.Printf("could not fetch URL: %v", err)
88 | }
89 | }()
90 | }
91 | wg.Wait()
92 | }
93 |
94 | func fetchCmd(cmd *cobra.Command, args []string) {
95 | e, err := fetch(urlFlag)
96 | if err != nil {
97 | log.Fatalf("could not fetch URL: %v", err)
98 | }
99 | // Print digest to stdout.
100 | fmt.Printf("%s\n", e.Digest)
101 | }
102 |
103 | func fetch(urlString string) (*index.IndexEntry, error) {
104 | log.Print(urlString)
105 | parsedURL, err := url.Parse(urlString)
106 | if err != nil {
107 | return nil, fmt.Errorf("invalid URL: %w", err)
108 | }
109 |
110 | if parsedURL.User != nil {
111 | return nil, fmt.Errorf("non-empty user in URL")
112 | }
113 | if parsedURL.Fragment != "" {
114 | return nil, fmt.Errorf("non-empty fragment in URL")
115 | }
116 |
117 | urlString = parsedURL.String()
118 | log.Printf("fetching %q", urlString)
119 |
120 | res, err := http.Get(urlString)
121 | if err != nil {
122 | return nil, fmt.Errorf("could not fetch URL %q: %w", urlString, err)
123 | }
124 | if res.StatusCode != http.StatusOK {
125 | return nil, fmt.Errorf("invalid error code %d (%s)", res.StatusCode, res.Status)
126 | }
127 |
128 | data, err := ioutil.ReadAll(res.Body)
129 | if err != nil {
130 | return nil, fmt.Errorf("could not read HTTP body: %w", err)
131 | }
132 | digest := utils.ComputeDigest(data)
133 |
134 | l := filepath.Join(indexFlag, index.DigestToPath(digest), index.EntryFilename)
135 |
136 | var e index.IndexEntry
137 | if _, err := os.Stat(l); err == nil {
138 | log.Printf("index entry existing: %q", l)
139 | bytes, err := ioutil.ReadFile(l)
140 | if err != nil {
141 | return nil, fmt.Errorf("could not read index entry: %w", err)
142 | }
143 | err = json.Unmarshal(bytes, &e)
144 | if err != nil {
145 | return nil, fmt.Errorf("could not unmarshal JSON for index entry: %w", err)
146 | }
147 |
148 | if sort.SearchStrings(e.URLS, urlString) != -1 {
149 | // URL already in the indexed entry.
150 | } else {
151 | // Add the new URL.
152 | e.URLS = append(e.URLS, urlString)
153 | sort.Strings(e.URLS)
154 | }
155 |
156 | // Fix all fields just in case.
157 | e.MediaType = http.DetectContentType(data)
158 | e.Digest = string(digest)
159 | e.Size = len(data)
160 | } else {
161 | e = index.IndexEntry{
162 | MediaType: http.DetectContentType(data),
163 | Digest: string(digest),
164 | Size: len(data),
165 | URLS: []string{urlString},
166 | }
167 | }
168 | log.Printf("index entry to create: %+v", e)
169 |
170 | es, err := json.Marshal(e)
171 | if err != nil {
172 | return nil, fmt.Errorf("could not marshal JSON: %w", err)
173 | }
174 | err = os.MkdirAll(filepath.Dir(l), 0755)
175 | if err != nil {
176 | return nil, fmt.Errorf("could not create file: %w", err)
177 | }
178 | err = ioutil.WriteFile(l, es, 0644)
179 | if err != nil {
180 | return nil, fmt.Errorf("could not write to file: %w", err)
181 | }
182 |
183 | log.Printf("index entry created: %q", l)
184 |
185 | return &e, nil
186 | }
187 |
188 | func main() {
189 | var rootCmd = &cobra.Command{Use: "indexer"}
190 |
191 | serverCmd :=
192 | &cobra.Command{
193 | Use: "server",
194 | Short: "Run the indexer server",
195 | Run: server,
196 | }
197 | serverCmd.PersistentFlags().StringVar(&indexFlag, "index", "", "path to index repository")
198 | serverCmd.PersistentFlags().StringVar(&firebaseProject, "firebase-project", "", "Firebase project name")
199 | serverCmd.PersistentFlags().StringVar(&firebaseCredentials, "firebase-credentials", "", "file with Firebase credentials")
200 | serverCmd.PersistentFlags().IntVar(&concurrency, "concurrency", 10, "HTTP fetch concurrency")
201 | rootCmd.AddCommand(serverCmd)
202 |
203 | getCmd := &cobra.Command{
204 | Use: "fetch",
205 | Short: "Fetch and index a single URL",
206 | Run: fetchCmd,
207 | }
208 | getCmd.PersistentFlags().StringVar(&indexFlag, "index", "", "path to index repository")
209 | getCmd.PersistentFlags().StringVar(&urlFlag, "url", "", "url of the entry to index")
210 | rootCmd.AddCommand(getCmd)
211 |
212 | rootCmd.Execute()
213 | }
214 |
--------------------------------------------------------------------------------
/utils/node.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2022 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package utils
17 |
18 | import (
19 | "bytes"
20 | "encoding/binary"
21 | "fmt"
22 | "io"
23 | "log"
24 | "strconv"
25 | "strings"
26 |
27 | "github.com/ipfs/go-cid"
28 | "github.com/multiformats/go-varint"
29 | )
30 |
31 | type DAGNode struct {
32 | Bytes []byte
33 | Links []cid.Cid
34 | }
35 |
36 | const (
37 | TypeRaw = cid.Raw
38 | TypeDAG = 0x70
39 | )
40 |
41 | type Path []Selector
42 |
43 | // Index of the link to follow
44 | type Selector uint
45 |
46 | var order = binary.BigEndian
47 |
48 | const (
49 | FieldTypeInt = 0
50 | FieldTypeBytes = 1
51 | FieldTypeMsg = 2
52 | )
53 |
54 | type Field struct {
55 | ID uint64
56 | Type uint64
57 |
58 | UintValue uint64
59 | BytesValue []byte
60 | }
61 |
62 | func ReadUint64(b *bytes.Reader) (uint64, error) {
63 | return varint.ReadUvarint(b)
64 | }
65 |
66 | func WriteUint64(b *bytes.Buffer, i uint64) error {
67 | _, err := b.Write(varint.ToUvarint(i))
68 | return err
69 | }
70 |
71 | func DecodeField(b *bytes.Reader) (*Field, error) {
72 | fieldID, err := ReadUint64(b)
73 | if err == io.EOF {
74 | return nil, err
75 | } else if err != nil {
76 | return nil, fmt.Errorf("could not read field ID: %w", err)
77 | }
78 | fieldType, err := ReadUint64(b)
79 | if err != nil {
80 | return nil, fmt.Errorf("could not read field type: %w", err)
81 | }
82 | switch fieldType {
83 | case FieldTypeInt:
84 | fieldValue, err := ReadUint64(b)
85 | if err != nil {
86 | return nil, fmt.Errorf("could not read field value: %w", err)
87 | }
88 | return &Field{
89 | ID: fieldID,
90 | Type: fieldType,
91 | UintValue: fieldValue,
92 | BytesValue: nil,
93 | }, nil
94 | case FieldTypeBytes:
95 | fieldLength, err := ReadUint64(b)
96 | if err != nil {
97 | return nil, fmt.Errorf("could not read field length: %w", err)
98 | }
99 | fieldValue := make([]byte, fieldLength)
100 | if fieldLength > 0 {
101 | n, err := b.Read(fieldValue)
102 | if err != nil {
103 | return nil, fmt.Errorf("could not read field value: %w", err)
104 | }
105 | if n != int(fieldLength) {
106 | return nil, fmt.Errorf("could not read field value, read %d bytes, expected %d", n, fieldLength)
107 | }
108 | }
109 | return &Field{
110 | ID: fieldID,
111 | Type: fieldType,
112 | UintValue: 0,
113 | BytesValue: fieldValue,
114 | }, nil
115 | case FieldTypeMsg:
116 | fieldValue, err := ReadUint64(b)
117 | if err != nil {
118 | return nil, fmt.Errorf("could not read field value: %w", err)
119 | }
120 | return &Field{
121 | ID: fieldID,
122 | Type: fieldType,
123 | UintValue: fieldValue,
124 | BytesValue: nil,
125 | }, nil
126 | default:
127 | return nil, fmt.Errorf("unknown field type: %d", fieldType)
128 | }
129 | }
130 |
131 | func EncodeField(b *bytes.Buffer, f *Field) error {
132 | if err := WriteUint64(b, f.ID); err != nil {
133 | return fmt.Errorf("could not write field ID: %w", err)
134 | }
135 | if err := WriteUint64(b, f.Type); err != nil {
136 | return fmt.Errorf("could not write field type: %w", err)
137 | }
138 | switch f.Type {
139 | case FieldTypeInt:
140 | if err := WriteUint64(b, f.UintValue); err != nil {
141 | return fmt.Errorf("could not write field value: %w", err)
142 | }
143 | case FieldTypeBytes:
144 | if err := WriteUint64(b, uint64(len(f.BytesValue))); err != nil {
145 | return fmt.Errorf("could not write field length: %w", err)
146 | }
147 | if _, err := b.Write(f.BytesValue); err != nil {
148 | return fmt.Errorf("could not write field bytes: %w", err)
149 | }
150 | case FieldTypeMsg:
151 | if err := WriteUint64(b, f.UintValue); err != nil {
152 | return fmt.Errorf("could not write field value: %w", err)
153 | }
154 | default:
155 | return fmt.Errorf("unknown field type: %d", f.Type)
156 | }
157 | return nil
158 | }
159 |
160 | func EncodeLink(b *bytes.Buffer, link cid.Cid) error {
161 | _, err := link.WriteBytes(b)
162 | return err
163 | }
164 |
165 | func DecodeLink(b *bytes.Reader) (cid.Cid, error) {
166 | _, link, err := cid.CidFromReader(b)
167 | return link, err
168 | }
169 |
170 | func ParseDAGNode(b []byte) (*DAGNode, error) {
171 | if len(b) < 16 {
172 | return nil, fmt.Errorf("invalid DAGNode, too short: %d", len(b))
173 | }
174 | // https://fuchsia.dev/fuchsia-src/reference/fidl/language/wire-format#envelopes
175 | log.Printf("ParseDAGNode: %x", b)
176 | bytesNum := order.Uint64(b[0:8])
177 | log.Printf("bytesNum: %d", bytesNum)
178 | linksNum := order.Uint64(b[8:16])
179 | log.Printf("linksNum: %d", linksNum)
180 | linkReader := bytes.NewReader(b[16+bytesNum:])
181 | links := make([]cid.Cid, linksNum)
182 | for i := 0; i < int(linksNum); i++ {
183 | link, err := DecodeLink(linkReader)
184 | if err != nil {
185 | return nil, fmt.Errorf("could not decode link #%d: %w", i, err)
186 | }
187 | links[i] = link
188 | }
189 | return &DAGNode{
190 | Bytes: b[16 : 16+bytesNum],
191 | Links: links,
192 | }, nil
193 | }
194 |
195 | func SerializeDAGNode(node *DAGNode) ([]byte, error) {
196 | b := &bytes.Buffer{}
197 | {
198 | x := make([]byte, 8)
199 | order.PutUint64(x, uint64(len(node.Bytes)))
200 | b.Write(x)
201 | }
202 | {
203 | x := make([]byte, 8)
204 | order.PutUint64(x, uint64(len(node.Links)))
205 | b.Write(x)
206 | }
207 | b.Write(node.Bytes)
208 | for i, link := range node.Links {
209 | if err := EncodeLink(b, link); err != nil {
210 | return nil, fmt.Errorf("could not encode link #%d: %w", i, err)
211 | }
212 | }
213 | bb := b.Bytes()
214 | log.Printf("SerializeDAGNode: %x", bb)
215 | return bb, nil
216 | }
217 |
218 | // Parse a selector of the form "1"
219 | func ParseSelector(s string) (Selector, error) {
220 | index, err := strconv.Atoi(s)
221 | if err != nil {
222 | return 0, fmt.Errorf("invalid selector: %w", err)
223 | }
224 | return Selector(index), nil
225 | }
226 |
227 | func PrintSelector(s Selector) string {
228 | return fmt.Sprintf("%d", s)
229 | }
230 |
231 | func ParsePath(s string) (Path, error) {
232 | selectors := []Selector{}
233 | for _, s := range strings.Split(s, "/") {
234 | if s == "" {
235 | continue
236 | }
237 | selector, err := ParseSelector(s)
238 | if err != nil {
239 | return nil, fmt.Errorf("invalid selector: %w", err)
240 | }
241 | selectors = append(selectors, selector)
242 | }
243 | return selectors, nil
244 | }
245 |
246 | func PrintPath(path Path) string {
247 | out := ""
248 | for _, s := range path {
249 | out += "/" + PrintSelector(s)
250 | }
251 | return out
252 | }
253 |
--------------------------------------------------------------------------------
/cmd/ent-web/main.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2022 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package main
17 |
18 | import (
19 | "context"
20 | "flag"
21 | "fmt"
22 | "net/http"
23 | "strings"
24 | "time"
25 |
26 | "github.com/BurntSushi/toml"
27 | "github.com/gin-gonic/gin"
28 | "github.com/google/ent/log"
29 | "github.com/google/ent/nodeservice"
30 | "github.com/google/ent/utils"
31 | "github.com/ipfs/go-cid"
32 | "github.com/multiformats/go-multihash"
33 | "golang.org/x/net/http2"
34 | "golang.org/x/net/http2/h2c"
35 | )
36 |
37 | const (
38 | wwwSegment = "www"
39 | )
40 |
41 | var (
42 | domainName string
43 | objectGetter nodeservice.ObjectGetter
44 | )
45 |
46 | var configPath = flag.String("config", "", "path to config file")
47 |
48 | func main() {
49 | flag.Parse()
50 |
51 | ctx := context.Background()
52 | if *configPath == "" {
53 | log.Errorf(ctx, "must specify config")
54 | return
55 | }
56 | log.Infof(ctx, "loading config from %q", *configPath)
57 | config := readConfig()
58 | log.Infof(ctx, "loaded config: %#v", config)
59 |
60 | domainName = config.DomainName
61 |
62 | log.InitLog(config.ProjectID)
63 |
64 | objectGetter = getMultiplexObjectGetter(config)
65 |
66 | router := gin.Default()
67 | router.LoadHTMLGlob("templates/*")
68 | router.GET("/*path", webGetHandler)
69 |
70 | s := &http.Server{
71 | Addr: config.ListenAddress,
72 | Handler: h2c.NewHandler(router, &http2.Server{}),
73 | ReadTimeout: 5 * time.Second,
74 | WriteTimeout: 10 * time.Second,
75 | MaxHeaderBytes: 1 << 20,
76 | }
77 |
78 | log.Infof(ctx, "server running")
79 | log.Criticalf(ctx, "%v", s.ListenAndServe())
80 | }
81 |
82 | func webGetHandler(c *gin.Context) {
83 | ctx := c
84 | hostSegments := hostSegments(c.Request.Host)
85 | if len(hostSegments) == 0 {
86 | log.Warningf(ctx, "no host segments")
87 | pathSegments := strings.Split(strings.TrimPrefix(c.Param("path"), "/"), "/")
88 | log.Warningf(ctx, "path segments: %#v", pathSegments)
89 | if len(pathSegments) == 1 {
90 | base := pathSegments[0]
91 | digest, err := utils.ParseDigest(base)
92 | if err != nil {
93 | log.Warningf(ctx, "could not parse digest: %s", err)
94 | c.AbortWithStatus(http.StatusNotFound)
95 | return
96 | }
97 | link := cid.NewCidV1(utils.TypeDAG, multihash.Multihash(digest))
98 | c.Redirect(http.StatusMovedPermanently, fmt.Sprintf("http://%s.www.%s", link.String(), domainName))
99 | return
100 | }
101 | }
102 | if len(hostSegments) != 2 {
103 | log.Warningf(ctx, "invalid host segments: %#v", hostSegments)
104 | c.AbortWithStatus(http.StatusNotFound)
105 | return
106 | }
107 | if hostSegments[len(hostSegments)-1] != wwwSegment {
108 | log.Warningf(ctx, "invalid host segments: %#v", hostSegments)
109 | c.AbortWithStatus(http.StatusNotFound)
110 | return
111 | }
112 |
113 | rootLink, err := cid.Parse(hostSegments[0])
114 | if err != nil {
115 | log.Warningf(ctx, "could not parse root link from %q: %s", hostSegments[0], err)
116 | c.AbortWithStatus(http.StatusNotFound)
117 | return
118 | }
119 |
120 | path := strings.Split(strings.TrimPrefix(c.Param("path"), "/"), "/")
121 | log.Debugf(ctx, "root link: %s", rootLink.String())
122 | log.Debugf(ctx, "path: %#v", path)
123 | target, err := traverseString(ctx, objectGetter, rootLink, path)
124 | if err != nil {
125 | log.Warningf(ctx, "could not traverse: %s", err)
126 | c.AbortWithStatus(http.StatusNotFound)
127 | return
128 | }
129 | log.Debugf(ctx, "target: %s", target.String())
130 | c.Header("ent-digest", target.String())
131 | nodeRaw, err := objectGetter.Get(ctx, utils.Digest(target.Hash()))
132 | if err != nil {
133 | log.Warningf(ctx, "could not get blob %s: %s", target, err)
134 | c.Abort()
135 | return
136 | }
137 | switch target.Type() {
138 | case utils.TypeRaw:
139 | contentType := http.DetectContentType(nodeRaw)
140 | log.Debugf(ctx, "content type: %s", contentType)
141 | c.Data(http.StatusOK, contentType, nodeRaw)
142 | case utils.TypeDAG:
143 | renderDag(c, rootLink, target, nodeRaw, path)
144 | default:
145 | log.Warningf(ctx, "invalid target type: %s", target.Type())
146 | c.AbortWithStatus(http.StatusNotFound)
147 | }
148 | }
149 |
150 | func renderDag(c *gin.Context, root cid.Cid, target cid.Cid, nodeRaw []byte, path []string) {
151 | ctx := c
152 | node, err := utils.ParseDAGNode(nodeRaw)
153 | if err != nil {
154 | log.Warningf(ctx, "could not parse dag node: %s", err)
155 | c.AbortWithStatus(http.StatusNotFound)
156 | return
157 | }
158 |
159 | parents := []UILink{}
160 | parents = append(parents, UILink{
161 | Name: root.String(),
162 | URL: "/",
163 | })
164 | for i, s := range path {
165 | parents = append(parents, UILink{
166 | Name: s,
167 | URL: "/" + strings.Join(path[:i+1], "/"),
168 | })
169 | }
170 |
171 | links := []UILink{}
172 | names := strings.Split(string(node.Bytes), "\n")
173 | prefix := "/" + strings.Join(path, "/")
174 | if !strings.HasSuffix(prefix, "/") {
175 | prefix += "/"
176 | }
177 | for i, link := range node.Links {
178 | links = append(links, UILink{
179 | Name: names[i],
180 | Raw: link.Type() == utils.TypeRaw,
181 | URL: prefix + names[i],
182 | })
183 | }
184 |
185 | c.HTML(http.StatusOK, "basic.html", gin.H{
186 | "links": links,
187 | "parents": parents,
188 | "root": target.String(),
189 | })
190 | }
191 |
192 | type UILink struct {
193 | Name string
194 | Raw bool
195 | URL string
196 | }
197 |
198 | func hostSegments(host string) []string {
199 | host = strings.TrimSuffix(host, domainName)
200 | host = strings.TrimSuffix(host, ".")
201 | hostSegments := strings.Split(host, ".")
202 | if len(hostSegments) > 0 && hostSegments[0] == "" {
203 | return hostSegments[1:]
204 | } else {
205 | return hostSegments
206 | }
207 | }
208 |
209 | func traverseString(ctx context.Context, og nodeservice.ObjectGetter, link cid.Cid, segments []string) (cid.Cid, error) {
210 | if len(segments) == 0 || link.Type() == utils.TypeRaw {
211 | return link, nil
212 | } else {
213 | digest := utils.Digest(link.Hash())
214 | nodeRaw, err := og.Get(ctx, digest)
215 | if err != nil {
216 | return cid.Cid{}, fmt.Errorf("could not get blob %s: %w", digest, err)
217 | }
218 | node, err := utils.ParseDAGNode(nodeRaw)
219 | if err != nil {
220 | return cid.Cid{}, fmt.Errorf("could not parse node %s: %w", digest, err)
221 | }
222 | selector := segments[0]
223 | if selector != "" {
224 | names := strings.Split(string(node.Bytes), "\n")
225 | // Find the name corresponding to the selector
226 | log.Debugf(ctx, "selector: %v", selector)
227 | log.Debugf(ctx, "names: %#v", names)
228 | linkIndex := -1
229 | for i, name := range names {
230 | if name == selector {
231 | linkIndex = i
232 | break
233 | }
234 | }
235 | if linkIndex == -1 {
236 | return cid.Cid{}, fmt.Errorf("could not find link %s/%v", digest, selector)
237 | }
238 | next := node.Links[linkIndex]
239 | if err != nil {
240 | return cid.Cid{}, fmt.Errorf("could not traverse %s/%v: %w", digest, selector, err)
241 | }
242 | log.Debugf(ctx, "next: %v", next)
243 | return traverseString(ctx, og, next, segments[1:])
244 | } else {
245 | return link, nil
246 | }
247 | }
248 | }
249 |
250 | func readConfig() Config {
251 | config := Config{}
252 | _, err := toml.DecodeFile(*configPath, &config)
253 | if err != nil {
254 | panic(err)
255 | }
256 | return config
257 | }
258 |
259 | func getMultiplexObjectGetter(config Config) nodeservice.ObjectGetter {
260 | inner := make([]nodeservice.Inner, 0)
261 | for _, remote := range config.Remotes {
262 | inner = append(inner, nodeservice.NewRemote(remote.Name, remote.URL, remote.APIKey))
263 | }
264 | return nodeservice.Sequence{
265 | Inner: inner,
266 | }
267 | }
268 |
--------------------------------------------------------------------------------
/proto/dag_pb.pb.go:
--------------------------------------------------------------------------------
1 | // Code generated by protoc-gen-go. DO NOT EDIT.
2 | // versions:
3 | // protoc-gen-go v1.28.1
4 | // protoc v3.21.8
5 | // source: proto/dag_pb.proto
6 |
7 | package ent
8 |
9 | import (
10 | protoreflect "google.golang.org/protobuf/reflect/protoreflect"
11 | protoimpl "google.golang.org/protobuf/runtime/protoimpl"
12 | reflect "reflect"
13 | sync "sync"
14 | )
15 |
16 | const (
17 | // Verify that this generated code is sufficiently up-to-date.
18 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
19 | // Verify that runtime/protoimpl is sufficiently up-to-date.
20 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
21 | )
22 |
23 | type PBLink struct {
24 | state protoimpl.MessageState
25 | sizeCache protoimpl.SizeCache
26 | unknownFields protoimpl.UnknownFields
27 |
28 | // binary CID (with no multibase prefix) of the target object
29 | Hash []byte `protobuf:"bytes,1,opt,name=Hash,proto3,oneof" json:"Hash,omitempty"`
30 | // UTF-8 string name
31 | Name *string `protobuf:"bytes,2,opt,name=Name,proto3,oneof" json:"Name,omitempty"`
32 | // cumulative size of target object
33 | Tsize *uint64 `protobuf:"varint,3,opt,name=Tsize,proto3,oneof" json:"Tsize,omitempty"`
34 | }
35 |
36 | func (x *PBLink) Reset() {
37 | *x = PBLink{}
38 | if protoimpl.UnsafeEnabled {
39 | mi := &file_proto_dag_pb_proto_msgTypes[0]
40 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
41 | ms.StoreMessageInfo(mi)
42 | }
43 | }
44 |
45 | func (x *PBLink) String() string {
46 | return protoimpl.X.MessageStringOf(x)
47 | }
48 |
49 | func (*PBLink) ProtoMessage() {}
50 |
51 | func (x *PBLink) ProtoReflect() protoreflect.Message {
52 | mi := &file_proto_dag_pb_proto_msgTypes[0]
53 | if protoimpl.UnsafeEnabled && x != nil {
54 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
55 | if ms.LoadMessageInfo() == nil {
56 | ms.StoreMessageInfo(mi)
57 | }
58 | return ms
59 | }
60 | return mi.MessageOf(x)
61 | }
62 |
63 | // Deprecated: Use PBLink.ProtoReflect.Descriptor instead.
64 | func (*PBLink) Descriptor() ([]byte, []int) {
65 | return file_proto_dag_pb_proto_rawDescGZIP(), []int{0}
66 | }
67 |
68 | func (x *PBLink) GetHash() []byte {
69 | if x != nil {
70 | return x.Hash
71 | }
72 | return nil
73 | }
74 |
75 | func (x *PBLink) GetName() string {
76 | if x != nil && x.Name != nil {
77 | return *x.Name
78 | }
79 | return ""
80 | }
81 |
82 | func (x *PBLink) GetTsize() uint64 {
83 | if x != nil && x.Tsize != nil {
84 | return *x.Tsize
85 | }
86 | return 0
87 | }
88 |
89 | type PBNode struct {
90 | state protoimpl.MessageState
91 | sizeCache protoimpl.SizeCache
92 | unknownFields protoimpl.UnknownFields
93 |
94 | // refs to other objects
95 | Links []*PBLink `protobuf:"bytes,2,rep,name=Links,proto3" json:"Links,omitempty"`
96 | // opaque user data
97 | Data []byte `protobuf:"bytes,1,opt,name=Data,proto3,oneof" json:"Data,omitempty"`
98 | }
99 |
100 | func (x *PBNode) Reset() {
101 | *x = PBNode{}
102 | if protoimpl.UnsafeEnabled {
103 | mi := &file_proto_dag_pb_proto_msgTypes[1]
104 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
105 | ms.StoreMessageInfo(mi)
106 | }
107 | }
108 |
109 | func (x *PBNode) String() string {
110 | return protoimpl.X.MessageStringOf(x)
111 | }
112 |
113 | func (*PBNode) ProtoMessage() {}
114 |
115 | func (x *PBNode) ProtoReflect() protoreflect.Message {
116 | mi := &file_proto_dag_pb_proto_msgTypes[1]
117 | if protoimpl.UnsafeEnabled && x != nil {
118 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
119 | if ms.LoadMessageInfo() == nil {
120 | ms.StoreMessageInfo(mi)
121 | }
122 | return ms
123 | }
124 | return mi.MessageOf(x)
125 | }
126 |
127 | // Deprecated: Use PBNode.ProtoReflect.Descriptor instead.
128 | func (*PBNode) Descriptor() ([]byte, []int) {
129 | return file_proto_dag_pb_proto_rawDescGZIP(), []int{1}
130 | }
131 |
132 | func (x *PBNode) GetLinks() []*PBLink {
133 | if x != nil {
134 | return x.Links
135 | }
136 | return nil
137 | }
138 |
139 | func (x *PBNode) GetData() []byte {
140 | if x != nil {
141 | return x.Data
142 | }
143 | return nil
144 | }
145 |
146 | var File_proto_dag_pb_proto protoreflect.FileDescriptor
147 |
148 | var file_proto_dag_pb_proto_rawDesc = []byte{
149 | 0x0a, 0x12, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x64, 0x61, 0x67, 0x5f, 0x70, 0x62, 0x2e, 0x70,
150 | 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x03, 0x65, 0x6e, 0x74, 0x22, 0x71, 0x0a, 0x06, 0x50, 0x42, 0x4c,
151 | 0x69, 0x6e, 0x6b, 0x12, 0x17, 0x0a, 0x04, 0x48, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28,
152 | 0x0c, 0x48, 0x00, 0x52, 0x04, 0x48, 0x61, 0x73, 0x68, 0x88, 0x01, 0x01, 0x12, 0x17, 0x0a, 0x04,
153 | 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x04, 0x4e, 0x61,
154 | 0x6d, 0x65, 0x88, 0x01, 0x01, 0x12, 0x19, 0x0a, 0x05, 0x54, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03,
155 | 0x20, 0x01, 0x28, 0x04, 0x48, 0x02, 0x52, 0x05, 0x54, 0x73, 0x69, 0x7a, 0x65, 0x88, 0x01, 0x01,
156 | 0x42, 0x07, 0x0a, 0x05, 0x5f, 0x48, 0x61, 0x73, 0x68, 0x42, 0x07, 0x0a, 0x05, 0x5f, 0x4e, 0x61,
157 | 0x6d, 0x65, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x54, 0x73, 0x69, 0x7a, 0x65, 0x22, 0x4d, 0x0a, 0x06,
158 | 0x50, 0x42, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x21, 0x0a, 0x05, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x18,
159 | 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x65, 0x6e, 0x74, 0x2e, 0x50, 0x42, 0x4c, 0x69,
160 | 0x6e, 0x6b, 0x52, 0x05, 0x4c, 0x69, 0x6e, 0x6b, 0x73, 0x12, 0x17, 0x0a, 0x04, 0x44, 0x61, 0x74,
161 | 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x04, 0x44, 0x61, 0x74, 0x61, 0x88,
162 | 0x01, 0x01, 0x42, 0x07, 0x0a, 0x05, 0x5f, 0x44, 0x61, 0x74, 0x61, 0x42, 0x10, 0x5a, 0x0e, 0x67,
163 | 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x6e, 0x74, 0x62, 0x06, 0x70,
164 | 0x72, 0x6f, 0x74, 0x6f, 0x33,
165 | }
166 |
167 | var (
168 | file_proto_dag_pb_proto_rawDescOnce sync.Once
169 | file_proto_dag_pb_proto_rawDescData = file_proto_dag_pb_proto_rawDesc
170 | )
171 |
172 | func file_proto_dag_pb_proto_rawDescGZIP() []byte {
173 | file_proto_dag_pb_proto_rawDescOnce.Do(func() {
174 | file_proto_dag_pb_proto_rawDescData = protoimpl.X.CompressGZIP(file_proto_dag_pb_proto_rawDescData)
175 | })
176 | return file_proto_dag_pb_proto_rawDescData
177 | }
178 |
179 | var file_proto_dag_pb_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
180 | var file_proto_dag_pb_proto_goTypes = []interface{}{
181 | (*PBLink)(nil), // 0: ent.PBLink
182 | (*PBNode)(nil), // 1: ent.PBNode
183 | }
184 | var file_proto_dag_pb_proto_depIdxs = []int32{
185 | 0, // 0: ent.PBNode.Links:type_name -> ent.PBLink
186 | 1, // [1:1] is the sub-list for method output_type
187 | 1, // [1:1] is the sub-list for method input_type
188 | 1, // [1:1] is the sub-list for extension type_name
189 | 1, // [1:1] is the sub-list for extension extendee
190 | 0, // [0:1] is the sub-list for field type_name
191 | }
192 |
193 | func init() { file_proto_dag_pb_proto_init() }
194 | func file_proto_dag_pb_proto_init() {
195 | if File_proto_dag_pb_proto != nil {
196 | return
197 | }
198 | if !protoimpl.UnsafeEnabled {
199 | file_proto_dag_pb_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
200 | switch v := v.(*PBLink); i {
201 | case 0:
202 | return &v.state
203 | case 1:
204 | return &v.sizeCache
205 | case 2:
206 | return &v.unknownFields
207 | default:
208 | return nil
209 | }
210 | }
211 | file_proto_dag_pb_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
212 | switch v := v.(*PBNode); i {
213 | case 0:
214 | return &v.state
215 | case 1:
216 | return &v.sizeCache
217 | case 2:
218 | return &v.unknownFields
219 | default:
220 | return nil
221 | }
222 | }
223 | }
224 | file_proto_dag_pb_proto_msgTypes[0].OneofWrappers = []interface{}{}
225 | file_proto_dag_pb_proto_msgTypes[1].OneofWrappers = []interface{}{}
226 | type x struct{}
227 | out := protoimpl.TypeBuilder{
228 | File: protoimpl.DescBuilder{
229 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
230 | RawDescriptor: file_proto_dag_pb_proto_rawDesc,
231 | NumEnums: 0,
232 | NumMessages: 2,
233 | NumExtensions: 0,
234 | NumServices: 0,
235 | },
236 | GoTypes: file_proto_dag_pb_proto_goTypes,
237 | DependencyIndexes: file_proto_dag_pb_proto_depIdxs,
238 | MessageInfos: file_proto_dag_pb_proto_msgTypes,
239 | }.Build()
240 | File_proto_dag_pb_proto = out.File
241 | file_proto_dag_pb_proto_rawDesc = nil
242 | file_proto_dag_pb_proto_goTypes = nil
243 | file_proto_dag_pb_proto_depIdxs = nil
244 | }
245 |
--------------------------------------------------------------------------------
/cmd/ent-server/main.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2021 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package main
17 |
18 | import (
19 | "context"
20 | "flag"
21 | "fmt"
22 | "net/http"
23 | "os"
24 | "strings"
25 | "time"
26 |
27 | "cloud.google.com/go/firestore"
28 | "cloud.google.com/go/storage"
29 | "github.com/BurntSushi/toml"
30 | "github.com/gin-contrib/cors"
31 | "github.com/gin-gonic/gin"
32 | "github.com/go-redis/redis/v8"
33 | "github.com/google/ent/datastore"
34 | "github.com/google/ent/log"
35 | "github.com/google/ent/nodeservice"
36 | "github.com/google/ent/objectstore"
37 | pb "github.com/google/ent/proto"
38 | "github.com/google/ent/utils"
39 | "github.com/ipfs/go-cid"
40 | "golang.org/x/net/http2"
41 | "golang.org/x/net/http2/h2c"
42 | "google.golang.org/grpc"
43 | "google.golang.org/grpc/metadata"
44 | )
45 |
46 | var (
47 | blobStore nodeservice.ObjectStore
48 | store Store
49 |
50 | apiKeyToUser = map[string]*User{}
51 | )
52 |
53 | const (
54 | // See https://cloud.google.com/endpoints/docs/openapi/openapi-limitations#api_key_definition_limitations
55 | apiKeyHeader = "x-api-key"
56 | )
57 |
58 | var configPath = flag.String("config", "", "path to config file")
59 |
60 | type UINode struct {
61 | Kind string
62 | Value string
63 | Digest string
64 | Links []UILink
65 | URL string
66 | ParentURL string
67 | PathSegments []UIPathSegment
68 | WWWURL string
69 | }
70 |
71 | type UILink struct {
72 | Selector utils.Selector
73 | Digest string
74 | Raw bool
75 | URL string
76 | }
77 |
78 | type UIPathSegment struct {
79 | Selector utils.Selector
80 | Name string
81 | URL string
82 | }
83 |
84 | func readConfig() Config {
85 | config := Config{}
86 | _, err := toml.DecodeFile(*configPath, &config)
87 | if err != nil {
88 | panic(err)
89 | }
90 | return config
91 | }
92 |
93 | func initStore(ctx context.Context, projectID string) *firestore.Client {
94 | firestoreClient, err := firestore.NewClient(context.Background(), projectID)
95 | if err != nil {
96 | log.Criticalf(ctx, "Failed to create client: %v", err)
97 | os.Exit(1)
98 | }
99 | res, err := firestoreClient.Doc("test/test").Set(ctx, map[string]interface{}{})
100 | if err != nil {
101 | log.Criticalf(ctx, "Failed to write to firestore: %v", err)
102 | os.Exit(1)
103 | }
104 | log.Infof(ctx, "Wrote to firestore: %v", res)
105 | return firestoreClient
106 | }
107 |
108 | func main() {
109 | flag.Parse()
110 |
111 | ctx := context.Background()
112 | if *configPath == "" {
113 | log.Errorf(ctx, "must specify config")
114 | os.Exit(1)
115 | }
116 | log.Infof(ctx, "loading config from %q", *configPath)
117 | config := readConfig()
118 | log.Infof(ctx, "loaded config: %#v", config)
119 |
120 | log.InitLog(config.ProjectID)
121 |
122 | for _, user := range config.Users {
123 | // Must make a copy first, or else the map will point to the same.
124 | u := user
125 | apiKeyToUser[user.APIKey] = &u
126 | }
127 | for apiKey, user := range apiKeyToUser {
128 | log.Infof(ctx, "user %q: %q %d", redact(apiKey), user.Name, user.ID)
129 | }
130 |
131 | var ds datastore.DataStore
132 |
133 | if config.CloudStorageEnabled {
134 | objectsBucketName := config.CloudStorageBucket
135 | if objectsBucketName == "" {
136 | log.Errorf(ctx, "must specify Cloud Storage bucket name")
137 | os.Exit(1)
138 | }
139 | log.Infof(ctx, "using Cloud Storage bucket: %q", objectsBucketName)
140 | storageClient, err := storage.NewClient(ctx)
141 | if err != nil {
142 | log.Errorf(ctx, "could not create Cloud Storage client: %v", err)
143 | os.Exit(1)
144 | }
145 | ds = datastore.Cloud{
146 | Client: storageClient,
147 | BucketName: objectsBucketName,
148 | }
149 | } else {
150 | log.Infof(ctx, "using local file system")
151 | ds = datastore.File{
152 | DirName: "data/objects",
153 | }
154 | }
155 |
156 | if config.RedisEnabled {
157 | log.Infof(ctx, "using Redis: %q", config.RedisEndpoint)
158 | rdb := redis.NewClient(&redis.Options{
159 | Addr: config.RedisEndpoint,
160 | })
161 | err := rdb.Set(ctx, "key", "value", 0).Err()
162 | if err != nil {
163 | log.Errorf(ctx, "could not connect to Redis: %v", err)
164 | } else {
165 | ds = datastore.Memcache{
166 | Inner: ds,
167 | RDB: rdb,
168 | }
169 | }
170 | }
171 |
172 | if config.BigqueryEnabled {
173 | bigqueryDataset := config.BigqueryDataset
174 | log.Infof(ctx, "bigquery dataset: %q", bigqueryDataset)
175 | InitBigquery(ctx, config.ProjectID, config.BigqueryDataset)
176 | }
177 |
178 | blobStore = objectstore.Store{
179 | Inner: ds,
180 | }
181 |
182 | fs := initStore(ctx, config.ProjectID)
183 | store = Store{
184 | c: fs,
185 | }
186 |
187 | gin.SetMode(config.GinMode)
188 | router := gin.Default()
189 |
190 | corsConfig := cors.DefaultConfig()
191 | corsConfig.AllowAllOrigins = true
192 | router.Use(cors.New(corsConfig))
193 |
194 | router.RedirectTrailingSlash = false
195 | router.RedirectFixedPath = false
196 |
197 | router.GET("/raw/:digest", rawGetHandler)
198 | router.PUT("/raw", rawPutHandler)
199 |
200 | grpServer := grpc.NewServer()
201 | pb.RegisterEntServer(grpServer, grpcServer{})
202 | router.Any("/ent.server.api.Ent/*any", gin.WrapH(grpServer))
203 |
204 | s := &http.Server{
205 | Addr: config.ListenAddress,
206 | Handler: h2c.NewHandler(router, &http2.Server{}),
207 | ReadTimeout: 600 * time.Second,
208 | WriteTimeout: 600 * time.Second,
209 | MaxHeaderBytes: 1 << 20,
210 | }
211 | log.Infof(ctx, "server running")
212 | err := s.ListenAndServe()
213 | fmt.Printf("server exited: %v", err)
214 | log.Criticalf(ctx, "%v", err)
215 | }
216 |
217 | func indexHandler(c *gin.Context) {
218 | c.HTML(http.StatusOK, "index.tmpl", gin.H{})
219 | }
220 |
221 | func parsePath(p string) []string {
222 | if p == "/" || p == "" {
223 | return []string{}
224 | } else {
225 | return strings.Split(strings.TrimPrefix(p, "/"), "/")
226 | }
227 | }
228 |
229 | func parseHost(p string) []string {
230 | if p == "/" || p == "" {
231 | return []string{}
232 | } else {
233 | return strings.Split(strings.TrimPrefix(p, "/"), "/")
234 | }
235 | }
236 |
237 | func fetchNodes(ctx context.Context, base cid.Cid, depth uint) ([][]byte, error) {
238 | log.Debugf(ctx, "fetching nodes for %v, depth %d", base, depth)
239 | var nodes [][]byte
240 |
241 | blob, err := blobStore.Get(ctx, utils.Digest(base.Hash()))
242 | if err != nil {
243 | return nil, fmt.Errorf("error getting blob %q: %w", base.Hash(), err)
244 | }
245 |
246 | nodes = append(nodes, blob)
247 |
248 | // Nothing to recurse here.
249 | if base.Type() == utils.TypeRaw || depth == 0 {
250 | return nodes, nil
251 | }
252 |
253 | dagNode, err := utils.ParseDAGNode(blob)
254 | if err != nil {
255 | log.Warningf(ctx, "error parsing blob %q: %s", base, err)
256 | return nodes, nil
257 | }
258 |
259 | for _, link := range dagNode.Links {
260 | nn, err := fetchNodes(ctx, link, depth-1)
261 | if err != nil {
262 | log.Warningf(ctx, "error fetching nodes: %s", err)
263 | continue
264 | }
265 | nodes = append(nodes, nn...)
266 | }
267 | return nodes, nil
268 | }
269 |
270 | func getAPIKey(c *gin.Context) string {
271 | return c.Request.Header.Get(apiKeyHeader)
272 | }
273 |
274 | func getAPIKeyGRPC(c context.Context) string {
275 | md, _ := metadata.FromIncomingContext(c)
276 | vv := md.Get(apiKeyHeader)
277 | if len(vv) == 0 {
278 | return ""
279 | }
280 | return vv[0]
281 | }
282 |
283 | func traverse(ctx context.Context, digest utils.Digest, segments []utils.Selector) (utils.Digest, error) {
284 | if len(segments) == 0 {
285 | return digest, nil
286 | } else {
287 | nodeRaw, err := blobStore.Get(ctx, digest)
288 | if err != nil {
289 | return utils.Digest{}, fmt.Errorf("could not get blob %s: %w", digest, err)
290 | }
291 | node, err := utils.ParseDAGNode(nodeRaw)
292 | if err != nil {
293 | return utils.Digest{}, fmt.Errorf("could not parse node %s: %w", digest, err)
294 | }
295 | selector := segments[0]
296 | next := node.Links[selector]
297 | if err != nil {
298 | return utils.Digest{}, fmt.Errorf("could not traverse %s/%v: %w", digest, selector, err)
299 | }
300 | log.Debugf(ctx, "next: %v", next)
301 | return traverse(ctx, utils.Digest(next.Hash()), segments[1:])
302 | }
303 | }
304 |
305 | func BaseLogItem(c *gin.Context) LogItem {
306 | return LogItem{
307 | Timestamp: time.Now(),
308 | IP: c.ClientIP(),
309 | UserAgent: c.Request.UserAgent(),
310 | RequestMethod: c.Request.Method,
311 | RequestURI: c.Request.RequestURI,
312 | }
313 | }
314 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # 🌳 Ent
2 |
3 | Ent (named after the [Ent](https://en.wikipedia.org/wiki/Ent) species from _The
4 | Lord of the Rings_ by J. R. R. Tolkien) is an experimental universal, scalable,
5 | general purpose, Content-Addressable Store (CAS) to explore verifiable data
6 | structures, policies and graphs.
7 |
8 | This is not an officially supported Google product.
9 |
10 | ## Content-Addressability
11 |
12 | Ent encourages a model in which files are referred to by their digest (as a
13 | proxy for their content), instead of which server they happen to be located
14 | (which is what a URL normally is for).
15 |
16 | For example, instead of referring to the image below by its URL
17 | `https://upload.wikimedia.org/wikipedia/commons/thumb/4/48/The_Calling_of_Saint_Matthew-Caravaggo_%281599-1600%29.jpg/405px-The_Calling_of_Saint_Matthew-Caravaggo_%281599-1600%29.jpg`,
18 | in Ent it would be referred to by its digest
19 | `sha256:f3e737f4d50fbf6bb6053e3b8c72d6bf7f1a7229aacf2e9b4c97e9dd27cb1dcf`.
20 |
21 | 
22 |
23 | The digest of a file is a stable cryptographic identifier for the actual data
24 | that is contained in the file, and does not depend on which server the file
25 | happens to be hosted on, or at what path. If at some point in the future the
26 | file were to disappear from the original location and be made available at a
27 | different location, the original URL would stop working, but the digest of the
28 | file would remain the same, and can be used to refer to that file forever.
29 |
30 | Additionally, using a digest to refer to a file is useful for security and
31 | trustworthiness: if someone sends you the digest of a file to download (e.g. a
32 | program to install on your computer), you can be sure that, by resolving that
33 | digest to an actual file via Ent, the resulting file is exactly the one that the
34 | sender intended, without having to trust the Ent Server, the Ent Index or the
35 | server where the file is ultimately hosted.
36 |
37 | ## Installation
38 |
39 | The Ent CLI can be built and installed with the following command, after having
40 | cloned this repository locally:
41 |
42 | ```bash
43 | go install ./cmd/ent
44 | ```
45 |
46 | And is then available via the binary called `ent`:
47 |
48 | ```bash
49 | ent help
50 | ```
51 |
52 | ## Examples
53 |
54 | In order to fetch a file with a given digest, the `ent get` subcommand can be
55 | used.
56 |
57 | You can try the following command in your terminal, which fetches the text of
58 | the _Treasure Island_ book:
59 |
60 | ```console
61 | $ ent get sha256:4c350163715b7b1d0fc3bcbf11bfffc0cf2d107f69253f237111a7480809e192 | head
62 | The Project Gutenberg EBook of Treasure Island, by Robert Louis Stevenson
63 |
64 | This eBook is for the use of anyone anywhere in the United States and most
65 | other parts of the world at no cost and with almost no restrictions
66 | whatsoever. You may copy it, give it away or re-use it under the terms of
67 | the Project Gutenberg License included with this eBook or online at
68 | www.gutenberg.org. If you are not located in the United States, you'll have
69 | to check the laws of the country where you are located before using this ebook.
70 |
71 | Title: Treasure Island
72 | ```
73 |
74 | The Ent CLI queries the default Ent index
75 | (https://github.com/tiziano88/ent-index) to resolve the digest to a URL, and
76 | then fetches the file at that URL, and also verifies that it corresponds to the
77 | expected digest. It first buffers the entire file internally in order to verify
78 | its digest, and only prints it to stdout if it does match the expected digest.
79 |
80 | You can also manually double check that the returned file does in fact
81 | correspond to the expected digest:
82 |
83 | ```console
84 | $ ent get sha256:4c350163715b7b1d0fc3bcbf11bfffc0cf2d107f69253f237111a7480809e192 | sha256sum
85 | 4c350163715b7b1d0fc3bcbf11bfffc0cf2d107f69253f237111a7480809e192 -
86 | ```
87 |
88 | ## Ent Server
89 |
90 | An Ent Server provides access to an underlying Ent store via an HTTP-based REST
91 | API.
92 |
93 | An Ent Server may be running locally (on port 27333 by default), or remotely.
94 |
95 | Some Ent Servers require the user to be authenticated in order for the user to
96 | read and / or write, which is performed via an API key.
97 |
98 | ### JSON HTTP API
99 |
100 | The JSON API allows retrieving and creating multiple objects at once. All the
101 | methods below are invoked via HTTP `POST` methods.
102 |
103 | - `/api/v1/blobs/get`
104 |
105 | Get one or more objects by their digest.
106 |
107 | - `/api/v1/blobs/put`
108 |
109 | Put one or more objects.
110 |
111 | ### Raw HTTP API
112 |
113 | The raw HTTP API is meant to be used by existing basic tools without requiring
114 | any special serialization.
115 |
116 | The API supports the following HTTP operations:
117 |
118 | - `GET /raw/:digest`
119 | - `PUT /raw`
120 |
121 | For instance, this API may be used directly from the terminal via `curl`:
122 |
123 | - Get an object by digest:
124 |
125 | ```console
126 | $ curl --header 'x-api-key: xxx' localhost:27333/raw/sha256:4c350163715b7b1d0fc3bcbf11bfffc0cf2d107f69253f237111a7480809e192 | sha256sum
127 | % Total % Received % Xferd Average Speed Time Time Time Current
128 | Dload Upload Total Spent Left Speed
129 | 100 390k 0 390k 0 0 5818k 0 --:--:-- --:--:-- --:--:-- 5832k
130 | 4c350163715b7b1d0fc3bcbf11bfffc0cf2d107f69253f237111a7480809e192 -
131 | ```
132 |
133 | - Put an object:
134 |
135 | ```console
136 | $ curl --header 'x-api-key: yyy' --upload-file README.md --head localhost:27333/raw
137 | % Total % Received % Xferd Average Speed Time Time Time Current
138 | Dload Upload Total Spent Left Speed
139 | 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0HTTP/1.1 100 Continue
140 |
141 | HTTP/1.1 201 Created
142 | Location: /raw/sha256:c1a4c83dfeca632af8dcac3591f4b01a303342cf0bae0a63d9a5d7688b0e77cc
143 | Date: Thu, 10 Mar 2022 18:00:45 GMT
144 | Content-Length: 0
145 |
146 | 100 8013 0 0 100 8013 0 132k --:--:-- --:--:-- --:--:-- 134k
147 | ```
148 |
149 | ## Ent Index
150 |
151 | An Ent index is a "cheap" way to provide access to existing (location-addressed)
152 | content on the internet, but in a content-addressable way.
153 |
154 | It consists of a static website, which serves an entry for each digest, listing
155 | one or more "traditional" URLs which may provide the file in question.
156 |
157 | For instance, it may be serialized as a Git repository with a directory
158 | structure corresponding to the digest of each entry, and a JSON file for each
159 | entry that lists one or more URLs at which the object may be found.
160 |
161 | The directory path is obtained by grouping sets of two digits from the digest,
162 | and creating a nested folder for each of them; this is in order to limit the
163 | number of files or directories inside each directory, since that would otherwise
164 | not scale when there are millions of entries in the index.
165 |
166 | For instance, the file with digest
167 | `sha256:4c350163715b7b1d0fc3bcbf11bfffc0cf2d107f69253f237111a7480809e192` is
168 | stored in the Ent index under the file
169 | `/sha256/4c/35/01/63/71/5b/7b/1d/0f/c3/bc/bf/11/bf/ff/c0/cf/2d/10/7f/69/25/3f/23/71/11/a7/48/08/09/e1/92/entry.json`,
170 | which contains the following entry:
171 |
172 | https://github.com/tiziano88/ent-index/blob/fddaa4b78ec4f4ba1e2c1e3e1c0b5ae9b06565e2/sha256/4c/35/01/63/71/5b/7b/1d/0f/c3/bc/bf/11/bf/ff/c0/cf/2d/10/7f/69/25/3f/23/71/11/a7/48/08/09/e1/92/entry.json#L1
173 |
174 | Note that the Ent index only stores URLs, not actual data, under the
175 | _assumption_ that each URL will keep pointing to the same file forever.
176 |
177 | The client querying the index is responsible to verify that the target file
178 | still corresponds to the expected digest; if this validation fails, it means
179 | that the URL was moved to point to a diferent file after it was added to the Ent
180 | index.
181 |
182 | ### Updating the index
183 |
184 | Currently, entries may be added to the default index by creating a comment in
185 | https://github.com/tiziano88/ent-index/issues/1 containing the URL of the file
186 | to index. A GitHub actions is then triggered that fetches the file, creates an
187 | appropriate entry in the index, and commits that back into the git repository.
188 |
189 | You can try this by picking a URL of an existing file, and creating a comment in
190 | https://github.com/tiziano88/ent-index/issues/1 ; after a few minutes, the
191 | GitHub action should post another comment in reply, confirming that the entry
192 | was correctly incorporated in the index, and printing out its digest, which may
193 | then be used with the Ent CLI as above.
194 |
195 | If the URL stops pointing to the file that was originally indexed, the Ent CLI
196 | will detect that and produce an error.
197 |
198 | There is no process for cleaning up / fixing inconsistent entries in the index
199 | (yet).
200 |
201 | ## Comparison with other systems
202 |
203 | ### IPFS
204 |
205 | https://ipfs.io/
206 |
207 | IPFS aims to be a fully decentralized and censorship-resistant protocol and
208 | ecosystem, which heavily relies on content-addressability.
209 |
--------------------------------------------------------------------------------
/cmd/ent-server/grpc.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2021 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package main
17 |
18 | import (
19 | "bytes"
20 | "context"
21 | "io"
22 | "time"
23 |
24 | "cloud.google.com/go/storage"
25 | "github.com/google/ent/log"
26 | pb "github.com/google/ent/proto"
27 | "github.com/google/ent/utils"
28 | "google.golang.org/grpc/codes"
29 | "google.golang.org/grpc/status"
30 | )
31 |
32 | func redact(s string) string {
33 | if len(s) > 4 {
34 | return s[:4] + "..."
35 | }
36 | return s
37 | }
38 |
39 | type grpcServer struct {
40 | pb.UnimplementedEntServer
41 | }
42 |
43 | var _ pb.EntServer = grpcServer{}
44 |
45 | // GetEntry implements ent.EntServer
46 | func (grpcServer) GetEntry(req *pb.GetEntryRequest, s pb.Ent_GetEntryServer) error {
47 | ctx := s.Context()
48 | log.Infof(ctx, "GetEntry req: %s", req)
49 | accessItem := &LogItemGet{
50 | // TODO
51 | Source: SourceAPI,
52 | }
53 | defer LogGet(ctx, accessItem)
54 |
55 | apiKey := getAPIKeyGRPC(ctx)
56 | log.Debugf(ctx, "apiKey: %q", redact(apiKey))
57 | user := apiKeyToUser[apiKey]
58 | if user == nil {
59 | log.Warningf(ctx, "invalid API key: %q", redact(apiKey))
60 | return status.Errorf(codes.PermissionDenied, "invalid API key: %q", redact(apiKey))
61 | }
62 | log.Debugf(ctx, "user: %q %d", user.Name, user.ID)
63 | log.Debugf(ctx, "perms: read:%v write:%v", user.CanRead, user.CanWrite)
64 | if !user.CanRead {
65 | log.Warningf(ctx, "user %d does not have read permission", user.ID)
66 | return status.Errorf(codes.PermissionDenied, "user %d does not have read permission", user.ID)
67 | }
68 | accessItem.UserID = int64(user.ID)
69 |
70 | digest := utils.DigestFromProto(req.Digest)
71 | log.Debugf(ctx, "digest: %q", digest.String())
72 |
73 | log.Debugf(ctx, "getting blob: %q", digest.String())
74 | blob, err := blobStore.Get(ctx, digest)
75 | if err == storage.ErrObjectNotExist {
76 | log.Warningf(ctx, "blob not found: %q", digest.String())
77 | return status.Errorf(codes.NotFound, "blob not found: %q", digest.String())
78 | } else if err != nil {
79 | log.Warningf(ctx, "could not get blob: %s", err)
80 | return status.Errorf(codes.Internal, "could not get blob: %s", err)
81 | }
82 | log.Debugf(ctx, "got blob: %q", digest.String())
83 |
84 | err = s.Send(&pb.GetEntryResponse{
85 | Entry: &pb.GetEntryResponse_Metadata{
86 | Metadata: &pb.EntryMetadata{
87 | Digests: []*pb.Digest{
88 | utils.DigestToProto(digest),
89 | },
90 | },
91 | },
92 | })
93 | if err != nil {
94 | log.Warningf(ctx, "could not send response: %s", err)
95 | return status.Errorf(codes.Internal, "could not send response: %s", err)
96 | }
97 |
98 | err = s.Send(&pb.GetEntryResponse{
99 | Entry: &pb.GetEntryResponse_Chunk{
100 | Chunk: &pb.Chunk{
101 | Data: blob,
102 | },
103 | },
104 | })
105 | if err != nil {
106 | log.Warningf(ctx, "could not send response: %s", err)
107 | return status.Errorf(codes.Internal, "could not send response: %s", err)
108 | }
109 |
110 | return nil
111 | }
112 |
113 | func (grpcServer) GetEntryMetadata(ctx context.Context, req *pb.GetEntryMetadataRequest) (*pb.GetEntryMetadataResponse, error) {
114 | log.Infof(ctx, "HasEntry req: %s", req)
115 | accessItem := &LogItemGet{
116 | // TODO
117 | Source: SourceAPI,
118 | }
119 | defer LogGet(ctx, accessItem)
120 |
121 | apiKey := getAPIKeyGRPC(ctx)
122 | log.Debugf(ctx, "apiKey: %q", redact(apiKey))
123 | user := apiKeyToUser[apiKey]
124 | if user == nil {
125 | log.Warningf(ctx, "invalid API key: %q", redact(apiKey))
126 | return nil, status.Errorf(codes.PermissionDenied, "invalid API key: %q", redact(apiKey))
127 | }
128 | log.Debugf(ctx, "user: %q %d", user.Name, user.ID)
129 | log.Debugf(ctx, "perms: read:%v write:%v", user.CanRead, user.CanWrite)
130 | if !user.CanRead {
131 | log.Warningf(ctx, "user %d does not have read permission", user.ID)
132 | return nil, status.Errorf(codes.PermissionDenied, "user %d does not have read permission", user.ID)
133 | }
134 | accessItem.UserID = int64(user.ID)
135 |
136 | digest := utils.DigestFromProto(req.Digest)
137 | log.Debugf(ctx, "digest: %q", digest.String())
138 |
139 | log.Debugf(ctx, "getting blob: %q", digest.String())
140 | ok, err := blobStore.Has(ctx, digest)
141 | if err != nil {
142 | log.Warningf(ctx, "could not get blob: %s", err)
143 | return nil, status.Errorf(codes.Internal, "could not get blob: %s", err)
144 | }
145 | log.Debugf(ctx, "got blob: %q = %v", digest.String(), ok)
146 |
147 | if !ok {
148 | return nil, status.Errorf(codes.NotFound, "blob not found: %q", digest.String())
149 | }
150 |
151 | res := &pb.GetEntryMetadataResponse{
152 | Metadata: &pb.EntryMetadata{
153 | Digests: []*pb.Digest{
154 | utils.DigestToProto(digest),
155 | },
156 | },
157 | }
158 |
159 | return res, nil
160 | }
161 |
162 | // PutEntry implements ent.EntServer
163 | func (grpcServer) PutEntry(s pb.Ent_PutEntryServer) error {
164 | ctx := s.Context()
165 | accessItem := &LogItemPut{
166 | // TODO
167 | Source: SourceAPI,
168 | }
169 | defer LogPut(ctx, accessItem)
170 |
171 | apiKey := getAPIKeyGRPC(ctx)
172 | log.Debugf(ctx, "apiKey: %q", redact(apiKey))
173 | user := apiKeyToUser[apiKey]
174 | if user == nil {
175 | log.Warningf(ctx, "invalid API key: %q", redact(apiKey))
176 | return status.Errorf(codes.PermissionDenied, "invalid API key: %q", redact(apiKey))
177 | }
178 | log.Debugf(ctx, "user: %q %d", user.Name, user.ID)
179 | log.Debugf(ctx, "perms: read:%v write:%v", user.CanRead, user.CanWrite)
180 | if !user.CanWrite {
181 | log.Warningf(ctx, "user %d does not have write permission", user.ID)
182 | return status.Errorf(codes.PermissionDenied, "user %d does not have write permission", user.ID)
183 | }
184 | accessItem.UserID = int64(user.ID)
185 |
186 | // TODO: Use correct size.
187 | blob := make([]byte, 0, 1024*1024)
188 |
189 | next := true
190 | for next {
191 | req, err := s.Recv()
192 | if err == io.EOF {
193 | log.Infof(ctx, "received EOF")
194 | blob = append(blob, req.GetChunk().GetData()...)
195 | next = false
196 | } else if err != nil {
197 | log.Warningf(ctx, "could not receive request: %s", err)
198 | return status.Errorf(codes.Internal, "could not receive request: %s", err)
199 | } else {
200 | blob = append(blob, req.GetChunk().GetData()...)
201 | }
202 | }
203 |
204 | digest := utils.ComputeDigest(blob)
205 |
206 | exists, err := blobStore.Has(ctx, digest)
207 | if err != nil {
208 | log.Errorf(ctx, "error checking blob existence: %s", err)
209 | accessItem.NotCreated = append(accessItem.NotCreated, digest.String())
210 | }
211 | if exists {
212 | log.Infof(ctx, "blob %q already exists", digest)
213 | accessItem.NotCreated = append(accessItem.NotCreated, digest.String())
214 | // We count the blob as created, even though it already existed.
215 | } else {
216 | log.Infof(ctx, "adding blob: %q", digest)
217 | digest1, err := blobStore.Put(ctx, blob)
218 | if !bytes.Equal(digest1, digest) {
219 | log.Errorf(ctx, "mismatching digest, expected %q, got %q", digest.String(), digest1.String())
220 | }
221 | accessItem.Digest = append(accessItem.Digest, digest1.String())
222 | if err != nil {
223 | log.Errorf(ctx, "error adding blob: %s", err)
224 | accessItem.NotCreated = append(accessItem.NotCreated, digest1.String())
225 | }
226 | log.Infof(ctx, "added blob: %q", digest1.String())
227 | accessItem.Created = append(accessItem.Created, digest1.String())
228 | }
229 | res := &pb.PutEntryResponse{
230 | Metadata: &pb.EntryMetadata{
231 | Digests: []*pb.Digest{
232 | utils.DigestToProto(digest),
233 | },
234 | },
235 | }
236 | err = s.SendAndClose(res)
237 | if err != nil {
238 | log.Warningf(ctx, "could not send response: %s", err)
239 | return status.Errorf(codes.Internal, "could not send response: %s", err)
240 | }
241 | return nil
242 | }
243 |
244 | // GetTag implements ent.EntServer
245 | func (grpcServer) GetTag(ctx context.Context, req *pb.GetTagRequest) (*pb.GetTagResponse, error) {
246 | log.Debugf(ctx, "req: %s", req)
247 |
248 | entry, err := store.GetMapEntry(ctx, req.PublicKey, req.Label)
249 | if err != nil {
250 | log.Errorf(ctx, "could not get tag: %s", err)
251 | return nil, status.Errorf(codes.Internal, "could not get tag: %s", err)
252 | }
253 | if entry == nil {
254 | log.Debugf(ctx, "tag not found")
255 | return &pb.GetTagResponse{}, nil
256 | }
257 | log.Infof(ctx, "tag found: %v", entry)
258 | return &pb.GetTagResponse{
259 | SignedTag: &pb.SignedTag{
260 | Tag: &pb.Tag{
261 | Label: entry.Label,
262 | Target: &pb.Digest{
263 | Code: uint64(entry.Target.Code),
264 | Digest: entry.Target.Digest,
265 | },
266 | },
267 | },
268 | }, nil
269 |
270 | }
271 |
272 | // SetTag implements ent.EntServer
273 | func (grpcServer) SetTag(ctx context.Context, req *pb.SetTagRequest) (*pb.SetTagResponse, error) {
274 | log.Debugf(ctx, "req: %s", req)
275 |
276 | e := MapEntry{
277 | PublicKey: req.SignedTag.PublicKey,
278 | Label: req.SignedTag.Tag.Label,
279 | Target: Digest{
280 | Code: int64(req.SignedTag.Tag.Target.Code),
281 | Digest: req.SignedTag.Tag.Target.Digest,
282 | },
283 | EntrySignature: req.SignedTag.TagSignature,
284 | CreationTime: time.Now(),
285 | }
286 | // TODO: Check signature.
287 | err := store.SetMapEntry(ctx, &e)
288 | if err != nil {
289 | log.Errorf(ctx, "could not set tag: %s", err)
290 | return nil, status.Errorf(codes.Internal, "could not set tag: %s", err)
291 | }
292 | log.Infof(ctx, "set tag: %v", e)
293 |
294 | return &pb.SetTagResponse{}, nil
295 | }
296 |
--------------------------------------------------------------------------------
/schema/schema.go:
--------------------------------------------------------------------------------
1 | //
2 | // Copyright 2022 The Ent Authors.
3 | //
4 | // Licensed under the Apache License, Version 2.0 (the "License");
5 | // you may not use this file except in compliance with the License.
6 | // You may obtain a copy of the License at
7 | //
8 | // http://www.apache.org/licenses/LICENSE-2.0
9 | //
10 | // Unless required by applicable law or agreed to in writing, software
11 | // distributed under the License is distributed on an "AS IS" BASIS,
12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | // See the License for the specific language governing permissions and
14 | // limitations under the License.
15 |
16 | package schema
17 |
18 | import (
19 | "bytes"
20 | "context"
21 | "encoding/binary"
22 | "fmt"
23 | "io"
24 | "reflect"
25 | "strconv"
26 |
27 | "github.com/google/ent/log"
28 | "github.com/google/ent/nodeservice"
29 | "github.com/google/ent/utils"
30 | "github.com/ipfs/go-cid"
31 | "github.com/multiformats/go-multihash"
32 | )
33 |
34 | type Schema struct {
35 | // #0 is the root node kind.
36 | Kinds []Kind `ent:"0"`
37 | }
38 |
39 | type Kind struct {
40 | KindID uint32 `ent:"0"`
41 | Name string `ent:"1"`
42 | Fields []Field `ent:"2"`
43 | }
44 |
45 | type Field struct {
46 | FieldID uint32 `ent:"0"`
47 | Name string `ent:"1"`
48 | KindID uint32 `ent:"2"`
49 | Raw uint32 `ent:"3"`
50 | }
51 |
52 | func ResolveLink(o nodeservice.ObjectGetter, base utils.Digest, path []utils.Selector) (utils.Digest, error) {
53 | if len(path) == 0 {
54 | return base, nil
55 | } else {
56 | object, err := o.Get(context.Background(), base)
57 | if err != nil {
58 | return utils.Digest{}, fmt.Errorf("failed to get object: %v", err)
59 | }
60 | node, err := utils.ParseDAGNode(object)
61 | if err != nil {
62 | return utils.Digest{}, fmt.Errorf("failed to parse object: %v", err)
63 | }
64 | selector := path[0]
65 | newBase := node.Links[selector]
66 | return ResolveLink(o, utils.Digest(newBase.Hash()), path[1:])
67 | }
68 | }
69 |
70 | func GetFieldWithId(v interface{}, fieldID uint64) (reflect.Value, error) {
71 | rv := reflect.ValueOf(v).Elem()
72 | for i := 0; i < rv.NumField(); i++ {
73 | typeField := rv.Type().Field(i)
74 | tag := typeField.Tag.Get("ent")
75 | id, err := strconv.Atoi(tag)
76 | if err != nil {
77 | return reflect.Value{}, fmt.Errorf("failed to parse field id: %v", err)
78 | }
79 | if uint64(id) == fieldID {
80 | return rv.Field(i), nil
81 | }
82 | }
83 | return reflect.Value{}, fmt.Errorf("failed to find field with id: %v", fieldID)
84 | }
85 |
86 | func GetStruct(o nodeservice.ObjectGetter, digest utils.Digest, v interface{}) error {
87 | log.Debugf(context.Background(), "getting struct %s", digest.String())
88 | object, err := o.Get(context.Background(), digest)
89 | if err != nil {
90 | return fmt.Errorf("failed to get struct object: %v", err)
91 | }
92 | node, err := utils.ParseDAGNode(object)
93 | if err != nil {
94 | return fmt.Errorf("failed to parse struct object: %v", err)
95 | }
96 | log.Debugf(context.Background(), "parsed node: %+v", node)
97 | r := bytes.NewReader(node.Bytes)
98 | linkIndex := 0
99 | for {
100 | field, err := utils.DecodeField(r)
101 | if err == io.EOF {
102 | break
103 | } else if err != nil {
104 | return fmt.Errorf("failed to decode field: %v", err)
105 | }
106 | fieldValue, err := GetFieldWithId(v, field.ID)
107 | if err != nil {
108 | return fmt.Errorf("failed to get field with id: %v", err)
109 | }
110 | switch field.Type {
111 | case utils.FieldTypeInt:
112 | switch fieldValue.Kind() {
113 | case reflect.Slice:
114 | // Append to existing slice.
115 | fieldValue.Set(reflect.Append(fieldValue, reflect.ValueOf(field.UintValue).Convert(fieldValue.Type().Elem())))
116 | case reflect.Int32, reflect.Int64, reflect.Int:
117 | fieldValue.SetInt(int64(field.UintValue))
118 | case reflect.Uint32, reflect.Uint64, reflect.Uint:
119 | fieldValue.SetUint(uint64(field.UintValue))
120 | default:
121 | return fmt.Errorf("unexpected field type for int: %v", fieldValue.Kind())
122 | }
123 | case utils.FieldTypeBytes:
124 | switch fieldValue.Kind() {
125 | case reflect.Slice:
126 | switch fieldValue.Type().Elem().Kind() {
127 | case reflect.Uint8: // []byte
128 | fieldValue.SetBytes(field.BytesValue)
129 | case reflect.String: // []string
130 | fieldValue.Set(reflect.Append(fieldValue, reflect.ValueOf(string(field.BytesValue))))
131 | default:
132 | return fmt.Errorf("unexpected field type for bytes: %v", fieldValue.Type().Elem().Kind())
133 | }
134 | case reflect.String:
135 | fieldValue.SetString(string(field.BytesValue))
136 | default:
137 | return fmt.Errorf("unsupported field type: %v", fieldValue.Kind())
138 | }
139 | case utils.FieldTypeMsg:
140 | switch fieldValue.Kind() {
141 | case reflect.Slice:
142 | if field.UintValue != 1 {
143 | return fmt.Errorf("no presence bit for repeated field: %v", field.UintValue)
144 | }
145 | link := node.Links[linkIndex]
146 | linkIndex++
147 | v := reflect.New(fieldValue.Type().Elem())
148 | if err := GetStruct(o, utils.Digest(link.Hash()), v.Interface()); err != nil {
149 | return fmt.Errorf("failed to get struct: %v", err)
150 | }
151 | fieldValue.Set(reflect.Append(fieldValue, v.Elem()))
152 | case reflect.Ptr:
153 | case reflect.Struct:
154 | if field.UintValue != 1 {
155 | return fmt.Errorf("no presence bit for repeated field: %v", field.UintValue)
156 | }
157 | link := node.Links[linkIndex]
158 | linkIndex++
159 | if err := GetStruct(o, utils.Digest(link.Hash()), fieldValue.Addr().Interface()); err != nil {
160 | return fmt.Errorf("failed to get struct: %v", err)
161 | }
162 | }
163 | default:
164 | return fmt.Errorf("unsupported field type: %v", field.Type)
165 | }
166 | }
167 | return nil
168 | }
169 |
170 | func PutStruct(o nodeservice.ObjectStore, v interface{}) (utils.Digest, error) {
171 | rv := reflect.ValueOf(v)
172 | if rv.Kind() == reflect.Ptr {
173 | rv = rv.Elem()
174 | }
175 | bytes := &bytes.Buffer{}
176 | links := []cid.Cid{}
177 | // TODO: Traverse in order of field id.
178 | for i := 0; i < rv.NumField(); i++ {
179 | typeField := rv.Type().Field(i)
180 | tag := typeField.Tag.Get("ent")
181 | fieldID, err := strconv.Atoi(tag)
182 | if err != nil {
183 | return utils.Digest{}, fmt.Errorf("failed to parse field id: %v", err)
184 | }
185 | fieldValue := rv.Field(i)
186 | switch typeField.Type.Kind() {
187 | case reflect.Uint32, reflect.Uint64, reflect.Uint:
188 | log.Infof(nil, "putting int: %+v", fieldValue)
189 | utils.EncodeField(bytes, &utils.Field{
190 | ID: uint64(fieldID),
191 | Type: utils.FieldTypeInt,
192 | UintValue: fieldValue.Uint(),
193 | BytesValue: nil,
194 | })
195 | case reflect.String:
196 | log.Infof(nil, "putting string: %+v", fieldValue)
197 | utils.EncodeField(bytes, &utils.Field{
198 | ID: uint64(fieldID),
199 | Type: utils.FieldTypeBytes,
200 | UintValue: 0,
201 | BytesValue: []byte(fieldValue.String()),
202 | })
203 | case reflect.Struct:
204 | digest, err := PutStruct(o, fieldValue.Interface())
205 | if err != nil {
206 | return utils.Digest{}, fmt.Errorf("failed to put struct field: %v", err)
207 | }
208 | utils.EncodeField(bytes, &utils.Field{
209 | ID: uint64(fieldID),
210 | Type: utils.FieldTypeMsg,
211 | UintValue: 1, // Present
212 | BytesValue: nil,
213 | })
214 | links = append(links, cid.NewCidV1(utils.TypeDAG, multihash.Multihash(digest)))
215 | case reflect.Slice:
216 | switch typeField.Type.Elem().Kind() {
217 | case reflect.Uint32, reflect.Uint64, reflect.Uint:
218 | for i := 0; i < fieldValue.Len(); i++ {
219 | iv := fieldValue.Index(i).Uint()
220 | utils.EncodeField(bytes, &utils.Field{
221 | ID: uint64(fieldID),
222 | Type: utils.FieldTypeInt,
223 | UintValue: iv,
224 | BytesValue: nil,
225 | })
226 | }
227 | case reflect.String:
228 | for i := 0; i < fieldValue.Len(); i++ {
229 | iv := fieldValue.Index(i).String()
230 | utils.EncodeField(bytes, &utils.Field{
231 | ID: uint64(fieldID),
232 | Type: utils.FieldTypeBytes,
233 | UintValue: 0,
234 | BytesValue: []byte(iv),
235 | })
236 | }
237 | case reflect.Struct:
238 | for i := 0; i < fieldValue.Len(); i++ {
239 | iv := fieldValue.Index(i).Interface()
240 | digest, err := PutStruct(o, iv)
241 | if err != nil {
242 | return utils.Digest{}, fmt.Errorf("failed to put string field: %v", err)
243 | }
244 | utils.EncodeField(bytes, &utils.Field{
245 | ID: uint64(fieldID),
246 | Type: utils.FieldTypeMsg,
247 | UintValue: 1, // Present
248 | BytesValue: nil,
249 | })
250 | links = append(links, cid.NewCidV1(utils.TypeDAG, multihash.Multihash(digest)))
251 | }
252 | default:
253 | return utils.Digest{}, fmt.Errorf("unsupported field type: %v", typeField.Type.Elem().Kind())
254 | }
255 | default:
256 | return utils.Digest{}, fmt.Errorf("unsupported field type: %v", typeField.Type.Kind())
257 | }
258 | }
259 | node := utils.DAGNode{
260 | Bytes: bytes.Bytes(),
261 | Links: links,
262 | }
263 | b, err := utils.SerializeDAGNode(&node)
264 | if err != nil {
265 | return utils.Digest{}, fmt.Errorf("failed to serialize node: %v", err)
266 | }
267 | digest, err := o.Put(context.Background(), b)
268 | log.Infof(nil, "putting node: %+v -> %s", node, digest)
269 | return digest, err
270 | }
271 |
272 | func GetUint32(o nodeservice.ObjectGetter, digest utils.Digest) (uint32, error) {
273 | b, err := o.Get(context.Background(), digest)
274 | if err != nil {
275 | return 0, fmt.Errorf("failed to get struct object: %v", err)
276 | }
277 | v := binary.BigEndian.Uint32(b)
278 | return v, nil
279 | }
280 |
281 | func PutUint32(o nodeservice.ObjectStore, v uint32) (utils.Digest, error) {
282 | b := make([]byte, 4)
283 | binary.BigEndian.PutUint32(b, v)
284 | return o.Put(context.Background(), b)
285 | }
286 |
287 | func GetString(o nodeservice.ObjectGetter, digest utils.Digest) (string, error) {
288 | b, err := o.Get(context.Background(), digest)
289 | if err != nil {
290 | return "", fmt.Errorf("failed to get struct object: %v", err)
291 | }
292 | v := string(b)
293 | return v, nil
294 | }
295 |
296 | func PutString(o nodeservice.ObjectStore, v string) (utils.Digest, error) {
297 | b := []byte(v)
298 | return o.Put(context.Background(), b)
299 | }
300 |
--------------------------------------------------------------------------------
/proto/ent_server_api_grpc.pb.go:
--------------------------------------------------------------------------------
1 | // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
2 | // versions:
3 | // - protoc-gen-go-grpc v1.2.0
4 | // - protoc v3.21.8
5 | // source: proto/ent_server_api.proto
6 |
7 | package ent
8 |
9 | import (
10 | context "context"
11 | grpc "google.golang.org/grpc"
12 | codes "google.golang.org/grpc/codes"
13 | status "google.golang.org/grpc/status"
14 | )
15 |
16 | // This is a compile-time assertion to ensure that this generated file
17 | // is compatible with the grpc package it is being compiled against.
18 | // Requires gRPC-Go v1.32.0 or later.
19 | const _ = grpc.SupportPackageIsVersion7
20 |
21 | // EntClient is the client API for Ent service.
22 | //
23 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
24 | type EntClient interface {
25 | GetTag(ctx context.Context, in *GetTagRequest, opts ...grpc.CallOption) (*GetTagResponse, error)
26 | SetTag(ctx context.Context, in *SetTagRequest, opts ...grpc.CallOption) (*SetTagResponse, error)
27 | GetEntry(ctx context.Context, in *GetEntryRequest, opts ...grpc.CallOption) (Ent_GetEntryClient, error)
28 | GetEntryMetadata(ctx context.Context, in *GetEntryMetadataRequest, opts ...grpc.CallOption) (*GetEntryMetadataResponse, error)
29 | PutEntry(ctx context.Context, opts ...grpc.CallOption) (Ent_PutEntryClient, error)
30 | }
31 |
32 | type entClient struct {
33 | cc grpc.ClientConnInterface
34 | }
35 |
36 | func NewEntClient(cc grpc.ClientConnInterface) EntClient {
37 | return &entClient{cc}
38 | }
39 |
40 | func (c *entClient) GetTag(ctx context.Context, in *GetTagRequest, opts ...grpc.CallOption) (*GetTagResponse, error) {
41 | out := new(GetTagResponse)
42 | err := c.cc.Invoke(ctx, "/ent.server.api.Ent/GetTag", in, out, opts...)
43 | if err != nil {
44 | return nil, err
45 | }
46 | return out, nil
47 | }
48 |
49 | func (c *entClient) SetTag(ctx context.Context, in *SetTagRequest, opts ...grpc.CallOption) (*SetTagResponse, error) {
50 | out := new(SetTagResponse)
51 | err := c.cc.Invoke(ctx, "/ent.server.api.Ent/SetTag", in, out, opts...)
52 | if err != nil {
53 | return nil, err
54 | }
55 | return out, nil
56 | }
57 |
58 | func (c *entClient) GetEntry(ctx context.Context, in *GetEntryRequest, opts ...grpc.CallOption) (Ent_GetEntryClient, error) {
59 | stream, err := c.cc.NewStream(ctx, &Ent_ServiceDesc.Streams[0], "/ent.server.api.Ent/GetEntry", opts...)
60 | if err != nil {
61 | return nil, err
62 | }
63 | x := &entGetEntryClient{stream}
64 | if err := x.ClientStream.SendMsg(in); err != nil {
65 | return nil, err
66 | }
67 | if err := x.ClientStream.CloseSend(); err != nil {
68 | return nil, err
69 | }
70 | return x, nil
71 | }
72 |
73 | type Ent_GetEntryClient interface {
74 | Recv() (*GetEntryResponse, error)
75 | grpc.ClientStream
76 | }
77 |
78 | type entGetEntryClient struct {
79 | grpc.ClientStream
80 | }
81 |
82 | func (x *entGetEntryClient) Recv() (*GetEntryResponse, error) {
83 | m := new(GetEntryResponse)
84 | if err := x.ClientStream.RecvMsg(m); err != nil {
85 | return nil, err
86 | }
87 | return m, nil
88 | }
89 |
90 | func (c *entClient) GetEntryMetadata(ctx context.Context, in *GetEntryMetadataRequest, opts ...grpc.CallOption) (*GetEntryMetadataResponse, error) {
91 | out := new(GetEntryMetadataResponse)
92 | err := c.cc.Invoke(ctx, "/ent.server.api.Ent/GetEntryMetadata", in, out, opts...)
93 | if err != nil {
94 | return nil, err
95 | }
96 | return out, nil
97 | }
98 |
99 | func (c *entClient) PutEntry(ctx context.Context, opts ...grpc.CallOption) (Ent_PutEntryClient, error) {
100 | stream, err := c.cc.NewStream(ctx, &Ent_ServiceDesc.Streams[1], "/ent.server.api.Ent/PutEntry", opts...)
101 | if err != nil {
102 | return nil, err
103 | }
104 | x := &entPutEntryClient{stream}
105 | return x, nil
106 | }
107 |
108 | type Ent_PutEntryClient interface {
109 | Send(*PutEntryRequest) error
110 | CloseAndRecv() (*PutEntryResponse, error)
111 | grpc.ClientStream
112 | }
113 |
114 | type entPutEntryClient struct {
115 | grpc.ClientStream
116 | }
117 |
118 | func (x *entPutEntryClient) Send(m *PutEntryRequest) error {
119 | return x.ClientStream.SendMsg(m)
120 | }
121 |
122 | func (x *entPutEntryClient) CloseAndRecv() (*PutEntryResponse, error) {
123 | if err := x.ClientStream.CloseSend(); err != nil {
124 | return nil, err
125 | }
126 | m := new(PutEntryResponse)
127 | if err := x.ClientStream.RecvMsg(m); err != nil {
128 | return nil, err
129 | }
130 | return m, nil
131 | }
132 |
133 | // EntServer is the server API for Ent service.
134 | // All implementations must embed UnimplementedEntServer
135 | // for forward compatibility
136 | type EntServer interface {
137 | GetTag(context.Context, *GetTagRequest) (*GetTagResponse, error)
138 | SetTag(context.Context, *SetTagRequest) (*SetTagResponse, error)
139 | GetEntry(*GetEntryRequest, Ent_GetEntryServer) error
140 | GetEntryMetadata(context.Context, *GetEntryMetadataRequest) (*GetEntryMetadataResponse, error)
141 | PutEntry(Ent_PutEntryServer) error
142 | mustEmbedUnimplementedEntServer()
143 | }
144 |
145 | // UnimplementedEntServer must be embedded to have forward compatible implementations.
146 | type UnimplementedEntServer struct {
147 | }
148 |
149 | func (UnimplementedEntServer) GetTag(context.Context, *GetTagRequest) (*GetTagResponse, error) {
150 | return nil, status.Errorf(codes.Unimplemented, "method GetTag not implemented")
151 | }
152 | func (UnimplementedEntServer) SetTag(context.Context, *SetTagRequest) (*SetTagResponse, error) {
153 | return nil, status.Errorf(codes.Unimplemented, "method SetTag not implemented")
154 | }
155 | func (UnimplementedEntServer) GetEntry(*GetEntryRequest, Ent_GetEntryServer) error {
156 | return status.Errorf(codes.Unimplemented, "method GetEntry not implemented")
157 | }
158 | func (UnimplementedEntServer) GetEntryMetadata(context.Context, *GetEntryMetadataRequest) (*GetEntryMetadataResponse, error) {
159 | return nil, status.Errorf(codes.Unimplemented, "method GetEntryMetadata not implemented")
160 | }
161 | func (UnimplementedEntServer) PutEntry(Ent_PutEntryServer) error {
162 | return status.Errorf(codes.Unimplemented, "method PutEntry not implemented")
163 | }
164 | func (UnimplementedEntServer) mustEmbedUnimplementedEntServer() {}
165 |
166 | // UnsafeEntServer may be embedded to opt out of forward compatibility for this service.
167 | // Use of this interface is not recommended, as added methods to EntServer will
168 | // result in compilation errors.
169 | type UnsafeEntServer interface {
170 | mustEmbedUnimplementedEntServer()
171 | }
172 |
173 | func RegisterEntServer(s grpc.ServiceRegistrar, srv EntServer) {
174 | s.RegisterService(&Ent_ServiceDesc, srv)
175 | }
176 |
177 | func _Ent_GetTag_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
178 | in := new(GetTagRequest)
179 | if err := dec(in); err != nil {
180 | return nil, err
181 | }
182 | if interceptor == nil {
183 | return srv.(EntServer).GetTag(ctx, in)
184 | }
185 | info := &grpc.UnaryServerInfo{
186 | Server: srv,
187 | FullMethod: "/ent.server.api.Ent/GetTag",
188 | }
189 | handler := func(ctx context.Context, req interface{}) (interface{}, error) {
190 | return srv.(EntServer).GetTag(ctx, req.(*GetTagRequest))
191 | }
192 | return interceptor(ctx, in, info, handler)
193 | }
194 |
195 | func _Ent_SetTag_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
196 | in := new(SetTagRequest)
197 | if err := dec(in); err != nil {
198 | return nil, err
199 | }
200 | if interceptor == nil {
201 | return srv.(EntServer).SetTag(ctx, in)
202 | }
203 | info := &grpc.UnaryServerInfo{
204 | Server: srv,
205 | FullMethod: "/ent.server.api.Ent/SetTag",
206 | }
207 | handler := func(ctx context.Context, req interface{}) (interface{}, error) {
208 | return srv.(EntServer).SetTag(ctx, req.(*SetTagRequest))
209 | }
210 | return interceptor(ctx, in, info, handler)
211 | }
212 |
213 | func _Ent_GetEntry_Handler(srv interface{}, stream grpc.ServerStream) error {
214 | m := new(GetEntryRequest)
215 | if err := stream.RecvMsg(m); err != nil {
216 | return err
217 | }
218 | return srv.(EntServer).GetEntry(m, &entGetEntryServer{stream})
219 | }
220 |
221 | type Ent_GetEntryServer interface {
222 | Send(*GetEntryResponse) error
223 | grpc.ServerStream
224 | }
225 |
226 | type entGetEntryServer struct {
227 | grpc.ServerStream
228 | }
229 |
230 | func (x *entGetEntryServer) Send(m *GetEntryResponse) error {
231 | return x.ServerStream.SendMsg(m)
232 | }
233 |
234 | func _Ent_GetEntryMetadata_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
235 | in := new(GetEntryMetadataRequest)
236 | if err := dec(in); err != nil {
237 | return nil, err
238 | }
239 | if interceptor == nil {
240 | return srv.(EntServer).GetEntryMetadata(ctx, in)
241 | }
242 | info := &grpc.UnaryServerInfo{
243 | Server: srv,
244 | FullMethod: "/ent.server.api.Ent/GetEntryMetadata",
245 | }
246 | handler := func(ctx context.Context, req interface{}) (interface{}, error) {
247 | return srv.(EntServer).GetEntryMetadata(ctx, req.(*GetEntryMetadataRequest))
248 | }
249 | return interceptor(ctx, in, info, handler)
250 | }
251 |
252 | func _Ent_PutEntry_Handler(srv interface{}, stream grpc.ServerStream) error {
253 | return srv.(EntServer).PutEntry(&entPutEntryServer{stream})
254 | }
255 |
256 | type Ent_PutEntryServer interface {
257 | SendAndClose(*PutEntryResponse) error
258 | Recv() (*PutEntryRequest, error)
259 | grpc.ServerStream
260 | }
261 |
262 | type entPutEntryServer struct {
263 | grpc.ServerStream
264 | }
265 |
266 | func (x *entPutEntryServer) SendAndClose(m *PutEntryResponse) error {
267 | return x.ServerStream.SendMsg(m)
268 | }
269 |
270 | func (x *entPutEntryServer) Recv() (*PutEntryRequest, error) {
271 | m := new(PutEntryRequest)
272 | if err := x.ServerStream.RecvMsg(m); err != nil {
273 | return nil, err
274 | }
275 | return m, nil
276 | }
277 |
278 | // Ent_ServiceDesc is the grpc.ServiceDesc for Ent service.
279 | // It's only intended for direct use with grpc.RegisterService,
280 | // and not to be introspected or modified (even as a copy)
281 | var Ent_ServiceDesc = grpc.ServiceDesc{
282 | ServiceName: "ent.server.api.Ent",
283 | HandlerType: (*EntServer)(nil),
284 | Methods: []grpc.MethodDesc{
285 | {
286 | MethodName: "GetTag",
287 | Handler: _Ent_GetTag_Handler,
288 | },
289 | {
290 | MethodName: "SetTag",
291 | Handler: _Ent_SetTag_Handler,
292 | },
293 | {
294 | MethodName: "GetEntryMetadata",
295 | Handler: _Ent_GetEntryMetadata_Handler,
296 | },
297 | },
298 | Streams: []grpc.StreamDesc{
299 | {
300 | StreamName: "GetEntry",
301 | Handler: _Ent_GetEntry_Handler,
302 | ServerStreams: true,
303 | },
304 | {
305 | StreamName: "PutEntry",
306 | Handler: _Ent_PutEntry_Handler,
307 | ClientStreams: true,
308 | },
309 | },
310 | Metadata: "proto/ent_server_api.proto",
311 | }
312 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 |
2 | Apache License
3 | Version 2.0, January 2004
4 | http://www.apache.org/licenses/
5 |
6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7 |
8 | 1. Definitions.
9 |
10 | "License" shall mean the terms and conditions for use, reproduction,
11 | and distribution as defined by Sections 1 through 9 of this document.
12 |
13 | "Licensor" shall mean the copyright owner or entity authorized by
14 | the copyright owner that is granting the License.
15 |
16 | "Legal Entity" shall mean the union of the acting entity and all
17 | other entities that control, are controlled by, or are under common
18 | control with that entity. For the purposes of this definition,
19 | "control" means (i) the power, direct or indirect, to cause the
20 | direction or management of such entity, whether by contract or
21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
22 | outstanding shares, or (iii) beneficial ownership of such entity.
23 |
24 | "You" (or "Your") shall mean an individual or Legal Entity
25 | exercising permissions granted by this License.
26 |
27 | "Source" form shall mean the preferred form for making modifications,
28 | including but not limited to software source code, documentation
29 | source, and configuration files.
30 |
31 | "Object" form shall mean any form resulting from mechanical
32 | transformation or translation of a Source form, including but
33 | not limited to compiled object code, generated documentation,
34 | and conversions to other media types.
35 |
36 | "Work" shall mean the work of authorship, whether in Source or
37 | Object form, made available under the License, as indicated by a
38 | copyright notice that is included in or attached to the work
39 | (an example is provided in the Appendix below).
40 |
41 | "Derivative Works" shall mean any work, whether in Source or Object
42 | form, that is based on (or derived from) the Work and for which the
43 | editorial revisions, annotations, elaborations, or other modifications
44 | represent, as a whole, an original work of authorship. For the purposes
45 | of this License, Derivative Works shall not include works that remain
46 | separable from, or merely link (or bind by name) to the interfaces of,
47 | the Work and Derivative Works thereof.
48 |
49 | "Contribution" shall mean any work of authorship, including
50 | the original version of the Work and any modifications or additions
51 | to that Work or Derivative Works thereof, that is intentionally
52 | submitted to Licensor for inclusion in the Work by the copyright owner
53 | or by an individual or Legal Entity authorized to submit on behalf of
54 | the copyright owner. For the purposes of this definition, "submitted"
55 | means any form of electronic, verbal, or written communication sent
56 | to the Licensor or its representatives, including but not limited to
57 | communication on electronic mailing lists, source code control systems,
58 | and issue tracking systems that are managed by, or on behalf of, the
59 | Licensor for the purpose of discussing and improving the Work, but
60 | excluding communication that is conspicuously marked or otherwise
61 | designated in writing by the copyright owner as "Not a Contribution."
62 |
63 | "Contributor" shall mean Licensor and any individual or Legal Entity
64 | on behalf of whom a Contribution has been received by Licensor and
65 | subsequently incorporated within the Work.
66 |
67 | 2. Grant of Copyright License. Subject to the terms and conditions of
68 | this License, each Contributor hereby grants to You a perpetual,
69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70 | copyright license to reproduce, prepare Derivative Works of,
71 | publicly display, publicly perform, sublicense, and distribute the
72 | Work and such Derivative Works in Source or Object form.
73 |
74 | 3. Grant of Patent License. Subject to the terms and conditions of
75 | this License, each Contributor hereby grants to You a perpetual,
76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77 | (except as stated in this section) patent license to make, have made,
78 | use, offer to sell, sell, import, and otherwise transfer the Work,
79 | where such license applies only to those patent claims licensable
80 | by such Contributor that are necessarily infringed by their
81 | Contribution(s) alone or by combination of their Contribution(s)
82 | with the Work to which such Contribution(s) was submitted. If You
83 | institute patent litigation against any entity (including a
84 | cross-claim or counterclaim in a lawsuit) alleging that the Work
85 | or a Contribution incorporated within the Work constitutes direct
86 | or contributory patent infringement, then any patent licenses
87 | granted to You under this License for that Work shall terminate
88 | as of the date such litigation is filed.
89 |
90 | 4. Redistribution. You may reproduce and distribute copies of the
91 | Work or Derivative Works thereof in any medium, with or without
92 | modifications, and in Source or Object form, provided that You
93 | meet the following conditions:
94 |
95 | (a) You must give any other recipients of the Work or
96 | Derivative Works a copy of this License; and
97 |
98 | (b) You must cause any modified files to carry prominent notices
99 | stating that You changed the files; and
100 |
101 | (c) You must retain, in the Source form of any Derivative Works
102 | that You distribute, all copyright, patent, trademark, and
103 | attribution notices from the Source form of the Work,
104 | excluding those notices that do not pertain to any part of
105 | the Derivative Works; and
106 |
107 | (d) If the Work includes a "NOTICE" text file as part of its
108 | distribution, then any Derivative Works that You distribute must
109 | include a readable copy of the attribution notices contained
110 | within such NOTICE file, excluding those notices that do not
111 | pertain to any part of the Derivative Works, in at least one
112 | of the following places: within a NOTICE text file distributed
113 | as part of the Derivative Works; within the Source form or
114 | documentation, if provided along with the Derivative Works; or,
115 | within a display generated by the Derivative Works, if and
116 | wherever such third-party notices normally appear. The contents
117 | of the NOTICE file are for informational purposes only and
118 | do not modify the License. You may add Your own attribution
119 | notices within Derivative Works that You distribute, alongside
120 | or as an addendum to the NOTICE text from the Work, provided
121 | that such additional attribution notices cannot be construed
122 | as modifying the License.
123 |
124 | You may add Your own copyright statement to Your modifications and
125 | may provide additional or different license terms and conditions
126 | for use, reproduction, or distribution of Your modifications, or
127 | for any such Derivative Works as a whole, provided Your use,
128 | reproduction, and distribution of the Work otherwise complies with
129 | the conditions stated in this License.
130 |
131 | 5. Submission of Contributions. Unless You explicitly state otherwise,
132 | any Contribution intentionally submitted for inclusion in the Work
133 | by You to the Licensor shall be under the terms and conditions of
134 | this License, without any additional terms or conditions.
135 | Notwithstanding the above, nothing herein shall supersede or modify
136 | the terms of any separate license agreement you may have executed
137 | with Licensor regarding such Contributions.
138 |
139 | 6. Trademarks. This License does not grant permission to use the trade
140 | names, trademarks, service marks, or product names of the Licensor,
141 | except as required for reasonable and customary use in describing the
142 | origin of the Work and reproducing the content of the NOTICE file.
143 |
144 | 7. Disclaimer of Warranty. Unless required by applicable law or
145 | agreed to in writing, Licensor provides the Work (and each
146 | Contributor provides its Contributions) on an "AS IS" BASIS,
147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 | implied, including, without limitation, any warranties or conditions
149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 | PARTICULAR PURPOSE. You are solely responsible for determining the
151 | appropriateness of using or redistributing the Work and assume any
152 | risks associated with Your exercise of permissions under this License.
153 |
154 | 8. Limitation of Liability. In no event and under no legal theory,
155 | whether in tort (including negligence), contract, or otherwise,
156 | unless required by applicable law (such as deliberate and grossly
157 | negligent acts) or agreed to in writing, shall any Contributor be
158 | liable to You for damages, including any direct, indirect, special,
159 | incidental, or consequential damages of any character arising as a
160 | result of this License or out of the use or inability to use the
161 | Work (including but not limited to damages for loss of goodwill,
162 | work stoppage, computer failure or malfunction, or any and all
163 | other commercial damages or losses), even if such Contributor
164 | has been advised of the possibility of such damages.
165 |
166 | 9. Accepting Warranty or Additional Liability. While redistributing
167 | the Work or Derivative Works thereof, You may choose to offer,
168 | and charge a fee for, acceptance of support, warranty, indemnity,
169 | or other liability obligations and/or rights consistent with this
170 | License. However, in accepting such obligations, You may act only
171 | on Your own behalf and on Your sole responsibility, not on behalf
172 | of any other Contributor, and only if You agree to indemnify,
173 | defend, and hold each Contributor harmless for any liability
174 | incurred by, or claims asserted against, such Contributor by reason
175 | of your accepting any such warranty or additional liability.
176 |
177 | END OF TERMS AND CONDITIONS
178 |
179 | APPENDIX: How to apply the Apache License to your work.
180 |
181 | To apply the Apache License to your work, attach the following
182 | boilerplate notice, with the fields enclosed by brackets "[]"
183 | replaced with your own identifying information. (Don't include
184 | the brackets!) The text should be enclosed in the appropriate
185 | comment syntax for the file format. We also recommend that a
186 | file or class name and description of purpose be included on the
187 | same "printed page" as the copyright notice for easier
188 | identification within third-party archives.
189 |
190 | Copyright [yyyy] [name of copyright owner]
191 |
192 | Licensed under the Apache License, Version 2.0 (the "License");
193 | you may not use this file except in compliance with the License.
194 | You may obtain a copy of the License at
195 |
196 | http://www.apache.org/licenses/LICENSE-2.0
197 |
198 | Unless required by applicable law or agreed to in writing, software
199 | distributed under the License is distributed on an "AS IS" BASIS,
200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201 | See the License for the specific language governing permissions and
202 | limitations under the License.
203 |
--------------------------------------------------------------------------------
/proto/ent_server_config.pb.go:
--------------------------------------------------------------------------------
1 | // Code generated by protoc-gen-go. DO NOT EDIT.
2 | // versions:
3 | // protoc-gen-go v1.28.1
4 | // protoc v3.21.8
5 | // source: proto/ent_server_config.proto
6 |
7 | package ent
8 |
9 | import (
10 | protoreflect "google.golang.org/protobuf/reflect/protoreflect"
11 | protoimpl "google.golang.org/protobuf/runtime/protoimpl"
12 | reflect "reflect"
13 | sync "sync"
14 | )
15 |
16 | const (
17 | // Verify that this generated code is sufficiently up-to-date.
18 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
19 | // Verify that runtime/protoimpl is sufficiently up-to-date.
20 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
21 | )
22 |
23 | type Config struct {
24 | state protoimpl.MessageState
25 | sizeCache protoimpl.SizeCache
26 | unknownFields protoimpl.UnknownFields
27 |
28 | ProjectId string `protobuf:"bytes,1,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"`
29 | ListenAddress string `protobuf:"bytes,2,opt,name=listen_address,json=listenAddress,proto3" json:"listen_address,omitempty"`
30 | DomainName string `protobuf:"bytes,3,opt,name=domain_name,json=domainName,proto3" json:"domain_name,omitempty"`
31 | Redis *Redis `protobuf:"bytes,4,opt,name=redis,proto3" json:"redis,omitempty"`
32 | BigQuery *BigQuery `protobuf:"bytes,5,opt,name=big_query,json=bigQuery,proto3" json:"big_query,omitempty"`
33 | CloudStorage *CloudStorage `protobuf:"bytes,6,opt,name=cloud_storage,json=cloudStorage,proto3" json:"cloud_storage,omitempty"`
34 | GinMode string `protobuf:"bytes,7,opt,name=gin_mode,json=ginMode,proto3" json:"gin_mode,omitempty"`
35 | LogLevel string `protobuf:"bytes,8,opt,name=log_level,json=logLevel,proto3" json:"log_level,omitempty"`
36 | Remote []*Remote `protobuf:"bytes,9,rep,name=remote,proto3" json:"remote,omitempty"`
37 | User []*User `protobuf:"bytes,10,rep,name=user,proto3" json:"user,omitempty"`
38 | }
39 |
40 | func (x *Config) Reset() {
41 | *x = Config{}
42 | if protoimpl.UnsafeEnabled {
43 | mi := &file_proto_ent_server_config_proto_msgTypes[0]
44 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
45 | ms.StoreMessageInfo(mi)
46 | }
47 | }
48 |
49 | func (x *Config) String() string {
50 | return protoimpl.X.MessageStringOf(x)
51 | }
52 |
53 | func (*Config) ProtoMessage() {}
54 |
55 | func (x *Config) ProtoReflect() protoreflect.Message {
56 | mi := &file_proto_ent_server_config_proto_msgTypes[0]
57 | if protoimpl.UnsafeEnabled && x != nil {
58 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
59 | if ms.LoadMessageInfo() == nil {
60 | ms.StoreMessageInfo(mi)
61 | }
62 | return ms
63 | }
64 | return mi.MessageOf(x)
65 | }
66 |
67 | // Deprecated: Use Config.ProtoReflect.Descriptor instead.
68 | func (*Config) Descriptor() ([]byte, []int) {
69 | return file_proto_ent_server_config_proto_rawDescGZIP(), []int{0}
70 | }
71 |
72 | func (x *Config) GetProjectId() string {
73 | if x != nil {
74 | return x.ProjectId
75 | }
76 | return ""
77 | }
78 |
79 | func (x *Config) GetListenAddress() string {
80 | if x != nil {
81 | return x.ListenAddress
82 | }
83 | return ""
84 | }
85 |
86 | func (x *Config) GetDomainName() string {
87 | if x != nil {
88 | return x.DomainName
89 | }
90 | return ""
91 | }
92 |
93 | func (x *Config) GetRedis() *Redis {
94 | if x != nil {
95 | return x.Redis
96 | }
97 | return nil
98 | }
99 |
100 | func (x *Config) GetBigQuery() *BigQuery {
101 | if x != nil {
102 | return x.BigQuery
103 | }
104 | return nil
105 | }
106 |
107 | func (x *Config) GetCloudStorage() *CloudStorage {
108 | if x != nil {
109 | return x.CloudStorage
110 | }
111 | return nil
112 | }
113 |
114 | func (x *Config) GetGinMode() string {
115 | if x != nil {
116 | return x.GinMode
117 | }
118 | return ""
119 | }
120 |
121 | func (x *Config) GetLogLevel() string {
122 | if x != nil {
123 | return x.LogLevel
124 | }
125 | return ""
126 | }
127 |
128 | func (x *Config) GetRemote() []*Remote {
129 | if x != nil {
130 | return x.Remote
131 | }
132 | return nil
133 | }
134 |
135 | func (x *Config) GetUser() []*User {
136 | if x != nil {
137 | return x.User
138 | }
139 | return nil
140 | }
141 |
142 | type Redis struct {
143 | state protoimpl.MessageState
144 | sizeCache protoimpl.SizeCache
145 | unknownFields protoimpl.UnknownFields
146 |
147 | Endpoint string `protobuf:"bytes,1,opt,name=endpoint,proto3" json:"endpoint,omitempty"`
148 | }
149 |
150 | func (x *Redis) Reset() {
151 | *x = Redis{}
152 | if protoimpl.UnsafeEnabled {
153 | mi := &file_proto_ent_server_config_proto_msgTypes[1]
154 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
155 | ms.StoreMessageInfo(mi)
156 | }
157 | }
158 |
159 | func (x *Redis) String() string {
160 | return protoimpl.X.MessageStringOf(x)
161 | }
162 |
163 | func (*Redis) ProtoMessage() {}
164 |
165 | func (x *Redis) ProtoReflect() protoreflect.Message {
166 | mi := &file_proto_ent_server_config_proto_msgTypes[1]
167 | if protoimpl.UnsafeEnabled && x != nil {
168 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
169 | if ms.LoadMessageInfo() == nil {
170 | ms.StoreMessageInfo(mi)
171 | }
172 | return ms
173 | }
174 | return mi.MessageOf(x)
175 | }
176 |
177 | // Deprecated: Use Redis.ProtoReflect.Descriptor instead.
178 | func (*Redis) Descriptor() ([]byte, []int) {
179 | return file_proto_ent_server_config_proto_rawDescGZIP(), []int{1}
180 | }
181 |
182 | func (x *Redis) GetEndpoint() string {
183 | if x != nil {
184 | return x.Endpoint
185 | }
186 | return ""
187 | }
188 |
189 | type BigQuery struct {
190 | state protoimpl.MessageState
191 | sizeCache protoimpl.SizeCache
192 | unknownFields protoimpl.UnknownFields
193 |
194 | Dataset string `protobuf:"bytes,1,opt,name=dataset,proto3" json:"dataset,omitempty"`
195 | }
196 |
197 | func (x *BigQuery) Reset() {
198 | *x = BigQuery{}
199 | if protoimpl.UnsafeEnabled {
200 | mi := &file_proto_ent_server_config_proto_msgTypes[2]
201 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
202 | ms.StoreMessageInfo(mi)
203 | }
204 | }
205 |
206 | func (x *BigQuery) String() string {
207 | return protoimpl.X.MessageStringOf(x)
208 | }
209 |
210 | func (*BigQuery) ProtoMessage() {}
211 |
212 | func (x *BigQuery) ProtoReflect() protoreflect.Message {
213 | mi := &file_proto_ent_server_config_proto_msgTypes[2]
214 | if protoimpl.UnsafeEnabled && x != nil {
215 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
216 | if ms.LoadMessageInfo() == nil {
217 | ms.StoreMessageInfo(mi)
218 | }
219 | return ms
220 | }
221 | return mi.MessageOf(x)
222 | }
223 |
224 | // Deprecated: Use BigQuery.ProtoReflect.Descriptor instead.
225 | func (*BigQuery) Descriptor() ([]byte, []int) {
226 | return file_proto_ent_server_config_proto_rawDescGZIP(), []int{2}
227 | }
228 |
229 | func (x *BigQuery) GetDataset() string {
230 | if x != nil {
231 | return x.Dataset
232 | }
233 | return ""
234 | }
235 |
236 | type CloudStorage struct {
237 | state protoimpl.MessageState
238 | sizeCache protoimpl.SizeCache
239 | unknownFields protoimpl.UnknownFields
240 |
241 | Bucket string `protobuf:"bytes,1,opt,name=bucket,proto3" json:"bucket,omitempty"`
242 | }
243 |
244 | func (x *CloudStorage) Reset() {
245 | *x = CloudStorage{}
246 | if protoimpl.UnsafeEnabled {
247 | mi := &file_proto_ent_server_config_proto_msgTypes[3]
248 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
249 | ms.StoreMessageInfo(mi)
250 | }
251 | }
252 |
253 | func (x *CloudStorage) String() string {
254 | return protoimpl.X.MessageStringOf(x)
255 | }
256 |
257 | func (*CloudStorage) ProtoMessage() {}
258 |
259 | func (x *CloudStorage) ProtoReflect() protoreflect.Message {
260 | mi := &file_proto_ent_server_config_proto_msgTypes[3]
261 | if protoimpl.UnsafeEnabled && x != nil {
262 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
263 | if ms.LoadMessageInfo() == nil {
264 | ms.StoreMessageInfo(mi)
265 | }
266 | return ms
267 | }
268 | return mi.MessageOf(x)
269 | }
270 |
271 | // Deprecated: Use CloudStorage.ProtoReflect.Descriptor instead.
272 | func (*CloudStorage) Descriptor() ([]byte, []int) {
273 | return file_proto_ent_server_config_proto_rawDescGZIP(), []int{3}
274 | }
275 |
276 | func (x *CloudStorage) GetBucket() string {
277 | if x != nil {
278 | return x.Bucket
279 | }
280 | return ""
281 | }
282 |
283 | type Remote struct {
284 | state protoimpl.MessageState
285 | sizeCache protoimpl.SizeCache
286 | unknownFields protoimpl.UnknownFields
287 |
288 | Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
289 | }
290 |
291 | func (x *Remote) Reset() {
292 | *x = Remote{}
293 | if protoimpl.UnsafeEnabled {
294 | mi := &file_proto_ent_server_config_proto_msgTypes[4]
295 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
296 | ms.StoreMessageInfo(mi)
297 | }
298 | }
299 |
300 | func (x *Remote) String() string {
301 | return protoimpl.X.MessageStringOf(x)
302 | }
303 |
304 | func (*Remote) ProtoMessage() {}
305 |
306 | func (x *Remote) ProtoReflect() protoreflect.Message {
307 | mi := &file_proto_ent_server_config_proto_msgTypes[4]
308 | if protoimpl.UnsafeEnabled && x != nil {
309 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
310 | if ms.LoadMessageInfo() == nil {
311 | ms.StoreMessageInfo(mi)
312 | }
313 | return ms
314 | }
315 | return mi.MessageOf(x)
316 | }
317 |
318 | // Deprecated: Use Remote.ProtoReflect.Descriptor instead.
319 | func (*Remote) Descriptor() ([]byte, []int) {
320 | return file_proto_ent_server_config_proto_rawDescGZIP(), []int{4}
321 | }
322 |
323 | func (x *Remote) GetName() string {
324 | if x != nil {
325 | return x.Name
326 | }
327 | return ""
328 | }
329 |
330 | type User struct {
331 | state protoimpl.MessageState
332 | sizeCache protoimpl.SizeCache
333 | unknownFields protoimpl.UnknownFields
334 |
335 | Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"`
336 | Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
337 | ApiKey string `protobuf:"bytes,3,opt,name=api_key,json=apiKey,proto3" json:"api_key,omitempty"`
338 | CanRead bool `protobuf:"varint,4,opt,name=can_read,json=canRead,proto3" json:"can_read,omitempty"`
339 | CanWrite bool `protobuf:"varint,5,opt,name=can_write,json=canWrite,proto3" json:"can_write,omitempty"`
340 | }
341 |
342 | func (x *User) Reset() {
343 | *x = User{}
344 | if protoimpl.UnsafeEnabled {
345 | mi := &file_proto_ent_server_config_proto_msgTypes[5]
346 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
347 | ms.StoreMessageInfo(mi)
348 | }
349 | }
350 |
351 | func (x *User) String() string {
352 | return protoimpl.X.MessageStringOf(x)
353 | }
354 |
355 | func (*User) ProtoMessage() {}
356 |
357 | func (x *User) ProtoReflect() protoreflect.Message {
358 | mi := &file_proto_ent_server_config_proto_msgTypes[5]
359 | if protoimpl.UnsafeEnabled && x != nil {
360 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
361 | if ms.LoadMessageInfo() == nil {
362 | ms.StoreMessageInfo(mi)
363 | }
364 | return ms
365 | }
366 | return mi.MessageOf(x)
367 | }
368 |
369 | // Deprecated: Use User.ProtoReflect.Descriptor instead.
370 | func (*User) Descriptor() ([]byte, []int) {
371 | return file_proto_ent_server_config_proto_rawDescGZIP(), []int{5}
372 | }
373 |
374 | func (x *User) GetId() int64 {
375 | if x != nil {
376 | return x.Id
377 | }
378 | return 0
379 | }
380 |
381 | func (x *User) GetName() string {
382 | if x != nil {
383 | return x.Name
384 | }
385 | return ""
386 | }
387 |
388 | func (x *User) GetApiKey() string {
389 | if x != nil {
390 | return x.ApiKey
391 | }
392 | return ""
393 | }
394 |
395 | func (x *User) GetCanRead() bool {
396 | if x != nil {
397 | return x.CanRead
398 | }
399 | return false
400 | }
401 |
402 | func (x *User) GetCanWrite() bool {
403 | if x != nil {
404 | return x.CanWrite
405 | }
406 | return false
407 | }
408 |
409 | var File_proto_ent_server_config_proto protoreflect.FileDescriptor
410 |
411 | var file_proto_ent_server_config_proto_rawDesc = []byte{
412 | 0x0a, 0x1d, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x65, 0x72, 0x76,
413 | 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12,
414 | 0x11, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x66,
415 | 0x69, 0x67, 0x22, 0xb7, 0x03, 0x0a, 0x06, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1d, 0x0a,
416 | 0x0a, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
417 | 0x09, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x0e,
418 | 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02,
419 | 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x41, 0x64, 0x64, 0x72,
420 | 0x65, 0x73, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x5f, 0x6e, 0x61,
421 | 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e,
422 | 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2e, 0x0a, 0x05, 0x72, 0x65, 0x64, 0x69, 0x73, 0x18, 0x04, 0x20,
423 | 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72,
424 | 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x65, 0x64, 0x69, 0x73, 0x52, 0x05, 0x72,
425 | 0x65, 0x64, 0x69, 0x73, 0x12, 0x38, 0x0a, 0x09, 0x62, 0x69, 0x67, 0x5f, 0x71, 0x75, 0x65, 0x72,
426 | 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x65,
427 | 0x72, 0x76, 0x65, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x42, 0x69, 0x67, 0x51,
428 | 0x75, 0x65, 0x72, 0x79, 0x52, 0x08, 0x62, 0x69, 0x67, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x44,
429 | 0x0a, 0x0d, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x18,
430 | 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76,
431 | 0x65, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x43, 0x6c, 0x6f, 0x75, 0x64, 0x53,
432 | 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x0c, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x53, 0x74, 0x6f,
433 | 0x72, 0x61, 0x67, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x67, 0x69, 0x6e, 0x5f, 0x6d, 0x6f, 0x64, 0x65,
434 | 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x67, 0x69, 0x6e, 0x4d, 0x6f, 0x64, 0x65, 0x12,
435 | 0x1b, 0x0a, 0x09, 0x6c, 0x6f, 0x67, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x08, 0x20, 0x01,
436 | 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x31, 0x0a, 0x06,
437 | 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x65,
438 | 0x6e, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67,
439 | 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x52, 0x06, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x12,
440 | 0x2b, 0x0a, 0x04, 0x75, 0x73, 0x65, 0x72, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e,
441 | 0x65, 0x6e, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69,
442 | 0x67, 0x2e, 0x55, 0x73, 0x65, 0x72, 0x52, 0x04, 0x75, 0x73, 0x65, 0x72, 0x22, 0x23, 0x0a, 0x05,
443 | 0x52, 0x65, 0x64, 0x69, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e,
444 | 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e,
445 | 0x74, 0x22, 0x24, 0x0a, 0x08, 0x42, 0x69, 0x67, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x18, 0x0a,
446 | 0x07, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07,
447 | 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x22, 0x26, 0x0a, 0x0c, 0x43, 0x6c, 0x6f, 0x75, 0x64,
448 | 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65,
449 | 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x22,
450 | 0x1c, 0x0a, 0x06, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d,
451 | 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x7b, 0x0a,
452 | 0x04, 0x55, 0x73, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
453 | 0x03, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20,
454 | 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x61, 0x70, 0x69,
455 | 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x61, 0x70, 0x69, 0x4b,
456 | 0x65, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x61, 0x6e, 0x5f, 0x72, 0x65, 0x61, 0x64, 0x18, 0x04,
457 | 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x63, 0x61, 0x6e, 0x52, 0x65, 0x61, 0x64, 0x12, 0x1b, 0x0a,
458 | 0x09, 0x63, 0x61, 0x6e, 0x5f, 0x77, 0x72, 0x69, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08,
459 | 0x52, 0x08, 0x63, 0x61, 0x6e, 0x57, 0x72, 0x69, 0x74, 0x65, 0x42, 0x10, 0x5a, 0x0e, 0x67, 0x69,
460 | 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x6e, 0x74, 0x62, 0x06, 0x70, 0x72,
461 | 0x6f, 0x74, 0x6f, 0x33,
462 | }
463 |
464 | var (
465 | file_proto_ent_server_config_proto_rawDescOnce sync.Once
466 | file_proto_ent_server_config_proto_rawDescData = file_proto_ent_server_config_proto_rawDesc
467 | )
468 |
469 | func file_proto_ent_server_config_proto_rawDescGZIP() []byte {
470 | file_proto_ent_server_config_proto_rawDescOnce.Do(func() {
471 | file_proto_ent_server_config_proto_rawDescData = protoimpl.X.CompressGZIP(file_proto_ent_server_config_proto_rawDescData)
472 | })
473 | return file_proto_ent_server_config_proto_rawDescData
474 | }
475 |
476 | var file_proto_ent_server_config_proto_msgTypes = make([]protoimpl.MessageInfo, 6)
477 | var file_proto_ent_server_config_proto_goTypes = []interface{}{
478 | (*Config)(nil), // 0: ent.server.config.Config
479 | (*Redis)(nil), // 1: ent.server.config.Redis
480 | (*BigQuery)(nil), // 2: ent.server.config.BigQuery
481 | (*CloudStorage)(nil), // 3: ent.server.config.CloudStorage
482 | (*Remote)(nil), // 4: ent.server.config.Remote
483 | (*User)(nil), // 5: ent.server.config.User
484 | }
485 | var file_proto_ent_server_config_proto_depIdxs = []int32{
486 | 1, // 0: ent.server.config.Config.redis:type_name -> ent.server.config.Redis
487 | 2, // 1: ent.server.config.Config.big_query:type_name -> ent.server.config.BigQuery
488 | 3, // 2: ent.server.config.Config.cloud_storage:type_name -> ent.server.config.CloudStorage
489 | 4, // 3: ent.server.config.Config.remote:type_name -> ent.server.config.Remote
490 | 5, // 4: ent.server.config.Config.user:type_name -> ent.server.config.User
491 | 5, // [5:5] is the sub-list for method output_type
492 | 5, // [5:5] is the sub-list for method input_type
493 | 5, // [5:5] is the sub-list for extension type_name
494 | 5, // [5:5] is the sub-list for extension extendee
495 | 0, // [0:5] is the sub-list for field type_name
496 | }
497 |
498 | func init() { file_proto_ent_server_config_proto_init() }
499 | func file_proto_ent_server_config_proto_init() {
500 | if File_proto_ent_server_config_proto != nil {
501 | return
502 | }
503 | if !protoimpl.UnsafeEnabled {
504 | file_proto_ent_server_config_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
505 | switch v := v.(*Config); i {
506 | case 0:
507 | return &v.state
508 | case 1:
509 | return &v.sizeCache
510 | case 2:
511 | return &v.unknownFields
512 | default:
513 | return nil
514 | }
515 | }
516 | file_proto_ent_server_config_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
517 | switch v := v.(*Redis); i {
518 | case 0:
519 | return &v.state
520 | case 1:
521 | return &v.sizeCache
522 | case 2:
523 | return &v.unknownFields
524 | default:
525 | return nil
526 | }
527 | }
528 | file_proto_ent_server_config_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
529 | switch v := v.(*BigQuery); i {
530 | case 0:
531 | return &v.state
532 | case 1:
533 | return &v.sizeCache
534 | case 2:
535 | return &v.unknownFields
536 | default:
537 | return nil
538 | }
539 | }
540 | file_proto_ent_server_config_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
541 | switch v := v.(*CloudStorage); i {
542 | case 0:
543 | return &v.state
544 | case 1:
545 | return &v.sizeCache
546 | case 2:
547 | return &v.unknownFields
548 | default:
549 | return nil
550 | }
551 | }
552 | file_proto_ent_server_config_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
553 | switch v := v.(*Remote); i {
554 | case 0:
555 | return &v.state
556 | case 1:
557 | return &v.sizeCache
558 | case 2:
559 | return &v.unknownFields
560 | default:
561 | return nil
562 | }
563 | }
564 | file_proto_ent_server_config_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
565 | switch v := v.(*User); i {
566 | case 0:
567 | return &v.state
568 | case 1:
569 | return &v.sizeCache
570 | case 2:
571 | return &v.unknownFields
572 | default:
573 | return nil
574 | }
575 | }
576 | }
577 | type x struct{}
578 | out := protoimpl.TypeBuilder{
579 | File: protoimpl.DescBuilder{
580 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
581 | RawDescriptor: file_proto_ent_server_config_proto_rawDesc,
582 | NumEnums: 0,
583 | NumMessages: 6,
584 | NumExtensions: 0,
585 | NumServices: 0,
586 | },
587 | GoTypes: file_proto_ent_server_config_proto_goTypes,
588 | DependencyIndexes: file_proto_ent_server_config_proto_depIdxs,
589 | MessageInfos: file_proto_ent_server_config_proto_msgTypes,
590 | }.Build()
591 | File_proto_ent_server_config_proto = out.File
592 | file_proto_ent_server_config_proto_rawDesc = nil
593 | file_proto_ent_server_config_proto_goTypes = nil
594 | file_proto_ent_server_config_proto_depIdxs = nil
595 | }
596 |
--------------------------------------------------------------------------------