├── demo ├── deb-package │ ├── debian │ │ ├── compat │ │ ├── rules │ │ ├── control │ │ └── starlight.service │ ├── Dockerfile │ └── generate-changelog.sh ├── chart │ ├── .gitignore │ ├── .helmignore │ ├── templates │ │ ├── serviceaccount.yaml │ │ ├── service-registry.yaml │ │ ├── pvc-proxy.yaml │ │ ├── pvc-registry.yaml │ │ ├── service-proxy.yaml │ │ ├── ingress.yaml │ │ ├── daemonset-edge.yaml │ │ ├── deployment-registry.yaml │ │ ├── _helpers.tpl │ │ ├── NOTES.txt │ │ └── deployment-proxy.yaml │ ├── Chart.yaml │ └── values.yaml ├── config │ ├── scripts │ │ ├── hello.py │ │ ├── hello.js │ │ ├── Hello.java │ │ └── hello.go │ ├── entrypoint-js.sh │ ├── entrypoint-py.sh │ ├── entrypoint-hello.sh │ ├── entrypoint-memcached.sh │ ├── entrypoint-redis.sh │ ├── entrypoint-java.sh │ ├── starlight-snapshotter-entrypoint.sh │ ├── entrypoint-go.sh │ ├── my.cnf │ ├── sysctl.conf │ ├── containerd.config.toml │ └── all.env ├── compose │ ├── .gitignore │ ├── README.md │ └── docker-compose-example.yaml ├── terraform │ ├── locals.tf │ ├── .gitignore │ ├── providers.tf │ ├── outputs.tf │ ├── variables.tf │ ├── terraform.tfvars │ └── .terraform.lock.hcl ├── .gitignore ├── k3s-reset.sh ├── reset.sh └── convert-vanilla.sh ├── design └── starlight-image.md ├── .dockerignore ├── client ├── fs │ ├── instance_test.go │ ├── flag_darwin.go │ ├── flag_linux.go │ ├── tracer_test.go │ ├── instance.go │ └── fs.go ├── config_test.go ├── snapshotter │ ├── plugin_test.go │ ├── operator.go │ └── operator_test.go ├── api │ └── daemon.proto └── config.go ├── docs ├── images │ ├── starlight-workflow.png │ ├── metadatadb-screenshot.png │ └── provisioning-time-wan.png ├── terraform.md └── deprecated-v0.1 │ ├── starlight-workflow.md │ └── starlight-snapshotter.md ├── staticcheck.conf ├── proxy ├── config_test.go ├── api_test.go ├── database_test.go ├── config.go └── extractor_test.go ├── test ├── environment_test.go ├── helper.go └── environment.go ├── .github ├── ISSUE_TEMPLATE │ ├── question.md │ └── task.md └── workflows │ ├── versioning.yml │ ├── test.yml │ ├── helm-chart.yml │ ├── debian-package.yml │ └── docker-image.yml ├── util ├── common │ ├── image.go │ ├── errors.go │ └── cache.go ├── receive │ ├── types_darwin.go │ ├── types_linux.go │ ├── helper.go │ └── image.go ├── random.go ├── version.go ├── json.go ├── db.go ├── writer.go ├── log.go ├── config.go ├── send │ └── image.go └── convertor_test.go ├── .env_example ├── .gitignore ├── cmd ├── ctr-starlight │ ├── auth │ │ └── options.go │ ├── convert │ │ ├── options.go │ │ └── convert.go │ ├── version │ │ └── version.go │ ├── ping │ │ └── ping.go │ ├── listproxy │ │ └── list_proxy.go │ ├── report │ │ └── report.go │ ├── optimizer │ │ └── optimizer.go │ ├── notify │ │ └── notify.go │ ├── addproxy │ │ └── add_proxy.go │ ├── pull │ │ └── pull.go │ ├── main.go │ └── reset │ │ └── reset.go ├── starlight-proxy │ └── main.go └── starlight-daemon │ └── main.go ├── Dockerfile ├── DETAILS.md └── go.mod /demo/deb-package/debian/compat: -------------------------------------------------------------------------------- 1 | 10 2 | -------------------------------------------------------------------------------- /demo/chart/.gitignore: -------------------------------------------------------------------------------- 1 | *.tar.gz 2 | *.gz 3 | *.tgz -------------------------------------------------------------------------------- /design/starlight-image.md: -------------------------------------------------------------------------------- 1 | #Starlight Image Format 2 | -------------------------------------------------------------------------------- /demo/config/scripts/hello.py: -------------------------------------------------------------------------------- 1 | print("Hello from Python!") -------------------------------------------------------------------------------- /demo/deb-package/debian/rules: -------------------------------------------------------------------------------- 1 | dh $@ --with systemd 2 | -------------------------------------------------------------------------------- /demo/config/entrypoint-js.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | node /app/hello.js -------------------------------------------------------------------------------- /demo/config/entrypoint-py.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | python /app/hello.py -------------------------------------------------------------------------------- /demo/compose/.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | data_* 3 | docker-compose.yaml 4 | -------------------------------------------------------------------------------- /demo/config/entrypoint-hello.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | echo "hello" 3 | echo "" -------------------------------------------------------------------------------- /demo/config/entrypoint-memcached.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | memcached -m 128 -vv -------------------------------------------------------------------------------- /demo/terraform/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | project_name = var.project_id 3 | } 4 | -------------------------------------------------------------------------------- /demo/config/scripts/hello.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | console.log("Hello from Node.js!"); -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | Dockerfile 2 | out 3 | vendor 4 | sandbox 5 | 6 | .github 7 | my-values.yaml -------------------------------------------------------------------------------- /demo/config/entrypoint-redis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | /usr/local/bin/redis-server --protected-mode no -------------------------------------------------------------------------------- /demo/config/entrypoint-java.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | cd /app || exit 3 | javac Hello.java 4 | java Hello -------------------------------------------------------------------------------- /demo/config/starlight-snapshotter-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | containerd & 3 | starlight-grpc run -------------------------------------------------------------------------------- /client/fs/instance_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package fs 7 | -------------------------------------------------------------------------------- /demo/config/entrypoint-go.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | go build -o /bin/hello /app/hello.go 3 | chmod 777 /bin/hello 4 | /bin/hello -------------------------------------------------------------------------------- /docs/images/starlight-workflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc256/starlight/HEAD/docs/images/starlight-workflow.png -------------------------------------------------------------------------------- /demo/config/my.cnf: -------------------------------------------------------------------------------- 1 | 2 | [mysqld] 3 | # SSL Settings 4 | skip-ssl=1 5 | ssl-verify-server-cert=0 6 | ssl-mode=DISABLED 7 | ssl=0 -------------------------------------------------------------------------------- /docs/images/metadatadb-screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc256/starlight/HEAD/docs/images/metadatadb-screenshot.png -------------------------------------------------------------------------------- /docs/images/provisioning-time-wan.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc256/starlight/HEAD/docs/images/provisioning-time-wan.png -------------------------------------------------------------------------------- /client/fs/flag_darwin.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package fs 7 | 8 | const ( 9 | UnmountFlag = 0x0 10 | ) 11 | -------------------------------------------------------------------------------- /demo/config/scripts/Hello.java: -------------------------------------------------------------------------------- 1 | public class Hello 2 | { 3 | public static void main(String[] args) { 4 | System.out.println("Hello from Java!"); 5 | } 6 | } -------------------------------------------------------------------------------- /demo/.gitignore: -------------------------------------------------------------------------------- 1 | #shell script (excluding some local test cases) 2 | *.sh 3 | !config/*.sh 4 | !deb-package/*.sh 5 | temp.sh 6 | !convert* 7 | 8 | *.class 9 | 10 | k8s -------------------------------------------------------------------------------- /staticcheck.conf: -------------------------------------------------------------------------------- 1 | checks = ["all", "-S1036", "-S1005", "-SA4006", "-SA4006", "-SA9003", "-SA4009", "-S1019", "-U1000", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022", "-ST1023"] -------------------------------------------------------------------------------- /client/fs/flag_linux.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package fs 7 | 8 | import "syscall" 9 | 10 | const ( 11 | UnmountFlag = syscall.MNT_FORCE | syscall.MNT_DETACH 12 | ) 13 | -------------------------------------------------------------------------------- /proxy/config_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package proxy 7 | 8 | import ( 9 | "fmt" 10 | "testing" 11 | ) 12 | 13 | func TestLoadConfig(t *testing.T) { 14 | cfg, _, _, _ := LoadConfig("") 15 | fmt.Println(cfg) 16 | } 17 | -------------------------------------------------------------------------------- /demo/k3s-reset.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # remove containerd filesystems 4 | rm -rf /var/lib/rancher/k3s/agent/containerd 5 | 6 | # remove starlight layer cache 7 | rm -rf /var/lib/starlight/layers 8 | 9 | 10 | # systemctl restart k3s-agent 11 | # systemctl stop containerd -------------------------------------------------------------------------------- /test/environment_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestHasLoginRegistry(t *testing.T) { 9 | fmt.Printf("HasLoginAWSECR: %v\n", HasLoginAWSECR()) 10 | fmt.Printf("HasLoginStarlightGoharbor: %v\n", HasLoginStarlightGoharbor()) 11 | } 12 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/question.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: questions 3 | about: Describe this issue here. 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | Describe your question here 11 | --- 12 | 13 | XXXX 14 | 15 | 16 | What kind of hardward you have? 17 | --- 18 | - 2 VM 19 | - one edge node 20 | - ... -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/task.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: task 3 | about: Describe this issue here. 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | Description 11 | --- 12 | 13 | XXXX 14 | 15 | 16 | Support Documents 17 | --- 18 | - XXXX 19 | - XXXX 20 | - XXXX 21 | 22 | 23 | Acceptance Criteria 24 | --- 25 | - XXXX 26 | - XXXX 27 | - XXXX 28 | -------------------------------------------------------------------------------- /util/common/image.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package common 7 | 8 | type DeltaImageMetadata struct { 9 | ManifestSize int64 10 | ConfigSize int64 11 | StarlightHeaderSize int64 12 | ContentLength int64 13 | OriginalLength int64 14 | 15 | Digest string 16 | StarlightDigest string 17 | } 18 | -------------------------------------------------------------------------------- /client/config_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package client 7 | 8 | import ( 9 | "fmt" 10 | "testing" 11 | ) 12 | 13 | func TestParseProxyStrings1(t *testing.T) { 14 | k, c, err := ParseProxyStrings("starlight-shared,https,starlight.yuri.moe,,") 15 | if err != nil { 16 | t.Error(err) 17 | } 18 | fmt.Println(k, c) 19 | } 20 | -------------------------------------------------------------------------------- /demo/terraform/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore Terraform state files 2 | *.tfstate 3 | *.tfstate.backup 4 | *.tfplan 5 | 6 | # Ignore .terraform directory 7 | .terraform/ 8 | 9 | # Ignore override files 10 | override.tf 11 | override.tf.json 12 | *_override.tf 13 | *_override.tf.json 14 | 15 | # Ignore CLI configuration files 16 | .terraformrc 17 | terraform.rc 18 | 19 | .terraform.tfstate.lock.info -------------------------------------------------------------------------------- /.github/workflows/versioning.yml: -------------------------------------------------------------------------------- 1 | name: Versioning 2 | on: 3 | pull_request: 4 | types: [closed] 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | if: github.event.pull_request.merged 9 | steps: 10 | - name: Tag 11 | uses: K-Phoen/semver-release-action@master 12 | with: 13 | release_branch: master 14 | env: 15 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /proxy/api_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package proxy 7 | 8 | import ( 9 | "context" 10 | "testing" 11 | ) 12 | 13 | func TestStarlightProxy_Ping(t *testing.T) { 14 | proxy := NewStarlightProxy(context.TODO(), "http", "localhost:8090") 15 | //proxy.auth = *url.UserPassword("username", "password") 16 | 17 | if _, _, _, err := proxy.Ping(); err != nil { 18 | t.Error(err) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /demo/chart/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /demo/chart/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.enabled}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ .Values.serviceAccount.name }} 6 | namespace: {{ .Values.namespace }} 7 | {{- with .Values.serviceAccount.labels }} 8 | labels: 9 | {{- toYaml . | nindent 4 }} 10 | {{- end }} 11 | {{- with .Values.serviceAccount.annotations }} 12 | annotations: 13 | {{- toYaml . | nindent 4 }} 14 | {{- end }} 15 | {{- end}} -------------------------------------------------------------------------------- /demo/config/sysctl.conf: -------------------------------------------------------------------------------- 1 | # Adjust TCP windows size for high latency transmission 2 | # Calculate the window size needed on https://www.speedguide.net/bdp.php 3 | 4 | net.core.wmem_max=125829120 5 | net.core.rmem_max=125829120 6 | net.ipv4.tcp_rmem= 10240 87380 125829120 7 | net.ipv4.tcp_wmem= 10240 87380 125829120 8 | net.ipv4.tcp_window_scaling = 1 9 | net.ipv4.tcp_timestamps = 1 10 | net.ipv4.tcp_sack = 1 11 | net.ipv4.tcp_no_metrics_save = 1 12 | net.core.netdev_max_backlog = 10000 -------------------------------------------------------------------------------- /demo/deb-package/debian/control: -------------------------------------------------------------------------------- 1 | Source: starlight 2 | Section: devel 3 | Priority: optional 4 | Maintainer: Junlin Chen 5 | Build-Depends: build-essential, dh-systemd (>= 1.5), debhelper (>=10) 6 | Standards-Version: 0.0.0 7 | Homepage: https://github.com/mc256/starlight 8 | 9 | Package: starlight 10 | Architecture: amd64 11 | Recommends: containerd (>= 1.4.13) 12 | Depends: linux-headers-generic (>= 5.15.0) 13 | Description: Starlight is an accelerator for provisioning container-based applications. 14 | -------------------------------------------------------------------------------- /.env_example: -------------------------------------------------------------------------------- 1 | STARLIGHT_CONTAINER_REGISTRY=registry.yuri.moe 2 | STARLIGHT_SANDBOX_DIR=../.sandbox 3 | 4 | TEST_DOCKER_SOURCE_IMAGE=docker.io/library/redis:6.2.1 5 | 6 | TEST_HARBOR_REGISTRY=registry.yuri.moe 7 | TEST_HARBOR_IMAGE_FROM=registry.yuri.moe/starlight/test-harbor:test 8 | TEST_HARBOR_IMAGE_TO=registry.yuri.moe/starlight/test-harbor123:starlight2 9 | 10 | HARBOR_USERNAME= 11 | HARBOR_PASSWORD= 12 | 13 | TEST_ECR_IMAGE_FROM=public.ecr.aws/______/image-name:test 14 | TEST_ECR_IMAGE_TO=public.ecr.aws/______/image-name:starlight2 15 | 16 | 17 | -------------------------------------------------------------------------------- /demo/deb-package/debian/starlight.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Starlight snapshotter GRPC service 3 | Documentation=https://github.com/mc256/starlight 4 | After=network.target local-fs.target containerd.service 5 | Wants=containerd.service 6 | 7 | [Service] 8 | ExecStart=/usr/bin/starlight-daemon 9 | 10 | Type=simple 11 | KillMode=process 12 | Restart=always 13 | RestartSec=5 14 | 15 | LimitNPROC=infinity 16 | LimitCORE=infinity 17 | LimitNOFILE=infinity 18 | 19 | TasksMax=infinity 20 | OOMScoreAdjust=-999 21 | 22 | [Install] 23 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /demo/chart/templates/service-registry.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.registry.enabled}} 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: {{ include "starlight.fullname-registry" . }} 6 | namespace: {{ .Values.namespace }} 7 | labels: 8 | {{- include "starlight.registryLabels" . | nindent 4 }} 9 | spec: 10 | type: {{ .Values.service.type }} 11 | ports: 12 | - port: 5000 13 | name: container-registry 14 | targetPort: 5000 15 | protocol: TCP 16 | selector: 17 | {{- include "starlight.registrySelectorLabels" . | nindent 4 }} 18 | {{- end}} -------------------------------------------------------------------------------- /demo/config/containerd.config.toml: -------------------------------------------------------------------------------- 1 | subreaper = true 2 | oom_score = -999 3 | 4 | [grpc] 5 | gid = 1001 6 | 7 | [debug] 8 | level = "trace" 9 | gid = 1001 10 | 11 | [metrics] 12 | address = "127.0.0.1:1338" 13 | 14 | [plugins.linux] 15 | runtime = "runc" 16 | shim_debug = true 17 | 18 | [proxy_plugins] 19 | [proxy_plugins.starlight] 20 | type = "snapshot" 21 | address = "/run/starlight-grpc/starlight-snapshotter.socket" 22 | [proxy_plugins.stargz] 23 | type = "snapshot" 24 | address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.socket" -------------------------------------------------------------------------------- /util/receive/types_darwin.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package receive 7 | 8 | import ( 9 | "github.com/hanwen/go-fuse/v2/fuse" 10 | "syscall" 11 | ) 12 | 13 | func (r *ReferencedFile) SetBlockSize(out *fuse.Attr) { 14 | out.Blocks = out.Size / uint64(4096) 15 | if out.Size%uint64(4096) > 0 { 16 | out.Blocks++ 17 | } 18 | } 19 | 20 | func Creat(path string, perm uint32) (int, error) { 21 | return syscall.Open(path, syscall.O_CREAT|syscall.O_WRONLY|syscall.O_TRUNC, perm) 22 | } 23 | 24 | func FileMode(m uint16) uint32 { 25 | return uint32(m) 26 | } 27 | -------------------------------------------------------------------------------- /util/random.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package util 7 | 8 | import ( 9 | "fmt" 10 | "math/rand" 11 | "time" 12 | ) 13 | 14 | var ( 15 | letters = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") 16 | ) 17 | 18 | func randSequence(n int) string { 19 | rand.Seed(time.Now().UnixNano()) 20 | b := make([]byte, n) 21 | for i := range b { 22 | b[i] = letters[rand.Intn(len(letters))] 23 | } 24 | return string(b) 25 | } 26 | 27 | func GetRandomId(prefix string) string { 28 | return fmt.Sprintf("%s-%s", prefix, randSequence(10)) 29 | } 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | 17 | .out/ 18 | out/ 19 | .sandbox/ 20 | sandbox/ 21 | 22 | # GoLand and VSCode 23 | .idea 24 | # VSCode configuration example: https://gist.github.com/mc256/357a867b8b21bbcc571c5469d127a1d8 25 | .vscode 26 | 27 | vendor/ 28 | 29 | my-values.yaml 30 | 31 | # environment file 32 | .env -------------------------------------------------------------------------------- /demo/config/all.env: -------------------------------------------------------------------------------- 1 | MYSQL_ROOT_PASSWORD=12345678abc 2 | MYSQL_DATABASE=wptest 3 | MYSQL_USER=wptest 4 | MYSQL_PASSWORD=12345678wptest 5 | POSTGRES_PASSWORD=12345678abc 6 | CASSANDRA_BROADCAST_ADDRESS=127.0.0.1 7 | CASSANDRA_LISTEN_ADDRESS=127.0.0.1 8 | CASSANDRA_RPC_ADDRESS=127.0.0.1 9 | MONGO_INITDB_ROOT_USERNAME=root 10 | MONGO_INITDB_ROOT_PASSWORD=12345678abc 11 | RABBITMQ_DEFAULT_USER=user 12 | RABBITMQ_DEFAULT_PASS=12345678abc 13 | REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY=/data 14 | WORDPRESS_DB_HOST=db 15 | WORDPRESS_DB_NAME=wptest 16 | WORDPRESS_DB_USER=wptest 17 | WORDPRESS_DB_PASSWORD=12345678wptest 18 | FLINK_PROPERTIES="jobmanager.rpc.address: jobmanager" -------------------------------------------------------------------------------- /cmd/ctr-starlight/auth/options.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package auth 7 | 8 | import "github.com/urfave/cli/v2" 9 | 10 | var ( 11 | ProxyFlags = []cli.Flag{ 12 | &cli.StringFlag{ 13 | Name: "profile", 14 | Aliases: []string{"p"}, 15 | Value: "", 16 | Usage: "profile name for connecting to the starlight proxy server in the configuration file, " + 17 | "if leave empty will use the default profile", 18 | }, 19 | &cli.StringFlag{ 20 | Name: "quiet", 21 | Aliases: []string{"q"}, 22 | Value: "", 23 | Usage: "do not print any message unless error occurs", 24 | }, 25 | } 26 | ) 27 | -------------------------------------------------------------------------------- /util/receive/types_linux.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package receive 7 | 8 | import ( 9 | "syscall" 10 | 11 | "github.com/hanwen/go-fuse/v2/fuse" 12 | ) 13 | 14 | func (r *ReferencedFile) SetBlockSize(out *fuse.Attr) { 15 | out.Blksize = uint32(r.ChunkSize) 16 | if r.ChunkSize == 0 { 17 | out.Blksize = 4096 18 | } 19 | out.Blocks = out.Size / uint64(out.Blksize) 20 | if out.Size%uint64(out.Blksize) > 0 { 21 | out.Blocks++ 22 | } 23 | 24 | out.Padding = 0 25 | } 26 | 27 | func Creat(path string, perm uint32) (int, error) { 28 | return syscall.Creat(path, perm) 29 | } 30 | 31 | func FileMode(m uint32) uint32 { 32 | return m 33 | } 34 | -------------------------------------------------------------------------------- /demo/terraform/providers.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | aws = { 4 | version = "~> 5.35.0" 5 | source = "hashicorp/aws" 6 | } 7 | } 8 | } 9 | 10 | provider "aws" { 11 | profile = "default" 12 | # Hard-coded credentials are not recommended in any Terraform configuration and 13 | # risks secret leakage should this file ever be committed to a public version control system. 14 | # Instead, use environment variables or shared credentials file 15 | # ref: https://registry.terraform.io/providers/-/aws/latest/docs#environment-variables 16 | 17 | # Please set the credentials in the environment variables or in $HOME/.aws/credentials and $HOME/.aws/config files 18 | } 19 | -------------------------------------------------------------------------------- /demo/deb-package/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:stable 2 | 3 | WORKDIR /go/src/app 4 | 5 | ARG ARCH=amd64 6 | ARG UPLOAD_URL=http://127.0.0.1:35001/ 7 | ARG APT_UPLOAD_AUTH=XXXX 8 | 9 | ENV GO111MODULE=on 10 | ENV REGISTRY=registry2 11 | ENV LOGLEVEL=info 12 | ENV PATH="/usr/local/go/bin:${PATH}" 13 | ENV EMAIL=webmaster@mc256.dev 14 | ENV DEBIAN_FRONTEND=noninteractive 15 | ENV TZ=Etc/UTC 16 | 17 | RUN apt update -y && apt upgrade -y 18 | RUN apt install -y tzdata curl wget git dpkg-dev gpg build-essential debhelper bind9-utils 19 | RUN wget -q https://go.dev/dl/go1.20.8.linux-$ARCH.tar.gz && rm -rf /usr/local/go && tar -C /usr/local -xzf go1.20.8.linux-$ARCH.tar.gz 20 | COPY . . 21 | RUN ARCH=$ARCH && make create-deb-package.$ARCH upload-deb-package.$ARCH 22 | -------------------------------------------------------------------------------- /demo/chart/templates/pvc-proxy.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.postgres.persistence.enabled }} 2 | kind: PersistentVolumeClaim 3 | apiVersion: v1 4 | metadata: 5 | name: {{ include "starlight.fullname" . }} 6 | labels: 7 | {{- include "starlight.proxyLabels" . | nindent 4 }} 8 | annotations: 9 | {{- if .Values.postgres.persistence.storageClass }} 10 | volume.beta.kubernetes.io/storage-class: {{ .Values.postgres.persistence.storageClass | quote }} 11 | {{- else }} 12 | volume.alpha.kubernetes.io/storage-class: default 13 | {{- end }} 14 | spec: 15 | accessModes: 16 | {{- range .Values.postgres.persistence.accessModes }} 17 | - {{ . }} 18 | {{- end }} 19 | resources: 20 | requests: 21 | storage: {{ .Values.postgres.persistence.size | quote }} 22 | {{- end -}} 23 | -------------------------------------------------------------------------------- /demo/chart/templates/pvc-registry.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.registry.persistence.enabled }} 2 | kind: PersistentVolumeClaim 3 | apiVersion: v1 4 | metadata: 5 | name: {{ include "starlight.fullname-registry" . }} 6 | labels: 7 | {{- include "starlight.proxyLabels" . | nindent 4 }} 8 | annotations: 9 | {{- if .Values.registry.persistence.storageClass }} 10 | volume.beta.kubernetes.io/storage-class: {{ .Values.registry.persistence.storageClass | quote }} 11 | {{- else }} 12 | volume.alpha.kubernetes.io/storage-class: default 13 | {{- end }} 14 | spec: 15 | accessModes: 16 | {{- range .Values.registry.persistence.accessModes }} 17 | - {{ . }} 18 | {{- end }} 19 | resources: 20 | requests: 21 | storage: {{ .Values.registry.persistence.size | quote }} 22 | {{- end -}} 23 | -------------------------------------------------------------------------------- /demo/deb-package/generate-changelog.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | VERSION_FORMAT='^v\?[0-9.-]\+' 3 | function logentry() { 4 | local previous=$1 5 | local version=$2 6 | local version_number=`echo $2 | sed 's/v//g'` 7 | local version_number=`[[ -z "$version_number" ]] && echo "0.0.0" || echo $version_number` 8 | echo "starlight ($version_number) unstable; urgency=low" 9 | echo 10 | git --no-pager log --format=" * %s" $previous${previous:+..}$version 11 | echo 12 | git --no-pager log --format=" -- %an <%ae> %aD" -n 1 $version 13 | echo 14 | } 15 | 16 | git tag --sort "-version:refname" | grep "$VERSION_FORMAT" | ( 17 | read version; while read previous; do 18 | logentry $previous $version 19 | version="$previous" 20 | done 21 | logentry "" $version 22 | ) -------------------------------------------------------------------------------- /demo/chart/templates/service-proxy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "starlight.fullname" . }} 5 | namespace: {{ .Values.namespace }} 6 | labels: 7 | {{- include "starlight.proxyLabels" . | nindent 4 }} 8 | spec: 9 | type: {{ .Values.service.type }} 10 | ports: 11 | - port: 8090 12 | name: starlightproxy 13 | targetPort: 8090 14 | protocol: TCP 15 | {{- if .Values.postgres.enabled}} 16 | - port: 5432 17 | name: postgres 18 | targetPort: 5432 19 | protocol: TCP 20 | {{- if .Values.adminer.enabled}} 21 | - port: 8080 22 | name: adminer 23 | targetPort: 8080 24 | protocol: TCP 25 | {{- end}} 26 | {{- end}} 27 | selector: 28 | {{- include "starlight.proxySelectorLabels" . | nindent 4 }} 29 | -------------------------------------------------------------------------------- /demo/config/scripts/hello.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The starlight Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | 16 | file created by maverick in 2021 17 | */ 18 | 19 | package main 20 | 21 | import "fmt" 22 | 23 | func main() { 24 | fmt.Println("Hello from Golang!") 25 | } 26 | -------------------------------------------------------------------------------- /demo/compose/README.md: -------------------------------------------------------------------------------- 1 | # Docker Compose Dev environment 2 | 3 | The `docker-compose-example.yaml` file is an example for launching a local development environment. 4 | You could edit it and save it as `docker-compose.yaml` file 5 | 6 | | service needed for developing | starlight-proxy | db | registry | | 7 | | ----------------------------- | --------------- | ------------------ | ----------------- | ----------- | 8 | | ctr-starlight | ✅ | ✅ | ✅ | plus daemon | 9 | | starlight-daemon | ✅ | ✅ | ✅ | | 10 | | starlight-proxy | | ✅ | ✅ | | 11 | | ---------------- | --------------- | ------------------ | ----------------- | ---- | 12 | -------------------------------------------------------------------------------- /demo/reset.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ## This script reset the experiment environment 3 | ## Please cd into the project's root directory and run this script using this command: 4 | ## sudo ./demo/reset.sh 5 | 6 | STARLIGHT_SNAPSHOTTER_ROOT=/var/lib/starlight/ 7 | 8 | # Stop Starlight and containerd 9 | systemctl stop containerd 10 | systemctl stop starlight 11 | systemctl disable containerd 12 | systemctl disable starlight 13 | pkill -9 'containerd' | true 14 | pkill -9 'starlight' | true 15 | 16 | 17 | # Clear containerd folder 18 | rm -rf /var/lib/containerd 19 | 20 | # Clear starlight folder 21 | if [ -d "${STARLIGHT_SNAPSHOTTER_ROOT}sfs/" ] ; then 22 | find "${STARLIGHT_SNAPSHOTTER_ROOT}sfs/" \ 23 | -maxdepth 1 -mindepth 1 -type d -exec sudo umount -f "{}/m" \; 24 | fi 25 | rm -rf "${STARLIGHT_SNAPSHOTTER_ROOT}" 26 | 27 | # Remove Redis data folder 28 | rm -rf /tmp/test-redis-data 29 | 30 | 31 | # Restart the service 32 | #./out/starlight-grpc run --server=starlight.yuri.moe --log-level=debug & 33 | #containerd & 34 | -------------------------------------------------------------------------------- /test/helper.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | "os" 8 | ) 9 | 10 | func PrettyPrintJson(obj interface{}) { 11 | j, _ := json.MarshalIndent(obj, "", "\t") 12 | fmt.Println(string(j)) 13 | } 14 | 15 | type FakeResponseWriter struct { 16 | header http.Header 17 | id string 18 | } 19 | 20 | func (w *FakeResponseWriter) Header() http.Header { 21 | return w.header 22 | } 23 | 24 | func (w *FakeResponseWriter) Write(b []byte) (int, error) { 25 | err := os.WriteFile(fmt.Sprintf("../sandbox/%s.tar.gz", w.id), b, 0644) 26 | if err != nil { 27 | return 0, err 28 | } 29 | return len(b), nil 30 | } 31 | 32 | func (w *FakeResponseWriter) WriteHeader(statusCode int) { 33 | b, _ := json.MarshalIndent(w.header, "", " ") 34 | _ = os.WriteFile(fmt.Sprintf("../sandbox/%s.json", w.id), b, 0644) 35 | fmt.Printf("status code: %d", statusCode) 36 | } 37 | 38 | func NewFakeResponseWriter(id string) *FakeResponseWriter { 39 | return &FakeResponseWriter{ 40 | header: make(http.Header), 41 | id: id, 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /demo/terraform/outputs.tf: -------------------------------------------------------------------------------- 1 | output "cloud-instance-id" { 2 | description = "The ec2 instance id" 3 | value = aws_instance.starlight_cloud.id 4 | sensitive = false 5 | } 6 | 7 | output "cloud-instance-public-ip" { 8 | description = "The ec2 instance public ip" 9 | value = aws_instance.starlight_cloud.public_ip 10 | sensitive = false 11 | } 12 | 13 | output "cloud-instance-private-ip" { 14 | description = "The ec2 instance private ip" 15 | value = aws_instance.starlight_cloud.private_ip 16 | sensitive = false 17 | } 18 | 19 | output "edge-instance-id" { 20 | description = "The ec2 instance id" 21 | value = aws_instance.starlight_edge.id 22 | sensitive = false 23 | } 24 | 25 | output "edge-instance-public-ip" { 26 | description = "The ec2 instance public ip" 27 | value = aws_instance.starlight_edge.public_ip 28 | sensitive = false 29 | } 30 | 31 | output "edge-instance-private-ip" { 32 | description = "The ec2 instance private ip" 33 | value = aws_instance.starlight_edge.private_ip 34 | sensitive = false 35 | } 36 | 37 | -------------------------------------------------------------------------------- /util/version.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The starlight Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | 16 | file created by maverick in 2021 17 | */ 18 | 19 | package util 20 | 21 | import ( 22 | "fmt" 23 | 24 | "github.com/urfave/cli/v2" 25 | ) 26 | 27 | var Version = "0.0.0" 28 | 29 | func VersionAction(context *cli.Context) error { 30 | fmt.Printf("starlight version %s", Version) 31 | return nil 32 | } 33 | 34 | func VersionCommand() *cli.Command { 35 | cmd := cli.Command{ 36 | Name: "version", 37 | Usage: "Show version information", 38 | Action: VersionAction, 39 | } 40 | return &cmd 41 | } 42 | -------------------------------------------------------------------------------- /util/json.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The starlight Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | 16 | file created by maverick in 2021 17 | */ 18 | 19 | package util 20 | 21 | import ( 22 | "encoding/json" 23 | "os" 24 | ) 25 | 26 | func ExportToJsonFile(v interface{}, p string) error { 27 | fh, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE, 0644) 28 | if err != nil { 29 | return err 30 | } 31 | defer fh.Close() 32 | 33 | buf, err := json.MarshalIndent(v, "", "\t") 34 | if err != nil { 35 | return err 36 | } 37 | 38 | _, err = fh.Write(buf) 39 | if err != nil { 40 | return err 41 | } 42 | 43 | return fh.Sync() 44 | } 45 | -------------------------------------------------------------------------------- /demo/chart/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: starlight 3 | description: A Helm chart for Kubernetes 4 | 5 | # A chart can be either an 'application' or a 'library' chart. 6 | # 7 | # Application charts are a collection of templates that can be packaged into versioned archives 8 | # to be deployed. 9 | # 10 | # Library charts provide useful utilities or functions for the chart developer. They're included as 11 | # a dependency of application charts to inject those utilities and functions into the rendering 12 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 13 | type: application 14 | 15 | # This is the chart version. This version number should be incremented each time you make changes 16 | # to the chart and its templates, including the app version. 17 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 18 | version: 0.0.0 19 | 20 | # This is the version number of the application being deployed. This version number should be 21 | # incremented each time you make changes to the application. Versions are not expected to 22 | # follow Semantic Versioning. They should reflect the version the application is using. 23 | # It is recommended to use it with quotes. 24 | appVersion: "1.16.0" 25 | -------------------------------------------------------------------------------- /docs/terraform.md: -------------------------------------------------------------------------------- 1 | # Setup Starlight Experiment using Terraform 2 | 3 | ## Prerequisites 4 | - [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) 5 | - [AWS account](https://aws.amazon.com/) You will need to setup programmatic access to AWS (e.g. set up credentials in `$HOME/.aws/config` and `$HOME/.aws/credentials`). 6 | 7 | 8 | ## Install 9 | 1. Clone the repository 10 | ```shell 11 | git clone https://github.com/mc256/starlight.git 12 | cd starlight/demo/terraform 13 | ``` 14 | 15 | 2. Initialize Terraform 16 | ```shell 17 | terraform init 18 | ``` 19 | 20 | 3. Modify `terraform.tfvars` to your needs. 21 | 22 | 23 | 4. Apply the configuration 24 | ```shell 25 | terraform apply 26 | ``` 27 | 28 | 5. Wait for the infrastructure to be created. This may take a few minutes. After the infrastructure is create you can see there is a `.completed` file in the home directory. 29 | 30 | ## Experiment 31 | 32 | 1. SSH into the Starlight CLI Tool pods in the edge node. 33 | ```shell 34 | ssh -i ubuntu@ 35 | ``` 36 | 37 | 2. Run the experiment 38 | 39 | 40 | 41 | ## Uninstall 42 | 43 | 1. Destroy the infrastructure 44 | ```shell 45 | terraform destroy 46 | ``` -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ######################################################################### 2 | # starlight-proxy 3 | ######################################################################### 4 | FROM golang:1.20 AS starlight-proxy-build 5 | 6 | WORKDIR /go/src/app 7 | COPY . . 8 | 9 | ENV GO111MODULE=on 10 | ENV PRODUCTION=true 11 | 12 | RUN make change-version-number set-production starlight-proxy-for-alpine 13 | ######################################################################### 14 | FROM alpine:3.12 AS starlight-proxy 15 | 16 | COPY --from=starlight-proxy-build /go/src/app/out/ /opt/ 17 | WORKDIR /opt 18 | EXPOSE 8090 19 | CMD ["/opt/starlight-proxy"] 20 | 21 | ######################################################################### 22 | # ctr-starlight 23 | ######################################################################### 24 | FROM golang:1.20 AS starlight-cli-build 25 | 26 | WORKDIR /go/src/app 27 | COPY . . 28 | 29 | ENV GO111MODULE=on 30 | ENV PRODUCTION=true 31 | 32 | RUN make change-version-number set-production ctr-starlight-for-alpine 33 | ######################################################################### 34 | FROM alpine:3.12 AS starlight-cli 35 | 36 | COPY --from=starlight-cli-build /go/src/app/out/ /opt/ 37 | WORKDIR /opt 38 | EXPOSE 8090 39 | CMD ["/opt/ctr-starlight"] 40 | -------------------------------------------------------------------------------- /demo/terraform/variables.tf: -------------------------------------------------------------------------------- 1 | variable "default_tags" { 2 | default = { 3 | project = "starlight-experiment" 4 | environment = "dev" 5 | } 6 | } 7 | 8 | variable "ssh_key_name" { 9 | type = string 10 | description = "the ssh key to access the ec2 instance" 11 | } 12 | 13 | variable "ssh_public_key" { 14 | type = string 15 | description = "the private key to access the ec2 instance. If not provided, we assume the public key is already in the AWS account" 16 | default = "" 17 | } 18 | 19 | variable "cloud_instance_type" { 20 | type = string 21 | default = "m5a.large" 22 | description = "the instance type to use" 23 | } 24 | 25 | variable "edge_instance_type" { 26 | type = string 27 | default = "t2.micro" 28 | description = "the instance type to use" 29 | } 30 | 31 | variable "project_id" { 32 | type = string 33 | default = "starlight" 34 | description = "the project name" 35 | } 36 | 37 | variable "cloud_ebs_size_in_gb" { 38 | type = number 39 | default = 20 40 | description = "the ebs size in gb" 41 | } 42 | 43 | variable "edge_ebs_size_in_gb" { 44 | type = number 45 | default = 10 46 | description = "the ebs size in gb" 47 | } 48 | 49 | 50 | variable "starlight_version" { 51 | type = string 52 | default = "0.6.2" 53 | description = "the version of the starlight software to deploy" 54 | } 55 | -------------------------------------------------------------------------------- /client/snapshotter/plugin_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package snapshotter 7 | 8 | import ( 9 | "fmt" 10 | "testing" 11 | 12 | "github.com/opencontainers/go-digest" 13 | "github.com/opencontainers/image-spec/identity" 14 | ) 15 | 16 | func Test_diffIds(t *testing.T) { 17 | // t.Skip("for dev only") 18 | 19 | diffs := []digest.Digest{"sha256:bad4c2b21ac784690edb9fab9336c06f429523122db8258386b0816ea9ccff13", 20 | "sha256:541172e54f7a947125460b76b5788026aa93b9cfb6a9ee132159bd792fc5a213", 21 | "sha256:a0daea3da3a891e0907836e15387b107376b938918839afd555a93f56a04a437", 22 | "sha256:c21bfa4fc8f93fcf8cee7f410285c9b0830c39e581d122668f1bfac657d01539", 23 | "sha256:2f02d7c1705eec01163defce0c73bad60ef4696b9fd2e009bf64f15425e3cb9b", 24 | "sha256:56b9282b4ec2b115a530fb08958abc2dee400cebc1befbdc3d2a70ee8a7afc97"} 25 | chainId := identity.ChainID(diffs) 26 | //fmt.Println(chainId) 27 | 28 | if chainId != "sha256:b2defa2545685fad9251740b472ecc07e0334f652a973ec0f37bf55ba9917b70" { 29 | t.Fatal("failed") 30 | } 31 | // sha256:b2defa2545685fad9251740b472ecc07e0334f652a973ec0f37bf55ba9917b70 32 | } 33 | 34 | func Test_diffIds2(t *testing.T) { 35 | t.Skip("for dev only") 36 | 37 | diffs := []digest.Digest{"sha256:0ee7044be87efdc00d5c40caa3193d2192eb09e8d136cb9cc2ab9aa82864b6c1", 38 | "sha256:0ee7044be87efdc00d5c40caa3193d2192eb09e8d136cb9cc2ab9aa82864b6c1"} 39 | chainId := identity.ChainID(diffs) 40 | fmt.Println(chainId) 41 | } 42 | -------------------------------------------------------------------------------- /demo/terraform/terraform.tfvars: -------------------------------------------------------------------------------- 1 | # required 2 | ssh_key_name = "starlight-key" 3 | 4 | # please replace with your own public key 5 | # this is the key for accessing the EC2 instances, if empty, we assume the key above is already created 6 | ssh_public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAgEA08XpxYFsakw1hVCcFYTVvXMF+sxyF9PQ16dF+D1xP3Vr5MGq275FIyKQs0oQxKCCBtUGMIgCVrQ+V3CVu9CsfBTHoJlk1nW0dfJcB3j9kG3+3NRIHZ+FTrIO5hvMWJ+SzXTIDYgdcKLJ/YIboqBI4Rm/rNx7fVIib0K+43RcD4YRbwcMXK1lWkZ2czj+ixKmX5P0EeTLNz2J6XEsLT3V39B4QoXLKJM3TTjlRx2w980UI2lUED+3aW8jeW6Y9v9TUW5ZiZHxFLULQGD4UoivxLHUZCVJbnSaBX5r1yiE2jOvlhCXgaEa/EyYt6gBacMMC/qdIIX6nsts5+K4i/+37Jixd1qyJ5tWcxY9tUQZeiD1kuL0qy/mKtR9ONZLOUFvmWXG7t5i9axVaIyjj+Yb/4PXvEVd3jZ0jC8gcPjkkjOV22CLCCLgwtUHcDTE/OM8/oUem4Tnd/9blBjd47RfuHTdyVsujLwue5hpUo64E3JDSdVee0s1Yso1Wx8ZEfhJgqfAc+E5gSprY2pdFZUwffN52I8+72OfNnsmw1h10LqvN5Xpu+12eARr/LQHY/0E45kJRAcFbAgjUwKmKtdf0UAepb/AwLF37I9UT4uLxs56dvw4z1rJQDGQJdm8GVv0CJ8pzjBsYFYzf43Mp4+lYJ+V7BBMMo5YLa6em/bqi5s= mc256" 7 | 8 | # recommended to change to machine with more memory 9 | # 10 | # Current setting is tide to AWS free tier limit 750hours of t3.micro (1GB memory). 11 | cloud_instance_type = "t3.micro" 12 | edge_instance_type = "t3.micro" 13 | 14 | 15 | # EBS volume size in GB 16 | # Cloud will need more space for storing the container image and metadata than the edge. 17 | # Please adjust the size according to your needs. 18 | # 19 | # Current setting is tide to AWS EBS free tier limit 30GB 20 | cloud_ebs_size_in_gb = 20 21 | edge_ebs_size_in_gb = 10 22 | -------------------------------------------------------------------------------- /client/fs/tracer_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The starlight Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | 16 | file created by maverick in 2021 17 | */ 18 | 19 | package fs 20 | 21 | import ( 22 | "encoding/json" 23 | "os" 24 | "path" 25 | "testing" 26 | 27 | "github.com/mc256/starlight/util" 28 | ) 29 | 30 | // Test trace collection 31 | func TestLoadTraces(t *testing.T) { 32 | ctx := util.ConfigLogger() 33 | tc, err := NewTraceCollection(ctx, os.TempDir()) 34 | if err != nil { 35 | t.Fatalf("failed to create trace collection: %v", err) 36 | } 37 | 38 | buf, err := json.MarshalIndent(tc, "", "\t") 39 | if err != nil { 40 | t.Fatalf("failed to marshal json: %v", err) 41 | } 42 | _ = os.WriteFile(path.Join(os.TempDir(), "group-optimize.json"), buf, 0644) 43 | 44 | // check if file exists 45 | _, err = os.Stat(path.Join(os.TempDir(), "group-optimize.json")) 46 | if err != nil { 47 | t.Fatalf("file does not exists: %v", err) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /demo/terraform/.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/aws" { 5 | version = "5.35.0" 6 | constraints = "~> 5.35.0" 7 | hashes = [ 8 | "h1:KlFlsBQpmSzE+vrYnXQeYEwX/K2E/yUIf5bX4ilOS7Q=", 9 | "zh:3a2a6f40db82d30ea8c5e3e251ca5e16b08e520570336e7e342be823df67e945", 10 | "zh:420a23b69b412438a15b8b2e2c9aac2cf2e4976f990f117e4bf8f630692d3949", 11 | "zh:4d8b887f6a71b38cff77ad14af9279528433e279eed702d96b81ea48e16e779c", 12 | "zh:4edd41f8e1c7d29931608a7b01a7ae3d89d6f95ef5502cf8200f228a27917c40", 13 | "zh:6337544e2ded5cf37b55a70aa6ce81c07fd444a2644ff3c5aad1d34680051bdc", 14 | "zh:668faa3faaf2e0758bf319ea40d2304340f4a2dc2cd24460ddfa6ab66f71b802", 15 | "zh:79ddc6d7c90e59fdf4a51e6ea822ba9495b1873d6a9d70daf2eeaf6fc4eb6ff3", 16 | "zh:885822027faf1aa57787f980ead7c26e7d0e55b4040d926b65709b764f804513", 17 | "zh:8c50a8f397b871388ff2e048f5eb280af107faa2e8926694f1ffd9f32a7a7cdf", 18 | "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", 19 | "zh:a2f5d2553df5573a060641f18ee7585587047c25ba73fd80617f59b5893d22b4", 20 | "zh:c43833ae2a152213ee92eb5be7653f9493779eddbe0ce403ea49b5f1d87fd766", 21 | "zh:dab01527a3a55b4f0f958af6f46313d775e27f9ad9d10bedbbfea4a35a06dc5f", 22 | "zh:ed49c65620ec42718d681a7fc00c166c295ff2795db6cede2c690b83f9fb3e65", 23 | "zh:f0a358c0ae1087c466d0fbcc3b4da886f33f881a145c3836ec43149878b86a1a", 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /cmd/ctr-starlight/convert/options.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The starlight Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | 16 | file created by maverick in 2022 17 | */ 18 | 19 | package convert 20 | 21 | import ( 22 | "github.com/urfave/cli/v2" 23 | ) 24 | 25 | var ( 26 | Flags = []cli.Flag{ 27 | &cli.BoolFlag{ 28 | Name: "insecure-source", 29 | Usage: "use HTTP for the source image", 30 | Value: false, 31 | Required: false, 32 | }, 33 | &cli.BoolFlag{ 34 | Name: "insecure-destination", 35 | Usage: "use HTTP for the destination image", 36 | Value: false, 37 | Required: false, 38 | }, 39 | &cli.StringFlag{ 40 | Name: "platform", 41 | Usage: "chose what platforms to convert, e.g. 'linux/amd64,linux/arm64', use comma to separate. " + 42 | "'all' means all platforms in the source image. " + 43 | "If image is not multi-platform, this flag will be ignored. ", 44 | Value: "all", 45 | Required: false, 46 | }, 47 | } 48 | ) 49 | -------------------------------------------------------------------------------- /util/db.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The starlight Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | 16 | file created by maverick in 2021 17 | */ 18 | 19 | package util 20 | 21 | import ( 22 | "context" 23 | "encoding/binary" 24 | "github.com/containerd/containerd/log" 25 | bolt "go.etcd.io/bbolt" 26 | ) 27 | 28 | const ( 29 | metadataStorage = "metadata.db" 30 | ) 31 | 32 | func OpenDatabase(ctx context.Context, dbPath string) (*bolt.DB, error) { 33 | // Open Database 34 | db, err := bolt.Open(dbPath, 0600, nil) 35 | if err != nil { 36 | log.G(ctx).Fatal(err) 37 | return nil, err 38 | } 39 | return db, nil 40 | } 41 | 42 | func Int32ToB(v uint32) []byte { 43 | b := make([]byte, 4) 44 | binary.BigEndian.PutUint32(b, v) 45 | return b 46 | } 47 | 48 | func Int64ToB(v uint64) []byte { 49 | b := make([]byte, 8) 50 | binary.BigEndian.PutUint64(b, v) 51 | return b 52 | } 53 | 54 | func BToInt32(v []byte) uint32 { 55 | return binary.BigEndian.Uint32(v) 56 | } 57 | 58 | func BToInt64(v []byte) uint64 { 59 | return binary.BigEndian.Uint64(v) 60 | } 61 | -------------------------------------------------------------------------------- /test/environment.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "path" 8 | "strings" 9 | 10 | "github.com/joho/godotenv" 11 | ) 12 | 13 | // IsDevEnvironment returns true if the STARLIGHT_ENV environment variable starts with "DEV". 14 | // export STARLIGHT_ENV=dev 15 | func IsDevEnvironment() bool { 16 | return strings.HasPrefix(strings.ToUpper(os.Getenv("STARLIGHT_ENV")), "DEV") 17 | } 18 | 19 | func checkDockerConfig(domainName string) bool { 20 | type dockerConfig struct { 21 | Auths map[string]struct{} 22 | } 23 | 24 | home, err := os.UserHomeDir() 25 | if err != nil { 26 | return false 27 | } 28 | 29 | buf, err := os.ReadFile(fmt.Sprintf("%s/.docker/config.json", home)) 30 | if err != nil { 31 | return false 32 | } 33 | 34 | cfg := &dockerConfig{} 35 | err = json.Unmarshal(buf, cfg) 36 | if err != nil { 37 | return false 38 | } 39 | 40 | _, has := cfg.Auths[domainName] 41 | return has 42 | } 43 | 44 | func HasLoginAWSECR() bool { 45 | // aws ecr-public get-login-password --region us-east-1 --profile $AWS_PROFILE | docker login --username AWS --password-stdin $DOMAIN 46 | return checkDockerConfig("public.ecr.aws") 47 | } 48 | 49 | func HasLoginStarlightGoharbor() bool { 50 | return checkDockerConfig(os.Getenv("STARLIGHT_CONTAINER_REGISTRY")) 51 | } 52 | 53 | func LoadEnvironmentVariables() { 54 | // in case the running path is not the root of the project 55 | dir, _ := os.Getwd() 56 | for !strings.HasSuffix(dir, "starlight") { 57 | dir = path.Dir(dir) 58 | if dir == "/" { 59 | break 60 | } 61 | } 62 | godotenv.Load(path.Join(dir, ".env")) 63 | } 64 | -------------------------------------------------------------------------------- /util/receive/helper.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package receive 7 | 8 | import ( 9 | "archive/tar" 10 | "os" 11 | "syscall" 12 | ) 13 | 14 | // --------------------------------------------------------------------- 15 | // Helper Methods 16 | 17 | // modeOfEntry gets system's mode bits from TOCEntry 18 | // From estargz 19 | func modeOfEntry(e *ReferencedFile) uint32 { 20 | // Permission bits 21 | res := uint32(e.Stat().Mode() & os.ModePerm) 22 | 23 | // File type bits 24 | switch e.Stat().Mode() & os.ModeType { 25 | case os.ModeDevice: 26 | res |= syscall.S_IFBLK 27 | case os.ModeDevice | os.ModeCharDevice: 28 | res |= syscall.S_IFCHR 29 | case os.ModeDir: 30 | res |= syscall.S_IFDIR 31 | case os.ModeNamedPipe: 32 | res |= syscall.S_IFIFO 33 | case os.ModeSymlink: 34 | res |= syscall.S_IFLNK 35 | case os.ModeSocket: 36 | res |= syscall.S_IFSOCK 37 | default: // regular file. 38 | res |= syscall.S_IFREG 39 | } 40 | 41 | // SUID, SGID, Sticky bits 42 | // Stargz package doesn't provide these bits so let's calculate them manually 43 | // here. TOCEntry.Mode is a copy of tar.Header.Mode so we can understand the 44 | // mode using that package. 45 | // See also: 46 | // - https://github.com/google/crfs/blob/71d77da419c90be7b05d12e59945ac7a8c94a543/stargz/stargz.go#L706 47 | hm := (&tar.Header{Mode: e.Mode}).FileInfo().Mode() 48 | if hm&os.ModeSetuid != 0 { 49 | res |= syscall.S_ISUID 50 | } 51 | if hm&os.ModeSetgid != 0 { 52 | res |= syscall.S_ISGID 53 | } 54 | if hm&os.ModeSticky != 0 { 55 | res |= syscall.S_ISVTX 56 | } 57 | 58 | return res 59 | } 60 | -------------------------------------------------------------------------------- /cmd/ctr-starlight/version/version.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package version 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | "time" 12 | 13 | pb "github.com/mc256/starlight/client/api" 14 | "github.com/mc256/starlight/util" 15 | "github.com/urfave/cli/v2" 16 | "google.golang.org/grpc" 17 | "google.golang.org/grpc/credentials/insecure" 18 | ) 19 | 20 | func getVersion(client pb.DaemonClient) (string, error) { 21 | ctx, cancel := context.WithTimeout(context.Background(), time.Minute) 22 | defer cancel() 23 | resp, err := client.GetVersion(ctx, &pb.Request{}) 24 | if err != nil { 25 | return "", err 26 | } 27 | return resp.Version, nil 28 | } 29 | 30 | func Action(context *cli.Context) error { 31 | fmt.Printf("ctr-starlight %s\n", util.Version) 32 | 33 | // Dial to the daemon 34 | address := context.String("address") 35 | opts := grpc.WithTransportCredentials(insecure.NewCredentials()) 36 | conn, err := grpc.Dial(address, opts) 37 | if err != nil { 38 | fmt.Printf("connect to starlight daemon failed: %v\n", err) 39 | return nil 40 | } 41 | defer conn.Close() 42 | 43 | // Get Version 44 | v, err := getVersion(pb.NewDaemonClient(conn)) 45 | if err != nil { 46 | fmt.Printf("failed to obtain starlight daemon version: %v\n", err) 47 | return nil 48 | } 49 | fmt.Printf("starlight-daemon %s\n", v) 50 | 51 | return nil 52 | } 53 | 54 | func Command() *cli.Command { 55 | cmd := cli.Command{ 56 | Name: "version", 57 | Usage: "Show version information", 58 | Action: func(context *cli.Context) error { 59 | return Action(context) 60 | }, 61 | } 62 | return &cmd 63 | } 64 | -------------------------------------------------------------------------------- /demo/compose/docker-compose-example.yaml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | ################################################################ 4 | # proxy 5 | ################################################################ 6 | proxy: 7 | image: "ghcr.io/mc256/starlight/proxy:latest" 8 | ports: 9 | - 8090:8090 10 | command: 11 | - /opt/starlight-proxy 12 | environment: 13 | - STARLIGHT_HOST=0.0.0.0 14 | - STARLIGHT_PORT=8090 15 | - LOG_LEVEL=info 16 | - DB_CONNECTION_STRING=postgres://postgres:postgres@db:5432/postgres?sslmode=disable 17 | restart: always 18 | depends_on: 19 | - db 20 | - registry 21 | ################################################################ 22 | # db 23 | ################################################################ 24 | db: 25 | image: postgres:latest 26 | ports: 27 | - 5432:5432 28 | environment: 29 | - POSTGRES_PASSWORD=postgres 30 | - POSTGRES_USER=postgres 31 | - POSTGRES_DB=postgres 32 | #volumes: 33 | # - "./data_db:/var/lib/postgresql/data" 34 | dbadmin: 35 | image: adminer:latest 36 | ports: 37 | - 8080:8080 38 | environment: 39 | - ADMINER_DEFAULT_SERVER=postgres 40 | depends_on: 41 | - db 42 | ################################################################ 43 | # registry 44 | ################################################################ 45 | registry: 46 | image: registry:2 47 | ports: 48 | - 5001:5000 # Change to avoid conflict with Apple AirTunes server 49 | environment: 50 | - REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY=/data 51 | #volumes: 52 | #- "./data_registry:/data:rw" 53 | restart: always 54 | -------------------------------------------------------------------------------- /util/writer.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The starlight Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | 16 | file created by maverick in 2021 17 | */ 18 | 19 | package util 20 | 21 | import ( 22 | "crypto/sha256" 23 | "hash" 24 | "io" 25 | ) 26 | 27 | type CountWriter struct { 28 | w io.Writer 29 | count int64 30 | } 31 | 32 | func (c *CountWriter) Write(p []byte) (n int, err error) { 33 | n, err = c.w.Write(p) 34 | c.count += int64(n) 35 | return 36 | } 37 | 38 | func (c *CountWriter) GetWrittenSize() int64 { 39 | return c.count 40 | } 41 | 42 | func NewCountWriter(w io.Writer) (cw *CountWriter) { 43 | cw = &CountWriter{ 44 | w: w, 45 | count: 0, 46 | } 47 | return 48 | } 49 | 50 | type DigestWriter struct { 51 | w io.Writer 52 | digest hash.Hash 53 | } 54 | 55 | func (d *DigestWriter) Write(p []byte) (n int, err error) { 56 | n, err = d.w.Write(p) 57 | if err != nil { 58 | return n, err 59 | } 60 | n, err = d.digest.Write(p) 61 | if err != nil { 62 | return n, err 63 | } 64 | return 65 | } 66 | 67 | func (d *DigestWriter) GetDigest() []byte { 68 | return d.digest.Sum(nil) 69 | } 70 | 71 | func NewDigestWriter(w io.Writer) (dw *DigestWriter) { 72 | dw = &DigestWriter{ 73 | w: w, 74 | digest: sha256.New(), 75 | } 76 | return 77 | } 78 | -------------------------------------------------------------------------------- /docs/deprecated-v0.1/starlight-workflow.md: -------------------------------------------------------------------------------- 1 | # How Starlight Works (Overview) 2 | ![starlight-workflow](images/starlight-workflow.png) 3 | 4 | Once the user issues a worker `PULL` command to download a set of containers ①, 5 | the command is received by the standard **containerd** daemon. 6 | **containerd** then forwards the command to the **Starlight snapshotter** daemon ②, 7 | and waits for confirmation that the requested images have been found. 8 | The Starlight snapshotter opens a connection to the **Starlight proxy** 9 | and sends the list of requested containers as well as the list of relevant containers that already exist on the worker ③. 10 | The proxy queries the directory database ④ for the list of files in the various layers of the 11 | requested container image, as well in the image already available on the worker. 12 | 13 | The proxy will then begin computing the **delta bundle** that includes the set of distinct compressed file contents that the worker does not already have, specifically organized to speed up deployment; 14 | In the background, the proxy also responds with HTTP 200 OK header to the snapshotter, which notifies **containerd** that the `PULL` phase has finished successfully; the snapshotter however, remains active and keeps the connection open to receive the data from the proxy. 15 | In the background, the proxy issues a series of requests to the registry ⑦ to retrieve the compressed contents of files needed for delta bundle. 16 | Once the contents of the delta bundle has been computed, the proxy creates a **Starlight manifest** (SLM) -- the list of file metadata, container manifests, and other required metadata -- and sends it to the snapshotter ⑤, 17 | which notifies **containerd** that the `PULL` phase has finished successfully. 18 | 19 | Please read our [NSDI '22 paper](https://www.usenix.org/conference/nsdi22/presentation/chen-jun-lin) for more details. 20 | -------------------------------------------------------------------------------- /proxy/database_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package proxy 7 | 8 | import ( 9 | "context" 10 | "encoding/json" 11 | "fmt" 12 | "os" 13 | "testing" 14 | 15 | "github.com/mc256/starlight/client/fs" 16 | "github.com/mc256/starlight/util/send" 17 | ) 18 | 19 | var ( 20 | db *Database 21 | cfg *Configuration 22 | ) 23 | 24 | func TestMain(m *testing.M) { 25 | cfg, _, _, _ = LoadConfig("") 26 | ctx := context.Background() 27 | var err error 28 | db, err = NewDatabase(ctx, cfg.PostgresConnectionString) 29 | if err != nil { 30 | fmt.Printf("failed to connect to database %v\n", err) 31 | } 32 | m.Run() 33 | } 34 | 35 | func TestDatabase_Init(t *testing.T) { 36 | ctx := context.Background() 37 | db, err := NewDatabase(ctx, cfg.PostgresConnectionString) 38 | if err != nil { 39 | t.Error(err) 40 | return 41 | } 42 | 43 | if err = db.InitDatabase(); err != nil { 44 | t.Error(err) 45 | return 46 | } 47 | fmt.Println("done") 48 | } 49 | 50 | func TestDatabase_GetFiles(t *testing.T) { 51 | fl, err := db.GetUniqueFiles([]*send.ImageLayer{{Serial: 33}, {Serial: 34}, {Serial: 35}}) 52 | if err != nil { 53 | t.Error(err) 54 | } 55 | for _, f := range fl { 56 | fmt.Println(f) 57 | } 58 | } 59 | 60 | func TestDatabase_GetFilesWithRanks(t *testing.T) { 61 | fl, err := db.GetFilesWithRanks(5) 62 | if err != nil { 63 | t.Error(err) 64 | } 65 | for _, f := range fl { 66 | fmt.Println(f) 67 | } 68 | } 69 | 70 | func TestDatabase_UpdateFileRanks(t *testing.T) { 71 | t.Skip("for dev only") 72 | 73 | p := "/tmp/group-optimize.json" 74 | b, err := os.ReadFile(p) 75 | if err != nil { 76 | t.Error(err) 77 | return 78 | } 79 | var col *fs.TraceCollection 80 | err = json.Unmarshal(b, &col) 81 | 82 | var arr [][][]int64 83 | arr, err = db.UpdateFileRanks(col) 84 | if err != nil { 85 | t.Error(err) 86 | return 87 | } 88 | fmt.Println(arr) 89 | } 90 | -------------------------------------------------------------------------------- /cmd/ctr-starlight/ping/ping.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package ping 7 | 8 | import ( 9 | "context" 10 | "errors" 11 | "fmt" 12 | 13 | pb "github.com/mc256/starlight/client/api" 14 | "github.com/urfave/cli/v2" 15 | "google.golang.org/grpc" 16 | "google.golang.org/grpc/credentials/insecure" 17 | ) 18 | 19 | func ping(client pb.DaemonClient, req *pb.PingRequest, quiet bool) { 20 | resp, err := client.PingTest(context.Background(), req) 21 | if err != nil { 22 | fmt.Printf("network test failed: %v\n", err) 23 | return 24 | } 25 | if resp.Success { 26 | if !quiet { 27 | fmt.Printf("ping test success: %v\nlatency: %d ms\n", resp.GetMessage(), resp.GetLatency()) 28 | } 29 | } else { 30 | fmt.Printf("network test failed: %v\n", resp.GetMessage()) 31 | } 32 | } 33 | 34 | func Action(ctx context.Context, c *cli.Context) (err error) { 35 | if c.NArg() > 1 { 36 | return errors.New("wrong number of arguments, expected container image reference") 37 | } 38 | 39 | profile := c.Args().First() 40 | 41 | // Dial to the daemon 42 | address := c.String("address") 43 | opts := grpc.WithTransportCredentials(insecure.NewCredentials()) 44 | conn, err := grpc.Dial(address, opts) 45 | if err != nil { 46 | fmt.Printf("connect to starlight daemon failed: %v\n", err) 47 | return nil 48 | } 49 | defer conn.Close() 50 | 51 | // ping 52 | ping(pb.NewDaemonClient(conn), &pb.PingRequest{ 53 | ProxyConfig: profile, 54 | }, c.Bool("quiet")) 55 | 56 | return nil 57 | } 58 | 59 | func Command() *cli.Command { 60 | ctx := context.Background() 61 | cmd := cli.Command{ 62 | Name: "test", 63 | Aliases: []string{"ping"}, 64 | Usage: "testing network connection between starlight proxy and starlight daemon, returns HTTP RTT in ms", 65 | Action: func(c *cli.Context) error { 66 | return Action(ctx, c) 67 | }, 68 | ArgsUsage: "[proxy-profile-name]", 69 | } 70 | return &cmd 71 | } 72 | -------------------------------------------------------------------------------- /util/log.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The starlight Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | 16 | file created by maverick in 2021 17 | */ 18 | 19 | package util 20 | 21 | import ( 22 | "context" 23 | "strings" 24 | "time" 25 | 26 | "github.com/containerd/containerd/log" 27 | "github.com/sirupsen/logrus" 28 | ) 29 | 30 | func ConfigLogger() (ctx context.Context) { 31 | return ConfigLoggerWithLevel("info") 32 | } 33 | 34 | func ConfigLoggerWithLevel(level string) (ctx context.Context) { 35 | level = strings.ToLower(level) 36 | 37 | // Logger 38 | ctx = context.Background() 39 | log.GetLogger(ctx).Logger.SetFormatter(&logrus.TextFormatter{ 40 | FullTimestamp: true, 41 | TimestampFormat: time.StampNano, 42 | //ForceColors: true, 43 | //DisableColors: false, 44 | }) 45 | 46 | switch level { 47 | 48 | case "fatal": 49 | log.GetLogger(ctx).Logger.SetLevel(logrus.FatalLevel) 50 | return 51 | case "error": 52 | log.GetLogger(ctx).Logger.SetLevel(logrus.ErrorLevel) 53 | return 54 | case "warning": 55 | log.GetLogger(ctx).Logger.SetLevel(logrus.WarnLevel) 56 | return 57 | case "info": 58 | log.GetLogger(ctx).Logger.SetLevel(logrus.InfoLevel) 59 | return 60 | case "debug": 61 | log.GetLogger(ctx).Logger.SetLevel(logrus.DebugLevel) 62 | return 63 | case "trace": 64 | log.GetLogger(ctx).Logger.SetLevel(logrus.TraceLevel) 65 | return 66 | default: 67 | log.GetLogger(ctx).Logger.SetLevel(logrus.InfoLevel) 68 | return 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /cmd/ctr-starlight/listproxy/list_proxy.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by cstria0106 in 2022 3 | 4 | */ 5 | 6 | package listproxy 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | "time" 12 | 13 | pb "github.com/mc256/starlight/client/api" 14 | "github.com/pkg/errors" 15 | "github.com/urfave/cli/v2" 16 | "google.golang.org/grpc" 17 | "google.golang.org/grpc/credentials/insecure" 18 | ) 19 | 20 | // listProxyProfile prints the list of proxy profiles in Starlight daemon configuration 21 | func listProxyProfile(client pb.DaemonClient) error { 22 | ctx, cancel := context.WithTimeout(context.Background(), time.Minute) 23 | defer cancel() 24 | 25 | resp, err := client.GetProxyProfiles(ctx, &pb.Request{}) 26 | if err != nil { 27 | return errors.Wrapf(err, "failed to list proxy profiles") 28 | } 29 | 30 | // get max length of the name 31 | maxLen := 0 32 | for _, profile := range resp.Profiles { 33 | if len(profile.Name) > maxLen { 34 | maxLen = len(profile.Name) 35 | } 36 | } 37 | 38 | // print 39 | for _, profile := range resp.Profiles { 40 | fmt.Printf("%*s %s://%s\n", maxLen+2, fmt.Sprintf("[%s]", profile.Name), profile.Protocol, profile.Address) 41 | } 42 | 43 | return nil 44 | } 45 | 46 | func Action(context *cli.Context) (err error) { 47 | // Dial to the daemon 48 | address := context.String("address") 49 | opts := grpc.WithTransportCredentials(insecure.NewCredentials()) 50 | conn, err := grpc.Dial(address, opts) 51 | if err != nil { 52 | fmt.Printf("connect to starlight daemon failed: %v\n", err) 53 | return nil 54 | } 55 | defer conn.Close() 56 | 57 | if context.NArg() != 0 { 58 | return fmt.Errorf("invalid number of arguments") 59 | } 60 | 61 | // send to proxy server 62 | if err = listProxyProfile(pb.NewDaemonClient(conn)); err != nil { 63 | return err 64 | } 65 | 66 | return nil 67 | } 68 | 69 | func Command() *cli.Command { 70 | cmd := cli.Command{ 71 | Name: "list-proxy", 72 | Aliases: []string{"ls", "lp"}, 73 | Usage: "list starlight proxy servers in the configuration", 74 | Action: func(c *cli.Context) error { 75 | return Action(c) 76 | }, 77 | ArgsUsage: "", 78 | } 79 | return &cmd 80 | } 81 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Go Testing 2 | on: 3 | # PR testing before merge 4 | pull_request: 5 | types: 6 | - opened 7 | - edited 8 | - synchronize 9 | # For Testing 10 | push: 11 | branches: 12 | - feature_debian_* 13 | # For Release 14 | workflow_run: 15 | workflows: ["Versioning"] 16 | types: 17 | - completed 18 | concurrency: 19 | group: ${{ github.workflow }}-${{ github.ref }} 20 | cancel-in-progress: true 21 | jobs: 22 | build: 23 | name: Test All 24 | runs-on: ubuntu-latest 25 | steps: 26 | - name: Check out repository code 27 | uses: actions/checkout@v3 28 | with: 29 | fetch-depth: 0 30 | - name: Install Go 31 | uses: actions/setup-go@v2 32 | with: 33 | go-version-file: './go.mod' 34 | - name: Login to Goharbor Container Registry 35 | # This is needed for one of the test cases to check whether Starlight can integrate with Goharbor 36 | # Please make sure that the registry is available 37 | # status is available at: http://status-production.mc256.workers.dev 38 | uses: docker/login-action@v2 39 | with: 40 | registry: ${{ secrets.GOHARBOR_REGISTRY }} 41 | username: ${{ secrets.GOHARBOR_USERNAME }} 42 | password: ${{ secrets.GOHARBOR_PASSWORD }} 43 | logout: true 44 | - name: Install Buildessentials 45 | run: | 46 | sudo apt update -y 47 | sudo apt upgrade -y 48 | sudo apt install build-essential make -y 49 | - name: Install Docker 50 | uses: docker/setup-buildx-action@v1 51 | - name: Test 52 | run: | 53 | mkdir -p `pwd`/sandbox/etc/starlight 54 | docker-compose -f demo/compose/docker-compose-example.yaml up -d 55 | echo '{"port": 8090,"address": "0.0.0.0","log_level": "info","postgres": "postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable","default_registry": "registry.yuri.moe","default_registry_alias": ["registry.yuri.moe"],"cache_timeout": 3600}' > `pwd`/sandbox/etc/starlight/starlight-proxy.json 56 | ls -al `pwd` 57 | make test -------------------------------------------------------------------------------- /DETAILS.md: -------------------------------------------------------------------------------- 1 | 2 | ## Starlight Proxy 3 | 4 | 5 | ### Prebuild Docker Image and Docker-Compose 6 | 7 | The prebuild image is available in 8 | 9 | ```url 10 | ghcr.io/mc256/starlight/proxy:latest 11 | registry.yuri.moe/starlight-proxy:latest 12 | ``` 13 | 14 | Please first clone this project and go to the proxy demo folder 15 | ```shell 16 | git clone https://github.com/mc256/starlight.git 17 | cd ./starlight/demo/compose/proxy 18 | ``` 19 | 20 | In `config.env` and then launch the proxy using `docker-compose up -d`. The proxy listens on port 8090. 21 | 22 | --- 23 | 24 | ## Starlight Snapshotter 25 | 26 | ### 1. Prerequisites 27 | 28 | Starlight depends on `containerd`. Please install `containerd` follow the instructions on [containerd's Github repository](https://github.com/containerd/containerd). 29 | 30 | To enable the Starlight snapshotter plugin, add the following configuration to `/etc/containerd/config.toml` 31 | 32 | ```yaml 33 | [proxy_plugins] 34 | [proxy_plugins.starlight] 35 | type = "snapshot" 36 | address = "/run/starlight-grpc/starlight-snapshotter.socket" 37 | ``` 38 | 39 | ### 2. Build From Source 40 | 41 | Checkout this repository 42 | 43 | ```shell 44 | git clone git@github.com:mc256/starlight.git 45 | ``` 46 | 47 | Build and install this project 48 | 49 | ```shell 50 | make && sudo make install 51 | ``` 52 | 53 | 54 | ### 3. Run this project 55 | 56 | ```shell 57 | starlight-grpc run --server $STARLIGHT_PROXY_ADDRESS --socket /run/starlight-grpc/starlight-snapshotter.socket 58 | ``` 59 | or uses systemd service in `./demo/deb-package/debian/starlight.service` 60 | 61 | --- 62 | 63 | ## Other configurations 64 | 65 | Latency can impact the available bandwith if the TCP window is too small. 66 | Please use the following configurations in `/etc/sysctl.conf` to increase the default TCP window size or compute a suitable configuration using https://www.speedguide.net/bdp.php. 67 | 68 | ```shell 69 | net.core.wmem_max=125829120 70 | net.core.rmem_max=125829120 71 | net.ipv4.tcp_rmem= 10240 87380 125829120 72 | net.ipv4.tcp_wmem= 10240 87380 125829120 73 | net.ipv4.tcp_window_scaling = 1 74 | net.ipv4.tcp_timestamps = 1 75 | net.ipv4.tcp_sack = 1 76 | net.ipv4.tcp_no_metrics_save = 1 77 | net.core.netdev_max_backlog = 10000 78 | ``` 79 | 80 | After setting the new TCP window in `/etc/sysctl.conf`, use `sysctl -p` to apply the configuration. 81 | -------------------------------------------------------------------------------- /demo/chart/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.ingress.enabled -}} 2 | {{- $fullName := include "starlight.fullname" . -}} 3 | {{- $fullNameRegistry := include "starlight.fullname-registry" . -}} 4 | {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} 5 | {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} 6 | {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} 7 | {{- end }} 8 | {{- end }} 9 | {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} 10 | apiVersion: networking.k8s.io/v1 11 | {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} 12 | apiVersion: networking.k8s.io/v1beta1 13 | {{- else -}} 14 | apiVersion: extensions/v1beta1 15 | {{- end }} 16 | kind: Ingress 17 | metadata: 18 | name: {{ $fullName }} 19 | namespace: {{ .Values.namespace }} 20 | labels: 21 | {{- include "starlight.proxyLabels" . | nindent 4 }} 22 | annotations: 23 | nginx.ingress.kubernetes.io/rewrite-target: /$2 24 | {{- with .Values.ingress.annotations }} 25 | {{- toYaml . | nindent 4 }} 26 | {{- end }} 27 | spec: 28 | {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} 29 | ingressClassName: {{ .Values.ingress.ingressClassName }} 30 | {{- end }} 31 | {{- with .Values.ingress.tls }} 32 | tls: 33 | {{- toYaml . | nindent 4 }} 34 | {{- end }} 35 | rules: 36 | {{- range .Values.ingress.hosts }} 37 | - host: {{ . | quote }} 38 | http: 39 | paths: 40 | {{- if $.Values.registry.enabled }} 41 | - path: /()(v2.*) 42 | pathType: Prefix 43 | backend: 44 | service: 45 | name: {{ $fullNameRegistry }} 46 | port: 47 | number: 5000 48 | {{- end }} 49 | - path: / 50 | pathType: Prefix 51 | backend: 52 | service: 53 | name: {{ $fullName }} 54 | port: 55 | number: 8090 56 | - path: /()(scanner.*|starlight.*|health-check.*) 57 | pathType: Prefix 58 | backend: 59 | service: 60 | name: {{ $fullName }} 61 | port: 62 | number: 8090 63 | {{- end }} 64 | {{- end }} 65 | 66 | 67 | -------------------------------------------------------------------------------- /util/common/errors.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The starlight Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | 16 | file created by maverick in 2021 17 | */ 18 | 19 | package common 20 | 21 | import ( 22 | "errors" 23 | "fmt" 24 | "strings" 25 | ) 26 | 27 | var ( 28 | ErrNotImplemented = errors.New("this feature has not yet been implemented") 29 | ErrLayerNotFound = errors.New("cannot find layer") 30 | ErrMountingPointNotFound = errors.New("cannot find mounting point") 31 | ErrNotConsolidated = errors.New("delta image has not yet been consolidated") 32 | ErrAlreadyConsolidated = errors.New("delta image has been consolidated already") 33 | ErrHashCollision = errors.New("found two files have the same hash but different size") 34 | ErrMergedImageNotFound = errors.New("the requested image has not been merged") 35 | ErrWrongImageFormat = errors.New("please use this format :") 36 | ErrOrphanNode = errors.New("an entry node has no parent") 37 | ErrNoRoPath = errors.New("entry does not have path to RO layer") 38 | ErrImageNotFound = errors.New("cannot find image") 39 | ErrNoManager = errors.New("no manager found") 40 | ErrUnknownSnapshotParameter = errors.New("snapshots should follow a standard format") 41 | ErrTocUnknown = errors.New("please prefetch the delta image") 42 | ) 43 | 44 | // Aggregate combines a list of errors into a single new error. 45 | func ErrorAggregate(errs []error) error { 46 | switch len(errs) { 47 | case 0: 48 | return nil 49 | case 1: 50 | return errs[0] 51 | default: 52 | points := make([]string, len(errs)+1) 53 | points[0] = fmt.Sprintf("%d error(s) occurred:", len(errs)) 54 | for i, err := range errs { 55 | points[i+1] = fmt.Sprintf("* %s", err) 56 | } 57 | return errors.New(strings.Join(points, "\n\t")) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /.github/workflows/helm-chart.yml: -------------------------------------------------------------------------------- 1 | name: Helm Chart 2 | on: 3 | # PR testing before merge 4 | pull_request: 5 | types: 6 | - opened 7 | - edited 8 | - synchronize 9 | # For Testing 10 | push: 11 | branches: 12 | - feature_helm_* 13 | - fix_helm_* 14 | # For Release 15 | workflow_run: 16 | workflows: ["Versioning"] 17 | types: 18 | - completed 19 | concurrency: 20 | group: ${{ github.workflow }}-${{ github.ref }} 21 | cancel-in-progress: true 22 | jobs: 23 | release: 24 | name: Helm Chart 25 | # depending on default permission settings for your org (contents being read-only or read-write for workloads), you will have to add permissions 26 | # see: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token 27 | permissions: 28 | packages: write 29 | contents: read 30 | runs-on: ubuntu-latest 31 | steps: 32 | - name: Check out the repo 33 | uses: actions/checkout@v3 34 | with: 35 | fetch-depth: 0 36 | - name: Login to GitHub Container Registry 37 | uses: docker/login-action@v1 38 | with: 39 | registry: ghcr.io 40 | username: ${{ github.repository_owner }} 41 | password: ${{ secrets.GITHUB_TOKEN }} 42 | logout: true 43 | - name: Configure Git 44 | run: | 45 | git config user.name "$GITHUB_ACTOR" 46 | git config user.email "$GITHUB_ACTOR@users.noreply.github.com" 47 | - name: Install Helm 48 | uses: azure/setup-helm@v1 49 | with: 50 | version: v3.8.1 51 | - name: Helm Chart Build 52 | shell: bash 53 | run: |- 54 | echo ${{ secrets.GITHUB_TOKEN }} | helm registry login -u ${{ github.repository_owner }} --password-stdin ghcr.io 55 | make helm-package 56 | env: 57 | HELM_EXPERIMENTAL_OCI: '1' 58 | - name: Helm Chart Push 59 | shell: bash 60 | if: ${{ github.event.workflow_run.conclusion == 'success' }} 61 | run: |- 62 | echo ${{ secrets.GITHUB_TOKEN }} | helm registry login -u ${{ github.repository_owner }} --password-stdin ghcr.io 63 | make push-helm-package 64 | env: 65 | HELM_EXPERIMENTAL_OCI: '1' 66 | - name: Helm Chart Logout 67 | shell: bash 68 | run: |- 69 | helm registry logout ghcr.io 70 | env: 71 | HELM_EXPERIMENTAL_OCI: '1' -------------------------------------------------------------------------------- /demo/chart/templates/daemonset-edge.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.edge.enabled}} 2 | apiVersion: apps/v1 3 | kind: DaemonSet 4 | metadata: 5 | name: {{ include "starlight.fullname-edge" . }} 6 | namespace: {{ .Values.namespace }} 7 | labels: 8 | {{- include "starlight.edgeLabels" . | nindent 4 }} 9 | kubernetes.io/cluster-service: "true" 10 | spec: 11 | selector: 12 | matchLabels: 13 | {{- include "starlight.edgeSelectorLabels" . | nindent 6 }} 14 | template: 15 | metadata: 16 | {{- with .Values.podAnnotations }} 17 | annotations: 18 | {{- toYaml . | nindent 8 }} 19 | {{- end }} 20 | labels: 21 | {{- include "starlight.edgeSelectorLabels" . | nindent 8 }} 22 | spec: 23 | {{- if .Values.serviceAccount.enabled}} 24 | serviceAccountName: {{ include "starlight.serviceAccountName" . }} 25 | {{- end }} 26 | {{- with .Values.imagePullSecrets }} 27 | imagePullSecrets: 28 | {{- toYaml . | nindent 8 }} 29 | {{- end }} 30 | securityContext: 31 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 32 | volumes: 33 | - name: socket 34 | hostPath: 35 | path: /run/starlight 36 | containers: 37 | ############################################################## 38 | ############################################################## 39 | - name: starlight-edge 40 | securityContext: 41 | {{- toYaml .Values.securityContext | nindent 12 }} 42 | image: "{{ .Values.edge.repository }}:{{ .Values.edge.tag | default .Chart.AppVersion }}" 43 | imagePullPolicy: {{ .Values.edge.imagePullPolicy }} 44 | command: 45 | {{- toYaml .Values.edge.command | nindent 12 }} 46 | resources: 47 | {{- toYaml .Values.edge.resources | nindent 12}} 48 | env: 49 | {{- toYaml .Values.edge.env | nindent 12}} 50 | volumeMounts: 51 | - name: socket 52 | mountPath: /run/starlight 53 | ############################################################## 54 | ############################################################## 55 | {{- with .Values.edgeNodeSelector }} 56 | nodeSelector: 57 | {{- toYaml . | nindent 8 }} 58 | {{- end }} 59 | {{- with .Values.edgeAffinity }} 60 | affinity: 61 | {{- toYaml . | nindent 8 }} 62 | {{- end }} 63 | {{- with .Values.edgeTolerations }} 64 | tolerations: 65 | {{- toYaml . | nindent 8 }} 66 | {{- end }} 67 | {{- end}} -------------------------------------------------------------------------------- /.github/workflows/debian-package.yml: -------------------------------------------------------------------------------- 1 | name: Debian Package 2 | on: 3 | # PR testing before merge 4 | pull_request: 5 | types: 6 | - opened 7 | - edited 8 | - synchronize 9 | # For Testing 10 | push: 11 | branches: 12 | - feature_debian_* 13 | # For Release 14 | workflow_run: 15 | workflows: ["Versioning"] 16 | types: 17 | - completed 18 | concurrency: 19 | group: ${{ github.workflow }}-${{ github.ref }} 20 | cancel-in-progress: true 21 | jobs: 22 | build: 23 | name: Starlight Daemon 24 | #runs-on: self-hosted 25 | runs-on: ubuntu-latest 26 | steps: 27 | - name: Check out repository code 28 | uses: actions/checkout@v3 29 | with: 30 | fetch-depth: 0 31 | - name: Get SemVer 32 | id: get-version 33 | run: | 34 | echo "semver="`git describe --tags --match "v*" | cut -d '-' -f 1 || echo "v0.0.0"` >> $GITHUB_OUTPUT 35 | echo "major="`git describe --tags --match "v*" | cut -d '-' -f 1 | sed -e "s/^v//" | cut -d '.' -f 1` >> $GITHUB_OUTPUT 36 | echo "minor="`git describe --tags --match "v*" | cut -d '-' -f 1 | sed -e "s/^v//" | cut -d '.' -f 2` >> $GITHUB_OUTPUT 37 | echo "patch="`git describe --tags --match "v*" | cut -d '-' -f 1 | sed -e "s/^v//" | cut -d '.' -f 3` >> $GITHUB_OUTPUT 38 | - name: Build deb package 39 | # This needs container image from registry.yuri.moe. It is a small image that you can upload your deb package to. and save in a patial directory. 40 | # Please make sure that the registry is available 41 | # status is available at: http://status-production.mc256.workers.dev 42 | env: 43 | APT_UPLOAD_AUTH: ${{ secrets.APT_UPLOAD_AUTH }} 44 | run: | 45 | sudo apt update -y 46 | sudo apt upgrade -y 47 | sudo apt install net-tools 48 | curl -fsSL https://get.docker.com -o /tmp/get-docker.sh 49 | sh /tmp/get-docker.sh 50 | docker run --privileged --rm tonistiigi/binfmt --install all 51 | docker run -d --hostname helper --expose 8080 --name helper -v "$(pwd)"/sandbox:/app/upload:rw registry.yuri.moe/public/helper:latest 52 | export UPLOAD_URL=http://`docker inspect helper | grep "IPAddress" | grep -o -E '[0-9.]+' | head -n 1`:8080/ 53 | make docker-buildx-multi-arch 54 | - name: Release 55 | uses: softprops/action-gh-release@v1 56 | if: ${{ github.event.workflow_run.conclusion == 'success' }} 57 | with: 58 | tag_name: ${{ steps.get-version.outputs.semver }} 59 | files: ./sandbox/*.deb -------------------------------------------------------------------------------- /client/fs/instance.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package fs 7 | 8 | import ( 9 | "context" 10 | "github.com/containerd/containerd/log" 11 | "github.com/hanwen/go-fuse/v2/fs" 12 | "github.com/hanwen/go-fuse/v2/fuse" 13 | iofs "io/fs" 14 | "os" 15 | "syscall" 16 | "time" 17 | ) 18 | 19 | type ImageManager interface { 20 | GetPathByStack(stack int64) string 21 | GetPathBySerial(stack int64) string 22 | LookUpFile(stack int64, filename string) ReceivedFile 23 | LogTrace(stack int64, filename string, access, complete time.Time) 24 | } 25 | 26 | // Instance should be created using 27 | type Instance struct { 28 | ctx context.Context 29 | 30 | Root ReceivedFile 31 | rootInode *StarlightFsNode 32 | 33 | stack int64 34 | mountPoint string 35 | 36 | manager ImageManager 37 | server *fuse.Server 38 | } 39 | 40 | func (fi *Instance) GetMountPoint() string { return fi.mountPoint } 41 | func (fi *Instance) GetServer() *fuse.Server { return fi.server } 42 | 43 | // Teardown unmounts the file system and close the logging file if there is one writing 44 | // should you need this function, please consider using Manager.Teardown instead. 45 | func (fi *Instance) Teardown() error { 46 | return fi.GetServer().Unmount() 47 | } 48 | 49 | func (fi *Instance) Serve() { 50 | fi.server.Serve() 51 | } 52 | 53 | func NewInstance(ctx context.Context, m ImageManager, root ReceivedFile, stack int64, dir string, options *fs.Options, debug bool) (fi *Instance, err error) { 54 | fi = &Instance{ 55 | ctx: ctx, 56 | manager: m, 57 | stack: stack, 58 | } 59 | 60 | fi.rootInode = &StarlightFsNode{fs.Inode{}, root, fi} 61 | fi.mountPoint = dir 62 | 63 | one := time.Second 64 | 65 | options.EntryTimeout = &one 66 | options.AttrTimeout = &one 67 | options.AllowOther = true 68 | options.Name = "starlightfs" 69 | options.Options = []string{"suid", "ro"} 70 | options.DirectMount = true 71 | options.NullPermissions = true 72 | options.RememberInodes = false 73 | 74 | if debug { 75 | options.Debug = true 76 | } 77 | 78 | if _, err = os.Stat(dir); err != nil && err.(*iofs.PathError).Err == syscall.ENOTCONN { 79 | // if the directory exists, it means that the snapshot is already created 80 | // or there is a problem unmounting the snapshot 81 | // Transport endpoint is not connected 82 | err = syscall.Unmount(dir, UnmountFlag) 83 | log.G(ctx). 84 | WithError(err). 85 | WithField("dir", dir). 86 | Warn("fs: cleanup disconnected mountpoint") 87 | } 88 | 89 | rawFs := fs.NewNodeFS(fi.rootInode, options) 90 | fi.server, err = fuse.NewServer(rawFs, dir, &options.MountOptions) 91 | 92 | return fi, err 93 | } 94 | -------------------------------------------------------------------------------- /cmd/ctr-starlight/report/report.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The starlight Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | 16 | file created by maverick in 2021 17 | */ 18 | 19 | package report 20 | 21 | import ( 22 | "context" 23 | "fmt" 24 | "time" 25 | 26 | pb "github.com/mc256/starlight/client/api" 27 | "github.com/mc256/starlight/cmd/ctr-starlight/auth" 28 | "github.com/urfave/cli/v2" 29 | "google.golang.org/grpc" 30 | "google.golang.org/grpc/credentials/insecure" 31 | ) 32 | 33 | // report uses the daemon client to report traces, it does not report directly to the proxy 34 | // Because it does not handle the credential and authentication of the proxy. 35 | func report(client pb.DaemonClient, req *pb.ReportTracesRequest, quiet bool) error { 36 | ctx, cancel := context.WithTimeout(context.Background(), time.Minute) 37 | defer cancel() 38 | resp, err := client.ReportTraces(ctx, req) 39 | if err != nil { 40 | return fmt.Errorf("report traces failed: %v", err) 41 | } 42 | if resp.Success { 43 | if !quiet { 44 | fmt.Printf("reported traces: %s\n", resp.Message) 45 | } 46 | } else { 47 | fmt.Printf("report traces failed: %s\n", resp.Message) 48 | } 49 | return nil 50 | } 51 | 52 | func Action(ctx context.Context, c *cli.Context) (err error) { 53 | // Dial to the daemon 54 | address := c.String("address") 55 | opts := grpc.WithTransportCredentials(insecure.NewCredentials()) 56 | conn, err := grpc.Dial(address, opts) 57 | if err != nil { 58 | fmt.Printf("connect to starlight daemon failed: %v\n", err) 59 | return nil 60 | } 61 | defer conn.Close() 62 | 63 | // report 64 | return report(pb.NewDaemonClient(conn), &pb.ReportTracesRequest{ 65 | ProxyConfig: c.String("profile"), 66 | }, c.Bool("quiet")) 67 | } 68 | 69 | func Command() *cli.Command { 70 | ctx := context.Background() 71 | cmd := cli.Command{ 72 | Name: "report", 73 | Usage: "Report filesystem traces back to Starlight Proxy to speed up other similar deployment", 74 | Action: func(c *cli.Context) error { 75 | return Action(ctx, c) 76 | }, 77 | Flags: auth.ProxyFlags, 78 | ArgsUsage: "", 79 | } 80 | return &cmd 81 | } 82 | -------------------------------------------------------------------------------- /cmd/ctr-starlight/optimizer/optimizer.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package optimizer 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | "time" 12 | 13 | pb "github.com/mc256/starlight/client/api" 14 | "github.com/urfave/cli/v2" 15 | "google.golang.org/grpc" 16 | "google.golang.org/grpc/credentials/insecure" 17 | ) 18 | 19 | func optimizer(client pb.DaemonClient, req *pb.OptimizeRequest, quiet bool) error { 20 | ctx, cancel := context.WithTimeout(context.Background(), time.Minute) 21 | defer cancel() 22 | resp, err := client.SetOptimizer(ctx, req) 23 | if err != nil { 24 | return fmt.Errorf("set optimizer status failed: %v", err) 25 | } 26 | if resp.Success { 27 | if !quiet { 28 | fmt.Printf("set optimizer: %s\n", resp.Message) 29 | for k, v := range resp.Okay { 30 | fmt.Printf("\t%s: %s - okay \n", k, v) 31 | } 32 | for k, v := range resp.Failed { 33 | fmt.Printf("\t%s: %v \n", k, v) 34 | } 35 | } 36 | } else { 37 | fmt.Printf("set optimizer status failed: %s\n", resp.Message) 38 | } 39 | return nil 40 | } 41 | 42 | func Action(ctx context.Context, c *cli.Context) (err error) { 43 | if c.NArg() != 1 { 44 | return fmt.Errorf("wrong number of arguments, expected 1, got %d", c.NArg()) 45 | } 46 | action := c.Args().First() 47 | var a bool 48 | switch action { 49 | case "on": 50 | a = true 51 | break 52 | case "off": 53 | a = false 54 | break 55 | default: 56 | return fmt.Errorf("operation should be 'on' or 'off'") 57 | } 58 | 59 | // Dial to the daemon 60 | address := c.String("address") 61 | opts := grpc.WithTransportCredentials(insecure.NewCredentials()) 62 | conn, err := grpc.Dial(address, opts) 63 | if err != nil { 64 | fmt.Printf("connect to starlight daemon failed: %v\n", err) 65 | return nil 66 | } 67 | defer conn.Close() 68 | 69 | // set optimizer 70 | return optimizer(pb.NewDaemonClient(conn), &pb.OptimizeRequest{ 71 | Enable: a, 72 | Group: c.String("group"), 73 | }, c.Bool("quiet")) 74 | } 75 | 76 | func Command() *cli.Command { 77 | ctx := context.Background() 78 | return &cli.Command{ 79 | Name: "optimizer", 80 | Usage: `collect filesystem traces to find out the priorities of files.`, 81 | Action: func(c *cli.Context) error { 82 | return Action(ctx, c) 83 | }, 84 | Flags: []cli.Flag{ 85 | &cli.StringFlag{ 86 | Name: "group", 87 | Aliases: []string{"g"}, 88 | Usage: "group name for collecting multiple traces from multiple containers", 89 | Value: "", 90 | }, 91 | 92 | &cli.StringFlag{ 93 | Name: "quiet", 94 | Aliases: []string{"q"}, 95 | Value: "", 96 | Usage: "do not print any message unless error occurs", 97 | }, 98 | }, 99 | ArgsUsage: "on|off", 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /proxy/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package proxy 7 | 8 | import ( 9 | "encoding/json" 10 | "os" 11 | "path" 12 | 13 | "github.com/google/uuid" 14 | "github.com/mc256/starlight/util" 15 | "github.com/pkg/errors" 16 | ) 17 | 18 | type Configuration struct { 19 | // server 20 | ListenPort int `json:"port"` 21 | ListenAddress string `json:"address"` 22 | LogLevel string `json:"log_level"` 23 | 24 | // database 25 | PostgresConnectionString string `json:"postgres"` 26 | 27 | // registry 28 | DefaultRegistry string `json:"default_registry"` 29 | DefaultRegistryAlias []string `json:"default_registry_alias"` 30 | 31 | // goharbor hook 32 | //EnableHarborScanner bool `json:"harbor"` 33 | //HarborApiKey string `json:"harbor_apikey"` 34 | 35 | // layer cache timeout (second) 36 | CacheTimeout int `json:"cache_timeout"` 37 | } 38 | 39 | func LoadConfig(cfgPath string) (c *Configuration, p string, n bool, error error) { 40 | c = NewConfig() 41 | 42 | if cfgPath == "" { 43 | etcPath := util.GetEtcConfigPath() 44 | if err := os.MkdirAll(etcPath, 0775); err != nil { 45 | error = errors.Wrapf(err, "cannot create config folder") 46 | return 47 | } 48 | 49 | p = path.Join(etcPath, "starlight-proxy.json") 50 | } else { 51 | p = cfgPath 52 | } 53 | 54 | b, err := os.ReadFile(p) 55 | n = false 56 | if err != nil { 57 | n = true 58 | 59 | buf, _ := json.Marshal(c) 60 | 61 | if err = os.WriteFile(p, buf, 0644); err == nil { 62 | return 63 | } else { 64 | error = errors.Wrapf(err, "cannot create config file") 65 | } 66 | } 67 | 68 | if err = json.Unmarshal(b, c); err != nil { 69 | error = errors.Wrapf(err, "cannot parse config file") 70 | } 71 | 72 | return 73 | } 74 | 75 | func (c *Configuration) SaveConfig() error { 76 | etcPath := util.GetEtcConfigPath() 77 | if err := os.MkdirAll(etcPath, 0775); err != nil { 78 | return errors.Wrapf(err, "cannot create config folder") 79 | } 80 | 81 | p := path.Join(etcPath, "starlight-proxy.json") 82 | buf, _ := json.MarshalIndent(c, " ", " ") 83 | err := os.WriteFile(p, buf, 0644) 84 | if err == nil { 85 | return nil 86 | } 87 | 88 | return nil 89 | } 90 | 91 | func NewConfig() *Configuration { 92 | uuid.EnableRandPool() 93 | return &Configuration{ 94 | ListenPort: 8090, 95 | ListenAddress: "0.0.0.0", 96 | LogLevel: "info", 97 | PostgresConnectionString: "postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable", 98 | DefaultRegistry: "127.0.0.1:9000", 99 | DefaultRegistryAlias: []string{ 100 | "localhost:9000", 101 | }, 102 | 103 | //EnableHarborScanner: false, 104 | //HarborApiKey: uuid.New().String(), 105 | 106 | CacheTimeout: 3600, 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /demo/convert-vanilla.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script requires installation of Docker 4 | # (perhaps running this on the registry server) 5 | # TODO: change it to use the `ctr` command 6 | 7 | 8 | declare -a ImageList=( 9 | "ubuntu:18.04" 10 | "ubuntu:20.04" 11 | "alpine:3.12.7" 12 | "alpine:3.13.5" 13 | "busybox:1.32.1" 14 | "busybox:1.33.0" 15 | "debian:oldstable" 16 | "debian:stable" 17 | "centos:7" 18 | "centos:8" 19 | "fedora:32" 20 | "fedora:33" 21 | "oraclelinux:7" 22 | "oraclelinux:8" 23 | "mysql:8.0.20" 24 | "mysql:8.0.21" 25 | "mysql:8.0.22" 26 | "mysql:8.0.23" 27 | "mysql:8.0.24" 28 | "mariadb:10.4" 29 | "mariadb:10.5" 30 | "cassandra:3.11.10" 31 | "cassandra:3.11.9" 32 | "redis:5.0" 33 | "redis:6.0" 34 | "redis:6.2" 35 | 36 | "ubuntu:focal-20210416" 37 | "ubuntu:focal-20210401" 38 | "alpine:3.13.5" 39 | "alpine:3.13.4" 40 | "busybox:1.32.1" 41 | "busybox:1.33.0" 42 | "busybox:1.32.1" 43 | "busybox:1.32.0" 44 | "debian:oldstable" 45 | "debian:stable" 46 | "centos:7" 47 | "centos:8" 48 | "fedora:32" 49 | "fedora:33" 50 | "oraclelinux:7" 51 | "oraclelinux:8" 52 | 53 | "mysql:8.0.20" 54 | "mysql:8.0.21" 55 | "mysql:8.0.22" 56 | "mysql:8.0.23" 57 | "mysql:8.0.24" 58 | "mariadb:10.5.9" 59 | "mariadb:10.5.8" 60 | 61 | "cassandra:3.11.10" 62 | "cassandra:3.11.9" 63 | "redis:5.0" 64 | "redis:6.0" 65 | "redis:6.2" 66 | "redis:6.2.2" 67 | "redis:6.2.1" 68 | "postgres:13.2" 69 | "postgres:13.1" 70 | "mongo:4.0.24" 71 | "mongo:4.0.23" 72 | 73 | "python:3.9.4" 74 | "python:3.9.3" 75 | "node:16-alpine3.12" 76 | "node:16-alpine3.11" 77 | "openjdk:16.0.1-jdk" 78 | "openjdk:11.0.11-9-jdk" 79 | "golang:1.16.3" 80 | "golang:1.16.2" 81 | 82 | "rabbitmq:3.8.14" 83 | "rabbitmq:3.8.13" 84 | 85 | "wordpress:php7.4-fpm" 86 | "wordpress:php7.3-fpm" 87 | 88 | 89 | "nextcloud:21.0.1-apache" 90 | "nextcloud:20.0.9-apache" 91 | 92 | "ghost:4.3.3-alpine" 93 | "ghost:3.42.5-alpine" 94 | "phpmyadmin:5.1.0-fpm-alpine" 95 | "phpmyadmin:5.0.4-fpm-alpine" 96 | 97 | "httpd:2.4.46" 98 | "httpd:2.4.43" 99 | 100 | "nginx:1.19.10" 101 | "nginx:1.20.0" 102 | 103 | "flink:1.12.3-scala_2.12-java8" 104 | "flink:1.12.3-scala_2.11-java8" 105 | 106 | "eclipse-mosquitto:2.0.10-openssl" 107 | "eclipse-mosquitto:2.0.9-openssl" 108 | 109 | "registry:2.7.1" 110 | "registry:2.7.0" 111 | 112 | "memcached:1.6.9" 113 | "memcached:1.6.8" 114 | ) 115 | 116 | for VAL in "${ImageList[@]}"; do 117 | docker pull "$VAL" 118 | docker image tag "$VAL" "localhost:5000/$VAL" 119 | docker push "localhost:5000/$VAL" 120 | done 121 | 122 | read -p "The following command will remove all the images in docker, if you want to keep, please click Ctrl+C. Otherwise, Press enter to continue" 123 | docker rmi -f $(docker images -q) -------------------------------------------------------------------------------- /demo/chart/templates/deployment-registry.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.registry.enabled }} 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: {{ include "starlight.fullname-registry" . }} 6 | namespace: {{ .Values.namespace }} 7 | labels: 8 | {{- include "starlight.registryLabels" . | nindent 4 }} 9 | kubernetes.io/cluster-service: "true" 10 | spec: 11 | selector: 12 | matchLabels: 13 | {{- include "starlight.registrySelectorLabels" . | nindent 6 }} 14 | template: 15 | metadata: 16 | {{- with .Values.podAnnotations }} 17 | annotations: 18 | {{- toYaml . | nindent 8 }} 19 | {{- end }} 20 | labels: 21 | {{- include "starlight.registrySelectorLabels" . | nindent 8 }} 22 | spec: 23 | {{- if .Values.serviceAccount.enabled}} 24 | serviceAccountName: {{ include "starlight.serviceAccountName" . }} 25 | {{- end }} 26 | {{- with .Values.imagePullSecrets }} 27 | imagePullSecrets: 28 | {{- toYaml . | nindent 8 }} 29 | {{- end }} 30 | securityContext: 31 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 32 | {{- if eq .Values.registry.persistence.enabled true }} 33 | volumes: 34 | - name: registry-pv 35 | persistentVolumeClaim: 36 | {{- $newClaimName := include "starlight.fullname-registry" . }} 37 | claimName: {{ .Values.registry.persistence.existingClaim | default $newClaimName }} 38 | {{- end }} 39 | containers: 40 | ############################################################## 41 | ############################################################## 42 | - name: container-registry 43 | securityContext: 44 | {{- toYaml .Values.securityContext | nindent 12 }} 45 | image: "{{ .Values.registry.repository }}:{{ .Values.registry.tag | default .Chart.AppVersion }}" 46 | imagePullPolicy: {{ .Values.registry.imagePullPolicy }} 47 | ports: 48 | - name: registry 49 | containerPort: 5000 50 | protocol: TCP 51 | livenessProbe: 52 | periodSeconds: 60 53 | httpGet: 54 | path: /v2/ 55 | port: 5000 56 | readinessProbe: 57 | httpGet: 58 | path: /v2/ 59 | port: 5000 60 | resources: 61 | {{- toYaml .Values.registry.resources | nindent 12}} 62 | ############################################################## 63 | ############################################################## 64 | {{- with .Values.cloudNodeSelector }} 65 | nodeSelector: 66 | {{- toYaml . | nindent 8 }} 67 | {{- end }} 68 | {{- with .Values.cloudAffinity }} 69 | affinity: 70 | {{- toYaml . | nindent 8 }} 71 | {{- end }} 72 | {{- with .Values.cloudTolerations }} 73 | tolerations: 74 | {{- toYaml . | nindent 8 }} 75 | {{- end }} 76 | {{- end}} -------------------------------------------------------------------------------- /client/snapshotter/operator.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package snapshotter 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | "github.com/containerd/containerd/log" 12 | "github.com/containerd/containerd/snapshots" 13 | "github.com/mc256/starlight/util" 14 | "github.com/pkg/errors" 15 | ) 16 | 17 | type OperatorClient interface { 18 | // GetFilesystemRoot gets the directory to store all layers 19 | GetFilesystemRoot() string 20 | 21 | // AddCompletedLayers find out completed layer and save it to Client's layerMap 22 | AddCompletedLayers(compressedLayerDigest string) 23 | } 24 | 25 | // Operator communicates with the containerd interface. 26 | // Because containerd would cache snapshot information, therefore, we need to communicate with 27 | // containerd to update the information we have. On the other side, the Plugin class communicates 28 | // with the snapshotter plugin interface 29 | type Operator struct { 30 | ctx context.Context 31 | client OperatorClient 32 | sn snapshots.Snapshotter 33 | } 34 | 35 | func NewOperator(ctx context.Context, client OperatorClient, sn snapshots.Snapshotter) *Operator { 36 | return &Operator{ 37 | ctx: ctx, 38 | client: client, 39 | sn: sn, 40 | } 41 | } 42 | 43 | func (op *Operator) ScanSnapshots() (err error) { 44 | return op.sn.Walk(op.ctx, func(ctx context.Context, info snapshots.Info) (err error) { 45 | log.G(op.ctx). 46 | WithField("snapshot", info.Name). 47 | WithField("parent", info.Parent). 48 | WithField("labels", info.Labels). 49 | Debug("found snapshot") 50 | return 51 | }) 52 | } 53 | 54 | func (op *Operator) AddSnapshot(name, parent, imageDigest, uncompressedDigest string, stack int64) (sn *snapshots.Info, err error) { 55 | var ( 56 | snn snapshots.Info 57 | ) 58 | 59 | randId := util.GetRandomId("prepare") 60 | 61 | // check name exists or not 62 | snn, err = op.sn.Stat(op.ctx, name) 63 | if err == nil { 64 | return &snn, nil 65 | } 66 | 67 | // parent -> key 68 | // be aware that the returned mount is Read-Only. 69 | _, err = op.sn.Prepare(op.ctx, randId, parent, snapshots.WithLabels(map[string]string{ 70 | util.SnapshotLabelRefImage: imageDigest, 71 | util.SnapshotLabelRefLayer: fmt.Sprintf("%d", stack), 72 | util.SnapshotLabelRefUncompressed: uncompressedDigest, 73 | })) 74 | if err != nil { 75 | return nil, errors.Wrapf(err, "failed to create snapshot") 76 | } 77 | 78 | // key -> name 79 | err = op.sn.Commit(op.ctx, name, randId, snapshots.WithLabels(map[string]string{ 80 | util.SnapshotLabelRefImage: imageDigest, 81 | util.SnapshotLabelRefLayer: fmt.Sprintf("%d", stack), 82 | util.SnapshotLabelRefUncompressed: uncompressedDigest, 83 | })) 84 | if err != nil { 85 | return nil, errors.Wrapf(err, "failed to commit snapshot") 86 | } 87 | 88 | // stat 89 | snn, err = op.sn.Stat(op.ctx, name) 90 | if err != nil { 91 | return nil, errors.Wrapf(err, "failed to stat snapshot") 92 | } 93 | return &snn, nil 94 | } 95 | -------------------------------------------------------------------------------- /util/common/cache.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package common 7 | 8 | import ( 9 | "bytes" 10 | "context" 11 | "fmt" 12 | "github.com/containerd/containerd/log" 13 | "github.com/google/go-containerregistry/pkg/authn" 14 | "github.com/google/go-containerregistry/pkg/name" 15 | v1 "github.com/google/go-containerregistry/pkg/v1" 16 | "github.com/google/go-containerregistry/pkg/v1/remote" 17 | "github.com/pkg/errors" 18 | "io" 19 | "sync" 20 | "time" 21 | ) 22 | 23 | type CacheInterface interface { 24 | Digest() name.Digest 25 | Size() int64 26 | } 27 | 28 | type LayerCache struct { 29 | Buffer *io.SectionReader 30 | 31 | Mutex sync.Mutex 32 | Ready bool 33 | UseCounter int 34 | LastUsed time.Time 35 | 36 | subscribers []*chan error 37 | 38 | digest name.Digest 39 | size int64 40 | } 41 | 42 | func (lc *LayerCache) String() string { 43 | lc.Mutex.Lock() 44 | defer lc.Mutex.Unlock() 45 | return fmt.Sprintf("%s:%v:%02d:%s", 46 | lc.digest, lc.Ready, lc.UseCounter, lc.LastUsed.Format(time.RFC3339Nano)) 47 | } 48 | 49 | func (lc *LayerCache) SetReady(err error) { 50 | lc.Mutex.Lock() 51 | defer lc.Mutex.Unlock() 52 | lc.Ready = true 53 | lc.notify(err) 54 | } 55 | 56 | func (lc *LayerCache) notify(err error) { 57 | for _, s := range lc.subscribers { 58 | if err != nil { 59 | *s <- err 60 | } 61 | close(*s) 62 | } 63 | } 64 | 65 | func (lc *LayerCache) Subscribe(errChan *chan error) { 66 | lc.Mutex.Lock() 67 | defer lc.Mutex.Unlock() 68 | if lc.Ready { 69 | close(*errChan) 70 | return 71 | } 72 | lc.subscribers = append(lc.subscribers, errChan) 73 | } 74 | 75 | func (lc *LayerCache) Load(ctx context.Context) (err error) { 76 | defer lc.SetReady(err) 77 | 78 | var l v1.Layer 79 | l, err = remote.Layer(lc.digest, remote.WithAuthFromKeychain(authn.DefaultKeychain)) 80 | if err != nil { 81 | return err 82 | } 83 | var rc io.ReadCloser 84 | rc, err = l.Compressed() 85 | if err != nil { 86 | log.G(ctx).WithField("layer", lc.String()).Error(errors.Wrapf(err, "failed to load layer")) 87 | return err 88 | } 89 | 90 | buf := new(bytes.Buffer) 91 | var n int64 92 | n, err = io.Copy(buf, rc) 93 | if err != nil { 94 | log.G(ctx).WithField("layer", lc.String()).Error(errors.Wrapf(err, "failed to load layer")) 95 | return err 96 | } 97 | if n != lc.size { 98 | err = fmt.Errorf("size unmatch expected %d, but got %d", lc.size, n) 99 | log.G(ctx).WithField("layer", lc.String()).Error(errors.Wrapf(err, "failed to load layer")) 100 | return err 101 | } 102 | rc.Close() 103 | 104 | lc.Buffer = io.NewSectionReader(bytes.NewReader(buf.Bytes()), 0, n) 105 | 106 | return nil 107 | } 108 | 109 | func NewLayerCache(layer CacheInterface) *LayerCache { 110 | return &LayerCache{ 111 | Buffer: nil, 112 | Mutex: sync.Mutex{}, 113 | Ready: false, 114 | 115 | UseCounter: 0, 116 | LastUsed: time.Now(), 117 | digest: layer.Digest(), 118 | size: layer.Size(), 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /cmd/ctr-starlight/notify/notify.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package notify 7 | 8 | import ( 9 | "context" 10 | "errors" 11 | "fmt" 12 | 13 | "github.com/containerd/containerd/log" 14 | "github.com/containerd/containerd/namespaces" 15 | "github.com/google/go-containerregistry/pkg/name" 16 | pb "github.com/mc256/starlight/client/api" 17 | "github.com/mc256/starlight/cmd/ctr-starlight/auth" 18 | "github.com/mc256/starlight/util" 19 | "github.com/urfave/cli/v2" 20 | "google.golang.org/grpc" 21 | "google.golang.org/grpc/credentials/insecure" 22 | ) 23 | 24 | func notify(ctx context.Context, client pb.DaemonClient, req *pb.NotifyRequest, quiet bool) error { 25 | resp, err := client.NotifyProxy(context.Background(), req) 26 | if err != nil { 27 | return fmt.Errorf("notify starlight proxy server failed: %v", err) 28 | } 29 | if resp.Success { 30 | if !quiet { 31 | //("notify starlight proxy server success: converted %s\n", resp.GetMessage()) 32 | log.G(ctx). 33 | WithField("reference", resp.GetMessage()). 34 | Infof("notify starlight proxy server success") 35 | } 36 | } else { 37 | return errors.New(resp.GetMessage()) 38 | } 39 | return nil 40 | } 41 | 42 | func SharedAction(ctx context.Context, c *cli.Context, reference name.Reference) (err error) { 43 | // Dial to the daemon 44 | address := c.String("address") 45 | opts := grpc.WithTransportCredentials(insecure.NewCredentials()) 46 | conn, err := grpc.Dial(address, opts) 47 | if err != nil { 48 | return fmt.Errorf("failed to connect starlight daemon: %v", err) 49 | } 50 | defer conn.Close() 51 | 52 | // notify 53 | return notify(ctx, pb.NewDaemonClient(conn), &pb.NotifyRequest{ 54 | ProxyConfig: c.String("profile"), 55 | Insecure: c.Bool("insecure") || c.Bool("insecure-destination"), 56 | Reference: reference.String(), 57 | }, c.Bool("quiet")) 58 | } 59 | 60 | func Action(ctx context.Context, c *cli.Context) (err error) { 61 | // logger 62 | ns := c.String("namespace") 63 | util.ConfigLoggerWithLevel(c.String("log-level")) 64 | ctx = namespaces.WithNamespace(ctx, ns) 65 | 66 | // Parse the reference 67 | options := []name.Option{} 68 | 69 | if c.NArg() != 1 { 70 | return errors.New("wrong number of arguments, expected container image reference") 71 | } 72 | 73 | argRef := c.Args().Get(0) 74 | if argRef == "" { 75 | return errors.New("no image reference provided") 76 | } 77 | 78 | reference, err := name.ParseReference(argRef, options...) 79 | return SharedAction(ctx, c, reference) 80 | } 81 | 82 | func Command() *cli.Command { 83 | ctx := context.Background() 84 | cmd := cli.Command{ 85 | Name: "notify", 86 | Usage: "Notify the Starlight Proxy that a new Starlight image is available", 87 | Action: func(c *cli.Context) error { 88 | return Action(ctx, c) 89 | }, 90 | Flags: append(auth.ProxyFlags, 91 | &cli.BoolFlag{ 92 | Name: "insecure", 93 | Usage: "use HTTP registry", 94 | Value: false, 95 | Required: false, 96 | }), 97 | ArgsUsage: "[flags] SourceImage StarlightImage", 98 | } 99 | return &cmd 100 | } 101 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/mc256/starlight 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/containerd/containerd v1.6.26 7 | github.com/containerd/continuity v0.3.0 8 | github.com/google/go-containerregistry v0.5.1 9 | github.com/google/uuid v1.3.0 10 | 11 | // resolved overlayfs OICTL issue in https://github.com/hanwen/go-fuse/pull/408 12 | // tested on Kernel 5.15.0-52-generic 13 | github.com/hanwen/go-fuse/v2 v2.1.1-0.20221003202731-4c25c9c1eece 14 | github.com/lib/pq v1.10.6 15 | github.com/opencontainers/go-digest v1.0.0 16 | github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b 17 | github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect 18 | github.com/pkg/errors v0.9.1 19 | github.com/sirupsen/logrus v1.9.3 20 | 21 | // newer version causes issue #24 22 | github.com/urfave/cli/v2 v2.3.0 23 | go.etcd.io/bbolt v1.3.7 24 | golang.org/x/net v0.23.0 // indirect 25 | golang.org/x/sync v0.3.0 26 | golang.org/x/sys v0.18.0 27 | google.golang.org/grpc v1.58.3 28 | ) 29 | 30 | require ( 31 | github.com/pelletier/go-toml v1.9.5 32 | google.golang.org/protobuf v1.33.0 33 | ) 34 | 35 | require ( 36 | github.com/VividCortex/ewma v1.2.0 // indirect 37 | github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect 38 | github.com/containerd/log v0.1.0 // indirect 39 | github.com/imdario/mergo v0.3.12 // indirect 40 | github.com/mattn/go-runewidth v0.0.14 // indirect 41 | github.com/rivo/uniseg v0.4.4 // indirect 42 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect 43 | ) 44 | 45 | require ( 46 | github.com/Microsoft/go-winio v0.5.2 // indirect 47 | github.com/Microsoft/hcsshim v0.9.10 // indirect 48 | github.com/containerd/cgroups v1.0.4 // indirect 49 | github.com/containerd/fifo v1.0.0 // indirect 50 | github.com/containerd/stargz-snapshotter/estargz v0.4.1 // indirect 51 | github.com/containerd/ttrpc v1.1.2 // indirect 52 | github.com/containerd/typeurl v1.0.2 // indirect 53 | github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect 54 | github.com/docker/cli v20.10.17+incompatible // indirect 55 | github.com/docker/distribution v2.8.2+incompatible // indirect 56 | github.com/docker/docker v25.0.6+incompatible // indirect 57 | github.com/docker/docker-credential-helpers v0.6.3 // indirect 58 | github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect 59 | github.com/gogo/googleapis v1.4.1 // indirect 60 | github.com/gogo/protobuf v1.3.2 // indirect 61 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 62 | github.com/golang/protobuf v1.5.3 // indirect 63 | github.com/joho/godotenv v1.5.1 64 | github.com/klauspost/compress v1.15.9 // indirect 65 | github.com/moby/locker v1.0.1 // indirect 66 | github.com/moby/sys/mountinfo v0.6.2 // indirect 67 | github.com/moby/sys/signal v0.7.0 // indirect 68 | github.com/opencontainers/runc v1.1.12 // indirect 69 | github.com/opencontainers/selinux v1.10.1 // indirect 70 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 71 | github.com/vbauerster/mpb/v8 v8.4.0 72 | go.opencensus.io v0.24.0 // indirect 73 | golang.org/x/text v0.14.0 // indirect 74 | ) 75 | 76 | replace github.com/containerd/containerd v1.6.18 => github.com/containerd/containerd v1.6.26 77 | -------------------------------------------------------------------------------- /cmd/ctr-starlight/addproxy/add_proxy.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package addproxy 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | pb "github.com/mc256/starlight/client/api" 12 | "github.com/urfave/cli/v2" 13 | "google.golang.org/grpc" 14 | "google.golang.org/grpc/credentials/insecure" 15 | "time" 16 | ) 17 | 18 | // addProxyProfile adds a proxy profile to the Starlight daemon configuration 19 | func addProxyProfile(client pb.DaemonClient, name, protocol, address, username, password string, quiet bool) error { 20 | ctx, cancel := context.WithTimeout(context.Background(), time.Minute) 21 | defer cancel() 22 | 23 | resp, err := client.AddProxyProfile(ctx, &pb.AuthRequest{ 24 | ProfileName: name, 25 | Protocol: protocol, 26 | Address: address, 27 | 28 | Username: username, 29 | Password: password, 30 | }) 31 | if err != nil { 32 | return err 33 | } 34 | if resp.Success { 35 | if !quiet { 36 | fmt.Printf("proxy profile %s added\n", name) 37 | } 38 | } else { 39 | fmt.Printf("failed to add proxy profile %s: %s\n", name, resp.Message) 40 | } 41 | return nil 42 | } 43 | 44 | func Action(context *cli.Context) (err error) { 45 | // Dial to the daemon 46 | address := context.String("address") 47 | opts := grpc.WithTransportCredentials(insecure.NewCredentials()) 48 | conn, err := grpc.Dial(address, opts) 49 | if err != nil { 50 | fmt.Printf("connect to starlight daemon failed: %v\n", err) 51 | return nil 52 | } 53 | defer conn.Close() 54 | 55 | // parse args 56 | var ( 57 | proxyName string 58 | proxyAddr string 59 | protocol string 60 | username string 61 | password string 62 | ) 63 | if context.NArg() == 3 { 64 | proxyName = context.Args().Get(0) 65 | protocol = context.Args().Get(1) 66 | proxyAddr = context.Args().Get(2) 67 | } else if context.NArg() == 4 { 68 | proxyName = context.Args().Get(0) 69 | protocol = context.Args().Get(1) 70 | proxyAddr = context.Args().Get(2) 71 | username = context.Args().Get(3) 72 | fmt.Printf("password for %s: ", username) 73 | _, err = fmt.Scanln(&password) 74 | if err != nil { 75 | return err 76 | } 77 | } else { 78 | return fmt.Errorf("invalid number of arguments") 79 | } 80 | 81 | // send to proxy server 82 | err = addProxyProfile(pb.NewDaemonClient(conn), 83 | proxyName, 84 | protocol, proxyAddr, 85 | username, password, 86 | context.Bool("quiet")) 87 | if err != nil { 88 | return err 89 | } 90 | 91 | return nil 92 | } 93 | 94 | func Command() *cli.Command { 95 | cmd := cli.Command{ 96 | Name: "add-proxy", 97 | Aliases: []string{"add", "ap"}, 98 | Usage: "add a starlight proxy server to the configuration", 99 | Action: func(c *cli.Context) error { 100 | return Action(c) 101 | }, 102 | Flags: []cli.Flag{ 103 | &cli.StringFlag{ 104 | Name: "password", 105 | Aliases: []string{"p"}, 106 | Value: "", 107 | Usage: "provide username and password for the proxy server", 108 | }, 109 | &cli.StringFlag{ 110 | Name: "quiet", 111 | Aliases: []string{"q"}, 112 | Value: "", 113 | Usage: "do not print any message unless error occurs", 114 | }, 115 | }, 116 | ArgsUsage: "proxy-name (http|https) proxy-address [username]", 117 | } 118 | return &cmd 119 | } 120 | -------------------------------------------------------------------------------- /proxy/extractor_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package proxy 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | "net/http" 12 | "os" 13 | "testing" 14 | 15 | "github.com/containerd/containerd/log" 16 | "github.com/mc256/starlight/test" 17 | ) 18 | 19 | func TestNewExtractor(t *testing.T) { 20 | ctx := context.Background() 21 | cfg, _, _, _ := LoadConfig("") 22 | server := &Server{ 23 | ctx: ctx, 24 | Server: http.Server{ 25 | Addr: fmt.Sprintf("%s:%d", cfg.ListenAddress, cfg.ListenPort), 26 | }, 27 | config: cfg, 28 | } 29 | 30 | ext, err := NewExtractor(server, "starlight/mariadb:10.9.2", true) 31 | if err != nil { 32 | t.Error(err) 33 | } 34 | fmt.Println(ext) 35 | } 36 | 37 | func TestExtractor_SaveToC_Goharbor(t *testing.T) { 38 | test.LoadEnvironmentVariables() 39 | if test.HasLoginStarlightGoharbor() == false { 40 | t.Skip(">>>>> Skip: no container registry credentials for goharbor") 41 | } 42 | 43 | ctx := context.Background() 44 | cfg, _, _, _ := LoadConfig("") 45 | 46 | if os.Getenv("TEST_HARBOR_REGISTRY") != "" { 47 | cfg.DefaultRegistry = os.Getenv("TEST_HARBOR_REGISTRY") 48 | } 49 | if os.Getenv("POSTGRES_CONNECTION_URL") != "" { 50 | cfg.PostgresConnectionString = os.Getenv("POSTGRES_CONNECTION_URL") 51 | } 52 | 53 | server := &Server{ 54 | ctx: ctx, 55 | Server: http.Server{ 56 | Addr: fmt.Sprintf("%s:%d", cfg.ListenAddress, cfg.ListenPort), 57 | }, 58 | config: cfg, 59 | } 60 | if db, err := NewDatabase(ctx, cfg.PostgresConnectionString); err != nil { 61 | log.G(ctx).Errorf("failed to connect to database: %v\n", err) 62 | } else { 63 | server.db = db 64 | } 65 | 66 | ext, err := NewExtractor(server, "starlight/mariadb:10.9.2", true) 67 | if err != nil { 68 | t.Error(err) 69 | } 70 | fmt.Println(ext) 71 | res, err := ext.SaveToC() 72 | if err != nil { 73 | t.Error(err) 74 | } 75 | fmt.Println(res) 76 | } 77 | 78 | func TestExtractor_SaveToC_AWSECR(t *testing.T) { 79 | test.LoadEnvironmentVariables() 80 | if test.HasLoginAWSECR() == false { 81 | t.Skip(">>>>> Skip: no container registry credentials for AWS ECR") 82 | } 83 | 84 | ctx := context.Background() 85 | cfg, _, _, _ := LoadConfig("") 86 | 87 | if os.Getenv("TEST_HARBOR_REGISTRY") != "" { 88 | cfg.DefaultRegistry = os.Getenv("TEST_HARBOR_REGISTRY") 89 | } 90 | if os.Getenv("POSTGRES_CONNECTION_URL") != "" { 91 | cfg.PostgresConnectionString = os.Getenv("POSTGRES_CONNECTION_URL") 92 | } 93 | 94 | if os.Getenv("TEST_ECR_IMAGE_TO") == "" { 95 | t.Skip(">>>>> Skip: no ECR image set in TEST_ECR_IMAGE_TO") 96 | } 97 | 98 | awsecrImage := os.Getenv("TEST_ECR_IMAGE_TO") 99 | 100 | server := &Server{ 101 | ctx: ctx, 102 | Server: http.Server{ 103 | Addr: fmt.Sprintf("%s:%d", cfg.ListenAddress, cfg.ListenPort), 104 | }, 105 | config: cfg, 106 | } 107 | if db, err := NewDatabase(ctx, cfg.PostgresConnectionString); err != nil { 108 | log.G(ctx).Errorf("failed to connect to database: %v\n", err) 109 | } else { 110 | server.db = db 111 | } 112 | 113 | ext, err := NewExtractor(server, awsecrImage, true) 114 | if err != nil { 115 | t.Error(err) 116 | } 117 | fmt.Println(ext) 118 | res, err := ext.SaveToC() 119 | if err != nil { 120 | t.Error(err) 121 | } 122 | fmt.Println(res) 123 | } 124 | -------------------------------------------------------------------------------- /client/api/daemon.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2015 gRPC authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // api is the interface between the Starlight client and the CLI tool. 16 | // use `make update-protobuf` to update the generated code. 17 | syntax = "proto3"; 18 | 19 | option go_package = "github.com/mc256/starlight/client/api"; 20 | 21 | package api; 22 | 23 | // Interface exported by the server. 24 | service Daemon { 25 | rpc GetVersion(Request) returns (Version) {} 26 | rpc PingTest(PingRequest) returns (PingResponse) {} 27 | rpc AddProxyProfile(AuthRequest) returns (AuthResponse) {} 28 | rpc GetProxyProfiles(Request) returns (GetProxyProfilesResponse) {} 29 | rpc NotifyProxy(NotifyRequest) returns (NotifyResponse) {} 30 | rpc PullImage(ImageReference) returns (ImagePullResponse) {} 31 | rpc SetOptimizer(OptimizeRequest) returns (OptimizeResponse) {} 32 | rpc ReportTraces(ReportTracesRequest) returns (ReportTracesResponse) {} 33 | } 34 | 35 | // GetVersion 36 | message Request{ 37 | } 38 | 39 | message Version{ 40 | string version = 1; 41 | } 42 | 43 | // Ping Test 44 | message PingRequest{ 45 | string proxyConfig = 1; 46 | } 47 | 48 | message PingResponse{ 49 | bool success = 1; 50 | string message = 2; 51 | int64 latency = 3; 52 | } 53 | 54 | // Add Auth 55 | message AuthRequest { 56 | string profileName = 1; 57 | string username = 2; 58 | string password = 3; 59 | string protocol = 4; 60 | string address = 5; 61 | } 62 | 63 | message AuthResponse { 64 | bool success = 1; 65 | string message = 2; 66 | } 67 | 68 | message GetProxyProfilesResponse { 69 | message Profile { 70 | string name = 1; 71 | string protocol = 2; 72 | string address = 3; 73 | } 74 | 75 | repeated Profile profiles = 1; 76 | } 77 | 78 | // Notify Proxy 79 | message NotifyRequest { 80 | string reference = 1; 81 | bool insecure = 2; 82 | string proxyConfig = 3; 83 | } 84 | 85 | message NotifyResponse { 86 | bool success = 1; 87 | string message = 2; 88 | } 89 | 90 | 91 | // Pull Image 92 | message ImageReference { 93 | string reference = 1; 94 | string base = 2; 95 | string proxyConfig = 3; 96 | string namespace = 4; 97 | bool disableEarlyStart = 5; 98 | } 99 | 100 | message ImagePullResponse { 101 | bool success = 1; 102 | string message = 2; 103 | string baseImage = 3; 104 | int64 totalImageSize = 4; 105 | int64 originalImageSize = 5; 106 | } 107 | 108 | // Optimize 109 | message OptimizeRequest { 110 | bool enable = 1; 111 | string group = 2; 112 | } 113 | 114 | message OptimizeResponse { 115 | bool success = 1; 116 | string message = 2; 117 | map okay = 3; 118 | map failed = 4; 119 | } 120 | 121 | // Report Traces 122 | message ReportTracesRequest { 123 | string proxyConfig = 1; 124 | bool remove = 2; 125 | } 126 | 127 | message ReportTracesResponse { 128 | bool success = 1; 129 | string message = 2; 130 | map okay = 3; 131 | map failed = 4; 132 | } -------------------------------------------------------------------------------- /demo/chart/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "starlight.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 48 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | 9 | 10 | {{/* 11 | Create a default fully qualified app name. 12 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 13 | If release name contains chart name it will be used as a full name. 14 | */}} 15 | {{- define "starlight.fullname" -}} 16 | {{- if .Values.fullnameOverride }} 17 | {{- .Values.fullnameOverride | trunc 48 | trimSuffix "-" }} 18 | {{- else }} 19 | {{- $name := default .Chart.Name .Values.nameOverride }} 20 | {{- if contains $name .Release.Name }} 21 | {{- .Release.Name | trunc 48 | trimSuffix "-" }} 22 | {{- else }} 23 | {{- printf "%s-%s" .Release.Name $name | trunc 48 | trimSuffix "-" }} 24 | {{- end }} 25 | {{- end }} 26 | {{- end }} 27 | 28 | 29 | 30 | {{- define "starlight.fullname-registry" -}} 31 | {{ include "starlight.fullname" . }}-registry 32 | {{- end }} 33 | 34 | 35 | {{- define "starlight.fullname-edge" -}} 36 | {{ include "starlight.fullname" . }}-edge 37 | {{- end }} 38 | 39 | 40 | {{/* 41 | Create chart name and version as used by the chart label. 42 | */}} 43 | {{- define "starlight.chart" -}} 44 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 45 | {{- end }} 46 | 47 | 48 | {{/* 49 | Common labels - Starlight Proxy 50 | */}} 51 | {{- define "starlight.proxyLabels" -}} 52 | helm.sh/chart: {{ include "starlight.chart" . }} 53 | {{ include "starlight.proxySelectorLabels" . }} 54 | {{- if .Chart.AppVersion }} 55 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 56 | {{- end }} 57 | app.kubernetes.io/managed-by: {{ .Release.Service }} 58 | {{- end }} 59 | 60 | 61 | {{/* 62 | Common labels - Registry 63 | */}} 64 | {{- define "starlight.registryLabels" -}} 65 | helm.sh/chart: {{ include "starlight.chart" . }} 66 | {{ include "starlight.registrySelectorLabels" . }} 67 | {{- if .Chart.AppVersion }} 68 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 69 | {{- end }} 70 | app.kubernetes.io/managed-by: {{ .Release.Service }} 71 | {{- end }} 72 | 73 | 74 | 75 | {{/* 76 | Common labels - edge 77 | */}} 78 | {{- define "starlight.edgeLabels" -}} 79 | helm.sh/chart: {{ include "starlight.chart" . }} 80 | {{ include "starlight.edgeSelectorLabels" . }} 81 | {{- if .Chart.AppVersion }} 82 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 83 | {{- end }} 84 | app.kubernetes.io/managed-by: {{ .Release.Service }} 85 | {{- end }} 86 | 87 | 88 | 89 | 90 | {{/* 91 | Selector labels - Starlight Proxy 92 | */}} 93 | {{- define "starlight.proxySelectorLabels" -}} 94 | app.kubernetes.io/name: {{ include "starlight.name" . }} 95 | app.kubernetes.io/instance: {{ .Release.Name }} 96 | app.kubernetes.io/component: proxy 97 | {{- end }} 98 | 99 | 100 | {{/* 101 | Selector labels - Registry 102 | */}} 103 | {{- define "starlight.registrySelectorLabels" -}} 104 | app.kubernetes.io/name: {{ include "starlight.name" . }} 105 | app.kubernetes.io/instance: {{ .Release.Name }} 106 | app.kubernetes.io/component: registry 107 | {{- end }} 108 | 109 | 110 | {{/* 111 | Selector labels - Edge 112 | */}} 113 | {{- define "starlight.edgeSelectorLabels" -}} 114 | app.kubernetes.io/name: {{ include "starlight.name" . }} 115 | app.kubernetes.io/instance: {{ .Release.Name }} 116 | app.kubernetes.io/component: edge 117 | {{- end }} 118 | 119 | 120 | {{/* 121 | Create the name of the service account to use 122 | */}} 123 | {{- define "starlight.serviceAccountName" -}} 124 | {{- if .Values.serviceAccount.create }} 125 | {{- default (include "starlight.fullname" .) .Values.serviceAccount.name }} 126 | {{- else }} 127 | {{- default "default" .Values.serviceAccount.name }} 128 | {{- end }} 129 | {{- end }} 130 | -------------------------------------------------------------------------------- /cmd/ctr-starlight/pull/pull.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package pull 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | "time" 12 | 13 | pb "github.com/mc256/starlight/client/api" 14 | "github.com/mc256/starlight/cmd/ctr-starlight/auth" 15 | "github.com/urfave/cli/v2" 16 | "github.com/vbauerster/mpb/v8/decor" 17 | "google.golang.org/grpc" 18 | "google.golang.org/grpc/credentials/insecure" 19 | ) 20 | 21 | func pullImage(client pb.DaemonClient, ref *pb.ImageReference, quiet bool) error { 22 | if ref.DisableEarlyStart { 23 | fmt.Printf("early start has been disabled, this may take a while\n") 24 | } 25 | start := time.Now() 26 | ctx, cancel := context.WithTimeout(context.Background(), time.Minute*30) 27 | defer cancel() 28 | resp, err := client.PullImage(ctx, ref) 29 | if err != nil { 30 | return fmt.Errorf("pull image failed: %v", err) 31 | } 32 | if resp.Success { 33 | if quiet { 34 | return nil 35 | } 36 | end := time.Now() 37 | 38 | if resp.GetMessage() == "ok" { 39 | if resp.GetBaseImage() == "" { 40 | fmt.Printf("requested to pull image %s in %dms \n", 41 | ref.Reference, 42 | end.Sub(start).Milliseconds(), 43 | ) 44 | } else { 45 | fmt.Printf("requested to pull image %s based on %s in %dms \n", 46 | ref.Reference, resp.GetBaseImage(), 47 | end.Sub(start).Milliseconds(), 48 | ) 49 | } 50 | } else { 51 | fmt.Printf("%s\n", resp.GetMessage()) 52 | } 53 | 54 | if resp.TotalImageSize > -1 { 55 | // https://github.com/containers/image/blob/1dca20b9a8417764c0ce9bd4680ce102259ea2ca/copy/progress_bars.go#L24 56 | // This only includes the HTTP response body, not the HTTP headers. 57 | skipSize := resp.OriginalImageSize - resp.TotalImageSize 58 | fmt.Printf("delta image %.1f / original %.1f (skipped: %.1f = %.2f%%)\n", 59 | decor.SizeB1024(resp.TotalImageSize), 60 | decor.SizeB1024(resp.OriginalImageSize), 61 | decor.SizeB1024(skipSize), 62 | float64(skipSize)/float64(resp.OriginalImageSize)*100, 63 | ) 64 | } 65 | } else { 66 | fmt.Printf("pull image failed: %s\n", resp.Message) 67 | } 68 | return nil 69 | } 70 | 71 | func Action(ctx context.Context, c *cli.Context) error { 72 | var base, ref string 73 | if c.NArg() == 1 { 74 | ref = c.Args().Get(0) 75 | } else if c.NArg() == 2 { 76 | base = c.Args().Get(0) 77 | ref = c.Args().Get(1) 78 | } else { 79 | return fmt.Errorf("wrong number of arguments, expected 1 or 2, got %d", c.NArg()) 80 | } 81 | 82 | // Dial to the daemon 83 | address := c.String("address") 84 | opts := grpc.WithTransportCredentials(insecure.NewCredentials()) 85 | conn, err := grpc.Dial(address, opts) 86 | if err != nil { 87 | fmt.Printf("connect to starlight daemon failed: %v\n", err) 88 | return nil 89 | } 90 | defer conn.Close() 91 | 92 | // pull image 93 | return pullImage(pb.NewDaemonClient(conn), &pb.ImageReference{ 94 | Reference: ref, 95 | Base: base, 96 | ProxyConfig: c.String("profile"), 97 | Namespace: c.String("namespace"), 98 | DisableEarlyStart: c.Bool("disable-early-start"), 99 | }, c.Bool("quiet")) 100 | } 101 | 102 | func Command() *cli.Command { 103 | ctx := context.Background() 104 | return &cli.Command{ 105 | Name: "pull", 106 | Usage: "pull image from starlight proxy server, if the base image is not provided, it will choose the latest" + 107 | " available image with the same name from the same starlight proxy", 108 | Action: func(c *cli.Context) error { 109 | return Action(ctx, c) 110 | }, 111 | Flags: append( 112 | auth.ProxyFlags, 113 | &cli.BoolFlag{ 114 | Name: "disable-early-start", 115 | Aliases: []string{"w"}, 116 | Value: false, 117 | Usage: "block until the entire image is pulled to the local filesystem", 118 | }, 119 | ), 120 | ArgsUsage: "[flags] [BaseImage] PullImage", 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /client/config.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "path" 8 | "strings" 9 | 10 | "github.com/google/uuid" 11 | "github.com/mc256/starlight/util" 12 | "github.com/pkg/errors" 13 | ) 14 | 15 | type ProxyConfig struct { 16 | Protocol string `json:"protocol"` 17 | Address string `json:"address"` 18 | 19 | // Auth 20 | Username string `json:"username,omitempty"` 21 | Password string `json:"password,omitempty"` 22 | } 23 | 24 | type Configuration struct { 25 | LogLevel string `json:"log_level"` 26 | ClientId string `json:"id"` 27 | 28 | // path to database 29 | Metadata string `json:"metadata"` 30 | 31 | // socket address 32 | Socket string `json:"socket"` 33 | DaemonType string `json:"daemon_type"` 34 | Daemon string `json:"daemon"` 35 | Containerd string `json:"containerd"` 36 | Namespace string `json:"default_namespace"` 37 | TracesDir string `json:"traces_dir"` 38 | 39 | // registry + proxy 40 | DefaultProxy string `json:"default_proxy"` 41 | FileSystemRoot string `json:"fs_root"` 42 | 43 | Proxies map[string]*ProxyConfig `json:"configs"` 44 | } 45 | 46 | func (c *Configuration) getProxy(name string) (pc *ProxyConfig, key string) { 47 | if name == "" { 48 | name = c.DefaultProxy 49 | } 50 | if p, has := c.Proxies[name]; has { 51 | return p, name 52 | } 53 | name = c.DefaultProxy 54 | return c.Proxies[name], name 55 | } 56 | 57 | func ParseProxyStrings(v string) (name string, c *ProxyConfig, err error) { 58 | sp := strings.Split(v, ",") 59 | if len(sp) < 3 { 60 | return "", nil, fmt.Errorf("failed to parse '%s'", v) 61 | } 62 | 63 | c = &ProxyConfig{ 64 | Protocol: sp[1], 65 | Address: sp[2], 66 | } 67 | if len(sp) == 4 { 68 | c.Username = sp[3] 69 | } 70 | if len(sp) == 5 { 71 | c.Password = sp[4] 72 | } 73 | return sp[0], c, nil 74 | } 75 | 76 | func LoadConfig(cfgPath string) (c *Configuration, p string, n bool, error error) { 77 | c = NewConfig() 78 | 79 | if cfgPath == "" { 80 | etcPath := util.GetEtcConfigPath() 81 | if err := os.MkdirAll(etcPath, 0775); err != nil { 82 | error = errors.Wrapf(err, "cannot create config folder") 83 | return 84 | } 85 | 86 | p = path.Join(etcPath, "starlight-daemon.json") 87 | 88 | } else { 89 | p = cfgPath 90 | } 91 | 92 | b, err := os.ReadFile(p) 93 | n = false 94 | if err != nil { 95 | n = true 96 | 97 | buf, _ := json.MarshalIndent(c, " ", " ") 98 | if err = os.WriteFile(p, buf, 0644); err == nil { 99 | return 100 | } else { 101 | error = errors.Wrapf(err, "cannot create config file") 102 | } 103 | } 104 | 105 | if err = json.Unmarshal(b, c); err != nil { 106 | error = errors.Wrapf(err, "cannot parse config file") 107 | } 108 | return 109 | } 110 | 111 | func (c *Configuration) SaveConfig() error { 112 | etcPath := util.GetEtcConfigPath() 113 | if err := os.MkdirAll(etcPath, 0775); err != nil { 114 | return errors.Wrapf(err, "cannot create config folder") 115 | } 116 | 117 | p := path.Join(etcPath, "starlight-daemon.json") 118 | buf, _ := json.MarshalIndent(c, " ", " ") 119 | err := os.WriteFile(p, buf, 0644) 120 | if err == nil { 121 | return nil 122 | } 123 | 124 | return nil 125 | } 126 | 127 | func NewConfig() *Configuration { 128 | uuid.EnableRandPool() 129 | return &Configuration{ 130 | LogLevel: "info", 131 | Metadata: "/var/lib/starlight/metadata.db", 132 | Socket: "/run/starlight/starlight-snapshotter.sock", 133 | DaemonType: "unix", 134 | Daemon: "/run/starlight/starlight-daemon.sock", 135 | Containerd: "/run/containerd/containerd.sock", 136 | DefaultProxy: "starlight-shared", 137 | FileSystemRoot: "/var/lib/starlight", 138 | TracesDir: "/var/lib/starlight/traces", 139 | Namespace: "default", 140 | ClientId: uuid.New().String(), 141 | 142 | Proxies: map[string]*ProxyConfig{ 143 | "starlight-shared": { 144 | Protocol: "https", 145 | Address: "starlight.yuri.moe", 146 | 147 | Username: "", 148 | Password: "", 149 | }, 150 | }, 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /cmd/ctr-starlight/convert/convert.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The starlight Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | 16 | file created by maverick in 2022 17 | */ 18 | 19 | package convert 20 | 21 | import ( 22 | "context" 23 | "errors" 24 | "fmt" 25 | 26 | "github.com/containerd/containerd/log" 27 | "github.com/containerd/containerd/namespaces" 28 | "github.com/google/go-containerregistry/pkg/authn" 29 | "github.com/google/go-containerregistry/pkg/name" 30 | "github.com/google/go-containerregistry/pkg/v1/remote" 31 | "github.com/mc256/starlight/cmd/ctr-starlight/auth" 32 | "github.com/mc256/starlight/cmd/ctr-starlight/notify" 33 | "github.com/mc256/starlight/util" 34 | "github.com/urfave/cli/v2" 35 | ) 36 | 37 | // Action - This Action does not require communicates to the Starlight daemon. 38 | func Action(ctx context.Context, c *cli.Context) error { 39 | // [flags] SourceImage StarlightImage 40 | if c.Args().Len() != 2 { 41 | return errors.New("wrong number of arguments") 42 | } 43 | 44 | srcImg := c.Args().Get(0) 45 | slImg := c.Args().Get(1) 46 | 47 | srcInsecure := c.Bool("insecure-source") 48 | dstInsecure := c.Bool("insecure-destination") 49 | 50 | // logger 51 | ns := c.String("namespace") 52 | util.ConfigLoggerWithLevel(c.String("log-level")) 53 | ctx = namespaces.WithNamespace(ctx, ns) 54 | 55 | // source 56 | srcOptions := []name.Option{} 57 | if srcInsecure { 58 | srcOptions = append(srcOptions, name.Insecure) 59 | } 60 | dstOptions := []name.Option{} 61 | if dstInsecure { 62 | dstOptions = append(dstOptions, name.Insecure) 63 | } 64 | 65 | // auth 66 | remoteOptions := []remote.Option{remote.WithAuthFromKeychain(authn.DefaultKeychain)} 67 | 68 | // config 69 | convertor, err := util.NewConvertor(ctx, srcImg, slImg, srcOptions, dstOptions, remoteOptions, c.String("platform")) 70 | if err != nil { 71 | log.G(ctx).WithError(err).Error("illegal image reference") 72 | return nil 73 | } 74 | 75 | // convert 76 | err = convertor.ToStarlightImage() 77 | if err != nil { 78 | log.G(ctx).WithError(err).Error("fail to convert the container image") 79 | return nil 80 | } 81 | log.G(ctx).Info("conversion completed") 82 | 83 | // notify 84 | if c.Bool("notify") { 85 | err = notify.SharedAction(ctx, c, convertor.GetDst()) 86 | if err != nil { 87 | log.G(ctx).WithError(err).Error("fail to notify the converted image") 88 | flags := " " 89 | if dstInsecure { 90 | flags += "--insecure " 91 | } 92 | if c.String("profile") != "" { 93 | flags += fmt.Sprintf("--profile=%s ", c.String("profile")) 94 | } 95 | return fmt.Errorf("fail to notify the converted image, try again using `ctr-starlight%snotify %s`", flags, slImg) 96 | } 97 | log.G(ctx).Info("notified starlight proxy") 98 | } 99 | 100 | return nil 101 | } 102 | 103 | func Command() *cli.Command { 104 | ctx := context.Background() 105 | cmd := cli.Command{ 106 | Name: "convert", 107 | Usage: "Convert typical container image (in .tar.gz or .tar format) to Starlight image format. " + 108 | "Credentials for private registry can be configured in $DOCKER_CONFIG.", 109 | Action: func(c *cli.Context) error { 110 | return Action(ctx, c) 111 | }, 112 | Flags: append( 113 | 114 | // Convert Flags 115 | Flags, 116 | 117 | // Report Flags 118 | append( 119 | auth.ProxyFlags, 120 | &cli.BoolFlag{ 121 | Name: "notify", 122 | Usage: "notify the converted image to the Starlight Proxy", 123 | Value: false, 124 | Required: false, 125 | }, 126 | )..., 127 | ), 128 | ArgsUsage: "[flags] SourceImage StarlightImage", 129 | } 130 | return &cmd 131 | } 132 | -------------------------------------------------------------------------------- /util/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The starlight Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | 16 | file created by maverick in 2021 17 | */ 18 | 19 | package util 20 | 21 | import ( 22 | "os" 23 | "path/filepath" 24 | ) 25 | 26 | const ( 27 | ImageNameLabel = "containerd.io/snapshot/remote/starlight/imageName.label" 28 | ImageTagLabel = "containerd.io/snapshot/remote/starlight/imageTag.label" 29 | OptimizeLabel = "containerd.io/snapshot/remote/starlight/optimize.label" 30 | OptimizeGroupLabel = "containerd.io/snapshot/remote/starlight/optimizeGroup.label" 31 | 32 | ProxyLabel = "containerd.io/snapshot/remote/starlight/proxy" 33 | SnapshotterLabel = "containerd.io/gc.ref.snapshot.starlight" 34 | 35 | UserRwLayerText = "containerd.io/layer/user-rw-layer" 36 | 37 | StarlightTOCDigestAnnotation = "containerd.io/snapshot/remote/starlight/toc.digest" 38 | StarlightTOCCreationTimeAnnotation = "containerd.io/snapshot/remote/starlight/toc.timestamp" 39 | 40 | // ImageMediaTypeManifestV2 for containerd image TYPE field 41 | ImageMediaTypeManifestV2 = "application/vnd.docker.distribution.manifest.v2+json" 42 | 43 | // ImageLabelPuller and ImageLabelStarlightMetadata are labels for containerd image 44 | ImageLabelPuller = "puller.containerd.io" 45 | ImageLabelStarlightMetadata = "metadata.starlight.mc256.dev" 46 | 47 | // ContentLabelStarlightMediaType is the media type of the content, can be manifest, config, or starlight 48 | ContentLabelStarlightMediaType = "mediaType.starlight.mc256.dev" 49 | // ContentLabelContainerdGC prevents containerd from removing the content 50 | ContentLabelContainerdGC = "containerd.io/gc.ref.content" 51 | ContentLabelSnapshotGC = "containerd.io/gc.ref.snapshot.starlight" 52 | ContentLabelCompletion = "complete.starlight.mc256.dev" 53 | 54 | // --------------------------------------------------------------------------------- 55 | // Snapshot labels have a prefix of "containerd.io/snapshot/" 56 | // or are the "containerd.io/snapshot.ref" label. 57 | // SnapshotLabelRefImage is the digest of the image manifest 58 | SnapshotLabelRefImage = "containerd.io/snapshot/starlight.ref.image" 59 | SnapshotLabelRefUncompressed = "containerd.io/snapshot/starlight.ref.uncompressed" 60 | SnapshotLabelRefLayer = "containerd.io/snapshot/starlight.ref.layer" 61 | 62 | // --------------------------------------------------------------------------------- 63 | // Switch to false in `Makefile` when build for production environment 64 | production = false 65 | // Please leave this line unchanged, we need exactly one space around the equal sign. 66 | 67 | // find the project root directory 68 | ProjectIdentifier = "module github.com/mc256/starlight" 69 | ) 70 | 71 | // FindProjectRoot returns the root directory of the Git project if exists. 72 | // otherwise, it returns os.Getwd(). 73 | // To identify whether a directory is a root directory, it check the `go.mod` file 74 | // please making sure the first line of the go mode file is: 75 | // ``` 76 | // module github.com/mc256/starlight 77 | // ``` 78 | func FindProjectRoot() string { 79 | r, err := os.Getwd() 80 | if err != nil { 81 | return "" 82 | } 83 | p := r 84 | 85 | for p != "/" && len(p) != 0 { 86 | f, _ := os.OpenFile(filepath.Join(p, "go.mod"), os.O_RDONLY, 0644) 87 | b := make([]byte, len(ProjectIdentifier)) 88 | _, _ = f.Read(b) 89 | if string(b) == ProjectIdentifier { 90 | return p 91 | } 92 | p = filepath.Join(p, "../") 93 | } 94 | return r 95 | } 96 | 97 | // GetEtcConfigPath return a path to the configuration json 98 | func GetEtcConfigPath() string { 99 | if production { 100 | return "/etc/starlight/" 101 | } else { 102 | return filepath.Join(FindProjectRoot(), "sandbox", "etc", "starlight") 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /cmd/ctr-starlight/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The starlight Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | 16 | file created by maverick in 2021 17 | */ 18 | 19 | package main 20 | 21 | import ( 22 | "fmt" 23 | "os" 24 | 25 | cmdAddProxy "github.com/mc256/starlight/cmd/ctr-starlight/addproxy" 26 | cmdConvert "github.com/mc256/starlight/cmd/ctr-starlight/convert" 27 | cmdInstall "github.com/mc256/starlight/cmd/ctr-starlight/install" 28 | cmdListProxy "github.com/mc256/starlight/cmd/ctr-starlight/listproxy" 29 | cmdNotify "github.com/mc256/starlight/cmd/ctr-starlight/notify" 30 | cmdOptimizer "github.com/mc256/starlight/cmd/ctr-starlight/optimizer" 31 | cmdPing "github.com/mc256/starlight/cmd/ctr-starlight/ping" 32 | cmdPull "github.com/mc256/starlight/cmd/ctr-starlight/pull" 33 | cmdReport "github.com/mc256/starlight/cmd/ctr-starlight/report" 34 | cmdReset "github.com/mc256/starlight/cmd/ctr-starlight/reset" 35 | cmdVersion "github.com/mc256/starlight/cmd/ctr-starlight/version" 36 | 37 | "github.com/mc256/starlight/util" 38 | "github.com/urfave/cli/v2" 39 | ) 40 | 41 | func init() { 42 | cli.VersionPrinter = func(c *cli.Context) { 43 | fmt.Println(c.App.Name, c.App.Version) 44 | } 45 | } 46 | 47 | func main() { 48 | app := NewApp() 49 | if err := app.Run(os.Args); err != nil { 50 | _, _ = fmt.Fprintf(os.Stderr, "ctr-starlight: %v\n", err) 51 | os.Exit(1) 52 | } 53 | os.Exit(0) 54 | } 55 | 56 | func NewApp() *cli.App { 57 | app := cli.NewApp() 58 | 59 | app.Name = "ctr-starlight" 60 | app.Version = util.Version 61 | app.Usage = `CLI tool for starlight daemon. 62 | 63 | This is a CLI tool that controls starlight-daemon. 64 | Please make sure that starlight-daemon is running before using this tool. 65 | For more information, please refer to the README.md file in the project repository. 66 | https://github.com/mc256/starlight 67 | ` 68 | app.Description = fmt.Sprintf("\n%s\n", app.Usage) 69 | 70 | app.EnableBashCompletion = true 71 | app.Flags = []cli.Flag{ 72 | &cli.StringFlag{ 73 | Name: "namespace", 74 | Aliases: []string{"n"}, 75 | Value: "default", 76 | DefaultText: "default", 77 | EnvVars: []string{"CONTAINERD_NAMESPACE"}, 78 | Usage: "namespace to use with commands (if using kubernetes, please specify `k8s.io`)", 79 | Required: false, 80 | }, 81 | &cli.StringFlag{ 82 | Name: "address", 83 | Aliases: []string{"a", "addr"}, 84 | Value: "unix:////run/starlight/starlight-daemon.sock", 85 | DefaultText: "unix:////run/starlight/starlight-daemon.sock", 86 | EnvVars: []string{"STARLIGHT_ADDRESS"}, 87 | Usage: "address to connect to starlight-daemon", 88 | Required: false, 89 | }, 90 | &cli.StringFlag{ 91 | Name: "log-level", 92 | Aliases: []string{"l"}, 93 | Usage: "log level for this command line tool", 94 | Value: "info", 95 | DefaultText: "info", 96 | Required: false, 97 | }, 98 | } 99 | app.Commands = []*cli.Command{ 100 | cmdInstall.Command(), // 0. configure starlight-daemon on the host (with containerd or k3s) 101 | cmdVersion.Command(), // 1. confirm the version of starlight-daemon 102 | cmdAddProxy.Command(), // 2. add starlight proxy to the daemon 103 | cmdListProxy.Command(), // 3. list proxy profiles in daemon 104 | cmdPing.Command(), // 4. ping the starlight proxy to see if it is alive 105 | cmdConvert.Command(), // 5. convert docker image to starlight image 106 | cmdNotify.Command(), // 6. notify the proxy that the starlight image is available 107 | cmdOptimizer.Command(), // 7. turn on/off filesystem traces 108 | cmdReport.Command(), // 8. upload filesystem traces to starlight proxy 109 | cmdReset.Command(), // !. reset starlight daemon 110 | cmdPull.Command(), // 9. pull starlight image 111 | } 112 | 113 | return app 114 | } 115 | -------------------------------------------------------------------------------- /util/send/image.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package send 7 | 8 | import ( 9 | "fmt" 10 | 11 | "github.com/google/go-containerregistry/pkg/name" 12 | "github.com/mc256/starlight/util/common" 13 | ) 14 | 15 | type ImageLayer struct { 16 | StackIndex int64 `json:"-"` 17 | UncompressedSize int64 `json:"s"` 18 | Serial int64 `json:"f"` 19 | Hash string `json:"h"` 20 | digest name.Digest 21 | Available bool `json:"-"` 22 | Blob *common.LayerCache `json:"-"` 23 | } 24 | 25 | func (il *ImageLayer) SetDigest(d name.Digest) { 26 | il.digest = d 27 | } 28 | 29 | func (il *ImageLayer) Digest() name.Digest { 30 | return il.digest 31 | } 32 | 33 | func (il *ImageLayer) Size() int64 { 34 | return il.UncompressedSize 35 | } 36 | 37 | func (il *ImageLayer) String() string { 38 | return fmt.Sprintf("[%05d:%02d]%s-%d", il.Serial, il.StackIndex, il.Hash, il.Size()) 39 | } 40 | 41 | type Content struct { 42 | // Files used to find the highest rank, WILL NOT BE EXPORTED. 43 | // please use RequestedFiles instead 44 | Files []*RankedFile `json:"-"` 45 | 46 | // Rank WILL NOT BE EXPORTED. We do not want to send it to the client. 47 | // highest rank of all the files using this content 48 | Rank float64 `json:"-"` 49 | 50 | // ------------------------------------------ 51 | // stack identify which layer should this content be placed, all the files will be referencing the content 52 | Stack int64 `json:"t"` 53 | 54 | // offset is non-zero if the file is in the delta bundle body 55 | Offset int64 `json:"o,omitempty"` 56 | 57 | // size is the size of the compressed content 58 | Size int64 `json:"s"` 59 | 60 | Chunks []*FileChunk `json:"c"` 61 | 62 | Digest string `json:"d"` 63 | } 64 | 65 | type ByRank []*Content 66 | 67 | func (b ByRank) Len() int { 68 | return len(b) 69 | } 70 | 71 | func (b ByRank) Less(i, j int) bool { 72 | return b[i].Rank < b[j].Rank 73 | } 74 | 75 | func (b ByRank) Swap(i, j int) { 76 | b[i], b[j] = b[j], b[i] 77 | } 78 | 79 | type SignalContent struct { 80 | *Content 81 | Signal chan interface{} 82 | } 83 | 84 | func NewSignalContent(c *Content) *SignalContent { 85 | return &SignalContent{ 86 | Content: c, 87 | Signal: make(chan interface{}), 88 | } 89 | } 90 | 91 | type RankedFile struct { 92 | File 93 | 94 | // rank of the file, smaller has the higher priority 95 | Rank float64 `json:"-"` 96 | 97 | // Stack in the existing image from bottom to top 98 | Stack int64 `json:"S"` 99 | 100 | // if the file is available on the client then ReferenceFsId is non-zero, 101 | // expecting the file is available on the client and can be accessed using the File.Digest . 102 | ReferenceFsId int64 `json:"R,omitempty"` 103 | 104 | // if the file is not available on the client then ReferenceFsId is zero and ReferenceStack is non-zero, 105 | // expecting the file content in the delta bundle body 106 | ReferenceStack int64 `json:"T,omitempty"` 107 | // if the file is not available on the client then PayloadOrder is non-zero shows when this file can be ready 108 | PayloadOrder int `json:"O,omitempty"` 109 | } 110 | 111 | type FileChunk struct { 112 | Offset int64 `json:"o"` 113 | ChunkOffset int64 `json:"c"` 114 | ChunkSize int64 `json:"h"` 115 | CompressedSize int64 `json:"s"` 116 | } 117 | 118 | type FileChunkParsing struct { 119 | Offset int64 `json:"offset"` 120 | ChunkOffset int64 `json:"chunkOffset"` 121 | ChunkSize int64 `json:"chunkSize"` 122 | CompressedSize int64 `json:"compressedSize"` 123 | } 124 | 125 | type File struct { 126 | common.TOCEntry 127 | ParsingChunks []*FileChunkParsing `json:"chunks,omitempty"` // compatible with old version 128 | FsId int64 `json:"-"` 129 | } 130 | 131 | type Image struct { 132 | Ref name.Reference `json:"-"` 133 | Serial int64 `json:"s"` 134 | Layers []*ImageLayer `json:"l"` 135 | } 136 | 137 | type LayerSource bool 138 | 139 | const ( 140 | FromDestination LayerSource = false 141 | FromSource = true 142 | ) 143 | 144 | func (i Image) String() string { 145 | return fmt.Sprintf("%d->%v", i.Serial, i.Layers) 146 | } 147 | 148 | type DeltaBundle struct { 149 | Source *Image `json:"s"` 150 | Destination *Image `json:"d"` 151 | 152 | // contents and BodyLength are computed by Builder.computeDelta() 153 | Contents []*Content `json:"c"` 154 | BodyLength int64 `json:"bl"` 155 | 156 | // RequestedFiles are all the files in the requested images 157 | // Use this to reconstruct the file system 158 | RequestedFiles []*RankedFile `json:"rf"` 159 | } 160 | -------------------------------------------------------------------------------- /cmd/ctr-starlight/reset/reset.go: -------------------------------------------------------------------------------- 1 | package reset 2 | 3 | import ( 4 | "context" 5 | "io/ioutil" 6 | "os" 7 | "os/exec" 8 | "path" 9 | "strings" 10 | 11 | "github.com/containerd/containerd/log" 12 | "github.com/mc256/starlight/util" 13 | "github.com/urfave/cli/v2" 14 | ) 15 | 16 | func terminateContainerd(ctx context.Context) error { 17 | log.G(ctx). 18 | Info("terminating containerd") 19 | 20 | out, _ := exec.Command("systemctl", "stop", "containerd").CombinedOutput() 21 | if outs := string(out); strings.TrimSpace(outs) != "" { 22 | log.G(ctx). 23 | Warnf("systemctl stop containerd: %s", outs) 24 | } 25 | 26 | if err := os.RemoveAll("/var/lib/containerd"); err != nil { 27 | log.G(ctx). 28 | WithError(err). 29 | Warnf("rm -rf /var/lib/containerd") 30 | } 31 | if err := os.RemoveAll("/run/containerd"); err != nil { 32 | log.G(ctx). 33 | WithError(err). 34 | Warnf("rm -rf /run/containerd") 35 | } 36 | 37 | out, _ = exec.Command("pkill", "-9", "containerd").CombinedOutput() 38 | if outs := string(out); strings.TrimSpace(outs) != "" { 39 | log.G(ctx). 40 | Warnf("pkill -9 containerd: %s", outs) 41 | } 42 | 43 | return nil 44 | } 45 | 46 | func unmountFs(ctx context.Context) error { 47 | log.G(ctx). 48 | Info("unmounting filesystems") 49 | 50 | // iterate all files in /var/lib/starlight 51 | files, err := ioutil.ReadDir("/var/lib/starlight/sfs") 52 | if err != nil { 53 | return nil 54 | } 55 | 56 | for _, f := range files { 57 | // if it is a directory 58 | if f.IsDir() { 59 | // if it is a mount point 60 | log.G(ctx). 61 | WithField("path", path.Join(f.Name(), "m")). 62 | Info("unmounting") 63 | 64 | if s, err := os.Stat(path.Join(f.Name(), "m")); err == nil && s.IsDir() { 65 | out, err := exec.Command("umount", path.Join(f.Name(), "m")).CombinedOutput() 66 | if outs := string(out); strings.TrimSpace(outs) != "" { 67 | log.G(ctx). 68 | WithField("path", path.Join(f.Name(), "m")). 69 | Warnf("umount: %s", outs) 70 | } 71 | if err != nil { 72 | out, _ := exec.Command("umount", "-l", path.Join(f.Name(), "m")).CombinedOutput() 73 | if outs := string(out); strings.TrimSpace(outs) != "" { 74 | log.G(ctx). 75 | WithField("path", path.Join(f.Name(), "m")). 76 | Warnf("umount -l: %s", outs) 77 | } 78 | } 79 | } 80 | } 81 | } 82 | 83 | return nil 84 | } 85 | 86 | func terminateStarlight(ctx context.Context) error { 87 | log.G(ctx). 88 | Info("terminating starlight") 89 | 90 | out, _ := exec.Command("systemctl", "stop", "starlight").CombinedOutput() 91 | if outs := string(out); strings.TrimSpace(outs) != "" { 92 | log.G(ctx). 93 | Warnf("systemctl stop starlight: %s", outs) 94 | } 95 | 96 | if err := os.RemoveAll("/var/lib/starlight"); err != nil { 97 | log.G(ctx). 98 | WithError(err). 99 | Warnf("rm -rf /var/lib/starlight") 100 | } 101 | if err := os.RemoveAll("/run/starlight"); err != nil { 102 | log.G(ctx). 103 | WithError(err). 104 | Warnf("rm -rf /run/starlight") 105 | } 106 | 107 | out, _ = exec.Command("pkill", "-9", "starlight-d").CombinedOutput() 108 | if outs := string(out); strings.TrimSpace(outs) != "" { 109 | log.G(ctx). 110 | Warnf("pkill -9 starlight-d: %s", outs) 111 | } 112 | 113 | return nil 114 | } 115 | 116 | func Action(ctx context.Context, c *cli.Context) error { 117 | util.ConfigLoggerWithLevel(c.String("log-level")) 118 | 119 | all := c.Bool("all") 120 | containerd := c.Bool("containerd") 121 | starlight := c.Bool("starlight") 122 | umount := c.Bool("umount") 123 | 124 | if all { 125 | containerd = true 126 | starlight = true 127 | umount = true 128 | } 129 | 130 | if containerd { 131 | _ = terminateContainerd(ctx) 132 | } 133 | 134 | if umount { 135 | _ = unmountFs(ctx) 136 | } 137 | 138 | if starlight { 139 | _ = terminateStarlight(ctx) 140 | } 141 | 142 | return nil 143 | } 144 | 145 | func Command() *cli.Command { 146 | ctx := context.Background() 147 | return &cli.Command{ 148 | Name: "reset", 149 | Usage: "reset all services for starlight daemon on the host (for benchmark or debug purpose)", 150 | Action: func(c *cli.Context) error { 151 | return Action(ctx, c) 152 | }, 153 | Flags: []cli.Flag{ 154 | // all 155 | &cli.BoolFlag{ 156 | Name: "all", 157 | Value: false, 158 | Usage: "reset all the known services for starlight daemon on the host", 159 | }, 160 | 161 | // 3rd party 162 | &cli.BoolFlag{ 163 | Name: "containerd", 164 | Value: false, 165 | Usage: "enable Starlight Daemon for containerd", 166 | }, 167 | 168 | // starlight 169 | &cli.BoolFlag{ 170 | Name: "starlight", 171 | Value: false, 172 | Usage: "reset starlight daemon", 173 | }, 174 | &cli.BoolFlag{ 175 | Name: "umount", 176 | Value: false, 177 | Usage: "unmount starlight rootfs", 178 | }, 179 | }, 180 | ArgsUsage: "", 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /demo/chart/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | For more info, please visit https://github.com/mc256/starlight 2 | 3 | _ _ _ _ _ 4 | | | | (_) | | | | 5 | ___| |_ __ _ _ __| |_ __ _| |__ | |_ 6 | / __| __/ _` | '__| | |/ _` | '_ \| __| 7 | \__ \ || (_| | | | | | (_| | | | | |_ 8 | |___/\__\__,_|_| |_|_|\__, |_| |_|\__| 9 | __/ | 10 | |___/ 11 | 12 | 13 | {{- if .Values.ingress.enabled }} 14 | Get the application URL by running these commands: 15 | {{- range $host := .Values.ingress.hosts }} 16 | Starlight Proxy : http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host }} 17 | {{- if $.Values.registry.enabled }} 18 | Container Registry : http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host }}/v2 19 | {{- end }} 20 | {{- end }} 21 | 22 | export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "starlight.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") 23 | echo http://$SERVICE_IP 24 | echo $SERVICE_IP 25 | 26 | Please point your domain to $SERVICE_IP in your hosts file or DNS server. 27 | 28 | The maximum upload size of the Nginx ingress has set to {{ index .Values.ingress.annotations "nginx.ingress.kubernetes.io/proxy-body-size" }}. 29 | {{- else }} 30 | You have not enabled ingress. 31 | {{- end }} 32 | 33 | {{- if .Values.postgres.enabled }} 34 | -------------------------------------------------------------------------------- 35 | You have enabled a local PostgresQL database. 36 | In production environment, please consider using an external postgresql with properly configured security and partition. 37 | {{- if .Values.adminer.enabled }} 38 | 39 | -------------------------------------------------------------------------------- 40 | A light-weight database management tool Adminer has been enabled, to view the database content: 41 | 42 | kubectl port-forward --namespace {{ .Release.Namespace }} svc/{{ include "starlight.fullname" . }} 8080:8080 43 | open it in http://localhost:8080/ and the default login credential is 'postgres'. 44 | 45 | {{- end }} 46 | {{- end }} 47 | 48 | {{ if .Values.edge.enabled}} 49 | -------------------------------------------------------------------------------- 50 | You have enabled edge server. 51 | The purpose of this deployment is to keep the Starlight CLI image on every edge node, 52 | so that whever the user uses initContainer, there is a starlight CLI container for that. 53 | 54 | You still need to install the Starlight daemon (containerd snapshotter) and enable it on the edge node. 55 | 56 | ON THE WORKER NODE: 57 | 1. Install the Starlight daemon 58 | 59 | export ARCH=$(dpkg --print-architecture) # one of amd64, arm64, armhf 60 | wget https://github.com/mc256/starlight/releases/download/v{{ .Chart.Version }}/starlight_{{ .Chart.Version }}_$ARCH.deb 61 | sudo dpkg -i starlight_{{ .Chart.Version }}_$ARCH.deb 62 | sudo systemctl enable starlight 63 | sudo systemctl start starlight 64 | 65 | 66 | 67 | 2. Configure the containerd to use Starlight as the snapshotter 68 | 69 | sudo mkdir -p /etc/containerd 70 | cat <>>>> Skip: no container registry credentials for goharbor") 19 | } 20 | if os.Getenv("TEST_DOCKER_SOURCE_IMAGE") == "" { 21 | t.Skip(">>>>> Skip: no TEST_DOCKER_SOURCE_IMAGE environment variable") 22 | } 23 | if os.Getenv("TEST_HARBOR_IMAGE_TO") == "" { 24 | t.Skip(">>>>> Skip: no TEST_HARBOR_IMAGE_TO environment variable") 25 | } 26 | srcRef := os.Getenv("TEST_DOCKER_SOURCE_IMAGE") 27 | dstRef := os.Getenv("TEST_HARBOR_IMAGE_TO") 28 | 29 | ctx := context.Background() 30 | 31 | c, err := NewConvertor(ctx, srcRef, dstRef, []name.Option{}, []name.Option{}, []remote.Option{}, "all") 32 | 33 | if err != nil { 34 | t.Fatal(err) 35 | } 36 | fmt.Println(c) 37 | 38 | } 39 | 40 | func TestToStarlightImageGoharbor(t *testing.T) { 41 | test.LoadEnvironmentVariables() 42 | if test.HasLoginStarlightGoharbor() == false { 43 | t.Skip(">>>>> Skip: no container registry credentials for goharbor") 44 | } 45 | if os.Getenv("TEST_DOCKER_SOURCE_IMAGE") == "" { 46 | t.Skip(">>>>> Skip: no TEST_DOCKER_SOURCE_IMAGE environment variable") 47 | } 48 | if os.Getenv("TEST_HARBOR_IMAGE_TO") == "" { 49 | t.Skip(">>>>> Skip: no TEST_HARBOR_IMAGE_TO environment variable") 50 | } 51 | srcRef := os.Getenv("TEST_DOCKER_SOURCE_IMAGE") 52 | dstRef := os.Getenv("TEST_HARBOR_IMAGE_TO") 53 | 54 | ctx := context.Background() 55 | 56 | c, err := NewConvertor(ctx, srcRef, dstRef, []name.Option{}, []name.Option{}, []remote.Option{ 57 | remote.WithAuthFromKeychain(authn.DefaultKeychain), 58 | }, "amd64") 59 | //}, "all") 60 | if err != nil { 61 | t.Fatal(err) 62 | } 63 | if err != nil { 64 | t.Fatal(err) 65 | } 66 | err = c.ToStarlightImage() 67 | if err != nil { 68 | t.Fatal(err) 69 | } 70 | } 71 | 72 | func TestToStarlightImageGoharborWithMultiplePlatforms(t *testing.T) { 73 | test.LoadEnvironmentVariables() 74 | if test.HasLoginStarlightGoharbor() == false { 75 | t.Skip(">>>>> Skip: no container registry credentials for goharbor") 76 | } 77 | if os.Getenv("TEST_DOCKER_SOURCE_IMAGE") == "" { 78 | t.Skip(">>>>> Skip: no TEST_DOCKER_SOURCE_IMAGE environment variable") 79 | } 80 | if os.Getenv("TEST_HARBOR_IMAGE_TO") == "" { 81 | t.Skip(">>>>> Skip: no TEST_HARBOR_IMAGE_TO environment variable") 82 | } 83 | srcRef := os.Getenv("TEST_DOCKER_SOURCE_IMAGE") 84 | dstRef := os.Getenv("TEST_HARBOR_IMAGE_TO") 85 | 86 | ctx := context.Background() 87 | 88 | c, err := NewConvertor(ctx, srcRef, dstRef, []name.Option{}, []name.Option{}, []remote.Option{ 89 | remote.WithAuthFromKeychain(authn.DefaultKeychain), 90 | }, "linux/arm64/v8,linux/arm/v7,linux/arm/v5") 91 | //}, "amd64,linux/arm64/v8,linux/arm/v7") 92 | 93 | if err != nil { 94 | t.Fatal(err) 95 | } 96 | if err != nil { 97 | t.Fatal(err) 98 | } 99 | err = c.ToStarlightImage() 100 | if err != nil { 101 | t.Fatal(err) 102 | } 103 | } 104 | 105 | func TestToStarlightImageGoharborWithAllPlatforms(t *testing.T) { 106 | test.LoadEnvironmentVariables() 107 | if test.HasLoginStarlightGoharbor() == false { 108 | t.Skip(">>>>> Skip: no container registry credentials for goharbor") 109 | } 110 | if os.Getenv("TEST_DOCKER_SOURCE_IMAGE") == "" { 111 | t.Skip(">>>>> Skip: no TEST_DOCKER_SOURCE_IMAGE environment variable") 112 | } 113 | if os.Getenv("TEST_HARBOR_IMAGE_TO") == "" { 114 | t.Skip(">>>>> Skip: no TEST_HARBOR_IMAGE_TO environment variable") 115 | } 116 | srcRef := os.Getenv("TEST_DOCKER_SOURCE_IMAGE") 117 | dstRef := os.Getenv("TEST_HARBOR_IMAGE_TO") 118 | 119 | ctx := context.Background() 120 | 121 | c, err := NewConvertor(ctx, srcRef, dstRef, []name.Option{}, []name.Option{}, []remote.Option{ 122 | remote.WithAuthFromKeychain(authn.DefaultKeychain), 123 | }, "all") 124 | 125 | if err != nil { 126 | t.Fatal(err) 127 | } 128 | if err != nil { 129 | t.Fatal(err) 130 | } 131 | err = c.ToStarlightImage() 132 | if err != nil { 133 | t.Fatal(err) 134 | } 135 | } 136 | 137 | func TestToStarlightImageECR(t *testing.T) { 138 | 139 | test.LoadEnvironmentVariables() 140 | if test.HasLoginAWSECR() == false { 141 | t.Skip(">>>>> Skip: no container registry credentials for ECR") 142 | } 143 | if os.Getenv("TEST_ECR_IMAGE_FROM") == "" { 144 | t.Skip(">>>>> Skip: no TEST_ECR_IMAGE_FROM environment variable") 145 | } 146 | if os.Getenv("TEST_ECR_IMAGE_TO") == "" { 147 | t.Skip(">>>>> Skip: no TEST_ECR_IMAGE_TO environment variable") 148 | } 149 | 150 | srcRef := os.Getenv("TEST_ECR_IMAGE_FROM") 151 | dstRef := os.Getenv("TEST_ECR_IMAGE_TO") 152 | 153 | ctx := context.Background() 154 | 155 | c, err := NewConvertor(ctx, srcRef, dstRef, []name.Option{}, []name.Option{}, []remote.Option{ 156 | remote.WithAuthFromKeychain(authn.DefaultKeychain), 157 | }, "all") 158 | if err != nil { 159 | t.Fatal(err) 160 | } 161 | if err != nil { 162 | t.Fatal(err) 163 | } 164 | err = c.ToStarlightImage() 165 | if err != nil { 166 | t.Fatal(err) 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /.github/workflows/docker-image.yml: -------------------------------------------------------------------------------- 1 | name: Docker Image 2 | on: 3 | # PR testing before merge 4 | pull_request: 5 | types: 6 | - opened 7 | - edited 8 | - synchronize 9 | # For Testing 10 | push: 11 | branches: 12 | - feature_container_* 13 | - fix_container_* 14 | - feature_docker_* 15 | - fix_docker_* 16 | # For Release 17 | workflow_run: 18 | workflows: ["Versioning"] 19 | types: 20 | - completed 21 | concurrency: 22 | group: ${{ github.workflow }}-${{ github.ref }} 23 | cancel-in-progress: true 24 | jobs: 25 | proxy: 26 | name: Starlight Proxy 27 | runs-on: ubuntu-latest 28 | steps: 29 | - name: Checkout 30 | uses: actions/checkout@v3 31 | with: 32 | fetch-depth: 0 33 | - name: Get SemVer 34 | id: get-version 35 | run: | 36 | echo "semver="`git describe --tags --match "v*" | cut -d '-' -f 1 || echo "v0.0.0"` >> $GITHUB_OUTPUT 37 | echo "major="`git describe --tags --match "v*" | cut -d '-' -f 1 | sed -e "s/^v//" | cut -d '.' -f 1` >> $GITHUB_OUTPUT 38 | echo "minor="`git describe --tags --match "v*" | cut -d '-' -f 1 | sed -e "s/^v//" | cut -d '.' -f 2` >> $GITHUB_OUTPUT 39 | echo "patch="`git describe --tags --match "v*" | cut -d '-' -f 1 | sed -e "s/^v//" | cut -d '.' -f 3` >> $GITHUB_OUTPUT 40 | - name: Set up QEMU 41 | uses: docker/setup-qemu-action@v2 42 | with: 43 | platforms: 'amd64,arm64,arm' 44 | - name: Set up Docker Buildx 45 | uses: docker/setup-buildx-action@v2 46 | - name: Docker meta 47 | id: meta 48 | uses: docker/metadata-action@v4 49 | with: 50 | # list of Docker images to use as base name for tags 51 | images: | 52 | ghcr.io/mc256/starlight/proxy 53 | # generate Docker tags based on the following events/attributes 54 | tags: | 55 | type=raw,value=latest,enable={{is_default_branch}} 56 | type=raw,value=${{ steps.get-version.outputs.major }}.${{ steps.get-version.outputs.minor }}.${{ steps.get-version.outputs.patch }} 57 | type=raw,value=${{ steps.get-version.outputs.major }}.${{ steps.get-version.outputs.minor }} 58 | type=raw,value=${{ steps.get-version.outputs.major }} 59 | type=sha 60 | - name: Login to GitHub Container Registry 61 | uses: docker/login-action@v2 62 | with: 63 | registry: ghcr.io 64 | username: ${{ github.repository_owner }} 65 | password: ${{ secrets.GITHUB_TOKEN }} 66 | logout: true 67 | - name: Build and push 68 | uses: docker/build-push-action@v3 69 | with: 70 | context: . 71 | target: starlight-proxy 72 | platforms: linux/amd64,linux/arm/v7,linux/arm64 73 | push: ${{ github.event.workflow_run.conclusion == 'success' }} 74 | tags: ${{ steps.meta.outputs.tags }} 75 | labels: ${{ steps.meta.outputs.labels }} 76 | cli: 77 | name: Starlight CLI 78 | runs-on: ubuntu-latest 79 | steps: 80 | - name: Checkout 81 | uses: actions/checkout@v3 82 | with: 83 | fetch-depth: 0 84 | - name: Get SemVer 85 | id: get-version 86 | run: | 87 | echo "semver="`git describe --tags --match "v*" | cut -d '-' -f 1 || echo "v0.0.0"` >> $GITHUB_OUTPUT 88 | echo "major="`git describe --tags --match "v*" | cut -d '-' -f 1 | sed -e "s/^v//" | cut -d '.' -f 1` >> $GITHUB_OUTPUT 89 | echo "minor="`git describe --tags --match "v*" | cut -d '-' -f 1 | sed -e "s/^v//" | cut -d '.' -f 2` >> $GITHUB_OUTPUT 90 | echo "patch="`git describe --tags --match "v*" | cut -d '-' -f 1 | sed -e "s/^v//" | cut -d '.' -f 3` >> $GITHUB_OUTPUT 91 | - name: Set up QEMU 92 | uses: docker/setup-qemu-action@v2 93 | with: 94 | platforms: 'amd64,arm64,arm' 95 | - name: Set up Docker Buildx 96 | uses: docker/setup-buildx-action@v2 97 | - name: Docker meta 98 | id: meta 99 | uses: docker/metadata-action@v4 100 | with: 101 | # list of Docker images to use as base name for tags 102 | images: | 103 | ghcr.io/mc256/starlight/cli 104 | # generate Docker tags based on the following events/attributes 105 | tags: | 106 | type=raw,value=latest,enable={{is_default_branch}} 107 | type=raw,value=${{ steps.get-version.outputs.major }}.${{ steps.get-version.outputs.minor }}.${{ steps.get-version.outputs.patch }} 108 | type=raw,value=${{ steps.get-version.outputs.major }}.${{ steps.get-version.outputs.minor }} 109 | type=raw,value=${{ steps.get-version.outputs.major }} 110 | type=sha 111 | - name: Login to GitHub Container Registry 112 | uses: docker/login-action@v2 113 | with: 114 | registry: ghcr.io 115 | username: ${{ github.repository_owner }} 116 | password: ${{ secrets.GITHUB_TOKEN }} 117 | - name: Build and push 118 | uses: docker/build-push-action@v3 119 | with: 120 | context: . 121 | target: starlight-cli 122 | platforms: linux/amd64,linux/arm/v7,linux/arm64 123 | push: ${{ github.event.workflow_run.conclusion == 'success' }} 124 | tags: ${{ steps.meta.outputs.tags }} 125 | labels: ${{ steps.meta.outputs.labels }} 126 | -------------------------------------------------------------------------------- /cmd/starlight-proxy/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The starlight Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | 16 | file created by maverick in 2021 17 | */ 18 | 19 | package main 20 | 21 | import ( 22 | "fmt" 23 | "os" 24 | "regexp" 25 | "sync" 26 | 27 | "github.com/containerd/containerd/log" 28 | "github.com/mc256/starlight/proxy" 29 | "github.com/mc256/starlight/util" 30 | "github.com/sirupsen/logrus" 31 | "github.com/urfave/cli/v2" 32 | ) 33 | 34 | func init() { 35 | cli.VersionPrinter = func(c *cli.Context) { 36 | fmt.Println(c.App.Name, c.App.Version) 37 | } 38 | } 39 | 40 | func main() { 41 | app := New() 42 | if err := app.Run(os.Args); err != nil { 43 | _, _ = fmt.Fprintf(os.Stderr, "starlight-proxy: \n%v\n", err) 44 | os.Exit(1) 45 | } 46 | } 47 | 48 | func ProtectPassword(c string) string { 49 | p := regexp.MustCompile(`:(.*)@`) 50 | return p.ReplaceAllString(c, ":********@") 51 | } 52 | 53 | func New() *cli.App { 54 | app := cli.NewApp() 55 | cfg := proxy.NewConfig() 56 | 57 | app.Name = "starlight-proxy" 58 | app.Version = util.Version 59 | app.Usage = `Starlight Proxy accelerates container deployments. 60 | 61 | This is a proxy server on the cloud side mediates between Starlight workers and any standard registry server. For more 62 | information about Starlight, please visit our repository at https://github.com/mc256/starlight 63 | 64 | *CLI options will override values in the config file if specified. 65 | ` 66 | app.Description = fmt.Sprintf("\n%s\n", app.Usage) 67 | 68 | app.EnableBashCompletion = true 69 | app.Flags = []cli.Flag{ 70 | &cli.StringFlag{ 71 | Name: "config", 72 | DefaultText: "/etc/starlight/starlight-proxy.json", 73 | Aliases: []string{"c"}, 74 | EnvVars: []string{"STARLIGHT_PROXY_CONFIG"}, 75 | Usage: "json configuration file.", 76 | Required: false, 77 | }, 78 | // ---- 79 | &cli.StringFlag{ 80 | Name: "host", 81 | EnvVars: []string{"STARLIGHT_HOST"}, 82 | DefaultText: cfg.ListenAddress, 83 | Usage: "host", 84 | Required: false, 85 | }, 86 | &cli.IntFlag{ 87 | Name: "port", 88 | Aliases: []string{"p"}, 89 | EnvVars: []string{"STARLIGHT_PORT"}, 90 | DefaultText: fmt.Sprintf("%d", cfg.ListenPort), 91 | Usage: "proxy port", 92 | Required: false, 93 | }, 94 | &cli.StringFlag{ 95 | Name: "log-level", 96 | Aliases: []string{"l"}, 97 | EnvVars: []string{"LOG_LEVEL"}, 98 | DefaultText: cfg.LogLevel, 99 | Usage: "Choose one log level (fatal, error, warning, info, debug, trace)", 100 | Required: false, 101 | }, 102 | // ---- 103 | &cli.StringFlag{ 104 | Name: "postgres", 105 | Aliases: []string{"db"}, 106 | EnvVars: []string{"DB_CONNECTION_STRING"}, 107 | DefaultText: ProtectPassword(cfg.PostgresConnectionString), 108 | Usage: "use PostgreSQL database backend for storing TOCs", 109 | Required: false, 110 | }, 111 | // ---- 112 | &cli.StringFlag{ 113 | Name: "registry", 114 | Aliases: []string{"r"}, 115 | EnvVars: []string{"REGISTRY"}, 116 | DefaultText: cfg.DefaultRegistry, 117 | Usage: "Default container registry", 118 | Required: false, 119 | }, 120 | } 121 | app.Action = func(c *cli.Context) error { 122 | return DefaultAction(c, cfg) 123 | } 124 | 125 | return app 126 | } 127 | 128 | func DefaultAction(context *cli.Context, cfg *proxy.Configuration) (err error) { 129 | var ( 130 | p string 131 | ne bool 132 | ) 133 | 134 | config := context.String("config") 135 | cfg, p, ne, err = proxy.LoadConfig(config) 136 | 137 | if l := context.String("log-level"); l != "" { 138 | cfg.LogLevel = l 139 | } 140 | c := util.ConfigLoggerWithLevel(cfg.LogLevel) 141 | log.G(c). 142 | WithField("version", util.Version). 143 | Info("starlight-proxy") 144 | 145 | if err != nil { 146 | log.G(c).WithFields(logrus.Fields{ 147 | "log": cfg.LogLevel, 148 | "path": p, 149 | "new": ne, 150 | }). 151 | WithError(err). 152 | Fatal("failed to load configuration") 153 | } else { 154 | log.G(c).WithFields(logrus.Fields{ 155 | "log": cfg.LogLevel, 156 | "path": p, 157 | "new": ne, 158 | }). 159 | Info("loaded configuration") 160 | } 161 | 162 | if port := context.Int("port"); port != 0 { 163 | cfg.ListenPort = port 164 | } 165 | if h := context.String("host"); h != "" { 166 | cfg.ListenAddress = h 167 | } 168 | 169 | if pc := context.String("postgres"); pc != "" { 170 | cfg.PostgresConnectionString = pc 171 | } 172 | 173 | if r := context.String("registry"); r != "" { 174 | cfg.DefaultRegistry = r 175 | } 176 | log.G(c).Infof("default backend registry: %s", cfg.DefaultRegistry) 177 | 178 | httpServerExitDone := &sync.WaitGroup{} 179 | httpServerExitDone.Add(1) 180 | 181 | _, _ = proxy.NewServer(c, httpServerExitDone, cfg) 182 | 183 | wait := make(chan interface{}) 184 | <-wait 185 | return nil 186 | } 187 | -------------------------------------------------------------------------------- /client/snapshotter/operator_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package snapshotter 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | "github.com/containerd/containerd" 12 | "github.com/containerd/containerd/snapshots" 13 | "github.com/containerd/containerd/snapshots/storage" 14 | "github.com/mc256/starlight/util" 15 | "testing" 16 | ) 17 | 18 | func TestSnapshotList(t *testing.T) { 19 | t.Skip("for dev only") 20 | 21 | socket := "/run/k3s/containerd/containerd.sock" 22 | ns := "k8s.io" 23 | 24 | client, err := containerd.New(socket, containerd.WithDefaultNamespace(ns)) 25 | if err != nil { 26 | t.Error(err) 27 | return 28 | } 29 | defer client.Close() 30 | 31 | ctx := context.Background() 32 | 33 | sns := client.SnapshotService("starlight") 34 | 35 | sns.Walk(ctx, func(ctx context.Context, info snapshots.Info) error { 36 | fmt.Println(info.Name, info.Parent) 37 | fmt.Println("-->", info.Labels[util.SnapshotLabelRefImage]) 38 | ssid, inf, p, err := storage.GetInfo(ctx, info.Name) 39 | if err != nil { 40 | fmt.Println(ssid, inf, p) 41 | } else { 42 | fmt.Println(err) 43 | } 44 | return nil 45 | }) 46 | 47 | /* 48 | 36c584644174b103238ecd17cf43487b38485bf321f64c8cf525fafc5580ab86 sha256:1021ef88c7974bfff89c5a0ec4fd3160daac6c48a075f74cff721f85dd104e68 49 | --> 50 | 40b6e9770e3ec864efd193670e72640ed5a9364f2d1287234fa4b8d7731a9cba sha256:68ec11adbdc694363fbd1026d772664049569ee10fc3109b0fc44cc890b60c2c 51 | --> 52 | 47aac9e355fe94ed399ea2dcbbac7e4c912eb936eb80941d8bf08537f2956c5e sha256:1021ef88c7974bfff89c5a0ec4fd3160daac6c48a075f74cff721f85dd104e68 53 | --> 54 | 9823e5be2c32d3b2547b9a5344d49628e38189a4beafbb6c7dee3858545b98f1 sha256:ef04ecfb1d007266224a5b5b67992c74afdf12e984433a6978d061c7b852ee10 55 | --> 56 | c7f1c4db6e6fdb52efd09abeb5e51e7af6dc6fc753e32c34e4f9f04544dcd6cc sha256:ef04ecfb1d007266224a5b5b67992c74afdf12e984433a6978d061c7b852ee10 57 | --> 58 | sha256:1021ef88c7974bfff89c5a0ec4fd3160daac6c48a075f74cff721f85dd104e68 59 | --> 60 | sha256:1ad27bdd166b922492031b1938a4fb2f775e3d98c8f1b72051dad0570a4dd1b5 61 | --> 62 | sha256:40cf597a9181e86497f4121c604f9f0ab208950a98ca21db883f26b0a548a2eb 63 | --> 64 | sha256:4ac94bd63114d70c68e73d408d559bbd681b7c5094715b1a38bd7b361312555f sha256:1ad27bdd166b922492031b1938a4fb2f775e3d98c8f1b72051dad0570a4dd1b5 65 | --> 66 | sha256:68ec11adbdc694363fbd1026d772664049569ee10fc3109b0fc44cc890b60c2c sha256:7ad6b790083aeda0ad3e5e2888d9cf1a64d6c5ff6b41487a3404bc6f5684a2d3 67 | --> sha256:c20f5ef500bc93f26fec480dfea1d4cbda2d39791d611031a3aaf3c096f35c73 68 | sha256:7ad6b790083aeda0ad3e5e2888d9cf1a64d6c5ff6b41487a3404bc6f5684a2d3 sha256:b11040115c1a7bad75c256bfad158a1bf2aec69418ffc1f49326ba59e33f69b2 69 | --> sha256:c20f5ef500bc93f26fec480dfea1d4cbda2d39791d611031a3aaf3c096f35c73 70 | sha256:a5878da390fcf0e6324ce5593f03ae9609caaa9bce0a522b504c00b91e579b46 71 | --> sha256:c20f5ef500bc93f26fec480dfea1d4cbda2d39791d611031a3aaf3c096f35c73 72 | sha256:b11040115c1a7bad75c256bfad158a1bf2aec69418ffc1f49326ba59e33f69b2 sha256:d44df6880f5a4e309d25a92dc413458a1f964493868c99000519dbc365abe1d7 73 | --> sha256:c20f5ef500bc93f26fec480dfea1d4cbda2d39791d611031a3aaf3c096f35c73 74 | sha256:d44df6880f5a4e309d25a92dc413458a1f964493868c99000519dbc365abe1d7 sha256:f8521e5ffbea025f78787bf0d407a4295dff5fcc3cc5c8936126fd17fa1819a8 75 | --> sha256:c20f5ef500bc93f26fec480dfea1d4cbda2d39791d611031a3aaf3c096f35c73 76 | sha256:ef04ecfb1d007266224a5b5b67992c74afdf12e984433a6978d061c7b852ee10 sha256:4ac94bd63114d70c68e73d408d559bbd681b7c5094715b1a38bd7b361312555f 77 | --> 78 | sha256:f8521e5ffbea025f78787bf0d407a4295dff5fcc3cc5c8936126fd17fa1819a8 sha256:a5878da390fcf0e6324ce5593f03ae9609caaa9bce0a522b504c00b91e579b46 79 | --> sha256:c20f5ef500bc93f26fec480dfea1d4cbda2d39791d611031a3aaf3c096f35c73 80 | 81 | 82 | sha256:c20f5ef500bc93f26fec480dfea1d4cbda2d39791d611031a3aaf3c096f35c73 83 | manifest 84 | 85 | complete.starlight.mc256.dev=2022-11-27T18:34:46Z, 86 | puller.containerd.io=starlight, 87 | mediaType.starlight.mc256.dev=manifest, 88 | containerd.io/gc.ref.snapshot.starlight/0=sha256:a5878da390fcf0e6324ce5593f03ae9609caaa9bce0a522b504c00b91e579b46, 89 | containerd.io/gc.ref.content.starlight=sha256:5c409db91624a306224d9a98efcfded2ae94da0a483cf2f1d40b8cb1ea776385, 90 | starlight.mc256.dev/distribution.source.in-cluster=starlight-registry.default.svc.cluster.local:5000/starlight/redis:6.2.1, 91 | containerd.io/gc.ref.content.config=sha256:982983f3c76aeb79bddb66239c878f011c147b15cac9d7c71abedf76e772c7c6, 92 | containerd.io/gc.ref.snapshot.starlight/5=sha256:68ec11adbdc694363fbd1026d772664049569ee10fc3109b0fc44cc890b60c2c, 93 | containerd.io/gc.ref.snapshot.starlight/4=sha256:7ad6b790083aeda0ad3e5e2888d9cf1a64d6c5ff6b41487a3404bc6f5684a2d3, 94 | containerd.io/gc.ref.snapshot.starlight/2=sha256:d44df6880f5a4e309d25a92dc413458a1f964493868c99000519dbc365abe1d7, 95 | containerd.io/gc.ref.snapshot.starlight/1=sha256:f8521e5ffbea025f78787bf0d407a4295dff5fcc3cc5c8936126fd17fa1819a8, 96 | containerd.io/gc.ref.snapshot.starlight/3=sha256:b11040115c1a7bad75c256bfad158a1bf2aec69418ffc1f49326ba59e33f69b2 97 | 98 | sha256:e813af18bfe9565a5a4b67e4e80be064c488c4630770951c97f240fa337192e8 945B 53 minutes 99 | containerd.io/gc.ref.content.config=sha256:4b167e69a056cad9f179b344d7208fefcbc99fee9d48c8988098d4af45cbc9ed, 100 | containerd.io/distribution.source.ghcr.io=mc256/starlight/cli, 101 | containerd.io/gc.ref.content.l.2=sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1, 102 | containerd.io/gc.ref.content.l.1=sha256:76fc34d44084250f3a5e66218596e2105e9e0537b69456999dccf1740be1795e, 103 | containerd.io/gc.ref.content.l.0=sha256:1b7ca6aea1ddfe716f3694edb811ab35114db9e93f3ce38d7dab6b4d9270cb0c 104 | 105 | */ 106 | } 107 | -------------------------------------------------------------------------------- /demo/chart/values.yaml: -------------------------------------------------------------------------------- 1 | # Starlight Proxy + Registry 2 | nameOverride: "" 3 | fullnameOverride: "" 4 | namespace: "default" 5 | 6 | ######################################################################## 7 | # CLOUD 8 | ######################################################################## 9 | # Image pull information 10 | starlightProxy: 11 | repository: "ghcr.io/mc256/starlight/proxy" 12 | tag: "0.5" 13 | imagePullPolicy: Always 14 | 15 | # dbConnection specify 16 | # Default is set to the postgres instance in the same deployment. 17 | dbConnection: "postgres://postgres:postgres@127.0.0.1:5432/postgres?sslmode=disable" 18 | # logLevel is the log level for the proxy, options are: trace, debug, info, warn, error, fatal 19 | logLevel: "info" 20 | # defaultRegistry is the default registry when the full image name is not specified 21 | defaultRegistry: "http://container-registry.default.svc.cluster.local:5000" 22 | resources: {} 23 | 24 | # If you are using a private registry, you need to specify the secret name here 25 | dockerConfigSecret: "" 26 | 27 | 28 | # --------------------------------------------------------------------- 29 | # Metadata database - PostgreSQL 30 | # --------------------------------------------------------------------- 31 | # if disabled, please specify an existing database using `dbConnection` in the previous section 32 | postgres: 33 | enabled: true 34 | imagePullPolicy: IfNotPresent 35 | tag: "latest" 36 | persistence: 37 | enabled: true 38 | existingClaim: "" 39 | storageClass: "" 40 | accessModes: [ReadWriteOnce] 41 | size: 20Gi 42 | resources: {} 43 | 44 | # --------------------------------------------------------------------- 45 | # Metadata database management tool 46 | # --------------------------------------------------------------------- 47 | # A simple web UI for metadata database. 48 | # This database is mainly for debugging and testing purposes. 49 | # If you are not curious about what is in the database, you can disable it. 50 | # postgres.enabled must be true 51 | adminer: 52 | enabled: true 53 | imagePullPolicy: IfNotPresent 54 | tag: "latest" 55 | resources: {} 56 | 57 | # --------------------------------------------------------------------- 58 | # Vanilla Container Registry 59 | # --------------------------------------------------------------------- 60 | # Recommend to use https://goharbor.io/ in production. 61 | # But this is good enough for testing. 62 | registry: 63 | enabled: true # If use other container registry, please specify `registryAddress` 64 | repository: "registry" 65 | imagePullPolicy: IfNotPresent 66 | tag: "latest" 67 | persistence: 68 | enabled: true 69 | existingClaim: "" 70 | storageClass: "" 71 | accessModes: [ReadWriteOnce] 72 | size: 20Gi 73 | resources: {} 74 | 75 | # --------------------------------------------------------------------- 76 | imagePullSecrets: [] 77 | 78 | # Starlight Proxy Default Environment Variables 79 | # registryAddress: goharbor-core.default.svc.cluster.local/starlight 80 | logLevel: info 81 | 82 | # Service 83 | service: 84 | # Recommended to use Cluster IP and put a reverse proxy in front of it. 85 | type: ClusterIP 86 | 87 | 88 | # Ingress 89 | ingress: 90 | enabled: true 91 | ingressClassName: "nginx" 92 | annotations: 93 | # kubernetes.io/tls-acme: "true" 94 | # set to the largest layer size for uploading container image 95 | nginx.ingress.kubernetes.io/proxy-body-size: 512m 96 | # if you want to protect the starlight proxy with basic auth, please uncomment the following lines 97 | # and create a secret named `registry-auth`: 98 | # nginx.ingress.kubernetes.io/auth-type: basic 99 | # nginx.ingress.kubernetes.io/auth-secret: registry-auth 100 | 101 | hosts: 102 | - starlight.lan 103 | tls: [] 104 | 105 | # select nodes in the cloud 106 | cloudTolerations: [] 107 | # allow pod deploying to master node 108 | #- key: node-role.kubernetes.io/master 109 | # effect: NoSchedule 110 | 111 | cloudNodeSelector: 112 | kubernetes.io/os: linux 113 | # kubernetes.io/arch: amd64 114 | # kubernetes.io/hostname: cloud 115 | 116 | cloudAffinity: {} 117 | 118 | ######################################################################## 119 | # EDGE 120 | ########################################################################\ 121 | edge: 122 | enabled: true 123 | # starlightCLI connects to the Starlight Daemon on the edge node via gRPC 124 | repository: "ghcr.io/mc256/starlight/cli" 125 | tag: "0.5" 126 | imagePullPolicy: Always 127 | # This keep the container running allows you to run some starlight-related commands on the edge node 128 | command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 60 ; done"] 129 | # Starlight CLI Default Environment Variables 130 | env: 131 | - name: CONTAINERD_NAMESPACE 132 | value: "k8s.io" 133 | - name: PATH 134 | value: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt" 135 | 136 | # select nodes in the edge 137 | edgeTolerations: [] 138 | 139 | edgeNodeSelector: 140 | kubernetes.io/os: linux 141 | # install starlight daemon on the edge node with label `starlight: true` 142 | # set the label with `kubectl label node YOUREDGENODE node-role.kubernetes.io/starlight=ture` 143 | node-role.kubernetes.io/starlight: "ture" 144 | # kubernetes.io/arch: arm64 145 | # kubernetes.io/hostname: edge 146 | 147 | edgeAffinity: {} 148 | 149 | ######################################################################## 150 | # OTHER CONFIGURATIONS 151 | ######################################################################## 152 | serviceAccount: 153 | # Specifies whether a service account should be created 154 | enabled: true 155 | # Annotations to add to the service account 156 | annotations: {} 157 | # Labels to add to the service account 158 | labels: {} 159 | # The name of the service account to use. 160 | # If not set and create is true, a name is generated using the fullname template 161 | name: "starlight" 162 | 163 | podAnnotations: {} 164 | 165 | podSecurityContext: {} 166 | # fsGroup: 2000 167 | 168 | securityContext: {} 169 | # capabilities: 170 | # drop: 171 | # - ALL 172 | # readOnlyRootFilesystem: true 173 | # runAsNonRoot: true 174 | # runAsUser: 1000 175 | -------------------------------------------------------------------------------- /docs/deprecated-v0.1/starlight-snapshotter.md: -------------------------------------------------------------------------------- 1 | # Starlight Snapshotter Plugin 2 | 3 | **⚠️ This document is outdated. Please use https://github.com/mc256/starlight/blob/master/docs/newbie.md instead** 4 | 5 | This is the **Step 2** to use Starlight: 6 | 7 | Set up the worker to be able to run Starlight. 8 | This involves 9 | installing **containerd** and the **Starlight snapshotter plugin**, 10 | configuring containerd to use the plugin, 11 | and starting the Starlight snapshotter daemon 12 | (you also need to tell the snapshotter the address of the proxy server). 13 | 14 | [⬅️ Back to README.md](https://github.com/mc256/starlight) 15 | 16 | --- 17 | 18 | ## Method 1. Install Pre-built Package (Recommended) 19 | 20 | Pre-build deb package is available for `amd64`, `armhf`, and `arm64`. 21 | 22 | ### 1. Install Starlight Snapshotter 23 | 24 | Download and install the `.deb` package from the [release page](https://github.com/mc256/starlight/releases). 25 | 26 | ```shell 27 | export ARCH=amd64 28 | export SL_VERSION=0.1.2 29 | wget "https://github.com/mc256/starlight/releases/download/v${SL_VERSION}/starlight-snapshotter_${SL_VERSION}_$ARCH.deb" 30 | sudo apt install -f "./starlight-snapshotter_${SL_VERSION}_$ARCH.deb" 31 | ``` 32 | 33 | Update systemd service file `/lib/systemd/system/starlight.service`. 34 | - Change `STARLIGHT_PROXY` to the address of the Starlight Proxy. 35 | - remove `--plain-http` if the Starlight Proxy is behind a HTTPS reverse proxy. 36 | ``` 37 | ExecStart=/usr/bin/starlight-grpc run --plain-http starlight.lan 38 | ``` 39 | 40 | Reload systemd service 41 | ```shell 42 | sudo systemctl daemon-reload 43 | sudo systemctl restart starlight-snapshotter 44 | ``` 45 | 46 | ### 2. Configure `contaienrd` 47 | 48 | Add configuration to `/etc/containerd/config.toml`. 49 | (If you have set other `proxy_plugins`, please manually edit the file) 50 | ```shell 51 | sudo mkdir /etc/containerd/ && \ 52 | cat < /dev/null 53 | [proxy_plugins] 54 | [proxy_plugins.starlight] 55 | type = "snapshot" 56 | address = "/run/starlight-grpc/starlight-snapshotter.socket" 57 | EOT 58 | ``` 59 | 60 | Restart `containerd` service 61 | ```shell 62 | sudo systemctl restart containerd 63 | ``` 64 | 65 | Verify the Starlight snapshotter plugin is functioning 66 | ```shell 67 | sudo ctr plugin ls | grep starlight 68 | # io.containerd.snapshotter.v1 starlight - ok 69 | ``` 70 | 71 | 72 | 🙌 That's it. Please proceed to the **Step 3**. 73 | 74 | [⬅️ Back to README.md](https://github.com/mc256/starlight#getting-started) 75 | 76 | 77 | 78 | --- 79 | 80 | 81 | 82 | ## Method 2. Build from source 83 | 84 | ### 1. Install Dependencies 85 | 86 | The worker machine is supposed to be far away (in latency) to the registry and proxy. 87 | Please install **containerd** and **Starlight snapshotter** on a new machine (or VM), not the same machine that runs the proxy or the registry. 88 | 89 | The worker machine needs `build-essential` and `containerd`. 90 | ```shell 91 | sudo apt update && sudo apt upgrade -y && \ 92 | sudo apt install -y build-essential containerd 93 | ``` 94 | 95 | Enable `containerd` 96 | ```shell 97 | sudo systemctl enable containerd && \ 98 | sudo systemctl start containerd 99 | ``` 100 | 101 | Verify `containerd` is running 102 | ```shell 103 | sudo systemctl status containerd 104 | # Active: active 105 | ``` 106 | 107 | Install Go https://go.dev/doc/install ➡️ 108 | ```shell 109 | wget https://go.dev/dl/go1.17.8.linux-amd64.tar.gz && \ 110 | sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf go1.17.8.linux-amd64.tar.gz 111 | ``` 112 | 113 | Add Go to the environment variable (You may want to change `.zshrc` or `.bashrc` file to permanently add this folder to the `PATH` environment variable) 114 | ```shell 115 | export PATH=$PATH:/usr/local/go/bin 116 | ``` 117 | 118 | Verify Go is available 119 | ```shell 120 | go version 121 | # go version go1.17.8 linux/amd64 122 | ``` 123 | 124 | ### 2. Clone and Build 125 | Clone the Starlight repository 126 | ```shell 127 | git clone https://github.com/mc256/starlight.git && \ 128 | cd starlight 129 | ``` 130 | 131 | Build the snapshotter plugin and CLI tool 132 | ```shell 133 | make starlight-grpc ctr-starlight 134 | ``` 135 | 136 | ### 3. Configure Starlight Snapshotter 137 | 138 | You need to find out the IP address / DNS of the Starlight Proxy server (in **Step 1**. [Find out how to install **Starlight proxy** ➡️](docs/starlight-proxy.md) ) 139 | 140 | ```shell 141 | # This is an example 142 | export STARLIGHT_PROXY=172.18.1.3:8090 143 | export REGISTRY=172.18.1.3:5000 144 | ``` 145 | 146 | Verify that the Starlight proxy is accessible from the worker. 147 | ```shell 148 | curl http://$STARLIGHT_PROXY 149 | # Starlight Proxy OK! 150 | ``` 151 | 152 | Install Starlight Snapshotter `systemd` service and CLI tool. 153 | Please follow the prompt, enter 154 | ```shell 155 | sudo make install install-systemd-service 156 | #Please enter Starlight Proxy address (example: proxy.mc256.dev:8090):172.18.1.3:8090 157 | #Enable HTTPS Certificate (requires load balancer like Nginx) (y/N):n 158 | #Created systemd service file (/lib/systemd/system/starlight.service) 159 | #Reloaded systemd daemon 160 | ``` 161 | 162 | Enable Starlight snapshotter service 163 | ```shell 164 | sudo systemctl enable starlight && \ 165 | sudo systemctl start starlight 166 | ``` 167 | 168 | Verify Starlight is running 169 | ```shell 170 | sudo systemctl status starlight 171 | # it should be "active". 172 | ``` 173 | 174 | ### 4. Configure `contaienrd` 175 | 176 | Add configuration to `/etc/containerd/config.toml`. 177 | (If you have set other `proxy_plugins`, please manually edit the file) 178 | ```shell 179 | sudo mkdir /etc/containerd/ && \ 180 | cat < /dev/null 181 | [proxy_plugins] 182 | [proxy_plugins.starlight] 183 | type = "snapshot" 184 | address = "/run/starlight-grpc/starlight-snapshotter.socket" 185 | EOT 186 | ``` 187 | 188 | Restart `containerd` service 189 | ```shell 190 | sudo systemctl restart containerd 191 | ``` 192 | 193 | Verify the Starlight snapshotter plugin is functioning 194 | ```shell 195 | sudo ctr plugin ls | grep starlight 196 | # io.containerd.snapshotter.v1 starlight - ok 197 | ``` 198 | 199 | 200 | 🙌 That's it. Please proceed to the **Step 3**. 201 | 202 | [⬅️ Back to README.md](https://github.com/mc256/starlight#getting-started) 203 | 204 | 205 | --- 206 | 207 | For more information, please see `ctr-starlight --help` and `starlight-grpc --help` 208 | -------------------------------------------------------------------------------- /util/receive/image.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package receive 7 | 8 | import ( 9 | "fmt" 10 | fuseFs "github.com/hanwen/go-fuse/v2/fs" 11 | "github.com/hanwen/go-fuse/v2/fuse" 12 | "github.com/mc256/starlight/client/fs" 13 | "github.com/mc256/starlight/util/common" 14 | "golang.org/x/sys/unix" 15 | "path/filepath" 16 | "syscall" 17 | "unsafe" 18 | ) 19 | 20 | type ImageLayer struct { 21 | Size int64 `json:"s"` 22 | Serial int64 `json:"f"` 23 | 24 | // Hash is the digest of the compressed layer 25 | Hash string `json:"h"` 26 | 27 | // path to the local storage 28 | Local string 29 | } 30 | 31 | func (il ImageLayer) String() string { 32 | return fmt.Sprintf("[%05d:%02d]%s-%d", il.Serial, -1, il.Hash, il.Size) 33 | } 34 | 35 | type Content struct { 36 | Signal chan interface{} `json:"-"` 37 | 38 | // ------------------------------------------ 39 | // stack identify which layer should this content be placed, all the files will be referencing the content 40 | Stack int64 `json:"t"` 41 | 42 | // offset is non-zero if the file is in the delta bundle body 43 | Offset int64 `json:"o,omitempty"` 44 | 45 | // size is the size of the compressed content 46 | Size int64 `json:"s"` 47 | 48 | Chunks []*FileChunk `json:"c"` 49 | 50 | Digest string `json:"d"` 51 | } 52 | 53 | func (c *Content) GetBaseDir() string { 54 | return filepath.Join(c.Digest[7:8], c.Digest[8:10], c.Digest[10:12]) 55 | } 56 | 57 | func (c *Content) GetPath() string { 58 | return filepath.Join(c.GetBaseDir(), c.Digest[12:]) 59 | } 60 | 61 | type ReferencedFile struct { 62 | File 63 | 64 | // Stack in the existing image from bottom to top, 0-indexed 65 | Stack int64 `json:"S"` 66 | 67 | // if the file is available on the client then ReferenceFsId is non-zero, 68 | // expecting the file is available on the client and can be accessed using the File.Digest . 69 | // (This is the Serial index in the database `filesystem.id`) 70 | ReferenceFsId int64 `json:"R,omitempty"` 71 | 72 | // if the file is not available on the client but on other layers in the requested image， 73 | // then ReferenceFsId is zero and ReferenceStack is non-zero, 74 | // expecting the file content in the delta bundle body 75 | // (This is Stack not Serial, 0-indexed) 76 | ReferenceStack int64 `json:"T,omitempty"` 77 | 78 | // PayloadOrder is set if the file is not available on the client, then PayloadOrder is non-zero. 79 | // It indicates the order of the file in the delta bundle body (payload) 80 | PayloadOrder int `json:"O,omitempty"` 81 | 82 | // if Ready is nil or closed, means the file is ready 83 | Ready *chan interface{} `json:"-"` 84 | 85 | stable fuseFs.StableAttr 86 | children []fs.ReceivedFile 87 | } 88 | 89 | // ------------------------------------------ 90 | // use in file system 91 | // 92 | 93 | func (r *ReferencedFile) GetChildren() []fs.ReceivedFile { 94 | return r.children 95 | } 96 | 97 | func (r *ReferencedFile) AppendChild(children fs.ReceivedFile) { 98 | if r.children == nil { 99 | r.children = make([]fs.ReceivedFile, 0) 100 | } 101 | r.children = append(r.children, children) 102 | } 103 | 104 | func (r *ReferencedFile) IsReady() bool { 105 | return r.Ready == nil 106 | } 107 | 108 | func (r *ReferencedFile) InitFuseStableAttr() { 109 | r.stable.Ino = uint64(uintptr(unsafe.Pointer(r))) 110 | r.stable.Gen = 0 111 | r.stable.Mode = modeOfEntry(r) 112 | } 113 | 114 | func (r *ReferencedFile) GetAttr(out *fuse.Attr) syscall.Errno { 115 | out.Ino = r.stable.Ino 116 | out.Size = uint64(r.Size) 117 | if r.IsDir() { 118 | out.Size = 4096 119 | } else if r.Type == "symlink" { 120 | out.Size = uint64(len(r.LinkName)) 121 | } 122 | r.SetBlockSize(out) 123 | mtime := r.ModTime() 124 | out.SetTimes(&mtime, &mtime, &mtime) 125 | out.Mode = r.stable.Mode 126 | out.Owner = fuse.Owner{Uid: uint32(r.UID), Gid: uint32(r.GID)} 127 | out.Rdev = uint32(unix.Mkdev(uint32(r.DevMajor), uint32(r.DevMinor))) 128 | out.Nlink = uint32(r.NumLink) 129 | if out.Nlink == 0 { 130 | out.Nlink = 1 // zero "NumLink" means one. 131 | } 132 | return 0 133 | } 134 | 135 | func (r *ReferencedFile) GetXAttrs() map[string][]byte { 136 | if r.Xattrs == nil { 137 | return make(map[string][]byte) 138 | } 139 | return r.Xattrs 140 | } 141 | 142 | func (r *ReferencedFile) GetName() string { 143 | return r.Name 144 | } 145 | 146 | func (r *ReferencedFile) GetStableAttr() *fuseFs.StableAttr { 147 | return &r.stable 148 | } 149 | 150 | func (r *ReferencedFile) GetLinkName() string { 151 | return r.LinkName 152 | } 153 | 154 | func (r *ReferencedFile) GetBaseDir() string { 155 | return filepath.Join(r.Digest[7:8], r.Digest[8:10], r.Digest[10:12]) 156 | } 157 | 158 | func (r *ReferencedFile) GetRealPath() string { 159 | return filepath.Join(r.GetBaseDir(), r.Digest[12:]) 160 | } 161 | 162 | func (r *ReferencedFile) WaitForReady() { 163 | <-*r.Ready 164 | } 165 | 166 | func (r *ReferencedFile) IsReferencingRequestedImage() (stack int64, yes bool) { 167 | if r.ReferenceFsId != 0 { 168 | return 0, false 169 | } 170 | 171 | // in the payload 172 | if r.ReferenceStack != 0 { 173 | // different layer 174 | return r.ReferenceStack, true 175 | } 176 | // same layer 177 | return r.Stack, true 178 | } 179 | 180 | func (r *ReferencedFile) IsReferencingLocalFilesystem() (serial int64, yes bool) { 181 | if r.ReferenceFsId != 0 { 182 | return r.ReferenceFsId, true 183 | } 184 | return 0, false 185 | } 186 | 187 | // InPayload returns true if the content of the file is in the delta bundle (payload) 188 | func (r *ReferencedFile) InPayload() bool { 189 | return r.PayloadOrder > 0 190 | } 191 | 192 | type FileChunk struct { 193 | Offset int64 `json:"o"` 194 | ChunkOffset int64 `json:"c"` 195 | ChunkSize int64 `json:"h"` 196 | CompressedSize int64 `json:"s"` 197 | } 198 | 199 | type File struct { 200 | common.TOCEntry 201 | Chunks []*FileChunk `json:"c,omitempty"` 202 | FsId int64 `json:"-"` 203 | } 204 | 205 | type Image struct { 206 | Serial int64 `json:"s"` 207 | Layers []*ImageLayer `json:"l"` 208 | } 209 | 210 | func (i Image) String() string { 211 | return fmt.Sprintf("%d->%v", i.Serial, i.Layers) 212 | } 213 | 214 | type DeltaBundle struct { 215 | Source *Image `json:"s"` 216 | Destination *Image `json:"d"` 217 | 218 | // contents and BodyLength are computed by Builder.computeDelta() 219 | Contents []*Content `json:"c"` 220 | BodyLength int64 `json:"bl"` 221 | 222 | // RequestedFiles are all the files in the requested images 223 | // Use this to reconstruct the file system 224 | RequestedFiles []*ReferencedFile `json:"rf"` 225 | } 226 | -------------------------------------------------------------------------------- /demo/chart/templates/deployment-proxy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "starlight.fullname" . }} 5 | namespace: {{ .Values.namespace }} 6 | labels: 7 | {{- include "starlight.proxyLabels" . | nindent 4 }} 8 | kubernetes.io/cluster-service: "true" 9 | spec: 10 | selector: 11 | matchLabels: 12 | {{- include "starlight.proxySelectorLabels" . | nindent 6 }} 13 | template: 14 | metadata: 15 | {{- with .Values.podAnnotations }} 16 | annotations: 17 | {{- toYaml . | nindent 8 }} 18 | {{- end }} 19 | labels: 20 | {{- include "starlight.proxySelectorLabels" . | nindent 8 }} 21 | spec: 22 | {{- if .Values.serviceAccount.enabled}} 23 | serviceAccountName: {{ include "starlight.serviceAccountName" . }} 24 | {{- end }} 25 | {{- with .Values.imagePullSecrets }} 26 | imagePullSecrets: 27 | {{- toYaml . | nindent 8 }} 28 | {{- end }} 29 | securityContext: 30 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 31 | volumes: 32 | {{- if and (eq .Values.postgres.enabled true) (eq .Values.postgres.persistence.enabled true) }} 33 | - name: starlight-pv 34 | persistentVolumeClaim: 35 | {{- $newClaimName := include "starlight.fullname" .}} 36 | claimName: {{ .Values.postgres.persistence.existingClaim | default $newClaimName }} 37 | {{- end }} 38 | - name: dockerconfig 39 | secret: 40 | secretName: dockerconfig 41 | optional: true 42 | containers: 43 | ############################################################## 44 | ############################################################## 45 | - name: starlight-proxy 46 | securityContext: 47 | {{- toYaml .Values.securityContext | nindent 12 }} 48 | image: "{{ .Values.starlightProxy.repository }}:{{ .Values.starlightProxy.tag | default .Chart.AppVersion }}" 49 | imagePullPolicy: {{ .Values.starlightProxy.imagePullPolicy }} 50 | ports: 51 | - name: starlightproxy 52 | containerPort: 8090 53 | protocol: TCP 54 | livenessProbe: 55 | initialDelaySeconds: 5 56 | periodSeconds: 10 57 | httpGet: 58 | path: /health-check 59 | port: 8090 60 | readinessProbe: 61 | initialDelaySeconds: 5 62 | periodSeconds: 10 63 | httpGet: 64 | path: /health-check 65 | port: 8090 66 | resources: 67 | {{- toYaml .Values.starlightProxy.resources | nindent 12}} 68 | #command: ["/bin/sh", "-ec", "while :; do echo '.'; sleep 60 ; done"] 69 | command: ["/opt/starlight-proxy"] 70 | # args: ["--config", "/etc/starlight/starlight-proxy.json"] 71 | env: 72 | - name: STARLIGHT_HOST 73 | value: "0.0.0.0" 74 | - name: STARLIGHT_PORT 75 | value: "8090" 76 | - name: LOG_LEVEL 77 | value: {{ .Values.starlightProxy.logLevel | quote | default "info" }} 78 | - name: DB_CONNECTION_STRING 79 | value: {{ .Values.starlightProxy.dbConnection | quote | default "postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable" }} 80 | - name: REGISTRY 81 | value: {{ .Values.starlightProxy.defaultRegistry | quote | default "http://container-registry.default.svc.cluster.local:5000" }} 82 | - name: DOCKER_CONFIG 83 | value: /opt/.docker/config.json 84 | volumeMounts: 85 | - mountPath: /opt/.docker/config.json 86 | name: dockerconfig 87 | subPath: config.json 88 | 89 | {{- if .Values.postgres.enabled }} 90 | ############################################################ 91 | - name: starlight-metadata 92 | securityContext: 93 | {{- toYaml .Values.securityContext | nindent 12 }} 94 | image: "postgres:{{ .Values.postgres.tag | default "latest" }}" 95 | imagePullPolicy: {{ .Values.postgres.imagePullPolicy }} 96 | ports: 97 | - name: postgres 98 | containerPort: 5432 99 | protocol: TCP 100 | livenessProbe: 101 | periodSeconds: 60 102 | exec: 103 | command: 104 | - /bin/sh 105 | - -c 106 | - exec pg_isready -U "postgres" -h 127.0.0.1 -p 5432 107 | readinessProbe: 108 | exec: 109 | command: 110 | - /bin/sh 111 | - -c 112 | - exec pg_isready -U "postgres" -h 127.0.0.1 -p 5432 113 | env: 114 | - name: POSTGRES_PASSWORD 115 | value: postgres 116 | {{- if and (eq .Values.postgres.enabled true) (eq .Values.postgres.persistence.enabled true) }} 117 | volumeMounts: 118 | - mountPath: /var/lib/postgresql/data 119 | name: starlight-pv 120 | {{- end}} 121 | resources: 122 | {{- toYaml .Values.postgres.resources | nindent 12 }} 123 | {{- end }} 124 | {{- if and (eq .Values.postgres.enabled true) (eq .Values.adminer.enabled true) }} 125 | ############################################################ 126 | - name: adminer 127 | image: "adminer:{{ .Values.adminer.tag | default "latest" }}" 128 | imagePullPolicy: {{ .Values.adminer.imagePullPolicy }} 129 | ports: 130 | - name: adminer 131 | containerPort: 8080 132 | protocol: TCP 133 | env: 134 | - name: ADMINER_DEFAULT_SERVER 135 | value: postgres 136 | - name: test 137 | value: {{include "starlight.fullname-registry" . }} 138 | securityContext: 139 | runAsUser: 1000 140 | allowPrivilegeEscalation: false 141 | resources: 142 | {{- toYaml .Values.adminer.resources | nindent 12 }} 143 | {{- end }} 144 | ############################################################## 145 | ############################################################## 146 | {{- with .Values.cloudNodeSelector }} 147 | nodeSelector: 148 | {{- toYaml . | nindent 8 }} 149 | {{- end }} 150 | {{- with .Values.cloudAffinity }} 151 | affinity: 152 | {{- toYaml . | nindent 8 }} 153 | {{- end }} 154 | {{- with .Values.cloudTolerations }} 155 | tolerations: 156 | {{- toYaml . | nindent 8 }} 157 | {{- end }} 158 | -------------------------------------------------------------------------------- /client/fs/fs.go: -------------------------------------------------------------------------------- 1 | /* 2 | file created by Junlin Chen in 2022 3 | 4 | */ 5 | 6 | package fs 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | "github.com/containerd/containerd/log" 12 | "github.com/hanwen/go-fuse/v2/fs" 13 | "github.com/hanwen/go-fuse/v2/fuse" 14 | "github.com/sirupsen/logrus" 15 | "path/filepath" 16 | "strings" 17 | "syscall" 18 | "time" 19 | ) 20 | 21 | const ( 22 | DebugTrace = false 23 | ) 24 | 25 | type ReceivedFile interface { 26 | GetChildren() []ReceivedFile 27 | AppendChild(children ReceivedFile) 28 | IsReady() bool 29 | GetAttr(out *fuse.Attr) syscall.Errno 30 | GetXAttrs() map[string][]byte 31 | GetName() string 32 | GetStableAttr() *fs.StableAttr 33 | GetLinkName() string 34 | GetRealPath() string 35 | WaitForReady() 36 | 37 | // IsReferencingRequestedImage returns stack number where the actual content located 38 | // if the file is available in the local filesystem then yes is false 39 | IsReferencingRequestedImage() (stack int64, yes bool) 40 | 41 | // IsReferencingLocalFilesystem can not return true if IsReferencingRequestedImage returns true 42 | IsReferencingLocalFilesystem() (serial int64, yes bool) 43 | } 44 | 45 | type StarlightFsNode struct { 46 | fs.Inode 47 | ReceivedFile 48 | instance *Instance 49 | } 50 | 51 | func (n *StarlightFsNode) getFile(p string) ReceivedFile { 52 | return n.instance.manager.LookUpFile(n.instance.stack, p) 53 | } 54 | 55 | func (n *StarlightFsNode) getRealPath() (string, error) { 56 | // 1. not available, in the same layer 57 | // 2. not available, in other layers 58 | // 3. available, in local filesystem 59 | pp := n.GetRealPath() 60 | if stack, yes := n.ReceivedFile.IsReferencingRequestedImage(); yes { 61 | return filepath.Join(n.instance.manager.GetPathByStack(stack), pp), nil 62 | } 63 | if serial, yes := n.ReceivedFile.IsReferencingLocalFilesystem(); yes { 64 | return filepath.Join(n.instance.manager.GetPathBySerial(serial), pp), nil 65 | } 66 | 67 | return "", fmt.Errorf("fsnode: unknown file reference [%s]", n.GetName()) 68 | } 69 | 70 | func (n *StarlightFsNode) log(filename string, access, complete time.Time) { 71 | n.instance.manager.LogTrace(n.instance.stack, filename, access, complete) 72 | } 73 | 74 | var _ = (fs.NodeLookuper)((*StarlightFsNode)(nil)) 75 | 76 | func (n *StarlightFsNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) { 77 | f := n.getFile(filepath.Join(n.GetName(), name)) 78 | if f == nil { 79 | return nil, syscall.ENOENT 80 | } 81 | 82 | var attr fuse.Attr 83 | if err := f.GetAttr(&attr); err != 0 { 84 | return nil, err 85 | } 86 | out.Attr = attr 87 | return n.NewInode(ctx, &StarlightFsNode{ 88 | ReceivedFile: f, 89 | instance: n.instance, 90 | }, *f.GetStableAttr()), 0 91 | } 92 | 93 | var _ = (fs.NodeGetattrer)((*StarlightFsNode)(nil)) 94 | 95 | func (n *StarlightFsNode) Getattr(ctx context.Context, fh fs.FileHandle, out *fuse.AttrOut) syscall.Errno { 96 | var attr fuse.Attr 97 | if err := n.GetAttr(&attr); err != 0 { 98 | return err 99 | } 100 | out.Attr = attr 101 | return 0 102 | } 103 | 104 | var _ = (fs.NodeGetxattrer)((*StarlightFsNode)(nil)) 105 | 106 | func (n *StarlightFsNode) Getxattr(ctx context.Context, attr string, dest []byte) (uint32, syscall.Errno) { 107 | if val, hasVal := n.GetXAttrs()[attr]; hasVal { 108 | dest = val 109 | return uint32(len(val)), 0 110 | } 111 | 112 | return 0, fs.ENOATTR 113 | } 114 | 115 | var _ = (fs.NodeListxattrer)((*StarlightFsNode)(nil)) 116 | 117 | func (n *StarlightFsNode) Listxattr(ctx context.Context, dest []byte) (uint32, syscall.Errno) { 118 | xattrs := n.GetXAttrs() 119 | kl := make([]string, len(xattrs)) 120 | for k := range xattrs { 121 | kl = append(kl, k) 122 | } 123 | res := strings.Join(kl, "\x00") 124 | dest = []byte(res) 125 | 126 | return uint32(len(res)), 0 127 | } 128 | 129 | var _ = (fs.NodeReaddirer)((*StarlightFsNode)(nil)) 130 | 131 | func (n *StarlightFsNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) { 132 | children := n.GetChildren() 133 | cl := make([]fuse.DirEntry, 0, len(children)+2) 134 | for _, child := range children { 135 | var attr fuse.Attr 136 | if err := child.GetAttr(&attr); err != 0 { 137 | return nil, err 138 | } 139 | cl = append(cl, fuse.DirEntry{ 140 | Mode: attr.Mode, 141 | Name: filepath.Base(child.GetName()), 142 | Ino: attr.Ino, 143 | }) 144 | } 145 | 146 | // link to myself and parent 147 | // . 148 | attr := n.GetStableAttr() 149 | cl = append(cl, fuse.DirEntry{ 150 | Mode: attr.Mode, 151 | Name: ".", 152 | Ino: attr.Ino, 153 | }) 154 | 155 | // .. 156 | f := n.getFile(filepath.Join(n.GetName(), "..")) 157 | if f != nil { 158 | attr = f.GetStableAttr() 159 | } 160 | cl = append(cl, fuse.DirEntry{ 161 | Mode: attr.Mode, 162 | Name: "..", 163 | Ino: attr.Ino, 164 | }) 165 | 166 | return fs.NewListDirStream(cl), 0 167 | } 168 | 169 | var _ = (fs.NodeReadlinker)((*StarlightFsNode)(nil)) 170 | 171 | func (n *StarlightFsNode) Readlink(ctx context.Context) ([]byte, syscall.Errno) { 172 | buf, err := syscall.ByteSliceFromString(n.GetLinkName()) 173 | if err != nil { 174 | return nil, fs.ToErrno(err) 175 | } 176 | return buf, 0 177 | } 178 | 179 | var _ = (fs.NodeOpener)((*StarlightFsNode)(nil)) 180 | 181 | func (n *StarlightFsNode) Open(ctx context.Context, flags uint32) (fs.FileHandle, uint32, syscall.Errno) { 182 | r, err := n.getRealPath() 183 | if err != nil { 184 | log.G(ctx).WithFields(logrus.Fields{ 185 | "_s": n.instance.stack, 186 | "_r": r, 187 | }).Error("open") 188 | return nil, 0, syscall.ENODATA 189 | } 190 | 191 | access := time.Now() 192 | if !n.IsReady() { 193 | n.WaitForReady() 194 | } 195 | complete := time.Now() 196 | name := n.GetName() 197 | n.log(name, access, complete) 198 | 199 | log.G(ctx).WithFields(logrus.Fields{ 200 | "f": name, 201 | "_s": n.instance.stack, 202 | "_r": r, 203 | }).Trace("open") 204 | 205 | fd, err := syscall.Open(r, int(flags), 0) 206 | if err != nil { 207 | return nil, 0, fs.ToErrno(err) 208 | } 209 | return fs.NewLoopbackFile(fd), fuse.FOPEN_KEEP_CACHE, 0 210 | } 211 | 212 | var _ = (fs.NodeFsyncer)((*StarlightFsNode)(nil)) 213 | 214 | func (n *StarlightFsNode) Fsync(ctx context.Context, f fs.FileHandle, flags uint32) syscall.Errno { 215 | r, err := n.getRealPath() 216 | if err != nil { 217 | log.G(ctx).WithFields(logrus.Fields{ 218 | "_s": n.instance.stack, 219 | "_r": r, 220 | }).Error("fsync") 221 | return syscall.ENODATA 222 | } 223 | 224 | access := time.Now() 225 | if !n.IsReady() { 226 | n.WaitForReady() 227 | } 228 | complete := time.Now() 229 | name := n.GetName() 230 | n.log(name, access, complete) 231 | 232 | log.G(ctx).WithFields(logrus.Fields{ 233 | "f": name, 234 | "_s": n.instance.stack, 235 | "_r": r, 236 | }).Trace("fsync") 237 | 238 | fd, err := syscall.Open(r, int(flags), 0) 239 | if err != nil { 240 | return fs.ToErrno(err) 241 | } 242 | f = fs.NewLoopbackFile(fd) 243 | return f.(fs.FileFsyncer).Fsync(ctx, flags) 244 | 245 | } 246 | -------------------------------------------------------------------------------- /cmd/starlight-daemon/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The starlight Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | 16 | file created by maverick in 2021 17 | */ 18 | 19 | package main 20 | 21 | import ( 22 | "fmt" 23 | "github.com/containerd/containerd/log" 24 | "github.com/mc256/starlight/client" 25 | "github.com/mc256/starlight/util" 26 | "github.com/sirupsen/logrus" 27 | "github.com/urfave/cli/v2" 28 | "os" 29 | "os/signal" 30 | "syscall" 31 | ) 32 | 33 | func init() { 34 | cli.VersionPrinter = func(c *cli.Context) { 35 | fmt.Println(c.App.Name, c.App.Version) 36 | } 37 | } 38 | 39 | func main() { 40 | app := New() 41 | if err := app.Run(os.Args); err != nil { 42 | _, _ = fmt.Fprintf(os.Stderr, "starlight-daemon: \n%v\n", err) 43 | os.Exit(1) 44 | } 45 | } 46 | 47 | func New() *cli.App { 48 | app := cli.NewApp() 49 | cfg := client.NewConfig() 50 | 51 | app.Name = "starlight-daemon" 52 | app.Version = util.Version 53 | app.Usage = `Daemon for faster container-based application deployment. 54 | 55 | This is a plugin that can be used with containerd to accelerate container-based application deployments. 56 | To enable the plugin, please add plugin configurations to containerd config.toml. 57 | You can also verify the plugin status by running "ctr plugins ls". 58 | For more information, please refer to the README.md file in the project repository. 59 | https://github.com/mc256/starlight 60 | 61 | *CLI options will override values in the config file if specified.` 62 | app.Description = fmt.Sprintf("\n%s\n", app.Usage) 63 | 64 | app.EnableBashCompletion = true 65 | app.Flags = []cli.Flag{ 66 | &cli.StringFlag{ 67 | Name: "config", 68 | DefaultText: "/etc/starlight/starlight-daemon.json", 69 | Aliases: []string{"c"}, 70 | EnvVars: []string{"STARLIGHT_DAEMON_CONFIG"}, 71 | Usage: "json configuration file. CLI parameter will override values in the config file if specified", 72 | Required: false, 73 | }, 74 | &cli.StringFlag{ 75 | Name: "log-level", 76 | Aliases: []string{"l"}, 77 | EnvVars: []string{"LOG_LEVEL"}, 78 | DefaultText: cfg.LogLevel, 79 | Usage: "Choose one log level (fatal, error, warning, info, debug, trace)", 80 | Required: false, 81 | }, 82 | // ---- 83 | &cli.StringFlag{ 84 | Name: "metadata", 85 | DefaultText: cfg.Metadata, 86 | Aliases: []string{"m"}, 87 | EnvVars: []string{"METADATA"}, 88 | Usage: "path to store image metadata", 89 | Required: false, 90 | }, 91 | &cli.StringFlag{ 92 | Name: "socket", 93 | DefaultText: cfg.Socket, 94 | Usage: "gRPC socket address", 95 | Required: false, 96 | }, 97 | &cli.StringFlag{ 98 | Name: "default", 99 | DefaultText: cfg.DefaultProxy, 100 | Aliases: []string{"d"}, 101 | Usage: "name of the default proxy", 102 | }, 103 | &cli.StringFlag{ 104 | Name: "fs-root", 105 | DefaultText: cfg.FileSystemRoot, 106 | Aliases: []string{"fs"}, 107 | Usage: "path to store uncompress image layers", 108 | Required: false, 109 | }, 110 | &cli.StringFlag{ 111 | Name: "id", 112 | DefaultText: cfg.ClientId, 113 | Usage: "identifier for the client", 114 | Required: false, 115 | }, 116 | // ---- 117 | &cli.StringSliceFlag{ 118 | Name: "proxy", 119 | Aliases: []string{"p"}, 120 | Usage: "proxy of the configuration use comma (',') to separate components, and use another tag for other proxies, name,protocol,address,username,password", 121 | Required: false, 122 | DefaultText: "starlight-shared,https,starlight.yuri.moe,,", 123 | }, 124 | } 125 | app.Action = func(c *cli.Context) error { 126 | return DefaultAction(c, cfg) 127 | } 128 | 129 | return app 130 | } 131 | 132 | func DefaultAction(context *cli.Context, cfg *client.Configuration) (err error) { 133 | var ( 134 | p string 135 | ne bool 136 | ) 137 | 138 | config := context.String("config") 139 | cfg, p, ne, err = client.LoadConfig(config) 140 | 141 | if l := context.String("log-level"); l != "" { 142 | cfg.LogLevel = l 143 | } 144 | c := util.ConfigLoggerWithLevel(cfg.LogLevel) 145 | log.G(c). 146 | WithField("version", util.Version). 147 | Info("starlight-daemon") 148 | 149 | if err != nil { 150 | log.G(c).WithFields(logrus.Fields{ 151 | "log": cfg.LogLevel, 152 | "path": p, 153 | "new": ne, 154 | }). 155 | WithError(err). 156 | Fatal("failed to load configuration") 157 | } else { 158 | log.G(c).WithFields(logrus.Fields{ 159 | "log": cfg.LogLevel, 160 | "path": p, 161 | "new": ne, 162 | }). 163 | Info("loaded configuration") 164 | } 165 | 166 | if id := context.String("id"); id != "" { 167 | cfg.ClientId = id 168 | } 169 | if m := context.String("metadata"); m != "" { 170 | cfg.Metadata = m 171 | } 172 | if s := context.String("socket"); s != "" { 173 | cfg.Socket = s 174 | } 175 | if r := context.String("fs-root"); r != "" { 176 | cfg.FileSystemRoot = r 177 | } 178 | if d := context.String("default"); d != "" { 179 | cfg.DefaultProxy = d 180 | } 181 | parr := context.StringSlice("proxy") 182 | if len(parr) != 0 { 183 | for _, v := range parr { 184 | if k, vv, err := client.ParseProxyStrings(v); err != nil { 185 | log.G(c). 186 | WithError(err). 187 | Error("failed to parse proxy flag") 188 | } else { 189 | cfg.Proxies[k] = vv 190 | } 191 | } 192 | } 193 | 194 | // Client 195 | var slc *client.Client 196 | slc, err = client.NewClient(c, cfg) 197 | if err != nil { 198 | log.G(c). 199 | WithError(err). 200 | Fatal("failed to create starlight daemon client") 201 | os.Exit(1) 202 | return 203 | } 204 | 205 | // Snapshotter 206 | go func() { 207 | err = slc.InitSnapshotter() 208 | if err != nil { 209 | log.G(c). 210 | WithError(err). 211 | Fatal("failed to initialize snapshotter service") 212 | os.Exit(1) 213 | return 214 | } 215 | slc.StartSnapshotter() 216 | }() 217 | 218 | // Image Service 219 | go func() { 220 | err = slc.InitCLIServer() 221 | if err != nil { 222 | log.G(c). 223 | WithError(err). 224 | Fatal("failed to initialize CLI service") 225 | os.Exit(1) 226 | return 227 | } 228 | slc.StartCLIServer() 229 | 230 | }() 231 | 232 | wait := make(chan interface{}) 233 | si := make(chan os.Signal, 1) 234 | signal.Notify(si, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL) 235 | go func() { 236 | <-si 237 | slc.Close() 238 | close(wait) 239 | }() 240 | <-wait 241 | return nil 242 | } 243 | --------------------------------------------------------------------------------