├── assigner ├── .gitignore ├── Makefile ├── scripts │ ├── start_assigner │ └── peer_id_from_priv_key.go ├── config │ ├── init.go │ ├── policy.go │ ├── daemon.go │ ├── logging.go │ └── assignment.go ├── core │ └── order_test.go ├── main.go ├── server │ └── option.go └── command │ └── init.go ├── version.json ├── .gitignore ├── .github ├── workflows │ ├── go-test-config.json │ ├── tagpush.yml │ ├── releaser.yml │ ├── go-check.yml │ ├── go-test.yml │ ├── release-check.yml │ ├── cd.yml │ └── publish-ghcr.yml ├── dependabot.yml └── pull_request_template.md ├── doc ├── assigner.png ├── assigner.graffle └── config.md ├── deploy ├── manifests │ ├── base │ │ ├── storetheindex-single │ │ │ ├── identity.key │ │ │ ├── README.md │ │ │ ├── labels.yaml │ │ │ ├── service.yaml │ │ │ ├── kustomization.yaml │ │ │ └── deployment.yaml │ │ ├── promtail │ │ │ ├── auth.env │ │ │ ├── kustomization.yaml │ │ │ ├── rbac.yaml │ │ │ └── daemonset.yaml │ │ ├── lookout │ │ │ ├── kustomization.yaml │ │ │ └── deployment.yaml │ │ ├── k6-operator │ │ │ └── kustomization.yaml │ │ ├── monitoring │ │ │ └── kustomization.yaml │ │ ├── foundationdb │ │ │ ├── crds │ │ │ │ └── kustomization.yaml │ │ │ ├── fdbmeter │ │ │ │ ├── kustomization.yaml │ │ │ │ └── deployment.yaml │ │ │ └── namespaced-manager │ │ │ │ ├── kustomization.yaml │ │ │ │ └── rbac.yaml │ │ ├── dhfind │ │ │ ├── pdb.yaml │ │ │ ├── service.yaml │ │ │ ├── labels.yaml │ │ │ ├── kustomization.yaml │ │ │ └── deployment.yaml │ │ ├── caskadht │ │ │ ├── pdb.yaml │ │ │ ├── service.yaml │ │ │ ├── labels.yaml │ │ │ ├── kustomization.yaml │ │ │ └── deployment.yaml │ │ ├── cassette │ │ │ ├── pdb.yaml │ │ │ ├── service.yaml │ │ │ ├── labels.yaml │ │ │ ├── kustomization.yaml │ │ │ └── deployment.yaml │ │ ├── heyfil │ │ │ ├── kustomization.yaml │ │ │ ├── service.yaml │ │ │ └── deployment.yaml │ │ ├── indexstar │ │ │ ├── pdb.yaml │ │ │ ├── kustomization.yaml │ │ │ ├── service.yaml │ │ │ └── deployment.yaml │ │ ├── storetheindex │ │ │ ├── pdb.yaml │ │ │ ├── labels.yaml │ │ │ ├── indexer-service.yaml │ │ │ ├── kustomization.yaml │ │ │ └── indexer-statefulset.yaml │ │ ├── telemetry │ │ │ ├── pdb.yaml │ │ │ ├── kustomization.yaml │ │ │ ├── service.yaml │ │ │ ├── labels.yaml │ │ │ └── deployment.yaml │ │ ├── assigner │ │ │ ├── kustomization.yaml │ │ │ ├── service.yaml │ │ │ └── deployment.yaml │ │ ├── cert-manager │ │ │ └── kustomization.yaml │ │ ├── aws-ebs-csi-driver │ │ │ └── kustomization.yaml │ │ ├── external-dns │ │ │ └── kustomization.yaml │ │ ├── envoy │ │ │ ├── envoy-pdb.yaml │ │ │ ├── kustomization.yaml │ │ │ ├── envoy-config.yaml │ │ │ └── envoy-deployment.yaml │ │ ├── external-snapshotter │ │ │ ├── kustomization.yaml │ │ │ └── snapshot-controller │ │ │ │ └── kustomization.yaml │ │ ├── ingress-nginx │ │ │ └── kustomization.yaml │ │ ├── cluster-autoscaler │ │ │ └── kustomization.yaml │ │ └── flux-system │ │ │ └── kustomization.yaml │ └── prod │ │ └── us-east-2 │ │ ├── tenant │ │ └── ipni │ │ │ ├── namespace.yaml │ │ │ ├── kustomization.yaml │ │ │ ├── sf-ingress.yaml │ │ │ ├── sf2-ingress.yaml │ │ │ └── berg-ingress.yaml │ │ └── cluster │ │ ├── promtail │ │ ├── namespace.yaml │ │ ├── .sops.yaml │ │ ├── kustomization.yaml │ │ └── auth.env │ │ ├── external-dns │ │ ├── namespace.yaml │ │ ├── sa-patch.yaml │ │ ├── kustomization.yaml │ │ └── deploy-patch.yaml │ │ ├── kube-system │ │ ├── volume-snapshot-class.yaml │ │ ├── kustomization.yaml │ │ ├── io2-storage-class.yaml │ │ └── gp3-storage-class.yaml │ │ ├── cluster-autoscaler │ │ ├── kustomization.yaml │ │ └── patch.yaml │ │ ├── external-snapshotter │ │ └── kustomization.yaml │ │ ├── cert-manager │ │ ├── kustomization.yaml │ │ ├── issuer.yaml │ │ └── sa-patch.yaml │ │ ├── flux-system │ │ ├── kust-ctrlr-sa-patch.yaml │ │ ├── kustomization.yaml │ │ └── cluster-cd.yaml │ │ ├── aws-ebs-csi-driver │ │ ├── kustomization.yaml │ │ └── patch.yaml │ │ ├── ingress-nginx │ │ ├── kustomization.yaml │ │ ├── pod-monitor.yaml │ │ └── patch.yaml │ │ ├── kustomization.yaml │ │ └── monitoring │ │ ├── kustomization.yaml │ │ ├── patch.yaml │ │ └── prometheus-patch.yaml └── infrastructure │ ├── prod │ └── us-east-2 │ │ ├── versions.tf │ │ ├── route53.tf │ │ ├── main.tf │ │ ├── backend.tf │ │ ├── ebs_csi_driver.tf │ │ ├── s3_adstore_berg.tf │ │ ├── outputs.tf │ │ ├── external_dns.tf │ │ ├── s3_adstore.tf │ │ ├── autoscaler.tf │ │ ├── cert_manager.tf │ │ └── monitoring.tf │ ├── common │ ├── ecr.tf │ ├── backend.tf │ ├── main.tf │ ├── outputs.tf │ ├── pl-grafana.tf │ ├── .terraform.lock.hcl │ └── github_actions.tf │ └── modules │ └── ecr │ ├── variables.tf │ └── main.tf ├── peerutil └── doc.go ├── carstore ├── error.go ├── const.go └── option.go ├── FUNDING.json ├── command ├── loadtest.go ├── gc.go ├── loadgen │ ├── pseudorand.go │ └── config_test.go ├── assigner.go ├── init_test.go ├── log.go ├── datastore_test.go └── flags.go ├── test └── load │ ├── run.sh │ ├── load.py │ └── README.md ├── admin ├── model │ └── model.go └── client │ └── option.go ├── Makefile ├── internal ├── freeze │ ├── device_unix.go │ └── device_windows.go ├── revision │ └── revision.go ├── registry │ ├── errors.go │ ├── option.go │ └── sequences.go ├── ingest │ ├── error.go │ ├── seg_sync_test.go │ ├── hamt_ingest_test.go │ ├── mirror.go │ └── invalid_mh_ingest_test.go ├── metrics │ └── pprof │ │ └── pprof.go ├── libp2pserver │ └── message_writer.go └── httpserver │ └── util.go ├── fsutil ├── disk │ ├── usage_test.go │ ├── usage_openbsd.go │ ├── usage_unix.go │ ├── usage_windows.go │ └── usage.go └── fsutil.go ├── Dockerfile ├── config ├── reverse_index.go ├── peering.go ├── config_test.go ├── bootstrap_test.go ├── logging.go ├── mirror.go ├── datastore.go ├── init_test.go ├── finder.go ├── addresses.go ├── init.go ├── policy.go ├── types.go └── identity.go ├── server ├── find │ ├── index.html │ ├── header.go │ ├── options.go │ └── cached_stats.go ├── admin │ ├── options.go │ └── config_handler.go └── ingest │ └── options.go ├── filestore ├── option.go ├── config.go └── filestore.go ├── scripts ├── peer_id_from_priv_key │ └── main.go ├── gen_identity │ └── gen-identity.go ├── cross_announce │ └── main.go └── compare_providers │ └── main.go ├── version.go ├── main.go └── rate └── map.go /assigner/.gitignore: -------------------------------------------------------------------------------- 1 | /assigner 2 | -------------------------------------------------------------------------------- /version.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "v0.9.2" 3 | } 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /ipnigc 2 | /storetheindex 3 | *~ 4 | *.car 5 | *.out 6 | -------------------------------------------------------------------------------- /.github/workflows/go-test-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "skip32bit": true 3 | } 4 | -------------------------------------------------------------------------------- /doc/assigner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipni/storetheindex/HEAD/doc/assigner.png -------------------------------------------------------------------------------- /deploy/manifests/base/storetheindex-single/identity.key: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /doc/assigner.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipni/storetheindex/HEAD/doc/assigner.graffle -------------------------------------------------------------------------------- /deploy/manifests/base/promtail/auth.env: -------------------------------------------------------------------------------- 1 | PROMTAIL_CLIENT_USERNAME= 2 | PROMTAIL_CLIENT_PASSWORD= 3 | -------------------------------------------------------------------------------- /peerutil/doc.go: -------------------------------------------------------------------------------- 1 | // Package peerutil provides utilities around peer ID values. 2 | package peerutil 3 | -------------------------------------------------------------------------------- /deploy/infrastructure/prod/us-east-2/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.1.7" 3 | } 4 | -------------------------------------------------------------------------------- /carstore/error.go: -------------------------------------------------------------------------------- 1 | package carstore 2 | 3 | import "errors" 4 | 5 | var ErrHAMT = errors.New("hamt entries not supported") 6 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/tenant/ipni/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | 4 | metadata: 5 | name: ipni 6 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/promtail/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | 4 | metadata: 5 | name: promtail 6 | -------------------------------------------------------------------------------- /deploy/infrastructure/common/ecr.tf: -------------------------------------------------------------------------------- 1 | module "ecr_ue2" { 2 | source = "../modules/ecr" 3 | 4 | repositories = [] 5 | tags = local.tags 6 | } 7 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/external-dns/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | 4 | metadata: 5 | name: external-dns 6 | -------------------------------------------------------------------------------- /deploy/infrastructure/prod/us-east-2/route53.tf: -------------------------------------------------------------------------------- 1 | resource "aws_route53_zone" "prod_external" { 2 | name = "${local.environment_name}.cid.contact" 3 | } 4 | -------------------------------------------------------------------------------- /deploy/manifests/base/lookout/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - deployment.yaml 6 | 7 | commonLabels: 8 | app: lookout 9 | -------------------------------------------------------------------------------- /deploy/infrastructure/common/backend.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | backend "s3" { 3 | bucket = "sti-dev-terraform-state" 4 | key = "sti/common/terraform.tfstate" 5 | region = "us-east-2" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /deploy/infrastructure/prod/us-east-2/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = "us-east-2" 3 | allowed_account_ids = ["407967248065"] 4 | default_tags { 5 | tags = local.tags 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /deploy/manifests/base/k6-operator/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - github.com/grafana/k6-operator/config/default?ref=v0.0.8 6 | -------------------------------------------------------------------------------- /deploy/manifests/base/monitoring/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - github.com/prometheus-operator/kube-prometheus?ref=v0.10.0 6 | -------------------------------------------------------------------------------- /deploy/infrastructure/prod/us-east-2/backend.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | backend "s3" { 3 | bucket = "sti-dev-terraform-state" 4 | key = "sti/prod/us-east-2/terraform.tfstate" 5 | region = "us-east-2" 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /deploy/manifests/base/foundationdb/crds/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - github.com/FoundationDB/fdb-kubernetes-operator/config/crd?ref=v1.20.1 -------------------------------------------------------------------------------- /deploy/manifests/base/dhfind/pdb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: policy/v1 2 | kind: PodDisruptionBudget 3 | metadata: 4 | name: dhfind 5 | spec: 6 | maxUnavailable: 1 7 | selector: 8 | matchLabels: 9 | app: dhfind 10 | -------------------------------------------------------------------------------- /deploy/manifests/base/caskadht/pdb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: policy/v1 2 | kind: PodDisruptionBudget 3 | metadata: 4 | name: caskadht 5 | spec: 6 | maxUnavailable: 1 7 | selector: 8 | matchLabels: 9 | app: caskadht 10 | -------------------------------------------------------------------------------- /deploy/manifests/base/cassette/pdb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: policy/v1 2 | kind: PodDisruptionBudget 3 | metadata: 4 | name: cassette 5 | spec: 6 | maxUnavailable: 1 7 | selector: 8 | matchLabels: 9 | app: cassette 10 | -------------------------------------------------------------------------------- /deploy/manifests/base/heyfil/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - deployment.yaml 6 | - service.yaml 7 | 8 | commonLabels: 9 | app: heyfil 10 | -------------------------------------------------------------------------------- /deploy/manifests/base/indexstar/pdb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: policy/v1 2 | kind: PodDisruptionBudget 3 | metadata: 4 | name: indexstar 5 | spec: 6 | maxUnavailable: 1 7 | selector: 8 | matchLabels: 9 | app: indexstar 10 | -------------------------------------------------------------------------------- /deploy/manifests/base/storetheindex/pdb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: policy/v1 2 | kind: PodDisruptionBudget 3 | metadata: 4 | name: indexer 5 | spec: 6 | maxUnavailable: 1 7 | selector: 8 | matchLabels: 9 | app: indexer 10 | -------------------------------------------------------------------------------- /deploy/manifests/base/telemetry/pdb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: policy/v1 2 | kind: PodDisruptionBudget 3 | metadata: 4 | name: telemetry 5 | spec: 6 | maxUnavailable: 1 7 | selector: 8 | matchLabels: 9 | app: telemetry 10 | -------------------------------------------------------------------------------- /deploy/infrastructure/common/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = "us-east-1" 3 | alias = "use1" 4 | allowed_account_ids = [local.account_id] 5 | default_tags { 6 | tags = local.tags 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /deploy/manifests/base/assigner/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - deployment.yaml 6 | - service.yaml 7 | 8 | commonLabels: 9 | app: assigner 10 | -------------------------------------------------------------------------------- /deploy/manifests/base/cert-manager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - https://github.com/cert-manager/cert-manager/releases/download/v1.7.2/cert-manager.yaml 6 | -------------------------------------------------------------------------------- /deploy/manifests/base/storetheindex-single/README.md: -------------------------------------------------------------------------------- 1 | # Single Indexer node 2 | 3 | PVC volume with name `data` must be specified by the downstream overlay. 4 | By default the data is stored in temporary volumes and will be lost on restart. -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/kube-system/volume-snapshot-class.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: snapshot.storage.k8s.io/v1 2 | kind: VolumeSnapshotClass 3 | metadata: 4 | name: csi-aws-vsc 5 | driver: ebs.csi.aws.com 6 | deletionPolicy: Delete 7 | -------------------------------------------------------------------------------- /deploy/manifests/base/aws-ebs-csi-driver/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - github.com/kubernetes-sigs/aws-ebs-csi-driver/deploy/kubernetes/overlays/stable?ref=v1.12.1 6 | -------------------------------------------------------------------------------- /deploy/manifests/base/dhfind/service.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: dhfind 5 | spec: 6 | ports: 7 | - name: http 8 | port: 80 9 | targetPort: http 10 | selector: 11 | app: dhfind 12 | -------------------------------------------------------------------------------- /FUNDING.json: -------------------------------------------------------------------------------- 1 | { 2 | "drips": { 3 | "ethereum": { 4 | "ownedBy": "0xcB2E7F53F446AFA859192194Fc2E7eeCA20f6285" 5 | }, 6 | "filecoin": { 7 | "ownedBy": "0xcB2E7F53F446AFA859192194Fc2E7eeCA20f6285" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /carstore/const.go: -------------------------------------------------------------------------------- 1 | package carstore 2 | 3 | // File suffixes. 4 | const ( 5 | CarFileSuffix = ".car" 6 | GzipFileSuffix = ".gz" 7 | HeadFileSuffix = ".head" 8 | ) 9 | 10 | // Compression methods. 11 | const ( 12 | Gzip = "gzip" 13 | ) 14 | -------------------------------------------------------------------------------- /deploy/manifests/base/caskadht/service.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: caskadht 5 | spec: 6 | ports: 7 | - name: http 8 | port: 80 9 | targetPort: http 10 | selector: 11 | app: caskadht 12 | -------------------------------------------------------------------------------- /deploy/manifests/base/cassette/service.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: cassette 5 | spec: 6 | ports: 7 | - name: http 8 | port: 80 9 | targetPort: http 10 | selector: 11 | app: cassette 12 | -------------------------------------------------------------------------------- /deploy/manifests/base/telemetry/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - deployment.yaml 6 | - service.yaml 7 | - pdb.yaml 8 | 9 | transformers: 10 | - labels.yaml 11 | -------------------------------------------------------------------------------- /deploy/manifests/base/telemetry/service.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: telemetry 5 | spec: 6 | ports: 7 | - name: http 8 | port: 80 9 | targetPort: http 10 | selector: 11 | app: telemetry 12 | -------------------------------------------------------------------------------- /deploy/manifests/base/external-dns/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | namespace: external-dns 5 | 6 | resources: 7 | - https://github.com/kubernetes-sigs/external-dns.git//kustomize?ref=v0.11.0 8 | -------------------------------------------------------------------------------- /deploy/manifests/base/indexstar/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - deployment.yaml 6 | - service.yaml 7 | - pdb.yaml 8 | 9 | commonLabels: 10 | app: indexstar 11 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/tenant/ipni/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | namespace: ipni 4 | resources: 5 | - berg-ingress.yaml 6 | - sf-ingress.yaml 7 | - sf2-ingress.yaml 8 | - namespace.yaml -------------------------------------------------------------------------------- /deploy/manifests/base/envoy/envoy-pdb.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: policy/v1 2 | kind: PodDisruptionBudget 3 | metadata: 4 | name: envoy 5 | labels: 6 | app: envoy 7 | spec: 8 | selector: 9 | matchLabels: 10 | app: envoy 11 | maxUnavailable: 1 12 | -------------------------------------------------------------------------------- /deploy/manifests/base/external-snapshotter/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - github.com/kubernetes-csi/external-snapshotter/client/config/crd?ref=v6.1.0 6 | - snapshot-controller 7 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/cluster-autoscaler/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - ../../../../base/cluster-autoscaler 6 | 7 | patchesStrategicMerge: 8 | - patch.yaml 9 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/external-dns/sa-patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: external-dns 5 | annotations: 6 | eks.amazonaws.com/role-arn: "arn:aws:iam::407967248065:role/prod/us-east-2/external_dns" 7 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/external-snapshotter/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - ../../../../base/external-snapshotter 6 | 7 | replicas: 8 | - count: 0 9 | name: snapshot-controller -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/kube-system/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - aws-auth.yaml 6 | - io2-storage-class.yaml 7 | - gp3-storage-class.yaml 8 | - volume-snapshot-class.yaml 9 | -------------------------------------------------------------------------------- /deploy/manifests/base/foundationdb/fdbmeter/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - deployment.yaml 6 | 7 | images: 8 | - name: fdbmeter 9 | newName: ghcr.io/masih/fdbmeter 10 | newTag: v0.0.4 -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/cert-manager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - ../../../../base/cert-manager 6 | - issuer.yaml 7 | 8 | patchesStrategicMerge: 9 | - sa-patch.yaml 10 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 2 3 | updates: 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | interval: "weekly" 8 | - package-ecosystem: "gomod" 9 | directory: "/" 10 | schedule: 11 | interval: "weekly" 12 | -------------------------------------------------------------------------------- /deploy/manifests/base/assigner/service.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: assigner 5 | spec: 6 | ports: 7 | - name: http 8 | port: 3001 9 | targetPort: http 10 | selector: 11 | app: assigner 12 | type: ClusterIP 13 | clusterIP: None 14 | -------------------------------------------------------------------------------- /deploy/manifests/base/indexstar/service.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: indexstar 5 | spec: 6 | ports: 7 | - name: http 8 | port: 8080 9 | targetPort: http 10 | selector: 11 | app: indexstar 12 | type: ClusterIP 13 | clusterIP: None 14 | -------------------------------------------------------------------------------- /deploy/manifests/base/ingress-nginx/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | namespace: ingress-nginx 5 | 6 | resources: 7 | - https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.2/deploy/static/provider/aws/deploy.yaml 8 | -------------------------------------------------------------------------------- /deploy/manifests/base/external-snapshotter/snapshot-controller/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | namespace: kube-system 5 | 6 | resources: 7 | - github.com/kubernetes-csi/external-snapshotter/deploy/kubernetes/snapshot-controller?ref=v6.1.0 8 | -------------------------------------------------------------------------------- /command/loadtest.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "github.com/urfave/cli/v2" 5 | ) 6 | 7 | var LoadtestCmd = &cli.Command{ 8 | Name: "loadtest", 9 | Usage: "Perform load testing with an indexer", 10 | Subcommands: []*cli.Command{ 11 | loadGenCmd, 12 | loadGenVerifyCmd, 13 | }, 14 | } 15 | -------------------------------------------------------------------------------- /deploy/manifests/base/storetheindex/labels.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: builtin 2 | kind: LabelTransformer 3 | metadata: 4 | name: labels 5 | labels: 6 | app.kubernetes.io/part-of: storetheindex 7 | app.kubernetes.io/managed-by: kustomization 8 | fieldSpecs: 9 | - path: metadata/labels 10 | create: true 11 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/flux-system/kust-ctrlr-sa-patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: kustomize-controller 5 | namespace: flux-system 6 | annotations: 7 | eks.amazonaws.com/role-arn: "arn:aws:iam::407967248065:role/prod/us-east-2/kustomize_controller" 8 | -------------------------------------------------------------------------------- /test/load/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | NUM_WORKERS=4 3 | START=1 4 | # Start locust workers 5 | while [[ $i -le $NUM_WORKERS ]] 6 | do 7 | echo "Starting worker$number" 8 | locust -f load.py --worker & 9 | ((i = i + 1)) 10 | done 11 | # Start locust master 12 | locust -f load.py --master 13 | -------------------------------------------------------------------------------- /deploy/manifests/base/heyfil/service.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: heyfil 5 | spec: 6 | ports: 7 | - name: metrics 8 | port: 8080 9 | targetPort: metrics 10 | - name: api 11 | port: 8081 12 | targetPort: api 13 | selector: 14 | app: heyfil 15 | -------------------------------------------------------------------------------- /deploy/manifests/base/storetheindex-single/labels.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: builtin 2 | kind: LabelTransformer 3 | metadata: 4 | name: labels 5 | labels: 6 | app.kubernetes.io/part-of: storetheindex 7 | app.kubernetes.io/managed-by: kustomization 8 | fieldSpecs: 9 | - path: metadata/labels 10 | create: true 11 | -------------------------------------------------------------------------------- /deploy/manifests/base/cluster-autoscaler/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - https://raw.githubusercontent.com/kubernetes/autoscaler/cluster-autoscaler-chart-9.16.2/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yaml 6 | -------------------------------------------------------------------------------- /deploy/manifests/base/envoy/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - envoy-deployment.yaml 6 | - envoy-pdb.yaml 7 | 8 | configMapGenerator: 9 | - name: envoy-config 10 | behavior: create 11 | files: 12 | - envoy-config.yaml 13 | -------------------------------------------------------------------------------- /deploy/manifests/base/foundationdb/namespaced-manager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - deployment.yaml 6 | - rbac.yaml 7 | 8 | images: 9 | - name: manager 10 | newName: foundationdb/fdb-kubernetes-operator 11 | newTag: v1.20.1 -------------------------------------------------------------------------------- /deploy/manifests/base/envoy/envoy-config.yaml: -------------------------------------------------------------------------------- 1 | # Envoy config; see: https://www.envoyproxy.io/docs/envoy/latest/configuration/configuration 2 | # Override this file in your kustomization overlay; example: 3 | # configMapGenerator: 4 | # - name: envoy-config 5 | # behavior: replace 6 | # files: 7 | # - envoy-config.yaml -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/aws-ebs-csi-driver/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - ../../../../base/aws-ebs-csi-driver 6 | 7 | patchesStrategicMerge: 8 | - patch.yaml 9 | 10 | replicas: 11 | - count: 0 12 | name: ebs-csi-controller -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/promtail/.sops.yaml: -------------------------------------------------------------------------------- 1 | creation_rules: 2 | - path_regex: '.+\.env' 3 | kms: 'arn:aws:kms:us-east-2:407967248065:alias/prod/us-east-2/cluster' 4 | - path_regex: '.+\.y(a)?ml' 5 | encrypted_regex: '^(data|stringData)$' 6 | kms: 'arn:aws:kms:us-east-2:407967248065:alias/prod/us-east-2/cluster' 7 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/external-dns/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | namespace: external-dns 5 | 6 | resources: 7 | - ../../../../base/external-dns 8 | - namespace.yaml 9 | 10 | patchesStrategicMerge: 11 | - deploy-patch.yaml 12 | - sa-patch.yaml 13 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/ingress-nginx/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - ../../../../base/ingress-nginx 6 | - pod-monitor.yaml 7 | 8 | patchesStrategicMerge: 9 | - patch.yaml 10 | 11 | replicas: 12 | - name: ingress-nginx-controller 13 | count: 1 14 | -------------------------------------------------------------------------------- /deploy/manifests/base/caskadht/labels.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: builtin 2 | kind: LabelTransformer 3 | metadata: 4 | name: labels 5 | labels: 6 | app.kubernetes.io/name: caskadht 7 | app.kubernetes.io/component: routing 8 | app.kubernetes.io/part-of: ipni 9 | app.kubernetes.io/managed-by: kustomization 10 | fieldSpecs: 11 | - path: metadata/labels 12 | create: true 13 | -------------------------------------------------------------------------------- /deploy/manifests/base/cassette/labels.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: builtin 2 | kind: LabelTransformer 3 | metadata: 4 | name: labels 5 | labels: 6 | app.kubernetes.io/name: cassette 7 | app.kubernetes.io/component: routing 8 | app.kubernetes.io/part-of: ipni 9 | app.kubernetes.io/managed-by: kustomization 10 | fieldSpecs: 11 | - path: metadata/labels 12 | create: true 13 | -------------------------------------------------------------------------------- /deploy/manifests/base/dhfind/labels.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: builtin 2 | kind: LabelTransformer 3 | metadata: 4 | name: labels 5 | labels: 6 | app.kubernetes.io/name: dhfind 7 | app.kubernetes.io/component: routing 8 | app.kubernetes.io/part-of: ipni 9 | app.kubernetes.io/managed-by: kustomization 10 | fieldSpecs: 11 | - path: metadata/labels 12 | create: true 13 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - ingress-nginx 6 | - kube-system 7 | - flux-system 8 | - external-dns 9 | - cert-manager 10 | - cluster-autoscaler 11 | - monitoring 12 | - aws-ebs-csi-driver 13 | - promtail 14 | - external-snapshotter 15 | -------------------------------------------------------------------------------- /deploy/manifests/base/telemetry/labels.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: builtin 2 | kind: LabelTransformer 3 | metadata: 4 | name: labels 5 | labels: 6 | app.kubernetes.io/name: telemetry 7 | app.kubernetes.io/component: routing 8 | app.kubernetes.io/part-of: ipni 9 | app.kubernetes.io/managed-by: kustomization 10 | fieldSpecs: 11 | - path: metadata/labels 12 | create: true 13 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/monitoring/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - ../../../../base/monitoring 6 | 7 | patchesStrategicMerge: 8 | - patch.yaml 9 | - prometheus-patch.yaml 10 | 11 | replicas: 12 | - name: grafana 13 | count: 0 14 | - name: prometheus-adapter 15 | count: 1 -------------------------------------------------------------------------------- /deploy/manifests/base/storetheindex-single/service.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: indexer 5 | spec: 6 | ports: 7 | - name: finder 8 | port: 3000 9 | targetPort: finder 10 | - name: ingest 11 | port: 3001 12 | targetPort: ingest 13 | selector: 14 | app: indexer-single 15 | type: ClusterIP 16 | clusterIP: None 17 | -------------------------------------------------------------------------------- /admin/model/model.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "github.com/libp2p/go-libp2p/core/peer" 5 | ) 6 | 7 | type Assigned struct { 8 | Publisher peer.ID 9 | Continued peer.ID `json:",omitempty"` 10 | } 11 | 12 | type Handoff struct { 13 | FrozenID peer.ID 14 | FrozenURL string 15 | } 16 | 17 | type Status struct { 18 | Frozen bool 19 | ID peer.ID 20 | Usage float64 21 | } 22 | -------------------------------------------------------------------------------- /.github/workflows/tagpush.yml: -------------------------------------------------------------------------------- 1 | name: Tag Push Checker 2 | 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | 8 | permissions: 9 | contents: read 10 | issues: write 11 | 12 | concurrency: 13 | group: ${{ github.workflow }}-${{ github.ref }} 14 | cancel-in-progress: true 15 | 16 | jobs: 17 | releaser: 18 | uses: ipdxco/unified-github-workflows/.github/workflows/tagpush.yml@v1.0 19 | -------------------------------------------------------------------------------- /.github/workflows/releaser.yml: -------------------------------------------------------------------------------- 1 | name: Releaser 2 | 3 | on: 4 | push: 5 | paths: [ 'version.json' ] 6 | workflow_dispatch: 7 | 8 | permissions: 9 | contents: write 10 | 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.sha }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | releaser: 17 | uses: ipdxco/unified-github-workflows/.github/workflows/releaser.yml@v1.0 18 | -------------------------------------------------------------------------------- /command/gc.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "github.com/ipni/storetheindex/command/gc" 5 | "github.com/urfave/cli/v2" 6 | ) 7 | 8 | var GCCmd = &cli.Command{ 9 | Name: "gc", 10 | Usage: "IPNI indexer garbage collector", 11 | Description: "Remove deleted indexes from value store.", 12 | Subcommands: []*cli.Command{ 13 | gc.DaemonCmd, 14 | gc.ProviderCmd, 15 | }, 16 | } 17 | -------------------------------------------------------------------------------- /assigner/Makefile: -------------------------------------------------------------------------------- 1 | BIN := assigner 2 | 3 | .PHONY: all build clean test 4 | 5 | all: vet test build 6 | 7 | build: 8 | go build -ldflags "-X 'github.com/ipni/storetheindex/internal/version.GitVersion=$(git rev-list -1 HEAD)'" 9 | 10 | install: 11 | go install 12 | 13 | lint: 14 | golangci-lint run 15 | 16 | test: 17 | go test ./... 18 | 19 | vet: 20 | go vet ./... 21 | 22 | clean: 23 | go clean 24 | -------------------------------------------------------------------------------- /command/loadgen/pseudorand.go: -------------------------------------------------------------------------------- 1 | package loadgen 2 | 3 | import ( 4 | "io" 5 | "math/rand" 6 | ) 7 | 8 | type pr struct { 9 | rand.Source 10 | } 11 | 12 | func newPseudoRandReaderFrom(src rand.Source) io.Reader { 13 | return &pr{src} 14 | } 15 | 16 | func (r *pr) Read(p []byte) (n int, err error) { 17 | for i := 0; i < len(p); i++ { 18 | p[i] = byte(r.Int63()) 19 | } 20 | return len(p), nil 21 | } 22 | -------------------------------------------------------------------------------- /deploy/manifests/base/dhfind/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - deployment.yaml 6 | - service.yaml 7 | - pdb.yaml 8 | 9 | transformers: 10 | - labels.yaml 11 | 12 | configMapGenerator: 13 | - name: dhfind-env-vars 14 | behavior: create 15 | literals: 16 | - GOLOG_LOG_LEVEL=INFO 17 | - GOLOG_LOG_FMT=json 18 | -------------------------------------------------------------------------------- /deploy/manifests/base/caskadht/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - deployment.yaml 6 | - service.yaml 7 | - pdb.yaml 8 | 9 | transformers: 10 | - labels.yaml 11 | 12 | configMapGenerator: 13 | - name: caskadht-env-vars 14 | behavior: create 15 | literals: 16 | - GOLOG_LOG_LEVEL=INFO 17 | - GOLOG_LOG_FMT=json 18 | -------------------------------------------------------------------------------- /deploy/manifests/base/cassette/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | resources: 5 | - deployment.yaml 6 | - service.yaml 7 | - pdb.yaml 8 | 9 | transformers: 10 | - labels.yaml 11 | 12 | configMapGenerator: 13 | - name: cassette-env-vars 14 | behavior: create 15 | literals: 16 | - GOLOG_LOG_LEVEL=INFO 17 | - GOLOG_LOG_FMT=json 18 | -------------------------------------------------------------------------------- /deploy/manifests/base/storetheindex/indexer-service.yaml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | metadata: 4 | name: indexer 5 | labels: 6 | app: indexer 7 | spec: 8 | ports: 9 | - name: finder 10 | port: 3000 11 | targetPort: finder 12 | - name: ingest 13 | port: 3001 14 | targetPort: ingest 15 | selector: 16 | app: indexer 17 | type: ClusterIP 18 | clusterIP: None 19 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/flux-system/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | namespace: flux-system 5 | 6 | resources: 7 | - ../../../../base/flux-system 8 | - cluster-cd.yaml 9 | 10 | patchesStrategicMerge: 11 | - kust-ctrlr-sa-patch.yaml 12 | 13 | replicas: 14 | - name: helm-controller 15 | count: 0 16 | - name: notification-controller 17 | count: 0 -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BIN := storetheindex 2 | 3 | .PHONY: all build clean test 4 | 5 | all: vet test build 6 | 7 | build: 8 | go build 9 | 10 | docker: Dockerfile clean 11 | docker build . --force-rm -f Dockerfile -t storetheindex:$(shell git rev-parse --short HEAD) 12 | 13 | install: 14 | go install 15 | 16 | lint: 17 | golangci-lint run 18 | 19 | test: 20 | go test ./... 21 | 22 | vet: 23 | go vet ./... 24 | 25 | clean: 26 | go clean 27 | -------------------------------------------------------------------------------- /.github/workflows/go-check.yml: -------------------------------------------------------------------------------- 1 | name: Go Checks 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: ["main"] 7 | workflow_dispatch: 8 | 9 | permissions: 10 | contents: read 11 | 12 | concurrency: 13 | group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} 14 | cancel-in-progress: true 15 | 16 | jobs: 17 | go-check: 18 | uses: ipdxco/unified-github-workflows/.github/workflows/go-check.yml@v1.0 19 | -------------------------------------------------------------------------------- /command/assigner.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "github.com/ipni/storetheindex/assigner/command" 5 | "github.com/urfave/cli/v2" 6 | ) 7 | 8 | var AssignerCmd = &cli.Command{ 9 | Name: "assigner", 10 | Usage: "Start a network indexer assigner service daemon", 11 | Description: "The assigner service is responsible for assigning content advertisement publishers to indexers.", 12 | Subcommands: []*cli.Command{ 13 | command.DaemonCmd, 14 | command.InitCmd, 15 | }, 16 | } 17 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/cert-manager/issuer.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cert-manager.io/v1 2 | kind: ClusterIssuer 3 | metadata: 4 | name: letsencrypt 5 | spec: 6 | acme: 7 | server: https://acme-v02.api.letsencrypt.org/directory 8 | email: bedrock@protocol.ai 9 | privateKeySecretRef: 10 | name: letsencrypt 11 | solvers: 12 | - dns01: 13 | route53: 14 | region: us-east-2 15 | selector: 16 | dnsZones: 17 | - prod.cid.contact 18 | -------------------------------------------------------------------------------- /deploy/manifests/base/foundationdb/fdbmeter/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | labels: 5 | app: fdbmeter 6 | name: fdbmeter 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | app: fdbmeter 12 | template: 13 | metadata: 14 | labels: 15 | app: fdbmeter 16 | spec: 17 | containers: 18 | - name: fdbmeter 19 | image: fdbmeter 20 | ports: 21 | - containerPort: 40080 22 | name: http -------------------------------------------------------------------------------- /.github/workflows/go-test.yml: -------------------------------------------------------------------------------- 1 | name: Go Test 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: ["main"] 7 | workflow_dispatch: 8 | 9 | permissions: 10 | contents: read 11 | 12 | concurrency: 13 | group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} 14 | cancel-in-progress: true 15 | 16 | jobs: 17 | go-test: 18 | uses: ipdxco/unified-github-workflows/.github/workflows/go-test.yml@v1.0 19 | with: 20 | go-versions: '["this"]' 21 | -------------------------------------------------------------------------------- /.github/workflows/release-check.yml: -------------------------------------------------------------------------------- 1 | name: Release Checker 2 | 3 | on: 4 | pull_request_target: 5 | paths: [ 'version.json' ] 6 | types: [ opened, synchronize, reopened, labeled, unlabeled ] 7 | workflow_dispatch: 8 | 9 | permissions: 10 | contents: write 11 | pull-requests: write 12 | 13 | concurrency: 14 | group: ${{ github.workflow }}-${{ github.ref }} 15 | cancel-in-progress: true 16 | 17 | jobs: 18 | release-check: 19 | uses: ipdxco/unified-github-workflows/.github/workflows/release-check.yml@v1.0 20 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/cert-manager/sa-patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: cert-manager 5 | namespace: cert-manager 6 | annotations: 7 | eks.amazonaws.com/role-arn: "arn:aws:iam::407967248065:role/prod/us-east-2/cert_manager" 8 | 9 | --- 10 | apiVersion: v1 11 | kind: ServiceAccount 12 | metadata: 13 | name: cert-manager-webhook 14 | namespace: cert-manager 15 | annotations: 16 | eks.amazonaws.com/role-arn: "arn:aws:iam::407967248065:role/prod/us-east-2/cert_manager" 17 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/ingress-nginx/pod-monitor.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: monitoring.coreos.com/v1 2 | kind: PodMonitor 3 | metadata: 4 | name: nginx 5 | namespace: ingress-nginx 6 | spec: 7 | selector: 8 | matchLabels: 9 | app.kubernetes.io/component: controller 10 | app.kubernetes.io/instance: ingress-nginx 11 | app.kubernetes.io/name: ingress-nginx 12 | namespaceSelector: 13 | matchNames: 14 | - ingress-nginx 15 | podMetricsEndpoints: 16 | - path: /metrics 17 | port: prometheus 18 | -------------------------------------------------------------------------------- /internal/freeze/device_unix.go: -------------------------------------------------------------------------------- 1 | //go:build freebsd || linux || darwin || openbsd || (aix && !cgo) 2 | 3 | package freeze 4 | 5 | import ( 6 | "fmt" 7 | "os" 8 | "syscall" 9 | ) 10 | 11 | func deviceNumber(path string) (int, error) { 12 | fi, err := os.Stat(path) 13 | if err != nil { 14 | return 0, fmt.Errorf("cannot stat %q: %w", path, err) 15 | } 16 | sysStatAny := fi.Sys() 17 | if sysStatAny == nil { 18 | return -1, nil 19 | } 20 | sysStat, ok := sysStatAny.(*syscall.Stat_t) 21 | if !ok { 22 | return -1, nil 23 | } 24 | return int(sysStat.Dev), nil 25 | } 26 | -------------------------------------------------------------------------------- /deploy/infrastructure/prod/us-east-2/ebs_csi_driver.tf: -------------------------------------------------------------------------------- 1 | module "ebs_csi_controller_role" { 2 | source = "registry.terraform.io/terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" 3 | version = "5.20.0" 4 | 5 | create_role = true 6 | 7 | role_name = "ebs_csi_controller" 8 | role_path = local.iam_path 9 | provider_url = module.eks.oidc_provider 10 | 11 | role_policy_arns = [ 12 | "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy", 13 | ] 14 | 15 | oidc_fully_qualified_subjects = [ 16 | "system:serviceaccount:kube-system:ebs-csi-controller-sa", 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /assigner/scripts/start_assigner: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # This script is the entrypoint for the assigner docker container 4 | set -e 5 | user=storetheindex 6 | repo="$ASSIGNER_PATH" 7 | 8 | if [ `id -u` -eq 0 ]; then 9 | echo "Changing user to $user" 10 | # ensure folder is writable 11 | su-exec "$user" test -w "$repo" || chown -R -- "$user" "$repo" 12 | # restart script with new privileges 13 | exec su-exec "$user" "$0" "$@" 14 | fi 15 | 16 | # 2nd invocation with regular user 17 | 18 | if [ -e "$repo/config" ]; then 19 | echo "Found assigner repo at $repo" 20 | else 21 | assigner init 22 | fi 23 | 24 | exec assigner "$@" 25 | -------------------------------------------------------------------------------- /deploy/manifests/base/lookout/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: lookout 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: lookout 10 | template: 11 | metadata: 12 | labels: 13 | app: lookout 14 | spec: 15 | containers: 16 | - name: lookout 17 | image: lookout 18 | env: 19 | - name: GOLOG_LOG_LEVEL 20 | value: INFO 21 | - name: GOLOG_LOG_FMT 22 | value: json 23 | ports: 24 | - containerPort: 40080 25 | name: metrics 26 | -------------------------------------------------------------------------------- /deploy/manifests/base/promtail/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | namespace: promtail 5 | 6 | resources: 7 | - daemonset.yaml 8 | - rbac.yaml 9 | 10 | configMapGenerator: 11 | - name: promtail-config 12 | files: 13 | - config.yaml 14 | - name: promtail-env 15 | literals: 16 | - PROMTAIL_URL=http://localhost:1234/loki/api/v1/push 17 | - PROMTAIL_TENANT_ID= 18 | - PROMTAIL_LABEL_OWNER= 19 | - PROMTAIL_LABEL_ENV= 20 | - PROMTAIL_LABEL_REGION= 21 | 22 | secretGenerator: 23 | - name: promtail-auth 24 | envs: 25 | - auth.env 26 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Context 2 | 3 | 4 | ## Proposed Changes 5 | 6 | 7 | ## Tests 8 | 9 | 10 | ## Revert Strategy 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /assigner/config/init.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "io" 5 | 6 | sticfg "github.com/ipni/storetheindex/config" 7 | ) 8 | 9 | func Init(out io.Writer) (*Config, error) { 10 | identity, err := sticfg.CreateIdentity(out) 11 | if err != nil { 12 | return nil, err 13 | } 14 | 15 | return InitWithIdentity(identity) 16 | } 17 | 18 | func InitWithIdentity(identity sticfg.Identity) (*Config, error) { 19 | conf := &Config{ 20 | Version: Version, 21 | Assignment: NewAssignment(), 22 | Bootstrap: sticfg.NewBootstrap(), 23 | Daemon: NewDaemon(), 24 | Identity: identity, 25 | Logging: NewLogging(), 26 | } 27 | 28 | return conf, nil 29 | } 30 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/flux-system/cluster-cd.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: source.toolkit.fluxcd.io/v1beta2 2 | kind: GitRepository 3 | metadata: 4 | name: storetheindex 5 | spec: 6 | interval: 5m 7 | url: https://github.com/filecoin-project/storetheindex.git 8 | ref: 9 | branch: main 10 | --- 11 | apiVersion: kustomize.toolkit.fluxcd.io/v1beta2 12 | kind: Kustomization 13 | metadata: 14 | name: cluster 15 | spec: 16 | serviceAccountName: kustomize-controller 17 | decryption: 18 | provider: sops 19 | interval: 5m 20 | path: "./deploy/manifests/prod/us-east-2/cluster" 21 | sourceRef: 22 | kind: GitRepository 23 | name: storetheindex 24 | prune: true 25 | -------------------------------------------------------------------------------- /fsutil/disk/usage_test.go: -------------------------------------------------------------------------------- 1 | package disk_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ipni/storetheindex/fsutil/disk" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestUsage(t *testing.T) { 11 | tempDir := t.TempDir() 12 | us, err := disk.Usage(tempDir) 13 | require.NoError(t, err) 14 | 15 | t.Log("Path:", us.Path) 16 | require.Equal(t, tempDir, us.Path) 17 | 18 | t.Log("Total:", us.Total) 19 | require.NotZero(t, us.Total) 20 | 21 | t.Log("Free:", us.Free) 22 | require.NotZero(t, us.Free) 23 | 24 | t.Log("Used:", us.Used) 25 | require.NotZero(t, us.Used) 26 | 27 | t.Log("Percent:", us.Percent) 28 | require.Greater(t, us.Percent, 0.0) 29 | } 30 | -------------------------------------------------------------------------------- /deploy/manifests/base/heyfil/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: heyfil 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: heyfil 10 | template: 11 | metadata: 12 | labels: 13 | app: heyfil 14 | spec: 15 | containers: 16 | - name: heyfil 17 | image: heyfil 18 | env: 19 | - name: GOLOG_LOG_LEVEL 20 | value: INFO 21 | - name: GOLOG_LOG_FMT 22 | value: json 23 | ports: 24 | - containerPort: 8080 25 | name: metrics 26 | - containerPort: 8081 27 | name: api 28 | -------------------------------------------------------------------------------- /deploy/manifests/base/storetheindex/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | namespace: storetheindex 5 | 6 | resources: 7 | - indexer-statefulset.yaml 8 | - indexer-service.yaml 9 | - pdb.yaml 10 | - indexer-config.yaml 11 | 12 | transformers: 13 | - labels.yaml 14 | 15 | configMapGenerator: 16 | - name: indexer-env-vars 17 | behavior: create 18 | literals: 19 | - GOLOG_LOG_LEVEL=INFO 20 | - GOLOG_LOG_FMT=json 21 | - STORETHEINDEX_LOTUS_GATEWAY=wss://api.chain.love 22 | - STORETHEINDEX_PATH=/config 23 | - STORETHEINDEX_LISTEN_ADMIN=/ip4/0.0.0.0/tcp/3002 24 | - STORETHEINDEX_WATCH_CONFIG=true 25 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/promtail/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | namespace: promtail 5 | 6 | resources: 7 | - ../../../../base/promtail 8 | - namespace.yaml 9 | 10 | configMapGenerator: 11 | - name: promtail-env 12 | behavior: merge 13 | literals: 14 | - PROMTAIL_URL=https://logs-prod-us-central1.grafana.net/loki/api/v1/push 15 | - PROMTAIL_TENANT_ID=storetheindex 16 | - PROMTAIL_LABEL_OWNER=storetheindex 17 | - PROMTAIL_LABEL_ENV=prod 18 | - PROMTAIL_LABEL_REGION=us-east-2 19 | 20 | secretGenerator: 21 | - name: promtail-auth 22 | behavior: replace 23 | envs: 24 | - auth.env 25 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.25-bookworm as builder 2 | 3 | WORKDIR /storetheindex 4 | COPY go.* . 5 | RUN go mod download 6 | COPY . . 7 | 8 | RUN CGO_ENABLED=1 go build 9 | 10 | # Debug non-root image used as base in order to provide easier administration and debugging. 11 | FROM gcr.io/distroless/cc:debug-nonroot 12 | COPY --from=builder /storetheindex/storetheindex /usr/local/bin/ 13 | 14 | # Default port configuration: 15 | # - 3000 Finder interface 16 | # - 3001 Ingest interface 17 | # - 3002 Admin interface 18 | # - 3003 libp2p interface 19 | # Note: exposed ports below will have no effect if the default config is overridden. 20 | EXPOSE 3000-3003 21 | 22 | ENTRYPOINT ["/usr/local/bin/storetheindex"] 23 | CMD ["daemon"] 24 | -------------------------------------------------------------------------------- /config/reverse_index.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // ReverseIndexer stores the configuration for the reverse indexer. 4 | type ReverseIndexer struct { 5 | // Enabled is true if the reverse indexer is enabled. 6 | Enabled bool 7 | // StorePath is the path to the reverse index store. 8 | StorePath string 9 | } 10 | 11 | // NewReverseIndexer ReverseIndexer returns ReverseIndexer with values set to their defaults. 12 | func NewReverseIndexer() ReverseIndexer { 13 | return ReverseIndexer{ 14 | Enabled: false, 15 | StorePath: "reverse_index", 16 | } 17 | } 18 | 19 | func (rx *ReverseIndexer) populateUnset() { 20 | def := NewReverseIndexer() 21 | 22 | if rx.StorePath == "" { 23 | rx.StorePath = def.StorePath 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/aws-ebs-csi-driver/patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: ebs-csi-controller-sa 5 | namespace: kube-system 6 | annotations: 7 | eks.amazonaws.com/role-arn: "arn:aws:iam::407967248065:role/prod/us-east-2/ebs_csi_controller" 8 | 9 | --- 10 | apiVersion: apps/v1 11 | kind: DaemonSet 12 | metadata: 13 | name: ebs-csi-node 14 | namespace: kube-system 15 | spec: 16 | template: 17 | spec: 18 | tolerations: 19 | # Override default tolerations to allow all tains. Otherwise, the daemonset pods will not 20 | # run on nodes with specific taints; more specifically, nodegroups with "dedicated" taint. 21 | - operator: Exists 22 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/ingress-nginx/patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: ingress-nginx-controller 5 | namespace: ingress-nginx 6 | spec: 7 | template: 8 | spec: 9 | topologySpreadConstraints: 10 | - maxSkew: 1 11 | topologyKey: topology.kubernetes.io/zone 12 | whenUnsatisfiable: ScheduleAnyway 13 | containers: 14 | - name: controller 15 | ports: 16 | - name: prometheus 17 | containerPort: 10254 18 | resources: 19 | limits: 20 | cpu: "2" 21 | memory: 3Gi 22 | requests: 23 | cpu: "2" 24 | memory: 3Gi 25 | -------------------------------------------------------------------------------- /config/peering.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/ipni/go-libipni/mautil" 5 | "github.com/libp2p/go-libp2p/core/peer" 6 | ) 7 | 8 | // Peering configures the peering service. Peering is similar to bootstrapping, 9 | // but peering maintains connection with all peers configured in the peering 10 | // service. 11 | type Peering struct { 12 | // Peers lists the nodes to attempt to stay connected with. 13 | Peers []string 14 | } 15 | 16 | // NewPeering returns Peering with values set to their defaults. 17 | func NewPeering() Peering { 18 | return Peering{} 19 | } 20 | 21 | // PeerAddrs returns the peering peers as a list of AddrInfo. 22 | func (p Peering) PeerAddrs() ([]peer.AddrInfo, error) { 23 | return mautil.ParsePeers(p.Peers) 24 | } 25 | -------------------------------------------------------------------------------- /internal/revision/revision.go: -------------------------------------------------------------------------------- 1 | // Package revision provides the rcs revision, embedded by the compiler, as a 2 | // global variable. 3 | package revision 4 | 5 | import ( 6 | "encoding/json" 7 | "runtime/debug" 8 | ) 9 | 10 | var ( 11 | // Revision is the rcs revision embedded by the compiler. 12 | Revision string 13 | // RevisionJSON is the rcs revision, in JSON format. 14 | RevisionJSON []byte 15 | ) 16 | 17 | func init() { 18 | // Get the rcs revision from the build info. 19 | bi, ok := debug.ReadBuildInfo() 20 | if !ok { 21 | return 22 | } 23 | for i := range bi.Settings { 24 | if bi.Settings[i].Key == "vcs.revision" { 25 | Revision = bi.Settings[i].Value 26 | RevisionJSON, _ = json.Marshal(Revision) 27 | break 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /deploy/manifests/base/promtail/rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: promtail 5 | --- 6 | apiVersion: rbac.authorization.k8s.io/v1 7 | kind: ClusterRole 8 | metadata: 9 | name: promtail 10 | rules: 11 | - apiGroups: 12 | - "" 13 | resources: 14 | - nodes 15 | - nodes/proxy 16 | - services 17 | - endpoints 18 | - pods 19 | verbs: 20 | - get 21 | - list 22 | - watch 23 | --- 24 | apiVersion: rbac.authorization.k8s.io/v1 25 | kind: ClusterRoleBinding 26 | metadata: 27 | name: promtail 28 | roleRef: 29 | apiGroup: rbac.authorization.k8s.io 30 | kind: ClusterRole 31 | name: promtail 32 | subjects: 33 | - kind: ServiceAccount 34 | name: promtail 35 | namespace: promtail 36 | -------------------------------------------------------------------------------- /deploy/manifests/base/storetheindex-single/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | namespace: storetheindex 5 | 6 | generatorOptions: 7 | disableNameSuffixHash: true 8 | 9 | resources: 10 | - deployment.yaml 11 | - service.yaml 12 | 13 | transformers: 14 | - labels.yaml 15 | 16 | commonLabels: 17 | app: indexer-single 18 | 19 | secretGenerator: 20 | - name: identity 21 | files: 22 | - identity.key 23 | 24 | configMapGenerator: 25 | - name: env-vars 26 | behavior: create 27 | literals: 28 | - GOLOG_LOG_LEVEL=INFO 29 | - GOLOG_LOG_FMT=json 30 | - STORETHEINDEX_WATCH_CONFIG=true 31 | - name: config 32 | behavior: create 33 | files: 34 | - config=config.json 35 | -------------------------------------------------------------------------------- /server/find/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Network Indexer 7 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /command/loadgen/config_test.go: -------------------------------------------------------------------------------- 1 | package loadgen 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestParseEntriesPerAdGenerator(t *testing.T) { 9 | type testCase struct { 10 | desc string 11 | typ string 12 | } 13 | 14 | testCases := []testCase{ 15 | { 16 | "Normal distribution. stdev=2, mean=100", 17 | "Normal(2, 100)", 18 | }, 19 | { 20 | "Uniform distribution. start=20, end=100", 21 | "Uniform(20, 100)", 22 | }, 23 | { 24 | "Always. val=20", 25 | "Always(20)", 26 | }, 27 | } 28 | 29 | for _, tc := range testCases { 30 | t.Run(tc.desc, func(t *testing.T) { 31 | c := &Config{ 32 | EntriesPerAdType: tc.typ, 33 | } 34 | 35 | c.ParseEntriesPerAdGenerator() 36 | fmt.Println(c.EntriesPerAdGenerator()) 37 | }) 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /deploy/manifests/base/telemetry/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: telemetry 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: telemetry 9 | template: 10 | metadata: 11 | labels: 12 | app: telemetry 13 | spec: 14 | containers: 15 | - name: telemetry 16 | image: telemetry 17 | ports: 18 | - containerPort: 40080 19 | name: http 20 | - containerPort: 40081 21 | name: metrics 22 | readinessProbe: 23 | httpGet: 24 | port: http 25 | path: /ready 26 | initialDelaySeconds: 3 27 | failureThreshold: 3 28 | successThreshold: 1 29 | timeoutSeconds: 5 30 | periodSeconds: 10 31 | -------------------------------------------------------------------------------- /config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "path/filepath" 5 | "runtime" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestPath(t *testing.T) { 12 | const dir = "vstore" 13 | 14 | var absdir string 15 | if runtime.GOOS == "windows" { 16 | absdir = "c:\\tmp\vstore" 17 | } else { 18 | absdir = "/tmp/vstore" 19 | } 20 | 21 | path, err := Path("", dir) 22 | require.NoError(t, err) 23 | 24 | configRoot, err := PathRoot() 25 | require.NoError(t, err) 26 | 27 | require.Equal(t, filepath.Join(configRoot, dir), path) 28 | 29 | path, err = Path("altroot", dir) 30 | require.NoError(t, err) 31 | require.Equal(t, filepath.Join("altroot", dir), path) 32 | 33 | path, err = Path("altroot", absdir) 34 | require.NoError(t, err) 35 | require.Equal(t, filepath.Clean(absdir), path) 36 | } 37 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/external-dns/deploy-patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: external-dns 5 | namespace: external-dns 6 | spec: 7 | selector: 8 | matchLabels: 9 | app: external-dns 10 | strategy: 11 | type: Recreate 12 | template: 13 | metadata: 14 | labels: 15 | app: external-dns 16 | spec: 17 | containers: 18 | - name: external-dns 19 | args: 20 | - --source=service 21 | - --source=ingress 22 | - --domain-filter=prod.cid.contact 23 | - --provider=aws 24 | - --policy=upsert-only 25 | - --aws-zone-type=public 26 | - --registry=txt 27 | - --txt-owner-id=Z0812180J6HXR2V3GIFI # prod.cid.contact hosted zone ID; see terraform output `prod_cid_contact_zone_id` 28 | -------------------------------------------------------------------------------- /admin/client/option.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | ) 7 | 8 | type config struct { 9 | httpClient *http.Client 10 | } 11 | 12 | // Option is a function that sets a value in a config. 13 | type Option func(*config) error 14 | 15 | // getOpts creates a config and applies Options to it. 16 | func getOpts(opts []Option) (config, error) { 17 | cfg := config{ 18 | httpClient: http.DefaultClient, 19 | } 20 | for i, opt := range opts { 21 | if err := opt(&cfg); err != nil { 22 | return config{}, fmt.Errorf("option %d failed: %s", i, err) 23 | } 24 | } 25 | return cfg, nil 26 | } 27 | 28 | // WithClient allows creation of the http client using an underlying network 29 | // round tripper / client. 30 | func WithClient(c *http.Client) Option { 31 | return func(cfg *config) error { 32 | cfg.httpClient = c 33 | return nil 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /deploy/manifests/base/dhfind/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: dhfind 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: dhfind 9 | template: 10 | metadata: 11 | labels: 12 | app: dhfind 13 | spec: 14 | containers: 15 | - name: dhfind 16 | image: dhfind 17 | envFrom: 18 | - configMapRef: 19 | name: dhfind-env-vars 20 | ports: 21 | - containerPort: 40080 22 | name: http 23 | - containerPort: 40082 24 | name: metrics 25 | readinessProbe: 26 | httpGet: 27 | port: http 28 | path: /ready 29 | initialDelaySeconds: 3 30 | failureThreshold: 3 31 | successThreshold: 1 32 | timeoutSeconds: 5 33 | periodSeconds: 10 34 | -------------------------------------------------------------------------------- /deploy/manifests/base/caskadht/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: caskadht 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: caskadht 9 | template: 10 | metadata: 11 | labels: 12 | app: caskadht 13 | spec: 14 | containers: 15 | - name: caskadht 16 | image: caskadht 17 | envFrom: 18 | - configMapRef: 19 | name: caskadht-env-vars 20 | ports: 21 | - containerPort: 40080 22 | name: http 23 | - containerPort: 40081 24 | name: metrics 25 | readinessProbe: 26 | httpGet: 27 | port: http 28 | path: /ready 29 | initialDelaySeconds: 3 30 | failureThreshold: 3 31 | successThreshold: 1 32 | timeoutSeconds: 5 33 | periodSeconds: 10 34 | -------------------------------------------------------------------------------- /deploy/manifests/base/cassette/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: cassette 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: cassette 9 | template: 10 | metadata: 11 | labels: 12 | app: cassette 13 | spec: 14 | containers: 15 | - name: cassette 16 | image: cassette 17 | envFrom: 18 | - configMapRef: 19 | name: cassette-env-vars 20 | ports: 21 | - containerPort: 40080 22 | name: http 23 | - containerPort: 40081 24 | name: metrics 25 | readinessProbe: 26 | httpGet: 27 | port: http 28 | path: /ready 29 | initialDelaySeconds: 3 30 | failureThreshold: 3 31 | successThreshold: 1 32 | timeoutSeconds: 5 33 | periodSeconds: 10 34 | -------------------------------------------------------------------------------- /internal/registry/errors.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import "errors" 4 | 5 | var ( 6 | ErrAlreadyAssigned = errors.New("publisher already assigned to this indexer") 7 | ErrInProgress = errors.New("discovery already in progress") 8 | ErrCannotPublish = errors.New("publisher not allowed to publish to other provider") 9 | ErrFrozen = errors.New("indexer frozen") 10 | ErrMissingProviderAddr = errors.New("advertisement missing provider address") 11 | ErrNotAllowed = errors.New("peer not allowed by policy") 12 | ErrNoDiscovery = errors.New("discovery not available") 13 | ErrNotVerified = errors.New("provider cannot be verified") 14 | ErrPublisherNotAllowed = errors.New("publisher not allowed by policy") 15 | ErrTooSoon = errors.New("not enough time since previous discovery") 16 | ErrNoAssigner = errors.New("not configured to work with assigner service") 17 | ) 18 | -------------------------------------------------------------------------------- /internal/ingest/error.go: -------------------------------------------------------------------------------- 1 | package ingest 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type adIngestState string 8 | 9 | type adIngestError struct { 10 | state adIngestState 11 | err error 12 | } 13 | 14 | const ( 15 | adIngestIndexerErr adIngestState = "indexerErr" 16 | adIngestDecodingErr adIngestState = "decodingErr" 17 | adIngestMalformedErr adIngestState = "malformedErr" 18 | adIngestRegisterProviderErr adIngestState = "registerErr" 19 | adIngestSyncEntriesErr adIngestState = "syncEntriesErr" 20 | adIngestContentNotFound adIngestState = "contentNotFound" 21 | // Happens if there is an error during ingest of an entry chunk (rather than fetching it). 22 | adIngestEntryChunkErr adIngestState = "ingestEntryChunkErr" 23 | ) 24 | 25 | func (e adIngestError) Error() string { 26 | return fmt.Sprintf("%s: %s", e.state, e.err) 27 | } 28 | 29 | func (e adIngestError) Unwrap() error { 30 | return e.err 31 | } 32 | -------------------------------------------------------------------------------- /internal/freeze/device_windows.go: -------------------------------------------------------------------------------- 1 | package freeze 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "syscall" 7 | "unsafe" 8 | 9 | "golang.org/x/sys/windows" 10 | ) 11 | 12 | var devNum int 13 | var volNums = map[string]int{} 14 | 15 | var procGetVolumePathNameW = windows.NewLazySystemDLL("kernel32.dll").NewProc("GetVolumePathNameW") 16 | 17 | func deviceNumber(path string) (int, error) { 18 | _, err := os.Stat(path) 19 | if err != nil { 20 | return 0, fmt.Errorf("cannot stat %q: %w", path, err) 21 | } 22 | buf := make([]uint16, 200) 23 | r1, _, _ := procGetVolumePathNameW.Call( 24 | uintptr(unsafe.Pointer(windows.StringToUTF16Ptr(path))), 25 | uintptr(unsafe.Pointer(&buf[0])), 26 | uintptr(len(buf)), 27 | ) 28 | if r1 == 0 { 29 | return -1, nil 30 | } 31 | vol := syscall.UTF16ToString(buf) 32 | num, ok := volNums[vol] 33 | if ok { 34 | return num, nil 35 | } 36 | devNum++ 37 | volNums[vol] = devNum 38 | return devNum, nil 39 | } 40 | -------------------------------------------------------------------------------- /deploy/infrastructure/modules/ecr/variables.tf: -------------------------------------------------------------------------------- 1 | variable "repositories" { 2 | type = set(string) 3 | description = "The list of namespace and repository names in `/` format." 4 | } 5 | 6 | variable "tags" { 7 | type = map(string) 8 | description = "The tags applied to all created AWS resources." 9 | } 10 | 11 | variable "ecr_untagged_expiry_days" { 12 | type = number 13 | description = "The number of days after which to expire untagged pushed images" 14 | default = 7 15 | } 16 | 17 | variable "ecr_tag_immutability" { 18 | type = string 19 | description = "The immutability of container image tags published to ECR repository, either `MUTABLE` or `IMMUTABLE`." 20 | default = "IMMUTABLE" 21 | } 22 | 23 | variable "scan_on_push" { 24 | type = bool 25 | description = "Indicates whether images are scanned after being pushed to the repository." 26 | default = true 27 | } 28 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/monitoring/patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: prometheus-k8s 5 | rules: 6 | - apiGroups: 7 | - "" 8 | resources: 9 | - nodes/metrics 10 | verbs: 11 | - get 12 | - nonResourceURLs: 13 | - /metrics 14 | verbs: 15 | - get 16 | - apiGroups: 17 | - "" 18 | resources: 19 | - services 20 | - pods 21 | - endpoints 22 | verbs: 23 | - get 24 | - list 25 | - watch 26 | 27 | --- 28 | apiVersion: v1 29 | kind: ServiceAccount 30 | metadata: 31 | name: prometheus-k8s 32 | namespace: monitoring 33 | annotations: 34 | eks.amazonaws.com/role-arn: "arn:aws:iam::407967248065:role/prod/us-east-2/monitoring" 35 | --- 36 | apiVersion: monitoring.coreos.com/v1 37 | kind: Alertmanager 38 | metadata: 39 | name: main 40 | namespace: monitoring 41 | spec: 42 | replicas: 0 43 | -------------------------------------------------------------------------------- /carstore/option.go: -------------------------------------------------------------------------------- 1 | package carstore 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | // config contains all options for the server. 8 | type config struct { 9 | compAlg string 10 | } 11 | 12 | // Option is a function that sets a value in a config. 13 | type Option func(*config) error 14 | 15 | // getOpts creates a config and applies Options to it. 16 | func getOpts(opts []Option) (config, error) { 17 | var cfg config 18 | for i, opt := range opts { 19 | if err := opt(&cfg); err != nil { 20 | return config{}, fmt.Errorf("option %d error: %s", i, err) 21 | } 22 | } 23 | return cfg, nil 24 | } 25 | 26 | // WithCompress configures file compression. 27 | func WithCompress(alg string) Option { 28 | return func(c *config) error { 29 | switch alg { 30 | case Gzip, "gz": 31 | c.compAlg = Gzip 32 | case "", "none", "nil", "null": 33 | c.compAlg = "" 34 | default: 35 | return fmt.Errorf("unsupported compression: %s", alg) 36 | } 37 | return nil 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /fsutil/disk/usage_openbsd.go: -------------------------------------------------------------------------------- 1 | //go:build openbsd 2 | 3 | package disk 4 | 5 | import ( 6 | "syscall" 7 | 8 | "golang.org/x/sys/unix" 9 | ) 10 | 11 | func usage(path string) (*UsageStats, error) { 12 | var stat syscall.Statfs_t 13 | err := unix.Statfs(path, &stat) 14 | if err != nil { 15 | return nil, err 16 | } 17 | blockSize := stat.F_bsize 18 | 19 | // Total blocks only available to root. 20 | total := stat.F_blocks 21 | // Remaining free blocks usable by root. 22 | availToRoot := stat.F_bfree 23 | // Remaining free blocks usable by user. 24 | availToUser := stat.F_bavail 25 | // Total blocks being used. 26 | used := total - availToRoot 27 | // Total blocks available to user. 28 | totalUser := used + availToUser 29 | 30 | return &UsageStats{ 31 | Path: path, 32 | Percent: percentUsed(used, totalUser), 33 | Free: availToUser * uint64(blockSize), 34 | Total: total * uint64(blockSize), 35 | Used: used * uint64(blockSize), 36 | }, nil 37 | } 38 | -------------------------------------------------------------------------------- /fsutil/disk/usage_unix.go: -------------------------------------------------------------------------------- 1 | //go:build freebsd || linux || darwin || (aix && !cgo) 2 | 3 | package disk 4 | 5 | import ( 6 | "golang.org/x/sys/unix" 7 | ) 8 | 9 | func usage(path string) (*UsageStats, error) { 10 | var stat unix.Statfs_t 11 | err := unix.Statfs(path, &stat) 12 | if err != nil { 13 | return nil, err 14 | } 15 | blockSize := uint64(stat.Bsize) 16 | 17 | // Total blocks only available to root. 18 | total := uint64(stat.Blocks) 19 | // Remaining free blocks usable by root. 20 | availToRoot := uint64(stat.Bfree) 21 | // Remaining free blocks usable by user. 22 | availToUser := uint64(stat.Bavail) 23 | // Total blocks being used. 24 | used := total - availToRoot 25 | // Total blocks available to user. 26 | totalUser := used + availToUser 27 | 28 | return &UsageStats{ 29 | Path: path, 30 | Percent: percentUsed(used, totalUser), 31 | Free: availToUser * blockSize, 32 | Total: total * blockSize, 33 | Used: used * blockSize, 34 | }, nil 35 | } 36 | -------------------------------------------------------------------------------- /assigner/config/policy.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // Policy configures which peers are allowed and which are blocked. Announce 4 | // messages are accepted from allowed peers and the publisher assigned to an 5 | // indexer. Announce messagees from blocked peers are ignored. 6 | type Policy struct { 7 | // Allow is either false or true, and determines whether a peer is allowed 8 | // (true) or is blocked (false), by default. 9 | Allow bool 10 | // Except is a list of peer IDs that are exceptions to the Allow policy. 11 | // If Allow is true, then all peers are allowed except those listed in 12 | // Except. If Allow is false, then no peers are allowed except those listed 13 | // in Except. in other words, Allow=true means that Except is a deny-list 14 | // and Allow=false means that Except is an allow-list. 15 | Except []string 16 | } 17 | 18 | // NewPolicy returns Policy with values set to their defaults. 19 | func NewPolicy() Policy { 20 | return Policy{ 21 | Allow: true, 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /filestore/option.go: -------------------------------------------------------------------------------- 1 | package filestore 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type s3Config struct { 8 | endpoint string 9 | region string 10 | accessKey string 11 | secretKey string 12 | } 13 | 14 | type S3Option func(*s3Config) error 15 | 16 | func getOpts(opts []S3Option) (s3Config, error) { 17 | var cfg s3Config 18 | for i, opt := range opts { 19 | if err := opt(&cfg); err != nil { 20 | return s3Config{}, fmt.Errorf("option %d error: %s", i, err) 21 | } 22 | } 23 | return cfg, nil 24 | } 25 | 26 | func WithEndpoint(ep string) S3Option { 27 | return func(c *s3Config) error { 28 | c.endpoint = ep 29 | return nil 30 | } 31 | } 32 | 33 | func WithRegion(region string) S3Option { 34 | return func(c *s3Config) error { 35 | c.region = region 36 | return nil 37 | } 38 | } 39 | 40 | func WithKeys(accessKey, secretKey string) S3Option { 41 | return func(c *s3Config) error { 42 | c.accessKey = accessKey 43 | c.secretKey = secretKey 44 | return nil 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /assigner/config/daemon.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // Daemon stores daemon settings. 4 | type Daemon struct { 5 | // HTTPAddr is the HTTP host multiaddr for receiving direct announce 6 | // messages. Set to "none" to disable HTTP hosting. 7 | HTTPAddr string 8 | // P2PAddr is the libp2p host multiaddr for receiving announce messages. 9 | // Set to "none" to disable libp2p hosting. 10 | P2PAddr string 11 | // NoResourceManager disables the libp2p resource manager when true. 12 | NoResourceManager bool 13 | } 14 | 15 | // NewDaemon returns Addresses with values set to their defaults. 16 | func NewDaemon() Daemon { 17 | return Daemon{ 18 | HTTPAddr: "/ip4/0.0.0.0/tcp/3001", 19 | P2PAddr: "/ip4/0.0.0.0/tcp/3003", 20 | } 21 | } 22 | 23 | // populateUnset replaces zero-values in the config with default values. 24 | func (c *Daemon) populateUnset() { 25 | def := NewDaemon() 26 | 27 | if c.HTTPAddr == "" { 28 | c.HTTPAddr = def.HTTPAddr 29 | } 30 | if c.P2PAddr == "" { 31 | c.P2PAddr = def.P2PAddr 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /deploy/infrastructure/common/outputs.tf: -------------------------------------------------------------------------------- 1 | output "github_actions_role_arn" { 2 | value = module.github_actions_role.iam_role_arn 3 | } 4 | 5 | output "pl_grafana_user_password_encrypted" { 6 | value = module.pl_grafana_user.keybase_password_pgp_message 7 | } 8 | 9 | output "pl_grafana_user_access_key_id" { 10 | value = module.pl_grafana_user.iam_access_key_id 11 | } 12 | 13 | output "pl_grafana_user_access_key_secret_encrypted" { 14 | value = module.pl_grafana_user.iam_access_key_encrypted_secret 15 | } 16 | 17 | output "ipni_gh_mgmt_ro_user_access_key_id" { 18 | value = module.ipni_gh_mgmt_ro.iam_access_key_id 19 | } 20 | 21 | output "ipni_gh_mgmt_ro_user_access_key_secret_encrypted" { 22 | value = module.ipni_gh_mgmt_ro.iam_access_key_encrypted_secret 23 | } 24 | 25 | output "ipni_gh_mgmt_rw_user_access_key_id" { 26 | value = module.ipni_gh_mgmt_rw.iam_access_key_id 27 | } 28 | 29 | output "ipni_gh_mgmt_rw_user_access_key_secret_encrypted" { 30 | value = module.ipni_gh_mgmt_rw.iam_access_key_encrypted_secret 31 | } 32 | -------------------------------------------------------------------------------- /internal/registry/option.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | ) 7 | 8 | // regConfig contains all options for the server. 9 | type regConfig struct { 10 | freezeAtPercent float64 11 | freezeDirs []string 12 | } 13 | 14 | // Option is a function that sets a value in a regConfig. 15 | type Option func(*regConfig) error 16 | 17 | // getOpts creates a regConfig and applies Options to it. 18 | func getOpts(opts []Option) (regConfig, error) { 19 | var cfg regConfig 20 | for i, opt := range opts { 21 | if err := opt(&cfg); err != nil { 22 | return regConfig{}, fmt.Errorf("option %d error: %s", i, err) 23 | } 24 | } 25 | return cfg, nil 26 | } 27 | 28 | func WithFreezer(freezeDirs []string, freezeAtPercent float64) Option { 29 | return func(c *regConfig) error { 30 | if len(freezeDirs) != 0 && freezeAtPercent == 0 { 31 | return errors.New("cannot freeze at 0 percent usage") 32 | } 33 | c.freezeAtPercent = freezeAtPercent 34 | c.freezeDirs = freezeDirs 35 | return nil 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /scripts/peer_id_from_priv_key/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | 8 | "github.com/libp2p/go-libp2p/core/crypto" 9 | "github.com/libp2p/go-libp2p/core/peer" 10 | ) 11 | 12 | // Reads marshalled crypto.PrivKey from os.stdin and prints the peer.ID that corresponds to the key. 13 | // See: crypto.UnmarshalPrivateKey, peer.IDFromPrivateKey. 14 | // 15 | // For example, you can directly get the peer.ID from a sops-encrypted identity file as long as 16 | // your terminal session is authenticated to the storetheindex AWS account and has access 17 | // to the relevant KMS keys like this: 18 | // 19 | // sops -d indexer-0-identity.encrypted | go run //peer_id_from_priv_key.go 20 | func main() { 21 | all, err := io.ReadAll(os.Stdin) 22 | if err != nil { 23 | panic(err) 24 | } 25 | key, err := crypto.UnmarshalPrivateKey(all) 26 | if err != nil { 27 | panic(err) 28 | } 29 | pid, err := peer.IDFromPrivateKey(key) 30 | if err != nil { 31 | panic(err) 32 | } 33 | fmt.Println(pid.String()) 34 | } 35 | -------------------------------------------------------------------------------- /assigner/scripts/peer_id_from_priv_key.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | 8 | "github.com/libp2p/go-libp2p/core/crypto" 9 | "github.com/libp2p/go-libp2p/core/peer" 10 | ) 11 | 12 | // Reads marshalled crypto.PrivKey from os.stdin and prints the peer.ID that corresponds to the key. 13 | // See: crypto.UnmarshalPrivateKey, peer.IDFromPrivateKey. 14 | // 15 | // For example, you can directly get the peer.ID from a sops-encrypted identity file as long as 16 | // your terminal session is authenticated to the storetheindex AWS account and has access 17 | // to the relevant KMS keys like this: 18 | // 19 | // sops -d indexer-0-identity.encrypted | go run //peer_id_from_priv_key.go 20 | func main() { 21 | all, err := io.ReadAll(os.Stdin) 22 | if err != nil { 23 | panic(err) 24 | } 25 | key, err := crypto.UnmarshalPrivateKey(all) 26 | if err != nil { 27 | panic(err) 28 | } 29 | pid, err := peer.IDFromPrivateKey(key) 30 | if err != nil { 31 | panic(err) 32 | } 33 | fmt.Println(pid.String()) 34 | } 35 | -------------------------------------------------------------------------------- /deploy/infrastructure/modules/ecr/main.tf: -------------------------------------------------------------------------------- 1 | resource "aws_ecr_repository" "this" { 2 | for_each = var.repositories 3 | name = each.key 4 | tags = var.tags 5 | image_scanning_configuration { 6 | scan_on_push = var.scan_on_push 7 | } 8 | image_tag_mutability = var.ecr_tag_immutability 9 | } 10 | 11 | resource "aws_ecr_lifecycle_policy" "this" { 12 | for_each = var.repositories 13 | repository = each.key 14 | depends_on = [aws_ecr_repository.this] 15 | policy = < seqRetireInterval { 47 | go s.retire() 48 | } 49 | return nil 50 | } 51 | 52 | func (s *sequences) retire() { 53 | oldestAllowed := uint64(time.Now().Add(-s.maxAge).UnixNano()) 54 | active := make(map[peer.ID]uint64) 55 | 56 | s.mutex.Lock() 57 | defer s.mutex.Unlock() 58 | 59 | for id, seq := range s.seqs { 60 | if seq > oldestAllowed { 61 | active[id] = seq 62 | } 63 | } 64 | s.seqs = active 65 | s.lastRetire = time.Now() 66 | } 67 | -------------------------------------------------------------------------------- /server/ingest/options.go: -------------------------------------------------------------------------------- 1 | package ingest 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | const ( 9 | defaultWriteTimeout = 30 * time.Second 10 | defaultReadTimeout = 30 * time.Second 11 | ) 12 | 13 | // serverConfig contains all options for the server. 14 | type serverConfig struct { 15 | readTimeout time.Duration 16 | writeTimeout time.Duration 17 | version string 18 | } 19 | 20 | // Option is a function that sets a value in a serverConfig. 21 | type Option func(*serverConfig) error 22 | 23 | // getOpts creates a serverConfig and applies Options to it. 24 | func getOpts(opts []Option) (serverConfig, error) { 25 | cfg := serverConfig{ 26 | readTimeout: defaultReadTimeout, 27 | writeTimeout: defaultWriteTimeout, 28 | } 29 | 30 | for i, opt := range opts { 31 | if err := opt(&cfg); err != nil { 32 | return serverConfig{}, fmt.Errorf("option %d error: %s", i, err) 33 | } 34 | } 35 | return cfg, nil 36 | } 37 | 38 | // WithReadTimeout serverConfigures server read timeout. 39 | func WithReadTimeout(t time.Duration) Option { 40 | return func(c *serverConfig) error { 41 | c.readTimeout = t 42 | return nil 43 | } 44 | } 45 | 46 | // WithWriteTimeout serverConfigures server write timeout. 47 | func WithWriteTimeout(t time.Duration) Option { 48 | return func(c *serverConfig) error { 49 | c.writeTimeout = t 50 | return nil 51 | } 52 | } 53 | 54 | // WithVersion sets the version string used in /health output. 55 | func WithVersion(ver string) Option { 56 | return func(c *serverConfig) error { 57 | c.version = ver 58 | return nil 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /.github/workflows/cd.yml: -------------------------------------------------------------------------------- 1 | name: CD 2 | on: 3 | push: 4 | branches: 5 | - 'cd/prod' 6 | 7 | jobs: 8 | pull-request: 9 | name: Open PR to main 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v6 14 | - name: Create Pull Request 15 | uses: actions/github-script@v8 16 | with: 17 | script: | 18 | const head = process.env.GITHUB_REF_NAME; 19 | const env = head.replace(/cd\//g, ''); 20 | core.info(`Checking branch '${head}' for automated deployment to '${env}' environment...`); 21 | 22 | const { repo, owner } = context.repo; 23 | try { 24 | const result = await github.rest.pulls.create({ 25 | title: `Deploy latest to \`${env}\` environment`, 26 | owner, 27 | repo, 28 | head, 29 | base: 'main', 30 | body: `Approve changes and merge this PR to trigger deployment to \`${env}\` environment.` 31 | }); 32 | await github.rest.issues.addLabels({ 33 | owner, 34 | repo, 35 | issue_number: result.data.number, 36 | labels: ['cd', `env:${env}`] 37 | }); 38 | } catch (e) { 39 | if (!e.message.includes('A pull request already exists') && !e.message.includes('No commits between')) { 40 | throw e 41 | } 42 | core.info(`Skipped PR creation: ${e.message}`) 43 | } 44 | -------------------------------------------------------------------------------- /deploy/manifests/base/flux-system/kustomization.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: kustomize.config.k8s.io/v1beta1 2 | kind: Kustomization 3 | 4 | namespace: flux-system 5 | 6 | resources: 7 | - https://github.com/fluxcd/flux2/releases/download/v0.28.5/install.yaml 8 | 9 | patches: 10 | # Disallow cross namespace access, since the deployment follows a multi-tenant setup. 11 | - patch: | 12 | - op: add 13 | path: /spec/template/spec/containers/0/args/0 14 | value: --no-cross-namespace-refs=true 15 | target: 16 | kind: Deployment 17 | name: "(kustomize-controller|helm-controller|notification-controller|image-reflector-controller|image-automation-controller)" 18 | # Authenticate EKS with ECR automatically, used to pull images for deployment. 19 | - patch: | 20 | - op: add 21 | path: /spec/template/spec/containers/0/args/0 22 | value: --aws-autologin-for-ecr 23 | target: 24 | kind: Deployment 25 | name: image-reflector-controller 26 | # Require separate service account set up per tenant, since the deployment follows a multi-tenant setup. 27 | - patch: | 28 | - op: add 29 | path: /spec/template/spec/containers/0/args/0 30 | value: --default-service-account=default 31 | target: 32 | kind: Deployment 33 | name: "(kustomize-controller|helm-controller)" 34 | # Allow cluster-wide kustomizations to administer the cluster. 35 | - patch: | 36 | - op: add 37 | path: /spec/serviceAccountName 38 | value: kustomize-controller 39 | target: 40 | kind: Kustomization 41 | name: flux-system 42 | -------------------------------------------------------------------------------- /deploy/manifests/base/storetheindex/indexer-statefulset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: indexer 5 | spec: 6 | serviceName: "indexer" 7 | replicas: 1 8 | selector: 9 | matchLabels: 10 | app: indexer 11 | template: 12 | metadata: 13 | labels: 14 | app: indexer 15 | spec: 16 | securityContext: 17 | runAsUser: 10001 18 | fsGroup: 532 19 | containers: 20 | - name: indexer 21 | image: storetheindex 22 | envFrom: 23 | - configMapRef: 24 | name: indexer-env-vars 25 | ports: 26 | - containerPort: 3000 27 | name: finder 28 | - containerPort: 3001 29 | name: ingest 30 | - containerPort: 3002 31 | name: admin 32 | volumeMounts: 33 | - name: data 34 | mountPath: /data 35 | - name: config 36 | mountPath: /config 37 | readinessProbe: 38 | httpGet: 39 | port: finder 40 | path: /health 41 | initialDelaySeconds: 10 42 | failureThreshold: 3 43 | successThreshold: 1 44 | timeoutSeconds: 5 45 | periodSeconds: 10 46 | volumes: 47 | - name: config 48 | configMap: 49 | name: indexer-config 50 | volumeClaimTemplates: 51 | - metadata: 52 | name: data 53 | spec: 54 | accessModes: 55 | - ReadWriteOnce 56 | resources: 57 | requests: 58 | storage: 5Ti 59 | -------------------------------------------------------------------------------- /config/init_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "io" 7 | "path/filepath" 8 | "testing" 9 | 10 | crypto_pb "github.com/libp2p/go-libp2p/core/crypto/pb" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestCreateIdentity(t *testing.T) { 15 | id, err := CreateIdentity(io.Discard) 16 | require.NoError(t, err) 17 | pk, err := id.DecodePrivateKey("") 18 | require.NoError(t, err) 19 | require.Equal(t, crypto_pb.KeyType_Ed25519, pk.Type()) 20 | } 21 | 22 | func TestMarshalUnmarshal(t *testing.T) { 23 | cfg, err := Init(io.Discard) 24 | require.NoError(t, err) 25 | 26 | b, err := json.MarshalIndent(&cfg, " ", " ") 27 | require.NoError(t, err) 28 | 29 | t.Log("default config:\n", string(b)) 30 | 31 | cfg2 := Config{} 32 | err = json.Unmarshal(b, &cfg2) 33 | require.NoError(t, err) 34 | 35 | require.Equal(t, cfg.Identity.PeerID, cfg2.Identity.PeerID) 36 | require.Equal(t, cfg.Identity.PrivKey, cfg2.Identity.PrivKey) 37 | } 38 | 39 | func TestSaveLoad(t *testing.T) { 40 | tmpDir := t.TempDir() 41 | cfgFile, err := Path(tmpDir, "") 42 | require.NoError(t, err) 43 | 44 | require.Equal(t, tmpDir, filepath.Dir(cfgFile), "wrong root dir") 45 | 46 | cfg, err := Init(io.Discard) 47 | require.NoError(t, err) 48 | cfgBytes, err := Marshal(cfg) 49 | require.NoError(t, err) 50 | 51 | err = cfg.Save(cfgFile) 52 | require.NoError(t, err) 53 | 54 | cfg2, err := Load(cfgFile) 55 | require.NoError(t, err) 56 | cfg2Bytes, err := Marshal(cfg2) 57 | require.NoError(t, err) 58 | 59 | require.True(t, bytes.Equal(cfgBytes, cfg2Bytes), "config data different after being loaded") 60 | } 61 | -------------------------------------------------------------------------------- /config/finder.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type Finder struct { 8 | // ApiReadTimeout sets the HTTP server's maximum duration for reading the 9 | // entire request, including the body. A value of zero sets the default, 10 | // and a negative value means there will be no timeout. 11 | ApiReadTimeout Duration 12 | // ApiWriteTimeout sets the HTTP server's maximum duration before timing 13 | // out writes of the response. A value of zero sets the default and a 14 | // negative value means there will be no timeout. 15 | ApiWriteTimeout Duration 16 | // MaxConnections is maximum number of simultaneous connections that the 17 | // HTTP server will accept. A value of zero sets the default and a negative 18 | // value means there is no limit. 19 | MaxConnections int 20 | // Webpage is a domain to display when the homepage of the finder is 21 | // accessed over HTTP. 22 | Webpage string 23 | } 24 | 25 | func NewFinder() Finder { 26 | return Finder{ 27 | ApiReadTimeout: Duration(30 * time.Second), 28 | ApiWriteTimeout: Duration(30 * time.Second), 29 | MaxConnections: 8_000, 30 | Webpage: "https://web-ipni.cid.contact/", 31 | } 32 | } 33 | 34 | // populateUnset replaces zero-values in the config with default values. 35 | func (f *Finder) populateUnset() { 36 | def := NewFinder() 37 | if f.ApiReadTimeout == 0 { 38 | f.ApiReadTimeout = def.ApiReadTimeout 39 | } 40 | if f.ApiWriteTimeout == 0 { 41 | f.ApiWriteTimeout = def.ApiWriteTimeout 42 | } 43 | if f.MaxConnections == 0 { 44 | f.MaxConnections = def.MaxConnections 45 | } 46 | if f.Webpage == "" { 47 | f.Webpage = def.Webpage 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /deploy/manifests/prod/us-east-2/cluster/kube-system/gp3-storage-class.yaml: -------------------------------------------------------------------------------- 1 | # The cheapest gp3 volume config with 3000 IOPS and 125 MiB/s throughput. 2 | apiVersion: storage.k8s.io/v1 3 | kind: StorageClass 4 | metadata: 5 | name: gp3 6 | annotations: 7 | storageclass.kubernetes.io/is-default-class: "true" 8 | provisioner: ebs.csi.aws.com 9 | allowVolumeExpansion: true 10 | volumeBindingMode: WaitForFirstConsumer 11 | parameters: 12 | type: gp3 13 | --- 14 | # gp3 volume config with 5000 IOPS and 300 MiB/s throughput. 15 | apiVersion: storage.k8s.io/v1 16 | kind: StorageClass 17 | metadata: 18 | name: gp3-iops5k-t300 19 | provisioner: ebs.csi.aws.com 20 | allowVolumeExpansion: true 21 | volumeBindingMode: WaitForFirstConsumer 22 | parameters: 23 | # See: https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/parameters.md 24 | type: gp3 25 | iops: '5000' 26 | throughput: '300' 27 | allowAutoIOPSPerGBIncrease: 'true' 28 | --- 29 | # gp3 volume config with 10000 IOPS and 600 MiB/s throughput. 30 | apiVersion: storage.k8s.io/v1 31 | kind: StorageClass 32 | metadata: 33 | name: gp3-iops10k-t600 34 | provisioner: ebs.csi.aws.com 35 | allowVolumeExpansion: true 36 | volumeBindingMode: WaitForFirstConsumer 37 | parameters: 38 | type: gp3 39 | iops: '10000' 40 | throughput: '600' 41 | allowAutoIOPSPerGBIncrease: 'true' 42 | --- 43 | # gp3 volume config with 16000 IOPS and 1000 MiB/s throughput. 44 | apiVersion: storage.k8s.io/v1 45 | kind: StorageClass 46 | metadata: 47 | name: gp3-iops16k-t1000 48 | provisioner: ebs.csi.aws.com 49 | allowVolumeExpansion: true 50 | volumeBindingMode: WaitForFirstConsumer 51 | parameters: 52 | type: gp3 53 | iops: '16000' 54 | throughput: '1000' 55 | allowAutoIOPSPerGBIncrease: 'true' 56 | -------------------------------------------------------------------------------- /scripts/gen_identity/gen-identity.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/rand" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | "runtime" 10 | 11 | "github.com/libp2p/go-libp2p/core/crypto" 12 | "github.com/libp2p/go-libp2p/core/peer" 13 | ) 14 | 15 | func main() { 16 | flag.Usage = usage 17 | flag.Parse() 18 | 19 | err := genIdentity() 20 | if err != nil { 21 | fmt.Fprintln(os.Stderr, err) 22 | os.Exit(1) 23 | } 24 | } 25 | 26 | func usage() { 27 | const encFileName = "identity.key.encrypted" 28 | _, name, _, _ := runtime.Caller(0) 29 | name = filepath.Base(name) 30 | fmt.Fprintln(os.Stderr, "NAME:") 31 | fmt.Fprintln(os.Stderr, " ", name) 32 | fmt.Fprintln(os.Stderr) 33 | fmt.Fprintln(os.Stderr, "USAGE:") 34 | fmt.Fprintln(os.Stderr, " go run", name, "[command options]") 35 | fmt.Fprintln(os.Stderr) 36 | fmt.Fprintln(os.Stderr, "DESCRIPTION:") 37 | fmt.Fprintln(os.Stderr, " Generate an encrypted identity and write the data to stdout. The associated peer ID is printed to stderr.") 38 | fmt.Fprintln(os.Stderr) 39 | fmt.Fprintln(os.Stderr, " Example use:") 40 | fmt.Fprintf(os.Stderr, " sops -e <(go run %s) > %s\n", name, encFileName) 41 | fmt.Fprintln(os.Stderr) 42 | fmt.Fprintln(os.Stderr, "OPTIONS:") 43 | fmt.Fprintln(os.Stderr, " -help, -h Show help") 44 | } 45 | 46 | func genIdentity() error { 47 | k, _, err := crypto.GenerateEd25519Key(rand.Reader) 48 | if err != nil { 49 | return err 50 | } 51 | pid, err := peer.IDFromPrivateKey(k) 52 | if err != nil { 53 | return err 54 | } 55 | mk, err := crypto.MarshalPrivateKey(k) 56 | if err != nil { 57 | return err 58 | } 59 | os.Stdout.Write(mk) 60 | fmt.Fprintln(os.Stderr, "Peer ID for private key:", pid) 61 | return nil 62 | } 63 | -------------------------------------------------------------------------------- /internal/httpserver/util.go: -------------------------------------------------------------------------------- 1 | // Package httpserver provides functionality common to all storetheindex HTTP 2 | // servers 3 | package httpserver 4 | 5 | import ( 6 | "errors" 7 | "fmt" 8 | "net/http" 9 | "strings" 10 | 11 | logging "github.com/ipfs/go-log/v2" 12 | apierror "github.com/ipni/go-libipni/apierror" 13 | ) 14 | 15 | var log = logging.Logger("indexer/http") 16 | 17 | func MethodOK(w http.ResponseWriter, r *http.Request, method string) bool { 18 | if r.Method != method { 19 | w.Header().Set("Allow", method) 20 | if r.Method == http.MethodOptions { 21 | http.Error(w, "", http.StatusOK) 22 | } else { 23 | http.Error(w, "", http.StatusMethodNotAllowed) 24 | } 25 | return false 26 | } 27 | return true 28 | } 29 | 30 | func WriteJsonResponse(w http.ResponseWriter, status int, body []byte) { 31 | w.Header().Set("Content-Type", "application/json; charset=utf-8") 32 | if status != http.StatusOK { 33 | w.WriteHeader(status) 34 | } 35 | if _, err := w.Write(body); err != nil { 36 | log.Errorw("cannot write response", "err", err) 37 | http.Error(w, "", http.StatusInternalServerError) 38 | } 39 | } 40 | 41 | func HandleError(w http.ResponseWriter, err error, reqType string) { 42 | status := http.StatusBadRequest 43 | var apierr *apierror.Error 44 | if errors.As(err, &apierr) { 45 | if apierr.Status() >= 500 { 46 | msg := fmt.Sprintf("Cannot handle %s request", strings.ToUpper(reqType)) 47 | log.Errorw(msg, "err", apierr.Error(), "status", apierr.Status()) 48 | http.Error(w, "", apierr.Status()) 49 | return 50 | } 51 | status = apierr.Status() 52 | } 53 | msg := fmt.Sprintf("Bad %s request", strings.ToUpper(reqType)) 54 | log.Infow(msg, "err", err, "status", status) 55 | http.Error(w, err.Error(), status) 56 | } 57 | -------------------------------------------------------------------------------- /deploy/manifests/base/envoy/envoy-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: envoy 5 | labels: 6 | app: envoy 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | app: envoy 12 | progressDeadlineSeconds: 180 13 | template: 14 | metadata: 15 | labels: 16 | app: envoy 17 | spec: 18 | containers: 19 | - name: envoy 20 | image: envoyproxy/envoy:distroless-v1.26.1 21 | args: 22 | - --log-level 23 | - info 24 | - --config-path 25 | - '/config/envoy-config.yaml' 26 | env: 27 | - name: ENVOY_UID 28 | value: '0' 29 | securityContext: 30 | runAsUser: 10001 31 | runAsNonRoot: true 32 | allowPrivilegeEscalation: false 33 | readOnlyRootFilesystem: true 34 | capabilities: 35 | drop: 36 | - all 37 | ports: 38 | - name: http 39 | containerPort: 8080 40 | - name: admin 41 | containerPort: 9901 42 | livenessProbe: 43 | initialDelaySeconds: 3 44 | tcpSocket: 45 | port: admin 46 | readinessProbe: 47 | initialDelaySeconds: 3 48 | tcpSocket: 49 | port: admin 50 | resources: 51 | requests: 52 | cpu: 300m 53 | memory: 128Mi 54 | limits: 55 | cpu: 500m 56 | memory: 512Mi 57 | volumeMounts: 58 | - name: config 59 | mountPath: /config 60 | volumes: 61 | - name: config 62 | configMap: 63 | name: "envoy-config" 64 | -------------------------------------------------------------------------------- /filestore/filestore.go: -------------------------------------------------------------------------------- 1 | // Package filestore stores files in various types of storage systems. 2 | package filestore 3 | 4 | import ( 5 | "context" 6 | "io" 7 | "time" 8 | ) 9 | 10 | // File contains information about a stored file. 11 | type File struct { 12 | // Modified it the last modification time. 13 | Modified time.Time 14 | // Path is the path to the file relative to the root of the file store. 15 | // Path separators are always slash ('/') characters. 16 | Path string 17 | // Size if the number of bytes of data in the file. 18 | Size int64 19 | // URL is a URL where the file can be retrieved from, if available. 20 | URL string 21 | } 22 | 23 | // Interface is the interface supported by all file store implementations. All 24 | // Path arguments are relative to the root of the file store and always use 25 | // slash ('/') characters. 26 | type Interface interface { 27 | // Delete removes the specified file from storage. 28 | Delete(ctx context.Context, path string) error 29 | // Get retrieves the specified file from storage. Returns fs.ErrNotExist if 30 | // file is not count. 31 | Get(ctx context.Context, path string) (*File, io.ReadCloser, error) 32 | // Head gets information about the specified file in storage. Returns 33 | // fs.ErrNotExist if file is not count. 34 | Head(ctx context.Context, path string) (*File, error) 35 | // List returns a series of *File on the first channel returned. If an 36 | // error occurs, the first channel is closed and the error is returned on 37 | // the second channel. 38 | List(ctx context.Context, path string, recursive bool) (<-chan *File, <-chan error) 39 | // Put writes a file to storage. A nil reader creates an empty file. 40 | Put(ctx context.Context, path string, reader io.Reader) (*File, error) 41 | // Type returns the file store type. 42 | Type() string 43 | } 44 | -------------------------------------------------------------------------------- /config/addresses.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // Addresses stores the (string) multiaddr addresses for the node. 4 | type Addresses struct { 5 | // Admin is the admin http listen address. Set to "none" to disable this 6 | // server for both http and libp2p. 7 | Admin string 8 | // Finder is the finder http isten address. Set to "none" to disable this 9 | // server for both http and libp2p. 10 | Finder string 11 | // Ingest is the index data ingestion http listen address. Set to "none" 12 | // to disable this server for both http and libp2p. 13 | Ingest string 14 | // P2PMaddr is the libp2p host multiaddr for all servers. Set to "none" to 15 | // disable libp2p hosting. 16 | P2PAddr string 17 | // ReverseIndexer is the reverse indexer http listen address. Set to "none" to 18 | // disable this server for both http and libp2p. 19 | ReverseIndexer string 20 | // NoResourceManager disables the libp2p resource manager when true. 21 | NoResourceManager bool 22 | } 23 | 24 | // NewAddresses returns Addresses with values set to their defaults. 25 | func NewAddresses() Addresses { 26 | return Addresses{ 27 | Admin: "/ip4/127.0.0.1/tcp/3002", 28 | Finder: "/ip4/0.0.0.0/tcp/3000", 29 | Ingest: "/ip4/0.0.0.0/tcp/3001", 30 | P2PAddr: "/ip4/0.0.0.0/tcp/3003", 31 | ReverseIndexer: "0.0.0.0:3004", 32 | } 33 | } 34 | 35 | // populateUnset replaces zero-values in the config with default values. 36 | func (c *Addresses) populateUnset() { 37 | def := NewAddresses() 38 | 39 | if c.Admin == "" { 40 | c.Admin = def.Admin 41 | } 42 | if c.Finder == "" { 43 | c.Finder = def.Finder 44 | } 45 | if c.Ingest == "" { 46 | c.Ingest = def.Ingest 47 | } 48 | if c.P2PAddr == "" { 49 | c.P2PAddr = def.P2PAddr 50 | } 51 | if c.ReverseIndexer == "" { 52 | c.ReverseIndexer = def.ReverseIndexer 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /deploy/infrastructure/common/github_actions.tf: -------------------------------------------------------------------------------- 1 | resource "aws_iam_openid_connect_provider" "github" { 2 | client_id_list = [ 3 | "https://github.com/filecoin-project", 4 | "https://github.com/filecoin-shipyard", 5 | "sts.amazonaws.com" 6 | ] 7 | 8 | thumbprint_list = [ 9 | "6938fd4d98bab03faadb97b34396831e3780aea1", 10 | "1c58a3a8518e8759bf075b76b750d4f2df264fcd" 11 | ] 12 | url = "https://token.actions.githubusercontent.com" 13 | } 14 | 15 | data "aws_iam_policy_document" "github_actions" { 16 | statement { 17 | effect = "Allow" 18 | actions = [ 19 | "ecr:GetAuthorizationToken", 20 | "ecr:BatchCheckLayerAvailability", 21 | "ecr:CompleteLayerUpload", 22 | "ecr:InitiateLayerUpload", 23 | "ecr:PutImage", 24 | "ecr:UploadLayerPart", 25 | "ecr-public:BatchCheckLayerAvailability", 26 | "ecr-public:CompleteLayerUpload", 27 | "ecr-public:InitiateLayerUpload", 28 | "ecr-public:PutImage", 29 | "ecr-public:UploadLayerPart", 30 | "ecr-public:GetAuthorizationToken", 31 | "sts:GetServiceBearerToken", 32 | ] 33 | resources = ["*"] 34 | } 35 | } 36 | 37 | resource "aws_iam_policy" "github_actions" { 38 | name = "github_actions" 39 | policy = data.aws_iam_policy_document.github_actions.json 40 | path = local.iam_path 41 | } 42 | 43 | module "github_actions_role" { 44 | source = "registry.terraform.io/terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" 45 | version = "4.18.0" 46 | 47 | create_role = true 48 | 49 | role_name = "github_actions" 50 | role_path = local.iam_path 51 | provider_url = aws_iam_openid_connect_provider.github.url 52 | 53 | role_policy_arns = [ 54 | aws_iam_policy.github_actions.arn, 55 | ] 56 | 57 | oidc_subjects_with_wildcards = [ 58 | "repo:ipni/*:*" 59 | ] 60 | } 61 | -------------------------------------------------------------------------------- /deploy/manifests/base/storetheindex-single/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: indexer 5 | spec: 6 | # Do not run more than one replica; this overlay is designed to run a single instance only. 7 | replicas: 1 8 | strategy: 9 | rollingUpdate: 10 | maxSurge: 0 11 | maxUnavailable: 1 12 | selector: 13 | matchLabels: 14 | app: indexer 15 | template: 16 | metadata: 17 | labels: 18 | app: indexer 19 | spec: 20 | terminationGracePeriodSeconds: 300 21 | securityContext: 22 | runAsUser: 10001 23 | fsGroup: 532 24 | containers: 25 | - name: indexer 26 | image: storetheindex 27 | envFrom: 28 | - configMapRef: 29 | name: env-vars 30 | env: 31 | - name: STORETHEINDEX_PRIV_KEY_PATH 32 | value: /identity/identity.key 33 | - name: STORETHEINDEX_PATH 34 | value: /config 35 | ports: 36 | - containerPort: 3000 37 | name: finder 38 | - containerPort: 3001 39 | name: ingest 40 | - containerPort: 3002 41 | name: admin 42 | volumeMounts: 43 | - name: identity 44 | mountPath: /identity 45 | - name: config 46 | mountPath: /config 47 | readinessProbe: 48 | httpGet: 49 | port: finder 50 | path: /health 51 | initialDelaySeconds: 10 52 | failureThreshold: 3 53 | successThreshold: 1 54 | timeoutSeconds: 5 55 | periodSeconds: 10 56 | volumes: 57 | - name: identity 58 | secret: 59 | secretName: identity 60 | - name: config 61 | configMap: 62 | name: config 63 | -------------------------------------------------------------------------------- /deploy/manifests/base/promtail/daemonset.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: DaemonSet 3 | metadata: 4 | name: promtail 5 | spec: 6 | minReadySeconds: 10 7 | selector: 8 | matchLabels: 9 | name: promtail 10 | template: 11 | metadata: 12 | labels: 13 | name: promtail 14 | spec: 15 | containers: 16 | - name: promtail 17 | image: grafana/promtail:2.5.0 18 | args: 19 | - -config.file=/etc/promtail/config.yaml 20 | - -config.expand-env=true 21 | env: 22 | - name: HOSTNAME 23 | valueFrom: 24 | fieldRef: 25 | fieldPath: spec.nodeName 26 | envFrom: 27 | - configMapRef: 28 | name: promtail-env 29 | - secretRef: 30 | name: promtail-auth 31 | readinessProbe: 32 | httpGet: 33 | path: /ready 34 | port: http-metrics 35 | ports: 36 | - containerPort: 80 37 | name: http-metrics 38 | securityContext: 39 | privileged: true 40 | runAsUser: 0 41 | volumeMounts: 42 | - name: config 43 | mountPath: /etc/promtail 44 | - name: varlog 45 | mountPath: /var/log 46 | - name: varlibdockercontainers 47 | mountPath: /var/lib/docker/containers 48 | readOnly: true 49 | serviceAccountName: promtail 50 | tolerations: 51 | - effect: NoSchedule 52 | operator: Exists 53 | volumes: 54 | - configMap: 55 | name: promtail-config 56 | name: config 57 | - hostPath: 58 | path: /var/log 59 | name: varlog 60 | - hostPath: 61 | path: /var/lib/docker/containers 62 | name: varlibdockercontainers 63 | -------------------------------------------------------------------------------- /deploy/manifests/base/assigner/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: assigner 5 | spec: 6 | # Do not run more than one replica; this overlay is designed to run a single instance only. 7 | replicas: 1 8 | selector: 9 | matchLabels: 10 | app: assigner 11 | # Terminate previous assigner before rolling out new ones to avoid conflicts between assignments. 12 | strategy: 13 | type: Recreate 14 | # Explicitly override the default rolling update configuration. Otherwise, the manifest will be 15 | # invalid when strategy.type is set to Recreate. 16 | # See: 17 | # - https://github.com/kubernetes/kubernetes/issues/24198 18 | rollingUpdate: null 19 | template: 20 | metadata: 21 | labels: 22 | app: assigner 23 | spec: 24 | terminationGracePeriodSeconds: 30 25 | securityContext: 26 | runAsUser: 10001 27 | fsGroup: 532 28 | containers: 29 | - name: assigner 30 | image: storetheindex 31 | args: 32 | - 'assigner' 33 | - 'daemon' 34 | env: 35 | - name: GOLOG_LOG_LEVEL 36 | value: INFO 37 | - name: GOLOG_LOG_FMT 38 | value: json 39 | - name: STORETHEINDEX_PRIV_KEY_PATH 40 | value: /identity/identity.key 41 | - name: ASSIGNER_PATH 42 | value: /config 43 | ports: 44 | - containerPort: 3001 45 | name: http 46 | - containerPort: 3003 47 | name: libp2p 48 | - containerPort: 8081 49 | name: metrics 50 | readinessProbe: 51 | httpGet: 52 | port: http 53 | path: /health 54 | initialDelaySeconds: 10 55 | failureThreshold: 3 56 | successThreshold: 1 57 | timeoutSeconds: 5 58 | periodSeconds: 10 59 | -------------------------------------------------------------------------------- /server/admin/config_handler.go: -------------------------------------------------------------------------------- 1 | package admin 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "sort" 7 | 8 | logging "github.com/ipfs/go-log/v2" 9 | "github.com/ipni/storetheindex/internal/httpserver" 10 | ) 11 | 12 | // setLogLevel sets the log level for a subsystem matched using regular expression. 13 | // Multiple subsystems and levels may be specified as query parameters where 14 | // the key signifies a rgular expression matching a subsystem, and value 15 | // signifies the desired level. At least on such pair must be specified. 16 | // 17 | // See: registerSetLogLevelHandler. 18 | func setLogLevel(w http.ResponseWriter, r *http.Request) { 19 | if !httpserver.MethodOK(w, r, http.MethodPost) { 20 | return 21 | } 22 | 23 | query := r.URL.Query() 24 | qSize := len(query) 25 | if qSize == 0 { 26 | http.Error(w, "at least one = query parameter must be specified", http.StatusBadRequest) 27 | return 28 | } 29 | 30 | for ss := range query { 31 | l := query.Get(ss) 32 | if l == "" { 33 | log.Errorw("Level not set for subsystem", "subsystem", ss) 34 | http.Error(w, 35 | fmt.Sprintf("level is not specified for subsystem: %s", ss), 36 | http.StatusBadRequest) 37 | return 38 | } 39 | if err := logging.SetLogLevelRegex(ss, l); err != nil { 40 | log.Errorw("Failed to set log level", "subsystem", ss, "level", l, "err", err) 41 | http.Error(w, err.Error(), http.StatusInternalServerError) 42 | return 43 | } 44 | log.Infow("Log level changed", "subsystem", ss, "level", l) 45 | } 46 | } 47 | 48 | // listLogSubSystems prints current logging subsystems one at a line. 49 | func listLogSubSystems(w http.ResponseWriter, r *http.Request) { 50 | if !httpserver.MethodOK(w, r, http.MethodGet) { 51 | return 52 | } 53 | 54 | subsystems := logging.GetSubsystems() 55 | sort.Strings(subsystems) 56 | for _, ss := range subsystems { 57 | _, _ = fmt.Fprintln(w, ss) 58 | } 59 | log.Debugw("Listed logging subsystems", "subsystems", subsystems) 60 | } 61 | -------------------------------------------------------------------------------- /rate/map.go: -------------------------------------------------------------------------------- 1 | package rate 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | ) 7 | 8 | type Rate struct { 9 | Count uint64 10 | Elapsed time.Duration 11 | Samples int 12 | 13 | observed bool 14 | } 15 | 16 | // Map records the cumulative rate, since the last call to Get or GetAll, 17 | // for each ID. 18 | type Map struct { 19 | mutex sync.Mutex 20 | rates map[string]Rate 21 | } 22 | 23 | func NewMap() *Map { 24 | return &Map{ 25 | rates: map[string]Rate{}, 26 | } 27 | } 28 | 29 | // Update adds the new rate information to the existing information. If the 30 | // existing rate information is marked as observed, then it is reset before 31 | // applying the new information. 32 | func (m *Map) Update(id string, count uint64, elapsed time.Duration) { 33 | if m == nil || count == 0 || elapsed == 0 { 34 | return 35 | } 36 | m.mutex.Lock() 37 | rate := m.rates[id] 38 | if rate.observed { 39 | rate = Rate{} 40 | } 41 | rate.Count += count 42 | rate.Elapsed += elapsed 43 | rate.Samples++ 44 | m.rates[id] = rate 45 | m.mutex.Unlock() 46 | } 47 | 48 | // Get reads the accumulated rate information for the specified ID. The 49 | // information is marked as observed and is reset at next write. 50 | func (m *Map) Get(id string) (Rate, bool) { 51 | if m == nil { 52 | return Rate{}, false 53 | } 54 | m.mutex.Lock() 55 | r, ok := m.rates[id] 56 | if ok && !r.observed { 57 | r.observed = true 58 | m.rates[id] = r 59 | } 60 | m.mutex.Unlock() 61 | return r, ok 62 | } 63 | 64 | // GetAll reads and removes all accumulated rate information. The 65 | // information is marked as observed and is reset at next write. 66 | func (m *Map) GetAll() map[string]Rate { 67 | if m == nil { 68 | return nil 69 | } 70 | m.mutex.Lock() 71 | if len(m.rates) == 0 { 72 | m.mutex.Unlock() 73 | return nil 74 | } 75 | out := make(map[string]Rate, len(m.rates)) 76 | for id, r := range m.rates { 77 | if !r.observed { 78 | r.observed = true 79 | m.rates[id] = r 80 | } 81 | out[id] = r 82 | } 83 | m.mutex.Unlock() 84 | return out 85 | } 86 | -------------------------------------------------------------------------------- /config/init.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "crypto/rand" 5 | "encoding/base64" 6 | "fmt" 7 | "io" 8 | 9 | "github.com/cockroachdb/pebble" 10 | "github.com/libp2p/go-libp2p/core/crypto" 11 | "github.com/libp2p/go-libp2p/core/peer" 12 | ) 13 | 14 | func Init(out io.Writer) (*Config, error) { 15 | identity, err := CreateIdentity(out) 16 | if err != nil { 17 | return nil, err 18 | } 19 | 20 | return InitWithIdentity(identity) 21 | } 22 | 23 | func InitWithIdentity(identity Identity) (*Config, error) { 24 | conf := &Config{ 25 | Version: Version, 26 | Addresses: NewAddresses(), 27 | Bootstrap: NewBootstrap(), 28 | Datastore: NewDatastore(), 29 | Discovery: NewDiscovery(), 30 | Finder: NewFinder(), 31 | Identity: identity, 32 | Indexer: NewIndexer(), 33 | ReverseIndexer: NewReverseIndexer(), 34 | Ingest: NewIngest(), 35 | Logging: NewLogging(), 36 | } 37 | 38 | if conf.Indexer.ValueStoreType == "pebble" { 39 | conf.Indexer.PebbleFormatMajorVersion = int(pebble.FormatNewest) 40 | } 41 | 42 | return conf, nil 43 | } 44 | 45 | // CreateIdentity initializes a new identity. 46 | func CreateIdentity(out io.Writer) (Identity, error) { 47 | ident := Identity{} 48 | 49 | var sk crypto.PrivKey 50 | var pk crypto.PubKey 51 | 52 | fmt.Fprintf(out, "generating ED25519 keypair...") 53 | priv, pub, err := crypto.GenerateEd25519Key(rand.Reader) 54 | if err != nil { 55 | return ident, err 56 | } 57 | fmt.Fprintln(out, "done") 58 | 59 | sk = priv 60 | pk = pub 61 | 62 | // currently storing key unencrypted. in the future we need to encrypt it. 63 | // TODO(security) 64 | skbytes, err := crypto.MarshalPrivateKey(sk) 65 | if err != nil { 66 | return ident, err 67 | } 68 | ident.PrivKey = base64.StdEncoding.EncodeToString(skbytes) 69 | 70 | id, err := peer.IDFromPublicKey(pk) 71 | if err != nil { 72 | return ident, err 73 | } 74 | ident.PeerID = id.String() 75 | fmt.Fprintln(out, "peer identity:", ident.PeerID) 76 | return ident, nil 77 | } 78 | -------------------------------------------------------------------------------- /deploy/infrastructure/prod/us-east-2/monitoring.tf: -------------------------------------------------------------------------------- 1 | resource "aws_prometheus_workspace" "monitoring" { 2 | alias = local.environment_name 3 | } 4 | 5 | data "aws_iam_policy_document" "monitoring" { 6 | statement { 7 | effect = "Allow" 8 | 9 | actions = [ 10 | "aps:RemoteWrite", 11 | "aps:QueryMetrics", 12 | "aps:GetSeries", 13 | "aps:GetLabels", 14 | "aps:GetMetricMetadata" 15 | ] 16 | 17 | resources = [aws_prometheus_workspace.monitoring.arn] 18 | } 19 | } 20 | 21 | resource "aws_iam_policy" "monitoring" { 22 | name = "${local.environment_name}_monitoring" 23 | policy = data.aws_iam_policy_document.monitoring.json 24 | } 25 | 26 | module "monitoring_role" { 27 | source = "registry.terraform.io/terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" 28 | version = "5.20.0" 29 | 30 | create_role = true 31 | 32 | role_name = "monitoring" 33 | role_path = local.iam_path 34 | provider_url = module.eks.oidc_provider 35 | 36 | role_policy_arns = [ 37 | aws_iam_policy.monitoring.arn, 38 | ] 39 | 40 | oidc_fully_qualified_subjects = ["system:serviceaccount:monitoring:prometheus-k8s"] 41 | } 42 | 43 | resource "aws_security_group" "vpc_tls" { 44 | name = "${local.environment_name}_vpc_tls" 45 | description = "Allow TLS inbound traffic" 46 | vpc_id = module.vpc.vpc_id 47 | 48 | ingress { 49 | description = "TLS from VPC" 50 | from_port = 443 51 | to_port = 443 52 | protocol = "tcp" 53 | cidr_blocks = [module.vpc.vpc_cidr_block] 54 | } 55 | } 56 | 57 | module "endpoints" { 58 | source = "registry.terraform.io/terraform-aws-modules/vpc/aws//modules/vpc-endpoints" 59 | version = "5.0.0" 60 | 61 | vpc_id = module.vpc.vpc_id 62 | security_group_ids = [aws_security_group.vpc_tls.id] 63 | subnet_ids = local.initial_private_subnet_ids 64 | 65 | endpoints = { 66 | aps-workspaces = { 67 | service = "aps-workspaces" 68 | vpc_endpoint_type = "Interface" 69 | private_dns_enabled = true 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /server/find/options.go: -------------------------------------------------------------------------------- 1 | package find 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | const ( 9 | defaultHomepage = "https://web-ipni.cid.contact/" 10 | defaultMaxConns = 8_000 11 | defaultReadTimeout = 30 * time.Second 12 | defaultWriteTimeout = 30 * time.Second 13 | ) 14 | 15 | // config contains all options for the server. 16 | type config struct { 17 | homepageURL string 18 | maxConns int 19 | readTimeout time.Duration 20 | writeTimeout time.Duration 21 | version string 22 | } 23 | 24 | // Option is a function that sets a value in a config. 25 | type Option func(*config) error 26 | 27 | // getOpts creates a config and applies Options to it. 28 | func getOpts(opts []Option) (config, error) { 29 | cfg := config{ 30 | homepageURL: defaultHomepage, 31 | maxConns: defaultMaxConns, 32 | readTimeout: defaultReadTimeout, 33 | writeTimeout: defaultWriteTimeout, 34 | } 35 | 36 | for i, opt := range opts { 37 | if err := opt(&cfg); err != nil { 38 | return config{}, fmt.Errorf("option %d error: %s", i, err) 39 | } 40 | } 41 | return cfg, nil 42 | } 43 | 44 | // WithHomepage config for API. 45 | func WithHomepage(URL string) Option { 46 | return func(c *config) error { 47 | c.homepageURL = URL 48 | return nil 49 | } 50 | } 51 | 52 | // MaxConnections config allowed by server. 53 | func WithMaxConnections(maxConnections int) Option { 54 | return func(c *config) error { 55 | c.maxConns = maxConnections 56 | return nil 57 | } 58 | } 59 | 60 | // WithReadTimeout configures server read timeout. 61 | func WithReadTimeout(t time.Duration) Option { 62 | return func(c *config) error { 63 | c.readTimeout = t 64 | return nil 65 | } 66 | } 67 | 68 | // WithWriteTimeout configures server write timeout. 69 | func WithWriteTimeout(t time.Duration) Option { 70 | return func(c *config) error { 71 | c.writeTimeout = t 72 | return nil 73 | } 74 | } 75 | 76 | // WithVersion sets the version string used in /health output. 77 | func WithVersion(ver string) Option { 78 | return func(c *config) error { 79 | c.version = ver 80 | return nil 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /command/init_test.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/ipni/storetheindex/config" 9 | "github.com/stretchr/testify/require" 10 | "github.com/urfave/cli/v2" 11 | ) 12 | 13 | func TestInit(t *testing.T) { 14 | // Set up a context that is canceled when the command is interrupted 15 | ctx, cancel := context.WithCancel(context.Background()) 16 | defer cancel() 17 | 18 | tempDir := t.TempDir() 19 | t.Setenv(config.EnvDir, tempDir) 20 | 21 | app := &cli.App{ 22 | Name: "indexer", 23 | Commands: []*cli.Command{ 24 | InitCmd, 25 | }, 26 | } 27 | 28 | const ( 29 | badAddr = "ip3/127.0.0.1/tcp/9999" 30 | cacheSize = 2701 31 | goodAddr = "/ip4/127.0.0.1/tcp/7777" 32 | goodAddr2 = "/ip4/127.0.0.1/tcp/17171" 33 | storeType = "pebble" 34 | topicName = "index/mytopic" 35 | ) 36 | 37 | err := app.RunContext(ctx, []string{"storetheindex", "init", "-listen-admin", badAddr}) 38 | require.Error(t, err) 39 | 40 | err = app.RunContext(ctx, []string{"storetheindex", "init", "-listen-finder", badAddr}) 41 | require.Error(t, err) 42 | 43 | err = app.RunContext(ctx, []string{"storetheindex", "init", "-listen-ingest", badAddr}) 44 | require.Error(t, err) 45 | 46 | args := []string{ 47 | "storetheindex", "init", 48 | "-listen-finder", goodAddr, 49 | "-listen-ingest", goodAddr2, 50 | "-cachesize", fmt.Sprint(cacheSize), 51 | "--pubsub-topic", topicName, 52 | "-store", storeType, 53 | } 54 | err = app.RunContext(ctx, args) 55 | require.NoError(t, err) 56 | 57 | cfg, err := config.Load("") 58 | require.NoError(t, err) 59 | 60 | require.Equal(t, goodAddr, cfg.Addresses.Finder, "finder listen address was not configured") 61 | require.Equal(t, goodAddr2, cfg.Addresses.Ingest, "ingest listen address was not configured") 62 | require.Equal(t, cacheSize, cfg.Indexer.CacheSize, "cache size was tno configured") 63 | require.Equal(t, storeType, cfg.Indexer.ValueStoreType, "value store type was not configured") 64 | require.Equal(t, topicName, cfg.Ingest.PubSubTopic) 65 | require.Equal(t, config.Version, cfg.Version) 66 | 67 | t.Log(cfg.String()) 68 | } 69 | -------------------------------------------------------------------------------- /test/load/README.md: -------------------------------------------------------------------------------- 1 | # Load tests 2 | 3 | ## Start a daemon 4 | To run load tests over `storetheindex` we need to first initialize and run the indexer daemon: 5 | 6 | ### Initialize and start a `storetheindex` daemon: 7 | ``` 8 | export STORETHEINDEX_PATH=/tmp/sti_test 9 | ./storetheindex init 10 | ./storetheindex daemon --cachesize --dir 11 | ``` 12 | 13 | ### Initialize and start an `index-provider` daemon 14 | ``` 15 | ./provider init 16 | edit ~/.index-provider/config 17 | ``` 18 | 19 | Configure the provider to announce to the indexer. 20 | ```json 21 | "DirectAnnounce": { 22 | "URLs": ["http://127.0.0.1:3001"] 23 | } 24 | ``` 25 | 26 | Start the daemon 27 | ``` 28 | ./provider daemon 29 | ``` 30 | 31 | ### Import sample car files into the provider, which will then go to the indexer indexer: 32 | ``` 33 | ./provider import car -i testdata/sample-v1.car 34 | ``` 35 | 36 | ## Run the test 37 | We can then start the load tests client. The load test script has `locust` and `numpy` as dependencies. Be sure that you have `python3` installed 38 | and that you run `pip install locust numpy`. 39 | 40 | - A test run with 4 workers and a master client can be easily run through the `./run.sh` script. Be sure to generate the test data as described above. The `run.sh` script starts the client workers. To configure the test run and visualize the results, go to locust UI at `http://localhost:8089`. 41 | - Locust can also be run from the CLI. To stress test a single endpoint, use [this tool](https://github.com/rakyll/hey) as follows: 42 | ``` 43 | ./hey_linux_amd64 -m GET -z -c /cid/bafkreigxvijvpvmt7xnk2nxzudha22jf7fawi2vbjpsnh7cejagquq6z4y 44 | 45 | # Example 46 | ./hey_linux_amd64 -m GET -z 30s -c 10000 http://127.0.0.1:3000/cid/bafkreigxvijvpvmt7xnk2nxzudha22jf7fawi2vbjpsnh7cejagquq6z4y 47 | ``` 48 | 49 | **_File Descriptor Limit_** 50 | 51 | When trying to run a huge load over the node, the local client may reach the OS file descriptor limit (even if this is set to the maximum limit allowed). To send more load to the node, the load generation needs to be distributed over multiple machines. 52 | -------------------------------------------------------------------------------- /command/log.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | 7 | "github.com/ipni/storetheindex/admin/client" 8 | "github.com/urfave/cli/v2" 9 | ) 10 | 11 | var subsystemsCmd = &cli.Command{ 12 | Name: "subsystems", 13 | Usage: "Lists the available logging subsystems", 14 | Action: listLogSubsystemsAction, 15 | } 16 | 17 | var setLevelCmd = &cli.Command{ 18 | Name: "setlevel", 19 | Usage: "Sets log level for logging subsystems", 20 | Description: `Configures log level for logging subystems specified as pairs of arguments. 21 | Subsystem may be specified as a regular expression. For example the 22 | following command will set the level for all subsystems to debug: 23 | 24 | storetheindex log setlevel '.*' debug`, 25 | ArgsUsage: " [ ]...", 26 | Action: setLogLevelAction, 27 | } 28 | 29 | var LogCmd = &cli.Command{ 30 | Name: "log", 31 | Usage: "Show log subsystems and modify log levels", 32 | Flags: []cli.Flag{indexerHostFlag}, 33 | Subcommands: []*cli.Command{ 34 | subsystemsCmd, 35 | setLevelCmd, 36 | }, 37 | } 38 | 39 | func setLogLevelAction(cctx *cli.Context) error { 40 | cl, err := client.New(cliIndexer(cctx, "admin")) 41 | if err != nil { 42 | return err 43 | } 44 | 45 | args := cctx.Args() 46 | if !args.Present() { 47 | return errors.New("at least one argument pair must be specified") 48 | } 49 | 50 | if args.Len()%2 != 0 { 51 | return fmt.Errorf(" argument must be specified as pairs") 52 | } 53 | 54 | sysLvl := make(map[string]string, args.Len()/2) 55 | for i := 0; i < args.Len(); i += 2 { 56 | subsys := args.Get(i) 57 | level := args.Get(i + 1) 58 | sysLvl[subsys] = level 59 | } 60 | return cl.SetLogLevels(cctx.Context, sysLvl) 61 | } 62 | 63 | func listLogSubsystemsAction(cctx *cli.Context) error { 64 | cl, err := client.New(cliIndexer(cctx, "admin")) 65 | if err != nil { 66 | return err 67 | } 68 | 69 | subSystems, err := cl.ListLogSubSystems(cctx.Context) 70 | if err != nil { 71 | return err 72 | } 73 | 74 | for _, ss := range subSystems { 75 | _, _ = fmt.Fprintln(cctx.App.Writer, ss) 76 | } 77 | return nil 78 | } 79 | -------------------------------------------------------------------------------- /.github/workflows/publish-ghcr.yml: -------------------------------------------------------------------------------- 1 | name: Container 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'main' 7 | tags: 8 | - 'v*' 9 | workflow_run: 10 | workflows: [ Releaser ] 11 | types: 12 | - completed 13 | pull_request: 14 | 15 | jobs: 16 | prepare-checkout: 17 | if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' 18 | name: Prepare ref 19 | runs-on: ubuntu-latest 20 | outputs: 21 | ref: ${{ github.event_name != 'workflow_run' && github.ref || steps.releaser.outputs.version }} 22 | steps: 23 | - name: Get Ref from releaser 24 | id: releaser 25 | if: github.event_name == 'workflow_run' 26 | uses: ipdxco/unified-github-workflows/.github/actions/inspect-releaser@v1.0 27 | with: 28 | artifacts-url: ${{ github.event.workflow_run.artifacts_url }} 29 | publish: 30 | name: Publish 31 | needs: [ prepare-checkout ] 32 | runs-on: ubuntu-latest 33 | permissions: 34 | contents: read 35 | packages: write 36 | steps: 37 | - name: Checkout 38 | uses: actions/checkout@v6 39 | with: 40 | ref: ${{ needs.prepare-checkout.outputs.ref }} 41 | - name: Set up Docker Buildx 42 | uses: docker/setup-buildx-action@v3 43 | - name: Log in to the Container registry 44 | uses: docker/login-action@v3 45 | with: 46 | registry: ghcr.io 47 | username: ${{ github.actor }} 48 | password: ${{ github.token }} 49 | - name: Extract metadata 50 | id: meta 51 | uses: docker/metadata-action@v5 52 | with: 53 | images: ghcr.io/${{ github.repository }} 54 | tags: | 55 | type=semver,pattern={{raw}} 56 | type=ref,event=branch 57 | type=raw,value=${{ needs.prepare-checkout.outputs.ref }} 58 | - name: Build and push Docker image 59 | uses: docker/build-push-action@v6 60 | with: 61 | context: . 62 | cache-from: type=gha 63 | cache-to: type=gha,mode=max 64 | push: ${{ github.event_name != 'pull_request' }} 65 | tags: ${{ steps.meta.outputs.tags }} 66 | labels: ${{ steps.meta.outputs.labels }} 67 | -------------------------------------------------------------------------------- /internal/ingest/seg_sync_test.go: -------------------------------------------------------------------------------- 1 | package ingest 2 | 3 | import ( 4 | "context" 5 | "math/rand" 6 | "testing" 7 | 8 | cidlink "github.com/ipld/go-ipld-prime/linking/cid" 9 | "github.com/ipni/storetheindex/config" 10 | "github.com/ipni/storetheindex/test/typehelpers" 11 | "github.com/libp2p/go-libp2p/core/peer" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestAdsSyncedViaSegmentsAreProcessed(t *testing.T) { 16 | t.Parallel() 17 | cfg := config.NewIngest() 18 | cfg.PubSubTopic = defaultTestIngestConfig.PubSubTopic 19 | cfg.SyncSegmentDepthLimit = 7 20 | 21 | te := setupTestEnv(t, true, func(opts *testEnvOpts) { 22 | opts.ingestConfig = &cfg 23 | }) 24 | rng := rand.New(rand.NewSource(1413)) 25 | var cb []typehelpers.EntryBuilder 26 | for i := 0; i < 50; i++ { 27 | chunkCount := rng.Int31n(50) 28 | ePerChunk := rng.Int31n(50) 29 | seed := rng.Int63() 30 | kindChunk := rng.Float32() > 0.5 // Flip a coin to decide what kind of entries to generate. 31 | if kindChunk { 32 | cb = append(cb, typehelpers.RandomEntryChunkBuilder{ChunkCount: uint8(chunkCount), EntriesPerChunk: uint8(ePerChunk), Seed: seed}) 33 | } else { 34 | cb = append(cb, typehelpers.RandomHamtEntryBuilder{MultihashCount: uint32(chunkCount * ePerChunk), Seed: seed}) 35 | } 36 | } 37 | 38 | headAd := typehelpers.RandomAdBuilder{ 39 | EntryBuilders: cb, 40 | }.Build(t, te.publisherLinkSys, te.publisherPriv) 41 | headAdCid := headAd.(cidlink.Link).Cid 42 | 43 | ctx := context.Background() 44 | te.publisher.SetRoot(headAdCid) 45 | mhs := typehelpers.AllMultihashesFromAdLink(t, headAd, te.publisherLinkSys) 46 | 47 | subject := te.ingester 48 | 49 | pubInfo := peer.AddrInfo{ 50 | ID: te.publisher.ID(), 51 | } 52 | gotHeadAd, err := subject.Sync(ctx, pubInfo, 0, false) 53 | require.NoError(t, err) 54 | require.Equal(t, headAdCid, gotHeadAd, "Expected latest synced cid to match head of ad chain") 55 | 56 | require.Eventually(t, func() bool { 57 | return checkAllIndexed(subject.indexer, pubInfo.ID, mhs) == nil 58 | }, testRetryTimeout, testRetryInterval, "Expected all ads from publisher to have been indexed.") 59 | 60 | require.Eventually(t, func() bool { 61 | latestSync, err := subject.GetLatestSync(pubInfo.ID) 62 | require.NoError(t, err) 63 | return latestSync.Equals(headAdCid) 64 | }, testRetryTimeout, testRetryInterval, "Expected all ads from publisher to have been indexed.") 65 | } 66 | -------------------------------------------------------------------------------- /assigner/command/init.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/ipni/storetheindex/assigner/config" 8 | sticfg "github.com/ipni/storetheindex/config" 9 | "github.com/ipni/storetheindex/fsutil" 10 | "github.com/urfave/cli/v2" 11 | ) 12 | 13 | var InitCmd = &cli.Command{ 14 | Name: "init", 15 | Usage: "Initialize or upgrade config file", 16 | Flags: initFlags, 17 | Action: initAction, 18 | } 19 | 20 | var initFlags = []cli.Flag{ 21 | &cli.BoolFlag{ 22 | Name: "no-bootstrap", 23 | Usage: "Do not configure bootstrap peers", 24 | EnvVars: []string{"NO_BOOTSTRAP"}, 25 | Required: false, 26 | }, 27 | &cli.StringFlag{ 28 | Name: "pubsub-topic", 29 | Usage: "Subscribe to this pubsub topic to receive advertisement notification", 30 | EnvVars: []string{"ASSIGNER_PUBSUB_TOPIC"}, 31 | Required: false, 32 | }, 33 | &cli.BoolFlag{ 34 | Name: "upgrade", 35 | Usage: "Upgrade the config file to the current version, saving the old config as config.prev, and ignoring other flags ", 36 | Aliases: []string{"u"}, 37 | Required: false, 38 | }, 39 | } 40 | 41 | func initAction(cctx *cli.Context) error { 42 | // Check that the config root exists and it writable. 43 | configRoot, err := config.PathRoot() 44 | if err != nil { 45 | return err 46 | } 47 | 48 | if err = fsutil.DirWritable(configRoot); err != nil { 49 | return err 50 | } 51 | 52 | configFile, err := config.Path(configRoot, "") 53 | if err != nil { 54 | return err 55 | } 56 | 57 | if cctx.Bool("upgrade") { 58 | cfg, err := config.Load(configFile) 59 | if err != nil { 60 | return err 61 | } 62 | prevVer := cfg.Version 63 | err = cfg.UpgradeConfig(configFile) 64 | if err != nil { 65 | return fmt.Errorf("cannot upgrade: %s", err) 66 | } 67 | fmt.Println("Upgraded", configFile, "from version", prevVer, "to", cfg.Version) 68 | return nil 69 | } 70 | 71 | fmt.Println("Initializing", progName, "at", configRoot) 72 | 73 | if fsutil.FileExists(configFile) { 74 | return sticfg.ErrInitialized 75 | } 76 | 77 | cfg, err := config.Init(os.Stderr) 78 | if err != nil { 79 | return err 80 | } 81 | 82 | noBootstrap := cctx.Bool("no-bootstrap") 83 | if noBootstrap { 84 | cfg.Bootstrap.Peers = []string{} 85 | cfg.Bootstrap.MinimumPeers = 0 86 | } 87 | 88 | topic := cctx.String("pubsub-topic") 89 | if topic != "" { 90 | cfg.Assignment.PubSubTopic = topic 91 | } 92 | 93 | return cfg.Save(configFile) 94 | } 95 | -------------------------------------------------------------------------------- /config/policy.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // Policy configures which peers are allowed to be providers, publish 4 | // advertisements and publish on behalf of other providers. The Allow policy 5 | // determines which peers the indexer will request advertisements from and 6 | // index content for. The Publish policy determines if a publisher may supply 7 | // an advertisement that has a provider that is different from the publisher. 8 | // 9 | // Publishers and providers are not the same. Publishers are peers that supply 10 | // data to the indexer. Providers are the peers that appear in advertisements 11 | // and are where retrieval clients get content from. 12 | type Policy struct { 13 | // Allow is either false or true, and determines whether a peer is allowed 14 | // (true) or is blocked (false), by default. If a peer if blocked, then it 15 | // cannot publish advertisements to this indexer or be listed as a provider 16 | // by this indexer. 17 | Allow bool 18 | // Except is a list of peer IDs that are exceptions to the Allow policy. 19 | // If Allow is true, then all peers are allowed except those listed in 20 | // Except. If Allow is false, then no peers are allowed except those listed 21 | // in Except. in other words, Allow=true means that Except is a deny-list 22 | // and Allow=false means that Except is an allow-list. 23 | Except []string 24 | // Publish is the default Allow policy when a provider has no policy in 25 | // PublisherForProviders. It determines if any peers are allowed to publish 26 | // advertisements for a provider with a differen peer ID. 27 | Publish bool 28 | // PublisherExcept is a list of peer IDs that are exceptions to the default 29 | // Publish policy. 30 | PublishExcept []string 31 | // PublishersForProvider is a list of policies specifying which publishers 32 | // are allowed to publish advertisements on behalf of a specified provider. 33 | PublishersForProvider []PublishersPolicy 34 | } 35 | 36 | type PublishersPolicy struct { 37 | // Provider is the provider peer ID this policy pertains to. 38 | Provider string 39 | // Allow determines whether a peer is allowed or blocked from publishing 40 | // advertisements on behalf of a provider with a differen peer ID. 41 | Allow bool 42 | // Except is a list of peer IDs that are exceptions to the Allow 43 | // policy. 44 | Except []string 45 | } 46 | 47 | // NewPolicy returns Policy with values set to their defaults. 48 | func NewPolicy() Policy { 49 | return Policy{ 50 | Allow: true, 51 | Publish: true, 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /config/types.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding" 5 | "fmt" 6 | "strconv" 7 | "strings" 8 | "time" 9 | ) 10 | 11 | // ByteSize wraps uint64 to provide json serialization and 12 | // deserialization. 13 | // 14 | // NOTE: the zero value encodes to an empty string. 15 | type ByteSize uint64 16 | 17 | const ( 18 | giSuffix = "Gi" 19 | giSize = 1 << 30 20 | miSuffix = "Mi" 21 | miSize = 1 << 20 22 | ) 23 | 24 | func (d *ByteSize) UnmarshalText(text []byte) error { 25 | str := strings.TrimSpace(string(text)) 26 | // If the value is emoty - defaulting to zero 27 | if len(str) == 0 { 28 | *d = ByteSize(0) 29 | return nil 30 | } 31 | // If there is less than two bytes - treating it as a number 32 | if len(str) <= 2 { 33 | n, err := strconv.Atoi(str) 34 | if err != nil { 35 | return err 36 | } 37 | *d = ByteSize(n) 38 | return nil 39 | } 40 | suffix := strings.ToLower(str[len(str)-2:]) 41 | multiplier := 1 42 | var n int 43 | var err error 44 | switch suffix { 45 | case strings.ToLower(miSuffix): 46 | n, err = strconv.Atoi(str[:len(str)-2]) 47 | if err != nil { 48 | return err 49 | } 50 | multiplier = miSize 51 | case strings.ToLower(giSuffix): 52 | n, err = strconv.Atoi(str[:len(str)-2]) 53 | if err != nil { 54 | return err 55 | } 56 | multiplier = giSize 57 | default: 58 | n, err = strconv.Atoi(str) 59 | } 60 | *d = ByteSize(n * multiplier) 61 | return err 62 | } 63 | 64 | func (d ByteSize) MarshalText() ([]byte, error) { 65 | return []byte(d.String()), nil 66 | } 67 | 68 | func (d ByteSize) String() string { 69 | if d%giSize == 0 { 70 | return fmt.Sprintf("%d%s", d/giSize, giSuffix) 71 | } else if d%miSize == 0 { 72 | return fmt.Sprintf("%d%s", d/miSize, miSuffix) 73 | } else { 74 | return fmt.Sprintf("%d", d) 75 | } 76 | } 77 | 78 | // Duration wraps time.Duration to provide json serialization and 79 | // deserialization. 80 | // 81 | // NOTE: the zero value encodes to an empty string. 82 | type Duration time.Duration 83 | 84 | func (d *Duration) UnmarshalText(text []byte) error { 85 | dur, err := time.ParseDuration(string(text)) 86 | *d = Duration(dur) 87 | return err 88 | } 89 | 90 | func (d Duration) MarshalText() ([]byte, error) { 91 | return []byte(time.Duration(d).String()), nil 92 | } 93 | 94 | func (d Duration) String() string { 95 | return time.Duration(d).String() 96 | } 97 | 98 | var _ encoding.TextUnmarshaler = (*Duration)(nil) 99 | var _ encoding.TextMarshaler = (*Duration)(nil) 100 | -------------------------------------------------------------------------------- /server/find/cached_stats.go: -------------------------------------------------------------------------------- 1 | package find 2 | 3 | import ( 4 | "context" 5 | "sync/atomic" 6 | "time" 7 | 8 | "github.com/ipni/go-indexer-core" 9 | "github.com/ipni/go-libipni/find/model" 10 | ) 11 | 12 | // avgMhSize is a slight overcount over the expected size of a multihash as a 13 | // way of estimating the number of entries in the primary value store. 14 | const avgMhSize = 40 15 | 16 | type ( 17 | cachedStats struct { 18 | ctx context.Context 19 | cancel context.CancelFunc 20 | 21 | indexer indexer.Interface 22 | ticker *time.Ticker 23 | 24 | latest atomic.Value // Stores *latestStats 25 | } 26 | latestStats struct { 27 | entriesEstimate int64 28 | entriesCount int64 29 | errEntriesEstimate error 30 | errEntriesCount error 31 | } 32 | ) 33 | 34 | func newCachedStats(indexer indexer.Interface, tick time.Duration) *cachedStats { 35 | c := &cachedStats{ 36 | indexer: indexer, 37 | ticker: time.NewTicker(tick), 38 | } 39 | c.ctx, c.cancel = context.WithCancel(context.Background()) 40 | c.latest.Store(&latestStats{}) 41 | c.start() 42 | return c 43 | } 44 | 45 | func (c *cachedStats) start() { 46 | go func() { 47 | c.refresh() 48 | for { 49 | select { 50 | case <-c.ctx.Done(): 51 | return 52 | case <-c.ticker.C: 53 | c.refresh() 54 | } 55 | } 56 | }() 57 | } 58 | 59 | func (c *cachedStats) refresh() { 60 | lastResult := c.latest.Load().(*latestStats) 61 | var s *indexer.Stats 62 | var newResult latestStats 63 | // Only check the `stats` endpoint once; if the valuestore does not support it there is no 64 | // point checking it at every cycle. 65 | if lastResult.errEntriesCount != indexer.ErrStatsNotSupported { 66 | s, newResult.errEntriesCount = c.indexer.Stats() 67 | if newResult.errEntriesCount == nil && s != nil { 68 | newResult.entriesCount = int64(s.MultihashCount) 69 | } 70 | } 71 | var size int64 72 | size, newResult.errEntriesEstimate = c.indexer.Size() 73 | if newResult.errEntriesEstimate == nil { 74 | newResult.entriesEstimate = size / avgMhSize 75 | } 76 | c.latest.Store(&newResult) 77 | } 78 | 79 | func (c *cachedStats) get() (s model.Stats, err error) { 80 | r := c.latest.Load().(*latestStats) 81 | s.EntriesEstimate = r.entriesEstimate 82 | s.EntriesCount = r.entriesCount 83 | 84 | if r.errEntriesCount != nil && r.errEntriesCount != indexer.ErrStatsNotSupported { 85 | log.Warn("Failed to get EntriesCount", "err", r.errEntriesCount) 86 | } 87 | 88 | err = r.errEntriesEstimate 89 | return 90 | } 91 | 92 | func (c *cachedStats) close() { 93 | c.ticker.Stop() 94 | c.cancel() 95 | } 96 | -------------------------------------------------------------------------------- /fsutil/fsutil.go: -------------------------------------------------------------------------------- 1 | package fsutil 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io/fs" 7 | "os" 8 | "path/filepath" 9 | "time" 10 | ) 11 | 12 | // DirWritable checks if a directory is writable. If the directory does 13 | // not exist it is created with writable permission. 14 | func DirWritable(dir string) error { 15 | if dir == "" { 16 | return errors.New("directory not specified") 17 | } 18 | 19 | var err error 20 | dir, err = ExpandHome(dir) 21 | if err != nil { 22 | return err 23 | } 24 | fi, err := os.Stat(dir) 25 | if err != nil { 26 | if errors.Is(err, fs.ErrNotExist) { 27 | // Directory does not exist, so create it. 28 | err = os.Mkdir(dir, 0775) 29 | if err == nil { 30 | return nil 31 | } 32 | } 33 | if errors.Is(err, fs.ErrPermission) { 34 | err = fs.ErrPermission 35 | } 36 | return fmt.Errorf("directory not writable: %s: %w", dir, err) 37 | } 38 | if !fi.IsDir() { 39 | return fmt.Errorf("not a directory: %s", dir) 40 | } 41 | 42 | // Directory exists, check that a file can be written. 43 | file, err := os.CreateTemp(dir, "writetest") 44 | if err != nil { 45 | if errors.Is(err, fs.ErrPermission) { 46 | err = fs.ErrPermission 47 | } 48 | return fmt.Errorf("directory not writable: %s: %w", dir, err) 49 | } 50 | file.Close() 51 | return os.Remove(file.Name()) 52 | } 53 | 54 | // ExpandHome expands the path to include the home directory if the path is 55 | // prefixed with `~`. If it isn't prefixed with `~`, the path is returned 56 | // as-is. 57 | func ExpandHome(path string) (string, error) { 58 | if path == "" { 59 | return path, nil 60 | } 61 | 62 | if path[0] != '~' { 63 | return path, nil 64 | } 65 | 66 | if len(path) > 1 && path[1] != '/' && path[1] != '\\' { 67 | return "", errors.New("cannot expand user-specific home dir") 68 | } 69 | 70 | dir, err := os.UserHomeDir() 71 | if err != nil { 72 | return "", err 73 | } 74 | 75 | return filepath.Join(dir, path[1:]), nil 76 | } 77 | 78 | // FileChanged returns the modification time of a file and true if different 79 | // from the given time. 80 | func FileChanged(filePath string, modTime time.Time) (time.Time, bool, error) { 81 | fi, err := os.Stat(filePath) 82 | if err != nil { 83 | return modTime, false, fmt.Errorf("cannot stat file %s: %w", filePath, err) 84 | } 85 | if fi.ModTime() != modTime { 86 | return fi.ModTime(), true, nil 87 | } 88 | return modTime, false, nil 89 | } 90 | 91 | // FileExists return true if the file exists 92 | func FileExists(filename string) bool { 93 | _, err := os.Lstat(filename) 94 | return !errors.Is(err, os.ErrNotExist) 95 | } 96 | -------------------------------------------------------------------------------- /scripts/cross_announce/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "log" 8 | 9 | "github.com/ipfs/go-cid" 10 | findclient "github.com/ipni/go-libipni/find/client" 11 | ingestclient "github.com/ipni/go-libipni/ingest/client" 12 | ) 13 | 14 | func main() { 15 | source := flag.String("source", "", "Source indexer") 16 | target := flag.String("target", "", "Target indexer") 17 | help := flag.Bool("help", false, "Print usage") 18 | 19 | flag.Parse() 20 | 21 | if *help { 22 | fmt.Print(` 23 | Cross-announce announces all the providers from a given source indexer to a given target indexer. 24 | Simply, specify the source and target as the HTTP(S) IPNI indexer instance. Example: 25 | $ go run ./scripts/cross_announce/main.go --source https://one-indexer.example --target https://another-indexer.example 26 | `) 27 | return 28 | } 29 | 30 | if *source == "" || *target == "" { 31 | log.Fatal("both indexer instances must be specified") 32 | } 33 | 34 | sourcer, err := findclient.New(*source) 35 | if err != nil { 36 | log.Fatal(err) 37 | } 38 | 39 | targeter, err := ingestclient.New(*target) 40 | if err != nil { 41 | log.Fatal(err) 42 | } 43 | ctx := context.Background() 44 | 45 | fmt.Printf("Listing providers at %s...\n", *source) 46 | providers, err := sourcer.ListProviders(ctx) 47 | if err != nil { 48 | log.Fatal(err) 49 | } 50 | totalProviders := len(providers) 51 | fmt.Printf("\tFound %d provider(s).\n", totalProviders) 52 | fmt.Printf("Announcing providers to %s...\n", *target) 53 | 54 | var announcedSuccessfully int 55 | for i, provider := range providers { 56 | fmt.Printf("\t(%d/%d) ", (i + 1), totalProviders) 57 | switch { 58 | case provider.Publisher == nil, provider.Publisher.ID == "", len(provider.Publisher.Addrs) == 0: 59 | fmt.Printf("No publisher for provider %s; skipped announce.\n", provider.AddrInfo.ID) 60 | continue 61 | case cid.Undef.Equals(provider.LastAdvertisement): 62 | fmt.Printf("No last advertisement CID for provider %s; skipped announce.\n", provider.AddrInfo.ID) 63 | continue 64 | //TODO: add more filtering like by last seen or lag 65 | default: 66 | if err := targeter.Announce(ctx, provider.Publisher, provider.LastAdvertisement); err != nil { 67 | fmt.Printf("Failed to announce provider %s: %s \n", provider.AddrInfo.ID, err) 68 | continue 69 | } 70 | fmt.Printf("Successfully announced provider %s\n", provider.AddrInfo.ID) 71 | announcedSuccessfully++ 72 | } 73 | } 74 | fmt.Printf("Successfully announced %d out of %d discovered from source.", announcedSuccessfully, totalProviders) 75 | } 76 | -------------------------------------------------------------------------------- /scripts/compare_providers/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "log" 8 | 9 | findclient "github.com/ipni/go-libipni/find/client" 10 | "github.com/ipni/go-libipni/find/model" 11 | "github.com/libp2p/go-libp2p/core/peer" 12 | ) 13 | 14 | type compareProvidersStats struct { 15 | unknown uint 16 | latestAdMismatch uint 17 | latestAdMatch uint 18 | totalSourceProviders uint 19 | totalTargetProviders uint 20 | } 21 | 22 | func (s compareProvidersStats) print() { 23 | fmt.Println("Stats:") 24 | fmt.Printf(" OK: %d\n", s.latestAdMatch) 25 | fmt.Printf(" Unknown by target: %d\n", s.unknown) 26 | fmt.Printf(" Latest Ad Mismatch: %d\n", s.latestAdMismatch) 27 | fmt.Println(" --------------------------") 28 | fmt.Printf(" Total source providers: %d\n", s.totalSourceProviders) 29 | fmt.Printf(" Total target providers: %d\n", s.totalTargetProviders) 30 | } 31 | 32 | func main() { 33 | source := flag.String("source", "", "Source indexer") 34 | target := flag.String("target", "", "Target indexer") 35 | 36 | flag.Parse() 37 | 38 | if *source == "" || *target == "" { 39 | log.Fatal("both indexer instances must be specified") 40 | } 41 | 42 | sourceClient, err := findclient.New(*source) 43 | if err != nil { 44 | log.Fatal(err) 45 | } 46 | 47 | targetClient, err := findclient.New(*target) 48 | if err != nil { 49 | log.Fatal(err) 50 | } 51 | ctx := context.Background() 52 | 53 | sourceProvs, err := sourceClient.ListProviders(ctx) 54 | if err != nil { 55 | log.Fatal(err) 56 | } 57 | 58 | targetProvs, err := targetClient.ListProviders(ctx) 59 | if err != nil { 60 | log.Fatal(err) 61 | } 62 | 63 | targets := make(map[peer.ID]*model.ProviderInfo) 64 | for _, target := range targetProvs { 65 | if target.AddrInfo.ID == "" { 66 | continue 67 | } 68 | targets[target.AddrInfo.ID] = target 69 | } 70 | var stats compareProvidersStats 71 | for _, p := range sourceProvs { 72 | id := p.AddrInfo.ID 73 | fmt.Printf("%s: ", id) 74 | if other, exists := targets[id]; !exists { 75 | fmt.Println("Unknown by target indexer.") 76 | stats.unknown++ 77 | } else if p.LastAdvertisement != other.LastAdvertisement { 78 | fmt.Println("Mismatching latest ad") 79 | // TODO implement diagnosis of which is ahead/behind and by how many ads. 80 | stats.latestAdMismatch++ 81 | } else { 82 | fmt.Println("OK") 83 | stats.latestAdMatch++ 84 | } 85 | } 86 | stats.totalSourceProviders = uint(len(sourceProvs)) 87 | stats.totalTargetProviders = uint(len(targetProvs)) 88 | fmt.Println() 89 | stats.print() 90 | } 91 | -------------------------------------------------------------------------------- /command/datastore_test.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "os" 7 | "path/filepath" 8 | "testing" 9 | "time" 10 | 11 | "github.com/ipfs/go-datastore" 12 | "github.com/ipfs/go-datastore/query" 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | func TestCreateDatastore(t *testing.T) { 17 | tmpDir := t.TempDir() 18 | dsDir := filepath.Join(tmpDir, "testDataDir") 19 | _, _, err := createDatastore(context.Background(), dsDir, "unknown", false) 20 | require.Error(t, err) 21 | 22 | ds, path, err := createDatastore(context.Background(), dsDir, "levelds", false) 23 | require.NoError(t, err) 24 | require.NotNil(t, ds) 25 | require.Equal(t, dsDir, path) 26 | require.NoError(t, ds.Close()) 27 | 28 | checkFile := filepath.Join(dsDir, "check.test") 29 | err = os.WriteFile(checkFile, []byte("Hello"), 0666) 30 | require.NoError(t, err) 31 | 32 | // Check that ds directory is not removed. 33 | ds, _, err = createDatastore(context.Background(), dsDir, "levelds", false) 34 | require.NoError(t, err) 35 | require.NotNil(t, ds) 36 | require.NoError(t, ds.Close()) 37 | require.True(t, fileExists(checkFile)) 38 | 39 | // Check that ds directory is removed. 40 | ds, _, err = createDatastore(context.Background(), dsDir, "levelds", true) 41 | require.NoError(t, err) 42 | require.NotNil(t, ds) 43 | require.NoError(t, ds.Close()) 44 | require.False(t, fileExists(checkFile)) 45 | } 46 | 47 | func TestDeletePrefix(t *testing.T) { 48 | ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) 49 | defer cancel() 50 | 51 | tmpDir := t.TempDir() 52 | dsDir := filepath.Join(tmpDir, "testDataDir") 53 | ds, _, err := createDatastore(ctx, dsDir, "levelds", false) 54 | require.NoError(t, err) 55 | t.Cleanup(func() { 56 | ds.Close() 57 | }) 58 | 59 | const prefix = "testKeys" 60 | dsKey1 := datastore.NewKey(prefix + "/foo") 61 | err = ds.Put(ctx, dsKey1, []byte("One")) 62 | require.NoError(t, err) 63 | dsKey2 := datastore.NewKey(prefix + "/bar") 64 | err = ds.Put(ctx, dsKey2, []byte("Two")) 65 | require.NoError(t, err) 66 | err = ds.Sync(ctx, datastore.NewKey("")) 67 | require.NoError(t, err) 68 | 69 | q := query.Query{ 70 | Prefix: prefix, 71 | } 72 | results, err := ds.Query(ctx, q) 73 | require.NoError(t, err) 74 | ents, err := results.Rest() 75 | results.Close() 76 | require.NoError(t, err) 77 | require.Len(t, ents, 2) 78 | 79 | n, err := deletePrefix(ctx, ds, prefix) 80 | require.NoError(t, err) 81 | require.Equal(t, 2, n) 82 | 83 | results, err = ds.Query(ctx, q) 84 | require.NoError(t, err) 85 | ents, err = results.Rest() 86 | results.Close() 87 | require.NoError(t, err) 88 | require.Empty(t, ents) 89 | } 90 | 91 | func fileExists(filename string) bool { 92 | _, err := os.Lstat(filename) 93 | return !errors.Is(err, os.ErrNotExist) 94 | } 95 | -------------------------------------------------------------------------------- /internal/ingest/hamt_ingest_test.go: -------------------------------------------------------------------------------- 1 | package ingest 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | cidlink "github.com/ipld/go-ipld-prime/linking/cid" 8 | "github.com/ipni/storetheindex/test/typehelpers" 9 | "github.com/libp2p/go-libp2p/core/peer" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestIngester_IngestsMixedEntriesTypeSuccessfully(t *testing.T) { 14 | ctx := context.Background() 15 | te := setupTestEnv(t, true) 16 | 17 | // Generate an ad chain with mixed entries type and different shape HAMT. 18 | headAd := typehelpers.RandomAdBuilder{ 19 | EntryBuilders: []typehelpers.EntryBuilder{ 20 | typehelpers.RandomHamtEntryBuilder{BucketSize: 3, BitWidth: 5, MultihashCount: 500, Seed: 1}, 21 | typehelpers.RandomHamtEntryBuilder{BucketSize: 1, BitWidth: 3, MultihashCount: 500, Seed: 2}, 22 | typehelpers.RandomEntryChunkBuilder{ChunkCount: 13, EntriesPerChunk: 100, Seed: 3}, 23 | typehelpers.RandomHamtEntryBuilder{BucketSize: 10, BitWidth: 8, MultihashCount: 810, Seed: 4}, 24 | typehelpers.RandomEntryChunkBuilder{ChunkCount: 1, EntriesPerChunk: 1, Seed: 5}, 25 | typehelpers.RandomHamtEntryBuilder{BucketSize: 1, BitWidth: 3, MultihashCount: 1, Seed: 6}, 26 | }, 27 | }.Build(t, te.publisherLinkSys, te.publisherPriv) 28 | 29 | // Set the head on publisher. 30 | headAdCid := headAd.(cidlink.Link).Cid 31 | te.publisher.SetRoot(headAdCid) 32 | 33 | // Extract the list of all multihashes in the ad chain. 34 | mhs := typehelpers.AllMultihashesFromAdLink(t, headAd, te.publisherLinkSys) 35 | require.Len(t, mhs, 500+500+13*100+810+1+1) // Sanity check the total expected number of multihashes. 36 | 37 | subject := te.ingester 38 | 39 | pubInfo := peer.AddrInfo{ 40 | ID: te.publisher.ID(), 41 | } 42 | // Trigger a sync. 43 | gotHeadAd, err := subject.Sync(ctx, pubInfo, 0, false) 44 | require.NoError(t, err) 45 | require.Equal(t, headAdCid, gotHeadAd, "Expected latest synced cid to match head of ad chain") 46 | 47 | // Assert all indices are processed eventually 48 | require.Eventually(t, func() bool { 49 | return checkAllIndexed(subject.indexer, pubInfo.ID, mhs) == nil 50 | }, testRetryTimeout, testRetryInterval, "Expected all multihashes to have been indexed eventually") 51 | 52 | // Assert All ads are processed eventually 53 | require.Eventually(t, func() bool { 54 | latestSync, err := subject.GetLatestSync(pubInfo.ID) 55 | require.NoError(t, err) 56 | return latestSync.Equals(headAdCid) 57 | }, testRetryTimeout, testRetryInterval, "Expected all ads from publisher to have been indexed eventually") 58 | 59 | // Assert multihash indices correspond to the single expected provider. 60 | for _, mh := range mhs { 61 | gotIdx, b, err := subject.indexer.Get(mh) 62 | require.NoError(t, err) 63 | require.True(t, b) 64 | require.Equal(t, 1, len(gotIdx)) 65 | require.Equal(t, pubInfo.ID, gotIdx[0].ProviderID) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /deploy/manifests/base/foundationdb/namespaced-manager/rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: fdb-kubernetes-operator-controller-manager 5 | --- 6 | apiVersion: rbac.authorization.k8s.io/v1 7 | kind: Role 8 | metadata: 9 | name: fdb-kubernetes-operator-manager-role 10 | rules: 11 | - apiGroups: 12 | - "" 13 | resources: 14 | - configmaps 15 | - events 16 | - persistentvolumeclaims 17 | - pods 18 | - secrets 19 | - services 20 | verbs: 21 | - create 22 | - delete 23 | - get 24 | - list 25 | - patch 26 | - update 27 | - watch 28 | - apiGroups: 29 | - apps 30 | resources: 31 | - deployments 32 | verbs: 33 | - create 34 | - delete 35 | - get 36 | - list 37 | - patch 38 | - update 39 | - watch 40 | - apiGroups: 41 | - apps.foundationdb.org 42 | resources: 43 | - foundationdbbackups 44 | verbs: 45 | - create 46 | - delete 47 | - get 48 | - list 49 | - patch 50 | - update 51 | - watch 52 | - apiGroups: 53 | - apps.foundationdb.org 54 | resources: 55 | - foundationdbbackups/status 56 | verbs: 57 | - get 58 | - patch 59 | - update 60 | - apiGroups: 61 | - apps.foundationdb.org 62 | resources: 63 | - foundationdbclusters 64 | verbs: 65 | - create 66 | - delete 67 | - get 68 | - list 69 | - patch 70 | - update 71 | - watch 72 | - apiGroups: 73 | - apps.foundationdb.org 74 | resources: 75 | - foundationdbclusters/status 76 | verbs: 77 | - get 78 | - patch 79 | - update 80 | - apiGroups: 81 | - apps.foundationdb.org 82 | resources: 83 | - foundationdbrestores 84 | verbs: 85 | - create 86 | - delete 87 | - get 88 | - list 89 | - patch 90 | - update 91 | - watch 92 | - apiGroups: 93 | - apps.foundationdb.org 94 | resources: 95 | - foundationdbrestores/status 96 | verbs: 97 | - get 98 | - patch 99 | - update 100 | - apiGroups: 101 | - coordination.k8s.io 102 | resources: 103 | - leases 104 | verbs: 105 | - create 106 | - delete 107 | - get 108 | - list 109 | - patch 110 | - update 111 | - watch 112 | --- 113 | apiVersion: rbac.authorization.k8s.io/v1 114 | kind: RoleBinding 115 | metadata: 116 | name: fdb-kubernetes-operator-manager-rolebinding 117 | roleRef: 118 | apiGroup: rbac.authorization.k8s.io 119 | kind: Role 120 | name: fdb-kubernetes-operator-manager-role 121 | subjects: 122 | - kind: ServiceAccount 123 | name: fdb-kubernetes-operator-controller-manager -------------------------------------------------------------------------------- /internal/ingest/mirror.go: -------------------------------------------------------------------------------- 1 | package ingest 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "reflect" 7 | 8 | "github.com/ipfs/go-cid" 9 | "github.com/ipfs/go-datastore" 10 | "github.com/ipni/storetheindex/carstore" 11 | "github.com/ipni/storetheindex/config" 12 | "github.com/ipni/storetheindex/filestore" 13 | "github.com/libp2p/go-libp2p/core/peer" 14 | ) 15 | 16 | type adMirror struct { 17 | carReader *carstore.CarReader 18 | carWriter *carstore.CarWriter 19 | rdWrSame bool 20 | } 21 | 22 | func (m adMirror) canRead() bool { 23 | return m.carReader != nil 24 | } 25 | func (m adMirror) canWrite() bool { 26 | return m.carWriter != nil 27 | } 28 | 29 | func (m adMirror) cleanupAdData(ctx context.Context, adCid cid.Cid, skipEntries bool) error { 30 | return m.carWriter.CleanupAdData(ctx, adCid, skipEntries) 31 | } 32 | 33 | func (m adMirror) read(ctx context.Context, adCid cid.Cid, skipEntries bool) (*carstore.AdBlock, error) { 34 | return m.carReader.Read(ctx, adCid, skipEntries) 35 | } 36 | 37 | func (m adMirror) write(ctx context.Context, adCid cid.Cid, skipEntries, noOoverwrite bool) (*filestore.File, error) { 38 | return m.carWriter.Write(ctx, adCid, skipEntries, noOoverwrite) 39 | } 40 | 41 | func (m adMirror) writeHead(ctx context.Context, adCid cid.Cid, publisher peer.ID) (*filestore.File, error) { 42 | return m.carWriter.WriteHead(ctx, adCid, publisher) 43 | } 44 | 45 | func (m adMirror) readWriteSame() bool { 46 | return m.rdWrSame 47 | } 48 | 49 | func newMirror(cfgMirror config.Mirror, dstore datastore.Batching) (adMirror, error) { 50 | var m adMirror 51 | if cfgMirror.Write { 52 | switch writeStore, err := filestore.MakeFilestore(cfgMirror.Storage); { 53 | case err != nil: 54 | return m, fmt.Errorf("cannot create car file storage for mirror: %w", err) 55 | case writeStore != nil: 56 | m.carWriter, err = carstore.NewWriter(dstore, writeStore, carstore.WithCompress(cfgMirror.Compress)) 57 | if err != nil { 58 | return m, fmt.Errorf("cannot create mirror car file writer: %w", err) 59 | } 60 | default: 61 | log.Warnw("Mirror write is enabled with no storage backend", "backendType", cfgMirror.Storage.Type) 62 | } 63 | } 64 | if cfgMirror.Read { 65 | switch readStore, err := filestore.MakeFilestore(cfgMirror.Retrieval); { 66 | case err != nil: 67 | return m, fmt.Errorf("cannot create car file retrieval for mirror: %w", err) 68 | case readStore != nil: 69 | m.carReader, err = carstore.NewReader(readStore, carstore.WithCompress(cfgMirror.Compress)) 70 | if err != nil { 71 | return m, fmt.Errorf("cannot create mirror car file reader: %w", err) 72 | } 73 | default: 74 | log.Warnw("Mirror read is enabled with no retrieval backend", "backendType", cfgMirror.Retrieval.Type) 75 | } 76 | } 77 | 78 | m.rdWrSame = m.carWriter != nil && m.carReader != nil && reflect.DeepEqual(cfgMirror.Storage, cfgMirror.Retrieval) 79 | return m, nil 80 | } 81 | -------------------------------------------------------------------------------- /assigner/config/assignment.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "time" 5 | 6 | sticfg "github.com/ipni/storetheindex/config" 7 | ) 8 | 9 | // Assignment holds addresses of indexers to assign publishers to, policy 10 | // specifying which peers to allow announce messages from, and related 11 | // settings. 12 | type Assignment struct { 13 | // FilterIPs, when true, removes any private, loopback, or unspecified IP 14 | // addresses from provider and publisher addresses. 15 | FilterIPs bool 16 | // PoolInterval is how often to poll indexers for status. 17 | PollInterval sticfg.Duration 18 | // IndexerPool is the set of indexers the pool. 19 | IndexerPool []Indexer 20 | // Policy configures which peers are allowed and blocked. 21 | Policy Policy 22 | // PubSubTopic sets the topic name to which to subscribe for ingestion 23 | // announcements. 24 | PubSubTopic string 25 | // PresetReplication is the number of pre-assigned indexers to assign a 26 | // publisher to. See Indexer.PresetPeers. Any value < 1 defaults to 1. 27 | PresetReplication int 28 | // Replication is the number of indexers to assign each publisher to, when 29 | // the publisher does not have a preset assignment. A value <= 0 assigns 30 | // each publisher to one indexer. 31 | Replication int 32 | // ResendHttp, if true, resends announcements directly to assigned insexers. 33 | ResendHttp bool 34 | // ResendPubsub, if true, resends announcements over libp2p pubsub. 35 | ResendPubsub bool 36 | } 37 | 38 | type Indexer struct { 39 | // AdminURL is the base URL for the indexer's admin interface. 40 | AdminURL string 41 | // FindURL is the base URL for the indexer's find interface. 42 | FindURL string 43 | // IngestURL is the base URL for the indexer's ingest interface. 44 | IngestURL string 45 | // PresetPeers is a list of the peer IDs of pre-assigned publishers. A 46 | // publisher is assigned to n of the indexers that has the publisher in 47 | // PresetPeers, where n is PresetReplication. 48 | PresetPeers []string 49 | } 50 | 51 | func NewIndexer() Indexer { 52 | return Indexer{} 53 | } 54 | 55 | // NewDiscovery returns Discovery with values set to their defaults. 56 | func NewAssignment() Assignment { 57 | return Assignment{ 58 | PollInterval: sticfg.Duration(5 * time.Minute), 59 | Policy: NewPolicy(), 60 | PubSubTopic: "/indexer/ingest/mainnet", 61 | PresetReplication: 1, 62 | Replication: 1, 63 | ResendHttp: true, 64 | } 65 | } 66 | 67 | // populateUnset replaces zero-values in the config with default values. 68 | func (c *Assignment) populateUnset() { 69 | def := NewAssignment() 70 | 71 | if c.PollInterval == 0 { 72 | c.PollInterval = def.PollInterval 73 | } 74 | if c.PubSubTopic == "" { 75 | c.PubSubTopic = def.PubSubTopic 76 | } 77 | if c.PresetReplication <= 0 { 78 | c.PresetReplication = def.PresetReplication 79 | } 80 | if c.Replication <= 0 { 81 | c.Replication = def.Replication 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /config/identity.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/base64" 5 | "fmt" 6 | "os" 7 | 8 | ic "github.com/libp2p/go-libp2p/core/crypto" 9 | "github.com/libp2p/go-libp2p/core/peer" 10 | ) 11 | 12 | const ( 13 | IdentityTag = "Identity" 14 | PrivKeyTag = "PrivKey" 15 | PrivKeySelector = IdentityTag + "." + PrivKeyTag 16 | 17 | PrivateKeyPathEnvVar = "STORETHEINDEX_PRIV_KEY_PATH" 18 | ) 19 | 20 | // Identity tracks the configuration of the local node's identity. 21 | type Identity struct { 22 | // PeerID is the peer ID of the indexer that must match the given PrivKey. 23 | // If unspecified, it is automatically generated from the PrivKey. 24 | PeerID string 25 | // PrivKey represents the peer identity of the indexer. 26 | // 27 | // If unset, the key is loaded from the file at the path specified via 28 | // STORETHEINDEX_PRIV_KEY_PATH environment variable. 29 | PrivKey string `json:",omitempty"` 30 | } 31 | 32 | func (i Identity) Decode() (peer.ID, ic.PrivKey, error) { 33 | privKey, err := i.DecodePrivateKey("") 34 | if err != nil { 35 | return "", nil, fmt.Errorf("could not decode private key: %w", err) 36 | } 37 | 38 | peerIDFromPrivKey, err := peer.IDFromPrivateKey(privKey) 39 | if err != nil { 40 | return "", nil, fmt.Errorf("could not generate peer ID from private key: %w", err) 41 | } 42 | 43 | // If peer ID is specified in JSON config, then verify that it is: 44 | // 1. a valid peer ID, and 45 | // 2. consistent with the peer ID generated from private key. 46 | if i.PeerID != "" { 47 | peerID, err := peer.Decode(i.PeerID) 48 | if err != nil { 49 | return "", nil, fmt.Errorf("could not decode peer id: %w", err) 50 | } 51 | 52 | if peerID != "" && peerIDFromPrivKey != peerID { 53 | return "", nil, fmt.Errorf("provided peer ID must either match the peer ID generated from private key or be omitted: expected %s but got %s", peerIDFromPrivKey, peerID) 54 | } 55 | } 56 | 57 | return peerIDFromPrivKey, privKey, nil 58 | } 59 | 60 | // DecodePrivateKey is a helper to decode the user's PrivateKey. 61 | func (i Identity) DecodePrivateKey(passphrase string) (ic.PrivKey, error) { 62 | // TODO(security): currently storing key unencrypted. in the future we need to encrypt it. 63 | 64 | // If a value is supplied in JSON config then attempt to decode it as the private key. 65 | if i.PrivKey != "" { 66 | pkb, err := base64.StdEncoding.DecodeString(i.PrivKey) 67 | if err != nil { 68 | return nil, err 69 | } 70 | return ic.UnmarshalPrivateKey(pkb) 71 | } 72 | 73 | // Otherwise, load the private key from path supplied via env var. 74 | privKeyPath := os.Getenv(PrivateKeyPathEnvVar) 75 | if privKeyPath == "" { 76 | return nil, fmt.Errorf("private key not specified; it must be specified either in config or via %s env var", PrivateKeyPathEnvVar) 77 | } 78 | 79 | pkb, err := os.ReadFile(privKeyPath) 80 | if err != nil { 81 | return nil, err 82 | } 83 | return ic.UnmarshalPrivateKey(pkb) 84 | } 85 | -------------------------------------------------------------------------------- /internal/ingest/invalid_mh_ingest_test.go: -------------------------------------------------------------------------------- 1 | package ingest 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | cidlink "github.com/ipld/go-ipld-prime/linking/cid" 8 | "github.com/ipni/storetheindex/test/typehelpers" 9 | "github.com/libp2p/go-libp2p/core/peer" 10 | "github.com/multiformats/go-multihash" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestInvalidMultihashesAreNotIngested(t *testing.T) { 15 | te := setupTestEnv(t, true) 16 | 17 | headAd := typehelpers.RandomAdBuilder{ 18 | EntryBuilders: []typehelpers.EntryBuilder{ 19 | typehelpers.RandomEntryChunkBuilder{ChunkCount: 1, EntriesPerChunk: 1, Seed: 1, WithInvalidMultihashes: true}, 20 | typehelpers.RandomEntryChunkBuilder{ChunkCount: 1, EntriesPerChunk: 1, Seed: 2, WithInvalidMultihashes: false}, 21 | typehelpers.RandomEntryChunkBuilder{ChunkCount: 1, EntriesPerChunk: 1, Seed: 3, WithInvalidMultihashes: true}, 22 | typehelpers.RandomHamtEntryBuilder{MultihashCount: 1, Seed: 4, WithInvalidMultihashes: false}, 23 | typehelpers.RandomHamtEntryBuilder{MultihashCount: 1, Seed: 5, WithInvalidMultihashes: true}, 24 | typehelpers.RandomHamtEntryBuilder{MultihashCount: 1, Seed: 6, WithInvalidMultihashes: false}, 25 | }, 26 | }.Build(t, te.publisherLinkSys, te.publisherPriv) 27 | headAdCid := headAd.(cidlink.Link).Cid 28 | ctx := context.Background() 29 | te.publisher.SetRoot(headAdCid) 30 | mhs := typehelpers.AllMultihashesFromAdLink(t, headAd, te.publisherLinkSys) 31 | require.Len(t, mhs, 6) // 6 ads; one mh each. 32 | validMhs := []multihash.Multihash{mhs[1], mhs[3], mhs[5]} 33 | invalidMhs := []multihash.Multihash{mhs[0], mhs[2], mhs[4]} 34 | 35 | subject := te.ingester 36 | 37 | pubInfo := peer.AddrInfo{ 38 | ID: te.publisher.ID(), 39 | } 40 | gotHeadAd, err := subject.Sync(ctx, pubInfo, 0, false) 41 | require.NoError(t, err) 42 | 43 | require.Equal(t, headAdCid, gotHeadAd, "Expected latest synced cid to match head of ad chain") 44 | 45 | require.Eventually(t, func() bool { 46 | return checkAllIndexed(subject.indexer, pubInfo.ID, validMhs) == nil 47 | }, testRetryTimeout, testRetryInterval, "Expected only valid multihashes to be indexed") 48 | 49 | require.Eventually(t, func() bool { 50 | latestSync, err := subject.GetLatestSync(pubInfo.ID) 51 | require.NoError(t, err) 52 | return latestSync.Equals(headAdCid) 53 | }, testRetryTimeout, testRetryInterval, "Expected all ads from publisher to have been indexed") 54 | 55 | // Assert valid multihash indices correspond to the expected provider. 56 | for _, mh := range validMhs { 57 | gotIdx, b, err := subject.indexer.Get(mh) 58 | require.NoError(t, err) 59 | require.True(t, b) 60 | require.Equal(t, 1, len(gotIdx)) 61 | require.Equal(t, pubInfo.ID, gotIdx[0].ProviderID) 62 | } 63 | 64 | // Assert invalid multihashes are not indexed. 65 | for _, mh := range invalidMhs { 66 | _, b, err := subject.indexer.Get(mh) 67 | require.NoError(t, err) 68 | require.False(t, b) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /command/flags.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/ipni/storetheindex/config" 7 | "github.com/multiformats/go-multiaddr" 8 | "github.com/urfave/cli/v2" 9 | ) 10 | 11 | var indexerHostFlag = &cli.StringFlag{ 12 | Name: "indexer", 13 | Usage: "Host or host:port of indexer to use", 14 | EnvVars: []string{"INDEXER"}, 15 | Aliases: []string{"i"}, 16 | Required: false, 17 | } 18 | 19 | var cacheSizeFlag = &cli.Int64Flag{ 20 | Name: "cachesize", 21 | Usage: "Maximum number of multihashes that result cache can hold, -1 to disable cache", 22 | Required: false, 23 | } 24 | 25 | var listenAdminFlag = &cli.StringFlag{ 26 | Name: "listen-admin", 27 | Usage: "Admin HTTP API listen address or 'none' to disable, overrides config", 28 | EnvVars: []string{"STORETHEINDEX_LISTEN_ADMIN"}, 29 | Required: false, 30 | } 31 | var listenFindFlag = &cli.StringFlag{ 32 | Name: "listen-finder", 33 | Usage: "HTTP listen address or 'none' to disable, overrides config", 34 | EnvVars: []string{"STORETHEINDEX_LISTEN_FINDER"}, 35 | Required: false, 36 | } 37 | var listenIngestFlag = &cli.StringFlag{ 38 | Name: "listen-ingest", 39 | Usage: "Ingestion HTTP API listen address or 'none' to disable, overrides config", 40 | EnvVars: []string{"STORETHEINDEX_LISTEN_INGEST"}, 41 | Required: false, 42 | } 43 | var listenP2PFlag = &cli.StringFlag{ 44 | Name: "listen-p2p", 45 | Usage: "P2P listen address or 'none' to disable, overrides config", 46 | EnvVars: []string{"STORETHEINDEX_LISTEN_P2P"}, 47 | Required: false, 48 | } 49 | 50 | // cliIndexer reads the indexer host from CLI flag or from config. 51 | func cliIndexer(cctx *cli.Context, addrType string) string { 52 | idxr := cctx.String("indexer") 53 | if idxr != "" { 54 | return idxr 55 | } 56 | 57 | idxr = indexerHost(addrType) 58 | if idxr != "" { 59 | return idxr 60 | } 61 | 62 | return "localhost" 63 | } 64 | 65 | func indexerHost(addrType string) string { 66 | // No indexer given on command line, get from config. 67 | cfg, err := config.Load("") 68 | if err != nil { 69 | return "" 70 | } 71 | if cfg.Addresses.Finder == "" { 72 | return "" 73 | } 74 | var maddr multiaddr.Multiaddr 75 | switch addrType { 76 | case "find": 77 | maddr, err = multiaddr.NewMultiaddr(cfg.Addresses.Finder) 78 | case "admin": 79 | maddr, err = multiaddr.NewMultiaddr(cfg.Addresses.Admin) 80 | case "ingest": 81 | maddr, err = multiaddr.NewMultiaddr(cfg.Addresses.Ingest) 82 | default: 83 | return "" 84 | } 85 | if err != nil { 86 | return "" 87 | } 88 | return multiaddrHost(maddr) 89 | } 90 | 91 | func multiaddrHost(maddr multiaddr.Multiaddr) string { 92 | for _, proto := range []int{multiaddr.P_IP4, multiaddr.P_IP6} { 93 | addr, err := maddr.ValueForProtocol(proto) 94 | if err == nil { 95 | port, err := maddr.ValueForProtocol(multiaddr.P_TCP) 96 | if err == nil { 97 | addr = fmt.Sprint(addr, ":", port) 98 | } 99 | return addr 100 | } 101 | } 102 | return "" 103 | } 104 | --------------------------------------------------------------------------------