├── .editorconfig ├── .editorconfig-checked.json ├── .gitattributes ├── .github ├── ISSUE_TEMPLATE │ ├── 01-bug_report.md │ ├── 02-feature_request.md │ ├── 03-spec.md │ └── config.yml ├── dependabot.yaml ├── discord-embed-webhook.py ├── labeler.yml ├── requirements.txt └── workflows │ ├── build.yaml │ ├── docker-mediamtx.yaml │ ├── docker.yaml │ ├── issue-labeler.yml │ ├── pr-labeler.yml │ ├── release.yaml │ └── test.yaml ├── .gitignore ├── CHANGELOG.md ├── CHANGELOG_PENDING.md ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── PULL_REQUEST_TEMPLATE.md ├── README.md ├── VERSION ├── ai ├── file_worker.go └── worker │ ├── b64.go │ ├── b64_test.go │ ├── container.go │ ├── doc.go │ ├── docker.go │ ├── docker_test.go │ ├── multipart.go │ ├── runner.gen.go │ ├── utils.go │ ├── utils_test.go │ └── worker.go ├── box ├── aiModels-comfyui.json ├── aiModels-noop.json ├── box.md ├── box.sh ├── build-runner.sh ├── docker-compose.supabase.yml ├── frontend.sh ├── frontend.sql ├── gateway.sh ├── mediamtx.sh ├── mediamtx.yml ├── orchestrator.sh ├── stream.sh └── supabase.sh ├── build ├── chain.go ├── chain_dev.go ├── chain_mainnet.go ├── chain_rinkeby.go ├── const.go └── const_windows.go ├── ci_env.sh ├── clog ├── clog.go └── clog_test.go ├── cmd ├── devtool │ ├── README.md │ ├── devtool.go │ ├── devtool │ │ └── devtool_utils.go │ └── scripts │ │ └── create_multiple_transcoders.bash ├── livepeer │ ├── livepeer.go │ ├── livepeer_test.go │ └── starter │ │ ├── kafka.go │ │ ├── starter.go │ │ ├── starter_test.go │ │ └── test_fixtures │ │ └── perf_stats.json ├── livepeer_bench │ ├── livepeer_bench.go │ ├── transcodingOptions-netint.json │ └── transcodingOptions.json ├── livepeer_cli │ ├── livepeer_cli.go │ ├── wizard.go │ ├── wizard_bond.go │ ├── wizard_broadcast.go │ ├── wizard_deposit.go │ ├── wizard_eth.go │ ├── wizard_rounds.go │ ├── wizard_stats.go │ ├── wizard_stream.go │ ├── wizard_ticketbroker.go │ ├── wizard_ticketbroker_test.go │ ├── wizard_token.go │ └── wizard_transcoder.go ├── livepeer_router │ └── livepeer_router.go ├── scripts │ └── linkify_changelog.go └── simple_auth_server │ └── simple_auth_server.go ├── common ├── db.go ├── db_test.go ├── log.go ├── readfromfile.go ├── readfromfile_test.go ├── testutil.go ├── types.go ├── util.go ├── util_test.go └── videoprofile_ids.go ├── config.toml ├── core ├── accounting.go ├── accounting_test.go ├── ai.go ├── ai_orchestrator.go ├── ai_test.go ├── autoconvertedprice.go ├── autoconvertedprice_test.go ├── broadcaster.go ├── capabilities.go ├── capabilities_test.go ├── core_test.go ├── lb.go ├── lb_test.go ├── livepeernode.go ├── livepeernode_test.go ├── orch_test.go ├── orchestrator.go ├── os.go ├── playlistmanager.go ├── playlistmanager_test.go ├── stream_test.go ├── streamdata.go ├── test.phash ├── test.ts ├── test2.ts ├── test_segment.go ├── test_segment.sh ├── transcoder.go └── transcoder_test.go ├── crypto ├── verify.go └── verify_test.go ├── discovery ├── db_discovery.go ├── discovery.go ├── discovery_test.go ├── stub.go ├── suspensionqueue.go ├── suspensionqueue_test.go └── wh_discovery.go ├── doc ├── assets │ ├── ai-runner-container-lifecycle.jpg │ └── redeemer │ │ ├── eth-events.png │ │ ├── eth-events.txt │ │ ├── ticketflow.png │ │ └── ticketflow.txt ├── database.md ├── development.md ├── discovery.md ├── ethereum.md ├── go.md ├── gpu.md ├── httpcli.md ├── ingest.md ├── multi-o.md ├── networking.md ├── orchwebhook.md ├── payments.md ├── redeemer.md ├── releases.md ├── reliability.md ├── rtmpwebhookauth.md ├── selection.md ├── transcodingoptions.md ├── verification.md └── worker.md ├── docker ├── Dockerfile ├── Dockerfile.cuda-base ├── Dockerfile.mediamtx ├── crontab └── mediamtx-metrics.bash ├── etc └── ffmpeg_trans_test.sh ├── eth ├── README.md ├── accountmanager.go ├── accountmanager_test.go ├── backend.go ├── backend_test.go ├── blockwatch │ ├── LICENSE │ ├── README.md │ ├── block_watcher.go │ ├── block_watcher_test.go │ ├── client.go │ ├── fake_client.go │ ├── fake_log_client.go │ ├── stack.go │ ├── stack_test.go │ └── testdata │ │ ├── fake_client_basic_fixture.json │ │ ├── fake_client_block_poller_fixtures.json │ │ └── fake_client_fast_sync_fixture.json ├── client.go ├── client_test.go ├── client_ticketbroker.go ├── contracts │ ├── LivepeerGovernor.go │ ├── bondingManager.go │ ├── chainlink │ │ ├── AggregatorV3Interface.abi │ │ ├── AggregatorV3Interface.go │ │ └── AggregatorV3Interface.sol │ ├── controller.go │ ├── l1BondingManager.go │ ├── livepeerToken.go │ ├── livepeerTokenFaucet.go │ ├── minter.go │ ├── poll.go │ ├── roundsManager.go │ ├── serviceRegistry.go │ └── ticketBroker.go ├── gaspricemonitor.go ├── gaspricemonitor_test.go ├── helpers.go ├── helpers_test.go ├── noncemanager.go ├── noncemanager_test.go ├── pricefeed.go ├── pricefeed_test.go ├── rewardservice.go ├── rewardservice_test.go ├── roundinitializer.go ├── roundinitializer_test.go ├── stubclient.go ├── transactionManager.go ├── transactionManager_test.go ├── types │ ├── contracts.go │ ├── merkletree.go │ └── merkletree_test.go └── watchers │ ├── eventdecoder.go │ ├── eventdecoder_test.go │ ├── orchestratorwatcher.go │ ├── orchestratorwatcher_test.go │ ├── pricefeedwatcher.go │ ├── pricefeedwatcher_test.go │ ├── senderwatcher.go │ ├── senderwatcher_test.go │ ├── serviceRegistryWatcher.go │ ├── serviceRegistryWatcher_test.go │ ├── stub.go │ ├── timewatcher.go │ ├── timewatcher_test.go │ ├── topics.go │ ├── types.go │ ├── unbondingwatcher.go │ └── unbondingwatcher_test.go ├── go.mod ├── go.sum ├── install_ffmpeg.sh ├── liveai.openapi.yaml ├── media ├── mediamtx.go ├── ring.go ├── ring_test.go ├── rtmp2segment.go ├── rtmp2segment_windows.go ├── rtp_segmenter.go ├── rtp_segmenter_test.go ├── rw.go ├── rw_test.go ├── segment_reader.go ├── select_darwin.go ├── select_linux.go ├── whip_connection.go ├── whip_connection_test.go └── whip_server.go ├── monitor ├── census.go ├── census_test.go └── kafka.go ├── net ├── lp_rpc.pb.go ├── lp_rpc.proto ├── lp_rpc_grpc.pb.go ├── redeemer.pb.go ├── redeemer.proto ├── redeemer_grpc.pb.go ├── redeemer_grpc_mock.pb.go └── redeemer_mock.pb.go ├── pm ├── README.md ├── assets │ ├── diagram.png │ └── diagram.txt ├── broker.go ├── helpers.go ├── queue.go ├── queue_test.go ├── recipient.go ├── recipient_test.go ├── sender.go ├── sender_test.go ├── sendermonitor.go ├── sendermonitor_test.go ├── signer.go ├── sigverifier.go ├── sigverifier_test.go ├── stub.go ├── ticket.go ├── ticket_test.go ├── ticketstore.go ├── validator.go └── validator_test.go ├── prepare_mingw64.sh ├── print_version.sh ├── server ├── ai_http.go ├── ai_http_test.go ├── ai_live_video.go ├── ai_mediaserver.go ├── ai_pipeline_status.go ├── ai_process.go ├── ai_process_test.go ├── ai_session.go ├── ai_worker.go ├── ai_worker_test.go ├── auth.go ├── auth_test.go ├── broadcast.go ├── broadcast_test.go ├── cert.go ├── cert_test.go ├── cliserver_test.go ├── handlers.go ├── handlers_test.go ├── live_payment.go ├── live_payment_processor.go ├── live_payment_test.go ├── mediaserver.go ├── mediaserver_test.go ├── ot_rpc.go ├── ot_rpc_test.go ├── push_test.go ├── push_webhook_test.go ├── recordings_test.go ├── redeemer.go ├── redeemer_test.go ├── router.go ├── rpc.go ├── rpc_test.go ├── segment_rpc.go ├── segment_rpc_test.go ├── selection.go ├── selection_algorithm.go ├── selection_algorithm_test.go ├── selection_test.go ├── sessionpool_test.go ├── stub.go ├── suspensions.go ├── suspensions_test.go ├── test.flv ├── utils.go ├── webserver.go └── zero_frame_test.go ├── test.sh ├── test ├── ai │ ├── audio │ └── image └── e2e │ ├── binary_test.go │ ├── configure_orchestrator_test.go │ ├── deposit_broadcaster_test.go │ ├── e2e.go │ ├── http_push_broadcaster_test.go │ ├── register_orchestrator_test.go │ ├── remove_orchestrator_test.go │ └── test.flv ├── test_args.sh ├── test_docker.sh ├── test_e2e.sh ├── tools.go ├── transcode_demo.sh ├── trickle ├── README.md ├── local_publisher.go ├── local_subscriber.go ├── trickle_publisher.go ├── trickle_server.go └── trickle_subscriber.go └── verification ├── epic.go ├── epic_test.go ├── verify.go └── verify_test.go /.editorconfig: -------------------------------------------------------------------------------- 1 | # This file is the top-most EditorConfig file 2 | root = true 3 | 4 | # All Files 5 | [*] 6 | charset = utf-8 7 | indent_style = space 8 | indent_size = 4 9 | insert_final_newline = true 10 | trim_trailing_whitespace = true 11 | end_of_line = lf 12 | 13 | # Ignored files 14 | [*.{ts,phash,md}] 15 | charset = unset 16 | indent_style = unset 17 | indent_size = unset 18 | insert_final_newline = unset 19 | trim_trailing_whitespace = unset 20 | end_of_line = unset 21 | 22 | [{VERSION,LICENSE,*.txt}] 23 | insert_final_newline = unset 24 | 25 | # Makefiles/Dockerfile/golang files 26 | [{go.mod,Makefile,Dockerfile{,.cuda-base,.mediamtx},*.go}] 27 | indent_style = tab 28 | indent_size = 8 29 | 30 | # YAML/JSON/sh Files 31 | [*.{yml,yaml,sh,json,bash,proto}] 32 | indent_size = 2 33 | 34 | [{server/handlers_test,eth/accountmanager_test}.go] 35 | indent_style = unset 36 | -------------------------------------------------------------------------------- /.editorconfig-checked.json: -------------------------------------------------------------------------------- 1 | { 2 | "Verbose": false, 3 | "Debug": false, 4 | "IgnoreDefaults": false, 5 | "SpacesAfterTabs": false, 6 | "NoColor": false, 7 | "Exclude": [ 8 | "core/.*.ts", 9 | "net/.*.proto", 10 | ".*.md", 11 | "go.mod", 12 | "go.sum" 13 | ], 14 | "AllowedContentTypes": [], 15 | "PassedFiles": [], 16 | "Disable": { 17 | "EndOfLine": false, 18 | "Indentation": false, 19 | "InsertFinalNewline": false, 20 | "TrimTrailingWhitespace": false 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.png filter=lfs diff=lfs merge=lfs -text 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/01-bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | 5 | --- 6 | 7 | **Describe the bug** 8 | A clear and concise description of what the bug is. 9 | 10 | **To Reproduce** 11 | Steps to reproduce the behavior: 12 | 1. Go to '...' 13 | 2. Click on '....' 14 | 3. Scroll down to '....' 15 | 4. See error 16 | 17 | **Expected behavior** 18 | A clear and concise description of what you expected to happen. 19 | 20 | **Screenshots** 21 | If applicable, add screenshots to help explain your problem. 22 | 23 | **Desktop (please complete the following information):** 24 | - OS: [e.g. iOS] 25 | - Browser [e.g. chrome, safari] 26 | - Version [e.g. 22] 27 | 28 | **Smartphone (please complete the following information):** 29 | - Device: [e.g. iPhone6] 30 | - OS: [e.g. iOS8.1] 31 | - Browser [e.g. stock browser, safari] 32 | - Version [e.g. 22] 33 | 34 | **Additional context** 35 | Add any other context about the problem here. 36 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/02-feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | 5 | --- 6 | 7 | **Is your feature request related to a problem? Please describe.** 8 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 9 | 10 | **Describe the solution you'd like** 11 | A clear and concise description of what you want to happen. 12 | 13 | **Describe alternatives you've considered** 14 | A clear and concise description of any alternative solutions or features you've considered. 15 | 16 | **Additional context** 17 | Add any other context or screenshots about the feature request here. 18 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/03-spec.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature Spec 3 | about: Outline requirements and tasks for a new feature 4 | 5 | --- 6 | 7 | ## Abstract 8 | 9 | 10 | ## Motivation 11 | 12 | 13 | 14 | ## Proposed Solution 15 | 16 | 17 | 18 | ## Implementation Tasks and Considerations 19 | 20 | 21 | 22 | ## Testing Tasks and Considerations 23 | 24 | 25 | 26 | ## Known Unknowns 27 | 28 | 30 | 31 | ## Alternatives 32 | 33 | 35 | 36 | ## Additional Context 37 | 38 | 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: true 2 | contact_links: 3 | - name: Go-livepeer Question 4 | url: https://github.com/livepeer/go-livepeer/discussions 5 | about: Please ask and answer questions related to the go-livepeer software here. 6 | - name: Livepeer Question 7 | url: https://discord.gg/livepeer 8 | about: "Have a general Livepeer question? Join us in the Livepeer Discord server. We're here to help!" 9 | -------------------------------------------------------------------------------- /.github/dependabot.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | updates: 4 | - package-ecosystem: github-actions 5 | directory: / 6 | schedule: 7 | interval: daily 8 | 9 | - package-ecosystem: gomod 10 | directory: / 11 | schedule: 12 | interval: daily 13 | 14 | - package-ecosystem: docker 15 | directory: /docker 16 | schedule: 17 | interval: daily 18 | -------------------------------------------------------------------------------- /.github/discord-embed-webhook.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import json 4 | import pathlib 5 | from typing import Any 6 | 7 | from requests import Response 8 | from discord_webhook import DiscordEmbed, DiscordWebhook 9 | 10 | 11 | GITHUB_CONTEXT_JSON = os.environ.get("GITHUB_CONTEXT_JSON", "{}") 12 | DOWNLOAD_BASE_URL_TEMPLATE = ( 13 | "https://build.livepeer.live/go-livepeer/{version}/{filename}" 14 | ) 15 | 16 | 17 | def get_github_context_vars(context: dict[str, Any]) -> dict[str, str]: 18 | context_vars = {} 19 | event: dict[str, Any] = context["event"] 20 | if context.get("event_name") == "pull_request": 21 | pull_request_context: dict[str, Any] = event["pull_request"] 22 | head = pull_request_context.get("head", {}) 23 | repo = head.get("repo", {}) 24 | sha = head.get("sha") 25 | context_vars["title"] = head.get("ref") 26 | context_vars["sha"] = sha 27 | context_vars["commit_url"] = f'{repo.get("html_url")}/commit/{sha}' 28 | elif context.get("event_name") == "push": 29 | sha = event["after"] 30 | context_vars["title"] = context["ref_name"] 31 | context_vars["sha"] = sha 32 | context_vars["commit_url"] = event["compare"] 33 | return context_vars 34 | 35 | 36 | def populate_embeds(embed: DiscordEmbed, ref_name: str, checksums: list[str]): 37 | for line in checksums: 38 | _, filename = line.split() 39 | download_url = DOWNLOAD_BASE_URL_TEMPLATE.format( 40 | version=ref_name, 41 | filename=filename, 42 | ) 43 | title = filename.removeprefix("livepeer-").split(".")[0] 44 | print(f"Adding embed field name={title} value={download_url}") 45 | embed.add_embed_field(name=title, value=download_url, inline=False) 46 | 47 | 48 | def main(args): 49 | checksums = [] 50 | github_context: dict[str, Any] = json.loads(GITHUB_CONTEXT_JSON) 51 | context_vars = get_github_context_vars(github_context) 52 | checksums_file = pathlib.Path("releases") / f"{args.ref_name}_checksums.txt" 53 | checksums = checksums_file.read_text().splitlines() 54 | webhook = DiscordWebhook( 55 | url=args.discord_url, 56 | content=":white_check_mark: Build succeeded for go-livepeer :white_check_mark:", 57 | username="[BOT] Livepeer builder", 58 | ) 59 | webhook.add_file(filename=checksums_file.name, file=checksums_file.read_bytes()) 60 | embed = DiscordEmbed( 61 | title=context_vars.get("title"), 62 | description=args.git_commit, 63 | color=2928914, 64 | url=context_vars.get("commit_url"), 65 | ) 66 | embed.add_embed_field(name="Commit SHA", value=context_vars.get("sha")) 67 | embed.set_author(name=args.git_committer) 68 | populate_embeds(embed, args.ref_name, checksums) 69 | embed.set_timestamp() 70 | webhook.add_embed(embed) 71 | response: Response = webhook.execute() 72 | print("sending webhook with content:") 73 | print(webhook.json) 74 | # Fail the script if discord returns anything except OK status 75 | assert ( 76 | response.ok 77 | ), f"Discord webhook failed {response.status_code} {response.content}" 78 | 79 | 80 | if __name__ == "__main__": 81 | parser = argparse.ArgumentParser("Discord embed content generator for build system") 82 | parser.add_argument("--discord-url", help="Discord webhook URL") 83 | parser.add_argument("--ref-name", help="Tag/branch/commit for current build") 84 | parser.add_argument("--git-commit", help="git commit message") 85 | parser.add_argument("--git-committer", help="git commit author name") 86 | args = parser.parse_args() 87 | main(args) 88 | -------------------------------------------------------------------------------- /.github/labeler.yml: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - changed-files: 3 | - any-glob-to-any-file: 4 | - "**/go.mod" 5 | - "**/go.sum" 6 | - "**/requirements.txt" 7 | 8 | docker: 9 | - changed-files: 10 | - any-glob-to-any-file: 11 | - "docker/**" 12 | 13 | docs: 14 | - changed-files: 15 | - any-glob-to-any-file: 16 | - "doc/**" 17 | 18 | github_actions: 19 | - changed-files: 20 | - any-glob-to-any-file: 21 | - ".github/workflows/*.yml" 22 | - ".github/workflows/*.yaml" 23 | 24 | go: 25 | - changed-files: 26 | - any-glob-to-any-file: 27 | - "**/*.go" 28 | 29 | ai: 30 | - changed-files: 31 | - any-glob-to-any-file: 32 | - "ai/**" 33 | - "**/ai_*.go" 34 | -------------------------------------------------------------------------------- /.github/requirements.txt: -------------------------------------------------------------------------------- 1 | discord-webhook 2 | -------------------------------------------------------------------------------- /.github/workflows/docker-mediamtx.yaml: -------------------------------------------------------------------------------- 1 | name: MediaMTX Docker build 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - master 7 | paths: 8 | - 'docker/*mediamtx*' 9 | push: 10 | branches: 11 | - master 12 | paths: 13 | - 'docker/*mediamtx*' 14 | 15 | concurrency: 16 | group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} 17 | cancel-in-progress: true 18 | 19 | jobs: 20 | mediamtx: 21 | name: MediaMTX docker build 22 | if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository 23 | permissions: 24 | packages: write 25 | contents: read 26 | runs-on: ubuntu-latest 27 | steps: 28 | - name: Check out code 29 | uses: actions/checkout@v4 30 | with: 31 | fetch-depth: 0 32 | # Check https://github.com/livepeer/go-livepeer/pull/1891 33 | # for ref value discussion 34 | ref: ${{ github.event.pull_request.head.sha }} 35 | 36 | - name: Set up QEMU 37 | uses: docker/setup-qemu-action@v3 38 | 39 | - name: Set up Docker Buildx 40 | uses: docker/setup-buildx-action@v3 41 | 42 | - name: Extract metadata (tags, labels) for image 43 | id: meta 44 | uses: docker/metadata-action@v5 45 | with: 46 | images: | 47 | livepeerci/mediamtx 48 | ghcr.io/${{ github.repository }}/mediamtx 49 | tags: | 50 | type=sha 51 | type=ref,event=pr 52 | type=ref,event=tag 53 | type=sha,format=long 54 | type=ref,event=branch 55 | type=semver,pattern={{version}} 56 | type=semver,pattern={{major}}.{{minor}} 57 | type=raw,value=latest,enable={{is_default_branch}} 58 | type=raw,value=${{ github.event.pull_request.head.ref }} 59 | 60 | - name: Login to DockerHub 61 | uses: docker/login-action@v3 62 | with: 63 | username: ${{ secrets.CI_DOCKERHUB_USERNAME }} 64 | password: ${{ secrets.CI_DOCKERHUB_TOKEN }} 65 | 66 | - name: Log in to the Container registry 67 | uses: docker/login-action@v3 68 | with: 69 | registry: ghcr.io 70 | username: ${{ github.actor }} 71 | password: ${{ github.token }} 72 | 73 | - name: Build and push 74 | uses: docker/build-push-action@v6 75 | with: 76 | context: docker/ 77 | file: "docker/Dockerfile.mediamtx" 78 | platforms: linux/amd64 79 | provenance: mode=max 80 | sbom: true 81 | push: true 82 | tags: ${{ steps.meta.outputs.tags }} 83 | annotations: ${{ steps.meta.outputs.annotations }} 84 | labels: ${{ steps.meta.outputs.labels }} 85 | cache-from: type=gha 86 | cache-to: type=gha,mode=max 87 | -------------------------------------------------------------------------------- /.github/workflows/issue-labeler.yml: -------------------------------------------------------------------------------- 1 | name: Label issues 2 | 3 | on: 4 | issues: 5 | types: [opened, reopened] 6 | 7 | jobs: 8 | label_issues: 9 | if: ${{ github.event_name == 'issues' }} 10 | runs-on: ubuntu-latest 11 | permissions: 12 | issues: write 13 | steps: 14 | - name: Label issues 15 | uses: andymckay/labeler@master 16 | with: 17 | add-labels: "status: triage" 18 | repo-token: ${{ secrets.GITHUB_TOKEN }} 19 | ignore-if-assigned: false 20 | -------------------------------------------------------------------------------- /.github/workflows/pr-labeler.yml: -------------------------------------------------------------------------------- 1 | name: Label PRs 2 | on: 3 | pull_request: 4 | types: [opened, reopened] 5 | pull_request_target: 6 | types: [opened, reopened] 7 | 8 | jobs: 9 | label_pull_requests: 10 | runs-on: ubuntu-latest 11 | permissions: 12 | contents: read 13 | pull-requests: write 14 | steps: 15 | - uses: actions/checkout@v4 16 | - uses: actions/labeler@v5 17 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Create release on github 2 | 3 | on: 4 | workflow_run: 5 | workflows: 6 | - Build binaries 7 | types: 8 | - "completed" 9 | 10 | jobs: 11 | release: 12 | name: Prepare github release 13 | runs-on: ubuntu-24.04 14 | if: ${{ github.event.workflow_run.conclusion == 'success' }} 15 | steps: 16 | - name: checkout 17 | uses: actions/checkout@v4 18 | with: 19 | fetch-depth: 0 20 | ref: ${{ github.event.workflow_run.head_branch }} 21 | 22 | - name: Download artifacts from build stage 23 | uses: dawidd6/action-download-artifact@v6 24 | with: 25 | workflow: build.yaml 26 | run_id: ${{ github.event.workflow_run.id }} 27 | name: release-artifacts-.* 28 | name_is_regexp: true 29 | path: releases/ 30 | 31 | - name: Flatten all downloaded artifacts to a single directory 32 | shell: bash 33 | run: | 34 | cd releases/ 35 | find . -type f -exec mv '{}' ./ \; 36 | find . -type d -empty -delete 37 | 38 | - uses: actions-ecosystem/action-regex-match@v2 39 | id: match-tag 40 | with: 41 | text: ${{ github.event.workflow_run.head_branch }} 42 | regex: '^v([0-9]+\.\d+\.\d+)$' 43 | 44 | - name: Generate sha256 checksum and gpg signatures for release artifacts 45 | if: ${{ steps.match-tag.outputs.match != '' }} 46 | uses: livepeer/action-gh-checksum-and-gpg-sign@latest 47 | with: 48 | artifacts-dir: releases 49 | release-name: ${{ github.event.workflow_run.head_branch }} 50 | gpg-key: ${{ secrets.CI_GPG_SIGNING_KEY }} 51 | gpg-key-passphrase: ${{ secrets.CI_GPG_SIGNING_PASSPHRASE }} 52 | 53 | - name: Release to github 54 | uses: softprops/action-gh-release@v2 55 | if: ${{ steps.match-tag.outputs.match != '' }} 56 | with: 57 | generate_release_notes: true 58 | tag_name: ${{ github.event.workflow_run.head_branch }} 59 | files: | 60 | releases/* 61 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # IDE files 8 | *.vscode 9 | *.code-workspace 10 | .aider* 11 | 12 | # Test binary, build with `go test -c` 13 | *.test 14 | 15 | # Misc 16 | tmp 17 | 18 | # Output of the go coverage tool, specifically when used with LiteIDE 19 | *.out 20 | 21 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 22 | .glide/ 23 | 24 | eth/protocol 25 | 26 | # Our compiled binary 27 | /livepeer 28 | /test/e2e/livepeer 29 | /livepeer_cli 30 | /livepeer_router 31 | /livepeer_bench 32 | 33 | # Generated run scripts 34 | run_*.sh 35 | 36 | /.git.describe 37 | /livepeer-windows-amd64.zip 38 | 39 | #cmd/devtool generated directories and files 40 | /.lpdev2 41 | /.git.describe 42 | 43 | #windows build env 44 | /msys2 45 | /msys2-install 46 | /.build 47 | 48 | # mac autogenerated file 49 | .DS_Store 50 | 51 | # realtime AI box 52 | gateway.log 53 | orchestrator.log 54 | mediamtx.log 55 | 56 | box/supabase 57 | -------------------------------------------------------------------------------- /CHANGELOG_PENDING.md: -------------------------------------------------------------------------------- 1 | # Unreleased Changes 2 | 3 | ## v0.X.X 4 | 5 | ### Breaking Changes 🚨🚨 6 | 7 | ### Features ⚒ 8 | 9 | #### General 10 | 11 | #### Broadcaster 12 | 13 | #### Orchestrator 14 | 15 | #### Transcoder 16 | 17 | ### Bug Fixes 🐞 18 | 19 | #### CLI 20 | 21 | #### General 22 | 23 | #### Broadcaster 24 | 25 | #### Orchestrator 26 | 27 | #### Transcoder 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Livepeer 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | **What does this pull request do? Explain your changes. (required)** 2 | 3 | 4 | **Specific updates (required)** 5 | 6 | - 7 | - 8 | - 9 | 10 | **How did you test each of these updates (required)** 11 | 12 | 13 | 14 | **Does this pull request close any open issues?** 15 | 16 | 17 | 18 | **Checklist:** 19 | 20 | 21 | - [ ] Read the [contribution guide](./CONTRIBUTING.md) 22 | - [ ] `make` runs successfully 23 | - [ ] All tests in `./test.sh` pass 24 | - [ ] README and other documentation updated 25 | - [ ] [Pending changelog](./CHANGELOG_PENDING.md) updated 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 4 | 5 | [![go-livepeer](https://user-images.githubusercontent.com/555740/117340053-78210e80-ae6e-11eb-892c-d98085fe6824.png)](https://github.com/livepeer/go-livepeer) 6 | 7 | --- 8 | [![Go Report Card](https://goreportcard.com/badge/github.com/livepeer/go-livepeer)](https://goreportcard.com/report/github.com/livepeer/go-livepeer) 9 | [![Discord](https://img.shields.io/discord/423160867534929930.svg?style=flat-square)](https://discord.gg/livepeer) 10 | [![license](https://img.shields.io/badge/license-MIT-blue.svg?style=flat-square)](LICENSE) 11 | [![Contributions welcome](https://img.shields.io/badge/contributions-welcome-orange.svg?style=flat-square)](CONTRIBUTING.md) 12 | 13 | The Livepeer project aims to deliver a live video-streaming network protocol 14 | that is fully decentralized, highly scalable and crypto-token incentivized to 15 | serve as the live media layer in the decentralized development (Web3) stack. 16 | [Read our documentation](https://docs.livepeer.org/protocol/) to learn more about the protocol and its economic incentives. 17 | 18 | `go-livepeer` is a Go implementation of the [Livepeer](https://livepeer.org) protocol which powers the Livepeer Network. Specifically, `go-livepeer` contains implementations of Broadcaster, Orchestrator, and Transcoder nodes (roles) in the Livepeer Network ecosystem. 19 | 20 | 21 | 22 | ## Table of Contents 23 | 24 | - [Table of Contents](#table-of-contents) 25 | - [Requirements](#requirements) 26 | - [Getting Started](#getting-started) 27 | - [Contributing](#contributing) 28 | - [Resources](#resources) 29 | 30 | 31 | 32 | ## Requirements 33 | 34 | This project requires `go` and a unix shell. 35 | 36 | - [Installing and Managing Go](doc/go.md) 37 | 38 | 39 | ## Getting Started 40 | 41 | To get started, clone the repo and follow the [installation guide](https://docs.livepeer.org/guides/orchestrating/install-go-livepeer). 42 | 43 | Next, follow [the guide to set up a private ETH network with the Livepeer protocol deployed](cmd/devtool/README.md). 44 | 45 | ## Contributing 46 | 47 | Thanks for your interest in contributing to go-livepeer. There are many ways you can contribute to the project, even for non-developers. 48 | 49 | To start, take a few minutes to **[read the "Contributing to go-livepeer" guide](CONTRIBUTING.md)**. 50 | 51 | We look forward to your pull requests and / or involvement in our 52 | [issues page](https://github.com/livepeer/go-livepeer/issues) and hope to see 53 | your username on our 54 | [list of contributors](https://github.com/livepeer/go-livepeer/graphs/contributors) 55 | 🎉🎉🎉 56 | 57 | ## Resources 58 | 59 | To get a full idea of what Livepeer is about, be sure to take a look at these 60 | other resources: 61 | 62 | - 🌐 [The Livepeer Website](https://livepeer.org) 63 | - 📖 [The Livepeer Docs](https://livepeer.org/docs) 64 | - 🔭 [The 10-Minute Primer](https://livepeer.org/primer/) 65 | - ✍ [The Livepeer Blog](https://medium.com/livepeer-blog) 66 | - 💬 [The Livepeer Chat](https://discord.gg/livepeer) 67 | - ❓ [The Livepeer Forum](https://forum.livepeer.org/) 68 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.8.5 -------------------------------------------------------------------------------- /ai/file_worker.go: -------------------------------------------------------------------------------- 1 | package ai 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "os" 8 | 9 | "github.com/livepeer/go-livepeer/ai/worker" 10 | ) 11 | 12 | type FileWorker struct { 13 | files map[string]string 14 | } 15 | 16 | func NewFileWorker(files map[string]string) *FileWorker { 17 | return &FileWorker{files: files} 18 | } 19 | 20 | func (w *FileWorker) TextToImage(ctx context.Context, req worker.GenTextToImageJSONRequestBody) (*worker.ImageResponse, error) { 21 | fname, ok := w.files["text-to-image"] 22 | if !ok { 23 | return nil, errors.New("text-to-image response file not found") 24 | } 25 | 26 | data, err := os.ReadFile(fname) 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | var resp worker.ImageResponse 32 | if err := json.Unmarshal(data, &resp); err != nil { 33 | return nil, err 34 | } 35 | 36 | return &resp, nil 37 | } 38 | 39 | func (w *FileWorker) ImageToImage(ctx context.Context, req worker.GenImageToImageMultipartRequestBody) (*worker.ImageResponse, error) { 40 | fname, ok := w.files["image-to-image"] 41 | if !ok { 42 | return nil, errors.New("image-to-image response file not found") 43 | } 44 | 45 | data, err := os.ReadFile(fname) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | var resp worker.ImageResponse 51 | if err := json.Unmarshal(data, &resp); err != nil { 52 | return nil, err 53 | } 54 | 55 | return &resp, nil 56 | } 57 | 58 | func (w *FileWorker) ImageToVideo(ctx context.Context, req worker.GenImageToVideoMultipartRequestBody) (*worker.VideoResponse, error) { 59 | fname, ok := w.files["image-to-video"] 60 | if !ok { 61 | return nil, errors.New("image-to-video response file not found") 62 | } 63 | 64 | data, err := os.ReadFile(fname) 65 | if err != nil { 66 | return nil, err 67 | } 68 | 69 | var resp worker.VideoResponse 70 | if err := json.Unmarshal(data, &resp); err != nil { 71 | return nil, err 72 | } 73 | 74 | return &resp, nil 75 | } 76 | 77 | func (w *FileWorker) Upscale(ctx context.Context, req worker.GenUpscaleMultipartRequestBody) (*worker.ImageResponse, error) { 78 | fname, ok := w.files["upscale"] 79 | if !ok { 80 | return nil, errors.New("upscale response file not found") 81 | } 82 | 83 | data, err := os.ReadFile(fname) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | var resp worker.ImageResponse 89 | if err := json.Unmarshal(data, &resp); err != nil { 90 | return nil, err 91 | } 92 | 93 | return &resp, nil 94 | } 95 | 96 | func (w *FileWorker) Warm(ctx context.Context, containerName, modelID string) error { 97 | return nil 98 | } 99 | 100 | func (w *FileWorker) Stop(ctx context.Context, containerName string) error { 101 | return nil 102 | } 103 | -------------------------------------------------------------------------------- /ai/worker/b64.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "image" 7 | "image/gif" 8 | "image/jpeg" 9 | "image/png" 10 | "io" 11 | "os" 12 | 13 | "github.com/vincent-petithory/dataurl" 14 | ) 15 | 16 | func ReadImageB64DataUrl(url string, w io.Writer) error { 17 | dataURL, err := dataurl.DecodeString(url) 18 | if err != nil { 19 | return err 20 | } 21 | 22 | img, _, err := image.Decode(bytes.NewReader(dataURL.Data)) 23 | if err != nil { 24 | return err 25 | } 26 | 27 | switch dataURL.MediaType.ContentType() { 28 | case "image/png": 29 | err = png.Encode(w, img) 30 | case "image/jpg", "image/jpeg": 31 | err = jpeg.Encode(w, img, nil) 32 | case "image/gif": 33 | err = gif.Encode(w, img, nil) 34 | // Add cases for other image formats if necessary 35 | default: 36 | return fmt.Errorf("unsupported image format: %s", dataURL.MediaType.ContentType()) 37 | } 38 | 39 | return err 40 | } 41 | 42 | func SaveImageB64DataUrl(url, outputPath string) error { 43 | file, err := os.Create(outputPath) 44 | if err != nil { 45 | return err 46 | } 47 | defer file.Close() 48 | 49 | return ReadImageB64DataUrl(url, file) 50 | } 51 | 52 | func ReadAudioB64DataUrl(url string, w io.Writer) error { 53 | dataURL, err := dataurl.DecodeString(url) 54 | if err != nil { 55 | return err 56 | } 57 | 58 | w.Write(dataURL.Data) 59 | 60 | return nil 61 | } 62 | -------------------------------------------------------------------------------- /ai/worker/b64_test.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "bytes" 5 | "encoding/base64" 6 | "image" 7 | "image/color" 8 | "image/png" 9 | "os" 10 | "testing" 11 | 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestReadImageB64DataUrl(t *testing.T) { 16 | tests := []struct { 17 | name string 18 | dataURL string 19 | expectError bool 20 | }{ 21 | { 22 | name: "Valid PNG Image", 23 | dataURL: func() string { 24 | img := image.NewRGBA(image.Rect(0, 0, 1, 1)) 25 | img.Set(0, 0, color.RGBA{255, 0, 0, 255}) // Set a single red pixel 26 | var imgBuf bytes.Buffer 27 | err := png.Encode(&imgBuf, img) 28 | require.NoError(t, err) 29 | 30 | return "data:image/png;base64," + base64.StdEncoding.EncodeToString(imgBuf.Bytes()) 31 | }(), 32 | expectError: false, 33 | }, 34 | { 35 | name: "Unsupported Image Format", 36 | dataURL: "data:image/bmp;base64," + base64.StdEncoding.EncodeToString([]byte{ 37 | 0x42, 0x4D, // BMP header 38 | // ... (rest of the BMP data) 39 | }), 40 | expectError: true, 41 | }, 42 | { 43 | name: "Invalid Data URL", 44 | dataURL: "invalid-data-url", 45 | expectError: true, 46 | }, 47 | } 48 | 49 | for _, tt := range tests { 50 | t.Run(tt.name, func(t *testing.T) { 51 | var buf bytes.Buffer 52 | err := ReadImageB64DataUrl(tt.dataURL, &buf) 53 | if tt.expectError { 54 | require.Error(t, err) 55 | } else { 56 | require.NoError(t, err) 57 | require.NotEmpty(t, buf.Bytes()) 58 | } 59 | }) 60 | } 61 | } 62 | 63 | func TestSaveImageB64DataUrl(t *testing.T) { 64 | img := image.NewRGBA(image.Rect(0, 0, 1, 1)) 65 | img.Set(0, 0, color.RGBA{255, 0, 0, 255}) // Set a single red pixel 66 | var imgBuf bytes.Buffer 67 | err := png.Encode(&imgBuf, img) 68 | require.NoError(t, err) 69 | dataURL := "data:image/png;base64," + base64.StdEncoding.EncodeToString(imgBuf.Bytes()) 70 | 71 | outputPath := "test_output.png" 72 | defer os.Remove(outputPath) 73 | 74 | err = SaveImageB64DataUrl(dataURL, outputPath) 75 | require.NoError(t, err) 76 | 77 | // Verify that the file was created and is not empty 78 | fileInfo, err := os.Stat(outputPath) 79 | require.NoError(t, err) 80 | require.False(t, fileInfo.IsDir()) 81 | require.NotZero(t, fileInfo.Size()) 82 | } 83 | 84 | func TestReadAudioB64DataUrl(t *testing.T) { 85 | // Create a sample audio data and encode it as a data URL 86 | audioData := []byte{0x00, 0x01, 0x02, 0x03, 0x04} 87 | dataURL := "data:audio/wav;base64," + base64.StdEncoding.EncodeToString(audioData) 88 | 89 | var buf bytes.Buffer 90 | err := ReadAudioB64DataUrl(dataURL, &buf) 91 | require.NoError(t, err) 92 | require.Equal(t, audioData, buf.Bytes()) 93 | } 94 | -------------------------------------------------------------------------------- /ai/worker/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package `worker` hosts the main AI worker logic for managing or using runner 3 | containers for processing inference requests on the Livepeer AI subnet. The 4 | package allows interacting with the [AI runner containers], and it includes: 5 | 6 | - Golang API Bindings (./runner.gen.go): 7 | 8 | Generated from the AI runner's OpenAPI spec. To re-generate them run: `make ai_worker_codegen` 9 | 10 | - Worker (./worker.go): 11 | 12 | Listens for inference requests from the Livepeer AI subnet and routes them to the AI runner. 13 | 14 | - Docker Manager (./docker.go): 15 | 16 | Manages AI runner containers. For a state diagram showing the lifecycle of a container, see the /doc/worker.md file. 17 | 18 | [AI runner containers]: https://github.com/livepeer/ai-runner 19 | */ 20 | package worker 21 | -------------------------------------------------------------------------------- /ai/worker/utils.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "context" 5 | "github.com/Masterminds/semver/v3" 6 | "github.com/livepeer/go-livepeer/clog" 7 | ) 8 | 9 | // LowestVersion returns the lowest version of a given pipeline and model ID from a list of versions. 10 | func LowestVersion(versions []Version, pipeline string, modelId string) string { 11 | var res string 12 | var lowest *semver.Version 13 | 14 | for _, v := range versions { 15 | if v.Pipeline != pipeline || v.ModelId != modelId { 16 | continue 17 | } 18 | ver, err := semver.NewVersion(v.Version) 19 | if err != nil { 20 | clog.Warningf(context.Background(), "Invalid runner version '%s'", v) 21 | continue 22 | } 23 | if lowest == nil || ver.LessThan(lowest) { 24 | if lowest != nil { 25 | clog.Warningf(context.Background(), "Orchestrator has multiple versions set for the same pipeline and model ID. Using the lowest version: %s", ver) 26 | } 27 | lowest = ver 28 | res = v.Version 29 | } 30 | } 31 | return res 32 | } 33 | -------------------------------------------------------------------------------- /ai/worker/utils_test.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import "testing" 4 | 5 | func TestLowestVersion(t *testing.T) { 6 | versions := []Version{ 7 | {Pipeline: "pipeline1", ModelId: "model1", Version: "1.0.0"}, 8 | {Pipeline: "pipeline1", ModelId: "model1", Version: "2.0.0"}, 9 | {Pipeline: "pipeline2", ModelId: "model2", Version: "1.5.0"}, 10 | {Pipeline: "pipeline2", ModelId: "model2", Version: "0.1.0"}, 11 | {Pipeline: "pipeline2", ModelId: "model2", Version: "1.5.0"}, 12 | {Pipeline: "pipeline2", ModelId: "model2", Version: "0.0.2"}, 13 | } 14 | 15 | tests := []struct { 16 | pipeline string 17 | modelId string 18 | expected string 19 | }{ 20 | {"pipeline1", "model1", "1.0.0"}, 21 | {"pipeline2", "model2", "0.0.2"}, 22 | {"pipeline3", "model3", ""}, 23 | } 24 | 25 | for _, test := range tests { 26 | t.Run(test.pipeline+"_"+test.modelId, func(t *testing.T) { 27 | result := LowestVersion(versions, test.pipeline, test.modelId) 28 | if result != test.expected { 29 | t.Errorf("Expected %s, got %s", test.expected, result) 30 | } 31 | }) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /box/aiModels-comfyui.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "model_id": "comfyui", 4 | "pipeline": "live-video-to-video", 5 | "warm": true, 6 | "capacity": 1 7 | } 8 | ] 9 | -------------------------------------------------------------------------------- /box/aiModels-noop.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "model_id": "noop", 4 | "pipeline": "live-video-to-video", 5 | "warm": true, 6 | "capacity": 1 7 | } 8 | ] 9 | -------------------------------------------------------------------------------- /box/box.md: -------------------------------------------------------------------------------- 1 | # Realtime Video AI in a Box 2 | 3 | ## Requirements 4 | - Docker is installed (executing `docker` should succeed) 5 | - [ffmpeg](https://ffmpeg.org/) is installed (executing `ffmpeg` and `ffplay` should succeed) 6 | 7 | ## Usage (Linux AMD64) 8 | 9 | ``` 10 | export DOCKER=true 11 | ``` 12 | 13 | 1. Start everything with the following command 14 | ```bash 15 | make box 16 | ``` 17 | 2. Start streaming 18 | ```bash 19 | make box-stream 20 | ``` 21 | 22 | 3. Playback the stream 23 | ```bash 24 | make box-playback 25 | ``` 26 | 27 | ## Usage (M1 / Linux ARM64) 28 | 29 | It requires the following points: 30 | - go-livepeer compilation configuration (executing `make` should succeed) 31 | - [mediamtx](https://github.com/bluenviron/mediamtx) is installed (executing `mediamtx` should succeed) 32 | 33 | 1. Start everything with the following command 34 | ```bash 35 | make box 36 | ``` 37 | 2. Start streaming 38 | ```bash 39 | make box-stream 40 | ``` 41 | 42 | 3. Playback the stream 43 | ```bash 44 | make box-playback 45 | ``` 46 | 47 | ## Usage with ComfyUI Pipeline (requires GPU) 48 | 49 | ``` 50 | export DOCKER=true 51 | export PIPELINE=comfyui 52 | ``` 53 | 54 | 1. Download models 55 | 56 | ```bash 57 | cd ../ai-runner/runner 58 | ./dl_checkpoints.sh --tensorrt 59 | export AI_MODELS_DIR=$(pwd)/models 60 | ``` 61 | 62 | 2. Start everything with the following command 63 | ```bash 64 | make box 65 | ``` 66 | 3. Start streaming 67 | ```bash 68 | make box-stream 69 | ``` 70 | 71 | 4. Playback the stream 72 | ```bash 73 | make box-playback 74 | ``` 75 | 76 | ## Additional Configuration 77 | 78 | ### RTMP Output 79 | 80 | If you also want to send the inference output to an external RTMP endpoint, set the `RTMP_OUTPUT` env var: 81 | ```bash 82 | export RTMP_OUTPUT=rtmp://rtmp.livepeer.com/live/$STREAM_KEY 83 | ``` 84 | 85 | This one is only required for the `box-stream` command. It is useful when you cannot use the `box-playback` command to play the stream, for example when you are using a remote non-UI machine. 86 | 87 | ### Docker 88 | If you want to run the box in a docker container, set the `DOCKER` env var: 89 | ```bash 90 | export DOCKER=true 91 | ``` 92 | 93 | In general the Docker setup is simpler, but it's not possible to build the `go-liveeer` Docker image on M1 / Linux ARM64 machines. 94 | 95 | ### Rebuild 96 | 97 | By default, all dependencies are rebuilt every time you run the `make box` command. However, if you don't want this to happen, you can set the following env variable. 98 | 99 | ```bash 100 | export REBUILD=false 101 | ``` 102 | 103 | ### Each component separately 104 | 105 | You can also run each service separately. 106 | ```bash 107 | make box-gateway 108 | make box-orchestrator 109 | make box-mediamtx 110 | make box-stream 111 | make box-playback 112 | ``` 113 | 114 | ### Rebuilding runner 115 | To rebuild and restart the runner, run the following command: 116 | ```bash 117 | make box-runner 118 | ``` 119 | 120 | ## Frontend 121 | 122 | To start the frontend, run the following commands: 123 | ```bash 124 | make box-supabase 125 | make box-frontend 126 | ``` 127 | 128 | You can access the frontend with the following URL: http://localhost:3000/create?whipServer=http://127.0.0.1:5936/live/video-to-video/&pipeline=noop&videoJS=true 129 | 130 | Note that the box-frontend assumes you have the https://github.com/livepeer/pipelines/ repo cloned at the parent directory. 131 | Note also that you can start frontend together with `make box` by setting `export FRONTEND=true`. -------------------------------------------------------------------------------- /box/box.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | FRONTEND=${FRONTEND:-false} 5 | 6 | # Start multiple processes and output their logs to the console 7 | gateway() { 8 | echo "Starting Gateway..." 9 | ./box/gateway.sh | tee gateway.log 10 | } 11 | 12 | orchestrator() { 13 | echo "Starting Orchestrator..." 14 | ./box/orchestrator.sh | tee orchestrator.log 15 | } 16 | 17 | mediamtx() { 18 | echo "Starting MediaMTX..." 19 | ./box/mediamtx.sh | tee mediamtx.log 20 | } 21 | 22 | supabase() { 23 | echo "Starting Supabase..." 24 | ./box/supabase.sh | tee supabase.log 25 | } 26 | 27 | frontend() { 28 | echo "Starting Frontend..." 29 | ./box/frontend.sh | tee supabase.log 30 | } 31 | 32 | # Run processes in the background 33 | gateway & 34 | orchestrator & 35 | mediamtx & 36 | 37 | if [ "$DOCKER" = "true" ]; then 38 | supabase & 39 | mediamtx & 40 | fi 41 | 42 | # Wait for all background processes to finish 43 | wait 44 | 45 | echo "All processes have completed." 46 | -------------------------------------------------------------------------------- /box/build-runner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | PIPELINE=${PIPELINE:-noop} 5 | 6 | if [[ "$PIPELINE" != "noop" && "$PIPELINE" != "comfyui" ]]; then 7 | echo "Error: PIPELINE must be either 'noop' or 'comfyui'" 8 | exit 1 9 | fi 10 | 11 | # Switch to neighbour ai-runner directory 12 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 13 | cd "$SCRIPT_DIR/../../ai-runner/runner" 14 | 15 | VERSION="$(bash print_version.sh)" 16 | 17 | docker build -t livepeer/ai-runner:live-base -f docker/Dockerfile.live-base . 18 | if [ "${PIPELINE}" = "noop" ]; then 19 | docker build -t livepeer/ai-runner:live-app-noop -f docker/Dockerfile.live-app-noop --build-arg VERSION=${VERSION} . 20 | else 21 | docker build -t livepeer/ai-runner:live-base-${PIPELINE} -f docker/Dockerfile.live-base-${PIPELINE} . 22 | docker build -t livepeer/ai-runner:live-app-${PIPELINE} -f docker/Dockerfile.live-app__PIPELINE__ --build-arg PIPELINE=${PIPELINE} --build-arg VERSION=${VERSION} . 23 | fi 24 | 25 | docker stop live-video-to-video_${PIPELINE}_8900 || true 26 | -------------------------------------------------------------------------------- /box/frontend.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | LIVEPEER_PWD=$(pwd) 4 | 5 | cd ../pipelines/apps/app 6 | pnpm install 7 | 8 | if [ ! -f .env ]; then 9 | echo "Creating .env file..." 10 | cp .env.example .env 11 | fi 12 | 13 | export PGPASSWORD='your-super-secret-and-long-postgres-password' 14 | 15 | DB_PREPARED=$(psql -h 127.0.0.1 -p 5433 -U postgres -d postgres -tAc "SELECT 1 FROM information_schema.tables WHERE table_name='pipelines';") 16 | 17 | if [ "$DB_PREPARED" != "1" ]; then 18 | echo "Table pipelines does not exist, setting up the database..." 19 | psql -h 127.0.0.1 -p 5433 -U postgres -d postgres -c "CREATE EXTENSION pg_stat_monitor;" 20 | fi 21 | 22 | pnpm db:push 23 | 24 | MAIN_PIPELINE_EXISTS=$(psql -h 127.0.0.1 -p 5433 -U postgres -d postgres -tAc "SELECT 1 FROM pipelines where id = 'pip_DRQREDnSei4HQyC8';") 25 | if [ "$MAIN_PIPELINE_EXISTS" != "1" ]; then 26 | echo "Main Dreamshaper pipeline does not exist, creating..." 27 | psql -h 127.0.0.1 -p 5433 -U postgres -d postgres -f "${LIVEPEER_PWD}/box/frontend.sql" 28 | fi 29 | 30 | pnpm dev 31 | -------------------------------------------------------------------------------- /box/gateway.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | DOCKER=${DOCKER:-false} 5 | 6 | if [ "$DOCKER" = "false" ]; then 7 | LIVE_AI_WHIP_ADDR=":7280" LIVE_AI_ALLOW_CORS="1" ./livepeer -gateway -rtmpAddr :1936 -httpAddr :5936 -orchAddr localhost:8935 -v 6 -monitor 8 | else 9 | docker run -e LIVE_AI_WHIP_ADDR=":7280" -e LIVE_AI_ALLOW_CORS="1" --rm --name gateway --network host livepeer/go-livepeer -gateway -rtmpAddr :1936 -httpAddr :5936 -orchAddr localhost:8935 -v 6 -monitor 10 | fi 11 | -------------------------------------------------------------------------------- /box/mediamtx.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | DOCKER=${DOCKER:-false} 5 | 6 | if [ "$DOCKER" = "false" ]; then 7 | mediamtx ./box/mediamtx.yml 8 | else 9 | docker run --rm --name mediamtx --network host -v $(pwd)/box/mediamtx.yml:/mediamtx.yml livepeerci/mediamtx 10 | fi 11 | -------------------------------------------------------------------------------- /box/orchestrator.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | DOCKER=${DOCKER:-false} 5 | PIPELINE=${PIPELINE:-noop} 6 | AI_RUNNER_CONTAINERS_PER_GPU=${AI_RUNNER_CONTAINERS_PER_GPU:-1} 7 | 8 | DOCKER_HOSTNAME="172.17.0.1" 9 | if [[ "$(uname)" == "Darwin" ]]; then 10 | # Docker on macOS has a special host address 11 | DOCKER_HOSTNAME="host.docker.internal" 12 | fi 13 | 14 | NVIDIA="" 15 | AI_MODELS_DIR=${AI_MODELS_DIR:-} 16 | if [[ "$PIPELINE" != "noop" ]]; then 17 | NVIDIA="-nvidia all" 18 | if [[ "$AI_MODELS_DIR" = "" ]]; then 19 | AI_MODELS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd ../../ai-runner/runner/models && pwd )" 20 | fi 21 | AI_MODELS_DIR_FLAG="-aiModelsDir ${AI_MODELS_DIR}" 22 | fi 23 | 24 | if [ "$DOCKER" = "false" ]; then 25 | ./livepeer \ 26 | -orchestrator \ 27 | -aiWorker \ 28 | -aiModels ./box/aiModels-${PIPELINE}.json \ 29 | -aiRunnerContainersPerGPU ${AI_RUNNER_CONTAINERS_PER_GPU} \ 30 | ${AI_MODELS_DIR_FLAG} \ 31 | ${NVIDIA} \ 32 | -serviceAddr localhost:8935 \ 33 | -transcoder \ 34 | -v 6 \ 35 | -liveAITrickleHostForRunner "$DOCKER_HOSTNAME:8935" \ 36 | -monitor 37 | else 38 | docker run --rm --name orchestrator \ 39 | --network host \ 40 | -v /var/run/docker.sock:/var/run/docker.sock \ 41 | -v ./box/aiModels-${PIPELINE}.json:/opt/aiModels.json \ 42 | livepeer/go-livepeer \ 43 | -orchestrator \ 44 | -aiWorker \ 45 | -aiModels /opt/aiModels.json \ 46 | -aiRunnerContainersPerGPU ${AI_RUNNER_CONTAINERS_PER_GPU} \ 47 | ${AI_MODELS_DIR_FLAG} \ 48 | -serviceAddr 127.0.0.1:8935 \ 49 | -transcoder \ 50 | -v 6 \ 51 | -liveAITrickleHostForRunner '172.17.0.1:8935' \ 52 | -monitor 53 | fi 54 | -------------------------------------------------------------------------------- /box/stream.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | PIPELINE=${PIPELINE:-noop} 5 | STREAM_KEY="my-stream" 6 | STREAM_ID="my-stream-id" 7 | RTMP_OUTPUT=${RTMP_OUTPUT:-""} 8 | 9 | case "$1" in 10 | start) 11 | QUERY="pipeline=${PIPELINE}\&streamId=${STREAM_ID}" 12 | if [ -n "$RTMP_OUTPUT" ]; then 13 | QUERY="${QUERY}\&rtmpOutput=${RTMP_OUTPUT}" 14 | fi 15 | 16 | ffmpeg -re -f lavfi \ 17 | -i testsrc=size=1920x1080:rate=30,format=yuv420p \ 18 | -vf scale=1280:720 \ 19 | -c:v libx264 \ 20 | -b:v 1000k \ 21 | -x264-params keyint=60 \ 22 | -f flv rtmp://127.0.0.1:1935/${STREAM_KEY}?${QUERY} 23 | ;; 24 | playback) 25 | ffplay rtmp://127.0.0.1:1935/${STREAM_KEY}-out 26 | ;; 27 | *) 28 | echo "Usage: $0 {start|playback}" 29 | exit 1 30 | ;; 31 | esac 32 | -------------------------------------------------------------------------------- /box/supabase.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Download Docker scripts to run Supabase locally 4 | DEST="./box/supabase" 5 | if [ ! -d "$DEST" ]; then 6 | echo "Supabase directory: $DEST does not exist exists. Setting up..." 7 | cd ./box 8 | git clone --branch v1.24.09 --depth 1 https://github.com/supabase/supabase.git supabase-repo 9 | cp -r supabase-repo/docker ./supabase 10 | rm -rf supabase-repo 11 | cp docker-compose.supabase.yml supabase/docker-compose.yml 12 | cp supabase/.env.example supabase/.env 13 | cd .. 14 | fi 15 | 16 | cd ./box/supabase 17 | docker compose up 18 | -------------------------------------------------------------------------------- /build/chain.go: -------------------------------------------------------------------------------- 1 | package build 2 | 3 | // SupportedChains is an enum that indicates the chains supported by the node 4 | // All chains represented by an enum value less than or equal than this value are supported 5 | // All chains represented by an enum value greater than this value are not supported 6 | type SupportedChains int 7 | 8 | const ( 9 | // Dev is a development chain 10 | Dev SupportedChains = iota 11 | // Rinkeby is the Ethereum Rinkeby or Arbitrum Testnet test network chain 12 | Rinkeby 13 | // Mainnet is the Ethereum or Arbitrum main network chain 14 | Mainnet 15 | ) 16 | 17 | // ChainSupported returns whether the node can connect to the chain with the given ID 18 | func ChainSupported(chainID int64) bool { 19 | switch chainID { 20 | case 4, 421611: 21 | return Rinkeby <= HighestChain 22 | case 1, 42161: 23 | return Mainnet <= HighestChain 24 | default: 25 | return Dev <= HighestChain 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /build/chain_dev.go: -------------------------------------------------------------------------------- 1 | //go:build !mainnet && !rinkeby 2 | // +build !mainnet,!rinkeby 3 | 4 | package build 5 | 6 | const HighestChain = Dev 7 | -------------------------------------------------------------------------------- /build/chain_mainnet.go: -------------------------------------------------------------------------------- 1 | //go:build mainnet 2 | // +build mainnet 3 | 4 | package build 5 | 6 | const HighestChain = Mainnet 7 | -------------------------------------------------------------------------------- /build/chain_rinkeby.go: -------------------------------------------------------------------------------- 1 | //go:build rinkeby 2 | // +build rinkeby 3 | 4 | package build 5 | 6 | const HighestChain = Rinkeby 7 | -------------------------------------------------------------------------------- /build/const.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | // +build !windows 3 | 4 | package build 5 | 6 | const AcceptMultiline = "Ctrl+D" 7 | -------------------------------------------------------------------------------- /build/const_windows.go: -------------------------------------------------------------------------------- 1 | //go:build windows 2 | // +build windows 3 | 4 | package build 5 | 6 | const AcceptMultiline = "Ctrl+Z" 7 | -------------------------------------------------------------------------------- /ci_env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Script to populate some environment variables in various CI processes. Should be 4 | # invoked by `ci_env.sh [script-name]`. 5 | 6 | set -eo pipefail 7 | 8 | # Populate necessary Windows build path stuff 9 | if [[ $(uname) == *"MSYS"* ]]; then 10 | export PATH="/usr/bin:/mingw64/bin:$PATH" 11 | export HOME="/build" 12 | export C_INCLUDE_PATH="$HOME/compiled/lib:/mingw64/lib:/mingw64/lib:${C_INCLUDE_PATH:-}" 13 | mkdir -p $HOME 14 | 15 | export PATH="$HOME/compiled/bin":$PATH 16 | export PKG_CONFIG_PATH="/mingw64/lib/pkgconfig:$HOME/compiled/lib/pkgconfig" 17 | export GOROOT=/mingw64/lib/go 18 | export GOPATH=/mingw64 19 | fi 20 | 21 | # If we want to build with branch --> network support for any other networks, add them here! 22 | NETWORK_BRANCHES="dev rinkeby" 23 | 24 | branch="" 25 | if [[ "${TRAVIS_BRANCH:-}" != "" ]]; then 26 | branch="$TRAVIS_BRANCH" 27 | elif [[ "${GITHUB_REF_NAME:-}" != "" ]]; then 28 | branch="$GITHUB_REF_NAME" 29 | fi 30 | 31 | # By default we build with mainnet support 32 | # If we are on the dev branch then we do not build with Rinkeby or mainnet support 33 | # If we are on the rinkeby branch then we build with Rinkeby support, but not mainnet support 34 | export HIGHEST_CHAIN_TAG=mainnet 35 | for networkBranch in $NETWORK_BRANCHES; do 36 | if [[ $branch == "$networkBranch" ]]; then 37 | export HIGHEST_CHAIN_TAG=$networkBranch 38 | fi 39 | done 40 | 41 | # Allow non-tagged mainnet builds, but tagged releases should have mainnet support 42 | generatedVersion=$(./print_version.sh) 43 | definedVersion=$(cat VERSION) 44 | if [[ $HIGHEST_CHAIN_TAG != "mainnet" ]]; then 45 | if [[ $generatedVersion == $definedVersion ]]; then 46 | echo "disallowing semver tag release $generatedVersion on branch '$branch', should be 'mainnet'" 47 | exit 1 48 | fi 49 | fi 50 | 51 | export BUILD_TAGS="$HIGHEST_CHAIN_TAG" 52 | 53 | # Only build with experimental tag for non-semver tagged releases 54 | if [[ $generatedVersion != $definedVersion ]]; then 55 | export BUILD_TAGS="${BUILD_TAGS},experimental" 56 | fi 57 | 58 | if [[ "$CI" == "true" ]]; then 59 | echo "build-tags=${BUILD_TAGS}" >>"$GITHUB_OUTPUT" 60 | fi 61 | 62 | exec "$@" 63 | -------------------------------------------------------------------------------- /cmd/devtool/README.md: -------------------------------------------------------------------------------- 1 | # devtool 2 | 3 | An on-chain workflow testing tool that supports the following: 4 | 5 | - Automatically submitting the necessary setup transactions for each node type 6 | - Generating a Bash script with default CLI flags to start each node type 7 | 8 | ## Prerequisites 9 | 10 | - [Docker](https://docs.docker.com/get-docker/) 11 | - [Go](https://golang.org/doc/install) 12 | - [Go-livepeer](https://github.com/livepeer/go-livepeer) build from source (see [the docs](https://docs.livepeer.org/orchestrators/guides/install-go-livepeer#build-from-source)). 13 | 14 | ## Setting Up an On-Chain Development Environment 15 | 16 | ### Step 1: Set up a private ETH network with Livepeer protocol deployed 17 | 18 | ```bash 19 | docker pull livepeer/geth-with-livepeer-protocol:confluence 20 | docker run -p 8545:8545 -p 8546:8546 --name geth-with-livepeer-protocol livepeer/geth-with-livepeer-protocol:confluence 21 | ``` 22 | 23 | ### Step 2: Set up a broadcaster 24 | 25 | `go run cmd/devtool/devtool.go setup broadcaster` 26 | 27 | This command will submit the setup transactions for a broadcaster and generate the Bash script 28 | `run_broadcaster_.sh` which can be used to start a broadcaster node. 29 | 30 | ### Step 3: Set up an orchestrator/transcoder 31 | 32 | `go run cmd/devtool/devtool.go setup transcoder` 33 | 34 | This command will submit the setup transactions for an orchestrator/transcoder and generate the Bash scripts: 35 | 36 | - `run_orchestrator_with_transcoder_.sh` which can be used to start an orchestrator node that contains a transcoder (combined OT) 37 | - `run_orchestrator_standalone_.sh` and `run_transcoder_.sh` which can be used to start separate orchestrator and transcoder nodes (split O/T) 38 | 39 | ## Extra Resources 40 | 41 | ### Scripts 42 | 43 | Some helpful scripts are provided in the `scripts` directory. 44 | -------------------------------------------------------------------------------- /cmd/devtool/scripts/create_multiple_transcoders.bash: -------------------------------------------------------------------------------- 1 | # Script to create multiple transcoders on the ETH devnet. 2 | 3 | CLI_PORT=7935 4 | MEDIA_PORT=8935 5 | RTMP_PORT=1935 6 | BOND=50 7 | 8 | if [ -z "$1" ]; then 9 | echo "Usage: create_multiple_transcoders.bash " 10 | exit 1 11 | fi 12 | 13 | num_transcoders=$1 14 | 15 | # Run the go devtool transcoder script multiple times with different ports 16 | echo "Creating $num_transcoders transcoders..." 17 | for i in $(seq 1 $num_transcoders); do 18 | echo "Creating transcoder $i..." 19 | go run cmd/devtool/devtool.go --cliport $CLI_PORT --mediaport $MEDIA_PORT --rtmpport $RTMP_PORT -bond $BOND setup transcoder 20 | CLI_PORT=$((CLI_PORT + 10)) 21 | MEDIA_PORT=$((MEDIA_PORT + 10)) 22 | RTMP_PORT=$((RTMP_PORT + 10)) 23 | BOND=$((BOND ** 10)) 24 | done 25 | 26 | echo "Done creating $num_transcoders transcoders" 27 | -------------------------------------------------------------------------------- /cmd/livepeer/livepeer_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "os" 6 | "testing" 7 | 8 | "github.com/peterbourgon/ff/v3" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | // This test exists because we wanted to make our CLI argument casing consistent without 13 | // breaking backwards compatibility 14 | func TestParseAcceptsEitherDatadirCasing(t *testing.T) { 15 | // Reset between tests 16 | flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) 17 | 18 | // No Casing 19 | lpc := parseLivepeerConfig() 20 | 21 | err := ff.Parse(flag.CommandLine, []string{ 22 | "-datadir", "/some/data/dir", 23 | }) 24 | require.NoError(t, err) 25 | require.NotNil(t, lpc.Datadir) 26 | require.Equal(t, "/some/data/dir", *lpc.Datadir) 27 | 28 | // Reset between tests 29 | flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) 30 | 31 | // Camel Casing 32 | lpc = parseLivepeerConfig() 33 | 34 | err = ff.Parse(flag.CommandLine, []string{ 35 | "-dataDir", "/some/data/dir", 36 | }) 37 | require.NoError(t, err) 38 | require.NotNil(t, lpc.Datadir) 39 | require.Equal(t, "/some/data/dir", *lpc.Datadir) 40 | } 41 | -------------------------------------------------------------------------------- /cmd/livepeer/starter/kafka.go: -------------------------------------------------------------------------------- 1 | package starter 2 | 3 | import ( 4 | "github.com/golang/glog" 5 | lpmon "github.com/livepeer/go-livepeer/monitor" 6 | ) 7 | 8 | func startKafkaProducer(cfg LivepeerConfig) error { 9 | if *cfg.KafkaBootstrapServers == "" || *cfg.KafkaUsername == "" || *cfg.KafkaPassword == "" || *cfg.KafkaGatewayTopic == "" { 10 | glog.Warning("not starting Kafka producer as producer config values aren't present") 11 | return nil 12 | } 13 | 14 | var gatewayHost = "" 15 | if cfg.GatewayHost != nil { 16 | gatewayHost = *cfg.GatewayHost 17 | } 18 | 19 | return lpmon.InitKafkaProducer( 20 | *cfg.KafkaBootstrapServers, 21 | *cfg.KafkaUsername, 22 | *cfg.KafkaPassword, 23 | *cfg.KafkaGatewayTopic, 24 | gatewayHost, 25 | ) 26 | } 27 | -------------------------------------------------------------------------------- /cmd/livepeer/starter/test_fixtures/perf_stats.json: -------------------------------------------------------------------------------- 1 | { 2 | "0x001ffe939761eea3f37dd2223bd08401a3848bf3": { 3 | "FRA": { 4 | "success_rate": 0, 5 | "round_trip_score": 0, 6 | "score": 0 7 | }, 8 | "LAX": { 9 | "success_rate": 0.3333333333333333, 10 | "round_trip_score": 0.978674309814987, 11 | "score": 0.326224769938329 12 | }, 13 | "LON": { 14 | "success_rate": 0.3333333333333333, 15 | "round_trip_score": 0.9999999981139247, 16 | "score": 0.33333333270464155 17 | }, 18 | "MDW": { 19 | "success_rate": 1, 20 | "round_trip_score": 0.8356601580708897, 21 | "score": 0.8356601580708897 22 | }, 23 | "NYC": { 24 | "success_rate": 0.6666666666666666, 25 | "round_trip_score": 0.9564037252220472, 26 | "score": 0.6376024834813647 27 | }, 28 | "PRG": { 29 | "success_rate": 0.6666666666666666, 30 | "round_trip_score": 0.9988698987407547, 31 | "score": 0.6659132658271698 32 | }, 33 | "SAO": { 34 | "success_rate": 0.3333333333333333, 35 | "round_trip_score": 0.8955986338422629, 36 | "score": 0.29853287794742095 37 | }, 38 | "SIN": { 39 | "success_rate": 1, 40 | "round_trip_score": 0.9969482179442755, 41 | "score": 0.9969482179442755 42 | } 43 | }, 44 | "0x00803b76dc924ceabf4380a6f9edc2ddd3c90f38": { 45 | "FRA": { 46 | "success_rate": 1, 47 | "round_trip_score": 0.6646347113088987, 48 | "score": 0.6646347113088987 49 | }, 50 | "LAX": { 51 | "success_rate": 0.8222222222222223, 52 | "round_trip_score": 0.381062716451423, 53 | "score": 0.3133182335267256 54 | }, 55 | "LON": { 56 | "success_rate": 1, 57 | "round_trip_score": 0.7694480079804097, 58 | "score": 0.7694480079804097 59 | }, 60 | "MDW": { 61 | "success_rate": 0.6222222222222222, 62 | "round_trip_score": 0.36531156012968535, 63 | "score": 0.22730497074735978 64 | }, 65 | "NYC": { 66 | "success_rate": 1, 67 | "round_trip_score": 0.543865046753563, 68 | "score": 0.543865046753563 69 | }, 70 | "PRG": { 71 | "success_rate": 1, 72 | "round_trip_score": 0.6681529487891555, 73 | "score": 0.6681529487891555 74 | }, 75 | "SAO": { 76 | "success_rate": 0.6888888888888888, 77 | "round_trip_score": 0.33652629465036343, 78 | "score": 0.23182922520358365 79 | }, 80 | "SIN": { 81 | "success_rate": 0.6, 82 | "round_trip_score": 0.3958106005746348, 83 | "score": 0.23748636034478088 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /cmd/livepeer_bench/transcodingOptions-netint.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "240p0", 4 | "fps": 0, 5 | "bitrate": 250000, 6 | "width": 426, 7 | "height": 240, 8 | "gop": "1", 9 | "encoder": "HEVC" 10 | }, 11 | { 12 | "name": "360p0", 13 | "fps": 0, 14 | "bitrate": 800000, 15 | "width": 640, 16 | "height": 360, 17 | "gop": "1", 18 | "encoder": "HEVC" 19 | }, 20 | { 21 | "name": "480p0", 22 | "fps": 0, 23 | "bitrate": 1600000, 24 | "width": 854, 25 | "height": 480, 26 | "gop": "1", 27 | "encoder": "HEVC" 28 | }, 29 | { 30 | "name": "720p0", 31 | "fps": 0, 32 | "bitrate": 3000000, 33 | "width": 1280, 34 | "height": 720, 35 | "gop": "1", 36 | "encoder": "HEVC" 37 | } 38 | ] 39 | -------------------------------------------------------------------------------- /cmd/livepeer_bench/transcodingOptions.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "240p0", 4 | "fps": 0, 5 | "bitrate": 250000, 6 | "width": 426, 7 | "height": 240, 8 | "profile": "h264constrainedhigh", 9 | "gop": "1" 10 | }, 11 | { 12 | "name": "360p0", 13 | "fps": 0, 14 | "bitrate": 800000, 15 | "width": 640, 16 | "height": 360, 17 | "profile": "h264constrainedhigh", 18 | "gop": "1" 19 | }, 20 | { 21 | "name": "480p0", 22 | "fps": 0, 23 | "bitrate": 1600000, 24 | "width": 854, 25 | "height": 480, 26 | "profile": "h264constrainedhigh", 27 | "gop": "1" 28 | }, 29 | { 30 | "name": "720p0", 31 | "fps": 0, 32 | "bitrate": 3000000, 33 | "width": 1280, 34 | "height": 720, 35 | "profile": "h264constrainedhigh", 36 | "gop": "1" 37 | } 38 | ] 39 | -------------------------------------------------------------------------------- /cmd/livepeer_cli/wizard_deposit.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "math/big" 6 | 7 | "github.com/livepeer/go-livepeer/eth" 8 | ) 9 | 10 | func str2eth(v string) string { 11 | i, ok := big.NewInt(0).SetString(v, 10) 12 | if !ok { 13 | fmt.Printf("Could not convert %v to bigint", v) 14 | return "" 15 | } 16 | return eth.FormatUnits(i, "ETH") 17 | } 18 | -------------------------------------------------------------------------------- /cmd/livepeer_cli/wizard_eth.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | ) 7 | 8 | func (w *wizard) setMaxGasPrice() { 9 | fmt.Printf("Current maximum gas price: %v\n", w.maxGasPrice()) 10 | amount := w.readBigInt("Enter new maximum gas price in Wei (enter \"0\" for no maximum gas price)") 11 | 12 | val := url.Values{ 13 | "amount": {fmt.Sprintf("%v", amount.String())}, 14 | } 15 | 16 | httpPostWithParams(fmt.Sprintf("http://%v:%v/setMaxGasPrice", w.host, w.httpPort), val) 17 | } 18 | 19 | func (w *wizard) setMinGasPrice() { 20 | fmt.Printf("Current minimum gas price: %v\n", w.minGasPrice()) 21 | minGasPrice := w.readBigInt("Enter new minimum gas price in Wei") 22 | 23 | val := url.Values{ 24 | "minGasPrice": {fmt.Sprintf("%v", minGasPrice.String())}, 25 | } 26 | 27 | httpPostWithParams(fmt.Sprintf("http://%v:%v/setMinGasPrice", w.host, w.httpPort), val) 28 | } 29 | 30 | func (w *wizard) signMessage() { 31 | fmt.Printf("Enter or paste the message to sign: \n") 32 | msg := w.readMultilineString() 33 | val := url.Values{ 34 | "message": {msg}, 35 | } 36 | result, ok := httpPostWithParams(fmt.Sprintf("http://%v:%v/signMessage", w.host, w.httpPort), val) 37 | if !ok { 38 | fmt.Printf("Error signing message: %v\n", result) 39 | return 40 | } 41 | fmt.Println(fmt.Sprintf("\n\nSignature:\n0x%x", result)) 42 | } 43 | 44 | func (w *wizard) signTypedData() { 45 | fmt.Printf("Enter or paste the typed data to sign: \n") 46 | msg := w.readMultilineString() 47 | val := url.Values{ 48 | "message": {msg}, 49 | } 50 | headers := map[string]string{ 51 | "SigFormat": "data/typed", 52 | } 53 | result, ok := httpPostWithParamsHeaders(fmt.Sprintf("http://%v:%v/signMessage", w.host, w.httpPort), val, headers) 54 | if !ok { 55 | fmt.Printf("Error signing typed data: %v\n", result) 56 | return 57 | } 58 | fmt.Println(fmt.Sprintf("\n\nSignature:\n0x%x", result)) 59 | } 60 | -------------------------------------------------------------------------------- /cmd/livepeer_cli/wizard_rounds.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "math/big" 7 | "net/http" 8 | ) 9 | 10 | func (w *wizard) currentRound() (*big.Int, error) { 11 | resp, err := http.Get(fmt.Sprintf("http://%v:%v/currentRound", w.host, w.httpPort)) 12 | if err != nil { 13 | return nil, err 14 | } 15 | 16 | defer resp.Body.Close() 17 | 18 | if resp.StatusCode != http.StatusOK { 19 | return nil, fmt.Errorf("http response status %d", resp.StatusCode) 20 | } 21 | 22 | body, err := ioutil.ReadAll(resp.Body) 23 | if err != nil { 24 | return nil, err 25 | } 26 | cr, ok := new(big.Int).SetString(string(body), 10) 27 | if !ok { 28 | return nil, fmt.Errorf("could not parse current round from response") 29 | } 30 | 31 | return cr, nil 32 | } 33 | 34 | func (w *wizard) initializeRound() { 35 | httpPost(fmt.Sprintf("http://%v:%v/initializeRound", w.host, w.httpPort)) 36 | } 37 | -------------------------------------------------------------------------------- /cmd/livepeer_cli/wizard_stream.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | "os/exec" 8 | "time" 9 | 10 | "github.com/golang/glog" 11 | ) 12 | 13 | func (w *wizard) stream() { 14 | if w.httpPort == "" { 15 | fmt.Println("Need to specify http port") 16 | return 17 | } 18 | fmt.Println("Enter StreamID:") 19 | streamID := w.read() 20 | 21 | //Fetch local stream playlist - this will request from the network so we know the stream is here 22 | url := fmt.Sprintf("http://localhost:%v/stream/%v.m3u8", w.httpPort, streamID) 23 | fmt.Printf("Getting stream from: %v\n", url) 24 | res, err := http.Get(url) 25 | if err != nil || res.StatusCode != 200 { 26 | fmt.Printf("err: %v, status:%v", err, res.StatusCode) 27 | return 28 | } 29 | _, err = ioutil.ReadAll(res.Body) 30 | res.Body.Close() 31 | if err != nil { 32 | fmt.Println(err) 33 | return 34 | } 35 | 36 | fmt.Println("Here...") 37 | cmd := exec.Command("ffplay", "-timeout", "180000000", url) //timeout in 3 mins 38 | err = cmd.Start() 39 | if err != nil { 40 | glog.Infof("Couldn't start the stream. Make sure a local Livepeer node is running on port %v", w.httpPort) 41 | return 42 | } 43 | 44 | go func() { 45 | err = cmd.Wait() 46 | if err != nil { 47 | glog.Infof("Couldn't start the stream. Make sure a local Livepeer node is running on port %v", w.httpPort) 48 | fmt.Printf("Type `q` to stop streaming\n") 49 | return 50 | } 51 | }() 52 | 53 | fmt.Printf("Type `q` to stop streaming\n") 54 | end := w.read() 55 | if end == "q" { 56 | fmt.Println("Quitting broadcast...") 57 | cmd.Process.Kill() 58 | time.Sleep(time.Second) 59 | return 60 | } 61 | 62 | return 63 | } 64 | -------------------------------------------------------------------------------- /cmd/livepeer_cli/wizard_ticketbroker_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "math/big" 5 | "testing" 6 | 7 | "github.com/livepeer/go-livepeer/pm" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func createSender(deposit *big.Int, reserve *big.Int, withdrawRound *big.Int) (sender pm.SenderInfo) { 12 | sender.Deposit = deposit 13 | sender.WithdrawRound = withdrawRound 14 | sender.Reserve = &pm.ReserveInfo{ 15 | FundsRemaining: reserve, 16 | ClaimedInCurrentRound: big.NewInt(0), 17 | } 18 | 19 | return 20 | } 21 | 22 | func TestSenderStatus(t *testing.T) { 23 | assert := assert.New(t) 24 | 25 | // Test Empty 26 | s := createSender(big.NewInt(0), big.NewInt(0), big.NewInt(0)) 27 | ss := senderStatus(s, big.NewInt(0)) 28 | assert.Equal(Empty, ss) 29 | 30 | // Test Empty, but WithdrawRound > 0 31 | s = createSender(big.NewInt(0), big.NewInt(0), big.NewInt(5)) 32 | ss = senderStatus(s, big.NewInt(0)) 33 | assert.Equal(Empty, ss) 34 | 35 | // Test Unlocked when WithdrawRound = currentRound 36 | s = createSender(big.NewInt(7), big.NewInt(0), big.NewInt(5)) 37 | ss = senderStatus(s, big.NewInt(5)) 38 | assert.Equal(Unlocked, ss) 39 | 40 | // Test Unlocked when WithdrawRound < currentRound 41 | s = createSender(big.NewInt(7), big.NewInt(0), big.NewInt(5)) 42 | ss = senderStatus(s, big.NewInt(6)) 43 | assert.Equal(Unlocked, ss) 44 | 45 | // Test Unlocking 46 | s = createSender(big.NewInt(7), big.NewInt(0), big.NewInt(5)) 47 | ss = senderStatus(s, big.NewInt(3)) 48 | assert.Equal(Unlocking, ss) 49 | 50 | // Test Locked 51 | s = createSender(big.NewInt(7), big.NewInt(0), big.NewInt(0)) 52 | ss = senderStatus(s, big.NewInt(3)) 53 | assert.Equal(Locked, ss) 54 | } 55 | -------------------------------------------------------------------------------- /cmd/livepeer_cli/wizard_token.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "strings" 7 | ) 8 | 9 | func (w *wizard) transferTokens() { 10 | fmt.Printf("Current LPT balance: %v\n", w.getTokenBalance()) 11 | 12 | fmt.Printf("Enter recipient address (in hex i.e. 0xfoo) - ") 13 | to := w.readString() 14 | 15 | amount := w.readBigInt("Enter amount") 16 | 17 | val := url.Values{ 18 | "to": {fmt.Sprintf("%v", to)}, 19 | "amount": {fmt.Sprintf("%v", amount.String())}, 20 | } 21 | 22 | var input string 23 | userAccepted := false 24 | for !userAccepted { 25 | fmt.Printf("Are you sure you want to send %s LPTU to \"%s\"? (y/n) - ", val["amount"][0], val["to"][0]) 26 | 27 | input = w.readString() 28 | 29 | switch strings.ToLower(input) { 30 | case "y": 31 | userAccepted = true 32 | case "n": 33 | fmt.Printf("Transaction cancelled: Interrupted by user.\n") 34 | return 35 | default: 36 | fmt.Printf("Enter (y)es or (n)o \n") 37 | } 38 | } 39 | 40 | httpPostWithParams(fmt.Sprintf("http://%v:%v/transferTokens", w.host, w.httpPort), val) 41 | } 42 | 43 | func (w *wizard) requestTokens() { 44 | httpPost(fmt.Sprintf("http://%v:%v/requestTokens", w.host, w.httpPort)) 45 | } 46 | -------------------------------------------------------------------------------- /cmd/livepeer_router/livepeer_router.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "net/url" 6 | "os" 7 | "os/signal" 8 | "os/user" 9 | "path/filepath" 10 | "strings" 11 | 12 | "github.com/golang/glog" 13 | "github.com/livepeer/go-livepeer/server" 14 | ) 15 | 16 | func defaultAddr(addr, defaultHost, defaultPort string) string { 17 | if addr == "" { 18 | return defaultHost + ":" + defaultPort 19 | } 20 | if addr[0] == ':' { 21 | return defaultHost + addr 22 | } 23 | // not IPv6 safe 24 | if !strings.Contains(addr, ":") { 25 | return addr + ":" + defaultPort 26 | } 27 | return addr 28 | } 29 | 30 | func main() { 31 | flag.Set("logtostderr", "true") 32 | flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) 33 | 34 | datadir := flag.String("datadir", "", "Directory that data is stored in") 35 | httpAddr := flag.String("httpAddr", "", "Address (IP:port) to bind to for HTTP") 36 | serviceAddr := flag.String("serviceAddr", "", "Publicly accessible URI (IP:port or hostname) to receive requests at") 37 | orchAddr := flag.String("orchAddr", "", "Comma delimited list of orchestrator URIs (IP:port or hostname) to use") 38 | 39 | flag.Parse() 40 | 41 | usr, err := user.Current() 42 | if err != nil { 43 | glog.Exitf("Cannot find current user: %v", err) 44 | } 45 | 46 | if *datadir == "" { 47 | homedir := os.Getenv("HOME") 48 | if homedir == "" { 49 | homedir = usr.HomeDir 50 | } 51 | *datadir = filepath.Join(homedir, ".lpRouterData") 52 | } 53 | 54 | if _, err := os.Stat(*datadir); os.IsNotExist(err) { 55 | glog.Infof("Creating datadir: %v", *datadir) 56 | if err = os.MkdirAll(*datadir, 0755); err != nil { 57 | glog.Exitf("Error creating datadir: %v", err) 58 | } 59 | } 60 | 61 | if *serviceAddr == "" { 62 | glog.Exit("Missing -serviceAddr") 63 | } 64 | 65 | serviceURI, err := url.ParseRequestURI("https://" + *serviceAddr) 66 | if err != nil { 67 | glog.Exitf("Could not parse -serviceAddr: %v", err) 68 | } 69 | 70 | *httpAddr = defaultAddr(*httpAddr, "0.0.0.0", serviceURI.Port()) 71 | uri, err := url.ParseRequestURI("https://" + *httpAddr) 72 | if err != nil { 73 | glog.Exitf("Could not parse -httpAddr: %v", err) 74 | } 75 | 76 | var uris []*url.URL 77 | if len(*orchAddr) > 0 { 78 | for _, addr := range strings.Split(*orchAddr, ",") { 79 | addr = strings.TrimSpace(addr) 80 | if !strings.HasPrefix(addr, "http") { 81 | addr = "https://" + addr 82 | } 83 | uri, err := url.ParseRequestURI(addr) 84 | if err != nil { 85 | glog.Exitf("Could not parse orchestrator URI: %v", err) 86 | } 87 | uris = append(uris, uri) 88 | } 89 | } 90 | 91 | errCh := make(chan error) 92 | srv := server.NewRouter(uris) 93 | go func() { 94 | errCh <- srv.Start(uri, serviceURI, *datadir) 95 | }() 96 | 97 | c := make(chan os.Signal) 98 | signal.Notify(c, os.Interrupt) 99 | select { 100 | case <-c: 101 | glog.Infof("Shutting down router server") 102 | srv.Stop() 103 | case err := <-errCh: 104 | if err != nil { 105 | glog.Errorf("Router server error: %v", err) 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /cmd/scripts/linkify_changelog.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | "regexp" 9 | "strings" 10 | ) 11 | 12 | // Based on https://github.com/tendermint/tendermint/blob/master/scripts/linkify_changelog.py 13 | // This script goes through the provided file, and replaces any " \#", 14 | // with the valid markdown formatted link to it. e.g. 15 | // " [\#number](https://github.com/livepeer/go-livepeer/pull/)" 16 | // Note that if the number is for an issue, github will auto-redirect you when you click the link. 17 | // It is safe to run the script multiple times in succession. 18 | // 19 | // Example usage $ go run cmd/scripts/linkify_changelog.go CHANGELOG_PENDING.md 20 | func main() { 21 | if len(os.Args) < 2 { 22 | log.Fatal("Expected filename") 23 | } 24 | 25 | fname := os.Args[1] 26 | 27 | f, err := os.Open(fname) 28 | if err != nil { 29 | log.Fatal(err) 30 | } 31 | defer f.Close() 32 | 33 | re, err := regexp.Compile(`\\#([0-9]*)`) 34 | if err != nil { 35 | log.Fatal(err) 36 | } 37 | 38 | var lines []string 39 | scanner := bufio.NewScanner(f) 40 | for scanner.Scan() { 41 | line := re.ReplaceAllString(scanner.Text(), `[#$1](https://github.com/livepeer/go-livepeer/pull/$1)`) 42 | lines = append(lines, line) 43 | } 44 | 45 | if err := scanner.Err(); err != nil { 46 | log.Fatal(err) 47 | } 48 | 49 | if err := ioutil.WriteFile(fname, []byte(strings.Join(lines, "\n")), 0644); err != nil { 50 | log.Fatal(err) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /cmd/simple_auth_server/simple_auth_server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | "net/url" 9 | "path" 10 | "regexp" 11 | "strings" 12 | 13 | "github.com/livepeer/go-livepeer/core" 14 | ) 15 | 16 | type authWebhookReq struct { 17 | Url string `json:"url"` 18 | } 19 | 20 | type profile struct { 21 | Name string `json:"name"` 22 | Width int `json:"width"` 23 | Height int `json:"height"` 24 | Bitrate int `json:"bitrate"` 25 | FPS uint `json:"fps"` 26 | FPSDen uint `json:"fpsDen"` 27 | Profile string `json:"profile"` 28 | GOP string `json:"gop"` 29 | } 30 | 31 | type authWebhookResponse struct { 32 | ManifestID string `json:"manifestID"` 33 | Profiles []profile `json:"profiles"` 34 | } 35 | 36 | func main() { 37 | http.HandleFunc("/auth", func(w http.ResponseWriter, r *http.Request) { 38 | decoder := json.NewDecoder(r.Body) 39 | var req authWebhookReq 40 | err := decoder.Decode(&req) 41 | if err != nil { 42 | fmt.Printf("Error parsing URL: %v\n", err) 43 | w.WriteHeader(http.StatusForbidden) 44 | return 45 | } 46 | 47 | var mid core.ManifestID 48 | u, err := url.Parse(req.Url) 49 | mid = parseStreamID(u.String()).ManifestID 50 | 51 | if mid == "" { 52 | mid = core.RandomManifestID() 53 | fmt.Printf("Generated random manifestID: %v\n", mid) 54 | } else if mid == "fizz" { 55 | mid = "buzz" 56 | fmt.Printf("Detected \"fizz\" as manifestID. Crazy! Renaming to \"buzz\".\n") 57 | } 58 | fmt.Printf("Stream started with manifestID: %v\n", mid) 59 | 60 | resp := authWebhookResponse{ 61 | ManifestID: string(mid), 62 | Profiles: []profile{{ 63 | Name: "240p", 64 | Width: 426, 65 | Height: 240, 66 | Bitrate: 250000, 67 | FPS: 0, 68 | }}, 69 | } 70 | byteSlice, _ := json.Marshal(resp) 71 | w.Write(byteSlice) 72 | }) 73 | 74 | fmt.Println("Listening on localhost:8000/auth\nTry something crazy - stream with \"fizz\" as the manifestID.") 75 | err := http.ListenAndServe(":8000", nil) // set listen port 76 | if err != nil { 77 | log.Fatal("ListenAndServe: ", err) 78 | } 79 | } 80 | 81 | var StreamPrefix = regexp.MustCompile(`.*[ /](stream/)?`) 82 | 83 | func cleanStreamPrefix(reqPath string) string { 84 | return StreamPrefix.ReplaceAllString(reqPath, "") 85 | } 86 | 87 | func parseStreamID(reqPath string) core.StreamID { 88 | // remove extension and create streamid 89 | p := strings.TrimSuffix(reqPath, path.Ext(reqPath)) 90 | return core.SplitStreamIDString(cleanStreamPrefix(p)) 91 | } 92 | -------------------------------------------------------------------------------- /common/log.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | const SHORT = 4 4 | const DEBUG = 5 5 | const VERBOSE = 6 6 | -------------------------------------------------------------------------------- /common/readfromfile.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | ) 8 | 9 | // ReadFromFile attempts to read a file at the supplied location. 10 | // If it fails, then the original supplied string will be returned to the caller. 11 | // A valid string will always be returned, regardless of whether an error occurred. 12 | func ReadFromFile(s string) (string, error) { 13 | info, err := os.Stat(s) 14 | // Return string as-is if the Stat call returned any error 15 | if err != nil { 16 | return s, err 17 | } 18 | if info.IsDir() { 19 | // If the supplied string is a directory, return it along with an appropriate error. 20 | return s, fmt.Errorf("supplied path is a directory") 21 | } 22 | bytes, err := os.ReadFile(s) 23 | if err != nil { 24 | return s, err 25 | } 26 | txt := strings.TrimSpace(string(bytes)) 27 | 28 | if len(txt) <= 0 { 29 | return s, fmt.Errorf("supplied file is empty") 30 | } 31 | 32 | return txt, nil 33 | } 34 | -------------------------------------------------------------------------------- /common/readfromfile_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestReadFromFileNoFileExists(t *testing.T) { 13 | assert := assert.New(t) 14 | 15 | input := "/tmp/../../../nothing" 16 | expectedOutput := "/tmp/../../../nothing" 17 | 18 | // ReadFromFile should return the string it was supplied 19 | output, err := ReadFromFile(input) 20 | 21 | assert.NotNil(err) 22 | // ReadFromFile should return the originally supplied string 23 | assert.Equal(expectedOutput, output) 24 | } 25 | 26 | func TestReadFromFileDirectoryExists(t *testing.T) { 27 | assert := assert.New(t) 28 | 29 | input := "/tmp" 30 | expectedOutput := "/tmp" 31 | 32 | // ReadFromFile should return the string it was supplied 33 | output, err := ReadFromFile(input) 34 | 35 | assert.NotNil(err) 36 | // ReadFromFile should return the originally supplied string 37 | assert.Equal(expectedOutput, output) 38 | } 39 | 40 | func TestReadFromFileEmptyFileExists(t *testing.T) { 41 | assert := assert.New(t) 42 | require := require.New(t) 43 | 44 | tmpFile := "TestReadFromFileEmptyFileExists.txt" 45 | expectedOutput := "TestReadFromFileEmptyFileExists.txt" 46 | 47 | emptyFile, err := os.Create(tmpFile) 48 | emptyFile.Close() 49 | require.Nil(err) 50 | 51 | defer func() { 52 | err := os.Remove(tmpFile) 53 | require.Nil(err) 54 | }() 55 | 56 | // ReadFromFile should return an error 57 | output, err := ReadFromFile(tmpFile) 58 | 59 | assert.NotNil(err) 60 | // ReadFromFile should return the originally supplied string 61 | assert.Equal(expectedOutput, output) 62 | } 63 | 64 | func TestReadFromFile_FileExistsOneLine(t *testing.T) { 65 | assert := assert.New(t) 66 | require := require.New(t) 67 | 68 | input := `something` 69 | expectedOutput := "something" 70 | 71 | tmpFile := "TestReadFromFile_FileExistsOneLine.txt" 72 | 73 | file, err := os.Create(tmpFile) 74 | require.Nil(err) 75 | _, err = file.WriteString(fmt.Sprintf("%s\n", input)) 76 | file.Close() 77 | require.Nil(err) 78 | 79 | defer func() { 80 | err := os.Remove(tmpFile) 81 | require.Nil(err) 82 | }() 83 | 84 | output, err := ReadFromFile(tmpFile) 85 | 86 | assert.Nil(err) 87 | // ReadFromFile should the first line of the text file 88 | assert.Equal(expectedOutput, output) 89 | } 90 | 91 | // Check that we can reliably read a single line value from the file, even when it ends in a newline / whitespace 92 | func TestReadFromFile_FileExistsEndsInNewline(t *testing.T) { 93 | input := 94 | `something 95 | ` 96 | f, err := os.CreateTemp(os.TempDir(), "TestReadFromFile_FileExistsEndsInNewline*") 97 | require.NoError(t, err) 98 | defer os.Remove(f.Name()) 99 | 100 | _, err = f.WriteString(input) 101 | require.NoError(t, err) 102 | 103 | output, err := ReadFromFile(f.Name()) 104 | require.NoError(t, err) 105 | require.Equal(t, "something", output) 106 | } 107 | 108 | func TestReadFromFile_FileExistsMultiline(t *testing.T) { 109 | require := require.New(t) 110 | 111 | input := 112 | `something 113 | somethingelse` 114 | 115 | tmpFile := "TestReadFromFile_FileExistsMultiline.txt" 116 | 117 | file, err := os.Create(tmpFile) 118 | require.Nil(err) 119 | _, err = file.WriteString(fmt.Sprintf("%s\n", input)) 120 | file.Close() 121 | require.Nil(err) 122 | 123 | defer func() { 124 | err := os.Remove(tmpFile) 125 | require.Nil(err) 126 | }() 127 | 128 | output, err := ReadFromFile(tmpFile) 129 | 130 | require.NoError(err) 131 | // ReadFromFile should the first line of the text file 132 | require.Equal(input, output) 133 | } 134 | -------------------------------------------------------------------------------- /common/videoprofile_ids.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "encoding/hex" 5 | ) 6 | 7 | const VideoProfileIDSize = 8 8 | const VideoProfileIDBytes = 4 9 | 10 | type VideoProfileByteMap map[[VideoProfileIDBytes]byte]string 11 | 12 | var VideoProfileNameLookup = map[string]string{ 13 | "a7ac137a": "P720p60fps16x9", 14 | "49d54ea9": "P720p30fps16x9", 15 | "e4a64019": "P720p25fps16x9", 16 | "79332fe7": "P720p30fps4x3", 17 | "5ecf4b52": "P576p30fps16x9", 18 | "8b1843d6": "P576p25fps16x9", 19 | "93c717e7": "P360p30fps16x9", 20 | "7cd40fc7": "P360p25fps16x9", 21 | "b60382a0": "P360p30fps4x3", 22 | "c0a6517a": "P240p30fps16x9", 23 | "1301a7d0": "P240p25fps16x9", 24 | "d435c53a": "P240p30fps4x3", 25 | "fca40bf9": "P144p30fps16x9", 26 | "03f01d1f": "P144p25fps16x9", 27 | } 28 | 29 | func makeVideoProfileByteMap() VideoProfileByteMap { 30 | var ret = make(VideoProfileByteMap, 0) 31 | for k, v := range VideoProfileNameLookup { 32 | b, err := hex.DecodeString(k) 33 | if err != nil || len(b) != VideoProfileIDBytes { 34 | continue 35 | } 36 | var key [VideoProfileIDBytes]byte 37 | copy(key[:], b) 38 | ret[key] = v 39 | } 40 | return ret 41 | } 42 | 43 | var VideoProfileByteLookup = makeVideoProfileByteMap() 44 | -------------------------------------------------------------------------------- /config.toml: -------------------------------------------------------------------------------- 1 | ignoreGeneratedHeader = false 2 | severity = "warning" 3 | confidence = 0.8 4 | errorCode = 1 5 | warningCode = 0 6 | 7 | [rule.exported] 8 | Disabled = true 9 | 10 | [rule.package-comments] 11 | Disabled = true 12 | 13 | [rule.blank-imports] 14 | [rule.context-as-argument] 15 | [rule.context-keys-type] 16 | [rule.dot-imports] 17 | [rule.error-return] 18 | [rule.error-strings] 19 | [rule.error-naming] 20 | [rule.if-return] 21 | [rule.increment-decrement] 22 | [rule.var-naming] 23 | [rule.var-declaration] 24 | [rule.range] 25 | [rule.receiver-naming] 26 | [rule.time-naming] 27 | [rule.unexported-return] 28 | [rule.indent-error-flow] 29 | [rule.errorf] 30 | [rule.empty-block] 31 | [rule.superfluous-else] 32 | [rule.unused-parameter] 33 | [rule.unreachable-code] 34 | [rule.redefines-builtin-id] 35 | -------------------------------------------------------------------------------- /core/broadcaster.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | ethcommon "github.com/ethereum/go-ethereum/common" 5 | "github.com/ethereum/go-ethereum/crypto" 6 | ) 7 | 8 | // Broadcaster RPC interface implementation 9 | 10 | type broadcaster struct { 11 | node *LivepeerNode 12 | } 13 | 14 | func (bcast *broadcaster) Sign(msg []byte) ([]byte, error) { 15 | if bcast.node == nil || bcast.node.Eth == nil { 16 | return []byte{}, nil 17 | } 18 | return bcast.node.Eth.Sign(crypto.Keccak256(msg)) 19 | } 20 | func (bcast *broadcaster) Address() ethcommon.Address { 21 | if bcast.node == nil || bcast.node.Eth == nil { 22 | return ethcommon.Address{} 23 | } 24 | return bcast.node.Eth.Account().Address 25 | } 26 | func NewBroadcaster(node *LivepeerNode) *broadcaster { 27 | return &broadcaster{ 28 | node: node, 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /core/os.go: -------------------------------------------------------------------------------- 1 | /* 2 | Object store helper functions 3 | */ 4 | package core 5 | 6 | import ( 7 | "context" 8 | "crypto/tls" 9 | "fmt" 10 | "net/http" 11 | "time" 12 | 13 | "github.com/livepeer/go-livepeer/clog" 14 | "github.com/livepeer/go-livepeer/common" 15 | "github.com/livepeer/go-livepeer/net" 16 | "github.com/livepeer/go-tools/drivers" 17 | ) 18 | 19 | func DownloadData(ctx context.Context, uri string) ([]byte, error) { 20 | return downloadDataHTTP(ctx, uri) 21 | } 22 | 23 | var httpc = &http.Client{ 24 | Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}, 25 | Timeout: common.HTTPTimeout / 2, 26 | } 27 | 28 | func FromNetOsInfo(os *net.OSInfo) *drivers.OSInfo { 29 | if os == nil { 30 | return nil 31 | } 32 | return &drivers.OSInfo{ 33 | StorageType: drivers.OSInfo_StorageType(os.StorageType), 34 | S3Info: FromNetS3Info(os.S3Info), 35 | } 36 | } 37 | 38 | func FromNetS3Info(storage *net.S3OSInfo) *drivers.S3OSInfo { 39 | if storage == nil { 40 | return nil 41 | } 42 | return &drivers.S3OSInfo{ 43 | Host: storage.Host, 44 | Key: storage.Key, 45 | Policy: storage.Policy, 46 | Signature: storage.Signature, 47 | Credential: storage.Credential, 48 | XAmzDate: storage.XAmzDate, 49 | } 50 | } 51 | 52 | func ToNetOSInfo(os *drivers.OSInfo) *net.OSInfo { 53 | if os == nil { 54 | return nil 55 | } 56 | return &net.OSInfo{ 57 | StorageType: net.OSInfo_StorageType(os.StorageType), 58 | S3Info: ToNetS3Info(os.S3Info), 59 | } 60 | } 61 | 62 | func ToNetS3Info(storage *drivers.S3OSInfo) *net.S3OSInfo { 63 | if storage == nil { 64 | return nil 65 | } 66 | return &net.S3OSInfo{ 67 | Host: storage.Host, 68 | Key: storage.Key, 69 | Policy: storage.Policy, 70 | Signature: storage.Signature, 71 | Credential: storage.Credential, 72 | XAmzDate: storage.XAmzDate, 73 | } 74 | } 75 | 76 | func downloadDataHTTP(ctx context.Context, uri string) ([]byte, error) { 77 | clog.V(common.VERBOSE).Infof(ctx, "Downloading uri=%s", uri) 78 | started := time.Now() 79 | resp, err := httpc.Get(uri) 80 | if err != nil { 81 | clog.Errorf(ctx, "Error getting HTTP uri=%s err=%q", uri, err) 82 | return nil, err 83 | } 84 | defer resp.Body.Close() 85 | if resp.StatusCode != 200 { 86 | clog.Errorf(ctx, "Non-200 response for status=%v uri=%s", resp.Status, uri) 87 | return nil, fmt.Errorf(resp.Status) 88 | } 89 | body, err := common.ReadAtMost(resp.Body, common.MaxSegSize) 90 | if err != nil { 91 | clog.Errorf(ctx, "Error reading body uri=%s err=%q", uri, err) 92 | return nil, err 93 | } 94 | took := time.Since(started) 95 | clog.V(common.VERBOSE).Infof(ctx, "Downloaded uri=%s dur=%s bytes=%d", uri, took, len(body)) 96 | return body, nil 97 | } 98 | -------------------------------------------------------------------------------- /core/test.phash: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/livepeer/go-livepeer/e098564304ae51e6def882b0318b7591e4b8b93a/core/test.phash -------------------------------------------------------------------------------- /core/test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/livepeer/go-livepeer/e098564304ae51e6def882b0318b7591e4b8b93a/core/test.ts -------------------------------------------------------------------------------- /core/test2.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/livepeer/go-livepeer/e098564304ae51e6def882b0318b7591e4b8b93a/core/test2.ts -------------------------------------------------------------------------------- /core/test_segment.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | FILE="$1" 4 | 5 | echo "package core" > $FILE 6 | echo "// Auto generated by $0 DO NOT EDIT" >> $FILE 7 | 8 | echo "var testSegment_H264 = []byte{" >> $FILE 9 | ffmpeg -f lavfi -i color=white:s=144x144 -vframes 5 -c:v libx264 -f mpegts - | \ 10 | ffmpeg -f mpegts -i pipe:0 -c:v copy \ 11 | -f mpegts -metadata title='!' -metadata service_provider='!' - | gzip -9 | \ 12 | xxd -i | awk 'NR > 1 { print prev } { prev=$0 } END { ORS=""; print }' >> $FILE 13 | echo "}" >> $FILE 14 | 15 | echo "var testSegment_HEVC = []byte{" >> $FILE 16 | ffmpeg -f lavfi -i color=white:s=144x144 -vframes 5 -c:v libx265 -f mpegts - | gzip -9 | xxd -i | awk 'NR > 1 { print prev } { prev=$0 } END { ORS=""; print }' >> $FILE 17 | echo "}" >> $FILE 18 | 19 | echo "var testSegment_VP8 = []byte{" >> $FILE 20 | ffmpeg -f lavfi -i color=white:s=144x144 -vframes 5 -c:v libvpx -f webm - | gzip -9 | xxd -i | awk 'NR > 1 { print prev } { prev=$0 } END { ORS=""; print }' >> $FILE 21 | echo "}" >> $FILE 22 | 23 | echo "var testSegment_VP9 = []byte{" >> $FILE 24 | ffmpeg -f lavfi -i color=white:s=144x144 -vframes 5 -c:v libvpx-vp9 -f webm - | gzip -9 | xxd -i | awk 'NR > 1 { print prev } { prev=$0 } END { ORS=""; print }' >> $FILE 25 | echo "}" >> $FILE 26 | 27 | echo "var testSegment_H264_444_8bit = []byte{" >> $FILE 28 | ffmpeg -f lavfi -i color=white:s=144x144 -vframes 5 -pix_fmt yuv444p -c:v libx264 -f mpegts - | \ 29 | ffmpeg -f mpegts -i pipe:0 -c:v copy \ 30 | -f mpegts -metadata title='!' -metadata service_provider='!' - | gzip -9 | \ 31 | xxd -i | awk 'NR > 1 { print prev } { prev=$0 } END { ORS=""; print }' >> $FILE 32 | echo "}" >> $FILE 33 | 34 | echo "var testSegment_H264_422_8bit = []byte{" >> $FILE 35 | ffmpeg -f lavfi -i color=white:s=144x144 -vframes 5 -pix_fmt yuv422p -c:v libx264 -f mpegts - | \ 36 | ffmpeg -f mpegts -i pipe:0 -c:v copy \ 37 | -f mpegts -metadata title='!' -metadata service_provider='!' - | gzip -9 | \ 38 | xxd -i | awk 'NR > 1 { print prev } { prev=$0 } END { ORS=""; print }' >> $FILE 39 | echo "}" >> $FILE 40 | 41 | echo "var testSegment_H264_444_10bit = []byte{" >> $FILE 42 | ffmpeg -f lavfi -i color=white:s=144x144 -vframes 5 -pix_fmt yuv444p10le -c:v libx264 -f mpegts - | \ 43 | ffmpeg -f mpegts -i pipe:0 -c:v copy \ 44 | -f mpegts -metadata title='!' -metadata service_provider='!' - | gzip -9 | \ 45 | xxd -i | awk 'NR > 1 { print prev } { prev=$0 } END { ORS=""; print }' >> $FILE 46 | echo "}" >> $FILE 47 | 48 | echo "var testSegment_H264_422_10bit = []byte{" >> $FILE 49 | ffmpeg -f lavfi -i color=white:s=144x144 -vframes 5 -pix_fmt yuv422p10le -c:v libx264 -f mpegts - | \ 50 | ffmpeg -f mpegts -i pipe:0 -c:v copy \ 51 | -f mpegts -metadata title='!' -metadata service_provider='!' - | gzip -9 | \ 52 | xxd -i | awk 'NR > 1 { print prev } { prev=$0 } END { ORS=""; print }' >> $FILE 53 | echo "}" >> $FILE 54 | 55 | echo "var testSegment_H264_420_10bit = []byte{" >> $FILE 56 | ffmpeg -f lavfi -i color=white:s=144x144 -vframes 5 -pix_fmt yuv420p10be -c:v libx264 -f mpegts - | \ 57 | ffmpeg -f mpegts -i pipe:0 -c:v copy \ 58 | -f mpegts -metadata title='!' -metadata service_provider='!' - | gzip -9 | \ 59 | xxd -i | awk 'NR > 1 { print prev } { prev=$0 } END { ORS=""; print }' >> $FILE 60 | echo "}" >> $FILE 61 | 62 | gofmt -w $FILE 63 | -------------------------------------------------------------------------------- /crypto/verify.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import ( 4 | "errors" 5 | "math/big" 6 | 7 | "github.com/ethereum/go-ethereum/accounts" 8 | ethcommon "github.com/ethereum/go-ethereum/common" 9 | "github.com/ethereum/go-ethereum/crypto" 10 | ) 11 | 12 | var ( 13 | secp256k1N, _ = new(big.Int).SetString("fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141", 16) 14 | secp256k1halfN = new(big.Int).Div(secp256k1N, big.NewInt(2)) 15 | ) 16 | 17 | // VerifySig verifies that a ETH ECDSA signature over a given message 18 | // is produced by a given ETH address 19 | func VerifySig(addr ethcommon.Address, msg, sig []byte) bool { 20 | recovered, err := ecrecover(msg, sig) 21 | if err != nil { 22 | return false 23 | } 24 | 25 | return recovered == addr 26 | } 27 | 28 | func ecrecover(msg, sig []byte) (ethcommon.Address, error) { 29 | if len(sig) != 65 { 30 | return ethcommon.Address{}, errors.New("invalid signature length") 31 | } 32 | 33 | s := new(big.Int).SetBytes(sig[32:64]) 34 | if s.Cmp(secp256k1halfN) > 0 { 35 | return ethcommon.Address{}, errors.New("signature s value too high") 36 | } 37 | 38 | v := sig[64] 39 | if v != byte(27) && v != byte(28) { 40 | return ethcommon.Address{}, errors.New("signature v value must be 27 or 28") 41 | } 42 | 43 | // crypto.SigToPub() expects signature v value = 0/1 44 | // Copy the signature and convert its value to 0/1 45 | ethSig := make([]byte, 65) 46 | copy(ethSig[:], sig[:]) 47 | ethSig[64] -= 27 48 | 49 | ethMsg := accounts.TextHash(msg) 50 | pubkey, err := crypto.SigToPub(ethMsg, ethSig) 51 | if err != nil { 52 | return ethcommon.Address{}, err 53 | } 54 | 55 | return crypto.PubkeyToAddress(*pubkey), nil 56 | } 57 | -------------------------------------------------------------------------------- /crypto/verify_test.go: -------------------------------------------------------------------------------- 1 | package crypto 2 | 3 | import ( 4 | "testing" 5 | 6 | ethcommon "github.com/ethereum/go-ethereum/common" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestEcrecover(t *testing.T) { 11 | assert := assert.New(t) 12 | 13 | addr := ethcommon.HexToAddress("3BadDb1eeE2105893136A3F96c8a963E9C6309d6") 14 | msg := ethcommon.FromHex("b7da355477356fc4c47fcabcf232dc77a6db9b07b7e48b76261cc55cc8fbabb3") 15 | sig := ethcommon.FromHex("206443228e8f784bc3a122de0d85eb3ebff82d6a79cca26c7eeb907099a6404f6dff57bc6828f28bd6cd073c89d94cf3364204679ed8365fa45b5ee6af19a9841c") 16 | 17 | _, err := ecrecover(msg, sig[:64]) 18 | assert.EqualError(err, "invalid signature length") 19 | 20 | sigHighS := ethcommon.FromHex("e742ff452d41413616a5bf43fe15dd88294e983d3d36206c2712f39083d638bde0a0fc89be718fbc1033e1d30d78be1c68081562ed2e97af876f286f3453231d1b") 21 | _, err = ecrecover(msg, sigHighS) 22 | assert.EqualError(err, "signature s value too high") 23 | 24 | sigInvalidV := make([]byte, 65) 25 | copy(sigInvalidV[:], sigInvalidV[:]) 26 | sigInvalidV[64] -= 27 27 | _, err = ecrecover(msg, sigInvalidV) 28 | assert.EqualError(err, "signature v value must be 27 or 28") 29 | 30 | // Check that the correct address is recovered 31 | recovered, err := ecrecover(msg, sig) 32 | assert.Nil(err) 33 | assert.Equal(addr, recovered) 34 | 35 | // Check that the wrong address is recovered for a different message 36 | recovered, err = ecrecover(ethcommon.FromHex("foo"), sig) 37 | assert.Nil(err) 38 | assert.NotEqual(addr, recovered) 39 | } 40 | 41 | func TestVerifySig(t *testing.T) { 42 | assert := assert.New(t) 43 | 44 | addr := ethcommon.HexToAddress("3BadDb1eeE2105893136A3F96c8a963E9C6309d6") 45 | sig := ethcommon.FromHex("206443228e8f784bc3a122de0d85eb3ebff82d6a79cca26c7eeb907099a6404f6dff57bc6828f28bd6cd073c89d94cf3364204679ed8365fa45b5ee6af19a9841c") 46 | msg := ethcommon.FromHex("b7da355477356fc4c47fcabcf232dc77a6db9b07b7e48b76261cc55cc8fbabb3") 47 | 48 | // Check that verification passes 49 | assert.True(VerifySig(addr, msg, sig)) 50 | // Check that verification fails for a different message 51 | assert.False(VerifySig(addr, ethcommon.FromHex("foo"), sig)) 52 | } 53 | -------------------------------------------------------------------------------- /discovery/stub.go: -------------------------------------------------------------------------------- 1 | package discovery 2 | 3 | import ( 4 | "math/big" 5 | "net/url" 6 | 7 | ethcommon "github.com/ethereum/go-ethereum/common" 8 | "github.com/livepeer/go-livepeer/common" 9 | "github.com/livepeer/go-livepeer/core" 10 | lpTypes "github.com/livepeer/go-livepeer/eth/types" 11 | "github.com/livepeer/go-livepeer/net" 12 | "github.com/livepeer/go-livepeer/pm" 13 | ) 14 | 15 | type stubOrchestratorPool struct { 16 | uris []*url.URL 17 | bcast common.Broadcaster 18 | } 19 | 20 | func stringsToURIs(addresses []string) []*url.URL { 21 | var uris []*url.URL 22 | 23 | for _, addr := range addresses { 24 | uri, err := url.ParseRequestURI(addr) 25 | if err == nil { 26 | uris = append(uris, uri) 27 | } 28 | } 29 | return uris 30 | } 31 | 32 | func StubOrchestratorPool(addresses []string) *stubOrchestratorPool { 33 | uris := stringsToURIs(addresses) 34 | node, _ := core.NewLivepeerNode(nil, "", nil) 35 | bcast := core.NewBroadcaster(node) 36 | 37 | return &stubOrchestratorPool{bcast: bcast, uris: uris} 38 | } 39 | 40 | func StubOrchestrators(addresses []string) []*lpTypes.Transcoder { 41 | var orchestrators []*lpTypes.Transcoder 42 | 43 | for _, addr := range addresses { 44 | address := ethcommon.BytesToAddress([]byte(addr)) 45 | transc := &lpTypes.Transcoder{ 46 | ServiceURI: addr, 47 | Address: address, 48 | ActivationRound: big.NewInt(0), 49 | DeactivationRound: big.NewInt(0), 50 | } 51 | orchestrators = append(orchestrators, transc) 52 | } 53 | 54 | return orchestrators 55 | } 56 | 57 | type stubTicketParamsValidator struct { 58 | err error 59 | } 60 | 61 | func (s *stubTicketParamsValidator) ValidateTicketParams(params *pm.TicketParams) error { return s.err } 62 | 63 | type stubRoundsManager struct { 64 | round *big.Int 65 | } 66 | 67 | func (s *stubRoundsManager) LastInitializedRound() *big.Int { return s.round } 68 | 69 | type orchTest struct { 70 | EthereumAddr string 71 | ServiceURI string 72 | PricePerPixel int64 73 | } 74 | 75 | func toOrchTest(addr, serviceURI string, pricePerPixel int64) orchTest { 76 | return orchTest{EthereumAddr: addr, ServiceURI: serviceURI, PricePerPixel: pricePerPixel} 77 | } 78 | 79 | type stubSuspender struct { 80 | list map[string]int 81 | } 82 | 83 | func newStubSuspender() *stubSuspender { 84 | return &stubSuspender{make(map[string]int)} 85 | } 86 | 87 | func (s *stubSuspender) Suspended(orch string) int { 88 | return s.list[orch] 89 | } 90 | 91 | var capCompatString = []uint64{uint64(0x1337)} 92 | 93 | type stubCapabilities struct { 94 | isLegacy bool 95 | } 96 | 97 | func newStubCapabilities() *stubCapabilities { 98 | return &stubCapabilities{isLegacy: true} 99 | } 100 | func (s *stubCapabilities) CompatibleWith(caps *net.Capabilities) bool { 101 | return len(caps.Bitstring) > 0 && caps.Bitstring[0] == capCompatString[0] 102 | } 103 | func (s *stubCapabilities) LegacyOnly() bool { 104 | return s.isLegacy 105 | } 106 | func (s *stubCapabilities) ToNetCapabilities() *net.Capabilities { 107 | return &net.Capabilities{Bitstring: capCompatString} 108 | } 109 | -------------------------------------------------------------------------------- /discovery/suspensionqueue.go: -------------------------------------------------------------------------------- 1 | package discovery 2 | 3 | import ( 4 | "container/heap" 5 | "github.com/livepeer/go-livepeer/common" 6 | 7 | "github.com/livepeer/go-livepeer/net" 8 | ) 9 | 10 | // A suspensionQueue implements heap.Interface and holds suspensions. 11 | type suspensionQueue []*suspension 12 | 13 | // A suspension is the item we manage in the priority queue. 14 | type suspension struct { 15 | orch *net.OrchestratorInfo 16 | od *common.OrchestratorDescriptor 17 | penalty int 18 | } 19 | 20 | func (sq suspensionQueue) Len() int { return len(sq) } 21 | 22 | func (sq suspensionQueue) Less(i, j int) bool { 23 | return sq[i].penalty < sq[j].penalty 24 | } 25 | 26 | func (sq suspensionQueue) Swap(i, j int) { 27 | sq[i], sq[j] = sq[j], sq[i] 28 | } 29 | 30 | func (sq *suspensionQueue) Push(x interface{}) { 31 | item := x.(*suspension) 32 | *sq = append(*sq, item) 33 | } 34 | 35 | func (sq *suspensionQueue) Pop() interface{} { 36 | old := *sq 37 | n := len(old) 38 | item := old[n-1] 39 | old[n-1] = nil // avoid memory leak 40 | *sq = old[0 : n-1] 41 | return item 42 | } 43 | 44 | func newSuspensionQueue() *suspensionQueue { 45 | sq := &suspensionQueue{} 46 | heap.Init(sq) 47 | return sq 48 | } 49 | -------------------------------------------------------------------------------- /discovery/suspensionqueue_test.go: -------------------------------------------------------------------------------- 1 | package discovery 2 | 3 | import ( 4 | "container/heap" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestSuspensionQueue(t *testing.T) { 11 | assert := assert.New(t) 12 | sq := newSuspensionQueue() 13 | susp0 := &suspension{penalty: 4} 14 | heap.Push(sq, susp0) 15 | assert.Equal(sq.Len(), 1) 16 | assert.Equal(heap.Pop(sq).(*suspension), susp0) 17 | assert.Equal(sq.Len(), 0) 18 | 19 | susp1 := &suspension{penalty: 6} 20 | heap.Push(sq, susp0) 21 | assert.Equal(sq.Len(), 1) 22 | heap.Push(sq, susp1) 23 | assert.Equal(sq.Len(), 2) 24 | assert.True(sq.Less(0, 1)) 25 | assert.Equal(heap.Pop(sq).(*suspension), susp0) 26 | assert.Equal(sq.Len(), 1) 27 | assert.Equal(heap.Pop(sq).(*suspension), susp1) 28 | assert.Equal(sq.Len(), 0) 29 | 30 | susp2 := &suspension{penalty: 2} 31 | heap.Push(sq, susp0) 32 | assert.Equal(sq.Len(), 1) 33 | heap.Push(sq, susp1) 34 | assert.Equal(sq.Len(), 2) 35 | heap.Push(sq, susp2) 36 | assert.Equal(sq.Len(), 3) 37 | assert.Equal(heap.Pop(sq).(*suspension), susp2) 38 | assert.Equal(sq.Len(), 2) 39 | assert.Equal(heap.Pop(sq).(*suspension), susp0) 40 | assert.Equal(sq.Len(), 1) 41 | assert.Equal(heap.Pop(sq).(*suspension), susp1) 42 | assert.Equal(sq.Len(), 0) 43 | } 44 | -------------------------------------------------------------------------------- /doc/assets/ai-runner-container-lifecycle.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/livepeer/go-livepeer/e098564304ae51e6def882b0318b7591e4b8b93a/doc/assets/ai-runner-container-lifecycle.jpg -------------------------------------------------------------------------------- /doc/assets/redeemer/eth-events.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:bd23ba54de0a4ef25dc7c10afe42e3179de10047d43c30facc21d85bfb2fa42f 3 | size 284626 4 | -------------------------------------------------------------------------------- /doc/assets/redeemer/eth-events.txt: -------------------------------------------------------------------------------- 1 | title Ticket Redemption Service: Ethereum Events 2 | 3 | participantgroup #lightgreen **Orchestrator** 4 | participant Recipient 5 | participant Redeemer Client (SenderMonitor) 6 | end 7 | participantgroup #lightblue **Redeemer** 8 | participant Redeemer gRPC server 9 | participant LocalSenderMonitor 10 | participant SenderManager 11 | end 12 | participant Ethereum Node 13 | participantgroup 14 | 15 | Recipient->Redeemer Client (SenderMonitor):MaxFloat(sender) 16 | group MaxFloat(sender) 17 | 18 | note over Redeemer Client (SenderMonitor): No Local Cache 19 | Redeemer Client (SenderMonitor)->>Redeemer gRPC server: MaxFloat(sender) 20 | Redeemer gRPC server->LocalSenderMonitor: MaxFloat(sender) 21 | Redeemer gRPC server<--LocalSenderMonitor: MaxFloat 22 | Redeemer Client (SenderMonitor)<<--Redeemer gRPC server: MaxFloatUpdate 23 | Redeemer Client (SenderMonitor)-#blue>>Redeemer gRPC server: MonitorMaxFloat(sender) 24 | Redeemer gRPC server-#red>LocalSenderMonitor: SubscribeMaxFloatChange(sender) 25 | end 26 | Recipient<--Redeemer Client (SenderMonitor): MaxFloat 27 | 28 | entryspacing 3 29 | 30 | note over Ethereum Node: FundReserve(sender) Event 31 | entryspacing 0.1 32 | Ethereum Node-->(3)SenderManager:FundReserve 33 | SenderManager--#purple>(3)LocalSenderMonitor:ReserveChange(sender) 34 | 35 | group MaxFloat update 36 | Redeemer gRPC server(3)<#red--LocalSenderMonitor: SubscribeMaxFloatChange(sender) 37 | entryspacing 2 38 | Redeemer gRPC server->LocalSenderMonitor: MaxFloat(sender) 39 | Redeemer gRPC server<--LocalSenderMonitor: MaxFloat 40 | entryspacing 0.1 41 | Redeemer Client (SenderMonitor)(3)<<#blue--Redeemer gRPC server:MaxFloatUpdate 42 | entryspacing 2 43 | end 44 | 45 | entryspacing 4 46 | note over Ethereum Node: NewRound event 47 | entryspacing 0.1 48 | SenderManager<--Ethereum Node:NewRound 49 | entryspacing 1.5 50 | note over SenderManager: Pool size change 51 | entryspacing 0.1 52 | SenderManager--#a16545>(3)LocalSenderMonitor:PoolSizeChange 53 | loop i < len(LocalSenderMonitor.remoteSenders) 54 | entryspacing 1.5 55 | group MaxFloat update 56 | Redeemer gRPC server(3)<#red--LocalSenderMonitor: SubscribeMaxFloatChange(sender) 57 | entryspacing 2 58 | Redeemer gRPC server->LocalSenderMonitor: MaxFloat(ticket.sender) 59 | Redeemer gRPC server<--LocalSenderMonitor: MaxFloat 60 | entryspacing 0.1 61 | Redeemer Client (SenderMonitor)(3)<<#blue--Redeemer gRPC server:MaxFloatUpdate 62 | entryspacing 2 63 | end 64 | end 65 | -------------------------------------------------------------------------------- /doc/assets/redeemer/ticketflow.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:a3796206cf25858e8df232f436bb2265eb5a36fefabc0f0b59ca25a04c8e17ee 3 | size 371817 4 | -------------------------------------------------------------------------------- /doc/ethereum.md: -------------------------------------------------------------------------------- 1 | # Ethereum 2 | 3 | ## Reward 4 | 5 | The node can run a reward service that will automatically call a smart contract function to mint LPT rewards each round that the node's on-chain registered address is in the active set. Note that at the moment, only the on-chain registered address can call the smart contract function to mint LPT rewards. 6 | 7 | If the node detects that its address is registered on-chain, it will automatically start the reward service. The reward service can also be explicitly disabled by starting the node with `-reward=false` and explicitly enabled by starting the node with `-reward`. 8 | 9 | ## Round Initialization 10 | 11 | The node can run a round initialization service that will automatically call a smart contract function to initialize the current round. 12 | 13 | The round initialization service is disabled by default and can be enabled by starting the node with `-initializeRound`. 14 | 15 | ## Gas Prices 16 | 17 | After the EIP-1559 upgrade on Ethereum, the node treats the gas price as priority fee + base fee. 18 | 19 | ### Max gas price 20 | 21 | The `maxGasPrice` parameter makes sure the transaction fee never exceeds the specified limit. 22 | - If the current network gas price is higher than `maxGasPrice`, the transaction is not sent 23 | - The transaction parameter `maxFeePerGas` is set to `maxGasPrice` 24 | - **Note: As of v0.5.24, this is not true, but another release will be published to resolve this** 25 | 26 | The following options can be used to get the max gas price: 27 | 28 | - `curl localhost:7935/maxGasPrice` 29 | - Run `livepeer_cli` and observe the max gas price in the node stats 30 | 31 | The following options can be used to set the max gas price to ``, a Wei denominated value: 32 | 33 | - Start the node with `-maxGasPrice ` 34 | - `curl localhost:7935/setMaxGasPrice?maxGasPrice=` 35 | - Run `livepeer_cli` and select the set max gas price option 36 | 37 | ### Min gas price 38 | 39 | The following options can be used to get the min gas price: 40 | 41 | - `curl localhost:7935/minGasPrice` 42 | - Run `livepeer_cli` and observe the min gas price in the node stats 43 | 44 | The following options can be used to set the min gas price to ``, a Wei denominated value: 45 | 46 | - Start the node with `-minGasPrice ` 47 | - `curl localhost:7935/setMinGasPrice?minGasPrice=` 48 | - Run `livepeer_cli` and select the set min gas price option 49 | 50 | ### Known edge-cases 51 | A known edge-case that affects the initialization of new rounds and the ticket redemption occurs when the L2 block-rate is significantly slower than the L1 block-rate. 52 | This may result in: 53 | - go-livepeer not attempting to initialize a new round based on the expected `roundLength` interval - which is based on L1 block values. 54 | - go-livepeer attempting to redeem a winning ticket after its on-chain validity period already expired due to the block-rate difference between L1 and L2, which may result in the [`ticket is expired` error](https://github.com/livepeer/protocol/blob/confluence/contracts/pm/mixins/MixinTicketProcessor.sol#L64). 55 | - go-livepeer attempting to redeem a winning ticket after its [off-chain validity period](https://github.com/livepeer/go-livepeer/blob/3230eb1ac29fd86f88f1e6f768ff6bfbeef95572/pm/recipient.go#L24) already expired due to the block-rate difference between L1 and L2, which may result in the [`TicketParams expired` error](https://github.com/livepeer/go-livepeer/blob/master/pm/recipient.go#L17). 56 | -------------------------------------------------------------------------------- /doc/go.md: -------------------------------------------------------------------------------- 1 | # Installing and Managing Go 2 | 3 | ## Go 4 | 5 | Follow the instructions at https://go.dev/doc/install to download and install Go. 6 | 7 | If you are developing on Apple Silicon (M1), you will need to: 8 | 9 | - Use Go >= 1.16 as arm64 support was [introduced in 1.16](https://go.dev/doc/go1.16) 10 | - Download the `*-darwin-arm64` binary instead of the `*-darwin-amd64` binary on the [downloads page](https://go.dev/dl/). 11 | 12 | ### Managing Go Versions 13 | 14 | There are a few ways to manage different Go versions: 15 | 16 | 1. [go install](https://go.dev/doc/manage-install) 17 | 2. [gvm](https://github.com/moovweb/gvm) 18 | 19 | Using `gvm` has the benefit of automatically aliasing `go` to whichever version of Go you are currently using as opposed to having to use a command like `go1.10.7`. 20 | 21 | **gvm: Installing arm64 binaries on >= macOS 11** 22 | 23 | Until https://github.com/moovweb/gvm/pull/380 is merged, gvm does not support installing arm64 binaries on >= macOS 11 (i.e. Big Sur). A workaround for this issue is to install gvm using the fork for the PR: 24 | 25 | ``` 26 | # Download installer script 27 | curl -s -S -L https://raw.githubusercontent.com/moovweb/gvm/master/binscripts/gvm-installer 28 | # Run installer script using feature branch from fork 29 | SRC_REPO=https://github.com/jeremy-ebler-vineti/gvm.git bash gvm_installer feature/support-big-sur 30 | ``` 31 | 32 | Then, you can run the following command which should download the arm64 binary if you are on an arm64 machine: 33 | 34 | ``` 35 | gvm install -B 36 | ``` 37 | -------------------------------------------------------------------------------- /doc/gpu.md: -------------------------------------------------------------------------------- 1 | # GPU Support 2 | 3 | Livepeer supports decoding and encoding on NVIDIA GPUs on Linux and Windows. 4 | GPU transcoding can be enabled by starting Livepeer in `-transcoder` mode with 5 | the `-nvidia ` flag. The `` is a comma-separated 6 | numerical list of GPU devices that you wish to use for transcoding. If you are 7 | unsure of your GPU device, use the `nvidia-smi` utility. For example, to select 8 | devices 0, 2 and 4: 9 | 10 | ``` 11 | ./livepeer -transcoder -nvidia 0,2,4 12 | ``` 13 | 14 | Alternatively, if you want to use all the available NVIDIA GPUs on your system, 15 | you can set the flag like: 16 | 17 | ``` 18 | ./livepeer -transcoder -nvidia all 19 | ``` 20 | 21 | ### Limitations 22 | 23 | Currently the following limitations are observed: 24 | 25 | * **Device validity** Ensure valid devices are selected when starting up the node. Currently there is no start-up check to ensure device validity. 26 | 27 | * **YUV 4:2:0 input format** The pixel format of the source video must be in YUV 4:2:0 format (planar or 28 | interleaved). Anything else will return an error. 29 | 30 | * **CUDA Availability** If running the Livepeer binary, the CUDA shared libraries are expected to be installed in `/usr/local/cuda`. If the CUDA location differs on your machine, run the node with `LD_LIBRARY_PATH=` environment variable. 31 | 32 | So far, Livepeer has been tested to work with the following driver versions: 33 | 34 | CUDA | Nvidia 35 | --|-- 36 | 10.0.130 | 37 | 10.1 | 418.39 , 430.50 38 | 10.2 | 440.33.01, 440.118.02 39 | 11.1,11.2 | 460.39 40 | 41 | Nvidia's 450.xx drivers can occasionally lead to stuck transcoding sessions. 42 | Refer to this [forum post](https://forum.livepeer.org/t/working-around-occasional-transcoding-issues-with-nvidia-driver-450/1219) on how to switch to a different driver version. 43 | 44 | All Nvidia chipsets from [the Maxwell series](https://developer.nvidia.com/maxwell-compute-architecture) and later, that have NVDEC/NVENC cores, should theoretically be supported by go-livepeer. 45 | 46 | * **Driver Limits** Retail GPU cards may impose a software limit on the number of concurrent transcode sessions allowed on the system in official drivers. 47 | 48 | * **Linux Only** We've only tested this on Linux. We haven't tried other platforms; if it works elsewhere, especially on Windows or OSX, let us know! 49 | 50 | ### Running Tests 51 | 52 | A number of GPU unit tests are included. These may help verify your GPU setup. 53 | To run these tests, the Livepeer source code must be obtained; see the 54 | [install documentation](install.md) for details on setting up a build 55 | environment. Then the Livepeer unit test suite can be run with the `NV_DEVICE` 56 | environment variable. For example, to run the unit tests on GPU 1: 57 | 58 | ``` 59 | NV_DEVICE=1 bash test.sh 60 | ``` 61 | 62 | A more intensive set of GPU tests is available in the LPMS repository, which is vendored within `go-livepeer`. Refer to the [LPMS README](https://github.com/livepeer/lpms/blob/master/README.md) for details on how to run these tests. 63 | -------------------------------------------------------------------------------- /doc/httpcli.md: -------------------------------------------------------------------------------- 1 | # HTTP endpoint 2 | 3 | The Livepeer node exposes an HTTP interface for monitoring and managing the node. This is how the `livepeer_cli` tool interfaces with a running node. 4 | By default, the CLI listens to localhost:7935. This can be adjusted with the -cliAddr `:` flag. 5 | 6 | ## Available endpoints: 7 | 8 | 9 | 10 | `/getLogLevel` returns current verbosity level in the body of response 11 | 12 | `/setLogLevel` sets verbosity current level. Level to set should be provided in body of the request, encoded as `application/x-www-form-urlencoded`. Parameter should be named `loglevel`. 13 | It can be used from command like this: 14 | 15 | `curl -F loglevel=6 http://localhost:7935/setLogLevel` 16 | 17 | Log level should be integer from 0 to 6, where 6 means most verbose logging. 18 | -------------------------------------------------------------------------------- /doc/orchwebhook.md: -------------------------------------------------------------------------------- 1 | # Orchestrator Webhook 2 | 3 | Livepeer supports orchestrator discovery using a webhook. Webhook orchestrator discovery can be 4 | enabled by starting a Livepeer Broadcaster node with the `-orchWebhookUrl ` flag. 5 | The `` is a url that returns an array of objects that each contains an "address" key 6 | and a URL string value. 7 | 8 | For example: 9 | 10 | ```json 11 | [ 12 | {"address":"https://10.4.3.2:8935"}, 13 | {"address":"https://10.4.4.3:8935"}, 14 | {"address":"https://10.4.5.2:8935"} 15 | ] 16 | ``` 17 | 18 | The orchestrator webhook allows a Broadcaster node operator to periodically refresh its list of available orchestrators. 19 | The list is refreshed no more than once per minute or as needed, depending on streaming conditions. Refer to the [reliability documentation](https://github.com/livepeer/go-livepeer/blob/master/doc/reliability.md) for more information. 20 | -------------------------------------------------------------------------------- /doc/selection.md: -------------------------------------------------------------------------------- 1 | # Selection 2 | 3 | A broadcaster uses a *selection* algorithm to select the orchestrator to use for the next segment to be transcoded from a list of eligible active orchestrators. 4 | 5 | The selection algorithm is implemented by a selector that implements the [BroadcastSessionsSelector](https://github.com/livepeer/go-livepeer/blob/master/server/selection.go) interface. 6 | 7 | The current default selector implementation is the [MinLSSelectorWithRandFreq](https://github.com/livepeer/go-livepeer/blob/1af0a5182cd3a9aa38d961b6d1d104a3693ec814/server/selection.go#L118) which does the following: 8 | 9 | - Tracks "unknown sessions" and "known sessions" 10 | - A session is unknown if there is no latency score tracked yet i.e. no segment was transcoded by the orchestrator for this session yet 11 | - A session is known if there is a latency score tracked i.e. a segment was transcoded by the orchestrator for this session already 12 | - A latency score is calculated as the ratio between segment duration and the round trip response time (i.e. upload, transcode, download) 13 | - If there are no known sessions available, then select from the unknown sessions 14 | - If the best latency score of all known sessions does not meet the latency score threshold, then select from the unknown sessions 15 | - Otherwise, select from the known sessions 16 | 17 | **Selecting Unknown Sessions** 18 | 19 | - X% of the time select an unknown session randomly 20 | - The rest of the time, use stake weighted random selection to select the unknown session 21 | 22 | **Selecting Known Sessions** 23 | 24 | - Select the known session with the best latency score 25 | 26 | ## Future 27 | 28 | A few considerations for future iterations on selection algorithms: 29 | 30 | - Consider additional inputs outside of just speed and price including transcoding quality (i.e. lowest loss compared to original signal) and efficiency (i.e. highest quality per bit) 31 | - Previous work: 32 | - [Leaderboard scoring framework](https://livepeer.notion.site/Leaderboard-Score-Framework-a420c0e9b6e4408b81cf0d9ffcd9d40e) 33 | - [Presentation](https://www.youtube.com/watch?v=ZDCg5feDELA) on selection framework based on technical constraints and economic preferences 34 | - Should be able to express their preferences in a way that adjusts how much each input into the selection algorithm is weighted (i.e. weight speed over price, weight quality over speed, etc) and a list of weights (or a weights generation algorithm) could be used to customize how the selection algorithm works -------------------------------------------------------------------------------- /doc/verification.md: -------------------------------------------------------------------------------- 1 | # Verification 2 | 3 | The Livepeer node supports two types of verification when running with the `-gateway` flag: 4 | 5 | - Local verification 6 | - This currently involves pixel count and signature verification. 7 | - Pixel count verification ensures that the number of pixels that the orchestrator uses to charge the broadcaster for payments matches the number of pixels actually encoded by the orchestrator. The orchestrator reports the number of pixels encoded with each result returned to the broadcaster and the broadcaster compares this value with the actual number of pixels in the results. 8 | - Signature verification ensures that the results received are cryptographically signed using a known Ethereum account associated with an orchestrator. The Ethereum account used to sign the results may be an on-chain registered address or it may be an account specified in the address field of the `OrchestratorInfo` message sent to the broadcaster during discovery. 9 | - Tamper verification 10 | - This currently uses an external verifier that checks if a video has been tampered. 11 | 12 | Local verification is enabled by default when the node is connected to Rinkeby and mainnet and disabled by default when the node is running in off-chain mode. Local verification can be explicitly enabled by starting the node with `-localVerify` and can be explicitly disabled with `-localVerify=false`. 13 | 14 | Tamper verification is disabled by default and can be enabled by specifying `-verifierURL`. See this [guide](https://livepeer.org/docs/video-developers/how-to-guides/verification) for instructions on connecting the node to an external verifier that runs tamper verification. Note that when tamper verification is enabled, local verification is also enabled. -------------------------------------------------------------------------------- /doc/worker.md: -------------------------------------------------------------------------------- 1 | # AI Worker 2 | 3 | The AI Worker node manages [`ai-runner`](https://github.com/livepeer/ai-runner) containers running on the host system. 4 | These containers are started, monitored and stopped dynamically depending on the usage. 5 | 6 | This diagram describes the lifecycle of a container: 7 | 8 | ![ai-runner container lifecycle](./assets/ai-runner-container-lifecycle.jpg) 9 | 10 | Source: [Miro Board](https://miro.com/app/board/uXjVIZ0vO4k=/?share_link_id=987855784886) 11 | 12 | It can also be described by the following mermaid chart, but the rendered version is more confusing: 13 | ``` 14 | stateDiagram-v2 15 | direction TB 16 | [*] --> OFFLINE 17 | OFFLINE --> IDLE: Warm()->createCont() 18 | OFFLINE --> BORROWED: Borrow(ctx)->createCont() 19 | state RUNNING { 20 | [*] --> IDLE 21 | IDLE --> BORROWED: Borrow(ctx) 22 | BORROWED --> IDLE: BorrowCtx.Done() 23 | } 24 | hc: GET /health 25 | RUNNING --> hc 26 | state healthcheck <> 27 | hc --> healthcheck 28 | healthcheck --> OFFLINE: if error x2 29 | healthcheck --> RUNNING: if state=OK 30 | healthcheck --> IDLE: if state=IDLE 31 | ``` 32 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM --platform=$BUILDPLATFORM livepeerci/cuda:12.0.0-cudnn8-devel-ubuntu20.04 as build 2 | 3 | ARG TARGETARCH 4 | ARG BUILDARCH 5 | 6 | ENV GOARCH="$TARGETARCH" \ 7 | PATH="/usr/local/go/bin:/go/bin:${PATH}" \ 8 | PKG_CONFIG_PATH="/root/compiled/lib/pkgconfig" \ 9 | CPATH="/usr/local/cuda_${TARGETARCH}/include" \ 10 | LIBRARY_PATH="/usr/local/cuda_${TARGETARCH}/lib64" \ 11 | DEBIAN_FRONTEND="noninteractive" \ 12 | CGO_LDFLAGS="-L/usr/local/cuda_${TARGETARCH}/lib64" 13 | 14 | RUN apt update \ 15 | && apt install -yqq software-properties-common curl apt-transport-https lsb-release nasm \ 16 | && curl -fsSL https://dl.google.com/go/go1.21.5.linux-${BUILDARCH}.tar.gz | tar -C /usr/local -xz \ 17 | && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \ 18 | && add-apt-repository "deb [arch=${BUILDARCH}] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ 19 | && curl -fsSl https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \ 20 | && add-apt-repository "deb [arch=${BUILDARCH}] https://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs)-14 main" \ 21 | && apt update \ 22 | && apt -yqq install clang-14 clang-tools-14 lld-14 build-essential pkg-config autoconf git python docker-ce-cli pciutils gcc-multilib libgcc-8-dev-arm64-cross gcc-mingw-w64-x86-64 zlib1g zlib1g-dev libx264-dev 23 | 24 | RUN update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-14 30 \ 25 | && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 30 \ 26 | && update-alternatives --install /usr/bin/ld ld /usr/bin/lld-14 30 27 | 28 | RUN GRPC_HEALTH_PROBE_VERSION=v0.3.6 \ 29 | && curl -fsSL https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-${TARGETARCH} -o /usr/bin/grpc_health_probe \ 30 | && chmod +x /usr/bin/grpc_health_probe \ 31 | && ldconfig /usr/local/lib 32 | 33 | RUN FFMPEG_SHA=b76053d8bf322b197a9d07bd27bbdad14fd5bc15 git clone --depth 1 https://git.ffmpeg.org/ffmpeg.git /ffmpeg \ 34 | && cd /ffmpeg && git fetch --depth 1 origin ${FFMPEG_SHA} \ 35 | && git checkout ${FFMPEG_SHA} \ 36 | && ./configure --enable-gpl --enable-libx264 --prefix=build && make -j"$(nproc)" && make install 37 | 38 | ENV GOPATH=/go \ 39 | GO_BUILD_DIR=/build/ \ 40 | GOFLAGS="-mod=readonly" 41 | 42 | WORKDIR /src 43 | 44 | RUN mkdir -p /go \ 45 | && curl -fsSLO https://github.com/livepeer/livepeer-ml/releases/download/v0.3/tasmodel.pb 46 | 47 | COPY ./install_ffmpeg.sh ./install_ffmpeg.sh 48 | 49 | ARG BUILD_TAGS 50 | ENV BUILD_TAGS=${BUILD_TAGS} 51 | 52 | COPY go.mod go.sum ./ 53 | RUN go mod download 54 | 55 | RUN ./install_ffmpeg.sh \ 56 | && GO111MODULE=on go get -v github.com/golangci/golangci-lint/cmd/golangci-lint@v1.52.2 \ 57 | && go get -v github.com/jstemmer/go-junit-report 58 | 59 | COPY . . 60 | 61 | RUN make livepeer livepeer_cli livepeer_bench livepeer_router 62 | 63 | FROM --platform=$TARGETPLATFORM nvidia/cuda:12.0.0-cudnn8-runtime-ubuntu20.04 AS livepeer-amd64-base 64 | 65 | FROM --platform=$TARGETPLATFORM nvidia/cuda:12.0.0-cudnn8-runtime-ubuntu20.04 AS livepeer-arm64-base 66 | 67 | FROM livepeer-${TARGETARCH}-base 68 | 69 | ENV NVIDIA_DRIVER_CAPABILITIES=all 70 | 71 | RUN apt update && apt install -y libx264-155 72 | 73 | COPY --from=build /build/ /usr/local/bin/ 74 | COPY --from=build /usr/bin/grpc_health_probe /usr/local/bin/grpc_health_probe 75 | COPY --from=build /src/tasmodel.pb /tasmodel.pb 76 | COPY --from=build /usr/share/misc/pci.ids /usr/share/misc/pci.ids 77 | COPY --from=build /ffmpeg/build/ /usr/local 78 | RUN ldconfig /usr/local/lib 79 | 80 | ENTRYPOINT ["/usr/local/bin/livepeer"] 81 | -------------------------------------------------------------------------------- /docker/Dockerfile.cuda-base: -------------------------------------------------------------------------------- 1 | # livepeerci/cuda:12.0.0-cudnn8-devel-ubuntu20.04 2 | # 3 | # Base CUDA Develop image which contains CUDA SDK libs for the following architectures: linux amd64, linux arm64 4 | # 5 | # To build this image you need the following steps: 6 | # 1. Download NVIDIA CUDA SDK for ARM64, extract, and copy into cuda/arm64/usr/local/cuda/ 7 | 8 | FROM nvidia/cuda:12.0.0-cudnn8-devel-ubuntu20.04 9 | 10 | RUN mkdir -p /usr/local/cuda_arm64/lib64/ 11 | COPY cuda/arm64/usr/local/cuda/lib64/libnp* /usr/local/cuda_arm64/lib64/ 12 | COPY cuda/arm64/usr/local/cuda/include /usr/local/cuda_arm64/include 13 | 14 | RUN ln -s /usr/local/cuda /usr/local/cuda_amd64 15 | -------------------------------------------------------------------------------- /docker/Dockerfile.mediamtx: -------------------------------------------------------------------------------- 1 | ARG MEDIAMTX_VERSION="1.11.2-livepeer-3" 2 | 3 | FROM golang:1.23 AS builder 4 | 5 | # Install any build dependencies (e.g., git) 6 | RUN apt-get update && apt-get install -y --no-install-recommends \ 7 | git \ 8 | && rm -rf /var/lib/apt/lists/* 9 | 10 | # Use this branch while we have changes waiting for upstream 11 | ARG MEDIAMTX_VERSION 12 | WORKDIR /app 13 | RUN git clone --branch v${MEDIAMTX_VERSION} https://github.com/livepeer/mediamtx.git . 14 | 15 | # Download Go dependencies 16 | RUN go mod download 17 | 18 | # Disable CGO 19 | ENV CGO_ENABLED=0 20 | 21 | # Generate code and build 22 | RUN go generate ./... 23 | RUN go build -o mediamtx 24 | 25 | FROM ubuntu:24.04 26 | 27 | # we need curl in the image as it's later used in the runOnReady command 28 | RUN apt update \ 29 | && apt install -yqq \ 30 | ca-certificates \ 31 | curl \ 32 | cron \ 33 | && apt clean \ 34 | && rm -rf /var/lib/apt/lists/* /etc/cron.* 35 | 36 | COPY --chmod=0644 crontab /etc/crontab 37 | 38 | # Setup cron job for publishing metrics 39 | RUN mkdir -p /var/log/ \ 40 | && crontab /etc/crontab \ 41 | && touch /var/log/cron.log 42 | 43 | COPY --chmod=0755 mediamtx-metrics.bash /opt/mediamtx-metrics.bash 44 | 45 | # Copy artifacts from the builder stage 46 | ARG MEDIAMTX_VERSION 47 | ENV MEDIAMTX_VERSION=${MEDIAMTX_VERSION} 48 | COPY --from=builder /app/mediamtx /usr/local/bin/mediamtx 49 | COPY --from=builder /app/mediamtx.yml /etc/mediamtx/mediamtx.yml 50 | 51 | CMD [ "/bin/bash", "-c", "declare -p >> /etc/environment && cron && /usr/local/bin/mediamtx" ] 52 | -------------------------------------------------------------------------------- /docker/crontab: -------------------------------------------------------------------------------- 1 | SHELL=/bin/bash 2 | BASH_ENV=/etc/environment 3 | 4 | */5 * * * * /opt/mediamtx-metrics.bash >> /var/log/cron.log 2>&1 5 | -------------------------------------------------------------------------------- /docker/mediamtx-metrics.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | [ -v DEBUG ] && set -x 4 | 5 | set -euo pipefail 6 | 7 | if [ -v LP_PUBLISH_MEDIAMTX_METRICS ]; then 8 | METRIC_DATA=$( 9 | cat <&2 "No endpoint specified for publishing mediamtx metrics." 19 | exit 1 20 | fi 21 | echo "$METRIC_DATA" | curl -X POST --data-binary @- "$LP_PUBLISH_MEDIAMTX_METRICS_ENDPOINT" 22 | fi 23 | -------------------------------------------------------------------------------- /etc/ffmpeg_trans_test.sh: -------------------------------------------------------------------------------- 1 | if [ -d "./tran_result" ]; then 2 | echo "Need to remove tran_result first" 3 | exit -1 4 | fi 5 | 6 | mkdir -p tran_result 7 | lastsum='' 8 | 9 | for i in {1..2}; do 10 | echo "Transcoding for out$i" 11 | ffmpeg -i seg$i.ts -c:v libx264 -s 426:240 -r 30 -mpegts_copyts 1 -minrate 700k -maxrate 700k -bufsize 700k ./tran_result/out$i.ts 12 | sum=($(md5sum ./tran_result/out$i.ts)) 13 | if [[ "$lastsum" != '' && "$sum" != "$lastsum" ]]; then 14 | printf "\n\nDifferent MD5 - $lastsum != $sum(out$i)\n" 15 | exit -1 16 | fi 17 | lastsum="$sum" 18 | done 19 | 20 | echo "All Equal!" 21 | md5sum ./tran_result/out* 22 | -------------------------------------------------------------------------------- /eth/README.md: -------------------------------------------------------------------------------- 1 | # Generating Go bindings for contracts 2 | 3 | The `contracts` folder contains Go bindings for the Livepeer protocol smart contracts generated using the 4 | [abigen](https://github.com/ethereum/go-ethereum/tree/master/cmd/abigen) tool. 5 | 6 | If the smart contracts are updated you can generate new Go bindings by doing the following: 7 | 8 | ``` 9 | cd $GOPATH/src/github.com/livepeer/go-livepeer/eth 10 | git clone https://github.com/livepeer/protocol.git 11 | cd $GOPATH/src/github.com/livepeer/go-livepeer/eth/protocol 12 | yarn 13 | yarn compile 14 | cd $GOPATH/src/github.com/livepeer/go-livepeer/eth 15 | go generate client.go 16 | ``` 17 | -------------------------------------------------------------------------------- /eth/blockwatch/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2019 ZeroEx Intl. 2 | Modifications copyright 2019 Livepeer 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. -------------------------------------------------------------------------------- /eth/blockwatch/README.md: -------------------------------------------------------------------------------- 1 | # blockwatch 2 | 3 | Derived from [0x's blockwatch package](https://github.com/0xProject/0x-mesh/tree/development/ethereum/blockwatch) with a few modifications to allow it to be integrated into the Livepeer node. 4 | -------------------------------------------------------------------------------- /eth/blockwatch/fake_log_client.go: -------------------------------------------------------------------------------- 1 | package blockwatch 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "math/big" 7 | "sync/atomic" 8 | "time" 9 | 10 | "github.com/ethereum/go-ethereum" 11 | "github.com/ethereum/go-ethereum/common" 12 | "github.com/ethereum/go-ethereum/core/types" 13 | ) 14 | 15 | type filterLogsResponse struct { 16 | Logs []types.Log 17 | Err error 18 | } 19 | 20 | // fakeLogClient is a fake Client for testing code calling the `FilterLogs` method. 21 | // It allows the instatiator to specify `FilterLogs` responses for several block ranges. 22 | type fakeLogClient struct { 23 | count int64 24 | rangeToResponse map[string]filterLogsResponse 25 | } 26 | 27 | // newFakeLogClient instantiates a fakeLogClient for testing log fetching 28 | func newFakeLogClient(rangeToResponse map[string]filterLogsResponse) (*fakeLogClient, error) { 29 | return &fakeLogClient{count: 0, rangeToResponse: rangeToResponse}, nil 30 | } 31 | 32 | // HeaderByNumber fetches a block header by its number 33 | func (fc *fakeLogClient) HeaderByNumber(number *big.Int) (*MiniHeader, error) { 34 | return nil, errors.New("NOT_IMPLEMENTED") 35 | } 36 | 37 | // HeaderByHash fetches a block header by its block hash 38 | func (fc *fakeLogClient) HeaderByHash(hash common.Hash) (*MiniHeader, error) { 39 | return nil, errors.New("NOT_IMPLEMENTED") 40 | } 41 | 42 | // FilterLogs returns the logs that satisfy the supplied filter query 43 | func (fc *fakeLogClient) FilterLogs(q ethereum.FilterQuery) ([]types.Log, error) { 44 | // Add a slight delay to simulate an actual network request. This also gives 45 | // BlockWatcher.getLogsInBlockRange multi-requests to hit the concurrent request 46 | // limit semaphore and simulate more realistic conditions. 47 | time.Sleep(5 * time.Millisecond) 48 | r := toRange(q.FromBlock, q.ToBlock) 49 | res, ok := fc.rangeToResponse[r] 50 | if !ok { 51 | return nil, fmt.Errorf("Didn't find response for range %s but was expecting it to exist", r) 52 | } 53 | atomic.AddInt64(&fc.count, 1) 54 | return res.Logs, res.Err 55 | } 56 | 57 | // Count returns the number of times FilterLogs was called 58 | func (fc *fakeLogClient) Count() int { 59 | return int(fc.count) 60 | } 61 | 62 | func toRange(from, to *big.Int) string { 63 | r := fmt.Sprintf("%s-%s", from, to) 64 | return r 65 | } 66 | -------------------------------------------------------------------------------- /eth/blockwatch/stack.go: -------------------------------------------------------------------------------- 1 | package blockwatch 2 | 3 | import ( 4 | "math/big" 5 | "sync" 6 | 7 | ethcommon "github.com/ethereum/go-ethereum/common" 8 | "github.com/ethereum/go-ethereum/core/types" 9 | ) 10 | 11 | // MiniHeader is a succinct representation of an Ethereum block header 12 | type MiniHeader struct { 13 | Hash ethcommon.Hash 14 | Parent ethcommon.Hash 15 | Number *big.Int 16 | L1BlockNumber *big.Int 17 | Logs []types.Log 18 | } 19 | 20 | // MiniHeaderStore is an interface for a store that manages the state of a MiniHeader collection 21 | type MiniHeaderStore interface { 22 | FindLatestMiniHeader() (*MiniHeader, error) 23 | FindAllMiniHeadersSortedByNumber() ([]*MiniHeader, error) 24 | InsertMiniHeader(header *MiniHeader) error 25 | DeleteMiniHeader(hash ethcommon.Hash) error 26 | } 27 | 28 | // Stack allows performing basic stack operations on a stack of MiniHeaders. 29 | type Stack struct { 30 | // TODO(albrow): Use Transactions when db supports them instead of a mutex 31 | // here. There are cases where we need to make sure no modifications are made 32 | // to the database in between a read/write or read/delete. 33 | mut sync.Mutex 34 | store MiniHeaderStore 35 | limit int 36 | } 37 | 38 | // NewStack instantiates a new stack with the specified size limit. Once the size limit 39 | // is reached, adding additional blocks will evict the deepest block. 40 | func NewStack(store MiniHeaderStore, limit int) *Stack { 41 | return &Stack{ 42 | store: store, 43 | limit: limit, 44 | } 45 | } 46 | 47 | // Pop removes and returns the latest block header on the block stack. It 48 | // returns nil if the stack is empty. 49 | func (s *Stack) Pop() (*MiniHeader, error) { 50 | s.mut.Lock() 51 | defer s.mut.Unlock() 52 | latestMiniHeader, err := s.store.FindLatestMiniHeader() 53 | if err != nil { 54 | return nil, err 55 | } 56 | if latestMiniHeader == nil { 57 | return nil, nil 58 | } 59 | if err := s.store.DeleteMiniHeader(latestMiniHeader.Hash); err != nil { 60 | return nil, err 61 | } 62 | return latestMiniHeader, nil 63 | } 64 | 65 | // Push pushes a block header onto the block stack. If the stack limit is 66 | // reached, it will remove the oldest block header. 67 | func (s *Stack) Push(header *MiniHeader) error { 68 | s.mut.Lock() 69 | defer s.mut.Unlock() 70 | miniHeaders, err := s.store.FindAllMiniHeadersSortedByNumber() 71 | if err != nil { 72 | return err 73 | } 74 | if len(miniHeaders) == s.limit { 75 | oldestMiniHeader := miniHeaders[0] 76 | if err := s.store.DeleteMiniHeader(oldestMiniHeader.Hash); err != nil { 77 | return err 78 | } 79 | } 80 | if err := s.store.InsertMiniHeader(header); err != nil { 81 | return err 82 | } 83 | return nil 84 | } 85 | 86 | // Peek returns the latest block header from the block stack without removing 87 | // it. It returns nil if the stack is empty. 88 | func (s *Stack) Peek() (*MiniHeader, error) { 89 | return s.store.FindLatestMiniHeader() 90 | } 91 | 92 | // Inspect returns all the block headers currently on the stack 93 | func (s *Stack) Inspect() ([]*MiniHeader, error) { 94 | return s.store.FindAllMiniHeadersSortedByNumber() 95 | } 96 | -------------------------------------------------------------------------------- /eth/blockwatch/testdata/fake_client_basic_fixture.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "getLatestBlock": { 4 | "hash": "0x293b9ea024055a3e9eddbf9b9383dc7731744111894af6aa038594dc1b61f87f", 5 | "parent": "0x26b13ac89500f7fcdd141b7d1b30f3a82178431eca325d1cf10998f9d68ff5ba", 6 | "number": 5 7 | }, 8 | "getBlockByNumber": { 9 | "5": { 10 | "hash": "0x293b9ea024055a3e9eddbf9b9383dc7731744111894af6aa038594dc1b61f87f", 11 | "parent": "0x26b13ac89500f7fcdd141b7d1b30f3a82178431eca325d1cf10998f9d68ff5ba", 12 | "number": 5 13 | } 14 | }, 15 | "getBlockByHash": { 16 | "0x293b9ea024055a3e9eddbf9b9383dc7731744111894af6aa038594dc1b61f87f": { 17 | "hash": "0x293b9ea024055a3e9eddbf9b9383dc7731744111894af6aa038594dc1b61f87f", 18 | "parent": "0x26b13ac89500f7fcdd141b7d1b30f3a82178431eca325d1cf10998f9d68ff5ba", 19 | "number": 5 20 | } 21 | }, 22 | "getCorrectChain": [ 23 | { 24 | "hash": "0x293b9ea024055a3e9eddbf9b9383dc7731744111894af6aa038594dc1b61f87f", 25 | "parent": "0x26b13ac89500f7fcdd141b7d1b30f3a82178431eca325d1cf10998f9d68ff5ba", 26 | "number": 5 27 | } 28 | ], 29 | "blockEvents": [ 30 | { 31 | "type": 0, 32 | "blockHeader": { 33 | "hash": "0x293b9ea024055a3e9eddbf9b9383dc7731744111894af6aa038594dc1b61f87f", 34 | "parent": "0x26b13ac89500f7fcdd141b7d1b30f3a82178431eca325d1cf10998f9d68ff5ba", 35 | "number": 5 36 | } 37 | } 38 | ], 39 | "scenarioLabel": "FIND_NEXT_BLOCK" 40 | } 41 | ] 42 | -------------------------------------------------------------------------------- /eth/blockwatch/testdata/fake_client_fast_sync_fixture.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "getLatestBlock": { 4 | "hash": "0x382b25dd926322722bba2575b5092be661a99f26472e2b7c2bb3d51e6bd2b09c", 5 | "parent": "0xb57233b65a0da953da19d685d10a65e3124207fd1cf4694e3fc0f7279373bf5f", 6 | "number": 30 7 | }, 8 | "getBlockByNumber": { 9 | "5": { 10 | "hash": "0x293b9ea024055a3e9eddbf9b9383dc7731744111894af6aa038594dc1b61f87f", 11 | "parent": "0x26b13ac89500f7fcdd141b7d1b30f3a82178431eca325d1cf10998f9d68ff5ba", 12 | "number": 5 13 | }, 14 | "30": { 15 | "hash": "0x382b25dd926322722bba2575b5092be661a99f26472e2b7c2bb3d51e6bd2b09c", 16 | "parent": "0xb57233b65a0da953da19d685d10a65e3124207fd1cf4694e3fc0f7279373bf5f", 17 | "number": 30 18 | }, 19 | "29": { 20 | "hash": "0xb57233b65a0da953da19d685d10a65e3124207fd1cf4694e3fc0f7279373bf5f", 21 | "parent": "0x0f52679a1ff257072b0588234c9cf9727cc4fe3a973e5fa528474d2be638253b", 22 | "number": 29 23 | } 24 | }, 25 | "getBlockByHash": { 26 | "0x293b9ea024055a3e9eddbf9b9383dc7731744111894af6aa038594dc1b61f87f": { 27 | "hash": "0x293b9ea024055a3e9eddbf9b9383dc7731744111894af6aa038594dc1b61f87f", 28 | "parent": "0x26b13ac89500f7fcdd141b7d1b30f3a82178431eca325d1cf10998f9d68ff5ba", 29 | "number": 5 30 | }, 31 | "0x382b25dd926322722bba2575b5092be661a99f26472e2b7c2bb3d51e6bd2b09c": { 32 | "hash": "0x382b25dd926322722bba2575b5092be661a99f26472e2b7c2bb3d51e6bd2b09c", 33 | "parent": "0xb57233b65a0da953da19d685d10a65e3124207fd1cf4694e3fc0f7279373bf5f", 34 | "number": 30 35 | }, 36 | "0xb57233b65a0da953da19d685d10a65e3124207fd1cf4694e3fc0f7279373bf5f": { 37 | "hash": "0xb57233b65a0da953da19d685d10a65e3124207fd1cf4694e3fc0f7279373bf5f", 38 | "parent": "0x0f52679a1ff257072b0588234c9cf9727cc4fe3a973e5fa528474d2be638253b", 39 | "number": 29 40 | } 41 | }, 42 | "getCorrectChain": [], 43 | "blockEvents": [], 44 | "scenarioLabel": "FAST_SYNC" 45 | } 46 | ] 47 | -------------------------------------------------------------------------------- /eth/client_ticketbroker.go: -------------------------------------------------------------------------------- 1 | package eth 2 | 3 | import ( 4 | "math/big" 5 | 6 | ethcommon "github.com/ethereum/go-ethereum/common" 7 | "github.com/ethereum/go-ethereum/core/types" 8 | "github.com/livepeer/go-livepeer/eth/contracts" 9 | "github.com/livepeer/go-livepeer/pm" 10 | ) 11 | 12 | // FundDepositAndReserve funds a sender's deposit and reserve 13 | // This method wraps the underlying contract method in order to set the transaction options 14 | // value to the sum of the provided deposit and penalty escrow amounts 15 | func (c *client) FundDepositAndReserve(depositAmount, reserveAmount *big.Int) (*types.Transaction, error) { 16 | opts := c.transactOpts() 17 | opts.Value = new(big.Int).Add(depositAmount, reserveAmount) 18 | 19 | return c.ticketBroker.FundDepositAndReserve(opts, depositAmount, reserveAmount) 20 | } 21 | 22 | // FundDeposit funds a sender's deposit 23 | // This method wraps the underlying contract method in order to set the transaction options 24 | // value to the provided deposit amount 25 | func (c *client) FundDeposit(amount *big.Int) (*types.Transaction, error) { 26 | opts := c.transactOpts() 27 | opts.Value = amount 28 | 29 | return c.ticketBroker.FundDeposit(opts) 30 | } 31 | 32 | // FundReserve funds a sender's reserve 33 | // This method wraps the underlying contract method in order to set the transaction options 34 | // value to the provided reserve amount 35 | func (c *client) FundReserve(amount *big.Int) (*types.Transaction, error) { 36 | opts := c.transactOpts() 37 | opts.Value = amount 38 | 39 | return c.ticketBroker.FundReserve(opts) 40 | } 41 | 42 | // RedeemWinningTicket submits a ticket to be validated by the broker and if a valid winning ticket 43 | // the broker pays the ticket's face value to the ticket's recipient 44 | func (c *client) RedeemWinningTicket(ticket *pm.Ticket, sig []byte, recipientRand *big.Int) (*types.Transaction, error) { 45 | var recipientRandHash [32]byte 46 | copy(recipientRandHash[:], ticket.RecipientRandHash.Bytes()[:32]) 47 | 48 | return c.ticketBroker.RedeemWinningTicket( 49 | c.transactOpts(), 50 | contracts.MTicketBrokerCoreTicket{ 51 | Recipient: ticket.Recipient, 52 | Sender: ticket.Sender, 53 | FaceValue: ticket.FaceValue, 54 | WinProb: ticket.WinProb, 55 | SenderNonce: new(big.Int).SetUint64(uint64(ticket.SenderNonce)), 56 | RecipientRandHash: recipientRandHash, 57 | AuxData: ticket.AuxData(), 58 | }, 59 | sig, 60 | recipientRand, 61 | ) 62 | } 63 | 64 | // GetSenderInfo returns the info for a sender 65 | func (c *client) GetSenderInfo(addr ethcommon.Address) (*pm.SenderInfo, error) { 66 | info, err := c.ticketBroker.GetSenderInfo(c.callOpts(), addr) 67 | if err != nil { 68 | return nil, err 69 | } 70 | 71 | return &pm.SenderInfo{ 72 | Deposit: info.Sender.Deposit, 73 | WithdrawRound: info.Sender.WithdrawRound, 74 | Reserve: &pm.ReserveInfo{ 75 | FundsRemaining: info.Reserve.FundsRemaining, 76 | ClaimedInCurrentRound: info.Reserve.ClaimedInCurrentRound, 77 | }, 78 | }, nil 79 | } 80 | 81 | // IsUsedTicket checks if a ticket has been used 82 | // This method wraps the underlying contract method UsedTickets to allow callers to pass in 83 | // a ticket object 84 | func (c *client) IsUsedTicket(ticket *pm.Ticket) (bool, error) { 85 | var ticketHash [32]byte 86 | copy(ticketHash[:], ticket.Hash().Bytes()[:32]) 87 | 88 | return c.ticketBroker.UsedTickets(c.callOpts(), ticketHash) 89 | } 90 | -------------------------------------------------------------------------------- /eth/contracts/chainlink/AggregatorV3Interface.abi: -------------------------------------------------------------------------------- 1 | [{"inputs":[],"name":"decimals","outputs":[{"internalType":"uint8","name":"","type":"uint8"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"description","outputs":[{"internalType":"string","name":"","type":"string"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"uint80","name":"_roundId","type":"uint80"}],"name":"getRoundData","outputs":[{"internalType":"uint80","name":"roundId","type":"uint80"},{"internalType":"int256","name":"answer","type":"int256"},{"internalType":"uint256","name":"startedAt","type":"uint256"},{"internalType":"uint256","name":"updatedAt","type":"uint256"},{"internalType":"uint80","name":"answeredInRound","type":"uint80"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"latestRoundData","outputs":[{"internalType":"uint80","name":"roundId","type":"uint80"},{"internalType":"int256","name":"answer","type":"int256"},{"internalType":"uint256","name":"startedAt","type":"uint256"},{"internalType":"uint256","name":"updatedAt","type":"uint256"},{"internalType":"uint80","name":"answeredInRound","type":"uint80"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"version","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"}] 2 | -------------------------------------------------------------------------------- /eth/contracts/chainlink/AggregatorV3Interface.sol: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | // https://github.com/smartcontractkit/chainlink/blob/v2.9.1/contracts/src/v0.7/interfaces/AggregatorV3Interface.sol 3 | pragma solidity ^0.7.0; 4 | 5 | interface AggregatorV3Interface { 6 | function decimals() external view returns (uint8); 7 | 8 | function description() external view returns (string memory); 9 | 10 | function version() external view returns (uint256); 11 | 12 | // getRoundData and latestRoundData should both raise "No data present" 13 | // if they do not have data to report, instead of returning unset values 14 | // which could be misinterpreted as actual reported values. 15 | function getRoundData(uint80 _roundId) 16 | external 17 | view 18 | returns ( 19 | uint80 roundId, 20 | int256 answer, 21 | uint256 startedAt, 22 | uint256 updatedAt, 23 | uint80 answeredInRound 24 | ); 25 | 26 | function latestRoundData() 27 | external 28 | view 29 | returns ( 30 | uint80 roundId, 31 | int256 answer, 32 | uint256 startedAt, 33 | uint256 updatedAt, 34 | uint80 answeredInRound 35 | ); 36 | } 37 | -------------------------------------------------------------------------------- /eth/noncemanager.go: -------------------------------------------------------------------------------- 1 | package eth 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | 7 | ethcommon "github.com/ethereum/go-ethereum/common" 8 | ) 9 | 10 | // RemoteNonceReader is an interface that describes an object 11 | // capable of reading transaction nonces for ETH address from a remote source 12 | type RemoteNonceReader interface { 13 | PendingNonceAt(ctx context.Context, addr ethcommon.Address) (uint64, error) 14 | } 15 | 16 | type nonceLock struct { 17 | nonce uint64 18 | mu sync.Mutex 19 | } 20 | 21 | // NonceManager manages transaction nonces for multiple ETH addresses 22 | type NonceManager struct { 23 | nonces map[ethcommon.Address]*nonceLock 24 | mu sync.Mutex 25 | 26 | remoteReader RemoteNonceReader 27 | } 28 | 29 | // NewNonceManager creates an instance of a NonceManager 30 | func NewNonceManager(remoteReader RemoteNonceReader) *NonceManager { 31 | return &NonceManager{ 32 | nonces: make(map[ethcommon.Address]*nonceLock), 33 | remoteReader: remoteReader, 34 | } 35 | } 36 | 37 | // Lock locks the provided address. The caller should always call Lock before 38 | // calling Next or Update 39 | func (m *NonceManager) Lock(addr ethcommon.Address) { 40 | m.getNonceLock(addr).mu.Lock() 41 | } 42 | 43 | // Unlock unlocks the provided address. The caller should always call Unlock 44 | // after finishing calls to Next or Update 45 | func (m *NonceManager) Unlock(addr ethcommon.Address) { 46 | m.getNonceLock(addr).mu.Unlock() 47 | } 48 | 49 | // Next returns the next transaction nonce to be used for the provided address 50 | func (m *NonceManager) Next(addr ethcommon.Address) (uint64, error) { 51 | nonceLock := m.getNonceLock(addr) 52 | localNonce := nonceLock.nonce 53 | 54 | remoteNonce, err := m.remoteReader.PendingNonceAt(context.Background(), addr) 55 | if err != nil { 56 | return 0, err 57 | } 58 | 59 | // If remote nonce > local nonce, another client was likely used 60 | // to submit transactions such that the local nonce does not capture 61 | // transactions submitted by other clients 62 | if remoteNonce > localNonce { 63 | return remoteNonce, nil 64 | } 65 | 66 | return localNonce, nil 67 | } 68 | 69 | // Update uses the last nonce for the provided address to update the next transaction nonce 70 | func (m *NonceManager) Update(addr ethcommon.Address, lastNonce uint64) { 71 | nonceLock := m.getNonceLock(addr) 72 | localNonce := nonceLock.nonce 73 | 74 | if lastNonce == localNonce { 75 | nonceLock.nonce = localNonce + 1 76 | return 77 | } 78 | 79 | nonceLock.nonce = lastNonce + 1 80 | } 81 | 82 | func (m *NonceManager) getNonceLock(addr ethcommon.Address) *nonceLock { 83 | m.mu.Lock() 84 | defer m.mu.Unlock() 85 | 86 | if _, ok := m.nonces[addr]; !ok { 87 | m.nonces[addr] = new(nonceLock) 88 | } 89 | 90 | return m.nonces[addr] 91 | } 92 | -------------------------------------------------------------------------------- /eth/pricefeed.go: -------------------------------------------------------------------------------- 1 | package eth 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "math/big" 7 | "time" 8 | 9 | "github.com/ethereum/go-ethereum/accounts/abi/bind" 10 | "github.com/ethereum/go-ethereum/common" 11 | "github.com/ethereum/go-ethereum/ethclient" 12 | "github.com/livepeer/go-livepeer/eth/contracts/chainlink" 13 | ) 14 | 15 | type PriceData struct { 16 | RoundID int64 17 | Price *big.Rat 18 | UpdatedAt time.Time 19 | } 20 | 21 | // PriceFeedEthClient is an interface for fetching price data from a Chainlink 22 | // PriceFeed contract. 23 | type PriceFeedEthClient interface { 24 | Description() (string, error) 25 | FetchPriceData() (PriceData, error) 26 | } 27 | 28 | func NewPriceFeedEthClient(ethClient *ethclient.Client, priceFeedAddr string) (PriceFeedEthClient, error) { 29 | addr := common.HexToAddress(priceFeedAddr) 30 | priceFeed, err := chainlink.NewAggregatorV3Interface(addr, ethClient) 31 | if err != nil { 32 | return nil, fmt.Errorf("failed to create aggregator proxy: %w", err) 33 | } 34 | 35 | return &priceFeedClient{ 36 | client: ethClient, 37 | priceFeed: priceFeed, 38 | }, nil 39 | } 40 | 41 | type priceFeedClient struct { 42 | client *ethclient.Client 43 | priceFeed *chainlink.AggregatorV3Interface 44 | } 45 | 46 | func (c *priceFeedClient) Description() (string, error) { 47 | return c.priceFeed.Description(&bind.CallOpts{}) 48 | } 49 | 50 | func (c *priceFeedClient) FetchPriceData() (PriceData, error) { 51 | data, err := c.priceFeed.LatestRoundData(&bind.CallOpts{}) 52 | if err != nil { 53 | return PriceData{}, errors.New("failed to get latest round data: " + err.Error()) 54 | } 55 | 56 | decimals, err := c.priceFeed.Decimals(&bind.CallOpts{}) 57 | if err != nil { 58 | return PriceData{}, errors.New("failed to get decimals: " + err.Error()) 59 | } 60 | 61 | return computePriceData(data.RoundId, data.UpdatedAt, data.Answer, decimals), nil 62 | } 63 | 64 | // computePriceData transforms the raw data from the PriceFeed into the higher 65 | // level PriceData struct, more easily usable by the rest of the system. 66 | func computePriceData(roundID, updatedAt, answer *big.Int, decimals uint8) PriceData { 67 | // Compute a big.int which is 10^decimals. 68 | divisor := new(big.Int).Exp( 69 | big.NewInt(10), 70 | big.NewInt(int64(decimals)), 71 | nil) 72 | 73 | return PriceData{ 74 | RoundID: roundID.Int64(), 75 | Price: new(big.Rat).SetFrac(answer, divisor), 76 | UpdatedAt: time.Unix(updatedAt.Int64(), 0), 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /eth/pricefeed_test.go: -------------------------------------------------------------------------------- 1 | package eth 2 | 3 | import ( 4 | "math/big" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestComputePriceData(t *testing.T) { 11 | assert := assert.New(t) 12 | 13 | t.Run("valid data", func(t *testing.T) { 14 | roundID := big.NewInt(1) 15 | updatedAt := big.NewInt(1626192000) 16 | answer := big.NewInt(420666000) 17 | decimals := uint8(6) 18 | 19 | data := computePriceData(roundID, updatedAt, answer, decimals) 20 | 21 | assert.EqualValues(int64(1), data.RoundID, "Round ID didn't match") 22 | assert.Equal("210333/500", data.Price.RatString(), "The Price Rat didn't match") 23 | assert.Equal("2021-07-13 16:00:00 +0000 UTC", data.UpdatedAt.UTC().String(), "The updated at time did not match") 24 | }) 25 | 26 | t.Run("zero answer", func(t *testing.T) { 27 | roundID := big.NewInt(2) 28 | updatedAt := big.NewInt(1626192000) 29 | answer := big.NewInt(0) 30 | decimals := uint8(18) 31 | 32 | data := computePriceData(roundID, updatedAt, answer, decimals) 33 | 34 | assert.EqualValues(int64(2), data.RoundID, "Round ID didn't match") 35 | assert.Equal("0", data.Price.RatString(), "The Price Rat didn't match") 36 | assert.Equal("2021-07-13 16:00:00 +0000 UTC", data.UpdatedAt.UTC().String(), "The updated at time did not match") 37 | }) 38 | 39 | t.Run("zero decimals", func(t *testing.T) { 40 | roundID := big.NewInt(3) 41 | updatedAt := big.NewInt(1626192000) 42 | answer := big.NewInt(13) 43 | decimals := uint8(0) 44 | 45 | data := computePriceData(roundID, updatedAt, answer, decimals) 46 | 47 | assert.EqualValues(int64(3), data.RoundID, "Round ID didn't match") 48 | assert.Equal("13", data.Price.RatString(), "The Price Rat didn't match") 49 | assert.Equal("2021-07-13 16:00:00 +0000 UTC", data.UpdatedAt.UTC().String(), "The updated at time did not match") 50 | }) 51 | } 52 | -------------------------------------------------------------------------------- /eth/rewardservice.go: -------------------------------------------------------------------------------- 1 | package eth 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | 8 | "github.com/ethereum/go-ethereum/core/types" 9 | "github.com/golang/glog" 10 | "github.com/livepeer/go-livepeer/monitor" 11 | ) 12 | 13 | var ( 14 | ErrRewardServiceStarted = fmt.Errorf("reward service already started") 15 | ErrRewardServiceStopped = fmt.Errorf("reward service already stopped") 16 | ) 17 | 18 | type RewardService struct { 19 | client LivepeerEthClient 20 | working bool 21 | cancelWorker context.CancelFunc 22 | tw timeWatcher 23 | mu sync.Mutex 24 | } 25 | 26 | func NewRewardService(client LivepeerEthClient, tw timeWatcher) *RewardService { 27 | return &RewardService{ 28 | client: client, 29 | tw: tw, 30 | } 31 | } 32 | 33 | func (s *RewardService) Start(ctx context.Context) error { 34 | if s.working { 35 | return ErrRewardServiceStarted 36 | } 37 | 38 | cancelCtx, cancel := context.WithCancel(ctx) 39 | s.cancelWorker = cancel 40 | 41 | roundSink := make(chan types.Log, 10) 42 | sub := s.tw.SubscribeRounds(roundSink) 43 | defer sub.Unsubscribe() 44 | 45 | s.working = true 46 | defer func() { 47 | s.working = false 48 | }() 49 | 50 | for { 51 | select { 52 | case err := <-sub.Err(): 53 | if err != nil { 54 | glog.Errorf("Round subscription error err=%q", err) 55 | } 56 | case <-roundSink: 57 | go func() { 58 | err := s.tryReward() 59 | if err != nil { 60 | glog.Errorf("Error trying to call reward for round %v err=%q", s.tw.LastInitializedRound(), err) 61 | if monitor.Enabled { 62 | monitor.RewardCallError(err.Error()) 63 | } 64 | } 65 | }() 66 | case <-cancelCtx.Done(): 67 | glog.V(5).Infof("Reward service done") 68 | return nil 69 | } 70 | } 71 | } 72 | 73 | func (s *RewardService) Stop() error { 74 | if !s.working { 75 | return ErrRewardServiceStopped 76 | } 77 | 78 | s.cancelWorker() 79 | s.working = false 80 | 81 | return nil 82 | } 83 | 84 | func (s *RewardService) IsWorking() bool { 85 | return s.working 86 | } 87 | 88 | func (s *RewardService) tryReward() error { 89 | s.mu.Lock() 90 | defer s.mu.Unlock() 91 | 92 | currentRound := s.tw.LastInitializedRound() 93 | 94 | t, err := s.client.GetTranscoder(s.client.Account().Address) 95 | if err != nil { 96 | return err 97 | } 98 | 99 | if t.LastRewardRound.Cmp(currentRound) == -1 && t.Active { 100 | tx, err := s.client.Reward() 101 | if err != nil { 102 | return err 103 | } 104 | 105 | if err := s.client.CheckTx(tx); err != nil { 106 | return err 107 | } 108 | 109 | glog.Infof("Called reward for round %v", currentRound) 110 | 111 | return nil 112 | } 113 | 114 | return nil 115 | } 116 | -------------------------------------------------------------------------------- /eth/types/merkletree_test.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ethereum/go-ethereum/common" 7 | ) 8 | 9 | func TestVerifyProof(t *testing.T) { 10 | hashes := []common.Hash{ 11 | common.BytesToHash([]byte{5}), 12 | common.BytesToHash([]byte{6}), 13 | common.BytesToHash([]byte{7}), 14 | common.BytesToHash([]byte{8}), 15 | common.BytesToHash([]byte{9}), 16 | } 17 | 18 | root, proofs, err := NewMerkleTree(hashes) 19 | 20 | if err != nil { 21 | t.Fatalf("Failed to create merkle tree: %v", err) 22 | } 23 | 24 | for i, hash := range hashes { 25 | ok := VerifyProof(root.Hash, hash, proofs[i]) 26 | 27 | if !ok { 28 | t.Fatalf("Failed to verify merkle proof for %v: %v", hash.Hex(), err) 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /eth/watchers/eventdecoder.go: -------------------------------------------------------------------------------- 1 | package watchers 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/ethereum/go-ethereum/accounts/abi" 9 | "github.com/ethereum/go-ethereum/accounts/abi/bind" 10 | ethcommon "github.com/ethereum/go-ethereum/common" 11 | "github.com/ethereum/go-ethereum/core/types" 12 | ) 13 | 14 | // EventDecoder decodes logs into events for known contracts 15 | type EventDecoder struct { 16 | addr ethcommon.Address 17 | contract *bind.BoundContract 18 | topicToEventName map[ethcommon.Hash]string 19 | } 20 | 21 | // NewEventDecoder returns a new instance of EventDecoder with a contract binding to the provided ABI string 22 | func NewEventDecoder(addr ethcommon.Address, abiJSON string) (*EventDecoder, error) { 23 | abi, err := abi.JSON(strings.NewReader(abiJSON)) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | topicToEventName := make(map[ethcommon.Hash]string) 29 | for _, event := range abi.Events { 30 | topicToEventName[event.ID] = event.Name 31 | } 32 | 33 | return &EventDecoder{ 34 | addr: addr, 35 | // Create BoundContract without a backend because we just need to access 36 | // log unpacking without contract interaction 37 | contract: bind.NewBoundContract(addr, abi, nil, nil, nil), 38 | topicToEventName: topicToEventName, 39 | }, nil 40 | } 41 | 42 | // FindEventName returns the event name for a log. An error will be returned if the log is not emitted 43 | // from a known contract or if it does not map to a known event 44 | func (e *EventDecoder) FindEventName(log types.Log) (string, error) { 45 | if log.Address != e.addr { 46 | return "", errors.New("log not from known contract") 47 | } 48 | eventName, ok := e.topicToEventName[log.Topics[0]] 49 | if !ok { 50 | return "", fmt.Errorf("unknown event for %v", e.addr.Hex()) 51 | } 52 | 53 | return eventName, nil 54 | } 55 | 56 | // Decode decodes a log into an event struct. An error will be returned if the log is not emitted 57 | // from a known contract or if it does not map to a known event 58 | func (e *EventDecoder) Decode(eventName string, log types.Log, decodedLog interface{}) error { 59 | if log.Address != e.addr { 60 | return errors.New("log not from known contract") 61 | 62 | } 63 | return e.contract.UnpackLog(decodedLog, eventName, log) 64 | } 65 | -------------------------------------------------------------------------------- /eth/watchers/eventdecoder_test.go: -------------------------------------------------------------------------------- 1 | package watchers 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/ethereum/go-ethereum/common" 8 | ethcommon "github.com/ethereum/go-ethereum/common" 9 | "github.com/ethereum/go-ethereum/crypto" 10 | "github.com/livepeer/go-livepeer/eth/contracts" 11 | "github.com/stretchr/testify/assert" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestTopicToEventName_AllEventsIncluded(t *testing.T) { 16 | abi := "[{\"anonymous\": false,\"inputs\": [],\"name\": \"First\",\"type\": \"event\"},{\"anonymous\": false,\"inputs\": [],\"name\": \"Second\",\"type\": \"event\"},{\"anonymous\": false,\"inputs\": [],\"name\": \"Third\",\"type\": \"event\"}]" 17 | assert := assert.New(t) 18 | 19 | addr := ethcommon.HexToAddress("0x692a70d2e424a56d2c6c27aa97d1a86395877b3a") 20 | dec, err := NewEventDecoder(addr, abi) 21 | assert.Nil(err) 22 | assert.Equal(dec.topicToEventName[crypto.Keccak256Hash([]byte("First()"))], "First") 23 | assert.Equal(dec.topicToEventName[crypto.Keccak256Hash([]byte("Second()"))], "Second") 24 | assert.Equal(dec.topicToEventName[crypto.Keccak256Hash([]byte("Third()"))], "Third") 25 | 26 | } 27 | 28 | func TestEventDecoder_FindEventType(t *testing.T) { 29 | dec, err := NewEventDecoder(stubBondingManagerAddr, contracts.BondingManagerABI) 30 | require.Nil(t, err) 31 | 32 | assert := assert.New(t) 33 | 34 | // Test unknown contract address 35 | log := newStubBaseLog() 36 | log.Address = common.BytesToAddress([]byte("foo")) 37 | _, err = dec.FindEventName(log) 38 | assert.EqualError(err, "log not from known contract") 39 | 40 | // Test unknown contract event 41 | log.Address = stubBondingManagerAddr 42 | log.Topics = []common.Hash{common.BytesToHash([]byte("foo"))} 43 | _, err = dec.FindEventName(log) 44 | assert.EqualError(err, fmt.Sprintf("unknown event for %v", stubBondingManagerAddr.Hex())) 45 | 46 | // Test known contract address 47 | log = newStubUnbondLog() 48 | eventName, err := dec.FindEventName(log) 49 | assert.Nil(err) 50 | assert.Equal("Unbond", eventName) 51 | } 52 | 53 | func TestEventDecoder_Decode(t *testing.T) { 54 | dec, err := NewEventDecoder(stubBondingManagerAddr, contracts.BondingManagerABI) 55 | require.Nil(t, err) 56 | 57 | assert := assert.New(t) 58 | 59 | // Test unknown contract address 60 | log := newStubBaseLog() 61 | log.Address = common.BytesToAddress([]byte("foo")) 62 | 63 | var dummyEvent struct { 64 | foo string 65 | } 66 | err = dec.Decode("foo", log, dummyEvent) 67 | assert.EqualError(err, "log not from known contract") 68 | 69 | // Test known contract address 70 | log = newStubUnbondLog() 71 | 72 | var unbondEvent contracts.BondingManagerUnbond 73 | err = dec.Decode("Unbond", log, &unbondEvent) 74 | assert.Nil(err) 75 | } 76 | -------------------------------------------------------------------------------- /eth/watchers/serviceRegistryWatcher.go: -------------------------------------------------------------------------------- 1 | package watchers 2 | 3 | import ( 4 | ethcommon "github.com/ethereum/go-ethereum/common" 5 | "github.com/ethereum/go-ethereum/core/types" 6 | "github.com/golang/glog" 7 | "github.com/livepeer/go-livepeer/common" 8 | "github.com/livepeer/go-livepeer/eth" 9 | "github.com/livepeer/go-livepeer/eth/blockwatch" 10 | "github.com/livepeer/go-livepeer/eth/contracts" 11 | ) 12 | 13 | type ServiceRegistryWatcher struct { 14 | store common.OrchestratorStore 15 | dec *EventDecoder 16 | watcher BlockWatcher 17 | lpEth eth.LivepeerEthClient 18 | quit chan struct{} 19 | } 20 | 21 | func NewServiceRegistryWatcher(serviceRegistryAddr ethcommon.Address, watcher BlockWatcher, store common.OrchestratorStore, lpEth eth.LivepeerEthClient) (*ServiceRegistryWatcher, error) { 22 | dec, err := NewEventDecoder(serviceRegistryAddr, contracts.ServiceRegistryABI) 23 | if err != nil { 24 | return nil, err 25 | } 26 | 27 | return &ServiceRegistryWatcher{ 28 | store: store, 29 | dec: dec, 30 | watcher: watcher, 31 | lpEth: lpEth, 32 | quit: make(chan struct{}), 33 | }, nil 34 | } 35 | 36 | // Watch starts the event watching loop 37 | func (srw *ServiceRegistryWatcher) Watch() { 38 | events := make(chan []*blockwatch.Event, 10) 39 | sub := srw.watcher.Subscribe(events) 40 | defer sub.Unsubscribe() 41 | 42 | for { 43 | select { 44 | case <-srw.quit: 45 | return 46 | case err := <-sub.Err(): 47 | glog.Error(err) 48 | case events := <-events: 49 | srw.handleBlockEvents(events) 50 | } 51 | } 52 | } 53 | 54 | // Stop watching for events 55 | func (srw *ServiceRegistryWatcher) Stop() { 56 | close(srw.quit) 57 | } 58 | 59 | func (srw *ServiceRegistryWatcher) handleBlockEvents(events []*blockwatch.Event) { 60 | for _, event := range events { 61 | for _, log := range event.BlockHeader.Logs { 62 | if event.Type == blockwatch.Removed { 63 | log.Removed = true 64 | } 65 | if err := srw.handleLog(log); err != nil { 66 | glog.Error(err) 67 | } 68 | } 69 | } 70 | } 71 | 72 | func (srw *ServiceRegistryWatcher) handleLog(log types.Log) error { 73 | eventName, err := srw.dec.FindEventName(log) 74 | if err != nil { 75 | // Noop if we cannot find the event name 76 | return nil 77 | } 78 | 79 | switch eventName { 80 | case "ServiceURIUpdate": 81 | return srw.handleServiceURIUpdate(log) 82 | default: 83 | return nil 84 | } 85 | } 86 | 87 | func (srw *ServiceRegistryWatcher) handleServiceURIUpdate(log types.Log) error { 88 | var serviceURIUpdate contracts.ServiceRegistryServiceURIUpdate 89 | if err := srw.dec.Decode("ServiceURIUpdate", log, &serviceURIUpdate); err != nil { 90 | return err 91 | } 92 | 93 | if !log.Removed { 94 | return srw.store.UpdateOrch( 95 | &common.DBOrch{ 96 | EthereumAddr: serviceURIUpdate.Addr.String(), 97 | ServiceURI: serviceURIUpdate.ServiceURI, 98 | }, 99 | ) 100 | } 101 | uri, err := srw.lpEth.GetServiceURI(serviceURIUpdate.Addr) 102 | if err != nil { 103 | return err 104 | } 105 | return srw.store.UpdateOrch( 106 | &common.DBOrch{ 107 | EthereumAddr: serviceURIUpdate.Addr.String(), 108 | ServiceURI: uri, 109 | }, 110 | ) 111 | } 112 | -------------------------------------------------------------------------------- /eth/watchers/serviceRegistryWatcher_test.go: -------------------------------------------------------------------------------- 1 | package watchers 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/livepeer/go-livepeer/eth" 8 | "github.com/livepeer/go-livepeer/eth/blockwatch" 9 | lpTypes "github.com/livepeer/go-livepeer/eth/types" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestServiceRegistryWatcher_WatchAndStop(t *testing.T) { 14 | assert := assert.New(t) 15 | watcher := &stubBlockWatcher{} 16 | stubStore := &stubOrchestratorStore{} 17 | lpEth := ð.StubClient{} 18 | srw, err := NewServiceRegistryWatcher(stubServiceRegistryAddr, watcher, stubStore, lpEth) 19 | assert.Nil(err) 20 | 21 | go srw.Watch() 22 | time.Sleep(2 * time.Millisecond) 23 | 24 | // Test Stop 25 | srw.Stop() 26 | time.Sleep(2 * time.Millisecond) 27 | assert.True(watcher.sub.unsubscribed) 28 | } 29 | 30 | func TestServiceRegistryWatcher_HandleLog_HandleServiceURIUpdate(t *testing.T) { 31 | assert := assert.New(t) 32 | watcher := &stubBlockWatcher{} 33 | stubStore := &stubOrchestratorStore{} 34 | lpEth := ð.StubClient{ 35 | Orch: &lpTypes.Transcoder{ 36 | Address: stubTranscoder, 37 | ServiceURI: "http://127.0.0.1:1337", 38 | }, 39 | } 40 | srw, err := NewServiceRegistryWatcher(stubServiceRegistryAddr, watcher, stubStore, lpEth) 41 | assert.Nil(err) 42 | 43 | header := defaultMiniHeader() 44 | header.Logs = append(header.Logs, newStubServiceURIUpdateLog()) 45 | 46 | blockEvent := &blockwatch.Event{ 47 | Type: blockwatch.Added, 48 | BlockHeader: header, 49 | } 50 | 51 | go srw.Watch() 52 | defer srw.Stop() 53 | time.Sleep(2 * time.Millisecond) 54 | 55 | watcher.sink <- []*blockwatch.Event{blockEvent} 56 | time.Sleep(2 * time.Millisecond) 57 | assert.Equal(stubUpdatedServiceURI, stubStore.serviceURI) 58 | assert.Equal(stubStore.ethereumAddr, stubTranscoder.String()) 59 | 60 | // Test log removed 61 | blockEvent.Type = blockwatch.Removed 62 | watcher.sink <- []*blockwatch.Event{blockEvent} 63 | time.Sleep(2 * time.Millisecond) 64 | assert.Equal(lpEth.Orch.ServiceURI, stubStore.serviceURI) 65 | assert.Equal(lpEth.Orch.Address.String(), stubStore.ethereumAddr) 66 | } 67 | -------------------------------------------------------------------------------- /eth/watchers/topics.go: -------------------------------------------------------------------------------- 1 | package watchers 2 | 3 | import ( 4 | "github.com/ethereum/go-ethereum/common" 5 | "github.com/ethereum/go-ethereum/crypto" 6 | ) 7 | 8 | var eventSignatures = []string{ 9 | "Unbond(address,address,uint256,uint256,uint256)", 10 | "Rebond(address,address,uint256,uint256)", 11 | "WithdrawStake(address,uint256,uint256,uint256)", 12 | "NewRound(uint256,bytes32)", 13 | "DepositFunded(address,uint256)", 14 | "ReserveFunded(address,uint256)", 15 | "Withdrawal(address,uint256,uint256)", 16 | "WinningTicketTransfer(address,address,uint256)", 17 | "Unlock(address,uint256,uint256)", 18 | "UnlockCancelled(address)", 19 | "TranscoderActivated(address,uint256)", 20 | "TranscoderDeactivated(address,uint256)", 21 | "ServiceURIUpdate(address,string)", 22 | } 23 | 24 | // FilterTopics returns a list of topics to be used when filtering logs 25 | func FilterTopics() []common.Hash { 26 | topics := make([]common.Hash, len(eventSignatures)) 27 | for i, sig := range eventSignatures { 28 | topics[i] = crypto.Keccak256Hash([]byte(sig)) 29 | } 30 | 31 | return topics 32 | } 33 | -------------------------------------------------------------------------------- /eth/watchers/types.go: -------------------------------------------------------------------------------- 1 | package watchers 2 | 3 | import ( 4 | "github.com/ethereum/go-ethereum/core/types" 5 | "github.com/ethereum/go-ethereum/event" 6 | "github.com/livepeer/go-livepeer/eth/blockwatch" 7 | ) 8 | 9 | type BlockWatcher interface { 10 | Subscribe(sink chan<- []*blockwatch.Event) event.Subscription 11 | GetLatestBlock() (*blockwatch.MiniHeader, error) 12 | } 13 | 14 | type timeWatcher interface { 15 | SubscribeRounds(sink chan<- types.Log) event.Subscription 16 | } 17 | -------------------------------------------------------------------------------- /install_ffmpeg.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | echo 'WARNING: downloading and executing lpms/install_ffmpeg.sh, use it directly in case of issues' 3 | curl https://raw.githubusercontent.com/livepeer/lpms/e1872bf609de6befe3cfcdc7a464d1e3469ea843/install_ffmpeg.sh | bash -s $1 4 | -------------------------------------------------------------------------------- /liveai.openapi.yaml: -------------------------------------------------------------------------------- 1 | definitions: 2 | server.smokeTestRequest: 3 | properties: 4 | duration_secs: 5 | type: integer 6 | stream_url: 7 | type: string 8 | type: object 9 | info: 10 | contact: {} 11 | title: Live Video-To-Video AI 12 | version: 0.0.0 13 | paths: 14 | /live/video-to-video/{stream}/start: 15 | get: 16 | consumes: 17 | - multipart/form-data 18 | parameters: 19 | - description: Stream Key 20 | in: path 21 | name: stream 22 | required: true 23 | type: string 24 | - description: MediaMTX source ID, used for calls back to MediaMTX 25 | in: formData 26 | name: source_id 27 | required: true 28 | type: string 29 | - description: MediaMTX specific source type (webrtcSession/rtmpConn) 30 | in: formData 31 | name: source_type 32 | required: true 33 | type: string 34 | - description: Queryparams from the original ingest URL 35 | in: formData 36 | name: query 37 | required: true 38 | type: string 39 | responses: 40 | "200": 41 | description: OK 42 | summary: Start Live Video 43 | /live/video-to-video/{stream}/status: 44 | get: 45 | parameters: 46 | - description: Stream ID 47 | in: path 48 | name: stream 49 | required: true 50 | type: string 51 | responses: 52 | "200": 53 | description: OK 54 | summary: Get Live Stream Status 55 | /live/video-to-video/{stream}/update: 56 | post: 57 | parameters: 58 | - description: Stream Key 59 | in: path 60 | name: stream 61 | required: true 62 | type: string 63 | - description: update request 64 | in: body 65 | name: params 66 | required: true 67 | schema: 68 | type: string 69 | responses: 70 | "200": 71 | description: OK 72 | summary: Update Live Stream 73 | /live/video-to-video/smoketest: 74 | put: 75 | parameters: 76 | - description: smoke test request 77 | in: body 78 | name: request 79 | required: true 80 | schema: 81 | $ref: '#/definitions/server.smokeTestRequest' 82 | responses: 83 | "200": 84 | description: OK 85 | summary: Start Smoke Test 86 | swagger: "2.0" 87 | -------------------------------------------------------------------------------- /media/mediamtx.go: -------------------------------------------------------------------------------- 1 | package media 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "time" 10 | 11 | "github.com/livepeer/go-livepeer/clog" 12 | ) 13 | 14 | type MediaMTXClient struct { 15 | host string 16 | apiPassword string 17 | sourceID string 18 | sourceType string 19 | } 20 | 21 | func NewMediaMTXClient(host, apiPassword, sourceID, sourceType string) *MediaMTXClient { 22 | return &MediaMTXClient{ 23 | host: host, 24 | apiPassword: apiPassword, 25 | sourceID: sourceID, 26 | sourceType: sourceType, 27 | } 28 | } 29 | 30 | const ( 31 | mediaMTXControlPort = "9997" 32 | mediaMTXControlTimeout = 30 * time.Second 33 | mediaMTXControlUser = "admin" 34 | MediaMTXWebrtcSession = "webrtcsession" 35 | MediaMTXRtmpConn = "rtmpconn" 36 | ) 37 | 38 | func MediamtxSourceTypeToString(s string) (string, error) { 39 | switch s { 40 | case MediaMTXWebrtcSession: 41 | return "whip", nil 42 | case MediaMTXRtmpConn: 43 | return "rtmp", nil 44 | default: 45 | return "", errors.New("unknown media source") 46 | } 47 | } 48 | 49 | func getApiPath(sourceType string) (string, error) { 50 | var apiPath string 51 | switch sourceType { 52 | case MediaMTXWebrtcSession: 53 | apiPath = "webrtcsessions" 54 | case MediaMTXRtmpConn: 55 | apiPath = "rtmpconns" 56 | default: 57 | return "", fmt.Errorf("invalid sourceType: %s", sourceType) 58 | } 59 | return apiPath, nil 60 | } 61 | 62 | func (mc *MediaMTXClient) KickInputConnection(ctx context.Context) error { 63 | clog.V(8).Infof(ctx, "Kicking mediamtx input connection") 64 | apiPath, err := getApiPath(mc.sourceType) 65 | if err != nil { 66 | return err 67 | } 68 | 69 | ctx, cancel := context.WithTimeout(context.Background(), mediaMTXControlTimeout) 70 | defer cancel() 71 | req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("http://%s:%s/v3/%s/kick/%s", mc.host, mediaMTXControlPort, apiPath, mc.sourceID), nil) 72 | if err != nil { 73 | return fmt.Errorf("failed to create kick request: %w", err) 74 | } 75 | req.SetBasicAuth(mediaMTXControlUser, mc.apiPassword) 76 | resp, err := http.DefaultClient.Do(req) 77 | if err != nil { 78 | return fmt.Errorf("failed to kick connection: %w", err) 79 | } 80 | defer resp.Body.Close() 81 | if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusBadRequest { 82 | body, _ := io.ReadAll(resp.Body) 83 | return fmt.Errorf("kick connection failed with status code: %d body: %s", resp.StatusCode, body) 84 | } 85 | return nil 86 | } 87 | 88 | func (mc *MediaMTXClient) StreamExists() (bool, error) { 89 | apiPath, err := getApiPath(mc.sourceType) 90 | if err != nil { 91 | return false, err 92 | } 93 | ctx, cancel := context.WithTimeout(context.Background(), mediaMTXControlTimeout) 94 | defer cancel() 95 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://%s:%s/v3/%s/get/%s", mc.host, mediaMTXControlPort, apiPath, mc.sourceID), nil) 96 | if err != nil { 97 | return false, fmt.Errorf("failed to create get stream request: %w", err) 98 | } 99 | req.SetBasicAuth(mediaMTXControlUser, mc.apiPassword) 100 | resp, err := http.DefaultClient.Do(req) 101 | if err != nil { 102 | return false, fmt.Errorf("failed to get stream: %w", err) 103 | } 104 | defer resp.Body.Close() 105 | if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusBadRequest { 106 | body, _ := io.ReadAll(resp.Body) 107 | return false, fmt.Errorf("get stream failed with status code: %d body: %s", resp.StatusCode, body) 108 | } 109 | return true, nil 110 | } 111 | -------------------------------------------------------------------------------- /media/rtmp2segment_windows.go: -------------------------------------------------------------------------------- 1 | //go:build windows 2 | 3 | package media 4 | 5 | import "context" 6 | 7 | type MediaSegmenter struct { 8 | Workdir string 9 | MediaMTXClient *MediaMTXClient 10 | } 11 | 12 | func (ms *MediaSegmenter) RunSegmentation(ctx context.Context, in string, segmentHandler SegmentHandler) { 13 | // Not supported for Windows 14 | } 15 | 16 | func StartFileCleanup(ctx context.Context, workDir string) { 17 | // Not supported for Windows 18 | } 19 | -------------------------------------------------------------------------------- /media/rw.go: -------------------------------------------------------------------------------- 1 | package media 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "log/slog" 7 | "sync" 8 | ) 9 | 10 | type CloneableReader interface { 11 | io.Reader 12 | Clone() CloneableReader 13 | } 14 | 15 | type MediaWriter struct { 16 | mu *sync.Mutex 17 | cond *sync.Cond 18 | buffer *bytes.Buffer 19 | closed bool 20 | } 21 | 22 | type MediaReader struct { 23 | source *MediaWriter 24 | readPos int 25 | } 26 | 27 | func NewMediaWriter() *MediaWriter { 28 | mu := &sync.Mutex{} 29 | return &MediaWriter{ 30 | buffer: new(bytes.Buffer), 31 | cond: sync.NewCond(mu), 32 | mu: mu, 33 | } 34 | } 35 | 36 | func (mw *MediaWriter) Write(data []byte) (int, error) { 37 | mw.mu.Lock() 38 | defer mw.mu.Unlock() 39 | 40 | // Write to buffer 41 | n, err := mw.buffer.Write(data) 42 | 43 | // Signal waiting readers 44 | mw.cond.Broadcast() 45 | 46 | return n, err 47 | } 48 | 49 | func (mw *MediaWriter) readData(startPos int) ([]byte, bool) { 50 | mw.mu.Lock() 51 | defer mw.mu.Unlock() 52 | for { 53 | totalLen := mw.buffer.Len() 54 | if startPos < totalLen { 55 | data := mw.buffer.Bytes()[startPos:totalLen] 56 | return data, mw.closed 57 | } 58 | if startPos > totalLen { 59 | slog.Info("Invalid start pos, invoking eof") 60 | return nil, true 61 | } 62 | if mw.closed { 63 | return nil, true 64 | } 65 | // Wait for new data 66 | mw.cond.Wait() 67 | } 68 | } 69 | 70 | func (mw *MediaWriter) Close() { 71 | if mw == nil { 72 | return // sometimes happens, weird 73 | } 74 | mw.mu.Lock() 75 | defer mw.mu.Unlock() 76 | if !mw.closed { 77 | mw.closed = true 78 | mw.cond.Broadcast() 79 | } 80 | } 81 | 82 | func (mw *MediaWriter) MakeReader() CloneableReader { 83 | return &MediaReader{ 84 | source: mw, 85 | } 86 | } 87 | 88 | func (mr *MediaReader) Read(p []byte) (int, error) { 89 | data, eof := mr.source.readData(mr.readPos) 90 | toRead := len(p) 91 | if len(data) <= toRead { 92 | toRead = len(data) 93 | } else { 94 | // there is more data to read 95 | eof = false 96 | } 97 | 98 | copy(p, data[:toRead]) 99 | mr.readPos += toRead 100 | 101 | var err error = nil 102 | if eof { 103 | err = io.EOF 104 | } 105 | 106 | return toRead, err 107 | } 108 | 109 | func (mr *MediaReader) Clone() CloneableReader { 110 | return &MediaReader{ 111 | source: mr.source, 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /media/segment_reader.go: -------------------------------------------------------------------------------- 1 | package media 2 | 3 | import ( 4 | "io" 5 | "sync" 6 | ) 7 | 8 | type SegmentHandler func(reader CloneableReader) 9 | 10 | func NoopReader(reader CloneableReader) { 11 | // don't have to do anything here 12 | } 13 | 14 | type EOSReader struct{} 15 | 16 | func (r *EOSReader) Read(p []byte) (n int, err error) { 17 | return 0, io.EOF 18 | } 19 | func (r *EOSReader) Clone() CloneableReader { 20 | return r 21 | } 22 | 23 | type SwitchableSegmentReader struct { 24 | mu sync.RWMutex 25 | reader SegmentHandler 26 | seg CloneableReader 27 | } 28 | 29 | func NewSwitchableSegmentReader() *SwitchableSegmentReader { 30 | return &SwitchableSegmentReader{ 31 | reader: NoopReader, 32 | } 33 | } 34 | 35 | func (sr *SwitchableSegmentReader) SwitchReader(newReader SegmentHandler) { 36 | sr.mu.Lock() 37 | defer sr.mu.Unlock() 38 | sr.reader = newReader 39 | if sr.seg != nil { 40 | // immediately send the current segment instead of waiting for the next one 41 | // clone since current segment may have already been partially consumed 42 | sr.reader(sr.seg.Clone()) 43 | } 44 | } 45 | 46 | func (sr *SwitchableSegmentReader) Read(reader CloneableReader) { 47 | sr.mu.Lock() 48 | defer sr.mu.Unlock() 49 | sr.reader(reader) 50 | sr.seg = reader 51 | } 52 | 53 | func (sr *SwitchableSegmentReader) Close() { 54 | sr.mu.RLock() 55 | defer sr.mu.RUnlock() 56 | sr.reader(&EOSReader{}) 57 | } 58 | -------------------------------------------------------------------------------- /media/select_darwin.go: -------------------------------------------------------------------------------- 1 | //go:build darwin 2 | 3 | package media 4 | 5 | import "syscall" 6 | 7 | func crossPlatformSelect(nfd int, r, w, e *syscall.FdSet, timeout *syscall.Timeval) (int, error) { 8 | // On macOS, syscall.Select only returns an error 9 | err := syscall.Select(nfd, r, w, e, timeout) 10 | if err != nil { 11 | return -1, err // Return -1 in case of an error 12 | } 13 | // We need to manually count the number of ready descriptors in FdSets 14 | n := 0 15 | if r != nil { 16 | n += countReadyDescriptors(r, nfd) 17 | } 18 | if w != nil { 19 | n += countReadyDescriptors(w, nfd) 20 | } 21 | if e != nil { 22 | n += countReadyDescriptors(e, nfd) 23 | } 24 | return n, nil 25 | 26 | } 27 | 28 | // countReadyDescriptors manually counts the number of ready file descriptors in an FdSet 29 | func countReadyDescriptors(set *syscall.FdSet, nfd int) int { 30 | count := 0 31 | for fd := 0; fd < nfd; fd++ { 32 | if isSet(fd, set) { 33 | count++ 34 | } 35 | } 36 | return count 37 | } 38 | 39 | // isSet checks if a file descriptor is set in an FdSet 40 | func isSet(fd int, set *syscall.FdSet) bool { 41 | return set.Bits[fd/64]&(1<<(uint(fd)%64)) != 0 42 | } 43 | -------------------------------------------------------------------------------- /media/select_linux.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | 3 | package media 4 | 5 | import "syscall" 6 | 7 | func crossPlatformSelect(nfd int, r, w, e *syscall.FdSet, timeout *syscall.Timeval) (int, error) { 8 | return syscall.Select(nfd, r, w, e, timeout) 9 | } 10 | -------------------------------------------------------------------------------- /net/redeemer.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | import "net/lp_rpc.proto"; 3 | 4 | package net; 5 | option go_package = "./net"; 6 | 7 | service TicketRedeemer { 8 | rpc QueueTicket(Ticket) returns (QueueTicketRes) {} 9 | rpc MaxFloat(MaxFloatReq) returns (MaxFloatUpdate) {} 10 | rpc MonitorMaxFloat(MaxFloatReq) returns (stream MaxFloatUpdate) {} 11 | } 12 | 13 | message Ticket { 14 | TicketParams ticket_params = 1; 15 | bytes sender = 2; 16 | TicketExpirationParams expiration_params = 3; 17 | TicketSenderParams sender_params = 4; 18 | bytes recipient_rand = 5; 19 | } 20 | 21 | message QueueTicketRes {} 22 | 23 | message MaxFloatReq { 24 | bytes sender = 1; 25 | } 26 | 27 | message MaxFloatUpdate { 28 | bytes max_float = 1; 29 | } 30 | -------------------------------------------------------------------------------- /net/redeemer_mock.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: net/redeemer.pb.go 3 | 4 | // Package net is a generated GoMock package. 5 | package net 6 | -------------------------------------------------------------------------------- /pm/assets/diagram.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:d949ddab0fdb2e6ae6c14fc89e4f81edae57a4b0eaaeb6559b1c962306c685ee 3 | size 54765 4 | -------------------------------------------------------------------------------- /pm/assets/diagram.txt: -------------------------------------------------------------------------------- 1 | title Sending & Receiving Tickets 2 | 3 | participant ChangeMe1 4 | Recipient->Sender: ticketParams 5 | 6 | note over Sender: StartSession(ticketParams) 7 | 8 | note over Sender: tickets = CreateTicketBatch(n) 9 | 10 | Sender -> Recipient: n tickets 11 | 12 | note over Recipient: ReceiveTicket(ticket) for all n tickets 13 | 14 | note over Recipient: RedeemWinningTicket(ticket) for all winning tickets 15 | 16 | note over Sender: tickets = CreateTicketBatch(n) 17 | 18 | Sender -> Recipient: n tickets 19 | 20 | note over Recipient: ReceiveTicket(ticket) for all n tickets 21 | 22 | note over Recipient: RedeemWinningTicket(ticket) for all winning tickets -------------------------------------------------------------------------------- /pm/helpers.go: -------------------------------------------------------------------------------- 1 | package pm 2 | 3 | import ( 4 | "math/rand" 5 | 6 | ethcommon "github.com/ethereum/go-ethereum/common" 7 | ) 8 | 9 | // RandHash returns a random keccak256 hash 10 | func RandHash() ethcommon.Hash { 11 | return ethcommon.BytesToHash(RandBytes(32)) 12 | } 13 | 14 | // RandAddress returns a random ETH address 15 | func RandAddress() ethcommon.Address { 16 | return ethcommon.BytesToAddress(RandBytes(addressSize)) 17 | } 18 | 19 | // RandBytes returns a slice of random bytes with the size specified by the caller 20 | func RandBytes(size uint) []byte { 21 | x := make([]byte, size, size) 22 | for i := 0; i < len(x); i++ { 23 | x[i] = byte(rand.Uint32()) 24 | } 25 | return x 26 | } 27 | -------------------------------------------------------------------------------- /pm/signer.go: -------------------------------------------------------------------------------- 1 | package pm 2 | 3 | import "github.com/ethereum/go-ethereum/accounts" 4 | 5 | // Signer supports identifying as an Ethereum account owner, by providing the 6 | // Account and enabling message signing. 7 | type Signer interface { 8 | Sign(msg []byte) ([]byte, error) 9 | Account() accounts.Account 10 | } 11 | -------------------------------------------------------------------------------- /pm/sigverifier.go: -------------------------------------------------------------------------------- 1 | package pm 2 | 3 | import ( 4 | ethcommon "github.com/ethereum/go-ethereum/common" 5 | "github.com/livepeer/go-livepeer/crypto" 6 | ) 7 | 8 | // SigVerifier is an interface which describes an object capable 9 | // of verification of ECDSA signatures produced by ETH addresses 10 | type SigVerifier interface { 11 | // Verify checks if a provided signature over a message 12 | // is valid for a given ETH address 13 | Verify(addr ethcommon.Address, msg, sig []byte) bool 14 | } 15 | 16 | // DefaultSigVerifier is client-side-only implementation of sig verification, i.e. not relying on 17 | // any smart contract inputs. 18 | type DefaultSigVerifier struct { 19 | } 20 | 21 | // Verify checks if a provided signature over a message 22 | // is valid for a given ETH address 23 | func (sv *DefaultSigVerifier) Verify(addr ethcommon.Address, msg, sig []byte) bool { 24 | return crypto.VerifySig(addr, msg, sig) 25 | } 26 | 27 | // ApprovedSigVerifier is an implementation of the SigVerifier interface 28 | // that relies on an implementation of the Broker interface to provide a registry 29 | // mapping ETH addresses to approved signer sets. This implementation will 30 | // recover a ETH address from a signature and check if the recovered address 31 | // is approved 32 | // type ApprovedSigVerifier struct { 33 | // broker Broker 34 | // } 35 | 36 | // // NewApprovedSigVerifier returns an instance of an approved signature verifier 37 | // func NewApprovedSigVerifier(broker Broker) *ApprovedSigVerifier { 38 | // return &ApprovedSigVerifier{ 39 | // broker: broker, 40 | // } 41 | // } 42 | 43 | // // Verify checks if a provided signature over a message 44 | // // is valid for a given ETH address 45 | // func (sv *ApprovedSigVerifier) Verify(addr ethcommon.Address, msg, sig []byte) bool { 46 | // personalMsg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", 32, msg) 47 | // personalHash := crypto.Keccak256([]byte(personalMsg)) 48 | 49 | // pubkey, err := crypto.SigToPub(personalHash, sig) 50 | // if err != nil { 51 | // return false 52 | // } 53 | 54 | // rec := crypto.PubkeyToAddress(*pubkey) 55 | 56 | // if addr == rec { 57 | // // If recovered address matches, return early 58 | // return true 59 | // } 60 | 61 | // approved, err := sv.broker.IsApprovedSigner(addr, rec) 62 | // if err != nil { 63 | // return false 64 | // } 65 | 66 | // return approved 67 | // } 68 | -------------------------------------------------------------------------------- /pm/sigverifier_test.go: -------------------------------------------------------------------------------- 1 | package pm 2 | 3 | // func TestVerify(t *testing.T) { 4 | // msg := []byte("foo") 5 | // personalMsg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", 32, msg) 6 | // personalHash := crypto.Keccak256([]byte(personalMsg)) 7 | 8 | // senderPrivKey, err := crypto.GenerateKey() 9 | // if err != nil { 10 | // t.Fatal(err) 11 | // } 12 | 13 | // sender := crypto.PubkeyToAddress(senderPrivKey.PublicKey) 14 | 15 | // senderSig, err := crypto.Sign(personalHash, senderPrivKey) 16 | // if err != nil { 17 | // t.Fatal(err) 18 | // } 19 | 20 | // unapprovedSignerPrivKey, err := crypto.GenerateKey() 21 | // if err != nil { 22 | // t.Fatal(err) 23 | // } 24 | 25 | // unapprovedSignerSig, err := crypto.Sign(personalHash, unapprovedSignerPrivKey) 26 | 27 | // b := newStubBroker() 28 | 29 | // sv := NewApprovedSigVerifier(b) 30 | 31 | // // Test invalid signature for non-approved signer 32 | // if valid := sv.Verify(sender, msg, unapprovedSignerSig); valid { 33 | // t.Error("expected invalid signature for non-approved signer") 34 | // } 35 | 36 | // // Test valid signature for approved signer 37 | // approvedSignerPrivKey, err := crypto.GenerateKey() 38 | // if err != nil { 39 | // t.Fatal(err) 40 | // } 41 | 42 | // approvedSigner := crypto.PubkeyToAddress(approvedSignerPrivKey.PublicKey) 43 | 44 | // approvedSignerSig, err := crypto.Sign(personalHash, approvedSignerPrivKey) 45 | // if err != nil { 46 | // t.Fatal(err) 47 | // } 48 | 49 | // // Approve signer 50 | // b.ApproveSigners([]ethcommon.Address{approvedSigner}) 51 | 52 | // if valid := sv.Verify(sender, msg, approvedSignerSig); !valid { 53 | // t.Error("expected valid signature for approved signer") 54 | // } 55 | 56 | // // Test valid signature for sender 57 | // if valid := sv.Verify(sender, msg, senderSig); !valid { 58 | // t.Error("expected valid signature for sender") 59 | // } 60 | // } 61 | -------------------------------------------------------------------------------- /pm/ticketstore.go: -------------------------------------------------------------------------------- 1 | package pm 2 | 3 | import ( 4 | ethcommon "github.com/ethereum/go-ethereum/common" 5 | "math/big" 6 | ) 7 | 8 | // TicketStore is an interface which describes an object capable 9 | // of persisting tickets 10 | type TicketStore interface { 11 | // SelectEarliestWinningTicket selects the earliest stored winning ticket for a 'sender' 12 | // which is not yet redeemed 13 | SelectEarliestWinningTicket(sender ethcommon.Address, minCreationRound int64) (*SignedTicket, error) 14 | 15 | // RemoveWinningTicket removes a ticket 16 | RemoveWinningTicket(ticket *SignedTicket) error 17 | 18 | // StoreWinningTicket stores a signed ticket 19 | StoreWinningTicket(ticket *SignedTicket) error 20 | 21 | // MarkWinningTicketRedeemed stores the on-chain transaction hash and timestamp of redemption 22 | // This marks the ticket as being 'redeemed' 23 | MarkWinningTicketRedeemed(ticket *SignedTicket, txHash ethcommon.Hash) error 24 | 25 | // WinningTicketCount returns the amount of non-redeemed winning tickets for a sender in the TicketStore 26 | WinningTicketCount(sender ethcommon.Address, minCreationRound int64) (int, error) 27 | 28 | // IsOrchActive returns true if the given orchestrator addr is active in the given round 29 | IsOrchActive(addr ethcommon.Address, round *big.Int) (bool, error) 30 | } 31 | -------------------------------------------------------------------------------- /pm/validator.go: -------------------------------------------------------------------------------- 1 | package pm 2 | 3 | import ( 4 | "math/big" 5 | 6 | ethcommon "github.com/ethereum/go-ethereum/common" 7 | "github.com/ethereum/go-ethereum/crypto" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | var ( 12 | errInvalidTicketRecipient = errors.New("invalid ticket recipient") 13 | errInvalidTicketSender = errors.New("invalid ticket sender") 14 | errInvalidTicketRecipientRand = errors.New("invalid recipientRand for ticket recipientRandHash") 15 | errInvalidTicketSignature = errors.New("invalid ticket signature") 16 | errInvalidCreationRound = errors.New("invalid ticket creation round") 17 | errInvalidCreationRoundBlockHash = errors.New("invalid ticket creation round block hash") 18 | errIsUsedTicket = errors.New("ticket already used") 19 | ) 20 | 21 | // Validator is an interface which describes an object capable 22 | // of validating tickets 23 | type Validator interface { 24 | // ValidateTicket checks if a ticket is valid 25 | ValidateTicket(recipient ethcommon.Address, ticket *Ticket, sig []byte, recipientRand *big.Int) error 26 | 27 | // IsWinningTicket checks if a ticket won 28 | // Note: This method does not check if a ticket is valid which is done using ValidateTicket 29 | IsWinningTicket(ticket *Ticket, sig []byte, recipientRand *big.Int) bool 30 | } 31 | 32 | // validator is an implementation of the Validator interface 33 | type validator struct { 34 | sigVerifier SigVerifier 35 | tm TimeManager 36 | } 37 | 38 | // NewValidator returns an instance of a validator 39 | func NewValidator(sigVerifier SigVerifier, tm TimeManager) Validator { 40 | return &validator{ 41 | sigVerifier: sigVerifier, 42 | tm: tm, 43 | } 44 | } 45 | 46 | // ValidateTicket checks if a ticket is valid 47 | func (v *validator) ValidateTicket(recipient ethcommon.Address, ticket *Ticket, sig []byte, recipientRand *big.Int) error { 48 | if ticket.Recipient != recipient { 49 | return errInvalidTicketRecipient 50 | } 51 | 52 | if (ticket.Sender == ethcommon.Address{}) { 53 | return errInvalidTicketSender 54 | } 55 | 56 | if crypto.Keccak256Hash(ethcommon.LeftPadBytes(recipientRand.Bytes(), uint256Size)) != ticket.RecipientRandHash { 57 | return errInvalidTicketRecipientRand 58 | } 59 | 60 | if !v.sigVerifier.Verify(ticket.Sender, ticket.Hash().Bytes(), sig) { 61 | return errInvalidTicketSignature 62 | } 63 | 64 | return nil 65 | } 66 | 67 | // IsWinningTicket checks if a ticket won 68 | // Note: This method does not check if a ticket is valid which is done using IsValidTicket 69 | // A ticket wins if: 70 | // H(SIG(H(T)), T.RecipientRand) < T.WinProb 71 | func (v *validator) IsWinningTicket(ticket *Ticket, sig []byte, recipientRand *big.Int) bool { 72 | recipientRandBytes := ethcommon.LeftPadBytes(recipientRand.Bytes(), bytes32Size) 73 | res := new(big.Int).SetBytes(crypto.Keccak256(sig, recipientRandBytes)) 74 | 75 | return res.Cmp(ticket.WinProb) < 0 76 | } 77 | -------------------------------------------------------------------------------- /prepare_mingw64.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eo pipefail 4 | set -x 5 | 6 | DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 7 | 8 | export MINGW_INSTALLS=mingw64 9 | pacman -S --noconfirm --noprogressbar --ask=20 --needed mingw-w64-x86_64-binutils mingw-w64-x86_64-gcc mingw-w64-x86_64-pkg-config mingw-w64-x86_64-go mingw-w64-x86_64-nasm mingw-w64-x86_64-clang git make autoconf automake patch libtool texinfo gtk-doc zip 10 | gpg --keyserver keyserver.ubuntu.com --recv-keys F3599FF828C67298 249B39D24F25E3B6 2071B08A33BD3F06 29EE58B996865171 D605848ED7E69871 11 | 12 | echo "removing all dlls" 13 | # https://narkive.com/Fjlrbrjg:20.646.48 14 | find /mingw64 -name "*.dll.a" -exec rm -v {} \; 15 | -------------------------------------------------------------------------------- /print_version.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Prints the current build version of go-livepeer. Based on the following considerations: 4 | # 1. If the VERSION file matches the tag of our current commit, then we're the canonical 5 | # x.y.z version, and that will be our version string. 6 | # 2. Otherwise, our version string is x.y.z-SHA, provided by the VERSION file and 7 | # git describe --always --long --abbrev=8 --dirty 8 | 9 | set -e 10 | set -o nounset 11 | 12 | DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 13 | 14 | currentTag="$(git describe --tags)" 15 | currentVersion="$(cat "$DIR/VERSION")" 16 | currentSha="$(git describe --always --long --dirty --abbrev=8)" 17 | 18 | if [[ "$currentTag" == "v$currentVersion" ]]; then 19 | echo -en "$currentVersion" 20 | else 21 | echo -en "$currentVersion-$currentSha" 22 | fi 23 | -------------------------------------------------------------------------------- /server/ai_pipeline_status.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | type streamStatusStore struct { 8 | store map[string]map[string]interface{} 9 | mu sync.RWMutex 10 | } 11 | 12 | var StreamStatusStore = streamStatusStore{store: make(map[string]map[string]interface{})} 13 | var GatewayStatus = streamStatusStore{store: make(map[string]map[string]interface{})} 14 | 15 | // StoreStreamStatus updates the status for a stream 16 | func (s *streamStatusStore) Store(streamID string, status map[string]interface{}) { 17 | s.mu.Lock() 18 | s.store[streamID] = status 19 | s.mu.Unlock() 20 | } 21 | 22 | // ClearStreamStatus removes a stream's status from the store 23 | func (s *streamStatusStore) Clear(streamID string) { 24 | s.mu.Lock() 25 | delete(s.store, streamID) 26 | s.mu.Unlock() 27 | } 28 | 29 | // GetStreamStatus returns the current status for a stream 30 | func (s *streamStatusStore) Get(streamID string) (map[string]interface{}, bool) { 31 | s.mu.RLock() 32 | defer s.mu.RUnlock() 33 | status, exists := s.store[streamID] 34 | return status, exists 35 | } 36 | 37 | // StoreIfNotExists stores a status only if the streamID doesn't already exist or keyToCheck does not exist on the status 38 | func (s *streamStatusStore) StoreIfNotExists(streamID string, key string, status map[string]interface{}) { 39 | s.mu.Lock() 40 | defer s.mu.Unlock() 41 | existing, exists := s.store[streamID] 42 | if !exists { 43 | s.storeKey(streamID, key, status) 44 | return 45 | } 46 | if _, ok := existing[key]; !ok { 47 | s.storeKey(streamID, key, status) 48 | } 49 | } 50 | 51 | func (s *streamStatusStore) StoreKey(streamID, key string, status map[string]interface{}) { 52 | s.mu.Lock() 53 | defer s.mu.Unlock() 54 | s.storeKey(streamID, key, status) 55 | } 56 | 57 | func (s *streamStatusStore) storeKey(streamID, key string, status map[string]interface{}) { 58 | if _, ok := s.store[streamID]; !ok { 59 | s.store[streamID] = make(map[string]interface{}) 60 | } 61 | s.store[streamID][key] = status 62 | } 63 | -------------------------------------------------------------------------------- /server/cert.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "crypto/ecdsa" 5 | "crypto/elliptic" 6 | "crypto/rand" 7 | "crypto/x509" 8 | "encoding/pem" 9 | "math/big" 10 | "net/url" 11 | "os" 12 | "path/filepath" 13 | "time" 14 | 15 | "github.com/golang/glog" 16 | ) 17 | 18 | const certExpiry = 8765 * time.Hour // One year 19 | 20 | func genCert(host string, priv *ecdsa.PrivateKey) ([]byte, error) { 21 | serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) 22 | glog.Info("Generating cert for ", host) 23 | if err != nil { 24 | glog.Error("Could not generate serial ", err) 25 | return []byte{}, err 26 | } 27 | tmpl := x509.Certificate{ 28 | SerialNumber: serial, 29 | NotBefore: time.Now(), 30 | NotAfter: time.Now().Add(certExpiry), // XXX fix fix fix 31 | KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, 32 | ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, 33 | DNSNames: []string{host}, 34 | BasicConstraintsValid: true, 35 | IsCA: true, 36 | } 37 | cert, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv) 38 | if err != nil { 39 | glog.Error("Could not create certificate ", err) 40 | return []byte{}, err 41 | } 42 | return cert, nil 43 | } 44 | 45 | func genKey() (*ecdsa.PrivateKey, []byte, error) { 46 | key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) 47 | if err != nil { 48 | glog.Error("Unable to generate private key ", err) 49 | return nil, []byte{}, err 50 | } 51 | keyBytes, err := x509.MarshalECPrivateKey(key) 52 | if err != nil { 53 | glog.Error("Unable to marshal EC private key ", err) 54 | return nil, []byte{}, err 55 | } 56 | return key, keyBytes, nil 57 | } 58 | 59 | func writeFile(fname string, desc string, contents []byte) error { 60 | file, err := os.Create(fname) 61 | defer file.Close() 62 | if err != nil { 63 | glog.Errorf("Unable to create file %v: %v", fname, err) 64 | return err 65 | } 66 | err = pem.Encode(file, &pem.Block{Type: desc, Bytes: contents}) 67 | if err != nil { 68 | glog.Errorf("Unable to pem-encode %v: %v", fname, err) 69 | return err 70 | } 71 | return nil 72 | } 73 | 74 | func getCert(uri *url.URL, workDir string) (string, string, error) { 75 | // if cert doesn't exist, generate a selfsigned cert 76 | certFile := filepath.Join(workDir, "cert.pem") 77 | keyFile := filepath.Join(workDir, "key.pem") 78 | _, certErr := os.Stat(certFile) 79 | _, keyErr := os.Stat(keyFile) 80 | //if os.IsNotExist(certErr) || os.IsNotExist(keyErr) { 81 | // XXX for now, just generate a new cert every time. 82 | if true { 83 | glog.Info("Private key and cert not found. Generating") 84 | key, keyBytes, err := genKey() 85 | if err != nil { 86 | return "", "", err 87 | } 88 | err = writeFile(keyFile, "EC PRIVATE KEY", keyBytes) 89 | if err != nil { 90 | return "", "", err 91 | } 92 | cert, err := genCert(uri.Hostname(), key) 93 | if err != nil { 94 | return "", "", err 95 | } 96 | err = writeFile(certFile, "CERTIFICATE", cert) 97 | if err != nil { 98 | return "", "", err 99 | } 100 | } else if certErr != nil || keyErr != nil { 101 | glog.Error("Problem getting key/cert ", certErr, keyErr) 102 | err := certErr 103 | if keyErr != nil { 104 | err = keyErr 105 | } 106 | return "", "", err 107 | } 108 | return certFile, keyFile, nil 109 | } 110 | -------------------------------------------------------------------------------- /server/cert_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha1" 6 | "crypto/tls" 7 | "crypto/x509" 8 | "io" 9 | "net/url" 10 | "os" 11 | "testing" 12 | ) 13 | 14 | func sha1sum(fname string) ([]byte, error) { 15 | f, err := os.Open(fname) 16 | if err != nil { 17 | return []byte{}, err 18 | } 19 | defer f.Close() 20 | h := sha1.New() 21 | if _, err := io.Copy(h, f); err != nil { 22 | return []byte{}, err 23 | } 24 | return h.Sum(nil), nil 25 | } 26 | 27 | func sha1sums(t *testing.T, cert, key string) ([]byte, []byte, error) { 28 | kh, err := sha1sum(key) 29 | if err != nil { 30 | t.Error("Could not sha1 keyfile", err) 31 | return []byte{}, []byte{}, err 32 | } 33 | ch, err := sha1sum(cert) 34 | if err != nil { 35 | t.Error("Could not sha1 certfile", err) 36 | return []byte{}, []byte{}, err 37 | } 38 | return ch, kh, nil 39 | } 40 | 41 | func TestRPCCert(t *testing.T) { 42 | url, _ := url.Parse("https://livepeer.org") 43 | wd := t.TempDir() 44 | cf, kf, err := getCert(url, wd) 45 | if err != nil { 46 | t.Error("Could not get cert/key ", err) 47 | return 48 | } 49 | ch, kh, err := sha1sums(t, cf, kf) 50 | if err != nil { 51 | return 52 | } 53 | 54 | // ensure that the cert is valid and contains data we expect 55 | tlsCert, err := tls.LoadX509KeyPair(cf, kf) 56 | if err != nil { 57 | t.Error("Could not load cert/key pair", err) 58 | return 59 | } 60 | cert, err := x509.ParseCertificate(tlsCert.Certificate[0]) 61 | if err != nil { 62 | t.Error("Could not parse x509 cert", err) 63 | return 64 | } 65 | if cert.DNSNames[0] != url.Hostname() { 66 | t.Error("Cert did not have expected DNS name") 67 | return 68 | } 69 | 70 | // ensure that when invoking again, the same cert is returned 71 | /* XXX re-enable when we make certs persistent 72 | cf, kf, err = getCert(url, wd) 73 | if err != nil { 74 | t.Error("Could not get cert/key ", err) 75 | return 76 | } 77 | ch1, kh1, err := sha1sums(t, cf, kf) 78 | if err != nil { 79 | return 80 | } 81 | if !bytes.Equal(kh, kh1) || !bytes.Equal(ch, ch1) { 82 | t.Error("Mismatched cert checksum") 83 | return 84 | }*/ 85 | 86 | // ensure that when a cert is missing, a new key/cert is generated 87 | err = os.Remove(cf) 88 | if err != nil { 89 | t.Error(err) 90 | return 91 | } 92 | cf, kf, err = getCert(url, wd) 93 | if err != nil { 94 | t.Error("Could not get cert/key", err) 95 | return 96 | } 97 | ch2, kh2, err := sha1sums(t, cf, kf) 98 | if err != nil { 99 | return 100 | } 101 | if bytes.Equal(kh, kh2) || bytes.Equal(ch, ch2) { 102 | t.Error("Matched cert checksum") 103 | } 104 | 105 | // ensure when a key is missing, a new key/cert is generated 106 | err = os.Remove(kf) 107 | if err != nil { 108 | t.Error(err) 109 | return 110 | } 111 | cf, kf, err = getCert(url, wd) 112 | if err != nil { 113 | t.Error("Could not get cert/key", err) 114 | return 115 | } 116 | ch3, kh3, err := sha1sums(t, cf, kf) 117 | if err != nil { 118 | return 119 | } 120 | if bytes.Equal(kh, kh3) || bytes.Equal(ch, ch3) { 121 | t.Error("Matched cert checksum") 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /server/push_webhook_test.go: -------------------------------------------------------------------------------- 1 | //go:build !race 2 | 3 | package server 4 | 5 | import ( 6 | "encoding/json" 7 | "github.com/golang/glog" 8 | "github.com/stretchr/testify/assert" 9 | "io/ioutil" 10 | "net/http" 11 | "net/http/httptest" 12 | "testing" 13 | ) 14 | 15 | func TestPush_WebhookRequestURL(t *testing.T) { 16 | assert := assert.New(t) 17 | 18 | // wait for any earlier tests to complete 19 | assert.True(wgWait(&pushResetWg), "timed out waiting for earlier tests") 20 | 21 | s, cancel := setupServerWithCancel() 22 | defer serverCleanup(s) 23 | defer cancel() 24 | 25 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 26 | out, _ := ioutil.ReadAll(r.Body) 27 | var req authWebhookReq 28 | err := json.Unmarshal(out, &req) 29 | if err != nil { 30 | glog.Error("Error parsing URL: ", err) 31 | w.WriteHeader(http.StatusForbidden) 32 | return 33 | } 34 | assert.Equal(req.URL, "http://example.com/live/seg.ts") 35 | w.Write(nil) 36 | })) 37 | 38 | defer ts.Close() 39 | 40 | oldURL := AuthWebhookURL 41 | defer func() { AuthWebhookURL = oldURL }() 42 | AuthWebhookURL = mustParseUrl(t, ts.URL) 43 | handler, reader, w := requestSetup(s) 44 | req := httptest.NewRequest("POST", "/live/seg.ts", reader) 45 | handler.ServeHTTP(w, req) 46 | resp := w.Result() 47 | defer resp.Body.Close() 48 | 49 | // Server has empty sessions list, so it will return 503 50 | assert.Equal(503, resp.StatusCode) 51 | } 52 | -------------------------------------------------------------------------------- /server/stub.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "github.com/livepeer/go-livepeer/net" 5 | ) 6 | 7 | type StubCapabilityComparator struct { 8 | NetCaps *net.Capabilities 9 | IsLegacy bool 10 | } 11 | 12 | func (s *StubCapabilityComparator) ToNetCapabilities() *net.Capabilities { 13 | return s.NetCaps 14 | } 15 | 16 | func (s *StubCapabilityComparator) CompatibleWith(other *net.Capabilities) bool { 17 | // Implement the logic for compatibility check if needed 18 | return true 19 | } 20 | 21 | func (s *StubCapabilityComparator) LegacyOnly() bool { 22 | return s.IsLegacy 23 | } 24 | -------------------------------------------------------------------------------- /server/suspensions.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | // suspender is a list that keep track of suspender orchestrators 8 | // and the count until which they are suspended 9 | type suspender struct { 10 | mu sync.Mutex 11 | list map[string]int // list of orchestrator => refresh count at which the orchestrator is no longer suspended 12 | count int 13 | } 14 | 15 | // newSuspender returns the pointer to a new Suspender instance 16 | func newSuspender() *suspender { 17 | return &suspender{ 18 | list: make(map[string]int), 19 | } 20 | } 21 | 22 | // suspend an orchestrator for 'penalty' refreshes 23 | func (s *suspender) suspend(orch string, penalty int) { 24 | s.mu.Lock() 25 | defer s.mu.Unlock() 26 | s.list[orch] += penalty 27 | } 28 | 29 | // Suspended returns a non-zero value if the orchestrator is suspended 30 | // 'orch' is the service URI of the orchestrator 31 | // The value returned is the suspension penalty associated with the orchestrator whereby lower is better 32 | func (s *suspender) Suspended(orch string) int { 33 | s.mu.Lock() 34 | defer s.mu.Unlock() 35 | if s.list[orch] < s.count { 36 | delete(s.list, orch) 37 | } 38 | return s.list[orch] 39 | } 40 | 41 | // signalRefresh increases Suspender.count 42 | func (s *suspender) signalRefresh() { 43 | s.mu.Lock() 44 | defer s.mu.Unlock() 45 | s.count++ 46 | } 47 | -------------------------------------------------------------------------------- /server/suspensions_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestSuspender(t *testing.T) { 10 | assert := assert.New(t) 11 | s := newSuspender() 12 | 13 | s.suspend("foo", 5) 14 | assert.Equal(s.Suspended("foo"), 5) 15 | s.suspend("foo", 5) 16 | assert.Equal(s.Suspended("foo"), 10) 17 | s.count = 11 18 | assert.Equal(s.Suspended("foo"), 0) 19 | _, ok := s.list["foo"] 20 | assert.False(ok) 21 | 22 | s.signalRefresh() 23 | assert.Equal(s.count, 12) 24 | } 25 | -------------------------------------------------------------------------------- /server/test.flv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/livepeer/go-livepeer/e098564304ae51e6def882b0318b7591e4b8b93a/server/test.flv -------------------------------------------------------------------------------- /server/utils.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | // utils.go contains server utility functions. 4 | 5 | import ( 6 | "encoding/json" 7 | "net/http" 8 | 9 | "github.com/oapi-codegen/runtime" 10 | ) 11 | 12 | // Decoder for JSON requests. 13 | func jsonDecoder[T any](req *T, r *http.Request) error { 14 | return json.NewDecoder(r.Body).Decode(req) 15 | } 16 | 17 | // Decoder for Multipart requests. 18 | func multipartDecoder[T any](req *T, r *http.Request) error { 19 | multiRdr, err := r.MultipartReader() 20 | if err != nil { 21 | return err 22 | } 23 | return runtime.BindMultipart(req, *multiRdr) 24 | } 25 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eux 4 | 5 | # Test script to run all the tests except of e2e tests for continuous integration 6 | go test -coverprofile cover.out $(go list ./... | grep -v 'test/e2e') 7 | 8 | cd core 9 | # Be more strict with load balancer tests: run with race detector enabled 10 | go test -run LB_ -race 11 | # Be more strict with nvidia tests: run with race detector enabled 12 | go test -run Nvidia_ -race 13 | go test -run Capabilities_ -race 14 | cd .. 15 | 16 | # Be more strict with discovery tests: run with race detector enabled 17 | cd discovery 18 | go test -race 19 | cd .. 20 | 21 | # Be more strict with HTTP push tests: run with race detector enabled 22 | cd server 23 | go test -run TestSelectSession_ -race 24 | go test -run RegisterConnection -race 25 | cd .. 26 | 27 | cd media 28 | go test -race 29 | cd .. 30 | 31 | ./test_args.sh 32 | 33 | printf "\n\nAll Tests Passed\n\n" 34 | -------------------------------------------------------------------------------- /test/e2e/binary_test.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/stretchr/testify/assert" 6 | "os/exec" 7 | "testing" 8 | ) 9 | 10 | func TestMistJson(t *testing.T) { 11 | assert := assert.New(t) 12 | buildLivepeer(assert) 13 | // run 14 | lp := exec.Command("./livepeer", "-j") 15 | stdoutRes, err := lp.Output() 16 | assert.NoError(err) 17 | 18 | // parse output 19 | jsonMap := make(map[string](interface{})) 20 | err = json.Unmarshal(stdoutRes, &jsonMap) 21 | assert.NoError(err) 22 | // only check for name element 23 | assert.Contains(jsonMap, "name") 24 | } 25 | -------------------------------------------------------------------------------- /test/e2e/configure_orchestrator_test.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "math/big" 7 | "net/url" 8 | "testing" 9 | "time" 10 | 11 | "github.com/livepeer/go-livepeer/eth" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestConfigureOrchestrator(t *testing.T) { 16 | // given 17 | ctx, cancel := context.WithCancel(context.Background()) 18 | defer cancel() 19 | 20 | geth := setupGeth(t, ctx) 21 | defer terminateGeth(t, ctx, geth) 22 | 23 | oCtx, oCancel := context.WithCancel(context.Background()) 24 | o := startOrchestratorWithNewAccount(t, oCtx, geth) 25 | registerOrchestrator(t, o) 26 | 27 | // orchestrator needs to restart in order to be able to process rewards, so configuration is unlocked 28 | o = restartOrchestrator(t, ctx, oCancel, geth, o) 29 | defer o.stop() 30 | 31 | waitUntilOrchestratorIsConfigurable(t, o.dev.Client) 32 | 33 | // when 34 | configureOrchestrator(o, newCfg) 35 | 36 | // then 37 | assertOrchestratorConfigured(t, o, newCfg) 38 | } 39 | 40 | func restartOrchestrator(t *testing.T, ctx context.Context, cancel context.CancelFunc, geth *gethContainer, o *livepeer) *livepeer { 41 | o.stop() 42 | cancel() 43 | return startOrchestratorWithExistingAccount(t, ctx, geth, o.cfg.EthAcctAddr, o.cfg.Datadir) 44 | } 45 | 46 | func waitUntilOrchestratorIsConfigurable(t *testing.T, lpEth eth.LivepeerEthClient) { 47 | require := require.New(t) 48 | 49 | for { 50 | active, err := lpEth.IsActiveTranscoder() 51 | require.NoError(err) 52 | 53 | initialized, err := lpEth.CurrentRoundInitialized() 54 | require.NoError(err) 55 | 56 | t, err := lpEth.GetTranscoder(lpEth.Account().Address) 57 | require.NoError(err) 58 | rewardCalled := t.LastRewardRound.Cmp(big.NewInt(0)) > 0 59 | 60 | if active && initialized && rewardCalled { 61 | return 62 | } 63 | 64 | time.Sleep(2 * time.Second) 65 | } 66 | } 67 | 68 | func configureOrchestrator(o *livepeer, cfg *orchestratorConfig) { 69 | val := url.Values{ 70 | "blockRewardCut": {fmt.Sprintf("%v", cfg.BlockRewardCut)}, 71 | "feeShare": {fmt.Sprintf("%v", cfg.FeeShare)}, 72 | "serviceURI": {fmt.Sprintf("http://%v", cfg.ServiceURI)}, 73 | } 74 | 75 | for { 76 | if _, ok := httpPostWithParams(fmt.Sprintf("http://%s/setOrchestratorConfig", *o.cfg.CliAddr), val); ok { 77 | return 78 | } 79 | time.Sleep(200 * time.Millisecond) 80 | } 81 | } 82 | 83 | func assertOrchestratorConfigured(t *testing.T, o *livepeer, cfg *orchestratorConfig) { 84 | require := require.New(t) 85 | 86 | transPool, err := o.dev.Client.TranscoderPool() 87 | uri, errURI := o.dev.Client.GetServiceURI(o.dev.Client.Account().Address) 88 | 89 | require.NoError(err) 90 | require.NoError(errURI) 91 | require.Len(transPool, 1) 92 | trans := transPool[0] 93 | require.Equal(eth.FromPerc(cfg.FeeShare), trans.FeeShare) 94 | require.Equal(eth.FromPerc(cfg.BlockRewardCut), trans.RewardCut) 95 | require.Equal(fmt.Sprintf("http://%v", cfg.ServiceURI), uri) 96 | } 97 | -------------------------------------------------------------------------------- /test/e2e/deposit_broadcaster_test.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "context" 5 | "math/big" 6 | "testing" 7 | ) 8 | 9 | func TestDepositBroadcaster(t *testing.T) { 10 | ctx, cancel := context.WithCancel(context.Background()) 11 | defer cancel() 12 | 13 | geth := setupGeth(t, ctx) 14 | defer terminateGeth(t, ctx, geth) 15 | 16 | b := startBroadcasterWithNewAccount(t, ctx, geth) 17 | defer b.stop() 18 | 19 | // 100 ETH 20 | amount := new(big.Int).Mul(big.NewInt(100), new(big.Int).Exp(big.NewInt(10), big.NewInt(18), nil)) 21 | depositBroadcaster(t, b, amount) 22 | } 23 | -------------------------------------------------------------------------------- /test/e2e/http_push_broadcaster_test.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "context" 5 | "math/big" 6 | "testing" 7 | ) 8 | 9 | func TestHTTPPushBroadcaster(t *testing.T) { 10 | ctx, cancel := context.WithCancel(context.Background()) 11 | defer cancel() 12 | 13 | geth := setupGeth(t, ctx) 14 | defer terminateGeth(t, ctx, geth) 15 | 16 | o := startOrchestratorWithNewAccount(t, ctx, geth) 17 | defer o.stop() 18 | 19 | registerOrchestrator(t, o) 20 | requireOrchestratorRegisteredAndActivated(t, o) 21 | 22 | b := startBroadcasterWithNewAccount(t, ctx, geth) 23 | defer b.stop() 24 | 25 | amount := new(big.Int).Mul(big.NewInt(100), new(big.Int).Exp(big.NewInt(10), big.NewInt(18), nil)) 26 | depositBroadcaster(t, b, amount) 27 | 28 | // Sequential requests 29 | pushSegmentsBroadcaster(t, b, 3) 30 | } 31 | -------------------------------------------------------------------------------- /test/e2e/register_orchestrator_test.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | ) 7 | 8 | func TestRegisterOrchestrator(t *testing.T) { 9 | // given 10 | ctx, cancel := context.WithCancel(context.Background()) 11 | defer cancel() 12 | 13 | geth := setupGeth(t, ctx) 14 | defer terminateGeth(t, ctx, geth) 15 | 16 | o := startOrchestratorWithNewAccount(t, ctx, geth) 17 | defer o.stop() 18 | 19 | // when 20 | registerOrchestrator(t, o) 21 | 22 | // then 23 | requireOrchestratorRegisteredAndActivated(t, o) 24 | } 25 | -------------------------------------------------------------------------------- /test/e2e/remove_orchestrator_test.go: -------------------------------------------------------------------------------- 1 | package e2e 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "math/big" 9 | "net/http" 10 | "net/url" 11 | "testing" 12 | "time" 13 | 14 | "github.com/golang/glog" 15 | "github.com/livepeer/go-livepeer/common" 16 | "github.com/stretchr/testify/require" 17 | ) 18 | 19 | func TestRemoveOrchestrator(t *testing.T) { 20 | //given 21 | ctx, cancel := context.WithCancel(context.Background()) 22 | defer cancel() 23 | 24 | geth := setupGeth(t, ctx) 25 | defer terminateGeth(t, ctx, geth) 26 | 27 | o := startOrchestratorWithNewAccount(t, ctx, geth) 28 | defer o.stop() 29 | 30 | registerOrchestrator(t, o) 31 | waitUntilRoundInitialized(t, o.dev.Client) 32 | 33 | // when 34 | deactivateOrchestrator(o, big.NewInt(initialCfg.LptStake)) 35 | 36 | // then 37 | assertOrchestratorRemoved(t, o) 38 | } 39 | 40 | func deactivateOrchestrator(o *livepeer, initialStake *big.Int) { 41 | val := url.Values{ 42 | "amount": {fmt.Sprintf("%d", initialStake)}, 43 | } 44 | 45 | glog.Errorf("deactivate orchestrator") 46 | 47 | for { 48 | if _, ok := httpPostWithParams(fmt.Sprintf("http://%s/unbond", *o.cfg.CliAddr), val); ok { 49 | return 50 | } 51 | time.Sleep(200 * time.Millisecond) 52 | } 53 | } 54 | 55 | func assertOrchestratorRemoved(t *testing.T, o *livepeer) { 56 | require := require.New(t) 57 | 58 | lock := getUnbondingLock(o) 59 | require.Equal(0, lock.Amount.Cmp(big.NewInt(initialCfg.LptStake))) 60 | require.Equal(o.dev.Client.Account().Address, lock.Delegator) 61 | } 62 | 63 | func getUnbondingLock(o *livepeer) common.DBUnbondingLock { 64 | response, _ := http.Get(fmt.Sprintf("http://%s/unbondingLocks", *o.cfg.CliAddr)) 65 | defer response.Body.Close() 66 | 67 | payload, _ := ioutil.ReadAll(response.Body) 68 | 69 | var unbondingLocks = []common.DBUnbondingLock{} 70 | json.Unmarshal(payload, &unbondingLocks) 71 | return unbondingLocks[0] 72 | } 73 | -------------------------------------------------------------------------------- /test/e2e/test.flv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/livepeer/go-livepeer/e098564304ae51e6def882b0318b7591e4b8b93a/test/e2e/test.flv -------------------------------------------------------------------------------- /test_docker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | docker build -t go-livepeer-test . && docker run --rm --name go-livepeer-test-run go-livepeer-test 4 | -------------------------------------------------------------------------------- /test_e2e.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eux 4 | 5 | go test $(go list ./... | grep 'test/e2e') --timeout 15m 6 | -------------------------------------------------------------------------------- /tools.go: -------------------------------------------------------------------------------- 1 | //go:build tools 2 | // +build tools 3 | 4 | package tools 5 | 6 | import ( 7 | _ "github.com/ethereum/go-ethereum/cmd/abigen" 8 | _ "github.com/golang/mock/mockgen" 9 | ) 10 | -------------------------------------------------------------------------------- /transcode_demo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | url='localhost:8935' 4 | manifestID='current' 5 | 6 | print_usage() { 7 | printf "Usage: use -u to specify the url (default localhost:8935), use -m to specify manifestID (default current.m3u8)" 8 | } 9 | 10 | while getopts ":u:m:h" opt; do 11 | case ${opt} in 12 | u ) 13 | url=$OPTARG 14 | echo "url: $url" 15 | ;; 16 | m ) 17 | manifestID=$OPTARG 18 | echo "manifestID: $manifestID" 19 | ;; 20 | h ) 21 | print_usage 22 | ;; 23 | \? ) 24 | echo "Invalid option: $OPTARG" 1>&2 25 | ;; 26 | : ) 27 | echo "Invalid option: $OPTARG requires an argument" 1>&2 28 | ;; 29 | esac 30 | done 31 | 32 | sURI="$url/stream/$manifestID.m3u8" 33 | manifest=$(curl -s $sURI 2> /dev/null) 34 | if [ -z "$manifest" ] 35 | then 36 | echo "Empty response from $url/stream/$manifestID.m3u8. Make sure the -currentManifest flag is on, or use -h to get usage options." 37 | exit 38 | fi 39 | 40 | sid1=$(echo $manifest| cut -d' ' -f 4) 41 | sid2=$(echo $manifest| cut -d' ' -f 6) 42 | sid3=$(echo $manifest| cut -d' ' -f 8) 43 | ffplay "http://$url/stream/$sid1" & 44 | ffplay "http://$url/stream/$sid2" & 45 | ffplay "http://$url/stream/$sid3" & 46 | -------------------------------------------------------------------------------- /trickle/local_publisher.go: -------------------------------------------------------------------------------- 1 | package trickle 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "log/slog" 7 | "sync" 8 | ) 9 | 10 | // local (in-memory) publisher for trickle protocol 11 | 12 | type TrickleLocalPublisher struct { 13 | channelName string 14 | mimeType string 15 | server *Server 16 | 17 | mu *sync.Mutex 18 | seq int 19 | } 20 | 21 | func NewLocalPublisher(sm *Server, channelName string, mimeType string) *TrickleLocalPublisher { 22 | return &TrickleLocalPublisher{ 23 | channelName: channelName, 24 | server: sm, 25 | mu: &sync.Mutex{}, 26 | mimeType: mimeType, 27 | } 28 | } 29 | 30 | func (c *TrickleLocalPublisher) CreateChannel() { 31 | c.server.getOrCreateStream(c.channelName, c.mimeType, true) 32 | } 33 | 34 | func (c *TrickleLocalPublisher) Write(data io.Reader) error { 35 | stream := c.server.getOrCreateStream(c.channelName, c.mimeType, true) 36 | c.mu.Lock() 37 | seq := c.seq 38 | segment, exists := stream.getForWrite(seq) 39 | if exists { 40 | c.mu.Unlock() 41 | return errors.New("Entry already exists for this sequence") 42 | } 43 | 44 | // before we begin - let's pre-create the next segment 45 | nextSeq := c.seq + 1 46 | if _, exists = stream.getForWrite(nextSeq); exists { 47 | c.mu.Unlock() 48 | return errors.New("Next entry already exists in this sequence") 49 | } 50 | c.seq = nextSeq 51 | c.mu.Unlock() 52 | 53 | // now continue with the show 54 | buf := make([]byte, 1024*32) // 32kb to begin with 55 | totalRead := 0 56 | for { 57 | n, err := data.Read(buf) 58 | if n > 0 { 59 | segment.writeData(buf[:n]) 60 | totalRead += n 61 | } 62 | if err != nil { 63 | if err == io.EOF { 64 | break 65 | } 66 | slog.Info("Error reading published data", "channel", c.channelName, "seq", seq, "bytes written", totalRead, "err", err) 67 | } 68 | } 69 | segment.close() 70 | return nil 71 | } 72 | 73 | func (c *TrickleLocalPublisher) Close() error { 74 | return c.server.closeStream(c.channelName) 75 | } 76 | -------------------------------------------------------------------------------- /trickle/local_subscriber.go: -------------------------------------------------------------------------------- 1 | package trickle 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "log/slog" 7 | "strconv" 8 | "sync" 9 | ) 10 | 11 | // local (in-memory) subscriber for trickle protocol 12 | 13 | type TrickleData struct { 14 | Reader io.Reader 15 | Metadata map[string]string 16 | } 17 | 18 | type TrickleLocalSubscriber struct { 19 | channelName string 20 | server *Server 21 | 22 | mu *sync.Mutex 23 | seq int 24 | } 25 | 26 | func NewLocalSubscriber(sm *Server, channelName string) *TrickleLocalSubscriber { 27 | return &TrickleLocalSubscriber{ 28 | channelName: channelName, 29 | server: sm, 30 | mu: &sync.Mutex{}, 31 | seq: -1, 32 | } 33 | } 34 | 35 | func (c *TrickleLocalSubscriber) Read() (*TrickleData, error) { 36 | stream, exists := c.server.getStream(c.channelName) 37 | if !exists { 38 | return nil, errors.New("stream not found") 39 | } 40 | c.mu.Lock() 41 | defer c.mu.Unlock() 42 | segment, latestSeq, exists := stream.getForRead(c.seq) 43 | if !exists { 44 | return nil, errors.New("seq not found") 45 | } 46 | c.seq++ 47 | r, w := io.Pipe() 48 | go func() { 49 | subscriber := &SegmentSubscriber{ 50 | segment: segment, 51 | } 52 | for { 53 | data, eof := subscriber.readData() 54 | n, err := w.Write(data) 55 | if err != nil { 56 | slog.Info("Error writing", "channel", c.channelName, "seq", segment.idx, "err", err) 57 | return 58 | } 59 | if n != len(data) { 60 | slog.Info("Did not write enough data to local subscriber", "channel", c.channelName, "seq", segment.idx) 61 | return 62 | } 63 | if eof { 64 | // trigger eof on the reader 65 | w.Close() 66 | return 67 | } 68 | } 69 | }() 70 | return &TrickleData{ 71 | Reader: r, 72 | Metadata: map[string]string{ 73 | "Lp-Trickle-Latest": strconv.Itoa(latestSeq), 74 | "Lp-Trickle-Seq": strconv.Itoa(segment.idx), 75 | "Content-Type": stream.mimeType, 76 | }, // TODO take more metadata from http headers 77 | }, nil 78 | } 79 | --------------------------------------------------------------------------------