├── Rakefile ├── lib ├── stellar_core_commander │ ├── version.rb │ ├── cmd_result.rb │ ├── convert.rb │ ├── sequence_tracker.rb │ ├── concerns │ │ ├── named_objects.rb │ │ └── tracks_accounts.rb │ ├── cmd.rb │ ├── container.rb │ ├── commander.rb │ ├── horizon_commander.rb │ ├── local_process.rb │ ├── docker_process.rb │ ├── transaction_builder.rb │ ├── transactor.rb │ └── process.rb └── stellar_core_commander.rb ├── examples ├── merge_account.rb ├── inflation.rb ├── simple_payment.rb ├── onetime_signer.rb ├── bumpseq.rb ├── allow_trust.rb ├── load_generation.rb ├── load_generation_auto.rb ├── non_native_payment.rb ├── cross_host_simple_payment.rb ├── set_options.rb ├── history_testnet_catchup.rb ├── multi_host_simple_payment.rb ├── passive_offer.rb ├── history_generate_and_catchup.rb ├── pathed_payment.rb ├── trade.rb ├── path_payment_strict_send.rb └── version_mix_consensus.rb ├── Gemfile ├── .gitignore ├── test.rb ├── bin ├── hcc └── scc ├── stellar_core_commander.gemspec ├── CONTRIBUTING.md ├── README.md └── LICENSE.txt /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | 3 | -------------------------------------------------------------------------------- /lib/stellar_core_commander/version.rb: -------------------------------------------------------------------------------- 1 | module StellarCoreCommander 2 | VERSION = "0.0.13" 3 | end 4 | -------------------------------------------------------------------------------- /examples/merge_account.rb: -------------------------------------------------------------------------------- 1 | account :scott 2 | 3 | create_account :scott, :master 4 | 5 | close_ledger 6 | 7 | merge_account :scott, :master 8 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gemspec 3 | 4 | # gem 'stellar-base', path: "~/prj/stellar/ruby-stellar-base" 5 | # gem 'xdr', path: "~/src/stellar/ruby-xdr"# 6 | -------------------------------------------------------------------------------- /examples/inflation.rb: -------------------------------------------------------------------------------- 1 | account :scott 2 | 3 | create_account :scott, :master, 2_000_000_000 4 | 5 | close_ledger 6 | 7 | set_inflation_dest :scott, :scott 8 | inflation 9 | -------------------------------------------------------------------------------- /examples/simple_payment.rb: -------------------------------------------------------------------------------- 1 | account :scott 2 | 3 | create_account :scott, :master 4 | 5 | close_ledger 6 | 7 | payment :master, :scott, [:native, 1000] 8 | 9 | check_no_error_metrics 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /Gemfile.lock 4 | /_yardoc/ 5 | /coverage/ 6 | /doc/ 7 | /pkg/ 8 | /spec/reports/ 9 | /tmp/ 10 | *.bundle 11 | *.so 12 | *.o 13 | *.a 14 | mkmf.log 15 | out.sql 16 | .DS_Store 17 | *.swp 18 | -------------------------------------------------------------------------------- /test.rb: -------------------------------------------------------------------------------- 1 | require 'stellar_core_commander' 2 | require 'pry' 3 | 4 | bin = File.expand_path("~/src/stellar/stellar-core/bin/stellar-core") 5 | cmd = StellarCoreCommander::Commander.new(bin) 6 | 7 | cmd.cleanup_at_exit! 8 | p1 = cmd.make_process 9 | 10 | 11 | binding.pry -------------------------------------------------------------------------------- /examples/onetime_signer.rb: -------------------------------------------------------------------------------- 1 | account :scott 2 | puts "scott: #{get_account(:scott).address}" 3 | create_via_friendbot :scott 4 | wait 5 | 6 | pp get_account_info(:scott).signers 7 | 8 | add_onetime_signer :scott, "hello world", 1 9 | 10 | wait 11 | 12 | pp get_account_info(:scott).signers 13 | -------------------------------------------------------------------------------- /lib/stellar_core_commander/cmd_result.rb: -------------------------------------------------------------------------------- 1 | module StellarCoreCommander 2 | class CmdResult 3 | include Contracts 4 | 5 | attr_reader :success 6 | attr_reader :out 7 | 8 | Contract Bool, Maybe[String] => Any 9 | def initialize(success, out = None) 10 | @success = success 11 | @out = out 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /examples/bumpseq.rb: -------------------------------------------------------------------------------- 1 | use_manual_close 2 | 3 | account :scott 4 | puts "scott: #{get_account(:scott).address}" 5 | create_account :scott 6 | close_ledger 7 | 8 | seq = next_sequence get_account(:scott) 9 | puts "scott: #{seq}" 10 | 11 | bump_sequence :scott, seq + 10 12 | 13 | close_ledger 14 | 15 | seq = next_sequence get_account(:scott) 16 | puts "scott: #{seq}" -------------------------------------------------------------------------------- /examples/allow_trust.rb: -------------------------------------------------------------------------------- 1 | account :usd_gateway 2 | account :scott 3 | account :andrew 4 | 5 | create_account :usd_gateway 6 | create_account :scott 7 | create_account :andrew 8 | 9 | close_ledger 10 | 11 | require_trust_auth :usd_gateway 12 | 13 | close_ledger 14 | 15 | trust :scott, :usd_gateway, "USD" 16 | trust :andrew, :usd_gateway, "USD" 17 | 18 | close_ledger 19 | 20 | allow_trust :usd_gateway, :scott, "USD" 21 | -------------------------------------------------------------------------------- /examples/load_generation.rb: -------------------------------------------------------------------------------- 1 | process :node1, [:node1, :node2, :node3], await_sync: false 2 | process :node2, [:node1, :node2, :node3], await_sync: false 3 | process :node3, [:node1, :node2, :node3] 4 | 5 | on :node1 do 6 | generate_load_and_await_completion 1000, 1000, 30 7 | end 8 | 9 | on :node2 do 10 | check_integrity_against :node1 11 | end 12 | 13 | on :node3 do 14 | check_integrity_against :node1 15 | end 16 | -------------------------------------------------------------------------------- /examples/load_generation_auto.rb: -------------------------------------------------------------------------------- 1 | process :node1, [:node1, :node2, :node3], await_sync: false 2 | process :node2, [:node1, :node2, :node3], await_sync: false 3 | process :node3, [:node1, :node2, :node3] 4 | 5 | on :node1 do 6 | generate_load_and_await_completion 10000, 10000, :auto 7 | end 8 | 9 | on :node2 do 10 | check_integrity_against :node1 11 | end 12 | 13 | on :node3 do 14 | check_integrity_against :node1 15 | end 16 | -------------------------------------------------------------------------------- /examples/non_native_payment.rb: -------------------------------------------------------------------------------- 1 | account :usd_gateway 2 | account :scott 3 | account :andrew 4 | 5 | create_account :usd_gateway 6 | create_account :scott 7 | create_account :andrew 8 | 9 | close_ledger 10 | 11 | trust :scott, :usd_gateway, "USD" 12 | trust :andrew, :usd_gateway, "USD" 13 | 14 | close_ledger 15 | 16 | payment :usd_gateway, :scott, ["USD", :usd_gateway, 1000] 17 | 18 | close_ledger 19 | 20 | payment :scott, :andrew, ["USD", :usd_gateway, 500] 21 | -------------------------------------------------------------------------------- /lib/stellar_core_commander/convert.rb: -------------------------------------------------------------------------------- 1 | module StellarCoreCommander 2 | # 3 | # Generic format conversion module 4 | # 5 | module Convert 6 | require 'base64' 7 | 8 | def to_hex(string) 9 | string.unpack("H*").first 10 | end 11 | 12 | def from_hex(hex_string) 13 | [hex_string].pack("H*") 14 | end 15 | 16 | def to_base64(string) 17 | Base64.strict_encode64(string) 18 | end 19 | 20 | def from_base64(base64_string) 21 | Base64.strict_decode64(base64_string) 22 | end 23 | 24 | extend self 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /examples/cross_host_simple_payment.rb: -------------------------------------------------------------------------------- 1 | process :node1, [:node1, :node2, :node3], host: '192.168.99.105', await_sync: false 2 | process :node2, [:node1, :node2, :node3], host: '192.168.99.104', await_sync: false 3 | process :node3, [:node1, :node2, :node3], host: '192.168.99.103' 4 | 5 | account :alice 6 | account :bob 7 | 8 | puts "Running txs on node1" 9 | 10 | on :node1 do 11 | create_account :alice, :master 12 | create_account :bob, :master 13 | close_ledger 14 | payment :master, :bob, [:native, 100] 15 | close_ledger 16 | end 17 | 18 | payment :master, :alice, [:native, 100] 19 | 20 | -------------------------------------------------------------------------------- /examples/set_options.rb: -------------------------------------------------------------------------------- 1 | use_manual_close 2 | 3 | account :scott 4 | account :bartek 5 | create_account :scott 6 | create_account :bartek 7 | 8 | close_ledger 9 | 10 | kp = Stellar::KeyPair.random 11 | 12 | set_inflation_dest :scott, :bartek 13 | set_flags :scott, [:auth_required_flag] 14 | set_master_signer_weight :scott, 2 15 | set_thresholds :scott, low: 0, medium: 2, high: 2 16 | set_thresholds :scott, high: 1 17 | set_home_domain :scott, "nullstyle.com" 18 | add_signer :scott, kp, 1 19 | 20 | close_ledger 21 | 22 | clear_flags :scott, [:auth_required_flag] 23 | remove_signer :scott, kp 24 | -------------------------------------------------------------------------------- /examples/history_testnet_catchup.rb: -------------------------------------------------------------------------------- 1 | process :node1, [:testnet1, :testnet2, :testnet3], forcescp: false, debug: true, validate: false 2 | on :node1 do 3 | raise "node1 synced but failed to catch up" if ledger_num < 5 4 | $stderr.puts "caught up on node1: ledger #{ledger_num}" 5 | check_no_error_metrics 6 | end 7 | 8 | process :node2, [:testnet1, :testnet2, :testnet3], forcescp: false, catchup_complete: true, debug: true, validate: false 9 | on :node2 do 10 | raise "node2 synced but failed to catch up" if ledger_num < 5 11 | $stderr.puts "caught up on node2: ledger #{ledger_num}" 12 | check_no_error_metrics 13 | end 14 | -------------------------------------------------------------------------------- /lib/stellar_core_commander/sequence_tracker.rb: -------------------------------------------------------------------------------- 1 | 2 | module StellarCoreCommander 3 | class SequenceTracker 4 | include Contracts 5 | 6 | Contract RespondTo[:sequence_for] => Any 7 | def initialize(provider) 8 | @provider = provider 9 | @data = {} 10 | end 11 | 12 | Contract None => Any 13 | def reset 14 | @data = {} 15 | end 16 | 17 | Contract Stellar::KeyPair => Num 18 | def next_sequence(kp) 19 | current = @data[kp.address] || @provider.sequence_for(kp) 20 | nexts = current + 1 21 | @data[kp.address] = nexts 22 | nexts 23 | end 24 | end 25 | end 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /examples/multi_host_simple_payment.rb: -------------------------------------------------------------------------------- 1 | process :node1, [:node1, :node2, :node3], await_sync: false 2 | process :node2, [:node1, :node2, :node3], await_sync: false 3 | process :node3, [:node1, :node2, :node3] 4 | 5 | account :alice 6 | account :bob 7 | 8 | puts "Running txs on node1" 9 | 10 | on :node1 do 11 | create_account :alice, :master 12 | create_account :bob, :master 13 | close_ledger 14 | payment :master, :bob, [:native, 100] 15 | close_ledger 16 | end 17 | 18 | payment :master, :alice, [:native, 100] 19 | 20 | on :node2 do 21 | check_integrity_against :node1 22 | end 23 | 24 | on :node3 do 25 | check_integrity_against :node1 26 | end 27 | -------------------------------------------------------------------------------- /lib/stellar_core_commander/concerns/named_objects.rb: -------------------------------------------------------------------------------- 1 | 2 | module StellarCoreCommander 3 | module Concerns 4 | module NamedObjects 5 | include Contracts 6 | 7 | private 8 | Contract Symbol, Any => Any 9 | def add_named(name, object) 10 | @named ||= {}.with_indifferent_access 11 | if @named.has_key?(name) 12 | raise ArgumentError, "#{name} is already registered" 13 | end 14 | @named[name] = object 15 | object 16 | end 17 | 18 | Contract Symbol => Any 19 | def get_named(name) 20 | @named ||= {}.with_indifferent_access 21 | @named[name] 22 | end 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /examples/passive_offer.rb: -------------------------------------------------------------------------------- 1 | use_manual_close #use_manual_close causes scc to run a process with MANUAL_CLOSE=true 2 | 3 | account :usd_gateway 4 | account :eur_gateway 5 | account :scott 6 | 7 | create_account :usd_gateway 8 | create_account :eur_gateway 9 | create_account :scott 10 | 11 | close_ledger 12 | 13 | trust :scott, :usd_gateway, "USD" 14 | trust :scott, :eur_gateway, "EUR" 15 | 16 | close_ledger 17 | 18 | payment :usd_gateway, :scott, ["USD", :usd_gateway, 1000] 19 | payment :eur_gateway, :scott, ["EUR", :eur_gateway, 1000] 20 | 21 | close_ledger 22 | 23 | passive_offer :scott, {sell:["USD", :usd_gateway], for:["EUR", :eur_gateway]}, 500, 1.0 24 | passive_offer :scott, {buy:["USD", :usd_gateway], with:["EUR", :eur_gateway]}, 500, 1.0 25 | -------------------------------------------------------------------------------- /lib/stellar_core_commander.rb: -------------------------------------------------------------------------------- 1 | require "stellar_core_commander/version" 2 | require "active_support/all" 3 | require "stellar-base" 4 | require "stellar-sdk" 5 | require "contracts" 6 | require "faraday" 7 | require "faraday_middleware" 8 | require "fileutils" 9 | require "sequel" 10 | require "pg" 11 | require "uri" 12 | 13 | module StellarCoreCommander 14 | extend ActiveSupport::Autoload 15 | 16 | autoload :Commander 17 | 18 | autoload :Cmd 19 | autoload :CmdResult 20 | autoload :Process 21 | autoload :LocalProcess 22 | autoload :Container 23 | autoload :DockerProcess 24 | 25 | autoload :Transactor 26 | autoload :TransactionBuilder 27 | 28 | autoload :Convert 29 | 30 | autoload :HorizonCommander 31 | autoload :SequenceTracker 32 | 33 | module Concerns 34 | extend ActiveSupport::Autoload 35 | 36 | autoload :NamedObjects 37 | autoload :TracksAccounts 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/stellar_core_commander/concerns/tracks_accounts.rb: -------------------------------------------------------------------------------- 1 | 2 | module StellarCoreCommander 3 | module Concerns 4 | module TracksAccounts 5 | include Contracts 6 | 7 | Contract Symbol, Stellar::KeyPair => Any 8 | # 9 | # Registered an account for this scenario. Future calls may refer to 10 | # the name provided. 11 | # 12 | # @param name [Symbol] the name to register the keypair at 13 | # @param keypair=Stellar::KeyPair.random [Stellar::KeyPair] the keypair to use for this account 14 | # 15 | def account(name, keypair=Stellar::KeyPair.random) 16 | add_named name, keypair 17 | end 18 | 19 | Contract Symbol => Stellar::KeyPair 20 | def get_account(name) 21 | get_named(name).tap do |found| 22 | unless found.is_a?(Stellar::KeyPair) 23 | raise ArgumentError, "#{name.inspect} is not account" 24 | end 25 | end 26 | end 27 | 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /examples/history_generate_and_catchup.rb: -------------------------------------------------------------------------------- 1 | process :node1, [:node1], accelerate_time: true 2 | on :node1 do 3 | generate_load_and_await_completion 100, 100, 20 4 | retry_until_true retries: 100 do 5 | ledger_num > 100 6 | end 7 | check_no_error_metrics 8 | end 9 | 10 | process :node2_minimal, [:node1], forcescp: false, accelerate_time: true 11 | on :node2_minimal do 12 | retry_until_true retries: 100 do 13 | ledger_num > 120 14 | end 15 | check_integrity_against :node1 16 | end 17 | 18 | 19 | process :node2_complete, [:node1], forcescp: false, accelerate_time: true, catchup_complete: true 20 | on :node2_complete do 21 | retry_until_true retries: 100 do 22 | ledger_num > 120 23 | end 24 | check_integrity_against :node1 25 | end 26 | 27 | process :node2_recent, [:node1], forcescp: false, accelerate_time: true, catchup_recent: 30 28 | on :node2_recent do 29 | retry_until_true retries: 100 do 30 | ledger_num > 120 31 | end 32 | check_integrity_against :node1 33 | end 34 | -------------------------------------------------------------------------------- /examples/pathed_payment.rb: -------------------------------------------------------------------------------- 1 | account :usd_gateway 2 | account :eur_gateway 3 | account :scott 4 | account :bartek 5 | account :andrew 6 | 7 | create_account :usd_gateway, :master 8 | create_account :eur_gateway, :master 9 | create_account :scott, :master 10 | create_account :bartek, :master 11 | create_account :andrew, :master 12 | 13 | close_ledger 14 | 15 | trust :scott, :usd_gateway, "USD" 16 | trust :bartek, :eur_gateway, "EUR" 17 | trust :andrew, :usd_gateway, "USD" 18 | trust :andrew, :eur_gateway, "EUR" 19 | 20 | close_ledger 21 | 22 | payment :usd_gateway, :scott, ["USD", :usd_gateway, 1000] 23 | payment :usd_gateway, :andrew, ["USD", :usd_gateway, 200] 24 | payment :eur_gateway, :andrew, ["EUR", :eur_gateway, 200] 25 | payment :eur_gateway, :bartek, ["EUR", :eur_gateway, 1000] 26 | 27 | close_ledger 28 | 29 | offer :andrew, {buy:["USD", :usd_gateway], with:["EUR", :eur_gateway]}, 200, 1.0 30 | 31 | close_ledger 32 | 33 | payment :scott, :bartek, ["EUR", :eur_gateway, 10], with: ["USD", :usd_gateway, 10], path: [] 34 | -------------------------------------------------------------------------------- /examples/trade.rb: -------------------------------------------------------------------------------- 1 | use_manual_close #use_manual_close causes scc to run a process with MANUAL_CLOSE=true 2 | 3 | account :usd_gateway 4 | account :eur_gateway 5 | account :scott 6 | account :bartek 7 | 8 | create_account :usd_gateway, :master, 1000 9 | create_account :eur_gateway, :master, 1000 10 | create_account :scott, :master, 1000 11 | create_account :bartek, :master, 1000 12 | 13 | close_ledger 14 | 15 | trust :scott, :usd_gateway, "USD" 16 | trust :bartek, :usd_gateway, "USD" 17 | trust :scott, :eur_gateway, "EUR" 18 | trust :bartek, :eur_gateway, "EUR" 19 | 20 | close_ledger 21 | 22 | payment :usd_gateway, :scott, ["USD", :usd_gateway, 1000] 23 | payment :eur_gateway, :bartek, ["EUR", :eur_gateway, 1000] 24 | 25 | close_ledger 26 | 27 | offer :bartek, {buy:["USD", :usd_gateway], with:["EUR", :eur_gateway]}, 1000, 1.0 28 | 29 | close_ledger 30 | 31 | offer :scott, {sell:["USD", :usd_gateway], for:["EUR", :eur_gateway]}, 500, 1.0 32 | 33 | offer :scott, {sell:["USD", :usd_gateway], for: :native}, 500, 1.0 34 | -------------------------------------------------------------------------------- /examples/path_payment_strict_send.rb: -------------------------------------------------------------------------------- 1 | account :usd_gateway 2 | account :eur_gateway 3 | account :scott 4 | account :bartek 5 | account :andrew 6 | 7 | create_account :usd_gateway, :master 8 | create_account :eur_gateway, :master 9 | create_account :scott, :master 10 | create_account :bartek, :master 11 | create_account :andrew, :master 12 | 13 | close_ledger 14 | 15 | trust :scott, :usd_gateway, "USD" 16 | trust :bartek, :eur_gateway, "EUR" 17 | trust :andrew, :usd_gateway, "USD" 18 | trust :andrew, :eur_gateway, "EUR" 19 | 20 | close_ledger 21 | 22 | payment :usd_gateway, :scott, ["USD", :usd_gateway, 1000] 23 | payment :usd_gateway, :andrew, ["USD", :usd_gateway, 200] 24 | payment :eur_gateway, :andrew, ["EUR", :eur_gateway, 200] 25 | payment :eur_gateway, :bartek, ["EUR", :eur_gateway, 1000] 26 | 27 | close_ledger 28 | 29 | offer :andrew, {buy:["USD", :usd_gateway], with:["EUR", :eur_gateway]}, 200, 1.0 30 | 31 | close_ledger 32 | 33 | path_payment_strict_send :scott, :bartek, ["EUR", :eur_gateway, 10], with: ["USD", :usd_gateway, 10], path: [] 34 | 35 | close_ledger 36 | -------------------------------------------------------------------------------- /lib/stellar_core_commander/cmd.rb: -------------------------------------------------------------------------------- 1 | module StellarCoreCommander 2 | class Cmd 3 | include Contracts 4 | 5 | Contract String => Any 6 | def initialize(working_dir) 7 | @working_dir = working_dir 8 | end 9 | 10 | Contract String, ArrayOf[String] => CmdResult 11 | def run_and_capture(cmd, args) 12 | Dir.chdir @working_dir do 13 | stringArgs = args.map{|x| "'#{x}'"}.join(" ") 14 | out = `#{cmd} #{stringArgs}` 15 | CmdResult.new($?.exitstatus == 0, out) 16 | end 17 | end 18 | 19 | Contract String, ArrayOf[String] => CmdResult 20 | def run_and_redirect(cmd, args) 21 | args += [{ 22 | out: ["stellar-core.out.log", "a"], 23 | err: ["stellar-core.err.log", "a"], 24 | }] 25 | 26 | Dir.chdir @working_dir do 27 | system(cmd, *args) 28 | end 29 | CmdResult.new($?.exitstatus == 0, nil) 30 | end 31 | 32 | Contract String, ArrayOf[String] => CmdResult 33 | def run(cmd, args) 34 | Dir.chdir @working_dir do 35 | system(cmd, *args) 36 | end 37 | CmdResult.new($?.exitstatus == 0, nil) 38 | end 39 | 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /bin/hcc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'stellar_core_commander' 4 | require 'stellar-base' 5 | require 'slop' 6 | 7 | def run 8 | $opts = Slop.parse(ARGV, :help => true) do 9 | banner 'Usage: hcc [ -r RECIPE | -s ] ' 10 | 11 | on 's', 'shell', 12 | 'start a shell' 13 | on 'r', 'recipe', 14 | 'a recipe file', 15 | argument: true 16 | on 'H', 'host', 17 | 'sets the network passphrase to configure on all running processes', 18 | argument: true, 19 | default: "https://horizon-testnet.stellar.org" 20 | on 'network-passphrase', 21 | 'sets the network passphrase to configure on all running processes', 22 | argument: true, 23 | default: Stellar::Networks::TESTNET 24 | end 25 | 26 | Stellar::default_network = $opts[:"network-passphrase"] 27 | 28 | commander = make_commander 29 | 30 | if $opts[:shell] 31 | commander.start_shell 32 | else 33 | recipe = load_recipe 34 | commander.run_recipe recipe 35 | end 36 | end 37 | 38 | 39 | def make_commander 40 | StellarCoreCommander::HorizonCommander.new $opts[:host] 41 | end 42 | 43 | def load_recipe 44 | recipe = $opts[:recipe] 45 | 46 | if recipe.blank? 47 | $stderr.puts $opts 48 | exit 1 49 | end 50 | 51 | unless File.exist?(recipe) 52 | $stderr.puts "not found: #{recipe}" 53 | exit 1 54 | end 55 | 56 | recipe 57 | end 58 | 59 | run 60 | -------------------------------------------------------------------------------- /stellar_core_commander.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'stellar_core_commander/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "stellar_core_commander" 8 | spec.version = StellarCoreCommander::VERSION 9 | spec.authors = ["Scott Fleckenstein"] 10 | spec.email = ["nullstyle@gmail.com"] 11 | spec.summary = %q{A helper gem for scripting stellar-core and horizon} 12 | spec.homepage = "" 13 | spec.license = "Apache 2.0" 14 | 15 | spec.files = `git ls-files -z`.split("\x0") 16 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 17 | spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) 18 | spec.require_paths = ["lib"] 19 | 20 | spec.add_dependency "stellar-base", ">= 0.17.0" 21 | spec.add_dependency "stellar-sdk", ">= 0.5.0" 22 | spec.add_dependency "slop", "~> 3.6.0" 23 | spec.add_dependency "faraday", "~> 0.9.1" 24 | spec.add_dependency "faraday_middleware", "~> 0.9.1" 25 | spec.add_dependency "pg", "~> 0.18.1" 26 | spec.add_dependency "sequel", "~> 5.5.0" 27 | spec.add_dependency "activesupport", "~> 6" 28 | spec.add_dependency "contracts", "~> 0.16" 29 | spec.add_dependency "typhoeus", "~> 0.8.0" 30 | spec.add_dependency "pry", "~> 0.11.3" 31 | 32 | spec.add_development_dependency "bundler", "~> 1.7" 33 | spec.add_development_dependency "rake", "~> 10.0" 34 | end 35 | -------------------------------------------------------------------------------- /examples/version_mix_consensus.rb: -------------------------------------------------------------------------------- 1 | old_image = ENV["OLD_IMAGE"] 2 | new_image = ENV["NEW_IMAGE"] 3 | 4 | raise "missing ENV['OLD_IMAGE']" unless old_image 5 | raise "missing ENV['NEW_IMAGE']" unless new_image 6 | 7 | peers = [:oldnode1, :oldnode2, :newnode1, :newnode2] 8 | 9 | process :oldnode1, peers, docker_core_image: old_image, docker_pull: true, await_sync: false 10 | process :oldnode2, peers, docker_core_image: old_image, docker_pull: true, await_sync: false 11 | process :newnode1, peers, docker_core_image: new_image, docker_pull: true, await_sync: false 12 | process :newnode2, peers, docker_core_image: new_image, docker_pull: true 13 | 14 | account :alice 15 | account :bob 16 | 17 | base_balance=0 18 | 19 | on :oldnode2 do 20 | create_account :alice, :master 21 | create_account :bob, :master 22 | while not ((account_created :bob) and (account_created :alice)) 23 | $stderr.puts "Awaiting account-creation" 24 | close_ledger 25 | end 26 | base_balance = (balance :bob) 27 | $stderr.puts "oldnode2 bob balance: #{(balance :bob)}" 28 | $stderr.puts "oldnode2 alice balance: #{(balance :alice)}" 29 | payment :master, :bob, [:native, 1000] 30 | close_ledger 31 | end 32 | 33 | on :newnode1 do 34 | $stderr.puts "newnode1 bob balance: #{(balance :bob)}" 35 | $stderr.puts "newnode1 alice balance: #{(balance :alice)}" 36 | raise if (balance :bob) != (base_balance + (1000 * Stellar::ONE)) 37 | payment :master, :alice, [:native, 1000] 38 | close_ledger 39 | check_integrity_against :oldnode1 40 | check_integrity_against :oldnode2 41 | end 42 | 43 | on :oldnode1 do 44 | $stderr.puts "oldnode1 bob balance: #{(balance :bob)}" 45 | $stderr.puts "oldnode1 alice balance: #{(balance :alice)}" 46 | raise if (balance :alice) != (base_balance + (1000 * Stellar::ONE)) 47 | check_integrity_against :newnode1 48 | check_integrity_against :newnode2 49 | end 50 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to contribute 2 | 3 | Your contributions to the Stellar network will help improve the world’s financial 4 | infrastructure, faster. 5 | 6 | We want to make it as easy as possible to contribute changes that 7 | help the Stellar network grow and thrive. There are a few guidelines that we 8 | ask contributors to follow so that we can merge your changes quickly. 9 | 10 | ## Getting Started 11 | 12 | * Create a GitHub issue for your contribution, assuming one does not already exist. 13 | * Clearly describe the issue including steps to reproduce if it is a bug. 14 | * Fork the repository on GitHub 15 | 16 | ## Making Changes 17 | 18 | * Create a topic branch from where you want to base your work. 19 | * This is usually the master branch. 20 | * Please avoid working directly on the `master` branch. 21 | * Make sure you have added the necessary tests for your changes, and make sure all tests pass. 22 | 23 | ## Submitting Changes 24 | 25 | * Sign the Contributor License Agreement. 26 | * All content, comments, and pull requests must follow the [Stellar Community Guidelines](https://www.stellar.org/community-guidelines/). 27 | * Push your changes to a topic branch in your fork of the repository. 28 | * Submit a pull request to the [stellar_core_commander repository](https://github.com/stellar/stellar-tutorials) in the Stellar organization. 29 | * Include a descriptive [commit message](https://github.com/erlang/otp/wiki/Writing-good-commit-messages). 30 | * Changes contributed via pull request should focus on a single issue at a time. 31 | * Rebase your local changes against the master branch. Resolve any conflicts that arise. 32 | 33 | At this point you're waiting on us. We like to at least comment on pull requests within three 34 | business days (and, typically, one business day). We may suggest some changes or improvements or alternatives. 35 | 36 | ## Making Trivial Changes 37 | 38 | ### Documentation 39 | For changes of a trivial nature to comments and documentation, it is not 40 | always necessary to create a new GitHub issue. In this case, it is 41 | appropriate to start the first line of a commit with 'doc' instead of 42 | an issue number. 43 | 44 | # Additional Resources 45 | 46 | * [Bug tracker (Github)](https://github.com/stellar/stellar_core_commander/issues) 47 | * Contributor License Agreement 48 | * [Explore the API](http://docs.stellarhorizon.apiary.io/) 49 | * [Readme for stellar_core_commander](https://github.com/stellar/stellar_core_commander/blob/master/README.md) 50 | * #stellar-dev IRC channel on freenode.org 51 | * #dev channel on [Slack](http://slack.stellar.org) 52 | 53 | 54 | This document is inspired by: 55 | 56 | https://github.com/puppetlabs/puppet/blob/master/CONTRIBUTING.md 57 | 58 | https://github.com/thoughtbot/factory_girl_rails/blob/master/CONTRIBUTING.md 59 | 60 | https://github.com/rust-lang/rust/blob/master/CONTRIBUTING.md -------------------------------------------------------------------------------- /lib/stellar_core_commander/container.rb: -------------------------------------------------------------------------------- 1 | require 'uri' 2 | require 'set' 3 | require 'securerandom' 4 | 5 | module StellarCoreCommander 6 | 7 | class Container 8 | include Contracts 9 | 10 | attr_accessor :image 11 | 12 | Contract Cmd, ArrayOf[String], String, String, Maybe[Func[None => Any]] => Any 13 | def initialize(cmd, args, image, name, &at_shutdown) 14 | @cmd = cmd 15 | @args = args 16 | @image = image 17 | @name = name 18 | @at_shutdown = at_shutdown 19 | end 20 | 21 | Contract ArrayOf[String], ArrayOf[String] => CmdResult 22 | def launch(arguments, command) 23 | command(@cmd.method(:run_and_redirect), %W(run -d --name #{@name}) + arguments + [@image] + command) 24 | end 25 | 26 | Contract None => CmdResult 27 | def stop 28 | command(@cmd.method(:run_and_redirect), %W(stop #{@name})) 29 | end 30 | 31 | Contract ArrayOf[String] => CmdResult 32 | def exec(arguments) 33 | command(@cmd.method(:run_and_capture), %W(exec #{@name}) + arguments) 34 | end 35 | 36 | Contract None => CmdResult 37 | def logs 38 | command(@cmd.method(:run_and_redirect), %W(logs #{@name})) 39 | end 40 | 41 | Contract None => CmdResult 42 | def pull 43 | command(@cmd.method(:run_and_redirect), %W(pull #{@image})) 44 | end 45 | 46 | Contract None => CmdResult 47 | def dump_cores 48 | command(@cmd.method(:run), %W(run --volumes-from #{@name} --rm -e MODE=local #{@image} /utils/core_file_processor.py)) 49 | command(@cmd.method(:run), %W(cp #{@name}:/cores .)) 50 | end 51 | 52 | Contract None => CmdResult 53 | def shutdown 54 | $stderr.puts "removing container #{@name} (image #{@image})" 55 | return CmdResult.new(true) unless exists? 56 | 57 | begin 58 | if @at_shutdown.is_a? Proc and exists? 59 | @at_shutdown.call 60 | end 61 | rescue Exception => e 62 | $stderr.puts "Error during at_shutdown call: #{e.class.name}): #{e.message}" 63 | ensure 64 | retries = 5 65 | loop do 66 | res = command(@cmd.method(:run_and_redirect), %W(rm -f -v #{@name}), false) 67 | return CmdResult.new(true) unless exists? 68 | return res if res.success or retries == 0 69 | retries -= 1 70 | sleep 3 71 | end 72 | end 73 | end 74 | 75 | Contract None => Bool 76 | def exists? 77 | res = command(@cmd.method(:run_and_redirect), ['inspect', '-f', '{{.Name}}', @name], false) 78 | res.success 79 | end 80 | 81 | Contract None => Bool 82 | def running? 83 | res = command(@cmd.method(:run_and_capture), ['inspect', '-f', '{{.Name}} running: {{.State.Running}}', @name], false) 84 | 85 | res.success && res.out.include?('running: true') 86 | end 87 | 88 | Contract Method, ArrayOf[String], Maybe[Bool] => CmdResult 89 | def command(run_method, arguments, mustSucceed = true) 90 | res = docker(run_method, arguments) 91 | if mustSucceed 92 | raise "Could not execute '#{arguments.join(" ")}' on #{@name}" unless res.success 93 | end 94 | res 95 | end 96 | 97 | Contract Method, ArrayOf[String] => CmdResult 98 | def docker(run_method, args) 99 | run_method.call("docker", @args + args) 100 | end 101 | end 102 | end 103 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # StellarCoreCommander 2 | 3 | A helper gem for scripting a [stellar-core](https://github.com/stellar/stellar-core). This gem provides a system of creating isolated test networks into which you can play 4 | transactions and record results. 5 | 6 | The motivation for this project comes from the testing needs of [horizon](https://github.com/stellar/horizon). Horizon uses `scc` to record the various testing scenarios that its suite uses. 7 | 8 | 9 | ## Installation 10 | 11 | Add this line to your application's Gemfile: 12 | 13 | ```ruby 14 | gem 'stellar_core_commander' 15 | ``` 16 | 17 | And then execute: 18 | 19 | $ bundle 20 | 21 | Or install it yourself as: 22 | 23 | $ gem install stellar_core_commander 24 | 25 | ## Assumptions about environment 26 | 27 | At present `scc` makes a few assumptions about the environment it runs in that you should be aware. In the event that your own environment differs from the below assumptions, `scc` will definitely break. 28 | 29 | 1. The `which` command is available on your system. 30 | 2. A working `stellar-core` binary is available on your path (or specified using the `--stellar-core-bin` flag) 31 | 3. Your system has libsodium installed 32 | 33 | If you choose to use Postgres: 34 | 1. Postgresql is installed locally and `pg_dump`, `createdb` and `dropdb` are available on your PATH 35 | 2. Postgresql is running and the current user has passwordless access to it. Running `psql postgres -c "\l"` should confirm you're setup correctly 36 | 3. Your current user has access to create and drop postgres databases. Test using: `createdb foobar && dropdb foobar` 37 | 38 | ## Usage As Command Line Tool 39 | 40 | Installing `stellar_core_commander` installs the command line tool `scc`. `scc` 41 | takes a recipe file, spins up a test network, plays the defined transactions against it, then dumps the ledger database to stdout. `scc`'s usage is like so: 42 | 43 | ```bash 44 | $ scc -r my_recipe.rb > out.sql 45 | ``` 46 | 47 | The above command will play the recipe written in `my_recipe.rb`, verify that all transactions within the recipe have succeeded, then dump the ledger database to `out.sql` 48 | 49 | ## Usage as a Library 50 | 51 | TODO 52 | 53 | ## Writing Recipes 54 | 55 | The heart of `scc` is a recipe, which is just a ruby script that executes in a context 56 | that makes it easy to play transactions against the isolated test network. Lets look at a simple recipe: 57 | 58 | ```ruby 59 | account :scott 60 | payment :master, :scott, [:native, 1000_000000] 61 | ``` 62 | 63 | Let's look at each statement in turn. `account :scott` declares a new unfunded account and binds it to the name `:scott`, which we will use in the next statement. 64 | 65 | The next statement is more complex: `payment :master, :scott, [:native, 1000_000000]`. This statement encodes "Send 1000 lumens from the :master account to the :scott account". 66 | 67 | `:master` (sometimes also called the "root" account) is a special account that is created when a new ledger is initialized. We often use it in our recipes to fund other accounts, since in a ledgers initial state the :master account has all 100 billion lumen. 68 | 69 | You can also spin up multiple stellar-core instances using `process`, where you need to specify node name, quorum sets, and possibly options. If you wish to use SQLite, you need to specify the option `database_url`. For example, the following code spins up a stellar-core instance, and generates some artificial load: 70 | ```ruby 71 | process :node, [:node], host: ENV['DOCKER1'], database_url: "sqlite3://stellar.db" 72 | on :node do 73 | generate_load_and_await_completion :create, accounts, 0, 2, 100 74 | end 75 | ``` 76 | 77 | Note that current SQLite support is very limited 78 | ### Recipe Reference 79 | 80 | All recipe execute within the context of a `StellarCoreCommander::Transactor`. [See the code for all available methods](lib/stellar_core_commander/transactor.rb). 81 | 82 | ## Example Recipes 83 | 84 | See [examples](examples). 85 | 86 | ## Contributing 87 | 88 | 1. Fork it ( https://github.com/[my-github-username]/stellar_core_commander/fork ) 89 | 2. Create your feature branch (`git checkout -b my-new-feature`) 90 | 3. Commit your changes (`git commit -am 'Add some feature'`) 91 | 4. Push to the branch (`git push origin my-new-feature`) 92 | 5. Create a new Pull Request 93 | -------------------------------------------------------------------------------- /lib/stellar_core_commander/commander.rb: -------------------------------------------------------------------------------- 1 | require 'fileutils' 2 | 3 | module StellarCoreCommander 4 | 5 | # 6 | # Commander is the object that manages running stellar-core processes. It is 7 | # responsible for creating and cleaning Process objects 8 | # 9 | class Commander 10 | include Contracts 11 | 12 | attr_reader :process_options 13 | 14 | # 15 | # Creates a new core commander 16 | # 17 | Contract Or["local", "docker"], Numeric, String, Hash => Any 18 | def initialize(process_type, base_port, destination, process_options={}) 19 | @process_type = process_type 20 | @base_port = base_port 21 | @destination = destination 22 | @process_options = process_options 23 | @processes = [] 24 | 25 | if File.exist? @destination 26 | $stderr.puts "scc is not capable of running with an existing destination directory. Please rename or remove #{@destination} and try again" 27 | exit 1 28 | end 29 | end 30 | 31 | Contract Transactor, Symbol, ArrayOf[Symbol], Hash => Process 32 | # 33 | # make_process returns a new, unlaunched Process object, bound to a new 34 | # tmpdir 35 | def make_process(transactor, name, quorum, options={}) 36 | working_dir = File.join(@destination, name.to_s) 37 | FileUtils.mkpath(working_dir) 38 | 39 | process_options = @process_options.merge({ 40 | transactor: transactor, 41 | working_dir: working_dir, 42 | name: name, 43 | base_port: @base_port + @processes.map(&:required_ports).sum, 44 | identity: Stellar::KeyPair.random, 45 | quorum: quorum, 46 | manual_close: transactor.manual_close 47 | }).merge(options) 48 | 49 | if process_options.key? :database_url 50 | db = process_options[:database_url] 51 | $stderr.puts "manually configured DB: #{db}" 52 | end 53 | 54 | process_class = case @process_type 55 | when 'local' 56 | LocalProcess 57 | when 'docker' 58 | DockerProcess 59 | else 60 | raise "Unknown process type: #{@process_type}" 61 | end 62 | 63 | process_class.new(process_options).tap do |p| 64 | @processes << p 65 | end 66 | end 67 | 68 | Contract Transactor => Process 69 | def get_root_process(transactor) 70 | if @processes.size == 0 71 | make_process transactor, :node0, [:node0] 72 | end 73 | @processes[0] 74 | end 75 | 76 | Contract None => ArrayOf[Process] 77 | def start_all_processes 78 | stopped = @processes.select(&:stopped?) 79 | return [] if stopped.empty? 80 | 81 | stopped.each(&:prepare) 82 | stopped.each(&:setup) 83 | 84 | stopped.each do |p| 85 | next if p.running? 86 | 87 | $stderr.puts "running #{p.idname} (dir:#{p.working_dir})" 88 | p.run 89 | p.wait_for_ready 90 | 91 | end 92 | end 93 | 94 | Contract String, Symbol, Num, Num, Or[Symbol, Num], Num => Any 95 | def record_performance_metrics(fname, txtype, accounts, txs, txrate, batchsize) 96 | @processes.each do |p| 97 | p.record_performance_metrics "#{p.name}_#{fname}", txtype, accounts, txs, txrate, batchsize 98 | end 99 | end 100 | 101 | Contract None => ArrayOf[Process] 102 | def require_processes_in_sync 103 | @processes.each do |p| 104 | next unless p.await_sync? 105 | begin 106 | p.wait_for_ready unless p.synced? 107 | rescue Timeout::Error 108 | @processes.each do |p2| 109 | p2.dump_scp_state 110 | p2.dump_info 111 | p2.dump_metrics 112 | raise "process #{p.name} lost sync" 113 | end 114 | end 115 | end 116 | end 117 | 118 | Contract None => Bool 119 | def check_no_process_error_metrics 120 | @processes.each do |p| 121 | p.check_no_error_metrics 122 | end 123 | true 124 | end 125 | 126 | def cleanup 127 | @processes.each(&:cleanup) 128 | @processes = [] 129 | end 130 | 131 | def cleanup_at_exit!(clean_up_destination) 132 | at_exit do 133 | $stderr.puts "cleaning up #{@processes.length} processes" 134 | cleanup 135 | FileUtils.rm_rf @destination if clean_up_destination 136 | end 137 | end 138 | end 139 | end 140 | -------------------------------------------------------------------------------- /lib/stellar_core_commander/horizon_commander.rb: -------------------------------------------------------------------------------- 1 | require 'typhoeus' 2 | require 'typhoeus/adapters/faraday' 3 | 4 | module StellarCoreCommander 5 | 6 | class HorizonCommander 7 | include Contracts 8 | include Concerns::NamedObjects 9 | include Concerns::TracksAccounts 10 | 11 | Contract String => Any 12 | def initialize(endpoint) 13 | @endpoint = endpoint 14 | @open = [] 15 | @sequences = SequenceTracker.new(self) 16 | @conn = Faraday.new(:url => @endpoint) do |faraday| 17 | faraday.request :retry, max: 2 18 | faraday.use FaradayMiddleware::FollowRedirects 19 | faraday.adapter :typhoeus 20 | end 21 | 22 | @transaction_builder = TransactionBuilder.new(self) 23 | account :master, Stellar::KeyPair.master 24 | end 25 | 26 | Contract None => Any 27 | def start_shell() 28 | 29 | @conn.in_parallel do 30 | require 'pry' 31 | Pry.start(self, quiet: true, prompt: Pry::SIMPLE_PROMPT) 32 | wait 33 | end 34 | 35 | rescue => e 36 | crash_recipe e 37 | end 38 | 39 | 40 | Contract String => Any 41 | # 42 | # Runs the provided recipe against the process identified by @process 43 | # 44 | # @param recipe_path [String] path to the recipe file 45 | # 46 | def run_recipe(recipe_path) 47 | 48 | @conn.in_parallel do 49 | load_recipe recipe_path 50 | wait 51 | end 52 | 53 | rescue => e 54 | crash_recipe e 55 | end 56 | 57 | def load_recipe(path) 58 | recipe_content = IO.read(path) 59 | instance_eval recipe_content, path, 1 60 | end 61 | 62 | def wait 63 | $stderr.puts "waiting for all open txns" 64 | @conn.parallel_manager.run 65 | 66 | @open.each do |resp| 67 | unless resp.success? 68 | require 'pry'; binding.pry 69 | raise "transaction failed" 70 | end 71 | end 72 | 73 | @open = [] 74 | end 75 | 76 | 77 | 78 | Contract ArrayOf[Symbol] => Any 79 | def self.recipe_steps(names) 80 | names.each do |name| 81 | define_method name do |*args| 82 | envelope = @transaction_builder.send(name, *args) 83 | submit_transaction envelope 84 | end 85 | end 86 | end 87 | 88 | recipe_steps [ 89 | :payment, 90 | :create_account, 91 | :trust, 92 | :change_trust, 93 | :offer, 94 | :passive_offer, 95 | :set_options, 96 | :set_flags, 97 | :clear_flags, 98 | :require_trust_auth, 99 | :add_signer, 100 | :remove_signer, 101 | :add_onetime_signer, 102 | :path_payment_strict_send, 103 | :set_master_signer_weight, 104 | :set_thresholds, 105 | :set_inflation_dest, 106 | :set_home_domain, 107 | :allow_trust, 108 | :revoke_trust, 109 | :merge_account, 110 | :inflation, 111 | :set_data, 112 | :clear_data, 113 | :bump_sequence, 114 | ] 115 | 116 | delegate :next_sequence, to: :@sequences 117 | 118 | 119 | Contract Stellar::KeyPair => Num 120 | def sequence_for(account) 121 | resp = Typhoeus.get("#{@endpoint}/accounts/#{account.address}") 122 | raise "couldn't get sequence for #{account.address}" unless resp.success? 123 | body = ActiveSupport::JSON.decode resp.body 124 | body["sequence"].to_i 125 | end 126 | 127 | Contract Symbol => Any 128 | def create_via_friendbot(account) 129 | account = get_account account 130 | @open << @conn.get("friendbot", addr: account.address) 131 | end 132 | 133 | Contract None => Stellar::Client 134 | def sdk_client 135 | @client ||= Stellar::Client.new(horizon: @endpoint) 136 | end 137 | 138 | Contract Symbol => Hyperclient::Resource 139 | def get_account_info(name) 140 | sdk_account = Stellar::Account.new(get_account(name)) 141 | sdk_client.account_info(sdk_account) 142 | end 143 | 144 | private 145 | 146 | 147 | Contract Stellar::TransactionEnvelope, Or[nil, Proc] => Any 148 | def submit_transaction(envelope, &after_confirmation) 149 | b64 = envelope.to_xdr(:base64) 150 | @open << @conn.post("transactions", tx: b64) 151 | end 152 | 153 | Contract Exception => Any 154 | def crash_recipe(e) 155 | puts 156 | puts "Error! (#{e.class.name}): #{e.message}" 157 | puts 158 | puts e.backtrace. 159 | reject{|l| l =~ %r{gems/contracts-.+?/} }. # filter contract frames 160 | join("\n") 161 | puts 162 | 163 | exit 1 164 | end 165 | 166 | end 167 | end 168 | -------------------------------------------------------------------------------- /bin/scc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'stellar_core_commander' 4 | require 'stellar-base' 5 | require 'slop' 6 | require 'tmpdir' 7 | 8 | begin 9 | require 'pry' 10 | rescue LoadError ; # no-op 11 | end 12 | 13 | def run 14 | $opts = Slop.parse(ARGV, :help => true) do 15 | banner 'Usage: scc -r RECIPE' 16 | 17 | on 'stellar-core-bin', 18 | 'a path to a stellar-core executable (defaults to `which stellar-core`)', 19 | argument: true 20 | on 'r', 'recipe', 21 | 'a recipe file', 22 | argument: true 23 | on 'p', 'process', 24 | 'method for running stellar-core', 25 | argument: true, 26 | default: 'local' 27 | on 'w', 'wait', 28 | 'wait for TERM signal before shutting down and cleaning up', 29 | argument: false, 30 | default: false 31 | on 'docker-core-image', 32 | 'docker image to use for stellar-core', 33 | argument: true, 34 | default: 'stellar/stellar-core' 35 | on 'docker-state-image', 36 | 'docker image to use for state', 37 | argument: true, 38 | default: 'stellar/stellar-core-state' 39 | on 'docker-pull', 40 | 'docker pull images before use', 41 | default: false 42 | on 'atlas', 43 | 'atlas endpoint for publishing metrics (e.g., http://192.168.59.103:7101/api/v1/publish)', 44 | argument: true, 45 | default: nil 46 | on 'atlas-interval', 47 | 'number of seconds to wait between publishing metric payloads', 48 | argument: true, 49 | default: 1 50 | on 'destination', 51 | 'where to store logs and other artifacts (default: a temporary directory)', 52 | argument: true, 53 | default: nil 54 | on 'use-s3', 55 | 'share history via s3', 56 | argument: false, 57 | default: false 58 | on 's3-history-prefix', 59 | 's3 prefix to store temp history in (default: subdir of s3://history-stg.stellar.org/dev/scc)', 60 | argument: true, 61 | default: "s3://history-stg.stellar.org/dev/scc/#{Time.now.to_i}-#{rand 100000}" 62 | on 's3-history-region', 63 | 's3 region to store temp history in (default: eu-west-1)', 64 | argument: true, 65 | default: "eu-west-1" 66 | on 'dump-root-db', 67 | 'when true, dump to root processes sql database after recipe completion', 68 | argument: false, 69 | default: false 70 | on 'network-passphrase', 71 | 'sets the network passphrase to configure on all running processes', 72 | argument: true, 73 | default: Stellar::Networks::TESTNET 74 | on 'base-port', 75 | 'all processes created by scc will use ports starting from base-port', 76 | argument: true, 77 | default: 11625 78 | on 'allow-failed-transactions', 79 | 'allow failed transactions', 80 | argument: false, 81 | default: false 82 | end 83 | 84 | Stellar::default_network = $opts[:"network-passphrase"] 85 | 86 | recipe = load_recipe 87 | commander = make_commander 88 | 89 | #run recipe 90 | transactor = StellarCoreCommander::Transactor.new(commander) 91 | 92 | #run the recipe 93 | $stderr.puts "running recipe" 94 | transactor.run_recipe recipe 95 | $stderr.puts "recipe finished" 96 | 97 | if $opts[:"dump-root-db"] 98 | file = commander.get_root_process(transactor).dump_database 99 | $stdout.puts IO.read(file) 100 | end 101 | 102 | if $opts[:wait] 103 | $stderr.puts "Waiting for INT signal..." 104 | Signal.trap("INT"){ exit } 105 | sleep 106 | end 107 | end 108 | 109 | 110 | def make_commander 111 | opts = { 112 | stellar_core_bin: $opts[:"stellar-core-bin"], 113 | docker_core_image: $opts[:"docker-core-image"], 114 | docker_state_image: $opts[:"docker-state-image"], 115 | docker_pull: $opts[:"docker-pull"], 116 | atlas: $opts[:"atlas"], 117 | atlas_interval: $opts[:"atlas-interval"].to_i, 118 | use_s3: $opts[:"use-s3"], 119 | s3_history_region: $opts[:"s3-history-region"], 120 | s3_history_prefix: $opts[:"s3-history-prefix"], 121 | network_passphrase: $opts[:"network-passphrase"], 122 | allow_failed_transactions: $opts[:"allow-failed-transactions"] 123 | } 124 | 125 | destination = $opts[:"destination"] 126 | if destination 127 | clean_up_destination = false 128 | else 129 | destination = Dir.mktmpdir("scc") 130 | FileUtils.rmdir destination 131 | clean_up_destination = true 132 | end 133 | 134 | StellarCoreCommander::Commander.new($opts[:"process"], $opts[:"base-port"].to_i, destination, opts).tap do |c| 135 | c.cleanup_at_exit!(clean_up_destination) 136 | end 137 | end 138 | 139 | def load_recipe 140 | recipe = $opts[:recipe] 141 | 142 | if recipe.blank? 143 | $stderr.puts $opts 144 | exit 1 145 | end 146 | 147 | unless File.exist?(recipe) 148 | $stderr.puts "not found: #{recipe}" 149 | exit 1 150 | end 151 | 152 | recipe 153 | end 154 | 155 | run 156 | -------------------------------------------------------------------------------- /lib/stellar_core_commander/local_process.rb: -------------------------------------------------------------------------------- 1 | module StellarCoreCommander 2 | 3 | class LocalProcess < Process 4 | include Contracts 5 | 6 | attr_reader :pid 7 | 8 | def initialize(params) 9 | raise "`host` param is unsupported on LocalProcess, please use `-p docker` for this recipe." if params[:host] 10 | $stderr.puts "Warning: Ignoring `atlas` param since LocalProcess doesn't support this." if params[:atlas] 11 | 12 | super 13 | @stellar_core_bin = params[:stellar_core_bin] 14 | @database_url = params[:database_url].try(:strip) 15 | @cmd = Cmd.new(working_dir) 16 | 17 | setup_working_dir 18 | end 19 | 20 | Contract None => Any 21 | def forcescp 22 | res = @cmd.run_and_redirect "./stellar-core", ["force-scp"] 23 | raise "Could not set --forcescp" unless res.success 24 | end 25 | 26 | Contract None => Any 27 | def initialize_history 28 | Dir.mkdir(history_dir) unless File.exists?(history_dir) 29 | res = @cmd.run_and_redirect "./stellar-core", ["new-hist", @name.to_s] 30 | raise "Could not initialize history" unless res.success 31 | end 32 | 33 | Contract None => Any 34 | def initialize_database 35 | res = @cmd.run_and_redirect "./stellar-core", ["new-db"] 36 | raise "Could not initialize db" unless res.success 37 | end 38 | 39 | Contract None => Any 40 | def create_database 41 | res = @cmd.run_and_redirect "createdb", [database_name] 42 | raise "Could not create db: #{database_name}" unless res.success 43 | end 44 | 45 | Contract None => Any 46 | def drop_database 47 | res = @cmd.run_and_redirect "dropdb", [database_name] 48 | raise "Could not drop db: #{database_name}" unless res.success 49 | end 50 | 51 | Contract None => Any 52 | def write_config 53 | IO.write("#{@working_dir}/stellar-core.cfg", config) 54 | end 55 | 56 | Contract None => String 57 | def history_dir 58 | File.expand_path("#{working_dir}/../history-archives") 59 | end 60 | 61 | Contract None => Any 62 | def setup! 63 | write_config 64 | create_database unless @keep_database or is_sqlite 65 | initialize_database 66 | initialize_history 67 | end 68 | 69 | Contract None => Num 70 | def launch_process 71 | forcescp if @forcescp 72 | launch_stellar_core 73 | end 74 | 75 | 76 | Contract None => Bool 77 | def running? 78 | return false unless @pid 79 | ::Process.kill 0, @pid 80 | true 81 | rescue Errno::ESRCH 82 | false 83 | end 84 | 85 | Contract Bool => Bool 86 | def shutdown(graceful=true) 87 | return true if !running? 88 | 89 | if graceful 90 | ::Process.kill "INT", @pid 91 | else 92 | ::Process.kill "KILL", @pid 93 | end 94 | 95 | @wait_value == 0 96 | end 97 | 98 | Contract None => Any 99 | def cleanup 100 | database.disconnect unless is_sqlite 101 | dump_database unless is_sqlite 102 | dump_scp_state 103 | dump_info 104 | dump_metrics 105 | shutdown 106 | drop_database unless @keep_database or is_sqlite 107 | end 108 | 109 | Contract None => Any 110 | def dump_database 111 | fname = "#{working_dir}/database-#{Time.now.to_i}-#{rand 100000}.sql" 112 | $stderr.puts "dumping database to #{fname}" 113 | sql = `pg_dump #{database_name} --clean --if-exists --no-owner --no-acl --inserts` 114 | File.open(fname, 'w') {|f| f.write(sql) } 115 | fname 116 | end 117 | 118 | Contract None => String 119 | def default_database_url 120 | "postgres:///#{idname}" 121 | end 122 | 123 | def crash 124 | `kill -ABRT #{@pid}` 125 | end 126 | 127 | private 128 | def launch_stellar_core 129 | Dir.chdir @working_dir do 130 | @pid = ::Process.spawn("./stellar-core", "run", 131 | :out => "stdout.txt", 132 | :err => "stderr.txt") 133 | @wait = Thread.new { 134 | @wait_value = ::Process.wait(@pid); 135 | $stderr.puts "stellar-core process exited: #{@wait_value}" 136 | } 137 | end 138 | @pid 139 | end 140 | 141 | Contract None => String 142 | def config 143 | <<-EOS.strip_heredoc 144 | PEER_PORT=#{peer_port} 145 | HTTP_PORT=#{http_port} 146 | PUBLIC_HTTP_PORT=false 147 | NODE_SEED="#{@identity.seed}" 148 | #{"NODE_IS_VALIDATOR=true" if @validate} 149 | 150 | ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING=true 151 | #{"ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true" if @accelerate_time} 152 | #{"CATCHUP_COMPLETE=true" if @catchup_complete} 153 | #{"CATCHUP_RECENT=" + @catchup_recent.to_s if @catchup_recent} 154 | 155 | DATABASE="#{dsn}" 156 | PREFERRED_PEERS=#{peer_connections} 157 | 158 | #{"MANUAL_CLOSE=true" if manual_close?} 159 | #{"COMMANDS=[\"ll?level=debug\"]" if @debug} 160 | 161 | FAILURE_SAFETY=0 162 | UNSAFE_QUORUM=true 163 | 164 | NETWORK_PASSPHRASE="#{network_passphrase}" 165 | 166 | INVARIANT_CHECKS=#{invariants.to_s} 167 | 168 | [QUORUM_SET] 169 | THRESHOLD_PERCENT=51 170 | VALIDATORS=#{quorum} 171 | 172 | #{history_sources} 173 | EOS 174 | end 175 | 176 | Contract Symbol => String 177 | def one_history_source(n) 178 | dir = "#{history_dir}/#{n}" 179 | if n == @name 180 | <<-EOS.strip_heredoc 181 | [HISTORY.#{n}] 182 | get="cp #{dir}/{0} {1}" 183 | put="cp {0} #{dir}/{1}" 184 | mkdir="mkdir -p #{dir}/{0}" 185 | EOS 186 | else 187 | name = n.to_s 188 | get = "cp #{history_dir}/%s/{0} {1}" 189 | if SPECIAL_PEERS.has_key? n 190 | name = SPECIAL_PEERS[n][:name] 191 | get = SPECIAL_PEERS[n][:get] 192 | end 193 | get.sub!('%s', name) 194 | <<-EOS.strip_heredoc 195 | [HISTORY.#{name}] 196 | get="#{get}" 197 | EOS 198 | end 199 | end 200 | 201 | Contract None => String 202 | def history_sources 203 | @quorum.map {|n| one_history_source n}.join("\n") 204 | end 205 | 206 | def setup_working_dir 207 | if @stellar_core_bin.blank? 208 | search = `which stellar-core`.strip 209 | 210 | if $?.success? 211 | @stellar_core_bin = search 212 | else 213 | $stderr.puts "Could not find a `stellar-core` binary, please use --stellar-core-bin to specify" 214 | exit 1 215 | end 216 | end 217 | 218 | FileUtils.cp(@stellar_core_bin, "#{working_dir}/stellar-core") 219 | end 220 | 221 | end 222 | end 223 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [2015] [Stellar Development Foundation] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /lib/stellar_core_commander/docker_process.rb: -------------------------------------------------------------------------------- 1 | require 'uri' 2 | require 'set' 3 | require 'securerandom' 4 | 5 | module StellarCoreCommander 6 | 7 | class DockerProcess < Process 8 | include Contracts 9 | 10 | attr_reader :docker_core_image 11 | attr_reader :docker_state_image 12 | 13 | Contract({ 14 | docker_state_image: String, 15 | docker_core_image: String, 16 | docker_pull: Bool, 17 | setup_timeout: Maybe[Num] 18 | } => Any) 19 | def initialize(params) 20 | @docker_pull = params[:docker_pull] 21 | super 22 | 23 | @heka_container = Container.new(@cmd, docker_args, "stellar/heka", heka_container_name) 24 | @state_container = Container.new(@cmd, docker_args, params[:docker_state_image], state_container_name) do 25 | dump_database 26 | end 27 | @stellar_core_container = Container.new(@cmd, docker_args, params[:docker_core_image], container_name) do 28 | dump_data 29 | end 30 | 31 | @setup_timeout = params[:setup_timeout] || 300 32 | @hist_setup = false 33 | end 34 | 35 | Contract None => Num 36 | def required_ports 37 | 3 38 | end 39 | 40 | Contract None => Any 41 | def launch_heka_container 42 | $stderr.puts "launching heka container #{heka_container_name} from image #{@heka_container.image}" 43 | @heka_container.launch(%W(--net container:#{container_name} --volumes-from #{container_name} -d), []) 44 | end 45 | 46 | Contract None => Any 47 | def launch_state_container 48 | $stderr.puts "launching state container #{state_container_name} from image #{@state_container.image}" 49 | @state_container.launch(%W(-p #{postgres_port}:5432 --env-file stellar-core.env), 50 | %W(postgres --fsync=off --full_page_writes=off --shared_buffers=512MB --work_mem=32MB)) 51 | end 52 | 53 | Contract None => Any 54 | def shutdown_state_container 55 | @state_container.shutdown 56 | end 57 | 58 | Contract None => Any 59 | def shutdown_heka_container 60 | @heka_container.shutdown 61 | end 62 | 63 | Contract None => Any 64 | def shutdown_core_container 65 | @stellar_core_container.shutdown 66 | end 67 | 68 | Contract None => Any 69 | def write_config 70 | IO.write("#{working_dir}/stellar-core.env", config) 71 | end 72 | 73 | Contract None => Any 74 | def setup! 75 | write_config 76 | unless is_sqlite 77 | launch_state_container 78 | wait_for_port postgres_port 79 | end 80 | 81 | launch_stellar_core true 82 | launch_heka_container if atlas 83 | 84 | at_exit do 85 | cleanup 86 | end 87 | 88 | counter = @setup_timeout 89 | while running? 90 | $stderr.puts "waiting for #{state_container_name} to complete setup" 91 | counter -= 1 92 | raise "setup did not complete before timeout of #{@setup_timeout}" if counter == 0 93 | sleep 1.0 94 | end 95 | @stellar_core_container.shutdown 96 | end 97 | 98 | Contract None => Any 99 | def launch_process 100 | launch_stellar_core is_sqlite 101 | end 102 | 103 | Contract None => Bool 104 | def running? 105 | @stellar_core_container.running? 106 | end 107 | 108 | Contract None => Bool 109 | def heka_container_running? 110 | @heka_core_container.running? 111 | end 112 | 113 | Contract None => Bool 114 | def state_container_running? 115 | @state_container.running? 116 | end 117 | 118 | Contract None => Any 119 | def dump_data 120 | dump_logs 121 | dump_cores 122 | dump_scp_state 123 | dump_info 124 | dump_metrics 125 | end 126 | 127 | Contract None => Any 128 | def cleanup 129 | database.disconnect unless is_sqlite 130 | dump_system_stats 131 | shutdown_core_container 132 | shutdown_state_container unless is_sqlite 133 | shutdown_heka_container if atlas 134 | end 135 | 136 | Contract None => Any 137 | def stop 138 | shutdown_core_container 139 | end 140 | 141 | Contract({ 142 | docker_core_image: String, 143 | forcescp: Maybe[Bool] 144 | } => Any) 145 | def upgrade(params) 146 | stop 147 | 148 | @stellar_core_container.image = params[:docker_core_image] 149 | @forcescp = params.fetch(:forcescp, @forcescp) 150 | $stderr.puts "upgrading docker-core-image to #{docker_core_image}" 151 | launch_stellar_core false 152 | @await_sync = true 153 | wait_for_ready 154 | end 155 | 156 | Contract None => Any 157 | def dump_logs 158 | @stellar_core_container.logs 159 | end 160 | 161 | Contract None => Any 162 | def dump_system_stats 163 | fname = "#{working_dir}/../proc-stats-#{Time.now.to_i}-#{rand 100000}.txt" 164 | uptime = IO.read("/proc/uptime").split[0] 165 | idletime = IO.read("/proc/uptime").split[1] 166 | 167 | meminfo = IO.read("/proc/meminfo") 168 | netstat = IO.read("/proc/net/dev") 169 | stat = IO.read("/proc/stat") 170 | 171 | File.open(fname, 'w') {|f| f.write("Uptime: #{uptime} \nIdle time: #{idletime} \n 172 | Mem Info: #{meminfo}\n Net stat: #{netstat}\n Other stats: #{stat}")} 173 | rescue 174 | $stderr.puts "Unable to dump stats from /proc directory. Dump stats is currently supported on Linux only. Skipping this step..." 175 | nil 176 | end 177 | 178 | Contract None => Any 179 | def dump_cores 180 | @stellar_core_container.dump_cores 181 | end 182 | 183 | Contract None => Any 184 | def dump_database 185 | unless is_sqlite 186 | fname = "#{working_dir}/database-#{Time.now.to_i}-#{rand 100000}.sql" 187 | $stderr.puts "dumping database to #{fname}" 188 | res = @state_container.exec %W(pg_dump -U #{database_user} --clean --no-owner --no-privileges #{database_name}) 189 | File.open(fname, 'w') {|f| f.write(res.out.to_s) } 190 | fname 191 | else 192 | $stderr.puts "`dump database` is not supported with SQLite." 193 | end 194 | end 195 | 196 | Contract None => String 197 | def default_database_url 198 | @database_password ||= SecureRandom.hex 199 | "postgres://postgres:#{@database_password}@#{docker_host}:#{postgres_port}/stellar" 200 | end 201 | 202 | Contract None => Num 203 | def postgres_port 204 | base_port + 2 205 | end 206 | 207 | Contract None => String 208 | def container_name 209 | "scc-#{idname}" 210 | end 211 | 212 | Contract None => String 213 | def state_container_name 214 | "scc-state-#{idname}" 215 | end 216 | 217 | Contract None => String 218 | def heka_container_name 219 | "scc-heka-#{idname}" 220 | end 221 | 222 | Contract None => String 223 | def docker_host 224 | return host if host 225 | return URI.parse(ENV['DOCKER_HOST']).host if ENV['DOCKER_HOST'] 226 | DEFAULT_HOST 227 | end 228 | 229 | Contract None => String 230 | def hostname 231 | docker_host 232 | end 233 | 234 | Contract None => Bool 235 | def docker_pull? 236 | @docker_pull 237 | end 238 | 239 | Contract None => ArrayOf[String] 240 | def aws_credentials_volume 241 | if use_s3 and File.exists?("#{ENV['HOME']}/.aws") 242 | ["-v", "#{ENV['HOME']}/.aws:/root/.aws:ro"] 243 | else 244 | [] 245 | end 246 | end 247 | 248 | Contract None => Bool 249 | def use_s3 250 | if @use_s3 251 | true 252 | else 253 | if host and (@quorum.size > 1) 254 | $stderr.puts "WARNING: multi-peer with remote docker host, but no s3; history will not be shared" 255 | end 256 | false 257 | end 258 | end 259 | 260 | Contract None => ArrayOf[String] 261 | def shared_history_volume 262 | if use_s3 263 | [] 264 | else 265 | dir = File.expand_path("#{working_dir}/../history-archives") 266 | Dir.mkdir(dir) unless File.exists?(dir) 267 | ["-v", "#{dir}:/history"] 268 | end 269 | end 270 | 271 | Contract None => String 272 | def history_get_command 273 | cmds = Set.new 274 | localget = "cp /history/%s/{0} {1}" 275 | s3get = "aws s3 --region #{@s3_history_region} cp #{@s3_history_prefix}/%s/{0} {1}" 276 | 277 | if @initial_catchup 278 | @history_peers.each do |q| 279 | if SPECIAL_PEERS.has_key? q 280 | cmds.add SPECIAL_PEERS[q][:get] 281 | end 282 | end 283 | else 284 | @quorum.each do |q| 285 | if q == @name 286 | next 287 | end 288 | if SPECIAL_PEERS.has_key? q 289 | cmds.add SPECIAL_PEERS[q][:get] 290 | elsif use_s3 291 | cmds.add s3get 292 | else 293 | cmds.add localget 294 | end 295 | end 296 | 297 | if cmds.size == 0 298 | if use_s3 299 | cmds.add s3get 300 | else 301 | cmds.add localget 302 | end 303 | end 304 | end 305 | 306 | if cmds.size != 1 307 | raise "Conflicting get commands: #{cmds.to_a.inspect}" 308 | end 309 | <<-EOS.strip_heredoc 310 | HISTORY_GET=#{cmds.to_a.first} 311 | EOS 312 | end 313 | 314 | Contract None => String 315 | def history_put_commands 316 | if @initial_catchup 317 | "" 318 | elsif has_special_peers? 319 | "" 320 | else 321 | if use_s3 322 | <<-EOS.strip_heredoc 323 | HISTORY_PUT=aws s3 --region #{@s3_history_region} cp {0} #{@s3_history_prefix}/%s/{1} 324 | EOS 325 | else 326 | <<-EOS.strip_heredoc 327 | HISTORY_PUT=cp {0} /history/%s/{1} 328 | HISTORY_MKDIR=mkdir -p /history/%s/{0} 329 | EOS 330 | end 331 | end 332 | end 333 | 334 | def prepare 335 | $stderr.puts "preparing #{idname} (dir:#{working_dir})" 336 | return unless docker_pull? 337 | @state_container.pull 338 | @stellar_core_container.pull 339 | @heka_container.pull 340 | end 341 | 342 | def crash 343 | @stellar_core_container.exec %W(pkill -ABRT stellar-core) 344 | end 345 | 346 | private 347 | def launch_stellar_core fresh 348 | $stderr.puts "launching stellar-core container #{container_name} from image #{@stellar_core_container.image}" 349 | args = [] 350 | args += %W(--volumes-from #{state_container_name}) unless is_sqlite 351 | args += aws_credentials_volume 352 | args += shared_history_volume 353 | args += %W(-p #{http_port}:#{http_port} -p #{peer_port}:#{peer_port}) 354 | args += %W(--env-file stellar-core.env) 355 | command = %W(/start #{@name}) 356 | if is_sqlite 357 | command += ["nopsql"] 358 | end 359 | if fresh 360 | if @hist_setup 361 | command += ["newdb"] 362 | else 363 | command += ["fresh", "skipstart"] 364 | end 365 | end 366 | if @forcescp 367 | command += ["forcescp"] 368 | end 369 | # must be added last, as this is requirement of stellar/stellar-core docker image 370 | if fresh and @initial_catchup 371 | command += ["catchupat", "current"] 372 | end 373 | 374 | @stellar_core_container.launch(args, command) 375 | @hist_setup = true 376 | @stellar_core_container 377 | end 378 | 379 | Contract None => String 380 | def config 381 | if is_sqlite 382 | db_config = "DATABASE=#{database_url}" 383 | else 384 | db_config = "POSTGRES_DB=#{database_name}" 385 | end 386 | 387 | ( 388 | <<-EOS.strip_heredoc 389 | #{db_config} 390 | POSTGRES_PASSWORD=#{database_password} 391 | 392 | ENVIRONMENT=scc 393 | CLUSTER_NAME=#{recipe_name} 394 | HOSTNAME=#{idname} 395 | 396 | #{@name}_POSTGRES_PORT=#{postgres_port} 397 | #{@name}_PEER_PORT=#{peer_port} 398 | #{@name}_HTTP_PORT=#{http_port} 399 | #{@name}_NODE_SEED=#{identity.seed} 400 | NODE_IS_VALIDATOR=#{@validate} 401 | 402 | #{"MANUAL_CLOSE=true" if manual_close?} 403 | 404 | ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING=true 405 | #{"ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true" if @accelerate_time} 406 | #{"CATCHUP_COMPLETE=true" if @catchup_complete} 407 | #{"CATCHUP_RECENT=" + @catchup_recent.to_s if @catchup_recent} 408 | 409 | #{"ATLAS_ADDRESS=" + atlas if atlas} 410 | 411 | METRICS_INTERVAL=#{atlas_interval} 412 | 413 | #{"COMMANDS=[\"ll?level=debug\"]" if @debug} 414 | 415 | FAILURE_SAFETY=0 416 | UNSAFE_QUORUM=true 417 | THRESHOLD_PERCENT=51 418 | 419 | PREFERRED_PEERS=#{peer_connections} 420 | VALIDATORS=#{quorum} 421 | 422 | TARGET_PEER_CONNECTION=32 423 | 424 | HISTORY_PEERS=#{history_peer_names} 425 | 426 | NETWORK_PASSPHRASE=#{network_passphrase} 427 | 428 | INVARIANT_CHECKS=#{invariants.to_s} 429 | EOS 430 | ) + history_get_command + history_put_commands 431 | end 432 | 433 | def recipe_name 434 | File.basename($opts[:recipe], '.rb') 435 | rescue TypeError 436 | 'recipe_name_not_found' 437 | end 438 | 439 | def docker_port 440 | if ENV['DOCKER_HOST'] 441 | URI.parse(ENV['DOCKER_HOST']).port 442 | else 443 | 2376 444 | end 445 | end 446 | 447 | def docker_args 448 | if host 449 | ["-H", "tcp://#{docker_host}:#{docker_port}"] 450 | else 451 | [] 452 | end 453 | end 454 | end 455 | end 456 | -------------------------------------------------------------------------------- /lib/stellar_core_commander/transaction_builder.rb: -------------------------------------------------------------------------------- 1 | require 'bigdecimal' 2 | module StellarCoreCommander 3 | 4 | class TransactionBuilder 5 | include Contracts 6 | 7 | Asset = Or[ 8 | [String, Symbol], 9 | :native, 10 | ] 11 | Amount = Or[ 12 | [String, Symbol, Or[Num,String]], 13 | [:native, Or[Num,String]], 14 | ] 15 | 16 | OfferCurrencies = Or[ 17 | {sell:Asset, for: Asset}, 18 | {buy:Asset, with: Asset}, 19 | ] 20 | 21 | Byte = And[Num, lambda{|n| (0..255).include? n}] 22 | ThresholdByte = Byte 23 | MasterWeightByte = Byte 24 | 25 | Thresholds = { 26 | low: Maybe[ThresholdByte], 27 | medium: Maybe[ThresholdByte], 28 | high: Maybe[ThresholdByte], 29 | } 30 | 31 | SetOptionsArgs = { 32 | inflation_dest: Maybe[Symbol], 33 | clear_flags: Maybe[ArrayOf[Symbol]], 34 | set_flags: Maybe[ArrayOf[Symbol]], 35 | thresholds: Maybe[Thresholds], 36 | master_weight: Maybe[MasterWeightByte], 37 | home_domain: Maybe[String], 38 | signer: Maybe[Stellar::Signer], 39 | } 40 | 41 | SignerSpec = Or[Stellar::KeyPair] 42 | 43 | StellarBaseAsset = Or[[Symbol, String, Stellar::KeyPair], [:native]] 44 | 45 | MAX_LIMIT= BigDecimal.new((2**63)-1) / Stellar::ONE 46 | 47 | Contract Or[Transactor,HorizonCommander] => Any 48 | def initialize(transactor) 49 | @transactor = transactor 50 | end 51 | 52 | Memo = Or[ 53 | Integer, 54 | String, 55 | [:id, Integer], 56 | [:text, String], 57 | [:hash, String], 58 | [:return, String], 59 | ] 60 | 61 | CommonOptions = {memo: Maybe[Memo]} 62 | PaymentOptions = Or[ 63 | CommonOptions, 64 | CommonOptions.merge({path: ArrayOf[Asset], with:Amount}), 65 | ] 66 | 67 | Contract Symbol, Symbol, Amount, PaymentOptions => Any 68 | def payment(from, to, amount, options={}) 69 | from = get_account from 70 | to = get_account to 71 | 72 | attrs = { 73 | account: from, 74 | destination: to, 75 | memo: options[:memo], 76 | sequence: next_sequence(from), 77 | amount: normalize_amount(amount), 78 | } 79 | 80 | tx = if options[:with] 81 | attrs[:with] = normalize_amount(options[:with]) 82 | attrs[:path] = options[:path].map{|p| make_asset p} 83 | Stellar::Transaction.path_payment_strict_receive(attrs) 84 | else 85 | Stellar::Transaction.payment(attrs) 86 | end 87 | 88 | tx.to_envelope(from) 89 | end 90 | 91 | Contract Symbol, Symbol, Amount, PaymentOptions => Any 92 | def path_payment_strict_receive(from, to, amount, options={}) 93 | from = get_account from 94 | to = get_account to 95 | 96 | attrs = { 97 | account: from, 98 | destination: to, 99 | memo: options[:memo], 100 | sequence: next_sequence(from), 101 | amount: normalize_amount(amount), 102 | with: normalize_amount(options[:with]), 103 | path: options[:path].map{|p| make_asset p} 104 | } 105 | 106 | tx = Stellar::Transaction.path_payment_strict_receive(attrs) 107 | 108 | tx.to_envelope(from) 109 | end 110 | 111 | Contract Symbol, Symbol, Amount, PaymentOptions => Any 112 | def path_payment_strict_send(from, to, amount, options={}) 113 | from = get_account from 114 | to = get_account to 115 | 116 | attrs = { 117 | account: from, 118 | destination: to, 119 | memo: options[:memo], 120 | sequence: next_sequence(from), 121 | amount: normalize_amount(amount), 122 | with: normalize_amount(options[:with]), 123 | path: options[:path].map{|p| make_asset p} 124 | } 125 | 126 | tx = Stellar::Transaction.path_payment_strict_send(attrs) 127 | 128 | tx.to_envelope(from) 129 | end 130 | 131 | Contract Symbol, Symbol, Or[String,Num] => Any 132 | def create_account(account, funder=:master, starting_balance=1000) 133 | account = get_account account 134 | funder = get_account funder 135 | 136 | Stellar::Transaction.create_account({ 137 | account: funder, 138 | destination: account, 139 | sequence: next_sequence(funder), 140 | starting_balance: starting_balance, 141 | }).to_envelope(funder) 142 | end 143 | 144 | Contract Symbol, Symbol, String => Any 145 | def trust(account, issuer, code) 146 | change_trust account, issuer, code, MAX_LIMIT 147 | end 148 | 149 | Contract Symbol, Symbol, String, Num => Any 150 | def change_trust(account, issuer, code, limit) 151 | account = get_account account 152 | 153 | Stellar::Transaction.change_trust({ 154 | account: account, 155 | sequence: next_sequence(account), 156 | line: make_asset([code, issuer]), 157 | limit: limit 158 | }).to_envelope(account) 159 | end 160 | 161 | Contract Symbol, Symbol, String, Bool => Any 162 | def allow_trust(account, trustor, code, authorize=true) 163 | asset = make_asset([code, account]) 164 | account = get_account account 165 | trustor = get_account trustor 166 | 167 | 168 | Stellar::Transaction.allow_trust({ 169 | account: account, 170 | sequence: next_sequence(account), 171 | asset: asset, 172 | trustor: trustor, 173 | authorize: authorize, 174 | }).to_envelope(account) 175 | end 176 | 177 | Contract Symbol, Symbol, String => Any 178 | def revoke_trust(account, trustor, code) 179 | allow_trust(account, trustor, code, false) 180 | end 181 | 182 | Contract Symbol, OfferCurrencies, Or[String,Num], Or[String,Num] => Any 183 | def offer(account, currencies, amount, price) 184 | account = get_account account 185 | 186 | buying, selling, price, amount = extract_offer(currencies, price, amount) 187 | 188 | Stellar::Transaction.manage_offer({ 189 | account: account, 190 | sequence: next_sequence(account), 191 | selling: selling, 192 | buying: buying, 193 | amount: amount, 194 | price: price, 195 | }).to_envelope(account) 196 | end 197 | 198 | Contract Symbol, OfferCurrencies, Or[String,Num], Or[String,Num] => Any 199 | def passive_offer(account, currencies, amount, price) 200 | account = get_account account 201 | 202 | buying, selling, price, amount = extract_offer(currencies, price, amount) 203 | 204 | Stellar::Transaction.create_passive_offer({ 205 | account: account, 206 | sequence: next_sequence(account), 207 | selling: selling, 208 | buying: buying, 209 | amount: amount, 210 | price: price, 211 | }).to_envelope(account) 212 | end 213 | 214 | Contract Symbol, SetOptionsArgs => Any 215 | def set_options(account, args) 216 | account = get_account account 217 | 218 | params = { 219 | account: account, 220 | sequence: next_sequence(account), 221 | } 222 | 223 | if args[:inflation_dest].present? 224 | params[:inflation_dest] = get_account args[:inflation_dest] 225 | end 226 | 227 | if args[:set_flags].present? 228 | params[:set] = make_account_flags(args[:set_flags]) 229 | end 230 | 231 | if args[:clear_flags].present? 232 | params[:clear] = make_account_flags(args[:clear_flags]) 233 | end 234 | 235 | if args[:master_weight].present? 236 | params[:master_weight] = args[:master_weight] 237 | end 238 | 239 | if args[:thresholds].present? 240 | params[:low_threshold] = args[:thresholds][:low] 241 | params[:med_threshold] = args[:thresholds][:medium] 242 | params[:high_threshold] = args[:thresholds][:high] 243 | end 244 | 245 | if args[:home_domain].present? 246 | params[:home_domain] = args[:home_domain] 247 | end 248 | 249 | if args[:signer].present? 250 | params[:signer] = args[:signer] 251 | end 252 | 253 | tx = Stellar::Transaction.set_options(params) 254 | tx.to_envelope(account) 255 | end 256 | 257 | 258 | Contract Symbol, ArrayOf[Symbol] => Any 259 | def set_flags(account, flags) 260 | set_options account, set_flags: flags 261 | end 262 | 263 | Contract Symbol, ArrayOf[Symbol] => Any 264 | def clear_flags(account, flags) 265 | set_options account, clear_flags: flags 266 | end 267 | 268 | Contract Symbol => Any 269 | def require_trust_auth(account) 270 | set_flags account, [:auth_required_flag] 271 | end 272 | 273 | Contract Symbol, SignerSpec, Num => Any 274 | def add_signer(account, key, weight) 275 | sk = Stellar::SignerKey.new :signer_key_type_ed25519, key.raw_public_key 276 | 277 | set_options account, signer: Stellar::Signer.new({ 278 | key: sk, 279 | weight: weight 280 | }) 281 | end 282 | 283 | Contract Symbol, SignerSpec => Any 284 | def remove_signer(account, key) 285 | add_signer account, key, 0 286 | end 287 | 288 | Contract Symbol, String, Num => Any 289 | def add_onetime_signer(account, preimage, weight) 290 | set_options account, signer: Stellar::Signer.new({ 291 | key: Stellar::SignerKey.onetime_signer(preimage), 292 | weight: weight 293 | }) 294 | end 295 | 296 | Contract(Symbol, MasterWeightByte => Any) 297 | def set_master_signer_weight(account, weight) 298 | set_options account, master_weight: weight 299 | end 300 | 301 | Contract(Symbol, Thresholds => Any) 302 | def set_thresholds(account, thresholds) 303 | set_options account, thresholds: thresholds 304 | end 305 | 306 | Contract(Symbol, Symbol => Any) 307 | def set_inflation_dest(account, destination) 308 | set_options account, inflation_dest: destination 309 | end 310 | 311 | Contract(Symbol, String => Any) 312 | def set_home_domain(account, domain) 313 | set_options account, home_domain: domain 314 | end 315 | 316 | Contract Symbol, Symbol => Any 317 | def merge_account(account, into) 318 | account = get_account account 319 | into = get_account into 320 | 321 | tx = Stellar::Transaction.account_merge({ 322 | account: account, 323 | sequence: next_sequence(account), 324 | destination: into, 325 | }) 326 | 327 | tx.to_envelope(account) 328 | end 329 | 330 | 331 | Contract None => Any 332 | def inflation(account=:master) 333 | account = get_account account 334 | 335 | tx = Stellar::Transaction.inflation({ 336 | account: account, 337 | sequence: next_sequence(account), 338 | }) 339 | 340 | tx.to_envelope(account) 341 | end 342 | 343 | 344 | Contract Symbol, String, String => Any 345 | def set_data(account, name, value) 346 | account = get_account account 347 | 348 | tx = Stellar::Transaction.manage_data({ 349 | account: account, 350 | sequence: next_sequence(account), 351 | name: name, 352 | value: value, 353 | }) 354 | 355 | tx.to_envelope(account) 356 | end 357 | 358 | 359 | Contract Symbol, String => Any 360 | def clear_data(account, name) 361 | account = get_account account 362 | 363 | tx = Stellar::Transaction.manage_data({ 364 | account: account, 365 | sequence: next_sequence(account), 366 | name: name, 367 | }) 368 | 369 | tx.to_envelope(account) 370 | end 371 | 372 | Contract Symbol, Num => Any 373 | def bump_sequence(account, sequence) 374 | account = get_account account 375 | Stellar::Transaction.bump_sequence({ 376 | account: account, 377 | sequence: next_sequence(account), 378 | bump_to: sequence 379 | }).to_envelope(account) 380 | end 381 | 382 | private 383 | 384 | delegate :get_account, to: :@transactor 385 | delegate :next_sequence, to: :@transactor 386 | 387 | Contract Asset => StellarBaseAsset 388 | def make_asset(input) 389 | if input == :native 390 | return [:native] 391 | end 392 | 393 | code, issuer = *input 394 | issuer = get_account issuer 395 | 396 | [:alphanum4, code, issuer] 397 | end 398 | 399 | def make_account_flags(flags=nil) 400 | flags ||= [] 401 | flags.map{|f| Stellar::AccountFlags.send(f)} 402 | end 403 | 404 | Contract Thresholds => String 405 | def make_thresholds_word(thresholds) 406 | thresholds.values_at(:master_weight, :low, :medium, :high).pack("C*") 407 | end 408 | 409 | Contract Amount => Any 410 | def normalize_amount(amount) 411 | return amount if amount.first == :native 412 | 413 | amount = [:alphanum4] + amount 414 | amount[2] = get_account(amount[2]) # translate issuer to account 415 | 416 | amount 417 | end 418 | 419 | Contract OfferCurrencies, Or[String,Num], Or[String,Num] => [StellarBaseAsset, StellarBaseAsset, Or[String,Num], Or[String,Num]] 420 | def extract_offer(currencies, price, amount) 421 | if currencies.has_key?(:sell) 422 | buying = make_asset currencies[:for] 423 | selling = make_asset currencies[:sell] 424 | else 425 | buying = make_asset currencies[:buy] 426 | selling = make_asset currencies[:with] 427 | price, amount = invert_offer_price_and_amount(price, amount) 428 | end 429 | 430 | [buying, selling, price, amount] 431 | end 432 | 433 | Contract Or[String,Num], Or[String,Num] => [String, String] 434 | def invert_offer_price_and_amount(price, amount) 435 | price = BigDecimal.new(price, 7) 436 | price = (1 / price) 437 | 438 | amount = BigDecimal.new(amount, 7) 439 | amount = (amount / price).floor 440 | 441 | [price.to_s("F"), amount.to_s] 442 | end 443 | end 444 | end 445 | -------------------------------------------------------------------------------- /lib/stellar_core_commander/transactor.rb: -------------------------------------------------------------------------------- 1 | require 'fileutils' 2 | module StellarCoreCommander 3 | 4 | 5 | # 6 | # A transactor plays transactions against a stellar-core test node. 7 | # 8 | # 9 | class Transactor 10 | include Contracts 11 | include Concerns::NamedObjects 12 | include Concerns::TracksAccounts 13 | 14 | class FailedTransaction < StandardError ; end 15 | class MissingTransaction < StandardError ; end 16 | 17 | attr_reader :manual_close 18 | 19 | Contract Commander => Any 20 | def initialize(commander) 21 | @commander = commander 22 | @transaction_builder = TransactionBuilder.new(self) 23 | @manual_close = false 24 | 25 | account :master, Stellar::KeyPair.master 26 | end 27 | 28 | def require_process_running 29 | if @process.nil? 30 | @process = @commander.get_root_process self 31 | 32 | if get_named(@process.name).blank? 33 | add_named @process.name, @process 34 | end 35 | end 36 | 37 | @commander.start_all_processes 38 | @commander.require_processes_in_sync 39 | end 40 | 41 | def shutdown(*args) 42 | @process.shutdown *args 43 | end 44 | 45 | Contract String => Any 46 | # 47 | # Runs the provided recipe against the process identified by @process 48 | # 49 | # @param recipe_path [String] path to the recipe file 50 | # 51 | def run_recipe(recipe_path) 52 | recipe_content = IO.read(recipe_path) 53 | instance_eval recipe_content, recipe_path, 1 54 | rescue => e 55 | crash_recipe e 56 | end 57 | 58 | 59 | # Contract Nil => Any 60 | # # 61 | # # Runs the provided recipe against the process identified by @process 62 | # # 63 | # # @param recipe_path [String] path to the recipe file 64 | # # 65 | # def transaction(&block) 66 | # tx = TransactionBuilder.new(self) 67 | # tx.run_recipe block 68 | 69 | # envelope = tx.build 70 | # submit_transaction envelope 71 | # end 72 | 73 | Contract Symbol => Any 74 | # recipe_step is a helper method to define 75 | # a method that follows the common procedure of executing a recipe step: 76 | # 77 | # 1. ensure all processes are running 78 | # 2. build the envelope by forwarding to the operation builder 79 | # 3. submit the envelope to the process 80 | # 81 | # @param name [Symbol] the method to be defined and delegated to @transaction_builder 82 | def self.recipe_step(name) 83 | define_method name do |*args, &block| 84 | require_process_running 85 | envelope = @transaction_builder.send(name, *args) 86 | 87 | if block.present? 88 | block.call envelope 89 | end 90 | 91 | submit_transaction envelope 92 | end 93 | end 94 | 95 | 96 | # 97 | # @see StellarCoreCommander::TransactionBuilder#payment 98 | def payment(*args, &block) 99 | require_process_running 100 | envelope = @transaction_builder.payment(*args) 101 | 102 | if block.present? 103 | block.call envelope 104 | end 105 | 106 | submit_transaction envelope do |result| 107 | payment_result = result.result.results!.first.tr!.value 108 | raise FailedTransaction unless payment_result.code.value >= 0 109 | end 110 | end 111 | 112 | # 113 | # @see StellarCoreCommander::TransactionBuilder#path_payment_strict_send 114 | def path_payment_strict_receive(*args, &block) 115 | require_process_running 116 | envelope = @transaction_builder.path_payment_strict_receive(*args) 117 | 118 | if block.present? 119 | block.call envelope 120 | end 121 | 122 | submit_transaction envelope do |result| 123 | payment_result = result.result.results!.first.tr!.value 124 | raise FailedTransaction unless payment_result.code.value >= 0 125 | end 126 | end 127 | 128 | # 129 | # @see StellarCoreCommander::TransactionBuilder#path_payment_strict_send 130 | def path_payment_strict_send(*args, &block) 131 | require_process_running 132 | envelope = @transaction_builder.path_payment_strict_send(*args) 133 | 134 | if block.present? 135 | block.call envelope 136 | end 137 | 138 | submit_transaction envelope do |result| 139 | payment_result = result.result.results!.first.tr!.value 140 | raise FailedTransaction unless payment_result.code.value >= 0 141 | end 142 | end 143 | 144 | # @see StellarCoreCommander::TransactionBuilder#create_account 145 | recipe_step :create_account 146 | 147 | # @see StellarCoreCommander::TransactionBuilder#trust 148 | recipe_step :trust 149 | 150 | # @see StellarCoreCommander::TransactionBuilder#change_trust 151 | recipe_step :change_trust 152 | 153 | # @see StellarCoreCommander::TransactionBuilder#offer 154 | recipe_step :offer 155 | 156 | # @see StellarCoreCommander::TransactionBuilder#passive_offer 157 | recipe_step :passive_offer 158 | 159 | # @see StellarCoreCommander::TransactionBuilder#set_options 160 | recipe_step :set_options 161 | 162 | # @see StellarCoreCommander::TransactionBuilder#set_flags 163 | recipe_step :set_flags 164 | 165 | # @see StellarCoreCommander::TransactionBuilder#clear_flags 166 | recipe_step :clear_flags 167 | 168 | # @see StellarCoreCommander::TransactionBuilder#require_trust_auth 169 | recipe_step :require_trust_auth 170 | 171 | # @see StellarCoreCommander::TransactionBuilder#add_signer 172 | recipe_step :add_signer 173 | 174 | # @see StellarCoreCommander::TransactionBuilder#remove_signer 175 | recipe_step :remove_signer 176 | 177 | # @see StellarCoreCommander::TransactionBuilder#add_onetime_signer 178 | recipe_step :add_onetime_signer 179 | 180 | # @see StellarCoreCommander::TransactionBuilder#set_master_signer_weight 181 | recipe_step :set_master_signer_weight 182 | 183 | 184 | # @see StellarCoreCommander::TransactionBuilder#set_thresholds 185 | recipe_step :set_thresholds 186 | 187 | # @see StellarCoreCommander::TransactionBuilder#set_inflation_dest 188 | recipe_step :set_inflation_dest 189 | 190 | # @see StellarCoreCommander::TransactionBuilder#set_home_domain 191 | recipe_step :set_home_domain 192 | 193 | # @see StellarCoreCommander::TransactionBuilder#allow_trust 194 | recipe_step :allow_trust 195 | 196 | # @see StellarCoreCommander::TransactionBuilder#revoke_trust 197 | recipe_step :revoke_trust 198 | 199 | # @see StellarCoreCommander::TransactionBuilder#merge_account 200 | recipe_step :merge_account 201 | 202 | # @see StellarCoreCommander::TransactionBuilder#inflation 203 | recipe_step :inflation 204 | 205 | # @see StellarCoreCommander::TransactionBuilder#set_data 206 | recipe_step :set_data 207 | 208 | # @see StellarCoreCommander::TransactionBuilder#clear_data 209 | recipe_step :clear_data 210 | 211 | # @see StellarCoreCommander::OperationBuilder#bump_sequence 212 | recipe_step :bump_sequence 213 | 214 | Contract None => Any 215 | # 216 | # Triggers a ledger close. Any unvalidated transaction will 217 | # be validated, which will trigger an error if any fail to be validated 218 | # 219 | def close_ledger 220 | require_process_running 221 | nretries = 3 222 | loop do 223 | residual = [] 224 | @process.close_ledger 225 | @process.unverified.each do |eb| 226 | begin 227 | envelope, after_confirmation = *eb 228 | result = validate_transaction envelope 229 | after_confirmation.call(result) if after_confirmation 230 | rescue MissingTransaction 231 | $stderr.puts "Failed to validate tx: #{Convert.to_hex envelope.tx.hash}" 232 | $stderr.puts "could not be found in txhistory table on process #{@process.name}" 233 | residual << eb 234 | rescue FailedTransaction 235 | if not @commander.process_options[:allow_failed_transactions] 236 | $stderr.puts "Failed to validate tx: #{Convert.to_hex envelope.tx.hash}" 237 | $stderr.puts "failed result: #{result.to_xdr(:base64)}" 238 | residual << eb 239 | end 240 | end 241 | end 242 | if residual.empty? 243 | @process.unverified.clear 244 | break 245 | end 246 | if nretries == 0 247 | raise "Missing or failed txs after multiple close attempts" 248 | else 249 | $stderr.puts "retrying close" 250 | nretries -= 1 251 | @process.unverified = residual 252 | residual = [] 253 | end 254 | end 255 | end 256 | 257 | Contract None => Num 258 | def ledger_num 259 | require_process_running 260 | @process.ledger_num 261 | end 262 | 263 | Contract Num, Symbol => Any 264 | def catchup(ledger, mode=:minimal) 265 | require_process_running 266 | @process.catchup ledger, mode 267 | end 268 | 269 | Contract None => Any 270 | def crash 271 | @process.crash 272 | end 273 | 274 | Contract None => Any 275 | def cleanup_all 276 | @commander.cleanup 277 | @process = nil 278 | end 279 | 280 | Contract Symbol => Process 281 | def get_process(name) 282 | @named[name].tap do |found| 283 | unless found.is_a?(Process) 284 | raise ArgumentError, "#{name.inspect} is not process" 285 | end 286 | end 287 | end 288 | 289 | Contract Symbol, Num, Num, Or[Symbol, Num], Num => Any 290 | def start_load_generation(mode=:create, accounts=10000000, txs=10000000, txrate=500, batchsize=100) 291 | $stderr.puts "starting load generation: #{mode} mode, #{accounts} accounts, #{txs} txs, #{txrate} tx/s, #{batchsize} batchsize" 292 | @process.start_load_generation mode, accounts, txs, txrate, batchsize 293 | end 294 | 295 | Contract None => Bool 296 | def load_generation_complete 297 | @process.load_generation_complete 298 | end 299 | 300 | Contract Symbol, Num, Num, Or[Symbol, Num], Num => Any 301 | def generate_load_and_await_completion(mode, accounts, txs, txrate, batchsize) 302 | runs = @process.load_generation_runs 303 | start_load_generation mode, accounts, txs, txrate, batchsize 304 | num_retries = if mode == :create then accounts else txs end 305 | 306 | retry_until_true retries: num_retries do 307 | txs = @process.transactions_applied 308 | r = @process.load_generation_runs 309 | tps = @process.transactions_per_second 310 | ops = @process.operations_per_second 311 | $stderr.puts "loadgen runs: #{r}, ledger: #{ledger_num}, accounts: #{accounts}, txs: #{txs}, actual tx/s: #{tps} op/s: #{ops}" 312 | r != runs 313 | end 314 | end 315 | 316 | Contract None => Hash 317 | def metrics 318 | @process.metrics 319 | end 320 | 321 | Contract None => Any 322 | def clear_metrics 323 | @process.clear_metrics 324 | end 325 | 326 | Contract None => Num 327 | def protocol_version 328 | @process.get_protocol_version 329 | end 330 | 331 | Contract String, Symbol, Num, Num, Or[Symbol, Num], Num => Any 332 | def record_performance_metrics(fname, txtype, accounts, txs, txrate, batchsize) 333 | @commander.record_performance_metrics fname, txtype, accounts, txs, txrate, batchsize 334 | end 335 | 336 | Contract Symbol, ArrayOf[Symbol], Hash => Process 337 | def process(name, quorum=[name], options={}) 338 | 339 | if @manual_close and quorum.size != 1 340 | raise "Cannot use `process` with multi-node quorum, this recipe has previously declared `use_manual_close`." 341 | end 342 | 343 | $stderr.puts "creating process #{name}" 344 | p = @commander.make_process self, name, quorum, options 345 | $stderr.puts "process #{name} is #{p.idname}" 346 | add_named name, p 347 | end 348 | 349 | Contract Symbol, Proc => Any 350 | def on(process_name) 351 | require_process_running 352 | tmp = @process 353 | p = get_process process_name 354 | $stderr.puts "executing steps on #{p.idname}" 355 | @process = p 356 | yield 357 | ensure 358 | @process = tmp 359 | end 360 | 361 | def retry_until_true(**opts, &block) 362 | retries = opts[:retries] || 20 363 | timeout = opts[:timeout] || 3 364 | while retries > 0 365 | b = begin yield block end 366 | if b 367 | return b 368 | end 369 | retries -= 1 370 | $stderr.puts "sleeping #{timeout} secs, #{retries} retries left" 371 | sleep timeout 372 | end 373 | raise "Ran out of retries while waiting for success" 374 | end 375 | 376 | Contract Stellar::KeyPair => Num 377 | def next_sequence(account) 378 | require_process_running 379 | base_sequence = @process.sequence_for(account) 380 | inflight_count = @process.unverified.select{|e| e.first.tx.source_account == account.public_key}.length 381 | 382 | base_sequence + inflight_count + 1 383 | end 384 | 385 | Contract Or[Symbol, Stellar::KeyPair] => Bool 386 | def account_created(account) 387 | require_process_running 388 | if account.is_a?(Symbol) 389 | account = get_account(account) 390 | end 391 | begin 392 | @process.account_row(account) 393 | return true 394 | rescue 395 | return false 396 | end 397 | end 398 | 399 | Contract Or[Symbol, Stellar::KeyPair] => Num 400 | def balance(account) 401 | require_process_running 402 | if account.is_a?(Symbol) 403 | account = get_account(account) 404 | end 405 | raise "no process!" unless @process 406 | @process.balance_for(account) 407 | end 408 | 409 | Contract None => Any 410 | def use_manual_close() 411 | $stderr.puts "using manual_close mode" 412 | @manual_close = true 413 | end 414 | 415 | Contract None => Bool 416 | def check_no_error_metrics 417 | @commander.check_no_process_error_metrics 418 | end 419 | 420 | Contract Bool => Bool 421 | def check_no_error_metrics_param(internal_error) 422 | @commander.check_no_error_metrics_param(internal_error) 423 | end 424 | 425 | Contract Or[Num, String] => Any 426 | def set_upgrades(protocolversion) 427 | @process.set_upgrades protocolversion 428 | end 429 | 430 | Contract None => Any 431 | def check_uses_latest_protocol_version 432 | latest_protocol_version = @process.get_protocol_version 433 | current_protocol_version = @process.get_current_protocol_version 434 | if latest_protocol_version != current_protocol_version 435 | raise "protocol version not upgraded to #{latest_protocol_version}, still at #{current_protocol_version} with #{@process.ledger_num}" 436 | end 437 | end 438 | 439 | Contract ArrayOf[Or[Symbol, Process]] => Bool 440 | def check_equal_ledger_objects(processes) 441 | raise "no process!" unless @process 442 | for p in processes 443 | if p.is_a?(Symbol) 444 | p = get_process(p) 445 | end 446 | @process.check_equal_ledger_objects(p) 447 | end 448 | true 449 | end 450 | 451 | Contract Or[Symbol, Process] => Any 452 | def check_ledger_sequence_matches(other) 453 | raise "no process!" unless @process 454 | if other.is_a?(Symbol) 455 | other = get_process(other) 456 | end 457 | @process.check_ledger_sequence_matches(other) 458 | end 459 | 460 | Contract Or[Symbol, Process] => Any 461 | def check_integrity_against(other) 462 | check_no_error_metrics 463 | check_equal_ledger_objects [other] 464 | check_ledger_sequence_matches other 465 | end 466 | 467 | private 468 | 469 | Contract Stellar::TransactionEnvelope, Or[nil, Proc] => Any 470 | def submit_transaction(envelope, &after_confirmation) 471 | require_process_running 472 | b64 = envelope.to_xdr(:base64) 473 | 474 | # submit to process 475 | @process.submit_transaction b64 476 | 477 | # register envelope for validation after ledger is closed 478 | @process.unverified << [envelope, after_confirmation] 479 | end 480 | 481 | Contract Stellar::TransactionEnvelope => Stellar::TransactionResult 482 | def validate_transaction(envelope) 483 | raw_hash = envelope.tx.hash 484 | hex_hash = Convert.to_hex(raw_hash) 485 | base64_envelope = Convert.to_base64(Stellar::TransactionEnvelope.to_xdr(envelope)) 486 | 487 | base64_result = @process.transaction_result(hex_hash) 488 | 489 | raise MissingTransaction if base64_result.blank? 490 | 491 | raw_result = Convert.from_base64(base64_result) 492 | 493 | pair = Stellar::TransactionResultPair.from_xdr(raw_result) 494 | result = pair.result 495 | 496 | if not @commander.process_options[:allow_failed_transactions] 497 | # ensure success for every operation 498 | expected = Stellar::TransactionResultCode.tx_success 499 | actual = result.result.code 500 | raise "tx: #{hex_hash} #{base64_envelope} transaction failed: #{base64_result}" unless expected == actual 501 | end 502 | 503 | result 504 | end 505 | 506 | Contract Exception => Any 507 | def crash_recipe(e) 508 | puts 509 | puts "Error! (#{e.class.name}): #{e.message}" 510 | puts 511 | puts e.backtrace. 512 | reject{|l| l =~ %r{gems/contracts-.+?/} }. # filter contract frames 513 | join("\n") 514 | puts 515 | 516 | exit 1 517 | end 518 | 519 | end 520 | end 521 | -------------------------------------------------------------------------------- /lib/stellar_core_commander/process.rb: -------------------------------------------------------------------------------- 1 | require 'set' 2 | require 'socket' 3 | require 'timeout' 4 | require 'csv' 5 | 6 | module StellarCoreCommander 7 | 8 | class UnexpectedDifference < StandardError 9 | def initialize(kind, x, y) 10 | @kind = kind 11 | @x = x 12 | @y = y 13 | end 14 | def message 15 | "Unexpected difference in #{@kind}: #{@x} != #{@y}" 16 | end 17 | end 18 | 19 | class Process 20 | include Contracts 21 | 22 | class Crash < StandardError ; end 23 | class AlreadyRunning < StandardError ; end 24 | 25 | attr_reader :transactor 26 | attr_reader :working_dir 27 | attr_reader :name 28 | attr_reader :base_port 29 | attr_reader :identity 30 | attr_reader :server 31 | attr_accessor :unverified 32 | attr_reader :host 33 | attr_reader :atlas 34 | attr_reader :atlas_interval 35 | attr_reader :network_passphrase 36 | 37 | DEFAULT_HOST = '127.0.0.1' 38 | 39 | SPECIAL_PEERS = { 40 | :testnet1 => { 41 | :dns => "core-testnet1.stellar.org", 42 | :key => "GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y", 43 | :name => "core_testnet_001", 44 | :get => "wget -q http://history.stellar.org/prd/core-testnet/%s/{0} -O {1}" 45 | }, 46 | :testnet2 => { 47 | :dns => "core-testnet2.stellar.org", 48 | :key => "GCUCJTIYXSOXKBSNFGNFWW5MUQ54HKRPGJUTQFJ5RQXZXNOLNXYDHRAP", 49 | :name => "core_testnet_002", 50 | :get => "wget -q http://history.stellar.org/prd/core-testnet/%s/{0} -O {1}" 51 | }, 52 | :testnet3 => { 53 | :dns => "core-testnet3.stellar.org", 54 | :key => "GC2V2EFSXN6SQTWVYA5EPJPBWWIMSD2XQNKUOHGEKB535AQE2I6IXV2Z", 55 | :name => "core_testnet_003", 56 | :get => "wget -q http://history.stellar.org/prd/core-testnet/%s/{0} -O {1}" 57 | }, 58 | :pubnet1 => { 59 | :dns => "core-live4.stellar.org", 60 | :key => "GCGB2S2KGYARPVIA37HYZXVRM2YZUEXA6S33ZU5BUDC6THSB62LZSTYH", 61 | :name => "core_live_001", 62 | :get => "curl -sf http://history.stellar.org/prd/core-live/%s/{0} -o {1}" 63 | }, 64 | :pubnet2 => { 65 | :dns => "core-live5.stellar.org", 66 | :key => "GCM6QMP3DLRPTAZW2UZPCPX2LF3SXWXKPMP3GKFZBDSF3QZGV2G5QSTK", 67 | :name => "core_live_002", 68 | :get => "curl -sf http://history.stellar.org/prd/core-live/%s/{0} -o {1}" 69 | }, 70 | :pubnet3 => { 71 | :dns => "core-live6.stellar.org", 72 | :key => "GABMKJM6I25XI4K7U6XWMULOUQIQ27BCTMLS6BYYSOWKTBUXVRJSXHYQ", 73 | :name => "core_live_003", 74 | :get => "curl -sf http://history.stellar.org/prd/core-live/%s/{0} -o {1}" 75 | } 76 | } 77 | 78 | Contract({ 79 | transactor: Transactor, 80 | working_dir: String, 81 | name: Symbol, 82 | base_port: Num, 83 | identity: Stellar::KeyPair, 84 | quorum: ArrayOf[Symbol], 85 | peers: Maybe[ArrayOf[Symbol]], 86 | history_peers: Maybe[ArrayOf[Symbol]], 87 | manual_close: Maybe[Bool], 88 | await_sync: Maybe[Bool], 89 | accelerate_time: Maybe[Bool], 90 | catchup_complete: Maybe[Bool], 91 | catchup_recent: Maybe[Num], 92 | initial_catchup: Maybe[Bool], 93 | forcescp: Maybe[Bool], 94 | validate: Maybe[Bool], 95 | host: Maybe[String], 96 | atlas: Maybe[String], 97 | atlas_interval: Num, 98 | use_s3: Bool, 99 | s3_history_prefix: String, 100 | s3_history_region: String, 101 | database_url: Maybe[String], 102 | keep_database: Maybe[Bool], 103 | debug: Maybe[Bool], 104 | wait_timeout: Maybe[Num], 105 | network_passphrase: Maybe[String], 106 | protocol_version: Maybe[Or[Num, String]], 107 | delay_upgrade: Maybe[Bool], 108 | } => Any) 109 | def initialize(params) 110 | #config 111 | @transactor = params[:transactor] 112 | @working_dir = params[:working_dir] 113 | @cmd = Cmd.new(@working_dir) 114 | @name = params[:name] 115 | @base_port = params[:base_port] 116 | @identity = params[:identity] 117 | @quorum = params[:quorum] 118 | @peers = params[:peers] || @quorum 119 | @history_peers = params[:history_peers] || @peers 120 | @manual_close = params[:manual_close] || false 121 | @await_sync = @manual_close ? false : params.fetch(:await_sync, true) 122 | @accelerate_time = params[:accelerate_time] || false 123 | @catchup_complete = params[:catchup_complete] || false 124 | @catchup_recent = params[:catchup_recent] || false 125 | @initial_catchup = params[:initial_catchup] || false 126 | @forcescp = params.fetch(:forcescp, true) 127 | @validate = params.fetch(:validate, true) 128 | @host = params[:host] 129 | @atlas = params[:atlas] 130 | @atlas_interval = params[:atlas_interval] 131 | @use_s3 = params[:use_s3] 132 | @s3_history_region = params[:s3_history_region] 133 | @s3_history_prefix = params[:s3_history_prefix] 134 | @database_url = params[:database_url] 135 | @keep_database = params[:keep_database] 136 | @debug = params[:debug] 137 | @wait_timeout = params[:wait_timeout] || 10 138 | @network_passphrase = params[:network_passphrase] || Stellar::Networks::TESTNET 139 | @protocol_version = params[:protocol_version] || "latest" 140 | @delay_upgrade = params[:delay_upgrade] || false 141 | 142 | # state 143 | @unverified = [] 144 | @sequences = Hash.new {|hash, account| hash[account] = (account_row account)[:seqnum]} 145 | @is_setup = false 146 | 147 | if not @quorum.include? @name 148 | @quorum = @quorum + [@name] 149 | end 150 | 151 | if not @peers.include? @name 152 | @peers = @peers + [@name] 153 | end 154 | 155 | if not @history_peers.include? @name 156 | @history_peers = @history_peers + [@name] 157 | end 158 | 159 | @server = Faraday.new(url: "http://#{hostname}:#{http_port}") do |conn| 160 | conn.request :url_encoded 161 | conn.adapter Faraday.default_adapter 162 | end 163 | end 164 | 165 | Contract None => Bool 166 | def has_special_peers? 167 | @peers.any? {|q| SPECIAL_PEERS.has_key? q} 168 | end 169 | 170 | Contract ArrayOf[Symbol], Symbol, Bool, Proc => ArrayOf[String] 171 | def node_map_or_special_field(nodes, field, include_self) 172 | specials = nodes.select {|q| SPECIAL_PEERS.has_key? q} 173 | if specials.empty? 174 | (nodes.map do |q| 175 | if q != @name or include_self 176 | yield q 177 | end 178 | end).compact 179 | else 180 | specials.map {|q| SPECIAL_PEERS[q][field]} 181 | end 182 | end 183 | 184 | Contract None => ArrayOf[String] 185 | def quorum 186 | node_map_or_special_field @quorum, :key, @validate do |q| 187 | @transactor.get_process(q).identity.address 188 | end 189 | end 190 | 191 | Contract None => ArrayOf[String] 192 | def history_peer_names 193 | node_map_or_special_field @history_peers, :name, true do |q| 194 | q.to_s 195 | end 196 | end 197 | 198 | Contract None => ArrayOf[String] 199 | def peer_connections 200 | node_map_or_special_field @peers, :dns, false do |q| 201 | p = @transactor.get_process(q) 202 | "#{p.hostname}:#{p.peer_port}" 203 | end 204 | end 205 | 206 | Contract None => Num 207 | def required_ports 208 | 2 209 | end 210 | 211 | Contract None => String 212 | def idname 213 | "#{@name}-#{@base_port}-#{@identity.address[0..5]}" 214 | end 215 | 216 | Contract None => String 217 | def database_url 218 | if @database_url.present? 219 | @database_url.strip 220 | else 221 | default_database_url 222 | end 223 | end 224 | 225 | Contract None => Bool 226 | def is_sqlite 227 | database_url.start_with? "sqlite" 228 | end 229 | 230 | Contract None => URI::Generic 231 | def database_uri 232 | URI.parse(database_url) 233 | end 234 | 235 | Contract None => Maybe[String] 236 | def database_host 237 | database_uri.host 238 | end 239 | 240 | Contract None => String 241 | def database_name 242 | database_uri.path[1..-1] 243 | end 244 | 245 | Contract None => Sequel::Database 246 | def database 247 | @database ||= Sequel.connect(database_url) 248 | end 249 | 250 | Contract None => Maybe[String] 251 | def database_user 252 | database_uri.user 253 | end 254 | 255 | Contract None => Maybe[String] 256 | def database_password 257 | database_uri.password 258 | end 259 | 260 | Contract None => Num 261 | def get_protocol_version 262 | v = info.fetch("protocol_version", -1) 263 | raise "Unable to retrieve protocol version. Try again later or pass version manually." if v == -1 264 | v 265 | end 266 | 267 | Contract None => Num 268 | def get_current_protocol_version 269 | (info_field "ledger")["version"] 270 | rescue 271 | -1 272 | end 273 | 274 | Contract None => String 275 | def database_port 276 | database_uri.port || "5432" 277 | end 278 | 279 | Contract None => String 280 | def dsn 281 | if is_sqlite 282 | base = database_url 283 | else 284 | base = "postgresql://dbname=#{database_name} " 285 | base << " user=#{database_user}" if database_user.present? 286 | base << " password=#{database_password}" if database_password.present? 287 | base << " host=#{database_host} port=#{database_port}" if database_host.present? 288 | end 289 | base 290 | end 291 | 292 | Contract None => Any 293 | def wait_for_ready 294 | $stderr.puts "waiting #{sync_timeout} seconds for stellar-core #{idname} ( catchup_complete: #{@catchup_complete}, catchup_recent: #{@catchup_recent}, special_peers: #{has_special_peers?} )" 295 | Timeout.timeout(sync_timeout) do 296 | loop do 297 | break if synced? || (!await_sync? && !booting?) 298 | raise Process::Crash, "process #{name} has crashed while waiting for being #{await_sync? ? 'synced' : 'ready'}" if crashed? 299 | $stderr.puts "waiting until stellar-core #{idname} is #{await_sync? ? 'synced' : 'ready'} (state: #{info_field 'state'}, SCP quorum: #{scp_quorum_num}, Status: #{info_status})" 300 | sleep 1 301 | end 302 | end 303 | $stderr.puts "Wait is over! stellar-core #{idname} is #{await_sync? ? 'synced' : 'ready'} (state: #{info_field 'state'}, SCP quorum: #{scp_quorum_num}, Status: #{info_status})" 304 | end 305 | 306 | Contract None => Bool 307 | def manual_close? 308 | @manual_close 309 | end 310 | 311 | Contract None => Bool 312 | def close_ledger 313 | raise "#{idname}: close_ledger is not supported for SQLite DB. Please switch to PostgreSQL." if is_sqlite 314 | 315 | prev_ledger = latest_ledger 316 | next_ledger = prev_ledger + 1 317 | 318 | Timeout.timeout(close_timeout) do 319 | 320 | server.get("manualclose") if manual_close? 321 | 322 | loop do 323 | raise Process::Crash, "process #{name} has crashed while waiting for ledger close" if crashed? 324 | current_ledger = latest_ledger 325 | 326 | case 327 | when current_ledger >= next_ledger 328 | break 329 | else 330 | $stderr.puts "#{idname} waiting for ledger #{next_ledger} (current: #{current_ledger}, SCP quorum: #{scp_quorum_num}, Status: #{info_status})" 331 | sleep 0.5 332 | end 333 | end 334 | end 335 | @sequences.clear 336 | $stderr.puts "#{idname} closed #{latest_ledger}" 337 | 338 | true 339 | end 340 | 341 | Contract Or[Num, String] => Any 342 | def set_upgrades(protocolversion="latest") 343 | 344 | if protocolversion == "latest" 345 | version = get_protocol_version 346 | else 347 | version = protocolversion 348 | end 349 | 350 | timestamp = Time.now.utc.strftime('%FT%TZ') 351 | response = server.get("/upgrades?mode=set&upgradetime=#{timestamp}&maxtxsize=1000000&protocolversion=#{version}") 352 | response = response.body.downcase 353 | if response.include? "exception" 354 | $stderr.puts "Did not submit upgrades: #{response}" 355 | end 356 | end 357 | 358 | Contract Num, Symbol => Any 359 | def catchup(ledger, mode) 360 | server.get("/catchup?ledger=#{ledger}&mode=#{mode}") 361 | end 362 | 363 | Contract None => Hash 364 | def info 365 | info! 366 | rescue 367 | {} 368 | end 369 | 370 | def info! 371 | response = server.get("/info") 372 | body = ActiveSupport::JSON.decode(response.body) 373 | body["info"] 374 | end 375 | 376 | Contract String => Any 377 | def info_field(k) 378 | i = info 379 | i[k] 380 | rescue 381 | false 382 | end 383 | 384 | Contract None => Bool 385 | def synced? 386 | (info_field "state") == "Synced!" 387 | end 388 | 389 | Contract None => Bool 390 | def booting? 391 | s = (info_field "state") 392 | return !s || s == "Booting" 393 | end 394 | 395 | Contract None => Num 396 | def ledger_num 397 | (info_field "ledger")["num"] 398 | rescue 399 | 0 400 | end 401 | 402 | Contract None => Any 403 | def scp_quorum_info 404 | (info_field "quorum") 405 | rescue 406 | false 407 | end 408 | 409 | Contract None => String 410 | def info_status 411 | s = info_field "status" 412 | v = "#{s}" 413 | return v == "" ? "[]" : v 414 | rescue 415 | "[]" 416 | end 417 | 418 | Contract None => Num 419 | def scp_quorum_num 420 | q = scp_quorum_info 421 | # In 11.2 we changed the representation of 422 | # the /info endpoint's "quorum" field; it now 423 | # contains quorum.qset.ledger 424 | if q.has_key? "qset" 425 | q["qset"]["ledger"].to_i 426 | else 427 | q.keys[0].to_i 428 | end 429 | rescue 430 | 2 431 | end 432 | 433 | Contract None => Bool 434 | def await_sync? 435 | @await_sync 436 | end 437 | 438 | Contract None => Hash 439 | def metrics 440 | response = server.get("/metrics") 441 | body = ActiveSupport::JSON.decode(response.body) 442 | body["metrics"] 443 | rescue 444 | {} 445 | end 446 | 447 | Contract None => Any 448 | def clear_metrics 449 | server.get("/clearmetrics") 450 | rescue 451 | nil 452 | end 453 | 454 | Contract String => Num 455 | def metrics_count(k) 456 | m = metrics 457 | m[k]["count"] 458 | rescue 459 | 0 460 | end 461 | 462 | Contract String => Num 463 | def metrics_1m_rate(k) 464 | m = metrics 465 | m[k]["1_min_rate"] 466 | rescue 467 | 0 468 | end 469 | 470 | Contract String => Any 471 | def dump_server_query(s) 472 | fname = "#{working_dir}/#{s}-#{Time.now.to_i}-#{rand 100000}.json" 473 | $stderr.puts "dumping server query #{fname}" 474 | response = server.get("/#{s}") 475 | File.open(fname, 'w') {|f| f.write(response.body) } 476 | rescue 477 | nil 478 | end 479 | 480 | Contract None => Any 481 | def dump_metrics 482 | dump_server_query("metrics") 483 | end 484 | 485 | METRICS_HEADER = [ 486 | 'Time', 487 | 'Type', 488 | 'Accounts', 489 | 'Expected Txs', 490 | 'Applied Txs', 491 | 'Tx Rate', 492 | 'Batchsize', 493 | 'Txs/Ledger Mean', 494 | 'Txs/Ledger StdDev', 495 | 'Load Step Rate', 496 | 'Load Step Mean', 497 | 'Nominate Mean', 498 | 'Nominate Min', 499 | 'Nominate Max', 500 | 'Nominate StdDev', 501 | 'Nominate Median', 502 | 'Nominate 75th', 503 | 'Nominate 95th', 504 | 'Nominate 99th', 505 | 'Prepare Mean', 506 | 'Prepare Min', 507 | 'Prepare Max', 508 | 'Prepare StdDev', 509 | 'Prepare Median', 510 | 'Prepare 75th', 511 | 'Prepare 95th', 512 | 'Prepare 99th', 513 | 'Close Mean', 514 | 'Close Min', 515 | 'Close Max', 516 | 'Close StdDev', 517 | 'Close Median', 518 | 'Close 75th', 519 | 'Close 95th', 520 | 'Close 99th', 521 | 'Close Rate', 522 | ] 523 | 524 | Contract String, Symbol, Num, Num, Or[Symbol, Num], Num => Any 525 | def record_performance_metrics(fname, txtype, accounts, txs, txrate, batchsize) 526 | m = metrics 527 | fname = "#{working_dir}/#{fname}" 528 | timestamp = Time.now.strftime('%Y-%m-%d_%H:%M:%S.%L') 529 | 530 | run_data = [timestamp, txtype, accounts, txs, transactions_applied, txrate, batchsize] 531 | run_data.push(m["ledger.transaction.count"]["mean"]) 532 | run_data.push(m["ledger.transaction.count"]["stddev"]) 533 | 534 | if m.key?("loadgen.step.submit") 535 | run_data.push(m["loadgen.step.submit"]["mean_rate"]) 536 | run_data.push(m["loadgen.step.submit"]["mean"]) 537 | else 538 | run_data.push("NA") 539 | run_data.push("NA") 540 | end 541 | 542 | metric_fields = ["scp.timing.nominated", "scp.timing.externalized", "ledger.ledger.close"] 543 | metric_fields.each { |field| 544 | run_data.push(m[field]["mean"]) 545 | run_data.push(m[field]["min"]) 546 | run_data.push(m[field]["max"]) 547 | run_data.push(m[field]["stddev"]) 548 | run_data.push(m[field]["median"]) 549 | run_data.push(m[field]["75%"]) 550 | run_data.push(m[field]["95%"]) 551 | run_data.push(m[field]["99%"]) 552 | } 553 | 554 | run_data.push(m["ledger.ledger.close"]["mean_rate"]) 555 | 556 | write_csv fname, METRICS_HEADER unless File.file?(fname) 557 | if METRICS_HEADER.length == run_data.length 558 | write_csv fname, run_data 559 | else 560 | raise "#{@name}: METRICS_HEADER and run_data have different number of columns." 561 | end 562 | end 563 | 564 | Contract String, Array => Any 565 | def write_csv(fname, data) 566 | CSV.open(fname, 'a', {:col_sep => "\t"}) do |csv| 567 | csv << data 568 | end 569 | end 570 | 571 | Contract None => Any 572 | def dump_info 573 | dump_server_query("info") 574 | end 575 | 576 | Contract None => Any 577 | def dump_scp_state 578 | dump_server_query("scp") 579 | end 580 | 581 | Contract None => ArrayOf[String] 582 | def invariants 583 | [".*"] 584 | end 585 | 586 | Contract None => Bool 587 | def check_no_error_metrics 588 | check_no_error_metrics_param(true) 589 | end 590 | 591 | Contract Bool => Bool 592 | def check_no_error_metrics_param(internal_error) 593 | m = metrics 594 | metric_names = ["scp.envelope.invalidsig", 595 | "history.publish.failure", 596 | "ledger.invariant.failure"] 597 | if internal_error 598 | metric_names.append("ledger.transaction.internal-error") 599 | end 600 | for metric in metric_names 601 | c = m[metric]["count"] rescue 0 602 | if c != 0 603 | raise "nonzero metrics count for #{metric}: #{c}" 604 | end 605 | end 606 | 607 | true 608 | end 609 | 610 | Contract Symbol, Num, Num, Or[Symbol, Num], Num => Any 611 | def start_load_generation(mode, accounts, txs, txrate, batchsize) 612 | server.get("/generateload?mode=#{mode}&accounts=#{accounts}&txs=#{txs}&txrate=#{txrate}&batchsize=#{batchsize}") 613 | end 614 | 615 | Contract None => Num 616 | def load_generation_runs 617 | metrics_count "loadgen.run.complete" 618 | end 619 | 620 | Contract None => Num 621 | def transactions_applied 622 | metrics_count "ledger.transaction.apply" 623 | end 624 | 625 | Contract None => Num 626 | def transactions_per_second 627 | metrics_1m_rate "ledger.transaction.apply" 628 | end 629 | 630 | Contract None => Num 631 | def operations_per_second 632 | metrics_1m_rate "ledger.operation.apply" 633 | end 634 | 635 | Contract None => Num 636 | def http_port 637 | base_port 638 | end 639 | 640 | Contract None => Num 641 | def peer_port 642 | base_port + 1 643 | end 644 | 645 | Contract String => Any 646 | def submit_transaction(envelope_hex) 647 | response = server.get("tx", blob: envelope_hex) 648 | body = ActiveSupport::JSON.decode(response.body) 649 | 650 | if body["status"] == "ERROR" 651 | xdr = Convert.from_base64(body["error"]) 652 | result = Stellar::TransactionResult.from_xdr(xdr) 653 | raise "transaction on #{idname} failed: #{result.inspect}" 654 | end 655 | 656 | end 657 | 658 | Contract Stellar::KeyPair => Any 659 | def account_row(account) 660 | row = database[:accounts].where(:accountid => account.address).first 661 | raise "Missing account in #{idname}'s database: #{account.address}" unless row 662 | row 663 | end 664 | 665 | Contract Stellar::KeyPair => Num 666 | def sequence_for(account) 667 | @sequences[account] 668 | end 669 | 670 | Contract Stellar::KeyPair => Num 671 | def balance_for(account) 672 | (account_row account)[:balance] 673 | end 674 | 675 | Contract None => Num 676 | def latest_ledger 677 | database[:ledgerheaders].max(:ledgerseq) 678 | end 679 | 680 | Contract String => Any 681 | def db_store_state(name) 682 | database.select(:state).from(:storestate).filter(statename: name).first[:state] 683 | end 684 | 685 | Contract None => String 686 | def latest_ledger_hash 687 | s_lcl = db_store_state("lastclosedledger") 688 | t_lcl = database.select(:ledgerhash) 689 | .from(:ledgerheaders) 690 | .filter(:ledgerseq=>latest_ledger).first[:ledgerhash] 691 | raise "inconsistent last-ledger hashes in db: #{t_lcl} vs. #{s_lcl}" if t_lcl != s_lcl 692 | s_lcl 693 | end 694 | 695 | Contract None => Any 696 | def history_archive_state 697 | ActiveSupport::JSON.decode(db_store_state("historyarchivestate")) 698 | end 699 | 700 | Contract None => Num 701 | def account_count 702 | database.fetch("SELECT count(*) FROM accounts").first[:count] 703 | end 704 | 705 | Contract None => Num 706 | def trustline_count 707 | database.fetch("SELECT count(*) FROM trustlines").first[:count] 708 | end 709 | 710 | Contract None => Num 711 | def offer_count 712 | database.fetch("SELECT count(*) FROM offers").first[:count] 713 | end 714 | 715 | Contract None => ArrayOf[Any] 716 | def ten_accounts 717 | database.fetch("SELECT * FROM accounts ORDER BY accountid LIMIT 10").all 718 | end 719 | 720 | Contract None => ArrayOf[Any] 721 | def ten_offers 722 | database.fetch("SELECT * FROM offers ORDER BY sellerid LIMIT 10").all 723 | end 724 | 725 | Contract None => ArrayOf[Any] 726 | def ten_trustlines 727 | database.fetch("SELECT * FROM trustlines ORDER BY accountid, issuer, assetcode LIMIT 10").all 728 | end 729 | 730 | Contract String, Any, Any => nil 731 | def check_equal(kind, x, y) 732 | raise UnexpectedDifference.new(kind, x, y) if x != y 733 | end 734 | 735 | Contract String, ArrayOf[Any], ArrayOf[Any] => nil 736 | def check_equal_by_column(kind, x, y) 737 | x.zip(y).each do |rowx, rowy| 738 | rowx.each do |key, val| 739 | raise UnexpectedDifference.new(key, x, y) if (rowy.has_key?(key) and val != rowy[key]) 740 | end 741 | rowy.each do |key, val| 742 | raise UnexpectedDifference.new(key, x, y) if (rowx.has_key?(key) and val != rowx[key]) 743 | end 744 | end 745 | return 746 | end 747 | 748 | Contract Process => nil 749 | def check_equal_ledger_objects(other) 750 | raise "#{idname}: check_equal_ledger_objects is not supported with SQLite DB option." if is_sqlite 751 | 752 | check_equal "account count", account_count, other.account_count 753 | check_equal "trustline count", trustline_count, other.trustline_count 754 | check_equal "offer count", offer_count, other.offer_count 755 | 756 | check_equal_by_column "ten accounts", ten_accounts, other.ten_accounts 757 | check_equal_by_column "ten trustlines", ten_trustlines, other.ten_trustlines 758 | check_equal_by_column "ten offers", ten_offers, other.ten_offers 759 | end 760 | 761 | Contract Process => Any 762 | def check_ledger_sequence_matches(other) 763 | raise "#{idname}: check_ledger_sequence_matches is not supported with SQLite DB option." if is_sqlite 764 | 765 | q = "SELECT ledgerseq, ledgerhash FROM ledgerheaders ORDER BY ledgerseq" 766 | our_headers = database.fetch(q).all 767 | other_headers = other.database.fetch(q).all 768 | our_ledger_seq_numbers = our_headers.map.with_index { |x| x[:ledgerseq] } 769 | other_ledger_seq_numbers = other_headers.map { |x| x[:ledgerseq] } 770 | common_ledger_seq_numbers = our_ledger_seq_numbers.to_set & other_ledger_seq_numbers 771 | our_hash = {} 772 | our_headers.each do |row| 773 | if common_ledger_seq_numbers.include?(row[:ledgerseq]) 774 | our_hash[row[:ledgerseq]] = row[:ledgerhash] 775 | end 776 | end 777 | other_hash = {} 778 | other_headers.each do |row| 779 | if common_ledger_seq_numbers.include?(row[:ledgerseq]) 780 | other_hash[row[:ledgerseq]] = row[:ledgerhash] 781 | end 782 | end 783 | common_ledger_seq_numbers.each do |ledger_seq_numbers| 784 | check_equal "ledger hashes", other_hash[ledger_seq_numbers], our_hash[ledger_seq_numbers] 785 | end 786 | end 787 | 788 | Contract String => Maybe[String] 789 | def transaction_result(hex_hash) 790 | row = database[:txhistory].where(txid:hex_hash).first 791 | return if row.blank? 792 | row[:txresult] 793 | end 794 | 795 | Contract None => String 796 | def hostname 797 | host || DEFAULT_HOST 798 | end 799 | 800 | Contract None => Num 801 | def close_timeout 802 | 150.0 803 | end 804 | 805 | Contract None => Num 806 | def sync_timeout 807 | if has_special_peers? and @catchup_complete 808 | # catchup-complete can take quite a while on testnet or pubnet; for now, 809 | # give such tests 36 hours. May require a change in strategy later. 810 | 3600.0 * 36 811 | elsif has_special_peers? 812 | # testnet and pubnet have relatively more complex history 813 | # we give ourself: 814 | # 3 checkpoints + 100 minutes to apply buckets + 0.5 second per ledger replayed 815 | (5.0 * 64 * 3) + (100 * 60) + (@catchup_recent ? (0.5 * @catchup_recent): 0) 816 | else 817 | # Checkpoints are made every 64 ledgers = 320s on a normal network, 818 | # or every 8 ledgers = 8s on an accelerated-time network; we give you 819 | # 3 checkpoints to make it to a sync (~16min) before giving up. The 820 | # accelerated-time variant tends to need more tries due to S3 not 821 | # admitting writes instantaneously, so we do not use a tighter bound 822 | # for that case, just use the same 16min value, despite commonly 823 | # succeeding in 20s or less. 824 | 320.0 * 3 825 | end 826 | end 827 | 828 | Contract None => Bool 829 | def stopped? 830 | !running? 831 | end 832 | 833 | Contract None => Bool 834 | def launched? 835 | !!@launched 836 | end 837 | 838 | Contract None => Bool 839 | def crashed? 840 | launched? && stopped? 841 | end 842 | 843 | Contract Num => Bool 844 | def port_open?(port) 845 | begin 846 | Timeout::timeout(1) do 847 | begin 848 | $stderr.puts "#{idname} waiting for #{hostname}: #{port}" 849 | s = TCPSocket.new(hostname, port) 850 | s.close 851 | $stderr.puts "#{idname} ready on #{hostname}: #{port}" 852 | return true 853 | rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH 854 | return false 855 | end 856 | end 857 | rescue Timeout::Error 858 | end 859 | 860 | return false 861 | end 862 | 863 | Contract None => Bool 864 | def http_port_open? 865 | port_open? http_port 866 | end 867 | 868 | Contract None => Bool 869 | def peer_port_open? 870 | port_open? peer_port 871 | end 872 | 873 | Contract None => Any 874 | def prepare 875 | # noop by default, implement in subclass to customize behavior 876 | nil 877 | end 878 | 879 | Contract None => Any 880 | def run 881 | raise Process::AlreadyRunning, "already running!" if running? 882 | raise Process::Crash, "process #{name} has crashed. cannot run process again" if crashed? 883 | 884 | setup 885 | launch_process 886 | @launched = true 887 | 888 | wait_for_http 889 | if not @delay_upgrade 890 | set_upgrades @protocol_version 891 | end 892 | end 893 | 894 | Contract None => Any 895 | def setup 896 | if not @is_setup 897 | setup! 898 | @is_setup = true 899 | end 900 | end 901 | 902 | # Dumps the database of the process to the working directory, returning the path to the file written to 903 | Contract None => String 904 | def dump_database 905 | raise NotImplementedError, "implement in subclass" 906 | end 907 | 908 | private 909 | Contract None => Any 910 | def launch_process 911 | raise NotImplementedError, "implement in subclass" 912 | end 913 | 914 | Contract None => Any 915 | def setup! 916 | raise NotImplementedError, "implement in subclass" 917 | end 918 | 919 | Contract None => Any 920 | def wait_for_http 921 | wait_for_port http_port 922 | 923 | @wait_timeout.times do 924 | return if info! rescue sleep 1.0 925 | end 926 | 927 | raise "failed to get a successful info response after #{@wait_timeout} attempts" 928 | end 929 | 930 | Contract Num => Any 931 | def wait_for_port (port) 932 | @wait_timeout.times do 933 | return if port_open?(port) 934 | sleep 1.0 935 | end 936 | 937 | raise "port #{port} remained closed after #{@wait_timeout} attempts" 938 | end 939 | end 940 | end 941 | --------------------------------------------------------------------------------