├── .gitignore ├── .travis.yml ├── Gemfile ├── History.txt ├── LICENSE ├── README.md ├── Rakefile ├── bin ├── budlabel ├── budplot ├── budtimelines ├── budvis └── rebl ├── bud.gemspec ├── docs ├── README.md ├── bfs.md ├── bfs_arch.png ├── bloom-loop.png ├── cheat.md ├── getstarted.md ├── intro.md ├── modules.md ├── operational.md ├── rebl.md ├── ruby_hooks.md └── visualizations.md ├── examples ├── README.md ├── basics │ ├── hello.rb │ └── paths.rb └── chat │ ├── README.md │ ├── chat.rb │ ├── chat_protocol.rb │ └── chat_server.rb ├── lib ├── bud.rb └── bud │ ├── aggs.rb │ ├── bud_meta.rb │ ├── collections.rb │ ├── depanalysis.rb │ ├── errors.rb │ ├── executor │ ├── README.rescan │ ├── elements.rb │ ├── group.rb │ └── join.rb │ ├── graphs.rb │ ├── labeling │ ├── bloomgraph.rb │ ├── budplot_style.rb │ └── labeling.rb │ ├── lattice-core.rb │ ├── lattice-lib.rb │ ├── meta_algebra.rb │ ├── metrics.rb │ ├── monkeypatch.rb │ ├── rebl.rb │ ├── rewrite.rb │ ├── rtrace.rb │ ├── server.rb │ ├── source.rb │ ├── state.rb │ ├── storage │ ├── dbm.rb │ └── zookeeper.rb │ ├── version.rb │ ├── viz.rb │ └── viz_util.rb └── test ├── perf ├── join_bench.rb ├── ring.rb └── scratch_derive.rb ├── tc_aggs.rb ├── tc_attr_rewrite.rb ├── tc_callback.rb ├── tc_channel.rb ├── tc_collections.rb ├── tc_dbm.rb ├── tc_delta.rb ├── tc_errors.rb ├── tc_execmodes.rb ├── tc_exists.rb ├── tc_halt.rb ├── tc_inheritance.rb ├── tc_interface.rb ├── tc_joins.rb ├── tc_labeling.rb ├── tc_lattice.rb ├── tc_mapvariants.rb ├── tc_meta.rb ├── tc_metrics.rb ├── tc_module.rb ├── tc_nest.rb ├── tc_new_executor.rb ├── tc_notin.rb ├── tc_rebl.rb ├── tc_schemafree.rb ├── tc_sort.rb ├── tc_temp.rb ├── tc_terminal.rb ├── tc_timer.rb ├── tc_wc.rb ├── tc_zookeeper.rb ├── test_common.rb ├── text └── ulysses.txt └── ts_bud.rb /.gitignore: -------------------------------------------------------------------------------- 1 | Gemfile.lock 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 3 | - "1.9.3" 4 | - "2.2.0" 5 | - "2.3.4" 6 | - "2.4.1" 7 | 8 | before_install: 9 | - sudo apt-get update -qq 10 | - sudo apt-get install -qq -y graphviz 11 | 12 | script: bundle exec "cd test; ruby ts_bud.rb" 13 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | gemspec 3 | -------------------------------------------------------------------------------- /History.txt: -------------------------------------------------------------------------------- 1 | == 0.9.8 / ??? 2 | 3 | * Fix bug in the stratification algorithm 4 | * Fix bug with insertions into stdio inside bootstrap blocks 5 | * Improve error message when <~ applied to a collection type that doesn't 6 | support it (#316) 7 | * Fix rescan logic for Zookeeper-backed collections (#317) 8 | * Fix rescan logic for chained negation operators in a single rule 9 | * Support + operator for concatenating tuples 10 | * More consistent behavior for #==, #eql?, and #hash methods on Bud tuple 11 | objects (TupleStruct) 12 | * Fix intermittent EM::ConnectionNotBound exceptions during the Bud shutdown 13 | sequence 14 | * Improve stratification algorithm to be slightly more aggressive; that is, we 15 | can place certain rules in an earlier strata than we were previously 16 | (partially fixes #277) 17 | * Add Rakefile to simplify common development operations (Joel VanderWerf, #324) 18 | 19 | == 0.9.7 / 2013-04-22 20 | 21 | * Avoid raising an exception when Bud is used with Ruby 2.0. There isn't 22 | anything special that Bud itself needs to do to support Ruby 2.0, but 23 | RubyParser doesn't properly support 2.0 yet. So using 2.0-only syntax in Bud 24 | rules is unlikely to work (until RubyParser is updated), but no other problems 25 | have been observed. 26 | * Reject <= on collections from outside Bloom rules (#289, Aaron Davidson). The 27 | previous behavior was error-prone: inserting into a scratch collection via <= 28 | was not rejected but the inserted data would immediately be discarded at the 29 | beginning of the next tick. Hence, it is simpler to require <+ or <~ for all 30 | insertions from outside Bloom rules. 31 | * Fix bug in join operators whose qualifiers reference collections defined 32 | inside imported modules (#301) 33 | * Fix bug in join operators when the same table and column name appears on the 34 | LHS of multiple join predicates (#313) 35 | * Fix bug in outer joins with multiple predicates (#315) 36 | * Fix several bugs in rescan/invalidation logic for tables in the presence of 37 | upstream deletions (#303) 38 | * Fix regression in file_reader collections (#304) 39 | * Improve chaining of notin operators 40 | * Optimize join evaluation with multiple predicates 41 | * Optimize notin self joins 42 | * Improve error reporting for syntax errors on temp collections (#310) 43 | * Add Travis CI integration hooks (#300, Josh Rosen) 44 | 45 | Lattices: 46 | 47 | * Rename lmap#apply_monotone to lmap#apply. Also, allow #apply to take either 48 | monotone functions or morphisms, and improve error reporting. 49 | * Add lmap#filter. This takes an lmap whose values are elements of the lbool 50 | lattice and returns an lmap containing only those k/v pairs where the value is 51 | true. 52 | * More intuitive lattice equi-join syntax (lset#eqjoin). Rather than specifying 53 | the join predicate(s) using array indexes, instead allow them to be specified 54 | using a hash of field names, as in traditional Bloom joins. This only works 55 | when the lset contains Structs. 56 | * Remove lset#project, and allow lset#eqjoin to take zero predicates; in the 57 | latter case, eqjoin computes the Cartesian product. 58 | * Support deletion rules (<-) with lattice values on the RHS 59 | 60 | == 0.9.6 / 2013-02-25 61 | 62 | * Support syntax sugar for initializing lattices (#294). For example, rather 63 | than writing "foo <= Bud::MaxLattice.new(2)", you can now just write "foo <= 64 | 2". Note that, due to a bug in the superator gem, you cannot use this syntax 65 | with the <+ operator. 66 | * Allow nested lattice values to be sent over channels (#295, patch from Josh 67 | Rosen). Lattice values could previously be sent as a field value in a tuple 68 | delivered over a channel, but they could not be embedded arbitrarily deeply 69 | within tuples (e.g., lattices nested within an array could not previously be 70 | sent as a field value). 71 | * Support "Comparable" for Bud::Lattice values 72 | * Add support for lattices to rebl 73 | * Reject attempts to insert into stdio via <+ (#288) 74 | * Restore functionality of reading from stdio (i.e., stdin) with MRI 1.9 75 | * Improve MRI 1.8 compatibility 76 | * Require ruby_parser >= 3.1.0 77 | * Fix bug in bootstrap blocks for lattice values 78 | * Fix bug in rescan/invalidation for rules that reference lattice values and use 79 | the <+ or <~ operators (#290) 80 | * Fix bug in rescan logic for notin operator (#291) 81 | * Fix bug in notin operators involving self joins (#298) 82 | * Fix error when output from a notin operator was fed into a lattice 83 | 84 | == 0.9.5 / 2012-11-24 85 | 86 | * Lattice branch (Bloom^L) merged 87 | * Compatibility with recent versions of ruby_parser (3.0.2+) and ruby2ruby 88 | (2.0.1+). Older versions of these two gems are no longer supported 89 | * Add support for aggregate functions that take multiple input columns 90 | * Add built-in aggregate function accum_pair(x, y), which produces a Set of 91 | pairs (two-element arrays [x,y]) 92 | * Support user-specified code blocks in payloads(), argagg(), argmin() and 93 | argmax() 94 | * Change behavior of BudChannel#payloads for channels with two 95 | columns. Previously we returned a single *column* (scalar) value in this case; 96 | now we always return a tuple with k-1 columns 97 | * More consistent behavior for BudCollection#sort when used outside Bloom 98 | programs 99 | * Restore support for each_with_index() over Bud collections 100 | * Restore functionality of Zookeeper-backed Bud collections and fix 101 | incompatibility with recent (> 0.4.4) versions of the Zookeeper gem 102 | * Optimize parsing of Bloom statements, particularly for large Bloom programs 103 | * Fix bug in argagg state materialization 104 | * Fix bug in chaining argmin() or argmax() expressions 105 | * Fix bug in chaining notin() expressions 106 | 107 | == 0.9.4 / 2012-09-06 108 | 109 | * Optimize grouping performance 110 | * Fix regression in dbm-backed collections with MRI 1.8 111 | * Fix regression in grouping operator with MRI 1.8 (#280) 112 | * Fix bug in programs that applied non-monotonic operators to scratch 113 | collections under certain circumstances (#281) 114 | * Fix bug in "notin" with multiple qualifiers (#282) 115 | 116 | == 0.9.3 / 2012-08-20 117 | 118 | * Change behavior of accum() aggregate to return a Set, rather than an Array in 119 | an unspecified order 120 | * Fix several serious bugs in caching/invalidation of materialized operator 121 | state (#276, #278, #279) 122 | * Avoid possible spurious infinite loop with dbm-backed collections 123 | * Optimize aggregation/grouping performance 124 | * Fix bugs and improve performance for materialization of sort operator 125 | * Fix REBL regression with push-based runtime (#274) 126 | * Minor performance optimizations for simple projection rules 127 | * Remove dependency on gchart 128 | * Built-in support for code coverage with MRI 1.9 and SimpleCov 129 | 130 | == 0.9.2 / 2012-05-19 131 | 132 | * Add new aggregate functions: bool_and() and bool_or() 133 | * Fix bugs in notin() stratification and implementation (#271) 134 | * Fix a bug in processing multi-way joins defined inside modules 135 | * Fix two bugs in reduce() operator 136 | * Incorrect default value was sometimes returned 137 | * Didn't handle reduce() outputs that aren't tuples with two fields 138 | * Improve reduce() operator error reporting 139 | * Improve MRI 1.9 compatibility 140 | 141 | == 0.9.1 / 2012-04-10 142 | 143 | * Reject attempts to insert a tuple into a collection with more fields than are 144 | in the collection's schema 145 | * Previous behavior was to ignore additional fields, but this was found to be 146 | error-prone 147 | * Remove builtin support for BUST (web services API); this avoids the need to 148 | depend on the json, nestful and i18n gems 149 | 150 | == 0.9.0 / 2012-03-21 151 | 152 | * Major performance enhancements 153 | * Much, much faster: rewritten runtime that now uses a push-based dataflow 154 | * Operator state is cached; only deltas are updated across ticks in many cases 155 | * Joins that use collection keys can use collection storage for improved 156 | performance 157 | * Improved compatibility: Bud now works with MRI 1.9 (as well as 1.8.7) 158 | * Switched from ParseTree to ruby_parser 159 | * Rewritten Bloom module system 160 | * Tuples are now represented as Ruby Structs, rather than Arrays 161 | * This avoids the need to define column accessor methods by hand 162 | * Tests now use MiniTest rather than Test::Unit 163 | * Observe the following incompatibilities: 164 | * Support for "semi-structured" collections have been removed. That is, 165 | previously you could store extra field values into a tuple; those values 166 | would be collapsed into a single array that was tacked onto the end of the 167 | tuple. 168 | * Support for Bloom-based signal handling has been removed 169 | * Support for the "with" syntax has been removed 170 | * The Bloom-based "deployment" framework has been removed 171 | * Support for Tokyo Cabinet-backed collections has been removed 172 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011, Regents of the University of California 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | o Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | o Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | o Neither the name of the University of California, Berkeley nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 9 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/bloom-lang/bud.svg?branch=v0.9.8)](https://travis-ci.org/bloom-lang/bud) 2 | # Bud 3 | 4 | This is Bud, a.k.a. "Bloom Under Development". It is an initial cut at a Bloom 5 | DSL, using Ruby as a setting. 6 | 7 | See LICENSE for licensing information. 8 | 9 | Language cheatsheet in docs/cheat.md ; see the docs/ directory for other 10 | documentation. 11 | 12 | Main deficiencies at this point are: 13 | 14 | - No Ruby constraints: Within Bloom programs the full power of Ruby is also 15 | available, including mutable state. This allows programmers to get outside the 16 | Bloom framework and lose cleanliness. 17 | 18 | - Compatibility: Bud only works with Ruby (MRI) 1.8.7 and 1.9. Bud also has 19 | experimental support for Ruby 2.0. JRuby and other Ruby implementations are 20 | currently not supported. 21 | 22 | ## Installation 23 | 24 | To install the latest release: 25 | 26 | % gem install bud 27 | 28 | To build and install a new gem from the current development sources: 29 | 30 | % gem build bud.gemspec ; gem install bud*.gem 31 | 32 | Note that [GraphViz](http://www.graphviz.org/) must be installed. 33 | 34 | Simple example programs can be found in examples. A much larger set of example 35 | programs and libraries can be found in the bud-sandbox repository. 36 | 37 | To run the unit tests: 38 | 39 | % gem install minitest # unless already installed 40 | % cd test; ruby ts_bud.rb 41 | 42 | To run the unit tests and produce a code coverage report: 43 | 44 | % gem install simplecov # unless already installed 45 | % cd test; COVERAGE=1 ruby ts_bud.rb 46 | 47 | ## Optional Dependencies 48 | 49 | The bud gem has a handful of mandatory dependencies. It also has one optional 50 | dependency: if you wish to use Bud collections backed by Zookeeper, the 51 | "zookeeper" gem must be installed. 52 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rake' 2 | require 'rake/testtask' 3 | 4 | PRJ = "bud" 5 | 6 | def version 7 | @version ||= begin 8 | $LOAD_PATH.unshift 'lib' 9 | require 'bud/version' 10 | warn "Bud::VERSION not a string" unless Bud::VERSION.kind_of? String 11 | Bud::VERSION 12 | end 13 | end 14 | 15 | def tag 16 | @tag ||= "v#{version}" 17 | end 18 | 19 | desc "Run all tests" 20 | task :test => "test:unit" 21 | 22 | TESTS = FileList["test/tc_*.rb"] 23 | SLOW_TESTS = %w{ test/tc_execmodes.rb } 24 | 25 | namespace :test do 26 | desc "Run unit tests" 27 | Rake::TestTask.new :unit do |t| 28 | t.libs << "lib" 29 | t.ruby_opts = %w{ -C test } 30 | t.test_files = TESTS.sub('test/', '') 31 | ### it would be better to make each tc_*.rb not depend on pwd 32 | end 33 | 34 | desc "Run quick unit tests" 35 | Rake::TestTask.new :quick do |t| 36 | t.libs << "lib" 37 | t.ruby_opts = %w{ -C test } 38 | t.test_files = TESTS.exclude(*SLOW_TESTS).sub('test/', '') 39 | end 40 | 41 | desc "Run quick non-zk unit tests" 42 | Rake::TestTask.new :quick_no_zk do |t| 43 | t.libs << "lib" 44 | t.ruby_opts = %w{ -C test } 45 | t.test_files = TESTS. 46 | exclude('test/tc_zookeeper.rb'). 47 | exclude(*SLOW_TESTS). 48 | sub('test/', '') 49 | end 50 | end 51 | 52 | desc "Commit, tag, and push repo; build and push gem" 53 | task :release => "release:is_new_version" do 54 | require 'tempfile' 55 | 56 | sh "gem build #{PRJ}.gemspec" 57 | 58 | file = Tempfile.new "template" 59 | begin 60 | file.puts "release #{version}" 61 | file.close 62 | sh "git commit --allow-empty -a -v -t #{file.path}" 63 | ensure 64 | file.close unless file.closed? 65 | file.unlink 66 | end 67 | 68 | sh "git tag #{tag}" 69 | sh "git push" 70 | sh "git push --tags" 71 | 72 | sh "gem push #{tag}.gem" 73 | end 74 | 75 | namespace :release do 76 | desc "Diff to latest release" 77 | task :diff do 78 | latest = `git describe --abbrev=0 --tags --match 'v*'`.chomp 79 | sh "git diff #{latest}" 80 | end 81 | 82 | desc "Log to latest release" 83 | task :log do 84 | latest = `git describe --abbrev=0 --tags --match 'v*'`.chomp 85 | sh "git log #{latest}.." 86 | end 87 | 88 | task :is_new_version do 89 | abort "#{tag} exists; update version!" unless `git tag -l #{tag}`.empty? 90 | end 91 | end 92 | -------------------------------------------------------------------------------- /bin/budlabel: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'rubygems' 4 | require 'bud' 5 | require 'getopt/std' 6 | require 'bud/labeling/labeling' 7 | require 'bud/labeling/bloomgraph' 8 | require 'bud/labeling/budplot_style' 9 | 10 | $LOAD_PATH.unshift(".") 11 | 12 | @opts = Getopt::Std.getopts("r:i:p:O:CP") 13 | 14 | unless @opts["r"] and @opts["i"] 15 | puts "USAGE:" 16 | puts "-r REQUIRE" 17 | puts "-i INCLUDE" 18 | puts "[-p INCLUDE PATH]" 19 | puts "[-O Output a graphviz representation of the module in FMT format (pdf if not specified)." 20 | puts "-C Concise output -- Associate a single label with each output interface" 21 | puts "-P Path-based output -- For each output interface, attribute a label to paths from each input interface" 22 | exit 23 | end 24 | 25 | hreadable = { 26 | "D" => "Diffluent: Nondeterministic output contents.", 27 | "A" => "Asynchronous. Nondeterministic output orders.", 28 | "N" => "Nonmonotonic. Output contents are sensitive to input orders.", 29 | "Bot" => "Monotonic. Order-insensitive and retraction-free." 30 | } 31 | 32 | if @opts["p"] 33 | $LOAD_PATH.unshift @opts["p"] 34 | end 35 | 36 | require @opts["r"] 37 | c = Label.new(@opts["i"]) 38 | 39 | puts "--- Report for module #{@opts["i"]} ---" 40 | 41 | if @opts["C"] 42 | puts "---------------" 43 | puts "Output\t\tLabel" 44 | puts "---------------" 45 | c.output_report.each_pair do |k, v| 46 | puts [k, hreadable[v]].join("\t") 47 | end 48 | end 49 | 50 | if @opts["P"] 51 | c.path_report.each_pair do |output, inpaths| 52 | puts "" 53 | puts "--------------------" 54 | puts "Output\tInput\tLabel" 55 | puts "--------------------" 56 | puts output 57 | inpaths.each_pair do |inp, lbl| 58 | puts "\t#{inp}\t#{hreadable[lbl]}" 59 | end 60 | end 61 | end 62 | 63 | c.write_graph(@opts["O"]) if @opts["O"] 64 | -------------------------------------------------------------------------------- /bin/budplot: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'rubygems' 3 | require 'bud' 4 | require 'bud/bud_meta' 5 | require 'bud/graphs' 6 | require 'bud/meta_algebra' 7 | require 'bud/viz_util' 8 | require 'getopt/std' 9 | 10 | include VizUtil 11 | 12 | def is_constant?(m) 13 | begin 14 | return (eval("defined?(#{m})") == "constant") 15 | rescue SyntaxError 16 | return false 17 | end 18 | end 19 | 20 | def make_instance(mods) 21 | # If we're given a single identifier that names a class, just return an 22 | # instance of that class. Otherwise, define a bogus class that includes all 23 | # the module names specified by the user and return an instance. 24 | tmpserver = TCPServer.new('127.0.0.1', 0) # get a free port 25 | default_params = {:dbm_dir => "/tmp/budplot_dbm_" + SecureRandom.uuid.to_s, :port => tmpserver.addr[1]} 26 | 27 | 28 | mods.each do |m| 29 | unless is_constant? m 30 | puts "Error: unable to find definition for module or class \"#{m}\"" 31 | exit 32 | end 33 | 34 | mod_klass = eval m 35 | if mod_klass.class == Class 36 | if mods.length == 1 37 | return mod_klass.new(default_params) 38 | else 39 | puts "Error: cannot intermix class \"#{mod_klass}\" with modules" 40 | exit 41 | end 42 | elsif mod_klass.class != Module 43 | puts "Error: \"#{m}\" is not a module or class name" 44 | exit 45 | end 46 | end 47 | 48 | def_lines = ["class FooBar", 49 | "include Bud", 50 | "include MetaAlgebra", 51 | "include MetaReports", 52 | mods.map {|m| "include #{m}"}, 53 | "end" 54 | ] 55 | class_def = def_lines.flatten.join("\n") 56 | eval(class_def) 57 | f = FooBar.new(default_params) 58 | 3.times{ f.tick } 59 | f 60 | end 61 | 62 | def trace_counts(begins) 63 | complexity = {:data => {}, :coord => {}} 64 | if !begins[:start].nil? 65 | begins[:start].each_pair do |k, v| 66 | if @data and @data[k] 67 | complexity[:data][k] = @data[k].length 68 | end 69 | end 70 | 71 | begins[:finish].each_pair do |k, v| 72 | if @data and @data[k] 73 | complexity[:coord][k] = @data[k].length 74 | end 75 | end 76 | end 77 | complexity 78 | end 79 | 80 | def process(mods) 81 | d = make_instance(mods) 82 | 83 | interfaces = {} 84 | d.t_provides.to_a.each do |prov| 85 | interfaces[prov.interface] = prov.input 86 | end 87 | 88 | tabinf = {} 89 | inp = [] 90 | outp = [] 91 | priv = [] 92 | d.tables.each do |t| 93 | tab = t[0].to_s 94 | tabinf[tab] = t[1].class.to_s 95 | next if d.builtin_tables.has_key? t[0] 96 | 97 | if interfaces[tab].nil? 98 | priv << t 99 | else 100 | if interfaces[tab] 101 | inp << t 102 | else 103 | outp << t 104 | end 105 | end 106 | end 107 | 108 | viz_name = "bud_doc/" + mods.join("_") + "_viz" 109 | graph_from_instance(d, "#{viz_name}_collapsed", "bud_doc", true, nil, @data) 110 | graph_from_instance(d, "#{viz_name}_expanded", "bud_doc", false, nil, @data) 111 | begins = graph_from_instance(d, "#{viz_name}_expanded_dot", "bud_doc", false, "dot", @data) 112 | 113 | 114 | complexity = trace_counts(begins) 115 | # try to figure out the degree of the async edges 116 | deg = find_degrees(d, @data) 117 | unless deg.nil? 118 | deg.each_pair do |k, v| 119 | puts "DEGREE: #{k} = #{v.keys.length}" 120 | end 121 | end 122 | 123 | write_index(inp, outp, priv, viz_name, complexity) 124 | end 125 | 126 | def find_degrees(inst, data) 127 | degree = {} 128 | return if data.nil? 129 | data.each_pair do |k, v| 130 | tab = inst.tables[k.gsub("_snd", "").to_sym] 131 | if !tab.nil? 132 | if tab.class == Bud::BudChannel 133 | v.each_pair do |k2, v2| 134 | v2.each do |row| 135 | loc = row[tab.locspec_idx] 136 | degree[k] ||= {} 137 | degree[k][loc] = true 138 | end 139 | end 140 | end 141 | end 142 | end 143 | return degree 144 | end 145 | 146 | def write_index(inp, outp, priv, viz_name, cx) 147 | f = File.open("bud_doc/index.html", "w") 148 | f.puts "" 149 | f.puts "" 150 | 151 | f.puts "" 152 | f.puts "
" 153 | f.puts "

Input Interfaces

" 154 | do_table(f, inp) 155 | f.puts "		" 156 | f.puts "

Output Interfaces

" 157 | do_table(f, outp) 158 | f.puts "		" 159 | f.puts "

Trace Analysis Results

" 160 | f.puts "

Data Complexity

" 161 | do_cx(f, cx[:data]) 162 | f.puts "

Coordination Complexity

" 163 | do_cx(f, cx[:coord]) 164 | f.puts "
" 165 | f.puts "" 166 | f.close 167 | end 168 | 169 | def do_cx(f, cx) 170 | f.puts "" 171 | cx.each_pair do |k, v| 172 | f.puts "" 173 | end 174 | f.puts "
#{k}#{v.inspect}
" 175 | end 176 | 177 | def do_table(f, info) 178 | f.puts "" 179 | info.sort{|a, b| a[0].to_s <=> b[0].to_s}.each do |tbl_name, tbl_impl| 180 | next if tbl_impl.schema.nil? 181 | key_s = tbl_impl.key_cols.join(", ") 182 | val_s = tbl_impl.val_cols.join(", ") 183 | f.puts "" 184 | f.puts "" 185 | end 186 | f.puts "
#{tbl_name}#{key_s}#{val_s}
" 187 | end 188 | 189 | def get_trace_data 190 | data = nil 191 | 192 | if @opts["t"] 193 | data = {} 194 | traces = @opts['t'].class == String ? [@opts['t']] : @opts['t'] 195 | traces.each do |t| 196 | meta, da = get_meta2(t) 197 | da.each do |d| 198 | data[d[1]] ||= {} 199 | data[d[1]][d[0]] ||= [] 200 | data[d[1]][d[0]] << d[2] 201 | end 202 | end 203 | end 204 | data 205 | end 206 | 207 | if ARGV.length < 2 208 | puts "Usage: budplot [-I PATH_TO_RUBY_INCLUDES] LIST_OF_FILES LIST_OF_MODULES_OR_CLASSES" 209 | exit 210 | end 211 | 212 | @opts = Getopt::Std.getopts("I:t:") 213 | unless @opts["I"].nil? 214 | if @opts["I"].class == Array 215 | @opts["I"].each{|i| $:.unshift i} 216 | else 217 | $:.unshift @opts["I"] 218 | end 219 | end 220 | 221 | @data = get_trace_data 222 | `mkdir bud_doc` 223 | 224 | modules = [] 225 | ARGV.each do |arg| 226 | if File.exists? arg 227 | arg = File.expand_path arg 228 | require arg 229 | else 230 | modules << arg 231 | end 232 | end 233 | 234 | process(modules) 235 | -------------------------------------------------------------------------------- /bin/budtimelines: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'rubygems' 3 | require 'dbm' 4 | require 'bud' 5 | require 'bud/graphs' 6 | require 'bud/viz_util' 7 | require 'getopt/std' 8 | 9 | include VizUtil 10 | 11 | module Depends 12 | state do 13 | table :depends, [:bud_obj, :rid, :lhs, :op, :rhs, :nm, :in_body] 14 | end 15 | end 16 | 17 | class GlobalDepAnalyzer 18 | include Bud 19 | include Depends 20 | # this module's purpose and mechanism are very similar 21 | # to those of basic stratification. here, we are interested 22 | # in NM paths that DO cross temporal edges. 23 | 24 | state do 25 | table :depends_tc, [:lhs, :rhs, :via, :nm] 26 | end 27 | 28 | bloom do 29 | depends_tc <= depends{|d| [d.lhs, d.rhs, d.rhs, d.nm]} 30 | depends_tc <= (depends * depends_tc).pairs(:rhs => :lhs) do |d, tc| 31 | [d.lhs, tc.rhs, d.rhs, (d.nm or tc.nm)] 32 | end 33 | end 34 | end 35 | 36 | module TPSchema 37 | state do 38 | table :deltas, [:bud_time, :tab, :nm] 39 | table :zerod_cards, [:bud_time, :table, :cnt, :pred] 40 | table :nm_tab, [:table] 41 | table :collapsible_base, [:start, :fin] 42 | table :collapsible, [:start, :fin] 43 | scratch :collapsible_tmp, [:start, :fin] 44 | scratch :lcl_best_interval, [:start, :fin] 45 | table :best_interval, [:start, :fin] 46 | end 47 | end 48 | 49 | module DeltaLogic 50 | include TPSchema 51 | bloom do 52 | zerod_cards <= cardinalities{|c| c.to_a + [c.bud_time-1]} 53 | zerod_cards <= (times * depends).pairs do |t, d| 54 | unless cardinalities{|c| c[1] if c[0] == t.bud_time}.include? d[1] 55 | [t.bud_time, d[1], 0, t.bud_time - 1] 56 | end 57 | end 58 | 59 | nm_tab <= depends do |d| 60 | [d[1]] if d[4] 61 | end 62 | 63 | deltas <= (zerod_cards * zerod_cards).pairs(:table => :table, :bud_time => :pred) do |c1, c2| 64 | if c1.bud_time == c2.bud_time - 1 and c1.table == c2.table and c1.cnt != c2.cnt 65 | if nm_tab.include? [c1.table] 66 | [c2.bud_time, c1.table, true] 67 | else 68 | [c2.bud_time, c1.table, false] 69 | end 70 | end 71 | end 72 | end 73 | end 74 | 75 | module VanillaTraceProcessing 76 | include TPSchema 77 | include DeltaLogic 78 | 79 | state do 80 | scratch :tp, times.schema 81 | scratch :bi1, best_interval.schema 82 | end 83 | 84 | bloom do 85 | tp <= times.notin(deltas, :bud_time => :bud_time) {|t, d| true if d.nm} 86 | collapsible_base <= tp {|t| [t.bud_time-1, t.bud_time]} 87 | collapsible <= collapsible_base 88 | 89 | collapsible <= (collapsible_base * collapsible).pairs(:fin => :start) do |b, c| 90 | [b.start, c.fin] 91 | end 92 | 93 | bi1 <= collapsible.notin(collapsible, :start => :start) {|c1, c2| true if c2.fin > c1.fin} 94 | best_interval <= bi1.notin(collapsible, :fin => :fin) {|c1, c2| true if c2.start < c1.start} 95 | end 96 | end 97 | 98 | class SimpleTraceProcessor 99 | include Bud 100 | include TraceCardinality 101 | include Depends 102 | include VanillaTraceProcessing 103 | 104 | end 105 | 106 | def collapse(intervals, host, time) 107 | return time unless @opts["C"] 108 | # worth rethinking when the # of intervals/instance gets high 109 | intervals[host].each do |i| 110 | if time > i[0] and time < i[1] 111 | return i[1] 112 | end 113 | end 114 | return time 115 | end 116 | 117 | def usage 118 | puts "USAGE:" 119 | exit 120 | end 121 | 122 | usage unless ARGV[0] 123 | usage if ARGV[0] == '--help' 124 | 125 | @opts = Getopt::Std.getopts("CLo:") 126 | 127 | snd_info = {} 128 | rcv_info = {} 129 | clean_arg = [] 130 | intervals = {} 131 | 132 | 133 | da = GlobalDepAnalyzer.new 134 | 135 | ARGV.each do |arg_raw| 136 | elems = arg_raw.split("_") 137 | arg = elems[1..4].join("_") 138 | clean_arg << arg 139 | snd_info[arg] = [] 140 | rcv_info[arg] = [] 141 | 142 | meta, data = get_meta2("#{arg_raw}") 143 | tp = SimpleTraceProcessor.new 144 | 145 | meta[:depends].each do |m| 146 | tp.depends << m 147 | da.depends << m 148 | end 149 | 150 | data.each do |d| 151 | tp.full_info << d 152 | if meta[:tabinf].map{|m| m[0] if m[1] == "Bud::BudChannel"}.include? d[1] 153 | if d[1] =~ /_snd\z/ 154 | snd_info[arg] << d 155 | else 156 | rcv_info[arg] << d 157 | end 158 | elsif meta[:tabinf].map{|m| m[0] if m[1] == "Bud::BudPeriodic"}.include? d[1] 159 | end 160 | end 161 | 162 | tp.tick 163 | 164 | puts "entries in collapsible: #{tp.collapsible.length}" 165 | puts "entries in base: #{tp.collapsible_base.length}" 166 | puts "entries in deltas: #{tp.deltas.length}" 167 | 168 | intervals[arg] = [] 169 | tp.best_interval.each do |n| 170 | puts "BEST INTERVAL[#{arg}]: #{n.inspect}" 171 | intervals[arg] << n 172 | end 173 | end 174 | 175 | da.tick 176 | nmreach = {} 177 | da.depends_tc.each do |d| 178 | nmreach[d[0]] = {} unless nmreach[d[0]] 179 | if nmreach[d[0]][d[1]] 180 | nmreach[d[0]][d[1]] = d[3] or nmreach[d[0]][d[1]] 181 | else 182 | nmreach[d[0]][d[1]] = d[3] 183 | end 184 | end 185 | 186 | # our local intervals relations are too optimistic. to say that intervals[foo] = [2, 5] 187 | # is merely to say that nothing NM happened locally btw 2 and 5. it is only safe to collapse 188 | # 2 and 5 if during this interval, we could not have BOTH caused and perceived the results of 189 | # a NM deduction. we can (again, conservatively) ensure that this is not the case by showing 190 | # that from 2-5, there exist no messages A and B s.t. we sent A and received B in the interval 2..5 191 | # and B true will cause a DBM directory DBM_dir to be created (Class_ObjectId_Port)" 16 | puts "> budvis DBM_dir" 17 | puts "This will create a series of svg files in DBM_dir, the root of which will be named tm_0.svg. Open in a browser.\n" 18 | puts "e.g." 19 | puts "> ruby test/tc_carts.rb" 20 | puts "> budvis DBM_BCS_2159661360_" 21 | puts "> open DBM_BCS_2159661360_/tm_0.svg" 22 | puts "\nWith the SVG file open in a browser, you may navigate forward and backward in time" 23 | puts "by clicking the T and S nodes, respectively." 24 | exit 25 | end 26 | 27 | usage unless ARGV[0] 28 | usage if ARGV[0] == '--help' 29 | 30 | meta, data = get_meta2(BUD_DBM_DIR) 31 | 32 | # prune outbufs from tabinf 33 | tabinf = meta[:tabinf].find_all do |k| 34 | !(k[1] == "Bud::BudChannel" and k[0] =~ /_snd\z/) 35 | end 36 | 37 | vh = VizHelper.new(tabinf, meta[:cycle], meta[:depends], meta[:rules], ARGV[0], meta[:provides]) 38 | data.each do |d| 39 | vh.full_info << d 40 | end 41 | vh.tick 42 | vh.summarize(ARGV[0], meta[:schminf]) 43 | -------------------------------------------------------------------------------- /bin/rebl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'rubygems' 3 | require 'bud/rebl' 4 | 5 | ReblShell::run 6 | -------------------------------------------------------------------------------- /bud.gemspec: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift 'lib' 2 | require 'bud/version' 3 | 4 | Gem::Specification.new do |s| 5 | s.name = "bud" 6 | s.version = Bud::VERSION 7 | s.authors = ["Peter Alvaro", "Neil Conway", "Joseph M. Hellerstein", "William R. Marczak", "Sriram Srinivasan"] 8 | s.email = ["bloomdevs@gmail.com"] 9 | s.summary = "A prototype Bloom DSL for distributed programming." 10 | s.homepage = "http://www.bloom-lang.org" 11 | s.description = "A prototype of the Bloom distributed programming language as a Ruby DSL." 12 | s.license = "BSD-3-Clause" 13 | s.has_rdoc = true 14 | s.required_ruby_version = '>= 1.9.3' 15 | s.rubyforge_project = 'bloom-lang' 16 | 17 | s.files = Dir['lib/**/*'] + Dir['bin/*'] + Dir['docs/**/*'] + Dir['examples/**/*'] + %w[README.md LICENSE History.txt Rakefile] 18 | s.executables = %w[rebl budplot budvis budtimelines budlabel] 19 | s.default_executable = 'rebl' 20 | 21 | s.add_dependency 'backports', '= 3.8.0' 22 | s.add_dependency 'eventmachine', '= 1.2.5' 23 | s.add_dependency 'fastercsv', '= 1.5.5' 24 | s.add_dependency 'getopt', '= 1.4.3' 25 | s.add_dependency 'msgpack', '= 1.1.0' 26 | s.add_dependency 'ruby-graphviz', '= 1.2.3' 27 | s.add_dependency 'ruby2ruby', '= 2.4.4' 28 | s.add_dependency 'ruby_parser', '= 3.10.1' 29 | s.add_dependency 'superators19', '= 0.9.3' 30 | s.add_dependency 'syntax', '= 1.2.2' 31 | s.add_dependency 'uuid', '= 2.3.8' 32 | 33 | s.add_development_dependency 'minitest', '= 2.5.1' 34 | 35 | # Optional dependencies -- if we can't find these libraries, certain features 36 | # will be disabled. 37 | # s.add_dependency 'zookeeper', '>= 1.3.0' 38 | end 39 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | Bud: Bloom under development 2 | ============================ 3 | 4 | Welcome to the documentation for *Bud*, a prototype of Bloom under development. 5 | 6 | The documents here are organized to be read in any order, but you might like to 7 | try the following: 8 | 9 | * [intro](intro.md): A brief introduction to Bud and Bloom. 10 | * [getstarted](getstarted.md): A quickstart to teach you basic Bloom 11 | concepts, the use of `rebl` interactive terminal, and the embedding of Bloom 12 | code in Ruby via the `Bud` module. 13 | * [operational](operational.md): An operational view of Bloom, to provide 14 | a more detailed model of how Bloom code is evaluated by Bud. 15 | * [cheat](cheat.md): Full documentation of the language constructs in a concise "cheat sheet" style. 16 | * [modules](modules.md): An overview of Bloom's modularity features. 17 | * [ruby_hooks](ruby\_hooks.md): Bud module methods that allow you to 18 | interact with the Bud evaluator from other Ruby threads. 19 | * [visualizations](visualizations.md): Overview of the `budvis` and 20 | `budplot` tools for visualizing Bloom program analyses. 21 | * [bfs](bfs.md): A walkthrough of the Bloom distributed filesystem. 22 | 23 | In addition, the [bud-sandbox](http://github.com/bloom-lang/bud-sandbox) GitHub 24 | repository contains lots of useful libraries and example programs built using 25 | Bloom. 26 | 27 | Finally, the Bud gem ships with RubyDoc on the language constructs and runtime 28 | hooks provided by the Bud module. To see rdoc, run `gem server` from a command 29 | line and open [http://0.0.0.0:8808/](http://0.0.0.0:8808/) 30 | -------------------------------------------------------------------------------- /docs/bfs_arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bloom-lang/bud/cbcc907061ddd5630847f1587c87dd82a9e45f00/docs/bfs_arch.png -------------------------------------------------------------------------------- /docs/bloom-loop.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bloom-lang/bud/cbcc907061ddd5630847f1587c87dd82a9e45f00/docs/bloom-loop.png -------------------------------------------------------------------------------- /docs/intro.md: -------------------------------------------------------------------------------- 1 | # *Bud*: Ruby <~ Bloom # 2 | 3 | Bud is a prototype of the [*Bloom*](http://bloom-lang.org) language for distributed programming, embedded as a DSL in Ruby. "Bud" stands for *Bloom under development*. The current release is the initial alpha, targeted at "friends and family" who would like to engage at an early stage in the language design. 4 | 5 | ## Distributed Code in Bloom ## 6 | The goal of Bloom is to make distributed programming far easier than it has been with traditional languages. The key features of Bloom are: 7 | 8 | 1. *Disorderly Programming*: Traditional languages like Java and C are based on the [von Neumann model](http://en.wikipedia.org/wiki/Von_Neumann_architecture), where a program counter steps through individual instructions in order. Distributed systems don’t work like that. Much of the pain in traditional distributed programming comes from this mismatch: programmers are expected to bridge from an ordered programming model into a disorderly reality that executes their code. Bloom was designed to match--and to exploit--the disorderly reality of distributed systems.  Bloom programmers write code made of unordered collections of statements, and use explicit constructs to impose order when needed. 9 | 10 | 2. *A Collected Approach to Data Structures*: Taking a cue from successfully-parallelized models like MapReduce and SQL, the standard data structures in Bloom are *disorderly collections*, rather than scalar variables and nested structures like lists, queues and trees. Disorderly collection types reflect the realities of non-deterministic ordering inherent in distributed systems. Bloom provides simple, familiar syntax for manipulating these structures. In Bud, much of this syntax comes straight from Ruby, with a taste of MapReduce and SQL. 11 | 12 | 3. *CALM Consistency*: Bloom enables powerful compiler analysis techniques based on the [CALM principle](http://db.cs.berkeley.edu/papers/cidr11-bloom.pdf) to reason about the consistency of your distributed code. The Bud prototype includes program analysis tools that can point out precise *points of order* in your program: lines of code where a coordination library should be plugged in to ensure distributed consistency. 13 | 14 | 4. *Concise Code*: Bloom is a very high-level language, designed with distributed code in mind. As a result, Bloom programs tend to be far smaller (often [orders of magnitude](http://boom.cs.berkeley.edu) smaller) than equivalent programs in traditional imperative languages. 15 | 16 | ## Alpha Goals and Limitations ## 17 | 18 | We had three main goals in preparing this release. The first was to flesh out the shape of the Bloom language: initial syntax and semantics, and the "feel" of embedding it as a DSL. The second goal was to build tools for reasoning about Bloom programs: both automatic program analyses, and tools for surfacing those analyses to developers. 19 | 20 | The third goal was to start a feedback loop with developers interested in the potential of the ideas behind the language. We are optimistic that the principles underlying Bloom can make distributed programming radically simpler. But we realize that those ideas only matter if programmers can adopt them naturally. We intend Bud to be the beginning of an iterative design partnership with developers who see value in betting early on these ideas, and shaping the design of the language. 21 | 22 | In developing this alpha release, we explicitly set aside some issues that we intend to revisit in future. The first limitation is performance: Bud alpha is not intended to excel in single-node performance in terms of either latency, throughput or scale. We do expect major improvements on all these fronts in future releases: many of the known performance problems have known solutions that we've implemented in prior systems. The second main limitation involves integration issues embedding Bloom as a DSL in Ruby. In the spectrum from flexibility to purity, we leaned decidedly toward flexibility. The barriers between Ruby and Bloom code are very fluid in the alpha, and we do relatively little to prevent programmers from ad-hoc mixtures of the two. Aggressive use of Ruby within Bloom statements is likely to do something *interesting*, but not necessarily predictable or desirable. This is an area where we expect to learn more from experience, and make some more refined decisions for the beta release. 23 | 24 | ### Friends and Family: Come On In ### 25 | Although our team has many years of development experience, Bud is still open-source academic software built by a small group of researchers. 26 | 27 | This alpha is targeted at "friends and family", and at developers who'd like to become same. This is definitely the bleeding edge: we're in a rapid cycle of learning about this new style of programming, and exposing what we learn in new iterations of the language. If you'd like to jump on the wheel with us and play with Bud, we'd love your feedback--both success stories and constructive criticism. 28 | 29 | ## Getting Started ## 30 | We're shipping Bud with a [sandbox](http://github.com/bloom-lang/bud-sandbox) of libraries and example applications for distributed systems. These illustrate the language and how it can be used, and also can serve as mixins for new code you might want to write. You may be surprised at how short the provided Bud code is, but don't be fooled. 31 | 32 | To get you started with Bud, we've provided a [quick-start tutorial](getstarted.md) and a number of other docs you can find linked from the [README](README.md). 33 | 34 | We welcome both constructive criticism and (hopefully occasional) smoke-out-your-ears, hair-tearing shouts of frustration. Please point your feedback cannon at the [Bloom mailing list](http://groups.google.com/group/bloom-lang). 35 | 36 | Happy Blooming! 37 | -------------------------------------------------------------------------------- /docs/modules.md: -------------------------------------------------------------------------------- 1 | # Code Structuring and Reuse in Bud 2 | 3 | ## Language Support 4 | 5 | ### Ruby Mixins 6 | 7 | The basic unit of reuse in Bud is the mixin functionality provided by Ruby itself. Bud code is structured into modules, each of which may have its own __state__ and __bootstrap__ block and any number of __bloom__ blocks (described below). A module or class may mix in a Bud module via Ruby's _include_ statement. _include_ causes the specified module's code to be expanded into the local scope. 8 | 9 | ### Bloom Blocks 10 | 11 | While the order and grouping of Bud rules have no semantic significance, rules can be grouped and tagged within a single module using __bloom__ blocks. Bloom blocks serve two purposes: 12 | 13 | 1. Improving readability by grouping related or dependent rules together. 14 | 2. Supporting name-based overriding. 15 | 16 | (1) is self-explanatory. (2) represents one of several extensibility mechanisms provided by Bud. If a Module B includes a module A which contains a basket X, B may supply a bloom block X and in so doing replaces the set of rules defined by (A.)X with its own set. For example: 17 | 18 | require 'rubygems' 19 | require 'bud' 20 | 21 | module Hello 22 | state do 23 | interface input, :stim 24 | end 25 | bloom :hi do 26 | stdio <~ stim {|s| ["Hello, #{s.val}"]} 27 | end 28 | end 29 | 30 | module HelloTwo 31 | include Hello 32 | bloom :hi do 33 | stdio <~ stim{|s| ["Hello, #{s.key}"]} 34 | end 35 | end 36 | 37 | class HelloClass 38 | include Bud 39 | include HelloTwo 40 | end 41 | 42 | h = HelloClass.new 43 | h.run_bg 44 | h.sync_do{h.stim <+ [[1,2]]} 45 | 46 | The program above will print "Hello, 1", because the module HelloTwo overrides the bloom block named __hi__. If we give the bloom block in HelloTwo a distinct name, the program will print "Hello, 1" and "Hello, 2" (in no guaranteed order). 47 | 48 | 49 | ### The Bud Module Import System 50 | 51 | For simple programs, composing modules via _include_ is often sufficient. But the flat namespace provided by mixins can make it difficult or impossible to support certain types of reuse. Consider a module Q that provides a queue-like functionality via an input interface _enqueue_ and an output interface _dequeue_, each with a single attribute (payload). A later module may wish to employ two queues (say, to implement a scheduler). But it cannot include Q twice! It would be necessary to rewrite Q's interfaces so as to support multiple ``users.'' 52 | 53 | In addition to _include_, Bud supports the _import_ keyword, which instantiates a Bud module under a namespace alias. For example: 54 | 55 | module UserCode 56 | import Q => :q1 57 | import Q => :q2 58 | 59 | bloom do 60 | # insert into the first queue 61 | q1.enqueue <= [....] 62 | end 63 | end 64 | 65 | 66 | ## Techniques 67 | 68 | ### Structuring 69 | 70 | In summary, Bud extends the basic Ruby code structuring mechanisms (classes and modules) with bloom blocks, for finer-granularity grouping of rules 71 | within modules, and the import system, for scoped inclusion. 72 | 73 | ### Composition 74 | 75 | Basic code composition can achieved using the Ruby mixin system. If the flat namespace causes ambiguity (as above) or hinders readability, the Bud import system provides the ability to scope code inclusions. 76 | 77 | ### Extension and Overriding 78 | 79 | Extending the existing functionality of a Bud program can be achieved in a number of ways. The simplest (but arguably least flexible) is via bloom block overriding, as described in the Hello example above. 80 | 81 | The import system can be used to implement finer-grained overriding, at the collection level. Consider a module BlackBox that provides an input interface __iin__ and an output interface __iout__. Suppose that we wish to "use" BlackBox, but need to provide additional functionality. We may extend one or both of its interfaces by _import_-ing BlackBox, redeclaring the interfaces, and gluing them together. For example, the module UsesBlackBox shown below interposes additional logic (indicated by ellipses) upstream of BlackBox's input interface, and provides ``extended'' BlackBox functionality. 82 | 83 | module UsesBlackBox 84 | import BlackBox => :bb 85 | state do 86 | interface input, :iin 87 | interface output, :iout 88 | end 89 | 90 | bloom do 91 | [ .... ] <= iin 92 | bb.iin <= [ .... ] 93 | iout <= bb.iout 94 | end 95 | end 96 | 97 | ### Abstract Interfaces and Concrete Implementations 98 | 99 | In the previous example, UsesBlackBox extended the functionality of BlackBox by _interposing_ additional logic into its dataflow. 100 | It was able to do this transparently because both implementations had the same externally visible interface: inserting tuples into __iin__ causes tuples to appear in __iout__. In some (extremely underspecified) sense, the definition of this pair of interfaces constitutes an abstract contract which both implementations implement -- and the dependency of UsesBlackBox on Blackbox is just a detail of UsesBlackBox's implementation. 101 | 102 | The basic Ruby module system inherited by Bud may be used, by convention, to enable code reuse and hiding via the separation of abstract interfaces and concrete implementations. Instead of reiterating the schema definitions in multiple state blocks, we will often instead declare a protocol module as follows: 103 | 104 | module BBProtocol 105 | # Contract: do XXXXXXXXXXX 106 | state do 107 | interface input, :iin 108 | interface output, :iout 109 | end 110 | end 111 | 112 | Each implementation of the protocol would then include BBProtocol. Though the interpreter treats this as an ordinary Ruby mixin, the interpretation is that by including BBProtocol, both BlackBox and UsesBlackBox _implement_ the protocol. A downstream developer may then write code against the external interface, committing only when necessary to a fully-specified implmentation. -------------------------------------------------------------------------------- /docs/operational.md: -------------------------------------------------------------------------------- 1 | # An Operational View Of Bloom # 2 | You may ask yourself: well, what does a Bloom program *mean*? You may ask yourself: How do I read this Bloom code? ([You may tell yourself, this is not my beautiful house.](http://www.youtube.com/watch?v=I1wg1DNHbNU)) 3 | 4 | There is a formal answer to these questions about Bloom, but there's also a more more approachable answer. Briefly, the formal answer is that Bloom's semantics are based in finite model theory, via a temporal logic language called *Dedalus* that is described in [a paper from Berkeley](http://www.eecs.berkeley.edu/Pubs/TechRpts/2009/EECS-2009-173.html). 5 | 6 | While that's nice for proving theorems (and writing [program analysis tools](visualizations.md)), many programmers don't find model-theoretic discussions of semantics terribly helpful or even interesting. It's usually easier to think about how the language *works* at some level, so you can reason about how to use it. 7 | 8 | That's the goal of this document: to provide a relatively simple, hopefully useful intuition for how Bloom is evaluated. This is not the only way to evaluate Bloom, but it's the intuitive way to do it, and basically the way that the Bud implementation works (modulo some optimizations). 9 | 10 | ## Bloom Timesteps ## 11 | Bloom is designed to be run on multiple machines, with no assumptions about coordinating their behavior or resources. 12 | 13 | Each machine runs an evaluator that works in a loop, as depicted in this figure: 14 | 15 | ![Bloom Loop](bloom-loop.png?raw=true) 16 | 17 | Each iteration of this loop is a *timestep* for that node; each timestep is associated with a monotonically increasing timestamp (which is accessible via the `budtime` method in Bud). Timesteps and timestamps are not coordinated across nodes; any such coordination has to be programmed in the Bloom language itself. 18 | 19 | A Bloom timestep has 3 main phases (from left to right): 20 | 21 | 1. *setup*: All scratch collections are set to empty. Network messages and periodic timer events are received from the runtime and placed into their designated `channel` and `periodic` scratches, respectively, to be read in the rhs of statements. Note that a batch of multiple messages/events may be received at once. 22 | 2. *logic*: All Bloom statements for the program are evaluated. In programs with recursion through instantaneous merges (`<=`), the statements are repeatedly evaluated until a *fixpoint* is reached: i.e., no new lhs items are derived from any rhs. 23 | 3. *transition*: Items derived on the lhs of deferred operators (`<+`, `<-`, `<+-`) are placed into/deleted from their corresponding collections, and items derived on the lhs of asynchronous merge (`<~`) are handed off to external code (i.e., the local operating system) for processing. 24 | 25 | It is important to understand how the Bloom collection operators fit into these timesteps: 26 | 27 | * *Instantaneous* merge (`<=`) occurs within the fixpoint of phase 2. 28 | * *Deferred* operations include merge (`<+`), update (`<+-`), and delete (`<-`), and are handled in phase 3. Their effects become visible atomically to Bloom statements in phase 2 of the next timestep. 29 | * *Asynchronous* merge (`<~`) is initiated during phase 3, so it cannot affect the current timestep. When multiple items are on the rhs of an async merge, they may "appear" independently spread across multiple different future local timesteps. 30 | 31 | 32 | ## Atomicity: Timesteps and Deferred Operators ## 33 | 34 | The only instantaneous Bloom operator is a merge (`<=`), which can only introduce additional items into a collection--it cannot delete or change existing items. As a result, all state within a Bloom timestep is *immutable*: once an item is in a collection at timestep *T*, it stays in that collection throughout timestep *T*. (And forever after, the fact that the item was in that collection at timestep *T* remains true.) 35 | 36 | To get atomic state change in Bloom, you exploit the combination of two language features: 37 | 38 | 1. the immutability of state in a single timestep, and 39 | 2. the uninterrupted sequencing of consecutive timesteps. 40 | 41 | State "update" is achieved in Bloom via a pair of deferred statements, one positive and one negative, like so: 42 | 43 | buffer <+ [[1, "newval"]] 44 | buffer <- buffer {|b| b if b.key == 1} 45 | 46 | This atomically replaces the entry for key 1 with the value "newval" at the start of the next timestep. As syntax sugar for this common pattern, the deferred update operator can be used: 47 | 48 | buffer <+- [[1, "newval"]] 49 | 50 | This update statement removes (from the following timestep) any fact in `buffer` with the key `1`, and inserts (in the following timestep) a fact with the value `[1, "newval"]`. Note that "key" here refers to the key column(s) of the lhs relation: this example assumes `buffer` has a single key column. 51 | 52 | Any reasoning about atomicity in Bloom programs is built on this simple foundation. It's really all you need. In the bud-sandbox we show how to build more powerful atomicity constructs using it, including things like enforcing [ordering of items across timesteps](https://github.com/bloom-lang/bud-sandbox/tree/master/ordering), and protocols for [agreeing on ordering of distributed updates](https://github.com/bloom-lang/bud-sandbox/tree/master/paxos) across all nodes. 53 | 54 | ## Recursion in Bloom ## 55 | Because Bloom is data-driven rather than call-stack-driven, recursion may feel a bit unfamiliar at first. 56 | 57 | Have a look at the following classic "transitive closure" example, which computes multi-hop paths in a graph based on a collection of one-hop links: 58 | 59 | state do 60 | table :link, [:from, :to, :cost] 61 | table :path, [:from, :to, :cost] 62 | end 63 | 64 | bloom :make_paths do 65 | # base case: every link is a path 66 | path <= link {|e| [e.from, e.to, e.cost]} 67 | 68 | # recurse: path of length n+1 made by a link to a path of length n 69 | path <= (link*path).pairs(:to => :from) do |l,p| 70 | [l.from, p.to, l.cost + p.cost] 71 | end 72 | end 73 | 74 | The recursion in the second Bloom statement is easy to see: the lhs and rhs both contain the path collection, so path is defined in terms of itself. 75 | 76 | You can think of this being computed by reevaluating the bloom block over and over--within phase 2 of a single timestep--until no more new paths are found. In each iteration, we find new paths that are one hop longer than the longest paths found previously. When no new items are found in an iteration, we are at what's called a *fixpoint*, and we can move to phase 3. 77 | 78 | Hopefully that description is fairly easy to understand. You can certainly construct more complicated examples of recursion--just as you can in a traditional language (e.g., simultaneous recursion.) But understanding this example of simple recursion is probably sufficient for most needs. 79 | 80 | ## Non-monotonicity and Strata ## 81 | 82 | Consider augmenting the previous path-finding program to compute only the "highest-cost" paths between each source and destination, and print them out. We can do this by adding another statement to the above: 83 | 84 | bloom :print_highest do 85 | stdio <~ path.argmax([:from, :to], :cost) 86 | end 87 | 88 | The `argmax` expression in the rhs of this statement finds the items in path that have the maximum cost for each `[from, to]` pair. 89 | 90 | It's interesting to think about how to evaluate this statement. Consider what happens after a single iteration of the path-finding logic listed above. We will have 1-hop paths between some pairs. But there will likely be multi-hop paths between those pairs that cost more. So it would be premature after a single iteration to put anything out on stdio. In fact, we can't be sure what should go out to stdio until we have hit a fixpoint with respect to the path collection. That's because `argmax` is a logically *non-monotonic* operator: as we merge more items into its input collection, it may have to "retract" an output they would previously have produced. 91 | 92 | The Bud runtime takes care of this problem for you under the covers, by breaking your statements in *strata* (layers) via a process called *stratification*. The basic idea is simple. The goal is to postpone evaluating non-monotonic operators until fixpoint is reached on their input collections. Stratification basically breaks up the statements in a Bloom program into layers that are separated by non-monotonic operators, and evaluates the layers in order. (Note: the singular form of strata is *stratum*). 93 | 94 | For your reference, the basic non-monotonic Bloom operators include `group, reduce, argmin, argmax`. Also, statements that embed Ruby collection methods in their blocks are often non-monotonic--e.g., methods like `all?, empty?, include?, none?` and `size`. 95 | 96 | Note that it is possible to write a program in Bloom that is *unstratifiable*: there is no way to separate it into layers like this. This arises when some collection is recursively defined in terms of itself, and there is a non-monotonic method along the recursive dependency chain. A simple example of this is as follows: 97 | 98 | glass <= one_item {|t| ['full'] if glass.empty? } 99 | 100 | Consider the case where we start out with glass being empty. Then we know the fact `glass.empty?`, and the bloom statement says that `(glass.empty? => not glass.empty?)` which is equivalent to `(glass.empty? and not glass.empty?)` which is a contradiction. The Bud runtime detects cycles through non-monotonicity for you automatically when you instantiate your class. 101 | -------------------------------------------------------------------------------- /docs/rebl.md: -------------------------------------------------------------------------------- 1 | REBL stands for "Read Eval Bud Loop" and is Bud's REPL. Running REBL by typing in "rebl" causes the "rebl>" prompt to appear. Input is either a collection definition, Bud statement, or REBL command. REBL commands are prefixed by "/"; all other input is interpreted as Bud code. If the input begins with either "table", "scratch" or "channel" (the currently-supported collection types), then the input is processed as a collection definition. Otherwise, the input is processed as a Bud rule. 2 | 3 | Rules are not evaluated until a user enters one of the evaluation commands; either "/tick [x]" (tick x times, or once if optional argument x is not specified), or "/run" (runs in the background until a breakpoint is hit, execution stops, or a user inputs "/stop"). Rules may be added or deleted at any time, and collections may similarly be added at any time. 4 | 5 | A breakpoint is a Bud rule that has the "breakpoint" scratch on its left-hand-side. The "/run" command will stop executing at the end of any timestep where a "breakpoint" tuple was derived. Another invocation of "/run" will continue executing from the beginning of the next timestep. 6 | 7 | Let's step through two examples to better understand REBL. The first example is a centralized all-pairs-all-paths example in a graph, the second example is a distributed ping-pong example. These examples illustrate the use of the commands, which can be listed from within rebl by typing "/help". 8 | 9 | 10 | # Shortest Paths Example 11 | 12 | Let's start by declaring some collections. "link" represents directed edges in a graph. "path" represents all pairs of reachable nodes, with the next hop and cost of the path. 13 | 14 | rebl> table :link, [:from, :to, :cost] 15 | rebl> table :path, [:from, :to, :next, :cost] 16 | 17 | "/lscollections" confirms that the collections are defined. 18 | 19 | rebl> /lscollections 20 | 1: table :link, [:from, :to, :cost] 21 | 2: table :path, [:from, :to, :next, :cost] 22 | 23 | We now define some rules to populate "path" based on "link". 24 | 25 | rebl> path <= link {|e| [e.from, e.to, e.to, e.cost]} 26 | rebl> temp :j <= (link*path).pairs(:to => :from) 27 | rebl> path <= j { |l,p| [l.from, p.to, p.from, l.cost+p.cost] } 28 | 29 | Furthermore, we decide to print out paths to stdout. 30 | 31 | rebl> stdio <~ path.inspected 32 | 33 | We provide some initial data to "link". 34 | 35 | rebl> link <= [['a','b',1],['a','b',4],['b','c',1],['c','d',1],['d','e',1]] 36 | 37 | Ticking prints out the paths to stdout. 38 | 39 | rebl> /tick 40 | ["a", "c", "b", 5] 41 | ["a", "e", "b", 4] 42 | ["a", "b", "b", 1] 43 | ["b", "d", "c", 2] 44 | ["a", "c", "b", 2] 45 | ["a", "d", "b", 6] 46 | ["b", "e", "c", 3] 47 | ["b", "c", "c", 1] 48 | ["a", "d", "b", 3] 49 | ["c", "e", "d", 2] 50 | ["a", "e", "b", 7] 51 | ["a", "b", "b", 4] 52 | ["d", "e", "e", 1] 53 | ["c", "d", "d", 1] 54 | 55 | This might be a bit annoying though. Let's try to remove that rule using "/rmrule". First, we need to figure out which rule it is, using "/lsrules". 56 | 57 | rebl> /lsrules 58 | 5: link <= [['a','b',1],['a','b',4],['b','c',1],['c','d',1],['d','e',1]] 59 | 1: path <= link {|e| [e.from, e.to, e.to, e.cost]} 60 | 2: temp :j <= (link*path).pairs(:to => :from) 61 | 3: path <= j { |l,p| [l.from, p.to, p.from, l.cost+p.cost] } 62 | 4: stdio <~ path.inspected 63 | 64 | Looks like it's rule 4. 65 | 66 | rebl> /rmrule 4 67 | 68 | Note how ticking no longer prints out the paths. 69 | 70 | rebl> /tick 71 | 72 | 73 | # Ping-Pong Example 74 | 75 | We begin by starting up two instances of REBL on the same machine in different terminal windows. Take note of the port number printed when REBL starts up. For example, we might have: 76 | 77 | REBL1 : "Listening on localhost:33483" 78 | REBL2 : "Listening on localhost:48183" 79 | 80 | In each REBL, let us now define the following collections and rules: 81 | 82 | rebl> channel :ping, [:@dst, :src] 83 | rebl> ping <~ ping.map {|p| [p.src, p.dst]} 84 | 85 | In REBL1, type in an initial ping to start things off: 86 | 87 | rebl> ping <~ [["localhost:48183", ip_port]] 88 | 89 | Note that no messages are exchanged until we either type "/tick [x]" or "/run". Note that this program will run forever, as pings will contiuously be exchanged. Let's set a breakpoint so both REBLs break once they've received their first ping. In both REBLs, type: 90 | 91 | rebl> breakpoint <= ping 92 | 93 | Note that the schema of "breakpoint" is unimportant. 94 | 95 | Now, let us "/run" both REBLs. At a leisurely pace, type in "/run" to each REBL. Hmm, what happened? Type "/dump ping" on each REBL to see if it got a ping packet: 96 | 97 | rebl> /dump ping 98 | 99 | Pings are no longer being exchanged. Of course, if you want to remove a breakpoint, it is as simple as using "/lsrule" and "/rmrule x", where x is the rule ID shown by "/lsrule". If you type a "/run" command and want to stop execution, simply type the "/stop" command. -------------------------------------------------------------------------------- /docs/ruby_hooks.md: -------------------------------------------------------------------------------- 1 | ## Ruby/Bloom interactions in Bud ## 2 | Bud embeds Bloom as a [DSL](http://en.wikipedia.org/wiki/Domain-specific_language) in Ruby. On a given node, it is often useful to run a Ruby thread, and pass information between one vanilla Ruby thread, and another one running Bloom code. 3 | 4 | As described in the [Getting Started](getstarted.md) document, we embed Bloom code into Ruby by including the `Bud` module in some Ruby class, and putting Bloom blocks into the class. We then typically allocate a new instance of that class in Ruby, and invoke either its `run_fg` method or `run_bg` method. Since the Bud runtime typically runs indefinitely, the blocking `run_fg` call essentially hands the thread over to Bud. The `run_bg` call puts the Bud runtime into a second thread and returns to the caller. 5 | 6 | To support the `run_bg` case further, the Bud runtime provides hooks for Ruby threads to register code with the runtime for execution during Bloom timesteps. These hooks always run at the end of a timestep, after all Bloom statements have been processed for that timestep. There are two basic types of hooks: 7 | 8 | * The Bud module provides a Ruby method called `sync_do`, which takes a block of code, and hands it to the Bud runtime for execution at the end of a timestep. The Bud runtime is blocked during this execution, so the state of all Bud collections is fixed for the duration of the block of Ruby code. This is also a blocking call for the caller. Bud also supplies `async_do`, which hands off the code block to the Bud runtime to be executed, but returns immediately to the caller without waiting for the runtime. 9 | 10 | * The Bud module provides a Ruby method called `register_callback`. Given the name of a Bud collection, this method arranges for the given block of Ruby code to be invoked at the end of any timestep in which any tuples have been inserted into the specified collection. The code block is passed the collection as an argument; this provides a convenient way to examine the items inserted during that timestep. Note that because the Bud runtime is blocked while the callback is invoked, it can also examine any other Bud state freely. 11 | 12 | ## Bud and Ruby-driven Side Effects ## 13 | The Bloom language itself is a pure logic language based in Dedalus. Like any logic language, it has no notion of "mutable state" or "side effects". Each item that appears in a Bloom collection at timestep T will *forever* be recorded as having been in that collection at timestep T, at least in a formal sense. The temporal logic of Dedalus is a lot like a versioning system, where old versions of items are never removed. 14 | 15 | In the context of the existing Bud prototype, though, it easy to step outside the bright lines of pure Bloom using straight Ruby code and libraries. Methods like `sync_do` and callbacks allow Ruby code to be run that can mess with Bloom collections in a way that Bloom doesn't model. Similarly, Ruby blocks *within* the rhs of a Bloom statement (e.g. after a collection name or a `pairs` call) can produce visible side-effects in unpredictable ways. In particular, be aware that any side-effect-producing Ruby code within a Bloom block may be called an *arbitrary* number of times during a single Bloom timestep, as Bud works its way to fixpoint. 16 | 17 | If you must pollute your Bloom statements with side-effecting Ruby, please do it with *idempotent* side-effecting Ruby. Thank you. 18 | 19 | In future versions of the Bud runtime we plan to limit the Ruby that gets invoked in the Bloom context, and keep you safe from these tendencies. (As with a lot of the Bloom design, we hope to do this in a way that won't make you feel handicapped, just gently cradled in disorderly goodness.) -------------------------------------------------------------------------------- /docs/visualizations.md: -------------------------------------------------------------------------------- 1 | # Visualizations 2 | 3 | Bud programs compile naturally to dataflows, and dataflows have a natural 4 | graphical representation. Viewing the dataflow graph of a program can be useful 5 | to developers at various stages of program design, implementation and debugging. 6 | 7 | Bud ships with two visualization utilities, __budplot__ and __budvis__. Both 8 | use _GraphViz_ to draw a directed graph representing the program state and 9 | logic. __budplot__ provides a static analysis of the program, identifying 10 | sources and sinks of the dataflow, unconnected components, and "points of order" 11 | that correspond to logically nonmonotonic path edges. __budvis__ is an offline 12 | debugging tool that analyses the trace of a (local) Bud execution and provides 13 | an interactive representation of runtime state over time. 14 | 15 | ## Using budplot 16 | 17 | __budplot__ is a visual static analysis tool that aids in design and early 18 | implementation. The most common uses of __budplot__ are: 19 | 20 | 1. Visual sanity checking: does the dataflow look like I expected it to look? 21 | 2. Ensuring that a particular set of mixins is fully specified: e.g., did I forget to include a concrete implementation of a protocol required by other modules? 22 | The Bloom module system, abstract interfaces and concrete implementations are described in more detail in [modules.md](modules.md). 23 | 3. Identifying dead code 24 | 4. Experimenting with different module compositions 25 | 5. Identifying and iteratively refining a program's "points of order" 26 | 27 | To run __budplot__, specify a list of Ruby input files, followed by a list of 28 | Bud modules to be "mixed in" in the visualization. 29 | 30 | $ budplot 31 | Usage: budplot LIST_OF_FILES LIST_OF_MODULES 32 | 33 | For example: 34 | 35 | $ budplot kvs/kvs.rb ReplicatedKVS 36 | Warning: underspecified dataflow: ["my_id", true] 37 | Warning: underspecified dataflow: ["add_member", true] 38 | Warning: underspecified dataflow: ["send_mcast", true] 39 | Warning: underspecified dataflow: ["mcast_done", false] 40 | fn is ReplicatedKVS_viz_collapsed.svg 41 | $ open bud_doc/index.html 42 | 43 | `ReplicatedKVS` includes the `MulticastProtocol` and `MembershipProtocol` 44 | protocols, but does not specify which implementation of these abstractions to 45 | use. The program is underspecified, and this is represented in the resulting 46 | graph (`ReplicatedKVS_viz_collapsed.svg`) by a node labeled "??" in the 47 | dataflow. 48 | 49 | $ budplot kvs/kvs.rb ReplicatedKVS BestEffortMulticast StaticMembership 50 | fn is ReplicatedKVS_BestEffortMulticast_StaticMembership_viz_collapsed.svg 51 | $ open bud_doc/index.html 52 | 53 | ## Using budvis 54 | 55 | To enable tracing, we need to set `:trace => true` in the `Bud` constructor, and 56 | optionally provide a `:tag` to differentiate between traces by a human-readable 57 | name (rather than by `object_id`). I modified the unit test `test/DBM_kvs.rb` as 58 | follows: 59 | 60 | - v = BestEffortReplicatedKVS.new(@opts.merge(:port => 12345)) 61 | - v2 = BestEffortReplicatedKVS.new(@opts.merge(:port => 12346)) 62 | 63 | + v = BestEffortReplicatedKVS.new(@opts.merge(:port => 12345, :tag => 'dist_primary', :trace => true)) 64 | + v2 = BestEffortReplicatedKVS.new(@opts.merge(:port => 12346, :tag => 'dist_backup', :trace => true)) 65 | 66 | 67 | Then I ran the unit test: 68 | 69 | $ ruby test/tc_kvs.rb 70 | Loaded suite test/tc_kvs 71 | Started 72 | .Created directory: DBM_BestEffortReplicatedKVS_dist_primary_2160259460_ 73 | Created directory: DBM_BestEffortReplicatedKVS_dist_primary_2160259460_/bud_ 74 | Created directory: DBM_BestEffortReplicatedKVS_dist_backup_2159579740_ 75 | Created directory: DBM_BestEffortReplicatedKVS_dist_backup_2159579740_/bud_ 76 | .. 77 | Finished in 4.366793 seconds. 78 | 79 | 3 tests, 14 assertions, 0 failures, 0 errors 80 | 81 | Then I ran the visualization utility: 82 | 83 | $ budvis DBM_BestEffortReplicatedKVS_dist_primary_2160259460_/ 84 | 85 | And finally opened the (chronological) first output file: 86 | 87 | $ open -a /Applications/Google\ Chrome.app/ DBM_BestEffortReplicatedKVS_dist_primary_2160259460_/tm_0_expanded.svg 88 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | A few simple examples are included here, but for more detailed and useful examples please see the [bud-sandbox](http://github.com/bloom-lang/bud-sandbox) repository. -------------------------------------------------------------------------------- /examples/basics/hello.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'bud' 3 | 4 | class HelloWorld 5 | include Bud 6 | 7 | bloom do 8 | stdio <~ [["hello world!"]] 9 | end 10 | end 11 | 12 | HelloWorld.new.tick 13 | -------------------------------------------------------------------------------- /examples/basics/paths.rb: -------------------------------------------------------------------------------- 1 | # simple shortest paths 2 | # note use of program.tick at bottom to run a single timestep 3 | # and inspect relations 4 | require 'rubygems' 5 | require 'bud' 6 | 7 | class ShortestPaths 8 | include Bud 9 | 10 | state do 11 | table :link, [:from, :to, :cost] 12 | table :path, [:from, :to, :nxt, :cost] 13 | table :shortest, [:from, :to] => [:nxt, :cost] 14 | end 15 | 16 | # recursive rules to define all paths from links 17 | bloom :make_paths do 18 | # base case: every link is a path 19 | path <= link {|l| [l.from, l.to, l.to, l.cost]} 20 | 21 | # inductive case: make path of length n+1 by connecting a link to a path of 22 | # length n 23 | path <= (link*path).pairs(:to => :from) do |l,p| 24 | [l.from, p.to, l.to, l.cost+p.cost] 25 | end 26 | end 27 | 28 | # find the shortest path between each connected pair of nodes 29 | bloom :find_shortest do 30 | shortest <= path.argmin([path.from, path.to], path.cost) 31 | end 32 | end 33 | 34 | # compute shortest paths. 35 | program = ShortestPaths.new 36 | 37 | # populate our little example. we put two links between 'a' and 'b' 38 | # to see whether our shortest-paths code does the right thing. 39 | program.link <+ [['a', 'b', 1], 40 | ['a', 'b', 4], 41 | ['b', 'c', 1], 42 | ['c', 'd', 1], 43 | ['d', 'e', 1]] 44 | 45 | program.tick # one timestamp is enough for this simple program 46 | program.shortest.to_a.sort.each {|t| puts t.inspect} 47 | 48 | puts "----" 49 | 50 | # now lets add an extra link and recompute 51 | program.link <+ [['e', 'f', 1]] 52 | program.tick 53 | program.shortest.to_a.sort.each {|t| puts t.inspect} 54 | -------------------------------------------------------------------------------- /examples/chat/README.md: -------------------------------------------------------------------------------- 1 | To run the chat example, do each of the following in a different terminal: 2 | 3 | # ruby chat_server.rb 4 | 5 | # ruby chat.rb alice 6 | 7 | # ruby chat.rb bob 8 | 9 | # ruby chat.rb harvey 10 | 11 | Note that the "backports" gem should be installed. 12 | -------------------------------------------------------------------------------- /examples/chat/chat.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'backports' 3 | require 'bud' 4 | require_relative 'chat_protocol' 5 | 6 | class ChatClient 7 | include Bud 8 | include ChatProtocol 9 | 10 | def initialize(nick, server, opts={}) 11 | @nick = nick 12 | @server = server 13 | super opts 14 | end 15 | 16 | bootstrap do 17 | connect <~ [[@server, ip_port, @nick]] 18 | end 19 | 20 | bloom do 21 | mcast <~ stdio do |s| 22 | [@server, [ip_port, @nick, Time.new.strftime("%I:%M.%S"), s.line]] 23 | end 24 | 25 | stdio <~ mcast { |m| [pretty_print(m.val)] } 26 | end 27 | 28 | # format chat messages with color and timestamp on the right of the screen 29 | def pretty_print(val) 30 | str = "\033[34m"+val[1].to_s + ": " + "\033[31m" + (val[3].to_s || '') + "\033[0m" 31 | pad = "(" + val[2].to_s + ")" 32 | return str + " "*[66 - str.length,2].max + pad 33 | end 34 | end 35 | 36 | 37 | 38 | server = (ARGV.length == 2) ? ARGV[1] : ChatProtocol::DEFAULT_ADDR 39 | puts "Server address: #{server}" 40 | program = ChatClient.new(ARGV[0], server, :stdin => $stdin) 41 | program.run_fg 42 | -------------------------------------------------------------------------------- /examples/chat/chat_protocol.rb: -------------------------------------------------------------------------------- 1 | module ChatProtocol 2 | state do 3 | channel :connect, [:@addr, :client] => [:nick] 4 | channel :mcast 5 | end 6 | 7 | DEFAULT_ADDR = "127.0.0.1:12345" 8 | end 9 | -------------------------------------------------------------------------------- /examples/chat/chat_server.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'backports' 3 | require 'bud' 4 | require_relative 'chat_protocol' 5 | 6 | class ChatServer 7 | include Bud 8 | include ChatProtocol 9 | 10 | state { table :nodelist } 11 | 12 | bloom do 13 | nodelist <= connect { |c| [c.client, c.nick] } 14 | mcast <~ (mcast * nodelist).pairs { |m,n| [n.key, m.val] } 15 | end 16 | end 17 | 18 | # ruby command-line wrangling 19 | addr = ARGV.first ? ARGV.first : ChatProtocol::DEFAULT_ADDR 20 | ip, port = addr.split(":") 21 | puts "Server address: #{ip}:#{port}" 22 | program = ChatServer.new(:ip => ip, :port => port.to_i) 23 | program.run_fg 24 | -------------------------------------------------------------------------------- /lib/bud/aggs.rb: -------------------------------------------------------------------------------- 1 | module Bud 2 | ######## Agg definitions 3 | class Agg #:nodoc: all 4 | def init(val) 5 | val 6 | end 7 | 8 | # In order to support argagg, trans must return a pair: 9 | # 1. the running aggregate state 10 | # 2. a flag to indicate what the caller should do with the input tuple for argaggs 11 | # a. :ignore tells the caller to ignore this input 12 | # b. :keep tells the caller to save this input 13 | # c. :replace tells the caller to keep this input alone 14 | # d. :delete, [t1, t2, ...] tells the caller to delete the remaining tuples 15 | # For aggs that do not inherit from ArgExemplary, the 2nd part can simply be nil. 16 | def trans(the_state, val) 17 | return the_state, :ignore 18 | end 19 | 20 | def final(the_state) 21 | the_state 22 | end 23 | end 24 | 25 | # ArgExemplary aggs are used by argagg. Canonical examples are min/min (argmin/max) 26 | # They must have a trivial final method and be monotonic, i.e. once a value v 27 | # is discarded in favor of another, v can never be the final result. 28 | class ArgExemplary < Agg #:nodoc: all 29 | def tie(the_state, val) 30 | the_state == val 31 | end 32 | def final(the_state) 33 | the_state 34 | end 35 | end 36 | 37 | class Min < ArgExemplary #:nodoc: all 38 | def trans(the_state, val) 39 | if (the_state <=> val) < 0 40 | return the_state, :ignore 41 | elsif the_state == val 42 | return the_state, :keep 43 | else 44 | return val, :replace 45 | end 46 | end 47 | end 48 | # exemplary aggregate method to be used in Bud::BudCollection.group. 49 | # computes minimum of x entries aggregated. 50 | def min(x) 51 | [Min.new, x] 52 | end 53 | 54 | class Max < ArgExemplary #:nodoc: all 55 | def trans(the_state, val) 56 | if (the_state <=> val) > 0 57 | return the_state, :ignore 58 | elsif the_state == val 59 | return the_state, :keep 60 | else 61 | return val, :replace 62 | end 63 | end 64 | end 65 | # exemplary aggregate method to be used in Bud::BudCollection.group. 66 | # computes maximum of x entries aggregated. 67 | def max(x) 68 | [Max.new, x] 69 | end 70 | 71 | class BooleanAnd < ArgExemplary #:nodoc: all 72 | def trans(the_state, val) 73 | if val == false 74 | return val, :replace 75 | else 76 | return the_state, :ignore 77 | end 78 | end 79 | end 80 | 81 | def bool_and(x) 82 | [BooleanAnd.new, x] 83 | end 84 | 85 | class BooleanOr < ArgExemplary #:nodoc: all 86 | def trans(the_state, val) 87 | if val == true 88 | return val, :replace 89 | else 90 | return the_state, :ignore 91 | end 92 | end 93 | end 94 | 95 | def bool_or(x) 96 | [BooleanOr.new, x] 97 | end 98 | 99 | class Choose < ArgExemplary #:nodoc: all 100 | def trans(the_state, val) 101 | if the_state.nil? 102 | return val, :replace 103 | else 104 | return the_state, :ignore 105 | end 106 | end 107 | def tie(the_state, val) 108 | false 109 | end 110 | end 111 | 112 | # exemplary aggregate method to be used in Bud::BudCollection.group. 113 | # arbitrarily but deterministically chooses among x entries being aggregated. 114 | def choose(x) 115 | [Choose.new, x] 116 | end 117 | 118 | class ChooseOneRand < ArgExemplary #:nodoc: all 119 | def init(x=nil) # Vitter's reservoir sampling, sample size = 1 120 | the_state = {:cnt => 1, :val => x} 121 | end 122 | 123 | def trans(the_state, val) 124 | the_state[:cnt] += 1 125 | j = rand(the_state[:cnt]) 126 | if j < 1 # replace 127 | old_val = the_state[:val] 128 | the_state[:val] = val 129 | return the_state, :replace 130 | else 131 | return the_state, :ignore 132 | end 133 | end 134 | def tie(the_state, val) 135 | true 136 | end 137 | def final(the_state) 138 | the_state[:val] # XXX rand(@@reservoir_size will always be 0, since @@reservoir_size is 1 139 | end 140 | end 141 | 142 | # exemplary aggregate method to be used in Bud::BudCollection.group. 143 | # randomly chooses among x entries being aggregated. 144 | def choose_rand(x=nil) 145 | [ChooseOneRand.new, x] 146 | end 147 | 148 | class Sum < Agg #:nodoc: all 149 | def trans(the_state, val) 150 | return the_state + val, nil 151 | end 152 | end 153 | 154 | # aggregate method to be used in Bud::BudCollection.group. 155 | # computes sum of x entries aggregated. 156 | def sum(x) 157 | [Sum.new, x] 158 | end 159 | 160 | class Count < Agg #:nodoc: all 161 | def init(x=nil) 162 | 1 163 | end 164 | def trans(the_state, x=nil) 165 | return the_state + 1, nil 166 | end 167 | end 168 | 169 | # aggregate method to be used in Bud::BudCollection.group. 170 | # counts number of entries aggregated. argument is ignored. 171 | def count(x=nil) 172 | [Count.new] 173 | end 174 | 175 | class Avg < Agg #:nodoc: all 176 | def init(val) 177 | [val, 1] 178 | end 179 | def trans(the_state, val) 180 | retval = [the_state[0] + val] 181 | retval << (the_state[1] + 1) 182 | return retval, nil 183 | end 184 | def final(the_state) 185 | the_state[0].to_f / the_state[1] 186 | end 187 | end 188 | 189 | # aggregate method to be used in Bud::BudCollection.group. 190 | # computes average of a multiset of x values 191 | def avg(x) 192 | [Avg.new, x] 193 | end 194 | 195 | class Accum < Agg #:nodoc: all 196 | def init(x) 197 | [x].to_set 198 | end 199 | def trans(the_state, val) 200 | the_state << val 201 | return the_state, nil 202 | end 203 | end 204 | 205 | # aggregate method to be used in Bud::BudCollection.group. 206 | # accumulates all x inputs into a set. 207 | def accum(x) 208 | [Accum.new, x] 209 | end 210 | 211 | class AccumPair < Agg #:nodoc: all 212 | def init(fst, snd) 213 | [[fst, snd]].to_set 214 | end 215 | def trans(the_state, fst, snd) 216 | the_state << [fst, snd] 217 | return the_state, nil 218 | end 219 | end 220 | 221 | # aggregate method to be used in Bud::BudCollection.group. 222 | # accumulates x, y inputs into a set of pairs (two element arrays). 223 | def accum_pair(x, y) 224 | [AccumPair.new, x, y] 225 | end 226 | end 227 | -------------------------------------------------------------------------------- /lib/bud/depanalysis.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'bud' 3 | 4 | class DepAnalysis #:nodoc: all 5 | include Bud 6 | 7 | state do 8 | # Data inserted by client, usually from t_depends and t_provides 9 | scratch :depends, [:lhs, :op, :body, :neg, :in_body] 10 | scratch :providing, [:pred, :input] 11 | 12 | # Intermediate state 13 | scratch :depends_clean, [:lhs, :body, :neg, :temporal] 14 | 15 | scratch :depends_tc, [:lhs, :body, :via, :neg, :temporal] 16 | scratch :cycle, [:pred, :via, :neg, :temporal] 17 | scratch :underspecified, [:pred, :input] 18 | scratch :source, [:pred] 19 | scratch :sink, [:pred] 20 | end 21 | 22 | bloom :analysis do 23 | depends_clean <= depends do |d| 24 | is_temporal = (d.op.to_s =~ /<[\+\-\~]/) 25 | [d.lhs, d.body, d.neg, is_temporal] 26 | end 27 | 28 | # Compute the transitive closure of "depends_clean" to detect cycles in 29 | # the deductive fragment of the program. 30 | depends_tc <= depends_clean do |d| 31 | [d.lhs, d.body, d.body, d.neg, d.temporal] 32 | end 33 | depends_tc <= (depends_clean * depends_tc).pairs(:body => :lhs) do |b, r| 34 | [b.lhs, r.body, b.body, (b.neg or r.neg), (b.temporal or r.temporal)] 35 | end 36 | 37 | cycle <= depends_tc do |d| 38 | if d.lhs == d.body 39 | unless d.neg and !d.temporal 40 | [d.lhs, d.via, d.neg, d.temporal] 41 | end 42 | end 43 | end 44 | 45 | source <= providing do |p| 46 | if p.input and !depends_tc.map{|d| d.lhs}.include? p.pred 47 | [p.pred] 48 | end 49 | end 50 | 51 | sink <= providing do |p| 52 | if !p.input and !depends_tc.map{|d| d.body}.include? p.pred 53 | [p.pred] 54 | end 55 | end 56 | 57 | underspecified <= providing do |p| 58 | if p.input 59 | unless depends_tc.map{|d| d.body if d.lhs != d.body}.include? p.pred 60 | [p.pred, true] 61 | end 62 | else 63 | unless depends_tc.map{|dt| dt.lhs if dt.lhs != dt.body}.include? p.pred 64 | [p.pred, false] 65 | end 66 | end 67 | end 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /lib/bud/errors.rb: -------------------------------------------------------------------------------- 1 | module Bud 2 | # Root Bud exception type. 3 | class Error < StandardError; end 4 | 5 | # Raised (at runtime) when a type mismatch occurs (e.g., supplying a 6 | # non-Enumerable object to the RHS of a Bud statement). 7 | class TypeError < Error; end 8 | 9 | # Raised when a primary key constraint is violated. 10 | class KeyConstraintError < Error; end 11 | 12 | # Raised when the input program fails to compile (e.g., due to illegal 13 | # syntax). 14 | class CompileError < Error; end 15 | 16 | # Raised when the program is given in an illegal location (e.g., presented as 17 | # an eval block). 18 | class IllegalSourceError < Error; end 19 | 20 | # Raised when evaluation halts with outstanding callbacks. 21 | class ShutdownWithCallbacksError < Error; end 22 | end 23 | -------------------------------------------------------------------------------- /lib/bud/executor/README.rescan: -------------------------------------------------------------------------------- 1 | Notes on Invalidate and Rescan in Bud 2 | ===================================== 3 | 4 | (I'll use 'downstream' to mean rhs to lhs (like in budplot). In every stratum, 5 | data originates at scanned sources at the "top", winds its way through various 6 | PushElements and ends up in a collection at the "bottom". That is, data flows 7 | from "upstream" producers to "downstream" consumers. I'll also the term 8 | "elements" to mean both dataflow nodes (PushElements) and collections). 9 | 10 | Invalidation strategy works through two flags/signals, rescan and 11 | invalidate. Invalidation means a stateful PushElement or a scratch's contents 12 | are erased, or table is negated. Rescan means that tuples coming out of an 13 | element represent the entire collection (a full-scan), not just deltas. 14 | 15 | Earlier: all stateful elements were eagerly invalidated. 16 | Collections with state: scratches, interfaces, channels, terminal 17 | Elements with state: Group, join, sort, reduce, each_with_index 18 | 19 | Now: lazy invalidation where possible, based on the observation that the same 20 | state is often rederived downstream, which means that as long as there are no 21 | negations, one should be able to go on in incremental mode (working only on 22 | deltas, not on storage) from one tick to another. 23 | 24 | Observations: 25 | 26 | 1. There are two kinds of elements that are (or may be) invalidated at the 27 | beginning of every tick: source scratches (those that are not found on the 28 | lhs of any rule), and tables that process pending negations. 29 | 30 | 2. a. Invalidation implies rescan of its contents. 31 | 32 | b. Rescan of its contents implies invalidation of downstream nodes. 33 | 34 | c. Invalidation involves rebuilding of state, which means that if a node has 35 | multiple sources, it has to ask the other sources to rescan as well. 36 | 37 | Example: x,y,z are scratches 38 | z <= x.group(....) 39 | z <= y.sort {} 40 | 41 | If x is invalidated, it will rescan its contents. The group element then 42 | invalidates its state, and rebuilds itself as x is scanned. Since group is 43 | in rescan mode, z invalidates its state and is rebuilt from group. 44 | However, since part of z's state state comes from y.sort, it asks its 45 | source element (the sort node) for a rescan as well. 46 | 47 | This push-pull negotiation can be run until fixpoint, until the elements 48 | that need to be invalidated and rescanned is determined fully. 49 | 50 | 3. If a node is stateless, it passes the rescan request upstream, and the 51 | invalidations downstream. But if it is stateful, it need not pass a rescan 52 | request upstream. In the example above, only the sort node needs to rescan 53 | its buffer; y doesn't need to be scanned at all. 54 | 55 | 4. Solving the above constraints to a fixpoint at every tick is a huge 56 | overhead. So we determine the strategy at wiring time. 57 | 58 | bud.default_invalidate/default_rescan == set of elements that we know 59 | apriori will _always_ need the corresponding signal. 60 | 61 | scanner.invalidate_set/rescan_set == for each scanner, the set of elements 62 | to invalidate/rescan should that scanner's collection be negated. 63 | 64 | bud.prepare_invalidation_scheme works as follows. 65 | 66 | Start the process by determining which tables will invalidate at each tick, 67 | and which PushElements will rescan at the beginning of each tick. Then run 68 | rescan_invalidate_tc for a transitive closure, where each element gets to 69 | determine its own presence in the rescan and invalidate sets, depending on 70 | its source or target elements' presence in those sets. This creates the 71 | default sets. 72 | 73 | Then for each scanner, prime the pump by setting the scanner to rescan mode, 74 | and determine what effect it has on the system, by running 75 | rescan_invalidate_tc. All the elements that are not already in the default 76 | sets are those that need to be additionally informed at run time, should we 77 | discover that that scanner's collection has been negated at the beginning of 78 | each tick. 79 | 80 | The BUD_SAFE environment variable is used to force old-style behavior, where 81 | every cached element is invalidated and fully scanned once every tick. 82 | -------------------------------------------------------------------------------- /lib/bud/executor/group.rb: -------------------------------------------------------------------------------- 1 | require 'bud/executor/elements' 2 | 3 | module Bud 4 | class PushGroup < PushStatefulElement 5 | def initialize(elem_name, bud_instance, collection_name, 6 | keys_in, aggpairs_in, schema_in, &blk) 7 | if keys_in.nil? 8 | @keys = [] 9 | else 10 | @keys = keys_in.map{|k| k[1]} 11 | end 12 | # An aggpair is an array: [agg class instance, array of indexes of input 13 | # agg input columns]. The second field is nil for Count. 14 | @aggpairs = aggpairs_in.map do |ap| 15 | agg, *rest = ap 16 | if rest.empty? 17 | [agg, nil] 18 | else 19 | [agg, rest.map {|r| r[1]}] 20 | end 21 | end 22 | @groups = {} 23 | 24 | # Check whether we need to eliminate duplicates from our input (we might 25 | # see duplicates because of the rescan/invalidation logic, as well as 26 | # because we don't do duplicate elimination on the output of a projection 27 | # operator). We don't need to dupelim if all the args are exemplary. 28 | @elim_dups = @aggpairs.any? {|ap| not ap[0].kind_of? ArgExemplary} 29 | @input_cache = Set.new if @elim_dups 30 | 31 | super(elem_name, bud_instance, collection_name, schema_in, &blk) 32 | end 33 | 34 | def insert(item, source) 35 | if @elim_dups 36 | return if @input_cache.include? item 37 | @input_cache << item 38 | end 39 | 40 | key = item.values_at(*@keys) 41 | group_state = @groups[key] 42 | if group_state.nil? 43 | @groups[key] = @aggpairs.map do |ap| 44 | if ap[1].nil? 45 | ap[0].init(item) 46 | else 47 | ap[0].init(*item.values_at(*ap[1])) 48 | end 49 | end 50 | else 51 | @aggpairs.each_with_index do |ap, agg_ix| 52 | state_val = group_state[agg_ix] 53 | if ap[1].nil? 54 | trans_rv = ap[0].trans(state_val, item) 55 | else 56 | trans_rv = ap[0].trans(state_val, *item.values_at(*ap[1])) 57 | end 58 | group_state[agg_ix] = trans_rv[0] 59 | end 60 | end 61 | end 62 | 63 | def add_rescan_invalidate(rescan, invalidate) 64 | # XXX: need to understand why this is necessary; it is dissimilar to the 65 | # way other stateful non-monotonic operators are handled. 66 | rescan << self 67 | super 68 | end 69 | 70 | def invalidate_cache 71 | puts "#{self.class}/#{self.tabname} invalidated" if $BUD_DEBUG 72 | @groups.clear 73 | @input_cache.clear if @elim_dups 74 | end 75 | 76 | def flush 77 | # Don't emit fresh output unless a rescan is needed 78 | return unless @rescan 79 | @rescan = false 80 | 81 | @groups.each do |key, group_state| 82 | rv = key.clone 83 | @aggpairs.each_with_index do |ap, agg_ix| 84 | rv << ap[0].final(group_state[agg_ix]) 85 | end 86 | push_out(rv) 87 | end 88 | end 89 | end 90 | 91 | class PushArgAgg < PushGroup 92 | def initialize(elem_name, bud_instance, collection_name, keys_in, aggpairs_in, schema_in, &blk) 93 | unless aggpairs_in.length == 1 94 | raise Bud::Error, "multiple aggpairs #{aggpairs_in.map{|a| a.class.name}} in ArgAgg; only one allowed" 95 | end 96 | super(elem_name, bud_instance, collection_name, keys_in, aggpairs_in, schema_in, &blk) 97 | @winners = {} 98 | end 99 | 100 | public 101 | def invalidate_cache 102 | super 103 | @winners.clear 104 | end 105 | 106 | def insert(item, source) 107 | key = @keys.map{|k| item[k]} 108 | group_state = @groups[key] 109 | if group_state.nil? 110 | @groups[key] = @aggpairs.map do |ap| 111 | @winners[key] = [item] 112 | input_vals = item.values_at(*ap[1]) 113 | ap[0].init(*input_vals) 114 | end 115 | else 116 | @aggpairs.each_with_index do |ap, agg_ix| 117 | input_vals = item.values_at(*ap[1]) 118 | state_val, flag, *rest = ap[0].trans(group_state[agg_ix], *input_vals) 119 | group_state[agg_ix] = state_val 120 | 121 | case flag 122 | when :ignore 123 | # do nothing 124 | when :replace 125 | @winners[key] = [item] 126 | when :keep 127 | @winners[key] << item 128 | when :delete 129 | rest.each do |t| 130 | @winners[key].delete t 131 | end 132 | else 133 | raise Bud::Error, "strange result from argagg transition func: #{flag}" 134 | end 135 | end 136 | end 137 | end 138 | 139 | def flush 140 | # Don't emit fresh output unless a rescan is needed 141 | return unless @rescan 142 | @rescan = false 143 | 144 | @groups.each_key do |g| 145 | @winners[g].each do |t| 146 | push_out(t) 147 | end 148 | end 149 | end 150 | end 151 | end 152 | -------------------------------------------------------------------------------- /lib/bud/labeling/bloomgraph.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'bud' 3 | require 'graphviz' 4 | 5 | # A simple interface between graphviz and bud 6 | module BudGraph 7 | state do 8 | interface input, :bnode, [:name] => [:meta] 9 | interface input, :bedge, [:from, :to, :meta] 10 | end 11 | end 12 | 13 | module BloomGraph 14 | include BudGraph 15 | 16 | state do 17 | table :nodes, bnode.schema 18 | table :edges, bedge.schema 19 | end 20 | 21 | bloom do 22 | nodes <= bnode 23 | edges <= bedge 24 | end 25 | 26 | def finish(ignore, name, fmt=:pdf) 27 | it = ignore.to_set 28 | tick 29 | nodes.to_a.each do |n| 30 | unless it.include? n.name.to_sym 31 | @graph.add_nodes(n.name, n.meta) 32 | end 33 | end 34 | 35 | edges.to_a.each do |e| 36 | unless it.include? e.from.to_sym or it.include? e.to.to_sym 37 | @graph.add_edges(e.from, e.to, e.meta) 38 | end 39 | end 40 | @graph.output(fmt => name) 41 | end 42 | 43 | def initialize(opts={:type => :digraph}) 44 | @graph = GraphViz.new(:G, opts) 45 | super 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /lib/bud/labeling/budplot_style.rb: -------------------------------------------------------------------------------- 1 | require 'bud/labeling/labeling' 2 | 3 | module PDG 4 | include GuardedAsync 5 | # a bloomgraph program that plots a NM-and-async-aware PDG 6 | state do 7 | scratch :bodies, [:table] => [:tbl_type] 8 | scratch :source, [:pred] 9 | scratch :sink, [:pred] 10 | end 11 | 12 | bloom do 13 | bodies <= dep{|d| [d.body, coll_type(d.body)]} 14 | bodies <= dep{|d| [d.head, coll_type(d.head)]} 15 | 16 | bnode <= bodies do |b| 17 | shape = case b.tbl_type 18 | when Bud::BudTable then "rectangle" 19 | when Bud::LatticeWrapper then "triangle" 20 | else "oval" 21 | end 22 | [b.table, {:shape => shape}] 23 | end 24 | 25 | bedge <= dep do |d| 26 | line = d.label == "A" ? "dashed" : "solid" 27 | circle = d.label == "N" ? "veeodot" : "normal" 28 | [d.body, d.head, {:style => line, :arrowhead => circle, :penwidth => 4}] 29 | end 30 | end 31 | 32 | bloom :endpoints do 33 | source <= t_provides do |p| 34 | if p.input and !dep_tc.map{|d| d.head}.include? p.interface 35 | [p.interface] 36 | end 37 | end 38 | 39 | sink <= t_provides do |p| 40 | if !p.input and !dep_tc.map{|d| d.body}.include? p.interface 41 | [p.interface] 42 | end 43 | end 44 | 45 | bedge <= source{|s| ["S", s.pred, {}]} 46 | bedge <= sink{|s| [s.pred, "T", {}]} 47 | end 48 | 49 | bootstrap do 50 | bnode << ["S", {:shape => "diamond", :color => "blue"}] 51 | bnode << ["T", {:shape => "diamond", :color => "blue"}] 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /lib/bud/labeling/labeling.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'bud' 3 | 4 | module Validate 5 | state do 6 | scratch :dep, [:body, :head, :label] 7 | scratch :dep_tc, [:body, :head, :members] 8 | scratch :scc, [:pred, :cluster] 9 | scratch :scc_raw, scc.schema 10 | scratch :new_dep, [:body, :head, :label] 11 | scratch :labeled_path, [:body, :head, :path, :label] 12 | scratch :full_path, labeled_path.schema 13 | scratch :ndn, new_dep.schema 14 | scratch :iinterface, t_provides.schema 15 | scratch :ointerface, t_provides.schema 16 | scratch :iin, t_provides.schema 17 | scratch :iout, t_provides.schema 18 | end 19 | 20 | bloom do 21 | dep <= t_depends do |d| 22 | [d.body, d.lhs, labelof(d.op, d.nm)] 23 | end 24 | 25 | dep_tc <= dep do |d| 26 | [d.body, d.head, Set.new([d.body, d.head])] 27 | end 28 | dep_tc <= (dep * dep_tc).pairs(:head => :body) do |d, t| 29 | [d.body, t.head, t.members | [d.head]] 30 | end 31 | 32 | scc_raw <= dep_tc do |d| 33 | if d.head == d.body 34 | [d.head, d.members.to_a.sort] 35 | end 36 | end 37 | 38 | scc <= scc_raw.reduce(Hash.new) do |memo, i| 39 | memo[i.pred] ||= [] 40 | memo[i.pred] |= i.cluster 41 | memo 42 | end 43 | 44 | new_dep <= (dep * scc * scc).combos do |d, s1, s2| 45 | if d.head == s1.pred and d.body == s2.pred 46 | ["#{s2.cluster.join(",")}_IN", "#{s1.cluster.join(",")}_OUT", d.label] 47 | end 48 | end 49 | new_dep <= (dep * scc).pairs(:body => :pred) do |d, s| 50 | ["#{s.cluster.join(",")}_OUT", d.head, d.label] unless s.cluster.include? d.head 51 | end 52 | new_dep <= (dep * scc).pairs(:head => :pred) do |d, s| 53 | [d.body, "#{s.cluster.join(",")}_IN", d.label] unless s.cluster.include? d.body 54 | end 55 | 56 | ndn <= dep.notin(scc, :body => :pred) 57 | new_dep <= ndn.notin(scc, :head => :pred) 58 | end 59 | 60 | bloom :channel_inputs do 61 | temp :dummy_input <= t_provides do |p| 62 | if p.input and coll_type(p.interface) == Bud::BudChannel 63 | [p.interface] 64 | end 65 | end 66 | dep <= dummy_input{|i| ["#{i.first}_INPUT", i.first, "A"]} 67 | dep <= dummy_input{|i| ["#{i.first}_INPUT", i.first, "A"]} 68 | t_provides <= dummy_input{|i| ["#{i.first}_INPUT", true]} 69 | end 70 | 71 | bloom :full_paths do 72 | iin <= t_provides{|p| p if p.input} 73 | iout <= t_provides{|p| p if !p.input} 74 | iinterface <= iin.notin(new_dep, :interface => :head) 75 | ointerface <= iout.notin(new_dep, :interface => :body) 76 | 77 | labeled_path <= (new_dep * iinterface).pairs(:body => :interface) do |d, p| 78 | [d.body, d.head, [d.body, d.head], [d.label]] 79 | end 80 | labeled_path <= (labeled_path * new_dep).pairs(:head => :body) do |p, d| 81 | [p.body, d.head, p.path + [d.head], p.label + [d.label]] 82 | end 83 | 84 | full_path <= (labeled_path * ointerface).lefts(:head => :interface) 85 | end 86 | 87 | def validate 88 | dp = Set.new 89 | divergent_preds.each do |p| 90 | dp.add(p.coll) 91 | end 92 | report = [] 93 | full_path.to_a.each do |p| 94 | state = ["Bot"] 95 | start_a = -1 96 | p.label.each_with_index do |lbl, i| 97 | if lbl == "A" 98 | start_a = i + 1 99 | end 100 | os = state.first 101 | state = do_collapse(state, [lbl]) 102 | end 103 | if dp.include? p.head 104 | report << (p.to_a + [:unguarded, ["D"]]) 105 | else 106 | report << (p.to_a + [:path, state]) 107 | end 108 | end 109 | return report 110 | end 111 | 112 | def do_collapse(left, right) 113 | l = left.pop 114 | r = right.shift 115 | left + collapse(l, r) + right 116 | end 117 | 118 | def labelof(op, nm) 119 | if op == "<~" 120 | "A" 121 | elsif nm 122 | "N" 123 | else 124 | "Bot" 125 | end 126 | end 127 | 128 | def collapse(left, right) 129 | return [right] if left == 'Bot' 130 | return [left] if right == 'Bot' 131 | return [left] if left == right 132 | return ['D'] if left == 'D' or right == 'D' 133 | # CALM 134 | return ['D'] if left == 'A' and right =~ /N/ 135 | # sometimes we cannot reduce 136 | return [left, right] 137 | end 138 | end 139 | 140 | 141 | module GuardedAsync 142 | include Validate 143 | state do 144 | scratch :meet, [:chan, :partner, :at, :lpath, :rpath] 145 | scratch :meet_stg, meet.schema 146 | scratch :channel_race, [:chan, :partner, :to, :guarded] 147 | scratch :dep_tc_type, [:body, :head, :types] 148 | scratch :divergent_preds, [:coll] 149 | end 150 | 151 | bloom do 152 | dep_tc_type <= dep do |d| 153 | btab = coll_type(d.body) 154 | htab = coll_type(d.head) 155 | [d.body, d.head, Set.new([btab, htab])] 156 | end 157 | dep_tc_type <= (dep * dep_tc_type).pairs(:head => :body) do |d, t| 158 | htab = coll_type(d.head) 159 | [d.body, t.head, t.types | [htab]] 160 | end 161 | 162 | meet_stg <= (dep_tc_type * dep_tc_type).pairs(:head => :head) do |l, r| 163 | ltab = self.tables[l.body.to_sym] 164 | rtab = self.tables[r.body.to_sym] 165 | if ltab.class == Bud::BudChannel and rtab.class == Bud::BudChannel and l.body != r.body 166 | [l.body, r.body, l.head, l.types, r.types] 167 | end 168 | end 169 | 170 | meet <= meet_stg.notin(dep_tc_type, :chan => :body, :partner => :head) 171 | channel_race <= meet{|m| [m.chan, m.partner, m.at, guarded(m.lpath, m.rpath)]} 172 | divergent_preds <= channel_race{|r| [r.to] unless r.guarded} 173 | divergent_preds <= (channel_race * dep_tc_type).pairs(:to => :body){|r, t| [t.head] unless r.guarded} 174 | end 175 | 176 | def coll_type(nm) 177 | tab = self.tables[nm.to_sym] 178 | if tab.nil? 179 | tab = self.lattices[nm.to_sym] 180 | end 181 | tab.class 182 | end 183 | 184 | def guarded(lpath, rpath) 185 | if lpath.include? Bud::BudTable or lpath.include? Bud::LatticeWrapper 186 | if rpath.include? Bud::BudTable or rpath.include? Bud::LatticeWrapper 187 | return true 188 | end 189 | end 190 | false 191 | end 192 | end 193 | 194 | require 'bud/labeling/bloomgraph' 195 | require 'bud/labeling/budplot_style' 196 | 197 | module MetaMods 198 | include Validate 199 | include GuardedAsync 200 | include BloomGraph 201 | include PDG 202 | end 203 | 204 | class Label 205 | attr_reader :f 206 | 207 | def initialize(mod) 208 | @report = nil 209 | @mod = Object.const_get(mod) 210 | if @mod.class == Class 211 | nc = new_class_from_class(@mod) 212 | elsif @mod.class == Module 213 | nc = new_class(@mod) 214 | else 215 | raise "#{mod} neither class nor module" 216 | end 217 | @f = nc.new 218 | @f.tick 219 | end 220 | 221 | def validate 222 | @report = @f.validate if @report.nil? 223 | end 224 | 225 | def output_report 226 | validate 227 | rep = {} 228 | @report.each do |from, to, path, labels, reason, final| 229 | rep[to] ||= "Bot" 230 | rep[to] = disjunction(rep[to], final.last) 231 | end 232 | rep 233 | end 234 | 235 | def path_report 236 | validate 237 | zips = {} 238 | @report.each do |from, to, path, labels, reason, final| 239 | zips[to] ||= {} 240 | zips[to][from] ||= "Bot" 241 | zips[to][from] = disjunction(zips[to][from], final.last) 242 | end 243 | zips 244 | end 245 | 246 | def disjunction(l, r) 247 | both = [l, r] 248 | if both.include? "D" 249 | "D" 250 | elsif both.include? "N" 251 | if both.include? "A" 252 | return "D" 253 | else 254 | return "N" 255 | end 256 | elsif both.include? "A" 257 | return "A" 258 | else 259 | return "Bot" 260 | end 261 | end 262 | 263 | def new_class(mod) 264 | Class.new do 265 | include Bud 266 | include MetaMods 267 | include mod 268 | end 269 | end 270 | 271 | def new_class_from_class(cls) 272 | Class.new(cls) do 273 | include MetaMods 274 | end 275 | end 276 | 277 | def internal_tabs 278 | cls = Class.new do 279 | include Bud 280 | include MetaMods 281 | end 282 | cls.new.tables.keys 283 | end 284 | 285 | def write_graph(fmt=:pdf) 286 | f.finish(internal_tabs, "#{@mod.to_s}.#{fmt}", fmt) 287 | end 288 | end 289 | -------------------------------------------------------------------------------- /lib/bud/meta_algebra.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'bud' 3 | 4 | module MetaAlgebra 5 | state do 6 | table :alg_path, [:from, :to, :path, :last_rule, :tag, :lastop] 7 | scratch :clean_dep, [:body, :head, :rule_id] => [:tag, :lastop] 8 | 9 | scratch :rule_nm, [:rule_id] => [:tag] 10 | 11 | table :apj1, [:from, :head, :rule_id, :path, :tag, :tag2, :lastop] 12 | table :apj2, [:from, :head, :rule_id] => [:path, :tag, :lastop] 13 | table :seq_lattice, [:left, :right, :directional] 14 | table :seq_lattice_closure, [:left, :right, :directional, :dist] 15 | table :lub, [:left, :right] => [:result] 16 | table :seq_lattice_result, [:left, :right, :result] 17 | table :upper_bound, [:left, :right, :bound, :height] 18 | table :lps, alg_path.schema 19 | end 20 | 21 | bootstrap do 22 | seq_lattice <= [ 23 | [:M, :A, false], 24 | [:M, :N, false], 25 | [:A, :D, false], 26 | [:N, :D, false], 27 | [:N, :A, true] #, 28 | 29 | # disabled, for now 30 | #[:A, :D, true], 31 | #[:D, :G, false], 32 | #[:A, :G, false] 33 | ] 34 | 35 | end 36 | 37 | def max_of(a, b) 38 | if b > a 39 | b 40 | else 41 | a 42 | end 43 | end 44 | 45 | bloom :debug do 46 | #stdio <~ upper_bound{|b| ["UPPERB: #{b.inspect}"]} 47 | #stdio <~ seq_lattice_closure{|c| ["SLC: #{c.inspect}"]} 48 | #stdio <~ jlr {|j| ["JLR: #{j.inspect}"]} 49 | #stdio <~ lub {|l| ["LUB #{l.inspect}, left class #{l.left.class}"]} 50 | #stdio <~ clean_dep.inspected 51 | end 52 | 53 | bloom :lattice_rules do 54 | seq_lattice_closure <= seq_lattice {|l| [l.left, l.right, l.directional, 1]} 55 | seq_lattice_closure <= seq_lattice {|l| [l.left, l.left, false, 0]} 56 | seq_lattice_closure <= seq_lattice {|l| [l.right, l.right, false, 0]} 57 | seq_lattice_closure <= (seq_lattice_closure * seq_lattice).pairs(:right => :left) do |c, l| 58 | [c.left, l.right, (c.directional or l.directional), c.dist + 1] 59 | end 60 | 61 | # the join lattice is symmetric 62 | lub <= seq_lattice_closure {|l| [l.left, l.right, l.right]} 63 | lub <= seq_lattice_closure {|l| [l.right, l.left, l.right] unless l.directional} 64 | 65 | # still need a LUB for incomparable types. 66 | upper_bound <= (seq_lattice_closure * seq_lattice_closure).map do |c1, c2| 67 | if c1.right == c2.right and seq_lattice_closure.find_all{|c| c.left == c1.left and c.right == c2.left}.empty? 68 | unless c1.left == c1.right or c2.left == c2.right 69 | [c1.left, c2.left, c1.right, max_of(c1.dist, c2.dist) + 1] 70 | end 71 | end 72 | end 73 | 74 | temp :jlr <= upper_bound.argagg(:min, [upper_bound.left, upper_bound.right], upper_bound.height) 75 | lub <+ jlr {|j| [j.left, j.right, j.bound] unless lub.map{|l| [l.left, l.right]}.include? [j.left, j.right] } 76 | end 77 | 78 | def get_tag(nm, op) 79 | if nm and op == '<~' 80 | :D 81 | elsif nm 82 | :N 83 | elsif op == '<~' 84 | :A 85 | else 86 | :M 87 | end 88 | end 89 | 90 | def in_prefix(node, path) 91 | path.split("|").include? node 92 | end 93 | 94 | bloom :make_paths do 95 | rule_nm <= t_depends.reduce(Hash.new) do |memo, i| 96 | tag = get_tag(i.nm, i.op) 97 | if memo[i.rule_id].nil? or memo[i.rule_id] == :M 98 | memo[i.rule_id] = tag 99 | end 100 | memo 101 | end 102 | 103 | clean_dep <= (t_depends * rule_nm).pairs(:rule_id => :rule_id) do |dep, rn| 104 | unless dep.lhs == 'alg_path' 105 | [dep.body, dep.lhs, dep.rule_id, rn.tag, dep.op] 106 | end 107 | end 108 | 109 | alg_path <= clean_dep.map do |dep| 110 | [dep.body, dep.head, "#{dep.body}|#{dep.head}", dep.rule_id, dep.tag, dep.lastop] 111 | end 112 | 113 | lps <= (alg_path * t_provides).pairs(:from => :interface) do |a, p| 114 | if p.input 115 | a 116 | end 117 | end 118 | 119 | apj1 <= (alg_path * clean_dep).pairs(:to => :body) do |a, c| 120 | [a.from, c.head, c.rule_id, a.path, a.tag, c.tag, a.lastop] 121 | end 122 | apj2 <= (apj1 * lub).pairs(:tag => :left, :tag2 => :right) 123 | alg_path <= apj2.map do |p, l| 124 | unless in_prefix(p.head, p.path) 125 | [p.from, p.head, "#{p.path}|#{p.head}", p.rule_id, l.result, p.lastop] 126 | end 127 | end 128 | end 129 | end 130 | 131 | module MetaReports 132 | state do 133 | table :global_property, [:from, :to, :tag, :c1, :c2] 134 | scratch :paths, [:from, :to] => [:cnt] 135 | table :tags, [:from, :to, :tag, :cnt] 136 | table :d_begins, [:from, :tag, :path, :len, :lastop] 137 | table :ap, d_begins.schema 138 | scratch :a_preds, d_begins.schema + [:fullpath] 139 | end 140 | 141 | bloom :loci do 142 | # one approach: for every paths that 'turns D', identify the last async edge before 143 | # the critical transition. ordering this edge prevents diffluence. 144 | # find the first point of diffluence in each paths: d_begins already does this. 145 | # for each "D"-entry in d_begins, find the longest subpath ending in an async rule. 146 | a_preds <= (d_begins * ap).pairs(:from => :from) do |b, a| 147 | if a.len < b.len and a.tag == :A and b.path.index(a.path) == 0 and a.lastop == "<~" and b.tag == :D 148 | [a.from, a.tag, a.path, a.len, a.lastop, b.path] 149 | end 150 | end 151 | end 152 | 153 | bloom do 154 | paths <= alg_path.group([:from, :to], count(:tag)) 155 | tags <= alg_path.group([:from, :to, :tag], count()) 156 | global_property <= (paths * tags).pairs(:from => :from, :to => :to, :cnt => :cnt) do |p, t| 157 | [t.from, t.to, t.tag, p.cnt, t.cnt] 158 | end 159 | 160 | ap <= (alg_path * t_provides).pairs(:from => :interface) do |p, pr| 161 | if pr.input 162 | [p.from, p.tag, p.path, p.path.split("|").length, p.lastop] 163 | end 164 | end 165 | 166 | d_begins <= ap.argagg(:min, [:from, :tag], :len) 167 | end 168 | end 169 | 170 | -------------------------------------------------------------------------------- /lib/bud/metrics.rb: -------------------------------------------------------------------------------- 1 | require "bud/errors" 2 | if RUBY_VERSION <= "1.9" 3 | require 'faster_csv' 4 | $mod = FasterCSV 5 | else 6 | require 'csv' 7 | $mod = CSV 8 | end 9 | 10 | # Metrics are reported in a nested hash representing a collection of relational tables. 11 | # The metrics hash has the following form: 12 | # - key of the metrics hash is the name of the metric (table), e.g. "tickstats", "collections", "rules", etc. 13 | # - value of the metrics is itself a hash holding the rows of the table, keyed by key columns. 14 | # - It has the following form: 15 | # - key is a hash of key attributes, with the following form: 16 | # - key is the name of an attribute 17 | # - value is the attribute's value 18 | # - value is a single dependent value, e.g. a statistic like a count 19 | 20 | def report_metrics 21 | metrics.each do |k,v| 22 | if v.first 23 | csvstr = $mod.generate(:force_quotes=>true) do |csv| 24 | csv << [k.to_s] + v.first[0].keys + [:val] 25 | v.each do |row| 26 | csv << [nil] + row[0].values + [row[1]] 27 | end 28 | end 29 | puts csvstr 30 | end 31 | end 32 | end 33 | 34 | def initialize_stats 35 | return {{:name=>:count} => 0, {:name=>:mean} => 0, {:name=>:Q} => 0, {:name=>:stddev} => 0} 36 | end 37 | 38 | # see http://en.wikipedia.org/wiki/Standard_deviation#Rapid_calculation_methods 39 | def running_stats(stats, elapsed) 40 | raise Bud::Error, "running_stats called with negative elapsed time" if elapsed < 0 41 | stats[{:name=>:count}] += 1 42 | oldmean = stats[{:name=>:mean}] 43 | stats[{:name=>:mean}] = stats[{:name=>:mean}] + \ 44 | (elapsed - stats[{:name=>:mean}]) / stats[{:name=>:count}] 45 | stats[{:name=>:Q}] = stats[{:name=>:Q}] + \ 46 | (elapsed - oldmean) * (elapsed - stats[{:name=>:mean}]) 47 | stats[{:name=>:stddev}] = Math.sqrt(stats[{:name=>:Q}]/stats[{:name=>:count}]) 48 | stats 49 | end 50 | -------------------------------------------------------------------------------- /lib/bud/monkeypatch.rb: -------------------------------------------------------------------------------- 1 | # We monkeypatch Module to add support for Bloom's syntax additions: "state", 2 | # "bloom", and "bootstrap" blocks, plus the "import" statement. 3 | 4 | require 'bud/source' 5 | 6 | class Class 7 | def modules 8 | a = self.ancestors 9 | a[1..a.index(superclass)-1] 10 | end 11 | end 12 | 13 | $struct_classes = {} 14 | $struct_lock = Mutex.new 15 | 16 | # FIXME: Should likely override #hash and #eql? as well. 17 | class Bud::TupleStruct < Struct 18 | include Comparable 19 | 20 | def self.new_struct(cols) 21 | $struct_lock.synchronize { 22 | ($struct_classes[cols] ||= Bud::TupleStruct.new(*cols)) 23 | } 24 | end 25 | 26 | # XXX: This only considers two TupleStruct instances to be equal if they have 27 | # the same schema (column names) AND the same contents; unclear if structural 28 | # equality (consider only values, not column names) would be better. 29 | def <=>(o) 30 | if o.class == self.class 31 | self.each_with_index do |e, i| 32 | other = o[i] 33 | next if e == other 34 | return e <=> other 35 | end 36 | return 0 37 | elsif o.nil? 38 | return nil 39 | else 40 | raise "Comparison (<=>) between #{o.class} and #{self.class} not implemented" 41 | end 42 | end 43 | 44 | def ==(o) 45 | if o.class == self.class 46 | return super 47 | elsif o.class == Array 48 | return false if self.length != o.length 49 | self.each_with_index do |el, i| 50 | return false if el != o[i] 51 | end 52 | return true 53 | end 54 | false 55 | end 56 | 57 | def hash 58 | self.values.hash 59 | end 60 | 61 | def eql?(o) 62 | self == o 63 | end 64 | 65 | def +(o) 66 | self.to_ary + o.to_ary 67 | end 68 | 69 | def to_msgpack(out=nil) 70 | self.to_a.to_msgpack(out) 71 | end 72 | 73 | def inspect 74 | self.to_a.inspect 75 | end 76 | 77 | alias :to_s :inspect 78 | alias :to_ary :to_a 79 | end 80 | 81 | # XXX: TEMPORARY/UGLY hack to ensure that arrays and structs compare. This can be 82 | # removed once tests are rewritten. 83 | class Array 84 | alias :old_eq :== 85 | alias :old_eql? :eql? 86 | 87 | def ==(o) 88 | o = o.to_a if o.kind_of? Bud::TupleStruct 89 | self.old_eq(o) 90 | end 91 | 92 | def eql?(o) 93 | o = o.to_a if o.kind_of? Bud::TupleStruct 94 | self.old_eql?(o) 95 | end 96 | end 97 | 98 | 99 | $moduleWrapper = {} # module => wrapper class. See import below. 100 | class Module 101 | def modules 102 | ancestors[1..-1] 103 | end 104 | 105 | # import another module and assign to a qualifier symbol: import MyModule => :m 106 | def import(spec) 107 | raise Bud::CompileError unless (spec.class <= Hash and spec.length == 1) 108 | mod, local_name = spec.first 109 | raise Bud::CompileError unless (mod.class <= Module and local_name.class <= Symbol) 110 | if mod.class <= Class 111 | raise Bud::CompileError, "import must be used with a Module, not a Class" 112 | end 113 | 114 | # A statement like this: 115 | # import MyModule => :m 116 | # is translated as follows. First, module MyModule is made instantiable by wrapping it in a class 117 | # class MyModule__wrap__ 118 | # include Bud 119 | # include MyModule 120 | # end 121 | # 122 | # Then introduce a method "m", the import binding name, in the calling module/class 123 | # (the one with the import statement). This returns an instance of the wrapped class. 124 | # inst = MyModule__wrap__.new 125 | # def m 126 | # inst 127 | # end 128 | 129 | mod, local_name = spec.first 130 | 131 | if self.method_defined? local_name 132 | raise Bud::CompileError, "#{local_name} is already taken" 133 | else 134 | src = %Q{ 135 | def #{local_name} 136 | @#{local_name} 137 | end 138 | def #{local_name}=(val) 139 | raise Bud::Error, "type error: expecting an instance of #{mod}" unless val.kind_of? #{mod} 140 | @#{local_name} = val 141 | end 142 | } 143 | self.class_eval src 144 | end 145 | 146 | import_tbl = self.bud_import_table 147 | import_tbl[local_name] = mod 148 | end 149 | 150 | def bud_import_table() #:nodoc: all 151 | @bud_import_tbl ||= {} 152 | @bud_import_tbl 153 | end 154 | 155 | # the block of Bloom collection declarations. one per module. 156 | def state(&block) 157 | meth_name = Module.make_state_meth_name(self) 158 | define_method(meth_name, &block) 159 | end 160 | 161 | # a ruby block to be run before timestep 1. one per module. 162 | def bootstrap(&block) 163 | meth_name = "__bootstrap__#{Module.get_class_name(self)}".to_sym 164 | define_method(meth_name, &block) 165 | end 166 | 167 | # bloom statements to be registered with Bud runtime. optional +block_name+ 168 | # assigns a name for the block; this is useful documentation, and also allows 169 | # the block to be overridden in a child class. 170 | def bloom(block_name=nil, &block) 171 | # If no block name was specified, generate a unique name 172 | if block_name.nil? 173 | @block_id ||= 0 174 | block_name = "#{Module.get_class_name(self)}__#{@block_id}".to_sym 175 | @block_id += 1 176 | else 177 | unless block_name.class <= Symbol 178 | raise Bud::CompileError, "block name must be a symbol: #{block_name}" 179 | end 180 | end 181 | 182 | # Note that we don't encode the module name ("self") into the name of the 183 | # method. This allows named blocks to be overridden (via inheritance or 184 | # mixin) in the same way as normal Ruby methods. 185 | meth_name = "__bloom__#{block_name}" 186 | 187 | # Don't allow duplicate named bloom blocks to be defined within a single 188 | # module; this indicates a likely programmer error. 189 | if instance_methods(false).include?(meth_name) || 190 | instance_methods(false).include?(meth_name.to_sym) 191 | raise Bud::CompileError, "duplicate block name: '#{block_name}' in #{self}" 192 | end 193 | ast = Source.read_block(caller[0]) # pass in caller's location via backtrace 194 | 195 | # ast corresponds only to the statements of the block. Wrap it in a method 196 | # definition for backward compatibility for now. 197 | 198 | # If the block contained multiple statements, the AST will have a top-level 199 | # :block node. Since ruby_parser ASTs for method definitions don't contain 200 | # such a node, remove it. 201 | if ast.nil? 202 | ast = [] 203 | elsif ast.sexp_type == :block 204 | ast = ast.sexp_body 205 | else 206 | ast = [ast] 207 | end 208 | ast = s(:defn, meth_name.to_sym, s(:args), *ast) 209 | unless self.respond_to? :__bloom_asts__ 210 | def self.__bloom_asts__ 211 | @__bloom_asts__ ||= {} 212 | @__bloom_asts__ 213 | end 214 | end 215 | __bloom_asts__[meth_name] = ast 216 | define_method(meth_name.to_sym, &block) 217 | end 218 | 219 | # Return a string with a version of the class name appropriate for embedding 220 | # into a method name. Annoyingly, if you define class X nested inside 221 | # class/module Y, X's class name is the string "Y::X". We don't want to define 222 | # method names with semicolons in them, so just return "X" instead. 223 | private 224 | def self.get_class_name(klass) 225 | (klass.name.nil? or klass.name == "") \ 226 | ? "Anon#{klass.object_id}" \ 227 | : klass.name.split("::").last 228 | end 229 | 230 | # State method blocks are named using an auto-incrementing counter. This is to 231 | # ensure that we can rediscover the possible dependencies between these blocks 232 | # after module import (see Bud#call_state_methods). 233 | def self.make_state_meth_name(klass) 234 | @state_meth_id ||= 0 235 | r = "__state#{@state_meth_id}__#{Module.get_class_name(klass)}".to_sym 236 | @state_meth_id += 1 237 | return r 238 | end 239 | end 240 | 241 | 242 | module Enumerable 243 | public 244 | # We rewrite "map" calls in Bloom blocks to invoke the "pro" method 245 | # instead. This is fine when applied to a BudCollection; when applied to a 246 | # normal Enumerable, just treat pro as an alias for map. 247 | def pro(&blk) 248 | map(&blk) 249 | end 250 | end 251 | -------------------------------------------------------------------------------- /lib/bud/rtrace.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'bud/state' 3 | 4 | class RTrace #:nodoc: all 5 | attr_reader :table_recv, :table_send, :table_sleep 6 | 7 | def initialize(bud_instance) 8 | @bud_instance = bud_instance 9 | return if bud_instance.class == Stratification or 10 | @bud_instance.class == DepAnalysis 11 | @table_recv = Bud::BudTable.new(:t_recv_time, @bud_instance, [:pred, :tuple, :time]) 12 | @table_send = Bud::BudTable.new(:t_send_time, @bud_instance, [:pred, :tuple, :time]) 13 | @table_sleep = Bud::BudTable.new(:t_sleep_time, @bud_instance, [:time]) 14 | end 15 | 16 | def send(pred, datum) 17 | @table_send << [pred.to_s, datum, Time.now.to_f] 18 | end 19 | 20 | def recv(datum) 21 | @table_recv << [datum[0].to_s, datum[1], Time.now.to_f] 22 | end 23 | 24 | def sleep 25 | @table_sleep << [Time.now.to_f] 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /lib/bud/server.rb: -------------------------------------------------------------------------------- 1 | require 'socket' 2 | 3 | class Bud::BudServer < EM::Connection #:nodoc: all 4 | def initialize(bud, channel_filter) 5 | @bud = bud 6 | @channel_filter = channel_filter 7 | @filter_buf = {} 8 | @pac = MessagePack::Unpacker.new 9 | super 10 | end 11 | 12 | def receive_data(data) 13 | # Feed the received data to the deserializer 14 | @pac.feed_each(data) do |obj| 15 | recv_message(obj) 16 | end 17 | 18 | # apply the channel filter to each channel's pending tuples 19 | buf_leftover = {} 20 | @filter_buf.each do |tbl_name, buf| 21 | if @channel_filter 22 | accepted, saved = @channel_filter.call(tbl_name, buf) 23 | else 24 | accepted = buf 25 | saved = [] 26 | end 27 | 28 | unless accepted.empty? 29 | @bud.inbound[tbl_name] ||= [] 30 | @bud.inbound[tbl_name].concat(accepted) 31 | end 32 | buf_leftover[tbl_name] = saved unless saved.empty? 33 | end 34 | @filter_buf = buf_leftover 35 | 36 | begin 37 | @bud.tick_internal if @bud.running_async 38 | rescue Exception => e 39 | # If we raise an exception here, EM dies, which causes problems (e.g., 40 | # other Bud instances in the same process will crash). Ignoring the 41 | # error isn't best though -- we should do better (#74). 42 | puts "Exception handling network messages: #{e}" 43 | puts e.backtrace 44 | puts "Inbound messages:" 45 | @bud.inbound.each do |chn_name, t| 46 | puts " #{t.inspect} (channel: #{chn_name})" 47 | end 48 | @bud.inbound.clear 49 | end 50 | 51 | @bud.rtracer.sleep if @bud.options[:rtrace] 52 | end 53 | 54 | def recv_message(obj) 55 | unless (obj.class <= Array and obj.length == 3 and 56 | @bud.tables.include?(obj[0].to_sym) and 57 | obj[1].class <= Array and obj[2].class <= Array) 58 | raise Bud::Error, "bad inbound message of class #{obj.class}: #{obj.inspect}" 59 | end 60 | 61 | # Deserialize any nested marshalled values 62 | tbl_name, tuple, marshall_indexes = obj 63 | marshall_indexes.each do |i| 64 | if i < 0 || i >= tuple.length 65 | raise Bud::Error, "bad inbound message: marshalled value at index #{i}, #{obj.inspect}" 66 | end 67 | tuple[i] = Marshal.load(tuple[i]) 68 | end 69 | 70 | obj = [tbl_name, tuple] 71 | @bud.rtracer.recv(obj) if @bud.options[:rtrace] 72 | @filter_buf[obj[0].to_sym] ||= [] 73 | @filter_buf[obj[0].to_sym] << obj[1] 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /lib/bud/source.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | 3 | module Source 4 | $cached_file_info = Struct.new(:curr_file, :lines, :last_state_bloom_line).new 5 | 6 | # Reads the block corresponding to the location (string of the form 7 | # "file:line_num"). Returns an ast for the block. 8 | def Source.read_block(location) 9 | if location.start_with? '(' 10 | raise Bud::IllegalSourceError, "source must be present in a file; cannot read interactive shell or eval block" 11 | end 12 | location =~ /^(.*):(\d+)/ 13 | filename, num = $1, $2.to_i 14 | if filename.nil? 15 | raise Bud::IllegalSourceError, "couldn't determine filename from backtrace" 16 | end 17 | lines = cache(filename, num) 18 | # Note: num is 1-based. 19 | 20 | # for_current_ruby might object if the current Ruby version is not supported 21 | # by RubyParser; bravely try to continue on regardless 22 | parser = RubyParser.for_current_ruby rescue RubyParser.new 23 | stmt = "" # collection of lines that form one complete Ruby statement 24 | ast = nil 25 | lines[num .. -1].each do |l| 26 | next if l =~ /^\s*#/ 27 | if l =~ /^\s*([}]|end)/ 28 | # We found some syntax that looks like it might terminate the Ruby 29 | # statement. Hence, try to parse it; if we don't find a syntax error, 30 | # we're done. 31 | begin 32 | ast = parser.parse stmt 33 | break 34 | rescue 35 | ast = nil 36 | end 37 | end 38 | stmt += l + "\n" 39 | end 40 | ast 41 | end 42 | 43 | def Source.cache(filename, num) # returns array of lines 44 | if $cached_file_info.curr_file == filename 45 | retval = $cached_file_info.lines 46 | if $cached_file_info.last_state_bloom_line == num 47 | # have no use for the cached info any more. reset it. 48 | $cached_file_info.lines = [] 49 | $cached_file_info.curr_file = "" 50 | $cached_file_info.last_state_bloom_line = -1 51 | end 52 | else 53 | $cached_file_info.last_state_bloom_line = -1 54 | $cached_file_info.curr_file = filename 55 | $cached_file_info.lines = [] 56 | retval = [] 57 | File.open(filename, "r").each_with_index {|line, i| 58 | retval << line 59 | if line =~ /^ *(bloom|state)/ 60 | $cached_file_info.last_state_bloom_line = i 61 | end 62 | } 63 | $cached_file_info.lines = retval 64 | end 65 | retval # array of lines 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /lib/bud/state.rb: -------------------------------------------------------------------------------- 1 | module Bud 2 | ######## methods for registering collection types 3 | private 4 | def check_collection_name(name) 5 | if @tables.has_key? name or @lattices.has_key? name 6 | raise Bud::CompileError, "collection already exists: #{name}" 7 | end 8 | 9 | # Rule out collection names that use reserved words, including 10 | # previously-defined method names. 11 | reserved = eval "defined?(#{name})" 12 | unless reserved.nil? 13 | raise Bud::CompileError, "symbol :#{name} reserved, cannot be used as collection name" 14 | end 15 | end 16 | 17 | def define_collection(name) 18 | check_collection_name(name) 19 | 20 | self.singleton_class.send(:define_method, name) do |*args, &blk| 21 | if blk.nil? 22 | return @tables[name] 23 | else 24 | return @tables[name].pro(&blk) 25 | end 26 | end 27 | end 28 | 29 | def define_lattice(name) 30 | check_collection_name(name) 31 | 32 | self.singleton_class.send(:define_method, name) do |*args, &blk| 33 | if blk.nil? 34 | return @lattices[name] 35 | else 36 | return @lattices[name].pro(&blk) 37 | end 38 | end 39 | end 40 | 41 | public 42 | def input # :nodoc: all 43 | true 44 | end 45 | 46 | def output # :nodoc: all 47 | false 48 | end 49 | 50 | # declare a transient collection to be an input or output interface 51 | def interface(mode, name, schema=nil) 52 | define_collection(name) 53 | t_provides << [name.to_s, mode] 54 | @tables[name] = (mode ? BudInputInterface : BudOutputInterface).new(name, self, schema) 55 | end 56 | 57 | # declare an in-memory, non-transient collection. default schema [:key] => [:val]. 58 | def table(name, schema=nil) 59 | define_collection(name) 60 | @tables[name] = Bud::BudTable.new(name, self, schema) 61 | end 62 | 63 | # declare a collection-generating expression. default schema [:key] => [:val]. 64 | def coll_expr(name, expr, schema=nil) 65 | define_collection(name) 66 | @tables[name] = Bud::BudCollExpr.new(name, self, expr, schema) 67 | end 68 | 69 | # declare a syncronously-flushed persistent collection. default schema [:key] => [:val]. 70 | def sync(name, storage, schema=nil) 71 | define_collection(name) 72 | case storage 73 | when :dbm 74 | @tables[name] = Bud::BudDbmTable.new(name, self, schema) 75 | @dbm_tables[name] = @tables[name] 76 | else 77 | raise Bud::Error, "unknown synchronous storage engine #{storage.to_s}" 78 | end 79 | end 80 | 81 | def store(name, storage, schema=nil) 82 | define_collection(name) 83 | case storage 84 | when :zookeeper 85 | # treat "schema" as a hash of options 86 | options = schema 87 | raise Bud::Error, "Zookeeper tables require a :path option" if options[:path].nil? 88 | options[:addr] ||= "localhost:2181" 89 | @tables[name] = Bud::BudZkTable.new(name, options[:path], options[:addr], self) 90 | @zk_tables[name] = @tables[name] 91 | else 92 | raise Bud::Error, "unknown async storage engine #{storage.to_s}" 93 | end 94 | end 95 | 96 | # declare a transient collection. default schema [:key] => [:val] 97 | def scratch(name, schema=nil) 98 | define_collection(name) 99 | @tables[name] = Bud::BudScratch.new(name, self, schema) 100 | end 101 | 102 | def readonly(name, schema=nil) 103 | define_collection(name) 104 | @tables[name] = Bud::BudReadOnly.new(name, self, schema) 105 | end 106 | 107 | # declare a scratch in a bloom statement lhs. schema inferred from rhs. 108 | def temp(name) 109 | define_collection(name) 110 | # defer schema definition until merge 111 | @tables[name] = Bud::BudTemp.new(name, self, nil, true) 112 | end 113 | 114 | # declare a transient network collection. default schema [:address, :val] => [] 115 | def channel(name, schema=nil, loopback=false) 116 | define_collection(name) 117 | @tables[name] = Bud::BudChannel.new(name, self, schema, loopback) 118 | @channels[name] = @tables[name] 119 | end 120 | 121 | # declare a transient network collection that delivers facts back to the 122 | # current Bud instance. This is syntax sugar for a channel that always 123 | # delivers to the IP/port of the current Bud instance. Default schema 124 | # [:key] => [:val] 125 | def loopback(name, schema=nil) 126 | schema ||= {[:key] => [:val]} 127 | channel(name, schema, true) 128 | end 129 | 130 | # declare a collection to be read from +filename+. rhs of statements only 131 | def file_reader(name, filename) 132 | define_collection(name) 133 | @tables[name] = Bud::BudFileReader.new(name, filename, self) 134 | end 135 | 136 | # declare a collection to be auto-populated every +period+ seconds. schema [:key] => [:val]. 137 | # rhs of statements only. 138 | def periodic(name, period=1) 139 | define_collection(name) 140 | raise Bud::Error if @periodics.has_key? [name] 141 | @periodics << [name, period] 142 | @tables[name] = Bud::BudPeriodic.new(name, self) 143 | end 144 | 145 | def terminal(name) # :nodoc: all 146 | if defined?(@terminal) && @terminal != name 147 | raise Bud::Error, "can't register IO collection #{name} in addition to #{@terminal}" 148 | else 149 | @terminal = name 150 | end 151 | define_collection(name) 152 | @tables[name] = Bud::BudTerminal.new(name, self) 153 | @channels[name] = @tables[name] 154 | end 155 | 156 | # an alternative approach to declaring interfaces 157 | def interfaces(direction, collections) 158 | mode = case direction 159 | when :input then true 160 | when :output then false 161 | else 162 | raise Bud::CompileError, "unrecognized interface type #{direction}" 163 | end 164 | collections.each do |tab| 165 | t_provides << [tab.to_s, mode] 166 | end 167 | end 168 | 169 | # Define methods to implement the state declarations for every registered kind 170 | # of lattice. 171 | def load_lattice_defs 172 | Bud::Lattice.global_mfuncs.each do |m| 173 | next if RuleRewriter::MONOTONE_WHITELIST.include? m 174 | if Bud::BudCollection.instance_methods.include? m.to_s 175 | puts "monotone method #{m} conflicts with non-monotonic method in BudCollection" 176 | end 177 | end 178 | 179 | Bud::Lattice.global_morphs.each do |m| 180 | next if RuleRewriter::MONOTONE_WHITELIST.include? m 181 | if Bud::BudCollection.instance_methods.include? m.to_s 182 | puts "morphism #{m} conflicts with non-monotonic method in BudCollection" 183 | end 184 | end 185 | 186 | # Sanity-check lattice definitions 187 | # XXX: We should do this only once per lattice 188 | Bud::Lattice.lattice_kinds.each do |wrap_name, klass| 189 | unless klass.method_defined? :merge 190 | raise Bud::CompileError, "lattice #{wrap_name} does not define a merge function" 191 | end 192 | 193 | # If a method is marked as monotone in any lattice, every lattice that 194 | # declares a method of that name must also mark it as monotone. 195 | meth_list = klass.instance_methods(false).to_set 196 | Bud::Lattice.global_mfuncs.each do |m| 197 | next unless meth_list.include? m.to_s 198 | unless klass.mfuncs.include? m 199 | raise Bud::CompileError, "method #{m} in #{wrap_name} must be monotone" 200 | end 201 | end 202 | 203 | # Apply a similar check for morphs 204 | Bud::Lattice.global_morphs.each do |m| 205 | next unless meth_list.include? m.to_s 206 | unless klass.morphs.include? m 207 | raise Bud::CompileError, "method #{m} in #{wrap_name} must be a morph" 208 | end 209 | end 210 | 211 | # Similarly, check for non-monotone lattice methods that are found in the 212 | # builtin list of monotone operators. The "merge" method is implicitly 213 | # monotone (XXX: should it be declared as a morph or monotone function?) 214 | meth_list.each do |m_str| 215 | m = m_str.to_sym 216 | next unless RuleRewriter::MONOTONE_WHITELIST.include? m 217 | # XXX: ugly hack. We want to allow lattice class implementations to 218 | # define their own equality semantics. 219 | next if m == :== 220 | unless klass.mfuncs.include?(m) || klass.morphs.include?(m) || m == :merge 221 | raise Bud::CompileError, "method #{m} in #{wrap_name} must be monotone" 222 | end 223 | end 224 | 225 | # XXX: replace "self" with toplevel? 226 | self.singleton_class.send(:define_method, wrap_name) do |lat_name| 227 | define_lattice(lat_name) 228 | @lattices[lat_name] = Bud::LatticeWrapper.new(lat_name, klass, self) 229 | end 230 | end 231 | end 232 | end 233 | -------------------------------------------------------------------------------- /lib/bud/storage/dbm.rb: -------------------------------------------------------------------------------- 1 | require 'dbm' 2 | 3 | module Bud 4 | # Persistent table implementation based on dbm. 5 | class BudDbmTable < BudPersistentCollection # :nodoc: all 6 | def initialize(name, bud_instance, given_schema) 7 | dbm_dir = bud_instance.options[:dbm_dir] 8 | raise Bud::Error, "dbm support must be enabled via 'dbm_dir'" unless dbm_dir 9 | if bud_instance.port.nil? 10 | raise Bud::Error, "use of dbm storage requires an explicit port to be specified in Bud initialization options" 11 | end 12 | 13 | unless File.exists?(dbm_dir) 14 | Dir.mkdir(dbm_dir) 15 | puts "Created directory: #{dbm_dir}" unless bud_instance.options[:quiet] 16 | end 17 | dirname = "#{dbm_dir}/bud_#{bud_instance.port}" 18 | unless File.exists?(dirname) 19 | Dir.mkdir(dirname) 20 | puts "Created directory: #{dirname}" unless bud_instance.options[:quiet] 21 | end 22 | 23 | super(name, bud_instance, given_schema) 24 | @to_delete = [] 25 | @invalidated = true 26 | 27 | db_fname = "#{dirname}/#{name}.dbm" 28 | flags = DBM::WRCREAT 29 | if bud_instance.options[:dbm_truncate] == true 30 | flags |= DBM::NEWDB 31 | end 32 | @dbm = DBM.open(db_fname, 0666, flags) 33 | if @dbm.nil? 34 | raise Bud::Error, "failed to open dbm database '#{db_fname}': #{@dbm.errmsg}" 35 | end 36 | end 37 | 38 | def init_storage 39 | # XXX: we can't easily use the @storage infrastructure provided by 40 | # BudCollection; issue #33 41 | @storage = nil 42 | end 43 | 44 | def [](key) 45 | check_enumerable(key) 46 | key_s = MessagePack.pack(key) 47 | val_s = @dbm[key_s] 48 | if val_s 49 | return make_tuple(key, MessagePack.unpack(val_s)) 50 | else 51 | return @delta[key] 52 | end 53 | end 54 | 55 | def length 56 | @dbm.length + @delta.length 57 | end 58 | 59 | def has_key?(k) 60 | check_enumerable(k) 61 | key_s = MessagePack.pack(k) 62 | return true if @dbm.has_key? key_s 63 | return @delta.has_key? k 64 | end 65 | 66 | def include?(tuple) 67 | key = get_key_vals(tuple) 68 | value = self[key] 69 | return (value == tuple) 70 | end 71 | 72 | def make_tuple(k_ary, v_ary) 73 | t = @struct.new 74 | @key_colnums.each_with_index do |k,i| 75 | t[k] = k_ary[i] 76 | end 77 | val_cols.each_with_index do |c,i| 78 | t[cols.index(c)] = v_ary[i] 79 | end 80 | t 81 | end 82 | 83 | def each(&block) 84 | each_from([@delta], &block) 85 | each_storage(&block) 86 | end 87 | 88 | def each_raw(&block) 89 | each_storage(&block) 90 | end 91 | 92 | def each_from(bufs, &block) 93 | bufs.each do |b| 94 | if b == @storage then 95 | each_storage(&block) 96 | else 97 | b.each_value do |v| 98 | tick_metrics if bud_instance.options[:metrics] 99 | yield v 100 | end 101 | end 102 | end 103 | end 104 | 105 | def each_storage(&block) 106 | @dbm.each do |k,v| 107 | k_ary = MessagePack.unpack(k) 108 | v_ary = MessagePack.unpack(v) 109 | tick_metrics if bud_instance.options[:metrics] 110 | yield make_tuple(k_ary, v_ary) 111 | end 112 | end 113 | 114 | def flush 115 | end 116 | 117 | def close 118 | @dbm.close unless @dbm.nil? 119 | @dbm = nil 120 | end 121 | 122 | def merge_to_db(buf) 123 | buf.each do |key,tuple| 124 | merge_tuple_to_db(key, tuple) 125 | end 126 | end 127 | 128 | def merge_tuple_to_db(key, tuple) 129 | key_s = MessagePack.pack(key) 130 | if @dbm.has_key?(key_s) 131 | old_tuple = self[key] 132 | raise_pk_error(tuple, old_tuple) if tuple != old_tuple 133 | else 134 | val = val_cols.map{|c| tuple[cols.index(c)]} 135 | @dbm[key_s] = MessagePack.pack(val) 136 | end 137 | end 138 | 139 | # move deltas to on-disk storage, and new_deltas to deltas 140 | def tick_deltas 141 | unless @delta.empty? 142 | merge_to_db(@delta) 143 | @tick_delta.concat(@delta.values) if accumulate_tick_deltas 144 | @delta.clear 145 | end 146 | unless @new_delta.empty? 147 | # We allow @new_delta to contain duplicates but eliminate them here. We 148 | # can't just allow duplicate delta tuples because that might cause 149 | # spurious infinite delta processing loops. 150 | @new_delta.reject! {|key, val| self[key] == val} 151 | 152 | @delta = @new_delta 153 | @new_delta = {} 154 | end 155 | return !(@delta.empty?) 156 | end 157 | 158 | public 159 | def flush_deltas 160 | unless @delta.empty? 161 | merge_to_db(@delta) 162 | @tick_delta.concat(@delta.values) if accumulate_tick_deltas 163 | @delta.clear 164 | end 165 | merge_to_db(@new_delta) 166 | @new_delta = {} 167 | end 168 | 169 | # This is verbatim from BudTable. Need to DRY up. Should we be a subclass 170 | # of BudTable? 171 | public 172 | def pending_delete(o) 173 | if o.class <= Bud::PushElement 174 | o.wire_to(self, :delete) 175 | elsif o.class <= Bud::BudCollection 176 | o.pro.wire_to(self, :delete) 177 | else 178 | @to_delete.concat(o.map{|t| prep_tuple(t) unless t.nil?}) 179 | end 180 | end 181 | superator "<-" do |o| 182 | pending_delete(o) 183 | end 184 | 185 | def insert(tuple) 186 | key = get_key_vals(tuple) 187 | merge_tuple_to_db(key, tuple) 188 | end 189 | 190 | alias << insert 191 | 192 | # Remove to_delete and then move pending => delta. 193 | def tick 194 | deleted = nil 195 | @to_delete.each do |tuple| 196 | k = get_key_vals(tuple) 197 | k_str = MessagePack.pack(k) 198 | cols_str = @dbm[k_str] 199 | unless cols_str.nil? 200 | db_cols = MessagePack.unpack(cols_str) 201 | delete_cols = val_cols.map{|c| tuple[cols.index(c)]} 202 | if db_cols == delete_cols 203 | deleted ||= @dbm.delete k_str 204 | end 205 | end 206 | end 207 | @to_delete = [] 208 | 209 | @invalidated = !deleted.nil? 210 | unless @pending.empty? 211 | @delta = @pending 212 | @pending = {} 213 | end 214 | flush 215 | end 216 | 217 | def invalidate_cache 218 | end 219 | 220 | # XXX: shouldn't this check @delta as well? 221 | public 222 | def empty? 223 | @dbm.empty? 224 | end 225 | end 226 | end 227 | -------------------------------------------------------------------------------- /lib/bud/storage/zookeeper.rb: -------------------------------------------------------------------------------- 1 | begin 2 | require 'zookeeper' 3 | Bud::HAVE_ZOOKEEPER = true 4 | rescue LoadError 5 | end 6 | 7 | module Bud 8 | # Persistent table implementation based on Zookeeper. 9 | class BudZkTable < BudPersistentCollection # :nodoc: all 10 | def initialize(name, zk_path, zk_addr, bud_instance) 11 | unless defined? Bud::HAVE_ZOOKEEPER 12 | raise Bud::Error, "zookeeper gem is not installed: zookeeper-backed stores cannot be used" 13 | end 14 | 15 | super(name, bud_instance, [:key] => [:val, :opts]) 16 | 17 | @zk = Zookeeper.new(zk_addr) 18 | zk_path = zk_path.chomp("/") unless zk_path == "/" 19 | @zk_path = zk_path 20 | @base_path = @zk_path 21 | @base_path += "/" unless @zk_path.end_with? "/" 22 | @store_mutex = Mutex.new 23 | @zk_mutex = Mutex.new 24 | @next_storage = {} 25 | @saw_delta = false 26 | @child_watch_id = nil 27 | end 28 | 29 | def invalidate_at_tick 30 | true 31 | end 32 | 33 | def invalidate_cache 34 | end 35 | 36 | # Since the watcher callbacks might invoke EventMachine, we wait until after 37 | # EM startup to start watching for Zk events. 38 | def start_watchers 39 | # Watcher callbacks are invoked in a separate Ruby thread. Note that there 40 | # is a possible deadlock between invoking watcher callbacks and calling 41 | # close(): if we get a watcher event and a close at around the same time, 42 | # the close might fire first. Closing the Zk handle will block on 43 | # dispatching outstanding watchers, but it does so holding the @zk_mutex, 44 | # causing a deadlock. Hence, we just have the watcher callback spin on the 45 | # @zk_mutex, aborting if the handle is ever closed. 46 | @child_watcher = Zookeeper::Callbacks::WatcherCallback.new do 47 | while true 48 | break if @zk.closed? 49 | if @zk_mutex.try_lock 50 | get_and_watch unless @zk.closed? 51 | @zk_mutex.unlock 52 | break 53 | end 54 | end 55 | end 56 | 57 | @stat_watcher = Zookeeper::Callbacks::WatcherCallback.new do 58 | while true 59 | break if @zk.closed? 60 | if @zk_mutex.try_lock 61 | stat_and_watch unless @zk.closed? 62 | @zk_mutex.unlock 63 | break 64 | end 65 | end 66 | end 67 | 68 | stat_and_watch 69 | end 70 | 71 | def stat_and_watch 72 | r = @zk.stat(:path => @zk_path, :watcher => @stat_watcher) 73 | 74 | unless r[:stat].exists 75 | # The given @zk_path doesn't exist, so try to create it. Unclear 76 | # whether this is always the best behavior. 77 | r = @zk.create(:path => @zk_path) 78 | if r[:rc] != Zookeeper::ZOK and r[:rc] != Zookeeper::ZNODEEXISTS 79 | raise 80 | end 81 | end 82 | 83 | # Make sure we're watching for children 84 | get_and_watch unless @child_watch_id 85 | end 86 | 87 | def get_and_watch 88 | r = @zk.get_children(:path => @zk_path, :watcher => @child_watcher) 89 | return unless r[:stat].exists 90 | @child_watch_id = r[:req_id] 91 | 92 | # XXX: can we easily get snapshot isolation? 93 | new_children = {} 94 | r[:children].each do |c| 95 | child_path = @base_path + c 96 | 97 | get_r = @zk.get(:path => child_path) 98 | unless get_r[:stat].exists 99 | puts "ZK: failed to fetch child: #{child_path}" 100 | return 101 | end 102 | 103 | data = get_r[:data] 104 | # XXX: For now, conflate empty string values with nil values 105 | data ||= "" 106 | new_children[c] = [c, data] 107 | end 108 | 109 | # We successfully fetched all the children of @zk_path; arrange to install 110 | # the new data into @storage at the next Bud tick 111 | need_tick = false 112 | @store_mutex.synchronize { 113 | @next_storage = new_children 114 | if @storage != @next_storage 115 | need_tick = true 116 | @saw_delta = true 117 | end 118 | } 119 | 120 | # If we have new data, force a new Bud tick in the near future 121 | if need_tick and @bud_instance.running_async 122 | EventMachine::schedule { 123 | @bud_instance.tick_internal 124 | } 125 | end 126 | end 127 | 128 | def tick 129 | @store_mutex.synchronize { 130 | return unless @saw_delta 131 | @storage = @next_storage 132 | @next_storage = {} 133 | @saw_delta = false 134 | } 135 | end 136 | 137 | def flush 138 | each_from([@pending]) do |t| 139 | path = @base_path + t.key 140 | data = t.val 141 | ephemeral = false 142 | sequence = false 143 | 144 | opts = t.opts 145 | unless opts.nil? 146 | if opts[:ephemeral] == true 147 | ephemeral = true 148 | end 149 | if opts[:sequence] == true 150 | sequence = true 151 | end 152 | end 153 | 154 | r = @zk.create(:path => path, :data => data, 155 | :ephemeral => ephemeral, :sequence => sequence) 156 | if r[:rc] == Zookeeper::ZNODEEXISTS 157 | puts "Ignoring duplicate insert: #{t.inspect}" 158 | elsif r[:rc] != Zookeeper::ZOK 159 | puts "Failed create of #{path}: #{r.inspect}" 160 | end 161 | end 162 | @pending.clear 163 | end 164 | 165 | def close 166 | # See notes in start_watchers. 167 | @zk_mutex.synchronize { @zk.close } 168 | end 169 | 170 | superator "<~" do |o| 171 | pending_merge(o) 172 | end 173 | 174 | superator "<+" do |o| 175 | raise Bud::Error, "illegal use of <+ with zookeeper store '#{@tabname}' on left" 176 | end 177 | 178 | def <=(o) 179 | raise Bud::Error, "illegal use of <= with zookeeper store '#{@tabname}' on left" 180 | end 181 | 182 | def <<(o) 183 | raise Bud::Error, "illegal use of << with zookeeper store '#{@tabname}' on left" 184 | end 185 | end 186 | end 187 | -------------------------------------------------------------------------------- /lib/bud/version.rb: -------------------------------------------------------------------------------- 1 | module Bud 2 | VERSION = "0.9.9" 3 | end 4 | -------------------------------------------------------------------------------- /lib/bud/viz.rb: -------------------------------------------------------------------------------- 1 | require 'bud/state' 2 | 3 | class VizOnline #:nodoc: all 4 | attr_reader :logtab 5 | 6 | META_TABLES = %w[t_cycle t_depends t_provides t_rule_stratum t_rules t_stratum 7 | t_table_info t_table_schema t_underspecified].to_set 8 | 9 | def initialize(bud_instance) 10 | @bud_instance = bud_instance 11 | @bud_instance.options[:dbm_dir] = "DBM_#{@bud_instance.class}_#{bud_instance.options[:tag]}_#{bud_instance.object_id}_#{bud_instance.port}" 12 | @table_info = bud_instance.tables[:t_table_info] 13 | @table_schema = bud_instance.tables[:t_table_schema] 14 | @logtab = new_tab(:the_big_log, [:table, :time, :contents], bud_instance) 15 | tmp_set = [] 16 | @bud_instance.tables.each do |name, tbl| 17 | next if name == :the_big_log || name == :localtick 18 | # Temp collections don't have a schema until a fact has been inserted into 19 | # them; for now, we just include an empty schema for them in the viz 20 | if tbl.schema.nil? 21 | schema = [:a, :b, :c, :d] 22 | else 23 | schema = tbl.schema.clone 24 | end 25 | tmp_set << [name, schema, tbl.class.to_s] 26 | end 27 | 28 | tmp_set.each do |t| 29 | snd_alias = t[0].to_s + "_snd" 30 | @table_schema << [t[0], :c_bud_time, 0] 31 | t[1].each_with_index do |s, i| 32 | @table_schema << [t[0], s, i+1] 33 | if t[2] == "Bud::BudChannel" 34 | @table_schema << [snd_alias, s, i+1] 35 | end 36 | end 37 | if t[2] == "Bud::BudChannel" 38 | lts = "#{snd_alias}_vizlog".to_sym 39 | @table_info << [snd_alias, t[2]] 40 | end 41 | @table_info << [t[0], t[2]] 42 | end 43 | end 44 | 45 | def new_tab(name, schema, instance) 46 | ret = Bud::BudDbmTable.new(name, instance, schema) 47 | instance.tables[name] = ret 48 | return ret 49 | end 50 | 51 | def add_rows(collection, tab) 52 | collection.each do |row| 53 | if collection.class == Hash 54 | row = row[1] 55 | elsif collection.class == Bud::BudPeriodic 56 | row = row[0] 57 | end 58 | 59 | # t_depends, t_rule_stratum, and t_rules have Bud object as their first 60 | # field. Replace with a string, since Bud instances cannot be serialized. 61 | if row[0].class <= Bud 62 | row = row.to_a 63 | row = [row[0].class.to_s] + row[1..-1] 64 | end 65 | newrow = [tab, @bud_instance.budtime, row] 66 | begin 67 | @logtab << newrow 68 | rescue 69 | raise "ERROR! #{@logtab} << #{newrow.inspect} (etxt #{$!})" 70 | end 71 | end 72 | end 73 | 74 | def do_cards 75 | @bud_instance.tables.each do |t| 76 | tab = t[0] 77 | next if tab == :the_big_log 78 | next if @bud_instance.budtime > 0 and META_TABLES.include? tab.to_s 79 | add_rows(t[1], tab) 80 | if t[1].class == Bud::BudChannel 81 | add_rows(t[1].pending, "#{tab}_snd") 82 | end 83 | @logtab.tick 84 | end 85 | end 86 | end 87 | -------------------------------------------------------------------------------- /test/perf/join_bench.rb: -------------------------------------------------------------------------------- 1 | require "rubygems" 2 | require "bud" 3 | require "benchmark" 4 | 5 | JOIN_INPUT_SIZE = 20000 6 | NUM_RUNS = 10 7 | 8 | class HashJoinBench 9 | include Bud 10 | 11 | state do 12 | table :t1 13 | table :t2 14 | table :t3 15 | end 16 | 17 | bootstrap do 18 | # Only a single pair of tuples satisfy the join condition 19 | JOIN_INPUT_SIZE.times do |i| 20 | t1 << [i, i + 50000] 21 | t2 << [i + JOIN_INPUT_SIZE - 1, i] 22 | end 23 | end 24 | 25 | bloom do 26 | t3 <= (t1 * t2).pairs(:key => :key) 27 | stdio <~ t3 {|t1, t2| ["Join result: #{[t1,t2].inspect}"]} 28 | end 29 | end 30 | 31 | b = HashJoinBench.new 32 | b.tick 33 | t = Benchmark.measure do 34 | NUM_RUNS.times do 35 | b.t1 <- [[0, 50000]] 36 | b.t1 <+ [[0, 50000]] 37 | b.tick 38 | end 39 | end 40 | puts "Time taken for #{NUM_RUNS} joins: #{t}" 41 | -------------------------------------------------------------------------------- /test/perf/ring.rb: -------------------------------------------------------------------------------- 1 | require "rubygems" 2 | require "bud" 3 | 4 | RING_SIZE = 20 5 | NUM_CIRCUITS = 800 6 | NUM_MESSAGES = RING_SIZE * NUM_CIRCUITS 7 | 8 | class RingMember 9 | include Bud 10 | 11 | state do 12 | channel :pipe, [:@addr, :cnt, :val] 13 | scratch :kickoff, [:cnt, :val] 14 | table :next_guy, [:addr] 15 | table :last_cnt, [:cnt] 16 | scratch :done, [:cnt] 17 | end 18 | 19 | bloom :ring_msg do 20 | pipe <~ kickoff {|k| [ip_port, k.cnt, k.val]} 21 | pipe <~ (pipe * next_guy).pairs {|p,n| [n.addr, p.cnt + 1, p.val] if p.cnt < NUM_MESSAGES} 22 | done <= pipe {|p| [p.cnt] if p.cnt == NUM_MESSAGES} 23 | end 24 | 25 | # We do some minor computation as well as just routing the message onward 26 | bloom :update_log do 27 | last_cnt <+ pipe {|p| [p.cnt]} 28 | last_cnt <- (pipe * last_cnt).pairs {|p, lc| [lc.cnt]} 29 | end 30 | end 31 | 32 | def test_basic_ring 33 | ring = [] 34 | RING_SIZE.times do |i| 35 | ring[i] = RingMember.new 36 | ring[i].run_bg 37 | end 38 | q = Queue.new 39 | ring.last.register_callback(:done) do 40 | q.push(true) 41 | end 42 | 43 | ring.each_with_index do |r, i| 44 | next_idx = i + 1 45 | next_idx = 0 if next_idx == RING_SIZE 46 | next_addr = ring[next_idx].ip_port 47 | 48 | r.sync_do { 49 | r.next_guy <+ [[next_addr]] 50 | } 51 | end 52 | 53 | first = ring.first 54 | first.async_do { 55 | first.kickoff <+ [[1, "xyz"]] 56 | } 57 | 58 | # Wait for the "done" callback from the last member of the ring. 59 | q.pop 60 | 61 | ring.each_with_index do |r, i| 62 | # XXX: we need to do a final tick here to ensure that each Bud instance 63 | # applies pending <+ and <- derivations. See issue #50. 64 | r.sync_do 65 | r.stop 66 | expected = (NUM_MESSAGES - RING_SIZE) + i + 1 67 | raise unless r.last_cnt.first == [expected] 68 | end 69 | end 70 | 71 | test_basic_ring 72 | -------------------------------------------------------------------------------- /test/perf/scratch_derive.rb: -------------------------------------------------------------------------------- 1 | require "rubygems" 2 | require "bud" 3 | 4 | BENCH_LIMIT = 30000 5 | 6 | class ScratchBench 7 | include Bud 8 | 9 | state do 10 | scratch :t1, [:key] 11 | scratch :done 12 | end 13 | 14 | bloom do 15 | t1 <= t1 {|t| [t.key + 1] if t.key < BENCH_LIMIT} 16 | done <= t1 {|t| t if t.key >= BENCH_LIMIT} 17 | end 18 | end 19 | 20 | b = ScratchBench.new 21 | b.run_bg 22 | b.sync_do { 23 | b.t1 <+ [[0]] 24 | } 25 | b.stop 26 | -------------------------------------------------------------------------------- /test/tc_attr_rewrite.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | 3 | class TestCols < Minitest::Test 4 | class SimpleCols 5 | include Bud 6 | state do 7 | table :t1 8 | table :t2 9 | end 10 | 11 | bootstrap do 12 | t1 << [1,2] 13 | end 14 | 15 | bloom do 16 | t2 <= t1 {|t| [t.key, t.val]} 17 | end 18 | end 19 | 20 | def test_simple_cols 21 | program = SimpleCols.new 22 | program.tick 23 | assert_equal([[1,2]], program.t2.to_a) 24 | end 25 | 26 | class NestedCols 27 | include Bud 28 | state do 29 | table :t1 30 | table :t2 31 | table :t3 32 | end 33 | bootstrap do 34 | t1 << [1,2] 35 | t3 << [1,3] 36 | end 37 | 38 | bloom do 39 | t2 <= t1 {|t| [t.key, t.val] if t3.each{|x| t.key == x.key}} 40 | end 41 | end 42 | 43 | def test_nested_cols 44 | program = NestedCols.new 45 | program.tick 46 | assert_equal([[1,2]], program.t2.to_a) 47 | end 48 | 49 | class BadNestedCols 50 | include Bud 51 | state do 52 | table :t1 53 | table :t2 54 | table :t3, [:val, :key] 55 | end 56 | bootstrap do 57 | t1 << [1,2] 58 | t3 << [1,3] 59 | end 60 | 61 | bloom do 62 | t2 <= t1 {|t| [t.key, t.val] if t3.each{|t| t.key == t.val}} 63 | end 64 | end 65 | 66 | def test_bad_nested_cols 67 | assert_raises(Bud::CompileError) {BadNestedCols.new} 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /test/tc_callback.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | require 'timeout' 3 | 4 | class SimpleCb 5 | include Bud 6 | 7 | state do 8 | scratch :t1 9 | scratch :c1 10 | end 11 | 12 | bloom do 13 | c1 <= t1 14 | end 15 | end 16 | 17 | class CallbackAtNext 18 | include Bud 19 | 20 | state do 21 | scratch :t1 22 | scratch :c1 23 | end 24 | 25 | bloom do 26 | c1 <+ t1 27 | end 28 | end 29 | 30 | class TickingCallback 31 | include Bud 32 | 33 | state do 34 | periodic :tic, 0.1 35 | scratch :dummy 36 | end 37 | 38 | bloom do 39 | dummy <= tic 40 | end 41 | end 42 | 43 | class CallbackWithChannel 44 | include Bud 45 | 46 | state do 47 | channel :cin 48 | scratch :iout 49 | end 50 | 51 | bloom do 52 | iout <= cin 53 | end 54 | end 55 | 56 | class CallbackTest < Minitest::Test 57 | class Counter 58 | attr_reader :cnt 59 | 60 | def initialize 61 | @cnt = 0 62 | end 63 | 64 | def bump 65 | @cnt += 1 66 | end 67 | end 68 | 69 | def test_simple_cb 70 | c = SimpleCb.new 71 | call_tick = Counter.new 72 | tuple_tick = Counter.new 73 | c.register_callback(:c1) do |t| 74 | call_tick.bump 75 | t.length.times do 76 | tuple_tick.bump 77 | end 78 | end 79 | 80 | c.run_bg 81 | c.sync_do 82 | assert_equal(0, call_tick.cnt) 83 | assert_equal(0, tuple_tick.cnt) 84 | c.sync_do { 85 | c.t1 <+ [[5, 10]] 86 | } 87 | assert_equal(1, call_tick.cnt) 88 | assert_equal(1, tuple_tick.cnt) 89 | c.sync_do { 90 | c.t1 <+ [[10, 15], [20, 25]] 91 | } 92 | assert_equal(2, call_tick.cnt) 93 | assert_equal(3, tuple_tick.cnt) 94 | c.stop 95 | end 96 | 97 | def test_cb_at_next 98 | c = CallbackAtNext.new 99 | c.run_bg 100 | tick = Counter.new 101 | c.register_callback(:c1) do |t| 102 | tick.bump 103 | end 104 | 105 | c.sync_do { 106 | c.t1 <+ [[20, 30]] 107 | } 108 | assert_equal(0, tick.cnt) 109 | c.sync_do 110 | assert_equal(1, tick.cnt) 111 | 112 | c.stop 113 | end 114 | 115 | def test_missing_cb_error 116 | c = SimpleCb.new 117 | assert_raises(Bud::Error) do 118 | c.register_callback(:crazy) do 119 | raise RuntimeError 120 | end 121 | end 122 | end 123 | 124 | def test_blocking_on_callback 125 | c = SimpleCb.new 126 | c.run_bg 127 | tuples = [[1, 2]] 128 | c.sync_callback(:t1, tuples, :c1) do |cb| 129 | assert_equal(1, cb.length) 130 | end 131 | c.stop 132 | end 133 | 134 | def test_delta 135 | c = TickingCallback.new 136 | c.run_bg 137 | Timeout::timeout(5) {c.delta(:tic)} 138 | c.stop 139 | end 140 | 141 | def add_cb(b) 142 | tick = Counter.new 143 | id = b.register_callback(:c1) do 144 | tick.bump 145 | end 146 | return [tick, id] 147 | end 148 | 149 | def test_unregister_cb 150 | c = SimpleCb.new 151 | tick1, id1 = add_cb(c) 152 | tick2, id2 = add_cb(c) 153 | 154 | c.run_bg 155 | c.sync_do { 156 | c.t1 <+ [[100, 200]] 157 | } 158 | assert_equal(1, tick1.cnt) 159 | assert_equal(1, tick2.cnt) 160 | c.unregister_callback(id1) 161 | c.sync_do { 162 | c.t1 <+ [[200, 400]] 163 | } 164 | assert_equal(1, tick1.cnt) 165 | assert_equal(2, tick2.cnt) 166 | c.stop 167 | end 168 | 169 | def test_callback_with_channel 170 | c = CallbackWithChannel.new 171 | c.run_bg 172 | c.sync_callback(:cin, [[c.ip_port, "foo"]], :iout) 173 | assert(true) 174 | end 175 | 176 | def test_shutdown_cb 177 | cnt = 0 178 | c = TickingCallback.new 179 | c.on_shutdown do 180 | cnt += 1 181 | end 182 | c.run_bg 183 | assert_equal(0, cnt) 184 | c.stop 185 | assert_equal(1, cnt) 186 | end 187 | end 188 | -------------------------------------------------------------------------------- /test/tc_dbm.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | require 'fileutils' 3 | 4 | class DbmTest 5 | include Bud 6 | 7 | state do 8 | sync :t1, :dbm, [:k1, :k2] => [:v1, :v2] 9 | table :in_buf, [:k1, :k2, :v1, :v2] 10 | table :del_buf, [:k1, :k2, :v1, :v2] 11 | table :pending_buf, [:k1, :k2] => [:v1, :v2] 12 | table :pending_buf2, [:k1, :k2] => [:v1, :v2] 13 | 14 | scratch :t2, [:k] => [:v] 15 | scratch :t3, [:k] => [:v] 16 | scratch :t4, [:k] => [:v] 17 | sync :chain_start, :dbm, [:k] => [:v] 18 | sync :chain_del, :dbm, [:k] => [:v] 19 | 20 | sync :join_t1, :dbm, [:k] => [:v1, :v2] 21 | sync :join_t2, :dbm, [:k] => [:v1, :v2] 22 | scratch :cart_prod, [:k, :v1] 23 | scratch :join_res, [:k, :v1] 24 | end 25 | 26 | bloom do 27 | t1 <= in_buf 28 | t1 <- del_buf 29 | t1 <+ pending_buf 30 | t1 <+ pending_buf2 31 | end 32 | 33 | bloom :do_chain do 34 | t2 <= chain_start.map{|c| [c.k, c.v + 1]} 35 | t3 <= t2.map{|c| [c.k, c.v + 1]} 36 | t4 <= t3.map{|c| [c.k, c.v + 1]} 37 | chain_start <- chain_del 38 | end 39 | 40 | bloom :do_join do 41 | join_res <= (join_t1 * join_t2).pairs(:k => :k) 42 | cart_prod <= (join_t1 * join_t2) 43 | end 44 | end 45 | 46 | DBM_BUD_DIR = "#{Dir.pwd}/bud_tmp" 47 | 48 | def setup_bud 49 | rm_bud_dir 50 | end 51 | 52 | def cleanup_bud(b) 53 | b.stop unless b.nil? 54 | rm_bud_dir 55 | end 56 | 57 | def rm_bud_dir 58 | return unless File.directory? DBM_BUD_DIR 59 | FileUtils.rm_r(DBM_BUD_DIR) 60 | end 61 | 62 | class TestDbm < Minitest::Test 63 | def setup 64 | setup_bud 65 | @t = make_bud(true) 66 | end 67 | 68 | def teardown 69 | cleanup_bud(@t) 70 | @t = nil 71 | end 72 | 73 | def make_bud(truncate) 74 | DbmTest.new(:dbm_dir => DBM_BUD_DIR, :dbm_truncate => truncate, :quiet => true, :port => 65432) 75 | end 76 | 77 | def test_basic_ins 78 | assert_equal(0, @t.t1.length) 79 | @t.in_buf <+ [['1', '2', '3', '4'], 80 | ['1', '3', '3', '4']] 81 | @t.tick 82 | assert_equal(2, @t.t1.length) 83 | assert(@t.t1.include? ['1', '2', '3', '4']) 84 | assert(@t.t1.has_key? ['1', '2']) 85 | assert_equal(false, @t.t1.include?(['1', '2', '3', '5'])) 86 | end 87 | 88 | def test_key_conflict_delta 89 | @t.in_buf <+ [['1', '2', '3', '4'], 90 | ['1', '2', '3', '5']] 91 | assert_raises(Bud::KeyConstraintError) {@t.tick} 92 | end 93 | 94 | def test_key_conflict 95 | @t.in_buf <+ [['1', '2', '3', '4']] 96 | @t.tick 97 | @t.in_buf <+ [['1', '2', '3', '5']] 98 | assert_raises(Bud::KeyConstraintError) {@t.tick} 99 | end 100 | 101 | def test_key_merge 102 | @t.in_buf <+ [['1', '2', '3', '4'], 103 | ['1', '2', '3', '4'], 104 | ['1', '2', '3', '4'], 105 | ['1', '2', '3', '4'], 106 | ['5', '10', '3', '4'], 107 | ['6', '10', '3', '4'], 108 | ['6', '10', '3', '4']] 109 | 110 | @t.t1 <+ [['1', '2', '3', '4'], 111 | ['1', '2', '3', '4']] 112 | 113 | @t.tick 114 | assert_equal(3, @t.t1.length) 115 | end 116 | 117 | def test_truncate 118 | @t.in_buf <+ [['1', '2', '3', '4'], 119 | ['1', '3', '3', '4']] 120 | @t.tick 121 | assert_equal(2, @t.t1.length) 122 | 123 | @t.stop 124 | @t = make_bud(true) 125 | 126 | assert_equal(0, @t.t1.length) 127 | @t.in_buf <+ [['1', '2', '3', '4'], 128 | ['1', '3', '3', '4']] 129 | @t.tick 130 | assert_equal(2, @t.t1.length) 131 | end 132 | 133 | def test_persist 134 | @t.in_buf <+ [[1, 2, 3, 4], 135 | [5, 10, 3, 4]] 136 | @t.tick 137 | assert_equal(2, @t.t1.length) 138 | 139 | 10.times do |i| 140 | @t.stop 141 | @t = make_bud(false) 142 | @t.in_buf <+ [[6, 10 + i, 3, 4]] 143 | @t.tick 144 | assert_equal(3 + i, @t.t1.length) 145 | end 146 | end 147 | 148 | def test_pending_ins 149 | @t.pending_buf <+ [['1', '2', '3', '4']] 150 | @t.tick 151 | assert_equal(0, @t.t1.length) 152 | @t.tick 153 | assert_equal(1, @t.t1.length) 154 | end 155 | 156 | def test_pending_key_conflict 157 | @t.pending_buf <+ [['1', '2', '3', '4']] 158 | @t.pending_buf2 <+ [['1', '2', '3', '5']] 159 | assert_raises(Bud::KeyConstraintError) {@t.tick} 160 | end 161 | 162 | def test_basic_del 163 | @t.t1 <+ [['1', '2', '3', '4'], 164 | ['1', '3', '3', '4'], 165 | ['2', '4', '3', '4']] 166 | @t.tick 167 | assert_equal(3, @t.t1.length) 168 | 169 | @t.del_buf <+ [['2', '4', '3', '4']] # should delete 170 | @t.tick 171 | assert_equal(3, @t.t1.length) 172 | @t.tick 173 | assert_equal(2, @t.t1.length) 174 | 175 | @t.del_buf <+ [['1', '3', '3', '5']] # shouldn't delete 176 | @t.tick 177 | assert_equal(2, @t.t1.length) 178 | @t.tick 179 | assert_equal(2, @t.t1.length) 180 | 181 | @t.del_buf <+ [['1', '3', '3', '4']] # should delete 182 | @t.tick 183 | assert_equal(2, @t.t1.length) 184 | @t.tick 185 | assert_equal(1, @t.t1.length) 186 | end 187 | 188 | def test_chain 189 | @t.chain_start <+ [[5, 10], 190 | [10, 15]] 191 | @t.tick 192 | assert_equal(2, @t.t2.length) 193 | assert_equal(2, @t.t3.length) 194 | assert_equal(2, @t.t4.length) 195 | assert_equal([10,18], @t.t4[[10]]) 196 | 197 | @t.chain_del <+ [[5,10]] 198 | @t.tick 199 | assert_equal(2, @t.chain_start.length) 200 | assert_equal(2, @t.t2.length) 201 | assert_equal(2, @t.t3.length) 202 | assert_equal(2, @t.t4.length) 203 | @t.tick 204 | assert_equal(1, @t.chain_start.length) 205 | assert_equal(1, @t.t2.length) 206 | assert_equal(1, @t.t3.length) 207 | assert_equal(1, @t.t4.length) 208 | end 209 | 210 | def test_cartesian_product 211 | @t.join_t1 <+ [[12, 50, 100],[15, 50, 120]] 212 | @t.join_t2 <+ [[12, 70, 150], [6, 20, 30]] 213 | 214 | @t.tick 215 | assert_equal(4, @t.cart_prod.length) 216 | 217 | @t.join_t2 <+ [[6, 20, 30], #dup 218 | [18, 70, 150]] 219 | 220 | @t.tick 221 | assert_equal(6, @t.cart_prod.length) 222 | end 223 | 224 | def test_join 225 | @t.join_t1 <+ [[12, 50, 100], 226 | [15, 50, 120]] 227 | @t.join_t2 <+ [[12, 70, 150], 228 | [6, 20, 30]] 229 | @t.tick 230 | 231 | assert_equal(1, @t.join_res.length) 232 | end 233 | end 234 | 235 | class DbmNest 236 | include Bud 237 | 238 | state { 239 | scratch :in_buf, [:k1, :k2] => [:v1] 240 | table :t1, [:k1] => [:v1] 241 | sync :t2, :dbm, [:k1, :k2] => [:v1, :v2] 242 | } 243 | 244 | bootstrap do 245 | t1 << [5, 10] 246 | end 247 | 248 | bloom do 249 | t2 <= (in_buf * t1).pairs {|b, t| [b.k1, b.k2, b.v1, t]} 250 | end 251 | end 252 | 253 | class TestNestedDbm < Minitest::Test 254 | def setup 255 | setup_bud 256 | @t = make_bud 257 | end 258 | 259 | def teardown 260 | cleanup_bud(@t) 261 | @t = nil 262 | end 263 | 264 | def make_bud 265 | DbmNest.new(:dbm_dir => DBM_BUD_DIR, :dbm_truncate => true, :quiet => true, :port => 65432) 266 | end 267 | 268 | def test_basic_nest 269 | @t.run_bg 270 | 271 | @t.sync_do { 272 | @t.in_buf <+ [[10, 20, 30]] 273 | } 274 | @t.sync_do { 275 | # We can store nested tuples inside DBM tables, but we lose the ability to 276 | # access named columns after deserialization. 277 | assert_equal([10, 20, 30, [5, 10]], @t.t2.first) 278 | } 279 | 280 | @t.stop 281 | end 282 | end 283 | 284 | class DbmBootstrap 285 | include Bud 286 | 287 | state do 288 | sync :t1, :dbm 289 | end 290 | 291 | bootstrap do 292 | t1 << [5, 10] 293 | t1 << [10,15] 294 | end 295 | end 296 | 297 | class TestDbmBootstrap < Minitest::Test 298 | def setup 299 | setup_bud 300 | @t = make_bud 301 | end 302 | 303 | def teardown 304 | cleanup_bud(@t) 305 | @t = nil 306 | end 307 | 308 | def make_bud 309 | DbmBootstrap.new(:dbm_dir => DBM_BUD_DIR, :dbm_truncate => false, :quiet => true, :port => 65432) 310 | end 311 | 312 | def test_basic 313 | def check_t 314 | @t.run_bg 315 | @t.sync_do { 316 | assert_equal([[5, 10], [10, 15]], @t.t1.to_a.sort) 317 | } 318 | @t.stop 319 | end 320 | 321 | check_t 322 | @t = make_bud 323 | check_t 324 | end 325 | end 326 | 327 | class DbmCycleDelta 328 | include Bud 329 | 330 | state do 331 | sync :t1, :dbm 332 | end 333 | 334 | bloom do 335 | t1 <= t1 336 | end 337 | end 338 | 339 | class TestDbmDelta < Minitest::Test 340 | def setup 341 | setup_bud 342 | @t = make_bud 343 | end 344 | 345 | def teardown 346 | cleanup_bud(@t) 347 | @t = nil 348 | end 349 | 350 | def make_bud 351 | DbmCycleDelta.new(:dbm_dir => DBM_BUD_DIR, :dbm_truncate => false, :quiet => true, :port => 65432) 352 | end 353 | 354 | def test_sum_delta 355 | @t.run_bg 356 | @t.sync_do { 357 | @t.t1 <+ [[10, 20]] 358 | } 359 | @t.sync_do { 360 | assert_equal([[10, 20]], @t.t1.to_a) 361 | } 362 | @t.stop 363 | end 364 | end 365 | -------------------------------------------------------------------------------- /test/tc_delta.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | 3 | class DeltaTest 4 | include Bud 5 | 6 | state do 7 | table :orig, [:k1, :k2] => [] 8 | scratch :scr, [:k1, :k2] => [] 9 | table :result, [:k1, :k2] => [] 10 | end 11 | 12 | bootstrap do 13 | orig <= [['a', 'b']] 14 | orig <= [['a', 'c']] 15 | end 16 | 17 | bloom do 18 | scr <= orig 19 | result <= scr 20 | end 21 | end 22 | 23 | class DeltaJoinTest 24 | include Bud 25 | 26 | state do 27 | table :orig, [:from, :to] 28 | scratch :link, [:from, :to] 29 | scratch :path, [:from, :to] 30 | scratch :hashpath, [:from, :to] 31 | end 32 | 33 | bootstrap do 34 | orig <= [['a', 'b'], ['b', 'c'], ['c', 'd']] 35 | end 36 | 37 | bloom :paths do 38 | link <= orig 39 | path <= link 40 | path <= (link * path).pairs {|l,p| [l.from, p.to] if l.to == p.from} 41 | hashpath <= link 42 | hashpath <= (link * path).pairs(:to => :from) {|l,p| [l.from, p.to]} 43 | end 44 | end 45 | 46 | class Delta3JoinTest 47 | include Bud 48 | 49 | state do 50 | table :orig, [:from, :to] 51 | table :wanted, [:node] 52 | scratch :link, [:from, :to] 53 | scratch :path, [:from, :to] 54 | scratch :hashpath, [:from, :to] 55 | end 56 | 57 | bootstrap do 58 | orig <= [['a', 'b'], ['b', 'c'], ['c', 'd']] 59 | wanted <= [['a'], ['b'], ['c']] 60 | end 61 | 62 | bloom :paths do 63 | link <= orig 64 | path <= link 65 | path <= (link * path * wanted).combos {|l,p,w| [l.from, p.to] if l.to == p.from and l.from == w.node} 66 | hashpath <= link 67 | hashpath <= (link * path * wanted).combos(link.to => path.from, link.from => wanted.node) {|l,p| [l.from, p.to]} 68 | end 69 | end 70 | 71 | class TestDelta < Minitest::Test 72 | def test_transitivity 73 | program = DeltaTest.new 74 | program.tick 75 | assert_equal(2, program.result.length) 76 | end 77 | 78 | def test_one 79 | program = DeltaJoinTest.new 80 | program.tick 81 | assert_equal(6, program.path.length) 82 | assert_equal(6, program.hashpath.length) 83 | end 84 | 85 | def test_three 86 | program = Delta3JoinTest.new 87 | program.tick 88 | assert_equal(6, program.path.length) 89 | assert_equal(6, program.hashpath.length) 90 | end 91 | end 92 | -------------------------------------------------------------------------------- /test/tc_errors.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | require 'tempfile' 3 | 4 | class TestErrorHandling < Minitest::Test 5 | class EmptyBud 6 | include Bud 7 | end 8 | 9 | def test_do_sync_error 10 | b = EmptyBud.new 11 | b.run_bg 12 | 3.times { 13 | assert_raises(ZeroDivisionError) { 14 | b.sync_do { 15 | puts 5 / 0 16 | } 17 | } 18 | } 19 | 20 | b.stop 21 | end 22 | 23 | class IllegalOp 24 | include Bud 25 | 26 | state do 27 | table :t1 28 | end 29 | 30 | bloom do 31 | t1 < t1 {|t| [t.key + 1, t.val + 1]} 32 | end 33 | end 34 | 35 | def test_illegal_op_error 36 | assert_raises(Bud::CompileError) { IllegalOp.new } 37 | end 38 | 39 | class IllegalAsyncOp 40 | include Bud 41 | 42 | state do 43 | table :t1 44 | end 45 | 46 | bloom do 47 | t1 <~ t1 {|x| ["foo"]} 48 | end 49 | end 50 | 51 | def test_illegal_async_op 52 | assert_raises(Bud::CompileError) { IllegalAsyncOp.new.tick } 53 | end 54 | 55 | class IllegalAsyncLattice 56 | include Bud 57 | 58 | state do 59 | lmap :m1 60 | end 61 | 62 | bloom do 63 | m1 <~ m1 64 | end 65 | end 66 | 67 | def test_illegal_async_lattice 68 | assert_raises(Bud::CompileError) { IllegalAsyncLattice.new.tick } 69 | end 70 | 71 | class InsertInBloomBlock 72 | include Bud 73 | 74 | state do 75 | table :t1 76 | end 77 | 78 | bloom do 79 | t1 << [5, 10] 80 | end 81 | end 82 | 83 | def test_insert_in_bloom_error 84 | assert_raises(Bud::CompileError) { InsertInBloomBlock.new } 85 | end 86 | 87 | class MissingTable 88 | include Bud 89 | 90 | state do 91 | table :t1 92 | end 93 | 94 | bloom do 95 | t2 <= t1 96 | end 97 | end 98 | 99 | class BadSchemy 100 | include Bud 101 | 102 | state do 103 | table :num, ["key"] => [] 104 | end 105 | end 106 | 107 | def test_bad_schemy 108 | assert_raises(Bud::Error) do 109 | p = BadSchemy.new 110 | p.tick 111 | end 112 | end 113 | 114 | class SchemyConflict 115 | include Bud 116 | 117 | state do 118 | table :num, [:map] => [] 119 | end 120 | end 121 | 122 | def test_schemy_conflict 123 | assert_raises(Bud::Error) do 124 | p = SchemyConflict.new 125 | p.tick 126 | end 127 | end 128 | 129 | def test_missing_table_error 130 | assert_raises(Bud::CompileError) { MissingTable.new } 131 | end 132 | 133 | class PrecedenceError 134 | include Bud 135 | 136 | state do 137 | table :foo 138 | table :bar 139 | table :baz 140 | end 141 | 142 | bloom do 143 | foo <= baz 144 | # Mistake: <= binds more tightly than "or" 145 | foo <= (bar.first and baz.first) or [] 146 | end 147 | end 148 | 149 | def test_precedence_error 150 | assert_raises(Bud::CompileError) { PrecedenceError.new } 151 | end 152 | 153 | class VarShadowError 154 | include Bud 155 | 156 | state do 157 | table :t1 158 | table :t2 159 | end 160 | 161 | bloom do 162 | temp :t2 <= (t1 * t1) 163 | end 164 | end 165 | 166 | def test_var_shadow_error 167 | assert_raises(Bud::CompileError) { VarShadowError.new } 168 | end 169 | 170 | def test_bloom_block_error 171 | defn = "class BloomBlockError\ninclude Bud\nbloom \"blockname\" do\nend\n\nend\n" 172 | assert_raises(Bud::CompileError) {eval(defn)} 173 | end 174 | 175 | def test_dup_blocks 176 | src = "class DupBlocks\ninclude Bud\nbloom :foo do\nend\nbloom :foo do\nend\nend\n" 177 | f = Tempfile.new("dup_blocks.rb") 178 | f.write(src) 179 | f.close 180 | assert_raises(Bud::CompileError) { load f.path } 181 | end 182 | 183 | class EvalError 184 | include Bud 185 | 186 | state do 187 | scratch :t1 188 | scratch :t2 189 | end 190 | 191 | bloom do 192 | t2 <= t1 { |t| [t.key, 5 / t.val]} 193 | end 194 | end 195 | 196 | def test_eval_error 197 | e = EvalError.new 198 | e.run_bg 199 | 200 | assert_raises(ZeroDivisionError) { 201 | e.sync_do { 202 | e.t1 <+ [[5, 0]] 203 | } 204 | } 205 | 206 | e.stop 207 | end 208 | 209 | class BadGroupingCols 210 | include Bud 211 | 212 | state do 213 | table :t1 214 | end 215 | 216 | bootstrap do 217 | t1 << [1,1] 218 | end 219 | 220 | bloom do 221 | temp :t2 <= t1.group(["key"], min(:val)) 222 | end 223 | end 224 | 225 | def test_bad_grouping_cols 226 | p = BadGroupingCols.new 227 | assert_raises(Bud::Error) {p.tick} 228 | end 229 | 230 | class BadJoinTabs 231 | include Bud 232 | state do 233 | table :t1 234 | table :t2 235 | table :t3 236 | end 237 | bootstrap do 238 | t1 << [1,1] 239 | t2 << [2,2] 240 | end 241 | 242 | bloom do 243 | temp :out <= (t1*t2).pairs(t3.key => t2.val) 244 | end 245 | end 246 | 247 | def test_bad_join_tabs 248 | p = BadJoinTabs.new 249 | assert_raises(Bud::CompileError) {p.tick} 250 | end 251 | 252 | class BadNextChannel 253 | include Bud 254 | state do 255 | channel :c1 256 | end 257 | bloom do 258 | c1 <+ [["doh"]] 259 | end 260 | end 261 | 262 | def test_bad_next_channel 263 | p = BadNextChannel.new 264 | assert_raises(Bud::CompileError) {p.tick} 265 | end 266 | 267 | class BadStdio 268 | include Bud 269 | bloom do 270 | stdio <= [["phooey"]] 271 | end 272 | end 273 | 274 | def test_bad_stdio 275 | p = BadStdio.new 276 | assert_raises(Bud::CompileError) {p.tick} 277 | end 278 | 279 | class BadFileReader1 280 | include Bud 281 | state do 282 | file_reader :fd, "/tmp/foo#{Process.pid}" 283 | end 284 | bloom do 285 | fd <= [['no!']] 286 | end 287 | end 288 | 289 | def test_bad_file_reader_1 290 | File.open("/tmp/foo#{Process.pid}", 'a') 291 | p = BadFileReader1.new 292 | assert_raises(Bud::CompileError){p.tick} 293 | end 294 | 295 | class BadFileReader2 296 | include Bud 297 | state do 298 | file_reader :fd, "/tmp/foo#{Process.pid}" 299 | end 300 | bloom do 301 | fd <+ [['no!']] 302 | end 303 | end 304 | 305 | def test_bad_file_reader_2 306 | File.open("/tmp/foo#{Process.pid}", 'a') 307 | assert_raises(Bud::CompileError) { BadFileReader2.new.tick} 308 | end 309 | 310 | class BadFileReader3 311 | include Bud 312 | state do 313 | file_reader :fd, "/tmp/foo#{Process.pid}" 314 | end 315 | bloom do 316 | fd <~ [['no!']] 317 | end 318 | end 319 | 320 | def test_bad_file_reader_3 321 | File.open("/tmp/foo#{Process.pid}", 'a') 322 | assert_raises(Bud::CompileError) { BadFileReader3.new.tick} 323 | end 324 | 325 | class BadOp 326 | include Bud 327 | state do 328 | table :foo 329 | table :bar 330 | end 331 | bloom do 332 | foo + bar 333 | end 334 | end 335 | 336 | def test_bad_op 337 | assert_raises(Bud::CompileError) { BadOp.new } 338 | end 339 | 340 | class BadTerminal 341 | include Bud 342 | state {terminal :joeio} 343 | bloom do 344 | joeio <~ [["hi"]] 345 | end 346 | end 347 | 348 | def test_bad_terminal 349 | assert_raises(Bud::Error) { BadTerminal.new } 350 | end 351 | 352 | module SyntaxBase 353 | state do 354 | table :foo 355 | table :bar 356 | end 357 | end 358 | 359 | class SyntaxTest1 360 | include Bud 361 | include SyntaxBase 362 | 363 | bloom :foobar do 364 | foo = bar 365 | end 366 | end 367 | 368 | def test_parsetime_error 369 | begin 370 | SyntaxTest1.new 371 | assert(false) 372 | rescue 373 | assert_equal(Bud::CompileError, $!.class) 374 | # fragile assertion? (whitespace etc) 375 | assert_equal("illegal operator: '=' in rule block \"__bloom__foobar\"\nCode: foo = bar", $!.to_s) 376 | end 377 | end 378 | end 379 | -------------------------------------------------------------------------------- /test/tc_execmodes.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | require 'socket' 3 | require 'timeout' 4 | 5 | class Vacuous 6 | include Bud 7 | end 8 | 9 | class ExecModeTest < Minitest::Test 10 | def test_shutdown_em 11 | c = Vacuous.new 12 | c.run_bg 13 | c.stop(true) 14 | assert_equal(false, EventMachine::reactor_running?) 15 | end 16 | 17 | def test_term 18 | kill_with_signal("TERM") 19 | kill_with_signal("TERM") 20 | end 21 | 22 | def test_int 23 | kill_with_signal("INT") 24 | kill_with_signal("INT") 25 | end 26 | 27 | class AckWhenReady 28 | include Bud 29 | 30 | state do 31 | scratch :dummy 32 | periodic :timer, 0.1 33 | end 34 | 35 | bloom do 36 | dummy <= timer { 37 | @ack_io.puts ip_port 38 | @ack_io.puts "ready" 39 | [10, 20] 40 | } 41 | end 42 | end 43 | 44 | def kill_with_signal(sig) 45 | read, write = IO.pipe 46 | c = AckWhenReady.new 47 | c.instance_variable_set('@ack_io', write) 48 | q = Queue.new 49 | c.on_shutdown do 50 | q.push(true) 51 | end 52 | c.run_bg 53 | Timeout::timeout(6) do 54 | _ = read.readline 55 | _ = read.readline 56 | Process.kill(sig, $$) 57 | q.pop 58 | end 59 | assert(q.empty?) 60 | read.close ; write.close 61 | 62 | # XXX: hack. There currently isn't a convenient way to block until the kill 63 | # signal has been completely handled (on_shutdown callbacks are invoked 64 | # before the end of the Bud shutdown process). Since we don't want to run 65 | # another test until EM has shutdown, we can at least wait for that. 66 | begin 67 | EventMachine::reactor_thread.join(10) 68 | rescue NoMethodError 69 | end 70 | end 71 | 72 | def test_sigint_child 73 | 2.times { kill_child_with_signal(Vacuous, "INT") } 74 | end 75 | 76 | def test_sigterm_child 77 | 2.times { kill_child_with_signal(Vacuous, "TERM") } 78 | end 79 | 80 | def kill_child_with_signal(parent_class, signal) 81 | read, write = IO.pipe 82 | parent = parent_class.new 83 | parent.run_bg 84 | pid = Bud.do_fork do 85 | p = AckWhenReady.new 86 | p.instance_variable_set('@ack_io', write) 87 | p.run_fg 88 | end 89 | _ = read.readline 90 | _ = read.readline 91 | Process.kill(signal, pid) 92 | _, status = Process.waitpid2(pid) 93 | assert(!status.signaled?) # Should have caught the signal 94 | assert(status.exited?) 95 | rubyMajorVersion = RUBY_VERSION.split('.')[0].to_i 96 | if (rubyMajorVersion < 2) 97 | assert_equal(0, status.exitstatus) 98 | else 99 | # Should be 1 on Ruby 2, but for some reason is non-deterministically 0 during testing 100 | assert_equal(true, (status.exitstatus == 1 or status.exitstatus == 0)) 101 | end 102 | parent.stop 103 | read.close ; write.close 104 | end 105 | 106 | def test_fg_bg_mix 107 | c1 = Vacuous.new 108 | c2 = Vacuous.new 109 | c1.run_bg 110 | cnt = 0 111 | t = Thread.new { 112 | c2.run_fg 113 | cnt += 1 114 | } 115 | sleep 0.1 116 | c1.stop 117 | c2.stop 118 | t.join 119 | assert_equal(1, cnt) 120 | end 121 | 122 | class AckOnBootWithShutdown 123 | include Bud 124 | 125 | bootstrap do 126 | @ack_io.puts ip_port 127 | @ack_io.puts "ready" 128 | on_shutdown do 129 | @ack_io.puts "done" 130 | end 131 | end 132 | end 133 | 134 | def test_fg_crash_shutdown_cb 135 | read, write = IO.pipe 136 | 137 | child_pid = Bud.do_fork do 138 | out_buf = StringIO.new 139 | $stdout = out_buf 140 | x = AckOnBootWithShutdown.new 141 | x.instance_variable_set('@ack_io', write) 142 | x.run_fg 143 | end 144 | 145 | child_ip_port = read.readline.rstrip 146 | child_ip, child_port = child_ip_port.split(":") 147 | result = read.readline.rstrip 148 | assert_equal("ready", result) 149 | 150 | # Shoot garbage at the Bud instance in the child process, which should cause 151 | # it to shutdown 152 | sock = UDPSocket.open 153 | sock.send("1234", 0, child_ip, child_port) 154 | sock.close 155 | 156 | Timeout::timeout(5) do 157 | result = read.readline.rstrip 158 | assert_equal("done", result) 159 | end 160 | read.close ; write.close 161 | Process.waitpid(child_pid) 162 | end 163 | 164 | def test_interrogate1 165 | c = Vacuous.new 166 | assert_raises(Bud::Error) {c.int_ip_port} 167 | end 168 | 169 | def test_interrogate2 170 | c = Vacuous.new 171 | c.run_bg 172 | assert_kind_of(String, c.int_ip_port) 173 | end 174 | 175 | def test_extra_stoppage 176 | c = Vacuous.new 177 | c.run_bg 178 | 5.times { c.stop } 179 | end 180 | 181 | def test_extra_startage 182 | c = Vacuous.new 183 | c.run_bg 184 | 5.times do 185 | assert_raises(Bud::Error) { c.run_bg } 186 | end 187 | c.stop 188 | end 189 | 190 | def test_stop_no_start 191 | c = Vacuous.new 192 | 5.times { c.stop } 193 | end 194 | end 195 | 196 | class ThreePhase 197 | include Bud 198 | 199 | state do 200 | loopback :c1 201 | scratch :s1, [] => [:v] 202 | loopback :c2 203 | 204 | scratch :c1_done, [] => [:v] 205 | scratch :s1_done, [] => [:v] 206 | scratch :c2_done, [] => [:v] 207 | end 208 | 209 | bootstrap do 210 | c1 <~ [["foo", 1]] 211 | end 212 | 213 | bloom :p1 do 214 | c1 <~ c1 {|t| [t.key, t.val + 1] if t.val < 50} 215 | c1_done <= c1 {|t| [t.val] if t.val >= 50} 216 | end 217 | 218 | bloom :p2 do 219 | s1 <+ s1 {|s| [s.v + 1] if s.v < 3} 220 | s1_done <= s1 {|s| [s.v] if s.v >= 3} 221 | end 222 | 223 | bloom :p3 do 224 | c2 <~ s1_done {|t| ["foo", 0]} 225 | c2 <~ c2 {|t| [t.key, t.val + 1] if t.val < 17} 226 | c2_done <= c2 {|t| [t.val] if t.val >= 17} 227 | end 228 | end 229 | 230 | class TestPause < Minitest::Test 231 | def test_pause_threephase 232 | b = ThreePhase.new 233 | q = Queue.new 234 | cb_id = b.register_callback(:c1_done) do |t| 235 | q.push(t.to_a) 236 | end 237 | b.run_bg 238 | rv = q.pop 239 | assert_equal([50], rv.first.to_a) 240 | b.pause 241 | b.unregister_callback(cb_id) 242 | 243 | b.s1 <+ [[1]] 244 | b.tick 245 | assert_equal([[1]], b.s1.to_a) 246 | assert(b.s1_done.empty?) 247 | 248 | b.tick 249 | assert_equal([[2]], b.s1.to_a) 250 | assert(b.s1_done.empty?) 251 | 252 | b.tick 253 | assert_equal([[3]], b.s1.to_a) 254 | assert_equal([[3]], b.s1_done.to_a) 255 | 256 | b.register_callback(:c2_done) do |t| 257 | q.push(t.to_a) 258 | end 259 | b.run_bg 260 | b.sync_do # might need to force another tick 261 | rv = q.pop 262 | assert_equal([17], rv.first.to_a) 263 | 264 | b.stop 265 | end 266 | end 267 | -------------------------------------------------------------------------------- /test/tc_exists.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | 3 | class ExistTest 4 | include Bud 5 | 6 | state do 7 | table :notes 8 | table :memories 9 | table :dups, [:str] 10 | periodic :timer, 0.5 11 | channel :msgs 12 | end 13 | 14 | bloom do 15 | msgs <~ (notes * timer).lefts{|n| [ip_port, n]} 16 | memories <= msgs.payloads{|p| p.val} 17 | dups <= memories.map{|n| [n.inspect] if msgs.exists?{|m| n.val == m.val[1]}} 18 | end 19 | end 20 | 21 | class TestExists < Minitest::Test 22 | def test_conv 23 | p = ExistTest.new 24 | p.run_bg 25 | 26 | q = Queue.new 27 | p.register_callback(:msgs) do 28 | q.push(true) 29 | end 30 | 31 | p.sync_do { 32 | p.notes <+ [[1, 'what a lovely day']] 33 | } 34 | p.sync_do { 35 | p.notes <+ [[2, "I think I'll go for a walk"]] 36 | } 37 | 38 | # Wait for two messages 39 | 2.times { q.pop } 40 | 41 | p.stop 42 | assert_equal(2, p.memories.length) 43 | assert_equal('what a lovely day', p.memories.first.val) 44 | assert_equal(2, p.dups.length) 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /test/tc_halt.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | 3 | class Halts 4 | include Bud 5 | 6 | state do 7 | scratch :tbl, [:key] 8 | periodic :timer, 0.01 9 | end 10 | 11 | bootstrap do 12 | tbl << [1] 13 | end 14 | 15 | bloom do 16 | halt <= tbl {|t| t if t.key == 2} 17 | tbl <+ tbl {|t| [t.key+1]} 18 | end 19 | end 20 | 21 | class TestHalt < Minitest::Test 22 | def test_halt 23 | program = Halts.new 24 | program.run_bg 25 | assert_raises(Bud::ShutdownWithCallbacksError) {4.times{program.delta(:tbl)}} 26 | end 27 | 28 | def test_halt_fg 29 | run_fg_finished = false 30 | t = Thread.new do 31 | program = Halts.new 32 | program.run_fg 33 | run_fg_finished = true 34 | end 35 | t.join 36 | assert(run_fg_finished) 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /test/tc_inheritance.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | 3 | module SimpleModule 4 | state do 5 | table :boot_tbl 6 | end 7 | 8 | bootstrap do 9 | boot_tbl << [25, 50] 10 | end 11 | end 12 | 13 | class ParentBud 14 | include Bud 15 | include SimpleModule 16 | 17 | state { 18 | table :tbl 19 | } 20 | 21 | bootstrap do 22 | boot_tbl << [5, 10] 23 | end 24 | 25 | bloom :bundle do 26 | tbl <= [[2, 'a']] 27 | end 28 | end 29 | 30 | class ChildBud < ParentBud 31 | bootstrap do 32 | boot_tbl << [10, 20] 33 | end 34 | 35 | # Test overriding 36 | bloom :bundle do 37 | tbl <= [[2, 'b']] 38 | end 39 | end 40 | 41 | class TestSubclass < Minitest::Test 42 | def test_override 43 | p1 = ParentBud.new 44 | p2 = ChildBud.new 45 | p1.tick 46 | p2.tick 47 | 48 | assert_equal('a', p1.tbl[[2]].val) 49 | assert_equal('b', p2.tbl[[2]].val) 50 | 51 | assert_equal([[5, 10], [25, 50]], p1.boot_tbl.to_a.sort) 52 | assert_equal([[5, 10], [10, 20], [25, 50]], p2.boot_tbl.to_a.sort) 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /test/tc_interface.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | 3 | module MemberProtocol 4 | state do 5 | interface input, :add_member, [:req_id] => [:name, :addr] 6 | interface output, :result, [:req_id] => [:success] 7 | table :member, [:name] => [:addr] 8 | end 9 | end 10 | 11 | # Don't allow members whose names appear in "bad_people" 12 | module SelectiveMembership 13 | include MemberProtocol 14 | 15 | state do 16 | table :bad_people, [:name] 17 | scratch :good_add_reqs, [:req_id] => [:name, :addr] 18 | end 19 | 20 | bootstrap do 21 | bad_people <= [['foo'], ['bar']] 22 | end 23 | 24 | bloom do 25 | good_add_reqs <= add_member.map do |m| 26 | m unless bad_people.include? [m.name] 27 | end 28 | 29 | member <= good_add_reqs.map {|m| [m.name, m.addr]} 30 | result <= good_add_reqs.map {|m| [m.req_id, true]} 31 | result <= add_member.map {|m| [m.req_id, false] unless good_add_reqs.include? m} 32 | end 33 | end 34 | 35 | class SimpleClient 36 | include Bud 37 | include SelectiveMembership 38 | end 39 | 40 | class InterfaceTest < Minitest::Test 41 | def test_basic 42 | c = SimpleClient.new 43 | c.run_bg 44 | 45 | # Add a legal member 46 | c.sync_do { 47 | c.add_member <+ [[1, 'quux', c.ip_port]] 48 | } 49 | c.sync_do { 50 | assert_equal(1, c.result.length) 51 | assert_equal([1, true], c.result.first) 52 | } 53 | # Test that output interface flushed after tick 54 | c.sync_do { 55 | assert(c.result.empty?) 56 | } 57 | 58 | # Add two members, one is illegal 59 | c.sync_do { 60 | c.add_member <+ [[2, 'foo', c.ip_port], [3, 'baz', c.ip_port]] 61 | } 62 | c.sync_do { 63 | results = c.result.to_a.sort 64 | assert_equal([[2, false], [3, true]], results) 65 | } 66 | c.stop 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /test/tc_labeling.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | require 'bud/labeling/labeling' 3 | 4 | module TestState 5 | state do 6 | interface input, :i1 7 | interface input, :i2 8 | channel :c1 9 | channel :c2 10 | table :guard1 11 | table :guard2 12 | interface output, :response 13 | end 14 | end 15 | 16 | module TestBasic 17 | include TestState 18 | bloom do 19 | c1 <~ i1 20 | c2 <~ i2 21 | guard1 <= c1 22 | guard2 <= c2 23 | end 24 | end 25 | 26 | module TestNM 27 | include TestBasic 28 | bloom do 29 | response <= guard1.notin(guard2, :val => :val) 30 | end 31 | end 32 | 33 | module TestGroup 34 | include TestBasic 35 | bloom do 36 | response <= guard1.group([:val], count) 37 | end 38 | end 39 | 40 | module TestMono 41 | include TestBasic 42 | bloom do 43 | response <= (guard1 * guard2).lefts(:val => :val) 44 | end 45 | end 46 | 47 | module TestDeletion 48 | include TestMono 49 | state do 50 | interface input, :dguard 51 | channel :c3 52 | end 53 | bloom do 54 | c3 <~ dguard 55 | guard2 <- (guard2 * c3).lefts(:val => :val) 56 | end 57 | end 58 | 59 | module TestNestMod 60 | import TestNM => :tnm 61 | state do 62 | interface input, :inn1 63 | interface input, :inn2 64 | interface output, :outt 65 | end 66 | 67 | bloom do 68 | tnm.i1 <= inn1 69 | tnm.i2 <= inn2 70 | outt <= tnm.response 71 | end 72 | end 73 | 74 | # ``unguarded asynchrony'' is a hidden source of nondeterminism in otherwise 75 | # monotonic bloom programs. 76 | module JoinProto 77 | state do 78 | interface input, :ileft 79 | interface input, :iright 80 | interface output, :result 81 | end 82 | end 83 | 84 | module Buffers 85 | include JoinProto 86 | state do 87 | scratch :ls 88 | scratch :rs 89 | table :lt 90 | table :rt 91 | 92 | channel :lchan 93 | channel :rchan 94 | end 95 | 96 | bloom do 97 | ileft <= lchan 98 | iright <= rchan 99 | 100 | ls <= ileft 101 | rs <= iright 102 | lt <= ileft 103 | rt <= iright 104 | end 105 | end 106 | 107 | module BugButt 108 | include Buffers 109 | bloom do 110 | result <= (ls * rs).lefts 111 | end 112 | end 113 | 114 | module HalfGuard 115 | include Buffers 116 | bloom do 117 | result <= (lt * rs).lefts 118 | end 119 | end 120 | 121 | module FullGuard 122 | include Buffers 123 | bloom do 124 | result <= (lt * rt).lefts 125 | end 126 | end 127 | 128 | 129 | 130 | module BB 131 | include Validate 132 | include GuardedAsync 133 | include Bud 134 | end 135 | 136 | class RolledUp 137 | include BB 138 | include TestNM 139 | end 140 | 141 | class RollupMono 142 | include BB 143 | include TestMono 144 | end 145 | 146 | class RollGroup 147 | include BB 148 | include TestGroup 149 | end 150 | 151 | class RollDels 152 | include BB 153 | include TestDeletion 154 | end 155 | 156 | class RollNest 157 | include BB 158 | include TestNestMod 159 | end 160 | 161 | module BBG 162 | include BB 163 | state do 164 | interface input, :ul 165 | interface input, :ur 166 | end 167 | bloom do 168 | lchan <~ ul 169 | rchan <~ ur 170 | end 171 | end 172 | 173 | class RollHG 174 | include BBG 175 | include HalfGuard 176 | end 177 | 178 | class TestBlazes < Minitest::Test 179 | def test_label1 180 | r = RolledUp.new 181 | r.tick 182 | report = r.validate 183 | assert(report.map{|r| r.to_a.last}.include?(["D"]), "flow not reported as divergent : #{report}") 184 | end 185 | 186 | def test_label2 187 | r = RollGroup.new 188 | r.tick 189 | report = r.validate 190 | assert(report.map{|r| r.to_a.last}.include?(["D"]), "flow not reported as divergent") 191 | end 192 | 193 | def test_mono 194 | r = RollupMono.new 195 | r.tick 196 | report = r.validate 197 | assert(!report.map{|r| r.to_a.last}.include?(["D"]), "flow not reported as confluent: #{report}") 198 | end 199 | 200 | def test_deletion 201 | r = RollDels.new 202 | r.tick 203 | report = r.validate 204 | reps = report.map{|r| [r[0], r[1], r.last]} 205 | assert(reps.include?(["dguard", "response", ["D"]]), "deletion path not marked D") 206 | assert(reps.include?(["i2", "response", ["D"]]), "main path not marked D #{reps}") 207 | end 208 | 209 | def test_nesting 210 | r = RollNest.new 211 | r.tick 212 | report = r.validate 213 | assert(report.map{|r| r.to_a.last}.include?(["D"]), "flow not reported as divergent : #{report}") 214 | end 215 | 216 | def test_unguarded 217 | h = RollHG.new 218 | h.tick 219 | report = h.validate 220 | assert(report.map{|r| r.to_a.last}.include?(["D"]), "flow not reported as divergent : #{report}") 221 | end 222 | 223 | def test_labeler1 224 | l = Label.new("TestNM") 225 | assert_equal({"response" => "D"}, l.output_report) 226 | assert_equal({"response" => {"i1" => "A", "i2" => "D"}}, l.path_report) 227 | end 228 | 229 | def test_labeler2 230 | l = Label.new("TestGroup") 231 | assert_equal({"response" => "D"}, l.output_report) 232 | assert_equal({"response" => {"i1" => "D"}}, l.path_report) 233 | end 234 | 235 | def test_labeler3 236 | l = Label.new("TestMono") 237 | assert_equal({"response" => "A"}, l.output_report) 238 | assert_equal({"response" => {"i1" => "A", "i2" => "A"}}, l.path_report) 239 | end 240 | end 241 | 242 | 243 | # Tests covering just the GA part 244 | module Extry 245 | include Bud 246 | include GuardedAsync 247 | 248 | state do 249 | table :rem_race, channel_race.schema 250 | end 251 | bloom do 252 | rem_race <= channel_race 253 | end 254 | end 255 | 256 | class BugC 257 | include Extry 258 | include BugButt 259 | end 260 | 261 | class HalfGuardC 262 | include Extry 263 | include HalfGuard 264 | end 265 | 266 | class FullGuardC 267 | include Extry 268 | include FullGuard 269 | end 270 | 271 | class TestBlazes < Minitest::Test 272 | def test_bug 273 | c = BugC.new 274 | c.tick 275 | assert_equal([["rchan", "lchan", "result", false], 276 | ["lchan", "rchan", "result", false]].to_set, 277 | c.rem_race.map{|r| r.to_a}.to_set) 278 | end 279 | 280 | def test_hg 281 | c = HalfGuardC.new 282 | c.tick 283 | assert_equal([["rchan", "lchan", "result", false], 284 | ["lchan", "rchan", "result", false]].to_set, 285 | c.rem_race.map{|r| r.to_a}.to_set) 286 | end 287 | 288 | def test_full 289 | c = FullGuardC.new 290 | c.tick 291 | assert_equal([["rchan", "lchan", "result", true], 292 | ["lchan", "rchan", "result", true]].to_set, 293 | c.rem_race.map{|r| r.to_a}.to_set) 294 | end 295 | end 296 | -------------------------------------------------------------------------------- /test/tc_mapvariants.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | 3 | # Check that maps over constant ranges aren't converted to semi-map 4 | class LeaveMapAlone 5 | include Bud 6 | 7 | state do 8 | table :num, [:num] 9 | end 10 | 11 | bloom do 12 | num <= (1..5).map{|i| [i]} 13 | end 14 | end 15 | 16 | class AllMapsAreOne 17 | include Bud 18 | 19 | state do 20 | scratch :out, [:val] 21 | scratch :snout, [:val] 22 | scratch :clout, [:val] 23 | scratch :inski 24 | end 25 | 26 | bootstrap {inski <= [[1,1], [2,2], [3,3]]} 27 | 28 | bloom do 29 | out <= inski {|i| [i.val]} 30 | snout <= inski.map {|i| [i.val]} 31 | clout <= inski.pro {|i| [i.val]} 32 | end 33 | end 34 | 35 | class StillAnnoying 36 | include Bud 37 | 38 | state do 39 | scratch :out, [:val] 40 | scratch :inski 41 | end 42 | 43 | bloom :rules do 44 | temp :k <= inski 45 | out <= k.map {|t| [t.val]} 46 | end 47 | end 48 | 49 | class LessAnnoying < StillAnnoying 50 | include Bud 51 | 52 | bloom :rules do 53 | temp :tmpy <= inski 54 | out <= tmpy {|t| [t.val]} 55 | end 56 | end 57 | 58 | class TestMapVariants < Minitest::Test 59 | def test_leave_map_alone 60 | program = LeaveMapAlone.new 61 | program.tick 62 | assert_equal([[1],[2],[3],[4],[5]], program.num.to_a.sort) 63 | end 64 | 65 | def test_all_maps 66 | p = AllMapsAreOne.new 67 | p.tick 68 | assert_equal(3, p.out.length) 69 | assert_equal(p.out.to_a, p.snout.to_a) 70 | assert_equal(p.out.to_a, p.clout.to_a) 71 | end 72 | end 73 | 74 | class TestProEnumerable < Minitest::Test 75 | class SortIdAssign 76 | include Bud 77 | 78 | state do 79 | interface input, :in_t, [:payload] 80 | interface output, :out_t, [:ident] => [:payload] 81 | end 82 | 83 | bloom do 84 | out_t <= in_t.sort.each_with_index.map {|a, i| [i, a]} 85 | end 86 | end 87 | 88 | def test_sort_pro 89 | p = SortIdAssign.new 90 | p.run_bg 91 | r = p.sync_callback(:in_t, [[5], [1], [100], [6]], :out_t) 92 | assert_equal([[0, [1]], [1, [5]], [2, [6]], [3, [100]]], r.to_a.sort) 93 | p.stop 94 | end 95 | end 96 | -------------------------------------------------------------------------------- /test/tc_metrics.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | require 'stringio' 3 | 4 | class MetricsTest 5 | include Bud 6 | 7 | state do 8 | table :t1 9 | scratch :s1 10 | end 11 | 12 | bloom do 13 | s1 <= t1 14 | end 15 | end 16 | 17 | 18 | class TestMetrics < Minitest::Test 19 | def test_metrics 20 | out, err = capture_io do 21 | p = MetricsTest.new(:metrics => true) 22 | 5.times { 23 | p.sync_do { p.t1 <+ [[p.budtime, 5]] } 24 | } 25 | p.stop 26 | end 27 | 28 | assert(out.include? %Q{"","count","5"}) 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /test/tc_nest.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | 3 | class Nesting 4 | include Bud 5 | 6 | state do 7 | table :nested_people, [:p_id, :firstname, :lastname, :hobbies] 8 | table :has_hobby, [:person_id, :name] 9 | table :meta, [:name, :tab] 10 | scratch :flat, [:p_id, :firstname, :lastname, :hobby] 11 | scratch :renested, nested_people.key_cols => nested_people.val_cols 12 | scratch :np2, [:firstname, :lastname, :hobbies] 13 | end 14 | 15 | bootstrap do 16 | nested_people <= [[1, 'Nick', 'Machiavelli', ['scheming', 'books']]] 17 | nested_people <= [[2, 'Chris', 'Columbus', ['sailing', 'books']]] 18 | has_hobby <= [[1, 'scheming'], [1, 'books'], [2, 'sailing'], [2, 'books']] 19 | meta <= [["nested_people", nested_people], ["has_hobby", has_hobby]] 20 | end 21 | 22 | bloom :simple_nesting do 23 | flat <= nested_people.flat_map do |p| 24 | p.hobbies.map { |h| [p.p_id, p.firstname, p.lastname, h] } 25 | end 26 | end 27 | 28 | bloom :simple_renest do 29 | renested <= flat.group([flat.p_id, flat.firstname, flat.lastname], accum(flat.hobby)) 30 | end 31 | 32 | bloom :structured_nesting do 33 | np2 <= meta.flat_map do |m| 34 | m.tab.map {|t| [t.firstname, t.lastname, t.hobbies] if m.name == 'nested_people'} 35 | end 36 | end 37 | end 38 | 39 | class TestNest < Minitest::Test 40 | def test_nest 41 | u = Nesting.new 42 | u.tick 43 | assert_equal([[1, "Nick", "Machiavelli", "books"], 44 | [1, "Nick", "Machiavelli", "scheming"], 45 | [2, "Chris", "Columbus", "books"], 46 | [2, "Chris", "Columbus", "sailing"]].sort, 47 | u.flat.to_a.sort) 48 | 49 | a = u.renested.map{|t| [t[0], t[1], t[2], t[3].sort]} 50 | assert_equal([[1, "Nick", "Machiavelli", ["books", "scheming"]], 51 | [2, "Chris", "Columbus", ["books", "sailing"]]].sort, 52 | a.sort) 53 | assert_equal([["Nick", "Machiavelli", ["scheming","books"]], 54 | ["Chris", "Columbus", ["sailing", "books"]]].sort, 55 | u.np2.to_a.sort) 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /test/tc_rebl.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | require 'bud/rebl' 3 | require 'stringio' 4 | require 'timeout' 5 | 6 | def capture_stdout 7 | $stdout = StringIO.new("", "w") 8 | begin 9 | yield 10 | $stdout.string 11 | ensure 12 | $stdout = STDOUT 13 | end 14 | end 15 | 16 | class ReblTester 17 | attr_reader :lib 18 | 19 | def initialize 20 | @lib = ReblShell::setup 21 | end 22 | 23 | def exec_rebl(str) 24 | return capture_stdout do 25 | $stdin = StringIO.new(str) 26 | ReblShell::rebl_loop(@lib, true) 27 | end 28 | end 29 | end 30 | 31 | # TODO: add the following testcases: 32 | # * test persistent store functionality 33 | 34 | class TestRebl < Minitest::Test 35 | def test_rebl_pingpong 36 | the_line = nil 37 | rt1 = nil 38 | ip_port1 = nil 39 | rt2 = nil 40 | ip_port2 = nil 41 | # Ignore the welcome messages. 42 | capture_stdout do 43 | rt1 = ReblTester.new 44 | rt2 = ReblTester.new 45 | ip_port1 = "#{rt1.lib.ip}:#{rt1.lib.port}" 46 | ip_port2 = "#{rt2.lib.ip}:#{rt2.lib.port}" 47 | end 48 | 49 | # Set up ping rules, send initial ping from rt1 to rt2 50 | rt1.exec_rebl("channel :ping, [:@dst, :src]") 51 | rt1.exec_rebl("ping <~ ping.map {|p| [p.src, p.dst]}") 52 | rt2.exec_rebl("channel :ping, [:@dst, :src]") 53 | rt2.exec_rebl("ping <~ ping.map {|p| [p.src, p.dst]}") 54 | rt1.exec_rebl("ping <~ [['#{ip_port2}', ip_port]]") 55 | rt1.exec_rebl("stdio <~ [(@budtime == 50) ? ['hit'] : nil]") 56 | rt2.exec_rebl("/run") 57 | 58 | # Start up the node, and wait for the bud time to go up to 50 (non-lazy mode) 59 | begin 60 | read, $stdout = IO.pipe 61 | $stdin = StringIO.new("/run") 62 | ReblShell::rebl_loop(rt1.lib, true) 63 | Timeout::timeout(30) do 64 | the_line = read.readline 65 | end 66 | ensure 67 | $stdout = STDOUT 68 | end 69 | assert_equal("hit\n", the_line) 70 | 71 | # Now perform a stop on both nodes 72 | rt1.exec_rebl("/stop") 73 | rt2.exec_rebl("/stop") 74 | 75 | # Check their timestamps 76 | stop_time1 = rt1.lib.rebl_class_inst.budtime 77 | stop_time2 = rt2.lib.rebl_class_inst.budtime 78 | 79 | begin 80 | # Now, test the breakpoint functionality 81 | rt1.exec_rebl("rebl_breakpoint <= [{50 => [true]}[@budtime]]") 82 | rt2.exec_rebl("/run") 83 | read, $stdout = IO.pipe 84 | $stdin = StringIO.new("/run") 85 | ReblShell::rebl_loop(rt1.lib, true) 86 | Timeout::timeout(30) do 87 | the_line = read.readline 88 | end 89 | ensure 90 | $stdout = STDOUT 91 | end 92 | assert_equal("hit\n", the_line) 93 | 94 | # Now perform a stop on both nodes 95 | rt1.exec_rebl("/stop") 96 | rt2.exec_rebl("/stop") 97 | 98 | # Check their timestamps 99 | stop_time1 = rt1.lib.rebl_class_inst.budtime 100 | stop_time2 = rt2.lib.rebl_class_inst.budtime 101 | end 102 | 103 | def start_tester 104 | rt = nil 105 | # Ignore the welcome messages. 106 | capture_stdout do 107 | rt = ReblTester.new 108 | end 109 | return rt 110 | end 111 | 112 | def test_rebl_shortestpaths 113 | rt = start_tester 114 | 115 | # Check to see if help mode works 116 | rt.exec_rebl("/help") 117 | 118 | # Declarations 119 | rt.exec_rebl("table :link, [:from, :to, :cost]") 120 | rt.exec_rebl("table :path, [:from, :to, :nxt, :cost]") 121 | 122 | # Check lscollections 123 | expected_output = "1: table :link, [:from, :to, :cost]\n2: table :path, [:from, :to, :nxt, :cost]\n" 124 | actual_output = rt.exec_rebl("/lscollections") 125 | assert_equal(expected_output, actual_output) 126 | 127 | # Now add some rules 128 | rt.exec_rebl("path <= link {|e| [e.from, e.to, e.to, e.cost]}") 129 | rt.exec_rebl("temp :k <= (link*path).pairs(:to => :from)") 130 | rt.exec_rebl("path <= k { |l,p| [l.from, p.to, p.from, l.cost+p.cost] }") 131 | rt.exec_rebl("stdio <~ [['foo']]") 132 | actual_output = rt.exec_rebl("/tick 3") 133 | 134 | # Check to make sure stdio thing is printing 135 | assert_equal("foo\nfoo\nfoo\n", actual_output) 136 | rt.exec_rebl("/rmrule 4") 137 | actual_output = rt.exec_rebl("/tick 3") 138 | # Check to make sure removed stdio rule no longer prints 139 | assert_equal("", actual_output) 140 | 141 | # Now check the rules we've got 142 | expected_output = "1: path <= link {|e| [e.from, e.to, e.to, e.cost]}\n2: temp :k <= (link*path).pairs(:to => :from)\n3: path <= k { |l,p| [l.from, p.to, p.from, l.cost+p.cost] }\n" 143 | actual_output = rt.exec_rebl("/lsrules") 144 | assert_equal(expected_output, actual_output) 145 | 146 | # Now add some links and tick 147 | rt.exec_rebl("link <= [['a','b',1],['a','b',4],['b','c',1],['c','d',1],['d','e',1]]") 148 | rt.exec_rebl("/tick") 149 | 150 | # Check dump functionality 151 | expected_output = "[\"a\", \"b\", \"b\", 1]\n[\"a\", \"b\", \"b\", 4]\n[\"a\", \"c\", \"b\", 2]\n[\"a\", \"c\", \"b\", 5]\n[\"a\", \"d\", \"b\", 3]\n[\"a\", \"d\", \"b\", 6]\n[\"a\", \"e\", \"b\", 4]\n[\"a\", \"e\", \"b\", 7]\n[\"b\", \"c\", \"c\", 1]\n[\"b\", \"d\", \"c\", 2]\n[\"b\", \"e\", \"c\", 3]\n[\"c\", \"d\", \"d\", 1]\n[\"c\", \"e\", \"d\", 2]\n[\"d\", \"e\", \"e\", 1]\n" 152 | actual_output = rt.exec_rebl("/dump path") 153 | assert_equal(expected_output, actual_output) 154 | 155 | # Add a new collection and rule for shortest paths, and tick 156 | rt.exec_rebl("table :shortest, [:from, :to] => [:nxt, :cost]") 157 | rt.exec_rebl("shortest <= path.argmin([path.from, path.to], path.cost)") 158 | rt.exec_rebl("/tick") 159 | 160 | # Now, remove all of the rules, and tick 161 | rt.exec_rebl("/rmrule 4") 162 | rt.exec_rebl("/rmrule 3") 163 | rt.exec_rebl("/rmrule 1") 164 | rt.exec_rebl("/rmrule 2") 165 | rt.exec_rebl("/tick") 166 | 167 | # Now check the contents of shortest to make sure that rule removal doesn't 168 | # cause un-derivation of previously derived tuples 169 | expected_output = "[\"a\", \"b\", \"b\", 1]\n[\"a\", \"c\", \"b\", 2]\n[\"a\", \"d\", \"b\", 3]\n[\"a\", \"e\", \"b\", 4]\n[\"b\", \"c\", \"c\", 1]\n[\"b\", \"d\", \"c\", 2]\n[\"b\", \"e\", \"c\", 3]\n[\"c\", \"d\", \"d\", 1]\n[\"c\", \"e\", \"d\", 2]\n[\"d\", \"e\", \"e\", 1]\n" 170 | actual_output = rt.exec_rebl("/dump shortest") 171 | assert_equal(expected_output, actual_output) 172 | end 173 | 174 | # Issue 274 -- check that cached state is invalidated correctly when 175 | # reinstanciating Bud. 176 | def test_no_leftovers 177 | rt = start_tester 178 | rt.exec_rebl("scratch :passing_clouds") 179 | rt.exec_rebl(%{passing_clouds <= [[3, "Nimbus"], [2, "Cumulonimbus"]]}) 180 | rt.exec_rebl(%{stdio <~ passing_clouds.inspected}) 181 | actual_output = rt.exec_rebl("/tick") 182 | assert_match(/3.*Nimbus/, actual_output) 183 | assert_match(/2.*Cumulonimbus/, actual_output) 184 | rt.exec_rebl("/rmrule 1") 185 | actual_output = rt.exec_rebl("/tick") 186 | assert_match(/\s*/, actual_output) 187 | end 188 | 189 | def test_rebl_lattice 190 | rt = start_tester 191 | rt.exec_rebl("lset :s1") 192 | rt.exec_rebl("lmax :m1") 193 | 194 | expected = "1: lset :s1\n2: lmax :m1\n" 195 | assert_equal(expected, rt.exec_rebl("/lscollections")) 196 | 197 | rt.exec_rebl("m1 <= s1.size") 198 | rt.exec_rebl("s1 <= [[1], [2], [3]]") 199 | assert_equal(expected, rt.exec_rebl("/lscollections")) 200 | 201 | rt.exec_rebl("/tick") 202 | assert_equal("\n", rt.exec_rebl("/dump m1")) 203 | 204 | # We expect that removing a rule does that remove previously derived 205 | # conclusions 206 | rt.exec_rebl("/rmrule 1") 207 | rt.exec_rebl("s1 <= [[4], [5], [6]]") 208 | rt.exec_rebl("/tick") 209 | assert_equal("\n", rt.exec_rebl("/dump m1")) 210 | assert_equal("\n", rt.exec_rebl("/dump s1")) 211 | end 212 | end 213 | -------------------------------------------------------------------------------- /test/tc_schemafree.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | 3 | class SchemaFree 4 | include Bud 5 | 6 | state do 7 | table :notes 8 | scratch :stats 9 | interface input, :send_me 10 | channel :msgs 11 | end 12 | 13 | bloom do 14 | notes <= msgs.payloads {|p| p.val} 15 | msgs <~ send_me 16 | end 17 | end 18 | 19 | class TestSFree < Minitest::Test 20 | def test_bloom 21 | p = SchemaFree.new 22 | p.run_bg 23 | 24 | q = Queue.new 25 | p.register_callback(:msgs) do 26 | q.push(true) 27 | end 28 | 29 | p.sync_do { 30 | p.send_me <+ [[p.ip_port, [[123, 1], 'what a lovely day']]] 31 | } 32 | p.sync_do { 33 | p.send_me <+ [[p.ip_port, [[123, 2], "I think I'll go for a walk"]]] 34 | } 35 | 36 | 2.times { q.pop } 37 | 38 | p.stop 39 | assert_equal(2, p.notes.length) 40 | assert_equal(123, p.notes.first.key[0]) 41 | assert_equal('what a lovely day', p.notes.first.val) 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /test/tc_sort.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | 3 | class TestSort < Minitest::Test 4 | class SortDelay 5 | include Bud 6 | 7 | state do 8 | interface input, :in_t, [:payload] 9 | interface output, :out_t, [:ident] => [:payload] 10 | scratch :delaybuf, [:ident] => [:payload] 11 | end 12 | 13 | bloom do 14 | delaybuf <= in_t.sort.each_with_index.map {|a, i| [i, a] } 15 | out_t <= delaybuf 16 | end 17 | end 18 | 19 | class SortTuples 20 | include Bud 21 | state do 22 | scratch :tab, [:a, :b] 23 | scratch :out, [:i] => [:a, :b] 24 | end 25 | bloom do 26 | out <= tab.sort{|t1,t2| t1.a == t2.a ? t1.b <=> t2.b : t1.a <=> t2.a}.each_with_index{|tup, i| [i, tup.a, tup.b]} 27 | end 28 | end 29 | 30 | def test_sort_simple 31 | p = SortTuples.new 32 | p.tab <+ [ 33 | [20, 20], 34 | [20, 30], 35 | [1, 5], 36 | [1, 10] 37 | ] 38 | p.tick 39 | out = p.out.map{|t| t.to_a} 40 | assert_equal([[0, 1, 5], [1, 1, 10], [2, 20, 20], [3, 20, 30]], out.sort{|a,b| a[0] <=> b[0]}) 41 | end 42 | 43 | def test_sort_pro 44 | p = SortDelay.new 45 | p.run_bg 46 | r = p.sync_callback(:in_t, [[5], [1], [100], [6]], :out_t) 47 | assert_equal([[0, [1]], [1, [5]], [2, [6]], [3, [100]]], r.to_a.sort) 48 | p.stop 49 | end 50 | 51 | class SortRescan 52 | include Bud 53 | 54 | state do 55 | table :t1 56 | scratch :s1 57 | scratch :sort_res 58 | end 59 | 60 | bloom do 61 | sort_res <= t1.sort 62 | sort_res <= s1 63 | end 64 | end 65 | 66 | def test_sort_rescan 67 | i = SortRescan.new 68 | i.t1 <+ [[3, 4], [5, 10]] 69 | i.tick 70 | assert_equal([[3, 4], [5, 10]], i.sort_res.to_a.sort) 71 | i.tick 72 | assert_equal([[3, 4], [5, 10]], i.sort_res.to_a.sort) 73 | i.s1 <+ [[1, 1]] 74 | i.t1 <+ [[9, 9]] 75 | i.tick 76 | assert_equal([[1, 1], [3, 4], [5, 10], [9, 9]], i.sort_res.to_a.sort) 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /test/tc_terminal.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | require 'stringio' 3 | 4 | 5 | class StdinReader 6 | include Bud 7 | state do 8 | scratch :saw_input, stdio.schema 9 | end 10 | 11 | bloom do 12 | saw_input <= stdio 13 | end 14 | end 15 | 16 | class StdioEcho 17 | include Bud 18 | 19 | bloom do 20 | stdio <~ stdio {|s| ["Saw: #{s.line}"]} 21 | end 22 | end 23 | 24 | class StdioBootstrap 25 | include Bud 26 | 27 | bootstrap do 28 | stdio <~ [["hello from bootstrap!"]] 29 | end 30 | end 31 | 32 | class TestTerminal < Minitest::Test 33 | def test_stdin 34 | input_lines = ["line1", "line2", "line3"] 35 | input_str = input_lines.join("\n") + "\n" 36 | input_buf = StringIO.new(input_str) 37 | q = Queue.new 38 | b = StdinReader.new(:stdin => input_buf) 39 | b.register_callback(:saw_input) do |tbl| 40 | tbl.to_a.each {|t| q.push(t)} 41 | end 42 | b.run_bg 43 | rv = [] 44 | input_lines.length.times { rv << q.pop } 45 | assert_equal(input_lines.map{|l| [l]}.sort, rv.sort) 46 | b.stop 47 | end 48 | 49 | def test_stdio_pipe 50 | in_read, in_write = IO.pipe 51 | out_read, out_write = IO.pipe 52 | 53 | b = StdioEcho.new(:stdin => in_read, :stdout => out_write) 54 | b.run_bg 55 | 56 | ["foo", "bar", "baz"].each do |str| 57 | in_write.puts(str) 58 | rv = out_read.gets 59 | assert_equal("Saw: #{str}\n", rv) 60 | end 61 | 62 | b.stop 63 | end 64 | 65 | def test_stdio_bootstrap 66 | output_buf = StringIO.new 67 | b = StdioBootstrap.new(:stdout => output_buf) 68 | b.tick 69 | assert_equal("hello from bootstrap!\n", output_buf.string) 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /test/tc_timer.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | 3 | class TemporalBudTest 4 | include Bud 5 | 6 | state do 7 | periodic :tik, 0.1 8 | table :log, tik.schema 9 | end 10 | 11 | bloom do 12 | log <= tik {|t| [t.key, t.val - 100]} 13 | end 14 | end 15 | 16 | class TestTimer < Minitest::Test 17 | def test_timer 18 | b = TemporalBudTest.new 19 | q = Queue.new 20 | b.register_callback(:tik) do |t| 21 | assert_equal(1, t.length) 22 | tup = t.to_a.first 23 | assert(tup.val < Time.now) 24 | q.push(tup) 25 | end 26 | b.run_bg 27 | 28 | 5.times { q.pop } 29 | b.stop 30 | end 31 | 32 | def test_timer_tick 33 | b = TemporalBudTest.new 34 | tick_cnt = 0 35 | b.register_callback(:tik) do |t| 36 | tick_cnt += 1 37 | end 38 | 39 | b.tick 40 | sleep 0.4 41 | b.tick 42 | assert_equal(1, tick_cnt) 43 | sleep 0.4 44 | b.tick 45 | assert_equal(2, tick_cnt) 46 | b.stop 47 | end 48 | end 49 | 50 | class BudClockExample 51 | include Bud 52 | 53 | state do 54 | table :t1 55 | scratch :in_tbl, [:val] 56 | scratch :in_tbl2, [:val] 57 | end 58 | 59 | bloom do 60 | t1 <= in_tbl {|t| [t.val, bud_clock]} 61 | t1 <= in_tbl2 {|t| [t.val, bud_clock]} 62 | end 63 | end 64 | 65 | class TestBudClock < Minitest::Test 66 | def test_bud_clock 67 | b = BudClockExample.new 68 | b.run_bg 69 | b.sync_do { 70 | b.in_tbl <+ [[5]] 71 | b.in_tbl2 <+ [[5]] 72 | } 73 | b.stop 74 | assert_equal(1, b.t1.length) 75 | end 76 | 77 | def test_bud_clock_outside_tick 78 | b = BudClockExample.new 79 | b.run_bg 80 | assert_raises(Bud::Error) do 81 | b.sync_do { 82 | puts "Current Bud clock: #{b.bud_clock}" 83 | } 84 | end 85 | b.stop 86 | end 87 | end 88 | -------------------------------------------------------------------------------- /test/tc_wc.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | 3 | class WordCount1 4 | include Bud 5 | 6 | attr_reader :pattern 7 | 8 | def initialize(pattern, *options) 9 | super(*options) 10 | @pattern = pattern 11 | end 12 | 13 | state do 14 | file_reader :txt, 'text/ulysses.txt' 15 | scratch :wc, [:word] => [:cnt] 16 | end 17 | 18 | bloom do 19 | wc <= txt.flat_map do |t| 20 | t.text.split.enum_for(:each_with_index).map {|w, i| [t.lineno, i, w]} 21 | end.rename(:loo, [:lineno, :wordno, :word]).group([:word], count) 22 | end 23 | end 24 | 25 | class TestWC1 < Minitest::Test 26 | def test_wc1 27 | program = WordCount1.new(/[Bb]loom/) 28 | program.tick 29 | assert_equal(23, program.wc[["yes"]].cnt) 30 | end 31 | end 32 | 33 | 34 | class WordCount2 35 | include Bud 36 | 37 | attr_reader :pattern 38 | 39 | def initialize(pattern, *options) 40 | super(*options) 41 | @pattern = pattern 42 | end 43 | 44 | state do 45 | file_reader :txt, 'text/ulysses.txt' 46 | scratch :words, [:lineno, :wordno] => [:word] 47 | scratch :wc, [:word] => [:cnt] 48 | end 49 | 50 | bloom do 51 | words <= txt.flat_map do |t| 52 | t.text.split.enum_for(:each_with_index).map {|w, i| [t.lineno, i, w]} 53 | end 54 | wc <= words.reduce(Hash.new) do |memo, t| 55 | memo[t.word] ||= 0 56 | memo[t.word] += 1 57 | memo 58 | end 59 | end 60 | end 61 | 62 | class TestWC2 < Minitest::Test 63 | def test_wc2 64 | program = WordCount2.new(/[Bb]loom/) 65 | program.tick 66 | assert_equal(23, program.wc[["yes"]].cnt) 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /test/tc_zookeeper.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | 3 | unless defined? Bud::HAVE_ZOOKEEPER 4 | puts "Skipping Zk test: no zookeeper Gem installed" 5 | raise 6 | end 7 | 8 | # Check whether ZK is running 9 | def zk_running? 10 | begin 11 | z = Zookeeper.new("localhost:2181") 12 | z.close 13 | return true 14 | rescue Exception 15 | return false 16 | end 17 | end 18 | 19 | unless zk_running? 20 | puts "Skipping Zk test: cannot connect to Zookeeper on localhost:2181" 21 | raise 22 | end 23 | 24 | ZK_ROOT = "/foo" 25 | 26 | class ZkMirror 27 | include Bud 28 | 29 | state do 30 | store :t1, :zookeeper, :path => ZK_ROOT, :addr => 'localhost:2181' 31 | table :dummy 32 | scratch :t1_is_empty 33 | end 34 | 35 | bootstrap do 36 | dummy << [1,1] 37 | end 38 | 39 | # XXX: This is a hack: we want t1_is_empty to have a tuple iff t1.empty? is 40 | # true 41 | bloom do 42 | t1_is_empty <= dummy {|t| t if t1.empty?} 43 | end 44 | end 45 | 46 | class TestZk < Minitest::Test 47 | def setup 48 | zk_delete(ZK_ROOT) 49 | end 50 | 51 | def zk_delete(path) 52 | z = Zookeeper.new("localhost:2181") 53 | zk_rm_r(z, path) 54 | z.close 55 | end 56 | 57 | def zk_rm_r(z, root) 58 | r = z.get_children(:path => root) 59 | return unless r[:stat].exists 60 | r[:children].each do |c| 61 | zk_rm_r(z, "#{root}/#{c}") 62 | end 63 | z.delete(:path => root) 64 | end 65 | 66 | def test_one_zk 67 | b = ZkMirror.new 68 | b.run_bg 69 | b.sync_do { 70 | assert_equal([], b.t1.to_a.sort) 71 | } 72 | 73 | tuples = [["xyz", "zzz"]] 74 | b.sync_callback(:t1, tuples, :t1) 75 | 76 | b.sync_do { 77 | assert_equal(tuples.sort, b.t1.to_a.sort) 78 | } 79 | b.stop 80 | end 81 | 82 | def test_mirror 83 | b1, b2 = ZkMirror.new, ZkMirror.new 84 | b1.run_bg 85 | b2.run_bg 86 | 87 | tuples = [["k1", "ggg"], ["k2", "ggg"]] 88 | q = Queue.new 89 | c = b2.register_callback(:t1) do |t| 90 | if t.length == tuples.length 91 | q.push(true) 92 | end 93 | end 94 | 95 | b1.sync_do { 96 | b1.t1 <~ tuples 97 | } 98 | 99 | q.pop 100 | b2.unregister_callback(c) 101 | 102 | b2.sync_do { 103 | assert_equal(tuples.sort, b2.t1.to_a.sort) 104 | } 105 | 106 | c = b2.register_callback(:t1) do |t| 107 | q.push(true) if t.length == 1 108 | end 109 | zk_delete(ZK_ROOT + "/k1") 110 | q.pop 111 | b2.unregister_callback(c) 112 | b2.sync_do { 113 | assert_equal([["k2", "ggg"]], b2.t1.to_a.sort) 114 | } 115 | 116 | c = b2.register_callback(:t1_is_empty) do |t| 117 | q.push(true) 118 | end 119 | zk_delete(ZK_ROOT + "/k2") 120 | q.pop 121 | b2.unregister_callback(c) 122 | 123 | b2.sync_do { 124 | assert_equal([], b2.t1.to_a.sort) 125 | } 126 | 127 | b1.stop 128 | b2.stop 129 | end 130 | 131 | def test_ephemeral 132 | b1, b2 = ZkMirror.new, ZkMirror.new 133 | b1.run_bg 134 | b2.run_bg 135 | 136 | b2.sync_do { 137 | b2.t1 <~ [["baz", "xyz"]] 138 | } 139 | 140 | q = Queue.new 141 | c = b2.register_callback(:t1) do |t| 142 | q.push(true) if t.length == 2 143 | end 144 | b1.sync_do { 145 | b1.t1 <~ [["foo", "bar", {:ephemeral => true}]] 146 | } 147 | q.pop 148 | b2.unregister_callback(c) 149 | 150 | b2.sync_do { 151 | assert_equal([["baz", "xyz"], ["foo", "bar"]], b2.t1.to_a.sort) 152 | } 153 | 154 | c = b2.register_callback(:t1) do |t| 155 | q.push(true) if t.length == 1 156 | end 157 | b1.stop 158 | q.pop 159 | b2.unregister_callback(c) 160 | b2.sync_do { 161 | assert_equal([["baz", "xyz"]], b2.t1.to_a.sort) 162 | } 163 | 164 | b2.stop 165 | end 166 | 167 | def test_sequence 168 | b = ZkMirror.new 169 | b.run_bg 170 | 171 | q = Queue.new 172 | c = b.register_callback(:t1) do |t| 173 | q.push(true) if t.length == 3 174 | end 175 | b.sync_do { b.t1 <~ [["a_", "kkk", {:sequence => true}]] } 176 | b.sync_do { b.t1 <~ [["b_", "kkk", {:sequence => true}]] } 177 | b.sync_do { b.t1 <~ [["c_", "kkk", {:sequence => true}]] } 178 | 179 | q.pop 180 | b.unregister_callback(c) 181 | 182 | b.sync_do { 183 | assert_equal([["a_0000000000", "kkk"], 184 | ["b_0000000001", "kkk"], 185 | ["c_0000000002", "kkk"]], 186 | b.t1.to_a.sort) 187 | } 188 | 189 | b.stop 190 | end 191 | end 192 | -------------------------------------------------------------------------------- /test/test_common.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | 3 | if ENV["COVERAGE"] 4 | require 'simplecov' 5 | SimpleCov.command_name 'minitest' 6 | SimpleCov.root '../' 7 | SimpleCov.start 8 | end 9 | 10 | # Prefer Bud from local source tree to any version in RubyGems 11 | $:.unshift File.join(File.dirname(__FILE__), "..", "lib") 12 | $:.unshift "." 13 | require 'bud' 14 | 15 | gem 'minitest' # Use the rubygems version of MT, not builtin (if on 1.9) 16 | require 'minitest/autorun' 17 | -------------------------------------------------------------------------------- /test/ts_bud.rb: -------------------------------------------------------------------------------- 1 | require './test_common' 2 | 3 | # In "quick mode", don't bother running some of the more expensive tests 4 | if ARGV.first and ARGV.first.downcase == "quick" 5 | $quick_mode = true 6 | end 7 | 8 | require 'tc_aggs' 9 | require 'tc_attr_rewrite' 10 | require 'tc_callback' 11 | require 'tc_channel' 12 | require 'tc_collections' 13 | require 'tc_dbm' 14 | require 'tc_delta' 15 | require 'tc_errors' 16 | require 'tc_execmodes' unless $quick_mode 17 | require 'tc_exists' 18 | require 'tc_halt' 19 | require 'tc_inheritance' 20 | require 'tc_interface' 21 | require 'tc_joins' 22 | require 'tc_labeling' 23 | require 'tc_lattice' 24 | require 'tc_mapvariants' 25 | require 'tc_meta' 26 | require 'tc_metrics' 27 | require 'tc_module' 28 | require 'tc_nest' 29 | require 'tc_new_executor' 30 | require 'tc_notin' 31 | require 'tc_rebl' 32 | require 'tc_schemafree' 33 | require 'tc_sort' 34 | require 'tc_temp' 35 | require 'tc_terminal' 36 | require 'tc_timer' 37 | require 'tc_wc' 38 | --------------------------------------------------------------------------------