├── LICENSE ├── README.rst ├── compile.sh ├── create_node.sh ├── delete_domain.sh ├── doc ├── html4css1.css ├── intro.txt ├── make_html.sh └── ringodoc.txt ├── list_domains.sh ├── ring ├── bfile │ ├── LICENCE │ ├── Makefile │ ├── README │ ├── c_src │ │ ├── FILE_drv.c │ │ └── Makefile │ ├── config │ │ ├── Makefile │ │ ├── acconfig.h │ │ ├── aclocal.m4 │ │ ├── config.guess │ │ ├── config.h.in │ │ ├── config.sub │ │ ├── configure.in │ │ ├── include.mk.in │ │ └── install-sh │ ├── ebin │ │ └── a │ ├── priv │ │ └── a │ ├── src │ │ ├── ._bfile.app.src │ │ ├── Makefile │ │ ├── bfile.app.src │ │ └── bfile.erl │ └── vsn.mk ├── compile.sh ├── ebin │ └── ringo.app ├── ringo_debug.sh ├── ringocmd.sh ├── src │ ├── bin_util.erl │ ├── lrucache.erl │ ├── make_boot.erl │ ├── ringo_debug.erl │ ├── ringo_domain.erl │ ├── ringo_external.erl │ ├── ringo_index.erl │ ├── ringo_indexdomain.erl │ ├── ringo_main.erl │ ├── ringo_node.erl │ ├── ringo_node.hrl │ ├── ringo_reader.erl │ ├── ringo_store.hrl │ ├── ringo_sync.erl │ ├── ringo_syncdomain.erl │ ├── ringo_util.erl │ ├── ringo_writer.erl │ └── trunc_io.erl ├── start_ringo.sh ├── test-system.sh ├── test.sh └── test │ ├── ringo_cmd.erl │ ├── system_test.py │ ├── test_index.erl │ ├── test_readwrite.erl │ └── test_sync.erl ├── ringogw ├── compile.sh ├── ebin │ └── ringogw.app ├── lighttpd.conf ├── py │ ├── ringo_reader.py │ ├── ringodisco.py │ └── ringogw.py ├── src │ ├── handle_chunkstat.erl │ ├── handle_data.erl │ ├── handle_domains.erl │ ├── handle_ring.erl │ ├── json.erl │ ├── make_boot.erl │ ├── mochi_dispatch.erl │ ├── ringogw.erl │ ├── ringogw_util.erl │ ├── scgi.erl │ ├── scgi_server.erl │ └── trunc_io.erl ├── start_ringogw.sh └── web │ ├── index.html │ ├── jquery-1.2.1.js │ ├── jquery-dom.js │ ├── reset-fonts-grids.css │ ├── ringomon.js │ ├── style.css │ └── typewatch1.1.js └── start_nodes.sh /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Copyright (c) 2008, Nokia Corporation. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are 7 | met: 8 | 9 | * Redistributions of source code must retain the above copyright 10 | notice, this list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright 13 | notice, this list of conditions and the following disclaimer in the 14 | documentation and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the author nor the names of its contributors 17 | may be used to endorse or promote products derived from this 18 | software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 26 | TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 27 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 28 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 29 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 30 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | 2 | Ringo - Distributed key/value storage for immutable data 3 | ======================================================== 4 | 5 | Ringo is an experimental, distributed, replicating key-value store based 6 | on consistent hashing and immutable data. Unlike many general-purpose 7 | databases, Ringo is designed for a specific use case: For archiving 8 | small (less than 4KB) or medium-size data items (<100MB) in real-time 9 | so that the data can survive K - 1 disk breaks, where K is the desired 10 | number of replicas, without any downtime, in a manner that scales to 11 | terabytes of data. In addition to storing, Ringo should be able to 12 | retrieve individual or small sets of data items with low latencies 13 | (<10ms) and provide a convenient on-disk format for bulk data access. 14 | 15 | WARNING: Ringo should not be used yet as a primary storage for critical 16 | data. Due to the fact that Ringo treats all data immutable, data 17 | corruption or loss should be improbable. However, Ringo needs more 18 | testing in real-world settings before we can be reasonably sure that it 19 | works as advertised. To make this happen, feel free to try it out, adapt 20 | it to your needs, and report your experiences! 21 | 22 | If you find Ringo interesting, you might also want to check out a paper 23 | about Amazon's Dynamo 24 | 25 | http://s3.amazonaws.com/AllThingsDistributed/sosp/amazon-dynamo-sosp2007.pdf 26 | 27 | and another Dynamo-like system, Scalaris, which is also implemented in 28 | Erlang: http://code.google.com/p/scalaris/ 29 | 30 | 31 | Requirements 32 | '''''''''''' 33 | 34 | Backend system (required): 35 | 36 | - Erlang R12B or newer 37 | - C toolchain and autotools (for building bfile) 38 | - Lighttpd or other web server that supports SCGI 39 | 40 | Test harness / Python interface (optional): 41 | 42 | - pkill command (Debian package procps) 43 | - Python 44 | - Python module pycurl (Debian package python-pycurl) 45 | - Python module cjson (Debian package python-cjson) 46 | 47 | 48 | Directories 49 | ''''''''''' 50 | 51 | doc/ Documentation 52 | 53 | ring/ Ringo backend 54 | ring/bfile High-performance replacement for the Erlang's standard file 55 | module 56 | ring/src Backend sources 57 | ring/test Test harness 58 | 59 | ringogw/ Ringo web frontend 60 | ringogw/src Frontend sources 61 | ringogw/web Web interface 62 | ringogw/py Python interfaces for the Ringo frontend and for the Disco 63 | map/reduce framework 64 | 65 | 66 | Compiling 67 | ''''''''' 68 | 69 | cd ringo 70 | ./compile.sh 71 | 72 | 73 | Starting Ringo 74 | '''''''''''''' 75 | 76 | First you need to initialize a number of virtual nodes - at least one is 77 | required. A virtual node is defined by an empty directory whose name is 78 | a random 128-bit integer in hexadecimal notation. 79 | 80 | A convenience script, create_node.sh, is provided that can be used to 81 | setup a virtual node. For instance, 82 | 83 | ringo/create_node.sh trurl /data/ringo 84 | 85 | sets up a new virtual node on the host trurl to the directory 86 | /data/ringo. The script uses ssh to log in to the host. It asks for a 87 | password unless key-based ssh authentication is properly set up (which 88 | is recommended). You may run this script on different nodes as many 89 | times as you like. 90 | 91 | You need to list all hostnames that may possibly host a Ringo node 92 | in a file called ~/.hosts.erlang. For further information about this 93 | file, see an Erlang manual page at "man 3erl net_adm". For instance the 94 | following command creates the required file for the virtual node that 95 | was initialized above: 96 | 97 | echo "'trurl'." > ~/.hosts.erlang 98 | 99 | After a number of virtual nodes have been initialized, Ringo may be 100 | started. Again, a simple script is provided that starts up all virtual 101 | nodes on a specified host. For instance, 102 | 103 | ringo/start_nodes.sh trurl /data/ringo 104 | 105 | starts up all nodes on the host trurl. After a while, the ring should 106 | be up and running. 107 | 108 | The web frontend provides a convenient way to monitor status of the 109 | system. An example configuration file for the Lighttpd web server is 110 | provided at ringo/ringogw/lighttpd.conf that communicates with the 111 | frontend process over SCGI. The following script starts up the 112 | web server and the frontend process: 113 | 114 | ringo/ringogw/start_ringogw.sh 115 | 116 | Now you should see the status page at http://localhost:15000. On the 117 | status page, you can click nodes on the leftmost panel to see domains 118 | that they contain. You can click a domain on the middle panel to see its 119 | replicas. By clicking a replica, you can see its status. You might need 120 | to wait for 10 seconds or so, and reload the page, to see new nodes and 121 | domains appear. 122 | 123 | 124 | Usage 125 | ''''' 126 | 127 | You can create domain, put keys and get keys from Ringo using simple 128 | HTTP requests. Assuming that you have started a ring as instructed 129 | above, you can create a new domain called "foobar" with the following 130 | POST request. Here curl is used to make a request but any other HTTP 131 | client would work as well: 132 | 133 | curl -d "" http://localhost:15000/mon/data/foobar?create 134 | 135 | You can put a new key-value pair to the domain with following POST 136 | request: 137 | 138 | curl -d "testvalue" http://localhost:15000/mon/data/foobar/testkey 139 | 140 | and retrieve the value given the key with a GET request as follows: 141 | 142 | curl http://localhost:15000/mon/data/foobar/testkey 143 | 144 | this returns all values assigned with the key "testkey". If only one 145 | value is required, the parameter ?single can be used: 146 | 147 | curl http://localhost:15000/mon/data/foobar/testkey?single 148 | 149 | A Python class Ringo is provided at ringo/ringogw/py/ringogw.py that 150 | encapsulates the above HTTP requests in Python function calls. 151 | 152 | An experimental interface for Disco, an an open-source implementation 153 | of the Map/Reduce framework (http://discoproject.org), can be found at 154 | ringo/ringogw/py/ringodisco.py. This function, which implements the 155 | Disco's map reader interface, makes it possible to use data stored in 156 | Ringo as input to a Disco job. The function provides a particularly 157 | efficient way of accessing data directly from Ringo's live data files. 158 | 159 | 160 | Running the Test Harness 161 | '''''''''''''''''''''''' 162 | 163 | Ringo comes with a set of tests that cover all main features of the 164 | system. You need to start the ringogw web frontend process to run 165 | the tests, as shown above. After the frontend has started, you can 166 | run the tests with the following command: 167 | 168 | cd ringo/ring/; ./test-system.sh 169 | 170 | 171 | Contact information 172 | ''''''''''''''''''' 173 | 174 | Bug reports, patches, comments etc. are welcome! Contact person is 175 | Ville Tuulos who can be reached at 176 | 177 | ville.h.tuulos -a- nokia.com 178 | 179 | or on the IRC channel #discoproject at Freenode. 180 | 181 | 182 | 183 | .. image:: https://d2weczhvl823v0.cloudfront.net/tuulos/ringo/trend.png 184 | :alt: Bitdeli badge 185 | :target: https://bitdeli.com/free 186 | 187 | -------------------------------------------------------------------------------- /compile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | echo "Compiling backend.." 4 | echo "++ compiling bfile module.." 5 | (cd ring/bfile; make) 6 | echo "++ compiling Ringo.." 7 | (cd ring/; ./compile.sh) 8 | echo "Compiling frontend.." 9 | (cd ringogw/; ./compile.sh) 10 | echo "All ok!" 11 | -------------------------------------------------------------------------------- /create_node.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z $1 ]; then 4 | echo "Usage: create_node.sh [host] [ringo_data]" 5 | exit 1 6 | fi 7 | 8 | node=`dd if=/dev/urandom bs=1024 count=1 2>/dev/null | md5sum | cut -d ' ' -f 1` 9 | ssh $1 "mkdir -p $2/$node" 10 | 11 | -------------------------------------------------------------------------------- /delete_domain.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z $1 ]; then 4 | echo "Usage: delete_domain.sh [node] [ringo_data] [domainid]" 5 | exit 1 6 | fi 7 | 8 | ssh $1 "find $2 -iname 'rdomain-$3' -exec rm -Rf \{\} \;" 9 | -------------------------------------------------------------------------------- /doc/html4css1.css: -------------------------------------------------------------------------------- 1 | /* 2 | :Author: David Goodger 3 | :Contact: goodger@users.sourceforge.net 4 | :Date: $Date: 2005-12-18 01:56:14 +0100 (Sun, 18 Dec 2005) $ 5 | :Revision: $Revision: 4224 $ 6 | :Copyright: This stylesheet has been placed in the public domain. 7 | 8 | Default cascading style sheet for the HTML output of Docutils. 9 | 10 | See http://docutils.sf.net/docs/howto/html-stylesheets.html for how to 11 | customize this style sheet. 12 | */ 13 | 14 | body{ 15 | text-align: center; 16 | color: #000000; 17 | font-family: Verdana, "Bitstream Vera Sans", sans-serif; 18 | line-height: 1.4em; 19 | } 20 | 21 | .document{ 22 | width: 750px; 23 | margin-left: auto; 24 | margin-right: auto; 25 | text-align: justify; 26 | } 27 | 28 | /* used to remove borders from tables and images */ 29 | .borderless, table.borderless td, table.borderless th { 30 | border: 0 } 31 | 32 | table.borderless td, table.borderless th { 33 | /* Override padding for "table.docutils td" with "! important". 34 | The right padding separates the table cells. */ 35 | padding: 0 0.5em 0 0 ! important } 36 | 37 | .first { 38 | /* Override more specific margin styles with "! important". */ 39 | margin-top: 0 ! important } 40 | 41 | .last, .with-subtitle { 42 | margin-bottom: 0 ! important } 43 | 44 | .hidden { 45 | display: none } 46 | 47 | a.toc-backref { 48 | text-decoration: none ; 49 | color: black } 50 | 51 | blockquote.epigraph { 52 | margin: 2em 5em ; } 53 | 54 | dl.docutils dd { 55 | margin-bottom: 0.5em } 56 | 57 | /* Uncomment (and remove this text!) to get bold-faced definition list terms 58 | dl.docutils dt { 59 | font-weight: bold } 60 | */ 61 | 62 | div.abstract { 63 | margin: 2em 5em } 64 | 65 | div.abstract p.topic-title { 66 | font-weight: bold ; 67 | text-align: center } 68 | 69 | div.admonition, div.attention, div.caution, div.danger, div.error, 70 | div.hint, div.important, div.note, div.tip, div.warning { 71 | margin: 2em ; 72 | border: medium outset ; 73 | padding: 1em } 74 | 75 | div.admonition p.admonition-title, div.hint p.admonition-title, 76 | div.important p.admonition-title, div.note p.admonition-title, 77 | div.tip p.admonition-title { 78 | font-weight: bold ; 79 | font-family: sans-serif } 80 | 81 | div.attention p.admonition-title, div.caution p.admonition-title, 82 | div.danger p.admonition-title, div.error p.admonition-title, 83 | div.warning p.admonition-title { 84 | color: red ; 85 | font-weight: bold ; 86 | font-family: sans-serif } 87 | 88 | /* Uncomment (and remove this text!) to get reduced vertical space in 89 | compound paragraphs. 90 | div.compound .compound-first, div.compound .compound-middle { 91 | margin-bottom: 0.5em } 92 | 93 | div.compound .compound-last, div.compound .compound-middle { 94 | margin-top: 0.5em } 95 | */ 96 | 97 | div.dedication { 98 | margin: 2em 5em ; 99 | text-align: center ; 100 | font-style: italic } 101 | 102 | div.dedication p.topic-title { 103 | font-weight: bold ; 104 | font-style: normal } 105 | 106 | div.figure { 107 | margin-left: 2em ; 108 | margin-right: 2em } 109 | 110 | div.footer, div.header { 111 | clear: both; 112 | font-size: smaller } 113 | 114 | div.line-block { 115 | display: block ; 116 | margin-top: 1em ; 117 | margin-bottom: 1em } 118 | 119 | div.line-block div.line-block { 120 | margin-top: 0 ; 121 | margin-bottom: 0 ; 122 | margin-left: 1.5em } 123 | 124 | div.sidebar { 125 | margin-left: 1em ; 126 | border: medium outset ; 127 | padding: 1em ; 128 | background-color: #ffffee ; 129 | width: 40% ; 130 | float: right ; 131 | clear: right } 132 | 133 | div.sidebar p.rubric { 134 | font-family: sans-serif ; 135 | font-size: medium } 136 | 137 | div.system-messages { 138 | margin: 5em } 139 | 140 | div.system-messages h1 { 141 | color: red } 142 | 143 | div.system-message { 144 | border: medium outset ; 145 | padding: 1em } 146 | 147 | div.system-message p.system-message-title { 148 | color: red ; 149 | font-weight: bold } 150 | 151 | div.topic { 152 | margin: 2em } 153 | 154 | h1 { font-size: 120%; color: #333;} 155 | 156 | h1.section-subtitle, h2.section-subtitle, h3.section-subtitle, 157 | h4.section-subtitle, h5.section-subtitle, h6.section-subtitle { 158 | margin-top: 0.4em } 159 | 160 | h1.title { margin-bottom: 2em; margin-top: 2em; color: #8f138b;} 161 | 162 | .section h1 { font-size: 100%; color: #8f138b;} 163 | .fn-backref, .citation-reference, .reference { color: #488f13; } 164 | 165 | 166 | h2.subtitle { 167 | text-align: center } 168 | 169 | hr.docutils { 170 | width: 75% } 171 | 172 | img.align-left { 173 | clear: left } 174 | 175 | img.align-right { 176 | clear: right } 177 | 178 | ol.simple, ul.simple { 179 | margin-bottom: 1em } 180 | 181 | ol.arabic { 182 | list-style: decimal } 183 | 184 | ol.loweralpha { 185 | list-style: lower-alpha } 186 | 187 | ol.upperalpha { 188 | list-style: upper-alpha } 189 | 190 | ol.lowerroman { 191 | list-style: lower-roman } 192 | 193 | ol.upperroman { 194 | list-style: upper-roman } 195 | 196 | p.attribution { 197 | text-align: right ; 198 | margin-left: 50% } 199 | 200 | p.caption { 201 | font-style: italic } 202 | 203 | p.credits { 204 | font-style: italic ; 205 | font-size: smaller } 206 | 207 | p.label { 208 | white-space: nowrap } 209 | 210 | p.rubric { 211 | font-weight: bold ; 212 | font-size: larger ; 213 | color: maroon ; 214 | text-align: center } 215 | 216 | p.sidebar-title { 217 | font-family: sans-serif ; 218 | font-weight: bold ; 219 | font-size: larger } 220 | 221 | p.sidebar-subtitle { 222 | font-family: sans-serif ; 223 | font-weight: bold } 224 | 225 | p.topic-title { 226 | font-weight: bold } 227 | 228 | pre.address { 229 | margin-bottom: 0 ; 230 | margin-top: 0 ; 231 | font-family: serif ; 232 | font-size: 100% } 233 | 234 | pre.literal-block, pre.doctest-block { 235 | margin-left: 2em ; 236 | margin-right: 2em ; 237 | line-height: 1em; 238 | background-color: #e4ecff } 239 | 240 | span.classifier { 241 | font-family: sans-serif ; 242 | font-style: oblique } 243 | 244 | span.classifier-delimiter { 245 | font-family: sans-serif ; 246 | font-weight: bold } 247 | 248 | span.interpreted { 249 | font-family: sans-serif } 250 | 251 | span.option { 252 | white-space: nowrap } 253 | 254 | span.pre { 255 | white-space: pre } 256 | 257 | span.problematic { 258 | color: red } 259 | 260 | span.section-subtitle { 261 | /* font-size relative to parent (h1..h6 element) */ 262 | font-size: 80% } 263 | 264 | table.citation { 265 | border-left: solid 1px gray; 266 | margin-left: 1px } 267 | 268 | table.docinfo { 269 | margin: 2em 4em } 270 | 271 | table.docutils { 272 | margin-top: 0.5em ; 273 | margin-bottom: 0.5em } 274 | 275 | table.footnote { 276 | border-left: solid 1px black; 277 | margin-left: 1px } 278 | 279 | table.docutils td, table.docutils th, 280 | table.docinfo td, table.docinfo th { 281 | padding-left: 0.5em ; 282 | padding-right: 0.5em ; 283 | vertical-align: top } 284 | 285 | table.docutils th.field-name, table.docinfo th.docinfo-name { 286 | font-weight: bold ; 287 | text-align: left ; 288 | white-space: nowrap ; 289 | padding-left: 0 } 290 | 291 | h1 tt.docutils, h2 tt.docutils, h3 tt.docutils, 292 | h4 tt.docutils, h5 tt.docutils, h6 tt.docutils { 293 | font-size: 100% } 294 | 295 | tt.docutils { 296 | background-color: #fffce4 } 297 | 298 | ul.auto-toc { 299 | list-style-type: none } 300 | -------------------------------------------------------------------------------- /doc/intro.txt: -------------------------------------------------------------------------------- 1 | 2 | Ringo 3 | ===== 4 | 5 | Ringo is an experimental, distributed, replicating key-value store based 6 | on consistent hashing and immutable data. Unlike many general-purpose 7 | databases, Ringo is designed for a specific use case: For archiving 8 | small (less than 4KB) or medium-size data items (<100MB) in real-time 9 | so that the data can survive K - 1 disk breaks, where K is the desired 10 | number of replicas, without any downtime, in a manner that scales to 11 | terabytes of data. In addition to storing, Ringo should be able to 12 | retrieve individual or small sets of data items with low latencies 13 | (<10ms) and provide a convenient on-disk format for bulk data access. 14 | 15 | Ringo is implemented in Erlang [Erlang]_, a functional language that is designed 16 | for building robust fault-tolerant distributed applications. 17 | 18 | API 19 | --- 20 | 21 | Ringo supports the following operations on data, using a REST-style 22 | web interface. 23 | 24 | - Create a domain, which initializes a new set of items using a POST 25 | request: 26 | 27 | http://ringo/mon/data/domain_name?create 28 | 29 | - Put an item to a domain, which appends a new key-value pair to an 30 | existing domain with a POST request. The address encodes the key and 31 | the request contains the corresponding value: 32 | 33 | http://ringo/mon/data/domain_name/key 34 | 35 | - Get items from a domain, which returns all values for the given key 36 | from the given domain with a GET request: 37 | 38 | http://ringo/mon/data/domain_name/key 39 | 40 | Note that the above operations can only add new items, or access existing 41 | items in the system but never modify them. Furthermore, Ringo doesn't 42 | change or move data internally in any way once it has been written to 43 | disk. This should guarantee that Ringo never corrupts data. Even if the 44 | local filesystem corrupts, which Ringo uses to store its data, the data 45 | can be recovered from redundant copies on replica nodes. 46 | 47 | Distributed data storage 48 | ------------------------ 49 | 50 | Ringo is designed to work on a cluster of servers. It is based on 51 | consistent hashing, which connects many independent processes on many 52 | independent servers to a single consistent system. It is possible 53 | to add and remove servers from the system on the fly without any 54 | interruptions. This eases maintenance of the system and makes it 55 | fault-tolerant, as there aren't any single points of failure. As the 56 | system lacks a central bookkeeping mechanism, and it doesn't rely on any 57 | global data structures, Ringo is inherently scalable. Simple chunking 58 | mechanism takes care of load balancing and data distribution. 59 | 60 | Ringo shares many characteristics with Amazon Dynamo [Dynamo]_, Amazon's 61 | internal, non-public storage system. However, whereas Dynamo is designed 62 | for the needs of real-time web services, Ringo is leaned towards long-term 63 | data archival and analysis, where data may originate from web or mobile 64 | services in real-time. 65 | 66 | Efficient bulk data access 67 | -------------------------- 68 | 69 | Ringo supports efficient bulk data access to the data through a direct, 70 | lockless access to the DB files. An interface to the raw data is 71 | provided for Disco [Disco]_, an implementation of the Map-Reduce 72 | framework for distributed computing, which makes it possible to process 73 | and analyze large-amounts of data stored in Ringo in an efficient 74 | manner. Ringo is thus especially suitable for Web services which receive 75 | seldomly changing data from users, and which need to generate reports, 76 | or to build models or indices periodically based on the user data or log 77 | files. 78 | 79 | At this point, Ringo is an experimental system and not suitable for 80 | production use as such. The largest Ringo deployment this far runs on 81 | 30 servers, each having two 500GB hard disks, in a cluster maintained by 82 | the Data Insight team in NRC Palo Alto. 83 | 84 | Main features 85 | ------------- 86 | 87 | - A clean REST-style Web interface to the data. 88 | 89 | - Fault-tolerant: Supports K-way redundancy. 90 | 91 | - Efficient data synchronization via hash trees. 92 | 93 | - Scales to terabytes of data and hundreds of individual disks. 94 | 95 | - Maintains inverted indices for all keys for fast access to data. 96 | 97 | - Disks and servers may be added to or removed from the system in a 98 | straightforward manner, without any interruptions. 99 | 100 | - Raw data can be read efficienly through direct, lockless access to 101 | the DB files. 102 | 103 | - Disco interface provided for DB files, for distributed data processing 104 | and analysis. 105 | 106 | - Separate Web APIs and a Web UI are provided for system monitoring and 107 | control. 108 | 109 | 110 | References 111 | ---------- 112 | 113 | .. [Erlang] http://www.erlang.org/ 114 | .. [Dynamo] http://s3.amazonaws.com/AllThingsDistributed/sosp/amazon-dynamo-sosp2007.pdf 115 | .. [Disco] file://disco-intro.html 116 | 117 | Contact person: ville.h.tuulos@nokia.com 118 | -------------------------------------------------------------------------------- /doc/make_html.sh: -------------------------------------------------------------------------------- 1 | rst2html --stylesheet-path=html4css1.css ringodoc.txt ringodoc.html 2 | rst2html --stylesheet-path=html4css1.css intro.txt intro.html 3 | -------------------------------------------------------------------------------- /list_domains.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z $1 ]; then 4 | echo "Usage: list_domains.sh [node] [ringo_data]" 5 | exit 1 6 | fi 7 | 8 | for f in `ssh $1 find $2 -iname 'rdomain-*'`; do 9 | T=`basename $f` 10 | echo ${T:8} 11 | done 12 | -------------------------------------------------------------------------------- /ring/bfile/LICENCE: -------------------------------------------------------------------------------- 1 | 2 | Copyright (c) 2006, Claes Wikstrom, klacke@hyber.org 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | * Neither the name of "bfile" nor the names of its contributors may be 14 | used to endorse or promote products derived from this software without 15 | specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /ring/bfile/Makefile: -------------------------------------------------------------------------------- 1 | 2 | 3 | all: 4 | (cd config;$(MAKE)) 5 | (cd src;$(MAKE)) 6 | -(cd c_src;$(MAKE) -k) 7 | $(MAKE) appfile 8 | 9 | clean: 10 | (cd src;$(MAKE) clean) 11 | (cd c_src;$(MAKE) clean) 12 | (cd config; $(MAKE) clean) 13 | 14 | 15 | install: all 16 | (cd c_src; $(MAKE) install) 17 | 18 | conf_clean: 19 | (cd config; $(MAKE) clean) 20 | 21 | appfile: 22 | (cd src;$(MAKE) ../ebin/bfile.app) 23 | 24 | -------------------------------------------------------------------------------- /ring/bfile/README: -------------------------------------------------------------------------------- 1 | An interface to fast FILE I/O 2 | 3 | It's based on an old and hacked version 4 | of the BSD FILE* 5 | 6 | To install, type make; make install 7 | and it shuld install itself as an app in your 8 | erlang dir. 9 | 10 | See the source src/bfile.erl for API 11 | 12 | Here's an example shell session: 13 | 14 | 15 | 2> bfile:load_driver(). 16 | ok 17 | 4> {ok, Fd} = bfile:fopen("Makefile", "r"). 18 | {ok,{bfile,#Port<0.98>}} 19 | 5> bfile:fgets(Fd). 20 | {line,<<10>>} 21 | 6> bfile:fgets(Fd). 22 | {line,<<10>>} 23 | 7> bfile:fgets(Fd). 24 | {line,<<97,108,108,58,32,10>>} 25 | 14> bfile:fread(Fd, 10000). 26 | {ok,<<10,10,105,110,115,116,97,108,108,58,32,97,108,108,10,9,40,99,100,32,99,95,115,114,99,59,32,...>>} 27 | 15> bfile:fread(Fd, 10000). 28 | eof 29 | -------------------------------------------------------------------------------- /ring/bfile/c_src/Makefile: -------------------------------------------------------------------------------- 1 | 2 | include ../config/include.mk 3 | 4 | ## don't build this under win32 at all 5 | ifdef WIN32 6 | 7 | PRIV_FILES = 8 | 9 | else 10 | 11 | PRIV_FILES=../priv/FILE_drv.so 12 | 13 | endif 14 | 15 | 16 | CFLAGS += -I$(ERL_C_INCLUDE_DIR) -I../config -I. 17 | 18 | # 19 | # Targets 20 | # 21 | 22 | all: $(PRIV_FILES) 23 | 24 | clean: 25 | -rm -f $(PRIV_FILES) FILE_drv.o 26 | 27 | 28 | install: 29 | install -d $(ERLDIR)/lib/bfile 30 | cp -r `pwd` $(ERLDIR)/lib/bfile 31 | cp -r `pwd`/../ebin $(ERLDIR)/lib/bfile 32 | cp -r `pwd`/../priv $(ERLDIR)/lib/bfile 33 | 34 | 35 | ../priv/FILE_drv.so: FILE_drv.o 36 | $(LD_SHARED) -o $@ FILE_drv.o $(LIBS) 37 | 38 | 39 | FILE_drv.o: FILE_drv.c 40 | $(CC) -o $@ -c -fpic $(CFLAGS) -DDYNAMIC_DRIVER FILE_drv.c 41 | 42 | 43 | -------------------------------------------------------------------------------- /ring/bfile/config/Makefile: -------------------------------------------------------------------------------- 1 | MK_INCLUDE=include.mk 2 | CONFIG_H=config.h 3 | 4 | all: config.status $(MK_INCLUDE) $(CONFIG_H) 5 | 6 | config.status: configure 7 | ./configure 8 | 9 | $(CONFIG_H) $(MK_INCLUDE): config.status include.mk.in config.h.in 10 | ./config.status 11 | 12 | configure: configure.in 13 | autoheader 14 | autoconf 15 | 16 | clean: 17 | -rm -f config.cache config.log config.status configure \ 18 | $(MK_INCLUDE) $(CONFIG_H) 19 | -rm -rf autom4te.cache 20 | 21 | 22 | -------------------------------------------------------------------------------- /ring/bfile/config/acconfig.h: -------------------------------------------------------------------------------- 1 | #ifndef _CONFIG_H_ 2 | #define _CONFIG_H_ 3 | 4 | #ifndef WIN32 5 | 6 | @TOP@ 7 | 8 | /* 9 | * This file contains prototypes for config.h.in which autoheader 10 | * could not figure by itself. I.e. if you write your own test which 11 | * defines a macro you will probably have to put it here, but if you 12 | * use any standard test AC_* it will be detected by autoheader. 13 | */ 14 | 15 | #undef WIN32 16 | 17 | /* Define to a string defining your target. ($target in configure.in) */ 18 | #undef CPU_VENDOR_OS 19 | 20 | 21 | /* Define to the full path of the ifconfig program */ 22 | #undef IFCONFIG 23 | 24 | /* Define to the full path of the route program */ 25 | #undef ROUTE 26 | 27 | /* Define to the full path of the arp program */ 28 | #undef ARP 29 | 30 | 31 | /* 32 | * These four below should definately be done away with!! 33 | */ 34 | 35 | /* Define if your target OS is a BSD derivative */ 36 | #undef BSD 37 | 38 | /* Define if your target OS is Linux */ 39 | #undef LINUX 40 | 41 | /* Define if your target OS is SunOS 5.x */ 42 | #undef SOLARIS 43 | 44 | /* Define if your target OS is BSD/OS 4.x */ 45 | #undef BSDI 46 | 47 | 48 | /* 49 | * What are these??? 50 | */ 51 | 52 | /* ? */ 53 | #undef USE_IFALIAS 54 | 55 | /* Define if you wish to use the bpf interface */ 56 | #undef USE_BPF 57 | 58 | /* Define if you wish to use the dlpi interface */ 59 | #undef USE_DLPI 60 | 61 | /* ? */ 62 | #undef USE_SOCKET 63 | 64 | 65 | /* Define if prototypes for malloc can be found in stdlib.h */ 66 | #undef STDLIB_MALLOC 67 | 68 | /* Define if your include files defines a prototype for sys_errlist[] */ 69 | #undef HAVE_SYS_ERRLIST 70 | 71 | /* This isn't used anywhere (that I could find) so I don't know what it means*/ 72 | #undef IFCONFIG_REQUIRES_DOWN_ADDRESS 73 | 74 | /* Define if your OS have broken cmsg fields in the msghdr struct (Linux) */ 75 | #undef BROKEN_CMSG_FIELDS 76 | 77 | /* Define if sockaddr structure has sa_len member */ 78 | #undef SA_LEN_IN_SOCKADDR 79 | 80 | 81 | 82 | /* 83 | * Provide a common way to refer to ints with specific size. (Is this 84 | * used everywhere?) The names used are {u_,}int{8,16,32,64}_t unless 85 | * they are defined in sys/types.h they are defined in ints.h using 86 | * the BIT macros. e.g. if int8_t isn't defined in sys/types.h int8_t 87 | * is typedef:ed to BIT8. 88 | * 89 | */ 90 | 91 | /* Define to a basic signed type that is 8 bits in size */ 92 | #undef BIT8 93 | 94 | /* Define to a basic signed type that is 16 bits in size */ 95 | #undef BIT16 96 | 97 | /* Define to a basic signed type that is 32 bits in size */ 98 | #undef BIT32 99 | 100 | /* Define to a basic signed type that is 64 bits in size */ 101 | #undef BIT64 102 | 103 | /* Define if sys/types.h defines this type */ 104 | #undef HAVE_int8_t 105 | 106 | /* Define if sys/types.h defines this type */ 107 | #undef HAVE_u_int8_t 108 | 109 | /* Define if sys/types.h defines this type */ 110 | #undef HAVE_int16_t 111 | 112 | /* Define if sys/types.h defines this type */ 113 | #undef HAVE_u_int16_t 114 | 115 | /* Define if sys/types.h defines this type */ 116 | #undef HAVE_int32_t 117 | 118 | /* Define if sys/types.h defines this type */ 119 | #undef HAVE_u_int32_t 120 | 121 | /* Define if sys/types.h defines this type */ 122 | #undef HAVE_int64_t 123 | 124 | /* Define if sys/types.h defines this type */ 125 | #undef HAVE_u_int64_t 126 | 127 | 128 | 129 | 130 | 131 | 132 | /* */ 133 | #undef ETHER_HEADER_USES_ETHER_ADDR 134 | 135 | /* */ 136 | #undef HAVE_DLIOCRAW 137 | 138 | /* */ 139 | #undef HAVE_ETHERADDRL 140 | 141 | /* */ 142 | #undef HAVE_ETHER_ADDR_LEN 143 | 144 | /* */ 145 | #undef HAVE_MSGHDR_MSG_CONTROL 146 | 147 | /* */ 148 | #undef HAVE_SIOCGARP 149 | 150 | /* */ 151 | #undef HAVE_SIOCGIFCONF 152 | 153 | /* */ 154 | #undef HAVE_SIOCGIFHWADDR 155 | 156 | /* */ 157 | #undef HAVE_arpreq 158 | 159 | /* */ 160 | #undef HAVE_caddr_t 161 | 162 | /* */ 163 | #undef HAVE_ether_header 164 | 165 | /* */ 166 | #undef HAVE_ethhdr 167 | 168 | /* */ 169 | #undef HAVE_ifnet 170 | 171 | /* */ 172 | #undef HAVE_in_addr 173 | 174 | /* */ 175 | #undef HAVE_sockaddr 176 | 177 | /* */ 178 | #undef HAVE_sockaddr_dl 179 | 180 | /* */ 181 | #undef HAVE_sockaddr_in 182 | 183 | /* */ 184 | #undef NEED_HAVE_sockaddr_dl 185 | 186 | 187 | 188 | /* 189 | * I have yet to figure out what all this need_* stuff is for, shouldn't 190 | * it suffice with using have_* ??? 191 | */ 192 | 193 | /* */ 194 | #undef NEED_LINUX_SOCKIOS_H 195 | 196 | /* */ 197 | #undef NEED_NETINET_IF_ETHER_H 198 | 199 | /* */ 200 | #undef NEED_NETINET_IN_H 201 | 202 | /* */ 203 | #undef NEED_NET_ETHERNET_H 204 | 205 | /* */ 206 | #undef NEED_NET_IF_ARP_H 207 | 208 | /* */ 209 | #undef NEED_NET_IF_DL_H 210 | 211 | /* */ 212 | #undef NEED_NET_IF_H 213 | 214 | /* */ 215 | #undef NEED_SYS_BITYPES_H 216 | 217 | /* */ 218 | #undef NEED_SYS_DLPI_H 219 | 220 | /* */ 221 | #undef NEED_SYS_ETHERNET_H 222 | 223 | /* */ 224 | #undef NEED_SYS_SOCKETIO_H 225 | 226 | /* */ 227 | #undef NEED_SYS_SOCKET_H 228 | 229 | /* */ 230 | #undef NEED_SYS_SOCKIO_H 231 | 232 | /* */ 233 | #undef NEED_SYS_TYPES_H 234 | 235 | /* if we have the openssl with engine support */ 236 | #undef HAVE_SSL_ENGINE 237 | /* ... with Rainbow patches */ 238 | #undef HAVE_RAINBOW_PATCHES 239 | /* ... with Rainbow's libswift */ 240 | #undef HAVE_SWIFT 241 | /* ... with one of our HW checks */ 242 | #undef HAVE_LOCAL_ENGINE_SETUP 243 | #undef HAVE_SSL_HW_CHECK 244 | /* ... with patch to get cfg password from card */ 245 | #undef HAVE_ENGINE_GET_PASSWORD 246 | /* ... with our buffer patches */ 247 | #undef HAVE_SSL_BUFFER_CB 248 | 249 | /* */ 250 | #undef ISD_SYSTEM_VSN 251 | 252 | /* eventpoll */ 253 | #undef HAVE_KPOLL 254 | 255 | /* Have/use netfilter (a.k.a. iptables) */ 256 | #undef HAVE_NETFILTER 257 | 258 | /* Non-standard support for "non-local connect()" in Linux 2.4 */ 259 | #undef HAVE_NONLOCAL_CONNECT 260 | 261 | /* atomic asmebler op in asm/atomic.h ? */ 262 | #undef HAVE_ATOMIC_OPS 263 | 264 | 265 | @BOTTOM@ 266 | 267 | #endif /* WIN32 */ 268 | #endif /* _CONFIG_H_ */ 269 | 270 | -------------------------------------------------------------------------------- /ring/bfile/config/aclocal.m4: -------------------------------------------------------------------------------- 1 | dnl ---------------------------------------------------------------------- 2 | dnl 3 | dnl BT_MSG_CONTROL checks for msg_control member in msghdr and that 4 | dnl the cmsg fields aren't broken... 5 | dnl 6 | 7 | AC_DEFUN(BT_MSG_CONTROL, 8 | [ 9 | AC_CACHE_CHECK([for msg_control member in msghdr], 10 | bt_cv_have_msghdr_msg_control, 11 | [AC_TRY_COMPILE([#include 12 | #include ], 13 | [struct msghdr msg; 14 | msg.msg_control;], 15 | bt_cv_have_msghdr_msg_control=yes, bt_cv_have_msghdr_msg_control=no)]) 16 | if test $bt_cv_have_msghdr_msg_control = yes; then 17 | AC_DEFINE(HAVE_MSGHDR_MSG_CONTROL) 18 | fi 19 | 20 | if test $bt_cv_have_msghdr_msg_control = yes; then 21 | AC_MSG_CHECKING(for broken CMSG_FIELDS) 22 | case "$target_os" in 23 | linux*) 24 | AC_DEFINE(BROKEN_CMSG_FIELDS) 25 | AC_MSG_RESULT(yes) 26 | ;; 27 | *) 28 | AC_MSG_RESULT(no) 29 | ;; 30 | esac 31 | fi 32 | ]) 33 | -------------------------------------------------------------------------------- /ring/bfile/config/config.h.in: -------------------------------------------------------------------------------- 1 | /* config.h.in. Generated from configure.in by autoheader. */ 2 | #ifndef _CONFIG_H_ 3 | #define _CONFIG_H_ 4 | 5 | #ifndef WIN32 6 | 7 | 8 | /* 9 | * This file contains prototypes for config.h.in which autoheader 10 | * could not figure by itself. I.e. if you write your own test which 11 | * defines a macro you will probably have to put it here, but if you 12 | * use any standard test AC_* it will be detected by autoheader. 13 | */ 14 | 15 | #undef WIN32 16 | 17 | /* Define to a string defining your target. ($target in configure.in) */ 18 | #undef CPU_VENDOR_OS 19 | 20 | 21 | /* Define to the full path of the ifconfig program */ 22 | #undef IFCONFIG 23 | 24 | /* Define to the full path of the route program */ 25 | #undef ROUTE 26 | 27 | /* Define to the full path of the arp program */ 28 | #undef ARP 29 | 30 | 31 | /* 32 | * These four below should definately be done away with!! 33 | */ 34 | 35 | /* Define if your target OS is a BSD derivative */ 36 | #undef BSD 37 | 38 | /* Define if your target OS is Linux */ 39 | #undef LINUX 40 | 41 | /* Define if your target OS is SunOS 5.x */ 42 | #undef SOLARIS 43 | 44 | /* Define if your target OS is BSD/OS 4.x */ 45 | #undef BSDI 46 | 47 | 48 | /* 49 | * What are these??? 50 | */ 51 | 52 | /* ? */ 53 | #undef USE_IFALIAS 54 | 55 | /* Define if you wish to use the bpf interface */ 56 | #undef USE_BPF 57 | 58 | /* Define if you wish to use the dlpi interface */ 59 | #undef USE_DLPI 60 | 61 | /* ? */ 62 | #undef USE_SOCKET 63 | 64 | 65 | /* Define if prototypes for malloc can be found in stdlib.h */ 66 | #undef STDLIB_MALLOC 67 | 68 | /* Define if your include files defines a prototype for sys_errlist[] */ 69 | #undef HAVE_SYS_ERRLIST 70 | 71 | /* This isn't used anywhere (that I could find) so I don't know what it means*/ 72 | #undef IFCONFIG_REQUIRES_DOWN_ADDRESS 73 | 74 | /* Define if your OS have broken cmsg fields in the msghdr struct (Linux) */ 75 | #undef BROKEN_CMSG_FIELDS 76 | 77 | /* Define if sockaddr structure has sa_len member */ 78 | #undef SA_LEN_IN_SOCKADDR 79 | 80 | 81 | 82 | /* 83 | * Provide a common way to refer to ints with specific size. (Is this 84 | * used everywhere?) The names used are {u_,}int{8,16,32,64}_t unless 85 | * they are defined in sys/types.h they are defined in ints.h using 86 | * the BIT macros. e.g. if int8_t isn't defined in sys/types.h int8_t 87 | * is typedef:ed to BIT8. 88 | * 89 | */ 90 | 91 | /* Define to a basic signed type that is 8 bits in size */ 92 | #undef BIT8 93 | 94 | /* Define to a basic signed type that is 16 bits in size */ 95 | #undef BIT16 96 | 97 | /* Define to a basic signed type that is 32 bits in size */ 98 | #undef BIT32 99 | 100 | /* Define to a basic signed type that is 64 bits in size */ 101 | #undef BIT64 102 | 103 | /* Define if sys/types.h defines this type */ 104 | #undef HAVE_int8_t 105 | 106 | /* Define if sys/types.h defines this type */ 107 | #undef HAVE_u_int8_t 108 | 109 | /* Define if sys/types.h defines this type */ 110 | #undef HAVE_int16_t 111 | 112 | /* Define if sys/types.h defines this type */ 113 | #undef HAVE_u_int16_t 114 | 115 | /* Define if sys/types.h defines this type */ 116 | #undef HAVE_int32_t 117 | 118 | /* Define if sys/types.h defines this type */ 119 | #undef HAVE_u_int32_t 120 | 121 | /* Define if sys/types.h defines this type */ 122 | #undef HAVE_int64_t 123 | 124 | /* Define if sys/types.h defines this type */ 125 | #undef HAVE_u_int64_t 126 | 127 | 128 | 129 | 130 | 131 | 132 | /* */ 133 | #undef ETHER_HEADER_USES_ETHER_ADDR 134 | 135 | /* */ 136 | #undef HAVE_DLIOCRAW 137 | 138 | /* */ 139 | #undef HAVE_ETHERADDRL 140 | 141 | /* */ 142 | #undef HAVE_ETHER_ADDR_LEN 143 | 144 | /* */ 145 | #undef HAVE_MSGHDR_MSG_CONTROL 146 | 147 | /* */ 148 | #undef HAVE_SIOCGARP 149 | 150 | /* */ 151 | #undef HAVE_SIOCGIFCONF 152 | 153 | /* */ 154 | #undef HAVE_SIOCGIFHWADDR 155 | 156 | /* */ 157 | #undef HAVE_arpreq 158 | 159 | /* */ 160 | #undef HAVE_caddr_t 161 | 162 | /* */ 163 | #undef HAVE_ether_header 164 | 165 | /* */ 166 | #undef HAVE_ethhdr 167 | 168 | /* */ 169 | #undef HAVE_ifnet 170 | 171 | /* */ 172 | #undef HAVE_in_addr 173 | 174 | /* */ 175 | #undef HAVE_sockaddr 176 | 177 | /* */ 178 | #undef HAVE_sockaddr_dl 179 | 180 | /* */ 181 | #undef HAVE_sockaddr_in 182 | 183 | /* */ 184 | #undef NEED_HAVE_sockaddr_dl 185 | 186 | 187 | 188 | /* 189 | * I have yet to figure out what all this need_* stuff is for, shouldn't 190 | * it suffice with using have_* ??? 191 | */ 192 | 193 | /* */ 194 | #undef NEED_LINUX_SOCKIOS_H 195 | 196 | /* */ 197 | #undef NEED_NETINET_IF_ETHER_H 198 | 199 | /* */ 200 | #undef NEED_NETINET_IN_H 201 | 202 | /* */ 203 | #undef NEED_NET_ETHERNET_H 204 | 205 | /* */ 206 | #undef NEED_NET_IF_ARP_H 207 | 208 | /* */ 209 | #undef NEED_NET_IF_DL_H 210 | 211 | /* */ 212 | #undef NEED_NET_IF_H 213 | 214 | /* */ 215 | #undef NEED_SYS_BITYPES_H 216 | 217 | /* */ 218 | #undef NEED_SYS_DLPI_H 219 | 220 | /* */ 221 | #undef NEED_SYS_ETHERNET_H 222 | 223 | /* */ 224 | #undef NEED_SYS_SOCKETIO_H 225 | 226 | /* */ 227 | #undef NEED_SYS_SOCKET_H 228 | 229 | /* */ 230 | #undef NEED_SYS_SOCKIO_H 231 | 232 | /* */ 233 | #undef NEED_SYS_TYPES_H 234 | 235 | /* if we have the openssl with engine support */ 236 | #undef HAVE_SSL_ENGINE 237 | /* ... with Rainbow patches */ 238 | #undef HAVE_RAINBOW_PATCHES 239 | /* ... with Rainbow's libswift */ 240 | #undef HAVE_SWIFT 241 | /* ... with one of our HW checks */ 242 | #undef HAVE_LOCAL_ENGINE_SETUP 243 | #undef HAVE_SSL_HW_CHECK 244 | /* ... with patch to get cfg password from card */ 245 | #undef HAVE_ENGINE_GET_PASSWORD 246 | /* ... with our buffer patches */ 247 | #undef HAVE_SSL_BUFFER_CB 248 | 249 | /* */ 250 | #undef ISD_SYSTEM_VSN 251 | 252 | /* eventpoll */ 253 | #undef HAVE_KPOLL 254 | 255 | /* Have/use netfilter (a.k.a. iptables) */ 256 | #undef HAVE_NETFILTER 257 | 258 | /* Non-standard support for "non-local connect()" in Linux 2.4 */ 259 | #undef HAVE_NONLOCAL_CONNECT 260 | 261 | /* atomic asmebler op in asm/atomic.h ? */ 262 | #undef HAVE_ATOMIC_OPS 263 | 264 | 265 | 266 | /* Description */ 267 | #undef DARWIN 268 | 269 | /* Define to 1 if you have the header file. */ 270 | #undef HAVE_INTTYPES_H 271 | 272 | /* Define to 1 if you have the header file. */ 273 | #undef HAVE_MALLOC_H 274 | 275 | /* Define to 1 if you have the header file. */ 276 | #undef HAVE_MEMORY_H 277 | 278 | /* Define to 1 if you have the header file. */ 279 | #undef HAVE_STDINT_H 280 | 281 | /* Define to 1 if you have the header file. */ 282 | #undef HAVE_STDLIB_H 283 | 284 | /* Define to 1 if you have the header file. */ 285 | #undef HAVE_STRINGS_H 286 | 287 | /* Define to 1 if you have the header file. */ 288 | #undef HAVE_STRING_H 289 | 290 | /* Define to 1 if you have the header file. */ 291 | #undef HAVE_SYS_STAT_H 292 | 293 | /* Define to 1 if you have the header file. */ 294 | #undef HAVE_SYS_TYPES_H 295 | 296 | /* Define to 1 if you have the header file. */ 297 | #undef HAVE_UNISTD_H 298 | 299 | /* Define to the address where bug reports for this package should be sent. */ 300 | #undef PACKAGE_BUGREPORT 301 | 302 | /* Define to the full name of this package. */ 303 | #undef PACKAGE_NAME 304 | 305 | /* Define to the full name and version of this package. */ 306 | #undef PACKAGE_STRING 307 | 308 | /* Define to the one symbol short name of this package. */ 309 | #undef PACKAGE_TARNAME 310 | 311 | /* Define to the version of this package. */ 312 | #undef PACKAGE_VERSION 313 | 314 | /* Define to 1 if you have the ANSI C header files. */ 315 | #undef STDC_HEADERS 316 | 317 | #endif /* WIN32 */ 318 | #endif /* _CONFIG_H_ */ 319 | 320 | -------------------------------------------------------------------------------- /ring/bfile/config/configure.in: -------------------------------------------------------------------------------- 1 | AC_INIT 2 | 3 | dnl work out who the cpu, vendor and OS are 4 | AC_CANONICAL_SYSTEM 5 | AC_DEFINE_UNQUOTED(CPU_VENDOR_OS, "$target") 6 | 7 | dnl Programs 8 | 9 | AC_PROG_CC 10 | AC_PATH_PROG(ERL, erl) 11 | AC_PATH_PROG(ERLC, erlc) 12 | ERLBINDIR=`dirname $ERL` ; ERLBINDIR=`dirname $ERLBINDIR`/lib/erlang/bin 13 | 14 | ERLDIR=`awk -F= '/ROOTDIR=/ { print [$]2; exit; }' $ERL` 15 | AC_SUBST(ERL) 16 | AC_SUBST(ERLC) 17 | AC_SUBST(ERLBINDIR) 18 | AC_SUBST(ERLDIR) 19 | 20 | if test ! -d "$ERLDIR" ; then 21 | AC_MSG_ERROR([Broken Erlang installation, $ERLDIR does not exist!]) 22 | fi 23 | 24 | dnl C header files 25 | 26 | AC_CONFIG_HEADER(config.h:config.h.in) 27 | 28 | AC_CHECK_HEADERS(malloc.h) 29 | 30 | BT_MSG_CONTROL 31 | 32 | case "$target_os" in 33 | *cygwin*) 34 | : 35 | dnl fix this later 36 | ;; 37 | linux*) 38 | AC_DEFINE(LINUX) 39 | LD_SHARED="ld -shared" 40 | ;; 41 | *bsd*) 42 | AC_DEFINE(BSD) 43 | LD_SHARED="ld -Bshareable" 44 | ;; 45 | *solaris*) 46 | AC_DEFINE(SOLARIS) 47 | LD_SHARED="ld -G" 48 | ;; 49 | *darwin*) 50 | AC_DEFINE([DARWIN], [], [Description]) 51 | LD_SHARED="cc -bundle -flat_namespace -undefined suppress" 52 | ;; 53 | *) 54 | LD_SHARED="ld -shared" 55 | ;; 56 | esac 57 | 58 | AC_SUBST(LD_SHARED) 59 | 60 | 61 | dnl libnsl and libsocket tests borrowed from ethereal's autoconf scheme. 62 | dnl # msh@cis.ufl.edu says -lnsl (and -lsocket) are needed for his 386/AT, 63 | dnl # to get the SysV transport functions. 64 | dnl # chad@anasazi.com says the Pyramid MIS-ES running DC/OSx (SVR4) 65 | dnl # needs -lnsl. 66 | dnl # The nsl library prevents programs from opening the X display 67 | dnl # on Irix 5.2, according to dickey@clark.net. 68 | AC_CHECK_FUNC(gethostbyname, , 69 | AC_CHECK_LIB(nsl, gethostbyname, NSL_LIBS="-lnsl")) 70 | AC_SUBST(NSL_LIBS) 71 | dnl # lieder@skyler.mavd.honeywell.com says without -lsocket, 72 | dnl # socket/setsockopt and other routines are undefined under SCO ODT 73 | dnl # 2.0. But -lsocket is broken on IRIX 5.2 (and is not necessary 74 | dnl # on later versions), says simon@lia.di.epfl.ch: it contains 75 | dnl # gethostby* variants that don't use the nameserver (or something). 76 | dnl # -lsocket must be given before -lnsl if both are needed. 77 | dnl # We assume that if connect needs -lnsl, so does gethostbyname. 78 | AC_CHECK_FUNC(connect, , 79 | AC_CHECK_LIB(socket, connect, SOCKET_LIBS="-lsocket", 80 | AC_MSG_ERROR(Function 'socket' not found.), $NSL_LIBS)) 81 | AC_SUBST(SOCKET_LIBS) 82 | 83 | dnl 84 | dnl End. 85 | 86 | AC_OUTPUT(include.mk) 87 | 88 | 89 | -------------------------------------------------------------------------------- /ring/bfile/config/include.mk.in: -------------------------------------------------------------------------------- 1 | ## -*- makefile -*- 2 | 3 | ###################################################################### 4 | ## C 5 | 6 | CC := @CC@ 7 | CFLAGS := @CFLAGS@ @DEFS@ 8 | 9 | LD_SHARED := @LD_SHARED@ 10 | 11 | ###################################################################### 12 | ## Erlang 13 | 14 | ERL = @ERL@ 15 | ERLC = @ERLC@ 16 | ERLDIR = @ERLDIR@ 17 | 18 | ERL_C_INCLUDE_DIR := $(ERLDIR)/usr/include 19 | 20 | 21 | ERLC_FLAGS := -W 22 | 23 | ifndef no_debug_info 24 | ERLC_FLAGS += +debug_info 25 | endif 26 | 27 | ifdef debug 28 | ERLC_FLAGS += -Ddebug 29 | endif 30 | 31 | EBIN_DIR := ../ebin 32 | DOC_DIR := ../doc 33 | EMULATOR := beam 34 | 35 | ERL_SOURCES := $(wildcard *.erl) 36 | ERL_HEADERS := $(wildcard *.hrl) $(wildcard ../include/*.hrl) 37 | ERL_OBJECTS := $(ERL_SOURCES:%.erl=$(EBIN_DIR)/%.$(EMULATOR)) 38 | ERL_DOCUMENTS := $(ERL_SOURCES:%.erl=$(DOC_DIR)/%.html) 39 | 40 | # Hmm, don't know if you are supposed to like this better... ;-) 41 | APPSCRIPT = '$$vsn=shift; $$mods=""; while(@ARGV){ $$_=shift; s/^([A-Z].*)$$/\'\''$$1\'\''/; $$mods.=", " if $$mods; $$mods .= $$_; } while(<>) { s/%VSN%/$$vsn/; s/%MODULES%/$$mods/; print; }' 42 | 43 | 44 | ../ebin/%.app: %.app.src ../vsn.mk Makefile 45 | perl -e $(APPSCRIPT) "$(VSN)" $(MODULES) < $< > $@ 46 | 47 | ../ebin/%.appup: %.appup 48 | cp $< $@ 49 | 50 | 51 | $(EBIN_DIR)/%.$(EMULATOR): %.erl 52 | $(ERLC) $(ERLC_FLAGS) -o $(EBIN_DIR) $< 53 | 54 | # generate documentation with edoc: 55 | # this is still not the proper way to do it, but it works 56 | # (see the wumpus application for an example) 57 | 58 | $(DOC_DIR)/%.html: %.erl 59 | ${ERL} -noshell \ 60 | -pa ../../syntax_tools/ebin \ 61 | -pa ../../edoc/ebin \ 62 | -pa ../../xmerl/ebin \ 63 | -pa ../../ucs/ebin \ 64 | -run edoc file $< -run init stop 65 | mv *.html $(DOC_DIR) 66 | 67 | # C linking items 68 | 69 | NSL_LIBS = @NSL_LIBS@ 70 | SOCKET_LIBS = @SOCKET_LIBS@ 71 | 72 | # Miscellaneous 73 | 74 | GROFF = @GROFF@ 75 | PS2PDF = @PS2PDF@ 76 | -------------------------------------------------------------------------------- /ring/bfile/config/install-sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # install - install a program, script, or datafile 4 | # This comes from X11R5 (mit/util/scripts/install.sh). 5 | # 6 | # Copyright 1991 by the Massachusetts Institute of Technology 7 | # 8 | # Permission to use, copy, modify, distribute, and sell this software and its 9 | # documentation for any purpose is hereby granted without fee, provided that 10 | # the above copyright notice appear in all copies and that both that 11 | # copyright notice and this permission notice appear in supporting 12 | # documentation, and that the name of M.I.T. not be used in advertising or 13 | # publicity pertaining to distribution of the software without specific, 14 | # written prior permission. M.I.T. makes no representations about the 15 | # suitability of this software for any purpose. It is provided "as is" 16 | # without express or implied warranty. 17 | # 18 | # Calling this script install-sh is preferred over install.sh, to prevent 19 | # `make' implicit rules from creating a file called install from it 20 | # when there is no Makefile. 21 | # 22 | # This script is compatible with the BSD install script, but was written 23 | # from scratch. It can only install one file at a time, a restriction 24 | # shared with many OS's install programs. 25 | 26 | 27 | # set DOITPROG to echo to test this script 28 | 29 | # Don't use :- since 4.3BSD and earlier shells don't like it. 30 | doit="${DOITPROG-}" 31 | 32 | 33 | # put in absolute paths if you don't have them in your path; or use env. vars. 34 | 35 | mvprog="${MVPROG-mv}" 36 | cpprog="${CPPROG-cp}" 37 | chmodprog="${CHMODPROG-chmod}" 38 | chownprog="${CHOWNPROG-chown}" 39 | chgrpprog="${CHGRPPROG-chgrp}" 40 | stripprog="${STRIPPROG-strip}" 41 | rmprog="${RMPROG-rm}" 42 | mkdirprog="${MKDIRPROG-mkdir}" 43 | 44 | transformbasename="" 45 | transform_arg="" 46 | instcmd="$mvprog" 47 | chmodcmd="$chmodprog 0755" 48 | chowncmd="" 49 | chgrpcmd="" 50 | stripcmd="" 51 | rmcmd="$rmprog -f" 52 | mvcmd="$mvprog" 53 | src="" 54 | dst="" 55 | dir_arg="" 56 | 57 | while [ x"$1" != x ]; do 58 | case $1 in 59 | -c) instcmd="$cpprog" 60 | shift 61 | continue;; 62 | 63 | -d) dir_arg=true 64 | shift 65 | continue;; 66 | 67 | -m) chmodcmd="$chmodprog $2" 68 | shift 69 | shift 70 | continue;; 71 | 72 | -o) chowncmd="$chownprog $2" 73 | shift 74 | shift 75 | continue;; 76 | 77 | -g) chgrpcmd="$chgrpprog $2" 78 | shift 79 | shift 80 | continue;; 81 | 82 | -s) stripcmd="$stripprog" 83 | shift 84 | continue;; 85 | 86 | -t=*) transformarg=`echo $1 | sed 's/-t=//'` 87 | shift 88 | continue;; 89 | 90 | -b=*) transformbasename=`echo $1 | sed 's/-b=//'` 91 | shift 92 | continue;; 93 | 94 | *) if [ x"$src" = x ] 95 | then 96 | src=$1 97 | else 98 | # this colon is to work around a 386BSD /bin/sh bug 99 | : 100 | dst=$1 101 | fi 102 | shift 103 | continue;; 104 | esac 105 | done 106 | 107 | if [ x"$src" = x ] 108 | then 109 | echo "install: no input file specified" 110 | exit 1 111 | else 112 | true 113 | fi 114 | 115 | if [ x"$dir_arg" != x ]; then 116 | dst=$src 117 | src="" 118 | 119 | if [ -d $dst ]; then 120 | instcmd=: 121 | else 122 | instcmd=mkdir 123 | fi 124 | else 125 | 126 | # Waiting for this to be detected by the "$instcmd $src $dsttmp" command 127 | # might cause directories to be created, which would be especially bad 128 | # if $src (and thus $dsttmp) contains '*'. 129 | 130 | if [ -f $src -o -d $src ] 131 | then 132 | true 133 | else 134 | echo "install: $src does not exist" 135 | exit 1 136 | fi 137 | 138 | if [ x"$dst" = x ] 139 | then 140 | echo "install: no destination specified" 141 | exit 1 142 | else 143 | true 144 | fi 145 | 146 | # If destination is a directory, append the input filename; if your system 147 | # does not like double slashes in filenames, you may need to add some logic 148 | 149 | if [ -d $dst ] 150 | then 151 | dst="$dst"/`basename $src` 152 | else 153 | true 154 | fi 155 | fi 156 | 157 | ## this sed command emulates the dirname command 158 | dstdir=`echo $dst | sed -e 's,[^/]*$,,;s,/$,,;s,^$,.,'` 159 | 160 | # Make sure that the destination directory exists. 161 | # this part is taken from Noah Friedman's mkinstalldirs script 162 | 163 | # Skip lots of stat calls in the usual case. 164 | if [ ! -d "$dstdir" ]; then 165 | defaultIFS=' 166 | ' 167 | IFS="${IFS-${defaultIFS}}" 168 | 169 | oIFS="${IFS}" 170 | # Some sh's can't handle IFS=/ for some reason. 171 | IFS='%' 172 | set - `echo ${dstdir} | sed -e 's@/@%@g' -e 's@^%@/@'` 173 | IFS="${oIFS}" 174 | 175 | pathcomp='' 176 | 177 | while [ $# -ne 0 ] ; do 178 | pathcomp="${pathcomp}${1}" 179 | shift 180 | 181 | if [ ! -d "${pathcomp}" ] ; 182 | then 183 | $mkdirprog "${pathcomp}" 184 | else 185 | true 186 | fi 187 | 188 | pathcomp="${pathcomp}/" 189 | done 190 | fi 191 | 192 | if [ x"$dir_arg" != x ] 193 | then 194 | $doit $instcmd $dst && 195 | 196 | if [ x"$chowncmd" != x ]; then $doit $chowncmd $dst; else true ; fi && 197 | if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dst; else true ; fi && 198 | if [ x"$stripcmd" != x ]; then $doit $stripcmd $dst; else true ; fi && 199 | if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dst; else true ; fi 200 | else 201 | 202 | # If we're going to rename the final executable, determine the name now. 203 | 204 | if [ x"$transformarg" = x ] 205 | then 206 | dstfile=`basename $dst` 207 | else 208 | dstfile=`basename $dst $transformbasename | 209 | sed $transformarg`$transformbasename 210 | fi 211 | 212 | # don't allow the sed command to completely eliminate the filename 213 | 214 | if [ x"$dstfile" = x ] 215 | then 216 | dstfile=`basename $dst` 217 | else 218 | true 219 | fi 220 | 221 | # Make a temp file name in the proper directory. 222 | 223 | dsttmp=$dstdir/#inst.$$# 224 | 225 | # Move or copy the file name to the temp name 226 | 227 | $doit $instcmd $src $dsttmp && 228 | 229 | trap "rm -f ${dsttmp}" 0 && 230 | 231 | # and set any options; do chmod last to preserve setuid bits 232 | 233 | # If any of these fail, we abort the whole thing. If we want to 234 | # ignore errors from any of these, just make sure not to ignore 235 | # errors from the above "$doit $instcmd $src $dsttmp" command. 236 | 237 | if [ x"$chowncmd" != x ]; then $doit $chowncmd $dsttmp; else true;fi && 238 | if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dsttmp; else true;fi && 239 | if [ x"$stripcmd" != x ]; then $doit $stripcmd $dsttmp; else true;fi && 240 | if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dsttmp; else true;fi && 241 | 242 | # Now rename the file to the real destination. 243 | 244 | $doit $rmcmd -f $dstdir/$dstfile && 245 | $doit $mvcmd $dsttmp $dstdir/$dstfile 246 | 247 | fi && 248 | 249 | 250 | exit 0 251 | -------------------------------------------------------------------------------- /ring/bfile/ebin/a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuulos/ringo/57a8e510a5b4fa77c8ab6f2dbf5795b41fbc631f/ring/bfile/ebin/a -------------------------------------------------------------------------------- /ring/bfile/priv/a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tuulos/ringo/57a8e510a5b4fa77c8ab6f2dbf5795b41fbc631f/ring/bfile/priv/a -------------------------------------------------------------------------------- /ring/bfile/src/._bfile.app.src: -------------------------------------------------------------------------------- 1 | Mac OS X  2 RTEXT -------------------------------------------------------------------------------- /ring/bfile/src/Makefile: -------------------------------------------------------------------------------- 1 | 2 | include ../config/include.mk 3 | 4 | include ../vsn.mk 5 | VSN=$(FD_SERVER_VSN) 6 | 7 | ifeq ($(TYPE),debug) 8 | DEBUG_FLAGS = -Ddebug 9 | else 10 | DEBUG_FLAGS = 11 | endif 12 | 13 | MODULES=bfile 14 | 15 | EBIN = ../ebin 16 | EBIN_FILES = $(MODULES:%=$(EBIN)/%.$(EMULATOR)) $(EBIN)/bfile.app 17 | 18 | 19 | all: $(EBIN_FILES) 20 | 21 | debug: 22 | $(MAKE) DEBUG=-DDEBUG 23 | clean: 24 | rm -rf $(EBIN_FILES) 25 | 26 | -------------------------------------------------------------------------------- /ring/bfile/src/bfile.app.src: -------------------------------------------------------------------------------- 1 | {application,bfile, 2 | [{description,"Fast FILE interface"}, 3 | {vsn,"%VSN%"}, 4 | {modules,[%MODULES%]}, 5 | {applications,[kernel,stdlib]}, 6 | {env, []}, 7 | {mod,{bfile,[]}}]}. 8 | -------------------------------------------------------------------------------- /ring/bfile/src/bfile.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% File : bfile.erl 3 | %%% Author : Claes Wikstrom 4 | %%% Purpose : Interface to stdio buffered FILE io 5 | %%% Created : 22 Nov 1999 by Claes Wikstrom 6 | %%%---------------------------------------------------------------------- 7 | 8 | -module(bfile). 9 | -vsn("$Revision: 1.1 $ "). 10 | -author('klacke@kaja.klacke.net'). 11 | 12 | -export([ 13 | load_driver/0, 14 | fopen/2, 15 | fclose/1, 16 | fread/2, 17 | fwrite/2, 18 | feof/1, 19 | ferror/1, 20 | set_linebuf_size/2, 21 | fseek/3, 22 | ftell/1, 23 | ftruncate/1, 24 | fflush/1, 25 | frewind/1, 26 | fgetc/1, 27 | fungetc/2, 28 | fgets/1, 29 | gets/1, 30 | pwrite/3, 31 | pread/3 32 | ]). 33 | 34 | 35 | %% Opcodes 36 | -define(OPEN, $o). 37 | -define(CLOSE, $c). 38 | -define(READ, $r). 39 | -define(WRITE, $w). 40 | -define(SEEK, $s). 41 | -define(TELL, $t). 42 | -define(TRUNCATE, $T). 43 | -define(FLUSH, $f). 44 | -define(OEOF, $e). 45 | -define(ERROR, $E). 46 | -define(GETC, $g). 47 | -define(GETS, $G). 48 | -define(GETS2, $2). 49 | -define(SET_LINEBUF_SIZE, $S). 50 | -define(UNGETC, $u). 51 | 52 | 53 | %% ret codes 54 | -define(VALUE, $v). 55 | -define(FLINE, $L). 56 | -define(OK, $o). 57 | -define(I32, $O). 58 | -define(NOLINE,$N). 59 | -define(FERROR, $E). 60 | -define(REOF, $x). 61 | 62 | -define(int32(X), 63 | [((X) bsr 24) band 16#ff, ((X) bsr 16) band 16#ff, 64 | ((X) bsr 8) band 16#ff, (X) band 16#ff]). 65 | %% Bytes to unsigned 66 | -define(u32(X3,X2,X1,X0), 67 | (((X3) bsl 24) bor ((X2) bsl 16) bor ((X1) bsl 8) bor (X0))). 68 | %% Bytes to signed 69 | -define(i32(X3,X2,X1,X0), 70 | (?u32(X3,X2,X1,X0) - 71 | (if (X3) > 127 -> 16#100000000; true -> 0 end))). 72 | 73 | load_driver() -> 74 | Dir = filename:join([filename:dirname(code:which(bfile)),"..", "priv"]), 75 | erl_ddll:load_driver(Dir, "FILE_drv"). 76 | 77 | 78 | %% Flags = "r" | "w" | ... etc, see fopen(3) 79 | %% Ret: {ok, Fd} | {error, Reason} 80 | 81 | fopen(Fname, Flags) -> 82 | P = open_port({spawn, 'FILE_drv'}, [binary]), 83 | erlang_port_command(P, [?OPEN, Fname, [0], Flags, [0]]), 84 | case recp(P) of 85 | ok -> 86 | {ok, {bfile, P}}; 87 | Err -> 88 | unlink(P), 89 | exit(P, die), 90 | Err 91 | end. 92 | 93 | %% void() 94 | fclose({bfile, Fd}) -> 95 | unlink(Fd), 96 | catch erlang:port_close(Fd). 97 | 98 | %% {ok, #Bin} | {error, Reason} | eof 99 | fread({bfile, Fd}, Sz) -> 100 | erlang_port_command(Fd, [?READ|?int32(Sz)]), 101 | recp(Fd). 102 | 103 | %% ok | {error, Reason} 104 | fwrite({bfile, Fd}, IoList) -> 105 | erlang_port_command(Fd, [?WRITE , IoList]), 106 | recp(Fd). 107 | 108 | %% ok | {error, Reason} 109 | pwrite(BFd, Pos, IoList) -> 110 | case fseek(BFd, Pos, seek_set) of 111 | ok -> 112 | fwrite(BFd, IoList); 113 | Error -> 114 | Error 115 | end. 116 | 117 | %% {ok, #Bin} | {error, Reason} | eof 118 | pread(BFd, Pos, Sz) -> 119 | case fseek(BFd, Pos, seek_set) of 120 | ok -> 121 | fread(BFd, Sz); 122 | Error -> 123 | Error 124 | end. 125 | 126 | %% bool 127 | feof({bfile, Fd}) -> 128 | erlang_port_command(Fd, [?OEOF]), 129 | bool(recp(Fd)). 130 | 131 | %% bool 132 | ferror({bfile, Fd}) -> 133 | erlang_port_command(Fd, [?ERROR]), 134 | bool(recp(Fd)). 135 | 136 | 137 | %% void() 138 | set_linebuf_size({bfile, Fd}, Sz) -> 139 | erlang_port_command(Fd, [?SET_LINEBUF_SIZE|?int32(Sz)]), 140 | recp(Fd). 141 | 142 | %% Whence == seek_set | seek_cur || seek_end 143 | %% ok | {error, Reason} 144 | fseek({bfile, Fd}, Offs, Whence) -> 145 | erlang_port_command(Fd, [?SEEK, ?int32(Offs), whence_enc(Whence)]), 146 | recp(Fd). 147 | 148 | %% {ok, Int} | {error, Reason} 149 | ftell({bfile, Fd}) -> 150 | erlang_port_command(Fd, [?TELL]), 151 | recp(Fd). 152 | 153 | %% ok | {error, Reason} 154 | ftruncate({bfile, Fd}) -> 155 | erlang_port_command(Fd, [?TRUNCATE]), 156 | recp(Fd). 157 | 158 | %% ok | {error, Reason} 159 | fflush({bfile, Fd}) -> 160 | erlang_port_command(Fd, [?FLUSH]), 161 | recp(Fd). 162 | 163 | %% ok | {error, Reason} 164 | frewind(BFd) -> 165 | fseek(BFd, 0, seek_set). 166 | 167 | %% {ok, Char} | {error, Reason} | eof 168 | fgetc({bfile, Fd}) -> 169 | erlang_port_command(Fd, [?GETC]), 170 | recp(Fd). 171 | 172 | %% ok | {error, Reason} 173 | fungetc({bfile, Fd}, Char) -> 174 | erlang_port_command(Fd, [?UNGETC, Char]), 175 | recp(Fd). 176 | 177 | %% {line, #Bin} | {noline, #Bin} | {error, Reason} | eof 178 | %% including newline 179 | fgets({bfile, Fd}) -> 180 | erlang_port_command(Fd, [?GETS]), 181 | recp(Fd). 182 | 183 | %% {line, #Bin} | {noline, #Bin} | {error, Reason} | eof 184 | %% not including newline 185 | gets({bfile, Fd}) -> 186 | erlang_port_command(Fd, [?GETS2]), 187 | recp(Fd). 188 | 189 | 190 | whence_enc(seek_set) -> 191 | 1; 192 | whence_enc(seek_cur) -> 193 | 2; 194 | whence_enc(seek_end) -> 195 | 3. 196 | 197 | 198 | bool({ok, 1}) -> 199 | true; 200 | bool({ok, 0}) -> 201 | false. 202 | 203 | 204 | recp(P) when port(P) -> 205 | receive 206 | {P, {data, [?VALUE|Bin]}} -> 207 | {ok, Bin}; 208 | {P, {data, [?FLINE|Bin]}} -> 209 | {line, Bin}; 210 | {P, {data, [?OK|_]}} -> 211 | ok; 212 | {P, {data, [?I32|Bin]}} -> 213 | [X1,X2,X3,X4] = binary_to_list(Bin), 214 | {ok, ?i32(X1, X2, X3, X4)}; 215 | {P, {data, [?NOLINE|Bin]}} -> 216 | {noline, Bin}; 217 | {P, {data, [?FERROR|Err]}} -> 218 | {error, list_to_atom(Err)}; 219 | {P, {data, [?REOF|_]}} -> 220 | eof 221 | end; 222 | 223 | 224 | recp(_PP) -> 225 | {error, badarg}. 226 | 227 | erlang_port_command(P, C) -> 228 | erlang:port_command(P, C). 229 | 230 | -------------------------------------------------------------------------------- /ring/bfile/vsn.mk: -------------------------------------------------------------------------------- 1 | BFILE_VSN=1.0 2 | -------------------------------------------------------------------------------- /ring/compile.sh: -------------------------------------------------------------------------------- 1 | erlc -o ebin src/*.erl 2 | #erlc +native +"{hipe, [o3]}" -o ebin src/*.erl 3 | erl -pa ebin -noshell -run make_boot write_scripts 4 | erlc -o ebin test/ringo_cmd.erl 5 | mv ringo.boot ebin 6 | -------------------------------------------------------------------------------- /ring/ebin/ringo.app: -------------------------------------------------------------------------------- 1 | {application, ringo, [ 2 | {description, "Ringo"}, 3 | {vsn, "1"}, 4 | {modules, [ringo_node, 5 | ringo_util, 6 | ringo_main]}, 7 | {registered, [ringo_node]}, 8 | {applications, [kernel, stdlib]}, 9 | {mod, {ringo_main, []}} 10 | ]}. 11 | 12 | -------------------------------------------------------------------------------- /ring/ringo_debug.sh: -------------------------------------------------------------------------------- 1 | erl -noshell -pa bfile/ebin -pa ebin -run bfile load_driver -run ringo_debug $1 $2 -run erlang halt 2 | -------------------------------------------------------------------------------- /ring/ringocmd.sh: -------------------------------------------------------------------------------- 1 | 2 | erl -sname tst -setcookie ringobingo -pa test -pa ebin/ -run ringo_cmd $@ 3 | -------------------------------------------------------------------------------- /ring/src/bin_util.erl: -------------------------------------------------------------------------------- 1 | -module(bin_util). 2 | -export([member64/2, to_list64/1, encode_kvsegment/1, decode_kvsegment/1, bits/1, pad/1]). 3 | -export([find_kv/2]). 4 | 5 | member64(Bin, Val) when is_binary(Bin), is_integer(Val) -> 6 | member64(Bin, <>); 7 | 8 | member64(Bin, Val) when is_binary(Bin), is_binary(Val) -> 9 | member64_1(Bin, Val). 10 | 11 | member64_1(<<>>, _) -> false; 12 | member64_1(<>, Val) when X == Val -> true; 13 | member64_1(<<_:8/binary, R/binary>>, Val) -> member64_1(R, Val). 14 | 15 | to_list64(B) -> to_list64(B, []). 16 | to_list64(<>, Res) -> to_list64(R, [X|Res]); 17 | to_list64(<<>>, Res) -> Res. 18 | 19 | encode_kvsegment([]) -> <<>>; 20 | encode_kvsegment(L) -> 21 | B = bits(lists:max([V || {_, V} <- L])), 22 | D = << <> || {K, V} <- L >>, 23 | pad(<<(length(L)):32, B:5, D/bits>>). 24 | 25 | decode_kvsegment(<<>>) -> []; 26 | decode_kvsegment(Seg) -> 27 | <> = Seg, 28 | S = N * (32 + B), 29 | <> = X, 30 | [{K, V} || <> <= D]. 31 | 32 | %%% 33 | %%% Binary search for a kvsegment 34 | %%% 35 | 36 | find_kv(Key, <<>>) -> {Key, none}; 37 | find_kv(Key, Seg) -> 38 | <> = Seg, 39 | S = N * (32 + B), 40 | <> = X, 41 | choose(D, Key, middle(D, 32 + B), 32 + B). 42 | 43 | middle(<<>>, _) -> none; 44 | middle(Seg, ItemSize) -> 45 | N = bit_size(Seg) div ItemSize, 46 | P = (N div 2) * ItemSize, 47 | <<_:P/bits, Middle:32, _/bits>> = Seg, 48 | {P, Middle}. 49 | 50 | choose(Seg, Key, {P, Middle}, ItemSize) when Key > Middle -> 51 | PP = P + ItemSize, 52 | <<_:PP, NSeg/bits>> = Seg, 53 | choose(NSeg, Key, middle(NSeg, ItemSize), ItemSize); 54 | 55 | choose(Seg, Key, {P, Middle}, ItemSize) when Key < Middle -> 56 | <> = Seg, 57 | choose(NSeg, Key, middle(NSeg, ItemSize), ItemSize); 58 | 59 | choose(Seg, Key, {P, _}, ItemSize) -> 60 | <<_:P/bits, Item:ItemSize/bits, _/bits>> = Seg, 61 | S = ItemSize - 32, 62 | <> = Item, 63 | {Key, Value}; 64 | 65 | choose(<<>>, Key, _, _) -> {Key, none}. 66 | 67 | %%% 68 | %%% 69 | %%% 70 | 71 | pad(X) when is_binary(X) -> X; 72 | pad(X) -> 73 | P = 8 - (bit_size(X) rem 8), 74 | <>. 75 | 76 | bits(X) -> bits(X, 0). 77 | bits(0, N) -> N; 78 | bits(X, N) -> bits(X bsr 1, N + 1). 79 | -------------------------------------------------------------------------------- /ring/src/lrucache.erl: -------------------------------------------------------------------------------- 1 | -module(lrucache). 2 | -export([new/0, update/2, is_empty/1, get_lru/1]). 3 | 4 | %%% 5 | %%% Thanks to Heli Tuulos for this beautiful design of LRU cache: 6 | %%% Cache is a double-linked list providing constant-time access to entries 7 | %%% via a dictionary. The LRU item is always the head and can be retrieved 8 | %%% with get_lru(). Accessing an item moves it to to the tail (update()). 9 | %%% 10 | %%% This implementation is based on the dict module to avoid copying 11 | %%% of large binary keys, as would happen with ets. 12 | %%% 13 | 14 | new() -> {gb_trees:empty(), nil, nil}. 15 | 16 | is_empty({_, nil, nil}) -> true; 17 | is_empty(_) -> false. 18 | 19 | % empty cache 20 | get_lru({_, nil, _}) -> nil; 21 | 22 | % one key 23 | get_lru({_, H, H}) -> {H, new()}; 24 | 25 | % normal case 26 | get_lru({D, H, T}) -> 27 | {Prev, _} = gb_trees:get(H, D), 28 | {Prev0, _} = gb_trees:get(Prev, D), 29 | {H, {gb_trees:enter(Prev, {Prev0, nil}, 30 | gb_trees:delete(H, D)), Prev, T}}. 31 | 32 | update(Key, {D, _, _} = LRU) -> 33 | update(Key, gb_trees:lookup(Key, D), LRU). 34 | 35 | % first key in the cache 36 | update(Key, none, {D, nil, nil}) -> 37 | {gb_trees:enter(Key, {nil, nil}, D), Key, Key}; 38 | 39 | % new key 40 | update(Key, none, LRU) -> add_tail(Key, LRU); 41 | 42 | % update tail 43 | update(_, {value, {nil, _}}, LRU) -> LRU; 44 | 45 | % update head 46 | update(Key, {value, {Prev, nil}}, {D, _, T}) -> 47 | {Prev0, _} = gb_trees:get(Prev, D), 48 | add_tail(Key, {gb_trees:enter(Prev, {Prev0, nil}, D), Key, T}); 49 | 50 | % existing key in the middle 51 | update(Key, {value, {Prev, Next}}, {D, H, T}) -> 52 | {Prev0, _} = gb_trees:get(Prev, D), 53 | {_, Next0} = gb_trees:get(Next, D), 54 | D0 = gb_trees:enter(Prev, {Prev0, Next}, D), 55 | add_tail(Key, {gb_trees:enter(Next, {Prev, Next0}, D0), H, T}). 56 | 57 | add_tail(Key, {D, H, T}) -> 58 | {nil, Next} = gb_trees:get(T, D), 59 | D0 = gb_trees:enter(T, {Key, Next}, D), 60 | {gb_trees:enter(Key, {nil, T}, D0), H, Key}. 61 | -------------------------------------------------------------------------------- /ring/src/make_boot.erl: -------------------------------------------------------------------------------- 1 | -module(make_boot). 2 | -export([write_scripts/0]). 3 | 4 | write_scripts() -> 5 | Erts = erlang:system_info(version), 6 | {value, {kernel, _, Kernel}} = lists:keysearch(kernel, 1, 7 | application:loaded_applications()), 8 | {value, {stdlib, _, Stdlib}} = lists:keysearch(stdlib, 1, 9 | application:loaded_applications()), 10 | 11 | Rel = "{release, {\"Ringo\", \"1\"}, {erts, \"~s\"}, [" 12 | "{kernel, \"~s\"}, {stdlib, \"~s\"}, {ringo, \"1\"}]}.", 13 | 14 | {ok, Fs} = file:open("ringo.rel", [write]), 15 | io:format(Fs, Rel, [Erts, Kernel, Stdlib]), 16 | file:close(Fs), 17 | 18 | systools:make_script("ringo", [local]), 19 | halt(). 20 | 21 | 22 | -------------------------------------------------------------------------------- /ring/src/ringo_debug.erl: -------------------------------------------------------------------------------- 1 | -module(ringo_debug). 2 | -export([dump_ids/1, dump_merkle/1, dump_iblock/1]). 3 | 4 | dump_ids(DBName) -> 5 | io:fwrite("# Date Time SyncID EntryID Leaf~n"), 6 | ringo_reader:fold(fun(_, _, _, {Time, EntryID}, _, N) -> 7 | Tstamp = ringo_util:format_timestamp( 8 | {Time div 1000000, Time rem 1000000, 0}), 9 | {Leaf, <>} = ringo_sync:sync_id(EntryID, Time), 10 | io:fwrite("~b ~s ~.16B ~.16B ~b~n", 11 | [N, Tstamp, SyncID, EntryID, Leaf]), 12 | N + 1 13 | end, 0, DBName). 14 | 15 | dump_merkle(DBName) -> 16 | {LeafHashes, LeafIDs} = ringo_sync:make_leaf_hashes_and_ids(DBName), 17 | Tree = ringo_sync:build_merkle_tree(LeafHashes), 18 | print_level(Tree, 0), 19 | dict:fold(fun(Leaf, List, _) -> 20 | io:fwrite("L ~b > ", [Leaf]), 21 | print_leaves(List) 22 | end, 0, LeafIDs). 23 | 24 | dump_iblock(IBName) -> 25 | {ok, Dex} = ringo_reader:read_file(IBName), 26 | {SingleSeg, MultiSeg, Offsets} = ringo_index:deserialize(Dex), 27 | lists:foreach(fun({Key, Offs}) -> 28 | io:fwrite("~b ~b~n", [Key, Offs]) 29 | end, bin_util:decode_kvsegment(SingleSeg)), 30 | lists:foreach(fun({Key, Offs}) -> 31 | <<_:Offs/bits, V/bits>> = Offsets, 32 | io:fwrite("~b ", [Key]), 33 | lists:foreach(fun(X) -> 34 | io:fwrite("~b ", [X]) 35 | end, ring_index:decode_poslist(V)), 36 | io:fwrite("~n") 37 | end, bin_util:decode_kvsegment(MultiSeg)). 38 | 39 | print_leaves(List) -> 40 | lists:map(fun(<>) -> 41 | io:fwrite("~b ", [X]) 42 | end, bin_util:to_list64(List)), 43 | io:fwrite("~n"). 44 | 45 | print_level([], _) -> ok; 46 | print_level([Level|Rest], H) -> 47 | lists:foldl(fun(E, N) -> 48 | io:fwrite("T ~b ~b ~b~n", [H, N, E]), 49 | N + 1 50 | end, 0, Level), 51 | print_level(Rest, H + 1). 52 | 53 | 54 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /ring/src/ringo_external.erl: -------------------------------------------------------------------------------- 1 | -module(ringo_external). 2 | -export([fetch_external/2, check_external/1]). 3 | 4 | -include("ringo_store.hrl"). 5 | 6 | %%% fetch_external() is a permanent process that takes care of copying external 7 | %%% files to this node as a part of the syncing process. In the normal put / 8 | %%% replica put case this is not needed. 9 | 10 | fetch_external(Domain, Home) -> 11 | fetch_external(Domain, Home, [], {none, none}). 12 | 13 | fetch_external(Domain, Home, [{_, {_, TmpFile, _}} = Entry|FetchList], 14 | {none, none}) -> 15 | 16 | {_, Ref} = spawn_monitor(fun() -> fetch(Entry) end), 17 | fetch_external(Domain, Home, FetchList, {Ref, TmpFile}); 18 | 19 | fetch_external(Domain, Home, FetchList, {Worker, TmpFile} = P) -> 20 | receive 21 | {fetch, {From, Entry}} -> 22 | fetch_external(Domain, Home, [{From, 23 | parse_entry(Home, Entry)}|FetchList], P); 24 | {'DOWN', Worker, _, _, normal} when FetchList == []-> 25 | error_logger:info_report( 26 | {"File", TmpFile, "fetched ok"}), 27 | gen_server:cast(Domain, update_domain_size), 28 | fetch_external(Domain, Home, FetchList, {none, none}); 29 | {'DOWN', Worker, _, _, normal} -> 30 | fetch_external(Domain, Home, FetchList, {none, none}); 31 | {'DOWN', Worker, _, _, Reason} -> 32 | error_logger:info_report( 33 | {"File", TmpFile, "error", Reason}), 34 | file:delete(TmpFile), 35 | fetch_external(Domain, Home, FetchList, {none, none}) 36 | end. 37 | 38 | parse_entry(Home, Entry) -> 39 | {_, _, _, _, {ext, {_CRC, SrcFile}}} = ringo_reader:decode(Entry), 40 | FetchID = integer_to_list(random:uniform(4294967295)), 41 | TmpFile = filename:join(Home, lists:flatten( 42 | [SrcFile, "-", FetchID, ".partial"])), 43 | DstFile = filename:join(Home, SrcFile), 44 | {SrcFile, TmpFile, DstFile}. 45 | 46 | fetch({From, {SrcFile, TmpFile, DstFile}}) -> 47 | {ok, SrcIO} = gen_server:call(From, {get_file_handle, SrcFile}), 48 | link(SrcIO), 49 | {ok, DstIO} = file:open(TmpFile, [write, binary, raw]), 50 | T = now(), 51 | {ok, _} = file:copy(SrcIO, DstIO), 52 | error_logger:info_report({"File", SrcFile, "copied in", 53 | timer:now_diff(now(), T) div 1000, "ms"}), 54 | file:rename(TmpFile, DstFile), 55 | file:close(SrcIO), 56 | file:close(DstIO). 57 | 58 | 59 | %%% 60 | %%% check_external is a periodic process that goes through the DB file, and for 61 | %%% each external entry, checks that the corresponding file really exists on 62 | %%% the domain directory. If it doesn't, it tries to find a copy of the file 63 | %%% from another node using a find_file request. Once a copy has been found, 64 | %%% it sends a fetch request to the fetch_external process (above). 65 | %%% 66 | %%% Note that the find_file request is made only for the first missing file. 67 | %%% If multiple files are missing, we opportunistically assume that the same 68 | %%% node might have a copy of them, too. If the assumption fails, next time 69 | %%% we will make a find_file request for the second missing file etc. In the 70 | %%% worst case, finding N missing files requires N check_external() runs and 71 | %%% N find_file requests to be found. 72 | %%% 73 | 74 | check_external(This) -> 75 | error_logger:info_report({"Check external starts"}), 76 | #domain{dbname = DBName, stats = Stats, home = Home, extproc = Ext} 77 | = gen_server:call(This, get_current_state), 78 | % Note that this doesn't sync iblocks since ringo_reader:fold() ignores 79 | % them. If ringo_reader:fold() gets a flag that allows iblocks to be 80 | % included in the iteration, we could sync them here as well. 81 | {NExt, NMiss, _} = ringo_reader:fold(fun(_, _, _, _, Entry, Nfo) -> 82 | check_entry(ringo_reader:is_external(Entry), 83 | Home, Entry, This, Nfo, Ext) 84 | end, {0, 0, none}, DBName), 85 | error_logger:info_report({"Check external finishes:", 86 | NExt, "external", NMiss, "missing"}), 87 | if NMiss > 0 -> 88 | error_logger:info_report({"Update domain size"}), 89 | gen_server:cast(This, update_domain_size); 90 | true -> ok 91 | end, 92 | ringo_domain:stats_buffer_add(Stats, external_entries, {NExt, NMiss}). 93 | 94 | % internal entry, do nothing 95 | check_entry(false, _, _, _, Nfo, _) -> Nfo; 96 | % external entry, check that the file exists 97 | check_entry(true, Home, Entry, This, {N, M, FileSrc}, Ext) -> 98 | % CRC checkin could be added here to detect random disk corruption 99 | {_, _, _, _, {ext, {_CRC, ExtFile}}} = ringo_reader:decode(Entry), 100 | case file:read_file_info(filename:join(Home, ExtFile)) of 101 | {ok, _} -> {N + 1, M, FileSrc}; 102 | _ -> {N + 1, M + 1, handle_missing_file(This, 103 | Entry, ExtFile, FileSrc, Ext)} 104 | end. 105 | 106 | % We don't know yet from which node we can fetch the missing files -> 107 | % Find it out with a find_file request. 108 | handle_missing_file(This, Entry, ExtFile, none, Ext) -> 109 | error_logger:warning_report({"File", ExtFile, "missing"}), 110 | gen_server:cast(This, {find_file, ExtFile, {self(), node()}, 0}), 111 | receive 112 | {file_found, ExtFile, FileSrc} -> handle_missing_file(This, 113 | Entry, ExtFile, FileSrc, Ext); 114 | _ -> none 115 | after 30000 -> none 116 | end; 117 | 118 | % We already know that FileSrc matched some previous missing file, so quite 119 | % likely it has this Entry as well. If not, fetch() will fail, but maybe next 120 | % time we will make a find_file call specifically for this Entry. 121 | handle_missing_file(_, Entry, _, FileSrc, Ext) -> 122 | Ext ! {fetch, {FileSrc, Entry}}, 123 | FileSrc. 124 | 125 | 126 | -------------------------------------------------------------------------------- /ring/src/ringo_index.erl: -------------------------------------------------------------------------------- 1 | -module(ringo_index). 2 | -export([build_index/3, fetch_entry/4, new_dex/0, add_item/3, serialize/1]). 3 | -export([deserialize/1, dexhash/1, find_key/2, find_key/3, decode_poslist/1]). 4 | 5 | -include("ringo_store.hrl"). 6 | 7 | % overwrite flag: index only the newest instance of this key 8 | 9 | build_index(DBName, StartPos, Num) -> 10 | ringo_reader:fold(fun 11 | (_, _, _, _, _, {N, _, _} = A, _) when N == Num -> 12 | throw({eof, A}); 13 | (Key, _, _Flags, _, Entry, {N, Dex, _}, Pos) -> 14 | {N + 1, add_item(Dex, Key, Pos), Pos + size(Entry)} 15 | end, {0, new_dex(), 0}, DBName, true, StartPos). 16 | 17 | fetch_entry(DB, Home, Key, Offset) -> 18 | case ringo_reader:read_entry(DB, Offset) of 19 | % key matches -- a valid external entry 20 | {Time, _, Flags, Key, V, _} when ?FLAG_UP(Flags, ?EXT_FLAG) -> 21 | case ringo_reader:read_external(Home, V) of 22 | {ok, Value} -> {Time, Key, Value}; 23 | _ -> invalid_entry 24 | end; 25 | % key matches -- a valid internal entry 26 | {Time, _, _, Key, Value, _} -> {Time, Key, Value}; 27 | % entry corrupted -- should not happen 28 | invalid_entry -> invalid_entry; 29 | % hash collision -- ignore this entry 30 | _ -> ignore 31 | end. 32 | 33 | new_dex() -> gb_trees:empty(). 34 | 35 | dexhash(Key) -> 36 | <> = erlang:md5(Key), Hash. 37 | 38 | add_item(Dex, Key, Pos) -> 39 | Hash = dexhash(Key), 40 | gb_trees:enter(Hash, add_pos(gb_trees:lookup(Hash, Dex), Pos), Dex). 41 | 42 | add_pos(none, Pos) -> 43 | {Pos, <>}; 44 | 45 | add_pos({value, {PrevPos, Lst}}, Pos) -> 46 | {Pos, <>}. 47 | 48 | serialize(Dex) -> 49 | L = gb_trees:to_list(Dex), 50 | {Single, Multi} = lists:partition(fun 51 | ({_, {_, <<_:32>>}}) -> true; 52 | (_) -> false 53 | end, L), 54 | SingleSeg = bin_util:encode_kvsegment( 55 | [{K, V} || {K, {_, <>}} <- Single]), 56 | {KeysOffs, _} = lists:mapfoldl(fun({K, {_, V}}, Offs) -> 57 | % We mark end of the offset list with an elias-encoded value 1. 58 | % This is safe, since real offsets are always at least 40 bytes, 59 | % because of entry headers etc. 60 | VA = <>, 61 | {{{K, Offs}, VA}, Offs + bit_size(VA)} 62 | end, 0, Multi), 63 | {MKeys, MOffs} = lists:unzip(KeysOffs), 64 | MultiSeg = bin_util:encode_kvsegment(MKeys), 65 | Offsets = bin_util:pad(list_to_bitstring(MOffs)), 66 | [<<(size(SingleSeg)):32, (size(MultiSeg)):32, (size(Offsets)):32>>, 67 | SingleSeg, MultiSeg, Offsets]. 68 | 69 | deserialize(Dex) -> 70 | <> = Dex, 74 | {SingleSeg, MultiSeg, Offsets}. 75 | 76 | find_key(Key, Dex) -> find_key(Key, Dex, true). 77 | 78 | find_key(Key, Dex, DoDecode) when is_binary(Key) -> 79 | find_key(dexhash(Key), Dex, DoDecode); 80 | 81 | % non-serialized index. NB: Make sure that when-clause matches with the actual 82 | % data structure used by the active index -- gb_trees is a tuple. 83 | find_key(Key, Dex, DoDecode) when is_tuple(Dex) -> 84 | case gb_trees:lookup(Key, Dex) of 85 | none -> {Key, []}; 86 | {value, {_, Lst}} when DoDecode == true -> 87 | {Key, decode_poslist(Lst)}; 88 | {value, {_, Lst}} -> {Key, Lst} 89 | end; 90 | 91 | % serialized index 92 | find_key(Key, Dex, DoDecode) when is_binary(Dex) -> 93 | <> = Dex, 97 | 98 | {Key, R} = bin_util:find_kv(Key, SingleSeg), 99 | if R == none -> 100 | {Key, Offs} = bin_util:find_kv(Key, MultiSeg), 101 | if Offs == none -> {Key, []}; 102 | DoDecode == true -> 103 | <<_:Offs/bits, V/bits>> = Offsets, 104 | {Key, decode_poslist(V)}; 105 | true -> 106 | <<_:Offs/bits, V/bits>> = Offsets, 107 | {Key, V} 108 | end; 109 | DoDecode == true -> {Key, [R]}; 110 | true -> {Key, <>} 111 | end. 112 | 113 | decode_poslist(<>) -> decode_poslist(B, [P]). 114 | decode_poslist(<<>>, L) -> lists:reverse(L); 115 | decode_poslist(B, [P|_] = L) -> 116 | case elias_decode(B) of 117 | {1, _} -> lists:reverse(L); 118 | {D, Rest} -> decode_poslist(Rest, [P + D|L]) 119 | end. 120 | 121 | elias_encode(0) -> throw(zero_elias); 122 | elias_encode(X) -> <>. 123 | 124 | elias_decode(B) -> elias_decode(B, 1). 125 | elias_decode(B, N) -> 126 | case B of 127 | <<1:N, _/bits>> -> 128 | M = N - 1, 129 | <<_:M, X:N, R/bits>> = B, 130 | {X, R}; 131 | _ -> elias_decode(B, N + 1) 132 | end. 133 | 134 | 135 | 136 | 137 | -------------------------------------------------------------------------------- /ring/src/ringo_main.erl: -------------------------------------------------------------------------------- 1 | -module(ringo_main). 2 | -behaviour(supervisor). 3 | -behaviour(application). 4 | 5 | -compile([verbose, report_errors, report_warnings, trace, debug_info]). 6 | -define(MAX_R, 10). 7 | -define(MAX_T, 60). 8 | 9 | -export([init/1, start/2, stop/1]). 10 | 11 | conf(P) -> 12 | case application:get_env(P) of 13 | undefined -> error_logger:error_report( 14 | [{"Parameter missing", P}]), 15 | exit(normal); 16 | {ok, Val} -> Val 17 | end. 18 | 19 | parse_path([Id|_]) -> Id; 20 | parse_path(_) -> error_logger:error_report( 21 | {"Invalid ringo_home path", conf(ringo_home)}), 22 | exit(normal). 23 | 24 | check_nodename(Id) -> 25 | [T|_] = string:tokens(atom_to_list(node()), "@"), 26 | E = "ringo-" ++ Id, 27 | if T =/= E -> 28 | error_logger:error_report( 29 | {"Node name should be", E, "not", T}), 30 | exit(normal); 31 | true -> ok 32 | end. 33 | 34 | start(_Type, _Args) -> 35 | Home = conf(ringo_home), 36 | Id = parse_path(lists:reverse(string:tokens(Home, "/"))), 37 | check_nodename(Id), 38 | supervisor:start_link(ringo_main, [Home, erlang:list_to_integer(Id, 16)]). 39 | 40 | init([Home, Id]) -> 41 | error_logger:info_report([{"RINGO NODE", Id, "BOOTS"}]), 42 | bfile:load_driver(), 43 | {ok, {{one_for_one, ?MAX_R, ?MAX_T}, 44 | [{ringo_node, {ringo_node, start_link, [Home, Id]}, 45 | permanent, 10, worker, dynamic}] 46 | }}. 47 | 48 | stop(_State) -> 49 | ok. 50 | 51 | -------------------------------------------------------------------------------- /ring/src/ringo_node.hrl: -------------------------------------------------------------------------------- 1 | 2 | -define(MAX_RING_SIZE, 50000). 3 | 4 | -record(rnode, {myid, previd, nextid, prevnode, nextnode, route, home}). 5 | -------------------------------------------------------------------------------- /ring/src/ringo_reader.erl: -------------------------------------------------------------------------------- 1 | -module(ringo_reader). 2 | 3 | -export([fold/3, fold/5, read_entry/2, is_external/1, decode/1]). 4 | -export([read_external/2, parse_flags/1, read_file/1]). 5 | -include("ringo_store.hrl"). 6 | 7 | -define(NI, :32/little). 8 | -record(iter, {db, f, prev, prev_head, acc, skipbad}). 9 | 10 | is_external(X) when is_list(X) -> is_external(iolist_to_binary(X)); 11 | is_external(<<_:16/binary, _:7, 1:1, _/binary>>) -> true; 12 | is_external(_) -> false. 13 | 14 | decode(X) when is_list(X) -> 15 | decode(iolist_to_binary(X)); 16 | 17 | decode(<>) -> 20 | 21 | Flags = parse_flags(FlagsB), 22 | Value = case proplists:is_defined(external, Flags) of 23 | true -> <> = ValueB, 24 | {ext, {FileCRC, binary_to_list(FileName)}}; 25 | false -> {int, ValueB} 26 | end, 27 | {Time, EntryID, Flags, Key, Value}. 28 | 29 | fold(F, Acc0, DBName) -> 30 | fold(F, Acc0, DBName, false, 0). 31 | 32 | fold(F, Acc0, DBName, WithPos, StartPos) -> 33 | {ok, DB} = bfile:fopen(DBName, "r"), 34 | if StartPos > 0 -> 35 | ok = bfile:fseek(DB, StartPos, seek_set); 36 | true -> ok 37 | end, 38 | try 39 | read_item(#iter{db = DB, f = F, prev = {0, 0}, skipbad = true, 40 | prev_head = StartPos, acc = Acc0}, WithPos) 41 | catch 42 | {eof, #iter{acc = Acc}} -> 43 | bfile:fclose(DB), 44 | Acc; 45 | % callback function may throw(eof) if it wants 46 | % to stop iterating 47 | {eof, Acc} -> 48 | bfile:fclose(DB), 49 | Acc 50 | end. 51 | 52 | read_item(#iter{f = F, prev = Prev, acc = Acc} = Q, WithPos) -> 53 | {NQ, {Time, EntryID, Flags, Key, Val, Entry}} = read_entry(Q), 54 | ID = {Time, EntryID}, 55 | % skip duplicate items 56 | AccN = if Prev == EntryID -> Acc; 57 | % skip iblocks 58 | ?FLAG_UP(Flags, ?IBLOCK_FLAG) -> Acc; 59 | true -> 60 | PFlags = parse_flags(Flags), 61 | if WithPos -> 62 | Pos = NQ#iter.prev_head - 8, 63 | F(Key, Val, PFlags, ID, Entry, Acc, Pos); 64 | true -> 65 | F(Key, Val, PFlags, ID, Entry, Acc) 66 | end 67 | end, 68 | read_item(NQ#iter{prev = EntryID, acc = AccN}, WithPos). 69 | 70 | read_entry(DB, Pos) -> 71 | ok = bfile:fseek(DB, Pos, seek_set), 72 | Q = #iter{db = DB, skipbad = false}, 73 | {_, R} = read_head(Q, read(Q, 8)), 74 | R. 75 | 76 | read_entry(Q) -> 77 | read_head(Q, read(Q, 8)). 78 | 79 | % If reading an entry fails, we have no guarantee on how many bytes 80 | % actually belong to this entry (in the extreme case only the magic head 81 | % is valid), so we have to backtrack to the field head and continue 82 | % seeking for the next magic head from there. Hence we need to 83 | % record position of the latest entry head below. 84 | read_head(#iter{db = DB} = Q, <> = PHead) -> 85 | {ok, Pos} = bfile:ftell(DB), 86 | NQ = Q#iter{prev_head = Pos}, 87 | Head = read(NQ, 7 * 4), 88 | read_body(NQ, Head, check(Head, HeadCRC), PHead); 89 | 90 | read_head(Q, _) -> seek_magic(Q). 91 | 92 | read_body(Q, <> = Head, true, PHead) -> 94 | 95 | <> = Body = 96 | read(Q, KeyLen + ValLen + 4), 97 | Entry = <>, 98 | validate(Q, {Time, EntryID, Flags, Key, Val, Entry}, 99 | check(Key, KeyCRC), check(Val, ValCRC), End); 100 | 101 | read_body(Q, _, false, _) -> seek_magic(Q). 102 | 103 | validate(Q, Ret, true, true, ?MAGIC_TAIL_B) -> {Q, Ret}; 104 | validate(Q, _, _, _, _) -> seek_magic(Q). 105 | 106 | check(Val, CRC) -> erlang:crc32(Val) == CRC. 107 | 108 | seek_magic(#iter{skipbad = false}) -> invalid_entry; 109 | 110 | seek_magic(#iter{db = DB, prev_head = 0} = Q) -> 111 | ok = bfile:fseek(DB, 1, seek_set), 112 | seek_magic(Q, read(Q, 8)); 113 | 114 | seek_magic(#iter{db = DB, prev_head = Pos} = Q) -> 115 | ok = bfile:fseek(DB, Pos - 7, seek_set), 116 | seek_magic(Q, read(Q, 8)). 117 | 118 | % Found a potentially valid entry head 119 | seek_magic(Q, <> = D) -> 120 | read_head(Q, D); 121 | 122 | % Skip a byte, continue seeking 123 | seek_magic(Q, <<_:1/binary, D/binary>>) -> 124 | E = read(Q, 1), 125 | seek_magic(Q, <>). 126 | 127 | read(#iter{db = DB} = Q, N) -> 128 | % fread() should return a short byte count only if an error 129 | % occurs or eof is reached, which should not happen here. 130 | % However, we may try to read a new entry that hasn't hit 131 | % the disk yet fully, causing a short byte count. This 132 | % shouldn't be considered an error. 133 | case bfile:fread(DB, N) of 134 | {ok, D} when size(D) == N -> D; 135 | {ok, _} -> throw({eof, Q}); 136 | eof -> throw({eof, Q}); 137 | Error -> throw(Error) 138 | end. 139 | 140 | %read(_, B, {ok, D}, N) when size(B) + size(D) == N -> 141 | % {ok, <>}; 142 | %read(F, B, {ok, D}, N) -> 143 | % B0 = <>, 144 | % read(F, B0, bfile:fread(F, N - size(B0)), N); 145 | %read(_, _, R, _) -> R. 146 | 147 | parse_flags(Flags) -> 148 | [S || {S, F} <- ?FLAGS, Flags band F > 0]. 149 | 150 | read_external(Home, <<_CRC:32, ExtFile/binary>>) -> 151 | ExtPath = filename:join(Home, binary_to_list(ExtFile)), 152 | case read_file(ExtPath) of 153 | {error, Reason} -> {io_error, Reason}; 154 | {ok, _Value} = R -> R 155 | %V = erlang:crc32(Value), 156 | %if V == CRC -> {ok, Value}; 157 | %true -> corrupted_file 158 | %end 159 | end. 160 | 161 | 162 | read_file(FName) -> 163 | case bfile:fopen(FName, "r") of 164 | {ok, F} -> read_file(F, <<>>, bfile:fread(F, 65536)); 165 | E -> E 166 | end. 167 | 168 | read_file(F, D, {ok, B}) -> 169 | read_file(F, <>, bfile:fread(F, 65536)); 170 | 171 | read_file(F, D, eof) -> 172 | bfile:fclose(F), {ok, D}. 173 | 174 | 175 | 176 | -------------------------------------------------------------------------------- /ring/src/ringo_store.hrl: -------------------------------------------------------------------------------- 1 | 2 | -define(RDONLY, 8#00400). 3 | -define(KEY_MAX, 4096). 4 | -define(VAL_INTERNAL_MAX, 4096). 5 | -define(MAGIC_HEAD, 16#47da66b5). 6 | -define(MAGIC_TAIL, 16#acc50f5d). 7 | -define(MAGIC_HEAD_B, <<16#47da66b5:32/little>>). 8 | -define(MAGIC_TAIL_B, <<16#acc50f5d:32/little>>). 9 | -define(EXT_FLAG, 1). 10 | -define(IBLOCK_FLAG, 2). 11 | -define(FLAGS, [{external, 1}, {iblock, 2}]). 12 | -define(FLAG_UP(Flags, Flag), Flags band Flag =/= 0). 13 | 14 | -define(NUM_MERKLE_LEAVES, 8192). 15 | 16 | -record(domain, {this, owner, home, host, id, db, size, full, num_entries, 17 | sync_tree, sync_ids, sync_inbox, sync_outbox, dbname, stats, info, 18 | nextnode, prevnode, extproc, index, max_repl_entries, 19 | domain_chunk_max}). 20 | -------------------------------------------------------------------------------- /ring/src/ringo_sync.erl: -------------------------------------------------------------------------------- 1 | -module(ringo_sync). 2 | 3 | -export([make_leaf_hashes_and_ids/1, make_leaf_hashes/1, 4 | make_leaf_hashes/3, build_merkle_tree/1, 5 | sync_id/2, sync_id_slot/1, update_leaf_ids/2, update_leaf_hashes/2, 6 | collect_leaves/2, in_leaves/2, diff_parents/3, count_entries/1, 7 | pick_children/3]). 8 | 9 | -include("ringo_store.hrl"). 10 | %%% 11 | %%% 12 | %%% 13 | 14 | % IDEA: Instead of binary lists, we could use Bloom filters to save 15 | % the ID lists. False positives don't matter -- we just accidentally 16 | % skip an entry that should have been synced. The trick is that if 17 | % we use _independent_ hashes (changing random salt) on each rsync 18 | % round, probability that the same entry will be skipped on N rounds 19 | % is p^N, which makes it practically certain that the entry will get 20 | % synchronized eventually. 21 | make_leaf_hashes_and_ids(empty) -> 22 | LeafIDs = dict:from_list( 23 | [{I, <<>>} || I <- lists:seq(0, ?NUM_MERKLE_LEAVES - 1)]), 24 | LeafHashes = ets:new(leaves, []), 25 | ets:insert(LeafHashes, 26 | [{I, 0} || I <- lists:seq(0, ?NUM_MERKLE_LEAVES - 1)]), 27 | {LeafHashes, LeafIDs}; 28 | 29 | make_leaf_hashes_and_ids(DBName) -> 30 | LeafIDs = dict:from_list( 31 | [{I, <<>>} || I <- lists:seq(0, ?NUM_MERKLE_LEAVES - 1)]), 32 | make_leaf_hashes(DBName, fun(Leaf, SyncID, LLeafIDs) -> 33 | Lst = dict:fetch(Leaf, LLeafIDs), 34 | % Binary append should be fast in R12B. This might be REALLY 35 | % slow on earlier revisions (see Erlang Efficiency Guide). 36 | dict:store(Leaf, <>, LLeafIDs) 37 | % to see that the append-optimization relly works, use the 38 | % inserted value for matching here, which should force the 39 | % binary to be copied. If the function becomes considerably 40 | % slower this way, we can assume that the optimization works. 41 | end, LeafIDs). 42 | 43 | make_leaf_hashes(DBName) -> 44 | {LeafHashes, _} = make_leaf_hashes(DBName, none, []), 45 | LeafHashes. 46 | 47 | make_leaf_hashes(DBName, F, Acc0) -> 48 | LeafHashes = ets:new(leaves, []), 49 | % create the leaves 50 | ets:insert(LeafHashes, 51 | [{I, 0} || I <- lists:seq(0, ?NUM_MERKLE_LEAVES - 1)]), 52 | AccF = ringo_reader:fold(fun(_, _, _, {Time, EntryID}, _, Acc) -> 53 | {Leaf, SyncID} = sync_id(EntryID, Time), 54 | update_leaf_hashes(LeafHashes, SyncID), 55 | if is_function(F) -> 56 | F(Leaf, SyncID, Acc); 57 | true -> ok 58 | end 59 | end, Acc0, DBName), 60 | {LeafHashes, AccF}. 61 | 62 | count_entries(LeafIDs) -> 63 | dict:fold(fun(_Leaf, IDList, N) -> 64 | N + size(IDList) div 8 65 | end, 0, LeafIDs). 66 | 67 | build_merkle_tree(LeafHashes) -> 68 | {_, Leaves} = lists:unzip(lists:sort(ets:tab2list(LeafHashes))), 69 | Tree = make_next_level(Leaves, [Leaves]), 70 | Tree. 71 | 72 | % update_leaf_hashes must implement a commutative (accumulated) 73 | % hashing function. It should be reasonably collision-free. However, 74 | % we can survive a modest number of collisions, since once a leaf gets 75 | % more IDs, its hash changes, and the probability of many consequent 76 | % collisions for the same leaf is hopefully small. 77 | % 78 | % Furthermore, by using only the EntryID part of SyncID for hashing, 79 | % input values are likely to be uniformly distributed, as EntryID is 80 | % defined as follows: EntryID = random:uniform(4294967295). 81 | 82 | update_leaf_hashes(LeafHashes, <<_Time:32, EntryID:32>> = SyncID) -> 83 | Leaf = sync_id_slot(SyncID), 84 | [{_, P}] = ets:lookup(LeafHashes, Leaf), 85 | ets:insert(LeafHashes, {Leaf, P bxor EntryID}). 86 | 87 | update_leaf_ids(LeafIDs, SyncID) -> 88 | Leaf = sync_id_slot(SyncID), 89 | Lst = dict:fetch(Leaf, LeafIDs), 90 | dict:store(Leaf, <>, LeafIDs). 91 | 92 | in_leaves(LeafIDs, SyncID) -> 93 | Leaf = sync_id_slot(SyncID), 94 | Lst = dict:fetch(Leaf, LeafIDs), 95 | bin_util:member64(Lst, SyncID). 96 | 97 | sync_id(EntryID, Time) -> 98 | % SyncID must have lots of entropy in the least significant 99 | % bits to ensure uniform allocation of the Merkle leaves 100 | Leaf = EntryID band (?NUM_MERKLE_LEAVES - 1), 101 | {Leaf, <>}. 102 | 103 | sync_id_slot(<<_Time:32, EntryID:32>>) -> 104 | EntryID band (?NUM_MERKLE_LEAVES - 1). 105 | 106 | make_next_level([_], Tree) -> Tree; 107 | make_next_level(Level, Tree) -> 108 | L = make_level(Level, []), 109 | make_next_level(L, [L|Tree]). 110 | 111 | make_level([], L) -> lists:reverse(L); 112 | make_level([X, Y|R], L) -> 113 | make_level(R, [erlang:crc32(<>)|L]). 114 | 115 | %%% 116 | %%% 117 | %%% 118 | 119 | diff_parents(H, RLevel, OTree) -> 120 | OLevel = lists:zip(lists:seq(0, 1 bsl (H - 1) - 1), 121 | lists:nth(H, OTree)), 122 | %io:fwrite("Rlevel ~w OLevel ~w~n", [RLevel, OLevel]), 123 | diff(OLevel, RLevel, []). 124 | 125 | diff(_, [], Res) -> 126 | lists:reverse(Res); 127 | diff([{N1, _}|_], [{N2, _}|_], Res) when N1 > N2 -> 128 | Res; 129 | diff([{N1, _}|R1], [{N2, _}|_] = L2, Res) when N1 < N2 -> 130 | diff(R1, L2, Res); 131 | diff([{N1, X1}|R1], [{_, X2}|R2], Res) when X1 =/= X2 -> 132 | diff(R1, R2, [N1|Res]); 133 | diff([_|R1], [_|R2], Res) -> 134 | diff(R1, R2, Res). 135 | 136 | %%% 137 | %%% 138 | %%% 139 | 140 | pick_children(H, Parents, Tree) -> 141 | Level = lists:zip(lists:seq(0, 1 bsl (H - 1) - 1), 142 | lists:nth(H, Tree)), 143 | pick(Level, Parents, 0, []). 144 | 145 | pick(_, [], _, Res) -> 146 | lists:reverse(Res); 147 | pick([X, Y|Level], [P|Parents], N, Res) when N == P -> 148 | pick(Level, Parents, N + 1, [Y, X|Res]); 149 | pick([_, _|Level], Parents, N, Res) -> 150 | pick(Level, Parents, N + 1, Res). 151 | 152 | %%% 153 | %%% 154 | %%% 155 | 156 | collect_leaves([], _) -> []; 157 | collect_leaves(LeafList, empty) -> [{N, []} || N <- LeafList]; 158 | collect_leaves(LeafList, DBName) -> 159 | LeafBag = ets:new(leaves, [bag]), 160 | ets:insert(LeafBag, [{N, empty} || N <- LeafList]), 161 | ringo_reader:fold(fun(_, _, _, {Time, EntryID}, _, _) -> 162 | {Leaf, SyncID} = sync_id(EntryID, Time), 163 | case ets:member(LeafBag, Leaf) of 164 | true -> ets:insert(LeafBag, {Leaf, SyncID}); 165 | false -> ok 166 | end 167 | end, ok, DBName), 168 | List = group_results(ets:tab2list(LeafBag)), 169 | ets:delete(LeafBag), 170 | List. 171 | 172 | %%% 173 | %%% 174 | %%% 175 | 176 | 177 | 178 | % This function converts lists of form: [{2, A}, {3, B}, {2, C}] 179 | % to form: [{2, [A, C]}, {3, [B]}] (shamelessly copied from 180 | % disco/handle_job.erl). 181 | group_results(List) -> 182 | lists:foldl(fun 183 | ({PartID, R}, []) -> 184 | [{PartID, [R]}]; 185 | ({PartID, R}, [{PrevID, Lst}|Rest]) when PrevID == PartID -> 186 | [{PartID, [R|Lst]}|Rest]; 187 | ({PartID, R}, [{PrevID, _}|_] = Q) when PrevID =/= PartID -> 188 | [{PartID, [R]}|Q] 189 | end, [], lists:keysort(1, List)). 190 | -------------------------------------------------------------------------------- /ring/src/ringo_syncdomain.erl: -------------------------------------------------------------------------------- 1 | -module(ringo_syncdomain). 2 | 3 | -export([resync/2, global_resync/1]). 4 | 5 | -include("ringo_store.hrl"). 6 | 7 | % Premises about resync: 8 | % 9 | % - We can't know for sure on which nodes all replicas of this domain exist. Nodes may 10 | % have disappeared, new nodes may have been added etc. Hence, we have to go 11 | % through the whole ring to find the replicas. 12 | % 13 | % - Since the ring may contain any arbitrary collection of nodes at any point of 14 | % time 15 | 16 | %%% 17 | %%% Synchronization 18 | %%% 19 | 20 | % Owner-end in synchronization -- just update the tree 21 | resync(This, owner) -> 22 | error_logger:info_report({"resync owner"}), 23 | #domain{dbname = DBName, db = DB, owner = true, stats = Stats} = 24 | gen_server:call(This, get_current_state), 25 | 26 | ringo_domain:stats_buffer_add(Stats, sync_time, nu), 27 | 28 | DBN = if DB == none -> empty; true -> DBName end, 29 | 30 | {[[Root]|_] = Tree, LeafIDsX, _} = update_sync_tree(This, DBN), 31 | gen_server:cast(This, {update_sync_data, Tree, LeafIDsX}), 32 | ringo_domain:stats_buffer_add(Stats, synctree_root, Root), 33 | flush_sync_outbox(This, DBN); 34 | 35 | % Replica-end in synchronization -- the active party 36 | resync(This, replica) -> 37 | error_logger:info_report({"resync replica"}), 38 | #domain{dbname = DBName, db = DB, id = DomainID, 39 | owner = false, stats = Stats} 40 | = gen_server:call(This, get_current_state), 41 | 42 | ringo_domain:stats_buffer_add(Stats, sync_time, nu), 43 | 44 | DBN = if DB == none -> empty; true -> DBName end, 45 | 46 | {[[Root]|_] = Tree, _, NumEntries} = update_sync_tree(This, DBN), 47 | ringo_domain:stats_buffer_add(Stats, synctree_root, Root), 48 | {ok, {_, OPid}, Distance} = find_owner(DomainID), 49 | % BUG: Shouldn't the domain die if find_owner fails? Now it seems 50 | % that only the resync process dies. Namely, how to make sure that 51 | % if there're two owners, one of them will surely die. 52 | DiffLeaves = merkle_sync(OPid, [{0, Root}], 1, Tree), 53 | ringo_domain:stats_buffer_add(Stats, diff_size, length(DiffLeaves)), 54 | if DiffLeaves == [] -> ok; 55 | true -> 56 | SyncIDs = ringo_sync:collect_leaves(DiffLeaves, DBName), 57 | gen_server:cast(OPid, 58 | {sync_pack, This, SyncIDs, NumEntries, Distance}) 59 | end, 60 | flush_sync_outbox(This, DBN). 61 | 62 | % merkle_sync compares the replica's Merkle tree to the owner's 63 | 64 | % Trees are in sync 65 | merkle_sync(_OPid, [], 2, _Tree) -> []; 66 | 67 | merkle_sync(OPid, Level, H, Tree) -> 68 | {ok, Diff} = gen_server:call(OPid, {sync_tree, H, Level}), 69 | if H < length(Tree) -> 70 | merkle_sync(OPid, ringo_sync:pick_children(H + 1, Diff, Tree), 71 | H + 1, Tree); 72 | true -> 73 | Diff 74 | end. 75 | 76 | % update_sync_tree scans entries in this DB, collects all entry IDs and 77 | % computes the leaf hashes. Entries in the inbox that don't exist in the DB 78 | % already are written to disk. Finally Merkle tree is re-built. 79 | 80 | update_sync_tree(This, DBName) -> 81 | error_logger:info_report({"update sync tree", DBName}), 82 | {ok, Inbox} = gen_server:call(This, {flush_syncbox, sync_inbox}), 83 | %error_logger:info_report({"INBOX", Inbox}), 84 | {LeafHashes, LeafIDs} = ringo_sync:make_leaf_hashes_and_ids(DBName), 85 | NumEntries = ringo_sync:count_entries(LeafIDs), 86 | gen_server:cast(This, {update_num_entries, NumEntries}), 87 | %error_logger:info_report({"LeafIDs", LeafIDs}), 88 | Entries = lists:filter(fun({SyncID, _, _}) -> 89 | not ringo_sync:in_leaves(LeafIDs, SyncID) 90 | end, Inbox), 91 | {LeafHashesX, LeafIDsX} = flush_sync_inbox(This, Entries, 92 | LeafHashes, LeafIDs), 93 | Tree = ringo_sync:build_merkle_tree(LeafHashesX), 94 | ets:delete(LeafHashesX), 95 | {Tree, LeafIDsX, NumEntries}. 96 | 97 | % flush_sync_inbox writes entries that have been sent to this replica 98 | % (or owner) to disk and updates the leaf hashes accordingly 99 | flush_sync_inbox(_, [], LeafHashes, LeafIDs) -> 100 | %error_logger:info_report({"flush inbox (empty)"}), 101 | {LeafHashes, LeafIDs}; 102 | 103 | flush_sync_inbox(This, [{SyncID, Entry, From}|Rest], LeafHashes, LeafIDs) -> 104 | %error_logger:info_report({"flush inbox"}), 105 | ringo_sync:update_leaf_hashes(LeafHashes, SyncID), 106 | LeafIDsX = ringo_sync:update_leaf_ids(LeafIDs, SyncID), 107 | gen_server:cast(This, {sync_write_entry, Entry, From}), 108 | flush_sync_inbox(This, Rest, LeafHashes, LeafIDsX). 109 | 110 | % flush_sync_outbox sends entries that exists on this replica or owner to 111 | % another node that requested the entry 112 | flush_sync_outbox(_, empty) -> 113 | %error_logger:info_report({"flush outbox (empty)"}), 114 | ok; 115 | 116 | flush_sync_outbox(This, DBName) -> 117 | %error_logger:info_report({"flush outbox"}), 118 | {ok, Outbox} = gen_server:call(This, {flush_syncbox, sync_outbox}), 119 | flush_sync_outbox_1(This, DBName, Outbox). 120 | 121 | flush_sync_outbox_1(_, _, []) -> ok; 122 | flush_sync_outbox_1(This, DBName, Outbox) -> 123 | Q = ets:new(x, [bag]), 124 | ets:insert(Q, Outbox), 125 | ringo_reader:fold(fun(_, _, _, {Time, EntryID}, Entry, _) -> 126 | {_, SyncID} = ringo_sync:sync_id(EntryID, Time), 127 | case ets:lookup(Q, SyncID) of 128 | [] -> ok; 129 | L -> Msg = {sync_put, SyncID, {Entry, {}}, This}, 130 | [gen_server:cast(To, Msg) || {_, To} <- L] 131 | end 132 | end, ok, DBName), 133 | ets:delete(Q). 134 | 135 | %%% 136 | %%% Global resync 137 | %%% 138 | %%% In the normal case N replica nodes that precede the domain owner 139 | %%% are responsible for re-syncing themselves with the owner. However, 140 | %%% if the ring goes through a major re-organization, there may be 141 | %%% replicas for this domain further away in the ring as well. 142 | %%% 143 | %%% Global_resync is activated periodically by the domain owner. It 144 | %%% initiates the find_owner operation, that finds the owner node for 145 | %%% the given DomainID. Since find_owner is started by the owner 146 | %%% itself, it has to circulate through the whole ring before reaching 147 | %%% back to this node. 148 | %%% 149 | %%% On its way, it instantiates a domain server for this domain on every 150 | %%% node. Servers on nodes that don't contain a replica of this domain 151 | %%% die away immediately. The others stay alive and start the normal 152 | %%% re_sync process (above) which eventually connects to the owner and 153 | %%% goes through the standard synchronization procedure. 154 | %%% 155 | %%% If a distant replica contains entries that are missing from the 156 | %%% owner, they are sent to it as usual. Otherwise, sync_pack-handler 157 | %%% recognizes the distant replica and kills it. 158 | %%% 159 | global_resync(DomainID) -> 160 | error_logger:info_report({"global resync"}), 161 | find_owner(DomainID). 162 | 163 | find_owner(DomainID) -> 164 | {ok, _, Next} = gen_server:call(ringo_node, get_neighbors), 165 | gen_server:cast({ringo_node, Next}, 166 | {{domain, DomainID}, {find_owner, self(), 1}}), 167 | receive 168 | {owner, DomainID, Owner, Distance} -> 169 | {ok, Owner, Distance} 170 | after 10000 -> 171 | error_logger:warning_report({ 172 | "Replica resync couldn't find the owner for domain", 173 | DomainID}), 174 | timeout 175 | end. 176 | 177 | -------------------------------------------------------------------------------- /ring/src/ringo_util.erl: -------------------------------------------------------------------------------- 1 | -module(ringo_util). 2 | 3 | -export([match/2, match/4, ringo_nodes/0, validate_ring/1, domain_id/1, 4 | domain_id/2, group_pairs/1, send_system_status/1, get_param/2, 5 | format_timestamp/1, sort_nodes/1, best_matching_node/2]). 6 | 7 | -include("ringo_node.hrl"). 8 | 9 | ringo_nodes() -> 10 | Hosts = case net_adm:host_file() of 11 | {error, _} -> throw("Could not read the hosts file"); 12 | Lst -> Lst 13 | end, 14 | 15 | % Nodes list should be in a consistent order, hence sorting 16 | lists:sort(lists:flatten(lists:map(fun(Host) -> 17 | H = atom_to_list(Host), 18 | case net_adm:names(Host) of 19 | {ok, Names} -> [list_to_atom(Name ++ "@" ++ H) || 20 | {Name, _} <- Names, 21 | string:str(Name, "ringo-") > 0]; 22 | _ -> [] 23 | end 24 | end, Hosts))). 25 | 26 | sort_nodes(Nodes) -> 27 | lists:keysort(1, lists:map(fun 28 | ({N, _} = T) -> {ID, _} = nodename_to_nodeid(N), {ID, T}; 29 | (N) -> nodename_to_nodeid(N) 30 | end, Nodes)). 31 | 32 | nodename_to_nodeid(N) -> 33 | [_, X1] = string:tokens(atom_to_list(N), "-"), 34 | [ID, _] = string:tokens(X1, "@"), 35 | {erlang:list_to_integer(ID, 16), N}. 36 | 37 | best_matching_node(_DomainID, []) -> {error, no_nodes}; 38 | 39 | best_matching_node(DomainID, [{NodeID, _}|_] = Nodes) -> 40 | best_matching_node(DomainID, Nodes, NodeID + 1). 41 | 42 | best_matching_node(DomainID, 43 | [{NodeID, Node}|[{NextID, _}|_] = Nodes], PrevID) -> 44 | 45 | Match = match(DomainID, NodeID, NextID, PrevID), 46 | if Match -> 47 | {ok, Node}; 48 | true -> 49 | best_matching_node(DomainID, Nodes, NodeID) 50 | end; 51 | 52 | best_matching_node(_, [{_, Node}], _) -> {ok, Node}. 53 | 54 | validate_ring(Ring) -> 55 | Min = lists:min(Ring), 56 | Prefix = lists:takewhile(fun(X) -> X =/= Min end, Ring), 57 | Suffix = Ring -- Prefix, 58 | {_, Bad} = lists:foldl(fun 59 | ({ID, _Node}, {}) -> 60 | {ID, []}; 61 | ({ID, _Node}, {PrevID, Rogues}) when ID > PrevID -> 62 | {ID, Rogues}; 63 | ({_ID, Node}, {PrevID, Rogues}) -> 64 | {PrevID, [Node|Rogues]} 65 | end, {}, Suffix ++ Prefix), 66 | Bad. 67 | 68 | domain_id(Name) -> domain_id(Name, 0). 69 | domain_id(Name, Chunk) when is_integer(Chunk), is_list(Name) -> 70 | <> = erlang:md5([integer_to_list(Chunk), " ", Name]), 71 | ID. 72 | 73 | % This function converts lists of form: [{2, A}, {3, B}, {2, C}] 74 | % to form: [{2, [A, C]}, {3, [B]}]. 75 | group_pairs(L) -> 76 | lists:foldl(fun 77 | ({PartID, R}, []) -> 78 | [{PartID, [R]}]; 79 | ({PartID, R}, [{PrevID, Lst}|Rest]) when PrevID == PartID -> 80 | [{PartID, [R|Lst]}|Rest]; 81 | ({PartID, R}, [{PrevID, _}|_] = Q) when PrevID =/= PartID -> 82 | [{PartID, [R]}|Q] 83 | end, [], lists:keysort(1, L)). 84 | 85 | send_system_status(Parent) -> 86 | [_, UseB, _, Used|_] = lists:reverse( 87 | string:tokens(os:cmd("df -h . | tail -1"), " ")), 88 | [_, UseI|_] = lists:reverse( 89 | string:tokens(os:cmd("df -i . | tail -1"), " ")), 90 | {value, {total, Mem}} = lists:keysearch(total, 1, erlang:memory()), 91 | Parent ! {node_results, {node(), {disk, {list_to_binary(UseB), 92 | list_to_binary(UseI), list_to_binary(Used), Mem}}}}. 93 | 94 | format_timestamp(Tstamp) -> 95 | {Date, Time} = calendar:now_to_local_time(Tstamp), 96 | DateStr = io_lib:fwrite("~w/~.2.0w/~.2.0w ", tuple_to_list(Date)), 97 | TimeStr = io_lib:fwrite("~.2.0w:~.2.0w:~.2.0w", tuple_to_list(Time)), 98 | list_to_binary(DateStr ++ TimeStr). 99 | 100 | get_param(Name, Default) -> 101 | case os:getenv(Name) of 102 | false -> Default; 103 | Value when is_integer(Default) -> list_to_integer(Value); 104 | Value -> Value 105 | end. 106 | 107 | %%% Match serves as a partitioning function for consistent hashing. Node 108 | %%% with ID = X serves requests in the range [X..X+1[ where X + 1 is X's 109 | %%% successor's ID. The last node serves requests in the range [X..inf[ 110 | %%% and the first node in the range [0..X + 1[. 111 | %%% 112 | %%% Returns true if ReqID belongs to MyID. 113 | 114 | match(ReqID, #rnode{myid = MyID, nextid = NextID, previd = PrevID}) -> 115 | match(ReqID, MyID, NextID, PrevID). 116 | 117 | % request matches a middle node 118 | match(ReqID, MyID, NextID, _PrevID) when ReqID >= MyID, ReqID < NextID -> true; 119 | 120 | % request matches the last node (MyID >= NextID guarantees that the seed, 121 | % which is connected to itself, matches) 122 | match(ReqID, MyID, NextID, _PrevID) when MyID >= NextID, ReqID >= MyID -> true; 123 | 124 | % request matches the first node (MyID =< PrevID guarantees that the seed, 125 | % which is connected to itself, matches) 126 | match(ReqID, MyID, _NextID, PrevID) when MyID =< PrevID, ReqID =< MyID -> true; 127 | 128 | match(_, _, _, _) -> false. 129 | -------------------------------------------------------------------------------- /ring/src/ringo_writer.erl: -------------------------------------------------------------------------------- 1 | 2 | -module(ringo_writer). 3 | 4 | -export([write_entry/3, make_entry/4, entry_size/1]). 5 | 6 | -include("ringo_store.hrl"). 7 | -include_lib("kernel/include/file.hrl"). 8 | 9 | entry_size({Entry, {}}) -> iolist_size(Entry); 10 | entry_size({Entry, {_, Value}}) -> iolist_size(Entry) + iolist_size(Value). 11 | 12 | write_entry(_Home, DB, {Entry, {}}) -> 13 | ok = bfile:fwrite(DB, Entry), ok; 14 | 15 | write_entry(Home, DB, {Entry, {ExtFile, Value}}) -> 16 | % Write first with a different name, rename then. This ensures that 17 | % resyncing won't copy partial files. BUT: When async-threads are 18 | % enabled write_file probably doesn't block and there's no way to 19 | % know when the bits have actually hit the disk, other than syncing 20 | % every time, hence renaming wouldn't help much. 21 | ok = bfile:fwrite(DB, Entry), 22 | ExtPath = filename:join(Home, ExtFile ++ ".partial"), 23 | ExtPathReal = filename:join(Home, ExtFile), 24 | {ok, F} = bfile:fopen(ExtPath, "w"), 25 | bfile:fwrite(F, Value), 26 | bfile:fclose(F), 27 | file:rename(ExtPath, ExtPathReal), 28 | ok. 29 | 30 | make_entry(EntryID, Key, Value, Flags) 31 | when is_binary(Key), is_binary(Value), size(Key) < ?KEY_MAX, 32 | size(Value) < ?VAL_INTERNAL_MAX, Flags =/= [iblock] -> 33 | 34 | {encode(Key, Value, EntryID, Flags), {}}; 35 | 36 | % Store value to a separate file 37 | make_entry(EntryID, Key, Value, Flags) 38 | when is_binary(Key), is_binary(Value), size(Key) < ?KEY_MAX -> 39 | 40 | CRC = erlang:crc32(Value), 41 | ExtFile = if Flags == [iblock] -> 42 | binary_to_list(Key); 43 | true -> 44 | io_lib:format("value-~.16b-~.16b", [EntryID, CRC]) 45 | end, 46 | Link = [<>, ExtFile], 47 | {encode(Key, list_to_binary(Link), EntryID, [external|Flags]), 48 | {ExtFile, Value}}; 49 | 50 | make_entry(_, Key, Value, _) -> 51 | error_logger:warning_report({"Invalid put request. Key", 52 | trunc_io:fprint(Key, 500), "Value", 53 | trunc_io:fprint(Value, 500)}), 54 | invalid_request. 55 | 56 | encode(Key, Value, EntryID, FlagList) when is_binary(Key), is_binary(Value) -> 57 | Flags = lists:foldl(fun(X, F) -> 58 | {value, {_, V}} = lists:keysearch(X, 1, ?FLAGS), 59 | F bor V 60 | end, 0, FlagList), 61 | {MSecs, Secs, _} = now(), 62 | 63 | Head = [pint(MSecs * 1000000 + Secs), 64 | pint(EntryID), 65 | pint(Flags), 66 | pint(erlang:crc32(Key)), 67 | pint(size(Key)), 68 | pint(erlang:crc32(Value)), 69 | pint(size(Value))], 70 | 71 | [?MAGIC_HEAD_B, pint(erlang:crc32(Head)), 72 | Head, Key, Value, ?MAGIC_TAIL_B]. 73 | 74 | % not quite right if Value is an external 75 | %encoded_size(Key, Value) when is_binary(Key), is_binary(Value), 76 | % size(Value) < ?VAL_INTERNAL_MAX -> 77 | % 78 | % 10 * 4 + iolist_size(Key) + iolist_size(Value); 79 | % 80 | %encoded_size(Key, Value) when is_binary(Key), is_binary(Value) -> 81 | % 27 + 10 * 4 + iolist_size(Key) + iolist_size(Value). 82 | 83 | 84 | pint(V) when V < (1 bsl 32) -> 85 | <>; 86 | pint(V) -> 87 | error_logger:warning_report({"Integer overflow in ringo_codec", V}), 88 | throw(integer_overflow). 89 | 90 | 91 | 92 | 93 | 94 | -------------------------------------------------------------------------------- /ring/src/trunc_io.erl: -------------------------------------------------------------------------------- 1 | %% ``The contents of this file are subject to the Erlang Public License, 2 | %% Version 1.1, (the "License"); you may not use this file except in 3 | %% compliance with the License. You should have received a copy of the 4 | %% Erlang Public License along with your Erlang distribution. If not, it can be 5 | %% retrieved via the world wide web at http://www.erlang.org/. 6 | %% 7 | %% Software distributed under the License is distributed on an "AS IS" 8 | %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See 9 | %% the License for the specific language governing rights and limitations 10 | %% under the License. 11 | %% 12 | %% The Initial Developer of the Original Code is Corelatus AB. 13 | %% Portions created by Corelatus are Copyright 2003, Corelatus 14 | %% AB. All Rights Reserved.'' 15 | % 16 | %% 17 | %% Module to print out terms for logging. Limits by length rather than depth. 18 | %% 19 | %% The resulting string may be slightly larger than the limit; the intention 20 | %% is to provide predictable CPU and memory consumption for formatting 21 | %% terms, not produce precise string lengths. 22 | %% 23 | %% Typical use: 24 | %% 25 | %% trunc_io:print(Term, 500). 26 | %% 27 | -module(trunc_io). 28 | -author('matthias@corelatus.se'). 29 | -export([print/2, fprint/2, safe/2]). % interface functions 30 | -export([perf/0, perf/3, perf1/0, test/0, test/2]). % testing functions 31 | 32 | 33 | %% Returns an flattened list containing the ASCII representation of the given 34 | %% term. 35 | fprint(T, Max) -> 36 | {L, _} = print(T, Max), 37 | lists:flatten(L). 38 | 39 | %% Same as print, but never crashes. 40 | %% 41 | %% This is a bit of a tradeoff. Print might conceivably crash if it's 42 | %% asked to print something it doesn't understand, for example some new 43 | %% data type in a future version of Erlang. If print crashes, we fall 44 | %% back to io_lib to format the term, but then the formatting is 45 | %% depth-limited instead of length limited, so you might run out 46 | %% memory printing it. Out of the frying pan and into the fire. 47 | %% 48 | safe(What, Len) -> 49 | case catch print(What, Len) of 50 | {L, Used} when list(L) -> {L, Used}; 51 | _ -> {"unable to print" ++ io_lib:write(What, 99)} 52 | end. 53 | 54 | %% Returns {List, Length} 55 | print(_, Max) when Max < 0 -> {"...", 3}; 56 | print(Tuple, Max) when tuple(Tuple) -> 57 | {TC, Len} = tuple_contents(Tuple, Max-2), 58 | {[${, TC, $}], Len + 2}; 59 | 60 | %% We assume atoms, floats, funs, integers, PIDs, ports and refs never need 61 | %% to be truncated. This isn't strictly true, someone could make an 62 | %% arbitrarily long bignum. Let's assume that won't happen unless someone 63 | %% is being malicious. 64 | %% 65 | print(Atom, _Max) when atom(Atom) -> 66 | L = atom_to_list(Atom), 67 | {L, length(L)}; 68 | 69 | print(<<>>, _Max) -> 70 | {"<<>>", 4}; 71 | 72 | print(Binary, Max) when binary(Binary) -> 73 | B = binary_to_list(Binary, 1, lists:min([Max, size(Binary)])), 74 | {L, Len} = alist_start(B, Max-4), 75 | {["<<", L, ">>"], Len}; 76 | 77 | print(Float, _Max) when float(Float) -> 78 | L = float_to_list(Float), 79 | {L, length(L)}; 80 | 81 | print(Fun, _Max) when function(Fun) -> 82 | L = erlang:fun_to_list(Fun), 83 | {L, length(L)}; 84 | 85 | print(Integer, _Max) when integer(Integer) -> 86 | L = integer_to_list(Integer), 87 | {L, length(L)}; 88 | 89 | print(Pid, _Max) when pid(Pid) -> 90 | L = pid_to_list(Pid), 91 | {L, length(L)}; 92 | 93 | print(Ref, _Max) when reference(Ref) -> 94 | L = erlang:ref_to_list(Ref), 95 | {L, length(L)}; 96 | 97 | print(Port, _Max) when port(Port) -> 98 | L = erlang:port_to_list(Port), 99 | {L, length(L)}; 100 | 101 | print(List, Max) when list(List) -> 102 | alist_start(List, Max). 103 | 104 | %% Returns {List, Length} 105 | tuple_contents(Tuple, Max) -> 106 | L = tuple_to_list(Tuple), 107 | list_body(L, Max). 108 | 109 | %% Returns {List, Length} 110 | list_body([], _) -> {[], 0}; 111 | list_body(_, Max) when Max < 4 -> {"...", 3}; 112 | list_body([H|T], Max) -> 113 | {List, Len} = print(H, Max), 114 | {Final, FLen} = list_bodyc(T, Max-Len-1), 115 | {[List|Final], FLen + Len + 1}; 116 | list_body(X, Max) -> %% improper list 117 | {List, Len} = print(X, Max - 2), 118 | {[$|,List,$]], Len + 2}. 119 | 120 | list_bodyc([], _) -> {[], 0}; 121 | list_bodyc(_, Max) when Max < 4 -> {"...", 3}; 122 | list_bodyc([H|T], Max) -> 123 | {List, Len} = print(H, Max), 124 | {Final, FLen} = list_bodyc(T, Max-Len-1), 125 | {[$,, List|Final], FLen + Len + 1}; 126 | list_bodyc(X,Max) -> %% improper list 127 | {List, Len} = print(X, Max - 2), 128 | {[$|,List,$]], Len + 2}. 129 | 130 | %% The head of a list we hope is ascii. Examples: 131 | %% 132 | %% [65,66,67] -> "ABC" 133 | %% [65,0,67] -> "A"[0,67] 134 | %% [0,65,66] -> [0,65,66] 135 | %% [65,b,66] -> "A"[b,66] 136 | %% 137 | alist_start([], _) -> {"[]", 2}; 138 | alist_start(_, Max) when Max < 4 -> {"...", 3}; 139 | alist_start([H|T], Max) when H >= 16#20, H =< 16#7e -> % definitely printable 140 | {L, Len} = alist([H|T], Max-1), 141 | {[$"|L], Len + 1}; 142 | alist_start([H|T], Max) when H == 9; H == 10; H == 13 -> % show as space 143 | {L, Len} = alist(T, Max-1), 144 | {[$ |L], Len + 1}; 145 | alist_start(L, Max) -> 146 | {R, Len} = list_body(L, Max-2), 147 | {[$[, R, $]], Len + 2}. 148 | 149 | alist([], _) -> {"\"", 1}; 150 | alist(_, Max) when Max < 5 -> {"...\"", 4}; 151 | alist([H|T], Max) when H >= 16#20, H =< 16#7e -> % definitely printable 152 | {L, Len} = alist(T, Max-1), 153 | {[H|L], Len + 1}; 154 | alist([H|T], Max) when H == 9; H == 10; H == 13 -> % show as space 155 | {L, Len} = alist(T, Max-1), 156 | {[$ |L], Len + 1}; 157 | alist(L, Max) -> 158 | {R, Len} = list_body(L, Max-2), 159 | {[$", $[, R, $]], Len + 2}. 160 | 161 | 162 | %%-------------------- 163 | %% The start of a test suite. So far, it only checks for not crashing. 164 | test() -> 165 | test(trunc_io, print). 166 | 167 | test(Mod, Func) -> 168 | Simple_items = [atom, 1234, 1234.0, {tuple}, [], [list], "string", self(), 169 | <<1,2,3>>, make_ref(), fun() -> ok end], 170 | F = fun(A) -> 171 | Mod:Func(A, 100), 172 | Mod:Func(A, 2), 173 | Mod:Func(A, 20) 174 | end, 175 | 176 | G = fun(A) -> 177 | case catch F(A) of 178 | {'EXIT', _} -> exit({failed, A}); 179 | _ -> ok 180 | end 181 | end, 182 | 183 | lists:foreach(G, Simple_items), 184 | 185 | Tuples = [ {1,2,3,a,b,c}, {"abc", def, 1234}, 186 | {{{{a},b,c,{d},e}},f}], 187 | 188 | Lists = [ [1,2,3,4,5,6,7], lists:seq(1,1000), 189 | [{a}, {a,b}, {a, [b,c]}, "def"], [a|b], [$a|$b] ], 190 | 191 | 192 | lists:foreach(G, Tuples), 193 | lists:foreach(G, Lists). 194 | 195 | perf() -> 196 | {New, _} = timer:tc(trunc_io, perf, [trunc_io, print, 1000]), 197 | {Old, _} = timer:tc(trunc_io, perf, [io_lib, write, 1000]), 198 | io:fwrite("New code took ~p us, old code ~p\n", [New, Old]). 199 | 200 | perf(M, F, Reps) when Reps > 0 -> 201 | test(M,F), 202 | perf(M,F,Reps-1); 203 | perf(_,_,_) -> 204 | done. 205 | 206 | %% Performance test. Needs a particularly large term I saved as a binary... 207 | perf1() -> 208 | {ok, Bin} = file:read_file("bin"), 209 | A = binary_to_term(Bin), 210 | {N, _} = timer:tc(trunc_io, print, [A, 1500]), 211 | {M, _} = timer:tc(io_lib, write, [A]), 212 | {N, M}. 213 | -------------------------------------------------------------------------------- /ring/start_ringo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z $1 ] 4 | then 5 | echo "Usage: start_ringo.sh [ringo_home]" 6 | exit 1 7 | fi 8 | if [ -z $RINGOSHELL ] 9 | then 10 | SHELL="-noshell -detached -heart" 11 | fi 12 | 13 | export HEART_COMMAND="$0 $@" 14 | 15 | RINGO=`dirname $0` 16 | ID=`basename $1` 17 | export ERL_CRASH_DUMP="ringo-$ID.erl_crash.dump" 18 | 19 | if [ `pgrep -f "ringo-$ID"` ] 20 | then 21 | echo "Node ringo-$ID already running" 22 | exit 1 23 | fi 24 | 25 | echo "Launching Ringo node [$ID]" 26 | erl $SHELL -setcookie ringobingo +K true -smp on -pa $RINGO/bfile/ebin \ 27 | -pa $RINGO/ebin -kernel error_logger "{file, \"ringo-$ID.log\"}"\ 28 | -boot ringo -sname "ringo-$ID" -ringo ringo_home "\"$1\"" 29 | -------------------------------------------------------------------------------- /ring/test-system.sh: -------------------------------------------------------------------------------- 1 | export PATH=$PATH:. 2 | export PYTHONPATH=../ringogw/py 3 | rm -Rf /tmp/ringotest-* 4 | 5 | if ! wget -O - localhost:15000/mon/ring/reset >/dev/null 2>/dev/null 6 | then 7 | echo "Ringogw doesn't seem to be running at http://localhost:15000" 8 | else 9 | python -u test/system_test.py localhost:15000 $@ 10 | fi 11 | -------------------------------------------------------------------------------- /ring/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function run { 4 | erl -pa ../bfile/ebin -pa ../ebin -noshell -run bfile load_driver -run $@ 5 | echo 6 | } 7 | 8 | echo 9 | #erlc +native +"{hipe, [o3]}" -I src/ -o test test/*.erl 10 | #erlc +native +"{hipe, [o3]}" -o ebin src/ringo_writer.erl src/ringo_reader.erl 11 | erlc -I src/ -o test test/*.erl 12 | 13 | SRC="src/ringo_writer.erl src/ringo_reader.erl\ 14 | src/trunc_io.erl src/ringo_sync.erl src/lrucache.erl\ 15 | src/ringo_index.erl src/bin_util.erl" 16 | 17 | if [[ -z $BEAM ]]; then 18 | echo "Compiling tests.. (Hipe)" 19 | erlc +native +"{hipe, [o3]}" -o ebin $SRC 20 | 21 | else 22 | echo "Compiling tests.." 23 | erlc -o ebin $SRC 24 | fi 25 | 26 | echo 27 | 28 | rm -Rf test/test_data 29 | mkdir test/test_data 30 | cd test 31 | 32 | 33 | if [[ -z $1 || $1 == "rw" ]]; then 34 | echo "*** Codec test ***" 35 | run test_readwrite codec_test 36 | echo "*** Write test ***" 37 | run test_readwrite write_test 100000 38 | echo "*** Read test ***" 39 | run test_readwrite read_test 100000 40 | echo "Writing more entries.." 41 | run test_readwrite write_test 100000 42 | run test_readwrite read_test 200000 43 | fi 44 | 45 | 46 | if [[ -z $1 || $1 == "corrupt" ]]; then 47 | echo "*** Corrupt test (should complain about 656. entry) ***" 48 | if [[ $1 == "corrupt" ]]; then 49 | run test_readwrite write_test 100000 50 | fi 51 | dd if=/dev/urandom conv=notrunc of=test_data/data\ 52 | seek=43345 bs=1 count=2000 2>/dev/null 53 | run test_readwrite read_test 100000 54 | fi 55 | 56 | if [[ -z $1 || $1 == "extfile" ]]; then 57 | echo "*** Extfile test ***" 58 | echo "Writing.." 59 | run test_readwrite extfile_write_test 1000 60 | echo "Reading.." 61 | run test_readwrite extfile_read_test 62 | fi 63 | 64 | if [[ -z $1 || $1 == "sync" ]]; then 65 | echo "*** Basic sync-tree test ***" 66 | run test_sync basic_tree_test 100000 67 | echo "*** ID-list test ***" 68 | run test_sync idlist_test 69 | echo "*** Diff test ***" 70 | run test_sync diff_test 1000 71 | echo "*** Order test ***" 72 | run test_sync order_test 1000 73 | fi 74 | 75 | if [[ -z $1 || $1 == "index" ]]; then 76 | echo "*** Build index test ***" 77 | run test_index buildindex_test 10000000 78 | run test_index buildindex_test 1000 79 | run test_index buildindex_test 10 80 | echo "*** Key-value segment encoding/decoding test ***" 81 | run test_index kv_test 82 | echo "*** Iblock de/serialization test ***" 83 | run test_index serialize_test 10000000 84 | run test_index serialize_test 1000 85 | run test_index serialize_test 1 86 | echo "*** DB access test ***" 87 | run test_index indexuse_test 10000000 88 | echo "*** LRU cache test ***" 89 | run test_index lrucache_test 90 | fi 91 | 92 | cd .. 93 | echo "ok" 94 | -------------------------------------------------------------------------------- /ring/test/ringo_cmd.erl: -------------------------------------------------------------------------------- 1 | -module(ringo_cmd). 2 | -export([create/1, put/1]). 3 | 4 | request(Name, Chunk, Msg) -> 5 | DomainID = ringo_util:domain_id(Name, Chunk), 6 | io:fwrite("DomainID: ~b~n", [DomainID]), 7 | [First|_] = Nodes = ringo_util:ringo_nodes(), 8 | io:fwrite("Nodes: ~w~n", [Nodes]), 9 | ok = gen_server:cast({ringo_node, First}, 10 | {match, DomainID, domain, x, Msg}), 11 | receive 12 | {ringo_reply, DomainID, {ok, M}} -> 13 | io:fwrite("Got ok: ~w~n", [M]); 14 | Other -> 15 | io:fwrite("Error: ~w~n", [Other]) 16 | after 5000 -> 17 | io:fwrite("Timeout~n", []) 18 | end. 19 | 20 | create([Name, NReplicas]) -> 21 | ReplN = list_to_integer(NReplicas), 22 | request(Name, 0, {new_domain, Name, 0, self(), [{nrepl, ReplN}]}), 23 | halt(). 24 | 25 | put([Name, Key, Value]) -> 26 | request(Name, 0, {put, list_to_binary(Key), 27 | list_to_binary(Value), [], self()}), 28 | halt(). 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /ring/test/test_index.erl: -------------------------------------------------------------------------------- 1 | -module(test_index). 2 | -export([buildindex_test/1, serialize_test/1, kv_test/0, indexuse_test/1]). 3 | -export([lrucache_test/0]). 4 | 5 | write_data(NumKeys) -> 6 | S = now(), 7 | %{ok, DB} = file:open("test_data/indexdata", [write, raw]), 8 | {ok, DB} = bfile:fopen("test_data/indexdata", "w"), 9 | Keys = lists:map(fun(_) -> 10 | EntryID = random:uniform(4294967295), 11 | N = random:uniform(NumKeys), 12 | Key = <<"KeyYek:", N:32>>, 13 | Entry = ringo_writer:make_entry(EntryID, Key, 14 | <<"ValueEulav:", N:32>>, []), 15 | ok = ringo_writer:write_entry("test_data", DB, Entry), 16 | Key 17 | end, lists:seq(1, 10000)), 18 | io:fwrite("Writing ~b items (max key ~b) took ~bms~n", 19 | [10000, NumKeys, round(timer:now_diff(now(), S) / 1000)]), 20 | bfile:fclose(DB), 21 | Keys. 22 | 23 | buildindex_test(Keys) when is_list(Keys) -> 24 | buildindex_test(list_to_integer(lists:flatten(Keys))); 25 | 26 | buildindex_test(Keys) -> 27 | write_data(Keys), 28 | S = now(), 29 | {_, Dex, _} = ringo_index:build_index("test_data/indexdata", 0, inf), 30 | Ser = ringo_index:serialize(Dex), 31 | {memory, Mem} = erlang:process_info(self(), memory), 32 | io:fwrite("Building index took ~bms~n", 33 | [round(timer:now_diff(now(), S) / 1000)]), 34 | io:fwrite("Process takes ~bK memory. Index takes ~bK.~n", 35 | [Mem div 1024, iolist_size(Ser) div 1024]), 36 | io:fwrite("~b keys in the index~n", [gb_trees:size(Dex)]), 37 | halt(). 38 | 39 | 40 | serialize_test(Keys) when is_list(Keys) -> 41 | serialize_test(list_to_integer(lists:flatten(Keys))); 42 | 43 | serialize_test(Keys) -> 44 | write_data(Keys), 45 | {_, Dex, _} = ringo_index:build_index("test_data/indexdata", 0, inf), 46 | S = now(), 47 | Ser = iolist_to_binary(ringo_index:serialize(Dex)), 48 | io:fwrite("Serialization took ~bms~n", 49 | [round(timer:now_diff(now(), S) / 1000)]), 50 | io:fwrite("Serialized index takes ~bK~n", [iolist_size(Ser) div 1024]), 51 | io:fwrite("~b keys in the index~n", [gb_trees:size(Dex)]), 52 | S2 = now(), 53 | lists:foreach(fun(ID) -> 54 | %io:fwrite("ID ~b~n", [ID]), 55 | {ID, [_|_]} = ringo_index:find_key(ID, Ser) 56 | end, gb_trees:keys(Dex)), 57 | io:fwrite("All keys found ok in ~bms~n", 58 | [round(timer:now_diff(now(), S2) / 1000)]), 59 | halt(). 60 | 61 | kv_test() -> 62 | lists:foreach(fun(I) -> 63 | L = [{X, X + 1} || X <- lists:seq(1, I)], 64 | S = bin_util:encode_kvsegment(L), 65 | lists:foreach(fun({K, _} = R) -> 66 | R = bin_util:find_kv(K, S) 67 | end, L) 68 | %io:fwrite("~b-item segment ok~n", [I]) 69 | end, lists:seq(1, 1000)), 70 | io:fwrite("Binary search for all segments ok~n", []), 71 | halt(). 72 | 73 | lrucache_test() -> 74 | S = now(), 75 | LRU = lists:foldl(fun(I, LRUx) -> 76 | lrucache:update(I, LRUx) 77 | end, lrucache:new(), lists:seq(1, 10000)), 78 | io:fwrite("10000 items updated in ~bms~n", 79 | [round(timer:now_diff(now(), S) / 1000)]), 80 | 81 | S0 = now(), 82 | LRU0 = lists:foldl(fun(I, LRUx) -> 83 | {I, LRUy} = lrucache:get_lru(LRUx), LRUy 84 | end, LRU, lists:seq(1, 10000)), 85 | io:fwrite("10000 lrus got in ~bms~n", 86 | [round(timer:now_diff(now(), S0) / 1000)]), 87 | 88 | true = lrucache:is_empty(LRU0), 89 | io:fwrite("is_empty() works~n", []), 90 | 91 | LRU1 = lists:foldl(fun(I, LRUx) -> 92 | lrucache:update(I, LRUx) 93 | end, lrucache:new(), lists:seq(1, 10000)), 94 | 95 | io:fwrite("Testing random access..~n", []), 96 | Hits = dict:from_list([{I, true} || I <- lists:seq(1, 10000)]), 97 | {Hits0, LRU2} = lists:foldl(fun(_, {Hx, LRUx}) -> 98 | N = random:uniform(9000), 99 | {dict:store(N, false, Hx), lrucache:update(N, LRUx)} 100 | end, {Hits, LRU1}, lists:seq(1, 10000)), 101 | 102 | lists:foldl(fun(_, LRUx) -> 103 | {I, LRUy} = lrucache:get_lru(LRUx), 104 | true = dict:fetch(I, Hits0), LRUy 105 | end, LRU2, lists:seq(1, 1000)), 106 | io:fwrite("Random access works~n", []), 107 | halt(). 108 | 109 | indexuse_test(NumKeys) when is_list(NumKeys) -> 110 | indexuse_test(list_to_integer(lists:flatten(NumKeys))); 111 | indexuse_test(NumKeys) -> 112 | Keys = write_data(NumKeys), 113 | {_, Dex, _} = ringo_index:build_index("test_data/indexdata", 0, inf), 114 | %{ok, DB} = file:open("test_data/indexdata", [read, raw, binary]), 115 | {ok, DB} = bfile:fopen("test_data/indexdata", "r"), 116 | Ser = iolist_to_binary(ringo_index:serialize(Dex)), 117 | S = now(), 118 | lists:foreach(fun(Key) -> 119 | {_, Offsets} = ringo_index:find_key(Key, Ser), 120 | E = [ringo_index:fetch_entry(DB, "test_data", Key, Offs) 121 | || Offs <- Offsets], 122 | [_|_] = lists:filter(fun 123 | ({_, K, _}) when K == Key -> true; 124 | (ignore) -> false 125 | end, E) 126 | end, Keys), 127 | io:fwrite("All keys read ok in ~bms~n", 128 | [round(timer:now_diff(now(), S) / 1000)]), 129 | halt(). 130 | 131 | -------------------------------------------------------------------------------- /ring/test/test_readwrite.erl: -------------------------------------------------------------------------------- 1 | -module(test_readwrite). 2 | -export([codec_test/0, write_test/1, read_test/1, extfile_write_test/1]). 3 | -export([extfile_read_test/0]). 4 | 5 | -include("ringo_store.hrl"). 6 | 7 | codec_test() -> 8 | EntryID = random:uniform(4294967295), 9 | Key = <<"dssdwe1we1234124e">>, 10 | Val = <<"dssddsaswqe-">>, 11 | Flags = [], 12 | 13 | io:fwrite("Encoding / decoding internal entry..~n", []), 14 | {Ent1, _} = ringo_writer:make_entry(EntryID, Key, Val, Flags), 15 | {_Time, EntryID, Flags, Key, {int, Val}} = ringo_reader:decode(Ent1), 16 | false = ringo_reader:is_external(Ent1), 17 | 18 | io:fwrite("Encoding / decoding external entry..~n", []), 19 | {ok, Bash} = file:read_file("/bin/bash"), 20 | {Ent2, _} = ringo_writer:make_entry(EntryID, Key, Bash, Flags), 21 | true = ringo_reader:is_external(Ent2), 22 | {_Time, EntryID, Flags2, Key, {ext, {_, _Path}}} = 23 | ringo_reader:decode(Ent2), 24 | 25 | case proplists:is_defined(external, Flags2) of 26 | true -> io:fwrite("Codec works ok.~n", []); 27 | false -> io:fwrite("Codec returned invalid flags: ~w~n", 28 | [Flags2]) 29 | end, 30 | halt(). 31 | 32 | 33 | write_test(Entries) when is_list(Entries) -> 34 | write_test(list_to_integer(lists:flatten(Entries))); 35 | 36 | write_test(Entries) -> 37 | %{ok, DB} = file:open("test_data/data", [append, raw]), 38 | {ok, DB} = bfile:fopen("test_data/data", "a"), 39 | S = now(), 40 | lists:foreach(fun(I) -> 41 | EntryID = random:uniform(4294967295), 42 | N = I + 100, 43 | Entry = ringo_writer:make_entry(EntryID, <<"KeyYek:", I:32>>, 44 | <<"ValueEulav:", N:32>>, []), 45 | ok = ringo_writer:write_entry("test_data", DB, Entry) 46 | end, lists:seq(1, Entries)), 47 | io:fwrite("Writing ~b items took ~bms~n", 48 | [Entries, round(timer:now_diff(now(), S) / 1000)]), 49 | halt(). 50 | 51 | read_test(Entries) when is_list(Entries) -> 52 | read_test(list_to_integer(lists:flatten(Entries))); 53 | 54 | read_test(Entries) -> 55 | S = now(), 56 | {_, N} = ringo_reader:fold(fun 57 | 58 | (<<"KeyYek:", N:32>>, <<"ValueEulav:", M:32>>, _, _, _, {C, T}) 59 | when N == C, M == C + 100 -> 60 | {C + 1, T + 1}; 61 | 62 | (<<"KeyYek:", N:32>>, <<"ValueEulav:", M:32>>, _, _, _, 63 | {_, T}) -> 64 | io:fwrite("~b. entry is invalid. Got keys ~b and ~b.~n", 65 | [T, N, M]), 66 | {N + 1, T + 1}; 67 | 68 | (Key, Val, _, _, _, {_, T}) -> 69 | io:fwrite("~b. entry is invalid. Key <~p> Val <~p>.~n", 70 | [T, Key, Val]), 71 | halt() 72 | 73 | end, {1, 0}, "test_data/data"), 74 | if N == Entries -> 75 | io:fwrite("~b entries read ok in ~bms~n", [N, 76 | round(timer:now_diff(now(), S) / 1000)]); 77 | true -> 78 | io:fwrite("Expected ~b entries, read ~b entries~n", 79 | [Entries, N]) 80 | end, 81 | halt(). 82 | 83 | 84 | extfile_write_test(Entries) when is_list(Entries) -> 85 | extfile_write_test(list_to_integer(lists:flatten(Entries))); 86 | 87 | extfile_write_test(Entries) -> 88 | %{ok, DB} = file:open("test_data/bigdata", [append, raw]), 89 | {ok, Bash} = file:read_file("/bin/bash"), 90 | {ok, DB} = bfile:fopen("test_data/bigdata", "a"), 91 | BashCRC = erlang:crc32(Bash), 92 | S = now(), 93 | lists:foreach(fun(_) -> 94 | EntryID = random:uniform(4294967295), 95 | ringo_writer:write_entry("test_data", DB, 96 | ringo_writer:make_entry(EntryID, 97 | <<"Bash:", BashCRC:32>>, Bash, [])), 98 | % entry with an equal EntryID, should be skipped 99 | ringo_writer:write_entry("test_data", DB, 100 | ringo_writer:make_entry(EntryID, 101 | <<"skipme">>, <<>>, [])), 102 | NextID = random:uniform(4294967295), 103 | ringo_writer:write_entry("test_data", 104 | DB, ringo_writer:make_entry(NextID, 105 | <<"small">>, <<"fall">>, [])) 106 | end, lists:seq(1, Entries)), 107 | io:fwrite("Writing ~b big items took ~bms~n", 108 | [Entries, round(timer:now_diff(now(), S) / 1000)]), 109 | halt(). 110 | 111 | extfile_read_test() -> 112 | S = now(), 113 | ringo_reader:fold(fun 114 | 115 | (<<"Bash:", BashCRC:32>>, Val, [external], ID, _, small) -> 116 | {ok, Bash} = ringo_reader:read_external( 117 | "test_data", Val), 118 | M = erlang:crc32(Bash), 119 | if M == BashCRC -> {big, ID}; 120 | true -> 121 | io:fwrite("Invalid checksum~n"), 122 | halt() 123 | end; 124 | 125 | (<<"small">>, <<"fall">>, [], _, _, {big, _}) -> 126 | small; 127 | 128 | (Key, _, _, ID, _, A) -> 129 | io:fwrite( 130 | "Invalid entry. Key <~p> Acc <~w> ID <~w>.~n", 131 | [Key, A, ID]), 132 | halt() 133 | 134 | end, small, "test_data/bigdata"), 135 | io:fwrite("Reading big and small items took ~bms~n", 136 | [round(timer:now_diff(now(), S) / 1000)]), 137 | halt(). 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | -------------------------------------------------------------------------------- /ring/test/test_sync.erl: -------------------------------------------------------------------------------- 1 | -module(test_sync). 2 | -export([basic_tree_test/1, idlist_test/0, diff_test/1, order_test/1]). 3 | 4 | -include("ringo_store.hrl"). 5 | 6 | basic_tree_test(Entries) when is_list(Entries) -> 7 | basic_tree_test(list_to_integer(lists:flatten(Entries))); 8 | 9 | basic_tree_test(Entries) -> 10 | %{ok, DB} = file:open("test_data/syncdata", [append, raw]), 11 | {ok, DB} = bfile:fopen("test_data/syncdata", "a"), 12 | lists:foreach(fun(_) -> 13 | EntryID = random:uniform(4294967295), 14 | Entry = ringo_writer:make_entry(EntryID, <<>>, <<>>, []), 15 | ok = ringo_writer:write_entry("test_data", DB, Entry) 16 | end, lists:seq(1, Entries)), 17 | 18 | S = now(), 19 | {LeafHashes, LeafLists} = ringo_sync:make_leaf_hashes_and_ids( 20 | "test_data/syncdata"), 21 | io:fwrite("Making ~b leaves and ids lists for ~b entries took ~bms~n", 22 | [ets:info(LeafHashes, size), Entries, 23 | round(timer:now_diff(now(), S) / 1000)]), 24 | 25 | Slots = lists:sort([round(size(V) / 8) || 26 | {_, V} <- dict:to_list(LeafLists)]), 27 | io:fwrite("Min slot ~b entries~n", [lists:min(Slots)]), 28 | io:fwrite("Median slot ~b entries~n", 29 | [lists:nth(round(length(Slots) / 2), Slots)]), 30 | io:fwrite("Max slot ~b entries~n", [lists:max(Slots)]), 31 | 32 | C = ringo_sync:count_entries(LeafLists), 33 | io:fwrite("Count_entries found ~b entries~n", [C]), 34 | 35 | Sum = lists:sum(Slots), 36 | if Sum == Entries, Sum == C -> 37 | io:fwrite("~b Entry IDs in total (ok)~n", [Sum]); 38 | true -> 39 | io:fwrite("~b Entry IDs, ~b according to count_entries, " 40 | "in total (should be ~b)~n", [Sum, C, Entries]), 41 | halt() 42 | end, 43 | 44 | S1 = now(), 45 | Tree = ringo_sync:build_merkle_tree(LeafHashes), 46 | lists:foldl(fun 47 | (L, N) when length(L) == N -> 48 | N * 2; 49 | (L, N) -> 50 | io:fwrite("Broken tree. Expected ~b nodes, got ~b", 51 | [length(L), N]), 52 | halt() 53 | end, 1, Tree), 54 | 55 | io:fwrite("Building a Merkle tree took ~bms~n", 56 | [round(timer:now_diff(now(), S1) / 1000)]), 57 | halt(). 58 | 59 | 60 | idlist_test() -> 61 | {_, LeafIDs} = ringo_sync:make_leaf_hashes_and_ids( 62 | "test_data/syncdata"), 63 | Leaves = ringo_sync:collect_leaves(lists:seq(0, 64 | ?NUM_MERKLE_LEAVES - 1), "test_data/syncdata"), 65 | S1 = now(), 66 | N = ringo_reader:fold(fun(_, _, _, {Time, EntryID}, _, N) -> 67 | {Leaf, SyncID} = ringo_sync:sync_id(EntryID, Time), 68 | 69 | {value, {_, L}} = lists:keysearch(Leaf, 1, Leaves), 70 | V1 = lists:member(SyncID, L), 71 | V2 = ringo_sync:in_leaves(LeafIDs, SyncID), 72 | 73 | if V1, V2 -> N + 1; 74 | true -> 75 | io:fwrite("~b. entry missing (id ~b, leaf ~b)~n", 76 | [N, EntryID, Leaf]), 77 | halt() 78 | end 79 | end, 0, "test_data/syncdata"), 80 | io:fwrite("~b IDs validated ok in ~bms~n", [N, 81 | round(timer:now_diff(now(), S1) / 1000)]), 82 | halt(). 83 | 84 | diff_test(Entries) when is_list(Entries) -> 85 | diff_test(list_to_integer(lists:flatten(Entries))); 86 | 87 | diff_test(Entries) -> 88 | Mods = lists:map(fun(_) -> 89 | EntryID = random:uniform(4294967295), 90 | ringo_sync:sync_id(EntryID, 0) 91 | end, lists:seq(1, Entries)), 92 | {CLeaves, SyncIDs} = lists:unzip(Mods), 93 | ChangedLeaves = lists:usort(CLeaves), 94 | 95 | {LeafHashes, _} = ringo_sync:make_leaf_hashes_and_ids( 96 | "test_data/syncdata"), 97 | [[RootA]|_] = TreeA = ringo_sync:build_merkle_tree(LeafHashes), 98 | [] = merkle_diff(1, [{0, RootA}], TreeA, TreeA), 99 | io:fwrite("Identical trees diff test passed.~n", []), 100 | 101 | [ringo_sync:update_leaf_hashes(LeafHashes, ID) || ID <- SyncIDs], 102 | [[RootB]|_] = TreeB = ringo_sync:build_merkle_tree(LeafHashes), 103 | 104 | S1 = now(), 105 | Diff2 = merkle_diff(1, [{0, RootA}], TreeA, TreeB), 106 | S2 = now(), 107 | Diff3 = merkle_diff(1, [{0, RootB}], TreeB, TreeA), 108 | 109 | if Diff2 == Diff3 -> 110 | io:fwrite("Symmetricity test passed.~n"); 111 | true -> 112 | io:fwrite("Symmetricity test FAILS!~n"), 113 | halt() 114 | end, 115 | 116 | Matches = Diff2 -- ChangedLeaves, 117 | if Matches == [], length(Diff2) == length(ChangedLeaves) -> 118 | io:fwrite( 119 | "Changes in ~b entries were detected correctly in ~bms.~n", 120 | [Entries, round(timer:now_diff(S2, S1) / 1000)]); 121 | true -> 122 | io:fwrite("Merkle_diff FAILS!~n"), 123 | io:fwrite("Found changes in ~w. Actually ~w were changed.", 124 | [Diff2, ChangedLeaves]) 125 | end, 126 | halt(). 127 | 128 | order_test(Entries) when is_list(Entries) -> 129 | order_test(list_to_integer(lists:flatten(Entries))); 130 | 131 | order_test(NEntries) -> 132 | {ok, DB1} = bfile:fopen("test_data/orderdata1", "a"), 133 | 134 | {_, Entries} = lists:unzip(lists:sort(lists:map(fun(_) -> 135 | EntryID = random:uniform(4294967295), 136 | Entry = ringo_writer:make_entry(EntryID, <<>>, <<>>, []), 137 | ok = ringo_writer:write_entry("test_data", DB1, Entry), 138 | {random:uniform(NEntries), Entry} 139 | end, lists:seq(1, NEntries)))), 140 | bfile:fclose(DB1), 141 | 142 | {ok, DB2} = bfile:fopen("test_data/orderdata2", "a"), 143 | lists:foreach(fun(Entry) -> 144 | ok = ringo_writer:write_entry("test_data", DB2, Entry) 145 | end, Entries), 146 | bfile:fclose(DB2), 147 | 148 | {LeafHashes1, _} = ringo_sync:make_leaf_hashes_and_ids( 149 | "test_data/orderdata1"), 150 | [[Root1]|_] = ringo_sync:build_merkle_tree(LeafHashes1), 151 | 152 | {LeafHashes2, _} = ringo_sync:make_leaf_hashes_and_ids( 153 | "test_data/orderdata2"), 154 | [[Root2]|_] = ringo_sync:build_merkle_tree(LeafHashes2), 155 | 156 | io:fwrite("Root1: ~b Root2: ~b~n", [Root1, Root2]), 157 | if Root1 == Root2 -> 158 | io:fwrite("Roots match although entries were written " 159 | "in different order. Good!~n"); 160 | true -> 161 | io:fwrite("Roots differ although entries are equal. Bad!~n") 162 | end, 163 | halt(). 164 | 165 | 166 | merkle_diff(2, [], _, _) -> []; 167 | merkle_diff(H, LevelA, TreeA, TreeB) -> 168 | Diff = ringo_sync:diff_parents(H, LevelA, TreeB), 169 | if H < length(TreeA) -> 170 | Children = ringo_sync:pick_children(H + 1, Diff, TreeA), 171 | merkle_diff(H + 1, Children, TreeA, TreeB); 172 | true -> 173 | Diff 174 | end. 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | -------------------------------------------------------------------------------- /ringogw/compile.sh: -------------------------------------------------------------------------------- 1 | #erlc +native +"{hipe, [o3]}" -I ../ring/src -o ebin src/*.erl 2 | #erlc -o ebin src/mochi_dispatch.erl 3 | 4 | erlc -I ../ring/src -o ebin src/*.erl 5 | erl -pa ebin -noshell -run make_boot write_scripts 6 | -------------------------------------------------------------------------------- /ringogw/ebin/ringogw.app: -------------------------------------------------------------------------------- 1 | {application, ringogw, [ 2 | {description, "Ringogw"}, 3 | {vsn, "1"}, 4 | {modules, [ringogw, 5 | scgi, 6 | json, 7 | scgi_server, 8 | handle_ring, 9 | trunc_io, 10 | handle_domains, 11 | handle_chunkstat]}, 12 | {registered, []}, 13 | {applications, [kernel, stdlib]}, 14 | {mod, {ringogw, []}} 15 | ]}. 16 | 17 | -------------------------------------------------------------------------------- /ringogw/lighttpd.conf: -------------------------------------------------------------------------------- 1 | # Debian lighttpd configuration file 2 | # 3 | 4 | ############ Options you really have to take care of #################### 5 | 6 | ## modules to load 7 | # mod_access, mod_accesslog and mod_alias are loaded by default 8 | # all other module should only be loaded if neccesary 9 | # - saves some time 10 | # - saves memory 11 | 12 | server.modules = ( 13 | "mod_access", 14 | "mod_alias", 15 | "mod_accesslog", 16 | "mod_cgi", 17 | "mod_scgi", 18 | # "mod_rewrite", 19 | # "mod_redirect", 20 | # "mod_status", 21 | # "mod_evhost", 22 | # "mod_compress", 23 | # "mod_usertrack", 24 | # "mod_rrdtool", 25 | # "mod_webdav", 26 | # "mod_expire", 27 | # "mod_flv_streaming", 28 | # "mod_evasive" 29 | ) 30 | 31 | scgi.server = ( "/mon/ring/" => (( 32 | "host" => "127.0.0.1", 33 | "port" => 15001, 34 | "check-local" => "disable", 35 | "disable-time" => 5, 36 | "broken-scriptfilename" => "enable") 37 | ), 38 | "/mon/domains/" => (( 39 | "host" => "127.0.0.1", 40 | "port" => 15001, 41 | "check-local" => "disable", 42 | "disable-time" => 5, 43 | "broken-scriptfilename" => "enable") 44 | ), 45 | "/mon/data/" => (( 46 | "host" => "127.0.0.1", 47 | "port" => 15001, 48 | "check-local" => "disable", 49 | "disable-time" => 5, 50 | "broken-scriptfilename" => "enable") 51 | ) 52 | ) 53 | 54 | 55 | ## a static document-root, for virtual-hosting take look at the 56 | ## server.virtual-* options 57 | server.document-root = "web" 58 | 59 | ## where to send error-messages to 60 | server.errorlog = "error.log" 61 | 62 | ## files to check for if .../ is requested 63 | index-file.names = ( "index.php", "index.html", 64 | "index.htm", "default.htm" ) 65 | 66 | 67 | ## Use the "Content-Type" extended attribute to obtain mime type if possible 68 | # mimetype.use-xattr = "enable" 69 | 70 | #### accesslog module 71 | accesslog.filename = "access.log" 72 | 73 | ## deny access the file-extensions 74 | # 75 | # ~ is for backupfiles from vi, emacs, joe, ... 76 | # .inc is often used for code includes which should in general not be part 77 | # of the document-root 78 | url.access-deny = ( "~", ".inc" ) 79 | 80 | 81 | 82 | ######### Options that are good to be but not neccesary to be changed ####### 83 | 84 | ## bind to port (default: 80) 85 | server.port = 15000 86 | 87 | ## bind to localhost only (default: all interfaces) 88 | ## server.bind = "localhost" 89 | 90 | ## error-handler for status 404 91 | #server.error-handler-404 = "/error-handler.html" 92 | #server.error-handler-404 = "/error-handler.php" 93 | 94 | ## to help the rc.scripts 95 | server.pid-file = "lighttpd.pid" 96 | 97 | ## 98 | ## Format: .html 99 | ## -> ..../status-404.html for 'File not found' 100 | #server.errorfile-prefix = "/var/www/" 101 | 102 | ## virtual directory listings 103 | dir-listing.encoding = "utf-8" 104 | server.dir-listing = "enable" 105 | 106 | ## send unhandled HTTP-header headers to error-log 107 | #debug.dump-unknown-headers = "enable" 108 | 109 | ### only root can use these options 110 | # 111 | # chroot() to directory (default: no chroot() ) 112 | #server.chroot = "/" 113 | 114 | ## change uid to (default: don't care) 115 | #server.username = "tuulos" 116 | 117 | ## change uid to (default: don't care) 118 | #server.groupname = "tuulos" 119 | 120 | #### compress module 121 | #compress.cache-dir = "/var/tmp/lighttpd/cache/compress/" 122 | #compress.filetype = ("text/plain", "text/html") 123 | 124 | #### status module 125 | # status.status-url = "/server-status" 126 | # status.config-url = "/server-config" 127 | 128 | #### url handling modules (rewrite, redirect, access) 129 | # url.rewrite = ( "^/$" => "/server-status" ) 130 | # url.redirect = ( "^/wishlist/(.+)" => "http://www.123.org/$1" ) 131 | 132 | # 133 | # define a pattern for the host url finding 134 | # %% => % sign 135 | # %0 => domain name + tld 136 | # %1 => tld 137 | # %2 => domain name without tld 138 | # %3 => subdomain 1 name 139 | # %4 => subdomain 2 name 140 | # 141 | # evhost.path-pattern = "/home/storage/dev/www/%3/htdocs/" 142 | 143 | #### expire module 144 | # expire.url = ( "/buggy/" => "access 2 hours", "/asdhas/" => "access plus 1 seconds 2 minutes") 145 | 146 | #### rrdtool 147 | # rrdtool.binary = "/usr/bin/rrdtool" 148 | # rrdtool.db-name = "/var/www/lighttpd.rrd" 149 | 150 | 151 | #### handle Debian Policy Manual, Section 11.5. urls 152 | #### and by default allow them only from localhost 153 | 154 | #### variable usage: 155 | ## variable name without "." is auto prefixed by "var." and becomes "var.bar" 156 | #bar = 1 157 | #var.mystring = "foo" 158 | 159 | ## integer add 160 | #bar += 1 161 | ## string concat, with integer cast as string, result: "www.foo1.com" 162 | #server.name = "www." + mystring + var.bar + ".com" 163 | ## array merge 164 | #index-file.names = (foo + ".php") + index-file.names 165 | #index-file.names += (foo + ".php") 166 | 167 | #cgi.assign = ( "/q" => "q.py" ) 168 | #### external configuration files 169 | ## mimetype mapping 170 | include_shell "/usr/share/lighttpd/create-mime.assign.pl" 171 | 172 | ## load enabled configuration files, 173 | ## read /etc/lighttpd/conf-available/README first 174 | #include_shell "/usr/share/lighttpd/include-conf-enabled.pl" 175 | -------------------------------------------------------------------------------- /ringogw/py/ringo_reader.py: -------------------------------------------------------------------------------- 1 | import ringodisco, sys 2 | 3 | for k, v in ringodisco.ringo_reader(file(sys.argv[1]), 0, ""): 4 | print k, v 5 | -------------------------------------------------------------------------------- /ringogw/py/ringodisco.py: -------------------------------------------------------------------------------- 1 | import ringogw 2 | 3 | 4 | def ringo_reader(fd, sze, fname): 5 | import struct, zlib 6 | MAGIC_HEAD = (0x47da66b5,) 7 | MAGIC_TAIL = (0xacc50f5d,) 8 | def read_really(s): 9 | t = 0 10 | buf = "" 11 | while t < s: 12 | r = fd.read(s - t) 13 | if not r: 14 | return buf 15 | t += len(r) 16 | buf += r 17 | return buf 18 | 19 | def check_body(head_body): 20 | time, entryid, flags, keycrc, keylen, valcrc, vallen =\ 21 | struct.unpack("= 8: 37 | if struct.unpack("= self.entrylen: 53 | r = self.cb(self.buf[:self.entrylen], self.out) 54 | self.buf = self.buf[self.entrylen:] 55 | self.entrylen = None 56 | else: 57 | # Wait for more data 58 | break 59 | 60 | def output(self): 61 | if self.buf: 62 | raise ReplyException("%d extra bytes in the stream" % 63 | len(self.buf)) 64 | else: 65 | return self.ret, self.out 66 | 67 | class Ringo: 68 | def __init__(self, host, keep_alive = True): 69 | if not host.startswith("http://"): 70 | host = "http://" + host 71 | self.host = host 72 | if keep_alive: 73 | self.curl = pycurl.Curl() 74 | self.keep_alive = keep_alive 75 | 76 | def request(self, url, data = None, verbose = False, 77 | retries = 0, decoder = DecodeJson): 78 | 79 | if self.keep_alive: 80 | curl = self.curl 81 | else: 82 | curl = pycurl.Curl() 83 | 84 | if url.startswith("http://"): 85 | purl = url 86 | else: 87 | purl = self.host + url 88 | 89 | curl.setopt(curl.URL, purl) 90 | if data == None: 91 | curl.setopt(curl.HTTPGET, 1) 92 | else: 93 | curl.setopt(curl.POST, 1) 94 | curl.setopt(curl.POSTFIELDS, data) 95 | curl.setopt(curl.HTTPHEADER, ["Expect:"]) 96 | dec = decoder() 97 | curl.setopt(curl.WRITEFUNCTION, dec.write) 98 | try: 99 | curl.perform() 100 | except pycurl.error, x: 101 | if verbose: 102 | print "Pycurl.error:", x 103 | return x 104 | 105 | code = curl.getinfo(curl.HTTP_CODE) 106 | if verbose: 107 | print "Request took %.2fms" %\ 108 | (curl.getinfo(curl.TOTAL_TIME) * 1000.0) 109 | 110 | # Request timeout 111 | if code == 408 and retries > 0: 112 | time.sleep(0.1) 113 | return self.request(url, data, verbose, 114 | retries - 1, decoder) 115 | else: 116 | return code, dec.output() 117 | 118 | 119 | def check_reply(self, reply): 120 | if reply[0] != 200 or reply[1][0] != 'ok': 121 | e = ReplyException("Invalid reply (code: %d): %s" %\ 122 | (reply[0], reply[1])) 123 | e.retcode = reply[0] 124 | e.retvalue = reply[1] 125 | raise e 126 | return reply[1][1:] 127 | 128 | def create(self, domain, nrepl, **kwargs): 129 | kwargs['decoder'] = DecodeJson 130 | url = "/mon/data/%s?create&nrepl=%d" % (domain, nrepl) 131 | if 'noindex' in kwargs: 132 | url += "&noindex=1" 133 | del kwargs['noindex'] 134 | if 'keycache' in kwargs: 135 | url += "&keycache=1" 136 | del kwargs['keycache'] 137 | return self.check_reply(self.request(url, "", **kwargs))[0] 138 | 139 | def put(self, domain, key, value, **kwargs): 140 | kwargs['decoder'] = DecodeJson 141 | return self.check_reply(self.request("/mon/data/%s/%s" %\ 142 | (domain, key), value, **kwargs)) 143 | 144 | def get(self, domain, key, **kwargs): 145 | url = "/mon/data/%s/%s" % (domain, key) 146 | kwargs['data'] = None 147 | if 'single' in kwargs: 148 | kwargs['decoder'] = DecodeRaw 149 | del kwargs['single'] 150 | code, val = self.request(url + "?single", **kwargs) 151 | if code != 200: 152 | e = ReplyException("Invalid reply (code: %d)"\ 153 | % code) 154 | e.retcode = code 155 | raise e 156 | return val 157 | else: 158 | if 'entry_callback' in kwargs: 159 | cb = kwargs['entry_callback'] 160 | kwargs['decoder'] = lambda: DecodeMulti(cb) 161 | del kwargs['entry_callback'] 162 | else: 163 | kwargs['decoder'] = DecodeMulti 164 | return self.check_reply(self.request(url, **kwargs))[0] 165 | -------------------------------------------------------------------------------- /ringogw/src/handle_chunkstat.erl: -------------------------------------------------------------------------------- 1 | -module(handle_chunkstat). 2 | -------------------------------------------------------------------------------- /ringogw/src/handle_domains.erl: -------------------------------------------------------------------------------- 1 | -module(handle_domains). 2 | -export([op/2, start_check_domains/0]). 3 | 4 | op("node", [{"name", NodeS}|_]) -> 5 | Node = list_to_existing_atom(NodeS), 6 | {json, [fetch_domaininfo(X) || X <- infopack_lookup({node, Node})]}; 7 | 8 | op("domain", [{"name", NameS}|_]) -> 9 | Name = list_to_binary(NameS), 10 | {json, [fetch_domaininfo(X) || X <- infopack_lookup({name, Name})]}; 11 | 12 | op("domain", [{"id", [$0, $x|IdS]}|_]) -> 13 | op("domain", [{"id", integer_to_list( 14 | erlang:list_to_integer(IdS, 16))}]); 15 | 16 | op("domain", [{"id", IdS}|_]) -> 17 | DomainID = list_to_integer(IdS), 18 | 19 | [{_, {Name, _, Chunk}}|_] = Repl = infopack_lookup({id, DomainID}), 20 | Nodes = [Node || {_, {_, Node, _}} <- Repl], 21 | 22 | gen_server:abcast(Nodes, ringo_node, {{domain, DomainID}, 23 | {get_status, self()}}), 24 | Status = receive_domainstatus([], length(Nodes)), 25 | 26 | {json, [list_to_binary(erlang:integer_to_list(DomainID, 16)), 27 | Name, Chunk, lists:map(fun(N) -> 28 | case lists:keysearch(N, 1, Status) of 29 | {value, {_, S}} -> {obj, [{node, N}|S]}; 30 | _ -> {obj, [{node, N}, {error, noreply}]} 31 | end 32 | end, Nodes)]}; 33 | 34 | op("reset", _Query) -> 35 | catch exit(whereis(check_domains), kill), 36 | {json, {ok, <<"killed">>}}. 37 | 38 | fetch_domaininfo({_, DomainID}) -> 39 | [{_, {Name, Node, Chunk}}|_] = Repl = infopack_lookup({id, DomainID}), 40 | Active = case catch ets:lookup(active_domains, DomainID) of 41 | [{_, A}] -> A; 42 | _ -> false 43 | end, 44 | {list_to_binary(erlang:integer_to_list(DomainID, 16)), 45 | Name, Node, Chunk, Active, length(Repl)}. 46 | 47 | start_check_domains() -> 48 | {ok, spawn_link(fun() -> register(check_domains, self()), 49 | check_domains() end)}. 50 | 51 | infopack_lookup(Key) -> 52 | case ets:lookup(infopack_cache, Key) of 53 | [] -> throw({http_error, 404, <<"Unknown domain">>}); 54 | X -> X 55 | end. 56 | 57 | check_domains() -> 58 | catch ets:new(infopack_cache, [named_table, bag]), 59 | catch ets:new(domains_t, [named_table]), 60 | 61 | Nodes = ringo_util:ringo_nodes(), 62 | lists:foreach(fun(Node) -> 63 | case catch gen_server:call({ringo_node, Node}, 64 | get_domainlist) of 65 | {ok, Domains} -> insert_and_index(Node, Domains); 66 | Error -> error_logger:warning_report( 67 | {"Couldn't get domainlist from", Node, Error}) 68 | end 69 | end, Nodes), 70 | catch ets:delete(active_domains), 71 | ets:rename(domains_t, active_domains), 72 | timer:sleep(10000), 73 | check_domains(). 74 | 75 | insert_and_index(Node, Domains) -> 76 | lists:foreach(fun({DomainID, _}) -> 77 | case ets:member(infopack_cache, DomainID) of 78 | false -> get_infopack(Node, DomainID); 79 | true -> ok 80 | end 81 | end, Domains), 82 | ets:insert(domains_t, Domains). 83 | 84 | get_infopack(Node, DomainID) -> 85 | case catch gen_server:call({ringo_node, Node}, 86 | {get_infopack, DomainID}) of 87 | {ok, Nfo} -> 88 | {value, {name, Name}} = lists:keysearch(name, 1, Nfo), 89 | {value, {chunk, Chunk}} = lists:keysearch(chunk, 1, Nfo), 90 | ets:insert(infopack_cache, {{name, Name}, DomainID}), 91 | ets:insert(infopack_cache, {{node, Node}, DomainID}), 92 | ets:insert(infopack_cache, {{id, DomainID}, {Name, Node, Chunk}}); 93 | Error -> error_logger:warning_report({"Couldn't get infopack for", 94 | DomainID, "from", Node, Error}) 95 | end. 96 | 97 | receive_domainstatus(L, 0) -> L; 98 | receive_domainstatus(L, N) -> 99 | receive 100 | {status, Node, E} -> 101 | receive_domainstatus([{Node, E}|L], N - 1); 102 | _ -> 103 | receive_domainstatus(L, N) 104 | after 2000 -> L 105 | end. 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | -------------------------------------------------------------------------------- /ringogw/src/handle_ring.erl: -------------------------------------------------------------------------------- 1 | -module(handle_ring). 2 | -export([op/2, start_check_node_status/0]). 3 | 4 | op("nodes", _Query) -> 5 | case catch ets:tab2list(node_status_table) of 6 | {'EXIT', _} -> {json, []}; 7 | L -> {json, check_ring(ringo_util:group_pairs(L))} 8 | end; 9 | 10 | op("reset", _Query) -> 11 | catch exit(whereis(check_node_status), kill), 12 | {json, {ok, <<"killed">>}}. 13 | 14 | start_check_node_status() -> 15 | {ok, spawn_link(fun() -> register(check_node_status, self()), 16 | check_node_status() end)}. 17 | 18 | check_node_status() -> 19 | ets:new(tmptable, [named_table, bag]), 20 | 21 | Nodes = ringo_util:ringo_nodes(), 22 | 23 | {OkNodes, BadNodes} = gen_server:multi_call(Nodes, 24 | ringo_node, get_neighbors, 2000), 25 | 26 | ets:insert(tmptable, [{N, {neighbors, {Prev, Next}}} || 27 | {N, {ok, Prev, Next}} <- OkNodes]), 28 | ets:insert(tmptable, [{N, {neighbors, timeout}} || N <- BadNodes]), 29 | 30 | lists:foreach(fun(Node) -> 31 | spawn(Node, ringo_util, send_system_status, [self()]) 32 | end, Nodes), 33 | ok = collect_results(), 34 | 35 | % We have an obvious race condition here: Check_node_status process may 36 | % delete node_status_table while a query handler is accessing it. We 37 | % just assume that this is an infrequent event and it doesn't matter if 38 | % a query fails occasionally as it is unlikely that many consequent 39 | % update requests would fail. 40 | catch ets:delete(node_status_table), 41 | ets:rename(tmptable, node_status_table), 42 | check_node_status(). 43 | 44 | collect_results() -> 45 | receive 46 | {node_results, NodeStatus} -> 47 | ets:insert(tmptable, NodeStatus), 48 | collect_results(); 49 | _ -> collect_results() 50 | after 10000 -> ok 51 | end. 52 | 53 | check_ring([]) -> []; 54 | check_ring(Nodes) -> 55 | % First sort the nodes according to ascending node ID 56 | {_, Sorted} = lists:unzip(ringo_util:sort_nodes(Nodes)), 57 | 58 | % Obtain the last node's information, which will be check against the 59 | % first one. 60 | [Last|_] = lists:reverse(Sorted), 61 | {_, {LastNode, LastNext}} = check_node(Last, {false, false}), 62 | % Check validity of each node's position in the ring, that is 63 | % prev and next entries match, and construct a JSON object as the 64 | % result. 65 | {Ret, _} = lists:mapfoldl(fun check_node/2, 66 | {LastNode, LastNext}, Sorted), 67 | Ret. 68 | 69 | check_node({Node, Attr}, {PrevNode, PrevNext}) -> 70 | case lists:keysearch(neighbors, 1, Attr) of 71 | {value, {neighbors, {Prev, Next}}} -> 72 | V = (PrevNode == Prev) and (PrevNext == Node), 73 | {{obj, [{node, Node}, {ok, V}|Attr]}, {Node, Next}}; 74 | {value, {neighbors, timeout}} -> 75 | {{obj, [{node, Node}, {ok, false}|Attr]}, {Node, Node}} 76 | end. 77 | 78 | -------------------------------------------------------------------------------- /ringogw/src/make_boot.erl: -------------------------------------------------------------------------------- 1 | -module(make_boot). 2 | -export([write_scripts/0]). 3 | 4 | write_scripts() -> 5 | Erts = erlang:system_info(version), 6 | {value, {kernel, _, Kernel}} = lists:keysearch(kernel, 1, 7 | application:loaded_applications()), 8 | {value, {stdlib, _, Stdlib}} = lists:keysearch(stdlib, 1, 9 | application:loaded_applications()), 10 | 11 | Rel = "{release, {\"Ringogw\", \"1\"}, {erts, \"~s\"}, [" 12 | "{kernel, \"~s\"}, {stdlib, \"~s\"}, {ringogw, \"1\"}]}.", 13 | 14 | {ok, Fs} = file:open("ringogw.rel", [write]), 15 | io:format(Fs, Rel, [Erts, Kernel, Stdlib]), 16 | file:close(Fs), 17 | 18 | systools:make_script("ringogw", [local]), 19 | halt(). 20 | 21 | 22 | -------------------------------------------------------------------------------- /ringogw/src/mochi_dispatch.erl: -------------------------------------------------------------------------------- 1 | -module(mochi_dispatch). 2 | -export([request/1]). 3 | 4 | % MAX_RECV_BODY can't be larger than 16M due to a gen_tcp:recv restriction in 5 | % raw mode: 6 | % http://www.erlang.org/pipermail/erlang-questions/2006-September/022907.html 7 | 8 | -define(MAX_RECV_BODY, (10 * 1024 * 1024)). 9 | 10 | request(Req) -> 11 | {ok, Dyn} = application:get_env(dynroot), 12 | {ok, Doc} = application:get_env(docroot), 13 | P = Req:get(path), 14 | inet:setopts(Req:get(socket), [{nodelay, true}]), 15 | case string:tokens(P, "/") of 16 | [Dyn, N, Script] -> serve_dynamic(Req, N, Script); 17 | [Dyn, N, Script|R] -> serve_dynamic(Req, N, [Script|R]); 18 | [] -> Req:serve_file("", Doc); 19 | E -> Req:serve_file(lists:last(E), Doc) 20 | end, 21 | % If this is a keep-alive session, this process is likely to serve 22 | % many requests. By flushing the inbox we make sure that no spurious 23 | % and unhandled messages accumulate in the process' inbox. 24 | ringogw_util:flush_inbox(). 25 | 26 | serve_dynamic(Req, N, Script) -> 27 | Mod = list_to_existing_atom("handle_" ++ N), 28 | Q = Req:parse_qs(), 29 | case Req:get(method) of 30 | 'GET' -> catch_op(Req, Mod, [Script, Q]); 31 | 'POST' -> catch_op(Req, Mod, [Script, Q, 32 | Req:recv_body(?MAX_RECV_BODY)]) 33 | end. 34 | 35 | catch_op(Req, Mod, Args) -> 36 | case catch apply(Mod, op, Args) of 37 | {http_error, Code, Error} -> 38 | error_logger:error_report({"HTTP error", Code, Error}), 39 | Req:respond({Code, [{"Content-Type", "text/plain"}], 40 | json:encode(Error)}); 41 | {'EXIT', Error} -> 42 | error_logger:error_report({"Request failed", Error}), 43 | Req:respond({500, [{"Content-Type", "text/plain"}], 44 | <<"\"Internal server error\"">>}); 45 | {json, Res} -> 46 | Req:ok({"text/plain", json:encode(Res)}); 47 | 48 | {data, Res} -> 49 | [_, Params] = Args, 50 | Req:ok({proplists:get_value("mime", 51 | Params, "application/octet-stream"), Res}); 52 | 53 | {chunked, ReplyGen} -> 54 | Req:respond({200, [{"Content-type", 55 | "application/octet-stream"}], chunked}), 56 | ringogw_util:chunked_reply( 57 | fun(B) -> Req:send(B) end, ReplyGen) 58 | end. 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /ringogw/src/ringogw.erl: -------------------------------------------------------------------------------- 1 | -module(ringogw). 2 | -behaviour(supervisor). 3 | -behaviour(application). 4 | 5 | -compile([verbose, report_errors, report_warnings, trace, debug_info]). 6 | -define(MAX_R, 10). 7 | -define(MAX_T, 60). 8 | 9 | -define(COMMON, [{active_node_updater, {handle_data, 10 | start_active_node_updater, []}, 11 | permanent, 10, worker, dynamic}, 12 | {chunk_cache, {handle_data, 13 | start_chunk_cache, []}, 14 | permanent, 10, worker, dynamic}, 15 | {check_domains, {handle_domains, 16 | start_check_domains, []}, 17 | permanent, 10, worker, dynamic}, 18 | {check_node_status, {handle_ring, 19 | start_check_node_status, []}, 20 | permanent, 10, worker, dynamic} 21 | ]). 22 | 23 | -export([init/1, start/2, stop/1]). 24 | 25 | 26 | conf(P) -> 27 | case application:get_env(P) of 28 | undefined -> exit(["Specify ", P]); 29 | {ok, Val} -> Val 30 | end. 31 | 32 | start(_Type, _Args) -> 33 | HttpMode = conf(httpmode), 34 | Port = conf(port), 35 | supervisor:start_link(ringogw, [HttpMode, Port]). 36 | 37 | init([mochi, Port]) -> 38 | error_logger:info_report([{"Ringo gateway starts (Mochi)"}]), 39 | {ok, {{one_for_one, ?MAX_R, ?MAX_T}, [ 40 | {mochiweb_http, {mochiweb_http, start, [[{port, Port}, 41 | {loop, {mochi_dispatch, request}}]]}, 42 | permanent, 10, worker, dynamic} 43 | ] ++ ?COMMON 44 | }}; 45 | 46 | init([scgi, Port]) -> 47 | error_logger:info_report([{"Ringo gateway starts (SCGI)"}]), 48 | {ok, {{one_for_one, ?MAX_R, ?MAX_T}, [ 49 | {scgi_server, {scgi_server, start_link, [Port]}, 50 | permanent, 10, worker, dynamic} 51 | ] ++ ?COMMON 52 | }}. 53 | 54 | stop(_State) -> 55 | ok. 56 | 57 | -------------------------------------------------------------------------------- /ringogw/src/ringogw_util.erl: -------------------------------------------------------------------------------- 1 | -module(ringogw_util). 2 | -export([chunked_reply/2, flush_inbox/0]). 3 | 4 | chunked_reply(Sender, ReplyGen) -> chunked_reply(Sender, ReplyGen, 0). 5 | chunked_reply(Sender, ReplyGen, N) -> 6 | case catch ReplyGen(N) of 7 | {entry, Entry} -> 8 | Sender(encode_chunk(Entry, <<"ok">>)), 9 | chunked_reply(Sender, ReplyGen, N); 10 | {next, N0} -> 11 | chunked_reply(Sender, ReplyGen, N0); 12 | done -> 13 | Sender(encode_chunk(done)); 14 | timeout -> 15 | Sender(encode_chunk(<<>>, <<"timeout">>)), 16 | Sender(encode_chunk(done)); 17 | invalid_domain -> 18 | Sender(encode_chunk(<<>>, <<"invalid domain">>)), 19 | Sender(encode_chunk(done)); 20 | {'EXIT', Error} -> 21 | error_logger:error_report( 22 | {"Chunked result generator failed", Error}), 23 | Sender(encode_chunk(<<>>, <<"error">>)), 24 | Sender(encode_chunk(done)) 25 | end. 26 | 27 | % last chunk 28 | encode_chunk(done) -> <<"0\r\n\r\n">>. 29 | encode_chunk(Data, Code) -> 30 | Prefixed = [io_lib:format("~b ", [size(Data)]), Code, " ", Data], 31 | [io_lib:format("~.16b\r\n", [iolist_size(Prefixed)]), Prefixed, "\r\n"]. 32 | 33 | flush_inbox() -> 34 | receive 35 | _ -> flush_inbox() 36 | after 0 -> ok 37 | end. 38 | -------------------------------------------------------------------------------- /ringogw/src/scgi.erl: -------------------------------------------------------------------------------- 1 | % Copyright (c) 2007 Ville H. Tuulos 2 | % 3 | % Permission is hereby granted, free of charge, to any person obtaining a copy 4 | % of this software and associated documentation files (the "Software"), to deal 5 | % in the Software without restriction, including without limitation the rights 6 | % to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | % copies of the Software, and to permit persons to whom the Software is 8 | % furnished to do so, subject to the following conditions: 9 | % 10 | % The above copyright notice and this permission notice shall be included in 11 | % all copies or substantial portions of the Software. 12 | % 13 | % THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | % IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | % FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | % AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | % LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | % OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | % THE SOFTWARE. 20 | 21 | 22 | -module(scgi). 23 | -export([receive_scgi_message/1, parse_scgi_message/1]). 24 | 25 | receive_scgi_message(Socket) -> 26 | receive_scgi_message(header, Socket, []). 27 | 28 | receive_scgi_message(header, Socket, Msg) -> 29 | case gen_tcp:recv(Socket, 1) of 30 | {ok, <<":">>} -> receive_scgi_message( 31 | body, Socket, list_to_integer( 32 | lists:reverse(lists:flatten(Msg)))); 33 | {ok, C} -> receive_scgi_message(header, Socket, 34 | [binary_to_list(C)|Msg]); 35 | _Other -> {error, invalid_scgi_header} 36 | end; 37 | 38 | receive_scgi_message(body, Socket, Length) -> 39 | case gen_tcp:recv(Socket, Length + 1) of 40 | {ok, Packet} -> {ok, parse_scgi_message( 41 | binary_to_list(Packet))}; 42 | _Other -> {error, invalid_scgi_body} 43 | end. 44 | 45 | parse_scgi_message([]) -> []; 46 | parse_scgi_message(Packet) -> 47 | {ok, A} = regexp:split(Packet, 0), 48 | parse_scgi_message(A, []). 49 | 50 | parse_scgi_message([], Lst) -> Lst; 51 | parse_scgi_message(Packet, Lst) -> 52 | [Key|A] = Packet, 53 | case A of 54 | [] -> Lst; 55 | [Value|B] -> parse_scgi_message(B, [{Key, Value}|Lst]) 56 | end. 57 | -------------------------------------------------------------------------------- /ringogw/src/scgi_server.erl: -------------------------------------------------------------------------------- 1 | % Copyright (c) 2007 Ville H. Tuulos 2 | % 3 | % Permission is hereby granted, free of charge, to any person obtaining a copy 4 | % of this software and associated documentation files (the "Software"), to deal 5 | % in the Software without restriction, including without limitation the rights 6 | % to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | % copies of the Software, and to permit persons to whom the Software is 8 | % furnished to do so, subject to the following conditions: 9 | % 10 | % The above copyright notice and this permission notice shall be included in 11 | % all copies or substantial portions of the Software. 12 | % 13 | % THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | % IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | % FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | % AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | % LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | % OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | % THE SOFTWARE. 20 | 21 | 22 | -module(scgi_server). 23 | -behaviour(gen_server). 24 | 25 | -compile([verbose, report_errors, report_warnings, trace, debug_info]). 26 | -define(TCP_OPTIONS, [binary, {active, false}, {reuseaddr, true}, {packet, raw}]). 27 | 28 | -export([start_link/1, stop/0, handle_request/2, scgi_worker/1]). 29 | 30 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 31 | terminate/2, code_change/3]). 32 | 33 | -define(ERROR_REDIRECT, "Location: /\r\n\r\n"). 34 | -define(ERROR_500, "HTTP/1.0 500 Error\r\n" 35 | "Status: 500\r\n\r\n" 36 | "\"500 - Internal Server Error\""). 37 | -define(ERROR, "HTTP/1.1 ~w Error\r\n" 38 | "Status: ~w\r\n" 39 | "Content-type: text/plain\r\n\r\n"). 40 | 41 | -define(HTTP_HEADER(X), ["HTTP/1.1 200 OK\n" 42 | "Status: 200 OK\n" 43 | "Content-type:", X, "\n\n"]). 44 | 45 | % scgi_server external interface 46 | 47 | start_link(Port) -> 48 | error_logger:info_report([{'SCGI SERVER STARTS'}]), 49 | case gen_server:start_link({local, scgi_server}, 50 | scgi_server, [], []) of 51 | {ok, Server} -> gen_server:call(scgi_server, {listen, Port}), 52 | {ok, Server}; 53 | {error, {already_started, Server}} -> {ok, Server} 54 | end. 55 | 56 | stop() -> 57 | gen_server:call(scgi_server, stop). 58 | 59 | % gen_server callbacks 60 | 61 | init(_Args) -> 62 | process_flag(trap_exit, true), 63 | {ok, {}}. 64 | 65 | handle_call({listen, Port}, _From, State) -> 66 | case catch gen_tcp:listen(Port, ?TCP_OPTIONS) of 67 | {ok, LSocket} -> spawn_link(fun() -> scgi_worker(LSocket) end), 68 | {reply, ok, LSocket}; 69 | Error -> {stop, {listen_failed, Error}, State} 70 | end; 71 | 72 | handle_call({new_worker, Worker}, _From, LSocket) -> 73 | spawn_link(fun() -> scgi_server:scgi_worker(LSocket) end), 74 | erlang:unlink(Worker), 75 | {reply, ok, LSocket}; 76 | 77 | handle_call(stop, _From, LSocket) -> {stop, stop_requested, LSocket}. 78 | 79 | handle_info({'EXIT', _Pid, _Reason}, LSocket) -> 80 | {noreply, LSocket}. 81 | 82 | % scgi stuff 83 | 84 | scgi_worker(LSocket) -> 85 | case gen_tcp:accept(LSocket) of 86 | {ok, Socket} -> 87 | gen_server:call(scgi_server, {new_worker, self()}), 88 | {ok, Msg} = scgi:receive_scgi_message(Socket), 89 | scgi_server:handle_request(Socket, Msg), 90 | gen_tcp:close(Socket) 91 | end. 92 | 93 | handle_request(Socket, Msg) -> 94 | case catch dispatch_request(Socket, Msg) of 95 | {json, Res} -> 96 | gen_tcp:send(Socket, [?HTTP_HEADER( 97 | "text/plain"), json:encode(Res)]); 98 | {data, Res} -> 99 | gen_tcp:send(Socket, [?HTTP_HEADER( 100 | "application/octet-stream"), Res]); 101 | {chunked, ReplyGen} -> 102 | % FIXME: Check how Lighttpd handles chunked output 103 | gen_tcp:send(Socket, [?HTTP_HEADER( 104 | "application/octet-stream")]), 105 | ringogw_util:chunked_reply(fun(B) -> 106 | gen_tcp:send(Socket, B) end, ReplyGen); 107 | {http_error, Code, Error} -> 108 | gen_tcp:send(Socket, [io_lib:format(?ERROR, [Code, Code]), 109 | json:encode(Error)]); 110 | {'EXIT', Error} -> 111 | error_logger:info_report(["Timeout", 112 | trunc_io:fprint(Error, 500)]), 113 | gen_tcp:send(Socket, ?ERROR_500); 114 | Error -> 115 | error_logger:info_report(["Error", 116 | trunc_io:fprint(Error, 500)]), 117 | gen_tcp:send(Socket, ?ERROR_500) 118 | end. 119 | 120 | dispatch_request(Socket, Msg) -> 121 | {value, {_, Path1}} = lists:keysearch("SCRIPT_NAME", 1, Msg), 122 | {value, {_, Path2}} = lists:keysearch("PATH_INFO", 1, Msg), 123 | {value, {_, Query}} = lists:keysearch("QUERY_STRING", 1, Msg), 124 | {value, {_, CLenStr}} = lists:keysearch("CONTENT_LENGTH", 1, Msg), 125 | {value, {_, Method}} = lists:keysearch("REQUEST_METHOD", 1, Msg), 126 | [_, N, Script] = case string:tokens( 127 | lists:flatten([Path1, Path2]), "/") of 128 | [_, _, _] = R -> R; 129 | [_, N1, S1|R] -> [none, N1, [S1|R]] 130 | end, 131 | 132 | Mod = list_to_existing_atom("handle_" ++ N), 133 | CLen = list_to_integer(CLenStr), 134 | 135 | if Method == "POST" -> 136 | if CLen > 0 -> 137 | {ok, PostData} = gen_tcp:recv(Socket, CLen, 30000); 138 | true -> 139 | PostData = <<>> 140 | end, 141 | Mod:op(Script, httpd:parse_query(Query), PostData); 142 | true -> 143 | Mod:op(Script, httpd:parse_query(Query)) 144 | end. 145 | 146 | % callback stubs 147 | 148 | terminate(_Reason, _State) -> {}. 149 | 150 | handle_cast(_Cast, State) -> {noreply, State}. 151 | 152 | code_change(_OldVsn, State, _Extra) -> {ok, State}. 153 | -------------------------------------------------------------------------------- /ringogw/src/trunc_io.erl: -------------------------------------------------------------------------------- 1 | %% ``The contents of this file are subject to the Erlang Public License, 2 | %% Version 1.1, (the "License"); you may not use this file except in 3 | %% compliance with the License. You should have received a copy of the 4 | %% Erlang Public License along with your Erlang distribution. If not, it can be 5 | %% retrieved via the world wide web at http://www.erlang.org/. 6 | %% 7 | %% Software distributed under the License is distributed on an "AS IS" 8 | %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See 9 | %% the License for the specific language governing rights and limitations 10 | %% under the License. 11 | %% 12 | %% The Initial Developer of the Original Code is Corelatus AB. 13 | %% Portions created by Corelatus are Copyright 2003, Corelatus 14 | %% AB. All Rights Reserved.'' 15 | % 16 | %% 17 | %% Module to print out terms for logging. Limits by length rather than depth. 18 | %% 19 | %% The resulting string may be slightly larger than the limit; the intention 20 | %% is to provide predictable CPU and memory consumption for formatting 21 | %% terms, not produce precise string lengths. 22 | %% 23 | %% Typical use: 24 | %% 25 | %% trunc_io:print(Term, 500). 26 | %% 27 | -module(trunc_io). 28 | -author('matthias@corelatus.se'). 29 | -export([print/2, fprint/2, safe/2]). % interface functions 30 | -export([perf/0, perf/3, perf1/0, test/0, test/2]). % testing functions 31 | 32 | 33 | %% Returns an flattened list containing the ASCII representation of the given 34 | %% term. 35 | fprint(T, Max) -> 36 | {L, _} = print(T, Max), 37 | lists:flatten(L). 38 | 39 | %% Same as print, but never crashes. 40 | %% 41 | %% This is a bit of a tradeoff. Print might conceivably crash if it's 42 | %% asked to print something it doesn't understand, for example some new 43 | %% data type in a future version of Erlang. If print crashes, we fall 44 | %% back to io_lib to format the term, but then the formatting is 45 | %% depth-limited instead of length limited, so you might run out 46 | %% memory printing it. Out of the frying pan and into the fire. 47 | %% 48 | safe(What, Len) -> 49 | case catch print(What, Len) of 50 | {L, Used} when list(L) -> {L, Used}; 51 | _ -> {"unable to print" ++ io_lib:write(What, 99)} 52 | end. 53 | 54 | %% Returns {List, Length} 55 | print(_, Max) when Max < 0 -> {"...", 3}; 56 | print(Tuple, Max) when tuple(Tuple) -> 57 | {TC, Len} = tuple_contents(Tuple, Max-2), 58 | {[${, TC, $}], Len + 2}; 59 | 60 | %% We assume atoms, floats, funs, integers, PIDs, ports and refs never need 61 | %% to be truncated. This isn't strictly true, someone could make an 62 | %% arbitrarily long bignum. Let's assume that won't happen unless someone 63 | %% is being malicious. 64 | %% 65 | print(Atom, _Max) when atom(Atom) -> 66 | L = atom_to_list(Atom), 67 | {L, length(L)}; 68 | 69 | print(<<>>, _Max) -> 70 | {"<<>>", 4}; 71 | 72 | print(Binary, Max) when binary(Binary) -> 73 | B = binary_to_list(Binary, 1, lists:min([Max, size(Binary)])), 74 | {L, Len} = alist_start(B, Max-4), 75 | {["<<", L, ">>"], Len}; 76 | 77 | print(Float, _Max) when float(Float) -> 78 | L = float_to_list(Float), 79 | {L, length(L)}; 80 | 81 | print(Fun, _Max) when function(Fun) -> 82 | L = erlang:fun_to_list(Fun), 83 | {L, length(L)}; 84 | 85 | print(Integer, _Max) when integer(Integer) -> 86 | L = integer_to_list(Integer), 87 | {L, length(L)}; 88 | 89 | print(Pid, _Max) when pid(Pid) -> 90 | L = pid_to_list(Pid), 91 | {L, length(L)}; 92 | 93 | print(Ref, _Max) when reference(Ref) -> 94 | L = erlang:ref_to_list(Ref), 95 | {L, length(L)}; 96 | 97 | print(Port, _Max) when port(Port) -> 98 | L = erlang:port_to_list(Port), 99 | {L, length(L)}; 100 | 101 | print(List, Max) when list(List) -> 102 | alist_start(List, Max). 103 | 104 | %% Returns {List, Length} 105 | tuple_contents(Tuple, Max) -> 106 | L = tuple_to_list(Tuple), 107 | list_body(L, Max). 108 | 109 | %% Returns {List, Length} 110 | list_body([], _) -> {[], 0}; 111 | list_body(_, Max) when Max < 4 -> {"...", 3}; 112 | list_body([H|T], Max) -> 113 | {List, Len} = print(H, Max), 114 | {Final, FLen} = list_bodyc(T, Max-Len-1), 115 | {[List|Final], FLen + Len + 1}; 116 | list_body(X, Max) -> %% improper list 117 | {List, Len} = print(X, Max - 2), 118 | {[$|,List,$]], Len + 2}. 119 | 120 | list_bodyc([], _) -> {[], 0}; 121 | list_bodyc(_, Max) when Max < 4 -> {"...", 3}; 122 | list_bodyc([H|T], Max) -> 123 | {List, Len} = print(H, Max), 124 | {Final, FLen} = list_bodyc(T, Max-Len-1), 125 | {[$,, List|Final], FLen + Len + 1}; 126 | list_bodyc(X,Max) -> %% improper list 127 | {List, Len} = print(X, Max - 2), 128 | {[$|,List,$]], Len + 2}. 129 | 130 | %% The head of a list we hope is ascii. Examples: 131 | %% 132 | %% [65,66,67] -> "ABC" 133 | %% [65,0,67] -> "A"[0,67] 134 | %% [0,65,66] -> [0,65,66] 135 | %% [65,b,66] -> "A"[b,66] 136 | %% 137 | alist_start([], _) -> {"[]", 2}; 138 | alist_start(_, Max) when Max < 4 -> {"...", 3}; 139 | alist_start([H|T], Max) when H >= 16#20, H =< 16#7e -> % definitely printable 140 | {L, Len} = alist([H|T], Max-1), 141 | {[$"|L], Len + 1}; 142 | alist_start([H|T], Max) when H == 9; H == 10; H == 13 -> % show as space 143 | {L, Len} = alist(T, Max-1), 144 | {[$ |L], Len + 1}; 145 | alist_start(L, Max) -> 146 | {R, Len} = list_body(L, Max-2), 147 | {[$[, R, $]], Len + 2}. 148 | 149 | alist([], _) -> {"\"", 1}; 150 | alist(_, Max) when Max < 5 -> {"...\"", 4}; 151 | alist([H|T], Max) when H >= 16#20, H =< 16#7e -> % definitely printable 152 | {L, Len} = alist(T, Max-1), 153 | {[H|L], Len + 1}; 154 | alist([H|T], Max) when H == 9; H == 10; H == 13 -> % show as space 155 | {L, Len} = alist(T, Max-1), 156 | {[$ |L], Len + 1}; 157 | alist(L, Max) -> 158 | {R, Len} = list_body(L, Max-2), 159 | {[$", $[, R, $]], Len + 2}. 160 | 161 | 162 | %%-------------------- 163 | %% The start of a test suite. So far, it only checks for not crashing. 164 | test() -> 165 | test(trunc_io, print). 166 | 167 | test(Mod, Func) -> 168 | Simple_items = [atom, 1234, 1234.0, {tuple}, [], [list], "string", self(), 169 | <<1,2,3>>, make_ref(), fun() -> ok end], 170 | F = fun(A) -> 171 | Mod:Func(A, 100), 172 | Mod:Func(A, 2), 173 | Mod:Func(A, 20) 174 | end, 175 | 176 | G = fun(A) -> 177 | case catch F(A) of 178 | {'EXIT', _} -> exit({failed, A}); 179 | _ -> ok 180 | end 181 | end, 182 | 183 | lists:foreach(G, Simple_items), 184 | 185 | Tuples = [ {1,2,3,a,b,c}, {"abc", def, 1234}, 186 | {{{{a},b,c,{d},e}},f}], 187 | 188 | Lists = [ [1,2,3,4,5,6,7], lists:seq(1,1000), 189 | [{a}, {a,b}, {a, [b,c]}, "def"], [a|b], [$a|$b] ], 190 | 191 | 192 | lists:foreach(G, Tuples), 193 | lists:foreach(G, Lists). 194 | 195 | perf() -> 196 | {New, _} = timer:tc(trunc_io, perf, [trunc_io, print, 1000]), 197 | {Old, _} = timer:tc(trunc_io, perf, [io_lib, write, 1000]), 198 | io:fwrite("New code took ~p us, old code ~p\n", [New, Old]). 199 | 200 | perf(M, F, Reps) when Reps > 0 -> 201 | test(M,F), 202 | perf(M,F,Reps-1); 203 | perf(_,_,_) -> 204 | done. 205 | 206 | %% Performance test. Needs a particularly large term I saved as a binary... 207 | perf1() -> 208 | {ok, Bin} = file:read_file("bin"), 209 | A = binary_to_term(Bin), 210 | {N, _} = timer:tc(trunc_io, print, [A, 1500]), 211 | {M, _} = timer:tc(io_lib, write, [A]), 212 | {N, M}. 213 | -------------------------------------------------------------------------------- /ringogw/start_ringogw.sh: -------------------------------------------------------------------------------- 1 | 2 | MODE=mochi 3 | PORT=15000 4 | SCGI=1 5 | 6 | if [ ! -z $SCGI ] 7 | then 8 | MODE=scgi 9 | PORT=15001 10 | echo "SCGI mode" 11 | fi 12 | 13 | export HEART_COMMAND="$0 $@" 14 | 15 | PATH=$PATH:/usr/sbin lighttpd -f lighttpd.conf -D & 16 | 17 | PATH=.:$PATH erl -heart -noshell -detached +K true -smp on -sname ringogw -setcookie ringobingo -pa ../mochiweb/ebin -pa ../ring/ebin -pa ebin -pa src -boot ringogw -ringogw httpmode $MODE -ringogw port $PORT -ringogw docroot \"web\" -ringogw dynroot \"mon\" -kernel error_logger '{file, "ringogw.log"}' -eval "[handle_ring, handle_domains, handle_data]" 18 | 19 | -------------------------------------------------------------------------------- /ringogw/web/index.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | ringo status 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 |
17 |

ringo status

18 |
19 |
20 |
21 |
22 |
Ring
23 |
24 |
25 |
26 |
disk: used / use% / inode use% / vm mem
27 |
28 |
29 |
Domains
30 |
31 |
32 | 33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
Domain status
41 |
42 |
43 | 44 | 45 | 46 | 50 | 51 |
this domainowner
52 |
53 |
54 |
55 |
56 | 57 |
58 |
59 |
60 | 61 | 62 | -------------------------------------------------------------------------------- /ringogw/web/jquery-dom.js: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * DOM node creation for jQuery. 4 | * 5 | * Author : Sean Gilbertson 6 | * Created : 2006-01-04 7 | * 8 | */ 9 | 10 | $.create = function() { 11 | if (arguments.length == 0) { 12 | return []; 13 | } 14 | 15 | var first_arg = arguments[0]; 16 | 17 | /* 18 | * In case someone passes in a null object, 19 | * assume that they want an empty string. 20 | */ 21 | if (first_arg == null) { 22 | first_arg = ""; 23 | } 24 | 25 | if (first_arg.constructor == String) { 26 | if (arguments.length > 1) { 27 | var second_arg = arguments[1]; 28 | 29 | if (second_arg.constructor == String) { 30 | var elt = document.createTextNode(first_arg); 31 | 32 | var elts = []; 33 | 34 | elts.push(elt); 35 | 36 | var siblings = $.create.apply(null, Array.prototype.slice.call(arguments, 1)); 37 | 38 | elts = elts.concat(siblings); 39 | 40 | return elts; 41 | } else { 42 | var elt = document.createElement(first_arg); 43 | 44 | /* 45 | * Set element attributes. 46 | */ 47 | var attributes = arguments[1]; 48 | 49 | for (var attr in attributes) { 50 | $(elt).attr(attr, attributes[attr]); 51 | } 52 | 53 | /* 54 | * Add children of this element. 55 | */ 56 | var children = arguments[2]; 57 | 58 | children = $.create.apply(null, children); 59 | 60 | $(elt).append(children); 61 | 62 | /* 63 | * If there are more siblings, render those too. 64 | */ 65 | if (arguments.length > 3) { 66 | var siblings = $.create.apply(null, Array.prototype.slice.call(arguments, 3)); 67 | 68 | return [elt].concat(siblings); 69 | } 70 | 71 | return elt; 72 | } 73 | } else { 74 | return document.createTextNode(first_arg); 75 | } 76 | } else { 77 | var elts = []; 78 | 79 | elts.push(first_arg); 80 | 81 | var siblings = $.create.apply(null, (Array.prototype.slice.call(arguments, 1))); 82 | 83 | elts = elts.concat(siblings); 84 | 85 | return elts; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /ringogw/web/reset-fonts-grids.css: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2008, Yahoo! Inc. All rights reserved. 3 | Code licensed under the BSD License: 4 | http://developer.yahoo.net/yui/license.txt 5 | version: 2.5.1 6 | */ 7 | html{color:#000;background:#FFF;}body,div,dl,dt,dd,ul,ol,li,h1,h2,h3,h4,h5,h6,pre,code,form,fieldset,legend,input,textarea,p,blockquote,th,td{margin:0;padding:0;}table{border-collapse:collapse;border-spacing:0;}fieldset,img{border:0;}address,caption,cite,code,dfn,em,strong,th,var{font-style:normal;font-weight:normal;}li{list-style:none;}caption,th{text-align:left;}h1,h2,h3,h4,h5,h6{font-size:100%;font-weight:normal;}q:before,q:after{content:'';}abbr,acronym {border:0;font-variant:normal;}sup {vertical-align:text-top;}sub {vertical-align:text-bottom;}input,textarea,select{font-family:inherit;font-size:inherit;font-weight:inherit;}input,textarea,select{*font-size:100%;}legend{color:#000;}body {font:13px/1.231 arial,helvetica,clean,sans-serif;*font-size:small;*font:x-small;}table {font-size:inherit;font:100%;}pre,code,kbd,samp,tt{font-family:monospace;*font-size:108%;line-height:100%;} 8 | body{text-align:center;}#ft{clear:both;}#doc,#doc2,#doc3,#doc4,.yui-t1,.yui-t2,.yui-t3,.yui-t4,.yui-t5,.yui-t6,.yui-t7{margin:auto;text-align:left;width:57.69em;*width:56.25em;min-width:750px;}#doc2{width:73.076em;*width:71.25em;}#doc3{margin:auto 10px;width:auto;}#doc4{width:74.923em;*width:73.05em;}.yui-b{position:relative;}.yui-b{_position:static;}#yui-main .yui-b{position:static;}#yui-main{width:100%;}.yui-t1 #yui-main,.yui-t2 #yui-main,.yui-t3 #yui-main{float:right;margin-left:-25em;}.yui-t4 #yui-main,.yui-t5 #yui-main,.yui-t6 #yui-main{float:left;margin-right:-25em;}.yui-t1 .yui-b{float:left;width:12.30769em;*width:12.00em;}.yui-t1 #yui-main .yui-b{margin-left:13.30769em;*margin-left:13.05em;}.yui-t2 .yui-b{float:left;width:13.8461em;*width:13.50em;}.yui-t2 #yui-main .yui-b{margin-left:14.8461em;*margin-left:14.55em;}.yui-t3 .yui-b{float:left;width:23.0769em;*width:22.50em;}.yui-t3 #yui-main .yui-b{margin-left:24.0769em;*margin-left:23.62em;}.yui-t4 .yui-b{float:right;width:13.8456em;*width:13.50em;}.yui-t4 #yui-main .yui-b{margin-right:14.8456em;*margin-right:14.55em;}.yui-t5 .yui-b{float:right;width:18.4615em;*width:18.00em;}.yui-t5 #yui-main .yui-b{margin-right:19.4615em;*margin-right:19.125em;}.yui-t6 .yui-b{float:right;width:23.0769em;*width:22.50em;}.yui-t6 #yui-main .yui-b{margin-right:24.0769em;*margin-right:23.62em;}.yui-t7 #yui-main .yui-b{display:block;margin:0 0 1em 0;}#yui-main .yui-b{float:none;width:auto;}.yui-gb .yui-u,.yui-g .yui-gb .yui-u,.yui-gb .yui-g,.yui-gb .yui-gb,.yui-gb .yui-gc,.yui-gb .yui-gd,.yui-gb .yui-ge,.yui-gb .yui-gf,.yui-gc .yui-u,.yui-gc .yui-g,.yui-gd .yui-u{float:left;}.yui-g .yui-u,.yui-g .yui-g,.yui-g .yui-gb,.yui-g .yui-gc,.yui-g .yui-gd,.yui-g .yui-ge,.yui-g .yui-gf,.yui-gc .yui-u,.yui-gd .yui-g,.yui-g .yui-gc .yui-u,.yui-ge .yui-u,.yui-ge .yui-g,.yui-gf .yui-g,.yui-gf .yui-u{float:right;}.yui-g div.first,.yui-gb div.first,.yui-gc div.first,.yui-gd div.first,.yui-ge div.first,.yui-gf div.first,.yui-g .yui-gc div.first,.yui-g .yui-ge div.first,.yui-gc div.first div.first{float:left;}.yui-g .yui-u,.yui-g .yui-g,.yui-g .yui-gb,.yui-g .yui-gc,.yui-g .yui-gd,.yui-g .yui-ge,.yui-g .yui-gf{width:49.1%;}.yui-gb .yui-u,.yui-g .yui-gb .yui-u,.yui-gb .yui-g,.yui-gb .yui-gb,.yui-gb .yui-gc,.yui-gb .yui-gd,.yui-gb .yui-ge,.yui-gb .yui-gf,.yui-gc .yui-u,.yui-gc .yui-g,.yui-gd .yui-u{width:32%;margin-left:1.99%;}.yui-gb .yui-u{*margin-left:1.9%;*width:31.9%;}.yui-gc div.first,.yui-gd .yui-u{width:66%;}.yui-gd div.first{width:32%;}.yui-ge div.first,.yui-gf .yui-u{width:74.2%;}.yui-ge .yui-u,.yui-gf div.first{width:24%;}.yui-g .yui-gb div.first,.yui-gb div.first,.yui-gc div.first,.yui-gd div.first{margin-left:0;}.yui-g .yui-g .yui-u,.yui-gb .yui-g .yui-u,.yui-gc .yui-g .yui-u,.yui-gd .yui-g .yui-u,.yui-ge .yui-g .yui-u,.yui-gf .yui-g .yui-u{width:49%;*width:48.1%;*margin-left:0;}.yui-g .yui-gb div.first,.yui-gb .yui-gb div.first{*margin-right:0;*width:32%;_width:31.7%;}.yui-g .yui-gc div.first,.yui-gd .yui-g{width:66%;}.yui-gb .yui-g div.first{*margin-right:4%;_margin-right:1.3%;}.yui-gb .yui-gc div.first,.yui-gb .yui-gd div.first{*margin-right:0;}.yui-gb .yui-gb .yui-u,.yui-gb .yui-gc .yui-u{*margin-left:1.8%;_margin-left:4%;}.yui-g .yui-gb .yui-u{_margin-left:1.0%;}.yui-gb .yui-gd .yui-u{*width:66%;_width:61.2%;}.yui-gb .yui-gd div.first{*width:31%;_width:29.5%;}.yui-g .yui-gc .yui-u,.yui-gb .yui-gc .yui-u{width:32%;_float:right;margin-right:0;_margin-left:0;}.yui-gb .yui-gc div.first{width:66%;*float:left;*margin-left:0;}.yui-gb .yui-ge .yui-u,.yui-gb .yui-gf .yui-u{margin:0;}.yui-gb .yui-gb .yui-u{_margin-left:.7%;}.yui-gb .yui-g div.first,.yui-gb .yui-gb div.first{*margin-left:0;}.yui-gc .yui-g .yui-u,.yui-gd .yui-g .yui-u{*width:48.1%;*margin-left:0;}s .yui-gb .yui-gd div.first{width:32%;}.yui-g .yui-gd div.first{_width:29.9%;}.yui-ge .yui-g{width:24%;}.yui-gf .yui-g{width:74.2%;}.yui-gb .yui-ge div.yui-u,.yui-gb .yui-gf div.yui-u{float:right;}.yui-gb .yui-ge div.first,.yui-gb .yui-gf div.first{float:left;}.yui-gb .yui-ge .yui-u,.yui-gb .yui-gf div.first{*width:24%;_width:20%;}.yui-gb .yui-ge div.first,.yui-gb .yui-gf .yui-u{*width:73.5%;_width:65.5%;}.yui-ge div.first .yui-gd .yui-u{width:65%;}.yui-ge div.first .yui-gd div.first{width:32%;}#bd:after,.yui-g:after,.yui-gb:after,.yui-gc:after,.yui-gd:after,.yui-ge:after,.yui-gf:after{content:".";display:block;height:0;clear:both;visibility:hidden;}#bd,.yui-g,.yui-gb,.yui-gc,.yui-gd,.yui-ge,.yui-gf{zoom:1;} -------------------------------------------------------------------------------- /ringogw/web/ringomon.js: -------------------------------------------------------------------------------- 1 | 2 | jQuery.fn.log = function (msg) { 3 | console.log("%s: %o", msg, this); 4 | return this; 5 | }; 6 | 7 | $(document).ready(function(){ 8 | $.getJSON("/mon/ring/nodes", update_ring); 9 | $.ajaxSetup({ 10 | cache: false, 11 | }) 12 | var options = { 13 | callback:filter_domains, 14 | wait:700, 15 | highlight:true, 16 | enterkey:true 17 | } 18 | $("#domainfilter").typeWatch(options); 19 | 20 | }); 21 | 22 | function nodename(x){ 23 | var s = x.split("@"); 24 | return [s[0].substr(6), s[1]]; 25 | } 26 | 27 | function update_ring(ringdata) 28 | { 29 | var pre_chosen = $("#ringlist > .chosen").attr("node"); 30 | $("#ringlist").html($.map(ringdata, function(node, i){ 31 | var mem = Math.round(node.disk[3] / (1024 * 1024)) + "M"; 32 | var warn = chosen = ""; 33 | if (!node.ok){ warn = "warn "; } 34 | if (node.node == pre_chosen) { chosen = "chosen "; } 35 | var N = $.create("div", {"class": "nodenfo"}, [ 36 | node.disk[2] + " / " + node.disk[0] + " / " + 37 | node.disk[1] + " / " + mem]); 38 | 39 | nextn = nodename(node.neighbors[1]); 40 | thisn = nodename(node.node); 41 | var T = $.create("div", {}, [$.create("span", {"class": "host"}, 42 | [thisn[1] + ":"]), $.create("span", {"class": "hexid"}, 43 | [thisn[0] + " \u279D " + nextn[0]])]); 44 | var R = $.create("div", {"class": "listitem " + warn + chosen}, 45 | [T, N]); 46 | $(R).click(function(){ 47 | $("#ringlist > .chosen").removeClass("chosen"); 48 | if($(this).hasClass("chosen")){ 49 | empty_domainlist(); 50 | }else{ 51 | $(this).addClass("chosen"); 52 | $.getJSON("/mon/domains/node?name=" + node.node, 53 | render_domainlist); 54 | } 55 | }).attr("node", node.node); 56 | return R; 57 | })); 58 | $("#ringtitle").html("Ring (" + ringdata.length + " nodes)") 59 | 60 | setTimeout(function(){ 61 | $.getJSON("/mon/ring/nodes", update_ring); 62 | }, 10000); 63 | } 64 | 65 | function empty_domainlist() 66 | { 67 | $("#domainlist > *").remove(); 68 | $("#domainfilter").val(""); 69 | $("#domaintitle").html("Domains"); 70 | } 71 | 72 | function render_domainlist(domainlist) 73 | { 74 | $("#domainlist").html($.map(domainlist, function(d, i){ 75 | R = render_domain(d[3], d[1], d[5], d[0], d[4]); 76 | $(R).click(function(){ 77 | $.getJSON("/mon/domains/domain?id=0x" + d[0], 78 | render_replicalist); 79 | }); 80 | return R; 81 | })); 82 | var n = domainlist.length; 83 | $("#domaintitle").html("Domains (showing " + n + "/" + n + ")"); 84 | } 85 | 86 | function render_replicalist(msg) 87 | { 88 | var id = msg[0]; 89 | var name = msg[1]; 90 | var chunk = msg[2]; 91 | var domain_info = msg[3]; 92 | var domain = render_domain(chunk, name, domain_info.length, id, true); 93 | $("#domainbox > .rtitle").html("Domains"); 94 | $(domain).addClass("chosen").click(function(){ 95 | var chosen = $("#ringlist > .chosen"); 96 | if (chosen.length) 97 | $.getJSON("/mon/domains/node?name=" + 98 | chosen.attr("node"), render_domainlist); 99 | else{ 100 | txt = $("#domainfilter").val(); 101 | $.getJSON("/mon/domains/domain?name=" + escape(txt), 102 | render_domainlist); 103 | } 104 | }); 105 | domain_info.sort(function(x, y){ 106 | return Number(y.owner) - Number(x.owner) 107 | }); 108 | var owner = {}; 109 | var replicas = $.map(domain_info, function(d, i){ 110 | var t = "replica on "; 111 | var st = ""; 112 | if (d.owner){ 113 | owner = d; 114 | t = "owner on "; 115 | st = "ownritem"; 116 | } else if (d.error){ 117 | t = "zombie on "; 118 | st = ""; 119 | } 120 | var thisn = nodename(d.node); 121 | R = $.create("div", {"class": "listitem replitem"}, [ 122 | $.create("span", {"class": st}, [t]), 123 | $.create("span", {"class": "host"}, [thisn[1] + ":"]), 124 | $.create("span", {"class": "hexid"}, [thisn[0]]) 125 | ]); 126 | $(R).click(function(){ 127 | $("#domainlist > .replitem").removeClass("chosen"); 128 | if($(this).hasClass("chosen")){ 129 | render_domaininfo({}); 130 | }else{ 131 | $(this).addClass("chosen"); 132 | render_domaininfo(domain_info[i], owner); 133 | } 134 | }); 135 | return R; 136 | }); 137 | $("#domainlist").html([domain].concat(replicas)); 138 | } 139 | 140 | function render_domain(chunk, name, num_repl, id, alive) 141 | { 142 | var C = $.create("span", {"class": "chunk"}, ["[" + chunk + "] "]); 143 | var N = $.create("span", {"class": "domain"}, [name]); 144 | var H = $.create("div", {"class": "domainnfo"}, 145 | [$.create("span", {}, [num_repl + " / "]), 146 | $.create("span", {}, [id])]); 147 | var alivec = "" 148 | if (!alive) 149 | alivec = "domain_inactive"; 150 | return $.create("div", {"class": "listitem " + alivec}, [C, N, H]); 151 | } 152 | 153 | function filter_domains(txt) 154 | { 155 | if ($("#ringlist > .chosen").length){ 156 | var all_num = $("#domainlist > *").length; 157 | $("#domainlist > *").css("display", "none"); 158 | var num = $("#domainlist > *:contains(" + txt + ")") 159 | .css("display", "block").length; 160 | $("#domaintitle").html( 161 | "Domains (showing " + num + "/" + all_num + ")"); 162 | }else{ 163 | $.getJSON("/mon/domains/domain?name=" + escape(txt), 164 | render_domainlist); 165 | } 166 | } 167 | 168 | function render_domaininfo(stat, owner) 169 | { 170 | 171 | $("#domainstat > table").hide() 172 | var rows = []; 173 | var aa = []; 174 | for (x in stat) 175 | if (x != "id") 176 | aa.push(x) 177 | $.each(aa, function(i, key){ 178 | 179 | var ext = ""; 180 | if (typeof(stat[key]) == typeof([])) 181 | ext = " \u2b0e"; 182 | 183 | var T = $.create("td", {"class": "rowhead"}, [key + ext]); 184 | var E = [T, $.create("td", {}, 185 | make_dcell(key, stat[key], true)), 186 | $.create("td", {}, 187 | make_dcell(key, owner[key], true))]; 188 | 189 | if (typeof(stat[key]) == typeof([])){ 190 | var N = [T, $.create("td", {}, 191 | make_dcell(key, stat[key], false)), 192 | $.create("td", {}, 193 | make_dcell(key, owner[key], false))]; 194 | var R = $.create("tr", {"class": "row_expands"}, N); 195 | $(R).click(function(){ 196 | if ($(this).hasClass("chosen")) 197 | $(this).removeClass("chosen").html(N); 198 | else 199 | $(this).addClass("chosen").html(E); 200 | }); 201 | }else{ 202 | var R = $.create("tr", {}, E); 203 | } 204 | rows.push(R); 205 | }); 206 | if (rows.length) 207 | $("#domainstat > table").show() 208 | $("#domainstat > table > tbody").html(rows); 209 | } 210 | 211 | preproc = {"node": 212 | function (x){ 213 | thisn = nodename(x); 214 | var X1 = $.create("span", {"class": "host"}, [thisn[1] + ":"]); 215 | var X2 = $.create("span", {"class": "hexid"}, [thisn[0]]); 216 | return [X1, X2]; 217 | }, 218 | "started": function (x){ 219 | return [$.create("span", 220 | {"class": "tstamp"}, [" (" + x + ")"])]; 221 | }, 222 | "sync_time": function (x) { return [""]; } 223 | }; 224 | 225 | 226 | function make_dcell(key, e, show_all) 227 | { 228 | if (key in preproc) 229 | pp = preproc[key]; 230 | else 231 | pp = function(x) { return [x.toString()]; } 232 | 233 | if (typeof(e) == typeof([])){ 234 | if (show_all) 235 | return $.makeArray($.map(e, function(d, i){ 236 | return $.create("div", {}, 237 | val_with_tstamp(pp(d[1]), d[0])); 238 | })); 239 | else 240 | return val_with_tstamp(pp(e[0][1]), e[0][0]); 241 | }else if(e){ 242 | return pp(e); 243 | }else 244 | return [""]; 245 | } 246 | 247 | function val_with_tstamp(v, t){ 248 | return [$.create("span", {}, [v.toString()]), 249 | $.create("span", {"class": "tstamp"}, 250 | [" (" + t + ")"])]; 251 | } 252 | -------------------------------------------------------------------------------- /ringogw/web/style.css: -------------------------------------------------------------------------------- 1 | 2 | body { 3 | background: #ffffff; 4 | color: #000000; 5 | font-family: Verdana, "Bitstream Vera Sans", sans-serif; 6 | line-height: 1.8em; 7 | margin-left: 20px; 8 | } 9 | 10 | #hd h1{ 11 | margin-top: 50px; 12 | font-size: 250%; 13 | height: 50px; 14 | } 15 | 16 | .rcolumn{ 17 | border: 1px solid #666; 18 | height: 500px; 19 | padding-top: 10px; 20 | } 21 | 22 | .rtitle{ 23 | background: #333; 24 | color: white; 25 | padding-left: 5px; 26 | text-align: left; 27 | } 28 | 29 | .listbox{ 30 | height: 100%; 31 | overflow: auto; 32 | white-space: nowrap; 33 | } 34 | 35 | .rcolumn input{ 36 | border: 1px solid #aaa; 37 | width: 90%; 38 | margin-bottom: 10px; 39 | } 40 | 41 | #nodelist{ 42 | } 43 | 44 | .legend{ 45 | color: #666; 46 | font-size: 90%; 47 | line-height: 1em; 48 | margin-top: 10px; 49 | margin-left: 5px; 50 | } 51 | 52 | #domainstat table{ 53 | margin-left: 20px; 54 | display: none; 55 | } 56 | 57 | #domainstat table thead th{ 58 | color: #8f138b; 59 | font-weight: bold; 60 | } 61 | 62 | #domainstat table .rowhead{ 63 | color: #8f138b; 64 | font-weight: bold; 65 | } 66 | 67 | #domainstat table td{ 68 | padding-right: 20px; 69 | vertical-align: top; 70 | } 71 | 72 | .row_expands{ 73 | cursor: pointer; 74 | } 75 | 76 | .row_expands:hover{ 77 | background: #FFDE00; 78 | } 79 | 80 | #domainbox{ 81 | text-align: center; 82 | } 83 | 84 | #domainlist{ 85 | text-align: left; 86 | } 87 | 88 | .domain{ 89 | color: #8f138b; 90 | } 91 | 92 | .chunk{ 93 | color: black; 94 | } 95 | 96 | .listitem{ 97 | cursor: pointer; 98 | padding-left: 10px; 99 | padding-right: 10px; 100 | line-height: 1em; 101 | padding-top: 5px; 102 | padding-bottom: 5px; 103 | 104 | } 105 | 106 | .chosen{ 107 | background: #FFF7BF; 108 | } 109 | 110 | #ownritem{ 111 | color: #8f138b; 112 | font-weight: bold; 113 | } 114 | 115 | .replitem{ 116 | padding-left: 25px; 117 | } 118 | 119 | .listitem:hover{ 120 | background: #FFDE00; 121 | } 122 | 123 | .domain_inactive{ 124 | background: #f6f6f6; 125 | } 126 | 127 | .warn{ 128 | background: #FFC9BF; 129 | } 130 | 131 | .nodenfo{ 132 | color: #8f138b; 133 | line-height: 1.0em; 134 | margin-left: 20px; 135 | font-size: 90%; 136 | } 137 | 138 | .domainnfo{ 139 | line-height: 1.0em; 140 | margin-left: 20px; 141 | font-size: 90%; 142 | } 143 | 144 | #ringlist{ 145 | } 146 | 147 | .hexid{ 148 | text-transform: uppercase; 149 | font-size: 90%; 150 | color: black; 151 | } 152 | 153 | .host{ 154 | color: #8f138b; 155 | } 156 | 157 | .tstamp{ 158 | color: #8f138b; 159 | font-size: 90%; 160 | } 161 | -------------------------------------------------------------------------------- /ringogw/web/typewatch1.1.js: -------------------------------------------------------------------------------- 1 | /* 2 | * TypeWatch 1.1.1 3 | * jQuery 1.1.3 + 4 | * 5 | * Examples/Docs: www.dennydotnet.com 6 | * Copyright(c) 2007 Denny Ferrassoli 7 | * Dual licensed under the MIT and GPL licenses: 8 | * http://www.opensource.org/licenses/mit-license.php 9 | * http://www.gnu.org/licenses/gpl.html 10 | */ 11 | 12 | jQuery.fn.extend({ 13 | typeWatch:function(options) { 14 | waitTextbox(this, options); 15 | } 16 | }); 17 | 18 | function waitTextbox(element, options) { 19 | element.each( 20 | 21 | function() { 22 | 23 | // Set to current element 24 | thisEl = jQuery(this); 25 | var winSaved = "typewatch_" + typewatch_uid++; 26 | var objWatch = {timer:null, text:null, cb:null, el:null, wait:null}; 27 | 28 | // Create js prop 29 | this.typewatchid = winSaved; 30 | 31 | // Must be text or textarea 32 | if (this.type.toUpperCase() == "TEXT" 33 | || this.nodeName.toUpperCase() == "TEXTAREA") { 34 | 35 | // Allocate timer element 36 | window[winSaved] = objWatch; 37 | var timer = window[winSaved]; 38 | 39 | // Defaults 40 | var _wait = 750; 41 | var _callback = function() { }; 42 | var _highlight = true; 43 | var _captureEnter = true; 44 | 45 | // Get options 46 | if (options) { 47 | if(options["wait"] != null) _wait = parseInt(options["wait"]); 48 | if(options["callback"] != null) _callback = options["callback"]; 49 | if(options["highlight"] != null) _highlight = options["highlight"]; 50 | if(options["enterkey"] != null) _captureEnter = options["enterkey"]; 51 | } 52 | 53 | // Set values 54 | timer.text = thisEl.val().toUpperCase(); 55 | timer.cb = _callback; 56 | timer.wait = _wait; 57 | timer.el = this; 58 | 59 | // Set focus action (highlight) 60 | if (_highlight) { 61 | thisEl.focus( 62 | function() { 63 | this.select(); 64 | }); 65 | } 66 | 67 | // Key watcher / clear and reset the timer 68 | thisEl.keydown( 69 | function(evt) { 70 | var thisWinSaved = this.typewatchid; 71 | var timer = window[thisWinSaved]; 72 | 73 | // Enter key only on INPUT controls 74 | if (evt.keyCode == 13 && this.type.toUpperCase() == "TEXT") { 75 | clearTimeout(timer.timer); 76 | timer.timer = setTimeout('typewatchCheck("' + thisWinSaved + '", true)', 1); 77 | return false; 78 | } 79 | 80 | clearTimeout(timer.timer); 81 | timer.timer = setTimeout('typewatchCheck("' + thisWinSaved + '", false)', timer.wait); 82 | }); 83 | } 84 | }); 85 | } 86 | 87 | function typewatchCheck( thisWinSaved, override ) { 88 | var timer = window[thisWinSaved]; 89 | var elTxt = $(timer.el).val(); 90 | 91 | // Fire if text > 2 AND text != saved txt OR if override AND text > 2 92 | if ((elTxt.length >= 0 && elTxt.toUpperCase() != timer.text) || (override && elTxt.length >= 0)) { 93 | timer.text = elTxt.toUpperCase(); 94 | timer.cb(elTxt); 95 | } 96 | } 97 | 98 | var typewatch_uid = 0; // unique counter 99 | -------------------------------------------------------------------------------- /start_nodes.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z $1 ]; then 4 | echo "Usage: start_nodes.sh [host] [ringo_data]" 5 | exit 1 6 | fi 7 | 8 | if [ ! -e ~/.hosts.erlang ]; then 9 | echo -e "\nYou don't seem to have an existing ~/.hosts.erlang file." 10 | echo "This file is required and it should contain a list of all possible hostnames" 11 | echo -e "that can contain active nodes in the ring. At the bare minimum, you can say:\n" 12 | echo -e "echo \"'localhost'.\" > ~/.hosts.erlang\n" 13 | echo "if you have only one node that is your localhost. See 'man 3erl net_adm' for" 14 | echo -e "further information about the .hosts.erlang file.\n" 15 | exit 1 16 | fi 17 | 18 | if [ -z $RIBGO_ROOT ]; then 19 | RINGO_ROOT=$(cd `dirname $0`; pwd) 20 | fi 21 | echo "RINGO_ROOT is $RINGO_ROOT" 22 | 23 | for id in `ssh $1 "ls -1 $2"` 24 | do 25 | echo "Starting ringo-$id" 26 | ssh $1 "$RINGO_ROOT/ring/start_ringo.sh $2/$id" 27 | sleep 1 28 | done 29 | 30 | --------------------------------------------------------------------------------