├── .gitignore ├── LICENSE ├── README.md ├── build.sbt └── src ├── main └── scala │ └── org │ └── spread │ └── core │ ├── SampleApplication.scala │ ├── algorithm │ └── Solve.scala │ ├── annotation │ └── Annotation.scala │ ├── constraint │ └── Constraint.scala │ ├── expression │ ├── Spread.scala │ ├── SpreadArithmetic.scala │ ├── SpreadLogic.scala │ └── Test.scala │ ├── language │ └── Annotation.scala │ ├── sequence │ ├── AnnotatedSequence.scala │ ├── AnnotatedTreeSequence.scala │ ├── ArraySequence.scala │ ├── MappedSequence.scala │ ├── OrderingSequence.scala │ ├── PairedSequence.scala │ ├── RangedSequence.scala │ ├── Sequence.scala │ └── VectorSequence.scala │ └── splithash │ ├── ClassLoader.scala │ ├── Hashing.scala │ ├── Reference.scala │ ├── SetHash.scala │ └── SplitHash.scala └── test └── scala └── org └── spread └── core └── test └── SeqSpecification.scala /.gitignore: -------------------------------------------------------------------------------- 1 | # All the compiler-generated stuff 2 | *.so 3 | *.jar 4 | *.war 5 | *.class 6 | 7 | # IDE-specific folders and files 8 | *.iml 9 | .idea/ 10 | .metadata/ 11 | .project/ 12 | project/ 13 | nb*.xml 14 | nbproject/ 15 | out/ 16 | target/ 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ### SPREAD 2 | 3 | I want to know *exactly* what my software is doing. Better still, I want *cryptographic proof* that my software executes each and every computation step correctly. 4 | 5 | Wouldn't you? 6 | 7 | Currently, I don't know what Windows 10 is doing (or it is *very* hard to find out) and I hate that. 8 | 9 | That's because most of Windows 10: 10 | 11 | * is compiled to machine code that bears no resemblance to the original source code, 12 | * hides intricate networks of mutable objects after abstract interfaces, 13 | * and destroys precious machine state after machine state. 14 | 15 | Welcome to SPREAD! 16 | 17 | In SPREAD, all data, machines states, code and computations are cryptographically authenticated. 18 | 19 | To put it mildly, some practical issues had to be resolved to make SPREAD a reality. First and foremost, SPREAD almost completely eradicates mutable state. 20 | 21 | Obviously, keeping *all* the state for authentication purposes would require enormous amounts of storage. SPREAD solves that issue by cryptographically 'signing' states *incrementally*, while still allowing full user-control over which ones need to be signed, and at what level of granularity. 22 | 23 | Alternatively, full machine states can also be incrementally stored by SPREAD. In turn, this allows the pervasive re-use of state that was calculated earlier. 24 | 25 | So SPREAD kinda acts like a spreadsheet. That's because spreadsheets also re-use the previous states of a workbook to optimally recalculate the next. 26 | 27 | Unlike SPREAD however, spreadsheets are incapable of keeping all their versions around. And typically, Excel plug-ins completely destroy the (otherwise) purely functional nature of spreadsheets. In contrast, SPREAD only permits referentially transparent functions as primitives. 28 | 29 | So can SPREAD be used as a drop-in Excel replacement? 30 | 31 | Not *exactly*. Actually not by a mile. 32 | 33 | However, the spreadsheet spirit is there. In fact, SPREAD can be considered the next-generation spreadsheet, and possibly the next-generation of software in general. 34 | 35 | #### Warning 36 | 37 | SPREAD is currently in extreme ALPHA stage. For the impatient, there is already some novelty to be found in the repository: 38 | 39 | #### [SplitHash](https://github.com/odipar/spread/blob/master/src/main/scala/org/spread/core/splithash/SplitHash.scala): an immutable, uniquely represented Sequence Authenticated Data Structure. 40 | 41 | ### Implementation 42 | 43 | SPREAD is currently implemented as a Scala DSL. This is not an absolute requirement for SPREAD to work, but Scala does bring some nice benefits to the table: 44 | 45 | * It runs on the Java Virtual Machine (JVM) 46 | * Purely functional (where appropriate) 47 | * Strongly typed at compile time 48 | * And it supports the creation of Domain Specific Languages (DSLs) 49 | 50 | (to be honest, I would have rather liked to implement SPREAD in the Avail Programming Language, but Avail's toolchain isn't just there yet). 51 | 52 | ### The anotomy of a simple SPREAD expression 53 | 54 | Now let's take a look at this standard Scala code: 55 | ```scala 56 | val e = (1 + 2) * (3 + 4) 57 | ``` 58 | A Scala compiler would compile the above source code into bytecode that would produce a single *Int * result when executed on the JVM. 59 | 60 | And this is the SPREAD equivalent of the same Scala code: 61 | ```scala 62 | val e = (1 !+ 2) !* (3 !+ 4) 63 | ``` 64 | Notice the exclamation `!` marks. In SPREAD, each operation that is prefixed by an exclamation mark *constructs* an expression, and is not directly evaluated. Consequently, to get to the canonical value of an expression, we need to explicitly request its *evaluation*. 65 | 66 | Now here is where the SPREAD magic comes in: the final result won't just be a single `Int`, but a full trace of all (sub)computations that lead up to that `Int`, including all the ancestor expressions! 67 | 68 | So what happens if we run this? 69 | 70 | ```scala 71 | println(e.fullEval) 72 | ``` 73 | we get the following result: 74 | ``` 75 | ((1 !+ 2) !* (3 !+ 4)) => 76 | (1 !+ 2) => 3 77 | (3 !+ 4) => 7 78 | (3 !* 7) 79 | (3 !* 7) => 21 80 | ``` 81 | 82 | Notice that the full trace holds the final value 21. Alternatively, we may choose to destroy the trace by requesting its head. 83 | 84 | Indeed, destroying data in SPREAD is certainly possible, but directly goes against the grain of SPREAD. That said, keeping all traces is not always an option (due to memory constraints). So in that case, it is better to cryptographically sign a trace, rather than to destroy it. With signing, it remains possible to fully authenticate every computation. 85 | 86 | Authenticating a trace can be achieved by prefixing a tilde `~` to any expression. When such expression is evaluated, it will yield a default 128 bit cryptographic hash of its trace, together with its final value. In similar fashion, more cryptographic bits can be acquired by prefixing more consecutive tildes. 87 | 88 | ### Fibonacci revisited 89 | 90 | To show how well this works in practice we first define the infamous fibonacci function in plain Scala: 91 | 92 | ```scala 93 | object fib extends Function1[Int,Int] { 94 | def apply(i: Int) = { 95 | if (i < 2) 1 96 | else fib(i-1) + fib(i-2) 97 | } 98 | } 99 | ``` 100 | Of course, real Scala wizards would abbreviate the above to: 101 | 102 | ```scala 103 | val fib: Int=>Int = i => { if (i < 2) 1 ; else fib(i-1) + fib(i-2) } 104 | ``` 105 | .. but that's slightly different from the first definition: the latter definition dynamically creates an anynomous function object, while the first creates an immutable top-level object. Currently, SPREAD only permits user-defined functions to be immutable top-level objects (this restriction will eventually be lifted). 106 | 107 | Now here is the same `fib` function in SPREAD format: 108 | ```scala 109 | object fib extends FA1[Int,Int] { 110 | def apply(i: $Int) = { 111 | if (!i < 2) 1 112 | else %(fib,!i - 1) !+ %(fib,!i - 2) 113 | } 114 | } 115 | ``` 116 | .. which I actually believe is not too bad: apart from the extra syntactic noise (due to limitations of the DSL) the SPREAD version pretty much matches with the idiomatic Scala version. 117 | 118 | now if we evaluate fib(4): 119 | 120 | ```scala 121 | println(%(fib,4).fullEval) 122 | ``` 123 | we get the following result: 124 | ``` 125 | fib(4) => (fib(3) !+ fib(2)) 126 | (fib(3) !+ fib(2)) => 127 | fib(3) => (fib(2) !+ fib(1)) 128 | (fib(2) !+ fib(1)) => 129 | fib(2) => (fib(1) !+ fib(0)) 130 | (fib(1) !+ fib(0)) => 131 | fib(1) => 1 132 | fib(0) => 1 133 | (1 !+ 1) 134 | (1 !+ 1) => 2 135 | fib(1) => 1 136 | (2 !+ 1) 137 | (2 !+ 1) => 3 138 | fib(2) => (fib(1) !+ fib(0)) 139 | (fib(1) !+ fib(0)) => 140 | fib(1) => 1 141 | fib(0) => 1 142 | (1 !+ 1) 143 | (1 !+ 1) => 2 144 | (3 !+ 2) 145 | (3 !+ 2) => 5 146 | ``` 147 | Just calculating `fib(4)` already generates a medium sized trace. Although in this case, keeping the trace of `fib(4)` would be relatively cheap, the naïve storage of `fib(25)` would incur 971138 items - which is just too much overhead. 148 | 149 | Fortunately, the size of a trace can be reduced by applying various evaluation strategies. Of course, which strategy to choose from depends on certain trade-offs that have to be made. 150 | ### Pruning traces 151 | Our first strategy is to prune a trace at certain levels. We can encode pruning directly in another version of the fib function: 152 | ```scala 153 | object fib2 extends FA1[Int,Int] { 154 | def apply(i: $Int) = { 155 | if (!i < 2) 1 156 | else if (!i < 5) ~%(fib,!i) 157 | else %(fib2,!i-1) + %(fib2,!i-2) 158 | } 159 | } 160 | ``` 161 | Notice that, when `fib2(i)` is called with `i < 5`, it applies the tilde(`~`) operator on `fib(i)`. 162 | 163 | So, 164 | ```scala 165 | println(%(fib2,6).fullEval) 166 | ``` 167 | .. gives us the following trace: 168 | ``` 169 | fib2(6) => (fib2(5) !+ fib2(4)) 170 | (fib2(5) !+ fib2(4)) => 171 | fib2(5) => (fib2(4) !+ fib2(3)) 172 | (fib2(4) !+ fib2(3)) => 173 | fib2(4) => ~fib(4) 174 | ~fib(4) => 175 | #B415F7B40A7F9A0A3C57E23CD6EC2ED6 => 5 176 | fib2(3) => ~fib(3) 177 | ~fib(3) => 178 | #C87382C86910AD69BB6DC1BB60577160 => 3 179 | (5 !+ 3) 180 | (5 !+ 3) => 8 181 | fib2(4) => ~fib(4) 182 | ~fib(4) => 183 | #B415F7B40A7F9A0A3C57E23CD6EC2ED6 => 5 184 | (8 !+ 5) 185 | (8 !+ 5) => 13 186 | ``` 187 | 188 | Notice that the fulll (sub)trace of fib(4) is now replaced by its cryptographic hash (indicated by the `#` prefix). 189 | ### Memoizing traces 190 | A possibly more advantageous strategy is to reduce the memory footprint via (dynamic) memoization. Memoization is nothing more than a special kind of index that maps function calls to traces, during evaluation. 191 | 192 | Memoization yields 2 major benefits: 193 | 194 | - the resulting trace of a function call can be re-used 195 | - (sub)traces can be structurally shared by reference 196 | 197 | Now if we take a closer look at `fib2(6)` it is easy to spot a potential re-use of `fib(4)`. Via memoization, we don't need to calculate fib(4) twice, but only once. 198 | 199 | Memoization works pretty good in practice for most algorithms. But it is not a silver bullet: memoization is only practical when past (sub)computations share a lot of structure with future (sub)computations. 200 | 201 | Memoization is easy to set up in SPREAD. We just need to associate an evaluation with a so-called `MemoizationContext`. During evaluation, SPREAD memoizes and re-uses all function calls via the associated `MemoizationContext`. 202 | 203 | There are currently three default Memoization strategies to choose from: 204 | 205 | - `EmptyContext` : this context doesn't store anything 206 | - `StrongMemoizationContext`: this context strongly references its items 207 | - `WeakMemoizationContext`: this context weakly references its items 208 | 209 | Here is an example that shows each strategy: 210 | ```scala 211 | val empty = EmptyMemoizationContext 212 | val strong = StrongMemoizationContext(HashMap()) 213 | val weak = WeakMemoizationContext(WeakHashMap()) 214 | 215 | var (result1,new_empty) = %(fib,25).fullEval(empty) 216 | var (result2,new_strong) = %(fib,25).fullEval(strong) 217 | var (result3,new_weak) = %(fib,25).fullEval(weak) 218 | 219 | result1 = null 220 | result2 = null 221 | result3 = null 222 | 223 | System.gc() 224 | ``` 225 | 226 | Although all three computations *must* return the same result, their computational and memory bounds are completely different. 227 | 228 | - The first computation is very inefficient as it cannot re-use previous `fib` calls. As a consequence, both the time complexity and the memory complexity are bound by `O(exp(n))`. However, memory will be garbage collected (GC'ed) and reclaimed as soon as the result becomes unreachable. 229 | 230 | - The second computation is much more very efficient, as it does re-use the previous `fib` calls. Through memoization, both time- and memory complexity becomes `O(log(n)*n)`. However, traces will **never** be GC'ed, as long as the context still holds **strong** references to them. 231 | 232 | - The third computation is as efficient as the second. But in this case, only *weak* references were held to every trace, so memory can be fully reclaimed when the result becomes unreachable. 233 | 234 | But remember that the trace of `fib(n)` grows exponentially with `n`. So how is it possible to get such a low memory bound of `O(n*log(n))`? 235 | ### Exponential Data Structures 236 | To get an idea of how to reach that bound we need to first consider the following simple example: 237 | ```scala 238 | def concat(x: List[Any], y: List[Any]): List[Any] = List(x,y) 239 | def size(x: List[Any]): Int= x match { 240 | case List(x: List[Any],y: List[Any]) => size(x) + size(y) 241 | case l: List[_] => l.size 242 | } 243 | 244 | var l: List[Any] = List(1,2,3,4) 245 | var i = 0 246 | while (i < 20) { 247 | l = concat(l,l) 248 | i = i + 1 249 | } 250 | println("size: " + size(l)) // 20 steps -> 4194304 251 | ``` 252 | With simple Lists we can already create `O(exp(n))` sized structures in `O(n)` time. But alas, to get to a specific element at a certain index also takes `O(n)`. 253 | 254 | For SPREAD, a more advanced Sequence Abstract Data Type (ADT) has been implemented: 255 | 256 | ```scala 257 | trait Sequence[X] { 258 | def size: Int 259 | def concat(o: Sequence[X]): Sequence[X] 260 | def split(i: Int): (Sequence[X], Sequence[X]) 261 | def first: X 262 | def last: X 263 | } 264 | ``` 265 | 266 | SPREAD works on top of this ADT, where all operations **must** be within time complexity bound `O(log(n))` (with the exception of `size` which **must** be `O(1)`). The Sequence ADT is used by SPREAD's evaluation algorithm that combines and concatenates (sub)traces into bigger traces. 267 | 268 | There are numerous known data structures that can be used to implement this ADT. Examples are: AVL trees, Skip Lists, Treaps, Finger Trees, etc. However, none of them can meet another very important constraint: 269 | 270 | *A certain ordered list of objects must always be uniquely and canonically be represented by the Sequence ADT.* 271 | 272 | This constraint **must** also be satisfied, otherwise SPREAD wouldn't be able to *incrementally* and cryptographically sign computations, data, states, etc. 273 | 274 | Now what, in essence, is the big deal breaker? The issue with standard cryptographic hashes is that they are not *incremental*. 275 | 276 | Here is an example. Let's say that you've just calculated the SHA256 hash of a big multi-terabyte file, which took you about 5 minutes. After that you change 1 byte in the middle of the file. 277 | 278 | The question is: how fast can you recalculate the SHA256 hash of that slightly modified file? 279 | 280 | ### SplitHash 281 | The first known Sequence ADT that is able to incrementally re-hash slightly modified data in `O(log(n))` is called SplitHash. 282 | 283 | What's interesting about SplitHash is that it isn't build on top of a single cryptographic hash type (like GIT's SHA1), but on top of 'infinite' hashes with the following API. 284 | 285 | ```scala 286 | trait Hash { 287 | def hashAt(i: Int): Int 288 | override def hashCode = hashAt(0) 289 | } 290 | ``` 291 | 292 | Pending crypthographic scrutiny, this API makes hash collisions very unlikely. Only objects that implement the Hash trait may be put into a SplitHash, so it is no surprise that all SPREAD primitives are Hashes, including SplitHash itself! 293 | 294 | ### Building them 295 | It's very easy to create and split SplitHashes. See if you can follow what is going on: 296 | ```scala 297 | val s1 = 1 ! 2 ! 3 ! 4 ! 5 ! 6 ! 7 ! 8 // construct 298 | val s2 = s1 ! s1 299 | val (s3,s4) = s2.split(12) 300 | val (s5,s6) = s2.split(2) 301 | val s7 = s4 ! s5 302 | val s8 = 5 ! 6 ! 7 ! 8 ! 1 ! 2 303 | 304 | println(s7 == s8) // true 305 | println(s7.hashAt(0) == s8.hashAt(0)) // true 306 | 307 | ``` 308 | 309 | So SplitHashes are used by SPREAD as the basic building block for traces. But they are also extremely useful to implement incremental algorithms for Sequences, such as fold, map, etc. 310 | ### Incremental Sum 311 | The implementation of a sum algorithm in SPREAD showcases everything what makes SPREAD so special. With this implementation we get: 312 | 313 | - An authenticated, proven sum 314 | - Scalable to bigger sums 315 | - Optionally incremental (spreadsheet like) 316 | 317 | Here is the implementation: 318 | ```scala 319 | object sum extends FA1[_SplitHash[Int],Int] { 320 | def apply(s: $SplitHash[Int]) = { 321 | val ss = !s 322 | if (ss.size == 1) ss.last 323 | else { 324 | val parts = ss.splitParts 325 | var ssum = %(sum,expr(parts(0))) 326 | var i = 1 327 | while (i < parts.length) { 328 | ssum = ssum !+ %(sum,expr(parts(i))) 329 | i = i + 1 330 | } 331 | ssum 332 | } 333 | } 334 | } 335 | ``` 336 | Yes I know, not the prettiest code. But don't bother too much about the syntactic sugar for now, let's put it to use: 337 | 338 | ```scala 339 | val context = WeakMemoizationContext(WeakHashMap()) 340 | val seq1 = 1 ! 2 ! 3 ! 4 ! 5 ! 6 ! 7 ! 8 341 | val seq2 = 1 ! 2 ! 3 ! 9 ! 5 ! 6 ! 7 ! 8 342 | 343 | val (sum1,context2) = %(sum,expr(seq1)).eval(context) 344 | val (sum2,context3) = %(sum,expr(seq2)).eval(context2) 345 | 346 | println(sum1) 347 | println(sum2) 348 | ``` 349 | What now follows is the output of the two sums: 350 | 351 | ### **WE FIRST LOOK AT THE TRACE OF THE FIRST SUM1. KEEP SCROLLING** 352 | ``` 353 | sum($(1 ! 2 ! 3 ! 4 ! 5 ! 6 ! 7 ! 8)) => (sum($(1 ! 2 ! 3 ! 4)) !+ sum($(5 ! 6 ! 7 ! 8))) 354 | (sum($(1 ! 2 ! 3 ! 4)) !+ sum($(5 ! 6 ! 7 ! 8))) => 355 | sum($(1 ! 2 ! 3 ! 4)) => (sum($(1 ! 2)) !+ sum($(3 ! 4))) 356 | (sum($(1 ! 2)) !+ sum($(3 ! 4))) => 357 | sum($(1 ! 2)) => (sum($(1)) !+ sum($(2))) 358 | (sum($(1)) !+ sum($(2))) => 359 | sum($(1)) => 1 360 | sum($(2)) => 2 361 | (1 !+ 2) 362 | (1 !+ 2) => 3 363 | sum($(3 ! 4)) => (sum($(3)) !+ sum($(4))) 364 | (sum($(3)) !+ sum($(4))) => 365 | sum($(3)) => 3 366 | sum($(4)) => 4 367 | (3 !+ 4) 368 | (3 !+ 4) => 7 369 | (3 !+ 7) 370 | (3 !+ 7) => 10 371 | sum($(5 ! 6 ! 7 ! 8)) => (sum($(5 ! 6)) !+ sum($(7 ! 8))) 372 | (sum($(5 ! 6)) !+ sum($(7 ! 8))) => 373 | sum($(5 ! 6)) => (sum($(5)) !+ sum($(6))) 374 | (sum($(5)) !+ sum($(6))) => 375 | sum($(5)) => 5 376 | sum($(6)) => 6 377 | (5 !+ 6) 378 | (5 !+ 6) => 11 379 | sum($(7 ! 8)) => (sum($(7)) !+ sum($(8))) 380 | (sum($(7)) !+ sum($(8))) => 381 | sum($(7)) => 7 382 | sum($(8)) => 8 383 | (7 !+ 8) 384 | (7 !+ 8) => 15 385 | (11 !+ 15) 386 | (11 !+ 15) => 26 387 | (10 !+ 26) 388 | (10 !+ 26) => 36 389 | ``` 390 | 391 | ### **NOW WE LOOK AT THE TRACE OF THE SECOND SUM2. KEEP SCROLLING** 392 | ``` 393 | sum($(1 ! 2 ! 3 ! 9 ! 5 ! 6 ! 7 ! 8)) => (sum($(1 ! 2 ! 3 ! 9)) !+ sum($(5 ! 6 ! 7 ! 8))) 394 | (sum($(1 ! 2 ! 3 ! 9)) !+ sum($(5 ! 6 ! 7 ! 8))) => 395 | sum($(1 ! 2 ! 3 ! 9)) => (sum($(1 ! 2)) !+ sum($(3 ! 9))) 396 | (sum($(1 ! 2)) !+ sum($(3 ! 9))) => 397 | sum($(1 ! 2)) => (sum($(1)) !+ sum($(2))) 398 | (sum($(1)) !+ sum($(2))) => 399 | sum($(1)) => 1 400 | sum($(2)) => 2 401 | (1 !+ 2) 402 | (1 !+ 2) => 3 403 | sum($(3 ! 9)) => (sum($(3)) !+ sum($(9))) 404 | (sum($(3)) !+ sum($(9))) => 405 | sum($(3)) => 3 406 | sum($(9)) => 9 407 | (3 !+ 9) 408 | (3 !+ 9) => 12 409 | (3 !+ 12) 410 | (3 !+ 12) => 15 411 | sum($(5 ! 6 ! 7 ! 8)) => (sum($(5 ! 6)) !+ sum($(7 ! 8))) 412 | (sum($(5 ! 6)) !+ sum($(7 ! 8))) => 413 | sum($(5 ! 6)) => (sum($(5)) !+ sum($(6))) 414 | (sum($(5)) !+ sum($(6))) => 415 | sum($(5)) => 5 416 | sum($(6)) => 6 417 | (5 !+ 6) 418 | (5 !+ 6) => 11 419 | sum($(7 ! 8)) => (sum($(7)) !+ sum($(8))) 420 | (sum($(7)) !+ sum($(8))) => 421 | sum($(7)) => 7 422 | sum($(8)) => 8 423 | (7 !+ 8) 424 | (7 !+ 8) => 15 425 | (11 !+ 15) 426 | (11 !+ 15) => 26 427 | (15 !+ 26) 428 | (15 !+ 26) => 41 429 | ``` 430 | ### **DID YOU NOTICE ANY DIFFERENCE BETWEEN THE TWO?** 431 | Of course you did! But did you also notice that there were some (sub)sums shared between `sum1` and `sum2`? 432 | 433 | Most notably the memoization of (sub)sum of `5 ! 6 ! 7 ! 8` was re-used. That's almost half of the sum. Of course, this is no coincidence. Indeed, it can be proven that - if only 1 element in a number sequence is changed - we only need to do `O(log(n))` extra work to re-calculate its sum. 434 | 435 | ### Conclusion 436 | For now, I'm done explaining the basics of SPREAD. I hope that you liked it. 437 | 438 | So what's next? 439 | 440 | Probably the coming two months I'll spend writing a formal paper on SplitHash. After that I'm planning to implement some radical new database technology on top of SPREAD: 441 | 442 | ### SPREAD: A database like Datomic but then with spreadsheet like capabilities! 443 | 444 | Oh yeah, and SPREAD *finally* supersedes the [Enchilada Programming Language](http://www.enchiladacode.nl). 445 | 446 | * * * 447 | 448 | Copyright 2016: Robbert van Dalen 449 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | name := "spread" 2 | 3 | version := "0.1" 4 | 5 | scalaVersion := "2.12.1" 6 | 7 | libraryDependencies += "org.spire-math" % "spire_2.12" % "0.13.0" 8 | 9 | libraryDependencies += "org.scala-lang" % "scala-compiler" % "2.12.1" 10 | 11 | libraryDependencies += "org.scala-lang" % "scala-reflect" % "2.12.1" 12 | 13 | libraryDependencies += "org.scalacheck" %% "scalacheck" % "1.13.4" % "test" 14 | 15 | -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/SampleApplication.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core 2 | 3 | import java.util.Comparator 4 | import java.util.concurrent.TimeUnit 5 | 6 | import org.spread.core.sequence.AnnotatedTreeSequence._ 7 | import org.spread.core.sequence.PairedSequence._ 8 | import org.spread.core.sequence.MappedSequence._ 9 | import org.spread.core.sequence.Sequence._ 10 | import org.spread.core.language.Annotation.sp 11 | import org.spread.core.sequence.ArraySequence.ArraySeq 12 | import org.spread.core.sequence.VectorSequence.VectorSeq 13 | import org.spread.core.sequence.RangedSequence._ 14 | import org.spread.core.sequence.OrderingSequence._ 15 | import org.spread.core.constraint.Constraint._ 16 | import org.spread.core.algorithm.Solve._ 17 | 18 | import scala.{specialized => sp2} 19 | import scala.language.{existentials, implicitConversions} 20 | import scala.language.experimental.macros 21 | import spire.implicits._ 22 | import org.spread.core.algorithm.Solve._ 23 | import org.spread.core.annotation.Annotation.{Annotator, Statistics, StatisticsAnnotator} 24 | import org.spread.core.constraint.Constraint.EqualStatP 25 | import org.spread.core.sequence.ArraySequence 26 | import org.spread.core.splithash.Hashing 27 | 28 | import scala.collection.immutable.HashSet 29 | import scala.reflect.ClassTag 30 | 31 | // 32 | // Exposing all the nice stuff 33 | // 34 | 35 | object SampleApplication { 36 | 37 | final def main(args: Array[String]): Unit = { 38 | import Selector._ 39 | import Combiner._ 40 | 41 | val c1 = createSeq((0 until 1000000).map(x => x).toArray) 42 | val c2 = createSeq((0 until 1000000).map(x => Hashing.siphash24(x,-x)).toArray) 43 | val c3 = createSeq((0 until 1000000).map(x => Hashing.siphash24(-x,x)).toArray) 44 | 45 | // table with pairwise columns 46 | val t1 = (c2 && c1).sort 47 | val t2 = (c3 && c1).sort 48 | 49 | val T1_C1 = t1.select(_.L) 50 | val T1_C1_1 = t1.select(_.L) 51 | 52 | val T1_C2 = t1.select(_.R) 53 | 54 | val T2_C1 = t2.select(_.L) 55 | val T2_C2 = t2.select(_.R) 56 | 57 | val p = (T1_C2 === T2_C2) AND (T1_C1 === c2(10000) OR T1_C1 === c2(20000)) AND (T2_C1 === c3(10000) OR T2_C1 === c3(20000)) 58 | 59 | val solver = defaultSolver 60 | 61 | for (k <- 1 to 10) { 62 | ss = 0 63 | val solution = defaultSolver.solve(p) 64 | println("number of solvers: " + ss) 65 | println("solution(t1): " + solution(t1)) 66 | //println("solution(t2): " + solution(t2)) 67 | 68 | } 69 | } 70 | } -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/algorithm/Solve.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.algorithm 2 | 3 | import org.spread.core.annotation.Annotation._ 4 | import org.spread.core.constraint.Constraint._ 5 | import org.spread.core.language.Annotation.sp 6 | import org.spread.core.sequence.AnnotatedSequence._ 7 | import org.spread.core.sequence.AnnotatedTreeSequence._ 8 | import org.spread.core.sequence.PairedSequence._ 9 | import org.spread.core.sequence.RangedSequence._ 10 | import org.spread.core.sequence.Sequence._ 11 | import org.spread.core.splithash.Hashing 12 | import spire.algebra.Order 13 | import spire.implicits._ 14 | 15 | import scala.reflect.ClassTag 16 | import scala.language.{existentials, implicitConversions} 17 | 18 | 19 | // 20 | // Constraint propagation relational join algorithm, based on user defined annotations 21 | // and a predicate DSL 22 | // 23 | // Copyright 2017: Robbert van Dalen 24 | // 25 | 26 | object Solve { 27 | type ASEQ[X,A,S <: AnnotatedSeq[X,A,S]] = AnnotatedSeq[X,A,S] 28 | type ASEL[X1,X2,A <: PVAL,S1 <: Seq[X1,S1],S2 <: ASEQ[X2,A,S2]] = AnnSelector[X1,X2,A,S1,S2] 29 | 30 | type ANYSEL = AnnSelector[X1,X2,A,S1,S2] forSome { 31 | type X1 32 | type X2 33 | type A <: PropValue 34 | type S1 <: Seq[X1,S1] 35 | type S2 <: AnnotatedSeq[X2,A,S2] 36 | } 37 | type PVAL = PropValue 38 | type ANYSEQ = Seq[X,S] forSome { type X ; type S <: Seq[X,S] } 39 | type ANYCEXPR = E forSome { type E <: CExpr[E] } 40 | 41 | trait Solver[S <: Solver[S]] { 42 | def init[X1,@sp X2,X3,@sp X4,A <: PVAL,S1 <: Seq[X1,S1],S2 <: ASEQ[X2,A,S2],S3 <: Seq[X3,S3],S4 <: ASEQ[X4,A,S4]] 43 | (r1: ASEL[X1,X2,A,S1,S2],r2: ASEL[X3,X4,A,S3,S4],cc: Prop[A]): S 44 | 45 | def isValid[X1,@sp X2,X3,@sp X4,A <: PVAL,S1 <: Seq[X1,S1],S2 <: ASEQ[X2,A,S2],S3 <: Seq[X3,S3],S4 <: ASEQ[X4,A,S4]] 46 | (r1: ASEL[X1,X2,A,S1,S2],r2: ASEL[X3,X4,A,S3,S4],cc: Prop[A]): Boolean 47 | 48 | def isSolved[X1,@sp X2,X3,@sp X4,A <: PVAL,S1 <: Seq[X1,S1],S2 <: ASEQ[X2,A,S2],S3 <: Seq[X3,S3],S4 <: ASEQ[X4,A,S4]] 49 | (r1: ASEL[X1,X2,A,S1,S2],r2: ASEL[X3,X4,A,S3,S4],cc: Prop[A]): Boolean 50 | 51 | def propagate[X1,@sp X2,X3,@sp X4,A <: PVAL,S1 <: Seq[X1,S1],S2 <: ASEQ[X2,A,S2],S3 <: Seq[X3,S3],S4 <: ASEQ[X4,A,S4]] 52 | (r1: ASEL[X1,X2,A,S1,S2],r2: ASEL[X3,X4,A,S3,S4],cc: Prop[A]): S 53 | 54 | def split: (S,S) 55 | 56 | def solve[E <: CExpr[E]](e: E): Map[ANYSEQ,LSEQ[NoAnnotation]] 57 | } 58 | 59 | trait CExpr[E <: CExpr[E]] { 60 | def isValid[S <: Solver[S]](s: S): Boolean 61 | def isSolved[S <: Solver[S]](s: S): Boolean 62 | def init[S <: Solver[S]](s: S): S 63 | def propagate[S <: Solver[S]](s: S): S 64 | def not: ANYCEXPR 65 | } 66 | 67 | case class CNotExpr[E <: CExpr[E]](e: E) extends CExpr[CNotExpr[E]] { 68 | val expr: ANYCEXPR = e.not 69 | def not = e.asInstanceOf[ANYCEXPR] 70 | 71 | def isValid[S <: Solver[S]](s: S) = expr.isValid(s) 72 | def isSolved[S <: Solver[S]](s: S) = expr.isSolved(s) 73 | def init[S <: Solver[S]](s: S) = expr.init(s) 74 | def propagate[S <: Solver[S]](s: S) = expr.propagate(s) 75 | } 76 | 77 | case class CAndExpr[L <: CExpr[L],R <: CExpr[R]](left: L,right: R) extends CExpr[CAndExpr[L,R]] { 78 | def isValid[S <: Solver[S]](s: S) = left.isValid(s) && right.isValid(s) 79 | def isSolved[S <: Solver[S]](s: S) = left.isSolved(s) && right.isSolved(s) 80 | def init[S <: Solver[S]](s: S) = right.init(left.init(s)) 81 | def propagate[S <: Solver[S]](s: S) = right.propagate(left.propagate(s)) 82 | def not: ANYCEXPR = COrExpr(left.not,right.not) 83 | } 84 | 85 | case class COrExpr[L <: CExpr[L],R <: CExpr[R]](left: L,right: R) extends CExpr[COrExpr[L,R]] { 86 | def isValid[S <: Solver[S]](s: S) = left.isValid(s) || right.isValid(s) 87 | def isSolved[S <: Solver[S]](s: S) = left.isSolved(s) || right.isSolved(s) 88 | def init[S <: Solver[S]](s: S) = right.init(left.init(s)) 89 | def propagate[S <: Solver[S]](s: S) = right.propagate(left.propagate(s)) 90 | def not: ANYCEXPR = CAndExpr(left.not,right.not) 91 | } 92 | 93 | trait CBinExpr[X1,@sp X2,X3,@sp X4,A <: PVAL,S1 <: Seq[X1,S1],S2 <: ASEQ[X2,A,S2],S3 <: Seq[X3,S3],S4 <: ASEQ[X4,A,S4], E <: CBinExpr[X1,X2,X3,X4,A,S1,S2,S3,S4,E]] 94 | extends CExpr[E] { 95 | def r1: ASEL[X1,X2,A,S1,S2] 96 | def r2: ASEL[X3,X4,A,S3,S4] 97 | def cc: Prop[A] 98 | 99 | def selectors = Set[ANYSEL](r1,r2) 100 | def isValid[S <: Solver[S]](s: S) = s.isValid(r1,r2,cc) 101 | def isSolved[S <: Solver[S]](s: S) = s.isSolved(r1,r2,cc) 102 | def init[S <: Solver[S]](s: S) = s.init(r1,r2,cc) 103 | def propagate[S <: Solver[S]](s: S) = s.propagate(r1,r2,cc) 104 | } 105 | 106 | class CEqual[X1,@sp X2,X3,@sp X4,A <: PVAL,S1 <: Seq[X1,S1],S2 <: ASEQ[X2,A,S2],S3 <: Seq[X3,S3],S4 <: ASEQ[X4,A,S4]] 107 | (val r1: ASEL[X1,X2,A,S1,S2], val r2: ASEL[X3,X4,A,S3,S4], val cc: EqualProp[A]) 108 | extends CBinExpr[X1,X2,X3,X4,A,S1,S2,S3,S4,CEqual[X1,X2,X3,X4,A,S1,S2,S3,S4]] { 109 | def not = CBinSyntax(r1).=!=(r2)(cc.notEqual).asInstanceOf[ANYCEXPR] 110 | } 111 | 112 | class CNotEqual[X1,@sp X2,X3,@sp X4,A <: PVAL,S1 <: Seq[X1,S1],S2 <: ASEQ[X2,A,S2],S3 <: Seq[X3,S3],S4 <: ASEQ[X4,A,S4]] 113 | (val r1: ASEL[X1,X2,A,S1,S2], val r2: ASEL[X3,X4,A,S3,S4], val cc: NotEqualProp[A]) 114 | extends CBinExpr[X1,X2,X3,X4,A,S1,S2,S3,S4,CNotEqual[X1,X2,X3,X4,A,S1,S2,S3,S4]] { 115 | def not = CBinSyntax(r1).===(r2)(cc.equal).asInstanceOf[ANYCEXPR] 116 | } 117 | 118 | class CGreaterEqual[X1,@sp X2,X3,@sp X4,A <: PVAL,S1 <: Seq[X1,S1],S2 <: ASEQ[X2,A,S2],S3 <: Seq[X3,S3],S4 <: ASEQ[X4,A,S4]] 119 | (val r1: ASEL[X1,X2,A,S1,S2], val r2: ASEL[X3,X4,A,S3,S4], val cc: GreaterEqualProp[A]) 120 | extends CBinExpr[X1,X2,X3,X4,A,S1,S2,S3,S4,CGreaterEqual[X1,X2,X3,X4,A,S1,S2,S3,S4]] { 121 | def not = CBinSyntax(r1).<(r2)(cc.lowerEqual,cc.notEqual).asInstanceOf[ANYCEXPR] 122 | } 123 | 124 | class CGreater[X1,@sp X2,X3,@sp X4,A <: PVAL,S1 <: Seq[X1,S1],S2 <: ASEQ[X2,A,S2],S3 <: Seq[X3,S3],S4 <: ASEQ[X4,A,S4]] 125 | (val r1: ASEL[X1,X2,A,S1,S2], val r2: ASEL[X3,X4,A,S3,S4], val c1: GreaterEqualProp[A], val c2: NotEqualProp[A]) 126 | extends CExpr[CGreater[X1,X2,X3,X4,A,S1,S2,S3,S4]] { 127 | 128 | val expr = CAndExpr(new CGreaterEqual(r1,r2,c1).asInstanceOf[ANYCEXPR],new CNotEqual(r1.copy,r2.copy,c2).asInstanceOf[ANYCEXPR]) 129 | 130 | def not = CBinSyntax(r1).<=(r2)(c1.lowerEqual) 131 | def isValid[S <: Solver[S]](s: S) = expr.isValid(s) 132 | def isSolved[S <: Solver[S]](s: S) = expr.isSolved(s) 133 | def init[S <: Solver[S]](s: S) = expr.init(s) 134 | def propagate[S <: Solver[S]](s: S) = expr.propagate(s) 135 | } 136 | 137 | class CLowerEqual[X1,@sp X2,X3,@sp X4,A <: PVAL,S1 <: Seq[X1,S1],S2 <: ASEQ[X2,A,S2],S3 <: Seq[X3,S3],S4 <: ASEQ[X4,A,S4]] 138 | (val r1: ASEL[X1,X2,A,S1,S2], val r2: ASEL[X3,X4,A,S3,S4], val cc: LowerEqualProp[A]) 139 | extends CBinExpr[X1,X2,X3,X4,A,S1,S2,S3,S4,CLowerEqual[X1,X2,X3,X4,A,S1,S2,S3,S4]] { 140 | 141 | def not: ANYCEXPR = CBinSyntax(r1).>(r2)(cc.greaterEqual,cc.notEqual).asInstanceOf[ANYCEXPR] 142 | } 143 | 144 | class CLower[X1,@sp X2,X3,@sp X4,A <: PVAL,S1 <: Seq[X1,S1],S2 <: ASEQ[X2,A,S2],S3 <: Seq[X3,S3],S4 <: ASEQ[X4,A,S4]] 145 | (val r1: ASEL[X1,X2,A,S1,S2], val r2: ASEL[X3,X4,A,S3,S4], val c1: LowerEqualProp[A], val c2: NotEqualProp[A]) extends 146 | CExpr[CLower[X1,X2,X3,X4,A,S1,S2,S3,S4]] { 147 | 148 | val expr = CAndExpr(new CLowerEqual(r1,r2,c1).asInstanceOf[ANYCEXPR],new CNotEqual(r1.copy,r2.copy,c2).asInstanceOf[ANYCEXPR]) 149 | 150 | def not = CBinSyntax(r1).>=(r2)(c1.greaterEqual) 151 | def isValid[S <: Solver[S]](s: S) = expr.isValid(s) 152 | def isSolved[S <: Solver[S]](s: S) = expr.isSolved(s) 153 | def init[S <: Solver[S]](s: S) = expr.init(s) 154 | def propagate[S <: Solver[S]](s: S) = expr.propagate(s) 155 | } 156 | 157 | implicit class CBinSyntax[X1,@sp X2,A <: PVAL,S1 <: Seq[X1,S1],S2 <: ASEQ[X2,A,S2]](r1: ASEL[X1,X2,A,S1,S2]) { 158 | def ===[X3,@sp X4,S3 <: Seq[X3,S3],S4 <: ASEQ[X4,A,S4]](r2: ASEL[X3,X4,A,S3,S4])(implicit cc: EqualProp[A]) = { 159 | new CEqual(r1.copy,r2.copy,cc) 160 | } 161 | def =!=[X3,@sp X4,S3 <: Seq[X3,S3],S4 <: ASEQ[X4,A,S4]](r2: ASEL[X3,X4,A,S3,S4])(implicit cc: NotEqualProp[A]) = { 162 | new CNotEqual(r1.copy,r2.copy,cc) // Non-equality needs to be constrained independently (no shared propagation) 163 | } 164 | def >=[X3,@sp X4,S3 <: Seq[X3,S3],S4 <: ASEQ[X4,A,S4]](r2: ASEL[X3,X4,A,S3,S4])(implicit cc: GreaterEqualProp[A]) = { 165 | new CGreaterEqual(r1.copy,r2.copy,cc) 166 | } 167 | def <=[X3,@sp X4,S3 <: Seq[X3,S3],S4 <: ASEQ[X4,A,S4]](r2: ASEL[X3,X4,A,S3,S4])(implicit cc: LowerEqualProp[A]) = { 168 | new CLowerEqual(r1.copy,r2.copy,cc) 169 | } 170 | def <[X3,@sp X4,S3 <: Seq[X3,S3],S4 <: ASEQ[X4,A,S4]](r2: ASEL[X3,X4,A,S3,S4])(implicit c1: LowerEqualProp[A], c2: NotEqualProp[A]) = { 171 | new CLower(r1.copy,r2.copy,c1,c2) 172 | } 173 | def >[X3,@sp X4,S3 <: Seq[X3,S3],S4 <: ASEQ[X4,A,S4]](r2: ASEL[X3,X4,A,S3,S4])(implicit c1: GreaterEqualProp[A], c2: NotEqualProp[A]) = { 174 | new CGreater(r1.copy,r2.copy,c1,c2) 175 | } 176 | } 177 | 178 | def not[E <: CExpr[E]](e: E) = CNotExpr(e) 179 | def NOT[E <: CExpr[E]](e: E) = CNotExpr(e) 180 | def !![E <: CExpr[E]](e: E) = CNotExpr(e) 181 | 182 | implicit class CExprSyntax[L <: CExpr[L]](left: L) { 183 | def and[R <: CExpr[R]](right: R): CAndExpr[L,R] = CAndExpr[L,R](left,right) 184 | def or[R <: CExpr[R]](right: R): COrExpr[L,R] = COrExpr[L,R](left,right) 185 | 186 | def AND[R <: CExpr[R]](right: R): CAndExpr[L,R] = CAndExpr[L,R](left,right) 187 | def OR[R <: CExpr[R]](right: R): COrExpr[L,R] = COrExpr[L,R](left,right) 188 | 189 | def &&[R <: CExpr[R]](right: R): CAndExpr[L,R] = CAndExpr[L,R](left,right) 190 | def ||[R <: CExpr[R]](right: R): COrExpr[L,R] = COrExpr[L,R](left,right) 191 | } 192 | 193 | def defaultSolver: DSolver = new DefaultSolver(Map(),Map()) 194 | 195 | case class SeqRange(start: Long, end: Long) { 196 | def size = (end-start+1) 197 | def applyRange = createRange(start,end) 198 | } 199 | 200 | trait DSolver extends Solver[DSolver] { 201 | def seqs: Map[ANYSEQ,SeqRange] 202 | def doms: Map[ANYSEL,PropValue] 203 | def propagate: DSolver 204 | 205 | def solve2[E <: CExpr[E]](e: E): Map[ANYSEQ,LSEQ[NoAnnotation]] 206 | def solve3[E <: CExpr[E]](e: E): Map[ANYSEQ,LSEQ[NoAnnotation]] 207 | } 208 | 209 | var ss: Long = 0 210 | 211 | case class DefaultSolver(seqs: Map[ANYSEQ,SeqRange], doms: Map[ANYSEL,PropValue]) extends DSolver { 212 | def self: DSolver = this 213 | 214 | def isValid[X1,@sp X2,X3,@sp X4,A <: PVAL,S1 <: Seq[X1,S1],S2 <: ASEQ[X2,A,S2],S3 <: Seq[X3,S3],S4 <: ASEQ[X4,A,S4]] 215 | (sel1: ASEL[X1,X2,A,S1,S2],sel2: ASEL[X3,X4,A,S3,S4],cc: Prop[A]) = { 216 | val a1 = doms(sel1) 217 | val a2 = doms(sel2) 218 | 219 | a1.isValid && a2.isValid && cc.isAnyValid(a1,a2) 220 | } 221 | 222 | def isSolved[X1,@sp X2,X3,@sp X4,A <: PVAL,S1 <: Seq[X1,S1],S2 <: ASEQ[X2,A,S2],S3 <: Seq[X3,S3],S4 <: ASEQ[X4,A,S4]] 223 | (sel1: ASEL[X1,X2,A,S1,S2],sel2: ASEL[X3,X4,A,S3,S4],cc: Prop[A]) = { 224 | val a1 = doms(sel1) 225 | val a2 = doms(sel2) 226 | 227 | if (a1.isValid && a2.isValid && cc.isAnySolved(a1,a2)) { 228 | val r1 = seqs(sel1.asSeq) 229 | val r2 = seqs(sel2.asSeq) 230 | 231 | val a3 = sel1().approxAnnotationRange(r1.start,r1.end) 232 | val a4 = sel2().approxAnnotationRange(r2.start,r2.end) 233 | 234 | cc.isAnySolved(a3,a4) 235 | } 236 | else false 237 | } 238 | 239 | def propagate: DSolver = { 240 | var ndoms = doms 241 | val iter = doms.iterator 242 | 243 | while(iter.hasNext) { 244 | val item = iter.next 245 | val dom = item._1 246 | val seq = dom.asSeq 247 | val range = seqs(seq) 248 | val annSeq = dom() 249 | val pval = item._2 250 | if (pval.isValid) { 251 | val ann = annSeq.approxAnnotationRange(range.start,range.end) 252 | val eqProp = annSeq.equal 253 | val annResult = eqProp.propagateAny(ann,pval)._1 254 | ndoms = ndoms + (dom -> annResult) 255 | } 256 | } 257 | DefaultSolver(seqs,ndoms) 258 | } 259 | 260 | def init[X1,@sp X2,X3,@sp X4,A <: PVAL,S1 <: Seq[X1,S1],S2 <: ASEQ[X2,A,S2],S3 <: Seq[X3,S3],S4 <: ASEQ[X4,A,S4]] 261 | (sel1: ASEL[X1,X2,A,S1,S2],sel2: ASEL[X3,X4,A,S3,S4],cc: Prop[A]): DSolver = { 262 | val s1 = sel1.asSeq 263 | val s2 = sel2.asSeq 264 | 265 | val nseqs = seqs + (s1 -> SeqRange(0,s1.size-1)) + (s2 -> SeqRange(0,s2.size-1)) 266 | val ndoms = doms + (sel1 -> sel1().annotation) + (sel2 -> sel2().annotation) 267 | 268 | DefaultSolver(nseqs,ndoms) 269 | } 270 | 271 | def propagate[X1,@sp X2,X3,@sp X4,A <: PVAL,S1 <: Seq[X1,S1],S2 <: ASEQ[X2,A,S2],S3 <: Seq[X3,S3],S4 <: ASEQ[X4,A,S4]] 272 | (sel1: ASEL[X1,X2,A,S1,S2],sel2: ASEL[X3,X4,A,S3,S4],cc: Prop[A]): DSolver = { 273 | val a1 = doms(sel1) 274 | val a2 = doms(sel2) 275 | 276 | if (a1.isValid && a2.isValid) { 277 | val (aa1,aa2) = cc.propagateAny(a1,a2) 278 | if ((a1 == aa1) && (a2 == aa2)) this 279 | else DefaultSolver(seqs,doms + (sel1 -> aa1) + (sel2 -> aa2)) 280 | } 281 | else this 282 | } 283 | 284 | def split: (DSolver,DSolver) = { 285 | val bestSeq = selectBestSplitCandidate 286 | val bestRange = seqs(bestSeq) 287 | val mid = (bestRange.start + bestRange.end + 1)/2 288 | 289 | val leftRange = SeqRange(bestRange.start,mid-1) 290 | val rightRange = SeqRange(mid,bestRange.end) 291 | 292 | val left = DefaultSolver(seqs + (bestSeq -> leftRange),doms) 293 | val right = DefaultSolver(seqs + (bestSeq -> rightRange),doms) 294 | 295 | (left.propagate,right.propagate) 296 | } 297 | 298 | def selectBestSplitCandidate: ANYSEQ = { 299 | val iterator = seqs.iterator 300 | var bestCandidate = seqs.last 301 | 302 | while (iterator.hasNext) { 303 | val item = iterator.next 304 | val s = item._2.size 305 | val cs = bestCandidate._2.size 306 | 307 | // Simple size heuristic (smallest > 1 is selected) 308 | // TODO: make heuristics pluggable 309 | if ((cs <= 1) || ((s > 1) && (s < cs))) { 310 | bestCandidate = item 311 | } 312 | } 313 | bestCandidate._1 314 | } 315 | 316 | def cartesianProduct = { 317 | var psize: Long = 1 318 | for (x <- seqs.values) { psize = psize * x.size } 319 | seqs.mapValues( x => repeat(x.applyRange,psize / x.size) ) 320 | } 321 | 322 | def repeat(s1: LSEQ[NoAnnotation],m: Long): LSEQ[NoAnnotation] = { 323 | if (m == 0) s1.emptySeq 324 | else if (m == 1) s1 325 | else { 326 | val m2 = m / 2 327 | val sr = repeat(s1,m2) 328 | sr.append(sr).append(repeat(s1,m - (m2 * 2))) 329 | } 330 | } 331 | 332 | def solve[E <: CExpr[E]](e: E): Map[ANYSEQ,LSEQ[NoAnnotation]] = { 333 | // init 334 | val s2 = e.init(self) 335 | val s3 = s2.propagate 336 | s3.solve2(e) 337 | } 338 | 339 | def solve2[E <: CExpr[E]](e: E): Map[ANYSEQ,LSEQ[NoAnnotation]] = { 340 | if (!e.isValid(self)) seqs.mapValues(x => defaultLongSeqFactory) 341 | else if (e.isSolved(self)) cartesianProduct 342 | else { 343 | e.propagate(self).solve3(e) 344 | } 345 | } 346 | 347 | def solve3[E <: CExpr[E]](e: E): Map[ANYSEQ,LSEQ[NoAnnotation]] = { 348 | ss = ss + 1 349 | 350 | val (m1,m2) = split 351 | val mm1 = m1.solve2(e) 352 | val mm2 = m2.solve2(e) 353 | mm1.transform((k,v) => v.append(mm2(k))) 354 | } 355 | } 356 | 357 | implicit def toConstantSel[@sp X: ClassTag](x: X)(implicit o1: Order[X]) = { 358 | import Selector._ 359 | val s: AnnTreeSeq[X,Statistics[X]] = createSeq(Array(x)) 360 | s.selectSame 361 | } 362 | 363 | def createSeq[@sp X: ClassTag](x: Array[X])(implicit o1: Order[X]): AnnTreeSeq[X,Statistics[X]] = { seqFactory[X].createSeq(x) } 364 | 365 | final def main(args: Array[String]): Unit = { 366 | import Selector._ 367 | import Combiner._ 368 | 369 | val c1 = createSeq((0 until 100000).map(x => x.toLong).toArray) 370 | val c2 = createSeq((0 until 100000).map(x => x.toLong).reverse.toArray) 371 | 372 | 373 | // table with pairwise columns 374 | val t1 = c1 375 | val t2 = c2 376 | 377 | val T1_C1 = t1.selectSame 378 | val T2_C1 = t2.selectSame 379 | 380 | val p = (T1_C1 === T2_C1) AND (T2_C1 > 10000L) AND (T2_C1 <= 10010L) 381 | 382 | println("START: ") 383 | 384 | val solver = defaultSolver 385 | 386 | for (k <- 1 to 10) { 387 | ss = 0 388 | val solution = defaultSolver.solve(p) 389 | println("number of solvers: " + ss) 390 | println("solution(t1): " + solution(t1)) 391 | println("solution(t2): " + solution(t2)) 392 | } 393 | 394 | 395 | } 396 | } -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/annotation/Annotation.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.annotation 2 | 3 | import org.spread.core.constraint.Constraint.PropValue 4 | 5 | import scala.language.{existentials, implicitConversions} 6 | import org.spread.core.language.Annotation.sp 7 | import org.spread.core.sequence.OrderingSequence.Order 8 | 9 | // 10 | // Sequence Annotators, most notably StatisticsAnnotator 11 | // 12 | // Copyright 2017: Robbert van Dalen 13 | // 14 | object Annotation { 15 | 16 | trait Annotated[@sp A] { 17 | def annotation: A 18 | } 19 | 20 | trait Annotator[@sp X,@sp A] { 21 | def none: A 22 | def one(x: X): A 23 | def manyX(start: Int, end: Int, d: Array[X]): A 24 | def manyA(start: Int, end: Int, d: Array[Annotated[A]]): A 25 | def append(r1: A, r2: A): A 26 | 27 | def manyX(d: Array[X]): A = manyX(0,d.length,d) 28 | def manyA(d: Array[Annotated[A]]): A = manyA(0,d.length,d) 29 | } 30 | 31 | trait RangeAnnotator[@sp X, @sp A] extends Annotator[X,A] { 32 | def range(from: X, to: X): A 33 | } 34 | 35 | trait NoAnnotation 36 | 37 | object NoAnnotation extends NoAnnotation { 38 | def isValid = false 39 | override def toString = "'" 40 | } 41 | 42 | case class NoAnnotator[@sp X]() extends RangeAnnotator[X,NoAnnotation] with PropValue { 43 | def none: NoAnnotation = NoAnnotation 44 | def one(x: X): NoAnnotation = NoAnnotation 45 | def manyX(start: Int, end: Int, d: Array[X]) = NoAnnotation 46 | def manyA(start: Int, end: Int, d: Array[Annotated[NoAnnotation]]) = NoAnnotation 47 | def append(a1: NoAnnotation, a2: NoAnnotation): NoAnnotation = NoAnnotation 48 | def isNone(a: NoAnnotation) = true 49 | def range(start: X, end: X) = NoAnnotation 50 | def isValid = false 51 | } 52 | 53 | 54 | trait Statistics[@sp X] extends PropValue { 55 | def lowerBound: X 56 | def upperBound: X 57 | def first: X 58 | def last: X 59 | def sorted: Boolean 60 | override def toString: String = { 61 | "<" + first + "," + lowerBound + "," + "sorted=" +sorted + "," + upperBound + "," + last + ">" 62 | } 63 | } 64 | 65 | case class InvalidStatistics[@sp X]() extends Statistics[X] { 66 | def error = sys.error("invalid statistics") 67 | def lowerBound = error 68 | def upperBound = error 69 | def first = error 70 | def last = error 71 | def sorted = error 72 | def isValid = false 73 | override def toString = ".." 74 | } 75 | 76 | case class StatisticsImpl3[@sp X] 77 | (lowerBound: X) extends Statistics[X] { 78 | def upperBound = lowerBound 79 | def first = lowerBound 80 | def last = upperBound 81 | def sorted = true 82 | def isValid = true 83 | } 84 | 85 | case class StatisticsImpl2[@sp X] 86 | (lowerBound: X,upperBound: X) extends Statistics[X] { 87 | def first = lowerBound 88 | def last = upperBound 89 | def sorted = true 90 | def isValid = true 91 | } 92 | 93 | case class StatisticsImpl[@sp X] 94 | (lowerBound: X,upperBound: X, first: X, last: X, sorted: Boolean) extends Statistics[X] { 95 | def isValid = true 96 | } 97 | 98 | def createStats[@sp X] 99 | (lowerBound: X,upperBound: X, first: X, last: X, sorted: Boolean): Statistics[X] = { 100 | 101 | if ((lowerBound == first) && (upperBound == last) && sorted) { 102 | if (lowerBound == upperBound) StatisticsImpl3(lowerBound) 103 | else StatisticsImpl2(lowerBound,upperBound) 104 | } 105 | else StatisticsImpl(lowerBound: X,upperBound: X, first: X, last: X, sorted: Boolean) 106 | } 107 | StatisticsAnnotator 108 | case class StatisticsAnnotator[@sp X](implicit ord: Order[X]) 109 | extends RangeAnnotator[X,Statistics[X]]{ 110 | 111 | // explicit implementation to avoid boxing ?? 112 | @inline final def min(v1: X, v2: X, o: Order[X]): X = { 113 | if (o.lt(v1,v2)) v1 114 | else v2 115 | } 116 | @inline final def max(v1: X, v2: X, o: Order[X]): X = { 117 | if (o.gt(v1,v2)) v1 118 | else v2 119 | } 120 | @inline final def gteqv(v1: X, v2: X, o: Order[X]): Boolean = { 121 | o.gteqv(v1,v2) 122 | } 123 | @inline final def lteqv(v1: X, v2: X, o: Order[X]): Boolean = { 124 | o.lteqv(v1,v2) 125 | } 126 | @inline final def lt(v1: X, v2: X, o: Order[X]): Boolean = { 127 | o.lt(v1,v2) 128 | } 129 | @inline final def gt(v1: X, v2: X, o: Order[X]): Boolean = { 130 | o.gt(v1,v2) 131 | } 132 | def range(start: X, end: X) = StatisticsImpl2(start,end) 133 | def ordering = ord 134 | def none: Statistics[X] = InvalidStatistics() 135 | def one(x: X) = StatisticsImpl2(x,x) 136 | def manyX(start: Int, end: Int, a: Array[X]): Statistics[X] ={ 137 | var lower = a(start) 138 | var upper = a(start) 139 | var msorted = true 140 | var i = start+1 141 | while (i < end) { 142 | val x = a(i) 143 | lower = min(lower,x,ord) 144 | upper = max(upper,x,ord) 145 | msorted = msorted && ord.lteqv(a(i - 1),x) 146 | i = i + 1 147 | } 148 | createStats(lower,upper,a(start),a(end-1),msorted) 149 | } 150 | // Some duplication of code for performance reasons 151 | def manyA(start: Int, end: Int, a: Array[Annotated[Statistics[X]]]): Statistics[X] ={ 152 | var s = a(start).annotation 153 | var lower = s.lowerBound 154 | var upper = s.upperBound 155 | var msorted = s.sorted 156 | var i = start+1 157 | while (i < end) { 158 | val x = a(i).annotation 159 | lower = min(lower,x.lowerBound,ord) 160 | upper = max(upper,x.upperBound,ord) 161 | msorted = msorted && x.sorted && ord.lteqv(a(i - 1).annotation.last,x.first) 162 | i = i + 1 163 | } 164 | createStats(lower,upper,s.first,a(end-1).annotation.last,msorted) 165 | } 166 | def append(s1: Statistics[X],s2: Statistics[X]): Statistics[X] ={ 167 | StatisticsImpl( 168 | min(s1.lowerBound,s2.lowerBound,ord), 169 | max(s1.upperBound,s2.upperBound,ord), 170 | s1.first,s2.last, 171 | s1.sorted && s2.sorted && ord.lteqv(s1.last,s2.first) 172 | ) 173 | } 174 | } 175 | 176 | implicit def stats[X](implicit ord: Order[X]) = StatisticsAnnotator[X]() 177 | } 178 | -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/constraint/Constraint.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.constraint 2 | 3 | import org.spread.core.annotation.Annotation.{NoAnnotation, Statistics, StatisticsAnnotator, createStats} 4 | import org.spread.core.sequence.PairedSequence._ 5 | 6 | import scala.language.{existentials, implicitConversions} 7 | import org.spread.core.language.Annotation.sp 8 | import org.spread.core.sequence.OrderingSequence.Order 9 | 10 | // 11 | // Standard constraint propagation objects 12 | // 13 | // Copyright 2017: Robbert van Dalen 14 | // 15 | object Constraint { 16 | 17 | trait PropValue { 18 | def isValid: Boolean 19 | } 20 | 21 | trait Prop[@sp X] { 22 | def isSolved(o1: X,o2: X): Boolean 23 | def isAnySolved(o1: Any,o2: Any) = isSolved(o1.asInstanceOf[X],o2.asInstanceOf[X]) 24 | def isValid(o1: X, o2: X): Boolean 25 | def isAnyValid(o1: Any, o2: Any) = isValid(o1.asInstanceOf[X],o2.asInstanceOf[X]) 26 | def propagate(o1: X, o2: X): (X,X) 27 | def propagateAny(o1: Any, o2: Any): (X,X) = propagate(o1.asInstanceOf[X],o2.asInstanceOf[X]) 28 | } 29 | 30 | trait EqualProp[@sp X] extends Prop[X] { 31 | def notEqual: NotEqualProp[X] 32 | } 33 | trait NotEqualProp[@sp X] extends Prop[X] { 34 | def equal: EqualProp[X] 35 | } 36 | trait GreaterEqualProp[@sp X] extends Prop[X] { 37 | def notEqual: NotEqualProp[X] 38 | def lowerEqual: LowerEqualProp[X] 39 | } 40 | trait LowerEqualProp[@sp X] extends Prop[X] { 41 | def notEqual: NotEqualProp[X] 42 | def greaterEqual: GreaterEqualProp[X] 43 | } 44 | 45 | trait StatisticsProp[@sp X] extends Prop[Statistics[X]] 46 | 47 | case object EqualNoAnn extends EqualProp[NoAnnotation] with NotEqualProp[NoAnnotation] { 48 | val noAnn = (NoAnnotation,NoAnnotation) 49 | def notEqual = this 50 | def equal = this 51 | def not = this 52 | def isSolved(o1: NoAnnotation,o2: NoAnnotation) = false 53 | def isValid(o1: NoAnnotation,o2: NoAnnotation) = false 54 | def propagate(o1: NoAnnotation,o2: NoAnnotation): (NoAnnotation,NoAnnotation) = noAnn 55 | } 56 | 57 | case class EqualStatP[@sp X](implicit ann: StatisticsAnnotator[X]) 58 | extends StatisticsProp[X] with EqualProp[Statistics[X]] { 59 | 60 | def ord = ann.ordering 61 | def notEqual = NotEqualStatP() 62 | 63 | def isSolved(o1: Statistics[X],o2: Statistics[X]) = { 64 | (o1.lowerBound == o2.upperBound) && (o1.upperBound == o2.lowerBound) 65 | } 66 | 67 | def isValid(o1: Statistics[X],o2: Statistics[X]) = { 68 | !(ann.gt(o1.lowerBound,o2.upperBound,ord) || ann.lt(o1.upperBound,o2.lowerBound,ord)) 69 | } 70 | 71 | def propagate(o1: Statistics[X], o2: Statistics[X]): (Statistics[X],Statistics[X]) = { 72 | val left = propagateOne(o1,o2) 73 | val right = propagateOne(o2,o1) 74 | (left,right) 75 | } 76 | 77 | def propagateOne(o1: Statistics[X], o2: Statistics[X]): Statistics[X] = { 78 | if (ann.gt(o1.lowerBound,o2.upperBound,ord)) ann.none 79 | else if (ann.lt(o1.upperBound,o2.lowerBound,ord)) ann.none 80 | else { 81 | // TODO: more tight bounds on first and last, we now set first=lower and last=upper 82 | val lower = ann.max(o1.lowerBound,o2.lowerBound,ord) 83 | val upper = ann.min(o1.upperBound,o2.upperBound,ord) 84 | val sorted = o1.sorted && o2.sorted 85 | createStats(lower,upper,lower,upper,sorted) 86 | } 87 | } 88 | override def toString = "EqualStatP" 89 | } 90 | 91 | case class NotEqualStatP[@sp X](implicit ann: StatisticsAnnotator[X]) 92 | extends StatisticsProp[X] with NotEqualProp[Statistics[X]] { 93 | 94 | def ord = ann.ordering 95 | def equal = EqualStatP() 96 | 97 | def isSolved(o1: Statistics[X],o2: Statistics[X]) = { 98 | (ann.gt(o1.lowerBound,o2.upperBound,ord) || ann.lt(o1.upperBound,o2.lowerBound,ord)) 99 | } 100 | 101 | def isValid(o1: Statistics[X],o2: Statistics[X]) = { 102 | !((o1.lowerBound == o2.upperBound) && (o1.upperBound == o2.lowerBound)) 103 | } 104 | 105 | def propagate(o1: Statistics[X], o2: Statistics[X]): (Statistics[X],Statistics[X]) = { 106 | val left = propagateOne(o1,o2) 107 | val right = propagateOne(o2,o1) 108 | (left,right) 109 | } 110 | 111 | def propagateOne(o1: Statistics[X], o2: Statistics[X]): Statistics[X] = { 112 | if(!isValid(o1,o2)) ann.none 113 | else { 114 | val lower = ann.min(o1.lowerBound,o2.lowerBound,ord) 115 | val upper = ann.max(o1.upperBound,o2.upperBound,ord) 116 | val sorted = o1.sorted && o2.sorted 117 | createStats(lower,upper,lower,upper,sorted) 118 | } 119 | } 120 | override def toString = "NotEqualStatP" 121 | } 122 | 123 | trait CompEqualStatP[@sp X] extends StatisticsProp[X] with EqualProp[Statistics[X]] { 124 | def ann: StatisticsAnnotator[X] 125 | def ord = ann.ordering 126 | 127 | def propagateGreaterEqual(o1: Statistics[X], o2: Statistics[X]): Statistics[X] = { 128 | if (!o1.isValid || !o1.isValid) ann.none 129 | else if (ann.gteqv(o1.lowerBound,o2.upperBound,ord)) o1 // (12,14) >= (6,10) 130 | else if (ann.lt(o1.upperBound,o2.lowerBound,ord)) ann.none // (5,8) >= (10,13) 131 | else { 132 | // TODO: more tight bounds on first and last, we now set first=lower and last=upper 133 | val lower = ann.max(o1.lowerBound,o2.lowerBound,ord) // (2,10) >= (7,15) => (7,10) 134 | val upper = o1.upperBound 135 | val sorted = o1.sorted && o2.sorted 136 | createStats(lower,upper,lower,upper,sorted) 137 | } 138 | } 139 | def propagateLowerEqual(o1: Statistics[X], o2: Statistics[X]): Statistics[X] = { 140 | if (!o1.isValid || !o1.isValid) ann.none 141 | else if (ann.gt(o1.lowerBound,o2.upperBound,ord)) ann.none // (10,13) <= (5,8) => none 142 | else if (ann.lteqv(o1.upperBound,o2.lowerBound,ord)) o1 // (6,10) <= (12,14) => (6,10) 143 | else { 144 | // TODO: more tight bounds on first and last, we now set first=lower and last=upper 145 | val lower = o1.lowerBound 146 | val upper = ann.min(o1.upperBound,o2.upperBound,ord) 147 | val sorted = o1.sorted && o2.sorted 148 | createStats(lower,upper,lower,upper,sorted) 149 | } 150 | } 151 | } 152 | 153 | case class GreaterEqualStatP[@sp X](implicit a: StatisticsAnnotator[X]) 154 | extends CompEqualStatP[X] with GreaterEqualProp[Statistics[X]] { 155 | 156 | def ann = a 157 | 158 | def lowerEqual = LowerEqualStatP[X]() 159 | def notEqual = NotEqualStatP[X]() 160 | 161 | def isSolved(o1: Statistics[X],o2: Statistics[X]) = ann.gteqv(o1.lowerBound,o2.upperBound,ord) 162 | def isValid(o1: Statistics[X],o2: Statistics[X]) = !ann.lt(o1.upperBound,o2.lowerBound,ord) 163 | 164 | def propagate(o1: Statistics[X], o2: Statistics[X]): (Statistics[X],Statistics[X]) = { 165 | val left = propagateGreaterEqual(o1,o2) 166 | val right = propagateLowerEqual(o2,o1) 167 | (left,right) 168 | } 169 | 170 | override def toString = "GreaterEqualStatP" 171 | } 172 | 173 | case class LowerEqualStatP[@sp X](implicit a: StatisticsAnnotator[X]) 174 | extends CompEqualStatP[X] with LowerEqualProp[Statistics[X]] { 175 | 176 | def ann = a 177 | 178 | def greaterEqual = GreaterEqualStatP[X]() 179 | def notEqual = NotEqualStatP[X]() 180 | 181 | def isSolved(o1: Statistics[X],o2: Statistics[X]) = ann.lteqv(o1.upperBound,o2.lowerBound,ord) 182 | def isValid(o1: Statistics[X],o2: Statistics[X]) = !ann.gt(o1.lowerBound,o2.upperBound,ord) 183 | 184 | def propagate(o1: Statistics[X], o2: Statistics[X]): (Statistics[X],Statistics[X]) = { 185 | val left = propagateLowerEqual(o1,o2) 186 | val right = propagateGreaterEqual(o2,o1) 187 | (left,right) 188 | } 189 | 190 | override def toString = "LowerEqualStatP" 191 | } 192 | 193 | implicit def equalStatProp[@sp X](implicit o: Order[X]): EqualProp[Statistics[X]] = { 194 | EqualStatP[X]()(StatisticsAnnotator[X]()) 195 | } 196 | 197 | implicit def notEqualStatProp[@sp X](implicit o: Order[X]): NotEqualProp[Statistics[X]] = { 198 | NotEqualStatP[X]()(StatisticsAnnotator[X]()) 199 | } 200 | 201 | implicit def greatereEqualStatProp[@sp X](implicit o: Order[X]): GreaterEqualProp[Statistics[X]] = { 202 | GreaterEqualStatP[X]()(StatisticsAnnotator[X]()) 203 | } 204 | 205 | implicit def lowerEqualStatProp[@sp X](implicit o: Order[X]): LowerEqualProp[Statistics[X]] = { 206 | LowerEqualStatP[X]()(StatisticsAnnotator[X]()) 207 | } 208 | } 209 | -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/expression/Spread.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.expression 2 | 3 | // SPREAD: Incremental Authenticated Computations 4 | // 5 | // Features: 6 | 7 | // 1) purely functional 8 | // 2) authenticated traces 9 | // 3) spreadsheet-like incremental computation via (weakly) memoized traces 10 | // 11 | // Copyright 2016: Robbert van Dalen 12 | // 13 | 14 | import SpreadLogic._ 15 | import org.spread.core.splithash.Hashing._ 16 | import org.spread.core.splithash.SplitHash._ 17 | 18 | import scala.collection.immutable.Map 19 | import scala.collection.mutable 20 | import scala.language.{existentials, implicitConversions} 21 | import scala.reflect.runtime.universe.TypeTag 22 | 23 | object Spread { 24 | 25 | // 26 | // A SPREAD expression carries the authenticated trace of computations that leads up to itself. 27 | // In turn, traces can be (weakly) memoized for re-use. 28 | // 29 | // For efficient concatenation and authentication, traces are stored as SplitHashes. 30 | // 31 | 32 | trait SPREAD[V] extends Hashable with Hash { 33 | type E <: SPREAD[_] 34 | type SH <: SplitHash[E,SH] 35 | 36 | def trace: SH 37 | } 38 | 39 | // Concrete default implementation 40 | trait Expr[V] extends SPREAD[V] { 41 | type E = Expr[_] 42 | type SH = SHNode[E] 43 | 44 | def trace: SH = ExprSHNode(this) 45 | def first: Expr[V] = trace.first.asInstanceOf[Expr[V]] 46 | def head: Expr[V] = trace.last.asInstanceOf[Expr[V]] 47 | def size = 1 48 | 49 | def hash = this 50 | def ===(o: Expr[V]) = this == o 51 | def !==(o: Expr[V]): Expr[Boolean] = %(equal2[V],this,o) 52 | 53 | def unary_~ : Expr[V] = SignedExpr(this,1) 54 | def quote : Expr[V] = Quoted(this) 55 | def fullEval: Expr[V] = fullEval(EmptyContext)._1 56 | def fullEval(c: EvaluationContext) = c.fullEval(this) 57 | 58 | def eval(c: EvaluationContext): (Expr[V],EvaluationContext) = c.eval(this) 59 | def _eval(c: EvaluationContext): (Expr[V],EvaluationContext) = (this,c) 60 | 61 | private var lazyProperties = 0 // lazy encoding of properties 62 | 63 | def properties = { 64 | if (lazyProperties == 0) { 65 | var p = 1 66 | p = p | ((_depth & ((1 << 30)-1)) << 1) 67 | p = p | (toInt(_containsVariable) << 30) 68 | p = p | (toInt(_containsQuote) << 31) 69 | lazyProperties = p 70 | } 71 | lazyProperties 72 | } 73 | def toInt(b: Boolean) = if (b) 1 ; else 0 74 | def toBool(i: Int) = if (i == 1) true ; else false 75 | 76 | def _containsVariable: Boolean = false 77 | def _containsQuote: Boolean = false 78 | def _depth: Int = 1 79 | 80 | def unquote: Expr[V] = %(unquote2,this) 81 | def _unquote: Expr[V] 82 | 83 | def bind[Y](s: Symbol, y: Expr[Y])(implicit t: TypeTag[Y]): Expr[V] = { 84 | val v: Variable[Y] = VariableImpl(s)(t) 85 | val b: Expr[Y] = y 86 | val e: Expr[V] = this 87 | 88 | %[V,Y,Y,V](bind2,e,v,b) 89 | } 90 | 91 | def _bindVariable[Y: TypeTag](s: Symbol, x: Expr[Y]): Expr[V] 92 | final def containsVariable: Boolean = toBool((properties >>> 30) & 1) 93 | final def containsQuote: Boolean = toBool((properties >>> 31) & 1) 94 | final def depth: Int = (properties & ((1 << 30)-1)) >>> 1 95 | } 96 | 97 | trait HashedExpr[V] extends Expr[V] { 98 | private var lHash = 0 99 | def lazyHash: Int 100 | override def hashCode = { 101 | if (lHash == 0) lHash = lazyHash 102 | lHash 103 | } 104 | } 105 | 106 | trait EvaluationContext { 107 | def eval[X](e: Expr[X]) = e._eval(this) 108 | 109 | def fullEval[X](e: Expr[X]): (Expr[X], EvaluationContext) = { 110 | val (e2,c2) = eval(e) 111 | if (e2 != e) c2.fullEval(e2) 112 | else (e2,c2) 113 | } 114 | } 115 | 116 | // A memoization context that is associated during evaluation 117 | trait MemoizationContext extends EvaluationContext { 118 | override def fullEval[V](e: Expr[V]): (Expr[V],EvaluationContext) = { 119 | val me = get(e) 120 | if (me != null) { 121 | if (traceReuse) { println("REUSED: " + e) } 122 | (me,this) 123 | } 124 | else { 125 | val (ev,c2: MemoizationContext) = fullEval2(e) 126 | if (ev != e) (ev,c2.put(e,ev)) 127 | else (ev,c2) 128 | } 129 | } 130 | def fullEval2[V](e: Expr[V]): (Expr[V],EvaluationContext) = { 131 | val (e2,c2: MemoizationContext) = eval(e) 132 | if (e2 != e) c2.fullEval2(e2) 133 | else (e2,c2) 134 | } 135 | 136 | def get[X](e: Expr[X]): Expr[X] = null 137 | def put[X](e1: Expr[X], e2: Expr[X]): EvaluationContext = this 138 | } 139 | 140 | object EmptyContext extends EvaluationContext 141 | 142 | case class StrongMemoizationContext(m: Map[Expr[_],Expr[_]]) extends MemoizationContext { 143 | override def get[X](e1: Expr[X]) = { 144 | m.get(e1) match { 145 | case None => null 146 | case Some(x) => (x.asInstanceOf[Expr[X]]) 147 | } 148 | } 149 | override def put[X](e1: Expr[X], e2: Expr[X]) = StrongMemoizationContext(m + (e1->e2)) 150 | } 151 | 152 | var traceReuse: Boolean = false 153 | 154 | case class WeakMemoizationContext(m: mutable.WeakHashMap[Expr[_],Expr[_]]) extends MemoizationContext { 155 | override def get[X](e1: Expr[X]) = { 156 | m.get(e1) match { 157 | case None => null 158 | case Some(x) => (x.asInstanceOf[Expr[X]]) 159 | } 160 | } 161 | override def put[X](e1: Expr[X], e2: Expr[X]) = WeakMemoizationContext(m += (e1->e2)) 162 | } 163 | 164 | val encoder = java.util.Base64.getEncoder 165 | 166 | case class CryptoSigned[X](a: Seq[Int]) extends Expr[X] { 167 | def hashAt(i: Int) = a(i % (a.size-1)) 168 | def parts = Array(this) 169 | override def toString = { 170 | var s = a.size * 4 171 | val b: Array[Byte] = new Array(s) 172 | var i = 0 173 | while (i < s) { 174 | var ii = i % 3 175 | var bb = (a(i >> 2) >>> (8 * ii)).toByte 176 | b(i) = bb 177 | i = i + 1 178 | } 179 | "#"+javax.xml.bind.DatatypeConverter.printHexBinary(b) 180 | } 181 | def _unquote = this 182 | def _bindVariable[Y: TypeTag](s: Symbol, x: Expr[Y]) = this 183 | override def _eval(c: EvaluationContext) = (this,c) 184 | } 185 | 186 | def copyHash(h: Hash, s: Int): Array[Int] = { 187 | var i = 0 188 | var a: Array[Int] = new Array(s) 189 | while (i < s) { 190 | a(i) = h.hashAt(i) 191 | i = i + 1 192 | } 193 | a 194 | } 195 | // Denotes an evaluation 196 | case class Eval[V](o: Expr[V], distance: Int) extends HashedExpr[V] { 197 | def lazyHash = siphash24(o.hashCode + magic_p1,distance - magic_p3) 198 | def hashAt(i: Int) = { 199 | if (i == 0) hashCode 200 | else siphash24(hashAt(i-1) - magic_p3,(magic_p2*distance) ^ o.hashCode) 201 | } 202 | override def _eval(c: EvaluationContext) = (o,c) 203 | def parts = o.parts 204 | def _unquote = this 205 | def _bindVariable[Y: TypeTag](s: Symbol, x: Expr[Y]) = this 206 | override def toString = o.toString + "@" + distance 207 | } 208 | 209 | trait Operator { override def toString = getClass().getSimpleName.replace("$","") } 210 | trait InfixOperator extends Operator 211 | trait PostfixOperator extends Operator 212 | 213 | // Nullary function that MUST evaluate to its canonical value in O(1) 214 | trait F0[X] extends Expr[X] with Hashable with Hash { 215 | def unary_! = value 216 | def value: X 217 | } 218 | // Unary (lazy) function 219 | trait FA1[A,X] extends (Expr[A] => Expr[X]) with CodeHash with Operator { 220 | def apply(a: Expr[A]): Expr[X] = a match { 221 | case (a: F0[A]) => apply2(a) 222 | case _ => %(this,a) 223 | } 224 | def apply2(a: F0[A]): Expr[X] 225 | } 226 | // Binary (lazy) function 227 | trait FA2[A,B,X] extends ((Expr[A],Expr[B]) => Expr[X]) with CodeHash with Operator { 228 | def apply(a: Expr[A], b: Expr[B]): Expr[X] = (a,b) match { 229 | case (aa: F0[A], bb: F0[B]) => apply2(aa,bb) 230 | case _ => %(this,a,b) 231 | } 232 | def apply2(a: F0[A], b: F0[B]): Expr[X] 233 | } 234 | 235 | // Ternary (lazy) function 236 | trait FA3[A,B,C,X] extends ((Expr[A],Expr[B],Expr[C]) => Expr[X]) with CodeHash with Operator { 237 | def apply(a: Expr[A], b: Expr[B], c: Expr[C]): Expr[X] = (a,b,c) match { 238 | case (aa: F0[A], bb: F0[B], cc: F0[C]) => apply2(aa,bb,cc) 239 | case _ => %(this,a,b,c) 240 | } 241 | def apply2(a: F0[A], b: F0[B], c: F0[C]): Expr[X] 242 | } 243 | 244 | // Denotes an unary function call 245 | private case class F1[A,X](f: FA1[A,X], v1: Expr[A]) extends HashedExpr[X] { 246 | override def toString = { 247 | if (f.isInstanceOf[PostfixOperator]) v1 + "." + f 248 | else f+"("+v1+")" 249 | } 250 | def lazyHash = siphash24(f.hashCode + magic_p2 ,v1.hashCode - magic_p3) 251 | def hashAt(index: Int) = { 252 | if (index == 0) hashCode 253 | else if (index == 1) (f.hashCode + v1.hashCode) ^ hashCode 254 | else { 255 | val nindex = index / 2 256 | 257 | if (hashCode > 0) siphash24(f.hash.hashAt(index - nindex) + magic_p3,v1.hash.hashAt(index) - (magic_p2 * hashCode)) 258 | else siphash24(f.hash.hashAt(nindex) + (magic_p1 * hashCode),v1.hash.hashAt(index - nindex) + magic_p3) 259 | } 260 | } 261 | override def _eval(c: EvaluationContext): (Expr[X], EvaluationContext) = (v1.head) match { 262 | case x1: F0[A] => (TracedExpr(node(Eval(this,1)).concat(node(f(x1)))),c) 263 | case x1 => { 264 | val (e1,c2) = x1.fullEval(c) 265 | 266 | val t1 = e1 == x1 267 | 268 | if (!t1) { 269 | var trace: SHNode[Expr[_]] = e1.trace 270 | trace = concat(trace,node(F1(f,e1.head))) 271 | trace = concat(node(Eval(this,trace.size)),trace) 272 | (TracedExpr(trace),c2) 273 | } 274 | else { 275 | val ev = f(x1) 276 | if (ev != this) (TracedExpr(node(Eval(this,1)).concat(node(ev))),c2) 277 | else (this,c) 278 | } 279 | } 280 | } 281 | 282 | def _unquote = { 283 | if (containsQuote) %(f,v1._unquote) 284 | else this 285 | } 286 | def _bindVariable[Y: TypeTag](s: Symbol, x: Expr[Y]) = { 287 | if (containsVariable) %(f,v1._bindVariable(s,x)) 288 | else this 289 | } 290 | override def _depth = v1.depth + 1 291 | override def _containsQuote = v1.containsQuote 292 | override def _containsVariable = v1.containsVariable 293 | def parts = Array(f,v1) 294 | } 295 | 296 | 297 | def tsize(t: SHNode[Expr[_]]): Int = { 298 | if (t == null) 0 299 | else t.size 300 | } 301 | 302 | // Denotes an binary function call 303 | private case class F2[A,B,X](f: FA2[A,B,X], v1: Expr[A], v2: Expr[B]) extends HashedExpr[X] { 304 | override def toString = f match { 305 | case i: InfixOperator => "(" + v1 + " " + f + " " + v2 + ")" 306 | case _ => f + "(" + v1 + "," + v2 + ")" 307 | } 308 | def lazyHash = siphash24(f.hashCode + magic_p1,siphash24(v1.hashCode + magic_p2 ,v2.hashCode - magic_p3)) 309 | def hashAt(index: Int) = { 310 | if (index == 0) hashCode 311 | else if (index == 1) (f.hashCode + v1.hashCode + v2.hashCode) ^ hashCode 312 | else { 313 | val nindex = index / 3 314 | val i2 = nindex * 2 315 | val i3 = index - i2 316 | 317 | if (hashCode > 0) siphash24(siphash24(f.hash.hashAt(nindex) + magic_p3,v1.hash.hashAt(i2) - (magic_p2 * hashCode)),v2.hash.hashAt(i3)) 318 | else siphash24(v2.hash.hashAt(i3) - magic_p3,siphash24(f.hash.hashAt(i2) + (magic_p1 * hashCode),v1.hash.hashAt(nindex) + magic_p3)) 319 | } 320 | } 321 | 322 | override def _eval(c: EvaluationContext): (Expr[X], EvaluationContext) = (v1.head,v2.head) match { 323 | case (x1: F0[A],x2: F0[B]) => (TracedExpr(node(Eval(this,1)).concat(node(f(x1,x2)))),c) 324 | case (x1,x2) => { 325 | val (e1,c2) = x1.fullEval(c) 326 | val (e2,c3) = x2.fullEval(c2) 327 | 328 | val t1 = e1 == x1 329 | val t2 = e2 == x2 330 | 331 | if (!t1 || !t2) { 332 | var trace: SHNode[Expr[_]] = null 333 | 334 | if (!t1) trace = concat(trace,e1.trace) 335 | if (!t2) trace = concat(trace,e2.trace) 336 | 337 | trace = concat(trace,node(F2(f,e1.head,e2.head))) 338 | trace = concat(node(Eval(this,trace.size)),trace) 339 | (TracedExpr(trace),c3) 340 | } 341 | else { 342 | val ev = f(x1,x2) 343 | 344 | if (ev != this) (TracedExpr(node(Eval(this,1)).concat(node(ev))),c3) 345 | else (this,c) 346 | } 347 | } 348 | } 349 | 350 | override def _depth = (v1.depth max v2.depth) + 1 351 | override def _containsQuote = v1.containsQuote || v2.containsQuote 352 | override def _containsVariable = v1.containsVariable || v2.containsVariable 353 | 354 | def _bindVariable[Y: TypeTag](s: Symbol, x: Expr[Y]) = { 355 | if (containsVariable) %(f,v1._bindVariable(s,x),v2._bindVariable(s,x)) 356 | else this 357 | } 358 | 359 | def _unquote = { 360 | if (containsQuote) %(f,v1._unquote,v2._unquote) 361 | else this 362 | } 363 | def parts = Array(f,v1,v2) 364 | } 365 | 366 | // Denotes an ternary function call 367 | private case class F3[A,B,C,X](f: FA3[A,B,C,X], v1: Expr[A], v2: Expr[B], v3: Expr[C]) extends HashedExpr[X] { 368 | override def toString = { 369 | if (f.isInstanceOf[PostfixOperator]) v1 + "." + f + "(" + v2 + "," + v3 + ")" 370 | else f + "(" + v1 + "," + v2 + "," + v3 + ")" 371 | } 372 | def lazyHash = siphash24(siphash24(f.hashCode + magic_p1,siphash24(v1.hashCode + magic_p2 ,v2.hashCode - magic_p3)),v2.hashCode * magic_p1) 373 | def hashAt(index: Int) = { 374 | if (index == 0) hashCode 375 | else if (index == 1) (f.hashCode + v1.hashCode + v2.hashCode ^ v3.hashCode) ^ hashCode 376 | else { 377 | val nindex = index / 4 378 | val i2 = nindex * 2 379 | val i3 = index * 3 380 | val i4 = index - nindex 381 | 382 | if (hashCode > 0) siphash24(siphash24(siphash24(f.hash.hashAt(nindex) + magic_p3,v1.hash.hashAt(i2) - (magic_p2 * hashCode)),v2.hash.hashAt(i3)),v3.hashAt(i4)) 383 | else siphash24(v3.hashCode.hashAt(i4),siphash24(v2.hash.hashAt(i3) - magic_p3,siphash24(f.hash.hashAt(i2) + (magic_p1 * hashCode),v1.hash.hashAt(nindex) + magic_p3))) 384 | } 385 | } 386 | override def _eval(c: EvaluationContext): (Expr[X], EvaluationContext) = (v1.head,v2.head,v3.head) match { 387 | case (x1: F0[A],x2: F0[B],x3: F0[C]) => (TracedExpr(node(Eval(this,1)).concat(node(f(x1,x2,x3)))),c) 388 | case (x1,x2,x3) => { 389 | val (e1,c2) = x1.fullEval(c) 390 | val (e2,c3) = x2.fullEval(c2) 391 | val (e3,c4) = x3.fullEval(c3) 392 | 393 | val t1 = e1 == x1 394 | val t2 = e2 == x2 395 | val t3 = e3 == x3 396 | 397 | if (!t1 || !t2 || !t3) { 398 | var trace: SHNode[Expr[_]] = null 399 | 400 | if (!t1) trace = concat(trace,e1.trace) 401 | if (!t2) trace = concat(trace,e2.trace) 402 | if (!t3) trace = concat(trace,e3.trace) 403 | 404 | trace = concat(trace,node(F3(f,e1.head,e2.head,e3.head))) 405 | trace = concat(node(Eval(this,trace.size)),trace) 406 | (TracedExpr(trace),c4) 407 | } 408 | else { 409 | // the arguments can't be reduced further but are not all F0[_], so apply the function still 410 | val ev = f(x1,x2,x3) 411 | 412 | if (ev != this) (TracedExpr(node(Eval(this,1)).concat(node(ev))),c4) 413 | else (this,c) 414 | } 415 | } 416 | } 417 | override def _depth = (v1.depth max v2.depth max v3.depth) + 1 418 | override def _containsQuote = v1.containsQuote || v2.containsQuote || v3.containsQuote 419 | override def _containsVariable = v1.containsVariable || v2.containsVariable | v3.containsVariable 420 | def _unquote = { 421 | if (containsQuote) { 422 | %(f,v1._unquote,v2._unquote,v3._unquote) 423 | } 424 | else this 425 | } 426 | def _bindVariable[Y: TypeTag](s: Symbol, x: Expr[Y]) = { 427 | if (containsVariable) %(f,v1._bindVariable(s,x),v2._bindVariable(s,x),v3._bindVariable(s,x)) 428 | else this 429 | } 430 | def parts = Array(f,v1,v2,v3) 431 | } 432 | 433 | case class ExprSHNode[X](o: Expr[_]) extends LeafNode[Expr[_]] { 434 | def value = o 435 | def lazyHash = siphash24(o.hashCode - magic_p3, magic_p2 * o.hashCode) 436 | def hashAt(i: Int) = { 437 | if (i == 0) hashCode 438 | else siphash24(o.hashAt(i), o.hashCode) 439 | } 440 | override def toString = o.toString 441 | } 442 | 443 | // A TracedExpr holds the history of computations that leads up to itself. 444 | case class TracedExpr[V](override val trace: SHNode[Expr[_]]) extends HashedExpr[V] { 445 | override def size = trace.size 446 | def lazyHash = siphash24(trace.hashCode - magic_p3, magic_p3 * trace.hashCode) 447 | def hashAt(i: Int) = { 448 | if (i == 0) hashCode 449 | else siphash24(trace.hashAt(i), magic_p2 * trace.hashCode) 450 | } 451 | override def _eval(c: EvaluationContext): (Expr[V], EvaluationContext) = { 452 | val r1 = head 453 | val (r2,cc) = c.eval(r1) 454 | if (r1 == r2) (this,cc) 455 | else (TracedExpr(concat(trace,r2.trace)),cc) 456 | } 457 | override def _containsQuote = head.containsQuote 458 | override def _containsVariable = head.containsVariable 459 | override def _depth = head.depth + 1 460 | 461 | override def _unquote = { 462 | if (containsQuote) head._unquote 463 | else this 464 | } 465 | def _bindVariable[Y: TypeTag](s: Symbol, x: Expr[Y]) = { 466 | if (containsVariable) head._bindVariable(s,x) 467 | else this 468 | } 469 | def parts = Array(this) 470 | override def toString = prettyPrint(trace,0,"") 471 | } 472 | 473 | // A SignedExpr holds the Expr to be crypto signed with hash size of (a * 128) bits 474 | case class SignedExpr[X](signed: Expr[X], bits_128: Int) extends HashedExpr[X]{ 475 | override def size = trace.size 476 | def lazyHash = siphash24(signed.hashCode * magic_p3,magic_p3 - bits_128.hashCode) 477 | def hashAt(i: Int) ={ 478 | if (i == 0) hashCode 479 | else siphash24(signed.hashAt(i) - magic_p1,magic_p1 * signed.hashCode) 480 | } 481 | override def _eval(c: EvaluationContext): (Expr[X],EvaluationContext) ={ 482 | val (ev: Expr[X],_) = EmptyContext.fullEval(signed) // we don't need to memoize the sub-trace 483 | val crypr: Expr[X] = CryptoSigned(copyHash(ev,bits_128 * 4)) 484 | val tt = node(Eval(this,2)).concat(node(crypr)).concat(node(ev.head)) 485 | (TracedExpr(tt),c) 486 | } 487 | def _unquote ={ 488 | if (containsQuote) SignedExpr(signed._unquote,bits_128) 489 | else this 490 | } 491 | def _bindVariable[Y: TypeTag](s: Symbol,x: Expr[Y]) ={ 492 | if (containsVariable) SignedExpr(signed._bindVariable(s,x),bits_128) 493 | else this 494 | } 495 | override def _containsQuote = signed.containsQuote 496 | override def _containsVariable = signed.containsVariable 497 | override def _depth = signed.depth + 1 498 | def parts = Array(this) 499 | override def unary_~ = SignedExpr(signed,bits_128 + 1) 500 | override def toString = wsp(bits_128,"~") + signed 501 | } 502 | 503 | case class LeafExpr[X <: Hashable](value: X) extends F0[X] { 504 | def lazyHash = siphash24(value.hash.hashCode * magic_p3, magic_p2 - value.hash.hashCode) 505 | def hashAt(i: Int) = { 506 | if (i == 0) lazyHash 507 | else siphash24(value.hash.hashAt(i) - magic_p2, magic_p3 * value.hashCode) 508 | } 509 | def parts = Array(this) 510 | def _unquote = this 511 | def _bindVariable[Y: TypeTag](s: Symbol, x: Expr[Y]) = this 512 | override def toString = "$("+value+")" 513 | } 514 | 515 | def node[X](e: Expr[X]) = ExprSHNode(e) 516 | def expr[X <: Hashable](e: X) = LeafExpr(e) 517 | 518 | def first[X](e: Expr[X]): Expr[X] = { 519 | if (e.size == 1) e 520 | else { 521 | val (Eval(e1,_)) = e.first 522 | e1 523 | } 524 | } 525 | 526 | def head[X](e: Expr[X]): Expr[X] = { 527 | if (e.size == 1) e 528 | else e.head 529 | } 530 | 531 | def trace2[X](e: Expr[X]): SHNode[Expr[_]] = { 532 | if (e.size == 1) null 533 | else e.trace 534 | } 535 | 536 | // smart constructors 537 | def %[A, X](f: FA1[A,X], a: Expr[A]): Expr[X] = { 538 | if (a.size > 1) { 539 | val a1 = first(a) 540 | val a2 = head(a) 541 | 542 | val t = trace2(a) 543 | TracedExpr(node(Eval(F1(f,a1),t.size+1)).concat(t).concat(node(F1(f,a2)))) 544 | } 545 | else F1(f,a) 546 | } 547 | def %[A, B, X](f: FA2[A,B,X], a: Expr[A], b: Expr[B]): Expr[X] = { 548 | if ((a.size > 1) || (b.size > 1)) { 549 | val a1 = first(a) 550 | val a2 = head(a) 551 | val b1 = first(b) 552 | val b2 = head(b) 553 | 554 | val t = concat(trace2(a),trace2(b)) 555 | TracedExpr(node(Eval(F2(f,a1,b1),t.size+1)).concat(t).concat(node(F2(f,a2,b2)))) 556 | } 557 | else F2(f,a,b) 558 | } 559 | def %[A, B, C, X](f: FA3[A,B,C,X], a: Expr[A], b: Expr[B], cc: Expr[C]): Expr[X] = { 560 | val c = cc 561 | if ((a.size > 1) || (b.size > 1) || (c.size > 1)) { 562 | val a1 = first(a) 563 | val a2 = head(a) 564 | val b1 = first(b) 565 | val b2 = head(b) 566 | val c1 = first(c) 567 | val c2 = head(c) 568 | 569 | val t = concat(concat(trace2(a),trace2(b)),trace2(c)) 570 | TracedExpr(node(Eval(F3(f,a1,b1,c1),t.size+1)).concat(t).concat(node(F3(f,a2,b2,c2)))) 571 | } 572 | else F3(f,a,b,c) 573 | } 574 | 575 | def ~%[A, X](f: FA1[A,X], a: Expr[A]): Expr[X] = ~(%(f,a)) 576 | def ~%[A, B, X](f: FA2[A,B,X], a: Expr[A], b: Expr[B]): Expr[X] = ~(%(f,a,b)) 577 | def ~%[A, B, C, X](f: FA3[A,B,C,X], a: Expr[A], b: Expr[B], c: Expr[C]): Expr[X] = ~(%(f,a,b,c)) 578 | 579 | // Gets the authenticated hash from the custum class loader 580 | trait CodeHash extends Hashable with Hash { 581 | var lazy_hash: Array[Int] = null 582 | 583 | def bhash: Array[Int] = { 584 | if (lazy_hash == null) { 585 | val loader = getClass.getClassLoader 586 | if (loader.getClass.getName.startsWith("spread.ClassLoader$AuthenticatedClassLoader")) { 587 | val m = loader.getClass.getMethod("cryptoSignClass", classOf[Class[_]]) 588 | lazy_hash = m.invoke(loader, getClass).asInstanceOf[Array[Int]] 589 | } 590 | else sys.error("SPREAD bytecode needs to be loaded by SPREADs authenticating classLoader") 591 | } 592 | lazy_hash 593 | } 594 | def parts = Array() 595 | def hash = this 596 | def hashAt(i: Int) = { 597 | if (i == 0) hashCode 598 | else bhash(i % (bhash.size-1)) 599 | } 600 | override def hashCode = bhash(0) 601 | } 602 | 603 | // Pretty print a trace (complicated!) 604 | def prettyPrint(e: SHNode[Expr[_]], depth: Int, prefix: String): String = { 605 | if (e == null) "" 606 | else { 607 | val s = { 608 | if (e.size == 1) prettyAtDepth(e.first,depth) 609 | else if (e.size == 2) prettyAtDepth(e.first,depth) + " => " + e.last 610 | else e.first match { 611 | case Eval(x,i) => { 612 | val (l,r) = e.split(i + 1) 613 | val (l2,r2) = l.split(1) 614 | 615 | if ((l2.size == 1) && (r2.size == 1)) prettyPrint(l2 ! r2,depth,"") + prettyPrint(r,depth,"\n") 616 | else prettyPrint(l2,depth,"") + prettyPrint(r2,depth + 1," =>\n") + prettyPrint(r,depth,"\n") 617 | } 618 | case x => prettyAtDepth(x,depth) 619 | } 620 | } 621 | prefix + s 622 | } 623 | } 624 | 625 | def prettyAtDepth(e: Expr[_], depth: Int): String = e match { 626 | case Eval(x,_) => wsp(depth) + x 627 | case TracedExpr(t) => wsp(depth) + "[" + "\n" + prettyPrint(t, depth+1,"") + "\n" + wsp(depth) + "]" 628 | case x => wsp(depth) + x 629 | } 630 | 631 | def wsp(d: Int): String = wsp(d,"\t") 632 | 633 | def wsp(d: Int, v: String): String = { 634 | if (d == 0) "" 635 | else if (d == 1) v 636 | else wsp(d/2,v) + wsp(d - (d/2),v) 637 | } 638 | 639 | trait Equal2[X] extends FA2[X,X,Boolean] with InfixOperator { 640 | def apply2(x: F0[X], y: F0[X]) = SpreadLogic.bexpr(x.value == y.value) 641 | override def toString = "!==" 642 | } 643 | 644 | object Equal3 extends Equal2[Nothing] 645 | 646 | def equal2[X]: FA2[X,X,Boolean] = Equal3.asInstanceOf[Equal2[X]] 647 | 648 | trait Unquote2[X] extends FA1[X,X] with PostfixOperator { 649 | override def apply(x: Expr[X]) = x._unquote 650 | def apply2(x: F0[X]) = apply(x) 651 | override def toString = "unquote" 652 | } 653 | 654 | object Unquote3 extends Unquote2[Nothing] 655 | def unquote2[X]: FA1[X,X] = Unquote3.asInstanceOf[FA1[X,X]] 656 | 657 | trait Bind2[X,Y] extends FA3[X,Y,Y,X] with PostfixOperator { 658 | override def apply(x: Expr[X], v: Expr[Y], b: Expr[Y]) = v match { 659 | case (v1: Variable[Y]) => x._bindVariable(v1.s,b)(v1.t) 660 | case _ => sys.error("no") 661 | } 662 | def apply2(x: F0[X], v: F0[Y], b: F0[Y]) = apply(x,v,b) 663 | override def toString = "bind" 664 | } 665 | object Bind3 extends Bind2[Nothing,Nothing] 666 | def bind2[X,Y]: FA3[X,Y,Y,X] = Bind3.asInstanceOf[FA3[X,Y,Y,X]] 667 | 668 | case class Quoted[X](e: Expr[X]) extends HashedExpr[X] { 669 | def lazyHash = siphash24(e.hashCode * magic_p1, e.hashCode * magic_p2 + e.hashCode) 670 | def hashAt(i: Int) = { 671 | if (i == 0) hashCode 672 | else if (i == 1) siphash24(e.hashCode * magic_p1, hashCode * magic_p2) 673 | else siphash24(hashAt(i-1), e.hashCode ^ hashCode) 674 | } 675 | override def _eval(c: EvaluationContext) = (this,c) 676 | def parts = Array(e) 677 | override def _depth = e.depth + 1 678 | override def _containsQuote = true 679 | override def _containsVariable = false 680 | def _unquote = e 681 | def _bindVariable[Y: TypeTag](s: Symbol, x: Expr[Y]) = this 682 | override def toString = e + ".quote" 683 | } 684 | 685 | trait Variable[X] extends Expr[X]{ 686 | def s: Symbol 687 | def t: TypeTag[X] 688 | def lazyHash = siphash24(s.hashCode * magic_p1,s.hashCode * magic_p2 + s.hashCode) 689 | def hashAt(i: Int) ={ 690 | if (i == 0) hashCode 691 | else if (i == 1) siphash24(s.hashCode * magic_p1,hashCode * magic_p2) 692 | else siphash24(hashAt(i - 1),s.hashCode ^ hashCode) 693 | } 694 | override def _eval(c: EvaluationContext) = (this,c) 695 | def parts = Array() 696 | override def _containsVariable = true 697 | def _bindVariable[Y](ss: Symbol, x: Expr[Y])(implicit ot: TypeTag[Y]) = { 698 | def evt = t.tpe 699 | def ovt = ot.tpe 700 | if ((ss == s) && (evt <:< ovt)) x.asInstanceOf[Expr[X]] // checks whether it's a subtype that can be bound 701 | else this 702 | } 703 | override def toString = s.toString 704 | } 705 | 706 | case class VariableImpl[X](s: Symbol)(implicit t2: TypeTag[X]) extends Variable[X] { 707 | def t: TypeTag[X] = t2 708 | def _unquote = this 709 | } 710 | } -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/expression/SpreadArithmetic.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.expression 2 | 3 | // 4 | // Integer expressions + evaluation + DSL 5 | // 6 | // Copyright 2016: Robbert van Dalen 7 | // 8 | 9 | import org.spread.core.expression.Spread._ 10 | import org.spread.core.splithash.Hashing._ 11 | 12 | import scala.language.implicitConversions 13 | import scala.reflect.runtime.universe.TypeTag 14 | 15 | object SpreadArithmetic { 16 | 17 | type _Int = Expr[Int] 18 | type $Int = F0[Int] 19 | 20 | trait IntExpr extends _Int { 21 | def unwrap: _Int 22 | 23 | def !+(o: _Int): _Int = %(add,unwrap,o) 24 | def !-(o: _Int): _Int = %(sub,unwrap,o) 25 | def !*(o: _Int): _Int = %(mul,unwrap,o) 26 | def !/(o: _Int): _Int = %(div,unwrap,o) 27 | 28 | // TODO: MORE primitives 29 | } 30 | 31 | case class IVarExpr(s: Symbol)(implicit t2: TypeTag[Int]) extends IntExpr with Variable[Int] { 32 | def t = t2 33 | def unwrap = this 34 | def _unquote = this 35 | } 36 | 37 | case class IExpr(value: Int) extends $Int with IntExpr { 38 | def unwrap = this 39 | def lazyHash = siphash24(value - magic_p1,value + magic_p2) 40 | def hashAt(index: Int) = { 41 | if (index == 0) hashCode 42 | else if (index == 1) siphash24(value + magic_p3, hashCode * magic_p2) 43 | else siphash24(hashCode * magic_p2, hashAt(index-1) - magic_p1) 44 | } 45 | def parts = Array() 46 | def _unquote = this 47 | def _bindVariable[Y: TypeTag](s: Symbol, x: Expr[Y]) = this 48 | override def toString = value.toString 49 | } 50 | 51 | trait BinIntOp extends FA2[Int,Int,Int] with InfixOperator 52 | 53 | trait add2 extends BinIntOp { 54 | def apply2(o1: $Int, o2: $Int) = IExpr(o1.value + o2.value) 55 | override def toString = "!+" 56 | } 57 | 58 | trait sub2 extends BinIntOp { 59 | def apply2(o1: $Int, o2: $Int) = IExpr(o1.value - o2.value) 60 | override def toString = "!-" 61 | } 62 | 63 | trait mul2 extends BinIntOp { 64 | def apply2(o1: $Int, o2: $Int) = IExpr(o1.value * o2.value) 65 | override def toString = "!*" 66 | } 67 | 68 | trait div2 extends BinIntOp { 69 | def apply2(o1: $Int, o2: $Int) = IExpr(o1.value / o2.value) 70 | override def toString = "!/" 71 | } 72 | 73 | object add extends add2 74 | object sub extends sub2 75 | object mul extends mul2 76 | object div extends div2 77 | 78 | case class IWrap(unwrap: _Int) extends IntExpr { 79 | def error = sys.error("Wrapper object. Should not be called.") 80 | def lazyHash = error 81 | def hashAt(i: Int) = error 82 | def parts = error 83 | def _unquote = error 84 | def _bindVariable[Y : TypeTag](s: Symbol, x: Expr[Y]) = error 85 | } 86 | 87 | def wrap(i: _Int): IntExpr = i match { 88 | case w: IWrap => w 89 | case _ => IWrap(i) 90 | } 91 | 92 | // Automatic conversion to bootstrap the DSL 93 | implicit def toVarExpr(s: Symbol)(implicit t2: TypeTag[Int]): IntExpr = IVarExpr(s) 94 | implicit def toIntExpr(i: Int): IntExpr = IExpr(i) 95 | implicit def toIntExpr2(i: _Int): IntExpr = wrap(i) 96 | } 97 | -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/expression/SpreadLogic.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.expression 2 | 3 | import org.spread.core.expression.Spread._ 4 | import org.spread.core.splithash.Hashing._ 5 | 6 | import scala.language.implicitConversions 7 | import scala.reflect.runtime.universe.TypeTag 8 | 9 | // 10 | // Boolean expressions + evaluation + DSL 11 | // 12 | // Copyright 2016: Robbert van Dalen 13 | // 14 | 15 | object SpreadLogic{ 16 | type _Boolean = Expr[Boolean] 17 | type $Boolean = F0[Boolean] 18 | 19 | trait BooleanExpr extends _Boolean { 20 | def unwrap: _Boolean 21 | 22 | def !&&(o: _Boolean): _Boolean = %(and,unwrap,o) 23 | def !||(o: _Boolean): _Boolean = %(or,unwrap,o) 24 | def !^^(o: _Boolean): _Boolean = %(xor,unwrap,o) 25 | 26 | def !?[X,Y](e1: Expr[X], e2: Expr[X]): Expr[X] = %(then3[X],unwrap,Either(e1,e2)) 27 | // TODO: MORE primitives 28 | } 29 | def !!(o: _Boolean): _Boolean = %(not,o) 30 | 31 | trait BinBoolOp extends FA2[Boolean,Boolean,Boolean] with InfixOperator 32 | 33 | case class Either[X](left: Expr[X], right: Expr[X]) extends F0[X] with HashedExpr[X] { 34 | val lazyHash = siphash24(left.hashCode * magic_p2 - magic_p3,right.hashCode * magic_p2 + magic_p1) 35 | def value = sys.error("There is no choice made") 36 | def hashAt(i: Int) = { 37 | if (i == 0) hashCode 38 | else if (i == 1) left.hashCode ^ right.hashCode + hashCode 39 | else siphash24(right.hash.hashAt(i-1) * magic_p2 - hashCode,right.hash.hashAt(i-1) - magic_p2 + hashCode) 40 | } 41 | def parts = Array(left,right) 42 | override def _depth = (left.depth max right.depth) + 1 43 | override def _containsQuote = left.containsQuote || left.containsQuote 44 | override def _containsVariable = left.containsVariable || right.containsVariable 45 | def _unquote = { 46 | if (containsQuote) Either(left._unquote,right._unquote) 47 | else this 48 | } 49 | def _bindVariable[Y : TypeTag](s: Symbol, x: Expr[Y]) = { 50 | if (containsVariable) Either(left._bindVariable(s,x),right._bindVariable(s,x)) 51 | else this 52 | } 53 | } 54 | 55 | trait BExpr extends $Boolean with BooleanExpr { 56 | def asInt: Int 57 | def unwrap = this 58 | def lazyHash = siphash24(asInt * magic_p2,asInt + magic_p2) 59 | def hashAt(index: Int) = { 60 | if (index == 0) hashCode 61 | else if (index == 1) siphash24(asInt * magic_p3, hashCode - magic_p2) 62 | else siphash24(hashCode - magic_p2, hashAt(index-1) + magic_p1) 63 | } 64 | def parts = Array() 65 | def _unquote = this 66 | def _bindVariable[Y : TypeTag](s: Symbol, x: Expr[Y]) = this 67 | override def toString = value.toString 68 | } 69 | 70 | object True extends BExpr { 71 | def asInt = magic_p1 72 | def value = true 73 | override def toString = "true" 74 | } 75 | 76 | object False extends BExpr { 77 | def asInt = magic_p3 78 | def value = false 79 | override def toString = "false" 80 | } 81 | 82 | def bexpr(b: Boolean): BExpr = { 83 | if (b) True 84 | else False 85 | } 86 | 87 | trait and2 extends BinBoolOp { 88 | def apply2(o1: $Boolean, o2: $Boolean) = bexpr(o1.value & o2.value) 89 | override def toString = "!&&" 90 | } 91 | 92 | trait or2 extends BinBoolOp { 93 | def apply2(o1: $Boolean, o2: $Boolean) = bexpr(o1.value | o2.value) 94 | override def toString = "!||" 95 | } 96 | 97 | trait xor2 extends BinBoolOp { 98 | def apply2(o1: $Boolean, o2: $Boolean) = bexpr(o1.value ^ o2.value) 99 | override def toString = "!^^" 100 | } 101 | 102 | trait not2 extends FA1[Boolean,Boolean] { 103 | def apply2(o1: $Boolean) = bexpr(!o1.value) 104 | override def toString = "!!" 105 | } 106 | 107 | trait then2[X] extends FA2[Boolean,X,X] { 108 | def apply2(c: $Boolean, e: F0[X]): Expr[X] = e match { 109 | case Either(e1,e2) => { 110 | if (c.value) e1 111 | else e2 112 | } 113 | case _ => e 114 | } 115 | override def toString = "!?" 116 | } 117 | 118 | object and extends and2 119 | object or extends or2 120 | object xor extends xor2 121 | object not extends not2 122 | object then22 extends then2[Nothing] 123 | 124 | def then3[X] = then22.asInstanceOf[then2[X]] 125 | 126 | case class BWrap(unwrap: _Boolean) extends BooleanExpr { 127 | def error = sys.error("Wrapper object. Should not be called.") 128 | def lazyHash = error 129 | def hashAt(i: Int) = error 130 | def parts = error 131 | def _unquote = error 132 | def _bindVariable[Y : TypeTag](s: Symbol, x: Expr[Y]) = error 133 | } 134 | 135 | def wrap(i: _Boolean): BooleanExpr = i match { 136 | case w: BWrap => w 137 | case _ => BWrap(i) 138 | } 139 | 140 | case class BVarExpr(s: Symbol)(implicit t2: TypeTag[Boolean]) extends BooleanExpr with Variable[Boolean] { 141 | def t = t2 142 | def unwrap = this 143 | def _unquote = this 144 | } 145 | 146 | // Automatic conversion to bootstrap the DSL 147 | implicit def toBVarExpr(s: Symbol)(implicit t2: TypeTag[Boolean]): BooleanExpr = BVarExpr(s) 148 | implicit def toBooleanExpr(b: Boolean): BooleanExpr = bexpr(b) 149 | implicit def toBooleanExpr2(b: _Boolean): BooleanExpr = wrap(b) 150 | } 151 | -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/expression/Test.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.expression 2 | 3 | import org.spread.core.expression.Spread._ 4 | import org.spread.core.expression.SpreadArithmetic._ 5 | import org.spread.core.expression.SpreadLogic._ 6 | import org.spread.core.splithash.ClassLoader 7 | import org.spread.core.splithash.SplitHash._ 8 | 9 | import scala.collection.immutable.HashMap 10 | import scala.collection.mutable.WeakHashMap 11 | import scala.language.{existentials, implicitConversions} 12 | 13 | // EXPOSITION: 14 | // 15 | // Authenticated re-usable computations = authenticated spreadsheets? 16 | // 17 | // Copyright 2016: Robbert van Dalen 18 | // 19 | 20 | object Test { 21 | final def main(args: Array[String]): Unit = { 22 | 23 | val l = ClassLoader.instance // SPREAD installs a custom ClassLoader to authenticate all the JVM bytecode 24 | val c = l.loadClass(new SubTest().getClass.getName) 25 | val i = c.newInstance 26 | 27 | val m = c.getMethod("main", classOf[Array[String]]) 28 | val params: Array[String] = null 29 | m.invoke(i, params) 30 | } 31 | 32 | class SubTest { 33 | val scontext = StrongMemoizationContext(HashMap()) 34 | val wcontext = WeakMemoizationContext(WeakHashMap()) 35 | val econtext = EmptyContext 36 | 37 | final def main(args: Array[String]): Unit ={ 38 | /*val a = %(fac,10) 39 | println(a.fullEval) 40 | */ 41 | /* { 42 | val s1 = 1 ! 2 ! 3 ! 4 ! 5 ! 6 ! 7 ! 8 43 | val s2 = s1 ! s1 44 | val (s3,s4) = s2.split(12) 45 | val (s5,s6) = s2.split(2) 46 | val s7 = s4 ! s5 47 | val s8 = 5 ! 6 ! 7 ! 8 ! 1 ! 2 48 | println(s7 == s8) // true 49 | println 50 | } 51 | { 52 | val a = (1 !+ 2) !* (3 !+ 4) 53 | val b = a.fullEval 54 | val c = b !+ b 55 | val d = a !+ a 56 | println(c.fullEval == d.fullEval) 57 | println("b: " + b) 58 | println 59 | } 60 | { 61 | val a = ('a !+ 2) !* (3 !+ 4) 62 | val b = (5 !+ 6) !* (7 !+ 8) 63 | val c = a !+ b 64 | println(c.bind('a,2).fullEval) 65 | println 66 | } 67 | { 68 | val a = (1 !+ 'a) !* (3 !+ 4.quote) 69 | val b = a.unquote.bind('a,(7 !- 5)).fullEval 70 | val c = a.unquote.fullEval 71 | val d = c.bind('a,(7 !- 5)) 72 | println(a) 73 | println 74 | println(b) 75 | println 76 | println(c) 77 | println 78 | println(d.fullEval) 79 | println 80 | } 81 | { 82 | val seq1 = 1 ! 2 ! 3 ! 4 ! 5 ! 6 ! 7 ! 8 83 | val seq2 = 1 ! 2 ! 3 ! 9 ! 5 ! 6 ! 7 ! 8 84 | 85 | val sum1 = %(sum,expr(seq1)) 86 | val sum2 = %(sum,expr(seq2)) 87 | 88 | traceReuse = true 89 | 90 | var (r1,_) = sum1.fullEval(wcontext) 91 | println(r1) 92 | 93 | var (r2,_) = sum2.fullEval(wcontext) 94 | println(r2) 95 | println 96 | } 97 | { 98 | traceReuse = true 99 | 100 | val fib1 = %(fib2,6) 101 | 102 | var (f1,_) = fib1.fullEval(econtext) 103 | println("slow: " + f1) 104 | 105 | var (f2,_) = fib1.fullEval(wcontext) 106 | println("fast: " + f2) 107 | 108 | if (f1 != f2) { 109 | sys.error("Internal inconsistency") 110 | } // the traces must be structurally equal 111 | println() 112 | } 113 | { 114 | val fac1 = %(fac,5) 115 | var (fc1,_) = fac1.fullEval(wcontext) 116 | println("fac(5): " + fc1) 117 | 118 | val fac22 = %(fac,7) 119 | var (fc2,_) = fac22.fullEval(wcontext) 120 | println("fac(7): " + fc2.head) 121 | println() 122 | 123 | val fac33 = %(fac2,5) 124 | var (fc3,_) = fac33.fullEval(wcontext) 125 | println("fac2(5): " + fc3) 126 | println() 127 | val (fc4,_) = fc3.unquote.fullEval(wcontext) 128 | println("fac2(5).unquote: " + fc4) 129 | println 130 | } */ 131 | /* val aa = 1 !+ 2 132 | val bb = 3 !* 'x 133 | val cc = aa !+ bb 134 | 135 | println(cc.fullEval(wcontext)._1.bind('x,2).fullEval(wcontext)._1) 136 | println(cc.bind('x,2).fullEval(wcontext)._2) */ 137 | /*val a = %(fac2,10) 138 | println("a: " + a.fullEval(wcontext)._1.unquote.fullEval(wcontext)._1) */ 139 | /* traceReuse = true 140 | val a = %(fib,20) 141 | println("a: " + a.fullEval(wcontext)._1.trace.size) */ 142 | 143 | println(fac3(10)) 144 | 145 | } 146 | 147 | object fac extends FA1[Int,Int]{ 148 | def apply2(i: $Int) ={ 149 | if (!i < 2) 1 150 | else i !* { 151 | if (!i < 5) ~%(fac,!i - 1) // We only capture the crypto hash for fac(i), i < 5 152 | else %(fac,!i - 1) // For i >= 5 we capture the full trace 153 | } 154 | } 155 | } 156 | 157 | object fac2 extends FA1[Int,Int]{ 158 | // Quoted + if then else 159 | def apply2(i: $Int) ={ 160 | (i !== 1) !?( // if (i == 1) 161 | 1.quote, // then quote 1 162 | i !* %(fac2,i !- 1) // else i * fac(i-1) 163 | ) 164 | } 165 | } 166 | 167 | def fac3(x: Int): Int = t1(fac3)(x) 168 | 169 | object t1 extends ((Int=>Int)=>(Int=>Int)) { 170 | def apply(f: Int=>Int) = { 171 | ( x => { if (x<1) 1 ; else x*f(x-1) } ) 172 | } 173 | } 174 | 175 | 176 | object fib extends FA1[Int,Int]{ 177 | def apply2(i: $Int) ={ 178 | if (!i < 2) 1 179 | else %(fib,!i - 1) !+ %(fib,!i - 2) 180 | } 181 | } 182 | 183 | object fib2 extends FA1[Int,Int]{ 184 | def apply2(i: $Int) ={ 185 | if (!i < 2) 1 186 | else if (!i < 5) ~%(fib,i) 187 | else %(fib2,!i - 1) !+ %(fib2,!i - 2) 188 | } 189 | } 190 | 191 | type _SplitHash[X] = SHNode[X] 192 | type $SplitHash[X] = F0[SHNode[X]] 193 | 194 | // We could easily have implemented sum with a generic fold 195 | // But for now we just explicitly show how to use the DSL and API 196 | object sum extends FA1[_SplitHash[Int],Int]{ 197 | def apply2(s: $SplitHash[Int]) ={ 198 | val ss = !s 199 | if (ss.size == 1) ss.last 200 | else { 201 | val parts = ss.splitParts 202 | var ssum = %(sum,expr(parts(0))) 203 | var i = 1 204 | while (i < parts.length) { 205 | ssum = ssum !+ %(sum,expr(parts(i))) 206 | i = i + 1 207 | } 208 | ssum 209 | } 210 | } 211 | } 212 | } 213 | 214 | } 215 | -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/language/Annotation.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.language 2 | import scala.Specializable._ 3 | 4 | // Here we can turn on/off scala specialization for all classes 5 | 6 | object Annotation { 7 | class sp() extends scala.specialized() 8 | //class sp() extends scala.annotation.StaticAnnotation 9 | } 10 | -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/sequence/AnnotatedSequence.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.sequence 2 | 3 | import org.spread.core.annotation.Annotation.{Annotated, Annotator, RangeAnnotator} 4 | import org.spread.core.constraint.Constraint.EqualProp 5 | import org.spread.core.language.Annotation.sp 6 | import org.spread.core.sequence.OrderingSequence._ 7 | import org.spread.core.sequence.Sequence._ 8 | 9 | import scala.language.{existentials, implicitConversions} 10 | import scala.reflect.ClassTag 11 | 12 | /** 13 | * Created by rapido on 24/02/17. 14 | */ 15 | object AnnotatedSequence { 16 | 17 | trait AnnotatedSeq[@sp X,A,S <: AnnotatedSeq[X,A,S]] extends SeqImpl[X,S] with Annotated[A] { 18 | def annotationRange(start: Long, end: Long): A 19 | def approxAnnotationRange(start: Long, end: Long): A 20 | def equal: EqualProp[A] 21 | } 22 | 23 | trait AnnOrdSeq[@sp X,A,S <: AnnOrdSeq[X,A,S]] extends AnnotatedSeq[X,A,S] with OrderingSeq[X,S] 24 | 25 | // Annotated Seq with representation 26 | trait AnnSeqWithRepr[@sp X,A,S <: AnnSeqWithRepr[X,A,S]] extends AnnotatedSeq[X,A,S] { 27 | type AS <: AnnSeqRepr[X,A,S] 28 | 29 | def repr: S#AS 30 | 31 | def create(s: S#AS): S 32 | def append[S2 <: S](o: S2): S = create(repr.append(o.repr)(self)) 33 | def split(o: Long): (S,S) = { val (l,r) = repr.split(o)(self); (create(l),create(r)) } 34 | def equalTo[S2 <: S](o: S2): Boolean = repr.equalToTree(o.repr)(self) 35 | def annotation = repr.annotation 36 | def annotationRange(start: Long, end: Long) = repr.annotationRange(start,end)(self) 37 | def approxAnnotationRange(start: Long, end: Long) = repr.approxAnnotationRange(start,end)(self) 38 | def size = repr.size 39 | def height = repr.height+1 40 | def first = repr.first(self) 41 | def last = repr.last(self) 42 | def apply(i: Long) = repr.apply(i)(self) 43 | } 44 | 45 | trait AnnOrdSeqWithRepr[@sp X,A,S <: AnnOrdSeqWithRepr[X,A,S]] 46 | extends AnnSeqWithRepr[X,A,S] with AnnOrdSeq[X,A,S] 47 | 48 | trait AnnSeqRepr[@sp X,A,S <: AnnSeqWithRepr[X,A,S]] extends Annotated[A] { 49 | def size: Long 50 | def height: Int 51 | def append[AS <: S#AS](o: AS)(implicit c: S): S#AS 52 | def split(o: Long)(implicit c: S): (S#AS,S#AS) 53 | def equalToTree[AS <: S#AS](o: AS)(implicit s: S): Boolean 54 | def annotationRange(start: Long, end: Long)(implicit c: S): A 55 | def approxAnnotationRange(start: Long, end: Long)(implicit c: S): A 56 | def first(implicit c: S): X 57 | def last(implicit c: S): X 58 | def apply(i: Long)(implicit c: S): X 59 | } 60 | 61 | trait AnnotationContext[@sp X,A] 62 | { 63 | type ANN <: Annotator[X,A] 64 | def ann: ANN 65 | def eq: EqualProp[A] 66 | def xTag: ClassTag[X] 67 | def aTag: ClassTag[A] 68 | } 69 | 70 | class AnnContextImpl[@sp X,A] 71 | (val ann: Annotator[X,A], val eq: EqualProp[A], val xTag: ClassTag[X], val aTag: ClassTag[A]) extends AnnotationContext[X,A] { 72 | type ANN = Annotator[X,A] 73 | } 74 | 75 | trait AnnotationOrderingContext[@sp X,A] extends AnnotationContext[X,A] 76 | { 77 | def ord: Order[X] 78 | } 79 | 80 | trait RangedAnnotationOrderingContext[@sp X,A] extends AnnotationOrderingContext[X,A] { 81 | type ANN <: RangeAnnotator[X,A] 82 | } 83 | 84 | class AnnOrdContextImpl[@sp X,A] 85 | (val ann: Annotator[X,A], val eq: EqualProp[A], val ord: Order[X], val xTag: ClassTag[X], val aTag: ClassTag[A]) 86 | extends AnnotationOrderingContext[X,A] { 87 | type ANN = Annotator[X,A] 88 | } 89 | 90 | class RangedAnnOrdContextImpl[@sp X,A] 91 | (val ann: RangeAnnotator[X,A], val eq: EqualProp[A], val ord: Order[X], val xTag: ClassTag[X], val aTag: ClassTag[A]) 92 | extends RangedAnnotationOrderingContext[X,A] { 93 | type ANN = RangeAnnotator[X,A] 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/sequence/AnnotatedTreeSequence.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.sequence 2 | 3 | import org.spread.core.annotation.Annotation._ 4 | import org.spread.core.constraint.Constraint._ 5 | import org.spread.core.sequence.Sequence._ 6 | import org.spread.core.sequence.OrderingSequence._ 7 | 8 | import scala.language.{existentials, implicitConversions} 9 | import scala.reflect.ClassTag 10 | import org.spread.core.language.Annotation.sp 11 | import org.spread.core.sequence.AnnotatedSequence._ 12 | import org.spread.core.sequence.OrderingSequence.Union 13 | 14 | object AnnotatedTreeSequence { 15 | 16 | trait AnnTreeSeq[@sp X,A] extends AnnOrdSeqWithRepr[X,A,AnnTreeSeq[X,A]] { 17 | type C = RangedAnnotationOrderingContext[X,A] 18 | type AS = BSeqTr[X,A] 19 | type SS = AnnTreeSeq[X,A] 20 | type SAS = SS#AS 21 | 22 | def context: C 23 | def annotator = context.ann 24 | def ordering = context.ord 25 | def equal = context.eq 26 | 27 | def tag = context.xTag 28 | def empty: SAS = EmptySeq[X,A]() 29 | 30 | implicit def xTag: ClassTag[X] = context.xTag 31 | implicit def aTag: ClassTag[A] = context.aTag 32 | 33 | 34 | @inline final def minWidth = 16 35 | @inline final def maxWidth = minWidth*4 36 | 37 | def createSeq(a: Array[X]): AnnTreeSeq[X,A] = createSeq(a,0,a.length) 38 | def toArray = { 39 | if (size < Int.MaxValue) { 40 | val array = new Array[X](size.toInt) 41 | repr.intoArray(array,0) 42 | array 43 | } 44 | else sys.error("sequence to big") 45 | } 46 | 47 | def createSeq(a: Array[X], start: Int, size: Int): AnnTreeSeq[X,A] = { 48 | if (size <= maxWidth) { 49 | val b = new Array[X](size.toInt) 50 | val end = start + size 51 | var i = start 52 | var ii = 0 53 | while (i < end) { 54 | b(ii) = a(i) 55 | i = i + 1 56 | ii = ii + 1 57 | } 58 | create(createLeaf(b)(this)) 59 | } 60 | else { 61 | val ms = size / 2 62 | createSeq(a,start,ms).append(createSeq(a,start+ms,size - ms)) 63 | } 64 | } 65 | def createLeaf(a: Array[X])(implicit c: SS) = { 66 | if (a.length == 0) empty 67 | else BSeqLeafImpl(a,annotator.manyX(a)) 68 | } 69 | 70 | def createTree[ARR <: Array[SAS]](a: ARR)(implicit c: SS): SAS = { 71 | val sz = new Array[Long](a.length) 72 | val ann = c.annotator.manyA(a.asInstanceOf[Array[Annotated[A]]]) 73 | 74 | var ts: Long = 0 75 | 76 | var i = 0 77 | var s = a.length 78 | 79 | while (i < s) { 80 | ts = ts + a(i).size; sz(i) = ts 81 | i = i + 1 82 | } 83 | 84 | BSeqTreeImpl(a,sz,ann) 85 | } 86 | 87 | def createPair[AA1 <: SAS, AA2 <: SAS](ss1: AA1,ss2: AA2)(implicit c: SS): SAS = { 88 | createTree(Array(ss1,ss2)) // TODO: create specialized Pair 89 | } 90 | 91 | def appendTrees(o1: BSeqTree[X,A],o2: BSeqTree[X,A])(implicit c: SS): SAS = { 92 | assert(o1.height == o2.height) // only append trees with same height 93 | if (o1.childCount >= minWidth && o2.childCount >= minWidth) createPair(o1,o2) 94 | else { 95 | val merged = o1.childs ++ o2.childs 96 | if (merged.length <= maxWidth) createTree(merged) 97 | else {val (l,r) = merged.splitAt((merged.length + 1) >> 1); createPair(createTree(l),createTree(r))} 98 | } 99 | } 100 | def appendLeafs(o1: BSeqLeaf[X,A],o2: BSeqLeaf[X,A])(implicit c: SS): SAS = { 101 | assert((o1.height == 0) && (o2.height == 0)) 102 | if (o1.size >= minWidth && o2.size >= minWidth) createPair(o1,o2) 103 | else { 104 | val tsize = o1.size + o2.size 105 | if (tsize <= maxWidth) createLeaf(o1.toArray ++ o2.toArray) 106 | else { 107 | if (o1.size > o2.size) { 108 | val (l,r) = o1.split(o1.size/2) 109 | createPair(l,r.append(o2)) 110 | } 111 | else { 112 | val (l,r) = o2.split(o2.size/2) 113 | createPair(o1.append(l),r) 114 | } 115 | } 116 | 117 | } 118 | } 119 | 120 | def asTree(t: SAS): BSeqTree[X,A] = t.asInstanceOf[BSeqTree[X,A]] 121 | def asLeaf(l: SAS): BSeqLeaf[X,A] = l.asInstanceOf[BSeqLeaf[X,A]] 122 | 123 | def sort = defaultSort2 124 | 125 | def defaultSort2: SS = { 126 | if (size <= 1) this 127 | else if (size <= (64*64)) { 128 | val a = toArray 129 | 130 | spire.math.Sorting.mergeSort(a)(ordering,tag) 131 | createSeq(a) 132 | } 133 | else repr match { 134 | case l: BSeqLeaf[X,A] => { 135 | var a = toArray 136 | spire.math.Sorting.mergeSort(a)(ordering,tag) 137 | 138 | create(createLeaf(a)(this)) 139 | } 140 | case t: BSeqTree[X,A] => { 141 | val childs: Array[SS] = t.childs.map(x => create(x)) 142 | sort(childs,0,childs.size) 143 | } 144 | } 145 | } 146 | 147 | def sort(s: Array[SS], start: Int, size: Int): SS = { 148 | if (size == 1) { s(start).sort } 149 | else { 150 | val m = size / 2 151 | combineSorted(sort(s,start,m),(sort(s,start+m,size-m)),Union) 152 | } 153 | } 154 | 155 | def append[AA1 <: SAS, AA2 <: SAS](ss1: AA1,ss2: AA2)(implicit c: SS): SAS = { 156 | if (ss2.size == 0) ss1 157 | else if (ss1.size == 0) ss2 158 | else if ((ss1.height == 0) && (ss2.height == 0)) appendLeafs(asLeaf(ss1),asLeaf(ss2)) 159 | else if (ss1.height == ss2.height) appendTrees(asTree(ss1),asTree(ss2)) 160 | else if (ss1.height > ss2.height) { 161 | val s1 = asTree(ss1) 162 | val newLast = s1.lastChild.append(ss2) 163 | if (newLast.height == s1.height) append(s1.withoutLastChild,newLast) 164 | else s1.replaceLastChild(newLast) 165 | } 166 | else { 167 | val s2 = asTree(ss2) 168 | val newFirst = ss1.append(s2.firstChild) 169 | if (newFirst.height == s2.height) append(newFirst,s2.withoutFirstChild) 170 | else s2.replaceFirstChild(newFirst) 171 | } 172 | } 173 | def equalTo[AA1 <: SAS, AA2 <: SAS](s1: AA1, s2: AA2)(implicit c: SS): Boolean = { 174 | // TODO: optimize this 175 | if (s1 eq s2) true 176 | else if (s1.size != s2.size) false 177 | else { 178 | if (s1.size == 1) asLeaf(s1).first == asLeaf(s2).first 179 | else { 180 | val i = (s1.size + 1) / 2 181 | val (l1,r1) = s1.split(i) 182 | val (l2,r2) = s2.split(i) 183 | l1.equalToTree(l2) && r1.equalToTree(r2) 184 | } 185 | } 186 | } 187 | override def iterator: SequenceIterator[X] = TreeSeqIterator[X,A]()(self) 188 | } 189 | 190 | trait BSeqTr[@sp X,A] extends AnnSeqRepr[X,A,AnnTreeSeq[X,A]] 191 | { 192 | type AS = BSeqTr[X,A] 193 | type SS = AnnTreeSeq[X,A] 194 | type SAS = SS#AS 195 | 196 | def getLeaf(i: Long): (BSeqLeaf[X,A],Int) 197 | def getLeaf2(i: Long): BSeqLeaf[X,A] 198 | def intoArray(dest: Array[X], i: Int): Int 199 | } 200 | 201 | trait BSeqTree[@sp X,A] extends BSeqTr[X,A] { 202 | def childs: Array[SAS] 203 | def sizes: Array[Long] 204 | val height = childs(0).height + 1 205 | def size = sizes(sizes.length - 1) 206 | def childCount = childs.length 207 | def firstChild = childs(0) 208 | def lastChild = childs(childCount - 1) 209 | def childAt(i: Int): SAS = childs(i) 210 | def setChild(i: Int,s: SAS)(implicit c: SS): SAS = {val nc = childs.clone; nc(i) = s; c.createTree(nc)} 211 | def replaceFirstChild(first: SAS)(implicit c: SS): SAS = setChild(0,first) 212 | def replaceLastChild(last: SAS)(implicit c: SS): SAS = setChild(childCount - 1,last) 213 | def withoutFirstChild(implicit c: SS): SAS = c.createTree(childs.slice(1,childCount)) 214 | def withoutLastChild(implicit c: SS): SAS = c.createTree(childs.slice(0,childCount - 1)) 215 | def offsetForChild(index: Int) = { 216 | if (index == 0) 0.toLong 217 | else sizes(index - 1) 218 | } 219 | def childAtIndex(index: Long): Int = { 220 | if (index < size && index >= 0) {var i = 0; while (sizes(i) <= index) {i = i + 1}; i} 221 | else sys.error("index out of bounds") 222 | } 223 | def append[AAS <: SAS](o: AAS)(implicit c: SS): SAS = c.append(this,o) 224 | def equalToTree[AS <: SAS](o: AS)(implicit c: SS): Boolean = c.equalTo(this,o) 225 | 226 | def split(i: Long)(implicit c: SS): (SAS,SAS) = { 227 | if (i >= size) (this,c.empty) 228 | else if (i < 0) (c.empty,this) 229 | else { 230 | val sIndex = childAtIndex(i) 231 | val offset = offsetForChild(sIndex) 232 | var (left,right) = childAt(sIndex).split(i - offset) 233 | for (i <- 0 until sIndex) left = childAt(sIndex - i - 1).append(left) 234 | for (i <- (sIndex + 1) until childCount) right = right.append(childAt(i)) 235 | (left,right) 236 | } 237 | } 238 | def annotationRange(start: Long,end: Long)(implicit c: SS): A = { 239 | if (end >= size) annotationRange(start,size - 1) 240 | else if (start < 0) annotationRange(0,end) 241 | else { 242 | val startIndex = childAtIndex(start) 243 | val startOffset = offsetForChild(startIndex) 244 | val endIndex = childAtIndex(end) 245 | val endOffset = offsetForChild(endIndex) 246 | val startChild = childAt(startIndex) 247 | if (startIndex == endIndex) startChild.annotationRange(start - startOffset,end - endOffset) 248 | else { 249 | val endChild = childAt(endIndex) 250 | var ann = startChild.annotationRange(start - startOffset,startChild.size) 251 | val arr = childs.asInstanceOf[Array[Annotated[A]]] 252 | 253 | if ((startIndex+1) < endIndex) { ann = c.annotator.append(ann,c.annotator.manyA(startIndex+1,endIndex,arr)) } 254 | 255 | c.annotator.append(ann,endChild.annotationRange(0,end - endOffset)) 256 | } 257 | } 258 | } 259 | def approxAnnotationRange(start: Long,end: Long)(implicit c: SS): A = { 260 | if (end >= size) approxAnnotationRange(start,size - 1) 261 | else if (start < 0) approxAnnotationRange(0,end) 262 | else { 263 | val startIndex = childAtIndex(start) 264 | val startOffset = offsetForChild(startIndex) 265 | val endIndex = childAtIndex(end) 266 | val endOffset = offsetForChild(endIndex) 267 | val startChild = childAt(startIndex) 268 | if (startIndex == endIndex) startChild.approxAnnotationRange(start - startOffset,end - endOffset) 269 | else { 270 | val arr = childs.asInstanceOf[Array[Annotated[A]]] 271 | c.annotator.manyA(startIndex,endIndex+1,arr) 272 | } 273 | } 274 | } 275 | 276 | def equalTo(o: SAS)(implicit c: SS): Boolean = c.equalTo(this,o) 277 | def first(implicit c: SS) = childs(0).first 278 | def last(implicit c: SS) = childs(childs.length-1).last 279 | def apply(i: Long)(implicit c: SS) = { 280 | val cc = childAtIndex(i) 281 | val o = offsetForChild(cc) 282 | childAt(cc)(i-o) 283 | } 284 | def getLeaf(i: Long): (BSeqLeaf[X,A],Int) = { 285 | val cc = childAtIndex(i) 286 | val o = offsetForChild(cc) 287 | childAt(cc).getLeaf(i-o) 288 | } 289 | def getLeaf2(i: Long): BSeqLeaf[X,A] = { 290 | val cc = childAtIndex(i) 291 | val o = offsetForChild(cc) 292 | childAt(cc).getLeaf2(i-o) 293 | } 294 | override def toString = childs.foldLeft("<")((x,y) => x + " " + y) + " >" 295 | } 296 | 297 | case class EmptySeq[@sp X,A]() extends BSeqTr[X,A] { 298 | def error = sys.error("empty") 299 | def annotation(implicit c: SS) = c.annotator.none 300 | def annotation = sys.error("no annotation") 301 | def size = 0.toLong 302 | def height = -1 303 | def split(i: Long)(implicit c: SS) = (this,this) 304 | def append[AAS <: SAS](o: AAS)(implicit c: SS): SAS = o 305 | def annotationRange(start: Long,end: Long)(implicit c: SS) = c.annotator.none 306 | def approxAnnotationRange(start: Long, end: Long)(implicit c: SS) = c.annotator.none 307 | def equalToTree[AAS <: SAS](o: AAS)(implicit c: SS): Boolean = (o.size == 0) 308 | def first(implicit c: SS) = error 309 | def last(implicit c: SS) = error 310 | def apply(i: Long)(implicit c: SS) = error 311 | def getLeaf(i: Long): (BSeqLeaf[X,A],Int) = error 312 | def getLeaf2(i: Long): BSeqLeaf[X,A] = error 313 | def intoArray(dest: Array[X], i: Int) = i 314 | override def toString = "<>" 315 | } 316 | 317 | trait BSeqLeaf[@sp X,A] extends BSeqTr[X,A] { 318 | def toArray: Array[X] 319 | def height = 0 320 | override def toString = toArray.foldLeft("<")((x,y) => x + " " + y) + " >" 321 | } 322 | 323 | case class BSeqLeafImpl[@sp X,A](array: Array[X],ann: A) 324 | extends BSeqLeaf[X,A] { 325 | def annotation = ann 326 | def toArray = array 327 | def size = array.length 328 | def some = array(array.length/2) 329 | def split(i: Long)(implicit c: SS) = { 330 | if (i >= size) (this,c.empty) 331 | else if (i < 0) (c.empty,this) 332 | else { 333 | val (left: Array[X],right: Array[X]) = array.splitAt(i.toInt) 334 | (c.createLeaf(left),c.createLeaf(right)) 335 | } 336 | } 337 | def annotationRange(start: Long,end: Long)(implicit c: SS): A = { 338 | if (end >= size) annotationRange(start,size - 1) 339 | else if (start < 0) annotationRange(0,end) 340 | else c.annotator.manyX(start.toInt,end.toInt+1,array) 341 | } 342 | def approxAnnotationRange(start: Long,end: Long)(implicit c: SS): A = { 343 | if (end >= size) annotationRange(start,size - 1) 344 | else if (start < 0) annotationRange(0,end) 345 | else c.annotator.manyX(start.toInt,end.toInt+1,array) 346 | } 347 | def equalToTree[AAS <: SAS](o: AAS)(implicit c: SS): Boolean = c.equalTo(this,o) 348 | def first(implicit c: SS) = array(0) 349 | def last(implicit c: SS) = array(array.length - 1) 350 | def append[AAS <: SAS](o: AAS)(implicit c: SS): SAS = c.append(this,o) 351 | def apply(i: Long)(implicit c: SS) = array(i.toInt) 352 | def getLeaf(i: Long): (BSeqLeaf[X,A],Int) = (this,i.toInt) 353 | def getLeaf2(i: Long): BSeqLeaf[X,A] = this 354 | def intoArray(dest: Array[X], i: Int): Int = { 355 | array.copyToArray(dest,i,array.length) 356 | array.length + i 357 | } 358 | } 359 | 360 | case class BSeqTreeImpl[@sp X,A, ARR <: Array[AnnTreeSeq[X,A]#AS]] 361 | (childs: ARR,sizes: Array[Long],ann: A) extends BSeqTree[X,A] { 362 | def annotation = ann 363 | def intoArray(dest: Array[X], i: Int): Int = { 364 | var ii = i 365 | var s = childs.length 366 | var idx = 0 367 | while (idx < s) { 368 | ii = childs(idx).intoArray(dest,ii) 369 | idx = idx + 1 370 | } 371 | ii 372 | } 373 | } 374 | 375 | trait AnnTreeSeqImpl[@sp X,A] extends AnnTreeSeq[X,A] { 376 | def self = this 377 | def emptySeq = new EmptyAnnotatedTreeSeq[X,A]()(context) 378 | def create(s: AnnTreeSeq[X,A]#AS) = new FullAnnotatedTreeSeq(s)(context) 379 | implicit def xtag = context.xTag 380 | } 381 | 382 | class EmptyAnnotatedTreeSeq[@sp X,A] 383 | (implicit c: RangedAnnotationOrderingContext[X,A]) extends AnnTreeSeqImpl[X,A] { 384 | def context = c 385 | def repr = empty 386 | } 387 | 388 | class FullAnnotatedTreeSeq[@sp X,A] 389 | (val repr: AnnTreeSeq[X,A]#AS)(implicit val c: RangedAnnotationOrderingContext[X,A]) extends AnnTreeSeqImpl[X,A] { 390 | def context = c 391 | } 392 | 393 | def seqFactory[@sp X](implicit ord: Order[X], ct: ClassTag[X], ca: ClassTag[Statistics[X]]): AnnTreeSeq[X,Statistics[X]] = { 394 | val ann = StatisticsAnnotator[X]() 395 | val eq = EqualStatP[X]()(ann) 396 | new EmptyAnnotatedTreeSeq[X,Statistics[X]]()(new RangedAnnOrdContextImpl[X,Statistics[X]](ann,eq,ord,ct,ca)) 397 | } 398 | 399 | def seqFactory2[@sp X](implicit ord: Order[X], ct: ClassTag[X], ca: ClassTag[Statistics[X]]): EmptyAnnotatedTreeSeq[X,Statistics[X]] = { 400 | val ann = StatisticsAnnotator[X]() 401 | val eq = EqualStatP[X]()(ann) 402 | val f = new EmptyAnnotatedTreeSeq[X,Statistics[X]]()(new RangedAnnOrdContextImpl[X,Statistics[X]](ann,eq,ord,ct,ca)) 403 | f 404 | } 405 | 406 | def defaultSeqFactory[@sp X](implicit ord: Order[X], ct: ClassTag[X]) = { 407 | val ann = NoAnnotator[X] 408 | val ca = implicitly[ClassTag[NoAnnotation]] 409 | val eq = EqualNoAnn 410 | new EmptyAnnotatedTreeSeq()(new RangedAnnOrdContextImpl(ann,eq,ord,ct,ca)) 411 | } 412 | 413 | 414 | private case class TreeSeqIterator[@sp X,@sp A](implicit seq: AnnTreeSeq[X,A]) extends SequenceIterator[X] { 415 | var pos: Long = 0 416 | var li: Int = seq.repr.getLeaf(0)._2 417 | var leaf: BSeqLeaf[X,A] = seq.repr.getLeaf(0)._1 418 | 419 | val size = seq.size 420 | var lsize: Long = seq.repr.getLeaf(0)._1.size 421 | 422 | def next: X = { 423 | if (li < lsize) { 424 | val v = leaf(li) ; li = li + 1 425 | v 426 | } 427 | else { 428 | pos = pos + lsize 429 | if (pos < size) { 430 | val (newleaf,newli) = seq.repr.getLeaf(pos) 431 | leaf = newleaf 432 | li = newli 433 | lsize = leaf.size 434 | next 435 | } 436 | else sys.error("no more elements") 437 | } 438 | } 439 | def position = pos 440 | def hasNext = { 441 | if (li < lsize) true 442 | else (pos+lsize) < size 443 | } 444 | def goto(i: Long) = { 445 | val (newleaf,newli) = seq.repr.getLeaf(pos) 446 | leaf = newleaf 447 | li = newli 448 | lsize = leaf.size 449 | pos = i 450 | } 451 | } 452 | 453 | def insertionSort[@sp X](input: Array[X], ordering: Order[X]): Array[X] = { 454 | var i = 1 455 | while (i < input.length) { 456 | var j = i 457 | var tmp = input(j) 458 | while (j > 0) { 459 | if (ordering.lt(input(j),input(j-1))) { 460 | tmp = input(j) 461 | input(j) = input(j - 1) 462 | input(j - 1) = tmp 463 | } 464 | j = j - 1 465 | } 466 | i = i + 1 467 | } 468 | input 469 | } 470 | } 471 | -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/sequence/ArraySequence.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.sequence 2 | 3 | import org.spread.core.annotation.Annotation.Annotator 4 | import org.spread.core.sequence.OrderingSequence._ 5 | import org.spread.core.sequence.AnnotatedTreeSequence.insertionSort 6 | 7 | import scala.reflect.ClassTag 8 | import scala.language.{existentials, implicitConversions} 9 | import org.spread.core.language.Annotation.sp 10 | import org.spread.core.sequence.Sequence.SequenceIterator 11 | import org.spread.core.sequence.OrderingSequence._ 12 | 13 | 14 | object ArraySequence { 15 | 16 | case class ArraySeq[@sp X](x: Array[X])(implicit ord: Order[X], xt: ClassTag[X]) extends OrderingSeq[X,ArraySeq[X]] { 17 | type S = ArraySeq[X] 18 | 19 | def self: S = this 20 | def size: Long = x.length 21 | def height = 0 22 | 23 | def tag = xt 24 | 25 | def ordering = ord 26 | 27 | def sort = { 28 | if (size < (64*64)){ 29 | val a = x.clone 30 | spire.math.Sorting.mergeSort(a) 31 | ArraySeq(a) 32 | } 33 | else defaultSort 34 | } 35 | def emptySeq = ArraySeq(Array()) 36 | def createSeq(a: Array[X]) = ArraySeq(a) 37 | def toArray = x.clone 38 | def append[S2 <: S](o: S2): S = ArraySeq(x ++ o.x) 39 | def split(o: Long) = { val (l,r) = x.splitAt(o.toInt) ; (ArraySeq(l),ArraySeq(r)) } 40 | def equalTo[S2 <: S](o: S2): Boolean = { 41 | val s = x.length 42 | val ox = o.x 43 | 44 | if (s == ox.length) { 45 | var i = 0 46 | var eq = true 47 | while ((i < s) && eq) { eq = (x(i) == ox(i)) ; i = i + 1} 48 | eq 49 | } 50 | else false 51 | } 52 | def apply(i: Long) = x(i.toInt) 53 | def first = x(0) 54 | def last = x(x.length-1) 55 | 56 | 57 | override def iterator: SequenceIterator[X] = SeqArrayIterator(x,x.length) 58 | } 59 | 60 | private case class SeqArrayIterator[@sp X](x: Array[X], size: Long) extends SequenceIterator[X] { 61 | private var pos = 0 62 | 63 | def next: X = { val v = x(pos) ; pos = pos + 1 ; v } 64 | def hasNext = pos < size 65 | def position = pos 66 | def goto(i: Long) = (pos = i.toInt) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/sequence/MappedSequence.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.sequence 2 | 3 | import org.spread.core.annotation.Annotation.{Annotator, NoAnnotation} 4 | import org.spread.core.sequence.Sequence._ 5 | import org.spread.core.language.Annotation.sp 6 | import org.spread.core.sequence.OrderingSequence._ 7 | import org.spread.core.sequence.RangedSequence._ 8 | 9 | import scala.language.{existentials, implicitConversions} 10 | import scala.reflect.ClassTag 11 | 12 | // 13 | // Lazy map Seq (wrapping another Seq) 14 | // 15 | // Copyright 2017: Robbert van Dalen 16 | // 17 | object MappedSequence { 18 | case class MapSeq[@sp X, @sp Y, S <: Seq[X,S]](source: S)(implicit f: X=>Y, yt: ClassTag[Y]) extends SeqImpl[Y,MapSeq[X,Y,S]] { 19 | type SS = MapSeq[X,Y,S] 20 | 21 | def self: SS = this 22 | def size: Long = source.size 23 | def height = source.height+1 24 | 25 | def tag = yt 26 | def create(c: S): SS = MapSeq[X,Y,S](c)(f,yt) 27 | def createSeq(a: Array[Y]) = ??? 28 | def toArray = source.toArray.map(f) 29 | def emptySeq = create(source.emptySeq) 30 | def append[S2 <: SS](o: S2): SS = create(source ++ o.source) 31 | def split(o: Long) = { val (l,r) = source.split(o.toInt) ; (create(l),create(r)) } 32 | def equalTo[S2 <: SS](o: S2): Boolean = source.equals(o.s) 33 | def first = f(source.first) 34 | def last = f(source.last) 35 | def apply(i: Long) = f(source(i)) 36 | } 37 | 38 | case class OrdSeq[@sp X, S <: Seq[X,S]](source: S)(implicit ord: Order[X], xt: ClassTag[X]) extends OrderingSeq[X,OrdSeq[X,S]] { 39 | type SS = OrdSeq[X,S] 40 | def self: SS = this 41 | def size: Long = source.size 42 | def height = source.height + 1 43 | def sort = defaultSort 44 | def ordering = ord 45 | def tag = xt 46 | def create(c: S): SS = OrdSeq[X,S](c)(ord,xt) 47 | def createSeq(a: Array[X]) = OrdSeq(source.createSeq(a)) 48 | def toArray: Array[X] = source.toArray 49 | def emptySeq = create(source.emptySeq) 50 | def append[S2 <: SS](o: S2): SS = create(source ++ o.source) 51 | def split(o: Long) = {val (l,r) = source.split(o.toInt); (create(l),create(r))} 52 | def equalTo[S2 <: SS](o: S2): Boolean = source.equals(o.s) 53 | def first = source.first 54 | def last = source.last 55 | def apply(i: Long) = source(i) 56 | } 57 | 58 | implicit class Mapper[@sp X, S <: Seq[X,S]](s: Seq[X,S]) { 59 | def lmap[@sp Y](f: X=>Y)(implicit ct: ClassTag[Y]) = MapSeq[X,Y,S](s.asInstanceOf[S])(f,ct) 60 | def order(implicit o: Order[X], ct: ClassTag[X]) = OrdSeq[X,S](s.asInstanceOf[S])(o,ct) 61 | } 62 | } -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/sequence/OrderingSequence.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.sequence 2 | 3 | import org.spread.core.language.Annotation.sp 4 | import org.spread.core.sequence.Sequence._ 5 | 6 | import scala.language.{existentials, implicitConversions} 7 | import scala.reflect.ClassTag 8 | 9 | // 10 | // Generic sort, union, difference, intersection 11 | // 12 | // Improved version of Enchilada semantics 13 | // 14 | // Copyright 2017: Robbert van Dalen 15 | // 16 | object OrderingSequence { 17 | 18 | import spire.implicits._ 19 | 20 | type Order[@sp X] = spire.algebra.Order[X] 21 | 22 | trait OrderingSeq[@sp X,S <: OrderingSeq[X,S]] extends SeqImpl[X,S] { 23 | 24 | // We use Spire's Order because it is specialized, and that we can re-use Spire's sort algorithms 25 | def ordering: spire.algebra.Order[X] 26 | 27 | // Generic sort, must be overridden 28 | def sort: S 29 | 30 | // Default merge sort 31 | def defaultSort: S = { 32 | if (size <= 1) self 33 | else { 34 | val (left,right) = split(size / 2) 35 | combineSorted(left.sort,right.sort,Union) 36 | } 37 | } 38 | 39 | def union(o: S): S = combineSorted(this.sort,o.sort,Union) 40 | def difference(o: S): S = combineSorted(this.sort,o.sort,Difference) 41 | def intersect(o: S): S = combineSorted(this.sort,o.sort,Intersect) 42 | 43 | def smallerIndex(elem: X): Long = { 44 | val ord = ordering 45 | var start: Long = 0 46 | var end = size 47 | 48 | while(start != end) { 49 | val mid = (start / 2) + (end / 2) 50 | if (ord.lt(this(mid),elem)) start = mid + 1 51 | else end = mid 52 | } 53 | end 54 | } 55 | 56 | def greaterIndex(elem: X): Long = { 57 | val ord = ordering 58 | var start: Long = 0 59 | var end = size 60 | 61 | while(start != end) { 62 | val mid = (start / 2) + (end / 2) 63 | if (ord.gt(this(mid),elem)) end = mid 64 | else start = mid + 1 65 | } 66 | end 67 | } 68 | 69 | def repeat(s1: S,m: Int): S = { 70 | if (m == 0) s1.emptySeq 71 | else if (m == 1) s1 72 | else { 73 | val m2 = m / 2 74 | val sr = repeat(s1,m2) 75 | sr.append(sr).append(repeat(s1,m - (m2 * 2))) 76 | } 77 | } 78 | 79 | def multiply(s1: S,m: Int): S = { 80 | if (s1.size == 0) s1 81 | else if (s1.size == 1) repeat(s1,m) 82 | else { 83 | val (l,r) = s1.split(s1.size / 2) 84 | multiply(l,m).append(multiply(r,m)) 85 | } 86 | } 87 | 88 | def union(s1: S,s2: S): S = combine(s1,s2,Union) 89 | def difference(s1: S,s2: S): S = combine(s1,s2,Difference) 90 | def intersect(s1: S,s2: S): S = combine(s1,s2,Intersect) 91 | def combine(s1: S,s2: S,op: MergeOperator): S = combineSorted(s1.sort,s2.sort,op) 92 | 93 | def combineSorted(s1: S,s2: S,op: MergeOperator): S = { 94 | if (s1.isEmpty) op.append[X,S](s1,s2) 95 | else if (s2.isEmpty) op.append[X,S](s1,s2) 96 | //else if (s1.equalTo(s2)) op.equal[X,S](s1) // TODO: faster equality check 97 | else { 98 | val ord = s1.ordering 99 | if (ord.lt(s1.last,s2.first)) op.append[X,S](s1,s2) 100 | else if (ord.lt(s2.last,s1.first)) op.append[X,S](s2,s1) 101 | else if (s1.size == 1 && s2.size == 1) { 102 | val c = ord.compare(s1.first,s2.first) 103 | if (c == 0) op.equal[X,S](s1) 104 | if (c < 0) op.append[X,S](s1,s2) 105 | else op.append[X,S](s2,s1) 106 | } 107 | else { 108 | if (((s1.size + s2.size) <= 65536)) op.smallMerge[X,S](s1,s2)(s1.tag) 109 | else if (s1.size >= s2.size) { 110 | val elem = s1(s1.size/2) 111 | 112 | val li1 = s1.smallerIndex(elem) 113 | val l1 = s1.split(li1) 114 | val gi1 = l1._2.greaterIndex(elem) 115 | val r1 = l1._2.split(gi1) 116 | 117 | val smaller1 = l1._1 118 | val same1 = r1._1 119 | val bigger1 = r1._2 120 | 121 | val li2 = s2.smallerIndex(elem) 122 | val l2 = s2.split(li2) 123 | val gi2 = l2._2.greaterIndex(elem) 124 | val r2 = l2._2.split(gi2) 125 | 126 | val smaller2 = l2._1 127 | val same2 = r2._1 128 | val bigger2 = r2._2 129 | 130 | val ml = combineSorted(smaller1,smaller2,op) 131 | val mm = op.equalElems[X,S](same1,same2) 132 | val mr = combineSorted(bigger1,bigger2,op) 133 | 134 | ml.append(mm).append(mr) 135 | } 136 | else combineSorted(s2,s1,op) 137 | } 138 | } 139 | } 140 | } 141 | 142 | trait MergeOperator { 143 | def equal[@sp X,S <: OrderingSeq[X,S]](s1: S): S 144 | def equalElems[@sp X,S <: OrderingSeq[X,S]](s1: S,s2: S): S 145 | def append[@sp X,S <: OrderingSeq[X,S]](s1: S,s2: S): S 146 | def not_equal[@sp X](result: Array[X], value: X, k: Int): Int 147 | def equal[@sp X](result: Array[X], value: X, k: Int): Int 148 | 149 | def smallMerge[@sp X: ClassTag,S <: OrderingSeq[X,S]](s1: S,s2: S) = { 150 | val result = new Array[X](s1.size.toInt + s2.size.toInt) 151 | val ord = s1.ordering 152 | 153 | val a = s1.toArray 154 | val b = s2.toArray 155 | 156 | var i = 0 157 | var j = 0 158 | var k = 0 159 | 160 | while (i < a.length && j < b.length) { 161 | val c = ord.compare(a(i),b(j)) 162 | if (c < 0) { k = not_equal(result,a(i),k) ; i = i + 1 } 163 | else if (c > 0) { k = not_equal(result,b(j),k) ; j = j + 1 } 164 | else { k = equal(result,a(i),k) ; k = equal(result,b(j),k) ; i = i + 1 ; j = j + 1 } 165 | } 166 | 167 | while (i < a.length) { k = not_equal(result,a(i),k) ; i = i + 1 } 168 | while (j < b.length) { k = not_equal(result,b(j),k) ; j = j + 1 } 169 | 170 | val rr = new Array[X](k) 171 | result.copyToArray(rr,0,k) 172 | s1.createSeq(rr) 173 | } 174 | } 175 | 176 | trait Union extends MergeOperator 177 | trait Difference extends MergeOperator 178 | trait Intersect extends MergeOperator 179 | 180 | case object Union extends Union { 181 | @inline final def equal[@sp X,S <: OrderingSeq[X,S]](s1: S) = s1.multiply(s1,2) 182 | @inline final def equalElems[@sp X,S <: OrderingSeq[X,S]](s1: S,s2: S) = s1.append(s2) 183 | @inline final def append[@sp X,S <: OrderingSeq[X,S]](s1: S,s2: S) = s1.append(s2) 184 | @inline final def not_equal[@sp X](result: Array[X], value: X, k: Int): Int = { result(k) = value ; k + 1 } 185 | @inline final def equal[@sp X](result: Array[X], value: X, k: Int): Int = { result(k) = value ; k + 1 } 186 | } 187 | 188 | case object Difference extends Difference { 189 | @inline final def equal[@sp X,S <: OrderingSeq[X,S]](s1: S) = s1.emptySeq 190 | @inline final def equalElems[@sp X,S <: OrderingSeq[X,S]](s1: S,s2: S) = { 191 | val s = Math.abs(s1.size - s2.size) 192 | s1.split(s)._1 193 | } 194 | @inline final def append[@sp X,S <: OrderingSeq[X,S]](s1: S,s2: S) = s1.append(s2) 195 | @inline final def not_equal[@sp X](result: Array[X], value: X, k: Int): Int = { result(k) = value ; k + 1 } 196 | @inline final def equal[@sp X](result: Array[X], value: X, k: Int): Int = k 197 | } 198 | 199 | case object Intersect extends Intersect { 200 | @inline final def equal[@sp X,S <: OrderingSeq[X,S]](s1: S) = s1 201 | @inline final def equalElems[@sp X,S <: OrderingSeq[X,S]](s1: S,s2: S) = s1.split(min(s1.size,s2.size))._1 202 | @inline final def append[@sp X,S <: OrderingSeq[X,S]](s1: S,s2: S) = s1.emptySeq 203 | @inline final def not_equal[@sp X](result: Array[X], value: X, k: Int): Int = k 204 | @inline final def equal[@sp X](result: Array[X], value: X, k: Int): Int = { result(k) = value ; k + 1 } 205 | } 206 | 207 | @inline final def min(l1: Long, l2: Long) = if (l1 < l2) l1 ; else l2 208 | } 209 | -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/sequence/PairedSequence.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.sequence 2 | 3 | import org.spread.core.annotation.Annotation.Annotator 4 | import org.spread.core.constraint.Constraint.PropValue 5 | import org.spread.core.sequence.Sequence._ 6 | 7 | import org.spread.core.language.Annotation.sp 8 | import org.spread.core.sequence.AnnotatedSequence._ 9 | import org.spread.core.sequence.OrderingSequence._ 10 | 11 | import scala.language.{existentials, implicitConversions} 12 | 13 | import scala.reflect.ClassTag 14 | 15 | object PairedSequence { 16 | trait PairedSeq[@sp X1,@sp X2,S1 <: Seq[X1,S1], S2 <: Seq[X2,S2], S <: PairedSeq[X1,X2,S1,S2,S]] 17 | extends Seq[(X1,X2),S] { 18 | 19 | { assert(left.size == right.size) } 20 | 21 | def left: S1 22 | def right: S2 23 | 24 | def L = left 25 | def R = right 26 | 27 | def emptySeq = create(left.emptySeq,right.emptySeq) 28 | def create(l: S1, r: S2): S 29 | 30 | implicit def xTag = left.tag 31 | implicit def yTag = right.tag 32 | 33 | def createSeq(a: Array[(X1,X2)]) = { 34 | // TODO: optimize 35 | val x1 = a.map(_._1).toArray 36 | val x2 = a.map(_._2).toArray 37 | 38 | create(left.createSeq(x1),right.createSeq(x2)) 39 | } 40 | 41 | def toArray: Array[(X1,X2)] = { 42 | val x1 = left.toArray 43 | val x2 = right.toArray 44 | val result = new Array[(X1,X2)](x1.length) 45 | 46 | var i = 0 47 | var s = x1.length 48 | while (i < s) { 49 | result(i) = (x1(i),x2(i)) 50 | i = i + 1 51 | } 52 | result 53 | } 54 | 55 | def append[S2 <: S](o: S2): S = create(left append o.left, right append o.right) 56 | def split(o: Long) = { 57 | val (ll,lr) = left.split(o) 58 | val (rl,rr) = right.split(o) 59 | (create(ll,rl),create(lr,rr)) 60 | } 61 | def equalTo[S2 <: S](o: S2): Boolean = left.equals(o.left) && right.equals(o.right) 62 | def size = left.size 63 | def height = (left.height max right.height) + 1 64 | def first = (left.first,right.first) 65 | def last = (left.last,right.last) 66 | def apply(i: Long) = (left(i),right(i)) 67 | } 68 | 69 | case class PairSeqImpl[@sp X1,@sp X2,S1 <: Seq[X1,S1], S2 <: Seq[X2,S2]](left: S1, right: S2, tag: ClassTag[(X1,X2)]) 70 | extends PairedSeq[X1,X2,S1,S2,PairSeqImpl[X1,X2,S1,S2]] { 71 | 72 | type S = PairSeqImpl[X1,X2,S1,S2] 73 | 74 | def create(l: S1, r: S2): S = PairSeqImpl(l,r,tag) 75 | def self = this 76 | } 77 | 78 | case class OrdPairSeqImpl[@sp X1,@sp X2,S1 <: OrderingSeq[X1,S1], S2 <: OrderingSeq[X2,S2]] 79 | (left: S1, right: S2, tag: ClassTag[(X1,X2)])(implicit ord: Order[(X1,X2)]) 80 | extends PairedSeq[X1,X2,S1,S2,OrdPairSeqImpl[X1,X2,S1,S2]] with OrderingSeq[(X1,X2),OrdPairSeqImpl[X1,X2,S1,S2]] { 81 | 82 | type S = OrdPairSeqImpl[X1,X2,S1,S2] 83 | type ORD = Order[(X1,X2)] 84 | 85 | def sort = defaultSort 86 | def ordering = ord 87 | def create(l: S1, r: S2): S = OrdPairSeqImpl[X1,X2,S1,S2](l,r,tag)(ord) 88 | def self = this 89 | } 90 | 91 | type AS[@sp X,A,S <: AnnotatedSeq[X,A,S]] = AnnotatedSeq[X,A,S] 92 | 93 | trait AnnPairSeq 94 | [@sp X1,@sp X2,A1,A2,S1 <: AS[X1,A1,S1], S2 <: AS[X2,A2,S2], S <: AnnPairSeq[X1,X2,A1,A2,S1,S2,S]] 95 | extends PairedSeq[X1,X2,S1,S2,S] with AnnotatedSeq[(X1,X2),(A1,A2),S] { 96 | def annotationRange(start: Long, end: Long) = (left.annotationRange(start,end),right.annotationRange(start,end)) 97 | def approxAnnotationRange(start: Long, end: Long) = (left.annotationRange(start,end),right.annotationRange(start,end)) 98 | def equal = ??? 99 | } 100 | 101 | case class AnnPairSeqImpl[@sp X1,@sp X2,A1,A2,S1 <: AS[X1,A1,S1], S2 <: AS[X2,A2,S2]] 102 | (left: S1, right: S2, tag: ClassTag[(X1,X2)]) extends AnnPairSeq[X1,X2,A1,A2,S1,S2,AnnPairSeqImpl[X1,X2,A1,A2,S1,S2]] { 103 | 104 | type S = AnnPairSeqImpl[X1,X2,A1,A2,S1,S2] 105 | 106 | def annotation = (left.annotation,right.annotation) 107 | def create(l: S1, r: S2): S = AnnPairSeqImpl(left,right,tag) 108 | def self = this 109 | } 110 | 111 | case class AnnOrdPairSeqImpl[@sp X1,@sp X2,A1,A2,S1 <: AS[X1,A1,S1], S2 <: AS[X2,A2,S2]] 112 | (left: S1, right: S2, tag: ClassTag[(X1,X2)])(implicit ord: Order[(X1,X2)]) 113 | extends AnnPairSeq[X1,X2,A1,A2,S1,S2,AnnOrdPairSeqImpl[X1,X2,A1,A2,S1,S2]] with 114 | AnnOrdSeq[(X1,X2),(A1,A2),AnnOrdPairSeqImpl[X1,X2,A1,A2,S1,S2]] { 115 | 116 | type S = AnnOrdPairSeqImpl[X1,X2,A1,A2,S1,S2] 117 | type ORD = Order[(X1,X2)] 118 | 119 | def sort = defaultSort 120 | def ordering = ord 121 | def annotation = (left.annotation,right.annotation) 122 | def create(l: S1, r: S2): S = AnnOrdPairSeqImpl[X1,X2,A1,A2,S1,S2](l,r,tag)(ord) 123 | def self = this 124 | } 125 | 126 | trait Prio4Combiner { 127 | trait Combiner4[@sp X1,S1 <: Seq[X1,S1]] { 128 | def s1: Seq[X1,S1] 129 | def &&[@sp X2,S2 <: Seq[X2,S2]](s2: Seq[X2,S2]) = { 130 | PairSeqImpl[X1,X2,S1,S2](s1.asInstanceOf[S1],s2.asInstanceOf[S2],implicitly[ClassTag[(X1,X2)]]) 131 | } 132 | } 133 | implicit class SimpleCombiner[@sp X1,S1 <: Seq[X1,S1]](val s1: Seq[X1,S1]) extends Combiner4[X1,S1] 134 | } 135 | 136 | trait Prio3Combiner extends Prio4Combiner { 137 | trait Combiner3[@sp X1,S1 <: OrderingSeq[X1,S1]] extends Combiner4[X1,S1] { 138 | def s1: OrderingSeq[X1,S1] 139 | def &&[@sp X2,S2 <: OrderingSeq[X2,S2]](s2: OrderingSeq[X2,S2])(implicit ord: Order[(X1,X2)]) = { 140 | OrdPairSeqImpl[X1,X2,S1,S2](s1.asInstanceOf[S1],s2.asInstanceOf[S2],implicitly[ClassTag[(X1,X2)]]) 141 | } 142 | } 143 | implicit class OrdCombiner[@sp X1,S1 <: OrderingSeq[X1,S1]](val s1: OrderingSeq[X1,S1]) 144 | extends Combiner3[X1,S1] 145 | } 146 | 147 | trait Prio2Combiner extends Prio3Combiner { 148 | trait Combiner2[@sp X1,A1,S1 <: AnnotatedSeq[X1,A1,S1]] extends Combiner4[X1,S1] { 149 | def s1: AnnotatedSeq[X1,A1,S1] 150 | def &&[@sp X2,A2,S2 <: AnnotatedSeq[X2,A2,S2]](s2: AnnotatedSeq[X2,A2,S2]) = { 151 | AnnPairSeqImpl[X1,X2,A1,A2,S1,S2](s1.asInstanceOf[S1],s2.asInstanceOf[S2],implicitly[ClassTag[(X1,X2)]]) 152 | } 153 | } 154 | implicit class AnnCombiner[@sp X1,A1,S1 <: AnnotatedSeq[X1,A1,S1]](val s1: AnnotatedSeq[X1,A1,S1]) 155 | extends Combiner2[X1,A1,S1] 156 | } 157 | 158 | trait Prio1Combiner extends Prio2Combiner { 159 | trait Combiner1[@sp X1,A1,S1 <: AnnOrdSeq[X1,A1,S1]] extends Combiner2[X1,A1,S1] with Combiner3[X1,S1] { 160 | def s1: AnnOrdSeq[X1,A1,S1] 161 | def &&[@sp X2,A2,S2 <: AnnOrdSeq[X2,A2,S2]] 162 | (s2: AnnOrdSeq[X2,A2,S2])(implicit ord: Order[(X1,X2)]) = { 163 | AnnOrdPairSeqImpl[X1,X2,A1,A2,S1,S2](s1.asInstanceOf[S1],s2.asInstanceOf[S2],implicitly[ClassTag[(X1,X2)]]) 164 | } 165 | } 166 | implicit class AnnOrdCombiner[@sp X1,A1,S1 <: AnnOrdSeq[X1,A1,S1]](val s1: AnnOrdSeq[X1,A1,S1]) 167 | extends Combiner1[X1,A1,S1] 168 | } 169 | 170 | object Combiner extends Prio1Combiner // combining (&&) AnnOrdSeq has highest prio (Prio1) 171 | 172 | // Selector has identity 173 | class Selector[X1,@sp X2, S1 <: Seq[X1,S1], S2 <: Seq[X2,S2]] 174 | (seq: S1, f: S1 => S2) { 175 | def copy: Selector[X1,X2,S1,S2] = new Selector(seq,f) 176 | def asSeq: Seq[X1,S1] = seq 177 | def apply(): S2 = f(seq) 178 | def apply(o: S1): S2 = f(o) 179 | } 180 | 181 | // AnnSelector has identity 182 | class AnnSelector[X1,@sp X2, A, S1 <: Seq[X1,S1], S2 <: AnnotatedSeq[X2,A,S2]] 183 | (seq: S1, f: S1 => S2) { 184 | def copy: AnnSelector[X1,X2,A,S1,S2] = new AnnSelector(seq,f) 185 | def asSeq: Seq[X1,S1] = seq 186 | def apply(): S2 = f(seq) 187 | def apply(o: S1): S2 = f(o) 188 | } 189 | 190 | trait Prio3Selector { 191 | implicit class Select3[@sp X1,A,S1 <: AnnotatedSeq[X1,A,S1]](seq: AnnotatedSeq[X1,A,S1]) { 192 | def selectSame = new AnnSelector[X1,X1,A,S1,S1](seq.asInstanceOf[S1],(x=>x)) 193 | } 194 | } 195 | 196 | trait Prio2Selector extends Prio3Selector { 197 | implicit class Select2[X1,S1 <: Seq[X1,S1]] 198 | (s: Seq[X1,S1]) { 199 | def select[@sp X2,S2 <: Seq[X2,S2]](f: S1 => Seq[X2,S2]) = { 200 | new Selector[X1,X2,S1,S2](s.asInstanceOf[S1], f.asInstanceOf[S1=>S2]) 201 | } 202 | } 203 | } 204 | 205 | trait Prio1Selector extends Prio2Selector { 206 | implicit class Select1[X1,S1 <: Seq[X1,S1]] 207 | (seq: Seq[X1,S1]) { 208 | def select[@sp X2,A,S2 <: AnnotatedSeq[X2,A,S2]] 209 | (f: S1 => AnnotatedSeq[X2,A,S2]) = { 210 | new AnnSelector[X1,X2,A,S1,S2](seq.asInstanceOf[S1], f.asInstanceOf[S1 => S2]) 211 | } 212 | } 213 | 214 | } 215 | 216 | object Selector extends Prio1Selector 217 | } 218 | 219 | -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/sequence/RangedSequence.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.sequence 2 | 3 | import org.spread.core.annotation.Annotation._ 4 | import org.spread.core.constraint.Constraint.{EqualNoAnn, EqualProp, EqualStatP} 5 | import org.spread.core.sequence.AnnotatedTreeSequence._ 6 | import org.spread.core.sequence.OrderingSequence._ 7 | 8 | import scala.language.{existentials, implicitConversions} 9 | import scala.reflect.ClassTag 10 | import org.spread.core.language.Annotation.sp 11 | import org.spread.core.sequence.AnnotatedSequence._ 12 | import spire.implicits._ 13 | 14 | object RangedSequence { 15 | 16 | type SL = Statistics[Long] 17 | 18 | trait LongTreeSeqImpl[A] 19 | extends AnnTreeSeq[Long,A] { 20 | 21 | def self = this 22 | def emptySeq: AnnTreeSeq[Long,A] = EmptyLongTreeSeq[A]()(context) 23 | def create(s: AnnTreeSeq[Long,A]#AS) = FullLongTreeSeq[A](s)(context) 24 | def create(lowerBound: Long,upperBound: Long) = FullLongTreeSeq[A](createRange(lowerBound,upperBound)(self))(context) 25 | 26 | def createRange(lowerBound: Long,upperBound: Long)(implicit c: SS) = { 27 | if (lowerBound > upperBound) c.empty 28 | else LongLeafRange[A](lowerBound,upperBound,c.annotator.range(lowerBound,upperBound)) 29 | } 30 | 31 | override def sort = repr match { 32 | case l: LongLeafRange[A] => this 33 | case _ => defaultSort2 34 | } 35 | 36 | override def createTree[ARR <: Array[SAS]](a: ARR)(implicit c: SS): SAS = { 37 | var i = 0 38 | for (ii <- 1 until a.length) { 39 | if (canMerge(a(i),a(ii))) { a(i) = merge(a(i),a(ii)) } 40 | else { i = i + 1 ; a(i) = a(ii) } 41 | } 42 | if (i == (a.length-1)) { super.createTree(a) } 43 | else { super.createTree(a.splitAt(i+1)._1) } // TODO: optimize array copy 44 | } 45 | 46 | def canMerge(s1: SAS, s2: SAS): Boolean = { 47 | if (s1.isInstanceOf[LongLeafRange[A]] && s2.isInstanceOf[LongLeafRange[A]]) { 48 | val l1 = s1.asInstanceOf[LongLeafRange[A]] 49 | val l2 = s2.asInstanceOf[LongLeafRange[A]] 50 | ((l1.upperBound+1)==l2.lowerBound) 51 | } 52 | else false 53 | } 54 | 55 | def merge(s1: SAS, s2: SAS)(implicit c: SS): SAS = { 56 | val l1 = s1.asInstanceOf[LongLeafRange[A]] 57 | val l2 = s2.asInstanceOf[LongLeafRange[A]] 58 | createRange(l1.lowerBound,l2.upperBound) 59 | } 60 | 61 | override def createLeaf(a: Array[Long])(implicit c: SS) = { 62 | if (a.length == 0) c.empty 63 | else { 64 | var sequential = true 65 | for (i <- 1 until a.length) { if ((a(i-1)+1) != a(i)) sequential = false } 66 | if (sequential) { createRange(a(0),a(a.length-1)) } 67 | else BSeqLeafImpl(a,c.annotator.manyX(a)) 68 | } 69 | } 70 | } 71 | 72 | case class EmptyLongTreeSeq[A](implicit c: RangedAnnotationOrderingContext[Long,A]) extends LongTreeSeqImpl[A] { 73 | def repr = empty 74 | def context = c 75 | } 76 | 77 | case class FullLongTreeSeq[A](repr: AnnTreeSeq[Long,A]#AS)(implicit c: RangedAnnotationOrderingContext[Long,A]) 78 | extends LongTreeSeqImpl[A] { 79 | def context = c 80 | } 81 | 82 | case class LongLeafRange[A](lowerBound: Long,upperBound: Long, ann: A) 83 | extends BSeqLeaf[Long,A]{ 84 | { assert(lowerBound <= upperBound) } 85 | 86 | def annotation = ann 87 | def createRange(lowerBound: Long,upperBound: Long)(implicit c: SS): SAS = { 88 | if (lowerBound > upperBound) c.empty 89 | else LongLeafRange(lowerBound,upperBound,c.annotator.range(lowerBound,upperBound)) 90 | } 91 | 92 | def split(i: Long)(implicit c: SS) = { 93 | if (i < 0) (c.empty,this) 94 | else if (i >= size) (this,c.empty) 95 | else { 96 | val l = createRange(lowerBound,lowerBound + i - 1) 97 | val r = createRange(lowerBound + i,upperBound) 98 | (l,r) 99 | } 100 | } 101 | 102 | def annotationRange(start: Long,end: Long)(implicit c: SS): A = { 103 | if (end >= size) annotationRange(start,size - 1) 104 | else if (start < 0) annotationRange(0,end) 105 | else c.annotator.range(lowerBound + start,lowerBound + end) 106 | } 107 | 108 | def approxAnnotationRange(start: Long,end: Long)(implicit c: SS): A = { 109 | if (end >= size) approxAnnotationRange(start,size - 1) 110 | else if (start < 0) approxAnnotationRange(0,end) 111 | else c.annotator.range(lowerBound + start,lowerBound + end) 112 | } 113 | 114 | def intoArray(dest: Array[Long], i: Int) = { 115 | var ri = i 116 | var ii = lowerBound.toInt 117 | while (ii <= upperBound) { 118 | dest(ri) = ii 119 | ri = ri + 1 120 | ii = ii + 1 121 | } 122 | ri 123 | } 124 | def equalToTree[AAS <: SAS](o: AAS)(implicit c: SS): Boolean = o match { 125 | case LongLeafRange(l,u,_) => (lowerBound == l) && (upperBound == u) 126 | case _ => c.equalTo(this,o) 127 | } 128 | def append[AAS <: SAS](o: AAS)(implicit c: SS): SAS = o match { 129 | case LongLeafRange(l,u,_) => { 130 | if ((upperBound+1) == l) createRange(lowerBound,u) 131 | else c.append(this ,o) 132 | } 133 | case _ => c.append(this,o) 134 | } 135 | 136 | def sorted = true 137 | def size = upperBound - lowerBound + 1 138 | def toArray = (lowerBound to upperBound).toArray 139 | def first = lowerBound 140 | def last = upperBound 141 | def first(implicit c: SS) = lowerBound 142 | def last(implicit c: SS) = upperBound 143 | def apply(i: Long)(implicit c: SS) = lowerBound+i 144 | def isValid = true 145 | def getLeaf(i: Long): (BSeqLeaf[Long,A],Int) = { 146 | (this,i.toInt) 147 | } 148 | def getLeaf2(i: Long): BSeqLeaf[Long,A] = this 149 | override def toString = "L"+lowerBound + ":" + upperBound 150 | } 151 | 152 | type LSEQ[A] = AnnTreeSeq[Long,A] 153 | 154 | def longSeqFactory[A](implicit ann: RangeAnnotator[Long,A], ca: ClassTag[A], eq: EqualProp[A]): LSEQ[A] = { 155 | val ord = implicitly[Order[Long]] 156 | val cx = implicitly[ClassTag[Long]] 157 | EmptyLongTreeSeq()(new RangedAnnOrdContextImpl[Long,A](ann,eq,ord,cx,ca)) 158 | } 159 | 160 | def defaultLongSeqFactory = { 161 | val ann = NoAnnotator[Long] 162 | val ca = implicitly[ClassTag[NoAnnotation]] 163 | val eq = EqualNoAnn 164 | longSeqFactory(ann,ca,eq) 165 | } 166 | 167 | def defaultAnnLongSeqFactory = { 168 | val ann = StatisticsAnnotator[Long]() 169 | val ca = implicitly[ClassTag[SL]] 170 | val eq = EqualStatP()(ann) 171 | longSeqFactory(ann,ca,eq) 172 | } 173 | 174 | def createRange(lb: Long, ub: Long): LSEQ[NoAnnotation] = defaultLongSeqFactory.asInstanceOf[LongTreeSeqImpl[NoAnnotation]].create(lb,ub) 175 | def createAnnRange(lb: Long, ub: Long): LSEQ[SL] = defaultAnnLongSeqFactory.asInstanceOf[LongTreeSeqImpl[SL]].create(lb,ub) 176 | } 177 | -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/sequence/Sequence.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.sequence 2 | 3 | import org.spread.core.annotation.Annotation.Annotator 4 | import org.spread.core.constraint.Constraint.EqualProp 5 | 6 | import scala.language.{existentials, implicitConversions} 7 | import org.spread.core.language.Annotation.sp 8 | 9 | import scala.reflect.ClassTag 10 | 11 | object Sequence { 12 | trait Seq[@sp X,S <: Seq[X,S]] { 13 | 14 | def self: S 15 | def emptySeq: S 16 | 17 | def append[S2 <: S](o: S2): S 18 | def split(o: Long): (S,S) 19 | def equalTo[S2 <: S](o: S2): Boolean 20 | 21 | def size: Long 22 | def height: Int 23 | 24 | def apply(i: Long): X 25 | def first: X 26 | def last: X 27 | 28 | def iterator: SequenceIterator[X] = ??? 29 | 30 | def tag: ClassTag[X] 31 | def createSeq(a: Array[X]): S 32 | def toArray: Array[X] 33 | 34 | def isEmpty: Boolean = (size == 0) 35 | def ++[S2 <: S](o: S2): S = append(o) 36 | } 37 | 38 | trait SequenceIterator[@sp X] { 39 | def size: Long 40 | 41 | def next: X 42 | def hasNext: Boolean 43 | 44 | def goto(i: Long) 45 | def position: Long 46 | } 47 | 48 | // Yet to be defined extra methods 49 | trait SeqImpl[X, S <: SeqImpl[X,S]] extends Seq[X,S] { } 50 | 51 | implicit class Show[X, S <: Seq[X,S]](val s: Seq[X,S]) extends AnyVal { 52 | def show: Unit = { 53 | println(s.getClass.getSimpleName + ":") 54 | val size = { if (s.size > 20) 20 ; else s.size.toInt } 55 | val part = s.split(size)._1 56 | for (i <- (0 until size)) { 57 | println(" " + part(i)) 58 | } 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/sequence/VectorSequence.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.sequence 2 | 3 | import org.spread.core.annotation.Annotation.Annotator 4 | import org.spread.core.sequence.Sequence._ 5 | 6 | import scala.language.{existentials, implicitConversions} 7 | import scala.reflect.ClassTag 8 | import org.spread.core.language.Annotation.sp 9 | import org.spread.core.sequence.OrderingSequence._ 10 | 11 | object VectorSequence { 12 | case class VectorSeq[@sp X](x: Vector[X])(implicit ord: Order[X], ct: ClassTag[X]) 13 | extends OrderingSeq[X,VectorSeq[X]] { 14 | type S = VectorSeq[X] 15 | 16 | def ordering = ord 17 | 18 | def self: S = this 19 | def size: Long = x.length 20 | def height = 0 21 | 22 | def tag = ct 23 | def createSeq(a: Array[X]) = VectorSeq[X](a.toVector) 24 | 25 | def sort = defaultSort 26 | def emptySeq = VectorSeq(Vector()) 27 | def toArray = x.toArray 28 | def append[S2 <: S](o: S2): S = VectorSeq(x ++ o.x) 29 | def split(o: Long) = { val (l,r) = x.splitAt(o.toInt) ; (VectorSeq(l),VectorSeq(r)) } 30 | def equalTo[S2 <: S](o: S2): Boolean = x.equals(o.x) 31 | def first = x(0) 32 | def last = x.last 33 | def apply(i: Long) = x(i.toInt) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/splithash/ClassLoader.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.splithash 2 | 3 | import java.io._ 4 | 5 | // A custom class-loader that keeps all loaded byte-code available for SPREAD to hash its own byte-code 6 | object ClassLoader { 7 | val instance = AuthenticatedClassLoader(AuthenticatedClassLoader.getClass.getClassLoader) 8 | 9 | case class AuthenticatedClassLoader(parent: java.lang.ClassLoader) extends java.lang.ClassLoader(parent) { 10 | var bytecode: Map[Class[_],Array[Byte]] = Map() 11 | 12 | override def loadClass(name: String) = { 13 | if (!name.startsWith("java") && !name.startsWith("sun")) { 14 | val file = name.replace('.',File.separatorChar) + ".class" 15 | var is = getClass().getClassLoader().getResourceAsStream(file) 16 | 17 | if (is != null) { 18 | val buffer = new ByteArrayOutputStream() 19 | var data: Array[Byte] = new Array(128) 20 | var n = 0 21 | 22 | n = is.read(data,0,data.length) 23 | 24 | while (n != -1) { 25 | buffer.write(data,0,n) 26 | n = is.read(data,0,data.length) 27 | } 28 | 29 | buffer.flush() 30 | var b: Array[Byte] = buffer.toByteArray 31 | 32 | val c: Class[_] = defineClass(name,b,0,b.length) 33 | resolveClass(c) 34 | bytecode = bytecode + (c->b) 35 | c 36 | } 37 | else super.loadClass(name) 38 | } 39 | else super.loadClass(name) 40 | } 41 | 42 | // For full crypto, we have to sign the full topologically sorted set of classes that this class depends on 43 | // But that would require byte-code analysis, so for now we just take the single class 44 | def cryptoSignClass(c: Class[_]): Array[Int] = { 45 | import java.nio.ByteBuffer 46 | import java.security.MessageDigest 47 | 48 | val b = bytecode.get(c).get 49 | val md = MessageDigest.getInstance("SHA-256") 50 | 51 | md.reset 52 | md.update(b) 53 | 54 | val bb = ByteBuffer.wrap(md.digest).asIntBuffer 55 | val ib: Array[Int] = new Array(bb.limit) 56 | bb.get(ib) 57 | 58 | ib 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/splithash/Hashing.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.splithash 2 | 3 | object Hashing { 4 | import Reference._ 5 | 6 | // A Hashable object 7 | trait Hashable extends Ref[Hashable] { 8 | def hash: Hash 9 | def parts: Array[Hashable] 10 | } 11 | 12 | // An 'infinitely' indexable and expandable Hash that *must* obey the following property: 13 | // The chance that two (slightly) different objects have equal hashes at index i 14 | // *must* exponentially decrease at higher indices. 15 | // 16 | // Hashes that don't exhibit this property may cause SplitHash to get stuck in an infinite loop. 17 | 18 | trait Hash { 19 | def hashAt(i: Int): Int 20 | } 21 | 22 | trait FiniteHash extends Hash { 23 | def size: Int 24 | def error = sys.error("hash out of bounds") 25 | } 26 | 27 | case class IntHash(h: Int) extends FiniteHash { 28 | override def hashCode = h 29 | def size = 1 30 | def hashAt(i: Int) = { 31 | if (i == 0) h 32 | else error 33 | } 34 | } 35 | 36 | case class LongHash(h1: Int, h2: Int) extends FiniteHash { 37 | override def hashCode = h1 38 | def size = 2 39 | def hashAt(i: Int) = { 40 | if (i == 0) h1 41 | else if (i == 1) h2 42 | else error 43 | } 44 | } 45 | 46 | case class ArrayHash(a: Array[Int]) extends FiniteHash { 47 | def size = a.size 48 | override def hashCode = a(0) 49 | def hashAt(i: Int) = { 50 | if (i < a.size) a(i) 51 | else error 52 | } 53 | } 54 | 55 | // Magic relative primes 56 | final val magic_p1 = 1664525 57 | final val magic_p2 = 22695477 58 | final val magic_p3 = 1103515245 59 | 60 | final def rotl(x: Long, b: Int): Long = (x << b) | (x >>> -b) 61 | 62 | final def siphash24(x1: Int, x2: Int): Int = { 63 | var v0 = 0x736f6d6570736575L 64 | var v1 = 0x646f72616e646f6dL 65 | var v2 = 0x6c7967656e657261L 66 | var v3 = 0x7465646279746573L 67 | 68 | def sipround() { 69 | v0 += v1 70 | v1 = rotl(v1, 13) ^ v0 71 | v0 = rotl(v0, 32) 72 | v2 += v3 73 | v3 = rotl(v3, 16) ^ v2 74 | v0 += v3 75 | v3 = rotl(v3, 21) ^ v0 76 | v2 += v1 77 | v1 = rotl(v1, 17) ^ v2 78 | v2 = rotl(v2, 32) 79 | } 80 | 81 | val m = rotl(x1,32) + x2 // combine the input ints into one long 82 | 83 | v3 ^= m 84 | sipround 85 | sipround 86 | v0 ^= m 87 | 88 | v2 ^= 0xff 89 | sipround 90 | sipround 91 | sipround 92 | sipround 93 | 94 | val r = v0 ^ v1 ^ v2 ^ v3 95 | 96 | (rotl(r,32) ^ r).toInt // munch the long into an int 97 | } 98 | } -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/splithash/Reference.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.splithash 2 | 3 | /** 4 | * Created by rapido on 31/01/16. 5 | */ 6 | object Reference { 7 | 8 | trait Ref[X] 9 | 10 | trait RefResolver { 11 | def resolve[X](r: Ref[X]): Ref[X] 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/splithash/SetHash.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.splithash 2 | 3 | // 4 | // Bagwell's Ideal Hash Tree with 'infinite' hashes (nibble based) 5 | // 6 | // Copyright 2016: Robbert van Dalen 7 | // 8 | 9 | object SetHash { 10 | import org.spread.core.splithash.Hashing._ 11 | 12 | trait SetHash[SH <: SetHash[SH]] extends Hashable { 13 | def put(e: Hashable): SH 14 | def hash(e: Hashable): FiniteHash // Returns the minimum unique FiniteHash to get back to the original Hashable 15 | def get(h: FiniteHash): Hashable 16 | // TODO: remove 17 | } 18 | 19 | case class HashNode(a: Array[Hashable], bits: Int) extends SetHash[HashNode] with Hash { 20 | def parts = a.clone 21 | def put(o: Hashable): HashNode = put(o,0) 22 | def put(o: Hashable ,nibble: Int): HashNode = { 23 | // TODO: Correct Type magic should prevent this 24 | if (o.isInstanceOf[HashNode]) { sys.error("NOT ALLOWED") } 25 | 26 | val i = getIndex(o,nibble) 27 | val ai = getArrayIndex(i) 28 | 29 | get2(ai) match { 30 | case n: HashNode => { 31 | // deeper 32 | val na = parts 33 | na(ai) = n.put(o,nibble + 1) 34 | HashNode(na,bits) 35 | } 36 | case null => { 37 | // new 38 | val aai = Integer.bitCount(bits & ((2 << i) - 1)) 39 | val sp = a.splitAt(aai) 40 | // TODO: Optimize 41 | HashNode(sp._1 ++ Array(o) ++ sp._2, bits | (1 << i)) 42 | } 43 | case x => { 44 | if (o != x) { 45 | // collision 46 | val na = parts 47 | na(ai) = HashNode(Array(),0).put(o,nibble + 1).put(a(ai),nibble + 1) 48 | HashNode(na,bits) 49 | } 50 | else this // the same object has been re-stored (hash-consing) 51 | } 52 | } 53 | } 54 | def get(o: Hashable, nibble: Int): Hashable = get2(getArrayIndex(getIndex(o,nibble))) match { 55 | case n: HashNode => n.get(o,nibble + 1) 56 | case x => { 57 | if (x == o) x 58 | else null 59 | } 60 | } 61 | 62 | def get(id: FiniteHash): Hashable = get(id,0) 63 | def get(id: FiniteHash, nibble: Int): Hashable = get2(getArrayIndex(getIndex(id,nibble))) match { 64 | case n: HashNode => n.get(id,nibble + 1) 65 | case x => x 66 | } 67 | 68 | def hash(o: Hashable) = { 69 | val r: List[Int] = id(o,0) 70 | if (r == null) null 71 | else { 72 | val s = r.size 73 | if (s == 1) IntHash(r.head) 74 | else if (s == 2) LongHash(r.head,r.tail.head) 75 | else ArrayHash(r.toArray) 76 | } 77 | } 78 | 79 | def id(o: Hashable, nibble: Int): List[Int] = get2(getArrayIndex(getIndex(o,nibble))) match { 80 | case n: HashNode => { 81 | if ((nibble & 3) == 0) { 82 | val r = n.id(o,nibble+1) 83 | if (r != null) { 84 | o.hash.hashAt(nibble >> 2) +: r 85 | } 86 | else null 87 | } 88 | else n.id(o,nibble + 1) 89 | } 90 | case x => { 91 | if (x == o) { 92 | if ((nibble & 3) == 0) List(o.hash.hashAt(nibble >> 2)) 93 | else List() 94 | } 95 | else null 96 | } 97 | } 98 | 99 | def getIndex(o: Hashable, nibble: Int): Int = (o.hash.hashAt(nibble >> 2) >> ((nibble & 3) * 4)) & 0xf 100 | def getIndex(o: FiniteHash, nibble: Int): Int = (o.hashAt(nibble >> 2) >> ((nibble & 3) * 4)) & 0xf 101 | def getArrayIndex(i: Int): Int ={ 102 | val b = bits & ((2 << i) - 1) 103 | if (((bits >> i) & 1) == 0) -1 104 | else Integer.bitCount(b) - 1 105 | } 106 | def get2(i: Int): Hashable = { 107 | if (i < 0) null 108 | else a(i) 109 | } 110 | 111 | import org.spread.core.splithash.Hashing._ 112 | 113 | // 64 bit hashes 114 | var lHash1: Long = 0 115 | override def hashCode = longHashCode.toInt 116 | 117 | def hash = this 118 | def longHashCode: Long = { 119 | if (lHash1 == 0) { 120 | var i = 0 121 | var s = a.length 122 | var h1 = magic_p2 123 | var h2 = magic_p3 124 | 125 | while (i < s) { 126 | h1 = siphash24(h1,a(i).hashCode) 127 | h2 = siphash24(a(i).hash.hashAt(1),h2) 128 | i = i + 1 129 | } 130 | lHash1 = (rotl(h1.toLong,32)) ^ h2 131 | } 132 | lHash1 133 | } 134 | 135 | override def hashAt(i: Int) = { 136 | if (i == 0) hashCode 137 | else if (i == 1) rotl(longHashCode,32).toInt 138 | else sys.error("DON'T KNOW WHAT TO DO YET") 139 | } 140 | 141 | override def toString = (a.toSeq,bits).toString 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /src/main/scala/org/spread/core/splithash/SplitHash.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.splithash 2 | 3 | import scala.language.{existentials, implicitConversions} 4 | 5 | object SplitHash { 6 | // 7 | // SplitHash is an immutable, uniquely represented Sequence ADS (Authenticated Data Structure). 8 | // It is based on a novel hashing scheme that was first introduced with SeqHash. 9 | // (See http://www.bu.edu/hic/files/2015/01/versum-ccs14.pdf). 10 | // 11 | // Like SeqHashes, SplitHashes can be concatenated in O(log(n)). 12 | // But SplitHash extends SeqHash by allowing Hashes to also be split in O(log(n)). 13 | // It also solves SeqHash's issue with repeating nodes by applying RLE (Run Length Encoding) compression. 14 | // To improve cache coherence and memory bandwidth, SplitHashes can be optionally chunked into n-ary trees. 15 | // 16 | // SplitHash is the first known History-Independent(HI) ADS that holds all these properties. 17 | // 18 | // Copyright 2016: Robbert van Dalen. 19 | // 20 | 21 | import org.spread.core.splithash.Hashing._ 22 | 23 | trait SplitHash[X, SH <: SplitHash[X,SH]] extends Hashable { 24 | def size: Int // O(1) 25 | def concat(other: SH): SH // O(log(size)) 26 | def split(at: Int): (SH, SH) // O(log(size)) 27 | 28 | def first: X // O(log(size)) 29 | def last: X // O(log(size)) 30 | def chunk: SH // O(unchunked) 31 | def splitParts: Array[SH] // O(1) 32 | 33 | // def map[Y](f: X => Y): Seq[Y] // O(size * f) 34 | // def fold[Y](s: Y, f: (X,Y) => Y): Y // O(size * f) 35 | } 36 | 37 | 38 | // A canonical tree SH(Split Hash)Node 39 | trait SHNode[X] extends SplitHash[X,SHNode[X]] with Hash { 40 | def size: Int 41 | def hash = this 42 | def concat(other: SHNode[X]) = SplitHash.concat(this,other) 43 | def split(at: Int) = { 44 | val l = leftSplit(this,at) 45 | val r = rightSplit(this,at) 46 | // nulls, assert((l.size + r.size) == size) 47 | (l,r) 48 | } 49 | def first: X 50 | def last: X 51 | 52 | // tree traversal 53 | def left: SHNode[X] 54 | def right: SHNode[X] 55 | def height: Int 56 | 57 | // Equality check that's based on content equality, rather than structure equality (==) 58 | // (used to detect node repetitions) 59 | def equalTo(other: SHNode[X]): Boolean = (this == other) 60 | 61 | // chunking 62 | def chunk: SHNode[X] 63 | def isChunked: Boolean 64 | def chunkHeight: Int 65 | 66 | def isMultipleOf(n: SHNode[X]): Boolean = mget(this).equalTo(mget(n)) 67 | def combine(n: SHNode[X]): SHNode[X] = { 68 | if (isMultipleOf(n)) RLENode(mget(this),msize(this) + msize(n)) 69 | else BinNode(this,n,(size + n.size)) 70 | } 71 | def combine2(n: SHNode[X]): SHNode[X] = { 72 | if (isMultipleOf(n)) RLENode(mget(this),msize(this) + msize(n)) 73 | else TempBinNode(this,n) 74 | } 75 | def parts: Array[Hashable] = splitParts.asInstanceOf[Array[Hashable]] 76 | def !(other: SHNode[X]): SHNode[X] = this.concat(other) 77 | } 78 | 79 | // Utility methods to detect and deal with consecutive, equal nodes 80 | private def mget[X](e1: SHNode[X]): SHNode[X] = e1 match { 81 | case RLENode(h,_) => h 82 | case _ => e1 83 | } 84 | private def msize[X](n: SHNode[X]): Int = n match { 85 | case RLENode(_,s) => s 86 | case _ => 1 87 | } 88 | 89 | // A Leaf node that holds a single element 90 | trait LeafNode[X] extends SHNode[X] { 91 | def left = null 92 | def right = null 93 | def height = 0 94 | def size = 1 95 | def value: X 96 | def first = value 97 | def last = value 98 | 99 | def chunk = this 100 | def isChunked = false 101 | def chunkHeight = 0 102 | 103 | def splitParts = Array() 104 | } 105 | 106 | case class IntNode(value: Int) extends LeafNode[Int] { 107 | override val hashCode = siphash24(value + magic_p1,value - magic_p2) 108 | def hashAt(index: Int) = { 109 | if (index == 0) hashCode 110 | else if (index == 1) siphash24(value + magic_p2, hashCode * magic_p1) 111 | else siphash24(hashCode * magic_p3, hashAt(index-1) - magic_p2) 112 | } 113 | override def toString = value.toString 114 | } 115 | 116 | trait BNode[X] extends SHNode[X] { 117 | override def equalTo(other: SHNode[X]): Boolean = { 118 | // BY DESIGN: fast hashcode in-equality check 119 | if (hashCode != other.hashCode) false 120 | // fast reference equality check (=true when references are equal) 121 | else if (this.eq(other)) true 122 | else ((left.equalTo(other.left)) && (right.equalTo(other.right))) 123 | } 124 | } 125 | 126 | var unlikely: Int = 0 127 | 128 | // A full binary node holds a hash (Int), size (Int), (chunk)height (Int) and its left and right sub-trees 129 | // There is some trickery to reduce the memory footprint and the lazy computation of hashes. 130 | case class BinNode[X](left: SHNode[X], right: SHNode[X], csize: Int) extends BNode[X] { 131 | 132 | def first = left.first 133 | def last = right.last 134 | def size = scala.math.abs(csize) 135 | def isChunked = csize < 0 // negative csize indicates that the node is chunked 136 | def heightE = 1 + (left.height max right.height) 137 | def chunkHeightE = 1 + (left.chunkHeight max right.chunkHeight) 138 | val heightEE = (heightE << 8) | chunkHeightE // encode both heights into one Int 139 | def height = heightEE >> 8 140 | def chunkHeight = heightEE & 0xff 141 | 142 | private var lHash = 0 // A lazy hash = 0 (trick borrowed from the Avail Programming Language) 143 | override def hashCode = { 144 | if (lHash == 0) lHash = siphash24(left.hashCode - magic_p2,right.hashCode + magic_p3) 145 | lHash // could be 0 again, but then we just recompute 0. 146 | } 147 | 148 | // We just keep munging bits recursively while traversing down the tree. 149 | // 150 | // The idea is that, when two unequal tree nodes collide in their hashCode, 151 | // then their left and right tree nodes should not collide with exponential 152 | // higher probability, given a certain hash length/depth. 153 | // 154 | // This is shaky cryptographically, but 'works' in 'practice'. 155 | // (That said, by using SipHash it *might* be cryptographically strong) 156 | // 157 | // TODO: We do need a cryptographic expert to have a stab at it. 158 | 159 | override def hashAt(index: Int) = { 160 | if (index == 0) hashCode 161 | else if (index == 1) (left.hashCode - right.hashCode) ^ hashCode // quick and dirty rehash 162 | else { 163 | // 64 bits or more are requested. This should normally not happen, unless 164 | // a client just wishes to calculate a bigger hash. 165 | unlikely = unlikely + 1 166 | val nindex = index / 2 167 | 168 | if (hashCode > 0) siphash24(left.hash.hashAt(nindex) - magic_p3,right.hash.hashAt(index - nindex) + (magic_p1 * hashCode)) 169 | else siphash24(right.hash.hashAt(nindex) - (magic_p3 * hashCode),left.hash.hashAt(index - nindex) + magic_p1) 170 | } 171 | } 172 | def chunk = { 173 | if (isChunked) this 174 | else { 175 | val l = left.chunk 176 | val r = right.chunk 177 | val nt = BinNode(l,r,-(l.size + r.size)) // remember, 'negative' size indicates that a node is chunked. 178 | 179 | if (nt.chunkHeight > 5) chunkTree(nt) // chunks of height = 6 (average size ~ 32) 180 | else nt 181 | } 182 | } 183 | def splitParts = Array(left,right) 184 | override def toString = left + " ! " + right 185 | } 186 | 187 | // A RLE(Run Length Encoded) node denotes the repetition of another node 188 | case class RLENode[X](node: SHNode[X], multiplicity: Int) extends SHNode[X] { 189 | def left = { 190 | if (multiplicity < 4) node 191 | else RLENode(node,multiplicity/2) 192 | } 193 | def right = { 194 | if (multiplicity < 3) node 195 | else RLENode(node,multiplicity - (multiplicity/2)) 196 | } 197 | def size = node.size * multiplicity 198 | def height = node.height 199 | def chunkHeight = node.chunkHeight 200 | override val hashCode = siphash24(node.hashCode + magic_p1 ,multiplicity - magic_p3) 201 | override def hashAt(index: Int) = { 202 | if (index > 0) siphash24(hashAt(index-1) + magic_p2 ,multiplicity - (magic_p3 * index)) 203 | else hashCode 204 | } 205 | def chunk = { 206 | if (!node.isChunked) RLENode(node.chunk,multiplicity) 207 | else this 208 | } 209 | def isChunked = node.isChunked 210 | def first = node.first 211 | def last = node.last 212 | def splitParts = Array(left,right) 213 | override def toString = multiplicity + ":" + node 214 | } 215 | 216 | // A temporary binary node that shouldn't be part of a canonical tree 217 | case class TempBinNode[X](left: SHNode[X], right: SHNode[X]) extends SHNode[X] { 218 | def error = sys.error("Internal inconsistency. Should not be called") 219 | val height = 1 + (left.height max right.height) // only the height is important 220 | def size = error 221 | def first = error 222 | def last = error 223 | def chunk = error 224 | def isChunked = error 225 | def chunkHeight = error 226 | def hashAt(i: Int) = error 227 | def splitParts = error 228 | } 229 | 230 | // Iterates sub-nodes in post-order, given a certain target height 231 | class LeftNodeIterator[X](var tree: SHNode[X], target_height: Int) extends Iterator[SHNode[X]] { 232 | var nstack: List[SHNode[X]] = List() 233 | 234 | def hasNext = (tree != null) || (!nstack.isEmpty) 235 | def next: SHNode[X] = { 236 | if (tree != null) { 237 | val t = tree 238 | tree = null 239 | leftMost(t) 240 | } 241 | else if (!nstack.isEmpty) { 242 | val head = nstack.head 243 | nstack = nstack.tail 244 | leftMost(head.right) 245 | } 246 | else null 247 | } 248 | def leftMost(node: SHNode[X]): SHNode[X] = { 249 | var leftNode = node 250 | while(leftNode.height > target_height) { 251 | nstack = leftNode +: nstack 252 | leftNode = leftNode.left 253 | } 254 | leftNode 255 | } 256 | } 257 | 258 | // Iterates sub-nodes in pre-order, given a certain target height 259 | class RightNodeIterator[X](var tree: SHNode[X], target_height: Int) extends Iterator[SHNode[X]] { 260 | var nstack: List[SHNode[X]] = List() 261 | 262 | def hasNext = (tree != null) || (!nstack.isEmpty) 263 | def next: SHNode[X] = { 264 | if (tree != null) { 265 | val t = tree 266 | tree = null 267 | rightMost(t) 268 | } 269 | else if (!nstack.isEmpty) { 270 | val head = nstack.head 271 | nstack = nstack.tail 272 | rightMost(head.left) 273 | } 274 | else null 275 | } 276 | def rightMost(node: SHNode[X]): SHNode[X] ={ 277 | var rightNode = node 278 | while (rightNode.height > target_height) { 279 | nstack = rightNode +: nstack 280 | rightNode = rightNode.right 281 | } 282 | rightNode 283 | } 284 | } 285 | 286 | // most used type 287 | type NArray[X] = Array[SHNode[X]] 288 | 289 | // Lazily consume the Iterator, whilst caching its elements at a certain index. 290 | class LazyIndexableIterator[X](it: Iterator[SHNode[X]]) { 291 | var values: NArray[X] = new Array(8) 292 | var size = 0 293 | 294 | def apply(i: Int): SHNode[X] = { 295 | if (i >= size) { 296 | if (i >= values.length) { 297 | // array exhausted, so extend it 298 | var vv: NArray[X] = new Array(values.length * 2) 299 | values.copyToArray(vv) 300 | values = vv 301 | } 302 | var ii = size 303 | while ((ii <= i) && (it.hasNext)) { 304 | values(ii) = it.next 305 | ii = ii + 1 306 | } 307 | size = ii 308 | assert(size <= values.length) 309 | } 310 | 311 | if (i >= size) null 312 | else values(i) 313 | } 314 | 315 | def firstReversed(s: Int) = { 316 | val firstR: NArray[X] = new Array(s) 317 | var i = 0 318 | while (i < s) { 319 | firstR(i) = values(s-i-1) 320 | i = i + 1 321 | } 322 | firstR 323 | } 324 | } 325 | 326 | // Build a temporary tree for left or right Fringe consumption only 327 | def to_tmp_tree[X](subtrees: NArray[X]): SHNode[X] = { 328 | if (subtrees.length == 0) null 329 | else { 330 | val s = subtrees.length 331 | var tree = subtrees(0) 332 | var i = 1 333 | while (i < s) { 334 | tree = tree.combine2(subtrees(i)) 335 | i = i + 1 336 | } 337 | tree 338 | } 339 | } 340 | 341 | def chunkTree[X](tree: SHNode[X]): SHNode[X] = { 342 | val (t,b) = chunkTree2(tree,List()) 343 | ChunkedNode(t.toArray.toSeq,b.toArray.reverse.toSeq,tree.hashCode,tree.size,tree.height) 344 | } 345 | 346 | // TODO: imperative version 347 | def unchunkTree[X](nodes: Seq[SHNode[X]], tree: Seq[Boolean], i1: Int, i2: Int): (Int,Int,SHNode[X]) = { 348 | if (tree(i2) == false) (i1+1,i2+1,nodes(i1)) 349 | else { 350 | val (ii1,ii2,l1) = unchunkTree(nodes,tree,i1,i2+1) 351 | val (iii1,iii2,r1) = unchunkTree(nodes,tree,ii1,ii2+1) 352 | 353 | (iii1,iii2,l1.combine(r1)) 354 | } 355 | } 356 | 357 | // Chunk a canonical tree into a ChunkedNode that contains all the sub-nodes with chunkHeight = 0. 358 | // TODO: imperative version 359 | def chunkTree2[X](node: SHNode[X], treeEnc: List[Boolean]): (List[SHNode[X]], List[Boolean]) = { 360 | if (node.chunkHeight == 0) (List(node),false +: treeEnc) 361 | else { 362 | val (lt,ll) = chunkTree2(node.left,true +: treeEnc) 363 | val (rt,lll) = chunkTree2(node.right,true +: ll) 364 | (lt ++ rt,lll) 365 | } 366 | } 367 | 368 | def unchunk[X](cn: ChunkedNode[X]): SHNode[X] = { 369 | val (_,_,result) = unchunkTree(cn.nodes,cn.tree,0,0) 370 | 371 | assert(result.size == cn.size) 372 | assert(result.hashCode == cn.hashCode) 373 | 374 | result 375 | } 376 | 377 | import java.lang.ref.WeakReference 378 | 379 | // A ChunkedNode that holds all the LeafNodes of the canonical tree it represents. 380 | // When lazily requested, it also caches the unchunked canonical tree in a WeakReference, 381 | // The unchunked version can be GC'ed anytime, as it can always be rebuild from the originating chunk. 382 | // ChunkedNode turns a binary tree into a n-ary tree, thus saving a lot of memory (bandwidth) 383 | 384 | case class ChunkedNode[X](nodes: Seq[SHNode[X]], tree: Seq[Boolean], h: Int, size: Int, height: Int) extends BNode[X] { 385 | // Note that we could also decide to weakly store the unchunked version into some kind of 386 | // FIFOCache to avoid GC trashing. For now we just rely on the GC to clean the WeakReference 387 | // before memory pressure becomes too high. 388 | var unchunked: WeakReference[_] = null 389 | 390 | def isChunked = true 391 | def getUnchunked: SHNode[X] = { 392 | if (unchunked != null) { 393 | val u = unchunked.get 394 | if (u != null) u.asInstanceOf[SHNode[X]] 395 | else getUnchunked2 396 | } 397 | else getUnchunked2 398 | } 399 | def getUnchunked2: SHNode[X] = { 400 | val uchunk = unchunk(this) 401 | unchunked = new WeakReference(uchunk) 402 | uchunk 403 | } 404 | 405 | def left = getUnchunked.left 406 | def right = getUnchunked.right 407 | def first = nodes(0).first 408 | def last = nodes(nodes.length-1).last 409 | override def hashCode = h 410 | def hashAt(i: Int) = { 411 | if (i == 0) hashCode 412 | else getUnchunked.hashAt(i) 413 | } 414 | def chunk = this 415 | def chunkHeight = 0 416 | def splitParts = nodes.toArray 417 | } 418 | 419 | final val Unknown: Byte = 0 420 | final val Merge: Byte = 1 421 | final val Fringe: Byte = 2 422 | 423 | // Determine the left and right Fringe of a canonical tree 424 | // 425 | // Notice that, unlike SeqHash, we don't need to hold on to the left and right Fringes, as we can always determine 426 | // the Fringes in O(log(n)) for each canonical tree at anytime. 427 | 428 | def leftFringe[X](tree: SHNode[X], height: Int): NArray[X] = { 429 | fringeVolatile2(new LazyIndexableIterator(new LeftNodeIterator(tree,height)),0).reverse 430 | } 431 | 432 | def rightFringe[X](tree: SHNode[X], height: Int): NArray[X] = { 433 | fringeVolatile2(new LazyIndexableIterator(new RightNodeIterator(tree,height)),1) 434 | } 435 | 436 | // Iteratively scan and deepen the fringe 'frontier' (width(average)=5) until the exact Fringe is found 437 | // We do this in order to minimally consume the LazyIndexableIterator 438 | def fringeVolatile2[X](elems: LazyIndexableIterator[X], direction: Byte): NArray[X] = { 439 | val width = 5 440 | var frontier = width 441 | var fringe: NArray[X] = null 442 | 443 | while (fringe == null) { 444 | val frontier1 = frontier+1 445 | 446 | val kinds: Array[Byte] = new Array(frontier1) // create reuseable arrays for two fringe determinations 447 | val hashes: Array[Int] = new Array(frontier1) 448 | 449 | val fringe1 = fringeVolatile3(elems,direction,frontier,kinds,hashes) 450 | resetKinds(kinds) 451 | // this is the tricky part as we *need* to check the edge difference: 452 | // nodes may be combined differently at edges that are off-by-one 453 | val fringe2 = fringeVolatile3(elems,direction,frontier1,kinds,hashes) // frontier+(off by one) 454 | if (fringe1 != fringe2) frontier = frontier + width // advance the frontier when there is a difference 455 | else fringe = elems.firstReversed(fringe1) 456 | } 457 | fringe 458 | } 459 | 460 | // Build the Fringe up to the frontier by lazily consuming the LazyIndexableIterator 461 | def fringeVolatile3[X](elems: LazyIndexableIterator[X], direction: Byte, frontier: Int, kind: Array[Byte], hashes: Array[Int]): Int ={ 462 | var min_frontier = frontier 463 | var other_direction = 1-direction 464 | var done = false 465 | var index = 1 466 | var bit_index = 0 467 | var int_index = 0 468 | kind(0) = Fringe 469 | 470 | while (!done) { 471 | done = true 472 | 473 | assert(min_frontier <= frontier) 474 | 475 | if (bit_index == 0) { 476 | // cache hashes of nodes that are unknown 477 | var ei = index 478 | var ii = bit_index >> 5 479 | while (ei < min_frontier && (elems(ei) != null)) { 480 | if (kind(ei) == Unknown) hashes(ei) = elems(ei).hash.hashAt(int_index) 481 | ei = ei + 1 482 | } 483 | int_index = int_index + 1 484 | } 485 | 486 | if (index < min_frontier) { 487 | val e1 = elems(index) 488 | if (e1 != null) { 489 | if ((kind(index) == Unknown) && (bitAt(hashes(index),bit_index) == direction)) { 490 | kind(index) = Fringe 491 | index = index + 1 492 | } 493 | if ((index < min_frontier) && (kind(index) == Unknown)) done = false 494 | } 495 | } 496 | 497 | if (!done) { 498 | var j = index 499 | var mf1 = min_frontier-1 500 | 501 | while (j < mf1) { 502 | if ((kind(j) == Unknown) && (kind(j+1) == Unknown)) { 503 | val e1 = elems(j) 504 | val e2 = elems(j+1) 505 | 506 | if ((e1 != null) && (e2 != null)) { 507 | if ((bitAt(hashes(j),bit_index) == other_direction) && (bitAt(hashes(j+1),bit_index) == direction)) { 508 | kind(j) = Merge 509 | kind(j+1) = Merge 510 | min_frontier = j 511 | } 512 | else done = false 513 | } 514 | } 515 | j = j + 1 516 | } 517 | } 518 | bit_index = (bit_index + 1) & 31 519 | } 520 | index 521 | } 522 | 523 | // wipe array for re-use 524 | def resetKinds(kind: Array[Byte]): Unit = { 525 | var i = 0 526 | var f = kind.size 527 | while (i < f) { 528 | kind(i) = Unknown 529 | i = i + 1 530 | } 531 | } 532 | 533 | final def bitAt(value: Int, index: Int): Int = ((value >>> (31-index)) & 1) 534 | 535 | case class LeftFringe[X](height: Int, top: NArray[X], fringes: List[NArray[X]]) 536 | case class RightFringe[X](height: Int, top: NArray[X], fringes: List[NArray[X]]) 537 | 538 | // Transform a canonical tree into right-catenable LeftFringe tree 539 | def transformLeft[X](t: SHNode[X]): LeftFringe[X] = { 540 | var tt = t 541 | var height = 0 542 | var leftFringes: List[NArray[X]] = List() 543 | var result: LeftFringe[X] = null 544 | 545 | while(result == null) { 546 | val lfringe = leftFringe(tt,height) 547 | val lfirst = first(height,lfringe.size,tt) 548 | if (lfirst != null) { 549 | leftFringes = lfringe +: leftFringes 550 | tt = lfirst 551 | height = height + 1 552 | } 553 | else result = LeftFringe(height,lfringe,leftFringes.reverse) 554 | } 555 | result 556 | } 557 | 558 | // Get the first n tree nodes at a certain height 559 | def first[X](hh: Int, n: Int, t: SHNode[X]): SHNode[X] = { 560 | val f = first2(hh,n,t,List()) 561 | val cm = compress(f._2.toArray.reverse) 562 | to_tmp_tree(cm) 563 | } 564 | 565 | // TODO: turn this into an iterative version 566 | def first2[X](hh: Int, s: Int, t: SHNode[X], st: List[SHNode[X]]): (Int,List[SHNode[X]]) = { 567 | if (t.height <= hh) (1,st) 568 | else { 569 | val (ls,nst) = first2(hh, s,t.left, st) 570 | if (ls < s) { 571 | val (rs,nst2) = first2(hh,s - ls,t.right, nst) 572 | (ls + rs,nst2) 573 | } 574 | else (ls,t.right +: nst) 575 | } 576 | } 577 | 578 | // Get the last n tree nodes at a certain height 579 | def last[X](hh: Int, n: Int, t: SHNode[X]): SHNode[X] = { 580 | val l = last2(hh,n,t,List()) 581 | val cm = compress(l._2.toArray[SHNode[X]]) 582 | to_tmp_tree(cm) 583 | } 584 | 585 | // TODO: turn this into an iterative version 586 | def last2[X](hh: Int, s: Int, t: SHNode[X], st: List[SHNode[X]]): (Int,List[SHNode[X]]) = { 587 | if (t.height <= hh) (1,st) 588 | else { 589 | val (rs,nst) = last2(hh, s,t.right, st) 590 | if (rs < s) { 591 | val (ls,nst2) = last2(hh,s - rs,t.left, nst) 592 | (ls + rs,nst2) 593 | } 594 | else (rs,t.left +: nst) 595 | } 596 | } 597 | 598 | // Transform a canonical tree into a left-catenable RightFringe tree 599 | def transformRight[X](t: SHNode[X]): RightFringe[X] = { 600 | var tt = t 601 | var height = 0 602 | var rightFringes: List[NArray[X]] = List() 603 | var result: RightFringe[X] = null 604 | 605 | while(result == null) { 606 | val rfringe = rightFringe(tt,height) 607 | val rlast = last(height,rfringe.size,tt) 608 | if (rlast != null) { 609 | rightFringes = rfringe +: rightFringes 610 | tt = rlast 611 | height = height + 1 612 | } 613 | else result = RightFringe(height,rfringe,rightFringes.reverse) 614 | } 615 | result 616 | } 617 | 618 | // Concatenate two canonical trees 619 | def concat[X](left: SHNode[X], right: SHNode[X]): SHNode[X] = { 620 | if (left == null) right 621 | else if (right == null) left 622 | else concat2(transformRight(left),transformLeft(right)) 623 | } 624 | 625 | // Compresses consecutive and equal (RLE) nodes into RLE nodes 626 | def compress[X](elems: NArray[X]): NArray[X] = { 627 | // first check whether we need to compress 628 | var i = 1 629 | var s = elems.size 630 | var compress = false 631 | while ((i < s) && !compress) { 632 | if (elems(i-1).isMultipleOf(elems(i))) compress = true 633 | i = i + 1 634 | } 635 | if (compress) compress2(elems) // yep, we do need to compress 636 | else elems 637 | } 638 | 639 | def compress2[X](elems: NArray[X]): NArray[X] = { 640 | if (elems.size == 0) elems 641 | else { 642 | val size = elems.size 643 | var stack: List[SHNode[X]] = List(elems(0)) 644 | var i = 1 645 | 646 | while (i < size) { 647 | val head = stack.head 648 | val elem = elems(i) 649 | if (head.isMultipleOf(elem)) { 650 | stack = head.combine(elem) +: stack.tail 651 | } 652 | else stack = elem +: stack 653 | i = i + 1 654 | } 655 | stack.toArray.reverse 656 | } 657 | } 658 | 659 | def doRound[X](elems: NArray[X]): NArray[X] = doRound2(compress(elems)) 660 | 661 | // Like SeqHash, but without the left and right fringe determination 662 | // Inspired by: https://github.com/jellevandenhooff/versum/blob/master/seqhash/hash.go 663 | 664 | def doRound2[X](elems: NArray[X]): NArray[X] = { 665 | val N = elems.length 666 | val kind: Array[Byte] = new Array(N) 667 | val hashes: Array[Int] = new Array(N) 668 | var done = false 669 | var merges = 0 670 | 671 | var bit_index = 0 672 | var int_index = 0 673 | val N1 = N - 1 674 | 675 | while (!done) { 676 | done = true 677 | 678 | if (bit_index == 0) { 679 | // cache hashes of nodes that are unknown 680 | var ei = 0 681 | while (ei < N) { 682 | if (kind(ei) == Unknown) hashes(ei) = elems(ei).hash.hashAt(int_index) 683 | ei = ei + 1 684 | } 685 | int_index = int_index + 1 686 | } 687 | 688 | var j = 0 689 | while (j < N1) { 690 | if ((kind(j) == Unknown) && (kind(j + 1) == Unknown)) { 691 | if ((bitAt(hashes(j),bit_index) == 1) && (bitAt(hashes(j + 1),bit_index) == 0)) { 692 | kind(j) = Merge 693 | kind(j+1) = Merge 694 | j = j + 1 695 | merges = merges + 1 696 | } 697 | else done = false 698 | } 699 | j = j + 1 700 | } 701 | bit_index = (bit_index + 1) & 31 702 | } 703 | 704 | var i = 0 705 | var ii = 0 706 | var result: NArray[X] = new Array(N-merges) 707 | 708 | while (i < N) { 709 | if (kind(i) == Unknown) result(ii) = elems(i) 710 | else { 711 | result(ii) = elems(i).combine(elems(i + 1)) 712 | i = i + 1 713 | } 714 | i = i + 1 715 | ii = ii + 1 716 | } 717 | 718 | result 719 | } 720 | 721 | final val eRight = RightFringe[Nothing](-1,Array(),List()) 722 | final val eLeft = LeftFringe[Nothing](-1,Array(),List()) 723 | 724 | final def emptyRight[X]: RightFringe[X] = eRight.asInstanceOf[RightFringe[X]] 725 | final def emptyLeft[X]: LeftFringe[X] = eLeft.asInstanceOf[LeftFringe[X]] 726 | 727 | // Concatenate the left and right Fringes into a canonical tree 728 | // TODO: optimize the concatenation of intermediate NArray[X] 729 | def concat2[X](left: RightFringe[X], right: LeftFringe[X]): SHNode[X] ={ 730 | var elems: NArray[X] = Array() 731 | var height = 0 732 | var done = false 733 | 734 | var lf = left.fringes 735 | var rf = right.fringes 736 | 737 | var lh = left.height 738 | var rh = right.height 739 | 740 | while (!done) { 741 | if (height < lh) { 742 | elems = lf.head ++ elems 743 | lf = lf.tail 744 | } 745 | else if (height == lh) elems = left.top ++ elems 746 | 747 | if (height < rh) { 748 | elems = elems ++ rf.head 749 | rf = rf.tail 750 | } 751 | else if (height == rh) elems = elems ++ right.top 752 | 753 | if ((height >= lh) && (height >= rh) && (elems.length == 1)) done = true 754 | else { 755 | elems = doRound(elems) 756 | height = height + 1 757 | } 758 | } 759 | 760 | elems(0) 761 | } 762 | 763 | // Split the left side of a canonical tree, 764 | // We first split the tree into a List of sub-trees. O(log(N)) 765 | // Then we combine those sub-trees into a non-canonical temporary tree. 766 | // This temporary tree is almost similar to the final tree, except that its right Fringe must be 'repaired'. 767 | // Most interestingly, the right Fringe can be repaired by just re-evaluating it. 768 | // 769 | // This is key to log(N) splitting. 770 | // 771 | def leftSplit[X](h: SHNode[X], size: Int): SHNode[X] = { 772 | if (size <= 0) null 773 | else if (size >= h.size) h 774 | else { 775 | val ls = leftSplit2(h,size) 776 | val cm = compress(ls.toArray[SHNode[X]]) 777 | concat2(transformRight(to_tmp_tree(cm)),emptyLeft) 778 | } 779 | } 780 | 781 | def leftSplit2[X](h: SHNode[X], pos: Int): List[SHNode[X]] = { 782 | if (pos == 0) List() 783 | else { 784 | val left = h.left 785 | if (pos >= left.size) left +: leftSplit2(h.right,pos-left.size) 786 | else leftSplit2(left,pos) 787 | } 788 | } 789 | 790 | // Split the right side of a canonical tree 791 | def rightSplit[X](h: SHNode[X], size: Int): SHNode[X] = { 792 | if (size <= 0) h 793 | else if (size >= h.size) null 794 | else { 795 | val rs = rightSplit2(h,h.size - size).toArray.reverse 796 | val cm = compress(rs) 797 | concat2(emptyRight,transformLeft(to_tmp_tree(cm))) 798 | } 799 | } 800 | 801 | def rightSplit2[X](h: SHNode[X], pos: Int): List[SHNode[X]] = { 802 | if (pos == 0) List() 803 | else { 804 | val right = h.right 805 | if (pos >= right.size) right +: rightSplit2(h.left,pos-right.size) 806 | else rightSplit2(right,pos) 807 | } 808 | } 809 | 810 | def emptySH[X]: SHNode[X] = null 811 | def intNode(i: Int): SHNode[Int] = IntNode(i) 812 | 813 | implicit def toIntNode(i: Int): SHNode[Int] = intNode(i) 814 | 815 | // ************************************* 816 | // EXPOSITION OF ALL THE NICE PROPERTIES 817 | // ************************************* 818 | final def main(args: Array[String]): Unit = { 819 | 820 | var s1 = emptySH[Int] 821 | var s2 = emptySH[Int] 822 | var s3 = emptySH[Int] 823 | 824 | var i = 0 825 | var n = 50000 826 | 827 | while (i < n) { 828 | var k1 = intNode(i) // forwards 829 | var k2 = intNode(n-i-1) // backwards 830 | var k3 = intNode(i % 63) // repetitions 831 | 832 | s1 = concat(s1,k1) 833 | s2 = concat(k2,s2) 834 | s3 = concat(s3,k3) 835 | 836 | i = i + 1 837 | if ((i % 1000) == 0) { 838 | println("concat i: " + i) 839 | 840 | s1 = s1.chunk 841 | s2 = s2.chunk 842 | s3 = s3.chunk 843 | } 844 | } 845 | 846 | if (s1 != s2) sys.error("Internal inconsistency") 847 | i = 1 848 | 849 | while (i < n) { 850 | // split into left and right 851 | val (ss1,ss2) = s1.split(i) 852 | // concatenate left and right -> should return original (unsplit) version 853 | val cc = ss1.concat(ss2).chunk 854 | // thus should be exactly the same. Equality check is also fast because the two trees share a lot of nodes 855 | if (cc != s1) sys.error("Internal inconsistency") 856 | 857 | // split repetition into left and right (note that repetitions are notoriously hard to get right!) 858 | val (rp1,rp2) = s3.split(i) 859 | val ccc = rp1.concat(rp2).chunk 860 | // Again, the concatenation of the parts should equal the original tree 861 | if (ccc != s3) sys.error("Internal inconsistency") 862 | 863 | i = i + 1 864 | if ((i % 1000) == 0) println("split i: " + i) 865 | } 866 | 867 | val b = n / 50 868 | n = n / b 869 | i = 0 870 | var ss = emptySH[Int] 871 | 872 | // Ultra fast block based concatenation 873 | while (i < n) { 874 | var ii = 0 875 | var block: NArray[Int] = new Array(b) 876 | var o = i * b 877 | 878 | println("fast block: " + o) 879 | 880 | while (ii < b) { 881 | block(ii) = intNode(o + ii) 882 | ii = ii + 1 883 | } 884 | 885 | // Build the block 886 | while (block.length > 1) { 887 | block = doRound(block) 888 | } 889 | 890 | val cs = block(0).chunk 891 | ss = concat(ss,cs) 892 | ss = ss.chunk 893 | 894 | i = i + 1 895 | } 896 | 897 | 898 | if (s1 != ss) sys.error("Internal inconsistency") 899 | 900 | // If we restrict ourselves to only concat, split, chunk and doRound, hash consumption shouldn't exceed 64 bits. 901 | // When unlikely does happen, you've either won the lottery or you are a cryptographer. 902 | 903 | println("unlikely > 64 bit consumption: " + unlikely) 904 | } 905 | } 906 | -------------------------------------------------------------------------------- /src/test/scala/org/spread/core/test/SeqSpecification.scala: -------------------------------------------------------------------------------- 1 | package org.spread.core.test 2 | 3 | import org.scalacheck.Arbitrary.arbitrary 4 | import org.scalacheck.Prop.{BooleanOperators, forAll} 5 | import org.scalacheck.{Arbitrary, Gen, Properties} 6 | import org.spread.core.annotation.Annotation._ 7 | import org.spread.core.constraint.Constraint._ 8 | import org.spread.core.sequence.AnnotatedTreeSequence._ 9 | import org.spread.core.sequence.RangedSequence._ 10 | import scala.language.{existentials, implicitConversions, postfixOps} 11 | import spire.implicits._ 12 | 13 | object SeqSpecification extends Properties("Seq") { 14 | type SSEQ = AnnTreeSeq[Long,Statistics[Long]] 15 | 16 | val factory = seqFactory[Long] 17 | implicit def arbitraryIntSeq: Arbitrary[SSEQ] = Arbitrary(longSeq) 18 | 19 | def longSeq: Gen[SSEQ] = for { 20 | l <- Gen.choose(0, 1000) 21 | a <- Gen.listOfN(l,arbitrary[Int]) 22 | } yield factory.createSeq(a.toArray.map(_.toLong)) 23 | 24 | property("appendSize") = forAll { (p1: SSEQ,p2: SSEQ) => 25 | p1.append(p2).size == p2.append(p1).size 26 | } 27 | 28 | property("split") = forAll { (p: SSEQ,index: Long) => 29 | val i = (index % (p.size+1))/2 30 | val (l1,r1) = p.split(i) // random split 31 | l1.append(r1).equalTo(p) 32 | } 33 | 34 | property("statsBounds") = forAll { (p1: SSEQ,p2: SSEQ) => 35 | val c = p1.append(p2) 36 | val ca = c.annotation 37 | val a1 = c.annotation 38 | 39 | ((p1.size > 0) ==> (a1.first == ca.first)) && 40 | ((p1.size > 0) ==> (a1.last == ca.last)) 41 | } 42 | 43 | property("statsMinMax") = forAll { (p1: SSEQ,p2: SSEQ) => 44 | if ((p1.size > 0) && (p2.size > 0)) { 45 | val c = p1.append(p2) 46 | 47 | val ca = c.annotation 48 | val a1 = p1.annotation 49 | val a2 = p2.annotation 50 | 51 | val l = min(a1.lowerBound, a2.lowerBound) 52 | val h = max(a1.upperBound, a2.upperBound) 53 | (l == ca.lowerBound) && (h == ca.upperBound) 54 | } 55 | else true 56 | } 57 | 58 | def min(l1: Long, l2: Long) = if (l1 < l2) l1 ; else l2 59 | def max(l1: Long, l2: Long) = if (l1 > l2) l1 ; else l2 60 | 61 | property("multiSet") = forAll { (p: SSEQ) => 62 | 63 | val s = p.sort 64 | val u = s union s 65 | val d = s difference u 66 | 67 | s.equalTo(d) 68 | } 69 | } --------------------------------------------------------------------------------