├── LICENSE.md ├── README.md ├── composer.json ├── ldba.php └── src ├── BuddySystem.php └── LinearHashingDb.php /LICENSE.md: -------------------------------------------------------------------------------- 1 | # MIT License 2 | 3 | Copyright (c) 2018 Simplito 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and 6 | associated documentation files (the "Software"), to deal in the Software without restriction, 7 | including without limitation the rights to use, copy, modify, merge, publish, distribute, 8 | sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all copies or substantial 12 | portions of the Software. 13 | 14 | **THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT 15 | NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 16 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 17 | DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT 18 | OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.** 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # LDBA - a fast, pure PHP, key-value database. 3 | 4 | ## Information 5 | 6 | LDBA is a high-performance, low-memory-footprint, single-file embedded database for key/value storage and retrieval written in pure PHP. 7 | 8 | It is inspired by Erlang's Dets and Berkeley DB sofware and includes implementation of [extended linear hashing](https://en.wikipedia.org/wiki/Linear_hashing) for fast key/value access and implementation of a fast [buddy storage allocator](https://en.wikipedia.org/wiki/Buddy_memory_allocation) for file space management. 9 | 10 | LDBA supports insertion and deletion of records and lookup by exact key match only. Applications may iterate over all records stored in a database, but 11 | the order in which they are returned is undefined. Fault tolerance is achieved by automatic crash recovery thanks to transactional style writes. The size of LDBA files cannot exceed 2GB. 12 | 13 | LDBA provides functions compatible with [php_dba](http://php.net/manual/en/book.dba.php) (Database Abstraction Layer) for easy adoption in existing software. 14 | 15 | Requirements: PHP 5.4+ 16 | 17 | The library is being used, for example, as base storage interface in the [PrivMX WebMail](https://privmx.com) software. 18 | 19 | 20 | ## Instalation 21 | 22 | Run the following command to install the lib via composer: 23 | ``` 24 | composer require simplito/ldba-php 25 | ``` 26 | 27 | 28 | ## An example 29 | 30 | ```php 31 | $dbh = ldba_open("test.ldb", "c"); 32 | if (ldba_exists("counter", $dbh)) { 33 | $counter = intval(ldba_fetch("counter", $dbh)); 34 | } else { 35 | $counter = 0; 36 | } 37 | ldba_replace("counter", $counter + 1); 38 | ldba_close($dbh); 39 | ``` 40 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "simplito/ldba-php", 3 | "description": "LDBA is a high-performance, low-memory-footprint, single-file embedded database for key/value storage", 4 | "license": "MIT", 5 | "authors": [ 6 | { 7 | "name": "Simplito Team", 8 | "email": "s.smyczynski@simplito.com", 9 | "homepage": "https://simplito.com" 10 | } 11 | ], 12 | "require": { 13 | "simplito/pson-php": "~1.0.0" 14 | }, 15 | "autoload": { 16 | "psr-4": { 17 | "simplito\\": "src" 18 | }, 19 | "files": ["ldba.php"] 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /ldba.php: -------------------------------------------------------------------------------- 1 | open($fname, $mode)) 8 | return $ldb; 9 | return false; 10 | } 11 | 12 | function ldba_close($dbh) { 13 | $dbh->close(); 14 | } 15 | 16 | function ldba_insert($key, $value, $dbh) { 17 | return $dbh->insert($key, $value); 18 | } 19 | 20 | function ldba_replace($key, $value, $dbh) { 21 | return $dbh->replace($key, $value); 22 | } 23 | 24 | function ldba_fetch($key, $dbh) { 25 | return $dbh->fetch($key); 26 | } 27 | 28 | function ldba_exists($key, $dbh) { 29 | return $dbh->exists($key); 30 | } 31 | 32 | function ldba_delete($key, $dbh) { 33 | return $dbh->delete($key); 34 | } 35 | 36 | function ldba_firstkey($dbh) { 37 | return $dbh->firstkey(); 38 | } 39 | 40 | function ldba_nextkey($dbh) { 41 | return $dbh->nextkey(); 42 | } 43 | 44 | function ldba_count($dbh) { 45 | return $dbh->count(); 46 | } 47 | 48 | -------------------------------------------------------------------------------- /src/BuddySystem.php: -------------------------------------------------------------------------------- 1 | base = $base; 12 | $this->logLevel = $logLevel; 13 | } 14 | 15 | public function count() { 16 | return count($this->offsets); 17 | } 18 | 19 | public function exists($offset) { 20 | $offset = ($offset - $this->base) >> $this->logLevel; 21 | return isset($this->offsets[$offset]); 22 | } 23 | 24 | public function insert($offset) { 25 | $offset = ($offset - $this->base) >> $this->logLevel; 26 | $this->offsets[$offset] = 1; 27 | $this->dirty = true; 28 | } 29 | 30 | public function delete($offset) { 31 | $offset = ($offset - $this->base) >> $this->logLevel; 32 | unset($this->offsets[$offset]); 33 | } 34 | 35 | public function takeFirst() { 36 | if ($this->dirty) { 37 | ksort($this->offsets); 38 | $this->dirty = false; 39 | } 40 | reset($this->offsets); 41 | $offset = key($this->offsets); 42 | unset($this->offsets[$offset]); 43 | $offset = ($offset << $this->logLevel) + $this->base; 44 | return $offset; 45 | } 46 | 47 | public function __toString() { 48 | $str = "logLevel: {$this->logLevel}, size: " . (1 << $this->logLevel) . ", base: " . $this->base . ", empty at (" . count($this->offsets) . "): "; 49 | foreach($this->offsets as $k => $v) { 50 | $str .= " " . ($k << $this->logLevel); 51 | } 52 | return $str; 53 | } 54 | 55 | public function getOffsets() { 56 | if ($this->dirty) { 57 | ksort($this->offsets); 58 | $this->dirty = false; 59 | } 60 | $offsets = array_keys($this->offsets); 61 | foreach($offsets as $i => $offset) { 62 | $offsets[$i] = ($offset << $this->logLevel) + $this->base; 63 | } 64 | return $offsets; 65 | } 66 | 67 | public function encode(\PSON\ByteBuffer $buffer) { 68 | $offsets = $this->getOffsets(); 69 | $buffer->writeUint8($this->logLevel); 70 | $buffer->writeUint32(count($offsets)); 71 | foreach($offsets as $offset) { 72 | $buffer->writeUint32($offset); 73 | } 74 | return $buffer; 75 | } 76 | 77 | public static function decode($base, \PSON\ByteBuffer $buffer) { 78 | $logLevel = $buffer->readUint8(); 79 | $level = new BuddyLevel($base, $logLevel); 80 | $count = $buffer->readUint32(); 81 | for($i = 0; $i < $count; ++$i) { 82 | $offset = $buffer->readUint32(); 83 | $level->insert($offset); 84 | } 85 | return $level; 86 | } 87 | } 88 | 89 | class BuddySystem { 90 | public $levels = array(); 91 | public $base = 0; 92 | 93 | function __construct($base) { 94 | $this->base = $base; 95 | for($i = 0; $i < 32; ++$i) { 96 | $this->levels[$i] = new BuddyLevel($base, $i); 97 | } 98 | $this->levels[31]->insert($base); 99 | } 100 | 101 | public static function logLevel($size) { 102 | // log10 is used intentionally, with just log the logLevel(1 << 29) returns 30 103 | return (int)ceil(log10($size)/log10(2)); 104 | } 105 | 106 | public function buddy($offset, $size) { 107 | $size = 1 << self::logLevel($size); 108 | $offset = $offset - $this->base; 109 | if ($offset & $size) { 110 | $offset -= $size; 111 | } else { 112 | $offset += $size; 113 | } 114 | return $offset + $this->base; 115 | } 116 | 117 | public function allocExplicit($offset, $size) { 118 | if ($offset < 0) { 119 | throw new \Exception("wrong offset argument $offset"); 120 | } 121 | if ($size < 1) { 122 | throw new \Exception("wrong size argument $size"); 123 | } 124 | $logLevel = self::logLevel($size); 125 | $level = $this->levels[$logLevel]; 126 | while (true) { 127 | if ($level->exists($offset)) { 128 | $level->delete($offset); 129 | break; 130 | } 131 | $levelSize = 1 << $level->logLevel; 132 | $buddy = $this->buddy($offset, $levelSize); 133 | $level->insert($buddy); 134 | $offset = min($offset, $buddy); 135 | 136 | $logLevel++; 137 | if ($logLevel > 31) { 138 | throw new \Exception("cannot alloc $size bytes!"); 139 | } 140 | $level = $this->levels[$logLevel]; 141 | } 142 | } 143 | 144 | public function alloc($size) { 145 | if ($size < 1) { 146 | throw new \Exception("wrong argument $size"); 147 | } 148 | $logLevel = self::logLevel($size); 149 | $level = $this->levels[$logLevel]; 150 | $stack = array(); 151 | while($level->count() == 0) { 152 | array_push($stack, $level); 153 | $logLevel++; 154 | if ($logLevel > 31) { 155 | throw new \Exception("cannot alloc $size bytes!"); 156 | } 157 | $level = $this->levels[$logLevel]; 158 | } 159 | $offset = $level->takeFirst(); 160 | while(count($stack) > 0) { 161 | $level = array_pop($stack); 162 | $level->insert($offset + (1 << $level->logLevel)); 163 | } 164 | return $offset; 165 | } 166 | 167 | public function free($offset, $size) { 168 | if ($offset < $this->base || $size < 1) 169 | throw new \Exception("wrong argument"); 170 | 171 | $logLevel = self::logLevel($size); 172 | while($logLevel < 32) { 173 | $level = $this->levels[$logLevel]; 174 | $buddy = $this->buddy($offset, $size); 175 | if ($level->exists($buddy)) { 176 | $level->delete($buddy); 177 | $logLevel++; 178 | $offset = min($offset, $buddy); 179 | $size = 1 << $logLevel; 180 | } else { 181 | $level->insert($offset); 182 | break; 183 | } 184 | } 185 | } 186 | 187 | public function __toStringOld() { 188 | $free = array(); 189 | $str = "base: {$this->base}\nfree:"; 190 | foreach($this->levels as $level) { 191 | $size = 1 << $level->logLevel; 192 | foreach($level->getOffsets() as $offset) { 193 | $free[$offset] = $offset + $size; 194 | } 195 | } 196 | ksort($free); 197 | foreach($free as $o => $s) { 198 | $str .= " <$o,$s>"; 199 | } 200 | 201 | return $str; 202 | } 203 | 204 | public function __toString() { 205 | $free = array(); 206 | $str = "base: {$this->base}, levels: \n"; 207 | foreach($this->levels as $level) { 208 | $str .= $level . "\n"; 209 | } 210 | return $str; 211 | } 212 | 213 | public function encode() { 214 | $buffer = new \PSON\ByteBuffer(); 215 | $buffer->writeUint32($this->base); 216 | for($i = 0; $i < 32; ++$i) { 217 | $this->levels[$i]->encode($buffer); 218 | } 219 | return $buffer->flip()->toBinary(); 220 | } 221 | 222 | public static function decode($str) { 223 | $buffer = \PSON\ByteBuffer::wrap($str); 224 | $base = $buffer->readUint32(); 225 | $levels = array(); 226 | for($i = 0; $i < 32; ++$i) { 227 | $levels[$i] = BuddyLevel::decode($base, $buffer); 228 | } 229 | $result = new BuddySystem($base); 230 | $result->levels = &$levels; 231 | return $result; 232 | } 233 | } 234 | -------------------------------------------------------------------------------- /src/LinearHashingDb.php: -------------------------------------------------------------------------------- 1 | reset(); 31 | } 32 | 33 | public function __destruct() { 34 | $this->close(); 35 | } 36 | 37 | public function open($fname, $mode) { 38 | $this->close(); 39 | 40 | $rmode = $mode[0]; 41 | $ltype = strchr($mode, "l") ? "l" : "d"; 42 | $wait = strchr($mode, "t") ? false : true; 43 | 44 | if ($rmode == "r") { 45 | $fmode = "rb"; 46 | $lmode = LOCK_SH; 47 | } elseif ($rmode == "w") { 48 | $fmode = "r+b"; 49 | $lmode = LOCK_EX; 50 | } elseif ($rmode == "c") { 51 | $fmode = "c+b"; 52 | $lmode = LOCK_EX; 53 | } elseif ($rmode == "n") { 54 | $fmode = "w+b"; 55 | $lmode = LOCK_EX; 56 | } else { 57 | throw new \Exception("invalid mode $mode"); 58 | } 59 | 60 | if (!$wait) { 61 | $lmode |= LOCK_NB; 62 | } 63 | 64 | if ($ltype == 'l') { 65 | $lfd = fopen("$fname.lck", "c+"); 66 | if (!$lfd) { 67 | return false; 68 | } 69 | if (!flock($lfd, $lmode)) { 70 | fclose($lfd); 71 | return false; 72 | } 73 | $this->lfd = $lfd; 74 | } 75 | 76 | $fd = fopen($fname, $fmode); 77 | if (!$fd) { 78 | return false; 79 | } 80 | 81 | if ($ltype == 'd' && !flock($fd, $lmode)) { 82 | fclose($fd); 83 | return false; 84 | } 85 | $this->fd = $fd; 86 | 87 | set_file_buffer($this->fd, 0); 88 | 89 | if (fseek($this->fd, 0, SEEK_END) != 0) { 90 | throw new \Exception("file seek error"); 91 | } 92 | $fsize = ftell($this->fd); 93 | if ($fsize == 0) { 94 | $this->initializeFile(); 95 | } else { 96 | $this->readHeader(); 97 | $this->readPageDirectory(); 98 | if ($rmode != "r") { 99 | if ($this->dirtyBuddySystem) { 100 | $this->restoreBuddySystem(); 101 | } 102 | else { 103 | $this->readBuddySystem(); 104 | } 105 | } 106 | } 107 | return true; 108 | } 109 | 110 | public function close() { 111 | if ($this->fd == null) 112 | return; 113 | 114 | $this->flush(); 115 | fclose($this->fd); 116 | 117 | $this->fd = null; 118 | if ($this->lfd != null) { 119 | fclose($this->lfd); 120 | $this->lfd = null; 121 | } 122 | $this->reset(); 123 | } 124 | 125 | public function insert($key, $value) { 126 | if ($this->fd == null) { 127 | throw new \Exception("invalid operation"); 128 | } 129 | if ($this->exists($key)) { 130 | return false; 131 | } 132 | return $this->replace($key, $value); 133 | } 134 | 135 | public function replace($key, $value) { 136 | if ($this->fd == null) { 137 | throw new \Exception("invalid operation"); 138 | } 139 | $slotNo = $this->slotNo($key); 140 | $slot = $this->getSlot($slotNo); 141 | $value = $this->prepareSlotValue($value); 142 | 143 | $replace = $slot->exists($key); 144 | if ($replace) { 145 | $oldValue = $slot->fetch($key); 146 | if ($value == $oldValue) 147 | return; 148 | $this->freeSlotValue($oldValue); 149 | } 150 | 151 | $slot->insert($key, $value); 152 | $this->changedSlots[$slotNo] = $slot; 153 | 154 | if ($replace && ord($oldValue[0]) == 1) { 155 | // the space of old big value is released in buddy system 156 | // so to make this replace more reliable we have to flush immediately 157 | $this->flush(); 158 | } 159 | 160 | if (!$replace) { 161 | $this->o++; 162 | if ($this->o > $this->n) { 163 | $this->grow(); 164 | $this->flush(); 165 | } 166 | } 167 | if (count($this->changedSlots) > 256) { 168 | $this->flush(); 169 | } 170 | } 171 | 172 | public function fetch($key) { 173 | if ($this->fd == null) { 174 | throw new \Exception("invalid operation"); 175 | } 176 | $slotNo = $this->slotNo($key); 177 | $slot = $this->getSlot($slotNo); 178 | $value = $slot->fetch($key); 179 | $value = $this->getSlotValue($value); 180 | $this->cleanSlotCache(); 181 | return $value; 182 | } 183 | 184 | public function exists($key) { 185 | if ($this->fd == null) { 186 | throw new \Exception("invalid operation"); 187 | } 188 | $slotNo = $this->slotNo($key); 189 | $slot = $this->getSlot($slotNo); 190 | $exists = $slot->exists($key); 191 | $this->cleanSlotCache(); 192 | return $exists; 193 | } 194 | 195 | public function delete($key) { 196 | if ($this->fd == null) { 197 | throw new \Exception("invalid operation"); 198 | } 199 | $slotNo = $this->slotNo($key); 200 | $slot = $this->getSlot($slotNo); 201 | if (!$slot->exists($key)) { 202 | return; 203 | } 204 | $value = $slot->fetch($key); 205 | $this->freeSlotValue($value); 206 | $slot->delete($key); 207 | $this->o--; 208 | $this->changedSlots[$slotNo] = $slot; 209 | if (count($this->changedSlots) > 256) { 210 | $this->flush(); 211 | } 212 | } 213 | 214 | public function firstkey() { 215 | if ($this->fd == null) { 216 | throw new \Exception("invalid operation"); 217 | } 218 | $this->iterator_slot = -1; 219 | $this->iterator_keys = array(); 220 | return $this->nextkey(); 221 | } 222 | 223 | public function nextkey() { 224 | if ($this->fd == null) { 225 | throw new \Exception("invalid operation"); 226 | } 227 | if ($this->iterator_slot >= $this->n) { 228 | return false; 229 | } 230 | $key = next($this->iterator_keys); 231 | if ($key !== false) { 232 | return $key; 233 | } 234 | 235 | ++$this->iterator_slot; 236 | if ($this->iterator_slot >= $this->n) { 237 | return false; 238 | } 239 | while($this->iterator_slot < $this->n) { 240 | $this->iterator_keys = $this->getSlot($this->iterator_slot)->getKeys(); 241 | if (count($this->iterator_keys) > 0) { 242 | $this->cleanSlotCache(); 243 | return current($this->iterator_keys); 244 | } 245 | ++$this->iterator_slot; 246 | } 247 | return false; 248 | } 249 | 250 | public function count() { 251 | if ($this->fd == null) { 252 | throw new \Exception("invalid operation"); 253 | } 254 | return $this->o; 255 | } 256 | 257 | private function reset() { 258 | $this->s = 0; 259 | $this->n = 256; 260 | $this->m = 256; 261 | $this->o = 0; 262 | $this->fsize = 0; 263 | $this->dirtyBuddySystem = false; 264 | 265 | $this->buddySystem = null; 266 | $this->pageTableDirectory = null; 267 | $this->pageTableCache = null; 268 | $this->pageCache = array(); 269 | $this->slotCache = array(); 270 | $this->slotLRU = array(); 271 | $this->changedSlots = array(); 272 | $this->fd = null; 273 | $this->lfd = null; 274 | $this->iterator_slot = -1; 275 | $this->iterator_keys = array(); 276 | } 277 | 278 | private function writeHeader() { 279 | $buf = new \PSON\ByteBuffer(256); 280 | $buf->writeBytes("LHDB"); 281 | $buf->writeUint32(1); 282 | $buf->writeUint8($this->dirtyBuddySystem ? 1 : 0); 283 | $buf->writeUint32($this->base); 284 | $buf->writeUint32($this->fsize); 285 | $buf->writeUint32($this->n); 286 | $buf->writeUint32($this->m); 287 | $buf->writeUint32($this->s); 288 | $buf->writeUint32($this->o); 289 | $header = $buf->flip()->toBinary(); 290 | $this->pwrite(0, $header); 291 | } 292 | 293 | private function readHeader() { 294 | $value = $this->pread(0, 256); 295 | $buf = \PSON\ByteBuffer::wrap($value); 296 | $magic = $buf->readBytes(4); 297 | if ($magic != "LHDB") { 298 | throw new \Exception("No LHBA file"); 299 | } 300 | $ver = $buf->readUint32(); 301 | if ($ver != 1) { 302 | throw new \Exception("Unsupported LHBA version " . $ver); 303 | } 304 | $this->dirtyBuddySystem = $buf->readUint8() !== 0; 305 | $this->base = $buf->readUint32(); 306 | $this->fsize = $buf->readUint32(); 307 | $this->n = $buf->readUint32(); 308 | $this->m = $buf->readUint32(); 309 | $this->s = $buf->readUint32(); 310 | $this->o = $buf->readUint32(); 311 | } 312 | 313 | private function allocPageTable($pageTableNo) { 314 | $offset = $this->buddySystem->alloc(512*4); 315 | $pageTable = new SplFixedArray(512); 316 | for($i = 0; $i < 512; ++$i) { 317 | $pageTable[$i] = 0; 318 | } 319 | $this->pageTableCache[$pageTableNo] = &$pageTable; 320 | $this->pageTableDirectory[$pageTableNo] = $offset; 321 | 322 | $this->pwrite($offset, str_repeat(chr(0),512*4)); 323 | $this->pwrite(256 + $pageTableNo * 4, pack("N", $offset)); 324 | 325 | return $offset; 326 | } 327 | 328 | public function setBuddySystemDirty() { 329 | if (!$this->dirtyBuddySystem) { 330 | $this->dirtyBuddySystem = true; 331 | $this->pwrite(8, chr(1)); 332 | } 333 | } 334 | 335 | private function unsetBuddySystemDirty() { 336 | if ($this->dirtyBuddySystem) { 337 | $this->dirtyBuddySystem = false; 338 | $this->pwrite(8, chr(0)); 339 | } 340 | } 341 | 342 | private function allocPage($pageNo) { 343 | $pageTableNo = $pageNo >> 9; 344 | $pageTableOff = $pageNo & 511; 345 | 346 | if ($this->pageTableDirectory[$pageTableNo] == 0) { 347 | $this->allocPageTable($pageTableNo); 348 | } 349 | 350 | $offset = $this->buddySystem->alloc(256*8); 351 | $page = new SplFixedArray(512); 352 | for($i = 0; $i < 512; ++$i) { 353 | $page[$i] = 0; 354 | } 355 | $this->pwrite($offset, str_repeat(chr(0), 256*8)); 356 | 357 | $this->pageCache[$pageNo] = $page; 358 | 359 | $ptoffset = $this->pageTableDirectory[$pageTableNo] + $pageTableOff * 4; 360 | $this->pwrite($ptoffset, pack("N", $offset)); 361 | 362 | $pageTable = &$this->getPageTable($pageTableNo); 363 | $pageTable[$pageTableOff] = $offset; 364 | 365 | return $offset; 366 | } 367 | 368 | private function readPageDirectory() { 369 | $str = $this->pread(256, 256 * 4); 370 | $buf = \PSON\ByteBuffer::wrap($str); 371 | $this->pageTableDirectory = new SplFixedArray(256); 372 | for($i = 0; $i < 256; ++$i) { 373 | $this->pageTableDirectory[$i] = $buf->readUint32(); 374 | } 375 | assert($this->pageTableDirectory[0] == 256 + 256 * 4); 376 | } 377 | 378 | private function &getPageTable($pageTableNo) { 379 | if (isset($this->pageTableCache[$pageTableNo])) { 380 | return $this->pageTableCache[$pageTableNo]; 381 | } 382 | $offset = $this->pageTableDirectory[$pageTableNo]; 383 | if ($offset <= 0 || $offset >= $this->fsize) { 384 | throw new \Exception("invalid page table data"); 385 | } 386 | $str = $this->pread($offset, 512 * 4); 387 | $buf = \PSON\ByteBuffer::wrap($str); 388 | $pageTable = new SplFixedArray(512); 389 | for($i = 0; $i < 512; ++$i) { 390 | $pageTable[$i] = $buf->readUint32(); 391 | } 392 | $this->pageTableCache[$pageTableNo] = $pageTable; 393 | return $this->pageTableCache[$pageTableNo]; 394 | } 395 | 396 | private function &getPage($pageNo) { 397 | if (isset($this->pageCache[$pageNo])) { 398 | return $this->pageCache[$pageNo]; 399 | } 400 | $pageTableNo = $pageNo >> 9; 401 | $pageTableOff = $pageNo & 511; 402 | $pageTable = &$this->getPageTable($pageTableNo); 403 | $offset = $pageTable[$pageTableOff]; 404 | if ($offset <= 0 || $offset >= $this->fsize) { 405 | throw new \Exception("invalid page data"); 406 | } 407 | $str = $this->pread($offset, 256*2*4); 408 | $buf = \PSON\ByteBuffer::wrap($str); 409 | $page = new SplFixedArray(512); 410 | for($i = 0; $i < 256; ++$i) { 411 | $page[2 * $i] = $buf->readUint32(); 412 | $page[2 * $i + 1] = $buf->readUint32(); 413 | } 414 | $this->pageCache[$pageNo] = $page; 415 | return $this->pageCache[$pageNo]; 416 | } 417 | 418 | private function cleanSlotCache() { 419 | $slotsCnt = count($this->slotCache); 420 | if ($slotsCnt > 1024) { 421 | if ($slotsCnt != count($this->slotLRU)) { 422 | throw new \Exception("unexpected error"); 423 | } 424 | reset($this->slotLRU); 425 | while($slotsCnt > 256) { 426 | $key = key($this->slotLRU); 427 | if (!isset($this->changedSlots[$key])) { 428 | unset($this->slotCache[$key]); 429 | unset($this->slotLRU[$key]); 430 | --$slotsCnt; 431 | continue; 432 | } 433 | if (next($this->slotLRU) === false) 434 | break; 435 | } 436 | } 437 | } 438 | 439 | private function getSlot($slotNo) { 440 | unset($this->slotLRU[$slotNo]); 441 | $this->slotLRU[$slotNo] = 1; 442 | 443 | if (isset($this->slotCache[$slotNo])) { 444 | return $this->slotCache[$slotNo]; 445 | } 446 | 447 | $pageNo = $slotNo >> 8; 448 | $pageOff = $slotNo & 255; 449 | $page = &$this->getPage($pageNo); 450 | $voffset = $page[2 * $pageOff]; 451 | $vsize = $page[2 * $pageOff + 1]; 452 | if ($vsize == 0) { 453 | $slot = new Slot(); 454 | } else { 455 | $value = $this->pread($voffset, $vsize); 456 | $slot = Slot::decode($value); 457 | } 458 | $this->slotCache[$slotNo] = $slot; 459 | return $slot; 460 | } 461 | 462 | private function writeUint32Array($array) { 463 | $buf = new \PSON\ByteBuffer(count($array) * 4); 464 | foreach($array as $value) { 465 | $buf->writeUint32($value); 466 | } 467 | $data = $buf->flip()->toBinary(); 468 | if (($rc = fwrite($this->fd, $data)) != strlen($data)) { 469 | throw new \Exception("write error"); 470 | } 471 | } 472 | 473 | private function initializeFile() { 474 | ftruncate($this->fd, $this->base); 475 | fseek($this->fd, 256); 476 | 477 | // page directory table starts at offset 256 478 | $this->pageTableDirectory = new SplFixedArray(256); 479 | for($i = 0; $i < 256; ++$i) { 480 | $this->pageTableDirectory[$i] = 0; 481 | } 482 | // first page directory starts at offset 1280 483 | $this->pageTableDirectory[0] = 256 + 256*4; 484 | $this->writeUint32Array($this->pageTableDirectory); 485 | 486 | $pageTable = new SplFixedArray(512); 487 | for($i = 0; $i < 512; ++$i) { 488 | $pageTable[$i] = 0; 489 | } 490 | // first page table starts at offset 256 +256*4 + 512*4 491 | $pageTable[0] = 256 + 256*4 + 512*4; 492 | $this->writeUint32Array($pageTable); 493 | 494 | $this->pageTableCache = array(0 => $pageTable); 495 | 496 | // a page contains 256 slots 497 | $page = new SplFixedArray(512); 498 | for($i = 0; $i < 512; ++$i) { 499 | $page[$i] = 0; 500 | } 501 | $this->writeUint32Array($page); 502 | 503 | $this->base = ftell($this->fd); 504 | $this->fsize = $this->base; 505 | 506 | $this->pageCache = array(0 => $page); 507 | $this->buddySystem = new \simplito\BuddySystem($this->base); 508 | 509 | $this->writeHeader(); 510 | $this->writeBuddySystem(); 511 | } 512 | 513 | private function pread($pos, $sz) { 514 | if (fseek($this->fd, $pos) != 0) { 515 | throw new \Exception("file seek error"); 516 | } 517 | $value = fread($this->fd, $sz); 518 | if (strlen($value) != $sz) { 519 | throw new \Exception("file read error"); 520 | } 521 | return $value; 522 | } 523 | 524 | private function pwrite($pos, $buf) { 525 | $sz = strlen($buf); 526 | if ($pos > $this->fsize) { 527 | $this->setBuddySystemDirty(); 528 | ftruncate($this->fd, $pos); 529 | } 530 | if ($pos + $sz > $this->fsize) { 531 | $this->setBuddySystemDirty(); 532 | $this->fsize = $pos + $sz; 533 | } 534 | if (fseek($this->fd, $pos) != 0) { 535 | throw new \Exception("file fseek error"); 536 | } 537 | if (($rc = fwrite($this->fd, $buf)) != $sz) { 538 | throw new \Exception("file write error"); 539 | } 540 | } 541 | 542 | private function slotNo($key) { 543 | return Utils::slotNo($key, $this->s, $this->m); 544 | } 545 | 546 | private function prepareSlotValue($value) { 547 | $vsize = strlen($value); 548 | if ($vsize > 512) { 549 | $voffset = $this->buddySystem->alloc($vsize); 550 | $this->pwrite($voffset, $value); 551 | 552 | $buf = new \PSON\ByteBuffer(9); 553 | $buf->writeUint8(1); 554 | $buf->writeUint32($voffset); 555 | $buf->writeUint32($vsize); 556 | $value = $buf->flip()->toBinary(); 557 | } else { 558 | $value = chr(0) . $value; 559 | } 560 | return $value; 561 | } 562 | 563 | private function getSlotValue($value) { 564 | $type = ord($value[0]); 565 | if ($type == 0) { 566 | return substr($value, 1); 567 | } elseif ($type != 1) { 568 | throw new \Exception("invalid slot value"); 569 | } 570 | $buf = \PSON\ByteBuffer::wrap($value); 571 | $buf->skip(1); 572 | $voffset = $buf->readUint32(); 573 | $vsize = $buf->readUint32(); 574 | $value = $this->pread($voffset, $vsize); 575 | return $value; 576 | } 577 | 578 | private function getSlotValueInfo($value) { 579 | $type = ord($value[0]); 580 | if ($type == 0) { 581 | return null; 582 | } elseif ($type != 1) { 583 | throw new \Exception("invalid slot value"); 584 | } 585 | $buf = \PSON\ByteBuffer::wrap($value); 586 | $buf->skip(1); 587 | $voffset = $buf->readUint32(); 588 | $vsize = $buf->readUint32(); 589 | return new SlotInfo($voffset, $vsize); 590 | } 591 | 592 | private function freeSlotValue($value) { 593 | $type = ord($value[0]); 594 | if ($type == 0) { 595 | return; 596 | } elseif ($type != 1) { 597 | throw new \Exception("invalid slot value"); 598 | } 599 | $buf = \PSON\ByteBuffer::wrap($value); 600 | $buf->skip(1); 601 | $voffset = $buf->readUint32(); 602 | $vsize = $buf->readUint32(); 603 | if ($voffset > 0 && $vsize > 0) { 604 | $this->buddySystem->free($voffset, $vsize); 605 | } 606 | } 607 | 608 | public function flush() { 609 | if (count($this->changedSlots) == 0) 610 | return; 611 | 612 | $this->setBuddySystemDirty(); 613 | foreach($this->changedSlots as $slotNo => $slot) { 614 | $pageNo = $slotNo >> 8; 615 | $pageOff = $slotNo & 255; 616 | 617 | $pageTableNo = $pageNo >> 9; 618 | $pageTableOff = $pageNo & 511; 619 | 620 | $pageTable = &$this->getPageTable($pageTableNo); 621 | $page = &$this->getPage($pageNo); 622 | 623 | $value = $slot->encode(); 624 | $vsize = strlen($value); 625 | if ($vsize == 0) { 626 | $voffset = 0; 627 | } else { 628 | $voffset = $this->buddySystem->alloc($vsize); 629 | $this->pwrite($voffset, $value); 630 | } 631 | $this->pwrite($pageTable[$pageTableOff] + 8 * $pageOff, pack("NN", $voffset, $vsize)); 632 | 633 | $ovoffset = $page[2 * $pageOff]; 634 | $ovsize = $page[2 * $pageOff + 1]; 635 | if ($ovsize != 0) { 636 | $this->buddySystem->free($ovoffset, $ovsize); 637 | } 638 | $page[2 * $pageOff] = $voffset; 639 | $page[2 * $pageOff + 1] = $vsize; 640 | } 641 | $this->writeHeader(); 642 | $this->writeBuddySystem(); 643 | $this->unsetBuddySystemDirty(); 644 | 645 | $this->changedSlots = array(); 646 | $this->cleanSlotCache(); 647 | } 648 | 649 | private function readBuddySystem() { 650 | $value = $this->pread($this->fsize, 4); 651 | $value = unpack("N", $value); 652 | $size = $value[1]; 653 | $value = $this->pread($this->fsize + 4, $size); 654 | $this->buddySystem = BuddySystem::decode($value); 655 | } 656 | 657 | private function restoreBuddySystem() { 658 | $this->buddySystem = new \simplito\BuddySystem($this->base); 659 | 660 | //Page table directory already allocated outside of buddy system 661 | foreach ($this->pageTableDirectory as $pageTableIndex => $pageTableOffset) { 662 | //End of page tables 663 | if ($pageTableOffset == 0) { 664 | break; 665 | } 666 | //First page table already allocated outside of buddy system 667 | if ($pageTableIndex != 0) { 668 | $this->buddySystem->allocExplicit($pageTableOffset, 512*4); 669 | } 670 | $pageTable = $this->getPageTable($pageTableIndex); 671 | foreach ($pageTable as $pageIndex => $pageOffset) { 672 | //End of pages 673 | if ($pageOffset == 0) { 674 | break; 675 | } 676 | //First page in first page table already allocated outside of buddy system 677 | if ($pageTableIndex != 0 || $pageIndex != 0) { 678 | $this->buddySystem->allocExplicit($pageOffset, 256*8); 679 | } 680 | $page = $this->getPage($pageTableIndex * 512 + $pageIndex); 681 | for ($slotIndex = 0; $slotIndex < 256; $slotIndex++) { 682 | $slotOffset = $page[2 * $slotIndex]; 683 | $slotSize = $page[2 * $slotIndex + 1]; 684 | //Non existing slot 685 | if ($slotSize == 0) { 686 | continue; 687 | } 688 | $this->buddySystem->allocExplicit($slotOffset, $slotSize); 689 | $slotData = $this->pread($slotOffset, $slotSize); 690 | $slot = Slot::decode($slotData); 691 | foreach ($slot->getKeys() as $key) { 692 | $entryValue = $slot->fetch($key); 693 | $info = $this->getSlotValueInfo($entryValue); 694 | if ($info != null) { 695 | $this->buddySystem->allocExplicit($info->offset, $info->size); 696 | } 697 | } 698 | } 699 | } 700 | } 701 | $this->writeBuddySystem(); 702 | $this->unsetBuddySystemDirty(); 703 | } 704 | 705 | private function writeBuddySystem() { 706 | $value = $this->buddySystem->encode(); 707 | $size = strlen($value); 708 | if (fseek($this->fd, $this->fsize) != 0) { 709 | throw new \Exception("file seek error"); 710 | } 711 | if (fwrite($this->fd, pack("N", $size)) != 4) { 712 | throw new \Exception("file write error"); 713 | } 714 | if (($rc = fwrite($this->fd, $value)) != $size) { 715 | throw new \Exception("file write error"); 716 | } 717 | fflush($this->fd); 718 | if (!ftruncate($this->fd, $this->fsize + 4 + $size)) { 719 | throw new \Exception("truncate error"); 720 | } 721 | } 722 | 723 | private function grow() { 724 | if ($this->n >= 33554432) { 725 | return; 726 | } 727 | 728 | // allocate new page 729 | $pageNo = $this->n >> 8; 730 | $this->allocPage($pageNo); 731 | $this->n += 256; 732 | 733 | // rehash 734 | for($i = $this->s; $i < $this->s + 256; ++$i) { 735 | $from = $this->getSlot($i); 736 | $to = $this->getSlot($i + $this->m); 737 | $keys = $from->getKeys(); 738 | $changed = 0; 739 | foreach($keys as $key) { 740 | $slotNo = Utils::slotNo($key, $this->s + 256, $this->m); 741 | if ($slotNo != $i) { 742 | assert($slotNo == $i + $this->m); 743 | ++$changed; 744 | $value = $from->fetch($key); 745 | $from->delete($key); 746 | $to->insert($key, $value); 747 | } 748 | } 749 | if ($changed > 0) { 750 | $this->changedSlots[$i] = $from; 751 | $this->changedSlots[$i + $this->m] = $to; 752 | } 753 | } 754 | $this->s += 256; 755 | if ($this->s == $this->m) { 756 | $this->m = $this->m * 2; 757 | $this->s = 0; 758 | } 759 | } 760 | 761 | public function showBuddySystem() { 762 | return $this->buddySystem ? "" . $this->buddySystem : "(null)"; 763 | } 764 | } 765 | 766 | class SlotInfo { 767 | 768 | public $offset; 769 | public $size; 770 | 771 | public function __construct($offset, $size) { 772 | $this->offset = $offset; 773 | $this->size = $size; 774 | } 775 | } 776 | 777 | class Slot { 778 | private $objects = array(); 779 | 780 | public function insert($key, $value) { 781 | if (isset($this->objects[$key]) && $this->objects[$key] == $value) 782 | return; 783 | $this->objects[$key] = $value; 784 | } 785 | 786 | public function fetch($key) { 787 | if (!isset($this->objects[$key])) { 788 | return false; 789 | } 790 | return $this->objects[$key]; 791 | } 792 | 793 | public function delete($key) { 794 | if (!isset($this->objects[$key])) 795 | return; 796 | unset($this->objects[$key]); 797 | } 798 | 799 | public function exists($key) { 800 | return isset($this->objects[$key]); 801 | } 802 | 803 | public function getKeys() { 804 | return array_keys($this->objects); 805 | } 806 | 807 | public function encode() { 808 | $sz = count($this->objects); 809 | if ($sz == 0) { 810 | return ""; 811 | } 812 | $buf = new \PSON\ByteBuffer(); 813 | $buf->writeUint16($sz); 814 | foreach($this->objects as $key => $value) { 815 | $key = (string)$key; 816 | $value = (string)$value; 817 | $sz = strlen($key); 818 | $buf->writeUint32($sz); 819 | $buf->writeBytes($key); 820 | $sz = strlen($value); 821 | $buf->writeUint32($sz); 822 | $buf->writeBytes($value); 823 | } 824 | return $buf->flip()->toBinary(); 825 | } 826 | 827 | public static function decode($value) { 828 | $objects = array(); 829 | $buf = \PSON\ByteBuffer::wrap($value); 830 | if ($buf->remaining() == 0) { 831 | $this->objects = array(); 832 | return; 833 | } 834 | $objs = $buf->readUint16(); 835 | for($i = 0; $i < $objs; ++$i) { 836 | $sz = $buf->readUint32(); 837 | $key = $buf->readBytes($sz); 838 | $sz = $buf-> readUint32(); 839 | $value = $buf->readBytes($sz); 840 | $objects[$key] = $value; 841 | } 842 | $slot = new Slot(); 843 | $slot->objects = $objects; 844 | return $slot; 845 | } 846 | } 847 | 848 | class Utils { 849 | public static function slotNo($key, $s, $m) { 850 | $hash = crc32($key); 851 | $slot = $hash & ($m - 1); 852 | if ($slot < $s) { 853 | $slot = $hash & (2 * $m - 1); 854 | } 855 | return $slot; 856 | } 857 | } 858 | --------------------------------------------------------------------------------