├── .gitignore ├── test_suite ├── test_2.txt ├── test_1.txt ├── test_3.txt └── cli.php ├── docs ├── ifds_text.md ├── paging_file_cache.md ├── ifds_specification.md └── ifds_conf.md ├── support_extra ├── deflate_stream.php ├── ifds_text.php └── ifds_conf.php ├── support └── paging_file_cache.php └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /test.php 2 | *.ifds 3 | -------------------------------------------------------------------------------- /test_suite/test_2.txt: -------------------------------------------------------------------------------- 1 | 1,2,3,4,"5, 2 | "", 3 | 6",,"",9 4 | 1,2,3,4,"5,"",6",7,8,9 5 | -------------------------------------------------------------------------------- /test_suite/test_1.txt: -------------------------------------------------------------------------------- 1 | Mary had a little lamb, its fleece was white as snow. 2 | Wherever Mary went, the lamb was sure to go. 3 | Then Mary had dinner. 4 | It was tasty. 5 | 6 | Lamb: The other, other, other white meat. 7 | -------------------------------------------------------------------------------- /test_suite/test_3.txt: -------------------------------------------------------------------------------- 1 | Once upon a time, there was this crazy guy named Thomas. 2 | And he ran a crazy company called CubicleSoft. 3 | 4 | CubicleSoft created awesome software. 5 | In PHP. 6 | 7 | It was awesome. 8 | Eventually people decided it was so awesome, they parted with some money. 9 | 10 | Don't forget to donate. 11 | 12 | http://cubiclesoft.com/donate/ 13 | -------------------------------------------------------------------------------- /docs/ifds_text.md: -------------------------------------------------------------------------------- 1 | IFDS_Text Class: 'support/ifds_text.php' 2 | ========================================= 3 | 4 | The IFDS_Text class is one possible implementation of an example replacement format for the classic text file format. Utilizes the Incredibly Flexible Data Storage (IFDS) file format for storing and reading extremely large text files. Can also transparently compress and decompress data. Released under a MIT or LGPL license, your choice. 5 | 6 | Example usage can be seen in the main IFDS documentation and the IFDS test suite. 7 | 8 | IFDS_Text::SetCompressionLevel($level) 9 | -------------------------------------- 10 | 11 | Access: public 12 | 13 | Parameters: 14 | 15 | * $level - An integer containing a compression level (0-9). Default is -1. 16 | 17 | Returns: Nothing. 18 | 19 | This function sets the compression level to pass to `DeflateStream::Compress()`. 20 | 21 | IFDS_Text::Create($pfcfilename, $options = array()) 22 | --------------------------------------------------- 23 | 24 | Access: public 25 | 26 | Parameters: 27 | 28 | * $pfcfilename - An instance of PagingFileCache, a string containing a filename, or a boolean of false. 29 | * $options - An array containing options to set (Default is array()). 30 | 31 | Returns: A standard array of information. 32 | 33 | This function creates an IFDS text format file with the specified options. 34 | 35 | The $options array accepts these options: 36 | 37 | * compress - A boolean that specifies whether or not data should be compressed (Default is false). 38 | * trail - A boolean that specifies whether or not there is a trailing newline at the end of the file (Default is true). 39 | * newline - A string containing the bytes that represent a newline (Default is "\n"). 40 | * charset - A string containing the character set (Default is "utf-8"). 41 | * mimetype - A string containing the MIME type (Default is "text/plain"). 42 | * language - A string containing the language (Default is "en-us"). 43 | * author - A string containing the author (Default is ""). 44 | 45 | IFDS_Text::Open($pfcfilename) 46 | ----------------------------- 47 | 48 | Access: public 49 | 50 | Parameters: 51 | 52 | * $pfcfilename - An instance of PagingFileCache, a string containing a filename, or a boolean of false. 53 | 54 | Returns: A standard array of information. 55 | 56 | This function opens an IFDS text format file. 57 | 58 | IFDS_Text::GetIFDS() 59 | -------------------- 60 | 61 | Access: public 62 | 63 | Parameters: None. 64 | 65 | Returns: The internal IFDS object. 66 | 67 | This function returns the internal IFDS object. Can be useful for custom modifications (e.g. adding metadata for a editor, compiler, or tool). 68 | 69 | IFDS_Text::Close() 70 | ------------------ 71 | 72 | Access: public 73 | 74 | Parameters: None. 75 | 76 | Returns: Nothing. 77 | 78 | This function closes an open IFDS text format file. 79 | 80 | IFDS_Text::Save($flush = true) 81 | ------------------------------ 82 | 83 | Access: public 84 | 85 | Parameters: 86 | 87 | * $flush - A boolean indicating whether or not to also flush data in the IFDS object (Default is true). 88 | 89 | Returns: A standard array of information. 90 | 91 | This function saves internal state information and optionally flushes the internal IFDS object data as well. 92 | 93 | IFDS_Text::GetNumLines() 94 | ------------------------ 95 | 96 | Access: public 97 | 98 | Parameters: None. 99 | 100 | Returns: An integer containing the number of lines in the file. 101 | 102 | This function uses the line lookup table to calculate the total number of lines in the file. 103 | 104 | IFDS_Text::WriteMetadata() 105 | -------------------------- 106 | 107 | Access: protected 108 | 109 | Parameters: None. 110 | 111 | Returns: A standard array of information. 112 | 113 | This internal function writes updated metadata to the named metadata object. 114 | 115 | IFDS_Text::GetNewline() 116 | ----------------------- 117 | 118 | Access: public 119 | 120 | Parameters: None. 121 | 122 | Returns: A string containing the bytes used for newlines. 123 | 124 | This function returns the bytes used for newlines in the file data. 125 | 126 | IFDS_Text::IsCompressEnabled() 127 | ------------------------------ 128 | 129 | Access: public 130 | 131 | Parameters: None. 132 | 133 | Returns: A boolean of true if data should be compressed when writing new data, false otherwise. 134 | 135 | This function returns whether or not data should be compressed when writing new data. 136 | 137 | IFDS_Text::SetCompressData($enable) 138 | ----------------------------------- 139 | 140 | Access: public 141 | 142 | Parameters: 143 | 144 | * $enable - A boolean indicating whether or not new data should be compressed. 145 | 146 | Returns: Nothing. 147 | 148 | This function enables/disables the data compression bit for the file. Only affects new data written to the file. 149 | 150 | IFDS_Text::IsTrailingNewlineEnabled() 151 | ------------------------------------- 152 | 153 | Access: public 154 | 155 | Parameters: None. 156 | 157 | Returns: A boolean of true if there is a trailing newline at the end of the file, false otherwise. 158 | 159 | This function returns whether or not there is a trailing newline at the end of the file. 160 | 161 | IFDS_Text::SetTrailingNewline($enable) 162 | -------------------------------------- 163 | 164 | Access: public 165 | 166 | Parameters: 167 | 168 | * $enable - A boolean indicating whether or not a trailing newline exists at the end of the file. 169 | 170 | Returns: A standard array of information. 171 | 172 | This function is used later to determine whether or not there is a trailing newline at the end of the last line in the file when reading data. 173 | 174 | IFDS_Text::GetCharset() 175 | ----------------------- 176 | 177 | Access: public 178 | 179 | Parameters: None. 180 | 181 | Returns: A string containing the character set of the file data. 182 | 183 | This function returns the character set for the data. 184 | 185 | IFDS_Text::SetCharset($charset) 186 | ------------------------------- 187 | 188 | Access: public 189 | 190 | Parameters: 191 | 192 | * $charset - A string containing the character set. 193 | 194 | Returns: A standard array of information. 195 | 196 | This function sets the character set for the file. 197 | 198 | IFDS_Text::GetMIMEType() 199 | ------------------------ 200 | 201 | Access: public 202 | 203 | Parameters: None. 204 | 205 | Returns: A string containing the MIME type of the file data. 206 | 207 | This function returns the MIME type for the data. 208 | 209 | IFDS_Text::SetMIMEType($mimetype) 210 | --------------------------------- 211 | 212 | Access: public 213 | 214 | Parameters: 215 | 216 | * $mimetype - A string containing the MIME type. 217 | 218 | Returns: A standard array of information. 219 | 220 | This function sets the MIME type for the file. 221 | 222 | IFDS_Text::GetLanguage() 223 | ------------------------ 224 | 225 | Access: public 226 | 227 | Parameters: None. 228 | 229 | Returns: A string containing the IANA language code of the file data. 230 | 231 | This function returns the IANA language code for the data. 232 | 233 | IFDS_Text::SetLanguage($language) 234 | --------------------------------- 235 | 236 | Access: public 237 | 238 | Parameters: 239 | 240 | * $language - A string containing the IANA language code. 241 | 242 | Returns: A standard array of information. 243 | 244 | This function sets the IANA language code for the file. 245 | 246 | IFDS_Text::GetAuthor() 247 | ---------------------- 248 | 249 | Access: public 250 | 251 | Parameters: None. 252 | 253 | Returns: A string containing the author or owner of the file. 254 | 255 | This function returns the author or owner of the file. 256 | 257 | IFDS_Text::SetAuthor($author) 258 | ----------------------------- 259 | 260 | Access: public 261 | 262 | Parameters: 263 | 264 | * $author - A string containing the author or owner of the file. 265 | 266 | Returns: A standard array of information. 267 | 268 | This function sets the author or owner of the file. 269 | 270 | IFDS_Text::CreateSuperTextChunk($chunknum) 271 | ------------------------------------------ 272 | 273 | Access: protected 274 | 275 | Parameters: 276 | 277 | * $chunknum - An integer containing the location of the new super text chunk. 278 | 279 | Returns: A standard array of information. 280 | 281 | This internal function creates and inserts a super text chunk. 282 | 283 | IFDS_Text::LoadSuperTextChunk($chunknum) 284 | ---------------------------------------- 285 | 286 | Access: protected 287 | 288 | Parameters: 289 | 290 | * $chunknum - An integer containing the super text chunk to load. 291 | 292 | Returns: A standard array of information. 293 | 294 | This internal function loads a super text chunk and extracts it. 295 | 296 | IFDS_Text::ReadDataInternal($obj) 297 | --------------------------------- 298 | 299 | Access: protected 300 | 301 | Parameters: 302 | 303 | * $obj - An instance of a IFDS_RefCountObj object to read data from. 304 | 305 | Returns: A standard array of information. 306 | 307 | This internal function reads and decompresses data from an IFDS_RefCountObj object. 308 | 309 | IFDS_Text::WriteDataInternal($obj, &$data) 310 | ------------------------------------------ 311 | 312 | Access: protected 313 | 314 | Parameters: 315 | 316 | * $obj - An instance of a IFDS_RefCountObj object to write data to. 317 | * $data - A string containing the data to write. 318 | 319 | Returns: A standard array of information. 320 | 321 | This internal function compresses (optional) and writes data to an IFDS_RefCountObj object. Any existing data is overwritten. 322 | 323 | IFDS_Text::WriteLines($lines, $offset, $removelines) 324 | ---------------------------------------------------- 325 | 326 | Access: public 327 | 328 | Parameters: 329 | 330 | * $lines - An array or a string containing lines to insert. 331 | * $offset - An integer containing the offset, in lines, to start removing/inserting lines. 332 | * $removelines - An integer containing the number of lines to remove at the offset before inserting lines at the offset. 333 | 334 | Returns: A standard array of information. 335 | 336 | This function removes lines and inserts new lines starting at the specified offset. 337 | 338 | IFDS_Text::ReadLines($offset, $numlines, $ramlimit = 10485760) 339 | -------------------------------------------------------------- 340 | 341 | Access: public 342 | 343 | Parameters: 344 | 345 | * $offset - An integer containing the offset, in lines, to start reading lines. 346 | * $numlines - An integer containing the maximum number of lines to read. 347 | * $ramlimit - An integer containing the maximum number of bytes to read (Default is 10485760). 348 | 349 | Returns: A standard array of information. 350 | 351 | This function reads lines starting at the specified offset. 352 | 353 | IFDS_Text::IFDSTextTranslate($format, ...) 354 | ------------------------------------------ 355 | 356 | Access: _internal_ static 357 | 358 | Parameters: 359 | 360 | * $format - A string containing valid sprintf() format specifiers. 361 | 362 | Returns: A string containing a translation. 363 | 364 | This internal static function takes input strings and translates them from English to some other language if CS_TRANSLATE_FUNC is defined to be a valid PHP function name. 365 | -------------------------------------------------------------------------------- /support_extra/deflate_stream.php: -------------------------------------------------------------------------------- 1 | open = false; 13 | } 14 | 15 | public function __destruct() 16 | { 17 | $this->Finalize(); 18 | } 19 | 20 | public static function IsSupported() 21 | { 22 | if (!is_bool(self::$supported)) 23 | { 24 | self::$supported = function_exists("stream_filter_append") && function_exists("stream_filter_remove") && function_exists("gzcompress"); 25 | if (self::$supported) 26 | { 27 | $data = self::Compress("test"); 28 | if ($data === false || $data === "") self::$supported = false; 29 | else 30 | { 31 | $data = self::Uncompress($data); 32 | if ($data === false || $data !== "test") self::$supported = false; 33 | } 34 | } 35 | } 36 | 37 | return self::$supported; 38 | } 39 | 40 | public static function Compress($data, $compresslevel = -1, $options = array()) 41 | { 42 | $ds = new DeflateStream; 43 | if (!$ds->Init("wb", $compresslevel, $options)) return false; 44 | if (!$ds->Write($data)) return false; 45 | if (!$ds->Finalize()) return false; 46 | $data = $ds->Read(); 47 | 48 | return $data; 49 | } 50 | 51 | public static function Uncompress($data, $options = array("type" => "auto")) 52 | { 53 | $ds = new DeflateStream; 54 | if (!$ds->Init("rb", -1, $options)) return false; 55 | if (!$ds->Write($data)) return false; 56 | if (!$ds->Finalize()) return false; 57 | $data = $ds->Read(); 58 | 59 | return $data; 60 | } 61 | 62 | public function Init($mode, $compresslevel = -1, $options = array()) 63 | { 64 | if ($mode !== "rb" && $mode !== "wb") return false; 65 | if ($this->open) $this->Finalize(); 66 | 67 | $this->fp = fopen("php://memory", "w+b"); 68 | if ($this->fp === false) return false; 69 | $this->compress = ($mode == "wb"); 70 | if (!isset($options["type"])) $options["type"] = "rfc1951"; 71 | 72 | if ($options["type"] == "rfc1950") $options["type"] = "zlib"; 73 | else if ($options["type"] == "rfc1952") $options["type"] = "gzip"; 74 | 75 | if ($options["type"] != "zlib" && $options["type"] != "gzip" && ($this->compress || $options["type"] != "auto")) $options["type"] = "raw"; 76 | $this->options = $options; 77 | 78 | // Add the deflate filter. 79 | if ($this->compress) $this->filter = stream_filter_append($this->fp, "zlib.deflate", STREAM_FILTER_WRITE, $compresslevel); 80 | else $this->filter = stream_filter_append($this->fp, "zlib.inflate", STREAM_FILTER_READ); 81 | 82 | $this->open = true; 83 | $this->indata = ""; 84 | $this->outdata = ""; 85 | 86 | if ($this->compress) 87 | { 88 | if ($this->options["type"] == "zlib") 89 | { 90 | $this->outdata .= "\x78\x9C"; 91 | $this->options["a"] = 1; 92 | $this->options["b"] = 0; 93 | } 94 | else if ($this->options["type"] == "gzip") 95 | { 96 | if (!class_exists("CRC32Stream", false)) require_once str_replace("\\", "/", dirname(__FILE__)) . "/crc32_stream.php"; 97 | 98 | $this->options["crc32"] = new CRC32Stream(); 99 | $this->options["crc32"]->Init(); 100 | $this->options["bytes"] = 0; 101 | 102 | $this->outdata .= "\x1F\x8B\x08"; 103 | $flags = 0; 104 | if (isset($this->options["filename"])) $flags |= 0x08; 105 | if (isset($this->options["comment"])) $flags |= 0x10; 106 | $this->outdata .= chr($flags); 107 | $this->outdata .= "\x00\x00\x00\x00"; 108 | $this->outdata .= "\x00"; 109 | $this->outdata .= "\x03"; 110 | 111 | if (isset($this->options["filename"])) $this->outdata .= str_replace("\x00", " ", $this->options["filename"]) . "\x00"; 112 | if (isset($this->options["comment"])) $this->outdata .= str_replace("\x00", " ", $this->options["comment"]) . "\x00"; 113 | } 114 | } 115 | else 116 | { 117 | $this->options["header"] = false; 118 | } 119 | 120 | return true; 121 | } 122 | 123 | public function Read() 124 | { 125 | $result = $this->outdata; 126 | $this->outdata = ""; 127 | 128 | return $result; 129 | } 130 | 131 | public function Write($data) 132 | { 133 | if (!$this->open) return false; 134 | 135 | if ($this->compress) 136 | { 137 | if ($this->options["type"] == "zlib") 138 | { 139 | // Adler-32. 140 | $y = strlen($data); 141 | for ($x = 0; $x < $y; $x++) 142 | { 143 | $this->options["a"] = ($this->options["a"] + ord($data[$x])) % 65521; 144 | $this->options["b"] = ($this->options["b"] + $this->options["a"]) % 65521; 145 | } 146 | } 147 | else if ($this->options["type"] == "gzip") 148 | { 149 | $this->options["crc32"]->AddData($data); 150 | $this->options["bytes"] = $this->ADD32($this->options["bytes"], strlen($data)); 151 | } 152 | 153 | $this->indata .= $data; 154 | while (strlen($this->indata) >= 65536) 155 | { 156 | fwrite($this->fp, substr($this->indata, 0, 65536)); 157 | $this->indata = substr($this->indata, 65536); 158 | 159 | $this->ProcessOutput(); 160 | } 161 | } 162 | else 163 | { 164 | $this->indata .= $data; 165 | $this->ProcessInput(); 166 | } 167 | 168 | return true; 169 | } 170 | 171 | // Finalizes the stream. 172 | public function Finalize() 173 | { 174 | if (!$this->open) return false; 175 | 176 | if (!$this->compress) $this->ProcessInput(true); 177 | 178 | if (strlen($this->indata) > 0) 179 | { 180 | fwrite($this->fp, $this->indata); 181 | $this->indata = ""; 182 | } 183 | 184 | // Removing the filter pushes the last buffer into the stream. 185 | stream_filter_remove($this->filter); 186 | $this->filter = false; 187 | 188 | $this->ProcessOutput(); 189 | 190 | fclose($this->fp); 191 | 192 | if ($this->compress) 193 | { 194 | if ($this->options["type"] == "zlib") $this->outdata .= pack("N", $this->SHL32($this->options["b"], 16) | $this->options["a"]); 195 | else if ($this->options["type"] == "gzip") $this->outdata .= pack("V", $this->options["crc32"]->Finalize()) . pack("V", $this->options["bytes"]); 196 | } 197 | 198 | $this->open = false; 199 | 200 | return true; 201 | } 202 | 203 | private function ProcessOutput() 204 | { 205 | rewind($this->fp); 206 | 207 | // Hack! Because ftell() on a stream with a filter is still broken even under the latest PHP a mere 11 years later. 208 | // See: https://bugs.php.net/bug.php?id=49874 209 | ob_start(); 210 | fpassthru($this->fp); 211 | $this->outdata .= ob_get_contents(); 212 | ob_end_clean(); 213 | 214 | rewind($this->fp); 215 | ftruncate($this->fp, 0); 216 | } 217 | 218 | private function ProcessInput($final = false) 219 | { 220 | // Automatically determine the type of data based on the header signature. 221 | if ($this->options["type"] == "auto") 222 | { 223 | if (strlen($this->indata) >= 3) 224 | { 225 | $zlibtest = unpack("n", substr($this->indata, 0, 2)); 226 | 227 | if (substr($this->indata, 0, 3) === "\x1F\x8B\x08") $this->options["type"] = "gzip"; 228 | else if ((ord($this->indata[0]) & 0x0F) == 8 && ((ord($this->indata[0]) & 0xF0) >> 4) < 8 && $zlibtest[1] % 31 == 0) $this->options["type"] = "zlib"; 229 | else $this->options["type"] = "raw"; 230 | } 231 | else if ($final) $this->options["type"] = "raw"; 232 | } 233 | 234 | if ($this->options["type"] == "gzip") 235 | { 236 | if (!$this->options["header"]) 237 | { 238 | if (strlen($this->indata) >= 10) 239 | { 240 | $idcm = substr($this->indata, 0, 3); 241 | $flg = ord($this->indata[3]); 242 | 243 | if ($idcm !== "\x1F\x8B\x08") $this->options["type"] = "ignore"; 244 | else 245 | { 246 | // Calculate the number of bytes to skip. If flags are set, the size can be dynamic. 247 | $size = 10; 248 | $y = strlen($this->indata); 249 | 250 | // FLG.FEXTRA 251 | if ($size && ($flg & 0x04)) 252 | { 253 | if ($size + 2 >= $y) $size = 0; 254 | else 255 | { 256 | $xlen = unpack("v", substr($this->indata, $size, 2)); 257 | $size = ($size + 2 + $xlen <= $y ? $size + 2 + $xlen : 0); 258 | } 259 | } 260 | 261 | // FLG.FNAME 262 | if ($size && ($flg & 0x08)) 263 | { 264 | $pos = strpos($this->indata, "\x00", $size); 265 | $size = ($pos !== false ? $pos + 1 : 0); 266 | } 267 | 268 | // FLG.FCOMMENT 269 | if ($size && ($flg & 0x10)) 270 | { 271 | $pos = strpos($this->indata, "\x00", $size); 272 | $size = ($pos !== false ? $pos + 1 : 0); 273 | } 274 | 275 | // FLG.FHCRC 276 | if ($size && ($flg & 0x02)) $size = ($size + 2 <= $y ? $size + 2 : 0); 277 | 278 | if ($size) 279 | { 280 | $this->indata = substr($this->indata, $size); 281 | $this->options["header"] = true; 282 | } 283 | } 284 | } 285 | } 286 | 287 | if ($this->options["header"] && strlen($this->indata) > 8) 288 | { 289 | fwrite($this->fp, substr($this->indata, 0, -8)); 290 | $this->indata = substr($this->indata, -8); 291 | 292 | $this->ProcessOutput(); 293 | } 294 | 295 | if ($final) $this->indata = ""; 296 | } 297 | else if ($this->options["type"] == "zlib") 298 | { 299 | if (!$this->options["header"]) 300 | { 301 | if (strlen($this->indata) >= 2) 302 | { 303 | $cmf = ord($this->indata[0]); 304 | $flg = ord($this->indata[1]); 305 | $cm = $cmf & 0x0F; 306 | $cinfo = ($cmf & 0xF0) >> 4; 307 | 308 | // Compression method 'deflate' ($cm = 8), window size - 8 ($cinfo < 8), no preset dictionaries ($flg bit 5), checksum validates. 309 | if ($cm != 8 || $cinfo > 7 || ($flg & 0x20) || (($cmf << 8 | $flg) % 31) != 0) $this->options["type"] = "ignore"; 310 | else 311 | { 312 | $this->indata = substr($this->indata, 2); 313 | $this->options["header"] = true; 314 | } 315 | } 316 | } 317 | 318 | if ($this->options["header"] && strlen($this->indata) > 4) 319 | { 320 | fwrite($this->fp, substr($this->indata, 0, -4)); 321 | $this->indata = substr($this->indata, -4); 322 | 323 | $this->ProcessOutput(); 324 | } 325 | 326 | if ($final) $this->indata = ""; 327 | } 328 | 329 | if ($this->options["type"] == "raw") 330 | { 331 | fwrite($this->fp, $this->indata); 332 | $this->indata = ""; 333 | 334 | $this->ProcessOutput(); 335 | } 336 | 337 | // Only set when an unrecoverable header error has occurred for gzip or zlib. 338 | if ($this->options["type"] == "ignore") $this->indata = ""; 339 | } 340 | 341 | private function SHL32($num, $bits) 342 | { 343 | if ($bits < 0) $bits = 0; 344 | 345 | return $this->LIM32((int)$num << $bits); 346 | } 347 | 348 | private function LIM32($num) 349 | { 350 | return (int)((int)$num & 0xFFFFFFFF); 351 | } 352 | 353 | private function ADD32($num, $num2) 354 | { 355 | $num = (int)$num; 356 | $num2 = (int)$num2; 357 | $add = ((($num >> 30) & 0x03) + (($num2 >> 30) & 0x03)); 358 | $num = ((int)($num & 0x3FFFFFFF) + (int)($num2 & 0x3FFFFFFF)); 359 | if ($num & 0x40000000) $add++; 360 | $num = (int)(($num & 0x3FFFFFFF) | (($add & 0x03) << 30)); 361 | 362 | return $num; 363 | } 364 | } 365 | ?> -------------------------------------------------------------------------------- /test_suite/cli.php: -------------------------------------------------------------------------------- 1 | $val) 15 | { 16 | if (!isset($options["rules"][$val])) unset($options["shortmap"][$key]); 17 | } 18 | foreach ($options["rules"] as $key => $val) 19 | { 20 | if (!isset($val["arg"])) $options["rules"][$key]["arg"] = false; 21 | if (!isset($val["multiple"])) $options["rules"][$key]["multiple"] = false; 22 | } 23 | 24 | if ($args === false) $args = $_SERVER["argv"]; 25 | else if (is_string($args)) 26 | { 27 | $args2 = $args; 28 | $args = array(); 29 | $inside = false; 30 | $currarg = ""; 31 | $y = strlen($args2); 32 | for ($x = 0; $x < $y; $x++) 33 | { 34 | $currchr = substr($args2, $x, 1); 35 | 36 | if ($inside === false && $currchr == " " && $currarg != "") 37 | { 38 | $args[] = $currarg; 39 | $currarg = ""; 40 | } 41 | else if ($currchr == "\"" || $currchr == "'") 42 | { 43 | if ($inside === false) $inside = $currchr; 44 | else if ($inside === $currchr) $inside = false; 45 | else $currarg .= $currchr; 46 | } 47 | else if ($currchr == "\\" && $x < $y - 1) 48 | { 49 | $x++; 50 | $currarg .= substr($args2, $x, 1); 51 | } 52 | else if ($inside !== false || $currchr != " ") 53 | { 54 | $currarg .= $currchr; 55 | } 56 | } 57 | 58 | if ($currarg != "") $args[] = $currarg; 59 | } 60 | 61 | $result = array("success" => true, "file" => array_shift($args), "opts" => array(), "params" => array()); 62 | 63 | // Look over shortmap to determine if options exist that are one byte (flags) and don't have arguments. 64 | $chrs = array(); 65 | foreach ($options["shortmap"] as $key => $val) 66 | { 67 | if (isset($options["rules"][$val]) && !$options["rules"][$val]["arg"]) $chrs[$key] = true; 68 | } 69 | 70 | $allowopt = true; 71 | $y = count($args); 72 | for ($x = 0; $x < $y; $x++) 73 | { 74 | $arg = $args[$x]; 75 | 76 | // Attempt to process an option. 77 | $opt = false; 78 | $optval = false; 79 | if ($allowopt && substr($arg, 0, 1) == "-") 80 | { 81 | $pos = strpos($arg, "="); 82 | if ($pos === false) $pos = strlen($arg); 83 | else $optval = substr($arg, $pos + 1); 84 | $arg2 = substr($arg, 1, $pos - 1); 85 | 86 | if (isset($options["rules"][$arg2])) $opt = $arg2; 87 | else if (isset($options["shortmap"][$arg2])) $opt = $options["shortmap"][$arg2]; 88 | else if ($x == 0) 89 | { 90 | // Attempt to process as a set of flags. 91 | $y2 = strlen($arg2); 92 | if ($y2 > 0) 93 | { 94 | for ($x2 = 0; $x2 < $y2; $x2++) 95 | { 96 | $currchr = substr($arg2, $x2, 1); 97 | 98 | if (!isset($chrs[$currchr])) break; 99 | } 100 | 101 | if ($x2 == $y2) 102 | { 103 | for ($x2 = 0; $x2 < $y2; $x2++) 104 | { 105 | $opt = $options["shortmap"][substr($arg2, $x2, 1)]; 106 | 107 | if (!$options["rules"][$opt]["multiple"]) $result["opts"][$opt] = true; 108 | else 109 | { 110 | if (!isset($result["opts"][$opt])) $result["opts"][$opt] = 0; 111 | $result["opts"][$opt]++; 112 | } 113 | } 114 | 115 | continue; 116 | } 117 | } 118 | } 119 | } 120 | 121 | if ($opt === false) 122 | { 123 | // Is a parameter. 124 | if (substr($arg, 0, 1) === "\"" || substr($arg, 0, 1) === "'") $arg = substr($arg, 1); 125 | if (substr($arg, -1) === "\"" || substr($arg, -1) === "'") $arg = substr($arg, 0, -1); 126 | 127 | $result["params"][] = $arg; 128 | 129 | if (!$options["allow_opts_after_param"]) $allowopt = false; 130 | } 131 | else if (!$options["rules"][$opt]["arg"]) 132 | { 133 | // Is a flag by itself. 134 | if (!$options["rules"][$opt]["multiple"]) $result["opts"][$opt] = true; 135 | else 136 | { 137 | if (!isset($result["opts"][$opt])) $result["opts"][$opt] = 0; 138 | $result["opts"][$opt]++; 139 | } 140 | } 141 | else 142 | { 143 | // Is an option. 144 | if ($optval === false) 145 | { 146 | $x++; 147 | if ($x == $y) break; 148 | $optval = $args[$x]; 149 | } 150 | 151 | if (substr($optval, 0, 1) === "\"" || substr($optval, 0, 1) === "'") $optval = substr($optval, 1); 152 | if (substr($optval, -1) === "\"" || substr($optval, -1) === "'") $optval = substr($optval, 0, -1); 153 | 154 | if (!$options["rules"][$opt]["multiple"]) $result["opts"][$opt] = $optval; 155 | else 156 | { 157 | if (!isset($result["opts"][$opt])) $result["opts"][$opt] = array(); 158 | $result["opts"][$opt][] = $optval; 159 | } 160 | } 161 | } 162 | 163 | return $result; 164 | } 165 | 166 | public static function CanGetUserInputWithArgs(&$args, $prefix) 167 | { 168 | return (($prefix !== false && isset($args["opts"][$prefix]) && is_array($args["opts"][$prefix]) && count($args["opts"][$prefix])) || count($args["params"])); 169 | } 170 | 171 | // Gets a line of input from the user. If the user supplies all information via the command-line, this could be entirely automated. 172 | public static function GetUserInputWithArgs(&$args, $prefix, $question, $default, $noparamsoutput = "", $suppressoutput = false, $callback = false, $callbackopts = false) 173 | { 174 | if (!self::CanGetUserInputWithArgs($args, $prefix) && $noparamsoutput != "") 175 | { 176 | echo "\n" . rtrim($noparamsoutput) . "\n\n"; 177 | 178 | $suppressoutput = false; 179 | $noparamsoutput = ""; 180 | } 181 | 182 | do 183 | { 184 | $prompt = ($suppressoutput ? "" : $question . ($default !== false ? " [" . $default . "]" : "") . ": "); 185 | 186 | if ($prefix !== false && isset($args["opts"][$prefix]) && is_array($args["opts"][$prefix]) && count($args["opts"][$prefix])) 187 | { 188 | $line = array_shift($args["opts"][$prefix]); 189 | if ($line === "") $line = $default; 190 | if (!$suppressoutput) echo $prompt . $line . "\n"; 191 | } 192 | else if (count($args["params"])) 193 | { 194 | $line = array_shift($args["params"]); 195 | if ($line === "") $line = $default; 196 | if (!$suppressoutput) echo $prompt . $line . "\n"; 197 | } 198 | else if (strtoupper(substr(php_uname("s"), 0, 3)) != "WIN" && function_exists("readline") && function_exists("readline_add_history")) 199 | { 200 | $line = readline($prompt); 201 | if ($line === false) exit(); 202 | 203 | $line = trim($line); 204 | if ($line === "") $line = $default; 205 | if ($line !== false && $line !== "") readline_add_history($line); 206 | } 207 | else 208 | { 209 | echo $prompt; 210 | fflush(STDOUT); 211 | $line = fgets(STDIN); 212 | if ($line === false || ($line === "" && feof(STDIN))) exit(); 213 | 214 | $line = trim($line); 215 | if ($line === "") $line = $default; 216 | } 217 | 218 | if ($line === false || (is_callable($callback) && !call_user_func_array($callback, array($line, &$callbackopts)))) 219 | { 220 | if ($line !== false) $line = false; 221 | else echo "Please enter a value.\n"; 222 | 223 | if (!self::CanGetUserInputWithArgs($args, $prefix) && $noparamsoutput != "") 224 | { 225 | echo "\n" . $noparamsoutput . "\n"; 226 | 227 | $noparamsoutput = ""; 228 | } 229 | 230 | $suppressoutput = false; 231 | } 232 | } while ($line === false); 233 | 234 | return $line; 235 | } 236 | 237 | // Obtains a valid line of input. 238 | public static function GetLimitedUserInputWithArgs(&$args, $prefix, $question, $default, $allowedoptionsprefix, $allowedoptions, $loop = true, $suppressoutput = false, $multipleuntil = false) 239 | { 240 | $noparamsoutput = $allowedoptionsprefix . "\n\n"; 241 | $size = 0; 242 | foreach ($allowedoptions as $key => $val) 243 | { 244 | if ($size < strlen($key)) $size = strlen($key); 245 | } 246 | 247 | foreach ($allowedoptions as $key => $val) 248 | { 249 | $newtab = str_repeat(" ", 2 + $size + 3); 250 | $noparamsoutput .= " " . $key . ":" . str_repeat(" ", $size - strlen($key)) . " " . str_replace("\n\t", "\n" . $newtab, $val) . "\n"; 251 | } 252 | 253 | $noparamsoutput .= "\n"; 254 | 255 | if ($default === false && count($allowedoptions) == 1) 256 | { 257 | reset($allowedoptions); 258 | $default = key($allowedoptions); 259 | } 260 | 261 | $results = array(); 262 | do 263 | { 264 | $displayed = (!count($args["params"])); 265 | $result = self::GetUserInputWithArgs($args, $prefix, $question, $default, $noparamsoutput, $suppressoutput); 266 | if (is_array($multipleuntil) && $multipleuntil["exit"] === $result) break; 267 | $result2 = false; 268 | if (!count($allowedoptions)) break; 269 | foreach ($allowedoptions as $key => $val) 270 | { 271 | if (!strcasecmp($key, $result) || !strcasecmp($val, $result)) $result2 = $key; 272 | } 273 | if ($loop) 274 | { 275 | if ($result2 === false) 276 | { 277 | echo "Please select an option from the list.\n"; 278 | 279 | $suppressoutput = false; 280 | } 281 | else if (is_array($multipleuntil)) 282 | { 283 | $results[$result2] = $result2; 284 | 285 | $question = $multipleuntil["nextquestion"]; 286 | $default = $multipleuntil["nextdefault"]; 287 | } 288 | } 289 | 290 | if ($displayed) $noparamsoutput = ""; 291 | } while ($loop && ($result2 === false || is_array($multipleuntil))); 292 | 293 | return (is_array($multipleuntil) ? $results : $result2); 294 | } 295 | 296 | // Obtains Yes/No style input. 297 | public static function GetYesNoUserInputWithArgs(&$args, $prefix, $question, $default, $noparamsoutput = "", $suppressoutput = false) 298 | { 299 | $default = (substr(strtoupper(trim($default)), 0, 1) === "Y" ? "Y" : "N"); 300 | 301 | $result = self::GetUserInputWithArgs($args, $prefix, $question, $default, $noparamsoutput, $suppressoutput); 302 | $result = (substr(strtoupper(trim($result)), 0, 1) === "Y"); 303 | 304 | return $result; 305 | } 306 | 307 | public static function GetHexDump($data) 308 | { 309 | $result = ""; 310 | 311 | $x = 0; 312 | $y = strlen($data); 313 | if ($y <= 256) $padwidth = 2; 314 | else if ($y <= 65536) $padwidth = 4; 315 | else if ($y <= 16777216) $padwidth = 6; 316 | else $padwidth = 8; 317 | 318 | $pad = str_repeat(" ", $padwidth); 319 | 320 | $data2 = str_split(strtoupper(bin2hex($data)), 32); 321 | foreach ($data2 as $line) 322 | { 323 | $result .= sprintf("%0" . $padwidth . "X", $x) . " | "; 324 | 325 | $line = str_split($line, 2); 326 | array_splice($line, 8, 0, ""); 327 | $result .= implode(" ", $line) . "\n"; 328 | 329 | $result .= $pad . " |"; 330 | $y2 = $x + 16; 331 | for ($x2 = 0; $x2 < 16 && $x < $y; $x2++) 332 | { 333 | $result .= " "; 334 | if ($x2 === 8) $result .= " "; 335 | 336 | $tempchr = ord($data[$x]); 337 | if ($tempchr === 0x09) $result .= "\\t"; 338 | else if ($tempchr === 0x0D) $result .= "\\r"; 339 | else if ($tempchr === 0x0A) $result .= "\\n"; 340 | else if ($tempchr === 0x00) $result .= "\\0"; 341 | else if ($tempchr < 32 || $tempchr > 126) $result .= " "; 342 | else $result .= " " . $data[$x]; 343 | 344 | $x++; 345 | } 346 | 347 | $result .= "\n"; 348 | } 349 | 350 | return $result; 351 | } 352 | 353 | // Outputs a JSON array (useful for captured output). 354 | public static function DisplayResult($result, $exit = true) 355 | { 356 | if (is_array($result)) echo json_encode($result, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n"; 357 | else echo $result . "\n"; 358 | 359 | if ($exit) exit(); 360 | } 361 | 362 | // Useful for reparsing remaining parameters as new arguments. 363 | public static function ReinitArgs(&$args, $newargs) 364 | { 365 | // Process the parameters. 366 | $options = array( 367 | "shortmap" => array( 368 | "?" => "help" 369 | ), 370 | "rules" => array( 371 | ) 372 | ); 373 | 374 | foreach ($newargs as $arg) $options["rules"][$arg] = array("arg" => true, "multiple" => true); 375 | $options["rules"]["help"] = array("arg" => false); 376 | 377 | $args = self::ParseCommandLine($options, array_merge(array(""), $args["params"])); 378 | 379 | if (isset($args["opts"]["help"])) self::DisplayResult(array("success" => true, "options" => array_keys($options["rules"]))); 380 | } 381 | 382 | // Tracks messages for a command-line interface app. 383 | private static $messages = array(); 384 | 385 | public static function LogMessage($msg, $data = null) 386 | { 387 | if (isset($data)) $msg .= "\n\t" . trim(str_replace("\n", "\n\t", json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES))); 388 | 389 | self::$messages[] = $msg; 390 | 391 | fwrite(STDERR, $msg . "\n"); 392 | } 393 | 394 | public static function DisplayError($msg, $result = false, $exit = true) 395 | { 396 | self::LogMessage(($exit ? "[Error] " : "") . $msg); 397 | 398 | if ($result !== false && is_array($result) && isset($result["error"]) && isset($result["errorcode"])) self::LogMessage("[Error] " . $result["error"] . " (" . $result["errorcode"] . ")", (isset($result["info"]) ? $result["info"] : null)); 399 | 400 | if ($exit) exit(); 401 | } 402 | 403 | public static function GetLogMessages($filters = array()) 404 | { 405 | if (is_string($filters)) $filters = array($filters); 406 | 407 | $result = array(); 408 | foreach (self::$messages as $message) 409 | { 410 | $found = (!count($filters)); 411 | foreach ($filters as $filter) 412 | { 413 | if (preg_match($filter, $message)) $found = true; 414 | } 415 | 416 | if ($found) $result[] = $message; 417 | } 418 | 419 | return $result; 420 | } 421 | 422 | public static function ResetLogMessages() 423 | { 424 | self::$messages = array(); 425 | } 426 | 427 | 428 | private static $timerinfo = array(); 429 | 430 | public static function StartTimer() 431 | { 432 | $ts = microtime(true); 433 | 434 | self::$timerinfo = array( 435 | "start" => $ts, 436 | "diff" => $ts 437 | ); 438 | } 439 | 440 | public static function UpdateTimer() 441 | { 442 | $ts = microtime(true); 443 | $diff = $ts - self::$timerinfo["diff"]; 444 | self::$timerinfo["diff"] = $ts; 445 | 446 | $result = array( 447 | "success" => true, 448 | "diff" => sprintf("%.2f", $diff), 449 | "total" => sprintf("%.2f", $ts - self::$timerinfo["start"]) 450 | ); 451 | 452 | return $result; 453 | } 454 | } 455 | ?> -------------------------------------------------------------------------------- /docs/paging_file_cache.md: -------------------------------------------------------------------------------- 1 | PagingFileCache Class: 'support/paging_file_cache.php' 2 | ======================================================= 3 | 4 | The PagingFileCache class applies a page cache layer over physical file and in-memory data. By default, each page is 4,096 bytes and the class will cache up to 2,048 pages (8MB) in RAM but the defaults can, of course, be adjusted as desired. This allows for very large, structured files to be quickly navigated, keeping the most frequently used pages in RAM. Released under a MIT or LGPL license, your choice. 5 | 6 | Supports reserved bytes to be able to extend this class to encrypt the data of each individual page. 7 | 8 | Also supports streaming for dynamically generating content, reading and writing lines, reading until a match is found, and reading and writing CSV records. 9 | 10 | Example usage can be seen in the test suite. 11 | 12 | PagingFileCache::GetPageSize() 13 | ------------------------------ 14 | 15 | Access: public 16 | 17 | Parameters: None. 18 | 19 | Returns: An integer containing the real page size minus reserved bytes. 20 | 21 | This function returns the actual amount of data per page. 22 | 23 | PagingFileCache::GetRealPageSize() 24 | ---------------------------------- 25 | 26 | Access: public 27 | 28 | Parameters: None. 29 | 30 | Returns: An integer containing the real page size. 31 | 32 | This function returns the real page size including reserved bytes. 33 | 34 | PagingFileCache::SetRealPageSize($size) 35 | --------------------------------------- 36 | 37 | Access: public 38 | 39 | Parameters: 40 | 41 | * $size - An integer containing the number of bytes to use for each page. 42 | 43 | Returns: Nothing. 44 | 45 | This function sets the real page size and recalculates the amount of data per page. 46 | 47 | PagingFileCache::GetPageReservedBytes() 48 | --------------------------------------- 49 | 50 | Access: public 51 | 52 | Parameters: None. 53 | 54 | Returns: An integer containing the number of reserved bytes per page. 55 | 56 | This function returns the number of reserved bytes per page. 57 | 58 | PagingFileCache::SetPageReservedBytes($numbytes) 59 | ------------------------------------------------ 60 | 61 | Access: public 62 | 63 | Parameters: 64 | 65 | * $numbytes - An integer containing the number of bytes to reserve per page. 66 | 67 | Returns: Nothing. 68 | 69 | This function sets the number of bytes to reserve per page and recalculates the amount of data per page. Reserved bytes are useful for encryption operations (e.g. padding) and validation operations (e.g. CRC-32). 70 | 71 | PagingFileCache::GetMaxCachedPages() 72 | ------------------------------------ 73 | 74 | Access: public 75 | 76 | Parameters: None. 77 | 78 | Returns: An integer containing the maximum number of cached pages to keep in memory. 79 | 80 | This function returns the maximum number of cached pages to keep in memory. 81 | 82 | PagingFileCache::SetMaxCachedPages($maxpages) 83 | --------------------------------------------- 84 | 85 | Access: public 86 | 87 | Parameters: 88 | 89 | * $maxpages - An integer containing the maximum number of cached pages to keep in memory. 90 | 91 | Returns: Nothing. 92 | 93 | This function sets the maximum number of cached pages to keep in memory. 94 | 95 | PagingFileCache::GetNumCachedPages() 96 | ------------------------------------ 97 | 98 | Access: public 99 | 100 | Parameters: None. 101 | 102 | Returns: The current number of cached pages. 103 | 104 | This function returns the current number of cached pages in the page map. 105 | 106 | PagingFileCache::GetRealPos($pos) 107 | --------------------------------- 108 | 109 | Access: public 110 | 111 | Parameters: 112 | 113 | * $pos - An integer containing a virtual position. 114 | 115 | Returns: An integer containing a real position. 116 | 117 | This function maps a virtual position to a real position. When pages contain reserved bytes, positions need to be adjusted to point at the actual location. 118 | 119 | PagingFileCache::GetVirtualPos($pos) 120 | ------------------------------------ 121 | 122 | Access: public 123 | 124 | Parameters: 125 | 126 | * $pos - An integer containing a real position. 127 | 128 | Returns: An integer containing a virtual position. 129 | 130 | This function maps a real position to a virtual position. 131 | 132 | Note that this function does not, by design, work properly when the position is in the reserved bytes space. 133 | 134 | PagingFileCache::SetData($data, $mode = PagingFileCache::PAGING_FILE_MODE_READ | PagingFileCache::PAGING_FILE_MODE_WRITE) 135 | ------------------------------------------------------------------------------------------------------------------------- 136 | 137 | Access: public 138 | 139 | Parameters: 140 | 141 | * $data - A string containing the raw data to use in place of a file. 142 | * $mode - An integer containing the mode to open the data as (Default is PagingFileCache::PAGING_FILE_MODE_READ | PagingFileCache::PAGING_FILE_MODE_WRITE). 143 | 144 | Returns: A standard array of information. 145 | 146 | This function sets the raw data to use in place of a file. Can be used to perform all operations in memory. 147 | 148 | The class becomes streaming enabled when $data is an empty string and $mode is write only (i.e. PagingFileCache::PAGING_FILE_MODE_WRITE). 149 | 150 | Example usage: 151 | 152 | ```php 153 | SetData("", PagingFileCache::PAGING_FILE_MODE_WRITE); 160 | if (!$result["success"]) 161 | { 162 | var_dump($result); 163 | 164 | exit(); 165 | } 166 | 167 | $data = ""; 168 | for ($x = 1; $x < 5000; $x++) 169 | { 170 | // Write data into the stream. 171 | $result = $pfc->Write("Line " . $x . "\n"); 172 | if (!$result["success"]) 173 | { 174 | var_dump($result); 175 | 176 | exit(); 177 | } 178 | 179 | // Collect data as it becomes available. 180 | $result = $pfc->GetData(); 181 | if ($result === false) 182 | { 183 | echo "Failed to get data.\n"; 184 | 185 | exit(); 186 | } 187 | 188 | $data .= $result; 189 | } 190 | 191 | // Finalize remaining data. 192 | $result = $pfc->Sync(true); 193 | if (!$result["success"]) 194 | { 195 | var_dump($result); 196 | 197 | exit(); 198 | } 199 | 200 | // Collect final data. 201 | $result = $pfc->GetData(); 202 | if ($result === false) 203 | { 204 | echo "Failed to get data.\n"; 205 | 206 | exit(); 207 | } 208 | 209 | $data .= $result; 210 | 211 | echo $data . "\n"; 212 | ?> 213 | ``` 214 | 215 | PagingFileCache::Open($filename, $mode = PagingFileCache::PAGING_FILE_MODE_READ | PagingFileCache::PAGING_FILE_MODE_WRITE) 216 | -------------------------------------------------------------------------------------------------------------------------- 217 | 218 | Access: public 219 | 220 | Parameters: 221 | 222 | * $filename - A string containing a filename to open. 223 | * $mode - An integer containing the mode to open the data as (Default is PagingFileCache::PAGING_FILE_MODE_READ | PagingFileCache::PAGING_FILE_MODE_WRITE). 224 | 225 | Returns: A standard array of information. 226 | 227 | This function opens the specified file. If the file doesn't exist, then it is created unless it is being opened read only. 228 | 229 | PagingFileCache::Close() 230 | ------------------------ 231 | 232 | Access: public 233 | 234 | Parameters: None. 235 | 236 | Returns: Nothing. 237 | 238 | This function performs a final synchronization and resets internal structures. Automatically called by the destructor. 239 | 240 | PagingFileCache::CanRead() 241 | -------------------------- 242 | 243 | Access: public 244 | 245 | Parameters: None. 246 | 247 | Returns: A boolean indicating whether or not the file cache is readable. 248 | 249 | This function returns whether or not the file cache is readable. 250 | 251 | PagingFileCache::CanWrite() 252 | --------------------------- 253 | 254 | Access: public 255 | 256 | Parameters: None. 257 | 258 | Returns: A boolean indicating whether or not the file cache is writable. 259 | 260 | This function returns whether or not the file cache is writable. 261 | 262 | PagingFileCache::CanSeek() 263 | -------------------------- 264 | 265 | Access: public 266 | 267 | Parameters: None. 268 | 269 | Returns: A boolean indicating whether or not the file cache is seekable. 270 | 271 | This function returns whether or not the file cache is seekable. 272 | 273 | PagingFileCache::GetCurrPos() 274 | ----------------------------- 275 | 276 | Access: public 277 | 278 | Parameters: None. 279 | 280 | Returns: An integer containing the current file cache virtual position. 281 | 282 | This function returns the current virtual position within the file cache. 283 | 284 | PagingFileCache::GetMaxPos() 285 | ---------------------------- 286 | 287 | Access: public 288 | 289 | Parameters: None. 290 | 291 | Returns: An integer containing the current file cache maximum virtual position. 292 | 293 | This function returns the current maximum virtual position within the file cache. 294 | 295 | PagingFileCache::Seek($offset, $whence = SEEK_SET) 296 | -------------------------------------------------- 297 | 298 | Access: public 299 | 300 | Parameters: 301 | 302 | * $offset - An integer containing an offset. 303 | * $whence - An integer containing one of SEEK_SET, SEEK_CUR, SEEK_END (Default SEEK_SET). 304 | 305 | Returns: A standard array of information. 306 | 307 | This function sets the virtual seek position if the file cache is seekable. 308 | 309 | PagingFileCache::Read($numbytes) 310 | -------------------------------- 311 | 312 | Access: public 313 | 314 | Parameters: 315 | 316 | * $numbytes - An integer containing the number of bytes to read. 317 | 318 | Returns: A standard array of information. 319 | 320 | This function reads in up to the specified number of bytes from the file cache from the current virtual position. Adjusts the current virtual position accordingly. If the end of the file cache is reached, eof will be set to true. 321 | 322 | Example usage: 323 | 324 | ```php 325 | Open("somefile.dat"); 332 | if (!$result["success"]) 333 | { 334 | var_dump($result); 335 | 336 | exit(); 337 | } 338 | 339 | // Read the data. 340 | $data = ""; 341 | do 342 | { 343 | $result = $pfc->Read(1024); 344 | if (!$result["success"]) 345 | { 346 | var_dump($result); 347 | 348 | exit(); 349 | } 350 | 351 | $data .= $result["data"]; 352 | } while (!$result["eof"]); 353 | 354 | $pfc->Close(); 355 | 356 | echo $data . "\n"; 357 | ?> 358 | ``` 359 | 360 | PagingFileCache::ReadUntil($matches, $options = array()) 361 | -------------------------------------------------------- 362 | 363 | Access: public 364 | 365 | Parameters: 366 | 367 | * $matches - An array containing one or more strings to match against. 368 | * $options - An array containing options (Default is array()). 369 | 370 | Returns: A standard array of information. 371 | 372 | This function reads data until it finds a match. 373 | 374 | The $options array accepts these options: 375 | 376 | * include_match - A boolean that indicates whether or not to include the match in the response data (Default is true). 377 | * rewind_match - A boolean that indicates whether or not to rewind to the start of the match (Default is false). 378 | * regex_match - A boolean that indicates whether or not to treat the strings in $matches as regular expressions (Default is false). 379 | * return_data - A boolean that indicates whether or not to return read data (Default is true). Can be used to seek to the start/end of some data. 380 | * min_buffer - An integer containing the size, in bytes, of the buffer to maintain for matching or a boolean to automatically calculate (Default is to auto-calculate). This should always be an integer when `regex_match` is true. 381 | 382 | PagingFileCache::ReadLine($includenewline = true) 383 | ------------------------------------------------- 384 | 385 | Access: public 386 | 387 | Parameters: 388 | 389 | * $includenewline - A boolean indicating whether or not to return the newline (Default is true). 390 | 391 | Returns: A standard array of information. 392 | 393 | This function calls `ReadUntil()` to retrieve content up to and including the next newline and returns it. 394 | 395 | PagingFileCache::ReadCSV($nulls = false, $separator = ",", $enclosure = "\"") 396 | ----------------------------------------------------------------------------- 397 | 398 | Access: public 399 | 400 | Parameters: 401 | 402 | * $nulls - A boolean indicating whether or not to treat empty columns as null (Default is false). 403 | * $separator - A string containing a single character column separator (Default is ,). 404 | * $enclosure - A string containing a single character enclosure start/end (Default is "). 405 | 406 | Returns: A standard array of information. 407 | 408 | This function reads a CSV record and returns it. 409 | 410 | PagingFileCache::Write($data) 411 | ----------------------------- 412 | 413 | Access: public 414 | 415 | Parameters: 416 | 417 | * $data - A string containing the data to write. 418 | 419 | Returns: A standard array of information. 420 | 421 | This function writes data to the file cache. 422 | 423 | PagingFileCache::WriteCSV($record) 424 | ---------------------------------- 425 | 426 | Access: public 427 | 428 | Parameters: 429 | 430 | * $record - An array of values containing the column data to write. 431 | 432 | Returns: A standard array of information. 433 | 434 | This function writes a record in the CSV file format. 435 | 436 | PagingFileCache::Sync($final = false) 437 | ------------------------------------- 438 | 439 | Access: public 440 | 441 | Parameters: 442 | 443 | * $final - A boolean indicating whether or not the final write operation has occurred (Default is false). 444 | 445 | Returns: A standard array of information. 446 | 447 | This function commits unwritten data to disk/memory. When $final is true, partial data at the end of the file is written and writing is disabled. 448 | 449 | PagingFileCache::GetData() 450 | -------------------------- 451 | 452 | Access: public 453 | 454 | Parameters: None. 455 | 456 | Returns: A string containing the current raw data in the file cache or a boolean of false if using a real file. 457 | 458 | This function returns the raw data set by `SetData()` plus any modifications. When in streaming/write only mode, this also clears the internal raw data and mapped pages and adjusts the base virtual position. 459 | 460 | PagingFileCache::InternalSeek($pos) 461 | ----------------------------------- 462 | 463 | Access: protected 464 | 465 | Parameters: 466 | 467 | * $pos - An integer containing the real position to seek to. 468 | 469 | Returns: A boolean indicating whether or not the seek operation was successful. 470 | 471 | This internal function seeks to a real position on disk. This does not change the virtual position information. 472 | 473 | PagingFileCache::UnloadExcessPages() 474 | ------------------------------------ 475 | 476 | Access: protected 477 | 478 | Parameters: None. 479 | 480 | Returns: Nothing. 481 | 482 | This internal function unloads about 25% of the loaded pages in memory when the total number of pages exceeds the maximum allowed. Modified pages are written to disk/memory before being unloaded. 483 | 484 | PagingFileCache::LoadPageForCurrPos() 485 | ------------------------------------- 486 | 487 | Access: protected 488 | 489 | Parameters: None. 490 | 491 | Returns: A standard array of information. 492 | 493 | This internal function loads the page for the current position from disk/memory. Returns the pagemap position and offset on success. 494 | 495 | PagingFileCache::PostLoadPageData(&$data, $pagepos) 496 | --------------------------------------------------- 497 | 498 | Access: protected 499 | 500 | Parameters: 501 | 502 | * $data - A string reference to the page data to load/modify. 503 | * $pagepos - An integer containing the page position in the page map. 504 | 505 | Returns: A boolean of true if page data was loaded successfully, false otherwise. 506 | 507 | This function is designed to be overridden by other classes that might decrypt or validate per-page data. 508 | 509 | PagingFileCache::SavePage($pagepos) 510 | ----------------------------------- 511 | 512 | Access: protected 513 | 514 | Parameters: 515 | 516 | * $pagepos - An integer containing the page position in the page map. 517 | 518 | Returns: A standard array of information. 519 | 520 | This internal function saves a modified page to disk/memory. 521 | 522 | PagingFileCache::PreSavePageData(&$data, $pagepos) 523 | -------------------------------------------------- 524 | 525 | Access: protected 526 | 527 | Parameters: 528 | 529 | * $data - A string reference to the page data to modify/save. 530 | * $pagepos - An integer containing the page position in the page map. 531 | 532 | Returns: A boolean of true if page data was loaded successfully, false otherwise. 533 | 534 | This function is designed to be overridden by other classes that might encrypt per-page data. 535 | 536 | PagingFileCache::PFCTranslate($format, ...) 537 | ------------------------------------------- 538 | 539 | Access: _internal_ static 540 | 541 | Parameters: 542 | 543 | * $format - A string containing valid sprintf() format specifiers. 544 | 545 | Returns: A string containing a translation. 546 | 547 | This internal static function takes input strings and translates them from English to some other language if CS_TRANSLATE_FUNC is defined to be a valid PHP function name. 548 | -------------------------------------------------------------------------------- /docs/ifds_specification.md: -------------------------------------------------------------------------------- 1 | Incredibly Flexible Data Storage (IFDS) Specification 2 | ----------------------------------------------------- 3 | 4 | This document contains the official specification for the Incredibly Flexible Data Storage (IFDS) file format. This specification is released under a Creative Commons 0 (CC0) license. 5 | 6 | The current version of this specification is: 1.0. 7 | 8 | All data is stored in big endian format. For example, a 32-bit integer containing the value of 1 would be stored as `00 00 00 01`. IFDS supports up to 2^64 storage, so implementations must use/support a minimum of 64-bit integers. 9 | 10 | Every IFDS file has a name mapping and object ID table. The free space table is optional and only created when an object/data is resized or moved. 11 | 12 | The IFDS file format is carefully designed such that external tools can read and verify files in the IFDS file format without having to understand the contents of the data being stored. 13 | 14 | IFDS Header 15 | ----------- 16 | 17 | Every IFDS file format starts with the IFDS header: 18 | 19 | ``` 20 | 0x80 | length + magic string (Default is "\x89IFDS\r\n\x00\x1A\n"). Magic string must end with "\r\n\x00\x1A\n". 21 | 1 byte IFDS major version (newer major version = completely incompatible with previous versions but also very unlikely to ever change in the first place) 22 | 1 byte IFDS minor version (newer minor version = probably minor/compatible changes) 23 | 2 byte app/custom format major version 24 | 2 byte app/custom format minor version 25 | 2 byte app/custom format patch/build number 26 | 4 byte enabled format features (bit field) 27 | 4 byte enabled app features (bit field) 28 | 8 byte base date (UNIX timestamp / 86400) 29 | 8 byte name table position 30 | 8 byte ID chunks table position 31 | 8 byte free space chunks table position 32 | 4 byte CRC-32 33 | ``` 34 | 35 | Format Features 36 | --------------- 37 | 38 | Various features can be enabled that alter the file format. Currently, three format features are defined. 39 | 40 | * Bit 0: Node IDs. The object/node ID is appended after the object size. Mostly redundant but useful when reading streaming content or objects that will be frequently added and removed. 41 | * Bit 1: Object ID table structure size. The object ID table has a 2 byte structure size to allow for performing a single read operation to load an object. Recommended for improved read performance. 42 | * Bit 2: Object ID table last access. The object ID table has a 2 byte last access day. Used to sort objects in large files when optimizing so that the most recently accessed objects generally appear earlier in the file. 43 | 44 | App Features 45 | ------------ 46 | 47 | Application-defined features can be specified. These features could be used to alter application-defined node types. 48 | 49 | Name Table (Key-ID map) 50 | ----------------------- 51 | 52 | The name table maps string keys to object IDs. 53 | 54 | ID Chunks Table (Fixed array) 55 | ----------------------------- 56 | 57 | The ID chunks table points to ID table entries structures. 58 | 59 | Format of each entry: 60 | 61 | ``` 62 | 8 byte ID table entries position 63 | 2 byte number of unassigned IDs 64 | ``` 65 | 66 | Supports a maximum of 65,536 entries in this fixed array. 67 | 68 | ID Table Entries (Fixed array) 69 | ------------------------------ 70 | 71 | Contains a pointer to an object, object size, and last access/modification information for each object ID. The actual size of each entry in this table can vary depending on Format Features and how large the file is. 72 | 73 | The first object ID is 1 but is stored at position 0. 2 => 1, 3 => 2, etc. 74 | 75 | Format of each entry: 76 | 77 | ``` 78 | 2/4/8 byte object position 79 | [2 byte structure size] 80 | [2 byte last access day] 81 | ``` 82 | 83 | The integer size of the object position is determined by the fixed array size. Structure size and last access day exist for each entry when relevant Format Features are enabled. Structure size feature is recommended for improved read performance. 84 | 85 | Supports a maximum of 65,536 entries in this fixed array. 86 | 87 | Free Space Chunks Table (Fixed array) 88 | ------------------------------------- 89 | 90 | The free space chunks table is not defined for all files. 91 | 92 | Format of each entry: 93 | 94 | ``` 95 | 8 byte free space table entries position 96 | 4 byte largest free space 97 | ``` 98 | 99 | The design of the free space structures has notable performance and security implications: 100 | 101 | * Position/size information can be imprecise or intentionally falsified. 102 | * Free space table structures can overlay valid data. 103 | 104 | Implementations must never trust the information found in the free space structures. When a problem is detected, just assume the space is actually full (i.e. 0 bytes free). 105 | 106 | There is no maximum number of entries in this fixed array but implementations should cap the number of allowed entries to the size of the file. 107 | 108 | Free Space Table Entries (Fixed array) 109 | -------------------------------------- 110 | 111 | Each free space table tracks up to 4.2GB of storage. Each entry in this table tracks the largest free space in a 65,536 byte chunk and the starting position of that space using just 4 bytes of data. 112 | 113 | Format of each entry: 114 | 115 | ``` 116 | 2 byte largest free space 117 | 2 byte first space start position 118 | ``` 119 | 120 | There are two combinations of free space and position that have special meaning: 121 | 122 | * 0x0000FFFF = 65,536 bytes free 123 | * 0xFFFFFFFF = 0 bytes free 124 | 125 | These special cases allows the values 0 through 65,536 to be represented in just 2 bytes. 126 | 127 | Supports a maximum of 65,536 entries in this fixed array. 128 | 129 | Free space tables are not defined for all files. When defined, entries not specified after the last entry are assumed to be full. 130 | 131 | Streaming 132 | --------- 133 | 134 | If the name table position and ID chunks table position of the header are zero, then this is a streamed file and critical data is located at the end of the file. 135 | 136 | Restrictions when generating streamed files: Insert before the end, deletions, and detach/attach are not allowed. Linked nodes "next" pointers are invalid. 137 | 138 | Note that even if the entire file hasn't been streamed, various internal aspects within the file may have been streamed during construction (e.g. interleaved data channels). 139 | 140 | The last node of a streamed file is a terminating DATA CHUNKS containing finalized header information: 141 | 142 | ``` 143 | 1 byte (0x3F) 144 | 1 byte (0x01) 145 | 2 byte data chunk size (16) 146 | 8 byte name table position 147 | 8 byte ID chunks table position 148 | 4 byte CRC-32 149 | ``` 150 | 151 | Upon opening a completed streaming file, load the 16 bytes from the end to get the name and ID chunks table positions. 152 | 153 | Opening an incomplete streaming file (e.g. a file being read over a network) is possible but with these restrictions: There is no way to obtain the last node, name table, or ID chunks table position in advance. However, an application can use the node type and encoder/decoder byte of a node to determine that node's purpose. Also, when the previous node ID for a node exists and is 0, the node is the start of the linked list for the structure. 154 | 155 | When the Node ID feature bit is set, the application can also keep track of the current node ID and the previous node IDs for each structure. 156 | 157 | Objects 158 | ------- 159 | 160 | Each object contains a data structure type byte (deleted, raw data, map, etc.), a data encoder + data type byte, 2 byte object size, optional format information (e.g. object ID), data structure information, and a 4 byte CRC-32. 161 | 162 | The maximum size of any object may not be over 32,767 bytes (2^15 - 1). This limit allows for a single read operation of both structure and the start of most DATA CHUNKS sections. 163 | 164 | The first byte of an object is the data structure type: 165 | 166 | * 0 - Deleted/Padding byte (not actually an object) 167 | * 1 - Raw data 168 | * 2 - Fixed array 169 | * 3 - Double-linked list and nodes 170 | * 4-31 - Reserved 171 | * 32-62 - Application-defined data structure 172 | * 63 (0x3F) - Reserved for DATA CHUNKS 173 | 174 | The two high bits of this byte are: 175 | 176 | * Bit 6 - Leaf node (For structures with nodes - e.g. linked list nodes) 177 | * Bit 7 - Streamed (Structure and object pointers need to be updated before the application can access the structure) 178 | 179 | If the data structure type is not 0 (i.e. not a Deleted/Padding byte), the next byte of an object is the data encoder/data type byte: 180 | 181 | * 0 - None (linked list header, NULL data, etc. - bits 6 and 7 must also be 0) 182 | * 1 - Raw data (No data decoding) 183 | * 2 - Key-ID map 184 | * 3 - Key-value map 185 | * 4-15 - Reserved 186 | * 16-63 - Application/implementation-defined (e.g. for a custom file format, 16 could be defined as JSON, 17 = JPEG, 18 = PNG, ...) 187 | 188 | The two high bits (6 and 7) of this byte are: 189 | 190 | * 0 - No data 191 | * 1 - INTERNAL DATA used (DATA CHUNKS not used) 192 | * 2 - DATA CHUNKS used, seekable (DATA locations table 0x02 follows immediately after this object) 193 | * 3 - DATA CHUNKS used, streaming, uses virtual channels (Interleaved DATA CHUNKS are used) 194 | 195 | When bits 6 and 7 of this byte are equal to 1, INTERNAL DATA is used. Data to be stored must fit inside the object. This is recommended only when using 3KB or less of data. 196 | 197 | INTERNAL DATA is placed at the end of the object with a 2 byte size appended to the data. This allows data to shrink and grow inside an object, which helps minimize file fragmentation. 198 | 199 | Users of IFDS implementations must never assume that position information in an IFDS file is static and won't change. All objects are free to move to other locations within an IFDS file - even outside the application the IFDS file is intended to be used with. 200 | 201 | Key-ID/Key-value Map 202 | -------------------- 203 | 204 | Each entry of a key-ID map is encoded as follows: 205 | 206 | ``` 207 | 2 byte length + key (high bit of length: 0 = signed integer, 1 = binary string/data) 208 | 4 byte object ID 209 | ``` 210 | 211 | Each entry of a key-value map is encoded as follows: 212 | 213 | ``` 214 | 2 byte length + key (high bit of length: 0 = signed integer, 1 = binary string/data) 215 | 2 byte length (minus high bit) + 2 byte second length (only if high bit of the first length is set) + value 216 | ``` 217 | 218 | For integer keys, when length is 1, 2, 4, or 8, that many bytes follow containing the actual key. Signed integers are stored. For all other small, positive integer keys from 0 through 32,767, the length is just used by itself as the key. String keys can be up to 32,767 bytes in length. 219 | 220 | The length of values of a key-value map can technically be up to 2.1GB but should be practically limited to just a few MB. Implementations and applications may enforce hard limits on the length that can be used for a single value. 221 | 222 | Object: Raw Data 223 | ----------------- 224 | 225 | Stores raw data. 226 | 227 | ``` 228 | 1 byte data structure type (1) 229 | 1 byte data encoder/decoder 230 | 2 byte structure size 231 | [4 byte node ID] 232 | INTERNAL DATA 233 | 4 byte CRC-32 234 | DATA CHUNKS 235 | ``` 236 | 237 | Object: Fixed Array 238 | -------------------- 239 | 240 | Stores sequential fixed-size entries. 241 | 242 | ``` 243 | 1 byte data structure type (2) 244 | 1 byte data encoder/decoder (1) 245 | 2 byte structure size 246 | [4 byte node ID] 247 | 4 byte entry size 248 | 4 byte num entries 249 | INTERNAL DATA 250 | 4 byte CRC-32 251 | DATA CHUNKS 252 | ``` 253 | 254 | The total amount of data stored is supposed to be `entry size * num entries`. However, users of fixed arrays should not rely on the number of entries to be correct and should validate that the entry size is correct before reading any data. 255 | 256 | Object: Double-linked List 257 | --------------------------- 258 | 259 | Stores the head of a double-linked list. 260 | 261 | ``` 262 | 1 byte data structure type (3) 263 | 1 byte data encoder/decoder (0) 264 | 2 byte structure size 265 | [4 byte node ID] 266 | 4 byte num nodes 267 | 4 byte first node ID 268 | 4 byte last node ID 269 | 4 byte CRC-32 270 | ``` 271 | 272 | This object generally does not store any data as the data in a linked list is usually stored in the linked list nodes themselves. 273 | 274 | Object: Linked List Node 275 | ------------------------- 276 | 277 | Stores a node for a double-linked list. 278 | 279 | ``` 280 | 1 byte data structure type (3 | 0x40) 281 | 1 byte data encoder/decoder 282 | 2 byte structure size 283 | [4 byte node ID] 284 | 4 byte prev node ID 285 | 4 byte next node ID 286 | INTERNAL DATA 287 | 4 byte CRC-32 288 | DATA CHUNKS 289 | ``` 290 | 291 | When attached to a linked list, the first and last node point at the linked list head ID (if it has one). 292 | 293 | When streaming, next node IDs should point at the linked list head ID. Next node IDs will need to be corrected before accessing a streamed linked list for the first time. 294 | 295 | Object: Application-defined Data Structure/Node 296 | ------------------------------------------------ 297 | 298 | Stores a custom data structure. 299 | 300 | ``` 301 | 1 byte data structure type (32-62) 302 | 1 byte data encoder/decoder 303 | 2 byte structure size 304 | [4 byte node ID] 305 | Application-defined type info 306 | INTERNAL DATA 307 | 4 byte CRC-32 308 | DATA CHUNKS 309 | ``` 310 | 311 | DATA CHUNKS 312 | ----------- 313 | 314 | Data that does not fit inside an object is stored in DATA CHUNKS. DATA CHUNKS come in two flavors: Dynamic seekable and static interleaved multi-channel. Dynamic seekable is tracked using a special DATA locations table, which allows for rapidly storing and retrieving up to 280TB of data scattered around any IFDS file. Static interleaved multi-channel doesn't support seeking but instead supports interleaved multi-channel data storage and can theoretically store up to 17EB. 315 | 316 | Note that dynamic seekable DATA CHUNKS can be stored anywhere in a file and can even be stored out of order. Data for an object can appear both before and after an object. 317 | 318 | If the first byte of an object's structure type is 0x3F, then it is actually DATA CHUNKS. 319 | 320 | Seekable DATA chunk: 321 | 322 | ``` 323 | 1 byte = 0x3F (Invalid data structure type specifically reserved for DATA CHUNKS that usually follow an object node) 324 | 1 byte = 0x00 and 0x01 (DATA chunk and DATA chunk w/ termination respectively) 325 | 2 byte data size - Values up to 65528 are valid. 326 | The 65528 limit allows exactly 65536 bytes to be retrieved per read operation for memory alignment but not disk read alignment purposes. 327 | Disk alignment isn't really possible anyway due to PagingFileCache. 328 | DATA 329 | 4 byte CRC-32 330 | ``` 331 | 332 | Interleaved DATA chunk: 333 | 334 | ``` 335 | 1 byte = 0x3F | 0x80 (DATA chunk with streaming/interleaved bit set) 336 | 1 byte = 0x00 and 0x01 (DATA chunk and DATA chunk w/ termination respectively) 337 | 2 byte data size - Values up to 65526 are valid. 338 | 2 byte virtual channel ID (application/implementation-defined) 339 | DATA 340 | 4 byte CRC-32 341 | ``` 342 | 343 | When interleaving, all DATA chunks for the object must be interleaved. This allows external tools to differentiate between interleaved and non-interleaved DATA CHUNKS. Non-interleaved is both seekable and streamable while interleaved is streamable only. No particular limit on data length. 344 | 345 | An interleaved channel is terminated when using 0x01. 346 | 347 | DATA CHUNKS itself always ends when channel 0 is terminated. Other channels MAY remain active across multiple data structure nodes in a single structure (application-defined). This allows for scenarios such as constructing two linked lists and flushing data accumulated for the second list every few seconds and then picking up again in the first list after that. 348 | 349 | Note that interleaved data may only be written one time, must be written sequentially, and is considered read only after channel 0 is closed. 350 | 351 | DATA locations table: 352 | 353 | ``` 354 | 1 byte = 0x3F 355 | 1 byte = 0x02 (DATA location table) 356 | 2 byte num entries 357 | (2 byte number of full seekable DATA chunks (65,536 bytes each) + 8 byte seekable DATA chunks start position) * num entries 358 | 2 byte last seekable DATA chunk size + 8 byte seekable DATA chunk start position 359 | 4 byte CRC-32 360 | ``` 361 | 362 | This table must appear immediately after a seekable object. When appending new max DATA chunks and num entries is 65535 (0xFFFF), merge down until num entries is 52268 (80%). Maximum data size supported is approximately 280TB. 363 | 364 | Implementation Issues 365 | --------------------- 366 | 367 | Implementing IFDS is not a simple, straightforward task. Even veteran software developers will find IFDS to be a challenge to implement both correctly and securely. The PHP reference implementation weighs in at around 5,600 lines of fairly dense code and is intended as a reference rather than being a highly optimized implementation of the IFDS specification (e.g. some math is expanded to better mirror the specification). 368 | 369 | Here are several difficult "gotchas" to watch out for when implementing IFDS in a new language/library: 370 | 371 | * False object and free space size reporting. Whether from data corruption or malicious intent, injecting false size information can lead to buffer overflows or corrupting other data. 372 | * Object ID map tables should not use object IDs. Almost certainly done with malicious intent, using object IDs in the object ID map can lead to data corruption. A well-written library will detect and clear the obvious attempt to cause problems. 373 | * There are a number of "chicken or the egg" problems: First, writing a IFDS-based file requires implementing the entire specification first, which makes a "test often" approach virtually impossible. Second, writing most objects such that they appear before their data requires knowing the size of the data but not all data sizes can be known in advance. Third, the free space mapping table, when defined, will tend to occupy free space. 374 | * Linked lists might have loops. It is possible to maliciously create a loop in a linked list. It is harder to detect loops in linked lists. 375 | * Deciding what to do with objects that are invalid. Every object has a CRC-32 associated with it. Ignoring invalid objects might create issues later on but simply failing when encountering an invalid object might cause a file to fail to load that would otherwise be fine. 376 | * Managing memory can get tricky. This file format is designed to handle extremely large data storage far beyond what today's OSes can generally handle. Actively managing and limiting system memory usage when reading and writing IFDS files is a very important feature of any IFDS library implementation. 377 | * Object IDs start at 1 but the ID table starts at 0. Off-by-one errors can result in reading from or writing to the wrong location. 378 | * Unassigned IDs only go up to 65,535. Each ID table supports up to 65,536 entries while the ID chunks table will only report up to 65,535 unassigned IDs. Implementations should really only care about zero vs. non-zero when locating the first available, unused object ID. 379 | * Merge down operations are exceptionally rare. The DATA locations table, when it gets full, should be merged down to make room for more data. The process effectively defragments a portion of DATA chunks by combining them. The earliest this process occurs is after 4.2GB of fragmented data has been written. Due to the rarity of this happening, merge down code may contain bugs and be less frequently tested. 380 | 381 | Implementation Patterns 382 | ----------------------- 383 | 384 | Creating an object follows this procedure: 385 | 386 | * Reserve/Assign an object ID. 387 | * If the object data size is known, attempt to find a free space that is large enough for the data but no more than 20% larger. 388 | * If the object data is of unknown size OR free space was not found, create the object at the end of the file. 389 | * Update the ID table entries structure with the new file position. 390 | 391 | Moving an object follows this procedure: 392 | 393 | * Attempt to find a free space that is large enough for the data. 394 | * If not found, copy the object to the end of the file, zero out the old bytes, and update the free space table. 395 | * Update the ID table entries structure with the new file position. 396 | 397 | Adding bytes to the free space table follows this procedure: 398 | 399 | * Fill the space with zeroes. 400 | * Find the largest size for each 65536 byte chunk. 401 | * Update the first space start position and largest size in the free space table. 402 | 403 | Deleting an object follows this procedure: 404 | 405 | * Verify that the object has detached all connections (e.g. linked list nodes). 406 | * Delete the DATA CHUNKS (if any). 407 | * If the object ID is not zero (anything but those in the header), zero the file position in the ID table entries structure and reduce assignment count in chunks table structure. 408 | * Update free space table. 409 | 410 | Optimizing the file follows this procedure (in a new, temporary file): 411 | 412 | * Write header. Set creation date to today. 413 | * Write name table. 414 | * Write placeholder ID chunks table and ID table entries structures. 415 | * Copy each object in order of next linked list node (when handling a list), last access day (descending), and object ID (ascending). Reset last access day to 0. Update positions in the ID table entries structure. Minimize the structure size of each object and recalculate the CRC-32. 416 | * Write ID chunks table and ID table entries structures. Reset last access day to 0. 417 | * Finalize header. 418 | * Delete original file. 419 | * Rename new file to original file. 420 | -------------------------------------------------------------------------------- /docs/ifds_conf.md: -------------------------------------------------------------------------------- 1 | IFDS_Conf and IFDS_ConfDef Classed: 'support/ifds_conf.php' 2 | ============================================================ 3 | 4 | The IFDS_Conf class is an example implementation of a possible replacement format for configuration files. Utilizes the Incredibly Flexible Data Storage (IFDS) file format for storing and reading application configuration information in a binary/machine readable format. Released under a MIT or LGPL license, your choice. 5 | 6 | The IFDS_ConfDef class is an example implementation of a potential file format for making it possible to create a universal application configuration tool. Utilizes the Incredibly Flexible Data Storage (IFDS) file format for storing and reading contexts, options, selectable option values, and multilingual documentation. 7 | 8 | Example usage can be seen in the main IFDS documentation and the IFDS test suite. 9 | 10 | IFDS_Conf::Create($pfcfilename, $options = array()) 11 | --------------------------------------------------- 12 | 13 | Access: public 14 | 15 | Parameters: 16 | 17 | * $pfcfilename - An instance of PagingFileCache, a string containing a filename, or a boolean of false. 18 | * $options - An array containing options to set (Default is array()). 19 | 20 | Returns: A standard array of information. 21 | 22 | This function creates an IFDS configuration format file with the specified options. 23 | 24 | The $options array accepts these options: 25 | 26 | * app - A string containing the application this configuration is intended to be used with (Default is ""). 27 | * ver - A string containing the version of the application this configuration is intended to be used with (Default is ""). 28 | * charset - A string containing the character set of Strings (Default is "utf-8"). 29 | 30 | IFDS_Conf::Open($pfcfilename) 31 | ----------------------------- 32 | 33 | Access: public 34 | 35 | Parameters: 36 | 37 | * $pfcfilename - An instance of PagingFileCache, a string containing a filename, or a boolean of false. 38 | 39 | Returns: A standard array of information. 40 | 41 | This function opens an IFDS configuration format file. 42 | 43 | IFDS_Conf::GetIFDS() 44 | -------------------- 45 | 46 | Access: public 47 | 48 | Parameters: None. 49 | 50 | Returns: The internal IFDS object. 51 | 52 | This function returns the internal IFDS object. Can be useful for custom modifications (e.g. adding metadata for a editor, compiler, or tool). 53 | 54 | IFDS_Conf::Close() 55 | ------------------ 56 | 57 | Access: public 58 | 59 | Parameters: None. 60 | 61 | Returns: Nothing. 62 | 63 | This function closes an open IFDS configuration format file. 64 | 65 | IFDS_Conf::Save($flush = true) 66 | ------------------------------ 67 | 68 | Access: public 69 | 70 | Parameters: 71 | 72 | * $flush - A boolean indicating whether or not to also flush data in the IFDS object (Default is true). 73 | 74 | Returns: A standard array of information. 75 | 76 | This function saves internal state information and optionally flushes the internal IFDS object data as well. 77 | 78 | IFDS_Conf::WriteMetadata() 79 | -------------------------- 80 | 81 | Access: protected 82 | 83 | Parameters: None. 84 | 85 | Returns: A standard array of information. 86 | 87 | This internal function writes updated metadata to the named metadata object. 88 | 89 | IFDS_Conf::GetApp() 90 | ------------------- 91 | 92 | Access: public 93 | 94 | Parameters: None. 95 | 96 | Returns: A string containing the application. 97 | 98 | This function returns the application this configuration file is intended to be used with. 99 | 100 | IFDS_Conf::SetApp($app) 101 | ----------------------- 102 | 103 | Access: public 104 | 105 | Parameters: 106 | 107 | * $app - A string containing the application. 108 | 109 | Returns: A standard array of information. 110 | 111 | This function sets the application this configuration file is intended to be used with. 112 | 113 | IFDS_Conf::GetVer() 114 | ------------------- 115 | 116 | Access: public 117 | 118 | Parameters: None. 119 | 120 | Returns: A string containing the application version. 121 | 122 | This function returns the version of the application this configuration file is intended to be used with. 123 | 124 | IFDS_Conf::SetVer($ver) 125 | ----------------------- 126 | 127 | Access: public 128 | 129 | Parameters: 130 | 131 | * $ver - A string containing the application version. 132 | 133 | Returns: A standard array of information. 134 | 135 | This function sets the version of the application this configuration file is intended to be used with. 136 | 137 | IFDS_Conf::GetCharset() 138 | ----------------------- 139 | 140 | Access: public 141 | 142 | Parameters: None. 143 | 144 | Returns: A string containing the character set for string options. 145 | 146 | This function returns the character set for string options. 147 | 148 | IFDS_Conf::SetCharset($charset) 149 | ------------------------------- 150 | 151 | Access: public 152 | 153 | Parameters: 154 | 155 | * $charset - A string containing the character set. 156 | 157 | Returns: A standard array of information. 158 | 159 | This function sets the character set for string options. 160 | 161 | IFDS_Conf::GetSectionsMap() 162 | --------------------------- 163 | 164 | Access: public 165 | 166 | Parameters: None. 167 | 168 | Returns: An array containing a section name to ID mapping. 169 | 170 | This function returns the section name to ID map. 171 | 172 | IFDS_Conf::CreateSection($name, $contextname) 173 | --------------------------------------------- 174 | 175 | Access: public 176 | 177 | Parameters: 178 | 179 | * $name - A string containing the name of the new section. Must be unique. 180 | * $contextname - A string containing the name of a configuration definition context. 181 | 182 | Returns: A standard array of information. 183 | 184 | This function creates a new configuration section. A context allows a generic configuration tool to manage the configuration file via the IFDS configuration definition file format. 185 | 186 | IFDS_Conf::DeleteSection($name) 187 | ------------------------------- 188 | 189 | Access: public 190 | 191 | Parameters: 192 | 193 | * $name - A string containing the name of the section to delete. 194 | 195 | Returns: A standard array of information. 196 | 197 | This function deletes a configuration section. 198 | 199 | IFDS_Conf::RenameSection($oldname, $newname) 200 | -------------------------------------------- 201 | 202 | Access: public 203 | 204 | Parameters: 205 | 206 | * $oldname - A string containing the name of the section to rename. 207 | * $newname - A string containing the new name of the section. Must be unique. 208 | 209 | Returns: A standard array of information. 210 | 211 | This function renames a configuration section. 212 | 213 | IFDS_Conf::ExtractTypeData(&$val, $pos, $type, $multiple) 214 | --------------------------------------------------------- 215 | 216 | Access: _internal_ static 217 | 218 | Parameters: 219 | 220 | * $val - A string containing data to extract type data from. 221 | * $pos - An integer containing the starting position in $val. 222 | * $type - An integer containing a valid data type. 223 | * $multiple - A boolean indicating whether or not to extract multiple values. 224 | 225 | Returns: An array containing extracted values. 226 | 227 | This internal function extracts type data for both the IFDS_Conf and IFDS_ConfDef classes. 228 | 229 | IFDS_Conf::GetSection($name) 230 | ---------------------------- 231 | 232 | Access: public 233 | 234 | Parameters: 235 | 236 | * $name - A string containing the name of the section to retrieve. 237 | 238 | Returns: A standard array of information. 239 | 240 | This function loads and extracts a configuration section. 241 | 242 | IFDS_Conf::AppendTypeData(&$val, $type, &$vals) 243 | ----------------------------------------------- 244 | 245 | Access: _internal_ static 246 | 247 | Parameters: 248 | 249 | * $val - A string to append type data to. 250 | * $type - An integer containing a valid data type. 251 | * $vals - An array containing values to append. 252 | 253 | Returns: Nothing. 254 | 255 | This internal function appends type data for both the IFDS_Conf and IFDS_ConfDef classes. 256 | 257 | IFDS_Conf::UpdateSection($obj, $options) 258 | ---------------------------------------- 259 | 260 | Access: public 261 | 262 | Parameters: 263 | 264 | * $obj - An instance of a IFDS_RefCountObj object to write section options to. 265 | * $options - An array containing section options. 266 | 267 | Returns: A standard array of information. 268 | 269 | This function prepares and writes section options to the section object. 270 | 271 | The empty string key in the $options array maps to a string containing the context. 272 | 273 | All other entries in the $options array maps a key to: 274 | 275 | * use - A boolean of true to use this option's value(s) or a boolean of false to use default value(s) or disable the section (for a section type). 276 | * type - An integer containing a valid data type. 277 | * vals - An array containing the values to store. 278 | 279 | Valid data types are one of: 280 | 281 | * IFDS_Conf::OPTION_TYPE_BOOL (0) - Boolean 282 | * IFDS_Conf::OPTION_TYPE_INT (1) - Integer (signed, variable size) 283 | * IFDS_Conf::OPTION_TYPE_FLOAT (2) - IEEE Float 284 | * IFDS_Conf::OPTION_TYPE_DOUBLE (3) - IEEE Double 285 | * IFDS_Conf::OPTION_TYPE_STRING (4) - String 286 | * IFDS_Conf::OPTION_TYPE_BINARY (5) - Binary data 287 | * IFDS_Conf::OPTION_TYPE_SECTION (6) - Section name 288 | 289 | IFDS_Conf::IFDSConfTranslate($format, ...) 290 | ------------------------------------------ 291 | 292 | Access: _internal_ static 293 | 294 | Parameters: 295 | 296 | * $format - A string containing valid sprintf() format specifiers. 297 | 298 | Returns: A string containing a translation. 299 | 300 | This internal static function takes input strings and translates them from English to some other language if CS_TRANSLATE_FUNC is defined to be a valid PHP function name. 301 | 302 | IFDS_ConfDef::Create($pfcfilename, $options = array()) 303 | ------------------------------------------------------ 304 | 305 | Access: public 306 | 307 | Parameters: 308 | 309 | * $pfcfilename - An instance of PagingFileCache, a string containing a filename, or a boolean of false. 310 | * $options - An array containing options to set (Default is array()). 311 | 312 | Returns: A standard array of information. 313 | 314 | This function creates an IFDS configuration definition format file with the specified options. 315 | 316 | The $options array accepts these options: 317 | 318 | * app - A string containing the application this configuration definition is intended to be used with (Default is ""). 319 | * ver - A string containing the version of the application this configuration definition is intended to be used with (Default is ""). 320 | * charset - A string containing the character set of Strings (Default is "utf-8"). 321 | 322 | IFDS_ConfDef::Open($pfcfilename) 323 | -------------------------------- 324 | 325 | Access: public 326 | 327 | Parameters: 328 | 329 | * $pfcfilename - An instance of PagingFileCache, a string containing a filename, or a boolean of false. 330 | 331 | Returns: A standard array of information. 332 | 333 | This function opens an IFDS configuration definition format file. 334 | 335 | IFDS_ConfDef::GetIFDS() 336 | ----------------------- 337 | 338 | Access: public 339 | 340 | Parameters: None. 341 | 342 | Returns: The internal IFDS object. 343 | 344 | This function returns the internal IFDS object. Can be useful for custom modifications (e.g. adding metadata for a editor, compiler, or tool). 345 | 346 | IFDS_ConfDef::Close() 347 | --------------------- 348 | 349 | Access: public 350 | 351 | Parameters: None. 352 | 353 | Returns: Nothing. 354 | 355 | This function closes an open IFDS configuration definition format file. 356 | 357 | IFDS_ConfDef::Save($flush = true) 358 | --------------------------------- 359 | 360 | Access: public 361 | 362 | Parameters: 363 | 364 | * $flush - A boolean indicating whether or not to also flush data in the IFDS object (Default is true). 365 | 366 | Returns: A standard array of information. 367 | 368 | This function saves internal state information and optionally flushes the internal IFDS object data as well. 369 | 370 | IFDS_ConfDef::WriteMetadata() 371 | ----------------------------- 372 | 373 | Access: protected 374 | 375 | Parameters: None. 376 | 377 | Returns: A standard array of information. 378 | 379 | This internal function writes updated metadata to the named metadata object. 380 | 381 | IFDS_ConfDef::GetApp() 382 | ---------------------- 383 | 384 | Access: public 385 | 386 | Parameters: None. 387 | 388 | Returns: A string containing the application. 389 | 390 | This function returns the application this configuration definition file is intended to be used with. 391 | 392 | IFDS_ConfDef::SetApp($app) 393 | -------------------------- 394 | 395 | Access: public 396 | 397 | Parameters: 398 | 399 | * $app - A string containing the application. 400 | 401 | Returns: A standard array of information. 402 | 403 | This function sets the application this configuration definition file is intended to be used with. 404 | 405 | IFDS_ConfDef::GetVer() 406 | ---------------------- 407 | 408 | Access: public 409 | 410 | Parameters: None. 411 | 412 | Returns: A string containing the application version. 413 | 414 | This function returns the version of the application this configuration definition file is intended to be used with. 415 | 416 | IFDS_ConfDef::SetVer($ver) 417 | -------------------------- 418 | 419 | Access: public 420 | 421 | Parameters: 422 | 423 | * $ver - A string containing the application version. 424 | 425 | Returns: A standard array of information. 426 | 427 | This function sets the version of the application this configuration definition file is intended to be used with. 428 | 429 | IFDS_ConfDef::GetCharset() 430 | -------------------------- 431 | 432 | Access: public 433 | 434 | Parameters: None. 435 | 436 | Returns: A string containing the character set for string options. 437 | 438 | This function returns the character set for string options. 439 | 440 | IFDS_ConfDef::SetCharset($charset) 441 | ---------------------------------- 442 | 443 | Access: public 444 | 445 | Parameters: 446 | 447 | * $charset - A string containing the character set. 448 | 449 | Returns: A standard array of information. 450 | 451 | This function sets the character set for string options. 452 | 453 | IFDS_ConfDef::GetContextsMap() 454 | ------------------------------ 455 | 456 | Access: public 457 | 458 | Parameters: None. 459 | 460 | Returns: An array containing a context name to ID mapping. 461 | 462 | This function returns the context name to ID map. The empty string key maps to a documentation object ID (Doc object ID) for the context. All other keys map to Option object IDs. 463 | 464 | IFDS_ConfDef::CreateContext($name) 465 | ---------------------------------- 466 | 467 | Access: public 468 | 469 | Parameters: 470 | 471 | * $name - A string containing the name of the context to create. 472 | 473 | Returns: A standard array of information. 474 | 475 | This function creates a new configuration context. Contexts contain options. Options specify data types, link to selectable values (or are freeform), and link to multilingual documentation objects. 476 | 477 | IFDS_ConfDef::DeleteContext($name) 478 | ---------------------------------- 479 | 480 | Access: public 481 | 482 | Parameters: 483 | 484 | * $name - A string containing the name of the context to create. 485 | 486 | Returns: A standard array of information. 487 | 488 | This function deletes a configuration context. Note that this only deletes the context object and removes it from the contexts map. Option objects and documentation objects are not deleted. 489 | 490 | IFDS_ConfDef::RenameContext($oldname, $newname) 491 | ----------------------------------------------- 492 | 493 | Access: public 494 | 495 | Parameters: 496 | 497 | * $oldname - A string containing the name of the context to rename. 498 | * $newname - A string containing the new name of the context. Must be unique. 499 | 500 | Returns: A standard array of information. 501 | 502 | This function renames a configuration context. 503 | 504 | IFDS_ConfDef::GetContext($name) 505 | ------------------------------- 506 | 507 | Access: public 508 | 509 | Parameters: 510 | 511 | * $name - A string containing the name of the context to retrieve. 512 | 513 | Returns: A standard array of information. 514 | 515 | This function retrieves a configuration context. 516 | 517 | IFDS_ConfDef::UpdateContext($obj, $optionsmap) 518 | ---------------------------------------------- 519 | 520 | Access: public 521 | 522 | Parameters: 523 | 524 | * $obj - An instance of a IFDS_RefCountObj object to write options to. 525 | * $options - An array containing options. 526 | 527 | Returns: A standard array of information. 528 | 529 | This function updates the context object. The empty string key maps to a documentation object ID (Doc object ID) for the context. All other keys map to Option object IDs. 530 | 531 | IFDS_ConfDef::GetOptionsList() 532 | ------------------------------ 533 | 534 | Access: public 535 | 536 | Parameters: None. 537 | 538 | Returns: An instance of a IFDS_RefCountObj object containing a linked list of all options. 539 | 540 | This function returns the IFDS linked list object containing all options. 541 | 542 | IFDS_ConfDef::CreateOption($type, $options = array()) 543 | ----------------------------------------------------- 544 | 545 | Access: public 546 | 547 | Parameters: 548 | 549 | * $type - An integer containing a valid data type. 550 | * $options - An array containing options for the option (Default is array()). 551 | 552 | Returns: A standard array of information. 553 | 554 | This function creates an option object with the specified type and options and attaches the object to the options linked list. 555 | 556 | Valid data types are one of: 557 | 558 | * IFDS_ConfDef::OPTION_TYPE_BOOL (0) - Boolean 559 | * IFDS_ConfDef::OPTION_TYPE_INT (1) - Integer (signed, variable size) 560 | * IFDS_ConfDef::OPTION_TYPE_FLOAT (2) - IEEE Float 561 | * IFDS_ConfDef::OPTION_TYPE_DOUBLE (3) - IEEE Double 562 | * IFDS_ConfDef::OPTION_TYPE_STRING (4) - String 563 | * IFDS_ConfDef::OPTION_TYPE_BINARY (5) - Binary data 564 | * IFDS_ConfDef::OPTION_TYPE_SECTION (6) - Section name 565 | 566 | The $options array accepts these options: 567 | 568 | * info - An integer containing a bitmask (Default is IFDS_ConfDef::OPTION_INFO_NORMAL). Possible values: IFDS_ConfDef::OPTION_INFO_NORMAL (0x00), IFDS_ConfDef::OPTION_INFO_DEPRECATED (0x01). 569 | * doc - An integer containing a documentation object ID (Default is 0). No documentation when 0. 570 | * values - An integer containing an option values object ID (Default is 0). Freeform when 0. Option values specify all possible/allowed values. 571 | * mimetype - A string containing a MIME type (Default is "application/octet-stream" for OPTION_TYPE_BINARY, "" for all other types). 572 | * defaults - An array containing option defaults (Default is array()). 573 | 574 | IFDS_ConfDef::DeleteOption($obj) 575 | -------------------------------- 576 | 577 | Access: public 578 | 579 | Parameters: 580 | 581 | * $obj - An instance of a IFDS_RefCountObj object containing an option to delete. 582 | 583 | Returns: A standard array of information. 584 | 585 | This function detaches and deletes an option object. Note that this only deletes the option object and removes it from the options linked list. Option Value objects and documentation objects are not deleted. 586 | 587 | IFDS_ConfDef::UpdateOption($obj, $type, $options = array()) 588 | ----------------------------------------------------------- 589 | 590 | Access: public 591 | 592 | Parameters: 593 | 594 | * $obj - An instance of a IFDS_RefCountObj object containing an option to update. 595 | * $type - An integer containing a valid data type. 596 | * $options - An array containing options for the option (Default is array()). 597 | 598 | Returns: A standard array of information. 599 | 600 | This function updates an option object with the specified type and options. See CreateOption() for details on $type and $options. 601 | 602 | IFDS_ConfDef::ExtractOptionData(&$data) 603 | --------------------------------------- 604 | 605 | Access: _internal_ static 606 | 607 | Parameters: 608 | 609 | * $data - A string containing option data to extract. 610 | 611 | Returns: An array of options on success, a boolean of false otherwise. 612 | 613 | This internal function extracts option data into an array. 614 | 615 | IFDS_ConfDef::GetOption($id) 616 | ---------------------------- 617 | 618 | Access: public 619 | 620 | Parameters: 621 | 622 | * $id - An integer containing an option object ID. 623 | 624 | Returns: A standard array of information. 625 | 626 | This function retrieves an option by its object ID. 627 | 628 | IFDS_ConfDef::CreateOptionValues($valuesmap) 629 | -------------------------------------------- 630 | 631 | Access: public 632 | 633 | Parameters: 634 | 635 | * $valuesmap - An array containing key-ID pairs to set. 636 | 637 | Returns: A standard array of information. 638 | 639 | This function creates a new option values object. Option values allow a configuration tool to display a list of selectable items, restrict data entry, or simply provide visual feedback that a value may be incorrect depending on data type, MIME type, or other factors. The keys are the values and the IDs are documentation object IDs. 640 | 641 | IFDS_ConfDef::DeleteOptionValues($obj) 642 | -------------------------------------- 643 | 644 | Access: public 645 | 646 | Parameters: 647 | 648 | * $obj - An instance of a IFDS_RefCountObj object containing an option values object to delete. 649 | 650 | Returns: A standard array of information. 651 | 652 | This function deletes an option values object. 653 | 654 | IFDS_ConfDef::UpdateOptionValues($obj, $valuesmap) 655 | -------------------------------------------------- 656 | 657 | Access: public 658 | 659 | Parameters: 660 | 661 | * $obj - An instance of a IFDS_RefCountObj object containing an option values object to update. 662 | * $valuesmap - An array containing key-ID pairs to set. 663 | 664 | Returns: A standard array of information. 665 | 666 | This function updates an option values object with new key-ID pairs. 667 | 668 | IFDS_ConfDef::GetOptionValues($id) 669 | ---------------------------------- 670 | 671 | Access: public 672 | 673 | Parameters: 674 | 675 | * $id - An integer containing an option values object ID. 676 | 677 | Returns: A standard array of information. 678 | 679 | This function retrieves the specified option values object and values map. 680 | 681 | IFDS_ConfDef::GetDocsList() 682 | --------------------------- 683 | 684 | Access: public 685 | 686 | Parameters: None. 687 | 688 | Returns: An instance of a IFDS_RefCountObj object containing a linked list of all documentation objects. 689 | 690 | This function returns the IFDS linked list object containing all documentation objects. 691 | 692 | IFDS_ConfDef::CreateDoc($langmap) 693 | --------------------------------- 694 | 695 | Access: public 696 | 697 | Parameters: 698 | 699 | * $langmap - An array containing IANA language codes mapped to strings to display. 700 | 701 | Returns: A standard array of information. 702 | 703 | This function creates a new documentation object and sets the language mapping. Can be used by a universal configuration tool to display localized, translated content to a user for contexts, options, and option values. 704 | 705 | IFDS_ConfDef::DeleteDoc($obj) 706 | ----------------------------- 707 | 708 | Access: public 709 | 710 | Parameters: 711 | 712 | * $obj - An instance of a IFDS_RefCountObj object containing a documentation object to delete. 713 | 714 | Returns: A standard array of information. 715 | 716 | This function deletes a documentation object. Note that this does not delete any references to the object. 717 | 718 | IFDS_ConfDef::UpdateDoc($obj, $langmap) 719 | --------------------------------------- 720 | 721 | Access: public 722 | 723 | Parameters: 724 | 725 | * $obj - An instance of a IFDS_RefCountObj object containing a documentation object to update. 726 | * $langmap - An array containing IANA language codes mapped to strings to display. 727 | 728 | Returns: A standard array of information. 729 | 730 | This function updates a documentation object with the specified language mapping. 731 | 732 | IFDS_ConfDef::GetDoc($id) 733 | ------------------------- 734 | 735 | Access: public 736 | 737 | Parameters: 738 | 739 | * $id - An integer containing a documentation object ID. 740 | 741 | Returns: A standard array of information. 742 | 743 | This function retrieves the specified documentation object and language map. 744 | 745 | IFDS_ConfDef::IFDSConfTranslate($format, ...) 746 | --------------------------------------------- 747 | 748 | Access: _internal_ static 749 | 750 | Parameters: 751 | 752 | * $format - A string containing valid sprintf() format specifiers. 753 | 754 | Returns: A string containing a translation. 755 | 756 | This internal static function takes input strings and translates them from English to some other language if CS_TRANSLATE_FUNC is defined to be a valid PHP function name. 757 | -------------------------------------------------------------------------------- /support/paging_file_cache.php: -------------------------------------------------------------------------------- 1 | open = false; 21 | $this->fp = false; 22 | $this->realpagesize = 4096; 23 | $this->reservedbytes = 0; 24 | $this->maxpages = 2048; 25 | 26 | $this->Close(); 27 | } 28 | 29 | public function __destroy() 30 | { 31 | $this->Close(); 32 | } 33 | 34 | public function GetPageSize() 35 | { 36 | return $this->pagesize; 37 | } 38 | 39 | public function GetRealPageSize() 40 | { 41 | return $this->realpagesize; 42 | } 43 | 44 | public function SetRealPageSize($size) 45 | { 46 | $this->realpagesize = $size; 47 | 48 | $this->pagesize = $this->realpagesize - $this->reservedbytes; 49 | } 50 | 51 | public function GetPageReservedBytes() 52 | { 53 | return $this->reservedbytes; 54 | } 55 | 56 | // The number of bytes guaranteed to be reserved per page. Useful for encryption operations. 57 | public function SetPageReservedBytes($numbytes) 58 | { 59 | $this->reservedbytes = $numbytes; 60 | 61 | $this->pagesize = $this->realpagesize - $this->reservedbytes; 62 | } 63 | 64 | public function GetMaxCachedPages() 65 | { 66 | return $this->maxpages; 67 | } 68 | 69 | public function SetMaxCachedPages($maxpages) 70 | { 71 | $this->maxpages = $maxpages; 72 | } 73 | 74 | public function GetNumCachedPages() 75 | { 76 | return count($this->pagemap); 77 | } 78 | 79 | // Maps a virtual position to a real position. 80 | public function GetRealPos($pos) 81 | { 82 | return ((int)($pos / $this->pagesize) * $this->realpagesize) + ($pos % $this->pagesize); 83 | } 84 | 85 | // Maps a real position to a virtual position. Note that this does not, by design, work properly when the position is in the reserved bytes space. 86 | public function GetVirtualPos($pos) 87 | { 88 | return ((int)($pos / $this->realpagesize) * $this->pagesize) + ($pos % $this->realpagesize); 89 | } 90 | 91 | public function SetData($data, $mode = self::PAGING_FILE_MODE_READ | self::PAGING_FILE_MODE_WRITE) 92 | { 93 | $this->Close(); 94 | 95 | $this->fp = str_split($data, $this->realpagesize); 96 | 97 | $this->open = true; 98 | 99 | $this->readable = ($mode & self::PAGING_FILE_MODE_READ ? true : false); 100 | $this->writable = ($mode & self::PAGING_FILE_MODE_WRITE ? true : false); 101 | $this->seekable = ($mode & self::PAGING_FILE_MODE_READ ? true : false); 102 | 103 | $this->realmaxpos = strlen($data); 104 | 105 | $this->maxpos = $this->GetVirtualPos($this->realmaxpos - $this->reservedbytes); 106 | 107 | return array("success" => true); 108 | } 109 | 110 | public function Open($filename, $mode = self::PAGING_FILE_MODE_READ | self::PAGING_FILE_MODE_WRITE) 111 | { 112 | $this->Close(); 113 | 114 | if ($mode & 0x03 === self::PAGING_FILE_MODE_READ | self::PAGING_FILE_MODE_WRITE) $mode2 = (file_exists($filename) ? "r+b" : "w+b"); 115 | else if ($mode & 0x03 === self::PAGING_FILE_MODE_READ) $mode2 = "rb"; 116 | else if ($mode & 0x03 === self::PAGING_FILE_MODE_WRITE) $mode2 = (file_exists($filename) ? "r+b" : "wb"); 117 | else return array("success" => false, "error" => self::PFCTranslate("Invalid mode specified."), "errorcode" => "invalid_mode"); 118 | 119 | $this->fp = @fopen($filename, $mode2); 120 | if ($this->fp === false) return array("success" => false, "error" => self::PFCTranslate("Unable to open the file."), "errorcode" => "fopen_failed"); 121 | 122 | $this->open = true; 123 | $this->readable = ($mode & self::PAGING_FILE_MODE_READ ? true : false); 124 | $this->writable = ($mode & self::PAGING_FILE_MODE_WRITE ? true : false); 125 | 126 | $metadata = stream_get_meta_data($this->fp); 127 | $this->seekable = $metadata["seekable"]; 128 | 129 | if (!$this->seekable) $this->realmaxpos = PHP_INT_MAX; 130 | else 131 | { 132 | fseek($this->fp, 0, SEEK_END); 133 | $this->realmaxpos = ftell($this->fp); 134 | fseek($this->fp, 0, SEEK_SET); 135 | } 136 | 137 | $this->maxpos = $this->GetVirtualPos($this->realmaxpos - $this->reservedbytes); 138 | 139 | return array("success" => true); 140 | } 141 | 142 | public function Close() 143 | { 144 | $this->Sync(true); 145 | 146 | if ($this->fp !== false && is_resource($this->fp)) fclose($this->fp); 147 | 148 | $this->fp = false; 149 | $this->readable = false; 150 | $this->writable = false; 151 | $this->seekable = false; 152 | $this->basepos = 0; 153 | $this->currpos = 0; 154 | $this->maxpos = 0; 155 | $this->realcurrpos = false; 156 | $this->realmaxpos = 0; 157 | $this->seekpos = 0; 158 | 159 | $this->pagemap = array(); 160 | $this->recentpages = array(); 161 | $this->pagesize = $this->realpagesize - $this->reservedbytes; 162 | } 163 | 164 | public function CanRead() 165 | { 166 | return $this->readable; 167 | } 168 | 169 | public function CanWrite() 170 | { 171 | return $this->writable; 172 | } 173 | 174 | public function CanSeek() 175 | { 176 | return $this->seekable; 177 | } 178 | 179 | public function GetCurrPos() 180 | { 181 | return $this->currpos; 182 | } 183 | 184 | public function GetMaxPos() 185 | { 186 | return $this->maxpos; 187 | } 188 | 189 | public function Seek($offset, $whence = SEEK_SET) 190 | { 191 | if (!$this->open) return array("success" => false, "error" => self::PFCTranslate("Unable to seek. File is not open."), "errorcode" => "file_not_open"); 192 | if (!$this->seekable) return array("success" => false, "error" => self::PFCTranslate("Unable to seek. File is not seekable."), "errorcode" => "file_not_seekable"); 193 | 194 | if ($whence === SEEK_SET) $this->currpos = $offset; 195 | else if ($whence === SEEK_CUR) $this->currpos += $offset; 196 | else if ($whence === SEEK_END) $this->currpos = $this->maxpos + $offset; 197 | else return array("success" => false, "error" => self::PFCTranslate("Invalid whence."), "errorcode" => "invalid_whence"); 198 | 199 | if ($this->currpos < 0) $this->currpos = 0; 200 | else if ($this->currpos > $this->maxpos) $this->currpos = $this->maxpos; 201 | 202 | $this->realcurrpos = false; 203 | 204 | return array("success" => true); 205 | } 206 | 207 | public function Read($numbytes) 208 | { 209 | if (!$this->open) return array("success" => false, "error" => self::PFCTranslate("Unable to read. File is not open."), "errorcode" => "file_not_open"); 210 | if (!$this->readable) return array("success" => false, "error" => self::PFCTranslate("Unable to read. File is not readable."), "errorcode" => "file_not_readable"); 211 | 212 | $data = ""; 213 | while ($numbytes > 0) 214 | { 215 | $result = $this->LoadPageForCurrPos(); 216 | if (!$result["success"]) return $result; 217 | 218 | $y = strlen($this->pagemap[$result["pos"]][0]); 219 | if ($y < 1 || $y <= $result["offset"]) break; 220 | 221 | $size = ($y > $result["offset"] + $numbytes ? $numbytes : $y - $result["offset"]); 222 | $data .= ($size === $y ? $this->pagemap[$result["pos"]][0] : substr($this->pagemap[$result["pos"]][0], $result["offset"], $size)); 223 | 224 | $numbytes -= $size; 225 | $this->currpos += $size; 226 | $this->realcurrpos = false; 227 | } 228 | 229 | return array("success" => true, "data" => $data, "eof" => ($numbytes > 0)); 230 | } 231 | 232 | public function ReadUntil($matches, $options = array()) 233 | { 234 | if (!$this->open) return array("success" => false, "error" => self::PFCTranslate("Unable to read. File is not open."), "errorcode" => "file_not_open"); 235 | if (!$this->readable) return array("success" => false, "error" => self::PFCTranslate("Unable to read. File is not readable."), "errorcode" => "file_not_readable"); 236 | 237 | if (!isset($options["include_match"])) $options["include_match"] = true; 238 | if (!isset($options["rewind_match"])) $options["rewind_match"] = false; 239 | if (!isset($options["regex_match"])) $options["regex_match"] = false; 240 | if (!isset($options["return_data"])) $options["return_data"] = true; 241 | 242 | if (!isset($options["min_buffer"]) || is_bool($options["min_buffer"])) 243 | { 244 | $options["min_buffer"] = 0; 245 | foreach ($matches as &$match) 246 | { 247 | $y = strlen($match); 248 | 249 | if ($options["min_buffer"] < $y) $options["min_buffer"] = $y; 250 | } 251 | } 252 | 253 | $origcurrpos = $this->currpos; 254 | $data = ""; 255 | $startpos = 0; 256 | $pos = false; 257 | $matchlen = 0; 258 | do 259 | { 260 | $result = $this->LoadPageForCurrPos(); 261 | if (!$result["success"]) return $result; 262 | 263 | $y = strlen($this->pagemap[$result["pos"]][0]); 264 | if ($y < 1 || $y <= $result["offset"]) return array("success" => true, "data" => $data, "eof" => true); 265 | 266 | $data .= substr($this->pagemap[$result["pos"]][0], $result["offset"]); 267 | $y2 = strlen($data); 268 | 269 | foreach ($matches as &$match) 270 | { 271 | if (!$options["regex_match"]) $pos2 = strpos($data, $match, $startpos); 272 | else if (!preg_match($match, $data, $matches2, PREG_OFFSET_CAPTURE, $startpos)) $pos2 = false; 273 | else $pos2 = $matches2[0][1]; 274 | 275 | if ($pos2 !== false && $pos2 < $y2 - strlen($match) && ($pos === false || $pos > $pos2)) 276 | { 277 | $pos = $pos2; 278 | 279 | $matchlen = strlen($match); 280 | } 281 | } 282 | 283 | if ($pos === false) 284 | { 285 | $this->currpos += $y - $result["offset"]; 286 | 287 | if ($options["return_data"]) 288 | { 289 | $startpos = $y2 - $options["min_buffer"]; 290 | 291 | if ($startpos < 0) $startpos = 0; 292 | } 293 | else if ($y2 > $options["min_buffer"]) 294 | { 295 | $origcurrpos += $y2 - $options["min_buffer"]; 296 | 297 | $data = substr($data, -$options["min_buffer"]); 298 | } 299 | } 300 | 301 | $this->realcurrpos = false; 302 | } while ($pos === false); 303 | 304 | $this->currpos = $origcurrpos + $pos + ($options["rewind_match"] ? 0 : $matchlen); 305 | 306 | if ($options["return_data"]) $data = substr($data, 0, $pos + ($options["include_match"] ? $matchlen : 0)); 307 | else $data = ($options["include_match"] ? substr($data, $pos, $matchlen) : ""); 308 | 309 | return array("success" => true, "data" => $data, "eof" => false); 310 | } 311 | 312 | public function ReadLine($includenewline = true) 313 | { 314 | $result = $this->ReadUntil(array("\r\n", "\r", "\n")); 315 | if (!$result["success"]) return $result; 316 | 317 | if (!$includenewline) $result["data"] = rtrim($result["data"], "\r\n"); 318 | 319 | return $result; 320 | } 321 | 322 | public function ReadCSV($nulls = false, $separator = ",", $enclosure = "\"") 323 | { 324 | $record = array(); 325 | $val = null; 326 | $enclosed = false; 327 | do 328 | { 329 | $result = $this->ReadLine(); 330 | if (!$result["success"]) return $result; 331 | 332 | $y = strlen($result["data"]); 333 | for ($x = 0; $x < $y; $x++) 334 | { 335 | if ($enclosed) 336 | { 337 | if ($result["data"][$x] !== $enclosure) $val .= $result["data"][$x]; 338 | else if ($x + 1 <= $y && $result["data"][$x + 1] === $enclosure) 339 | { 340 | $val .= $result["data"][$x]; 341 | $x++; 342 | } 343 | else $enclosed = false; 344 | } 345 | else if ($result["data"][$x] === $enclosure) 346 | { 347 | if ($val === null) $val = ""; 348 | 349 | $enclosed = true; 350 | } 351 | else if ($result["data"][$x] === "\r" || $result["data"][$x] === "\n") break; 352 | else if ($result["data"][$x] === $separator) 353 | { 354 | if (!$nulls && $val === null) $val = ""; 355 | 356 | $record[] = $val; 357 | 358 | $val = null; 359 | } 360 | else 361 | { 362 | if ($val === null) $val = ""; 363 | 364 | $val .= $result["data"][$x]; 365 | } 366 | } 367 | 368 | if ($result["eof"]) break; 369 | } while ($enclosed); 370 | 371 | if (!$nulls && $val === null) $val = ""; 372 | 373 | $record[] = $val; 374 | 375 | return array("success" => true, "record" => $record, "eof" => $result["eof"]); 376 | } 377 | 378 | public function Write($data) 379 | { 380 | if (!$this->open) return array("success" => false, "error" => self::PFCTranslate("Unable to write. File is not open."), "errorcode" => "file_not_open"); 381 | if (!$this->writable) return array("success" => false, "error" => self::PFCTranslate("Unable to write. File is not writable."), "errorcode" => "file_not_writable"); 382 | 383 | $data = (string)$data; 384 | $x = 0; 385 | $y = strlen($data); 386 | while ($x < $y) 387 | { 388 | if ($this->realcurrpos === false) $this->realcurrpos = $this->GetRealPos($this->currpos); 389 | 390 | $pagepos = $this->realcurrpos - ($this->realcurrpos % $this->realpagesize); 391 | 392 | if (!isset($this->pagemap[$pagepos])) 393 | { 394 | if ($pagepos === $this->realcurrpos && $this->realcurrpos === $this->realmaxpos) 395 | { 396 | // Page aligned and at the end of the file. Create a new page. 397 | $this->pagemap[$pagepos] = array("", self::PAGING_FILE_PAGE_LOADED); 398 | } 399 | else 400 | { 401 | // Attempt to load the page. 402 | $result = $this->LoadPageForCurrPos(); 403 | if (!$result["success"]) return $result; 404 | } 405 | } 406 | 407 | if ($this->pagemap[$pagepos][1] === self::PAGING_FILE_PAGE_NOT_LOADED) 408 | { 409 | if (!$this->PostLoadPageData($this->pagemap[$pagepos][0], $pagepos)) return array("success" => false, "error" => self::PFCTranslate("Unable to process loaded page data."), "errorcode" => "post_load_page_data_failed"); 410 | 411 | $this->pagemap[$pagepos][1] = self::PAGING_FILE_PAGE_LOADED; 412 | } 413 | 414 | unset($this->recentpages[$pagepos]); 415 | $this->recentpages[$pagepos] = true; 416 | 417 | 418 | // Copy data up to page size. 419 | $x2 = $this->currpos % $this->pagesize; 420 | $y2 = strlen($this->pagemap[$pagepos][0]); 421 | $diff = $y2 - $x2; 422 | if ($diff <= $y - $x) 423 | { 424 | $this->pagemap[$pagepos][0] = ($x2 === 0 ? "" : substr($this->pagemap[$pagepos][0], 0, $x2)); 425 | $this->pagemap[$pagepos][0] .= substr($data, $x, $diff); 426 | 427 | $x += $diff; 428 | $x2 = $y2; 429 | $this->currpos += $diff; 430 | } 431 | else if ($x2 === 0) 432 | { 433 | $diff = $y - $x; 434 | $tempdata = substr($this->pagemap[$pagepos][0], $y - $x); 435 | $this->pagemap[$pagepos][0] = ($x == 0 ? $data : substr($data, $x)); 436 | $this->pagemap[$pagepos][0] .= $tempdata; 437 | 438 | $x = $y; 439 | $x2 += $diff; 440 | $this->currpos += $diff; 441 | } 442 | else 443 | { 444 | // PHP is very slow when appending one byte at a time to a string. 445 | while ($x2 < $y2 && $x < $y) 446 | { 447 | $this->pagemap[$pagepos][0][$x2] = $data[$x]; 448 | 449 | $x++; 450 | $x2++; 451 | $this->currpos++; 452 | } 453 | } 454 | 455 | if ($y2 < $this->pagesize && $x < $y) 456 | { 457 | $size = ($this->pagesize - $y2 < $y - $x ? $this->pagesize - $y2 : $y - $x); 458 | $this->pagemap[$pagepos][0] .= ($x == 0 && $size == $y ? $data : substr($data, $x, $size)); 459 | 460 | $x += $size; 461 | $x2 += $size; 462 | $this->currpos += $size; 463 | } 464 | 465 | 466 | /* 467 | // Copy data up to page size. 468 | $x2 = $this->currpos % $this->pagesize; 469 | $y2 = strlen($this->pagemap[$pagepos][0]); 470 | $diff = $this->pagesize - $x2; 471 | $size = ($diff <= $y - $x ? $diff : $y - $x); 472 | $size2 = ($x2 + $size > $y2 ? $y2 - $x2 : $size); 473 | 474 | $this->pagemap[$pagepos][0] = substr_replace($this->pagemap[$pagepos][0], substr($data, $x, $size), $x2, $size2); 475 | 476 | $x += $size; 477 | $x2 += $size; 478 | $this->currpos += $size; 479 | */ 480 | 481 | /* 482 | // Copy data up to page size. 483 | $x2 = $this->currpos % $this->pagesize; 484 | $y2 = strlen($this->pagemap[$pagepos][0]); 485 | $diff = $this->pagesize - $x2; 486 | $size = ($diff <= $y - $x ? $diff : $y - $x); 487 | $size2 = ($x2 + $size > $y2 ? $y2 - $x2 : $size); 488 | 489 | str_splice($this->pagemap[$pagepos][0], $x2, $size2, $data, $x, $size); 490 | 491 | $x += $size; 492 | $x2 += $size; 493 | $this->currpos += $size; 494 | */ 495 | 496 | if ($this->pagemap[$pagepos][1] === self::PAGING_FILE_PAGE_LOADED) 497 | { 498 | $maxpagepos = $this->realmaxpos - ($this->realmaxpos % $this->realpagesize); 499 | 500 | $this->pagemap[$pagepos][1] = ($pagepos === $maxpagepos ? self::PAGING_FILE_PAGE_MODIFIED_LAST : self::PAGING_FILE_PAGE_MODIFIED); 501 | } 502 | 503 | // Adjust position. 504 | $this->realcurrpos = false; 505 | 506 | if ($this->maxpos < $this->currpos) 507 | { 508 | $this->maxpos = $this->currpos; 509 | $this->realmaxpos = $this->GetRealPos($this->maxpos); 510 | $this->realcurrpos = $this->realmaxpos; 511 | } 512 | 513 | // Write out the last page if it was filled. 514 | if ($x2 === $this->pagesize && $this->pagemap[$pagepos][1] === self::PAGING_FILE_PAGE_MODIFIED_LAST) 515 | { 516 | $result = $this->SavePage($pagepos); 517 | if (!$result["success"]) return $result; 518 | 519 | $this->UnloadExcessPages(); 520 | } 521 | } 522 | 523 | return array("success" => true, "bytes" => $y); 524 | } 525 | 526 | public function WriteCSV($record) 527 | { 528 | $data = array(); 529 | 530 | foreach ($record as $val) 531 | { 532 | $data[] = ($val === null ? "" : "\"" . str_replace(array("\r\n", "\r", "\""), array("\n", "\n", "\"\""), $val) . "\""); 533 | } 534 | 535 | $data = implode(",", $data) . "\n"; 536 | 537 | return $this->Write($data); 538 | } 539 | 540 | public function Sync($final = false) 541 | { 542 | if (!$this->open) return array("success" => false, "error" => self::PFCTranslate("Unable to sync. File is not open."), "errorcode" => "file_not_open"); 543 | if (!$this->writable) return array("success" => false, "error" => self::PFCTranslate("Unable to sync. File is not writable."), "errorcode" => "file_not_writable"); 544 | 545 | foreach ($this->recentpages as $pagepos => $val) 546 | { 547 | if ($this->pagemap[$pagepos][1] === self::PAGING_FILE_PAGE_MODIFIED || ($final && $this->pagemap[$pagepos][1] === self::PAGING_FILE_PAGE_MODIFIED_LAST)) 548 | { 549 | $result = $this->SavePage($pagepos); 550 | if (!$result["success"]) return $result; 551 | } 552 | } 553 | 554 | if ($final) $this->writable = false; 555 | 556 | return array("success" => true); 557 | } 558 | 559 | // Returns data only for non-filename instances. Clears the data in write-only mode. 560 | public function GetData() 561 | { 562 | if ($this->fp === false || is_resource($this->fp)) return false; 563 | 564 | $result = implode("", $this->fp); 565 | 566 | if (!$this->readable) 567 | { 568 | $y = strlen($result); 569 | $this->fp = array(); 570 | 571 | // Unload pages. 572 | $pagepos = $this->basepos - ($this->basepos % $this->realpagesize); 573 | $this->basepos += $y; 574 | $maxpagepos = $this->basepos - ($this->basepos % $this->realpagesize); 575 | 576 | for (; $pagepos < $maxpagepos; $pagepos += $this->realpagesize) 577 | { 578 | unset($this->pagemap[$pagepos]); 579 | unset($this->recentpages[$pagepos]); 580 | } 581 | } 582 | 583 | return $result; 584 | } 585 | 586 | // Seeks to a position in a file if possible. 587 | protected function InternalSeek($pos) 588 | { 589 | if ($pos !== $this->seekpos) 590 | { 591 | if (is_resource($this->fp)) 592 | { 593 | if ($this->seekable) 594 | { 595 | if (@fseek($this->fp, $pos, SEEK_SET) < 0) return false; 596 | } 597 | else if ($pos > $this->seekpos) 598 | { 599 | // Seek forward by reading data. 600 | $bytesleft = $pos - $this->seekpos; 601 | while ($bytesleft > 0 && !feof($this->fp)) 602 | { 603 | $tempdata = @fread($this->fp, ($bytesleft >= 1048576 ? 1048576 : $bytesleft)); 604 | if ($tempdata === false) return false; 605 | 606 | $bytesleft -= strlen($tempdata); 607 | } 608 | } 609 | else 610 | { 611 | return false; 612 | } 613 | } 614 | else if ($pos < $this->basepos) 615 | { 616 | return false; 617 | } 618 | 619 | $this->seekpos = $pos; 620 | } 621 | 622 | return true; 623 | } 624 | 625 | protected function UnloadExcessPages() 626 | { 627 | $y = count($this->pagemap); 628 | if ($y > $this->maxpages) 629 | { 630 | $num = (int)($y / 4); 631 | 632 | // Save modified page chunks. 633 | $num2 = 0; 634 | foreach ($this->recentpages as $pagepos => $val) 635 | { 636 | if ($this->pagemap[$pagepos][1] === self::PAGING_FILE_PAGE_MODIFIED) $this->SavePage($pagepos); 637 | 638 | if ($this->pagemap[$pagepos][1] !== self::PAGING_FILE_PAGE_MODIFIED_LAST) 639 | { 640 | unset($this->pagemap[$pagepos]); 641 | unset($this->recentpages[$pagepos]); 642 | } 643 | 644 | $num2++; 645 | if ($num2 >= $num) break; 646 | } 647 | } 648 | } 649 | 650 | protected function LoadPageForCurrPos() 651 | { 652 | if ($this->realcurrpos === false) $this->realcurrpos = $this->GetRealPos($this->currpos); 653 | 654 | $pagepos = $this->realcurrpos - ($this->realcurrpos % $this->realpagesize); 655 | 656 | // Load pages. 657 | if (!isset($this->pagemap[$pagepos])) 658 | { 659 | if (!$this->readable) return array("success" => false, "error" => self::PFCTranslate("Unable to read page chunk. File is not open for reading."), "errorcode" => "no_read"); 660 | 661 | if (!$this->InternalSeek($pagepos)) return array("success" => false, "error" => self::PFCTranslate("Unable to seek to page start."), "errorcode" => "seek_failed"); 662 | 663 | if (is_resource($this->fp)) 664 | { 665 | $data = ""; 666 | $size = $this->realpagesize; 667 | while ($size > 0 && !feof($this->fp)) 668 | { 669 | $tempdata = @fread($this->fp, $size); 670 | if ($tempdata === false) break; 671 | 672 | $data .= $tempdata; 673 | $size -= strlen($tempdata); 674 | $this->seekpos += strlen($tempdata); 675 | } 676 | } 677 | else 678 | { 679 | $pagenum = (int)($pagepos / $this->realpagesize); 680 | $data = (isset($this->fp[$pagenum]) ? $this->fp[$pagenum] : ""); 681 | $this->seekpos += strlen($data); 682 | } 683 | 684 | $y = strlen($data); 685 | if ($y > 0) $this->pagemap[$pagepos] = array($data, self::PAGING_FILE_PAGE_NOT_LOADED); 686 | else if (!isset($this->pagemap[$pagepos])) $this->pagemap[$pagepos] = array("", self::PAGING_FILE_PAGE_LOADED); 687 | 688 | $this->UnloadExcessPages(); 689 | } 690 | 691 | if ($this->pagemap[$pagepos][1] === self::PAGING_FILE_PAGE_NOT_LOADED) 692 | { 693 | if (!$this->PostLoadPageData($this->pagemap[$pagepos][0], $pagepos)) return array("success" => false, "error" => self::PFCTranslate("Unable to process loaded page data."), "errorcode" => "post_load_page_data_failed"); 694 | 695 | $this->pagemap[$pagepos][1] = self::PAGING_FILE_PAGE_LOADED; 696 | } 697 | 698 | unset($this->recentpages[$pagepos]); 699 | $this->recentpages[$pagepos] = true; 700 | 701 | return array("success" => true, "pos" => $pagepos, "offset" => $this->realcurrpos - $pagepos); 702 | } 703 | 704 | // Designed to be overridden by other classes that might decrypt per-page data. 705 | protected function PostLoadPageData(&$data, $pagepos) 706 | { 707 | return true; 708 | } 709 | 710 | // Saves sequential modified whole pages near to the specified page. 711 | protected function SavePage($pagepos) 712 | { 713 | if (isset($this->pagemap[$pagepos]) && ($this->pagemap[$pagepos][1] === self::PAGING_FILE_PAGE_MODIFIED || $this->pagemap[$pagepos][1] === self::PAGING_FILE_PAGE_MODIFIED_LAST)) 714 | { 715 | // Generate data to write. 716 | $data = $this->pagemap[$pagepos][0]; 717 | 718 | if (!$this->PreSavePageData($data, $pagepos)) return array("success" => false, "error" => self::PFCTranslate("Unable to process page data for saving."), "errorcode" => "pre_save_page_data_failed"); 719 | 720 | $this->pagemap[$pagepos][1] = self::PAGING_FILE_PAGE_LOADED; 721 | 722 | // Write data. 723 | if (!$this->InternalSeek($pagepos)) return array("success" => false, "error" => self::PFCTranslate("Unable to seek to page start."), "errorcode" => "seek_failed"); 724 | 725 | $y = strlen($data); 726 | 727 | if (is_resource($this->fp)) 728 | { 729 | $x = 0; 730 | while ($x < $y) 731 | { 732 | $x2 = @fwrite($this->fp, $data); 733 | if ($x2 < 1) break; 734 | 735 | $data = substr($data, $x2); 736 | $this->seekpos += $x2; 737 | $x += $x2; 738 | } 739 | 740 | if ($x < $y) return array("success" => false, "error" => self::PFCTranslate("Unable to write page data."), "errorcode" => "fwrite_failed"); 741 | } 742 | else 743 | { 744 | $this->fp[(int)($pagepos / $this->realpagesize)] = $data; 745 | 746 | $this->seekpos += $y; 747 | } 748 | } 749 | 750 | return array("success" => true); 751 | } 752 | 753 | // Designed to be overridden by other classes that might encrypt per-page data. 754 | protected function PreSavePageData(&$data, $pagepos) 755 | { 756 | return true; 757 | } 758 | 759 | public static function PFCTranslate() 760 | { 761 | $args = func_get_args(); 762 | if (!count($args)) return ""; 763 | 764 | return call_user_func_array((defined("CS_TRANSLATE_FUNC") && function_exists(CS_TRANSLATE_FUNC) ? CS_TRANSLATE_FUNC : "sprintf"), $args); 765 | } 766 | } 767 | ?> -------------------------------------------------------------------------------- /support_extra/ifds_text.php: -------------------------------------------------------------------------------- 1 | ifds = false; 17 | $this->compresslevel = -1; 18 | } 19 | 20 | public function __destruct() 21 | { 22 | $this->Close(); 23 | } 24 | 25 | public function SetCompressionLevel($level) 26 | { 27 | $this->compresslevel = (int)$level; 28 | } 29 | 30 | public function Create($pfcfilename, $options = array()) 31 | { 32 | $this->Close(); 33 | 34 | if (!class_exists("IFDS", false)) require_once str_replace("\\", "/", dirname(__FILE__)) . "/ifds.php"; 35 | 36 | $this->ifds = new IFDS(); 37 | $result = $this->ifds->Create($pfcfilename, 1, 0, 0, "TEXT"); 38 | if (!$result["success"]) return $result; 39 | 40 | $features = 0; 41 | if (isset($options["compress"]) && (bool)$options["compress"]) $features |= self::FEATURES_COMPRESS_DATA; 42 | if (!isset($options["trail"]) || (bool)$options["trail"]) $features |= self::FEATURES_TRAILING_NEWLINE; 43 | 44 | $this->ifds->SetAppFormatFeatures($features); 45 | 46 | // Store metadata. 47 | $this->metadata = array( 48 | "newline" => (isset($options["newline"]) ? (string)$options["newline"] : "\n"), 49 | "charset" => (isset($options["charset"]) ? (string)$options["charset"] : "utf-8"), 50 | "mimetype" => (isset($options["mimetype"]) ? (string)$options["mimetype"] : "text/plain"), 51 | "language" => (isset($options["language"]) ? (string)$options["language"] : "en-us"), 52 | "author" => (isset($options["author"]) ? (string)$options["author"] : "") 53 | ); 54 | 55 | $result = $this->ifds->CreateKeyValueMap("metadata"); 56 | if (!$result["success"]) return $result; 57 | 58 | $metadataobj = $result["obj"]; 59 | 60 | $result = $this->ifds->SetKeyValueMap($metadataobj, $this->metadata); 61 | if (!$result["success"]) return $result; 62 | 63 | // Create the root object. 64 | $result = $this->ifds->CreateFixedArray(8, "root"); 65 | if (!$result["success"]) return $result; 66 | 67 | $this->rootobj = $result["obj"]; 68 | 69 | return array("success" => true); 70 | } 71 | 72 | public function Open($pfcfilename) 73 | { 74 | $this->Close(); 75 | 76 | if (!class_exists("IFDS", false)) require_once str_replace("\\", "/", dirname(__FILE__)) . "/ifds.php"; 77 | 78 | $this->ifds = new IFDS(); 79 | $result = $this->ifds->Open($pfcfilename, "TEXT"); 80 | if (!$result["success"]) return $result; 81 | 82 | if ($result["header"]["fmt_major_ver"] != 1) return array("success" => false, "error" => self::IFDSTranslate("Invalid file header. Expected text format major version 1."), "errorcode" => "invalid_fmt_major_ver"); 83 | 84 | // Get metadata. 85 | $result = $this->ifds->GetObjectByName("metadata"); 86 | if (!$result["success"]) return $result; 87 | 88 | $metadataobj = $result["obj"]; 89 | 90 | $result = $this->ifds->GetKeyValueMap($metadataobj); 91 | if (!$result["success"]) return $result; 92 | 93 | $this->metadata = $result["map"]; 94 | 95 | if (!isset($this->metadata["newline"])) return array("success" => false, "error" => self::IFDSTextTranslate("Missing 'newline' in metadata."), "errorcode" => "invalid_metadata"); 96 | if (!isset($this->metadata["charset"])) return array("success" => false, "error" => self::IFDSTextTranslate("Missing 'charset' in metadata."), "errorcode" => "invalid_metadata"); 97 | if (!isset($this->metadata["mimetype"])) return array("success" => false, "error" => self::IFDSTextTranslate("Missing 'mimetype' in metadata."), "errorcode" => "invalid_metadata"); 98 | if (!isset($this->metadata["language"])) return array("success" => false, "error" => self::IFDSTextTranslate("Missing 'language' in metadata."), "errorcode" => "invalid_metadata"); 99 | 100 | // Load the root object. 101 | $result = $this->ifds->GetObjectByName("root"); 102 | if (!$result["success"]) return $result; 103 | 104 | $this->rootobj = $result["obj"]; 105 | 106 | if ($this->ifds->GetFixedArrayEntrySize($this->rootobj) !== 8) return array("success" => false, "error" => self::IFDSTextTranslate("Invalid root object."), "errorcode" => "invalid_root"); 107 | 108 | // Extract root entries. 109 | do 110 | { 111 | $result = $this->ifds->GetNextFixedArrayEntry($this->rootobj); 112 | if (!$result["success"]) return $result; 113 | 114 | $this->rootentries[] = array( 115 | "lines" => unpack("N", substr($result["data"], 0, 4))[1], 116 | "id" => unpack("N", substr($result["data"], 4, 4))[1], 117 | "obj" => false, 118 | "entries" => array() 119 | ); 120 | 121 | } while (!$result["end"]); 122 | 123 | return array("success" => true); 124 | } 125 | 126 | public function GetIFDS() 127 | { 128 | return $this->ifds; 129 | } 130 | 131 | public function Close() 132 | { 133 | if ($this->ifds !== false) 134 | { 135 | $this->Save(false); 136 | 137 | $this->ifds->Close(); 138 | 139 | $this->ifds = false; 140 | } 141 | 142 | $this->metadata = array(); 143 | $this->rootobj = false; 144 | $this->rootentries = array(); 145 | $this->mod = false; 146 | } 147 | 148 | public function Save($flush = true) 149 | { 150 | if ($this->ifds === false) return array("success" => false, "error" => self::IFDSTextTranslate("Unable to save information. File is not open."), "errorcode" => "file_not_open"); 151 | 152 | if ($this->mod) 153 | { 154 | $result = $this->ifds->Truncate($this->rootobj); 155 | if (!$result["success"]) return $result; 156 | 157 | foreach ($this->rootentries as $num => &$rinfo) 158 | { 159 | if (!$rinfo["lines"]) unset($this->rootentries[$num]); 160 | else 161 | { 162 | if ($rinfo["obj"] !== false) 163 | { 164 | $result = $this->ifds->Truncate($rinfo["obj"]); 165 | if (!$result["success"]) return $result; 166 | 167 | foreach ($rinfo["entries"] as $num2 => &$entry) 168 | { 169 | $data2 = pack("n", $entry["lines"]); 170 | $data2 .= pack("N", $entry["id"]); 171 | 172 | $result = $this->ifds->AppendFixedArrayEntry($rinfo["obj"], $data2); 173 | if (!$result["success"]) return $result; 174 | } 175 | } 176 | 177 | $data = pack("N", $rinfo["lines"]); 178 | $data .= pack("N", $rinfo["id"]); 179 | 180 | $result = $this->ifds->AppendFixedArrayEntry($this->rootobj, $data); 181 | if (!$result["success"]) return $result; 182 | } 183 | } 184 | 185 | $this->rootentries = array_values($this->rootentries); 186 | 187 | $this->mod = false; 188 | } 189 | 190 | if ($flush) 191 | { 192 | $result = $this->ifds->FlushAll(); 193 | if (!$result["success"]) return $result; 194 | } 195 | 196 | return array("success" => true); 197 | } 198 | 199 | public function GetNumLines() 200 | { 201 | $total = 0; 202 | 203 | foreach ($this->rootentries as &$rinfo) $total += $rinfo["lines"]; 204 | 205 | if ($this->IsTrailingNewlineEnabled()) $total++; 206 | 207 | return $total; 208 | } 209 | 210 | protected function WriteMetadata() 211 | { 212 | $result = $this->ifds->GetObjectByName("metadata"); 213 | if (!$result["success"]) return $result; 214 | 215 | $metadataobj = $result["obj"]; 216 | 217 | $result = $this->ifds->SetKeyValueMap($metadataobj, $this->metadata); 218 | 219 | return $result; 220 | } 221 | 222 | public function GetNewline() 223 | { 224 | return $this->metadata["newline"]; 225 | } 226 | 227 | public function IsCompressEnabled() 228 | { 229 | return ($this->ifds->GetAppFormatFeatures() & self::FEATURES_COMPRESS_DATA); 230 | } 231 | 232 | public function SetCompressData($enable) 233 | { 234 | $features = $this->ifds->GetAppFormatFeatures(); 235 | 236 | if ($enable) $features |= self::FEATURES_COMPRESS_DATA; 237 | else $features &= ~self::FEATURES_COMPRESS_DATA; 238 | 239 | $this->ifds->SetAppFormatFeatures($features); 240 | } 241 | 242 | public function IsTrailingNewlineEnabled() 243 | { 244 | return ($this->ifds->GetAppFormatFeatures() & self::FEATURES_TRAILING_NEWLINE); 245 | } 246 | 247 | public function SetTrailingNewline($enable) 248 | { 249 | $features = $this->ifds->GetAppFormatFeatures(); 250 | 251 | if ($enable) $features |= self::FEATURES_TRAILING_NEWLINE; 252 | else $features &= ~self::FEATURES_TRAILING_NEWLINE; 253 | 254 | $this->ifds->SetAppFormatFeatures($features); 255 | } 256 | 257 | public function GetCharset() 258 | { 259 | return $this->metadata["charset"]; 260 | } 261 | 262 | public function SetCharset($charset) 263 | { 264 | $this->metadata["charset"] = $charset; 265 | 266 | return $this->WriteMetadata(); 267 | } 268 | 269 | public function GetMIMEType() 270 | { 271 | return $this->metadata["mimetype"]; 272 | } 273 | 274 | public function SetMIMEType($mimetype) 275 | { 276 | $this->metadata["mimetype"] = $mimetype; 277 | 278 | return $this->WriteMetadata(); 279 | } 280 | 281 | public function GetLanguage() 282 | { 283 | return $this->metadata["language"]; 284 | } 285 | 286 | public function SetLanguage($language) 287 | { 288 | $this->metadata["language"] = $language; 289 | 290 | return $this->WriteMetadata(); 291 | } 292 | 293 | public function GetAuthor() 294 | { 295 | return (isset($this->metadata["author"]) ? $this->metadata["author"] : ""); 296 | } 297 | 298 | public function SetAuthor($author) 299 | { 300 | $this->metadata["author"] = $author; 301 | 302 | return $this->WriteMetadata(); 303 | } 304 | 305 | protected function CreateSuperTextChunk($chunknum) 306 | { 307 | $result = $this->ifds->CreateFixedArray(6); 308 | if (!$result["success"]) return $result; 309 | 310 | $obj = $result["obj"]; 311 | 312 | array_splice($this->rootentries, $chunknum, 0, array(array("lines" => 0, "id" => $obj->GetID(), "obj" => $obj, "entries" => array()))); 313 | 314 | $this->mod = true; 315 | 316 | return array("success" => true); 317 | } 318 | 319 | protected function LoadSuperTextChunk($chunknum) 320 | { 321 | $rinfo = &$this->rootentries[$chunknum]; 322 | 323 | if ($rinfo["obj"] !== false) return array("success" => true); 324 | 325 | $result = $this->ifds->GetObjectByID($rinfo["id"]); 326 | if (!$result["success"]) return $result; 327 | 328 | $obj = $result["obj"]; 329 | 330 | if ($this->ifds->GetFixedArrayEntrySize($obj) !== 6) return array("success" => false, "error" => self::IFDSTextTranslate("Invalid super text chunk object size."), "errorcode" => "invalid_super_text_chunk"); 331 | 332 | // Extract root entries. 333 | $total = 0; 334 | do 335 | { 336 | $result = $this->ifds->GetNextFixedArrayEntry($obj); 337 | if (!$result["success"]) return $result; 338 | 339 | $numlines = unpack("n", substr($result["data"], 0, 2))[1]; 340 | 341 | $total += $numlines; 342 | 343 | $rinfo["entries"][] = array( 344 | "lines" => $numlines, 345 | "id" => unpack("N", substr($result["data"], 2, 4))[1] 346 | ); 347 | 348 | } while (!$result["end"]); 349 | 350 | if ($rinfo["lines"] !== $total) 351 | { 352 | $rinfo["lines"] = $total; 353 | 354 | $this->mod = true; 355 | } 356 | 357 | return array("success" => true); 358 | } 359 | 360 | protected function ReadDataInternal($obj) 361 | { 362 | $result = $this->ifds->Seek($obj, 0); 363 | if (!$result["success"]) return $result; 364 | 365 | $result = $this->ifds->ReadData($obj); 366 | if (!$result["success"]) return $result; 367 | 368 | if ($obj->GetEncoder() === self::ENCODER_DEFLATE) 369 | { 370 | if (!class_exists("DeflateStream", false)) require_once str_replace("\\", "/", dirname(__FILE__)) . "/deflate_stream.php"; 371 | 372 | if (!DeflateStream::IsSupported()) return array("success" => false, "error" => self::IFDSTextTranslate("Unable to decompress data. Deflate/zlib stream filter support is not enabled."), "errorcode" => "zlib_support_not_enabled"); 373 | 374 | $result["data"] = DeflateStream::Uncompress($result["data"]); 375 | if ($result["data"] === false) return array("success" => false, "error" => self::IFDSTextTranslate("Unable to decompress data. Decompression failed."), "errorcode" => "uncompress_failed"); 376 | } 377 | 378 | return $result; 379 | } 380 | 381 | protected function WriteDataInternal($obj, &$data) 382 | { 383 | $result = $this->ifds->Seek($obj, 0); 384 | if (!$result["success"]) return $result; 385 | 386 | $y = strlen($data); 387 | 388 | // Attempt to compress the data. 389 | if ($this->IsCompressEnabled()) 390 | { 391 | if (class_exists("DeflateStream", false) || file_exists(str_replace("\\", "/", dirname(__FILE__)) . "/deflate_stream.php")) 392 | { 393 | require_once str_replace("\\", "/", dirname(__FILE__)) . "/deflate_stream.php"; 394 | 395 | if (DeflateStream::IsSupported()) 396 | { 397 | $data2 = DeflateStream::Compress($data, $this->compresslevel); 398 | if ($data2 !== false) 399 | { 400 | $y2 = strlen($data2); 401 | 402 | if ($y2 < $y) 403 | { 404 | $result = array("success" => true); 405 | 406 | if ($obj->GetEncoder() !== self::ENCODER_DEFLATE) $result = $this->ifds->SetObjectEncoder($obj, self::ENCODER_DEFLATE); 407 | 408 | if ($result["success"] && ($y2 < $obj->GetDataSize() || !$this->ifds->CanWriteData($obj))) $result = $this->ifds->Truncate($obj, ($this->ifds->CanWriteData($obj) ? $y2 : 0)); 409 | 410 | return $this->ifds->WriteData($obj, $data2); 411 | } 412 | } 413 | } 414 | } 415 | } 416 | 417 | // Write uncompressed data. 418 | if ($obj->GetEncoder() !== IFDS::ENCODER_RAW) $result = $this->ifds->SetObjectEncoder($obj, IFDS::ENCODER_RAW); 419 | 420 | if ($result["success"] && ($y < $obj->GetDataSize() || !$this->ifds->CanWriteData($obj))) 421 | { 422 | $result = $this->ifds->Truncate($obj, ($this->ifds->CanWriteData($obj) ? $y : 0)); 423 | if (!$result["success"]) return $result; 424 | } 425 | 426 | return $this->ifds->WriteData($obj, $data); 427 | } 428 | 429 | public function WriteLines($lines, $offset, $removelines) 430 | { 431 | if ($this->ifds === false) return array("success" => false, "error" => self::IFDSTextTranslate("Unable to write data. File is not open."), "errorcode" => "file_not_open"); 432 | 433 | if (is_array($lines)) 434 | { 435 | // Cleanup the lines. 436 | foreach ($lines as &$line) 437 | { 438 | if (strpos($line, $this->metadata["newline"]) !== false) $line = str_replace($this->metadata["newline"], "", $line); 439 | } 440 | 441 | unset($line); 442 | } 443 | else 444 | { 445 | $lines = (string)$lines; 446 | 447 | $lines = ($lines === "" ? array() : explode($this->metadata["newline"], $lines)); 448 | 449 | $y = count($lines); 450 | if ($y > 0 && $lines[$y - 1] === "") array_pop($lines); 451 | } 452 | 453 | $y = count($lines); 454 | 455 | // Find the offset in the root. 456 | $currline = 0; 457 | $y = count($this->rootentries); 458 | for ($x = 0; $x < $y && $offset > $currline + $this->rootentries[$x]["lines"]; $x++) $currline += $this->rootentries[$x]["lines"]; 459 | 460 | if ($x >= $y) 461 | { 462 | if ($x > 0) 463 | { 464 | $x--; 465 | 466 | $result = $this->LoadSuperTextChunk($x); 467 | if (!$result["success"]) return $result; 468 | } 469 | else 470 | { 471 | $result = $this->CreateSuperTextChunk($x); 472 | if (!$result["success"]) return $result; 473 | } 474 | 475 | $offset = $currline; 476 | 477 | $rinfo = &$this->rootentries[$x]; 478 | 479 | $y2 = count($rinfo["entries"]); 480 | $x2 = $y2; 481 | } 482 | else 483 | { 484 | // Find the offset in the super text chunk. 485 | $result = $this->LoadSuperTextChunk($x); 486 | if (!$result["success"]) return $result; 487 | 488 | $rinfo = &$this->rootentries[$x]; 489 | 490 | $y2 = count($rinfo["entries"]); 491 | for ($x2 = 0; $x2 < $y2 && $offset > $currline + $rinfo["entries"][$x2]["lines"]; $x2++) $currline += $rinfo["entries"][$x2]["lines"]; 492 | } 493 | 494 | // Attempt to append to the last entry if at the end. 495 | if ($x2 >= $y2 && $x2 > 0) 496 | { 497 | $x2--; 498 | 499 | $currline -= $rinfo["entries"][$x2]["lines"]; 500 | } 501 | 502 | $readsuper = $x; 503 | $nextread = $x2; 504 | $nextread2 = $y2; 505 | 506 | $writesuper = $x; 507 | $nextwrite = $x2; 508 | $nextwrite2 = $y2; 509 | $winfo = &$this->rootentries[$x]; 510 | 511 | $last = false; 512 | $size = 0; 513 | $newlinelen = strlen($this->metadata["newline"]); 514 | $lines2 = array(); 515 | $lines3 = array(); 516 | do 517 | { 518 | if ($currline < $offset || $removelines > 0) 519 | { 520 | // Load DATA chunk. 521 | if (!count($lines2)) 522 | { 523 | if ($nextread >= $nextread2) 524 | { 525 | if ($currline < $offset) $offset = $currline; 526 | else $removelines = 0; 527 | } 528 | else 529 | { 530 | $result = $this->ifds->GetObjectByID($rinfo["entries"][$nextread]["id"]); 531 | if (!$result["success"]) return $result; 532 | 533 | $obj = $result["obj"]; 534 | 535 | $result = $this->ReadDataInternal($obj); 536 | if (!$result["success"]) return $result; 537 | 538 | $lines2 = explode($this->metadata["newline"], $result["data"]); 539 | 540 | $y2 = count($lines2) - 1; 541 | if ($y2 >= 0 && $lines2[$y2] === "") array_pop($lines2); 542 | 543 | $rinfo["entries"][$nextread]["lines"] = $y2; 544 | 545 | $nextread++; 546 | 547 | if ($nextread >= $nextread2 && $readsuper < $y - 1) 548 | { 549 | $readsuper++; 550 | $rinfo = &$this->rootentries[$readsuper]; 551 | 552 | $nextread = 0; 553 | $nextread2 = count($rinfo["entries"]); 554 | } 555 | } 556 | } 557 | 558 | if (count($lines2)) 559 | { 560 | if ($currline < $offset) 561 | { 562 | foreach ($lines2 as $num => $line) 563 | { 564 | $size += strlen($line) + $newlinelen; 565 | 566 | $lines3[] = $line; 567 | 568 | $currline++; 569 | 570 | unset($lines2[$num]); 571 | 572 | if ($currline >= $offset || $size > 65527) break; 573 | } 574 | } 575 | else if ($removelines > 0) 576 | { 577 | foreach ($lines2 as $num => $line) 578 | { 579 | unset($lines2[$num]); 580 | 581 | $removelines--; 582 | 583 | if ($removelines < 1) break; 584 | } 585 | } 586 | } 587 | } 588 | else if (count($lines)) 589 | { 590 | // Append new lines. 591 | foreach ($lines as $num => $line) 592 | { 593 | $size += strlen($line) + $newlinelen; 594 | 595 | $lines3[] = $line; 596 | 597 | unset($lines[$num]); 598 | 599 | if ($size > 65527) break; 600 | } 601 | } 602 | else if (count($lines2)) 603 | { 604 | // Append remaining original lines that are left. 605 | foreach ($lines2 as $num => $line) 606 | { 607 | $size += strlen($line) + $newlinelen; 608 | 609 | $lines3[] = $line; 610 | 611 | unset($lines2[$num]); 612 | 613 | if ($size > 65527) break; 614 | } 615 | } 616 | else if ($size > 0) 617 | { 618 | // Force the remaining lines to be written. 619 | $last = true; 620 | } 621 | else 622 | { 623 | break; 624 | } 625 | 626 | // The 65527 size check is not an accident. 65528 bytes is the maximum IFDS DATA chunk size but requires a terminating DATA chunk to follow. 627 | // 65527 bytes allows the single DATA chunk to also be a terminating DATA chunk. 628 | if ($size > 65527 || $last) 629 | { 630 | if ($size > 65527) 631 | { 632 | // Attempt to split the chunk in approximately half. 633 | $size2 = 0; 634 | $lines4 = array(); 635 | foreach ($lines3 as $num => $line) 636 | { 637 | $size2 += strlen($line) + $newlinelen; 638 | 639 | $lines4[] = $line; 640 | 641 | unset($lines3[$num]); 642 | 643 | if ($size2 > 32767) break; 644 | } 645 | 646 | $size -= $size2; 647 | } 648 | else 649 | { 650 | $size = 0; 651 | $lines4 = $lines3; 652 | $lines3 = array(); 653 | } 654 | 655 | // Write the DATA chunk. 656 | if ($nextwrite === $nextwrite2 || ($writesuper === $readsuper && $nextwrite === $nextread)) 657 | { 658 | // Split the super text chunk. 659 | if ($nextwrite >= 65536) 660 | { 661 | $result = $this->CreateSuperTextChunk($writesuper + 1); 662 | if (!$result["success"]) return $result; 663 | 664 | if ($writesuper < $readsuper) $readsuper++; 665 | else $nextread -= 32768; 666 | 667 | $this->rootentries[$writesuper + 1]["entries"] = array_splice($winfo["entries"], 32768, $nextwrite - 32768); 668 | 669 | // Recalculate the number of lines. 670 | $winfo = &$this->rootentries[$writesuper]; 671 | 672 | $winfo["lines"] = 0; 673 | foreach ($winfo["entries"] as &$winfo2) $winfo["lines"] += $winfo2["lines"]; 674 | 675 | $writesuper++; 676 | 677 | $winfo = &$this->rootentries[$writesuper]; 678 | 679 | $winfo["lines"] = 0; 680 | foreach ($winfo["entries"] as &$winfo2) $winfo["lines"] += $winfo2["lines"]; 681 | 682 | $nextwrite -= 32768; 683 | $nextwrite2 = $nextwrite; 684 | } 685 | 686 | $result = $this->ifds->CreateRawData(IFDS::ENCODER_RAW); 687 | if (!$result["success"]) return $result; 688 | 689 | $obj = $result["obj"]; 690 | 691 | array_splice($winfo["entries"], $nextwrite, 0, array(array("lines" => 0, "id" => $obj->GetID()))); 692 | 693 | $nextwrite2++; 694 | 695 | if ($writesuper === $readsuper) 696 | { 697 | $nextread++; 698 | $nextread2 = $nextwrite2; 699 | } 700 | } 701 | 702 | $y2 = count($lines4); 703 | $diff = $y2 - $winfo["entries"][$nextwrite]["lines"]; 704 | 705 | $winfo["lines"] += $diff; 706 | $winfo["entries"][$nextwrite]["lines"] = $y2; 707 | 708 | $lines4[] = ""; 709 | 710 | $lines4 = implode($this->metadata["newline"], $lines4); 711 | 712 | $result = $this->ifds->GetObjectByID($winfo["entries"][$nextwrite]["id"]); 713 | if (!$result["success"]) return $result; 714 | 715 | $obj = $result["obj"]; 716 | 717 | $result = $this->WriteDataInternal($obj, $lines4); 718 | if (!$result["success"]) return $result; 719 | 720 | $nextwrite++; 721 | } 722 | 723 | } while (1); 724 | 725 | // Delete empty objects. 726 | while (($writesuper < $readsuper && $writesuper < $y) || $nextwrite < $nextwrite2) 727 | { 728 | while ($nextwrite < $nextwrite2) 729 | { 730 | $result = $this->ifds->DeleteObject($winfo["entries"][$nextwrite2 - 1]["id"]); 731 | if (!$result["success"]) return $result; 732 | 733 | unset($winfo["entries"][$nextwrite2 - 1]); 734 | 735 | $nextwrite2--; 736 | } 737 | 738 | $writesuper++; 739 | 740 | if ($writesuper < $y) 741 | { 742 | $winfo = &$this->rootentries[$writesuper]; 743 | 744 | $nextwrite = 0; 745 | $nextwrite2 = count($winfo["entries"]); 746 | } 747 | } 748 | 749 | $this->mod = true; 750 | 751 | return array("success" => true); 752 | } 753 | 754 | public function ReadLines($offset, $numlines, $ramlimit = 10485760) 755 | { 756 | if ($this->ifds === false) return array("success" => false, "error" => self::IFDSTextTranslate("Unable to write data. File is not open."), "errorcode" => "file_not_open"); 757 | 758 | // Find the offset in the root. 759 | $currline = 0; 760 | $y = count($this->rootentries); 761 | for ($x = 0; $x < $y && $offset > $currline + $this->rootentries[$x]["lines"]; $x++) $currline += $this->rootentries[$x]["lines"]; 762 | 763 | if ($x >= $y) return array("success" => true, "lines" => array(), "eof" => true); 764 | 765 | // Find the offset in the super text chunk. 766 | $result = $this->LoadSuperTextChunk($x); 767 | if (!$result["success"]) return $result; 768 | 769 | $rinfo = &$this->rootentries[$x]; 770 | 771 | $y2 = count($rinfo["entries"]); 772 | for ($x2 = 0; $x2 < $y2 && $offset > $currline + $rinfo["entries"][$x2]["lines"]; $x2++) $currline += $rinfo["entries"][$x2]["lines"]; 773 | 774 | // Read lines in until either the memory limit or the line limit is reached. 775 | $size = 0; 776 | $newlinelen = strlen($this->metadata["newline"]); 777 | $lines = array(); 778 | $lines2 = array(); 779 | while ($numlines > 0 && $size < $ramlimit) 780 | { 781 | // Load DATA chunk. 782 | if (!count($lines2)) 783 | { 784 | if ($x2 >= $y2) 785 | { 786 | if ($currline < $offset) $offset = $currline; 787 | else break; 788 | } 789 | else 790 | { 791 | $result = $this->ifds->GetObjectByID($rinfo["entries"][$x2]["id"]); 792 | if (!$result["success"]) return $result; 793 | 794 | $obj = $result["obj"]; 795 | 796 | $result = $this->ReadDataInternal($obj); 797 | if (!$result["success"]) return $result; 798 | 799 | $lines2 = explode($this->metadata["newline"], $result["data"]); 800 | 801 | $y3 = count($lines2) - 1; 802 | if ($y3 >= 0 && $lines2[$y3] === "") array_pop($lines2); 803 | 804 | if ($rinfo["entries"][$x2]["lines"] !== $y3) 805 | { 806 | $rinfo["entries"][$x2]["lines"] = $y3; 807 | 808 | $this->mod = true; 809 | } 810 | 811 | $x2++; 812 | 813 | if ($x2 >= $y2) 814 | { 815 | $x++; 816 | 817 | if ($x < $y) 818 | { 819 | $rinfo = &$this->rootentries[$x]; 820 | 821 | $x2 = 0; 822 | $y2 = count($rinfo["entries"]); 823 | } 824 | } 825 | } 826 | } 827 | 828 | if (count($lines2)) 829 | { 830 | if ($currline < $offset) 831 | { 832 | foreach ($lines2 as $num => $line) 833 | { 834 | $currline++; 835 | 836 | unset($lines2[$num]); 837 | 838 | if ($currline >= $offset) break; 839 | } 840 | } 841 | else 842 | { 843 | foreach ($lines2 as $num => $line) 844 | { 845 | $size += strlen($line) + $newlinelen; 846 | 847 | unset($lines2[$num]); 848 | 849 | $lines[] = $line; 850 | 851 | $numlines--; 852 | 853 | if ($numlines < 1 || $size >= $ramlimit) break; 854 | } 855 | } 856 | } 857 | } 858 | 859 | $eof = ($x >= $y && $x2 >= $y2 && !count($lines2)); 860 | 861 | if ($eof && $this->IsTrailingNewlineEnabled()) $lines[] = ""; 862 | 863 | return array("success" => true, "lines" => &$lines, "eof" => $eof); 864 | } 865 | 866 | public static function IFDSTextTranslate() 867 | { 868 | $args = func_get_args(); 869 | if (!count($args)) return ""; 870 | 871 | return call_user_func_array((defined("CS_TRANSLATE_FUNC") && function_exists(CS_TRANSLATE_FUNC) ? CS_TRANSLATE_FUNC : "sprintf"), $args); 872 | } 873 | } 874 | ?> -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Incredibly Flexible Data Storage (IFDS) File Format 2 | =================================================== 3 | 4 | The Incredibly Flexible Data Storage (IFDS) file format enables the rapid creation of highly scalable and flexible custom file formats. Create your own customized binary file format with ease with IFDS. 5 | 6 | Implementations of the [IFDS file format specification](docs/ifds_specification.md) internally handle all of the difficult bookkeeping bits that occur when inventing a brand new file format, which lets software developers focus on more important things like high level design and application development. See the use-cases and examples below that use the PHP reference implementation (MIT or LGPL, your choice) of the specification to understand what is possible with IFDS! 7 | 8 | [![Donate](https://cubiclesoft.com/res/donate-shield.png)](https://cubiclesoft.com/donate/) [![Discord](https://img.shields.io/discord/777282089980526602?label=chat&logo=discord)](https://cubiclesoft.com/product-support/github/) 9 | 10 | Features 11 | -------- 12 | 13 | * Custom magic strings (up to 123 bytes) and semantic file versioning. 14 | * Included default data structures: Raw storage, binary key-value/key-ID map, fixed array, linked list. 15 | * Extensible. Add your own low level data structures (supports up to 31 custom structures such as trees) and format encoders/decoders (supports up to 48). 16 | * Name to object ID map. Create "well-known" strings for your custom format to quickly retrieve root objects. 17 | * Supports up to 4.2 billion objects. Dynamically create, modify, and delete objects. 18 | * Supports four types of data storage per object: NULL, dynamic internal (up to 31KB; 2^15), dynamic seekable (up to 280TB; 2^48), and static interleaved multi-channel (up to 65,536 channels and up to 17EB; a little less than 2^64). 19 | * Compact object storage with minimal overhead. 20 | * Memory efficient. Most baseline read/write operations are carefully designed to fit into 65,536 byte chunks. Compatible with Paging File Cache (PFC). Exception: An object's DATA locations table can be as large as 655,378 bytes. 21 | * Verifiable. Every structure and data chunk uses a standard CRC-32 to readily detect partial data corruption within the file. 22 | * Streaming capable. Generate, read, and verify file data on the fly using minimal system resources. 23 | * Optimized. Free space is efficiently managed to aid in minimizing file fragmentation. Whole file content can be compacted and sorted by most recently accessed objects to allow for better sequential file access. 24 | * And more. 25 | 26 | Performance 27 | ----------- 28 | 29 | On an Intel Core i7 6th generation CPU, the following measurements were taken using the IFDS PHP reference implementation with the default PFC in-memory layer (can be replicated via the test suite's `-perftests` option): 30 | 31 | * Key-value objects created/encoded: 85,427/sec 32 | * Key-value objects read/decoded (sequential): 64,346/sec 33 | * Key-value objects read/decoded (random): 54,986/sec 34 | * DATA chunks sequential write rate: 543 MB/sec 35 | * DATA chunks sequential read rate: 241 MB/sec 36 | * DATA chunks random read rate: 302 MB/sec 37 | * DATA chunks random write rate: 37.0 MB/sec 38 | 39 | The IFDS PHP reference implementation is actually fairly inefficient due to both being implemented in PHP userland and a number of operations are intentionally not fully optimized so that the code is easier to read and understand. PHP itself doesn't perform all that well when modifying large binary data blobs as it is missing multiple inline string modifier functions. 40 | 41 | For an apples to oranges comparison, SQLite via PHP PDO on the same hardware can insert approximately 138,000 rows/sec (using transactions and commits) containing the same data into an in-memory SQLite database. SQLite is approximately 2 times faster than the PHP IFDS reference implementation for bulk insertions. However, if you need a database, then you should probably use an existing database. 42 | 43 | Use Cases 44 | --------- 45 | 46 | Here is a short list of ideas for using IFDS: 47 | 48 | * Invent a new, multi-layered image file format. See "JPEG-PNG-SVG" below. 49 | * Redesign text files and text editors to handle multi-TB file sizes. See "Redesign Text Files" below. 50 | * Replace configuration files with a universally compatible configuration format. See "Replace Configuration Files" below. 51 | * Streaming video container for holding multiple interleaved video and audio streams. 52 | * Executable files that can be easily modified. 53 | * Custom database-like indexes for large, external files like CSV and JSON-lines. 54 | * File compression container to replace ZIP archives. 55 | * A scalable virtual filesystem. 56 | * Spreadsheet storage for handling unlimited cells. 57 | * Custom databases. 58 | * Layer and access IFDS files inside IFDS files inside IFDS files. IFDS Inception! 59 | * Bundle documentation in the same file as source code but not inlined inside the source code itself. 60 | * Store and retrieve tiled images in a format similar to Google Maps for efficient memory usage when displaying massive image files. 61 | 62 | The possibilities are endless. 63 | 64 | Limitations 65 | ----------- 66 | 67 | * The underlying file system and/or the Operating System may limit the maximum size of a single file to something much less than 2^64. For example, NTFS limits individual files to 256TB, ext4's limit is 16TB, and XFS' limit is 8EB. The only known file system to date that supports files up to 2^64 in size is ZFS, but ZFS requires considerably more system resources to run well. 68 | * The 4.2 billion object limit is due to using 4 byte object IDs. The limitation is somewhat moot. Just storing 4.2 billion empty/NULL objects would require about 34.3GB of storage. 69 | * Page level disk alignment is not currently feasible due to Paging File Cache (PFC) compatibility. Using a PFC layer is recommended for performance. PFC is useful for applying transparent disk page encryption/decryption, Hamming codes for automatic error correction, and more. 70 | * By default, the PHP reference implementation of IFDS attempts to limit estimated RAM usage of the IFDS object cache to around 10MB (configurable). For files under 10MB, the entire file can be cached in RAM and a structured order can be maintained. For larger files, data and structures may be stored out of order, which can impact reading I/O performance later. A PFC layer can help alleviate performance related problems when processing very large files. 71 | * Interleaved multi-channel data storage can only be written one time, is mostly only written to the end of a file since the size is usually unknown, and can only be read sequentially. It is ideal for interleaved, multi-channel data that is written exactly one time and read many times (e.g. interleaved audio and video data for a streaming video file format). 72 | 73 | Use Case: JPEG-PNG-SVG 74 | ----------------------- 75 | 76 | This use case is a bit contrived but demonstrates combining three different existing file formats into a single, unified file without having to make major changes to an existing file format. 77 | 78 | The JPEG file format: 79 | 80 | * Is lossy. Multiple saves of the image loses more and more information. 81 | * Consistently produces the same file size for images of the same width, height, and JPEG quality setting. 82 | * Great at storing photos. Okay at storing gradients. 83 | * Terrible at storing line art and text. Lots of obvious artifacting appears around the lines. Especially noticeable on solid color backgrounds. 84 | * Supports millions of colors. 85 | * No transparency support. 86 | * Doesn't resize well to larger image sizes. 87 | 88 | The PNG file format: 89 | 90 | * Is lossless. Saving the image over and over will not lose information. 91 | * Produces very small files when storing line art and text. Excellent for fixed size icons where pixel perfect results matter. 92 | * Produces extremely large files when storing photos and nearly all gradients. 93 | * No artifacting around lines and text at 100% size. 94 | * Supports millions of colors. 95 | * 256 levels of transparency. 96 | * Doesn't resize well to larger image sizes. 97 | 98 | The SVG file format: 99 | 100 | * Is lossless. Saving the image over and over will not lose information. 101 | * Stores vector graphics - line art and text. Generally produces very small files and supports common, simple gradients. 102 | * Does not support storing photos or more complex gradients. 103 | * No artifacting around lines and text except at very small sizes (e.g. icons). 104 | * Supports millions of colors. 105 | * Theoretically unlimited levels of transparency. 106 | * Resizes to most sizes reasonably well. However, it does not resize down to small icon sizes very well (e.g. 16x16) when rendering complex, multicolor icons where pixel perfect results matter. 107 | 108 | There is no single image file format that works for all images. This is especially true for images that combine photos/gradients and line art. 109 | 110 | So let's make a combined image file format using IFDS and just four objects to store up to three images: 111 | 112 | * 'jpeg' object - Stores a JPEG image. 113 | * 'png' object - Stores a PNG image. 114 | * 'svg' object - Stores a SVG image. 115 | * 'metadata' object - Stores metadata. 116 | 117 | Using the PHP IFDS reference implementation: 118 | 119 | ```php 120 | Open("a.jps"); 131 | 132 | $ifds = new IFDS(); 133 | $result = $ifds->Create($pfc, 1, 0, 0, "JPEG-PNG-SVG"); 134 | if (!$result["success"]) CLI::DisplayError($result); 135 | 136 | // Create 'jpg' object. 137 | $result = $ifds->CreateRawData("jpg"); 138 | if (!$result["success"]) CLI::DisplayError($result); 139 | 140 | $jpgobj = $result["obj"]; 141 | 142 | $data = file_get_contents("a.jpg"); 143 | 144 | $result = $ifds->WriteData($jpgobj, $data); 145 | if (!$result["success"]) CLI::DisplayError($result); 146 | 147 | // Create 'png' object. 148 | $result = $ifds->CreateRawData("png"); 149 | if (!$result["success"]) CLI::DisplayError($result); 150 | 151 | $pngobj = $result["obj"]; 152 | 153 | $data = file_get_contents("a.png"); 154 | 155 | $result = $ifds->WriteData($pngobj, $data); 156 | if (!$result["success"]) CLI::DisplayError($result); 157 | 158 | // Create 'svg' object. 159 | $result = $ifds->CreateRawData("svg"); 160 | if (!$result["success"]) CLI::DisplayError($result); 161 | 162 | $svgobj = $result["obj"]; 163 | 164 | $data = file_get_contents("a.svg"); 165 | 166 | $result = $ifds->WriteData($svgobj, $data); 167 | if (!$result["success"]) CLI::DisplayError($result); 168 | 169 | // Create 'metadata' object. 170 | $result = $ifds->CreateKeyValueMap("metadata"); 171 | if (!$result["success"]) CLI::DisplayError($result); 172 | 173 | $metadataobj = $result["obj"]; 174 | 175 | $metadata = array( 176 | "width" => "500", 177 | "height" => "250", 178 | "desc" => "An amazing, multi-layered image with crisp text and line art!" 179 | ); 180 | 181 | $result = $ifds->SetKeyValueMap($metadataobj, $metadata); 182 | if (!$result["success"]) CLI::DisplayError($result); 183 | 184 | // Generally a good idea to explicitly call Close() so that all objects get flushed to disk. 185 | $ifds->Close(); 186 | ?> 187 | ``` 188 | 189 | The idea here is that the JPEG image stores photo portion while the PNG image stores the line art and the SVG stores a vector-scalable version of the line art. To display the image, the JPEG would be read and the PNG or SVG is then layered on top. The overall file size is only slightly larger than the combined size of the JPEG + PNG + SVG and the result would be a cleaner, crisper image when displayed on high density devices. 190 | 191 | Unfortunately, no existing software out there will currently read this brand new file format and display the contents (e.g. your favorite web browser or image editor won't read this file). It could be argued that the PNG format could handle this internally OR a simpler, combined format could be created. However, this is only meant as a very rudimentary example that barely scratches the surface of what can be accomplished with IFDS. 192 | 193 | Use Case: Redesign Text Files 194 | ------------------------------ 195 | 196 | Today's text files and text editors are stuck in the 1970's where storage, RAM, and CPU cycles were extremely limited and every single bit and byte actually mattered. Those extreme limitations generally no longer apply. However, "modern" text editors still act like they do. Editing files that are just a few MB in size dramatically slows down most text editors to a noticeable degree and gets exponentially worse with files in the 50MB+ range. What if someone needs to edit a multi-TB text file...over a network? 197 | 198 | There are several major problems with current text files and text editors: 199 | 200 | * Line endings. Text files use specific byte(s) to indicate the end of the line. This is metadata to the file but every text editor, compiler, interpreter, operating system, and software application has to figure it out over and over again for every single file. Line endings also rely on an extremely outdated and very specific portion of the ASCII character set with no flexibility for alternate byte sequences. 201 | * Counting the number of lines is difficult. Nearly all text editors load the full file data into RAM to determine...how many lines are in that file. Syntax highlighting is an inconsequential side effect. Reading in the whole file just to determine the number of lines in that file is crazy. 202 | * Line lengths. Lines are of arbitrary length, which makes it impossible to jump to a specific line in a large file without reading the entire file sequentially from the very beginning to the point of interest. 203 | * Character sets. Text files store data for multiple character sets and it is up to the text editor to determine which character set is in use and display the file with that character set. While most documents these days are stored as UTF-8, that isn't a guarantee. The character set that a document uses is metadata. Even HTML has a special tag called "meta" for faking metadata within the document to declare the character set. 204 | * No dedicated metadata section. There are so many "hacky workarounds" that have been applied over the years when it comes to metadata storage in text files. From simple, mostly benign things such as a full page of a commented out copyright statement (seriously, this doesn't belong at the top of your code) to storing text-editor specific formatting instructions in a comment to compiler/preprocessor file parsing instructions! Metadata does not belong anywhere in the main content of a text file. 205 | * Not suitable for log files. Text files are not a great storage medium for storing logs. The current solution for logs stored in text files is to rename and compress the files, which causes all kinds of issues. Log files should simply rotate within the file itself. 206 | * Not suitable for configuration files. Text files are a terrible storage medium for configuration data. See the next Use Case for a proper replacement for configuration files using IFDS. 207 | * Not suitable for anything but very small files. Text files with lines that exceed 1MB in length are severely problematic in most text editors. Text files 10MB and over take notably longer to load and save and start having serious problems with files 50MB and larger. Text files exceeding 1GB in total size tax most text editors and some even crash making the attempt. Forget opening a 1TB text file in 99% of all text editors out there let alone loading, editing, and saving the file in a reasonable amount of time over a network. Opening a file should be instantaneous, editing any portion of the file should be instantaneous, and saving should be - you guessed it - instantaneous. It also shouldn't matter what text editor is used. 208 | * Text files do not have compression support. Therefore, really large text files are just really large files occupying unnecessary space on disk. 209 | * Text files can't be internally digitally signed in a standard way. There's no option to have a dedicated signature section within the file. Even if there was, it wouldn't be able to be implemented in a universally consistent, standard way. 210 | 211 | IFDS could be used to solve all of those problems. 212 | 213 | ``` 214 | IFDS TEXT file format 215 | 216 | Root text object (Fixed array, points at Super text chunk objects that can store millions to billions of lines each) 217 | Each array entry: 218 | 4 byte num lines 219 | 4 byte Super chunk object ID 220 | 221 | Super text chunk object (Fixed array, max 65536 entries before splitting the Super chunk) 222 | Each array entry: 223 | 2 byte num lines 224 | 4 byte Chunk object ID (generally limited to 1 DATA chunk) 225 | 226 | Chunk object (Raw data) 227 | Encoding options: 1 = Raw data, 16 = Deflate compression 228 | 229 | Metadata 230 | newline = Sequence of bytes that denote a line ending (e.g. \r\n, \x00) 231 | charset = String containing the character set encoding used (e.g. utf-8) 232 | mimetype = String containing the primary MIME type of the file data (e.g. text/plain, text/html, application/json, text/x-cpp) 233 | language = String containing the IANA language code used in the file contents (e.g. en-us) 234 | author = String containing the author of the file (e.g. Bob's Document Farm) 235 | signature = Object ID of digital signature object (design and implementation is left as an exercise) 236 | ``` 237 | 238 | With this general structure, the average text editor could handle editing files up to 280TB in size before any notable problems would arise. This is an improvement of 5.6 million times greater when compared to today's average text editor, which begins to have notable problems at around 50MB. 239 | 240 | Example implementation usage: 241 | 242 | ```php 243 | Open("ifds.iphp"); 255 | 256 | $ifdstext = new IFDS_Text(); 257 | $result = $ifdstext->Create($pfc, array("compress" => true, "trail" => false, "mimetype" => "application/x-php")); 258 | if (!$result["success"]) CLI::DisplayError($result); 259 | 260 | // Write the data. 261 | $data = file_get_contents("support/ifds.php"); 262 | 263 | $result = $ifdstext->WriteLines($data, 0, 0); 264 | if (!$result["success"]) CLI::DisplayError($result); 265 | 266 | // Generally a good idea to explicitly call Close() so that all objects get flushed to disk. 267 | $ifdstext->Close(); 268 | ?> 269 | ``` 270 | 271 | Now every text editor, every CLI tool (grep, sed, git, etc.), and every library just needs to be updated to support the IFDS TEXT file format. Not difficult and most definitely won't cause anyone to get upset at all. 272 | 273 | Again, this simple example barely scratches the surface of IFDS. 274 | 275 | Use case: Replace Configuration Files 276 | -------------------------------------- 277 | 278 | Configuration files (e.g. INI, conf) are special, common cases of text files. They have a myriad of problems: 279 | 280 | * Inconsistent definitions across multiple applications. 281 | * The signal to noise ratio of "configuration key-values to comments" is always either too low or too high depending on skill level/understanding. Comments that do exist are generally nonsensical to first-time users and make navigation annoying later on to those who understand the options. 282 | * Configuration hierarchies are difficult to visualize and generally require spanning configurations across multiple, possibly hundreds to thousands of files (e.g. Nginx config files). This slows down application startup. 283 | * Error prone. A single option or value entered incorrectly results in wasted time, sometimes taking hours of debugging to figure out what went wrong. Also, in many cases, an invalid entry causes the application to fail to start, which results in downtime. 284 | * Each application that interfaces with any given configuration file has to do a lot of extra work to correctly parse each file for both syntax and structure. Again, this is error prone and likely to break. 285 | * There are no data type controls/option limits to allow a generic configuration editing tool to be created that works for all applications. 286 | * No support for binary data. 287 | * No multilingual support. Generally limited to the author's language. 288 | * Difficult to modify programmatically. 289 | * Difficult to upgrade. When a new version of an application comes out, it has to be infinitely backwards compatible with all kinds of previous configuration files that it used to support. 290 | * Configuration information is not actually suitable for storage in text files. Current configuration files are binary data being stored in a pleasingly laid out format suitable for humans to read in a text editor but the result is not compatible with how computer systems actually work. 291 | 292 | IFDS could be used to solve all of those problems. 293 | 294 | ``` 295 | IFDS CONF file format 296 | 297 | Sections (Key-ID map) 298 | The keys are section names. The object IDs link to Section objects. 299 | 300 | Section object (Key-Value map) 301 | The keys are option names. 302 | The empty string key's value is a string that specifies the Context for the section. 303 | For other keys, the values are as follows: 304 | 1 byte Status/Type (Bit 7: 0 = Use default value(s)/Disabled section, 1 = Use option value(s); Bit 6: Multiple values; Bits 0-6: Option Type) 305 | Remaining bytes are the option's value(s) (Big-endian storage; When using multiple values, string/binary data/section name/unknown is preceded by 4 byte size, other types preceded by 1 byte size) 306 | 307 | Metadata 308 | app = String containing the application this configuration is intended to be used with (e.g. Frank's App) 309 | ver = String containing the version of the application this configuration is intended to be used with (e.g. 1.0.3) 310 | charset = String containing the character set encoding used for Strings (e.g. utf-8) 311 | ``` 312 | 313 | ``` 314 | IFDS CONF-DEF file format 315 | 316 | Contexts (Key-ID map) 317 | A mapping of all Context objects. All keys are strings. 318 | 319 | Context object (Key-ID map) 320 | A mapping of all options for the context. 321 | The empty string maps to a Doc object (documentation object) for this context. 322 | All other keys map to Option objects. 323 | 324 | Options list (Linked list) 325 | 326 | Option object (Linked list node, Raw data) 327 | 1 byte Option Info (Bit 0 = Deprecated) 328 | 1 byte Option Type (0 = Boolean, 1 = Integer, 2 = IEEE Float, 3 = IEEE Double, 4 = String, 5 = Binary data, 6 = Section names; Bit 6 = Multiple values; Bit 7 is reserved) 329 | 4 byte Doc object ID (0 = No documentation) 330 | 4 byte Option Values object ID (0 = Freeform) 331 | 2 byte size of MIME type string + MIME type string for the data that immediately follows (e.g. 'int/bytes', 'int/bits', 'application/json', 'image/jpeg'), allows a configuration tool to adapt what users see/enter 332 | Remaining bytes are the application's default value(s) for the option (Big-endian storage; When using multiple values, string/binary data/section name/unknown is preceded by 4 byte size, other types preceded by 1 byte size) 333 | 334 | Option Values object (Key-ID map) 335 | The keys are the raw internal allowed values. The object IDs link to Doc objects. For Integer types, keys may be an allowed range of values and are stored as a string ("1-4"). 336 | 337 | Docs list (Linked list) 338 | 339 | Doc object (Linked list node, Key-Value map) 340 | The keys are lowercase IANA language keys (e.g. 'en-us'). Each value is the string containing the documentation to display in that language. 341 | If the string is a URI/URL, the configuration tool can display the URI/URL as-is and/or grab the content from that URI/URL and display it. 342 | 343 | Metadata 344 | app = String containing the application this configuration definition is intended to be used with (e.g. Frank's App) 345 | ver = String containing the version of the application this configuration definition is intended to be used with (e.g. 1.0.3) 346 | charset = String containing the character set encoding used for Strings (e.g. utf-8) 347 | ``` 348 | 349 | Example implementation usage: 350 | 351 | ```php 352 | Open("ifds.iini"); 364 | 365 | $ifdsconf = new IFDS_Conf(); 366 | $result = $ifdsconf->Create($pfc, array("app" => "PHP", "ver" => phpversion())); 367 | if (!$result["success"]) CLI::DisplayError($result); 368 | 369 | // Create a new configuration section. 370 | $result = $ifdsconf->CreateSection("PHP", "ini"); 371 | if (!$result["success"]) CLI::DisplayError($result); 372 | 373 | $iniobj = $result["obj"]; 374 | $iniopts = $result["options"]; 375 | 376 | // Copy all PHP variables to the section. 377 | $phpini = ini_get_all(); 378 | foreach ($phpini as $key => $info) 379 | { 380 | if (isset($info["global_value"])) $iniopts[$key] = array("use" => true, "type" => IFDS_Conf::OPTION_TYPE_STRING, "vals" => array($info["global_value"])); 381 | } 382 | 383 | $result = $ifdsconf->UpdateSection($iniobj, $iniopts); 384 | if (!$result["success"]) CLI::DisplayError($result); 385 | 386 | // Generally a good idea to explicitly call Close() so that all objects get flushed to disk. 387 | $ifdsconf->Close(); 388 | ?> 389 | ``` 390 | 391 | See the test suite for example usage of the `IFDS_ConfDef` class, which implements the IFDS CONF-DEF format for defining a configuration file that a generic tool could use to modify a IFDS CONF file. 392 | 393 | Now every OS just needs to be updated to support the IFDS CONF and IFDS CONF-DEF file formats with both CLI and GUI tools to make it easy and painless to manage application configurations. Not difficult and most definitely won't cause anyone to get upset at all. 394 | 395 | Also, once again, this simple example barely scratches the surface of IFDS. 396 | 397 | Documentation 398 | ------------- 399 | 400 | * [PagingFileCache class](docs/paging_file_cache.md) - Applies a page cache layer over physical file and in-memory data. 401 | * [IFDS class](docs/ifds.md) - The official reference implementation of the Incredibly Flexible Data Storage (IFDS) file format specification. 402 | * [IFDS specification](docs/ifds_specification.md) - The Incredibly Flexible Data Storage (IFDS) technical specification of the IFDS file format. 403 | * [IFDS_Text class](docs/ifds_text.md) - One possible implementation of an example replacement format for the classic text file format with nifty features such as support for massive file sizes (up to roughly 280TB) and transparent data compression and decompression. 404 | * [IFDS_Conf and IFDS_ConfDef classes](docs/ifds_conf.md) - An example implementation of a possible replacement format for configuration and configuration definition files respectively. 405 | -------------------------------------------------------------------------------- /support_extra/ifds_conf.php: -------------------------------------------------------------------------------- 1 | ifds = false; 27 | } 28 | 29 | public function __destruct() 30 | { 31 | $this->Close(); 32 | } 33 | 34 | public function Create($pfcfilename, $options = array()) 35 | { 36 | $this->Close(); 37 | 38 | if (!class_exists("IFDS", false)) require_once str_replace("\\", "/", dirname(__FILE__)) . "/ifds.php"; 39 | 40 | $this->ifds = new IFDS(); 41 | $result = $this->ifds->Create($pfcfilename, 1, 0, 0, "CONF"); 42 | if (!$result["success"]) return $result; 43 | 44 | // Store metadata. 45 | $this->metadata = array( 46 | "app" => (isset($options["app"]) ? (string)$options["app"] : ""), 47 | "ver" => (isset($options["ver"]) ? (string)$options["ver"] : ""), 48 | "charset" => (isset($options["charset"]) ? (string)$options["charset"] : "utf-8") 49 | ); 50 | 51 | $result = $this->ifds->CreateKeyValueMap("metadata"); 52 | if (!$result["success"]) return $result; 53 | 54 | $metadataobj = $result["obj"]; 55 | 56 | $result = $this->ifds->SetKeyValueMap($metadataobj, $this->metadata); 57 | if (!$result["success"]) return $result; 58 | 59 | // Create the sections object. 60 | $result = $this->ifds->CreateKeyIDMap("sections"); 61 | if (!$result["success"]) return $result; 62 | 63 | $this->sectionsobj = $result["obj"]; 64 | 65 | return array("success" => true); 66 | } 67 | 68 | public function Open($pfcfilename) 69 | { 70 | $this->Close(); 71 | 72 | if (!class_exists("IFDS", false)) require_once str_replace("\\", "/", dirname(__FILE__)) . "/ifds.php"; 73 | 74 | $this->ifds = new IFDS(); 75 | $result = $this->ifds->Open($pfcfilename, "CONF"); 76 | if (!$result["success"]) return $result; 77 | 78 | if ($result["header"]["fmt_major_ver"] != 1) return array("success" => false, "error" => self::IFDSTranslate("Invalid file header. Expected configuration format major version 1."), "errorcode" => "invalid_fmt_major_ver"); 79 | 80 | // Get metadata. 81 | $result = $this->ifds->GetObjectByName("metadata"); 82 | if (!$result["success"]) return $result; 83 | 84 | $metadataobj = $result["obj"]; 85 | 86 | $result = $this->ifds->GetKeyValueMap($metadataobj); 87 | if (!$result["success"]) return $result; 88 | 89 | $this->metadata = $result["map"]; 90 | 91 | if (!isset($this->metadata["app"])) return array("success" => false, "error" => self::IFDSConfTranslate("Missing 'app' in metadata."), "errorcode" => "invalid_metadata"); 92 | if (!isset($this->metadata["ver"])) return array("success" => false, "error" => self::IFDSConfTranslate("Missing 'ver' in metadata."), "errorcode" => "invalid_metadata"); 93 | if (!isset($this->metadata["charset"])) return array("success" => false, "error" => self::IFDSConfTranslate("Missing 'charset' in metadata."), "errorcode" => "invalid_metadata"); 94 | 95 | // Load the sections object. 96 | $result = $this->ifds->GetObjectByName("sections"); 97 | if (!$result["success"]) return $result; 98 | 99 | $this->sectionsobj = $result["obj"]; 100 | 101 | $result = $this->ifds->GetKeyValueMap($this->sectionsobj); 102 | if (!$result["success"]) return $result; 103 | 104 | $this->sectionsmap = $result["map"]; 105 | 106 | return array("success" => true); 107 | } 108 | 109 | public function GetIFDS() 110 | { 111 | return $this->ifds; 112 | } 113 | 114 | public function Close() 115 | { 116 | if ($this->ifds !== false) 117 | { 118 | $this->Save(false); 119 | 120 | $this->ifds->Close(); 121 | 122 | $this->ifds = false; 123 | } 124 | 125 | $this->metadata = array(); 126 | $this->sectionsobj = false; 127 | $this->sectionsmap = array(); 128 | $this->mod = false; 129 | } 130 | 131 | public function Save($flush = true) 132 | { 133 | if ($this->ifds === false) return array("success" => false, "error" => self::IFDSConfTranslate("Unable to save information. File is not open."), "errorcode" => "file_not_open"); 134 | 135 | if ($this->mod) 136 | { 137 | $result = $this->ifds->SetKeyValueMap($this->sectionsobj, $this->sectionsmap); 138 | if (!$result["success"]) return $result; 139 | 140 | $this->mod = false; 141 | } 142 | 143 | if ($flush) 144 | { 145 | $result = $this->ifds->FlushAll(); 146 | if (!$result["success"]) return $result; 147 | } 148 | 149 | return array("success" => true); 150 | } 151 | 152 | protected function WriteMetadata() 153 | { 154 | $result = $this->ifds->GetObjectByName("metadata"); 155 | if (!$result["success"]) return $result; 156 | 157 | $metadataobj = $result["obj"]; 158 | 159 | $result = $this->ifds->SetKeyValueMap($metadataobj, $this->metadata); 160 | 161 | return $result; 162 | } 163 | 164 | public function GetApp() 165 | { 166 | return $this->metadata["app"]; 167 | } 168 | 169 | public function SetApp($app) 170 | { 171 | $this->metadata["app"] = $app; 172 | 173 | return $this->WriteMetadata(); 174 | } 175 | 176 | public function GetVer() 177 | { 178 | return $this->metadata["ver"]; 179 | } 180 | 181 | public function SetVer($ver) 182 | { 183 | $this->metadata["ver"] = $ver; 184 | 185 | return $this->WriteMetadata(); 186 | } 187 | 188 | public function GetCharset() 189 | { 190 | return $this->metadata["charset"]; 191 | } 192 | 193 | public function SetCharset($charset) 194 | { 195 | $this->metadata["charset"] = $charset; 196 | 197 | return $this->WriteMetadata(); 198 | } 199 | 200 | public function GetSectionsMap() 201 | { 202 | return $this->sectionsmap; 203 | } 204 | 205 | public function CreateSection($name, $contextname) 206 | { 207 | if (isset($this->sectionsmap[$name])) return array("success" => false, "error" => self::IFDSConfTranslate("Unable to create section. Name already exists."), "errorcode" => "name_already_exists"); 208 | 209 | $result = $this->ifds->CreateKeyValueMap(); 210 | if (!$result["success"]) return $result; 211 | 212 | $obj = $result["obj"]; 213 | 214 | $this->sectionsmap[$name] = $obj->GetID(); 215 | 216 | $this->mod = true; 217 | 218 | $result["options"] = array("" => $contextname); 219 | 220 | return $result; 221 | } 222 | 223 | public function DeleteSection($name) 224 | { 225 | if (!isset($this->sectionsmap[$name])) return array("success" => false, "error" => self::IFDSConfTranslate("Unable to delete section. Section name does not exist."), "errorcode" => "name_not_found"); 226 | 227 | $result = $this->ifds->GetObjectByID($this->sectionsmap[$name]); 228 | if (!$result["success"]) return $result; 229 | 230 | $result = $this->ifds->DeleteObject($result["obj"]); 231 | if (!$result["success"]) return $result; 232 | 233 | unset($this->sectionsmap[$name]); 234 | 235 | $this->mod = true; 236 | 237 | return $result; 238 | } 239 | 240 | public function RenameSection($oldname, $newname) 241 | { 242 | if (!isset($this->sectionsmap[$oldname])) return array("success" => false, "error" => self::IFDSConfTranslate("Unable to rename section. Section name does not exist."), "errorcode" => "name_not_found"); 243 | if (isset($this->sectionsmap[$newname])) return array("success" => false, "error" => self::IFDSConfTranslate("Unable to rename section. Section name already exists."), "errorcode" => "name_already_exists"); 244 | 245 | $this->sectionsmap[$newname] = $this->sectionsmap[$oldname]; 246 | 247 | unset($this->sectionsmap[$oldname]); 248 | 249 | $this->mod = true; 250 | 251 | return array("success" => true); 252 | } 253 | 254 | public static function ExtractTypeData(&$val, $pos, $type, $multiple) 255 | { 256 | $vals = array(); 257 | 258 | $y = strlen($val); 259 | 260 | if (!$multiple && $pos >= $y && ($type === self::OPTION_TYPE_STRING || $type === self::OPTION_TYPE_BINARY)) $vals[] = ""; 261 | 262 | while ($pos < $y) 263 | { 264 | if (!$multiple) $size = $y - $pos; 265 | else 266 | { 267 | if ($type === self::OPTION_TYPE_STRING || $type === self::OPTION_TYPE_BINARY || $type === self::OPTION_TYPE_SECTION) 268 | { 269 | if ($pos + 3 >= $y) break; 270 | 271 | $size = unpack("N", substr($val, $pos, 4))[1]; 272 | $pos += 4; 273 | } 274 | else 275 | { 276 | $size = ord($val[$pos]); 277 | $pos++; 278 | } 279 | } 280 | 281 | $val2 = substr($val, $pos, $size); 282 | 283 | switch ($type) 284 | { 285 | case self::OPTION_TYPE_BOOL: 286 | { 287 | $vals[] = ($val2 !== str_repeat("\x00", $size)); 288 | 289 | break; 290 | } 291 | case self::OPTION_TYPE_INT: 292 | { 293 | // PHP unpack() doesn't appear to have an option for unpacking big endian signed integers. 294 | if ($size === 1) $vals[] = unpack("c", $val2)[1]; 295 | else if ($size === 2) $vals[] = unpack("s", pack("s", unpack("n", $val2)[1]))[1]; 296 | else if ($size === 4) $vals[] = unpack("l", pack("l", unpack("N", $val2)[1]))[1]; 297 | else if ($size === 8) $vals[] = unpack("q", pack("q", unpack("J", $val2)[1]))[1]; 298 | 299 | break; 300 | } 301 | case self::OPTION_TYPE_FLOAT: 302 | { 303 | $vals[] = unpack("G", $val2)[1]; 304 | 305 | break; 306 | } 307 | case self::OPTION_TYPE_DOUBLE: 308 | { 309 | $vals[] = unpack("E", $val2)[1]; 310 | 311 | break; 312 | } 313 | default: 314 | { 315 | $vals[] = $val2; 316 | 317 | break; 318 | } 319 | } 320 | 321 | $pos += $size; 322 | } 323 | 324 | return $vals; 325 | } 326 | 327 | public function GetSection($name) 328 | { 329 | if (!isset($this->sectionsmap[$name])) return array("success" => false, "error" => self::IFDSConfTranslate("Section name does not exist."), "errorcode" => "name_not_found"); 330 | 331 | $result = $this->ifds->GetObjectByID($this->sectionsmap[$name]); 332 | if (!$result["success"]) return $result; 333 | 334 | $obj = $result["obj"]; 335 | 336 | $result = $this->ifds->GetKeyValueMap($obj); 337 | if (!$result["success"]) return $result; 338 | 339 | $map = $result["map"]; 340 | 341 | $options = array(); 342 | foreach ($result["map"] as $key => $val) 343 | { 344 | if ($key === "") $options[$key] = $val; 345 | else 346 | { 347 | if (strlen($val) < 1) continue; 348 | 349 | $optinfo = ord($val[0]); 350 | 351 | $use = (bool)($optinfo & self::OPTION_STATUS_USE); 352 | $multiple = (bool)($optinfo & self::OPTION_TYPE_MULTIPLE); 353 | $type = ($optinfo & self::OPTION_TYPE_MASK); 354 | 355 | if ($type >= self::OPTION_TYPE_MAX) $type = self::OPTION_TYPE_BINARY; 356 | 357 | $vals = self::ExtractTypeData($val, 1, $type, $multiple); 358 | 359 | $options[$key] = array("use" => $use, "type" => $type, "vals" => $vals); 360 | } 361 | } 362 | 363 | return array("success" => true, "obj" => $obj, "options" => &$options); 364 | } 365 | 366 | public static function AppendTypeData(&$val, $type, &$vals) 367 | { 368 | $multiple = (count($vals) > 1); 369 | 370 | foreach ($vals as $val2) 371 | { 372 | switch ($type) 373 | { 374 | case self::OPTION_TYPE_BOOL: 375 | { 376 | if ($multiple) $val .= "\x01"; 377 | 378 | $val .= ($val2 ? "\x01" : "\x00"); 379 | 380 | break; 381 | } 382 | case self::OPTION_TYPE_INT: 383 | { 384 | if ($val2 >= -128 && $val2 <= 127) $val .= ($multiple ? "\x01" : "") . chr($val2); 385 | else if ($val2 >= -32768 && $val2 <= 32767) $val .= ($multiple ? "\x02" : "") . pack("n", $val2); 386 | else if ($val2 >= -2147483648 && $val2 <= 2147483647) $val .= ($multiple ? "\x04" : "") . pack("N", $val2); 387 | else $val .= ($multiple ? "\x08" : "") . pack("J", $val2); 388 | 389 | break; 390 | } 391 | case self::OPTION_TYPE_FLOAT: 392 | { 393 | $data = pack("G", $val2); 394 | 395 | if ($multiple) $val .= chr(strlen($data)); 396 | 397 | $val .= $data; 398 | 399 | break; 400 | } 401 | case self::OPTION_TYPE_DOUBLE: 402 | { 403 | $data = pack("E", $val2); 404 | 405 | if ($multiple) $val .= chr(strlen($data)); 406 | 407 | $val .= $data; 408 | 409 | break; 410 | } 411 | default: 412 | { 413 | if ($multiple) $val .= pack("N", strlen($val2)); 414 | 415 | $val .= $val2; 416 | 417 | break; 418 | } 419 | } 420 | } 421 | } 422 | 423 | public function UpdateSection($obj, $options) 424 | { 425 | $map = array(); 426 | foreach ($options as $key => &$info) 427 | { 428 | if ($key === "") $map[""] = $info; 429 | else 430 | { 431 | if (count($info["vals"]) < 1) continue; 432 | 433 | $type = $info["type"]; 434 | $multiple = (count($info["vals"]) > 1); 435 | 436 | if ($type >= self::OPTION_TYPE_MAX) $type = self::OPTION_TYPE_BINARY; 437 | 438 | $val = chr(($info["use"] ? self::OPTION_STATUS_USE : self::OPTION_STATUS_DEFAULT) | ($multiple ? self::OPTION_TYPE_MULTIPLE : 0x00) | $type); 439 | 440 | self::AppendTypeData($val, $type, $info["vals"]); 441 | 442 | $map[$key] = $val; 443 | } 444 | } 445 | 446 | return $this->ifds->SetKeyValueMap($obj, $map); 447 | } 448 | 449 | public static function IFDSConfTranslate() 450 | { 451 | $args = func_get_args(); 452 | if (!count($args)) return ""; 453 | 454 | return call_user_func_array((defined("CS_TRANSLATE_FUNC") && function_exists(CS_TRANSLATE_FUNC) ? CS_TRANSLATE_FUNC : "sprintf"), $args); 455 | } 456 | } 457 | 458 | class IFDS_ConfDef 459 | { 460 | protected $ifds, $metadata, $contextsobj, $contextsmap, $optionsobj, $docsobj, $mod; 461 | 462 | const OPTION_STATUS_DEFAULT = 0x00; 463 | const OPTION_STATUS_USE = 0x80; 464 | 465 | const OPTION_TYPE_MASK = 0x3F; 466 | const OPTION_TYPE_BOOL = 0; 467 | const OPTION_TYPE_INT = 1; 468 | const OPTION_TYPE_FLOAT = 2; 469 | const OPTION_TYPE_DOUBLE = 3; 470 | const OPTION_TYPE_STRING = 4; 471 | const OPTION_TYPE_BINARY = 5; 472 | const OPTION_TYPE_SECTION = 6; 473 | const OPTION_TYPE_MAX = 7; 474 | 475 | const OPTION_TYPE_MULTIPLE = 0x40; 476 | 477 | const OPTION_INFO_NORMAL = 0x00; 478 | const OPTION_INFO_DEPRECATED = 0x01; 479 | 480 | public function __construct() 481 | { 482 | $this->ifds = false; 483 | } 484 | 485 | public function __destruct() 486 | { 487 | $this->Close(); 488 | } 489 | 490 | public function Create($pfcfilename, $options = array()) 491 | { 492 | $this->Close(); 493 | 494 | if (!class_exists("IFDS", false)) require_once str_replace("\\", "/", dirname(__FILE__)) . "/ifds.php"; 495 | 496 | $this->ifds = new IFDS(); 497 | $result = $this->ifds->Create($pfcfilename, 1, 0, 0, "CONF-DEF"); 498 | if (!$result["success"]) return $result; 499 | 500 | // Store metadata. 501 | $this->metadata = array( 502 | "app" => (isset($options["app"]) ? (string)$options["app"] : ""), 503 | "ver" => (isset($options["ver"]) ? (string)$options["ver"] : ""), 504 | "charset" => (isset($options["charset"]) ? (string)$options["charset"] : "utf-8") 505 | ); 506 | 507 | $result = $this->ifds->CreateKeyValueMap("metadata"); 508 | if (!$result["success"]) return $result; 509 | 510 | $metadataobj = $result["obj"]; 511 | 512 | $result = $this->ifds->SetKeyValueMap($metadataobj, $this->metadata); 513 | if (!$result["success"]) return $result; 514 | 515 | // Create the contexts object. 516 | $result = $this->ifds->CreateKeyIDMap("contexts"); 517 | if (!$result["success"]) return $result; 518 | 519 | $this->contextsobj = $result["obj"]; 520 | 521 | // Create the options object. 522 | $result = $this->ifds->CreateLinkedList("options"); 523 | if (!$result["success"]) return $result; 524 | 525 | $this->optionsobj = $result["obj"]; 526 | 527 | // Create the docs object. 528 | $result = $this->ifds->CreateLinkedList("docs"); 529 | if (!$result["success"]) return $result; 530 | 531 | $this->docsobj = $result["obj"]; 532 | 533 | return array("success" => true); 534 | } 535 | 536 | public function Open($pfcfilename) 537 | { 538 | $this->Close(); 539 | 540 | if (!class_exists("IFDS", false)) require_once str_replace("\\", "/", dirname(__FILE__)) . "/ifds.php"; 541 | 542 | $this->ifds = new IFDS(); 543 | $result = $this->ifds->Open($pfcfilename, "CONF-DEF"); 544 | if (!$result["success"]) return $result; 545 | 546 | if ($result["header"]["fmt_major_ver"] != 1) return array("success" => false, "error" => self::IFDSTranslate("Invalid file header. Expected configuration definition format major version 1."), "errorcode" => "invalid_fmt_major_ver"); 547 | 548 | // Get metadata. 549 | $result = $this->ifds->GetObjectByName("metadata"); 550 | if (!$result["success"]) return $result; 551 | 552 | $metadataobj = $result["obj"]; 553 | 554 | $result = $this->ifds->GetKeyValueMap($metadataobj); 555 | if (!$result["success"]) return $result; 556 | 557 | $this->metadata = $result["map"]; 558 | 559 | if (!isset($this->metadata["app"])) return array("success" => false, "error" => self::IFDSConfTranslate("Missing 'app' in metadata."), "errorcode" => "invalid_metadata"); 560 | if (!isset($this->metadata["ver"])) return array("success" => false, "error" => self::IFDSConfTranslate("Missing 'ver' in metadata."), "errorcode" => "invalid_metadata"); 561 | if (!isset($this->metadata["charset"])) return array("success" => false, "error" => self::IFDSConfTranslate("Missing 'charset' in metadata."), "errorcode" => "invalid_metadata"); 562 | 563 | // Load the contexts object. 564 | $result = $this->ifds->GetObjectByName("contexts"); 565 | if (!$result["success"]) return $result; 566 | 567 | $this->contextsobj = $result["obj"]; 568 | 569 | $result = $this->ifds->GetKeyValueMap($this->contextsobj); 570 | if (!$result["success"]) return $result; 571 | 572 | $this->contextsmap = $result["map"]; 573 | 574 | // Load options object. 575 | $result = $this->ifds->GetObjectByName("options"); 576 | if (!$result["success"]) return $result; 577 | 578 | $this->optionsobj = $result["obj"]; 579 | 580 | // Load docs object. 581 | $result = $this->ifds->GetObjectByName("docs"); 582 | if (!$result["success"]) return $result; 583 | 584 | $this->docsobj = $result["obj"]; 585 | 586 | return array("success" => true); 587 | } 588 | 589 | public function GetIFDS() 590 | { 591 | return $this->ifds; 592 | } 593 | 594 | public function Close() 595 | { 596 | if ($this->ifds !== false) 597 | { 598 | $this->Save(false); 599 | 600 | $this->ifds->Close(); 601 | 602 | $this->ifds = false; 603 | } 604 | 605 | $this->metadata = array(); 606 | $this->contextsobj = false; 607 | $this->contextsmap = array(); 608 | $this->optionsobj = false; 609 | $this->docsobj = false; 610 | $this->mod = false; 611 | } 612 | 613 | public function Save($flush = true) 614 | { 615 | if ($this->ifds === false) return array("success" => false, "error" => self::IFDSConfTranslate("Unable to save information. File is not open."), "errorcode" => "file_not_open"); 616 | 617 | if ($this->mod) 618 | { 619 | $result = $this->ifds->SetKeyValueMap($this->contextsobj, $this->contextsmap); 620 | if (!$result["success"]) return $result; 621 | 622 | $this->mod = false; 623 | } 624 | 625 | if ($flush) 626 | { 627 | $result = $this->ifds->FlushAll(); 628 | if (!$result["success"]) return $result; 629 | } 630 | 631 | return array("success" => true); 632 | } 633 | 634 | protected function WriteMetadata() 635 | { 636 | $result = $this->ifds->GetObjectByName("metadata"); 637 | if (!$result["success"]) return $result; 638 | 639 | $metadataobj = $result["obj"]; 640 | 641 | $result = $this->ifds->SetKeyValueMap($metadataobj, $this->metadata); 642 | 643 | return $result; 644 | } 645 | 646 | public function GetApp() 647 | { 648 | return $this->metadata["app"]; 649 | } 650 | 651 | public function SetApp($app) 652 | { 653 | $this->metadata["app"] = $app; 654 | 655 | return $this->WriteMetadata(); 656 | } 657 | 658 | public function GetVer() 659 | { 660 | return $this->metadata["ver"]; 661 | } 662 | 663 | public function SetVer($ver) 664 | { 665 | $this->metadata["ver"] = $ver; 666 | 667 | return $this->WriteMetadata(); 668 | } 669 | 670 | public function GetCharset() 671 | { 672 | return $this->metadata["charset"]; 673 | } 674 | 675 | public function SetCharset($charset) 676 | { 677 | $this->metadata["charset"] = $charset; 678 | 679 | return $this->WriteMetadata(); 680 | } 681 | 682 | public function GetContextsMap() 683 | { 684 | return $this->contextsmap; 685 | } 686 | 687 | public function CreateContext($name) 688 | { 689 | if (isset($this->contextsmap[$name])) return array("success" => false, "error" => self::IFDSConfTranslate("Unable to create context. Context name already exists."), "errorcode" => "name_already_exists"); 690 | 691 | $result = $this->ifds->CreateKeyIDMap(); 692 | if (!$result["success"]) return $result; 693 | 694 | $obj = $result["obj"]; 695 | 696 | $this->contextsmap[$name] = $obj->GetID(); 697 | 698 | $this->mod = true; 699 | 700 | $result["options"] = array("" => 0); 701 | 702 | return $result; 703 | } 704 | 705 | public function DeleteContext($name) 706 | { 707 | if (!isset($this->contextsmap[$name])) return array("success" => false, "error" => self::IFDSConfTranslate("Unable to delete context. Context name does not exist."), "errorcode" => "name_not_found"); 708 | 709 | $result = $this->ifds->GetObjectByID($this->contextsmap[$name]); 710 | if (!$result["success"]) return $result; 711 | 712 | $result = $this->ifds->DeleteObject($result["obj"]); 713 | if (!$result["success"]) return $result; 714 | 715 | unset($this->contextsmap[$name]); 716 | 717 | $this->mod = true; 718 | 719 | return $result; 720 | } 721 | 722 | public function RenameContext($oldname, $newname) 723 | { 724 | if (!isset($this->contextsmap[$oldname])) return array("success" => false, "error" => self::IFDSConfTranslate("Unable to rename context. Context name does not exist."), "errorcode" => "name_not_found"); 725 | if (isset($this->contextsmap[$newname])) return array("success" => false, "error" => self::IFDSConfTranslate("Unable to rename context. Context name already exists."), "errorcode" => "name_already_exists"); 726 | 727 | $this->contextsmap[$newname] = $this->contextsmap[$oldname]; 728 | 729 | unset($this->contextsmap[$oldname]); 730 | 731 | $this->mod = true; 732 | 733 | return array("success" => true); 734 | } 735 | 736 | public function GetContext($name) 737 | { 738 | if (!isset($this->contextsmap[$name])) return array("success" => false, "error" => self::IFDSConfTranslate("Context name does not exist."), "errorcode" => "name_not_found"); 739 | 740 | $result = $this->ifds->GetObjectByID($this->contextsmap[$name]); 741 | if (!$result["success"]) return $result; 742 | 743 | $obj = $result["obj"]; 744 | 745 | if ($obj->GetEncoder() !== IFDS::ENCODER_KEY_ID_MAP) return array("success" => false, "error" => self::IFDSConfTranslate("Context is not a key-ID map."), "errorcode" => "invalid_data_method"); 746 | 747 | $result = $this->ifds->GetKeyValueMap($obj); 748 | if (!$result["success"]) return $result; 749 | 750 | return array("success" => true, "obj" => $obj, "options" => $result["map"]); 751 | } 752 | 753 | public function UpdateContext($obj, $optionsmap) 754 | { 755 | return $this->ifds->SetKeyValueMap($obj, $optionsmap); 756 | } 757 | 758 | public function GetOptionsList() 759 | { 760 | return $this->optionsobj; 761 | } 762 | 763 | public function CreateOption($type, $options = array()) 764 | { 765 | $result = $this->ifds->CreateLinkedListNode(IFDS::ENCODER_RAW); 766 | if (!$result["success"]) return $result; 767 | 768 | $obj = $result["obj"]; 769 | 770 | $result2 = $this->ifds->AttachLinkedListNode($this->optionsobj, $obj); 771 | if (!$result2["success"]) return $result2; 772 | 773 | $result2 = $this->UpdateOption($obj, $type, $options); 774 | if (!$result2["success"]) return $result2; 775 | 776 | return $result; 777 | } 778 | 779 | public function DeleteOption($obj) 780 | { 781 | return $this->ifds->DeleteLinkedListNode($this->optionsobj, $obj); 782 | } 783 | 784 | public function UpdateOption($obj, $type, $options = array()) 785 | { 786 | if ($type >= self::OPTION_TYPE_MAX) $type = self::OPTION_TYPE_BINARY; 787 | 788 | if (!isset($options["defaults"])) $options["defaults"] = array(); 789 | 790 | $multiple = (count($options["defaults"]) > 1); 791 | 792 | $data = chr(isset($options["info"]) ? (int)$options["info"] : self::OPTION_INFO_NORMAL); 793 | $data .= chr(($multiple ? self::OPTION_TYPE_MULTIPLE : 0x00) | $type); 794 | $data .= pack("N", (isset($options["doc"]) ? (int)$options["doc"] : 0)); 795 | $data .= pack("N", (isset($options["values"]) ? (int)$options["values"] : 0)); 796 | 797 | if (!isset($options["mimetype"])) $options["mimetype"] = ($type === self::OPTION_TYPE_BINARY ? "application/octet-stream" : ""); 798 | 799 | $data .= pack("n", strlen($options["mimetype"])); 800 | $data .= $options["mimetype"]; 801 | 802 | IFDS_Conf::AppendTypeData($data, $type, $options["defaults"]); 803 | 804 | $result = $this->ifds->Truncate($obj); 805 | if (!$result["success"]) return $result; 806 | 807 | return $this->ifds->WriteData($obj, $data); 808 | } 809 | 810 | public static function ExtractOptionData(&$data) 811 | { 812 | if (strlen($data) < 12) return false; 813 | 814 | $options = array(); 815 | $options["info"] = ord($data[0]); 816 | $options["type"] = (ord($data[1]) & self::OPTION_TYPE_MASK); 817 | $multiple = (ord($data[1]) & self::OPTION_TYPE_MULTIPLE); 818 | $options["doc"] = unpack("N", substr($data, 2, 4))[1]; 819 | $options["values"] = unpack("N", substr($data, 6, 4))[1]; 820 | 821 | $size = unpack("n", substr($data, 10, 2))[1]; 822 | $options["mimetype"] = substr($data, 12, $size); 823 | 824 | $options["defaults"] = IFDS_Conf::ExtractTypeData($data, 12 + $size, $options["type"], $multiple); 825 | 826 | return $options; 827 | } 828 | 829 | public function GetOption($id) 830 | { 831 | $result = $this->ifds->GetObjectByID($id); 832 | if (!$result["success"]) return $result; 833 | 834 | $obj = $result["obj"]; 835 | 836 | if ($obj->GetEncoder() !== IFDS::ENCODER_RAW) return array("success" => false, "error" => self::IFDSConfTranslate("Option is not raw data."), "errorcode" => "invalid_data_method"); 837 | 838 | $result = $this->ifds->ReadData($obj); 839 | if (!$result["success"]) return $result; 840 | 841 | $options = self::ExtractOptionData($result["data"]); 842 | if ($options === false) return array("success" => false, "error" => self::IFDSConfTranslate("Failed to extract option data."), "errorcode" => "extract_option_data_failed"); 843 | 844 | return array("success" => true, "obj" => $obj, "options" => &$options); 845 | } 846 | 847 | public function CreateOptionValues($valuesmap) 848 | { 849 | $result = $this->ifds->CreateKeyIDMap(); 850 | if (!$result["success"]) return $result; 851 | 852 | $obj = $result["obj"]; 853 | 854 | $result2 = $this->ifds->SetKeyValueMap($obj, $valuesmap); 855 | if (!$result2["success"]) return $result2; 856 | 857 | return $result; 858 | } 859 | 860 | public function DeleteOptionValues($obj) 861 | { 862 | return $this->ifds->DeleteObject($obj); 863 | } 864 | 865 | public function UpdateOptionValues($obj, $valuesmap) 866 | { 867 | return $this->ifds->SetKeyValueMap($obj, $valuesmap); 868 | } 869 | 870 | public function GetOptionValues($id) 871 | { 872 | $result = $this->ifds->GetObjectByID($id); 873 | if (!$result["success"]) return $result; 874 | 875 | $obj = $result["obj"]; 876 | 877 | if ($obj->GetEncoder() !== IFDS::ENCODER_KEY_ID_MAP) return array("success" => false, "error" => self::IFDSConfTranslate("Option values are not a key-ID map."), "errorcode" => "invalid_data_method"); 878 | 879 | $result = $this->ifds->GetKeyValueMap($obj); 880 | if (!$result["success"]) return $result; 881 | 882 | return array("success" => true, "obj" => $obj, "values" => $result["map"]); 883 | } 884 | 885 | public function GetDocsList() 886 | { 887 | return $this->docsobj; 888 | } 889 | 890 | public function CreateDoc($langmap) 891 | { 892 | $result = $this->ifds->CreateLinkedListNode(IFDS::ENCODER_KEY_VALUE_MAP); 893 | if (!$result["success"]) return $result; 894 | 895 | $obj = $result["obj"]; 896 | 897 | $result2 = $this->ifds->AttachLinkedListNode($this->docsobj, $obj); 898 | if (!$result2["success"]) return $result2; 899 | 900 | $result2 = $this->ifds->SetKeyValueMap($obj, $langmap); 901 | if (!$result2["success"]) return $result2; 902 | 903 | return $result; 904 | } 905 | 906 | public function DeleteDoc($obj) 907 | { 908 | return $this->ifds->DeleteLinkedListNode($this->docsobj, $obj); 909 | } 910 | 911 | public function UpdateDoc($obj, $langmap) 912 | { 913 | return $this->ifds->SetKeyValueMap($obj, $langmap); 914 | } 915 | 916 | public function GetDoc($id) 917 | { 918 | $result = $this->ifds->GetObjectByID($id); 919 | if (!$result["success"]) return $result; 920 | 921 | $obj = $result["obj"]; 922 | 923 | if ($obj->GetEncoder() !== IFDS::ENCODER_KEY_VALUE_MAP) return array("success" => false, "error" => self::IFDSConfTranslate("Documentation object is not a key-value map."), "errorcode" => "invalid_data_method"); 924 | 925 | $result = $this->ifds->GetKeyValueMap($obj); 926 | if (!$result["success"]) return $result; 927 | 928 | return array("success" => true, "obj" => $obj, "langmap" => $result["map"]); 929 | } 930 | 931 | public static function IFDSConfTranslate() 932 | { 933 | $args = func_get_args(); 934 | if (!count($args)) return ""; 935 | 936 | return call_user_func_array((defined("CS_TRANSLATE_FUNC") && function_exists(CS_TRANSLATE_FUNC) ? CS_TRANSLATE_FUNC : "sprintf"), $args); 937 | } 938 | } 939 | ?> --------------------------------------------------------------------------------