├── .gitignore ├── dub.sdl ├── source └── io │ ├── socket │ ├── package.d │ └── stream.d │ ├── buffer │ ├── package.d │ ├── traits.d │ └── fixed.d │ ├── package.d │ ├── file │ ├── package.d │ ├── pipe.d │ ├── stdio.d │ ├── temp.d │ ├── mmap.d │ ├── flags.d │ └── stream.d │ ├── stream │ ├── types.d │ ├── shim.d │ ├── interfaces.d │ ├── traits.d │ └── package.d │ ├── text.d │ └── range.d ├── .deploy_docs.sh ├── .travis.yml ├── LICENSE.md ├── README.md └── appveyor.yml /.gitignore: -------------------------------------------------------------------------------- 1 | /gh-pages/ 2 | *.swp 3 | *.lst 4 | *.o 5 | /.dub/* 6 | /docs/* 7 | /docs.json 8 | /dub.selections.json 9 | /libio.a 10 | /__test__library__ 11 | /__test__library__.exe 12 | /io.lib 13 | -------------------------------------------------------------------------------- /dub.sdl: -------------------------------------------------------------------------------- 1 | name "fio" 2 | description "IO streams implemented from the ground up." 3 | copyright "Copyright © 2014-2016, Jason White" 4 | authors "Jason White" 5 | targetType "library" 6 | license "BSL-1.0" 7 | -------------------------------------------------------------------------------- /source/io/socket/package.d: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright: Copyright Jason White, 2014-2016 3 | * License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). 4 | * Authors: Thayne McCombs 5 | */ 6 | module io.socket; 7 | 8 | public import io.socket.stream; 9 | -------------------------------------------------------------------------------- /source/io/buffer/package.d: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright: Copyright Jason White, 2014-2016 3 | * License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). 4 | * Authors: Jason White 5 | */ 6 | module io.buffer; 7 | 8 | public import io.buffer.traits, 9 | io.buffer.fixed; 10 | -------------------------------------------------------------------------------- /source/io/package.d: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright: Copyright Jason White, 2014-2016 3 | * License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). 4 | * Authors: Jason White 5 | */ 6 | module io; 7 | 8 | public import io.stream, 9 | io.file, 10 | io.socket, 11 | io.buffer, 12 | io.text, 13 | io.range; 14 | -------------------------------------------------------------------------------- /source/io/file/package.d: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright: Copyright Jason White, 2014-2016 3 | * License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). 4 | * Authors: Jason White 5 | */ 6 | module io.file; 7 | 8 | public import io.file.flags; 9 | public import io.file.stream; 10 | public import io.file.mmap; 11 | public import io.file.pipe; 12 | public import io.file.stdio; 13 | public import io.file.temp; 14 | -------------------------------------------------------------------------------- /.deploy_docs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ "$TRAVIS_REPO_SLUG" == "jasonwhite/io" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_BRANCH" == "master" ]; then 3 | git clone --recursive --branch=gh-pages https://github.com/${TRAVIS_REPO_SLUG}.git gh-pages 4 | 5 | cd gh-pages 6 | git config credential.helper "store --file=.git/credentials" 7 | echo "https://${GH_TOKEN}:@github.com" > .git/credentials 8 | git config --global user.name "travis-ci" 9 | git config --global user.email "travis@travis-ci.org" 10 | git config --global push.default simple 11 | 12 | echo -e "Generating DDoc...\n" 13 | sh ./generate.sh 14 | git add . 15 | git commit -m "Auto update docs from travis-ci build #$TRAVIS_BUILD_NUMBER" 16 | git push 17 | echo -e "Published DDoc to gh-pages.\n" 18 | fi 19 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: d 2 | 3 | os: 4 | - linux 5 | - osx 6 | 7 | d: 8 | - dmd 9 | - ldc 10 | 11 | after_success: 12 | - ./.deploy_docs.sh 13 | 14 | env: 15 | - secure: "Tw0dZnpksI3E21h52Rek3YyO7r92/605WWgFOMuY2/If7ygn4L58jfvTs6zCugpUUNn8cMfUgEULnJtgHhNWZGG+18z+7b+M98jfd1jYkfQX+kOdIoZbkuP1yh0sqW45xfXYDuRqbAFedyeQJl2AGpjG85jdV5NmNcfLsJpT9vLcP6iHJinMQS6KZ1Xd6Izc6xMa6XVrMKeuKb17otMUe8BSsO0xufGNkRMKH4nyjqwlg88giW7x274DmruHeCP5YsId0BEVcehXFufcZ9wYevkzWUKiPvgSGMCJPwzsCHr+Ig19ghwzmDrmscN9TQ/syJveOvtendSWHVV6mWdN2msMS3M68QVgYtuevPQEtZoiis6Re8UoT59MMef8i/srL6k7EQgagmRwTZInvkMXybFG31eJyXHRdTnX28NGqxX7vBzq6nc+HdFTh8Ic4AvL2yLJX++366z//1PiKmcIz7JwLGwjNPJ1eLiFapboTct3i2q13OnhrnfF+4A+6WYm44kt4wbirGGKCCWJ6tjSdgODYAbow0enoFrNa/+LQAlPZ6nOAveQ2Iaahk7UPu4Z3MFPZ4aUJQPMVzpy1kXAxwU7j6+/MiZQCdivdmGf9tuKELQYWs/ctGyUOgZ9ncSV2eWVQ2CcaIucvaQmZ246AZ73DQCvhUreo4v46iJH8LQ=" 16 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Boost Software License - Version 1.0 - August 17th, 2003 2 | 3 | Permission is hereby granted, free of charge, to any person or organization 4 | obtaining a copy of the software and accompanying documentation covered by 5 | this license (the "Software") to use, reproduce, display, distribute, 6 | execute, and transmit the Software, and to prepare derivative works of the 7 | Software, and to permit third-parties to whom the Software is furnished to 8 | do so, all subject to the following: 9 | 10 | The copyright notices in the Software and this entire statement, including 11 | the above license grant, this restriction and the following disclaimer, 12 | must be included in all copies of the Software, in whole or in part, and 13 | all derivative works of the Software, unless such copies or derivative 14 | works are solely in the form of machine-executable object code generated by 15 | a source language processor. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT 20 | SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE 21 | FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, 22 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 23 | DEALINGS IN THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /source/io/stream/types.d: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright: Copyright Jason White, 2014-2016 3 | * License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). 4 | * Authors: Jason White 5 | */ 6 | module io.stream.types; 7 | 8 | /** 9 | * Specifies how to access a stream. 10 | */ 11 | enum Access 12 | { 13 | /// Default access. Not very useful. 14 | none = 0, 15 | 16 | /// Allows only read operations on the stream. 17 | read = 1 << 0, 18 | 19 | /// Allows only write operations on the stream. 20 | write = 1 << 1, 21 | 22 | /// Allows data to be executed. This is only used for memory mapped files. 23 | execute = 1 << 2, 24 | 25 | /// Allows both read and write operations on the stream. 26 | readWrite = read | write, 27 | 28 | /// Complete access. 29 | all = read | write | execute, 30 | } 31 | 32 | /** 33 | * Relative position to seek from. 34 | */ 35 | enum From 36 | { 37 | /// Seek relative to the beginning of the stream. 38 | start, 39 | 40 | /// Seek relative to the current position in the stream. 41 | here, 42 | 43 | /// Seek relative to the end of the stream. 44 | end, 45 | } 46 | 47 | /** 48 | * Stream exceptions. 49 | */ 50 | class StreamException : Exception { this(string msg) { super(msg); } } 51 | 52 | /// Ditto 53 | class ReadException : StreamException { this(string msg) { super(msg); } } 54 | 55 | /// Ditto 56 | class WriteException : StreamException { this(string msg) { super(msg); } } 57 | 58 | /// Ditto 59 | class SeekException : StreamException { this(string msg) { super(msg); } } 60 | -------------------------------------------------------------------------------- /source/io/buffer/traits.d: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright: Copyright Jason White, 2014-2016 3 | * License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). 4 | * Authors: Jason White 5 | */ 6 | module io.buffer.traits; 7 | 8 | import io.stream.traits; 9 | 10 | /** 11 | * Checks if the stream can be buffered. A stream that is exclusively read from 12 | * or written to can always be buffered. However, when both reads and writes 13 | * must be buffered, the stream must also be seekable. There are no exceptions 14 | * to this last rule when buffering. 15 | */ 16 | enum isBufferable(Stream) = (isSource!Stream ^ isSink!Stream) || 17 | isSeekable!Stream; 18 | 19 | unittest 20 | { 21 | import io.stream.interfaces; 22 | 23 | interface A : Source {} 24 | static assert(isBufferable!A); 25 | 26 | interface B : Sink {} 27 | static assert(isBufferable!B); 28 | 29 | // Not possible. Stream must be seekable. 30 | interface C : SourceSink {} 31 | static assert(!isBufferable!C); 32 | 33 | interface D : Seekable!SourceSink {} 34 | static assert(isBufferable!D); 35 | 36 | interface E : Seekable!Source {} 37 | static assert(isBufferable!E); 38 | 39 | interface F : Seekable!Sink {} 40 | static assert(isBufferable!F); 41 | } 42 | 43 | /** 44 | * Checks if the stream can be flushed. 45 | */ 46 | enum isFlushable(Stream) = 47 | is(typeof({ 48 | Stream s = void; 49 | s.flush(); 50 | })); 51 | 52 | unittest 53 | { 54 | import io.stream.interfaces; 55 | struct A {} 56 | static assert(!isFlushable!A); 57 | 58 | struct B { void flush(); } 59 | static assert(isFlushable!B); 60 | } 61 | -------------------------------------------------------------------------------- /source/io/file/pipe.d: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright: Copyright Jason White, 2014-2016 3 | * License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). 4 | * Authors: Jason White 5 | */ 6 | module io.file.pipe; 7 | 8 | import io.file.stream; 9 | 10 | struct Pipe(F = File) 11 | if (is(F == struct)) 12 | { 13 | F readEnd; // Read end 14 | F writeEnd; // Write end 15 | } 16 | 17 | /** 18 | * Creates an unnamed, unidirectional pipe that can be written to on one end and 19 | * read from on the other. 20 | */ 21 | Pipe!F pipe(F = File)() 22 | if (is(F == struct)) 23 | { 24 | version (Posix) 25 | { 26 | import core.sys.posix.unistd : pipe; 27 | 28 | int[2] fd = void; 29 | sysEnforce(pipe(fd) != -1); 30 | return Pipe!F(F(fd[0]), F(fd[1])); 31 | } 32 | else version(Windows) 33 | { 34 | import core.sys.windows.windows : CreatePipe; 35 | 36 | F.Handle readEnd = void, writeEnd = void; 37 | sysEnforce(CreatePipe(&readEnd, &writeEnd, null, 0)); 38 | return Pipe!F(F(readEnd), F(writeEnd)); 39 | } 40 | else 41 | { 42 | static assert(false, "Unsupported platform."); 43 | } 44 | } 45 | 46 | /// 47 | unittest 48 | { 49 | import core.thread : Thread; 50 | 51 | immutable message = "The quick brown fox jumps over the lazy dog."; 52 | 53 | auto p = pipe(); 54 | 55 | void writer() 56 | { 57 | p.writeEnd.write(message); 58 | 59 | // Since this is buffered, it must be flushed in order to avoid a 60 | // deadlock. 61 | p.writeEnd.flush(); 62 | } 63 | 64 | auto thread = new Thread(&writer).start(); 65 | 66 | // Read the message from the other thread. 67 | char[message.length] buf; 68 | assert(buf[0 .. p.readEnd.read(buf)] == message); 69 | 70 | thread.join(); 71 | } 72 | -------------------------------------------------------------------------------- /source/io/stream/shim.d: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright: Copyright Jason White, 2014-2016 3 | * License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). 4 | * Authors: Jason White 5 | */ 6 | module io.stream.shim; 7 | 8 | import io.stream.traits; 9 | import io.stream.types; 10 | 11 | /** 12 | * Wraps a stream to provide useful higher-level functions. 13 | */ 14 | struct StreamShim(Stream) 15 | if (isStream!Stream) 16 | { 17 | Stream stream; 18 | 19 | alias stream this; 20 | 21 | /** 22 | * Copying is disabled. Reference counting should be used instead. 23 | */ 24 | @disable this(this); 25 | 26 | /** 27 | * Forwards arguments to super class. 28 | */ 29 | this(T...)(auto ref T args) 30 | { 31 | import std.functional : forward; 32 | stream = Stream(forward!args); 33 | } 34 | 35 | static if (isSource!Stream) 36 | { 37 | /** 38 | * Fills the given buffer with data from the stream. 39 | * 40 | * Note: This is not guaranteed to read the entire buffer from the 41 | * stream. If $(D T.sizeof) is larger than 1, it is possible that an 42 | * element is partially read. If a guarantee that the entire buffer is 43 | * filled, use $(D readExactly) instead. 44 | * 45 | * Returns: The number of bytes read. 46 | */ 47 | size_t read(T)(T[] buf) 48 | { 49 | return stream.read(cast(ubyte[])buf); 50 | } 51 | } 52 | 53 | static if (isSink!Stream) 54 | { 55 | /** 56 | * Writes an array of type T to the stream. 57 | * 58 | * Returns: The number of bytes written. 59 | * 60 | * Note: This is not guaranteed to write the entire buffer to the 61 | * stream. If $(D T.sizeof) is larger than 1, it is possible that an 62 | * element may not be fully written. If the guarantee that the entire 63 | * buffer is written to the stream, use $(D writeExactly) instead. 64 | */ 65 | size_t write(T)(in T[] buf) 66 | { 67 | return stream.write(cast(const(ubyte)[])buf); 68 | } 69 | 70 | alias put = write; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | :rotating_light: **This library is unmaintained. Please see https://github.com/MartinNowak/io instead.** :rotating_light: 2 | 3 | [buildbadge]: https://travis-ci.org/jasonwhite/io.svg?branch=master 4 | [buildstatus]: https://travis-ci.org/jasonwhite/io 5 | 6 | # IO Streams [![Build Status][buildbadge]][buildstatus] 7 | 8 | This is a POSIX and Windows compatible IO stream library for D. 9 | 10 | The primary purpose of this package is to provide a better API than what is 11 | currently available in the D standard library. Secondly, it is meant to be fast. 12 | All file operations are implemented using the low-level system calls provided by 13 | the operating system. 14 | 15 | ## Features 16 | 17 | * File streams 18 | * Memory mapped files 19 | * Pipes 20 | * Temporary files 21 | * Generic stream buffering 22 | * Text serialization to streams 23 | 24 | ## Examples 25 | 26 | ### Sorting Lines 27 | 28 | ```d 29 | import io, std.array, std.algorithm; 30 | 31 | void main() 32 | { 33 | stdin 34 | .byLineCopy 35 | .array 36 | .sort() 37 | .each!println; 38 | } 39 | ``` 40 | 41 | ### Temporary Files 42 | 43 | ```d 44 | import io; 45 | 46 | void main() 47 | { 48 | // Create a temporary file to write to. This returns the path to the file 49 | // and its file handle. The file is automatically deleted when the file 50 | // handle is closed. The file handle is closed when it falls out of scope. 51 | auto temp = tempFile(); 52 | 53 | // Print to standard output. 54 | println("Temporary file path: ", temp.path); 55 | 56 | // Write an arbitrary array to the stream. 57 | temp.file.write("Hello world!"); 58 | 59 | // Seek to the beginning. 60 | temp.file.position = 0; 61 | 62 | // Read in 5 characters. 63 | char[5] buf; 64 | temp.file.read(buf); 65 | assert(buf == "Hello"); 66 | } 67 | ``` 68 | 69 | ### Memory Maps 70 | 71 | ```d 72 | import io; 73 | import std.parallelism, std.random; 74 | 75 | void main() 76 | { 77 | // Creates a 1 GiB file 78 | auto f = File("big_random_file", FileFlags.writeNew); 79 | f.length = 1024^^3; // 1 GiB 80 | 81 | // Fill the file with random data using a memory map. 82 | auto map = f.memoryMap!size_t(Access.write); 83 | foreach (i, ref e; parallel(map[])) 84 | e = uniform!"[]"(size_t.min, size_t.max); 85 | } 86 | ``` 87 | 88 | ## Documentation 89 | 90 | Documentation can be found [here](https://jasonwhite.github.io/io/). 91 | 92 | ## License 93 | 94 | [Boost license](/LICENSE.md) 95 | -------------------------------------------------------------------------------- /source/io/file/stdio.d: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright: Copyright Jason White, 2014-2016 3 | * License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). 4 | * Authors: Jason White 5 | * 6 | * Description: 7 | * Provides access to the standard I/O streams: $(D stdin), $(D stdout), and $(D 8 | * stderr). Note that each of these streams are buffered. 9 | * 10 | * To avoid conflict with $(D std._stdio), the file handles of each stream are 11 | * duplicated on static construction and closed upon static destruction. 12 | */ 13 | module io.file.stdio; 14 | 15 | import io.file.stream; 16 | import io.buffer.fixed; 17 | 18 | __gshared 19 | { 20 | /** 21 | * Standard input stream. 22 | * 23 | * Example: 24 | * Counting the number of lines from standard input. 25 | * --- 26 | * import io; 27 | * size_t lines = 0; 28 | * foreach (line; stdin.byLine) 29 | * ++lines; 30 | * --- 31 | */ 32 | File stdin; 33 | 34 | /** 35 | * Standard output stream. 36 | * 37 | * Example: 38 | * --- 39 | * import io; 40 | * stdout.write("Hello world!\n"); 41 | * stdout.flush(); 42 | * --- 43 | */ 44 | File stdout; 45 | 46 | /** 47 | * Standard error stream. 48 | * 49 | * stderr is often used for writing error messages or printing status 50 | * updates. 51 | * 52 | * Example: 53 | * Prints a useful status message. 54 | * --- 55 | * import core.thread : Thread; 56 | * import core.time : dur; 57 | * 58 | * immutable status = `|/-\`; 59 | * 60 | * for (size_t i = 0; ; ++i) 61 | * { 62 | * Thread.sleep(dur!"msecs"(100)); 63 | * stderr.write("Reticulating splines... "); 64 | * stderr.write([status[i % status.length], '\r']); 65 | * stderr.flush(); 66 | * } 67 | * --- 68 | */ 69 | File stderr; 70 | } 71 | 72 | shared static this() 73 | { 74 | // Initialize stdio streams. 75 | version (Posix) 76 | { 77 | import core.sys.posix.unistd : dup; 78 | stdin = File.dup(0); 79 | stdout = File.dup(1); 80 | stderr = File.dup(2); 81 | } 82 | else version (Windows) 83 | { 84 | import core.sys.windows.windows; 85 | stdin = File.dup(GetStdHandle(STD_INPUT_HANDLE)); 86 | stdout = File.dup(GetStdHandle(STD_OUTPUT_HANDLE)); 87 | stderr = File.dup(GetStdHandle(STD_ERROR_HANDLE)); 88 | } 89 | } 90 | 91 | shared static ~this() 92 | { 93 | stderr.flush(); 94 | stdout.flush(); 95 | } 96 | -------------------------------------------------------------------------------- /source/io/stream/interfaces.d: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright: Copyright Jason White, 2014-2016 3 | * License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). 4 | * Authors: Jason White 5 | */ 6 | module io.stream.interfaces; 7 | 8 | import io.stream.types; 9 | import io.stream.traits; 10 | 11 | /** 12 | * A source is a stream that can be read from. 13 | */ 14 | interface Source 15 | { 16 | /** 17 | * Reads data into the specified buffer. The number of bytes read is 18 | * returned. 19 | */ 20 | size_t read(ubyte[] buf); 21 | } 22 | 23 | unittest 24 | { 25 | static assert( isSource!Source); 26 | static assert(!isSink!Source); 27 | static assert(!isSeekable!Source); 28 | } 29 | 30 | /** 31 | * A sink is a stream that can be written to. 32 | */ 33 | interface Sink 34 | { 35 | /** 36 | * Writes data to the stream. The number of bytes successfully written is 37 | * returned. 38 | */ 39 | size_t write(in ubyte[] data); 40 | 41 | /// Ditto 42 | alias put = write; 43 | } 44 | 45 | unittest 46 | { 47 | static assert(!isSource!Sink); 48 | static assert( isSink!Sink); 49 | static assert(!isSeekable!Sink); 50 | } 51 | 52 | /** 53 | * A stream that is both a Source and a Sink. 54 | */ 55 | interface SourceSink : Source, Sink {} 56 | 57 | /** 58 | * A seekable stream can move the read/write starting position in the stream. 59 | */ 60 | interface Seekable(Stream) : Stream 61 | { 62 | /** 63 | * Seeks to the specified offset relative to the given starting location. 64 | * 65 | * Params: 66 | * offset = The offset relative to $(D from). 67 | * from = The relative position to seek to. 68 | */ 69 | long seekTo(long offset, From from = From.start); 70 | } 71 | 72 | unittest 73 | { 74 | static assert( isSource!(Seekable!Source)); 75 | static assert( isSink!(Seekable!Sink)); 76 | static assert( isSource!(Seekable!SourceSink)); 77 | static assert( isSink!(Seekable!SourceSink)); 78 | static assert(!isSource!(Seekable!Sink)); 79 | static assert(!isSink!(Seekable!Source)); 80 | static assert( isSeekable!(Seekable!Source)); 81 | static assert( isSeekable!(Seekable!Sink)); 82 | static assert( isSeekable!(Seekable!SourceSink)); 83 | } 84 | 85 | unittest 86 | { 87 | static assert(is(Seekable!SourceSink : Source)); 88 | static assert(is(Seekable!SourceSink : Sink)); 89 | static assert(is(Seekable!Source : Source)); 90 | static assert(is(Seekable!Sink : Sink)); 91 | static assert(!is(Seekable!Source : Sink)); 92 | static assert(!is(Seekable!Sink : Source)); 93 | } 94 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | platform: x64 2 | environment: 3 | matrix: 4 | - DC: dmd 5 | DVersion: 2.071.0 6 | arch: x64 7 | - DC: dmd 8 | DVersion: 2.071.0 9 | arch: x86 10 | - DC: ldc 11 | DVersion: '1.0.0-alpha1' 12 | arch: x64 13 | - DC: ldc 14 | DVersion: 0.17.1 15 | arch: x64 16 | 17 | matrix: 18 | allow_failures: 19 | - DC: ldc 20 | 21 | skip_tags: true 22 | branches: 23 | only: 24 | - master 25 | 26 | install: 27 | - ps: function SetUpDCompiler 28 | { 29 | if($env:DC -eq "dmd"){ 30 | if($env:arch -eq "x86"){ 31 | $env:DConf = "m32"; 32 | } 33 | elseif($env:arch -eq "x64"){ 34 | $env:DConf = "m64"; 35 | } 36 | echo "downloading ..."; 37 | $env:toolchain = "msvc"; 38 | $version = $env:DVersion; 39 | Invoke-WebRequest "http://downloads.dlang.org/releases/2.x/$($version)/dmd.$($version).windows.7z" -OutFile "c:\dmd.7z"; 40 | echo "finished."; 41 | pushd c:\\; 42 | 7z x dmd.7z > $null; 43 | popd; 44 | } 45 | elseif($env:DC -eq "ldc"){ 46 | echo "downloading ..."; 47 | if($env:arch -eq "x86"){ 48 | $env:DConf = "m32"; 49 | } 50 | elseif($env:arch -eq "x64"){ 51 | $env:DConf = "m64"; 52 | } 53 | $env:toolchain = "msvc"; 54 | $version = $env:DVersion; 55 | Invoke-WebRequest "https://github.com/ldc-developers/ldc/releases/download/v$($version)/ldc2-$($version)-win64-msvc.zip" -OutFile "c:\ldc.zip"; 56 | echo "finished."; 57 | pushd c:\\; 58 | 7z x ldc.zip > $null; 59 | popd; 60 | } 61 | } 62 | - ps: SetUpDCompiler 63 | - powershell -Command Invoke-WebRequest https://code.dlang.org/files/dub-0.9.24-windows-x86.zip -OutFile dub.zip 64 | - 7z x dub.zip -odub > nul 65 | - set PATH=%CD%\%binpath%;%CD%\dub;%PATH% 66 | - dub --version 67 | 68 | before_build: 69 | - ps: if($env:arch -eq "x86"){ 70 | $env:compilersetupargs = "x86"; 71 | $env:Darch = "x86"; 72 | } 73 | elseif($env:arch -eq "x64"){ 74 | $env:compilersetupargs = "amd64"; 75 | $env:Darch = "x86_64"; 76 | } 77 | - ps : if($env:DC -eq "dmd"){ 78 | $env:PATH += ";C:\dmd2\windows\bin;"; 79 | } 80 | elseif($env:DC -eq "ldc"){ 81 | $version = $env:DVersion; 82 | $env:PATH += ";C:\ldc2-$($version)-win64-msvc\bin"; 83 | $env:DC = "ldc2"; 84 | } 85 | - ps: $env:compilersetup = "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall"; 86 | - '"%compilersetup%" %compilersetupargs%' 87 | 88 | build_script: 89 | - echo dummy build script - dont remove me 90 | 91 | test_script: 92 | - echo %PLATFORM% 93 | - echo %Darch% 94 | - echo %DC% 95 | - echo %PATH% 96 | - '%DC% --version' 97 | - dub test --arch=%Darch% --compiler=%DC% 98 | -------------------------------------------------------------------------------- /source/io/stream/traits.d: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright: Copyright Jason White, 2014-2016 3 | * License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). 4 | * Authors: Jason White 5 | */ 6 | module io.stream.traits; 7 | 8 | import io.stream.types : From; 9 | 10 | /** 11 | * Checks if a type is a source. A source is a stream that can be read from and 12 | * must define the member function $(D read). The stream can be either a class 13 | * or a struct. 14 | */ 15 | enum isSource(Stream) = 16 | is(typeof({ 17 | Stream s = void; 18 | ubyte[] buf; 19 | ulong n = s.read(buf); 20 | })); 21 | 22 | unittest 23 | { 24 | static struct A {} 25 | static assert(!isSource!A); 26 | 27 | static struct B 28 | { 29 | size_t read(ubyte[] buf) { return buf.length; } 30 | } 31 | 32 | static assert(isSource!B); 33 | 34 | static struct C 35 | { 36 | void read() {} 37 | } 38 | 39 | static assert(!isSource!C); 40 | } 41 | 42 | /** 43 | * Checks if a type is a sink. A sink is a stream that can be written to and must 44 | * define the member function $(D write). The stream can be either a class or a 45 | * struct. 46 | */ 47 | enum isSink(Stream) = 48 | is(typeof({ 49 | Stream s = void; 50 | immutable ubyte[] data; 51 | ulong n = s.write(data); 52 | })); 53 | 54 | unittest 55 | { 56 | static struct A {} 57 | static assert(!isSink!A); 58 | 59 | static struct B 60 | { 61 | size_t write(in ubyte[] data) { return 0; } 62 | } 63 | 64 | static assert(isSink!B); 65 | 66 | static struct C 67 | { 68 | void write() {} 69 | } 70 | 71 | static assert(!isSink!C); 72 | } 73 | 74 | /** 75 | * Checks if a type is seekable. A seekable stream must define the member 76 | * function $(D seek). The stream can be either a class or a struct. 77 | */ 78 | enum isSeekable(Stream) = 79 | is(typeof({ 80 | Stream s = void; 81 | auto pos = s.seekTo(0, From.start); 82 | })); 83 | 84 | unittest 85 | { 86 | static struct A {} 87 | static assert(!isSeekable!A); 88 | 89 | static struct B { 90 | long seekTo(long offset, From from) { return 0; } 91 | } 92 | static assert(isSeekable!B); 93 | 94 | static struct C { 95 | // Should return the current position. 96 | void seekTo(long offset, From from) {} 97 | } 98 | static assert(!isSeekable!C); 99 | } 100 | 101 | /** 102 | * Checks if the type is both a source and a sink. 103 | */ 104 | enum isSourceSink(Stream) = isSource!Stream && isSink!Stream; 105 | 106 | unittest 107 | { 108 | static struct A 109 | { 110 | size_t write(in ubyte[] data) { return 0; } 111 | } 112 | 113 | static assert(!isSourceSink!A); 114 | 115 | static struct B 116 | { 117 | size_t write(in ubyte[] data) { return 0; } 118 | size_t read(ubyte[] buf) { return buf.length; } 119 | } 120 | 121 | static assert(isSourceSink!B); 122 | } 123 | 124 | /** 125 | * Checks if the type is either a source or a sink (i.e., a stream). 126 | */ 127 | enum isStream(Stream) = isSource!Stream || isSink!Stream; 128 | 129 | unittest 130 | { 131 | static struct A 132 | { 133 | size_t write(in ubyte[] data) { return 0; } 134 | } 135 | 136 | static assert(isStream!A); 137 | 138 | static struct B 139 | { 140 | size_t read(ubyte[] buf) { return buf.length; } 141 | } 142 | 143 | static assert(isStream!B); 144 | 145 | static struct C 146 | { 147 | long seekTo(long offset, From from) { return 0; } 148 | } 149 | 150 | static assert(!isStream!C); 151 | } 152 | -------------------------------------------------------------------------------- /source/io/stream/package.d: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright: Copyright Jason White, 2014-2016 3 | * License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). 4 | * Authors: Jason White 5 | */ 6 | module io.stream; 7 | 8 | public import io.stream.types; 9 | public import io.stream.traits; 10 | public import io.stream.shim; 11 | 12 | import std.traits : isArray; 13 | 14 | /** 15 | * Reads exactly the number of bytes requested from the stream. Throws 16 | * an exception if it cannot be done. Returns the filled buffer. 17 | * 18 | * Note that, because it can potentially take multiple system calls to 19 | * complete the read, the read is not guaranteed to be atomic with 20 | * respect to other reads. 21 | * 22 | * Throws: ReadException if the given buffer cannot be completely filled. 23 | */ 24 | T[] readExactly(Stream, T)(auto ref Stream stream, T[] buf) 25 | if (isSource!(Stream)) 26 | { 27 | ubyte[] byteBuf = cast(ubyte[])buf; 28 | 29 | size_t totalRead = 0; 30 | while (totalRead < byteBuf.length) 31 | { 32 | if (immutable n = stream.read(byteBuf[totalRead .. $])) 33 | totalRead += n; 34 | else 35 | throw new ReadException( 36 | "Failed to fill entire buffer from stream" 37 | ); 38 | } 39 | 40 | return buf; 41 | } 42 | 43 | /** 44 | * Writes exactly the given buffer and no less. Throws an exception if 45 | * it cannot be done. 46 | * 47 | * Note that, because it can potentially take multiple system calls to 48 | * complete the write, the write is not guaranteed to be atomic with 49 | * respect to other writes. 50 | * 51 | * Throws: WriteException if the given buffer cannot be completely written. 52 | */ 53 | void writeExactly(Stream, T)(auto ref Stream stream, in T[] buf) 54 | if (isSink!Stream) 55 | { 56 | const(ubyte)[] byteBuf = cast(const(ubyte)[])buf; 57 | 58 | size_t total = 0; 59 | 60 | while (total < byteBuf.length) 61 | { 62 | if (immutable n = stream.write(byteBuf[total .. $])) 63 | total += n; 64 | else 65 | throw new WriteException( 66 | "Failed to write entire buffer to stream" 67 | ); 68 | } 69 | } 70 | 71 | // Ditto 72 | void writeExactly(Stream, T)(auto ref Stream stream, const auto ref T value) 73 | if (isSink!Stream && !isArray!T) 74 | { 75 | stream.writeExactly((cast(ubyte*)&value)[0 .. T.sizeof]); 76 | } 77 | 78 | /** 79 | * Reads the rest of the stream. 80 | */ 81 | T[] readAll(T=ubyte, Stream)(auto ref Stream stream, long upTo = long.max) 82 | if (isSource!Stream && isSeekable!Stream) 83 | { 84 | import std.algorithm : min; 85 | import std.array : uninitializedArray; 86 | 87 | immutable remaining = min((stream.length - stream.position)/T.sizeof, upTo); 88 | 89 | auto buf = uninitializedArray!(T[])(remaining); 90 | 91 | immutable bytesRead = stream.read(buf); 92 | 93 | return buf[0 .. bytesRead/T.sizeof]; 94 | } 95 | 96 | /** 97 | * Set the position (in bytes) of a stream. 98 | */ 99 | @property void position(Stream)(auto ref Stream stream, long offset) 100 | if (isSeekable!Stream) 101 | { 102 | stream.seekTo(offset, From.start); 103 | } 104 | 105 | /** 106 | * Gets the position (in bytes) of a stream. 107 | */ 108 | @property auto position(Stream)(auto ref Stream stream) 109 | if (isSeekable!Stream) 110 | { 111 | return stream.seekTo(0, From.here); 112 | } 113 | 114 | /** 115 | * Skip the specified number of bytes forward or backward. 116 | * 117 | * Returns: The position (in bytes) in the stream after the seek. 118 | */ 119 | long skip(Stream)(auto ref Stream stream, long offset) 120 | if (isSeekable!Stream) 121 | { 122 | return stream.seekTo(offset, From.here); 123 | } 124 | -------------------------------------------------------------------------------- /source/io/text.d: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright: Copyright Jason White, 2014-2016 3 | * License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). 4 | * Authors: Jason White 5 | */ 6 | module io.text; 7 | 8 | import io.stream; 9 | import io.file.stdio : stdout; 10 | import io.buffer.traits : isFlushable; 11 | 12 | import std.functional : forward; 13 | 14 | /** 15 | * Serializes the given arguments to a text representation without a trailing 16 | * new line. 17 | */ 18 | size_t print(Stream, T...)(Stream stream, auto ref T args) 19 | if (isSink!Stream) 20 | { 21 | import std.conv : to; 22 | 23 | size_t length; 24 | 25 | foreach (arg; args) 26 | length += stream.write(arg.to!string); 27 | 28 | static if (isFlushable!Stream) 29 | stream.flush(); 30 | 31 | return length; 32 | } 33 | 34 | /// Ditto 35 | size_t print(T...)(auto ref T args) 36 | if (T.length > 0 && !isSink!(T[0])) 37 | { 38 | return stdout.print(forward!args); 39 | } 40 | 41 | unittest 42 | { 43 | import io.file.stream; 44 | import std.typecons : tuple; 45 | import std.typetuple : TypeTuple; 46 | 47 | // First tuple value is expected output. Remaining are the types to be 48 | // printed. 49 | alias tests = TypeTuple!( 50 | tuple(""), 51 | tuple("Test", "Test"), 52 | tuple("[4, 8, 15, 16, 23, 42]", [4, 8, 15, 16, 23, 42]), 53 | tuple("The answer is 42", "The answer is ", 42), 54 | tuple("01234567890", 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0), 55 | ); 56 | 57 | auto tf = testFile(); 58 | 59 | foreach (t; tests) 60 | { 61 | immutable output = t[0]; 62 | 63 | { 64 | auto f = File(tf.name, FileFlags.writeEmpty); 65 | assert(f.print(t[1 .. $]) == output.length); 66 | } 67 | 68 | { 69 | char[output.length] buf; 70 | auto f = File(tf.name, FileFlags.readExisting); 71 | assert(f.read(buf) == buf.length); 72 | assert(buf == output); 73 | } 74 | } 75 | } 76 | 77 | /** 78 | * Serializes the given arguments to a text representation followed by a new 79 | * line. 80 | */ 81 | size_t println(T...)(auto ref T args) 82 | { 83 | return print(forward!args, '\n'); 84 | } 85 | 86 | unittest 87 | { 88 | import io.file.stream; 89 | import std.typecons : tuple; 90 | import std.typetuple : TypeTuple; 91 | 92 | // First tuple value is expected output. Remaining are the types to be 93 | // printed. 94 | alias tests = TypeTuple!( 95 | tuple("\n"), 96 | tuple("Test\n", "Test"), 97 | tuple("[4, 8, 15, 16, 23, 42]\n", [4, 8, 15, 16, 23, 42]), 98 | tuple("The answer is 42\n", "The answer is ", 42), 99 | tuple("01234567890\n", 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0), 100 | ); 101 | 102 | auto tf = testFile(); 103 | 104 | foreach (t; tests) 105 | { 106 | immutable output = t[0]; 107 | 108 | { 109 | auto f = File(tf.name, FileFlags.writeEmpty); 110 | assert(f.println(t[1 .. $]) == output.length); 111 | } 112 | 113 | { 114 | char[output.length] buf; 115 | auto f = File(tf.name, FileFlags.readExisting); 116 | assert(f.read(buf) == buf.length); 117 | assert(buf == output); 118 | } 119 | } 120 | } 121 | 122 | /** 123 | * Serializes the given arguments according to the given format specifier 124 | * string. 125 | */ 126 | void printf(Stream, T...)(Stream stream, string format, auto ref T args) 127 | if (isSink!Stream) 128 | { 129 | import std.format : formattedWrite; 130 | formattedWrite(stream, forward!(format, args)); 131 | } 132 | 133 | /// Ditto 134 | void printf(T...)(string format, auto ref T args) 135 | { 136 | stdout.printf(forward!(format, args)); 137 | } 138 | 139 | unittest 140 | { 141 | import io.file.stream; 142 | import std.typecons : tuple; 143 | import std.typetuple : TypeTuple; 144 | 145 | // First tuple value is expected output. Remaining are the types to be 146 | // printed. 147 | alias tests = TypeTuple!( 148 | tuple("", ""), 149 | tuple("Test", "Test"), 150 | tuple("The answer is 42.", "The answer is %d.", 42), 151 | tuple("Hello, my name is Inigo Montoya", "Hello, my name is %s %s...", 152 | "Inigo", "Montoya") 153 | ); 154 | 155 | auto tf = testFile(); 156 | 157 | foreach (t; tests) 158 | { 159 | immutable output = t[0]; 160 | 161 | { 162 | auto f = File(tf.name, FileFlags.writeEmpty); 163 | f.printf(t[1 .. $]); 164 | } 165 | 166 | { 167 | char[output.length] buf; 168 | auto f = File(tf.name, FileFlags.readExisting); 169 | assert(f.read(buf) == buf.length); 170 | assert(buf == output); 171 | } 172 | } 173 | } 174 | 175 | /** 176 | * Like $(D printf), but also writes a new line. 177 | */ 178 | void printfln(Stream, T...)(Stream stream, string format, auto ref T args) 179 | if (isSink!Stream) 180 | { 181 | stream.printf(forward!(format, args)); 182 | stream.print('\n'); 183 | } 184 | 185 | /// Ditto 186 | void printfln(T...)(string format, auto ref T args) 187 | { 188 | stdout.printfln(forward!(format, args)); 189 | } 190 | 191 | /** 192 | * Convenience function for returning a delimiter range that iterates over 193 | * lines. 194 | */ 195 | @property auto byLine(T = char, Stream)(Stream stream) 196 | if (isSource!Stream) 197 | { 198 | import io.range : splitter; 199 | return splitter!T(stream, '\n'); 200 | } 201 | 202 | /** 203 | * Like $(D byLine), but duplicates each line. Obviously, this is less efficient 204 | * than using $(D byLine). 205 | */ 206 | @property auto byLineCopy(T = char, Stream)(Stream stream) 207 | if (isSource!Stream) 208 | { 209 | import io.range : splitter; 210 | import std.algorithm.iteration : map; 211 | return splitter!T(stream, '\n').map!(l => l.idup); 212 | } 213 | -------------------------------------------------------------------------------- /source/io/buffer/fixed.d: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright: Copyright Jason White, 2014-2016 3 | * License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). 4 | * Authors: Jason White 5 | * 6 | * Description: 7 | * Buffers a stream using a fixed-size buffer. 8 | */ 9 | module io.buffer.fixed; 10 | 11 | import io.stream.types; 12 | import io.stream.traits; 13 | import io.buffer.traits; 14 | 15 | struct FixedBufferBase(Stream) 16 | if (is(Stream == struct) && isBufferable!Stream) 17 | { 18 | Stream stream; 19 | 20 | alias stream this; 21 | 22 | // Buffer to store the data to be read or written. 23 | private ubyte[] _buffer; 24 | 25 | // Current read/write position in the buffer. For writes, this is >0 if data 26 | // has been written to the buffer but not flushed. 27 | private size_t _position; 28 | 29 | @disable this(this); 30 | 31 | /** 32 | * Forwards arguments to super class. 33 | */ 34 | this(T...)(auto ref T args) 35 | { 36 | import std.functional : forward; 37 | stream = Stream(forward!args); 38 | _buffer.length = 8192; 39 | } 40 | 41 | /** 42 | * Upon destruction, any pending writes are flushed to the underlying 43 | * stream. 44 | */ 45 | ~this() 46 | { 47 | static if (isSink!Stream) 48 | flush(); 49 | } 50 | 51 | /** 52 | * Sets the size of the buffer. The default is 8192 bytes. This will only 53 | * succeed if no data has been buffered (e.g., just after construction). 54 | */ 55 | @property void bufferSize(size_t size) 56 | { 57 | if (_position > 0) return; 58 | 59 | static if (isSource!Stream) 60 | { 61 | if (_valid > 0) return; 62 | } 63 | 64 | _buffer.length = size; 65 | } 66 | 67 | /** 68 | * Gets the current buffer size. The default is 8192 bytes. 69 | */ 70 | @property size_t bufferSize() const pure nothrow 71 | { 72 | return _buffer.length; 73 | } 74 | 75 | static if (isSource!Stream) 76 | { 77 | // Last valid position in the buffer. This is 0 if no read data is 78 | // sitting in the buffer. 79 | private size_t _valid; 80 | 81 | /** 82 | * Initiates a read. This handles flushing any data previously written. 83 | */ 84 | static if (isSink!Stream) 85 | { 86 | private void beginRead() 87 | { 88 | flush(); 89 | } 90 | } 91 | else 92 | { 93 | // Nothing to do, this should be optimized away. 94 | private void beginRead() {} 95 | } 96 | 97 | private size_t readPartial(scope ubyte[] buf) 98 | { 99 | import std.algorithm : min; 100 | 101 | // Satisfy what can be copied so far from the buffer. 102 | immutable satisfiable = min(_valid - _position, buf.length); 103 | buf[0 .. satisfiable] = _buffer[_position .. _position + satisfiable]; 104 | _position += satisfiable; 105 | 106 | return satisfiable; 107 | } 108 | 109 | /** 110 | * Reads data from the stream into the given buffer. The number of bytes 111 | * read is returned. 112 | */ 113 | size_t read(scope ubyte[] buf) 114 | { 115 | beginRead(); 116 | 117 | immutable satisfied = readPartial(buf); 118 | if (satisfied == buf.length) 119 | return satisfied; 120 | 121 | buf = buf[satisfied .. $]; 122 | 123 | // Large read? Get it directly from the stream. 124 | if (buf.length >= _buffer.length) 125 | return satisfied + stream.read(buf); 126 | 127 | // Buffer is empty, fill it back up. 128 | immutable bytesRead = stream.read(_buffer); 129 | _position = 0; 130 | _valid = bytesRead; 131 | 132 | // Finish the copy 133 | return satisfied + readPartial(buf); 134 | } 135 | } 136 | 137 | static if (isSink!Stream) 138 | { 139 | /** 140 | * Initiates a write. This will handle seeking to the correct position 141 | * due to a previous read. 142 | */ 143 | static if (isSource!Stream) 144 | { 145 | private void beginWrite() 146 | { 147 | if (_valid == 0) return; 148 | 149 | // The length of the window indicates how much data hasn't 150 | // "really" been read from the stream. Just seek backwards that 151 | // distance. 152 | stream.seekTo(_position - _valid, From.here); 153 | _position = _valid = 0; 154 | } 155 | } 156 | else 157 | { 158 | // Nothing to do. This should be optimized away. 159 | private void beginWrite() {} 160 | } 161 | 162 | /* 163 | * Copies as much as possible to the stream buffer. The number of bytes 164 | * copied is returned. 165 | */ 166 | private size_t writePartial(in ubyte[] buf) 167 | { 168 | import std.algorithm : min; 169 | 170 | immutable satisfiable = min(_buffer.length - _position, buf.length); 171 | _buffer[_position .. _position + satisfiable] = buf[0 .. satisfiable]; 172 | _position += satisfiable; 173 | 174 | return satisfiable; 175 | } 176 | 177 | /** 178 | * Writes the given data to the buffered stream. When the internal 179 | * buffer is completely filled, it is flushed to the underlying stream. 180 | */ 181 | size_t write(in ubyte[] buf) 182 | { 183 | beginWrite(); 184 | 185 | immutable satisfied = writePartial(buf); 186 | if (satisfied == buf.length) 187 | return satisfied; 188 | 189 | const(ubyte)[] leftOver = buf[satisfied .. $]; 190 | 191 | // Buffer is full and there is more to write. Flush it. 192 | flush(); 193 | 194 | // Large write? Push it directly to the stream. 195 | if (leftOver.length >= _buffer.length) 196 | return satisfied + stream.write(leftOver); 197 | 198 | // Write the rest. 199 | return satisfied + writePartial(leftOver); 200 | } 201 | 202 | alias put = write; 203 | 204 | /** 205 | * Writes any pending data to the underlying stream. 206 | */ 207 | void flush() 208 | { 209 | static if (isSource!Stream) 210 | { 211 | if (_valid > 0) 212 | return; 213 | } 214 | 215 | if (_position > 0) 216 | { 217 | immutable bytesWritten = stream.write(_buffer[0 .. _position]); 218 | assert(bytesWritten == _position); 219 | _position = 0; 220 | } 221 | } 222 | } 223 | 224 | static if (isSeekable!Stream) 225 | { 226 | /** 227 | * Seeks to the given position relative to the given starting point. 228 | */ 229 | long seekTo(long offset, From from = From.start) 230 | { 231 | static if (isSource!Stream) 232 | { 233 | if (_valid > 0) 234 | { 235 | if (from == From.here) 236 | { 237 | // Can we seek within the buffer? 238 | // FIXME: Handle potential integer overflow. 239 | if (_position + offset < _valid) 240 | { 241 | _position += offset; 242 | return stream.seekTo(0, From.here) + (_position - _valid); 243 | } 244 | } 245 | 246 | // Invalidate the window 247 | _position = _valid = 0; 248 | } 249 | } 250 | 251 | static if (isSink!Stream) 252 | { 253 | flush(); 254 | } 255 | 256 | return stream.seekTo(offset, from); 257 | } 258 | } 259 | } 260 | 261 | import std.typecons : RefCounted, RefCountedAutoInitialize; 262 | alias FixedBuffer(Stream) = RefCounted!(FixedBufferBase!(Stream), RefCountedAutoInitialize.no); 263 | -------------------------------------------------------------------------------- /source/io/file/temp.d: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright: Copyright Jason White, 2014-2016 3 | * License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). 4 | * Authors: Jason White 5 | */ 6 | module io.file.temp; 7 | 8 | import io.stream; 9 | import io.file.stream; 10 | 11 | private version (Windows) 12 | { 13 | import core.sys.windows.windows; 14 | 15 | extern (Windows) nothrow export 16 | { 17 | UINT GetTempFileNameW( 18 | LPCWSTR lpPathName, 19 | LPCWSTR lpPrefixString, 20 | UINT uUnique, 21 | LPWSTR lpTempFileName 22 | ); 23 | 24 | DWORD GetTempPathW( 25 | DWORD nBufferLength, 26 | LPWSTR lpBuffer 27 | ); 28 | } 29 | } 30 | 31 | version (Posix) 32 | private const(char*) tempDirImpl() 33 | { 34 | import core.sys.posix.stdlib; 35 | import core.sys.posix.fcntl; 36 | 37 | static const(char*) isDir(const char *path) 38 | { 39 | stat_t statbuf = void; 40 | 41 | if (stat(path, &statbuf) == 0 && (statbuf.st_mode & S_IFMT) == S_IFDIR) 42 | return path; 43 | 44 | return null; 45 | } 46 | 47 | // TODO: Use secure_getenv, if available, instead? 48 | if (auto path = getenv("TMPDIR")) return path; 49 | if (auto path = getenv("TEMP")) return path; 50 | if (auto path = getenv("TMP")) return path; 51 | 52 | if (auto path = isDir("/tmp")) return path; 53 | if (auto path = isDir("/var/tmp")) return path; 54 | if (auto path = isDir("/usr/tmp")) return path; 55 | 56 | return null; 57 | } 58 | 59 | version (Windows) 60 | private wstring tempDirImpl() 61 | { 62 | static wchar[MAX_PATH] buf; 63 | immutable len = GetTempPathW(buf.length, buf.ptr); 64 | return cast(wstring)(buf[0 .. len]); 65 | } 66 | 67 | /** 68 | * Returns the path to a directory for temporary files. 69 | * 70 | * On Windows, this function returns the result of calling the Windows API 71 | * function 72 | * $(LINK2 http://msdn.microsoft.com/en-us/library/windows/desktop/aa364992.aspx, $(D GetTempPath)). 73 | * 74 | * On POSIX platforms, it searches through the following list of directories and 75 | * returns the first one which is found to exist: 76 | * $(OL 77 | * $(LI The directory given by the $(D TMPDIR) environment variable.) 78 | * $(LI The directory given by the $(D TEMP) environment variable.) 79 | * $(LI The directory given by the $(D TMP) environment variable.) 80 | * $(LI $(D /tmp)) 81 | * $(LI $(D /var/tmp)) 82 | * $(LI $(D /usr/tmp)) 83 | * ) 84 | * 85 | * On all platforms, $(D tempDir) returns $(D ".") on failure, representing the 86 | * current working directory. 87 | * 88 | * The return value of the function is cached, so the procedures described above 89 | * will only be performed the first time the function is called. All subsequent 90 | * runs will return the same string, regardless of whether environment variables 91 | * and directory structures have changed in the meantime. 92 | * 93 | * The POSIX $(D tempDir) algorithm is inspired by Python's 94 | * $(LINK2 http://docs.python.org/library/tempfile.html#tempfile.tempdir, $(D tempfile.tempdir)). 95 | */ 96 | T tempDir(T = string)() @trusted 97 | if (is(T : string) || is(T : wstring)) 98 | { 99 | import std.conv : to; 100 | 101 | static T cache; 102 | 103 | if (cache is null) 104 | { 105 | cache = tempDirImpl().to!T(); 106 | if (cache is null) cache = "."; 107 | } 108 | 109 | return cache; 110 | } 111 | 112 | /** 113 | * Struct representing a temporary file. Returned by $(LREF tempFile). 114 | */ 115 | struct TempFile(File, Path) 116 | { 117 | /** 118 | * The opened file stream. If $(D AutoDelete.yes) is specified, when this is 119 | * closed, the file is deleted. 120 | */ 121 | File file; 122 | 123 | /** 124 | * Path to the file. This is not guaranteed to exist if $(D AutoDelete.yes) 125 | * is specified. For example, on POSIX, the file is deleted as soon as it is 126 | * created such that, when the last file descriptor to it is closed, the 127 | * file is deleted. If $(D AutoDelete.no) is specified, this path $(I is) 128 | * guaranteed to exist. 129 | */ 130 | Path path; 131 | } 132 | 133 | /** 134 | * Used with $(LREF tempFile) to choose if a temporary file should be deleted 135 | * automatically when it is closed. 136 | */ 137 | enum AutoDelete 138 | { 139 | no, 140 | yes 141 | } 142 | 143 | /** 144 | * Creates a temporary file. The file is automatically deleted when it is no 145 | * longer referenced. The temporary file is always opened with both read and 146 | * write access. 147 | * 148 | * Params: 149 | * autoDelete = If set to $(D AutoDelete.yes) (the default), the file is 150 | * deleted from the file system after the file handle is closed. 151 | * Otherwise, the file must be deleted manually. 152 | * dir = Directory to create the temporary file in. By default, this is $(LREF 153 | * tempDir). 154 | * 155 | * Example: 156 | * Creates a temporary file and writes to it. 157 | * --- 158 | * auto f = tempFile.file; 159 | * assert(f.position == 0); 160 | * f.write("Hello"); 161 | * assert(f.position == 5); 162 | * --- 163 | * 164 | * Example: 165 | * Creates a temporary file, but doesn't delete it. This is useful to ensure a 166 | * uniquely named file exists so that it can be written to by another process. 167 | * --- 168 | * auto path = tempFile(AutoDelete.no).path; 169 | * --- 170 | */ 171 | version (Posix) 172 | TempFile!(F, string) tempFile(F = File)(AutoDelete autoDelete = AutoDelete.yes, 173 | string dir = tempDir) 174 | { 175 | /* Implementation note: Since Linux 3.11, there is the flag O_TMPFILE which 176 | * can be used to open a temporary file. This creates an unnamed inode in 177 | * the specified directory. Because the inode is unnamed, it will be 178 | * automatically deleted once the file descriptor is closed. In the future, 179 | * once Linux 3.11 is not so new, this flag could be used instead. 180 | */ 181 | 182 | import core.sys.posix.stdlib : mkstemp; 183 | import core.sys.posix.unistd : unlink; 184 | import std.exception : assumeUnique; 185 | 186 | char[] path = dir ~ "/XXXXXX\0".dup; 187 | 188 | int fd = mkstemp(path.ptr); 189 | sysEnforce(fd != File.InvalidHandle, 190 | "Failed to create temporary file '"~ path[0 .. $-1].idup ~"'" 191 | ); 192 | 193 | // Unlink the file to ensure it is deleted automatically when all 194 | // file descriptors referring to it are closed. 195 | if (autoDelete == AutoDelete.yes) 196 | sysEnforce(unlink(path.ptr) == 0, "Failed to unlink temporary file"); 197 | 198 | static if (is(F == class)) 199 | return typeof(return)(new F(fd), assumeUnique(path[0 .. $-1])); 200 | else 201 | return typeof(return)(F(fd), assumeUnique(path[0 .. $-1])); 202 | } 203 | 204 | version (Windows) 205 | TempFile!(F, T) tempFile(F = File, T = string)( 206 | AutoDelete autoDelete = AutoDelete.yes, T dir = tempDir!T) 207 | if ((is(T : string) || is(T : wstring))) 208 | { 209 | import std.conv : to; 210 | import std.utf : toUTF16z; 211 | import std.exception : assumeUnique; 212 | import core.stdc.wchar_ : wcslen; 213 | 214 | wchar[MAX_PATH] buf; 215 | sysEnforce( 216 | GetTempFileNameW(toUTF16z(dir), "tmp", 0, buf.ptr), 217 | "Failed to generate temporary file path" 218 | ); 219 | 220 | wchar[] path = buf[0 .. wcslen(buf.ptr)]; 221 | 222 | auto h = CreateFileW( 223 | // Temporary file name 224 | path.ptr, 225 | 226 | // Desired access 227 | GENERIC_READ | GENERIC_WRITE, 228 | 229 | // Share mode 230 | FILE_SHARE_DELETE | FILE_SHARE_READ | FILE_SHARE_WRITE, 231 | 232 | // Security attributes 233 | null, 234 | 235 | // Creation disposition. Note that GetTempFileName creates this file. 236 | CREATE_ALWAYS, 237 | 238 | // Flags and attributes 239 | FILE_ATTRIBUTE_NORMAL | FILE_ATTRIBUTE_TEMPORARY | 240 | ((autoDelete == AutoDelete.yes) ? FILE_FLAG_DELETE_ON_CLOSE : 0), 241 | 242 | // Template file 243 | null, 244 | ); 245 | 246 | sysEnforce( 247 | h != File.InvalidHandle, 248 | "Failed to create temporary file '"~ path.to!string ~"'" 249 | ); 250 | 251 | static if (is(F == class)) 252 | return typeof(return)(new F(h), assumeUnique(path).to!T); 253 | else 254 | return typeof(return)(F(h), assumeUnique(path).to!T); 255 | } 256 | 257 | unittest 258 | { 259 | auto f = tempFile.file; 260 | assert(f.position == 0); 261 | f.write("Hello"); 262 | assert(f.position == 5); 263 | } 264 | -------------------------------------------------------------------------------- /source/io/socket/stream.d: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright: Copyright Jason White, 2014-2016 3 | * License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). 4 | * Authors: Thayne McCombs 5 | */ 6 | module io.socket.stream; 7 | 8 | import io.stream; 9 | import std.socket; 10 | 11 | /** 12 | * A wrapper around a socket that provides stream functionality without buffering. 13 | */ 14 | struct UnbufferedSocketStreamBase 15 | { 16 | /** 17 | * Create a new Stream from an existing Socket. 18 | * 19 | * Params: 20 | * socket = The socket to make a stream from. 21 | * 22 | */ 23 | this(Socket socket) 24 | { 25 | _socket = socket; 26 | } 27 | 28 | unittest 29 | { 30 | auto pair = socketPair(); 31 | auto sock = UnbufferedSocketStream(pair[0]); 32 | assert(sock.isOpen); 33 | 34 | immutable ubyte[] data = [1,2,3,4,5]; 35 | ubyte[10] buff; 36 | 37 | sock.write(data); 38 | 39 | assert(pair[1].receive(buff) == 5); 40 | assert(buff[0..5] == data); 41 | } 42 | 43 | /** 44 | * Create a new SocketStream that is connected to `address` as 45 | * a client. The socket is created in streaming mode (obviously). 46 | * 47 | * Params: 48 | * address = The address to connect to. 49 | */ 50 | this(Address address) 51 | { 52 | _socket = new Socket(address.addressFamily, SocketType.STREAM); 53 | _socket.connect(address); 54 | } 55 | 56 | unittest 57 | { 58 | auto server = new TcpSocket(AddressFamily.INET); 59 | server.bind(new InternetAddress("localhost", InternetAddress.PORT_ANY)); 60 | server.listen(10); 61 | 62 | auto sock = SocketStream(server.localAddress); 63 | assert(sock.remoteAddress == server.localAddress); 64 | assert(sock.isOpen); 65 | } 66 | 67 | 68 | /** 69 | * Copying is disabled, because reference counting should be used instead. 70 | */ 71 | @disable this(this); 72 | 73 | /** 74 | * Returns true if the socket is alive. 75 | */ 76 | @property isOpen() @safe 77 | { 78 | return _socket && _socket.isAlive; 79 | } 80 | 81 | /** 82 | * Returns the underlying Socket. 83 | */ 84 | @property socket() @safe 85 | { 86 | return _socket; 87 | } 88 | 89 | /** 90 | * Reads data from the socket. 91 | * 92 | * Params: 93 | * buf = The buffer to read the data into. The length of the buffer 94 | * specifies how much data should be read. 95 | * 96 | * Returns: The number of bytes that were read. If the socket is blocking 97 | * wait for data to be available. If the remote side has closed 98 | * the connection 0 is returned. 99 | * 100 | * Throws: SocketException on failure. 101 | */ 102 | size_t read(scope ubyte[] buf) @safe 103 | in { assert(isOpen); } 104 | body 105 | { 106 | immutable n = _socket.receive(buf); 107 | socketEnforce(n != Socket.ERROR, "Failed to read from socket"); 108 | return n; 109 | } 110 | 111 | unittest 112 | { 113 | auto pair = socketPair(); 114 | auto sock = UnbufferedSocketStream(pair[0]); 115 | immutable ubyte[] data = [10,20,30,40]; 116 | pair[1].send(data); 117 | ubyte[10] buff; 118 | assert(sock.read(buff) == 4); 119 | assert(buff[0..4] == data); 120 | pair[1].close(); 121 | assert(sock.read(buff) == 0); 122 | } 123 | 124 | /** 125 | * Writes data to the socket. 126 | * 127 | * Params: 128 | * data = The data to write to the file. The length of the slice indicates 129 | * how much data should be written. 130 | * 131 | * Returns: The number of bytes that were written. 132 | * 133 | * Throws: SocketException on failure. 134 | */ 135 | size_t write(in ubyte[] data) @safe 136 | in { assert(isOpen); } 137 | body 138 | { 139 | immutable n = _socket.send(data); 140 | socketEnforce(n != Socket.ERROR, "Failed to write to socket"); 141 | return n; 142 | } 143 | 144 | unittest 145 | { 146 | auto pair = socketPair(); 147 | auto sock = UnbufferedSocketStream(pair[0]); 148 | immutable ubyte[] data = [5,9,10]; 149 | sock.write(data); 150 | ubyte[10] buff; 151 | assert(pair[1].receive(buff) == 3); 152 | assert(buff[0..3] == data); 153 | sock.write(data); 154 | sock.write(data); 155 | assert(pair[1].receive(buff) == 6); 156 | assert(buff[0..6] == data ~ data); 157 | } 158 | 159 | /// ditto 160 | alias put = write; 161 | 162 | /** 163 | * If the Socket is open, shut down both directions and close. 164 | * Otherwise, it does nothing. 165 | */ 166 | void close() @safe 167 | { 168 | import std.socket : SocketShutdown; 169 | if (isOpen) 170 | { 171 | _socket.shutdown(SocketShutdown.BOTH); 172 | _socket.close(); 173 | } 174 | } 175 | 176 | unittest 177 | { 178 | auto pair = socketPair(); 179 | auto sock = UnbufferedSocketStream(pair[0]); 180 | sock.close(); 181 | assert(!sock.isOpen); 182 | ubyte[1] buff; 183 | assert(pair[1].receive(buff) == 0); 184 | } 185 | 186 | /** 187 | * Detach the socket from this socket stream and return it. 188 | * 189 | * The stream is closed after becoming detached. 190 | * This can be used to avoid closing a socket when the stream is destroyed. 191 | */ 192 | Socket detach() @safe 193 | { 194 | scope(success) { _socket = null; } 195 | return _socket; 196 | } 197 | 198 | /// Ditto 199 | ~this() 200 | { 201 | close(); 202 | } 203 | 204 | alias _socket this; 205 | 206 | private: 207 | Socket _socket; 208 | } 209 | 210 | unittest 211 | { 212 | static assert(isSourceSink!UnbufferedSocketStreamBase); 213 | } 214 | 215 | /** 216 | * A stream that wraps a socket with buffered writes. 217 | */ 218 | struct SocketStreamBase { 219 | alias _stream this; 220 | 221 | @disable this(this); 222 | 223 | /** 224 | * Forwards argument to UnbufferedSocketStreamBase 225 | */ 226 | this(T...)(auto ref T args) 227 | { 228 | import std.functional : forward; 229 | _stream = UnbufferedSocketStreamBase(forward!args); 230 | _buffer.length = 8192; 231 | } 232 | 233 | 234 | /** 235 | * Sets the size of the buffer. The default is 8192 bytes. 236 | * If there is currently data in the buffer, it will be flushed. 237 | */ 238 | @property void bufferSize(size_t size) 239 | { 240 | if (_pos > 0) 241 | { 242 | flush(); 243 | } 244 | _buffer.length = size; 245 | } 246 | 247 | /** 248 | * Get the current buffer size. The default is 8192 bytes (8KB). 249 | */ 250 | @property size_t bufferSize() const pure nothrow @nogc 251 | { 252 | return _buffer.length; 253 | } 254 | 255 | /** 256 | * Upon destruction, any pending writes are flushed 257 | * to the socket. 258 | */ 259 | ~this() 260 | { 261 | // don't use flush because we want to avoid throwing an error 262 | _stream.socket.send(_buffer[0.._pos]); 263 | } 264 | 265 | /** 266 | * Writes any pending data to the socket. 267 | */ 268 | void flush() @safe 269 | { 270 | if (_pos > 0) 271 | { 272 | _stream.write(_buffer[0.._pos]); 273 | _pos = 0; 274 | } 275 | } 276 | 277 | /** 278 | * Write data to the stream, but buffer input 279 | * so that only sufficiently large packets are sent. 280 | */ 281 | size_t write(in ubyte[] buf) @safe 282 | { 283 | immutable satisfied = writePartial(buf); 284 | if (satisfied == buf.length) 285 | { 286 | return satisfied; 287 | } 288 | 289 | const(ubyte)[] leftOver = buf[satisfied .. $]; 290 | 291 | if (leftOver.length >= _buffer.length) 292 | { 293 | // leftOver is bigger than _buffer, write directly to socket 294 | return satisfied + _stream.write(leftOver); 295 | } 296 | else 297 | { 298 | return satisfied + writePartial(leftOver); 299 | } 300 | } 301 | 302 | unittest 303 | { 304 | auto pair = socketPair(); 305 | auto sock = SocketStream(pair[0]); 306 | auto other = pair[1]; 307 | other.blocking = false; 308 | sock.bufferSize = 10; 309 | 310 | ubyte[] data = [1,2,3,4,5]; 311 | ubyte[20] buff; 312 | 313 | sock.write(data); 314 | assert(other.receive(buff) == Socket.ERROR); 315 | assert(wouldHaveBlocked()); 316 | sock.write(data); 317 | assert(other.receive(buff) == 10); 318 | assert(buff[0..10] == data ~ data); 319 | sock.write(data); 320 | sock.flush(); 321 | buff = 0; 322 | assert(other.receive(buff) == 5); 323 | assert(buff[0..5] == data); 324 | } 325 | 326 | private size_t writePartial(in ubyte[] buf) @safe 327 | { 328 | import std.algorithm : min; 329 | immutable satisfiable = min(_buffer.length - _pos, buf.length); 330 | _buffer[_pos .. _pos + satisfiable] = buf[0 .. satisfiable]; 331 | _pos += satisfiable; 332 | 333 | if (_pos == _buffer.length) 334 | { 335 | // Buffer is full and there is more to write. Flush it. 336 | flush(); 337 | } 338 | 339 | return satisfiable; 340 | } 341 | 342 | /** 343 | * Reads data from the socket. 344 | * The stream itself doesn't buffer reading 345 | * because the OS already buffers when receiving 346 | * on a streaming socket. 347 | */ 348 | size_t read(scope ubyte[] buf) @safe 349 | { 350 | return _stream.read(buf); 351 | } 352 | 353 | 354 | private: 355 | UnbufferedSocketStreamBase _stream; 356 | ubyte[] _buffer; 357 | size_t _pos; 358 | } 359 | 360 | unittest 361 | { 362 | static assert(isSourceSink!SocketStreamBase); 363 | } 364 | 365 | import std.typecons : RefCounted, RefCountedAutoInitialize; 366 | alias UnbufferedSocketStream = RefCounted!(StreamShim!UnbufferedSocketStreamBase, RefCountedAutoInitialize.no); 367 | alias SocketStream = RefCounted!(StreamShim!SocketStreamBase, RefCountedAutoInitialize.no); 368 | 369 | unittest 370 | { 371 | static assert(isSourceSink!SocketStream); 372 | static assert(isSourceSink!UnbufferedSocketStream); 373 | } 374 | 375 | 376 | /** 377 | * Enforce that `check` is true, and throw a `SocketException` if it isn't. 378 | */ 379 | void socketEnforce(string file = __FILE__, size_t line = __LINE__)(bool check, lazy string msg = null) 380 | { 381 | if (!check) 382 | { 383 | throw new SocketOSException(msg, file, line); 384 | } 385 | } 386 | 387 | /** 388 | * Call `accept` on the socket and return the result as a `SocketStream`. 389 | */ 390 | SocketStream acceptStream(Socket socket) 391 | { 392 | return SocketStream(socket.accept()); 393 | } 394 | -------------------------------------------------------------------------------- /source/io/file/mmap.d: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright: Copyright Jason White, 2014-2016 3 | * License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). 4 | * Authors: Jason White 5 | * 6 | * Synopsis: 7 | * --- 8 | * // Creates a 1 GiB file containing random data. 9 | * import io.file; 10 | * import std.parallelism : parallel; 11 | * auto f = File("big_random_file.dat", FileFlags.writeNew); 12 | * f.length = 1024^^3; // 1 GiB 13 | * 14 | * auto map = f.memoryMap!size_t(Access.write); 15 | * foreach (i, ref e; parallel(map[])) 16 | * e = uniform!"[]"(size_t.min, size_t.max); 17 | * --- 18 | */ 19 | module io.file.mmap; 20 | 21 | public import io.file.stream; 22 | 23 | version (Posix) 24 | { 25 | import core.sys.posix.sys.mman; 26 | 27 | // Converts file access flags to POSIX protection flags. 28 | private @property int protectionFlags(Access access) pure nothrow 29 | { 30 | int flags = PROT_NONE; 31 | if (access & Access.read) flags |= PROT_READ; 32 | if (access & Access.write) flags |= PROT_WRITE; 33 | if (access & Access.execute) flags |= PROT_EXEC; 34 | return flags; 35 | } 36 | } 37 | else version (Windows) 38 | { 39 | import core.sys.windows.windows; 40 | 41 | enum FILE_MAP_EXECUTE = 0x0020; 42 | 43 | // Converts file access flags to Windows protection flags. 44 | private @property DWORD protectionFlags(Access access) pure nothrow 45 | { 46 | switch (access) 47 | { 48 | case Access.read: return PAGE_READONLY; 49 | case Access.write: return PAGE_READWRITE; 50 | case Access.readWrite: return PAGE_READWRITE; 51 | case Access.read | Access.execute: return PAGE_EXECUTE_READ; 52 | case Access.write | Access.execute: return PAGE_EXECUTE_READWRITE; 53 | case Access.readWrite | Access.execute: return PAGE_EXECUTE_READWRITE; 54 | default: return PAGE_READONLY; 55 | } 56 | } 57 | 58 | // Converts file access flags to Windows MapViewOfFileEx flags 59 | private @property DWORD mapViewFlags(Access access) pure nothrow 60 | { 61 | DWORD flags = 0; 62 | if (access & Access.read) flags |= FILE_MAP_READ; 63 | if (access & Access.write) flags |= FILE_MAP_WRITE; 64 | if (access & Access.execute) flags |= FILE_MAP_EXECUTE; 65 | return flags; 66 | } 67 | } 68 | else 69 | { 70 | static assert(false, "Not implemented on this platform."); 71 | } 72 | 73 | /** 74 | * A memory mapped file. This essentially allows a file to be used as if it were 75 | * a slice of memory. For many use cases, it is a very efficient means of 76 | * accessing a file. 77 | */ 78 | private struct MemoryMapImpl(T) 79 | { 80 | // Memory mapped data. 81 | T[] data; 82 | 83 | alias data this; 84 | 85 | version (Windows) 86 | private HANDLE fileMap = null; 87 | 88 | /** 89 | * Maps the contents of the specified file into memory. 90 | * 91 | * Params: 92 | * file = Open file to be mapped. The file may be closed after being 93 | * mapped to memory. The file must not be a terminal or a pipe. It 94 | * must have random access capabilities. 95 | * access = Access flags of the memory region. Read-only access by 96 | * default. 97 | * length = Length of the memory map in number of $(D T). If 0, the length 98 | * is taken to be the size of the file. 0 by default. 99 | * start = Offset within the file to start the mapping in bytes. 0 by 100 | * default. 101 | * share = If true, changes are visible to other processes. If false, 102 | * changes are not visible to other processes and are never 103 | * written back to the file. True by default. 104 | * address = The preferred memory address to map the file to. This is just 105 | * a hint, the system is may or may not use this address. If 106 | * null, the system chooses an appropriate address. Null by 107 | * default. 108 | * 109 | * Throws: SysException if the memory map could not be created. 110 | */ 111 | this(File file, Access access = Access.read, size_t length = 0, 112 | long start = 0, bool share = true, void* address = null) 113 | { 114 | import std.conv : to; 115 | 116 | if (length == 0) 117 | length = (file.length - start).to!size_t / T.sizeof; 118 | 119 | version (Posix) 120 | { 121 | int flags = share ? MAP_SHARED : MAP_PRIVATE; 122 | 123 | auto p = cast(T*)mmap( 124 | address, // Preferred base address 125 | length * T.sizeof, // Length of the memory map 126 | access.protectionFlags, // Protection flags 127 | flags, // Mapping flags 128 | file.handle, // File descriptor 129 | cast(off_t)start // Offset within the file 130 | ); 131 | 132 | sysEnforce(p != MAP_FAILED, "Failed to map file to memory"); 133 | 134 | data = p[0 .. length]; 135 | } 136 | else version (Windows) 137 | { 138 | immutable ULARGE_INTEGER maxSize = 139 | {QuadPart: cast(ulong)(length * T.sizeof)}; 140 | 141 | // Create the file mapping object 142 | fileMap = CreateFileMappingW( 143 | file.handle, // File handle 144 | null, // Security attributes 145 | access.protectionFlags, // Page protection flags 146 | maxSize.HighPart, // Maximum size (high-order bytes) 147 | maxSize.LowPart, // Maximum size (low-order bytes) 148 | null // Optional name to give the object 149 | ); 150 | 151 | sysEnforce(fileMap, "Failed to create file mapping object"); 152 | 153 | scope(failure) CloseHandle(fileMap); 154 | 155 | immutable ULARGE_INTEGER offset = {QuadPart: cast(ulong)start}; 156 | 157 | // Create a view into the file mapping 158 | auto p = cast(T*)MapViewOfFileEx( 159 | fileMap, // File mapping object 160 | access.mapViewFlags, // Desired access 161 | offset.HighPart, // File offset (high-order bytes) 162 | offset.LowPart, // File offset (low-order bytes) 163 | length * T.sizeof, // Number of bytes to map 164 | address, // Preferred base address 165 | ); 166 | 167 | sysEnforce(p, "Failed to map file to memory"); 168 | 169 | data = p[0 .. length]; 170 | } 171 | } 172 | 173 | /** 174 | * Unmaps the file from memory and writes back any changes to the file 175 | * system. 176 | */ 177 | ~this() 178 | { 179 | if (data is null) return; 180 | 181 | version (Posix) 182 | { 183 | sysEnforce( 184 | munmap(data.ptr, data.length * T.sizeof) == 0, 185 | "Failed to unmap memory" 186 | ); 187 | } 188 | else version (Windows) 189 | { 190 | sysEnforce( 191 | UnmapViewOfFile(data.ptr) != 0, 192 | "Failed to unmap memory" 193 | ); 194 | sysEnforce( 195 | CloseHandle(fileMap), 196 | "Failed to close file map object handle" 197 | ); 198 | } 199 | } 200 | 201 | /** 202 | * Synchronously writes any pending changes to the file on the file system. 203 | */ 204 | void flush() 205 | { 206 | version (Posix) 207 | { 208 | sysEnforce( 209 | msync(data.ptr, data.length * T.sizeof, MS_SYNC) == 0, 210 | "Failed to flush memory map" 211 | ); 212 | } 213 | else version (Windows) 214 | { 215 | // TODO: Make this synchronous 216 | sysEnforce( 217 | FlushViewOfFile(data.ptr, data.length * T.sizeof) != 0, 218 | "Failed to flush memory map" 219 | ); 220 | } 221 | } 222 | 223 | /** 224 | * Asynchronously writes any pending changes to the file on the file system. 225 | */ 226 | void flushAsync() 227 | { 228 | version (Posix) 229 | { 230 | sysEnforce( 231 | msync(data.ptr, data.length * T.sizeof, MS_ASYNC) == 0, 232 | "Failed to flush memory map" 233 | ); 234 | } 235 | else version (Windows) 236 | { 237 | sysEnforce( 238 | FlushViewOfFile(data.ptr, data.length * T.sizeof) != 0, 239 | "Failed to flush memory map" 240 | ); 241 | } 242 | } 243 | 244 | // Disable appends. It is possible to use mremap() on Linux to extend (or 245 | // contract) the length of the map. However, this is not a portable feature. 246 | @disable void opOpAssign(string op = "~")(const(T)[] rhs); 247 | } 248 | 249 | import std.typecons; 250 | alias MemoryMap(T) = RefCounted!(MemoryMapImpl!T, RefCountedAutoInitialize.no); 251 | 252 | /** 253 | * Convenience function for creating a memory map. 254 | */ 255 | auto memoryMap(T)(File file, Access access = Access.read, 256 | size_t length = 0, long start = 0, bool share = true, 257 | void* address = null) 258 | { 259 | return MemoryMap!T(file, access, length, start, share, address); 260 | } 261 | 262 | /// 263 | unittest 264 | { 265 | auto tf = testFile(); 266 | 267 | immutable newData = "The quick brown fox jumps over the lazy dog."; 268 | 269 | // Modify the file 270 | { 271 | auto f = File(tf.name, FileFlags.readWriteEmpty); 272 | f.length = newData.length; 273 | 274 | auto map = f.memoryMap!char(Access.readWrite); 275 | assert(map.length == newData.length); 276 | 277 | map[] = newData[]; 278 | 279 | assert(map[0 .. newData.length] == newData[]); 280 | } 281 | 282 | // Read the file back in 283 | { 284 | auto f = File(tf.name, FileFlags.readExisting); 285 | auto map = f.memoryMap!char(Access.read); 286 | assert(map.length == newData.length); 287 | assert(map[0 .. newData.length] == newData[]); 288 | } 289 | } 290 | 291 | unittest 292 | { 293 | import std.range : ElementType; 294 | static assert(is(ElementType!(MemoryMap!size_t) == size_t)); 295 | } 296 | 297 | unittest 298 | { 299 | import std.exception; 300 | 301 | auto tf = testFile(); 302 | 303 | auto f = File(tf.name, FileFlags.readWriteEmpty); 304 | assert(f.length == 0); 305 | assert(collectException!SysException(f.memoryMap!char(Access.readWrite))); 306 | } 307 | 308 | unittest 309 | { 310 | import io.file.temp; 311 | 312 | immutable int[] data = [4, 8, 15, 16, 23, 42]; 313 | 314 | auto f = tempFile.file; 315 | f.length = data.length * int.sizeof; 316 | 317 | auto map = f.memoryMap!int(Access.readWrite); 318 | map[] = data; 319 | assert(map[] == data); 320 | assert(map ~ [100, 200] == data ~ [100, 200]); 321 | } 322 | 323 | unittest 324 | { 325 | import io.file.temp; 326 | import std.parallelism, std.random; 327 | 328 | immutable N = 1024; 329 | 330 | auto f = tempFile.file; 331 | f.length = size_t.sizeof * N; 332 | 333 | auto map = f.memoryMap!size_t(Access.readWrite); 334 | assert(map.length == N); 335 | 336 | foreach (i, ref e; parallel(map[])) 337 | e = uniform!"[]"(size_t.min, size_t.max); 338 | } 339 | -------------------------------------------------------------------------------- /source/io/range.d: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright: Copyright Jason White, 2014-2016 3 | * License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). 4 | * Authors: Jason White 5 | * 6 | * Description: 7 | * This module provides _range interfaces for streams. This is useful for using 8 | * many of the _range operations in $(D std._range) and $(D std.algorithm). 9 | * 10 | * There is an important distinction between streams and ranges to be made. 11 | * Fundamentally, a stream is a unidirectional $(I stream) of bytes. That is, 12 | * there is no going backwards and there is no saving the current position (as 13 | * bidirectional and forward ranges can do). This provides a good mapping to 14 | * input ranges and output ranges. As streams only operate on raw bytes, ranges 15 | * provide an abstraction to operate on more complex data types. 16 | */ 17 | module io.range; 18 | 19 | import io.stream; 20 | 21 | 22 | /** 23 | * Range that reads up to a fixed size chunk of data from a stream at a time. 24 | */ 25 | struct ByChunk(Stream) 26 | if (isSource!Stream) 27 | { 28 | private 29 | { 30 | Stream _source; 31 | 32 | // Buffer to read in the data into. 33 | ubyte[] _buffer; 34 | 35 | // Length of valid data in the buffer 36 | size_t _valid; 37 | } 38 | 39 | /** 40 | * Initializes the range. A byte buffer with the given size is allocated to 41 | * hold the chunks. 42 | * 43 | * Params: 44 | * source = A stream that can be read from. 45 | * size = The size of each chunk to read at a time. 46 | */ 47 | this(Stream source, size_t size = 4096) 48 | { 49 | this(source, new ubyte[](size)); 50 | } 51 | 52 | /** 53 | * Initializes the range with the specified buffer. This is useful for 54 | * providing your own buffer that may be stack allocated. 55 | * 56 | * Params: 57 | * source = A stream that can be read from. 58 | * buffer = A byte array to hold each chunk as it is read. 59 | */ 60 | this(Stream source, ubyte[] buffer) 61 | { 62 | _source = source; 63 | _buffer = buffer; 64 | popFront(); 65 | } 66 | 67 | /** 68 | * Reads the next chunk from the stream. 69 | */ 70 | void popFront() 71 | { 72 | _valid = _source.read(_buffer); 73 | } 74 | 75 | /** 76 | * Returns: The current chunk of the stream. 77 | * 78 | * Note that a full chunk is not guaranteed to be returned. In the event of 79 | * a partial read from the stream, this will be less than the maximum chunk 80 | * size. Code should be impartial to the size of the returned chunk. 81 | */ 82 | const(ubyte)[] front() const pure 83 | { 84 | return _buffer[0 .. _valid]; 85 | } 86 | 87 | /** 88 | * Returns true if there are no more chunks to be read from the stream. 89 | */ 90 | bool empty() const pure nothrow 91 | { 92 | return _valid == 0; 93 | } 94 | } 95 | 96 | /** 97 | * Convenience function for creating a $(D ByChunk) range over a stream. 98 | * 99 | * Example: 100 | * --- 101 | * import std.digest.digest : digest; 102 | * import std.digest.sha : SHA1; 103 | * import io.file; 104 | * 105 | * // Hash a file, 4KiB chunks at a time 106 | * ubyte[4096] buf; 107 | * auto sha1 = digest!SHA1(File("foo").byChunk(buf)); 108 | * --- 109 | */ 110 | auto byChunk(Stream)(Stream stream, size_t size = 4096) 111 | if (isSource!Stream) 112 | { 113 | return ByChunk!Stream(stream, size); 114 | } 115 | 116 | /// Ditto 117 | auto byChunk(Stream)(Stream stream, ubyte[] buffer) 118 | if (isSource!Stream) 119 | { 120 | return ByChunk!Stream(stream, buffer); 121 | } 122 | 123 | unittest 124 | { 125 | import io.file.temp; 126 | import std.algorithm : equal; 127 | import std.array : join; 128 | 129 | immutable chunkSize = 4; 130 | immutable chunks = ["1234", "5678", "abcd", "efgh", "ij"]; 131 | immutable data = chunks.join(); 132 | 133 | auto f = tempFile.file; 134 | f.write(data); 135 | f.position = 0; 136 | 137 | assert(f.byChunk(4).equal(chunks)); 138 | } 139 | 140 | /** 141 | * Wraps a stream in a range interface such that blocks of a fixed size are read 142 | * from the source. It is assumed that the stream is buffered so that 143 | * performance is not adversely affected. 144 | * 145 | * Since streams and ranges are fundamentally different, this is useful for 146 | * performing range operations on streams. 147 | * 148 | * Note: This is an input range and cannot be saved with $(D save()). Thus, 149 | * usage of this should not be mixed with the underlying stream without first 150 | * seeking to a specific location in the stream. 151 | */ 152 | struct ByBlock(T, Stream) 153 | if (isSource!Stream) 154 | { 155 | private 156 | { 157 | Stream _source; 158 | 159 | // The current block in the stream. 160 | T _current; 161 | 162 | // Are we there yet? 163 | bool _empty = false; 164 | } 165 | 166 | /** 167 | * Initializes the range with a source stream. 168 | */ 169 | this(Stream source) 170 | { 171 | _source = source; 172 | 173 | // Prime the cannons. 174 | popFront(); 175 | } 176 | 177 | /** 178 | * Removes one block from the stream. 179 | */ 180 | void popFront() 181 | { 182 | try 183 | { 184 | _source.readExactly((cast(ubyte*)&_current)[0 .. T.sizeof]); 185 | } 186 | catch (ReadException e) 187 | { 188 | _empty = true; 189 | } 190 | } 191 | 192 | /** 193 | * Returns: The current block in the stream. 194 | */ 195 | @property ref const(T) front() const pure nothrow 196 | { 197 | return _current; 198 | } 199 | 200 | /** 201 | * The range is considered empty when less than $(D T.sizeof) bytes can be 202 | * read from the stream. 203 | * 204 | * Returns: True if there are no more blocks in the stream and false 205 | * otherwise. 206 | */ 207 | @property bool empty() const pure nothrow 208 | { 209 | return _empty; 210 | } 211 | } 212 | 213 | /** 214 | * Helper function for constructing a block range. 215 | * 216 | * Example: 217 | * --- 218 | * import std.algorithm : equal; 219 | * import std.range : take; 220 | * --- 221 | */ 222 | @property auto byBlock(T, Stream)(Stream stream) 223 | if (isSource!Stream) 224 | { 225 | return ByBlock!(T, Stream)(stream); 226 | } 227 | 228 | unittest 229 | { 230 | import io.file.temp; 231 | import std.algorithm : equal; 232 | 233 | static struct Data 234 | { 235 | int a, b, c; 236 | } 237 | 238 | immutable Data[] data = [ 239 | {1, 2, 3}, 240 | {4, 5, 6}, 241 | {7, 8, 9}, 242 | ]; 243 | 244 | // Write some data to the file 245 | auto f = tempFile.file; 246 | f.put(data); 247 | f.position = 0; 248 | 249 | // Read it back in, block-by-block 250 | assert(f.byBlock!Data.equal(data)); 251 | assert(f.byBlock!Data.empty); 252 | } 253 | 254 | import std.range : back, isBidirectionalRange; 255 | 256 | /** 257 | * Checks if the given region ends with the given separator. 258 | * 259 | * Returns: The number of elements that match. 260 | */ 261 | size_t endsWithSeparator(T, Separator) 262 | (const(T)[] region, const Separator separator) 263 | if (is(typeof(T.init == Separator.init) : bool)) 264 | { 265 | import std.range : back; 266 | return region.back == separator; 267 | } 268 | 269 | /// Ditto 270 | size_t endsWithSeparator(T, Separator) 271 | (const(T)[] region, Separator sep) 272 | if (isBidirectionalRange!Separator && 273 | is(typeof(T.init == sep.back.init) : bool)) 274 | { 275 | import std.range : back, empty, popBack; 276 | 277 | size_t common = 0; 278 | 279 | while (!region.empty && !sep.empty && region.back == sep.back) 280 | { 281 | region.popBack(); 282 | sep.popBack(); 283 | 284 | ++common; 285 | } 286 | 287 | return sep.empty ? common : 0; 288 | } 289 | 290 | /** 291 | * Checks if the given function can be used with $(D Splitter). 292 | */ 293 | enum isSplitFunction(alias fn, T, Separator) = 294 | is(typeof(fn(T[].init, Separator.init))); 295 | 296 | /// 297 | unittest 298 | { 299 | static assert(isSplitFunction!(endsWithSeparator, char, char)); 300 | static assert(isSplitFunction!(endsWithSeparator, char, string)); 301 | } 302 | 303 | /** 304 | * Splits a stream using a separator. The separator can be a single element or a 305 | * bidirectional range of elements. 306 | */ 307 | struct Splitter(T, Separator, Stream, alias splitFn = endsWithSeparator!(T, Separator)) 308 | if (isSource!Stream && isSplitFunction!(splitFn, T, Separator)) 309 | { 310 | private 311 | { 312 | import std.array : Appender; 313 | 314 | alias Region = Appender!(T[]); 315 | 316 | // The current region 317 | Region _region; 318 | 319 | // Block iterator 320 | ByBlock!(T, Stream) _blocks; 321 | 322 | bool _empty = false; 323 | 324 | // Element or range that separates regions. 325 | const Separator _separator; 326 | } 327 | 328 | this(Stream source, const Separator separator) 329 | { 330 | _blocks = source.byBlock!(T, Stream); 331 | _separator = separator; 332 | 333 | // Prime the cannons 334 | popFront(); 335 | } 336 | 337 | void popFront() 338 | { 339 | version (assert) 340 | { 341 | import core.exception : RangeError; 342 | if (empty) throw new RangeError(); 343 | } 344 | 345 | _region.clear(); 346 | 347 | if (_blocks.empty) 348 | { 349 | _empty = true; 350 | return; 351 | } 352 | 353 | while (!_blocks.empty) 354 | { 355 | _region.put(_blocks.front); 356 | 357 | if (auto len = splitFn(_region.data, _separator)) 358 | { 359 | assert(_region.data.length >= len); 360 | _region.shrinkTo(_region.data.length - len); 361 | break; 362 | } 363 | 364 | _blocks.popFront(); 365 | } 366 | 367 | _blocks.popFront(); 368 | } 369 | 370 | /** 371 | * Gets the current region in the stream. 372 | */ 373 | const(T)[] front() const pure nothrow 374 | { 375 | version (assert) 376 | { 377 | import core.exception : RangeError; 378 | if (empty) throw new RangeError(); 379 | } 380 | 381 | return _region.data; 382 | } 383 | 384 | /** 385 | * Returns true if there are no more regions in the splitter. 386 | */ 387 | bool empty() const pure nothrow 388 | { 389 | return _empty; 390 | } 391 | } 392 | 393 | unittest 394 | { 395 | import std.range.primitives; 396 | import io.file : File; 397 | 398 | alias S = File; 399 | 400 | static assert(isInputRange!(Splitter!(char, char, S))); 401 | static assert(isInputRange!(Splitter!(char, string, S))); 402 | static assert(isInputRange!(Splitter!(int, int, S))); 403 | static assert(isInputRange!(Splitter!(int, immutable(int)[], S))); 404 | 405 | static assert(!isOutputRange!(Splitter!(char, char, S), char)); 406 | static assert(!isForwardRange!(Splitter!(char, char, S))); 407 | static assert(!isBidirectionalRange!(Splitter!(char, char, S))); 408 | static assert(!isRandomAccessRange!(Splitter!(char, char, S))); 409 | } 410 | 411 | /** 412 | * Convenience function for returning a stream splitter. 413 | * 414 | * Params: 415 | * T = Type of each element in the stream. 416 | * stream = A sink stream that can be read from. 417 | * separator = An element or range of elements to split on. 418 | * 419 | * Example: 420 | * --- 421 | * // Get a list of words from standard input. 422 | * import io; 423 | * import std.algorithm : map, filter; 424 | * import std.array : array; 425 | * auto words = stdin.splitter!char(' ') 426 | * .filter!(w => w != "") 427 | * .map!(w => w.idup) 428 | * .array; 429 | * --- 430 | */ 431 | auto splitter(T = char, Separator, Stream)(Stream stream, Separator separator) 432 | if (isSource!Stream) 433 | { 434 | return Splitter!(T, Separator, Stream)(stream, separator); 435 | } 436 | 437 | unittest 438 | { 439 | // Test an empty split 440 | import io.file.temp; 441 | import std.algorithm : equal; 442 | assert(tempFile.file.splitter!char('\n').equal(string[].init)); 443 | } 444 | 445 | version (unittest) 446 | { 447 | void testSplitter(T, Separator)(const T[][] regions, Separator separator) 448 | { 449 | import io.file.temp; 450 | import std.array : join; 451 | import std.algorithm : equal; 452 | import std.traits : isArray; 453 | 454 | static if (isArray!Separator) 455 | auto joined = regions.join(separator); 456 | else 457 | auto joined = regions.join([separator]); 458 | 459 | auto f = tempFile.file; 460 | f.writeExactly(joined); 461 | f.position = 0; 462 | 463 | assert(f.splitter!T(separator).equal(regions)); 464 | assert(f.position == joined.length * T.sizeof); 465 | 466 | // Add a trailing separator at the end of the file 467 | static if (isArray!Separator) 468 | f.writeExactly(separator); 469 | else 470 | f.writeExactly([separator]); 471 | f.position = 0; 472 | assert(f.splitter!T(separator).equal(regions)); 473 | } 474 | } 475 | 476 | unittest 477 | { 478 | // Test different types of separators. 479 | immutable lines = [ 480 | "This is the first line", 481 | "", 482 | "That was a blank line.", 483 | "This is the penultimate line!", 484 | "This is the last line.", 485 | ]; 486 | 487 | testSplitter(lines, '\n'); 488 | testSplitter(lines, "\n"); 489 | testSplitter(lines, "\r\n"); 490 | testSplitter(lines, "\n\n"); 491 | testSplitter(lines, "||||"); 492 | } 493 | 494 | unittest 495 | { 496 | // Test combining with filter 497 | import io.file.temp; 498 | import std.algorithm : filter, equal; 499 | 500 | immutable data = "The quick brown fox jumps over the lazy dog "; 501 | immutable result = [ 502 | "The", "quick", "brown", "fox", "jumps", "over", "the", "lazy", "dog" 503 | ]; 504 | 505 | auto f = tempFile.file; 506 | f.writeExactly(data); 507 | f.position = 0; 508 | 509 | assert(f.splitter!char(' ').filter!(w => w != "").equal(result)); 510 | } 511 | -------------------------------------------------------------------------------- /source/io/file/flags.d: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright: Copyright Jason White, 2014-2016 3 | * License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). 4 | * Authors: Jason White 5 | */ 6 | module io.file.flags; 7 | 8 | public import io.stream : Access; 9 | 10 | /** 11 | * Specifies in what mode a file should be opened. These flags can be combined. 12 | */ 13 | enum Mode 14 | { 15 | /// Default mode. Not very useful. 16 | none = 0, 17 | 18 | /** 19 | * Opens an existing file. Unless combined with create, fails if the file 20 | * does not exist. 21 | */ 22 | open = 1 << 0, 23 | 24 | /** 25 | * Creates a new file. Fails if the file is opened without write access. 26 | * Fails if the file already exists and not combined with truncate or open. 27 | */ 28 | create = 1 << 1, 29 | 30 | /** 31 | * Opens the file if it already exists or creates it if it does not. 32 | */ 33 | openOrCreate = open | create, 34 | 35 | /** 36 | * Allows only appending to the end of the file. Seek operations only affect 37 | * subsequent reads. Upon writing, the file pointer gets set to the end of 38 | * the file. Requires write access to the file. 39 | */ 40 | append = 1 << 2, 41 | 42 | /** 43 | * Truncates the file. This has no effect if the file has been created anew. 44 | * Requires write access to the file. 45 | */ 46 | truncate = 1 << 3, 47 | } 48 | 49 | /** 50 | * Specifies what other processes are allowed to do to the file. These flags can 51 | * be combined. 52 | * 53 | * Windows_specific: 54 | * Currently only used by Windows. 55 | */ 56 | enum Share 57 | { 58 | /// Forbids sharing of the file. 59 | none = 0, 60 | 61 | /// Allows others to read from the file. 62 | read = 1 << 0, 63 | 64 | /// Allows others to write to the file. 65 | write = 1 << 1, 66 | 67 | /// Allows others to either read or write to the file. 68 | readWrite = read | write, 69 | 70 | /// Allows the file to deleted. 71 | remove = 1 << 2, 72 | } 73 | 74 | /** 75 | * File flags that determine how a file stream is created and used. 76 | * 77 | * Since all methods in this struct are pure, the high-level configuration flags 78 | * given here are converted to the platform-specific file flags at compile time. 79 | * 80 | * See_Also: 81 | * $(D io.file.stream) 82 | * 83 | * Example: 84 | * --- 85 | * // Creates the file "foobar", truncates it, and opens it in write-only mode 86 | * auto f = File("foobar", FileFlags.writeEmpty); 87 | * --- 88 | */ 89 | struct FileFlags 90 | { 91 | static immutable 92 | { 93 | /** 94 | * An existing file is opened with read access. This is likely the most 95 | * commonly used set of flags. 96 | */ 97 | FileFlags readExisting = FileFlags(Mode.open, Access.read); 98 | 99 | /** 100 | * An existing file is opened with write access. 101 | */ 102 | FileFlags writeExisting = FileFlags(Mode.open, Access.write); 103 | 104 | /** 105 | * A new file is created with write access. Fails if the file already 106 | * exists. 107 | */ 108 | FileFlags writeNew = FileFlags(Mode.create, Access.write); 109 | 110 | /** 111 | * A new file is either opened or created with write access. 112 | */ 113 | FileFlags writeAlways = FileFlags(Mode.openOrCreate, Access.write); 114 | 115 | /** 116 | * A new file is either opened or created, truncated if necessary, with 117 | * write access. This ensures that an $(I empty) file is opened. 118 | */ 119 | FileFlags writeEmpty = FileFlags(Mode.openOrCreate | Mode.truncate, Access.write); 120 | 121 | /** 122 | * An existing file is opened with read/write access. 123 | */ 124 | FileFlags readWriteExisting = FileFlags(Mode.open, Access.readWrite); 125 | 126 | /** 127 | * A new file is created with read/write access. Fails if the file 128 | * already exists. 129 | */ 130 | FileFlags readWriteNew = FileFlags(Mode.create, Access.readWrite); 131 | 132 | /** 133 | * A new file is either opened or created with read/write access. 134 | */ 135 | FileFlags readWriteAlways = FileFlags(Mode.openOrCreate, Access.readWrite); 136 | 137 | /** 138 | * A new file is either opened or created, truncated if necessary, with 139 | * read/write access. This ensures that an $(I empty) file is opened. 140 | */ 141 | FileFlags readWriteEmpty = FileFlags(Mode.openOrCreate | Mode.truncate, Access.readWrite); 142 | } 143 | 144 | version (Posix) 145 | { 146 | import core.sys.posix.fcntl; 147 | 148 | int flags; 149 | 150 | alias flags this; 151 | 152 | /** 153 | * Constructs the $(D FileFlag) with the given mode, access, and share 154 | * attributes. These high-level flags are converted to the 155 | * equivalent platform-specific flags that are needed when opening the 156 | * file. 157 | * 158 | * Params: 159 | * mode = Mode the file should be opened in. That is, to open, create, 160 | * append, or truncate the file. 161 | * access = The permissions on the file stream. That is, read access, 162 | * write access, or both. 163 | * share = The sharing permissions other processes are allowed on the 164 | * file stream when trying to open the same file. By default, 165 | * other processes are prevented from opening the file if they 166 | * request read, write, or delete access. If you wish to allow 167 | * other processes to read the file while it is open in this 168 | * process, set this to $(D Share.read). Currently only used 169 | * by Windows. 170 | * 171 | * Windows_specific: 172 | * The share parameter is currently only used by Windows. 173 | * 174 | * Example: 175 | * --- 176 | * immutable flags = FileFlags(Mode.open, Access.read); 177 | * --- 178 | */ 179 | this(Mode mode, 180 | Access access, 181 | Share share = Share.init) pure nothrow 182 | { 183 | // Disable buffering. Buffering is handled by $(D io.buffered). 184 | //flags |= O_DIRECT; // FIXME: O_DIRECT is not defined 185 | 186 | if ((mode & Mode.openOrCreate) == Mode.openOrCreate) 187 | flags |= O_CREAT; 188 | else if (mode & Mode.create) 189 | flags |= O_EXCL | O_CREAT; 190 | // Mode.open by default 191 | 192 | if (mode & Mode.truncate) 193 | flags |= O_TRUNC; 194 | 195 | if (mode & Mode.append) 196 | flags |= O_APPEND; 197 | 198 | if (access == Access.readWrite) 199 | flags |= O_RDWR; 200 | else if (access & Access.read) 201 | flags |= O_RDONLY; 202 | else if (access & Access.write) 203 | flags |= O_WRONLY; 204 | 205 | // Posix does not support Share flags. 206 | } 207 | 208 | unittest 209 | { 210 | with (FileFlags) 211 | { 212 | static assert(readExisting.flags == O_RDONLY); 213 | static assert(writeExisting.flags == O_WRONLY); 214 | static assert(writeNew.flags == (O_EXCL | O_CREAT | O_WRONLY)); 215 | static assert(writeAlways.flags == (O_CREAT | O_WRONLY)); 216 | static assert(writeEmpty.flags == (O_CREAT | O_TRUNC | O_WRONLY)); 217 | static assert(readWriteExisting.flags == O_RDWR); 218 | static assert(readWriteNew.flags == (O_CREAT | O_EXCL | O_RDWR)); 219 | static assert(readWriteAlways.flags == (O_CREAT | O_RDWR)); 220 | static assert(readWriteEmpty.flags == (O_CREAT | O_RDWR | O_TRUNC)); 221 | } 222 | } 223 | } 224 | else version (Windows) 225 | { 226 | import core.sys.windows.windows; 227 | 228 | DWORD access, share, mode; 229 | 230 | this(Mode mode, 231 | Access access, 232 | Share share = Share.init) pure nothrow 233 | { 234 | // Access flags 235 | if (access & Access.read) this.access |= GENERIC_READ; 236 | if (access & Access.write) this.access |= GENERIC_WRITE; 237 | if (mode & Mode.append) this.access |= FILE_APPEND_DATA; 238 | 239 | // Share flags 240 | if (share & Share.read) this.share |= FILE_SHARE_READ; 241 | if (share & Share.write) this.share |= FILE_SHARE_WRITE; 242 | if (share & Share.remove) this.share |= FILE_SHARE_DELETE; 243 | 244 | // Creation flags 245 | if (mode & Mode.truncate) 246 | { 247 | if (mode & Mode.create) 248 | this.mode = CREATE_ALWAYS; 249 | else 250 | this.mode = TRUNCATE_EXISTING; 251 | } 252 | else if ((mode & Mode.openOrCreate) == Mode.openOrCreate) 253 | this.mode = OPEN_ALWAYS; 254 | else if (mode & Mode.open) 255 | this.mode = OPEN_EXISTING; 256 | else if (mode & Mode.create) 257 | this.mode = CREATE_NEW; 258 | } 259 | } 260 | 261 | /** 262 | * Constructs the file flags from a mode string. 263 | * 264 | * This simply calls $(LREF FileFlags.parse). 265 | * 266 | * See_Also: 267 | * $(LREF FileFlags.parse) 268 | */ 269 | this(string mode) pure 270 | { 271 | this = parse(mode); 272 | } 273 | 274 | /// Ditto 275 | void opAssign(string mode) pure 276 | { 277 | this = parse(mode); 278 | } 279 | 280 | /// 281 | unittest 282 | { 283 | FileFlags ff = "wb+"; 284 | assert(ff == FileFlags.readWriteEmpty); 285 | assert(ff == FileFlags("wb+")); 286 | } 287 | 288 | /** 289 | * Parses an $(D fopen)-style mode string such as "r+". All possible mode 290 | * strings include: 291 | * 292 | * $(TABLE 293 | * $(TR $(TH Mode String) $(TH Meaning)) 294 | * $(TR $(TD $(D "wb")) $(TD Write truncated)) 295 | * $(TR $(TD $(D "wb+")) $(TD Read/write truncated)) 296 | * $(TR $(TD $(D "w+b")) $(TD Read/write truncated)) 297 | * $(TR $(TD $(D "wbx")) $(TD Write new)) 298 | * $(TR $(TD $(D "wb+x")) $(TD Read/write new)) 299 | * $(TR $(TD $(D "w+bx")) $(TD Read/write new)) 300 | * $(TR $(TD $(D "rb")) $(TD Read existing")) 301 | * $(TR $(TD $(D "rb+")) $(TD Read/write existing")) 302 | * $(TR $(TD $(D "r+b")) $(TD Read/write existing")) 303 | * $(TR $(TD $(D "ab")) $(TD Append new")) 304 | * $(TR $(TD $(D "ab+")) $(TD Append/read new")) 305 | * $(TR $(TD $(D "a+b")) $(TD Append/read new")) 306 | * ) 307 | * 308 | * The _mode strings accepted here differ from those accepted by $(D fopen). 309 | * Here, file streams are never opened in text _mode -- only binary mode. 310 | * Text handling functionality is built on top of low-level file streams. 311 | * It does not make sense to distinguish between text and binary modes here. 312 | * $(D fopen) opens all files in text _mode by default and the flag 'b' must 313 | * be specified in order to open in binary _mode. Thus, an exception is 314 | * thrown here if 'b' is omitted in the specified mode string. 315 | * 316 | * Note: It is not advisable to use fopen-style _mode strings. It is better 317 | * to use one of the predefined file flag configurations such as 318 | * $(LREF FileFlags.readExisting) for greater readability and intent of 319 | * meaning. 320 | */ 321 | static FileFlags parse(string mode) pure 322 | { 323 | // There are only 12 possible permutations of mode strings that we care 324 | // about. (There would be twice as many if the 'b' flag was optional.) 325 | // Thus, it is easier to just check for all 12 possible flags rather 326 | // than actually parsing the string. 327 | switch (mode) 328 | { 329 | case "wb": return FileFlags.writeEmpty; 330 | case "wb+": return FileFlags.readWriteEmpty; 331 | case "w+b": return FileFlags.readWriteEmpty; 332 | case "wbx": return FileFlags.writeNew; 333 | case "wb+x": return FileFlags.readWriteNew; 334 | case "w+bx": return FileFlags.readWriteNew; 335 | case "rb": return FileFlags.readExisting; 336 | case "rb+": return FileFlags.readWriteExisting; 337 | case "r+b": return FileFlags.readWriteExisting; 338 | case "ab": return FileFlags(Mode.openOrCreate | Mode.append, Access.write); 339 | case "ab+": return FileFlags(Mode.openOrCreate | Mode.append, Access.readWrite); 340 | case "a+b": return FileFlags(Mode.openOrCreate | Mode.append, Access.readWrite); 341 | default: 342 | throw new Exception("Invalid mode string"); 343 | } 344 | } 345 | 346 | /// 347 | unittest 348 | { 349 | static assert(FileFlags("wb") == FileFlags.writeEmpty); 350 | static assert(FileFlags("wb+") == FileFlags.readWriteEmpty); 351 | static assert(FileFlags("w+b") == FileFlags.readWriteEmpty); 352 | static assert(FileFlags("wbx") == FileFlags.writeNew); 353 | static assert(FileFlags("wb+x") == FileFlags.readWriteNew); 354 | static assert(FileFlags("w+bx") == FileFlags.readWriteNew); 355 | static assert(FileFlags("rb") == FileFlags.readExisting); 356 | static assert(FileFlags("rb+") == FileFlags.readWriteExisting); 357 | static assert(FileFlags("r+b") == FileFlags.readWriteExisting); 358 | static assert(FileFlags("ab") == FileFlags(Mode.openOrCreate | Mode.append, Access.write)); 359 | static assert(FileFlags("ab+") == FileFlags(Mode.openOrCreate | Mode.append, Access.readWrite)); 360 | static assert(FileFlags("a+b") == FileFlags(Mode.openOrCreate | Mode.append, Access.readWrite)); 361 | } 362 | 363 | unittest 364 | { 365 | import std.exception : collectException; 366 | 367 | immutable badModes = ["", "rw", "asdf", "+r", "b+", " r", "r+b "]; 368 | foreach (m; badModes) 369 | assert(collectException(FileFlags(m))); 370 | } 371 | } 372 | -------------------------------------------------------------------------------- /source/io/file/stream.d: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright: Copyright Jason White, 2014-2016 3 | * License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). 4 | * Authors: Jason White 5 | * 6 | * Description: 7 | * This module provides a low-level file stream class. 8 | * 9 | * Synopsis: 10 | * --- 11 | * // Open a new file in read/write mode. Throws an exception if the file exists. 12 | * auto f = File("myfile", FileFlags.readWriteNew); 13 | * 14 | * // Write an arbitrary array to the stream. 15 | * f.write("Hello world!"); 16 | * 17 | * // Seek to the beginning. 18 | * f.position = 0; 19 | * 20 | * // Read in 5 bytes. 21 | * char buf[5]; 22 | * f.read(buf); 23 | * assert(buf == "Hello"); 24 | * --- 25 | */ 26 | module io.file.stream; 27 | 28 | import io.stream; 29 | public import io.file.flags; 30 | 31 | version (unittest) 32 | { 33 | import file = std.file; // For easy file creation/deletion. 34 | import io.file.temp; 35 | } 36 | 37 | version (Posix) 38 | { 39 | import core.sys.posix.fcntl; 40 | import core.sys.posix.unistd; 41 | import core.sys.posix.sys.uio; 42 | import std.exception : ErrnoException; 43 | 44 | version (linux) 45 | { 46 | extern (C): @system: nothrow: 47 | ssize_t sendfile(int out_fd, int in_fd, off_t *offset, size_t count); 48 | } 49 | 50 | enum 51 | { 52 | SEEK_SET, 53 | SEEK_CUR, 54 | SEEK_END 55 | } 56 | 57 | // FIXME: This should be moved into a separate module. 58 | alias SysException = ErrnoException; 59 | } 60 | else version (Windows) 61 | { 62 | import core.sys.windows.windows; 63 | 64 | // These are not declared in core.sys.windows.windows 65 | extern (Windows) nothrow export 66 | { 67 | BOOL SetFilePointerEx( 68 | HANDLE hFile, 69 | long liDistanceToMove, 70 | long* lpNewFilePointer, 71 | DWORD dwMoveMethod 72 | ); 73 | 74 | BOOL FlushFileBuffers(HANDLE hFile); 75 | BOOL GetFileSizeEx(HANDLE hFile, long* lpFileSize); 76 | DWORD GetFileType(HANDLE hFile); 77 | 78 | enum 79 | { 80 | FILE_TYPE_UNKNOWN = 0x0000, 81 | FILE_TYPE_DISK = 0x0001, 82 | FILE_TYPE_CHAR = 0x0002, 83 | FILE_TYPE_PIPE = 0x0003, 84 | FILE_TYPE_REMOTE = 0x8000, 85 | } 86 | 87 | enum 88 | { 89 | DUPLICATE_CLOSE_SOURCE = 0x00000001, 90 | DUPLICATE_SAME_ACCESS = 0x00000002, 91 | } 92 | } 93 | 94 | // FIXME: This should be moved into a separate module. 95 | class SysException : Exception 96 | { 97 | uint errCode; 98 | 99 | this(string msg, string file = null, size_t line = 0) 100 | { 101 | import std.windows.syserror : sysErrorString; 102 | errCode = GetLastError(); 103 | super(msg ~ " (" ~ sysErrorString(errCode) ~ ")", file, line); 104 | } 105 | } 106 | } 107 | else 108 | { 109 | static assert(false, "Unsupported platform."); 110 | } 111 | 112 | // FIXME: This should be moved into a separate module. 113 | T sysEnforce(T, string file = __FILE__, size_t line = __LINE__) 114 | (T value, lazy string msg = null) 115 | { 116 | if (!value) throw new SysException(msg, file, line); 117 | return value; 118 | } 119 | 120 | /** 121 | * A cross-platform wrapper around low-level file operations. 122 | */ 123 | struct FileBase 124 | { 125 | /** 126 | * Platform-specific file handle. On Posix systems, this is the file 127 | * descriptor `int`. On Windows, this is a `HANDLE`. 128 | */ 129 | version (Posix) 130 | { 131 | alias Handle = int; 132 | enum Handle InvalidHandle = -1; 133 | } 134 | else version (Windows) 135 | { 136 | alias Handle = HANDLE; 137 | enum Handle InvalidHandle = INVALID_HANDLE_VALUE; 138 | } 139 | 140 | private Handle _h = InvalidHandle; 141 | 142 | // Name of the file, for debugging purposes only. 143 | debug const(char)[] name; 144 | 145 | /** 146 | * Opens or creates a file by name. By default, an existing file is opened 147 | * in read-only mode. 148 | * 149 | * Params: 150 | * name = Path to the file to open. 151 | * flags = How to open the file. 152 | * 153 | * Example: 154 | * --- 155 | * // Create a brand-new file and write to it. Throws an exception if the 156 | * // file already exists. The file is automatically closed when it falls 157 | * // out of scope. 158 | * auto f = File("filename", FileFlags.writeNew); 159 | * f.write("Hello world!"); 160 | * --- 161 | */ 162 | this(const(char)[] name, FileFlags flags = FileFlags.readExisting) 163 | { 164 | debug this.name = name; 165 | 166 | version (Posix) 167 | { 168 | import std.string : toStringz; 169 | 170 | _h = .open(toStringz(name), flags, 0b110_000_000); 171 | } 172 | else version (Windows) 173 | { 174 | import std.utf : toUTF16z; 175 | 176 | _h = .CreateFileW( 177 | name.toUTF16z(), // File name 178 | flags.access, // Desired access 179 | flags.share, // Share mode 180 | null, // Security attributes 181 | flags.mode, // Creation disposition 182 | FILE_ATTRIBUTE_NORMAL, // Flags and attributes 183 | null, // Template file handle 184 | ); 185 | } 186 | 187 | sysEnforce(_h != InvalidHandle, "Failed to open file '"~ name.idup ~"'"); 188 | } 189 | 190 | unittest 191 | { 192 | import std.exception : ce = collectException; 193 | 194 | // Ensure files are opened the way they are supposed to be opened. 195 | 196 | immutable data = "12345678"; 197 | ubyte[data.length] buf; 198 | 199 | auto tf = testFile(); 200 | 201 | // Make sure the file does *not* exist 202 | try .file.remove(tf.name); catch (Exception e) {} 203 | 204 | assert( File(tf.name, FileFlags.readExisting).ce); 205 | assert( File(tf.name, FileFlags.writeExisting).ce); 206 | assert(!File(tf.name, FileFlags.writeNew).ce); 207 | assert(!File(tf.name, FileFlags.writeAlways).ce); 208 | 209 | // Make sure the file *does* exist. 210 | .file.write(tf.name, data); 211 | 212 | assert(!File(tf.name, FileFlags.readExisting).ce); 213 | assert(!File(tf.name, FileFlags.writeExisting).ce); 214 | assert( File(tf.name, FileFlags.writeNew).ce); 215 | assert(!File(tf.name, FileFlags.writeAlways).ce); 216 | } 217 | 218 | /** 219 | * Takes control of a file handle. 220 | * 221 | * It is assumed that we have exclusive control over the file handle and will 222 | * be closed upon destruction as usual. If non-exclusive control is desired, 223 | * use $(D dup) instead. 224 | * 225 | * This function is useful in a couple of situations: 226 | * $(UL 227 | * $(LI 228 | * The file must be opened with special flags that cannot be obtained 229 | * via $(D FileFlags) 230 | * ) 231 | * $(LI 232 | * A special file handle must be opened (e.g., $(D stdout), a pipe). 233 | * ) 234 | * ) 235 | * 236 | * Params: 237 | * h = The handle to assume control over. For Posix, this is a file 238 | * descriptor ($(D int)). For Windows, this is an object handle ($(D 239 | * HANDLE)). 240 | */ 241 | this(Handle h) 242 | { 243 | _h = h; 244 | } 245 | 246 | /** 247 | * Duplicates the given platform-specific file handle. This is useful for 248 | * taking non-exclusive control over a file handle. 249 | */ 250 | static F dup(F = File)(Handle h) 251 | { 252 | version (Posix) 253 | { 254 | immutable new_h = .dup(h); 255 | sysEnforce(new_h != InvalidHandle, "Failed to duplicate handle"); 256 | return F(new_h); 257 | } 258 | else version (Windows) 259 | { 260 | Handle new_h = void; 261 | auto proc = GetCurrentProcess(); 262 | auto ret = DuplicateHandle( 263 | proc, // Process with the file handle 264 | h, // Handle to duplicate 265 | proc, // Process for the duplicated handle 266 | &new_h, // The duplicated handle 267 | 0, // Access flags, ignored 268 | true, // Allow this handle to be inherited 269 | DUPLICATE_SAME_ACCESS 270 | ); 271 | sysEnforce(ret, "Failed to duplicate handle"); 272 | return F(new_h); 273 | } 274 | } 275 | 276 | /** 277 | * Copying is disabled because references counting should be used instead. 278 | */ 279 | @disable this(this); 280 | 281 | /** 282 | * Closes the stream if it is open. Otherwise, it does nothing. 283 | */ 284 | void close() 285 | { 286 | // The file handle should only be invalid if the constructor throws an 287 | // exception. 288 | if (!isOpen) return; 289 | 290 | version (Posix) 291 | { 292 | sysEnforce(.close(_h) != -1, "Failed to close file"); 293 | } 294 | else version (Windows) 295 | { 296 | sysEnforce(CloseHandle(_h), "Failed to close file"); 297 | } 298 | 299 | _h = InvalidHandle; 300 | } 301 | 302 | /// Ditto 303 | ~this() 304 | { 305 | close(); 306 | } 307 | 308 | /** 309 | * Returns the internal file handle. On POSIX, this is a file descriptor. On 310 | * Windows, this is an object handle. 311 | */ 312 | @property Handle handle() pure nothrow 313 | { 314 | return _h; 315 | } 316 | 317 | /** 318 | * Returns true if the file is open. 319 | */ 320 | @property bool isOpen() 321 | { 322 | return _h != InvalidHandle; 323 | } 324 | 325 | /** 326 | * Reads data from the file. 327 | * 328 | * Params: 329 | * buf = The buffer to read the data into. The length of the buffer 330 | * specifies how much data should be read. 331 | * 332 | * Returns: The number of bytes that were read. 0 indicates that the end of 333 | * the file has been reached. 334 | */ 335 | size_t read(scope ubyte[] buf) 336 | in { assert(isOpen); } 337 | body 338 | { 339 | version (Posix) 340 | { 341 | immutable n = .read(_h, buf.ptr, buf.length); 342 | sysEnforce(n >= 0, "Failed to read from file"); 343 | return n; 344 | } 345 | else version (Windows) 346 | { 347 | DWORD n = void; 348 | sysEnforce( 349 | ReadFile(_h, buf.ptr, cast(uint)buf.length, &n, null), 350 | "Failed to read from file" 351 | ); 352 | return n; 353 | } 354 | } 355 | 356 | unittest 357 | { 358 | auto tf = testFile(); 359 | 360 | immutable data = "\r\n\n\r\n"; 361 | file.write(tf.name, data); 362 | 363 | char[data.length] buf; 364 | 365 | auto f = File(tf.name, FileFlags.readExisting); 366 | assert(buf[0 .. f.read(buf)] == data); 367 | } 368 | 369 | version (Posix) private size_t readv(iovec* iov, ubyte[][] bufs) 370 | { 371 | for (size_t i = 0; i < bufs.length; ++i) 372 | { 373 | iov[i].iov_base = cast(void*)bufs[i].ptr; 374 | iov[i].iov_len = bufs[i].length; 375 | } 376 | 377 | immutable n = .readv(_h, iov, cast(int)bufs.length); 378 | sysEnforce(n != -1, "Failed to read from file"); 379 | return n; 380 | } 381 | 382 | /** 383 | * Vectorized read. 384 | * 385 | * Params: 386 | * bufs = The buffers to read the data into. The length of each buffer 387 | * specifies how much data should be read. 388 | * 389 | * Returns: The number of bytes that were read. 0 indicates that the end of 390 | * the file has been reached. 391 | */ 392 | size_t readv(size_t StackSize = 32)(scope ubyte[][] bufs...) 393 | { 394 | import core.stdc.stdlib : malloc, free; 395 | 396 | version (Posix) 397 | { 398 | if (bufs.length <= StackSize) 399 | { 400 | iovec[StackSize] iov = void; 401 | return readv(iov.ptr, bufs); 402 | } 403 | else 404 | { 405 | // Not enough space in stack-allocated buffer, fallback to using 406 | // a heap-allocated buffer. 407 | iovec* iov = cast(iovec*)malloc(iovec.sizeof * bufs.length); 408 | scope (exit) free(iov); 409 | return readv(iov, bufs); 410 | } 411 | } 412 | else version (Windows) 413 | { 414 | // Dumb implementation for Windows. 415 | size_t totalLength = 0; 416 | 417 | foreach (buf; bufs) 418 | { 419 | if (immutable n = read(buf)) 420 | totalLength += n; 421 | else 422 | break; 423 | } 424 | 425 | return totalLength; 426 | } 427 | } 428 | 429 | unittest 430 | { 431 | auto tf = testFile(); 432 | 433 | immutable ubyte[] data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; 434 | 435 | ubyte[4] buf1; 436 | ubyte[3] buf2; 437 | ubyte[5] buf3; 438 | 439 | static assert(buf1.length + buf2.length + buf3.length == data.length); 440 | 441 | assert(File(tf.name, FileFlags.writeEmpty).write(data) == data.length); 442 | assert(File(tf.name, FileFlags.readExisting).readv(buf1, buf2, buf3) == data.length); 443 | assert(buf1 ~ buf2 ~ buf3 == data); 444 | 445 | assert(File(tf.name, FileFlags.writeEmpty).write(data) == data.length); 446 | assert(File(tf.name, FileFlags.readExisting).readv!2(buf1, buf2, buf3) == data.length); 447 | assert(buf1 ~ buf2 ~ buf3 == data); 448 | } 449 | 450 | /** 451 | * Writes data to the file. 452 | * 453 | * Params: 454 | * data = The data to write to the file. The length of the slice indicates 455 | * how much data should be written. 456 | * 457 | * Returns: The number of bytes that were written. 458 | */ 459 | size_t write(in ubyte[] data) 460 | in { assert(isOpen); } 461 | body 462 | { 463 | version (Posix) 464 | { 465 | immutable n = .write(_h, data.ptr, data.length); 466 | sysEnforce(n != -1, "Failed to write to file"); 467 | return n; 468 | } 469 | else version (Windows) 470 | { 471 | DWORD written = void; 472 | sysEnforce( 473 | WriteFile(_h, data.ptr, cast(uint)data.length, &written, null), 474 | "Failed to write to file" 475 | ); 476 | return written; 477 | } 478 | } 479 | 480 | alias put = write; 481 | 482 | unittest 483 | { 484 | auto tf = testFile(); 485 | 486 | immutable data = "\r\n\n\r\n"; 487 | char[data.length] buf; 488 | 489 | assert(File(tf.name, FileFlags.writeEmpty).write(data) == data.length); 490 | assert(File(tf.name, FileFlags.readExisting).read(buf)); 491 | assert(buf == data); 492 | } 493 | 494 | version (Posix) private size_t writev(iovec* iov, in ubyte[][] bufs) 495 | { 496 | for (size_t i = 0; i < bufs.length; ++i) 497 | { 498 | iov[i].iov_base = cast(void*)bufs[i].ptr; 499 | iov[i].iov_len = bufs[i].length; 500 | } 501 | 502 | immutable n = .writev(_h, iov, cast(int)bufs.length); 503 | sysEnforce(n != -1, "Failed to write to file"); 504 | return n; 505 | } 506 | 507 | /** 508 | * Vectorized write. 509 | * 510 | * Params: 511 | * bufs = The buffers to write the data from. The length of each buffer 512 | * specifies how much data should be written. 513 | * 514 | * Returns: The number of bytes that were written. 515 | */ 516 | size_t writev(size_t StackSize = 32)(in ubyte[][] bufs...) 517 | { 518 | import core.stdc.stdlib : malloc, free; 519 | 520 | version (Posix) 521 | { 522 | if (bufs.length <= StackSize) 523 | { 524 | iovec[StackSize] iov = void; 525 | return writev(iov.ptr, bufs); 526 | } 527 | else 528 | { 529 | // Not enough space in stack-allocated buffer, fallback to using 530 | // a heap-allocated buffer. 531 | iovec* iov = cast(iovec*)malloc(iovec.sizeof * bufs.length); 532 | scope (exit) free(iov); 533 | return writev(iov, bufs); 534 | } 535 | } 536 | else version (Windows) 537 | { 538 | // Dumb implementation for Windows. 539 | size_t totalLength = 0; 540 | 541 | foreach (buf; bufs) 542 | { 543 | if (immutable n = write(buf)) 544 | totalLength += n; 545 | else 546 | break; 547 | } 548 | 549 | return totalLength; 550 | } 551 | } 552 | 553 | unittest 554 | { 555 | auto tf = testFile(); 556 | 557 | immutable ubyte[] data1 = [1, 2, 3, 4, 5, 6, 7, 8]; 558 | immutable ubyte[] data2 = [9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]; 559 | immutable ubyte[] data3 = [21, 22, 23, 24]; 560 | immutable totalLength = data1.length + data2.length + data3.length; 561 | ubyte[totalLength] buf; 562 | 563 | assert(File(tf.name, FileFlags.writeEmpty).writev(data1, data2, data3) == totalLength); 564 | assert(File(tf.name, FileFlags.readExisting).read(buf) == buf.length); 565 | assert(buf == data1 ~ data2 ~ data3); 566 | 567 | assert(File(tf.name, FileFlags.writeEmpty).writev!2(data1, data2, data3) == totalLength); 568 | assert(File(tf.name, FileFlags.readExisting).read(buf) == buf.length); 569 | assert(buf == data1 ~ data2 ~ data3); 570 | } 571 | 572 | /** 573 | * Seeks relative to a position. 574 | * 575 | * Params: 576 | * offset = Offset relative to a reference point. 577 | * from = Optional reference point. 578 | */ 579 | long seekTo(long offset, From from = From.start) 580 | in { assert(isOpen); } 581 | body 582 | { 583 | version (Posix) 584 | { 585 | int whence = void; 586 | 587 | final switch (from) 588 | { 589 | case From.start: whence = SEEK_SET; break; 590 | case From.here: whence = SEEK_CUR; break; 591 | case From.end: whence = SEEK_END; break; 592 | } 593 | 594 | immutable pos = .lseek(_h, offset, whence); 595 | sysEnforce(pos != -1, "Failed to seek to position"); 596 | return pos; 597 | } 598 | else version (Windows) 599 | { 600 | DWORD whence = void; 601 | 602 | final switch (from) 603 | { 604 | case From.start: whence = FILE_BEGIN; break; 605 | case From.here: whence = FILE_CURRENT; break; 606 | case From.end: whence = FILE_END; break; 607 | } 608 | 609 | long pos = void; 610 | sysEnforce(SetFilePointerEx(_h, offset, &pos, whence), 611 | "Failed to seek to position"); 612 | return pos; 613 | } 614 | } 615 | 616 | unittest 617 | { 618 | auto tf = testFile(); 619 | 620 | auto f = File(tf.name, FileFlags.readWriteAlways); 621 | 622 | immutable data = "abcdefghijklmnopqrstuvwxyz"; 623 | assert(f.write(data) == data.length); 624 | 625 | assert(f.seekTo(5) == 5); 626 | assert(f.skip(5) == 10); 627 | assert(f.seekTo(-5, From.end) == data.length - 5); 628 | 629 | // Test offset beyond the end of the file 630 | assert(f.seekTo(int.max) == int.max); 631 | } 632 | 633 | /** 634 | * Gets the size of the file. 635 | * 636 | * Example: 637 | * --- 638 | * auto f = File("foobar", FileFlags.writeEmpty); 639 | * f.write("test"); 640 | * assert(f.length == 4); 641 | * --- 642 | */ 643 | @property long length() 644 | in { assert(isOpen); } 645 | body 646 | { 647 | version(Posix) 648 | { 649 | // Note that this uses stat to get the length of the file instead of 650 | // the seek method. This method is safer because it is atomic. 651 | stat_t stat = void; 652 | sysEnforce(.fstat(_h, &stat) != -1); 653 | return stat.st_size; 654 | } 655 | else version (Windows) 656 | { 657 | long size = void; 658 | sysEnforce(GetFileSizeEx(_h, &size)); 659 | return size; 660 | } 661 | } 662 | 663 | unittest 664 | { 665 | auto tf = testFile(); 666 | auto f = File(tf.name, FileFlags.writeEmpty); 667 | 668 | assert(f.length == 0); 669 | 670 | immutable data = "0123456789"; 671 | assert(f.write(data) == data.length); 672 | auto m = f.seekTo(3); 673 | 674 | assert(f.length == data.length); 675 | 676 | assert(f.position == m); 677 | } 678 | 679 | /** 680 | * Sets the length of the file. This can be used to truncate or extend the 681 | * length of the file. If the file is extended, the new segment is not 682 | * guaranteed to be initialized to zeros. 683 | * 684 | * Example: 685 | * --- 686 | * auto f = File("foobar", FileFlags.writeEmpty); 687 | * f.length = 42; 688 | * assert(f.length == 42); 689 | * --- 690 | */ 691 | @property void length(long len) 692 | in { assert(isOpen); } 693 | body 694 | { 695 | version (Posix) 696 | { 697 | sysEnforce( 698 | ftruncate(_h, len) == 0, 699 | "Failed to set the length of the file" 700 | ); 701 | } 702 | else version (Windows) 703 | { 704 | // FIXME: This is not thread safe because it requires 4 system calls. 705 | auto pos = seekTo(0, From.here); 706 | seekTo(len); // Seek to the correct position 707 | scope (exit) seekTo(pos); // Seek back 708 | 709 | sysEnforce( 710 | SetEndOfFile(_h), 711 | "Failed to set the length of the file" 712 | ); 713 | } 714 | } 715 | 716 | unittest 717 | { 718 | auto tf = testFile(); 719 | auto f = File(tf.name, FileFlags.writeEmpty); 720 | assert(f.length == 0); 721 | assert(f.position == 0); 722 | 723 | // Extend 724 | f.length = 100; 725 | assert(f.length == 100); 726 | assert(f.position == 0); 727 | 728 | // Truncate 729 | f.length = 0; 730 | assert(f.length == 0); 731 | assert(f.position == 0); 732 | } 733 | 734 | /** 735 | * Checks if the file refers to a terminal. 736 | * 737 | * Example: 738 | * --- 739 | * assert(stdin.isTerminal); 740 | * assert(stderr.isTerminal); 741 | * assert(stdout.isTerminal); 742 | * assert(!File("test", FileFlags.writeAlways).isTerminal); 743 | * --- 744 | */ 745 | @property bool isTerminal() 746 | in { assert(isOpen); } 747 | body 748 | { 749 | version (Posix) 750 | { 751 | return isatty(_h) == 1; 752 | } 753 | else version (Windows) 754 | { 755 | return GetFileType(_h) == FILE_TYPE_CHAR; 756 | } 757 | } 758 | 759 | unittest 760 | { 761 | auto tf = testFile(); 762 | assert(!File(tf.name, FileFlags.writeEmpty).isTerminal); 763 | } 764 | 765 | enum LockType 766 | { 767 | /** 768 | * Shared access to the locked file. Other processes can also access the 769 | * file. 770 | */ 771 | read, 772 | 773 | /** 774 | * Exclusive access to the locked file. No other processes may access 775 | * the file. 776 | */ 777 | readWrite, 778 | } 779 | 780 | version (Posix) 781 | { 782 | private int lockImpl(int operation, short type, 783 | long start, long length) 784 | { 785 | flock fl = { 786 | l_type: type, 787 | l_whence: SEEK_SET, 788 | l_start: start, 789 | l_len: (length == long.max) ? 0 : length, 790 | l_pid: -1, 791 | }; 792 | 793 | return .fcntl(_h, operation, &fl); 794 | } 795 | } 796 | else version (Windows) 797 | { 798 | private BOOL lockImpl(alias F, Flags...)( 799 | long start, long length, Flags flags) 800 | { 801 | import std.conv : to; 802 | 803 | immutable ULARGE_INTEGER 804 | liStart = {QuadPart: start.to!ulong}, 805 | liLength = {QuadPart: length.to!ulong}; 806 | 807 | OVERLAPPED overlapped = { 808 | Offset: liStart.LowPart, 809 | OffsetHigh: liStart.HighPart, 810 | hEvent: null, 811 | }; 812 | 813 | return F(_h, flags, 0, liLength.LowPart, liLength.HighPart, 814 | &overlapped); 815 | } 816 | } 817 | 818 | /** 819 | * Locks the specified file segment. If the file segment is already locked 820 | * by another process, waits until the existing lock is released. 821 | * 822 | * Note that this is a $(I per-process) lock. This locking mechanism should 823 | * not be used for thread-level synchronization. For that, use the $(D 824 | * synchronized) statement. 825 | */ 826 | void lock(LockType lockType = LockType.readWrite, 827 | long start = 0, long length = long.max) 828 | in { assert(isOpen); } 829 | body 830 | { 831 | version (Posix) 832 | { 833 | sysEnforce( 834 | lockImpl(F_SETLKW, 835 | lockType == LockType.readWrite ? F_WRLCK : F_RDLCK, 836 | start, length, 837 | ) != -1, "Failed to lock file" 838 | ); 839 | } 840 | else version (Windows) 841 | { 842 | sysEnforce( 843 | lockImpl!LockFileEx( 844 | start, length, 845 | lockType == LockType.readWrite ? LOCKFILE_EXCLUSIVE_LOCK : 0 846 | ), "Failed to lock file" 847 | ); 848 | } 849 | } 850 | 851 | /** 852 | * Like $(D lock), but returns false immediately if the lock is held by 853 | * another process. Returns true if the specified region in the file was 854 | * successfully locked. 855 | */ 856 | bool tryLock(LockType lockType = LockType.readWrite, 857 | long start = 0, long length = long.max) 858 | in { assert(isOpen); } 859 | body 860 | { 861 | version (Posix) 862 | { 863 | import core.stdc.errno; 864 | 865 | // Set the lock, return immediately if it's being held by another 866 | // process. 867 | if (lockImpl(F_SETLK, 868 | lockType == LockType.readWrite ? F_WRLCK : F_RDLCK, 869 | start, length) != 0 870 | ) 871 | { 872 | // Is another process is holding the lock? 873 | if (errno == EACCES || errno == EAGAIN) 874 | return false; 875 | else 876 | sysEnforce(false, "Failed to lock file"); 877 | } 878 | 879 | return true; 880 | } 881 | else version (Windows) 882 | { 883 | immutable flags = LOCKFILE_FAIL_IMMEDIATELY | ( 884 | (lockType == LockType.readWrite) ? LOCKFILE_EXCLUSIVE_LOCK : 0); 885 | if (!lockImpl!LockFileEx(start, length, flags)) 886 | { 887 | if (GetLastError() == ERROR_IO_PENDING || 888 | GetLastError() == ERROR_LOCK_VIOLATION) 889 | return false; 890 | else 891 | sysEnforce(false, "Failed to lock file"); 892 | } 893 | 894 | return true; 895 | } 896 | } 897 | 898 | void unlock(long start = 0, long length = long.max) 899 | in { assert(isOpen); } 900 | body 901 | { 902 | version (Posix) 903 | { 904 | sysEnforce( 905 | lockImpl(F_SETLK, F_UNLCK, start, length) != -1, 906 | "Failed to lock file" 907 | ); 908 | } 909 | else version (Windows) 910 | { 911 | sysEnforce(lockImpl!UnlockFileEx(start, length), 912 | "Failed to unlock file"); 913 | } 914 | } 915 | 916 | /** 917 | * Syncs all modified cached data of the file to disk. This includes data 918 | * written to the file as well as meta data (e.g., last modified time, last 919 | * access time). 920 | */ 921 | void sync() 922 | in { assert(isOpen); } 923 | body 924 | { 925 | version (Posix) 926 | { 927 | sysEnforce(fsync(_h) == 0); 928 | } 929 | else version (Windows) 930 | { 931 | sysEnforce(FlushFileBuffers(_h) != 0); 932 | } 933 | } 934 | 935 | /** 936 | * Copies the rest of this file to the other. The positions of both files 937 | * are appropriately incremented, as if one called read()/write() to copy 938 | * the file. The number of copied bytes is returned. 939 | */ 940 | version (linux) 941 | { 942 | size_t copyTo(File other, size_t n = ptrdiff_t.max) 943 | in { assert(isOpen && other.isOpen); } 944 | body 945 | { 946 | immutable written = .sendfile(other._h, _h, null, n); 947 | sysEnforce(written >= 0, "Failed to copy file."); 948 | return written; 949 | } 950 | 951 | unittest 952 | { 953 | import std.conv : to; 954 | 955 | auto a = tempFile.file; 956 | auto b = tempFile.file; 957 | immutable s = "This will be copied to the other file."; 958 | a.write(s); 959 | a.position = 0; 960 | a.copyTo(b); 961 | assert(a.position == s.length); 962 | 963 | b.position = 0; 964 | 965 | char[s.length] buf; 966 | assert(b.read(buf) == s.length); 967 | assert(buf == s); 968 | } 969 | } 970 | } 971 | 972 | unittest 973 | { 974 | static assert(isSink!FileBase); 975 | static assert(isSource!FileBase); 976 | static assert(isSeekable!FileBase); 977 | } 978 | 979 | import std.typecons; 980 | import io.buffer.fixed; 981 | alias UnbufferedFile = RefCounted!(StreamShim!FileBase, RefCountedAutoInitialize.no); 982 | alias File = RefCounted!(StreamShim!(FixedBufferBase!FileBase), RefCountedAutoInitialize.no); 983 | 984 | unittest 985 | { 986 | import io.buffer.traits; 987 | 988 | static assert(isSink!UnbufferedFile); 989 | static assert(isSource!UnbufferedFile); 990 | static assert(isSeekable!UnbufferedFile); 991 | static assert(isBufferable!UnbufferedFile); 992 | 993 | static assert(isSink!File); 994 | static assert(isSource!File); 995 | static assert(isSeekable!File); 996 | static assert(isFlushable!File); 997 | } 998 | 999 | unittest 1000 | { 1001 | auto tf = testFile(); 1002 | 1003 | immutable message = "The quick brown fox jumps over the lazy dog."; 1004 | 1005 | char[message.length] buf; 1006 | 1007 | foreach (bufSize; [0, 1, 2, 8, 16, 64, 256]) 1008 | { 1009 | { 1010 | auto f = File(tf.name, FileFlags.writeEmpty); 1011 | f.bufferSize = bufSize; 1012 | assert(f.bufferSize == bufSize); 1013 | assert(f.write(message) == message.length); 1014 | } 1015 | 1016 | { 1017 | auto f = File(tf.name, FileFlags.readExisting); 1018 | f.bufferSize = bufSize; 1019 | assert(f.bufferSize == bufSize); 1020 | assert(buf[0 .. f.read(buf)] == message); 1021 | } 1022 | } 1023 | } 1024 | 1025 | unittest 1026 | { 1027 | auto tf = testFile(); 1028 | 1029 | immutable data = "The quick brown fox jumps over the lazy dog."; 1030 | char[data.length] buffer; 1031 | 1032 | foreach (bufSize; [0, 1, 2, 8, 16, 64, 4096, 8192]) 1033 | { 1034 | auto f = File(tf.name, FileFlags.readWriteEmpty); 1035 | f.bufferSize = bufSize; 1036 | assert(f.bufferSize == bufSize); 1037 | 1038 | assert(f.write(data) == data.length); 1039 | f.position = 0; 1040 | assert(f.read(buffer) == buffer.length); 1041 | assert(buffer == data); 1042 | } 1043 | } 1044 | 1045 | version (unittest) 1046 | { 1047 | /** 1048 | * Generates a file name for testing and attempts to delete it on 1049 | * destruction. 1050 | */ 1051 | auto testFile(string file = __FILE__, size_t line = __LINE__) 1052 | { 1053 | import std.conv : text; 1054 | import std.path : baseName; 1055 | import std.file : tempDir; 1056 | 1057 | static struct TestFile 1058 | { 1059 | string name; 1060 | 1061 | alias name this; 1062 | 1063 | ~this() 1064 | { 1065 | // Don't care if this fails. 1066 | try .file.remove(name); catch (Exception e) {} 1067 | } 1068 | } 1069 | 1070 | return TestFile(text(tempDir, "/.deleteme-", baseName(file), ".", line)); 1071 | } 1072 | } 1073 | --------------------------------------------------------------------------------