├── .travis.yml ├── LICENSE ├── README.md ├── ci └── run-tests.go ├── file.go ├── file_metadata.go ├── file_test.go ├── go.mod ├── go.sum ├── hdf5.go ├── rbuffer.go └── testdata ├── empty-v0.h5 ├── empty-v3.h5 └── gen-create-empty-file.go /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | os: 3 | - linux 4 | 5 | env: 6 | - TAGS="-tags travis" 7 | 8 | cache: 9 | directories: 10 | - $HOME/.cache/go-build 11 | - $HOME/gopath/pkg/mod 12 | 13 | matrix: 14 | fast_finish: true 15 | allow_failures: 16 | - go: master 17 | include: 18 | - go: 1.9.x 19 | env: 20 | - TAGS="-tags travis" 21 | - go: 1.10.x 22 | env: 23 | - TAGS="-tags travis" 24 | - go: 1.11.x 25 | env: 26 | - TAGS="-tags travis" 27 | - COVERAGE="-cover -race" 28 | - go: master 29 | env: 30 | - TAGS="-tags travis" 31 | - COVERAGE="-race" 32 | - GO111MODULE="on" 33 | 34 | sudo: false 35 | 36 | addons: 37 | apt: 38 | packages: 39 | - libhdf5-serial-dev 40 | 41 | notifications: 42 | email: 43 | recipients: 44 | - binet@cern.ch 45 | on_success: always 46 | on_failure: always 47 | 48 | script: 49 | - go get -d -t -v ./... 50 | - go install -v $TAGS ./... 51 | - go run ./ci/run-tests.go $TAGS $COVERAGE 52 | 53 | after_success: 54 | - bash <(curl -s https://codecov.io/bash) 55 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright ©2018 The go-hdf5 Authors. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are met: 5 | * Redistributions of source code must retain the above copyright 6 | notice, this list of conditions and the following disclaimer. 7 | * Redistributions in binary form must reproduce the above copyright 8 | notice, this list of conditions and the following disclaimer in the 9 | documentation and/or other materials provided with the distribution. 10 | * Neither the name of the go-hdf5 project nor the names of its authors and 11 | contributors may be used to endorse or promote products derived from this 12 | software without specific prior written permission. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # hdf5 2 | 3 | [![Build Status](https://travis-ci.org/go-hdf5/hdf5.svg?branch=master)](https://travis-ci.org/go-hdf5/hdf5) 4 | [![codecov](https://codecov.io/gh/go-hdf5/hdf5/branch/master/graph/badge.svg)](https://codecov.io/gh/go-hdf5/hdf5) 5 | [![Go Report Card](https://goreportcard.com/badge/github.com/go-hdf5/hdf5)](https://goreportcard.com/report/github.com/go-hdf5/hdf5) 6 | [![GoDoc](https://godoc.org/github.com/go-hdf5/hdf5?status.svg)](https://godoc.org/github.com/go-hdf5/hdf5) 7 | 8 | `hdf5` is a pure Go implementation of the [HDF5](https://www.hdfgroup.org/) file format. 9 | 10 | ## License 11 | 12 | `hdf5` is released under the `BSD-3` license. 13 | 14 | ## Documentation 15 | 16 | Documentation for `hdf5` is served by [GoDoc](https://godoc.org/github.com/go-hdf5/hdf5). 17 | -------------------------------------------------------------------------------- /ci/run-tests.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The go-hdf5 Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build ignore 6 | 7 | package main 8 | 9 | import ( 10 | "bufio" 11 | "bytes" 12 | "flag" 13 | "io/ioutil" 14 | "log" 15 | "os" 16 | "os/exec" 17 | "strings" 18 | ) 19 | 20 | func main() { 21 | log.SetPrefix("ci: ") 22 | log.SetFlags(0) 23 | 24 | var ( 25 | race = flag.Bool("race", false, "enable race detector") 26 | cover = flag.Bool("cover", false, "enable code coverage") 27 | tags = flag.String("tags", "", "build tags") 28 | ) 29 | 30 | flag.Parse() 31 | 32 | out := new(bytes.Buffer) 33 | cmd := exec.Command("go", "list", "./...") 34 | cmd.Stdout = out 35 | cmd.Stderr = os.Stderr 36 | cmd.Stdin = os.Stdin 37 | 38 | err := cmd.Run() 39 | if err != nil { 40 | log.Fatal(err) 41 | } 42 | 43 | f, err := os.Create("coverage.txt") 44 | if err != nil { 45 | log.Fatal(err) 46 | } 47 | defer f.Close() 48 | 49 | args := []string{"test", "-v"} 50 | 51 | if *cover { 52 | args = append(args, "-coverprofile=profile.out", "-covermode=atomic") 53 | } 54 | if *tags != "" { 55 | args = append(args, "-tags="+*tags) 56 | } 57 | if *race { 58 | args = append(args, "-race") 59 | } 60 | args = append(args, "") 61 | 62 | scan := bufio.NewScanner(out) 63 | for scan.Scan() { 64 | pkg := scan.Text() 65 | if strings.Contains(pkg, "vendor") { 66 | continue 67 | } 68 | args[len(args)-1] = pkg 69 | cmd := exec.Command("go", args...) 70 | cmd.Stdin = os.Stdin 71 | cmd.Stdout = os.Stdout 72 | cmd.Stderr = os.Stderr 73 | err := cmd.Run() 74 | if err != nil { 75 | log.Fatal(err) 76 | } 77 | if *cover { 78 | profile, err := ioutil.ReadFile("profile.out") 79 | if err != nil { 80 | log.Fatal(err) 81 | } 82 | _, err = f.Write(profile) 83 | if err != nil { 84 | log.Fatal(err) 85 | } 86 | os.Remove("profile.out") 87 | } 88 | } 89 | 90 | err = f.Close() 91 | if err != nil { 92 | log.Fatal(err) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /file.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The go-hdf5 Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package hdf5 6 | 7 | import ( 8 | "bytes" 9 | "io" 10 | "os" 11 | 12 | "github.com/pkg/errors" 13 | ) 14 | 15 | // File represents an HDF5 file. 16 | type File struct { 17 | f *os.File 18 | 19 | super Superblock 20 | } 21 | 22 | // Open opens the named HDF5 file for reading. 23 | func Open(name string) (*File, error) { 24 | f, err := os.Open(name) 25 | if err != nil { 26 | return nil, err 27 | } 28 | 29 | // FIXME(sbinet): the signature+superblock may be located at 30 | // file offsets: 0, 512, 1024, 2048, ... 31 | buf := make([]byte, 8) 32 | _, err = io.ReadFull(f, buf) 33 | if err != nil { 34 | f.Close() 35 | return nil, errors.Wrapf(err, "could not read HDF5 signature") 36 | } 37 | 38 | if !bytes.Equal(buf, Signature[:]) { 39 | f.Close() 40 | return nil, ErrNotHDF5File 41 | } 42 | 43 | super, err := decodeSuperblock(f) 44 | if err != nil { 45 | f.Close() 46 | return nil, errors.Wrapf(err, "could not decode superblock") 47 | } 48 | 49 | return &File{f: f, super: super}, nil 50 | } 51 | 52 | func (f *File) Close() error { 53 | return f.f.Close() 54 | } 55 | -------------------------------------------------------------------------------- /file_metadata.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The go-hdf5 Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package hdf5 6 | 7 | import ( 8 | "encoding/binary" 9 | "io" 10 | 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | type Superblock interface { 15 | Version() byte 16 | Offsets() byte 17 | Lengths() byte 18 | 19 | decode(r io.Reader) error 20 | } 21 | 22 | type SuperblockV0 struct { 23 | version struct { 24 | FreeSpace byte 25 | SymTable byte 26 | _ byte 27 | SharedHeader byte 28 | } 29 | offsets byte 30 | lengths byte 31 | _ byte 32 | GroupLeafNode uint16 33 | GroupInternalNode uint16 34 | Flags uint32 35 | addr struct { 36 | base uint64 37 | free uint64 38 | eof uint64 39 | drv uint64 40 | } 41 | SymTable uint32 42 | } 43 | 44 | func (*SuperblockV0) Version() byte { return 0 } 45 | func (sblk *SuperblockV0) Offsets() byte { return sblk.offsets } 46 | func (sblk *SuperblockV0) Lengths() byte { return sblk.lengths } 47 | 48 | func (s *SuperblockV0) decode(r io.Reader) error { 49 | buf := make([]byte, 15) 50 | _, err := io.ReadFull(r, buf) 51 | if err != nil { 52 | return err 53 | } 54 | 55 | s.version.FreeSpace = buf[0] 56 | s.version.SymTable = buf[1] 57 | s.version.SharedHeader = buf[3] 58 | s.offsets = buf[4] 59 | s.lengths = buf[5] 60 | _ = buf[6] 61 | s.GroupLeafNode = binary.LittleEndian.Uint16(buf[7 : 7+2]) 62 | s.GroupInternalNode = binary.LittleEndian.Uint16(buf[9 : 9+2]) 63 | s.Flags = binary.LittleEndian.Uint32(buf[11:15]) 64 | 65 | rr := newRBuffer(r, s.offsets, s.lengths) 66 | s.addr.base = rr.readOffset() 67 | s.addr.free = rr.readOffset() 68 | s.addr.eof = rr.readOffset() 69 | s.addr.drv = rr.readOffset() 70 | s.SymTable = rr.readU32() 71 | 72 | return rr.err 73 | } 74 | 75 | type SuperblockV1 struct { 76 | version struct { 77 | FreeSpace byte 78 | SymTable byte 79 | } 80 | _ byte 81 | SharedHeader byte 82 | offsets byte 83 | lengths byte 84 | _ byte 85 | GroupLeafNode uint16 86 | GroupInternalNode uint16 87 | flags uint32 88 | IndexedStorage uint16 89 | _ uint16 90 | addr struct { 91 | base uint64 92 | free uint64 93 | eof uint64 94 | drv uint64 95 | } 96 | SymTable uint32 97 | } 98 | 99 | func (*SuperblockV1) Version() byte { return 1 } 100 | func (sblk *SuperblockV1) Offsets() byte { return sblk.offsets } 101 | func (sblk *SuperblockV1) Lengths() byte { return sblk.lengths } 102 | 103 | func (s *SuperblockV1) decode(r io.Reader) error { 104 | panic("not implemented") 105 | } 106 | 107 | type SuperblockV2 struct { 108 | offsets byte 109 | lengths byte 110 | flags uint32 111 | addr struct { 112 | base uint64 113 | super uint64 114 | eof uint64 115 | rootGroup uint64 116 | } 117 | chksum uint32 118 | } 119 | 120 | func (*SuperblockV2) Version() byte { return 2 } 121 | func (sblk *SuperblockV2) Offsets() byte { return sblk.offsets } 122 | func (sblk *SuperblockV2) Lengths() byte { return sblk.lengths } 123 | 124 | func (s *SuperblockV2) decode(r io.Reader) error { 125 | panic("not implemented") 126 | } 127 | 128 | type SuperblockV3 struct { 129 | offsets byte 130 | lengths byte 131 | flags uint32 132 | addr struct { 133 | base uint64 134 | super uint64 135 | eof uint64 136 | rootGroup uint64 137 | } 138 | chksum uint32 139 | } 140 | 141 | func (*SuperblockV3) Version() byte { return 3 } 142 | func (sblk *SuperblockV3) Offsets() byte { return sblk.offsets } 143 | func (sblk *SuperblockV3) Lengths() byte { return sblk.lengths } 144 | 145 | func (s *SuperblockV3) decode(r io.Reader) error { 146 | panic("not implemented") 147 | } 148 | 149 | func decodeSuperblock(r io.Reader) (Superblock, error) { 150 | var buf [8]byte 151 | _, err := io.ReadFull(r, buf[:1]) 152 | if err != nil { 153 | return nil, errors.Wrapf(err, "could not read superblock version") 154 | } 155 | 156 | var super Superblock 157 | switch buf[0] { 158 | case 0: 159 | super = &SuperblockV0{} 160 | case 1: 161 | super = &SuperblockV1{} 162 | case 2: 163 | super = &SuperblockV2{} 164 | case 3: 165 | super = &SuperblockV3{} 166 | default: 167 | return nil, ErrBadSuperblockVersion 168 | } 169 | 170 | err = super.decode(r) 171 | if err != nil { 172 | return nil, err 173 | } 174 | 175 | return super, nil 176 | } 177 | 178 | var ( 179 | _ Superblock = (*SuperblockV0)(nil) 180 | _ Superblock = (*SuperblockV1)(nil) 181 | _ Superblock = (*SuperblockV2)(nil) 182 | _ Superblock = (*SuperblockV3)(nil) 183 | ) 184 | -------------------------------------------------------------------------------- /file_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The go-hdf5 Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package hdf5 6 | 7 | import ( 8 | "math" 9 | "reflect" 10 | "testing" 11 | ) 12 | 13 | func TestOpen(t *testing.T) { 14 | f, err := Open("testdata/empty-v0.h5") 15 | if err != nil { 16 | t.Fatal(err) 17 | } 18 | defer f.Close() 19 | 20 | var want = new(SuperblockV0) 21 | want.version.FreeSpace = 0x0 22 | want.version.SymTable = 0x0 23 | want.version.SharedHeader = 0x0 24 | want.offsets = 0x8 25 | want.lengths = 0x8 26 | want.GroupLeafNode = 0x04 27 | want.GroupInternalNode = 0x10 28 | want.Flags = 0x0 29 | want.addr.base = 0x0 30 | want.addr.free = math.MaxUint64 31 | want.addr.eof = 0x320 32 | want.addr.drv = math.MaxUint64 33 | want.SymTable = 0x0 34 | 35 | if !reflect.DeepEqual(want, f.super) { 36 | t.Fatalf("invalid file:\ngot= %#v\nwant=%#v", f.super, want) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/go-hdf5/hdf5 2 | 3 | require github.com/pkg/errors v0.8.0 4 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= 2 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 3 | -------------------------------------------------------------------------------- /hdf5.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The go-hdf5 Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package hdf5 reads and writes files in the HDF5 format 6 | package hdf5 7 | 8 | import ( 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | //go:generate go run ./testdata/gen-create-empty-file.go 13 | 14 | var ( 15 | // Signature identifies a file as being an HDF5 file. 16 | Signature = [8]byte{0x89, 'H', 'D', 'F', '\r', '\n', 0x1a, '\n'} 17 | 18 | ErrNotHDF5File = errors.New("hdf5: not a HDF5 file") 19 | ErrBadSuperblockVersion = errors.New("hdf5: bad superblock version number") 20 | ) 21 | -------------------------------------------------------------------------------- /rbuffer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The go-hdf5 Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package hdf5 6 | 7 | import ( 8 | "encoding/binary" 9 | "io" 10 | 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | type rbuffer struct { 15 | r io.Reader 16 | err error 17 | buf []byte 18 | 19 | sz struct { 20 | offset func() uint64 21 | length func() uint64 22 | } 23 | } 24 | 25 | func newRBuffer(r io.Reader, offset, length byte) *rbuffer { 26 | rbuf := &rbuffer{r: r, buf: make([]byte, 8)} 27 | switch offset { 28 | case 4: 29 | rbuf.sz.offset = func() uint64 { return uint64(rbuf.readU32()) } 30 | case 8: 31 | rbuf.sz.offset = rbuf.readU64 32 | default: 33 | panic(errors.Errorf("hdf5: invalid offset size (%v)", offset)) 34 | } 35 | 36 | switch length { 37 | case 4: 38 | rbuf.sz.length = func() uint64 { return uint64(rbuf.readU32()) } 39 | case 8: 40 | rbuf.sz.length = rbuf.readU64 41 | default: 42 | panic(errors.Errorf("hdf5: invalid length size (%v)", length)) 43 | } 44 | 45 | return rbuf 46 | } 47 | 48 | func (r *rbuffer) Read(p []byte) (int, error) { 49 | if r.err != nil { 50 | return 0, r.err 51 | } 52 | var n int 53 | n, r.err = r.r.Read(p) 54 | return n, r.err 55 | } 56 | 57 | func (r *rbuffer) readOffset() uint64 { 58 | return r.sz.offset() 59 | } 60 | 61 | func (r *rbuffer) readLen() uint64 { 62 | return r.sz.length() 63 | } 64 | 65 | func (r *rbuffer) readU16() uint16 { 66 | const n = 2 67 | r.load(n) 68 | return binary.LittleEndian.Uint16(r.buf[:n]) 69 | } 70 | 71 | func (r *rbuffer) readU32() uint32 { 72 | const n = 4 73 | r.load(n) 74 | return binary.LittleEndian.Uint32(r.buf[:n]) 75 | } 76 | 77 | func (r *rbuffer) readU64() uint64 { 78 | const n = 8 79 | r.load(n) 80 | return binary.LittleEndian.Uint64(r.buf[:n]) 81 | } 82 | 83 | func (r *rbuffer) load(n int) { 84 | if r.err != nil { 85 | return 86 | } 87 | _, r.err = io.ReadFull(r, r.buf[:n]) 88 | } 89 | 90 | var ( 91 | _ io.Reader = (*rbuffer)(nil) 92 | ) 93 | -------------------------------------------------------------------------------- /testdata/empty-v0.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-hdf5/hdf5/af9b4abfc0710c005b0264137168bec468a93c20/testdata/empty-v0.h5 -------------------------------------------------------------------------------- /testdata/empty-v3.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-hdf5/hdf5/af9b4abfc0710c005b0264137168bec468a93c20/testdata/empty-v3.h5 -------------------------------------------------------------------------------- /testdata/gen-create-empty-file.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The go-hdf5 Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build ignore 6 | 7 | package main 8 | 9 | import ( 10 | "log" 11 | 12 | "gonum.org/v1/hdf5" 13 | ) 14 | 15 | func main() { 16 | createEmptyFile() 17 | createDataset_2x5() 18 | } 19 | 20 | func createEmptyFile() { 21 | const fname = "testdata/empty.h5" 22 | f, err := hdf5.CreateFile(fname, hdf5.F_ACC_TRUNC) 23 | if err != nil { 24 | log.Fatal(err) 25 | } 26 | err = f.Close() 27 | if err != nil { 28 | log.Fatal(err) 29 | } 30 | } 31 | 32 | func createDataset_2x5() { 33 | const fname = "testdata/dataset.h5" 34 | 35 | f, err := hdf5.CreateFile(fname, hdf5.F_ACC_TRUNC) 36 | if err != nil { 37 | log.Fatalf("CreateFile failed: %s", err) 38 | } 39 | defer f.Close() 40 | 41 | dims := []uint{2, 5} 42 | dspace, err := hdf5.CreateSimpleDataspace(dims, dims) 43 | if err != nil { 44 | log.Fatal(err) 45 | } 46 | defer dspace.Close() 47 | 48 | dset, err := f.CreateDataset("dset", hdf5.T_NATIVE_USHORT, dspace) 49 | if err != nil { 50 | log.Fatal(err) 51 | } 52 | defer dset.Close() 53 | 54 | data := [10]uint16{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} 55 | err = dset.Write(&data[0]) 56 | if err != nil { 57 | log.Fatal(err) 58 | } 59 | 60 | err = f.Close() 61 | if err != nil { 62 | log.Fatal(err) 63 | } 64 | } 65 | --------------------------------------------------------------------------------