├── netcdf ├── testdata │ ├── empty │ ├── bogus │ ├── hdf5.nc │ └── cdf.nc ├── cdf │ ├── testdata │ │ ├── testnull.cdl │ │ ├── solarforcing_small.nc │ │ ├── testempty.cdl │ │ ├── testbytesonly.cdl │ │ ├── testonedim.cdl │ │ ├── testshortsonly.cdl │ │ ├── testcdf5.cdl │ │ ├── testunlimited.cdl │ │ └── testmultidim.cdl │ ├── perf_test.go │ ├── slicer_test.go │ ├── verify_test.go │ └── cdfwriter.go ├── hdf5 │ ├── testdata │ │ ├── badmagic │ │ ├── bitfield.h5 │ │ ├── reference.h5 │ │ ├── testtypesbe.nc │ │ ├── testonedim.cdl │ │ ├── testbytesunlimited.cdl │ │ ├── testfilters.cdl │ │ ├── testshortsonly.cdl │ │ ├── small.cdl │ │ ├── testslicer.cdl │ │ ├── testopaque.cdl │ │ ├── testattrtypes.cdl │ │ ├── testunlimited.cdl │ │ ├── testarray.cdl │ │ ├── testsimple.cdl │ │ ├── testbytes.cdl │ │ ├── testempty.cdl │ │ ├── testgroups.cdl │ │ ├── testcompounds.cdl │ │ ├── testvtypes.cdl │ │ ├── testvlen.cdl │ │ ├── testattrs.cdl │ │ ├── testenum.cdl │ │ ├── testfills.cdl │ │ ├── testfills2.cdl │ │ ├── testtypes.cdl │ │ └── testlayout.cdl │ ├── amap.go │ ├── segment.go │ ├── time.go │ ├── slicer_test.go │ ├── layout_test.go │ ├── amap_test.go │ ├── checksum.go │ ├── bitfield.go │ ├── string.go │ ├── assert.go │ ├── opaque.go │ ├── reference.go │ ├── array.go │ ├── errors.go │ ├── types.go │ ├── readers_test.go │ ├── enum.go │ ├── floatingPoint.go │ ├── compound.go │ ├── fixedPoint.go │ ├── vlen.go │ ├── types_test.go │ └── readers.go ├── netcdf4_test.go ├── netcdf4.go ├── api │ └── api.go └── util │ ├── orderedmap_test.go │ └── orderedmap.go ├── go.mod ├── go.sum ├── internal ├── fillvalue.go ├── validname_test.go ├── validname.go ├── slicer.go └── logging.go ├── LICENSE └── README.md /netcdf/testdata/empty: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /netcdf/testdata/bogus: -------------------------------------------------------------------------------- 1 | bogus -------------------------------------------------------------------------------- /netcdf/cdf/testdata/testnull.cdl: -------------------------------------------------------------------------------- 1 | netcdf testnull { 2 | } 3 | -------------------------------------------------------------------------------- /netcdf/testdata/hdf5.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/batchatco/go-native-netcdf/HEAD/netcdf/testdata/hdf5.nc -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/badmagic: -------------------------------------------------------------------------------- 1 | 123456789 2 | 123456789 3 | 123456789 4 | 123456789 5 | 123456789 6 | 123456789 7 | 123456789 8 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/bitfield.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/batchatco/go-native-netcdf/HEAD/netcdf/hdf5/testdata/bitfield.h5 -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/reference.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/batchatco/go-native-netcdf/HEAD/netcdf/hdf5/testdata/reference.h5 -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testtypesbe.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/batchatco/go-native-netcdf/HEAD/netcdf/hdf5/testdata/testtypesbe.nc -------------------------------------------------------------------------------- /netcdf/cdf/testdata/solarforcing_small.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/batchatco/go-native-netcdf/HEAD/netcdf/cdf/testdata/solarforcing_small.nc -------------------------------------------------------------------------------- /netcdf/cdf/testdata/testempty.cdl: -------------------------------------------------------------------------------- 1 | netcdf testempty { 2 | dimensions: 3 | u = UNLIMITED ; // (0 currently) 4 | variables: 5 | int a(u); 6 | } 7 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testonedim.cdl: -------------------------------------------------------------------------------- 1 | netcdf onedim { 2 | dimensions: 3 | d1 = 1; 4 | variables: 5 | char c(d1); 6 | data: 7 | c = "a"; 8 | } 9 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/batchatco/go-native-netcdf 2 | 3 | go 1.18 4 | 5 | require github.com/batchatco/go-thrower v0.0.0-20200827035905-5cb7337f6be6 6 | -------------------------------------------------------------------------------- /netcdf/cdf/testdata/testbytesonly.cdl: -------------------------------------------------------------------------------- 1 | netcdf testunlimited { 2 | dimensions: 3 | d1 = UNLIMITED; 4 | variables: 5 | byte i8x1(d1); 6 | data: 7 | i8x1 = 12, 56; 8 | } 9 | -------------------------------------------------------------------------------- /netcdf/cdf/testdata/testonedim.cdl: -------------------------------------------------------------------------------- 1 | netcdf onedim { 2 | dimensions: 3 | d1 = 1; 4 | variables: 5 | char c; 6 | char c2(d1); 7 | data: 8 | c = 'a'; 9 | c2 = 'b'; 10 | } 11 | -------------------------------------------------------------------------------- /netcdf/cdf/testdata/testshortsonly.cdl: -------------------------------------------------------------------------------- 1 | netcdf testunlimited { 2 | dimensions: 3 | d1 = UNLIMITED; 4 | variables: 5 | short i16x1(d1); 6 | data: 7 | i16x1 = 9876, 5432, 7734; 8 | } 9 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testbytesunlimited.cdl: -------------------------------------------------------------------------------- 1 | netcdf testbytesunlimited { 2 | dimensions: 3 | d1 = UNLIMITED; 4 | variables: 5 | byte i8x1(d1); 6 | data: 7 | i8x1 = 12, 56; 8 | } 9 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testfilters.cdl: -------------------------------------------------------------------------------- 1 | netcdf filters { 2 | dimensions: 3 | dim = 10; 4 | variables: 5 | int nums(dim); 6 | data: 7 | nums = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9; 8 | } 9 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testshortsonly.cdl: -------------------------------------------------------------------------------- 1 | netcdf testunlimited { 2 | dimensions: 3 | d1 = UNLIMITED; 4 | variables: 5 | short i16x1(d1); 6 | data: 7 | i16x1 = 9876, 5432, 7734; 8 | } 9 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/small.cdl: -------------------------------------------------------------------------------- 1 | netcdf testtypes { 2 | dimensions: 3 | dim = 1; 4 | d1 = 2; 5 | d2 = 2; 6 | 7 | variables: 8 | int i32x2(d1, d2); 9 | 10 | data: 11 | i32x2 = -10000000, 10000000, -20000000, 20000000; 12 | } 13 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/batchatco/go-thrower v0.0.0-20200827035905-5cb7337f6be6 h1:gDf4IUqKDnH7F0XdgeYOBx2jlMKF/j9Xm42sISXpwqY= 2 | github.com/batchatco/go-thrower v0.0.0-20200827035905-5cb7337f6be6/go.mod h1:hJ9Ll7FOzcIr57sd7RHga7StcCVAL0vFBUsNpnGntNg= 3 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testslicer.cdl: -------------------------------------------------------------------------------- 1 | netcdf testslicer { 2 | dimensions: 3 | lat = 4; 4 | lon = 5; 5 | variables: 6 | byte tid(lat, lon); 7 | data: 8 | tid = 0, 1, 2, 3, 4, 9 | 5, 6, 6, 8, 9, 10 | 10, 11, 12, 13, 14, 11 | 15, 16, 17, 18, 19; 12 | } 13 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testopaque.cdl: -------------------------------------------------------------------------------- 1 | netcdf testopaque { 2 | types: 3 | opaque(5) opaque5; 4 | 5 | dimensions: 6 | dim = 5; 7 | 8 | variables: 9 | opaque5 v(dim); 10 | 11 | data: 12 | v = 0xdeadbeef01,0xdeadbeef02,0xdeadbeef03,0xdeadbeef04,0xdeadbeef05; 13 | } 14 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testattrtypes.cdl: -------------------------------------------------------------------------------- 1 | netcdf testattrtypes { 2 | 3 | variables: 4 | int :i = 0; 5 | float :f = 0; 6 | double :d = 0; 7 | string :s = "a"; 8 | 9 | int :i1 = 0, 0 ; 10 | float :f1 = 0, 0; 11 | double :d1 = 0, 0; 12 | string :s1 = "a", "b"; 13 | } 14 | -------------------------------------------------------------------------------- /netcdf/cdf/testdata/testcdf5.cdl: -------------------------------------------------------------------------------- 1 | netcdf testcdf5 { 2 | dimensions: 3 | d2 = 1; 4 | variables: 5 | ubyte ub(d2); 6 | ushort u16(d2); 7 | uint u32(d2); 8 | uint64 u64(d2); 9 | int64 i64(d2); 10 | data: 11 | ub = 1; 12 | u16 = 2; 13 | u32 = 3; 14 | u64 = 4; 15 | i64 = 5; 16 | } 17 | -------------------------------------------------------------------------------- /netcdf/cdf/testdata/testunlimited.cdl: -------------------------------------------------------------------------------- 1 | netcdf testunlimited { 2 | dimensions: 3 | d1 = UNLIMITED; 4 | d2 = 1; 5 | variables: 6 | byte i8x1(d1, d2); 7 | short i16x1(d1); 8 | int i32x1(d1); 9 | double dx1(d1); 10 | data: 11 | i8x1 = 12, 56; 12 | i16x1 = 9876, 5432; 13 | i32x1 = 12, 34; 14 | dx1 = 5.6e100, 7.8e100; 15 | } 16 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testunlimited.cdl: -------------------------------------------------------------------------------- 1 | netcdf testunlimited { 2 | dimensions: 3 | d1 = UNLIMITED; 4 | d2 = 1; 5 | variables: 6 | byte i8x1(d1, d2); 7 | short i16x1(d1); 8 | int i32x1(d1); 9 | int64 i64x1(d1); 10 | data: 11 | i8x1 = 12, 56; 12 | i16x1 = 9876, 5432; 13 | i32x1 = 12, 34; 14 | i64x1 = 56100, 78100; 15 | } 16 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testarray.cdl: -------------------------------------------------------------------------------- 1 | netcdf testarray { 2 | types: 3 | compound comp { 4 | int iArray(3); 5 | float fArray(2, 3); 6 | }; 7 | dimensions: 8 | dim = 2; 9 | variables: 10 | comp c(dim); 11 | data: 12 | c = 13 | {{1, 2, 3}, 14 | {4, 5, 6, 7, 8, 9}}, 15 | {{10, 11, 12}, 16 | {13, 14, 15, 16, 17, 18}}; 17 | } 18 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testsimple.cdl: -------------------------------------------------------------------------------- 1 | netcdf testsimple { 2 | types: 3 | compound AAA { 4 | short s; 5 | int i; 6 | }; 7 | compound BBB { 8 | float x; 9 | double y; 10 | }; 11 | 12 | variables: 13 | AAA anA; 14 | BBB aB; 15 | int scalar; 16 | 17 | data: 18 | anA = { 55, 5280 }; 19 | aB = { 98.6, -273.3 }; 20 | scalar = 5; 21 | } 22 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testbytes.cdl: -------------------------------------------------------------------------------- 1 | netcdf testbyts { 2 | dimensions: 3 | d1 = 1; 4 | d2 = 2; 5 | 6 | variables: 7 | byte b; 8 | ubyte ub; 9 | byte ba(d1); 10 | ubyte uba(d1); 11 | string s; 12 | string s2; 13 | char c; 14 | char c2(d1); 15 | char c3(d2); 16 | 17 | data: 18 | b = 1; 19 | ub = 2; 20 | ba = 3; 21 | uba = 4; 22 | s = "5"; 23 | s2 = "67"; 24 | c = '8'; 25 | c2 = '9'; 26 | c3 = 'a', 'b'; 27 | } 28 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testempty.cdl: -------------------------------------------------------------------------------- 1 | netcdf testempty { 2 | types: 3 | opaque(5) opaque5; 4 | compound alltypes { 5 | byte b; 6 | short s; 7 | int i; 8 | float f; 9 | double d; 10 | }; 11 | 12 | dimensions: 13 | u = UNLIMITED ; // (0 currently) 14 | variables: 15 | int a(u); 16 | opaque5 b(u); 17 | alltypes c(u); 18 | alltypes d(u,u); 19 | alltypes e(u,u,u); 20 | opaque5 f(u,u); 21 | opaque5 g(u,u,u); 22 | } 23 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testgroups.cdl: -------------------------------------------------------------------------------- 1 | netcdf testgroups { 2 | dimensions: 3 | d1 = 1; 4 | variables: 5 | int datamember(d1); 6 | data: 7 | datamember = 2; 8 | group: a { 9 | dimensions: 10 | d1 = 2; 11 | variables: 12 | int datamember(d1); 13 | data: 14 | datamember = 3, 4; 15 | 16 | group: b { 17 | dimensions: 18 | d1 = 3; 19 | variables: 20 | int datamember(d1); 21 | data: 22 | datamember = 5, 6, 7; 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /netcdf/cdf/testdata/testmultidim.cdl: -------------------------------------------------------------------------------- 1 | netcdf testmultidim { 2 | dimensions: 3 | d1 = 2 ; 4 | d2 = 2 ; 5 | d3 = 3 ; 6 | d4 = 4 ; 7 | variables: 8 | ushort val(d1, d2, d3, d4); 9 | data: 10 | val = 11 | 0, 1, 2, 3, 12 | 4, 5, 6, 7, 13 | 8, 9, 10, 11, 14 | 12, 13, 14, 15, 15 | 16, 17, 18, 19, 16 | 20, 21, 22, 23, 17 | 100, 101, 102, 103, 18 | 104, 105, 106, 107, 19 | 108, 109, 110, 111, 20 | 112, 113, 114, 115, 21 | 116, 117, 118, 119, 22 | 120, 121, 122, 123; 23 | } 24 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testcompounds.cdl: -------------------------------------------------------------------------------- 1 | netcdf testcompounds { 2 | types: 3 | compound Alltypes { 4 | byte B; 5 | short S; 6 | int I; 7 | float F; 8 | double D; 9 | }; 10 | compound Includes { 11 | Alltypes A; 12 | string S; 13 | }; 14 | compound Sametypes { 15 | int A; 16 | int B; 17 | int C; 18 | }; 19 | 20 | dimensions: 21 | dim = UNLIMITED; 22 | 23 | variables: 24 | Includes v(dim); 25 | Sametypes same(dim); 26 | 27 | data: 28 | v = {{'0', 1, 2, 3.0, 4.0}, "a"}, {{'1', 2, 3, 4.0, 5.0}, "b"}; 29 | 30 | same = {0,1,2},{3,4,5}; 31 | } 32 | -------------------------------------------------------------------------------- /internal/fillvalue.go: -------------------------------------------------------------------------------- 1 | // Internal API, not to be exported 2 | package internal 3 | 4 | import ( 5 | "io" 6 | ) 7 | 8 | type FillValueReader struct { 9 | repeat []byte 10 | repeatIndex int 11 | } 12 | 13 | func NewFillValueReader(repeat []byte) io.Reader { 14 | return &FillValueReader{repeat, 0} 15 | } 16 | 17 | func (fvr *FillValueReader) Read(p []byte) (int, error) { 18 | rl := len(fvr.repeat) 19 | ri := fvr.repeatIndex 20 | z := p 21 | if ri == 0 { 22 | for i := 0; i < len(p)-rl+1; i += rl { 23 | copy(z, fvr.repeat) 24 | z = z[rl:] 25 | } 26 | } 27 | for i := 0; i < len(z); i++ { 28 | z[i] = fvr.repeat[ri%rl] 29 | ri++ 30 | } 31 | fvr.repeatIndex = ri % rl 32 | return len(p), nil 33 | } 34 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testvtypes.cdl: -------------------------------------------------------------------------------- 1 | netcdf testvtypes { 2 | types: 3 | byte enum color { 4 | RED = 0, 5 | YELLOW = 1, 6 | GREEN = 2, 7 | CYAN = 3, 8 | BLUE = 4, 9 | MAGENTA = 5 10 | }; 11 | int(*) vint; 12 | compound easy { 13 | int firstEasy; 14 | int secondEasy; 15 | }; 16 | easy(*) easyVlen; 17 | compound tricky_t { 18 | int trickyInt; 19 | easyVlen trickVlen; 20 | }; 21 | dimensions: 22 | dim = 6; 23 | 24 | variables: 25 | color cl; 26 | easy e; 27 | easyVlen ev; 28 | tricky_t t; 29 | string c; 30 | byte b; 31 | ubyte ub; 32 | short s; 33 | ushort us; 34 | int i; 35 | uint ui; 36 | int64 i64; 37 | uint64 ui64; 38 | float f; 39 | double d; 40 | } 41 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testvlen.cdl: -------------------------------------------------------------------------------- 1 | netcdf testvlen { 2 | types: 3 | int(*) vint; 4 | compound Easy { 5 | int FirstEasy; 6 | int SecondEasy; 7 | }; 8 | Easy(*) EasyVlen; 9 | compound Tricky_t { 10 | int TrickyInt; 11 | EasyVlen TrickVlen; 12 | }; 13 | dimensions: 14 | dim = 6; 15 | 16 | variables: 17 | vint v(dim); 18 | Tricky_t v:Tricky = {1, {{2, 3}, {4, 5}, {6,7}}}; 19 | 20 | vint v2(dim); 21 | vint v2:Vint = {}, {1}, {2,3}, {4,5,6}, {7,8,9,10}, {11,12,13,14,15}; 22 | 23 | Tricky_t :Tricky = {1, {{2, 3}, {4, 5}, {6,7}}}; 24 | vint :Vint = {}, {1}, {2,3}, {4,5,6}, {7,8,9,10}, {11,12,13,14,15}; 25 | 26 | data: 27 | v = {}, {1}, {2,3}, {4,5,6}, {7,8,9,10}, {11,12,13,14,15}; 28 | v2 = {11,12,13,14,15}, {7,8,9,10}, {4,5,6}, {2,3}, {1}, {}; 29 | } 30 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testattrs.cdl: -------------------------------------------------------------------------------- 1 | netcdf testattrs { 2 | types: 3 | compound alltypes { 4 | byte b; 5 | short s; 6 | int i; 7 | float f; 8 | double d; 9 | }; 10 | byte enum color { 11 | RED = 0, 12 | YELLOW = 1, 13 | GREEN = 2, 14 | CYAN = 3, 15 | BLUE = 4, 16 | MAGENTA = 5 17 | }; 18 | 19 | variables: 20 | alltypes :all = {'0', 1, 2, 3.0, 4.0}; 21 | 22 | color :col = CYAN; 23 | 24 | char :c = 'c'; 25 | 26 | string :str = "hello"; 27 | 28 | float :f32 = 1; 29 | 30 | double :f64 = 2; 31 | 32 | byte :i8 = 3; 33 | 34 | ubyte :ui8 = 4; 35 | 36 | short :i16 = 5; 37 | 38 | ushort :ui16 = 6; 39 | 40 | int :i32 = 7; 41 | 42 | uint :ui32 = 8; 43 | 44 | int64 :i64 = 9; 45 | 46 | uint64 :ui64 = 10; 47 | data: 48 | group: subgroup { 49 | :groupattr = "foo"; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testenum.cdl: -------------------------------------------------------------------------------- 1 | netcdf testenum { 2 | types: 3 | byte enum color { 4 | RED = 0, 5 | YELLOW = 1, 6 | GREEN = 2, 7 | CYAN = 3, 8 | BLUE = 4, 9 | MAGENTA = 5 10 | }; 11 | ushort enum color2 { 12 | BLACK = 0, 13 | WHITE = 1 14 | }; 15 | int64 enum junk { 16 | FIRST = 1L, 17 | SECOND = 2, 18 | THIRD = 3, 19 | FOURTH = 4, 20 | FIFTH = 5, 21 | SIXTH = 6 22 | }; 23 | int enum junk2 { 24 | SEVENTH = 7, 25 | EIGHTH = 8 26 | }; 27 | 28 | dimensions: 29 | dim = 6; 30 | 31 | variables: 32 | color nodim; 33 | color c(dim); 34 | junk j(dim); 35 | color2 c2; 36 | junk2 j2; 37 | 38 | data: 39 | nodim = GREEN; 40 | 41 | c = RED, YELLOW, GREEN, CYAN, BLUE, MAGENTA; 42 | 43 | c2 = BLACK; 44 | 45 | j = FIRST, SECOND, THIRD, FOURTH, FIFTH, SIXTH; 46 | 47 | j2 = SEVENTH; 48 | } 49 | -------------------------------------------------------------------------------- /internal/validname_test.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import "testing" 4 | 5 | func TestGood(t *testing.T) { 6 | var goodStrings = []string{ 7 | "_", 8 | "a", 9 | "1", 10 | "0°", 11 | "byter", 12 | } 13 | for i := range goodStrings { 14 | if !IsValidNetCDFName(goodStrings[i]) { 15 | t.Error("name should be good", goodStrings[i]) 16 | return 17 | } 18 | } 19 | } 20 | 21 | func TestBad(t *testing.T) { 22 | var badStrings = []string{ 23 | "_ ", 24 | "/", 25 | "no/good", 26 | "\ta ", 27 | "1\t", 28 | "°", 29 | "°C", 30 | "\x08", 31 | "byte", "char", "string", 32 | "short", "int", "int64", "int64", 33 | "ushort", "uint", "uint64", "uint64", 34 | "float", "double", 35 | } 36 | for i := range badStrings { 37 | if IsValidNetCDFName(badStrings[i]) { 38 | t.Error("name should be bad", badStrings[i]) 39 | return 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testfills.cdl: -------------------------------------------------------------------------------- 1 | netcdf testfills { 2 | dimensions: 3 | dim = 1; 4 | d1 = 2; 5 | d2 = 2; 6 | 7 | variables: 8 | string str; 9 | string strx1(dim); 10 | string strx2(d1, d2); 11 | float f32; 12 | float f32x1(dim); 13 | float f32x2(d1, d2); 14 | double f64; 15 | double f64x1(dim); 16 | double f64x2(d1, d2); 17 | 18 | byte i8; 19 | byte i8x1(dim); 20 | byte i8x2(d1, d2); 21 | 22 | ubyte ui8; 23 | ubyte ui8x1(dim); 24 | ubyte ui8x2(d1, d2); 25 | 26 | short i16; 27 | short i16x1(dim); 28 | short i16x2(d1, d2); 29 | 30 | ushort ui16; 31 | ushort ui16x1(dim); 32 | ushort ui16x2(d1, d2); 33 | 34 | int i32; 35 | int i32x1(dim); 36 | int i32x2(d1, d2); 37 | 38 | uint ui32; 39 | uint ui32x1(dim); 40 | uint ui32x2(d1, d2); 41 | 42 | int64 i64; 43 | int64 i64x1(dim); 44 | int64 i64x2(d1, d2); 45 | 46 | uint64 ui64; 47 | uint64 ui64x1(dim); 48 | uint64 ui64x2(d1, d2); 49 | } 50 | -------------------------------------------------------------------------------- /internal/validname.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "regexp" 5 | ) 6 | 7 | const ( 8 | // A valid name must start with a letter, digit or underscore. 9 | // It may contain any character after that except control and slash. 10 | pattern = `^[\pL\pN_][^\pC/]*$` 11 | // It may not end with a whitespace character, or be a reserved word. 12 | antiPattern = `(\pZ|^(u?byte|char|string|u?short|u?int|u?int64|uint64|float|double|enum|opaque|compound))$` 13 | ) 14 | 15 | var ( 16 | re *regexp.Regexp 17 | antiRe *regexp.Regexp 18 | ) 19 | 20 | func init() { 21 | var err error 22 | re, err = regexp.Compile(pattern) 23 | if err != nil { 24 | panic(err) 25 | } 26 | antiRe, err = regexp.Compile(antiPattern) 27 | if err != nil { 28 | panic(err) 29 | } 30 | } 31 | 32 | // IsValidNetCDFName returns true if name is a valid NetCDF name. 33 | func IsValidNetCDFName(name string) bool { 34 | return re.MatchString(name) && !antiRe.MatchString(name) 35 | } 36 | -------------------------------------------------------------------------------- /netcdf/hdf5/amap.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | import ( 4 | "github.com/batchatco/go-native-netcdf/netcdf/util" 5 | ) 6 | 7 | // Adds callbacks to the standard OrderedMap so we can print 8 | // out more complex types than the default (structs, enums, etc.) 9 | type callback struct { 10 | h5 *HDF5 11 | om *util.OrderedMap 12 | } 13 | 14 | // CDL callback (regular) 15 | func (c *callback) regCallback(key string) (string, bool) { 16 | ret := c.h5.findGlobalAttrType(key) 17 | return ret, ret != "" 18 | } 19 | 20 | func (c *callback) goCallback(key string) (string, bool) { 21 | ret := c.h5.findGlobalAttrGoType(key) 22 | return ret, ret != "" 23 | } 24 | 25 | func newTypedAttributeMap(h5 *HDF5, keys []string, values map[string]interface{}) (*util.OrderedMap, error) { 26 | om, err := util.NewOrderedMap(keys, values) 27 | assertError(err == nil, err, "creating ordered map") 28 | if h5 != nil { 29 | c := callback{h5, om} 30 | om.SetTypeCallbacks(c.regCallback, c.goCallback) 31 | } 32 | return om, nil 33 | } 34 | -------------------------------------------------------------------------------- /netcdf/netcdf4_test.go: -------------------------------------------------------------------------------- 1 | package netcdf 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | var filenames = []string{"empty", "bogus", "cdf.nc", "hdf5.nc"} 9 | 10 | var errs = []error{ErrUnknown, ErrUnknown, nil, nil} 11 | 12 | func TestNew(t *testing.T) { 13 | for i, name := range filenames { 14 | f, err := os.Open("testdata/" + name) 15 | if err != nil { 16 | t.Error("os.Open", name, err) 17 | } else { 18 | g, err := New(f) 19 | if err != errs[i] { 20 | t.Error("New", name, "expected", errs[i], "got", err) 21 | } 22 | if g != nil { 23 | g.Close() 24 | } else { 25 | f.Close() 26 | } 27 | } 28 | } 29 | } 30 | 31 | func TestOpen(t *testing.T) { 32 | _, err := Open("testdata/bogus") 33 | if err != ErrUnknown { 34 | t.Error("expected error") 35 | } 36 | for i, name := range filenames { 37 | g, err := Open("testdata/" + name) 38 | if err != errs[i] { 39 | t.Error("Open", name, "expected", errs[i], "got", err) 40 | } 41 | if g != nil { 42 | g.Close() 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /netcdf/hdf5/segment.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | import ( 4 | "io" 5 | "sort" 6 | ) 7 | 8 | // segments are a bunch of discontiguous locations and lengths in 9 | // file to read data for an object. 10 | type segment struct { 11 | offset uint64 12 | length uint64 13 | r io.Reader // reader to use for this segment 14 | } 15 | 16 | // object representing all the segments 17 | type segments struct { 18 | segs []*segment 19 | } 20 | 21 | func newSegments() *segments { 22 | return &segments{make([]*segment, 0)} 23 | } 24 | 25 | // Standard sorting functions 26 | func (s *segments) Len() int { 27 | return len(s.segs) 28 | } 29 | 30 | func (s *segments) Swap(i, j int) { 31 | s.segs[i], s.segs[j] = s.segs[j], s.segs[i] 32 | } 33 | 34 | func (s *segments) Less(i, j int) bool { 35 | return s.segs[i].offset <= s.segs[j].offset 36 | } 37 | 38 | // other methods 39 | func (s *segments) get(i int) *segment { 40 | return s.segs[i] 41 | } 42 | 43 | func (s *segments) sort() { 44 | sort.Sort(s) 45 | } 46 | 47 | func (s *segments) append(thisSeg *segment) { 48 | s.segs = append(s.segs, thisSeg) 49 | } 50 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018-2020 Bradley Arthur Taylor 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /netcdf/cdf/perf_test.go: -------------------------------------------------------------------------------- 1 | package cdf 2 | 3 | // 1. Need test of invalid fill value (slice instead of scalar) 4 | // 2. Need test of V1 file. 5 | // 3. Need test of null in name. 6 | // 4. Need test of too many dimensions. 7 | 8 | import ( 9 | "testing" 10 | ) 11 | 12 | func common(b testing.TB, num int, slow convertType) { 13 | b.Helper() 14 | fileName := "testdata/solarforcing_small.nc" 15 | for i := 0; i < num; i++ { 16 | nc, err := Open(fileName) 17 | nc.(*CDF).slowConvert = slow 18 | if err != nil { 19 | b.Error(err) 20 | return 21 | } 22 | for _, n := range nc.ListVariables() { 23 | v, err := nc.GetVariable(n) 24 | if err != nil { 25 | b.Error("var", n, "missing", err) 26 | return 27 | } 28 | _ = v.Dimensions 29 | for _, k := range v.Attributes.Keys() { 30 | _, has := v.Attributes.Get(k) 31 | if !has { 32 | b.Error("Missing attr", k) 33 | return 34 | } 35 | } 36 | _ = v.Values 37 | } 38 | for _, n := range nc.Attributes().Keys() { 39 | _, has := nc.Attributes().Get(n) 40 | if !has { 41 | b.Error("Missing global attr", n) 42 | return 43 | } 44 | } 45 | nc.Close() 46 | } 47 | } 48 | 49 | func BenchmarkFast(b *testing.B) { 50 | common(b, 1000, fast) 51 | } 52 | 53 | func BenchmarkSlow(b *testing.B) { 54 | common(b, 1000, slow) 55 | } 56 | 57 | func TestPerf(t *testing.T) { 58 | common(t, 1, fast) 59 | } 60 | 61 | func TestSlow(t *testing.T) { 62 | common(t, 1, slow) 63 | } 64 | -------------------------------------------------------------------------------- /internal/slicer.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "github.com/batchatco/go-native-netcdf/netcdf/api" 5 | "github.com/batchatco/go-thrower" 6 | ) 7 | 8 | type slice struct { 9 | getSlice func(begin, end int64) (interface{}, error) 10 | length int64 11 | dimNames []string 12 | attrs api.AttributeMap 13 | cdlType string 14 | goType string 15 | } 16 | 17 | func (sl *slice) GetSlice(begin, end int64) (slice interface{}, err error) { 18 | defer thrower.RecoverError(&err) 19 | slice, err = sl.getSlice(begin, end) 20 | return slice, err 21 | } 22 | 23 | func (sl *slice) Values() (values interface{}, err error) { 24 | defer thrower.RecoverError(&err) 25 | values, err = sl.getSlice(0, sl.length) 26 | return values, err 27 | } 28 | 29 | func (sl *slice) Len() int64 { 30 | return sl.length 31 | } 32 | 33 | func (sl *slice) Attributes() api.AttributeMap { 34 | return sl.attrs 35 | } 36 | 37 | func (sl *slice) Dimensions() []string { 38 | return sl.dimNames 39 | } 40 | 41 | func (sl *slice) Type() string { 42 | return sl.cdlType 43 | } 44 | 45 | func (sl *slice) GoType() string { 46 | return sl.goType 47 | } 48 | 49 | func NewSlicer(getSlice func(begin, end int64) (interface{}, error), 50 | length int64, dimNames []string, attributes api.AttributeMap, 51 | cdlType string, goType string) api.VarGetter { 52 | return &slice{ 53 | getSlice: getSlice, 54 | length: length, 55 | dimNames: dimNames, 56 | attrs: attributes, 57 | cdlType: cdlType, 58 | goType: goType, 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /netcdf/netcdf4.go: -------------------------------------------------------------------------------- 1 | // Opens NetCDF files, regardless of type (CDF or HDF5) 2 | package netcdf 3 | 4 | import ( 5 | "errors" 6 | "io" 7 | "os" 8 | 9 | "github.com/batchatco/go-native-netcdf/netcdf/api" 10 | "github.com/batchatco/go-native-netcdf/netcdf/cdf" 11 | "github.com/batchatco/go-native-netcdf/netcdf/hdf5" 12 | ) 13 | 14 | const ( 15 | magicCDF = 'C' 16 | magicHDF = 0x89 17 | ) 18 | 19 | var ErrUnknown = errors.New("not a CDF or HDF5 file") 20 | 21 | // Open opens a NetCDF4 file by name 22 | func Open(fname string) (api.Group, error) { 23 | file, err := os.Open(fname) 24 | if err != nil { 25 | return nil, err 26 | } 27 | defer file.Close() 28 | 29 | kind, err := getKind(file) 30 | if err != nil { 31 | return nil, ErrUnknown 32 | } 33 | var g api.Group 34 | err = ErrUnknown 35 | switch kind { 36 | case magicCDF: 37 | g, err = cdf.Open(fname) 38 | case magicHDF: 39 | g, err = hdf5.Open(fname) 40 | } 41 | return g, err 42 | } 43 | 44 | // New is like Open, but takes an opened file instead of a filename. 45 | // If New returns no error, it has taken ownership of the file. Otherwise, it 46 | // is up to the caller to close the file. 47 | func New(file api.ReadSeekerCloser) (api.Group, error) { 48 | kind, err := getKind(file) 49 | if err != nil { 50 | return nil, ErrUnknown 51 | } 52 | var g api.Group 53 | err = ErrUnknown 54 | switch kind { 55 | case magicCDF: 56 | g, err = cdf.New(file) 57 | case magicHDF: 58 | g, err = hdf5.New(file) 59 | } 60 | return g, err 61 | } 62 | 63 | func getKind(file io.ReadSeeker) (byte, error) { 64 | var b [1]byte 65 | n, err := file.Read(b[:]) 66 | if n == 0 { 67 | return 0, err 68 | } 69 | _, err = file.Seek(0, io.SeekStart) 70 | return b[0], err 71 | } 72 | -------------------------------------------------------------------------------- /netcdf/hdf5/time.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | import ( 4 | "encoding/binary" 5 | "io" 6 | 7 | "github.com/batchatco/go-thrower" 8 | ) 9 | 10 | type timeManagerType struct{} 11 | 12 | var ( 13 | timeManager = timeManagerType{} 14 | _ typeManager = timeManager 15 | ) 16 | 17 | func (timeManagerType) cdlTypeString(sh sigHelper, name string, attr *attribute, origNames map[string]bool) string { 18 | // Not NetCDF 19 | fail("time") 20 | return "" 21 | } 22 | 23 | func (timeManagerType) goTypeString(sh sigHelper, name string, attr *attribute, origNames map[string]bool) string { 24 | // Not NetCDF 25 | fail("time") 26 | return "" 27 | } 28 | 29 | func (timeManagerType) alloc(hr heapReader, c caster, bf io.Reader, attr *attribute, dimensions []uint64) interface{} { 30 | fail("time") 31 | return nil 32 | } 33 | 34 | func (timeManagerType) defaultFillValue(obj *object, objFillValue []byte, undefinedFillValue bool) []byte { 35 | fail("time") 36 | return objFillValue 37 | } 38 | 39 | func (timeManagerType) parse(hr heapReader, c caster, attr *attribute, bitFields uint32, bf remReader, df remReader) { 40 | // This is disabled by default. Time is an obsolete type. 41 | if parseTime { 42 | logger.Info("time, len(data)=", df.Rem()) 43 | var endian binary.ByteOrder 44 | if bitFields == 0 { 45 | endian = binary.LittleEndian 46 | logger.Info("time little-endian") 47 | } else { 48 | endian = binary.BigEndian 49 | logger.Infof("time big-endian") 50 | } 51 | var bp int16 52 | err := binary.Read(bf, endian, &bp) 53 | thrower.ThrowIfError(err) 54 | logger.Info("time bit precision=", bp) 55 | if df.Rem() > 0 { 56 | fail("time") 57 | } 58 | } else { 59 | logger.Fatal("time type is obsolete") 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /netcdf/hdf5/slicer_test.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/batchatco/go-native-netcdf/netcdf/api" 8 | ) 9 | 10 | func TestSlicer(t *testing.T) { 11 | genName := ncGen(t, "testslicer") 12 | if genName == "" { 13 | t.Error(errorNcGen) 14 | return 15 | } 16 | defer os.Remove(genName) 17 | tidValues := [][]int8{ 18 | {0, 1, 2, 3, 4}, 19 | {5, 6, 6, 8, 9}, 20 | {10, 11, 12, 13, 14}, 21 | {15, 16, 17, 18, 19}, 22 | } 23 | contents := keyValList{ 24 | {"tid", "", api.Variable{ 25 | Values: tidValues, 26 | Attributes: nilMap, 27 | Dimensions: []string{"lat", "lon"}, 28 | }}, 29 | } 30 | 31 | // Now read and verify it 32 | nc, err := Open(genName) 33 | if err != nil { 34 | t.Error(err) 35 | return 36 | } 37 | defer nc.Close() 38 | 39 | slicer, err := nc.GetVarGetter("tid") 40 | if err != nil { 41 | t.Error(err) 42 | return 43 | } 44 | baseType := slicer.Type() 45 | // Grab slices of various sizes (including 0 and 4) and see if they match expected. 46 | for sliceSize := 0; sliceSize <= 4; sliceSize++ { 47 | for i := 0; i < (4 - sliceSize); i++ { 48 | slice, err := slicer.GetSlice(int64(i), int64(i+sliceSize)) 49 | if err != nil { 50 | t.Error(err) 51 | return 52 | } 53 | got := api.Variable{ 54 | Values: slice, 55 | Dimensions: slicer.Dimensions(), 56 | Attributes: slicer.Attributes(), 57 | } 58 | tid := contents[0] 59 | exp := keyValList{ 60 | {tid.name, "byte", api.Variable{ 61 | Values: tidValues[i : i+sliceSize], 62 | Dimensions: tid.val.Dimensions, 63 | Attributes: tid.val.Attributes, 64 | }}, 65 | } 66 | if !exp.check(t, "tid", baseType, got, true) { 67 | t.Error("fail") 68 | } 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testfills2.cdl: -------------------------------------------------------------------------------- 1 | netcdf testfills2 { 2 | dimensions: 3 | dim = 1; 4 | d1 = 2; 5 | d2 = 2; 6 | variables: 7 | string str; 8 | string str:_FillValue = "#"; 9 | string strx1(dim); 10 | string strx1:_FillValue = "#"; 11 | string strx2(d1, d2); 12 | string strx2:_FillValue = "#"; 13 | float f32; 14 | f32:_FillValue = 0; 15 | float f32x1(dim); 16 | f32x1:_FillValue = 0; 17 | float f32x2(d1, d2); 18 | f32x2:_FillValue = 0; 19 | double f64; 20 | f64:_FillValue = 0; 21 | double f64x1(dim); 22 | f64x1:_FillValue = 0; 23 | double f64x2(d1, d2); 24 | f64x2:_FillValue = 0; 25 | byte i8; 26 | i8:_FillValue = 0; 27 | byte i8x1(dim); 28 | i8x1:_FillValue = 0; 29 | byte i8x2(d1, d2); 30 | i8x2:_FillValue = 0; 31 | ubyte ui8; 32 | ui8:_FillValue = 0; 33 | ubyte ui8x1(dim); 34 | ui8x1:_FillValue = 0; 35 | ubyte ui8x2(d1, d2); 36 | ui8x2:_FillValue = 0; 37 | short i16; 38 | i16:_FillValue = 0; 39 | short i16x1(dim); 40 | i16x1:_FillValue = 0; 41 | short i16x2(d1, d2); 42 | i16x2:_FillValue = 0; 43 | ushort ui16; 44 | ui16:_FillValue = 0; 45 | ushort ui16x1(dim); 46 | ui16x1:_FillValue = 0; 47 | ushort ui16x2(d1, d2); 48 | ui16x2:_FillValue = 0; 49 | int i32; 50 | i32:_FillValue = 0; 51 | int i32x1(dim); 52 | i32x1:_FillValue = 0; 53 | int i32x2(d1, d2); 54 | i32x2:_FillValue = 0; 55 | uint ui32; 56 | ui32:_FillValue = 0; 57 | uint ui32x1(dim); 58 | ui32x1:_FillValue = 0; 59 | uint ui32x2(d1, d2); 60 | ui32x2:_FillValue = 0; 61 | int64 i64; 62 | i64:_FillValue = 0; 63 | int64 i64x1(dim); 64 | i64x1:_FillValue = 0; 65 | int64 i64x2(d1, d2); 66 | i64x2:_FillValue = 0; 67 | uint64 ui64; 68 | ui64:_FillValue = 0; 69 | uint64 ui64x1(dim); 70 | ui64x1:_FillValue = 0; 71 | uint64 ui64x2(d1, d2); 72 | ui64x2:_FillValue = 0; 73 | } 74 | -------------------------------------------------------------------------------- /netcdf/hdf5/layout_test.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/exec" 7 | "strings" 8 | "testing" 9 | ) 10 | 11 | func TestLayout(t *testing.T) { 12 | fileNameNoExt := "testlayout" 13 | genName := ncGen(t, fileNameNoExt) 14 | if genName == "" { 15 | t.Error(errorNcGen) 16 | return 17 | } 18 | defer os.Remove(genName) 19 | chunkedName := "testdata/" + fileNameNoExt + "_chunk.nc" 20 | cmdString := []string{"-l", "a:CHUNK=7x6", genName, chunkedName} 21 | cmd := exec.Command(h5RepackBinary, cmdString...) 22 | err := cmd.Run() 23 | if err != nil { 24 | s := strings.Join(cmdString, " ") 25 | t.Error(h5RepackBinary, s, ":", err) 26 | return 27 | } 28 | defer os.Remove(chunkedName) 29 | 30 | nc, err := Open(genName) 31 | if err != nil { 32 | t.Error("open", genName) 33 | return 34 | } 35 | defer nc.Close() 36 | vr, err := nc.GetVariable("a") 37 | if err != nil { 38 | t.Error("get gen var -- not found") 39 | return 40 | } 41 | vals, ok := vr.Values.([][]int32) 42 | if !ok { 43 | t.Error("get gen var -- wrong type") 44 | return 45 | } 46 | gnc, err := Open(chunkedName) 47 | if err != nil { 48 | t.Error("open", chunkedName) 49 | return 50 | } 51 | defer gnc.Close() 52 | gvr, err := gnc.GetVariable("a") 53 | if err != nil { 54 | t.Error("get chunked var -- not found") 55 | return 56 | } 57 | gvals, ok := gvr.Values.([][]int32) 58 | if !ok { 59 | t.Error("get chunked var -- wrong type") 60 | return 61 | } 62 | t.Log("[row,col] src,dst\n") 63 | for i := 0; i < len(vals); i++ { 64 | for j := 0; j < len(vals[i]); j++ { 65 | if vals[i][j] != gvals[i][j] { 66 | t.Error("value mismatch at", i, j, vals[i][j], gvals[i][j]) 67 | for k := 0; k < len(vals); k++ { 68 | fmt.Print(gvals[k], " ") 69 | } 70 | fmt.Println("") 71 | return 72 | } 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /netcdf/hdf5/amap_test.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | func TestGoType(t *testing.T) { 9 | fileName := "testattrtypes" 10 | genName := ncGen(t, fileName) 11 | if genName == "" { 12 | t.Error("Error opening", fileName, ":", errorNcGen) 13 | return 14 | } 15 | defer os.Remove(genName) 16 | nc, err := Open(genName) 17 | if err != nil { 18 | t.Error(err) 19 | return 20 | } 21 | defer nc.Close() 22 | om := nc.Attributes() 23 | rightTypes := map[string]string{ 24 | "i": "int32", 25 | "f": "float32", 26 | "d": "float64", 27 | "s": "string", 28 | "i1": "[]int32", 29 | "f1": "[]float32", 30 | "d1": "[]float64", 31 | "s1": "[]string", 32 | } 33 | for v, exp := range rightTypes { 34 | got, has := om.GetGoType(v) 35 | if !has { 36 | t.Errorf("Var %s is missing", v) 37 | continue 38 | } 39 | if got != exp { 40 | t.Errorf("wrong type for %s: got=%s exp=%s", v, got, exp) 41 | continue 42 | } 43 | } 44 | } 45 | 46 | func TestType(t *testing.T) { 47 | fileName := "testattrtypes" 48 | genName := ncGen(t, fileName) 49 | if genName == "" { 50 | t.Error("Error opening", fileName, ":", errorNcGen) 51 | return 52 | } 53 | defer os.Remove(genName) 54 | nc, err := Open(genName) 55 | if err != nil { 56 | t.Error(err) 57 | return 58 | } 59 | defer nc.Close() 60 | om := nc.Attributes() 61 | rightTypes := map[string]string{ 62 | "i": "int", 63 | "f": "float", 64 | "d": "double", 65 | "s": "string", 66 | "i1": "int(*)", 67 | "f1": "float(*)", 68 | "d1": "double(*)", 69 | "s1": "string(*)", 70 | } 71 | for v, exp := range rightTypes { 72 | got, has := om.GetType(v) 73 | if !has { 74 | t.Errorf("Var %s is missing", v) 75 | continue 76 | } 77 | if got != exp { 78 | t.Errorf("wrong type for %s: got=%s exp=%s", v, got, exp) 79 | continue 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testtypes.cdl: -------------------------------------------------------------------------------- 1 | netcdf testtypes { 2 | dimensions: 3 | dim = 1; 4 | d1 = 2; 5 | d2 = 2; 6 | 7 | variables: 8 | string str; 9 | string strx1(dim); 10 | string strx2(d1, d2); 11 | float f32; 12 | float f32x1(dim); 13 | float f32x2(d1, d2); 14 | double f64; 15 | double f64x1(dim); 16 | double f64x2(d1, d2); 17 | 18 | byte i8; 19 | byte i8x1(dim); 20 | byte i8x2(d1, d2); 21 | 22 | ubyte ui8; 23 | ubyte ui8x1(dim); 24 | ubyte ui8x2(d1, d2); 25 | 26 | short i16; 27 | short i16x1(dim); 28 | short i16x2(d1, d2); 29 | 30 | ushort ui16; 31 | ushort ui16x1(dim); 32 | ushort ui16x2(d1, d2); 33 | 34 | int i32; 35 | int i32x1(dim); 36 | int i32x2(d1, d2); 37 | 38 | uint ui32; 39 | uint ui32x1(dim); 40 | uint ui32x2(d1, d2); 41 | 42 | int64 i64; 43 | int64 i64x1(dim); 44 | int64 i64x2(d1, d2); 45 | 46 | uint64 ui64; 47 | uint64 ui64x1(dim); 48 | uint64 ui64x2(d1, d2); 49 | 50 | data: 51 | str = "a"; 52 | strx1 = "a"; 53 | strx2 = "ab", "cd", "ef", "gh"; 54 | f32 = -10.1; 55 | f32x1 = -10.1; 56 | f32x2 = -10.1, 10.1, -20.2, 20.2; 57 | 58 | f64 = -10.1; 59 | f64x1 = -10.1; 60 | f64x2 = -10.1, 10.1, -20.2, 20.2; 61 | 62 | i8 = -10; 63 | i8x1 = -10; 64 | i8x2 = -10, 10, -20, 20; 65 | 66 | ui8 = 10; 67 | ui8x1 = 10; 68 | ui8x2 = 10, 20, 20, 30; 69 | 70 | i16 = -10000; 71 | i16x1 = -10000; 72 | i16x2 = -10000, 10000, -20000, 20000; 73 | 74 | ui16 = 10000; 75 | ui16x1 = 10000; 76 | ui16x2 = 10000, 20000, 20000, 30000; 77 | 78 | i32 = -10000000; 79 | i32x1 = -10000000; 80 | i32x2 = -10000000, 10000000, -20000000, 20000000; 81 | 82 | ui32 = 10000000; 83 | ui32x1 = 10000000; 84 | ui32x2 = 10000000, 20000000, 20000000, 30000000; 85 | 86 | i64 = -10000000000; 87 | i64x1 = -10000000000; 88 | i64x2 = -10000000000, 10000000000, -20000000000, 20000000000; 89 | 90 | ui64 = 10000000000; 91 | ui64x1 = 10000000000; 92 | ui64x2 = 10000000000, 20000000000, 20000000000, 30000000000; 93 | } 94 | -------------------------------------------------------------------------------- /netcdf/hdf5/checksum.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | // Hash function used by HDF5 for checksums (Jenkins lookup3 hash function) 4 | 5 | func rot(x uint32, k uint32) uint32 { 6 | return x<>(32-k) 7 | } 8 | 9 | func mix(a, b, c *uint32) { 10 | *a -= *c 11 | *a ^= rot(*c, 4) 12 | *c += *b 13 | 14 | *b -= *a 15 | *b ^= rot(*a, 6) 16 | *a += *c 17 | 18 | *c -= *b 19 | *c ^= rot(*b, 8) 20 | *b += *a 21 | 22 | *a -= *c 23 | *a ^= rot(*c, 16) 24 | *c += *b 25 | 26 | *b -= *a 27 | *b ^= rot(*a, 19) 28 | *a += *c 29 | 30 | *c -= *b 31 | *c ^= rot(*b, 4) 32 | *b += *a 33 | } 34 | 35 | func final(a, b, c *uint32) { 36 | *c ^= *b 37 | *c -= rot(*b, 14) 38 | 39 | *a ^= *c 40 | *a -= rot(*c, 11) 41 | 42 | *b ^= *a 43 | *b -= rot(*a, 25) 44 | 45 | *c ^= *b 46 | *c -= rot(*b, 16) 47 | 48 | *a ^= *c 49 | *a -= rot(*c, 4) 50 | 51 | *b ^= *a 52 | *b -= rot(*a, 14) 53 | 54 | *c ^= *b 55 | *c -= rot(*b, 24) 56 | } 57 | 58 | // Computes the HDF5 checksum on 'vals'. nBytes is the number of signficant bytes in 59 | // 'vals', or 0 to compute it based upon the length of 'vals' (4*len(vals)). 60 | func hashInts(vals []uint32, nBytes uint32) uint32 { 61 | if nBytes == 0 { 62 | nBytes = uint32(4 * len(vals)) 63 | } 64 | a := 0xdeadbeef + nBytes 65 | b := a 66 | c := a 67 | for nBytes > 12 { 68 | a += vals[0] 69 | b += vals[1] 70 | c += vals[2] 71 | mix(&a, &b, &c) 72 | vals = vals[3:] 73 | nBytes -= 12 74 | } 75 | switch nBytes { 76 | case 12, 11, 10, 9: 77 | a += vals[0] 78 | b += vals[1] 79 | c += vals[2] 80 | case 8, 7, 6, 5: 81 | a += vals[0] 82 | b += vals[1] 83 | case 4, 3, 2, 1: 84 | a += vals[0] 85 | case 0: 86 | return c 87 | } 88 | final(&a, &b, &c) 89 | return c 90 | } 91 | 92 | func fletcher32(vals []uint16) uint32 { 93 | sum1 := uint32(0) 94 | sum2 := uint32(0) 95 | for _, v := range vals { 96 | sum1 = (sum1 + uint32(v)) % 65535 97 | sum2 = (sum2 + sum1) % 65535 98 | } 99 | return (uint32(sum2) << 16) | uint32(sum1) 100 | } 101 | -------------------------------------------------------------------------------- /netcdf/hdf5/testdata/testlayout.cdl: -------------------------------------------------------------------------------- 1 | netcdf testlayout { 2 | dimensions: 3 | LON = 20; 4 | LAT = 20; 5 | variables: 6 | int a(LAT, LON) ; 7 | data: 8 | 9 | a = 10 | 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 11 | 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 12 | 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 13 | 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 14 | 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 15 | 16 | 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 17 | 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 18 | 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 19 | 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 20 | 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 21 | 22 | 23 | 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 24 | 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 25 | 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 26 | 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 27 | 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 28 | 29 | 30 | 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 31 | 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 32 | 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 33 | 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 34 | 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399 35 | 36 | 37 | ; 38 | } 39 | -------------------------------------------------------------------------------- /netcdf/hdf5/bitfield.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | import ( 4 | "encoding/binary" 5 | "io" 6 | 7 | "github.com/batchatco/go-thrower" 8 | ) 9 | 10 | type bitfieldManagerType struct{} 11 | 12 | var ( 13 | bitfieldManager = bitfieldManagerType{} 14 | _ typeManager = bitfieldManager 15 | ) 16 | 17 | func (bitfieldManagerType) cdlTypeString(sh sigHelper, name string, attr *attribute, origNames map[string]bool) string { 18 | // Not NetCDF 19 | return "uchar" // same as uint8 20 | } 21 | 22 | func (bitfieldManagerType) goTypeString(sh sigHelper, typeName string, attr *attribute, origNames map[string]bool) string { 23 | // Not NetCDF 24 | return "uint8" // bitfield same as uint8 25 | } 26 | 27 | func (bitfieldManagerType) alloc(hr heapReader, c caster, bf io.Reader, attr *attribute, 28 | dimensions []uint64) interface{} { 29 | values := allocInt8s(bf, dimensions, false, nil) 30 | return values 31 | } 32 | 33 | func (bitfieldManagerType) defaultFillValue(obj *object, objFillValue []byte, undefinedFillValue bool) []byte { 34 | return objFillValue 35 | } 36 | 37 | func (bitfieldManagerType) parse(hr heapReader, c caster, attr *attribute, bitFields uint32, bf remReader, df remReader) { 38 | endian := hasFlag8(uint8(bitFields), 0) 39 | switch endian { 40 | case false: 41 | attr.endian = binary.LittleEndian 42 | case true: 43 | attr.endian = binary.BigEndian 44 | } 45 | loPad := hasFlag8(uint8(bitFields), 1) 46 | assert(!loPad, "low pad not supported") 47 | hiPad := hasFlag8(uint8(bitFields), 2) 48 | assert(!hiPad, "high pad not supported") 49 | bitOffset := read16(bf) 50 | checkVal(0, bitOffset, "bit offset must be zero") 51 | bitPrecision := read16(bf) 52 | logger.Infof("BitField offset %d, precision %d", bitOffset, bitPrecision) 53 | if df == nil || df.Rem() == 0 { 54 | logger.Infof("no data") 55 | return 56 | } 57 | logger.Info("bitfield rem: ", df.Rem()) 58 | if !allowBitfields { 59 | b := make([]byte, df.Rem()) 60 | read(df, b) 61 | logger.Infof("bitfield value: %#x", b) 62 | logger.Infof("Bitfields ignored") 63 | thrower.Throw(ErrBitfield) 64 | } 65 | if df.Rem() >= int64(attr.length) { 66 | attr.df = newResetReaderSave(df, df.Rem()) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /netcdf/cdf/slicer_test.go: -------------------------------------------------------------------------------- 1 | package cdf 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/batchatco/go-native-netcdf/netcdf/api" 8 | ) 9 | 10 | func TestSlicer(t *testing.T) { 11 | // Create the file data 12 | fileName := "testdata/testslicer.nc" 13 | _ = os.Remove(fileName) 14 | cw, err := OpenWriter(fileName) 15 | defer os.Remove(fileName) 16 | defer closeCW(t, &cw) // can be called twice 17 | if err != nil { 18 | t.Error(err) 19 | return 20 | } 21 | tidValues := [][]uint8{ 22 | {0, 1, 2, 3, 4}, 23 | {5, 6, 6, 8, 9}, 24 | {10, 11, 12, 13, 14}, 25 | {15, 16, 17, 18, 19}} 26 | contents := keyValList{ 27 | {"tid", "ubyte", "uint8", api.Variable{ 28 | Values: tidValues, 29 | Attributes: nilMap, 30 | Dimensions: []string{"lat", "lon"}}}, 31 | } 32 | for i := range contents { 33 | err := cw.AddVar(contents[i].name, contents[i].val) 34 | if err != nil { 35 | t.Error(err) 36 | return 37 | } 38 | } 39 | closeCW(t, &cw) // this writes out the data 40 | 41 | // Now read and verify it 42 | nc, err := Open(fileName) 43 | if err != nil { 44 | t.Error(err) 45 | return 46 | } 47 | defer nc.Close() 48 | 49 | slicer, err := nc.GetVarGetter("tid") 50 | if err != nil { 51 | t.Error(err) 52 | return 53 | } 54 | 55 | varType := slicer.Type() 56 | if varType != "ubyte" { 57 | t.Error("Type() is wrong", varType) 58 | } 59 | 60 | varType = slicer.GoType() 61 | if varType != "uint8" { 62 | t.Error("GoType() is wrong", varType) 63 | } 64 | 65 | // Grab slices of various sizes (including 0 and 4) and see if they match expected. 66 | for sliceSize := 0; sliceSize <= 4; sliceSize++ { 67 | for i := 0; i < (4 - sliceSize); i++ { 68 | slice, err := slicer.GetSlice(int64(i), int64(i+sliceSize)) 69 | if err != nil { 70 | t.Error(err) 71 | return 72 | } 73 | tid := contents[0] 74 | exp := keyValList{ 75 | {tid.name, "ubyte", "uint8", api.Variable{ 76 | Values: tidValues[i : i+sliceSize], 77 | Dimensions: tid.val.Dimensions, 78 | Attributes: tid.val.Attributes}}, 79 | } 80 | if !exp.check(t, "tid", slicer, slice) { 81 | t.Error("value mismatch", "sliceSize=", sliceSize) 82 | return 83 | } 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /netcdf/hdf5/string.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | import ( 4 | "io" 5 | "reflect" 6 | ) 7 | 8 | type stringManagerType struct{} 9 | 10 | var ( 11 | stringManager = stringManagerType{} 12 | _ typeManager = stringManager 13 | ) 14 | 15 | func (stringManagerType) cdlTypeString(sh sigHelper, name string, attr *attribute, origNames map[string]bool) string { 16 | return "string" 17 | } 18 | 19 | func (stringManagerType) goTypeString(sh sigHelper, name string, attr *attribute, origNames map[string]bool) string { 20 | return "string" 21 | } 22 | 23 | func (stringManagerType) alloc(hr heapReader, c caster, bf io.Reader, attr *attribute, 24 | dimensions []uint64) interface{} { 25 | logger.Info("regular string", len(dimensions), "dtlen=", attr.length) 26 | return allocRegularStrings(bf, dimensions, attr.length) // already converted 27 | } 28 | 29 | func (stringManagerType) defaultFillValue(obj *object, objFillValue []byte, undefinedFillValue bool) []byte { 30 | // return all zeros to get zero lengths 31 | return []byte{0} 32 | } 33 | 34 | func (stringManagerType) parse(hr heapReader, c caster, attr *attribute, bitFields uint32, bf remReader, df remReader) { 35 | checkVal(1, attr.dtversion, "Only support version 1 of string") 36 | logger.Info("string") 37 | padding := bitFields & 0b1111 38 | set := (bitFields >> 3) & 0b1111 39 | if df == nil { 40 | logger.Infof("no data") 41 | return 42 | } 43 | b := make([]byte, df.Rem()) 44 | read(df, b) 45 | logger.Infof("* string padding=%d set=%d b[%s]=%s", padding, set, 46 | attr.name, getString(b)) 47 | attr.value = getString(b) 48 | } 49 | 50 | // Regular strings are fixed length, as opposed to variable length ones 51 | func allocRegularStrings(bf io.Reader, dimLengths []uint64, dtlen uint32) interface{} { 52 | if len(dimLengths) == 0 { 53 | b := make([]byte, dtlen) 54 | read(bf, b) 55 | return getString(b) 56 | } 57 | thisDim := dimLengths[0] 58 | if len(dimLengths) == 1 { 59 | b := make([]byte, thisDim) 60 | read(bf, b) 61 | return getString(b) 62 | } 63 | vals := makeStringSlices(dimLengths) 64 | for i := uint64(0); i < thisDim; i++ { 65 | vals.Index(int(i)).Set(reflect.ValueOf(allocRegularStrings(bf, dimLengths[1:], dtlen))) 66 | } 67 | return vals.Interface() 68 | } 69 | -------------------------------------------------------------------------------- /netcdf/hdf5/assert.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | // Various kinds of assertions 4 | 5 | import ( 6 | "fmt" 7 | "io" 8 | 9 | "github.com/batchatco/go-thrower" 10 | ) 11 | 12 | // for padBytesCheck() 13 | const ( 14 | dontRound = false // the number is the number of pad bytes to check for. 15 | round = true // the number is the byte-boundary to check up to (1, 3 or 7). 16 | ) 17 | 18 | var ( 19 | logFunc = logger.Fatal // logging function for padBytesCheck() 20 | maybeFail = fail // fail function, can be disabled for testing 21 | ) 22 | 23 | // Panics if condition isn't met 24 | func assert(condition bool, msg string) { 25 | if condition { 26 | return 27 | } 28 | fail(msg) 29 | } 30 | 31 | // Warns if condition isn't met 32 | func warnAssert(condition bool, msg string) { 33 | if condition { 34 | return 35 | } 36 | logger.Warn(msg) 37 | } 38 | 39 | // Infos if condition isn't met 40 | func infoAssert(condition bool, msg string) { 41 | if condition { 42 | return 43 | } 44 | logger.Info(msg) 45 | } 46 | 47 | // Asserts with given error and message 48 | func assertError(condition bool, err error, msg string) { 49 | if condition { 50 | return 51 | } 52 | failError(err, msg) 53 | } 54 | 55 | // Panics always with message 56 | func fail(msg string) { 57 | failError(ErrInternal, msg) 58 | } 59 | 60 | // Warns with message 61 | func warn(msg string) { 62 | logger.Warn(msg) 63 | } 64 | 65 | // Panics with specified error and message 66 | func failError(err error, msg string) { 67 | logger.Error(msg) 68 | thrower.Throw(err) 69 | panic("never gets here") 70 | } 71 | 72 | // Check that pad bytes are zero up to byte boundary (pad32 = 1,3 or 7) 73 | func padBytes(bf io.Reader, pad32 int) { 74 | padBytesCheck(bf, pad32, round, logFunc) 75 | } 76 | 77 | // Check that a run of len pad bytes are zero 78 | func checkZeroes(bf io.Reader, len int) { 79 | padBytesCheck(bf, len, dontRound, logFunc) 80 | } 81 | 82 | // Check that pad bytes are zero. 83 | func padBytesCheck(obf io.Reader, pad32 int, round bool, 84 | logFunc func(v ...interface{})) bool { 85 | cbf := obf.(remReader) 86 | success := true 87 | var extra int 88 | if round { 89 | pad64 := int64(pad32) 90 | rounded := (cbf.Count() + pad64) & ^pad64 91 | extra = int(rounded) - int(cbf.Count()) 92 | } else { 93 | extra = pad32 94 | } 95 | if extra > 0 { 96 | logger.Info(cbf.Count(), "prepad", extra, "bytes") 97 | b := make([]byte, extra) 98 | read(cbf, b) 99 | for i := 0; i < int(extra); i++ { 100 | if b[i] != 0 { 101 | success = false 102 | } 103 | } 104 | if !success { 105 | logFunc(fmt.Sprintf("Reserved not zero len=%d 0x%x", extra, b)) 106 | } 107 | } 108 | return success 109 | } 110 | -------------------------------------------------------------------------------- /netcdf/hdf5/opaque.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "reflect" 7 | ) 8 | 9 | type opaqueManagerType struct{} 10 | 11 | type opaque []byte 12 | 13 | var ( 14 | opaqueManager = opaqueManagerType{} 15 | _ typeManager = opaqueManager 16 | ) 17 | 18 | func (opaqueManagerType) cdlTypeString(sh sigHelper, name string, attr *attribute, origNames map[string]bool) string { 19 | signature := fmt.Sprintf("opaque(%d)", attr.length) 20 | namedType := sh.findSignature(signature, name, origNames, cdlTypeString) 21 | if namedType != "" { 22 | return namedType 23 | } 24 | return signature 25 | } 26 | 27 | func (opaqueManagerType) goTypeString(sh sigHelper, typeName string, attr *attribute, origNames map[string]bool) string { 28 | signature := fmt.Sprintf("[%d]uint8", attr.length) // TODO 29 | namedType := sh.findSignature(signature, typeName, origNames, goTypeString) 30 | if namedType != "" { 31 | return namedType 32 | } 33 | return signature 34 | } 35 | 36 | func (opaqueManagerType) alloc(hr heapReader, c caster, bf io.Reader, attr *attribute, 37 | dimensions []uint64) interface{} { 38 | cast := c.cast(*attr) 39 | return allocOpaque(bf, dimensions, attr.length, cast) 40 | } 41 | 42 | func (opaqueManagerType) defaultFillValue(obj *object, objFillValue []byte, undefinedFillValue bool) []byte { 43 | return objFillValue 44 | } 45 | 46 | func (opaqueManagerType) parse(hr heapReader, c caster, attr *attribute, bitFields uint32, bf remReader, df remReader) { 47 | if bf.Rem() == 0 { 48 | logger.Info("No properties for opaque") 49 | return 50 | } 51 | plen := int(bf.Rem()) 52 | tag := make([]byte, plen) 53 | // not sure what the purpose of the tag is 54 | read(bf, tag) 55 | stringTag := getString(tag) 56 | logger.Info("tag=", stringTag) 57 | taglen := len(stringTag) 58 | for i := taglen; i < plen; i++ { 59 | checkVal(0, tag[i], 60 | fmt.Sprint("reserved byte should be zero: ", i)) 61 | } 62 | if df != nil && df.Rem() >= int64(attr.length) { 63 | attr.df = newResetReaderSave(df, df.Rem()) 64 | } 65 | } 66 | 67 | func allocOpaque(bf io.Reader, dimLengths []uint64, length uint32, 68 | cast reflect.Type) interface{} { 69 | if len(dimLengths) == 0 { 70 | if cast != nil { 71 | b := reflect.New(cast) 72 | read(bf, b.Interface()) 73 | return reflect.Indirect(b).Interface() 74 | } 75 | b := make([]byte, length) 76 | read(bf, b) 77 | return opaque(b) 78 | } 79 | thisDim := dimLengths[0] 80 | var ty reflect.Type 81 | if cast != nil { 82 | ty = cast 83 | } else { 84 | ty = reflect.TypeOf(opaque{}) 85 | } 86 | vals := makeSlices(ty, dimLengths) 87 | for i := uint64(0); i < thisDim; i++ { 88 | val := allocOpaque(bf, dimLengths[1:], length, cast) 89 | vals.Index(int(i)).Set(reflect.ValueOf(val)) 90 | } 91 | return vals.Interface() 92 | } 93 | -------------------------------------------------------------------------------- /netcdf/hdf5/reference.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "io" 7 | "reflect" 8 | 9 | "github.com/batchatco/go-thrower" 10 | ) 11 | 12 | type referenceManagerType struct{} 13 | 14 | var ( 15 | referenceManager = referenceManagerType{} 16 | _ typeManager = referenceManager 17 | ) 18 | 19 | func (referenceManagerType) cdlTypeString(sh sigHelper, name string, attr *attribute, origNames map[string]bool) string { 20 | // Not NetCDF 21 | return "uint64" // reference same as uint64 22 | } 23 | 24 | func (referenceManagerType) goTypeString(sh sigHelper, name string, attr *attribute, origNames map[string]bool) string { 25 | // Not NetCDF 26 | return "uint64" // reference same as uint64 27 | } 28 | 29 | func (referenceManagerType) alloc(hr heapReader, c caster, bf io.Reader, attr *attribute, 30 | dimensions []uint64) interface{} { 31 | return allocReferences(bf, dimensions) // already converted 32 | } 33 | 34 | func (referenceManagerType) defaultFillValue(obj *object, objFillValue []byte, undefinedFillValue bool) []byte { 35 | return objFillValue 36 | } 37 | 38 | func (referenceManagerType) parse(hr heapReader, c caster, attr *attribute, bitFields uint32, bf remReader, df remReader) { 39 | logger.Info("* reference") 40 | assertError(attr.dtversion == 1, ErrUnsupportedReferenceVersion, "Only support version 1 of reference") 41 | rType := bitFields & 0b1111 42 | switch rType { 43 | case 0: 44 | break 45 | case 1: 46 | break 47 | default: 48 | assert(df == nil, "references can't be attributes") 49 | maybeFail(fmt.Sprintf("invalid rtype value: %#b dtlength=%v", rType, attr.length)) 50 | return 51 | } 52 | logger.Info("* rtype=object") 53 | warnAssert((bitFields & ^uint32(0b1111)) == 0, "reserved must be zero") 54 | if df == nil { 55 | logger.Infof("no data") 56 | return 57 | } 58 | if !allowReferences { 59 | assert(df == nil, "references can't be attributes") 60 | logger.Infof("References ignored") 61 | thrower.Throw(ErrReference) 62 | } 63 | if df.Rem() >= int64(attr.length) { 64 | attr.df = newResetReaderSave(df, df.Rem()) 65 | } 66 | } 67 | 68 | func allocReferences(bf io.Reader, dimLengths []uint64) interface{} { 69 | if len(dimLengths) == 0 { 70 | var addr uint64 71 | err := binary.Read(bf, binary.LittleEndian, &addr) 72 | thrower.ThrowIfError(err) 73 | logger.Infof("Reference addr 0x%x", addr) 74 | return addr 75 | } 76 | 77 | thisDim := dimLengths[0] 78 | if len(dimLengths) == 1 { 79 | values := make([]uint64, thisDim) 80 | for i := range values { 81 | var addr uint64 82 | err := binary.Read(bf, binary.LittleEndian, &addr) 83 | thrower.ThrowIfError(err) 84 | logger.Infof("Reference addr[%d] 0x%x", i, addr) 85 | values[i] = addr 86 | } 87 | return values 88 | } 89 | vals := makeSlices(reflect.TypeOf(uint64(0)), dimLengths) 90 | for i := uint64(0); i < thisDim; i++ { 91 | vals.Index(int(i)).Set(reflect.ValueOf(allocReferences(bf, dimLengths[1:]))) 92 | } 93 | return vals.Interface() 94 | } 95 | -------------------------------------------------------------------------------- /netcdf/api/api.go: -------------------------------------------------------------------------------- 1 | // Package api is common to different implementations of NetCDF4 (CDF or HDF5) 2 | package api 3 | 4 | import "io" 5 | 6 | type ReadSeekerCloser interface { 7 | io.ReadSeeker 8 | io.Closer 9 | } 10 | 11 | type AttributeMap interface { 12 | // Ordered list of keys 13 | Keys() []string 14 | // Indexed lookup 15 | Get(key string) (val interface{}, has bool) 16 | 17 | GetType(key string) (string, bool) 18 | GetGoType(key string) (string, bool) 19 | } 20 | 21 | type Variable struct { 22 | Values interface{} 23 | Dimensions []string 24 | Attributes AttributeMap 25 | } 26 | 27 | type VarGetter interface { 28 | // Len() is the total length of the variable's slice. 29 | // Or returns 1 if it is a scalar. 30 | Len() int64 31 | 32 | // Values returns all the values of the variable. For very large variables, 33 | // it may be more appropriate to call GetSlice instead. 34 | Values() (interface{}, error) 35 | 36 | // GetSlice gets a (smaller) slice of the variable's slice 37 | // It's useful for variables which are very large and may not fit in memory. 38 | GetSlice(begin, end int64) (interface{}, error) 39 | 40 | Dimensions() []string 41 | 42 | Attributes() AttributeMap 43 | 44 | // Type returns the base type in CDL format, not including dimensions. 45 | Type() string 46 | // GoType returns the base type in Go format, not including dimensions. 47 | GoType() string 48 | } 49 | 50 | type Group interface { 51 | // Close closes this group and closes any underlying files if they are no 52 | // longer being used by any other groups. 53 | Close() 54 | 55 | // Attributes returns the global attributes for this group. 56 | Attributes() AttributeMap 57 | 58 | // ListVariables lists the variables in this group. 59 | ListVariables() []string 60 | 61 | // GetVariable returns the named variable or sets the error if not found. 62 | GetVariable(name string) (*Variable, error) 63 | 64 | // GetVarGetter is an function that returns an interface that allows you to get 65 | // smaller slices of a variable, in case the variable is very large and you want to 66 | // reduce memory usage. 67 | GetVarGetter(name string) (VarGetter, error) 68 | 69 | // ListSubgroups returns the names of the subgroups of this group 70 | ListSubgroups() []string 71 | 72 | // GetGroup gets the given group or returns an error if not found. 73 | // The group can start with "/" for absolute names, or relative. 74 | GetGroup(group string) (g Group, err error) 75 | 76 | // Experimental API to get user-defined type information 77 | 78 | // ListTypes returns the user-defined type names. 79 | ListTypes() []string 80 | 81 | // GetType gets the CDL description of the type and sets the bool to true if found. 82 | GetType(string) (string, bool) 83 | 84 | // GettGoType gets the Go description of the type and sets the bool to true if found. 85 | GetGoType(string) (string, bool) 86 | 87 | // ListDimensions lists the names of the dimensions in this group. 88 | ListDimensions() []string 89 | 90 | // GetDimension returns the size of the given dimension and sets 91 | // the bool to true if found. 92 | GetDimension(string) (uint64, bool) 93 | } 94 | -------------------------------------------------------------------------------- /internal/logging.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | // Internal logging utility. 4 | 5 | import ( 6 | "log" 7 | "os" 8 | "runtime/debug" 9 | "sync" 10 | ) 11 | 12 | type Logger struct { 13 | logLevel LogLevel 14 | logger *log.Logger 15 | lock sync.Mutex 16 | } 17 | 18 | type LogLevel int 19 | 20 | const ( 21 | // error levels that should almost always be printed 22 | LevelFatal LogLevel = iota // error that must stop the program (panics) 23 | LevelError // error that does not need to stop execution 24 | 25 | // debugging levels, okay to disable 26 | LevelWarn // something may be wrong, but not necessarily an error 27 | LevelInfo // nothing wrong, informational only 28 | 29 | // Production code by default only shows warnings and above. 30 | LogLevelDefault = LevelWarn 31 | 32 | // min, max levels for setting print level 33 | LevelMin = LevelFatal 34 | LevelMax = LevelInfo 35 | ) 36 | 37 | var ( 38 | levelToPrefix = []string{ 39 | "FATAL ", 40 | "ERROR ", 41 | "WARN ", 42 | "INFO ", 43 | } 44 | ) 45 | 46 | func NewLogger() *Logger { 47 | logger := log.New(os.Stderr, "", log.LstdFlags) 48 | return &Logger{logLevel: LogLevelDefault, logger: logger, lock: sync.Mutex{}} 49 | } 50 | 51 | func (l *Logger) LogLevel() LogLevel { 52 | return l.logLevel 53 | } 54 | 55 | // SetLogLevel returns the old level 56 | func (l *Logger) SetLogLevel(level LogLevel) LogLevel { 57 | if level < LevelMin || level > LevelMax { 58 | panic("trying to set invalid log level") 59 | } 60 | old := l.logLevel 61 | l.logLevel = level 62 | return old 63 | } 64 | 65 | func (l *Logger) output(level LogLevel, f func(...interface{}), v ...interface{}) { 66 | if level > l.logLevel { 67 | return 68 | } 69 | l.lock.Lock() 70 | defer l.lock.Unlock() 71 | l.logger.SetPrefix(levelToPrefix[level]) 72 | f(v...) 73 | } 74 | 75 | func (l *Logger) outputf(level LogLevel, f func(string, ...interface{}), format string, v ...interface{}) { 76 | if level > l.logLevel { 77 | return 78 | } 79 | l.lock.Lock() 80 | defer l.lock.Unlock() 81 | 82 | l.logger.SetPrefix(levelToPrefix[level]) 83 | f(format, v...) 84 | } 85 | 86 | func (l *Logger) Info(v ...interface{}) { 87 | l.output(LevelInfo, l.logger.Println, v...) 88 | } 89 | 90 | func (l *Logger) Infof(format string, v ...interface{}) { 91 | l.outputf(LevelInfo, l.logger.Printf, format, v...) 92 | } 93 | 94 | func (l *Logger) Warn(v ...interface{}) { 95 | l.output(LevelWarn, l.logger.Println, v...) 96 | } 97 | 98 | func (l *Logger) Warnf(format string, v ...interface{}) { 99 | l.outputf(LevelWarn, l.logger.Printf, format, v...) 100 | } 101 | 102 | func (l *Logger) Error(v ...interface{}) { 103 | l.output(LevelError, l.logger.Println, v...) 104 | } 105 | 106 | func (l *Logger) Errorf(format string, v ...interface{}) { 107 | l.outputf(LevelError, l.logger.Printf, format, v...) 108 | } 109 | 110 | func (l *Logger) Fatal(v ...interface{}) { 111 | log.Print(string(debug.Stack())) 112 | l.output(LevelFatal, l.logger.Fatalln, v...) 113 | } 114 | 115 | func (l *Logger) Fatalf(format string, v ...interface{}) { 116 | log.Print(string(debug.Stack())) 117 | l.outputf(LevelFatal, l.logger.Fatalf, format, v...) 118 | } 119 | -------------------------------------------------------------------------------- /netcdf/hdf5/array.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "strings" 7 | ) 8 | 9 | type arrayManagerType struct{} 10 | 11 | var ( 12 | arrayManager = arrayManagerType{} 13 | _ typeManager = arrayManager 14 | ) 15 | 16 | func (arrayManagerType) cdlTypeString(sh sigHelper, name string, attr *attribute, origNames map[string]bool) string { 17 | arrayAttr := attr.children[0] 18 | ty := cdlTypeString(arrayAttr.class, sh, name, arrayAttr, origNames) 19 | assert(ty != "", "unable to parse array attr") 20 | dStr := make([]string, len(arrayAttr.dimensions)) 21 | for i, d := range arrayAttr.dimensions { 22 | dStr[i] = fmt.Sprintf("%d", d) 23 | } 24 | dims := strings.Join(dStr, ",") 25 | signature := fmt.Sprintf("%s(%s)", ty, dims) 26 | namedType := sh.findSignature(signature, name, origNames, cdlTypeString) 27 | assert(namedType == "", "arrays are not named types") 28 | return signature 29 | } 30 | 31 | func (arrayManagerType) goTypeString(sh sigHelper, typeName string, attr *attribute, origNames map[string]bool) string { 32 | arrayAttr := attr.children[0] 33 | ty := goTypeString(arrayAttr.class, sh, typeName, arrayAttr, origNames) 34 | assert(ty != "", "unable to parse array attr") 35 | dStr := make([]string, len(arrayAttr.dimensions)) 36 | for i, d := range arrayAttr.dimensions { 37 | dStr[i] = fmt.Sprintf("[%d]", d) 38 | } 39 | dims := strings.Join(dStr, "") 40 | signature := fmt.Sprintf("%s%s", dims, ty) 41 | namedType := sh.findSignature(signature, typeName, origNames, goTypeString) 42 | assert(namedType == "", "arrays are not named types") 43 | return signature 44 | } 45 | 46 | func (arrayManagerType) alloc(hr heapReader, c caster, bf io.Reader, attr *attribute, 47 | dimensions []uint64) interface{} { 48 | logger.Info("orig dimensions=", attr.dimensions) 49 | logger.Info("Array length=", attr.length) 50 | logger.Info("Array dimensions=", dimensions) 51 | arrayAttr := attr.children[0] 52 | logger.Info("child dimensions=", arrayAttr.dimensions) 53 | logger.Info("childlength=", arrayAttr.length) 54 | newDimensions := append(dimensions, arrayAttr.dimensions...) 55 | arrayAttr.dimensions = newDimensions 56 | logger.Info("new dimensions=", newDimensions) 57 | cbf := bf.(remReader) 58 | logger.Info(cbf.Count(), "child length", arrayAttr.length) 59 | logger.Info(cbf.Count(), "array", "class", arrayAttr.class) 60 | return getDataAttr(hr, c, cbf, *arrayAttr) 61 | } 62 | 63 | func (arrayManagerType) defaultFillValue(obj *object, objFillValue []byte, undefinedFillValue bool) []byte { 64 | return objFillValue 65 | } 66 | 67 | func (arrayManagerType) parse(hr heapReader, c caster, attr *attribute, bitFields uint32, bf remReader, df remReader) { 68 | logger.Info("Array") 69 | dimensionality := read8(bf) 70 | logger.Info("dimensionality", dimensionality) 71 | switch attr.dtversion { 72 | case dtversionStandard, dtversionArray: 73 | checkZeroes(bf, 3) 74 | } 75 | dimensions := make([]uint64, dimensionality) 76 | for i := 0; i < int(dimensionality); i++ { 77 | dimensions[i] = uint64(read32(bf)) 78 | logger.Info("dim=", dimensions[i]) 79 | } 80 | logger.Info("dimensions=", dimensions) 81 | if attr.dtversion < 3 { 82 | for i := 0; i < int(dimensionality); i++ { 83 | perm := read32(bf) 84 | logger.Info("perm=", perm) 85 | } 86 | } 87 | var arrayAttr attribute 88 | printDatatype(hr, c, bf, nil, 0, &arrayAttr) 89 | arrayAttr.dimensions = dimensions 90 | attr.children = append(attr.children, &arrayAttr) 91 | if df != nil && df.Rem() > 0 { 92 | logger.Info("Using an array in an attribute") 93 | attr.df = newResetReaderSave(df, df.Rem()) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /netcdf/cdf/verify_test.go: -------------------------------------------------------------------------------- 1 | // Verify that ncdump can read the files we write 2 | package cdf 3 | 4 | import ( 5 | "io" 6 | "io/ioutil" 7 | "os" 8 | "os/exec" 9 | "strings" 10 | "testing" 11 | 12 | "github.com/batchatco/go-native-netcdf/netcdf/util" 13 | ) 14 | 15 | func getFiles(t *testing.T, path string, suffix string) map[string]bool { 16 | names := make(map[string]bool) 17 | files, err := ioutil.ReadDir(path) 18 | if err != nil { 19 | t.Log("Error opening", path, err) 20 | return nil 21 | } 22 | if !strings.HasSuffix(path, "/") { 23 | path = path + "/" 24 | } 25 | for _, file := range files { 26 | fullName := path + file.Name() 27 | if file.IsDir() { 28 | more := getFiles(t, fullName, suffix) 29 | for m := range more { 30 | names[m] = true 31 | } 32 | } else if strings.HasSuffix(file.Name(), suffix) { 33 | names[fullName] = true 34 | } 35 | } 36 | return names 37 | } 38 | 39 | func ncDump(t *testing.T, fname string) (success bool) { 40 | t.Helper() 41 | cmdString := []string{"-h", fname} 42 | cmd := exec.Command("ncdump", cmdString...) 43 | stdout, err := cmd.StdoutPipe() 44 | if err != nil { 45 | t.Log("ncdump error (exec)", err) 46 | return true 47 | } 48 | stderr, err := cmd.StderrPipe() 49 | if err != nil { 50 | t.Log("ncdump error (exec)", err) 51 | return true 52 | } 53 | err = cmd.Start() 54 | success = true 55 | defer func() { 56 | err := cmd.Wait() 57 | if err != nil { 58 | t.Log("ncdump error (wait)", err) 59 | success = false 60 | } 61 | }() 62 | if err != nil { 63 | t.Log("ncdump error (start)", err) 64 | return true 65 | } 66 | for { 67 | var b [1024 * 1024]byte 68 | _, err := stdout.Read(b[:]) 69 | if err == nil { 70 | continue 71 | } 72 | if err == io.EOF { 73 | break 74 | } 75 | t.Log(err) 76 | return false 77 | } 78 | errText, r := ioutil.ReadAll(stderr) 79 | if r == nil { 80 | if string(errText) != "" { 81 | t.Log(string(errText)) 82 | } 83 | } else { 84 | t.Log("readall err", r) 85 | return false 86 | } 87 | return true 88 | } 89 | 90 | func TestCompat(t *testing.T) { 91 | fileNames := getFiles(t, "testdata", ".cdl") 92 | gettingfiles: 93 | for fileName := range fileNames { 94 | baseName := fileName[:len(fileName)-4] 95 | genName := ncGen(t, baseName) 96 | if genName == "" { 97 | t.Error(errorNcGen) 98 | continue 99 | } 100 | defer os.Remove(genName) 101 | t.Log("TEST:", genName) 102 | nc, err := Open(genName) 103 | if err != nil { 104 | t.Error(err) 105 | continue 106 | } 107 | wname := baseName + "-written.nc" 108 | _ = os.Remove(wname) 109 | defer os.Remove(wname) 110 | cw, err := OpenWriter(wname) 111 | if err != nil { 112 | t.Error(err) 113 | continue 114 | } 115 | newKeys := make([]string, 0) 116 | newValues := make(map[string]interface{}) 117 | gattr := nc.Attributes() 118 | for _, attrName := range gattr.Keys() { 119 | v, _ := gattr.Get(attrName) 120 | newKeys = append(newKeys, attrName) 121 | newValues[attrName] = v 122 | } 123 | newAttrs, err := util.NewOrderedMap(newKeys, newValues) 124 | if err != nil { 125 | t.Error(err) 126 | return 127 | } 128 | cw.AddGlobalAttrs(newAttrs) 129 | for _, varName := range nc.ListVariables() { 130 | val, err := nc.GetVariable(varName) 131 | if err != nil { 132 | t.Error(err) 133 | continue gettingfiles 134 | } 135 | err = cw.AddVar(varName, *val) 136 | if err != nil { 137 | t.Error(err) 138 | continue gettingfiles 139 | } 140 | } 141 | cw.Close() 142 | nc.Close() 143 | if !ncDump(t, wname) { 144 | t.Error("can't ncdump", wname) 145 | continue 146 | } 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /netcdf/hdf5/errors.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | import "errors" 4 | 5 | var ( 6 | // ErrBadMagic is returned when the file is not an HDF5 file 7 | ErrBadMagic = errors.New("bad magic number") 8 | 9 | // ErrUnsupportedFilter is returned when an unrecognized filter is encountered 10 | ErrUnsupportedFilter = errors.New("unsupported filter found") 11 | 12 | // ErrUnsupportedCompression is returned for unsupported copmression schemes 13 | ErrUnknownCompression = errors.New("unknown compression") 14 | 15 | // ErrInternal is an internal error not otherwise specified here 16 | ErrInternal = errors.New("internal error") 17 | 18 | // ErrNotFound is returned for items requested that don't exist 19 | ErrNotFound = errors.New("not found") 20 | 21 | // ErrFletcherChecksum is returned for corrupted data failing the checksum 22 | ErrFletcherChecksum = errors.New("fletcher checksum failure") 23 | 24 | // ErrVersion is returned when the particular HDF5 version is not supported 25 | ErrVersion = errors.New("hdf5 version not supported") 26 | 27 | // ErrLinkType is returned for an unrecognized or unsupported link type 28 | ErrLinkType = errors.New("link type not supported") 29 | 30 | // ErrVirtualStorage is returned when the unsupported virtual storage feature is encountered 31 | ErrVirtualStorage = errors.New("virtual storage not supported") 32 | 33 | // ErrTruncated is returned when the file has fewer bytes than the superblock says 34 | ErrTruncated = errors.New("file is too small, may be truncated") 35 | 36 | // ErrOffsetSize is returned when offsets other than 64-bit are indicated. 37 | // Only 64-bit is supported in this implementation. 38 | ErrOffsetSize = errors.New("only 64-bit offsets are supported") 39 | 40 | // ErrDimensionality is returned when invalid dimensions are specified 41 | ErrDimensionality = errors.New("invalid dimensionality") 42 | 43 | // ErrDataspaceVersion is returned for unsupported dataspace versions 44 | ErrDataspaceVersion = errors.New("dataspace version not supported") 45 | 46 | // ErrCorrupted is returned when file inconsistencies are found 47 | ErrCorrupted = errors.New("corrupted file") 48 | 49 | // ErrLayout is returned for unsupported data layouts 50 | ErrLayout = errors.New("data layout version not supported") 51 | 52 | // ErrSuperblock is returned for unsupported superblock versions 53 | ErrSuperblock = errors.New("superblock extension not supported") 54 | 55 | // ErrBitfield is returned when bitfields are encountered. 56 | // Bitfields are valid HDF5, but not valid NetCDF4. 57 | ErrBitfield = errors.New("bitfields not supported") 58 | 59 | // ErrExternal is returned when requests for external files are encountered, which is 60 | // not supported. 61 | ErrExternal = errors.New("external data files not supported") 62 | 63 | // ErrFloatingPoint is returned when non-standard floating point is encountered 64 | ErrFloatingPoint = errors.New("non-standard floating point not handled") 65 | 66 | // ErrFloatingPointis returned when non-standard integers are encountered 67 | ErrFixedPoint = errors.New("non-standard fixed-point not handled") 68 | 69 | // ErrReference is returned when references are encountered. 70 | // References are valid HDF, but not valid NetCDF4. 71 | ErrReference = errors.New("unsupported reference type") 72 | 73 | // ErrNonExportedField is returned when a value cannot be assigned to user-supplied 74 | // struct because it has non-exported fields. 75 | ErrNonExportedField = errors.New("can't assign to non-exported field") 76 | 77 | // ErrUnsupportedDataTypeVersion is returned when a unsupported datatype version 78 | // is encountered. Only versions 1-3 are supported. 79 | ErrUnsupportedDataTypeVersion = errors.New("unsupported data type version") 80 | 81 | // ErrUnsupportedReferenceVersion is returned a reference version other than 1 82 | // is encountered. 83 | ErrUnsupportedReferenceVersion = errors.New("unsupported reference version") 84 | ) 85 | -------------------------------------------------------------------------------- /netcdf/testdata/cdf.nc: -------------------------------------------------------------------------------- 1 | CDF -------------------------------------------------------------------------------- /netcdf/hdf5/types.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "reflect" 7 | 8 | "github.com/batchatco/go-thrower" 9 | ) 10 | 11 | // Each type implements this interface. 12 | type typeManager interface { 13 | parse(hr heapReader, c caster, attr *attribute, bitFields uint32, f remReader, d remReader) 14 | defaultFillValue(obj *object, objFillValue []byte, undefinedFillValue bool) []byte 15 | alloc(hr heapReader, c caster, r io.Reader, attr *attribute, dimensions []uint64) interface{} 16 | cdlTypeString(sh sigHelper, name string, attr *attribute, origNames map[string]bool) string 17 | goTypeString(sh sigHelper, name string, attr *attribute, origNames map[string]bool) string 18 | } 19 | 20 | type heapReader interface { 21 | readGlobalHeap(heapAddress uint64, index uint32) (remReader, uint64) 22 | } 23 | 24 | type caster interface { 25 | cast(attr attribute) reflect.Type 26 | } 27 | 28 | type sigHelper interface { 29 | findSignature(signature string, name string, origNames map[string]bool, 30 | printer printerType) string 31 | } 32 | 33 | type printerType func(class uint8, helper sigHelper, name string, attr *attribute, 34 | origNames map[string]bool) string 35 | 36 | var dispatch = []typeManager{ 37 | // 0-4 38 | fixedPointManager, 39 | floatingPointManager, 40 | timeManager, 41 | stringManager, 42 | bitfieldManager, 43 | // 5-9 44 | opaqueManager, 45 | compoundManager, 46 | referenceManager, 47 | enumManager, 48 | vlenManager, 49 | // 10 50 | arrayManager, 51 | } 52 | 53 | func getDispatch(class uint8) typeManager { 54 | if int(class) >= len(dispatch) { 55 | fail(fmt.Sprintf("Unknown class: %d", class)) 56 | } 57 | return dispatch[class] 58 | } 59 | 60 | // parse is a wrapper around the table lookup of the type to get the interface 61 | func parse(class uint8, hr heapReader, c caster, attr *attribute, bitFields uint32, f remReader, d remReader) { 62 | getDispatch(class).parse(hr, c, attr, bitFields, f, d) 63 | } 64 | 65 | // defaultFillValue is a wrapper around the table lookup of the type to get the interface 66 | func defaultFillValue(class uint8, obj *object, objFillValue []byte, undefinedFillValue bool) []byte { 67 | return getDispatch(class).defaultFillValue(obj, objFillValue, undefinedFillValue) 68 | } 69 | 70 | func alloc(class uint8, hr heapReader, c caster, r io.Reader, attr *attribute, dimensions []uint64) interface{} { 71 | return getDispatch(class).alloc(hr, c, r, attr, dimensions) 72 | } 73 | 74 | func cdlTypeString(class uint8, sh sigHelper, name string, attr *attribute, origNames map[string]bool) string { 75 | return getDispatch(class).cdlTypeString(sh, name, attr, origNames) 76 | } 77 | 78 | func goTypeString(class uint8, sh sigHelper, name string, attr *attribute, origNames map[string]bool) string { 79 | return getDispatch(class).goTypeString(sh, name, attr, origNames) 80 | } 81 | 82 | func printDatatype(hr heapReader, c caster, bf remReader, df remReader, objCount int64, attr *attribute) { 83 | assert(bf.Rem() >= 8, "short data") 84 | b0 := read8(bf) 85 | b1 := read8(bf) 86 | b2 := read8(bf) 87 | b3 := read8(bf) 88 | bitFields := uint32(b1) | (uint32(b2) << 8) | (uint32(b3) << 16) 89 | dtversion := (b0 >> 4) & 0b1111 90 | dtclass := b0 & 0b1111 91 | dtlength := read32(bf) 92 | logger.Infof("* length=%d dtlength=%d dtversion=%d class=%s flags=%s", 93 | bf.Rem(), dtlength, 94 | dtversion, typeNames[dtclass], binaryToString(uint64(bitFields))) 95 | switch dtversion { 96 | case dtversionStandard: 97 | logger.Info("Standard datatype") 98 | case dtversionArray: 99 | logger.Info("Array-encoded datatype") 100 | case dtversionPacked: 101 | logger.Info("VAX and/or packed datatype") 102 | case dtversionV4: 103 | if maxDTVersion == dtversionV4 { 104 | // allowed 105 | logger.Info("Undocumented datatype version 4") 106 | break 107 | } 108 | fallthrough 109 | default: 110 | logger.Error(fmt.Sprint("Unknown datatype version: ", dtversion)) 111 | thrower.Throw(ErrUnsupportedDataTypeVersion) 112 | } 113 | attr.dtversion = dtversion 114 | attr.class = dtclass 115 | attr.length = dtlength 116 | assert(attr.length != 0, "attr length can't be zero") 117 | parse(dtclass, hr, c, attr, bitFields, bf, df) 118 | if df != nil && df.Rem() > 0 { 119 | // It is normal for there to be extra data, not sure why yet. 120 | // It does not break any unit tests, so the extra data seems unnecessary. 121 | logger.Info("did not read all data", df.Rem(), typeNames[dtclass]) 122 | skip(df, df.Rem()) 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /netcdf/hdf5/readers_test.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/batchatco/go-thrower" 8 | ) 9 | 10 | func TestFletcherOdd(t *testing.T) { 11 | const nbytes = 11 12 | // Create the input reader 13 | b := make([]byte, nbytes+4) 14 | // Put in the checksum 15 | copy(b[nbytes:], []byte{0x19, 0x1e, 0x50, 0x46}) 16 | // Then write the data 17 | for i := 0; i < nbytes; i++ { 18 | b[i] = byte(i) 19 | } 20 | r := bytes.NewReader(b) 21 | 22 | // create the Fletcher32 reader and verify we can read from it 23 | fl := newFletcher32Reader(r, uint64(len(b))) 24 | var b2 [nbytes]byte 25 | n, err := fl.Read(b2[:]) 26 | if err != nil { 27 | t.Error(err) 28 | return 29 | } 30 | if n != len(b2) { 31 | t.Error("Got", n, "expected", len(b2)) 32 | return 33 | } 34 | for i := 0; i < nbytes; i++ { 35 | if b2[i] != b[i] { 36 | t.Error("Got", b2[i], "at offset", i, "expected", b[i]) 37 | } 38 | } 39 | 40 | // try again with the old reader 41 | r = bytes.NewReader(b) 42 | fl = oldFletcher32Reader(r, uint64(len(b))) 43 | n, err = fl.Read(b2[:]) 44 | if err != nil { 45 | t.Error(err) 46 | return 47 | } 48 | if n != len(b2) { 49 | t.Error("Got", n, "expected", len(b2)) 50 | return 51 | } 52 | for i := 0; i < nbytes; i++ { 53 | if b2[i] != b[i] { 54 | t.Error("Got", b2[i], "at offset", i, "expected", b[i]) 55 | } 56 | } 57 | } 58 | 59 | func TestFletcherEven(t *testing.T) { 60 | const nbytes = 10 61 | // Create the input reader 62 | b := make([]byte, nbytes+4) 63 | // Put in the checksum 64 | copy(b[nbytes:], []byte{0x19, 0x14, 0x37, 0x28}) 65 | // Then write the data 66 | for i := 0; i < nbytes; i++ { 67 | b[i] = byte(i) 68 | } 69 | r := bytes.NewReader(b) 70 | 71 | // create the Fletcher32 reader and verify we can read from it 72 | fl := newFletcher32Reader(r, uint64(len(b))) 73 | var b2 [nbytes]byte 74 | n, err := fl.Read(b2[:]) 75 | if err != nil { 76 | t.Error(err) 77 | return 78 | } 79 | if n != len(b2) { 80 | t.Error("Got", n, "expected", len(b2)) 81 | return 82 | } 83 | for i := 0; i < nbytes; i++ { 84 | if b2[i] != b[i] { 85 | t.Error("Got", b2[i], "at offset", i, "expected", b[i]) 86 | } 87 | } 88 | 89 | // try again with the old reader 90 | r = bytes.NewReader(b) 91 | fl = oldFletcher32Reader(r, uint64(len(b))) 92 | n, err = fl.Read(b2[:]) 93 | if err != nil { 94 | t.Error(err) 95 | return 96 | } 97 | if n != len(b2) { 98 | t.Error("Got", n, "expected", len(b2)) 99 | return 100 | } 101 | for i := 0; i < nbytes; i++ { 102 | if b2[i] != b[i] { 103 | t.Error("Got", b2[i], "at offset", i, "expected", b[i]) 104 | } 105 | } 106 | } 107 | 108 | // Test that bad checksums properly fail 109 | func TestFletcherFail(t *testing.T) { 110 | const nbytes = 10 111 | // Create the input reader 112 | b := make([]byte, nbytes+4) 113 | // Put in the checksum 114 | copy(b[nbytes:], []byte{0xde, 0xad, 0xbe, 0xef}) 115 | // Then write the data 116 | for i := 0; i < nbytes; i++ { 117 | b[i] = byte(i) 118 | } 119 | 120 | err := func() (err error) { 121 | defer thrower.RecoverError(&err) 122 | r := bytes.NewReader(b) 123 | fl := newFletcher32Reader(r, uint64(len(b))) 124 | // create the Fletcher32 reader and verify we can read from it 125 | var b2 [nbytes]byte 126 | _, err = fl.Read(b2[:]) 127 | if err != nil { 128 | t.Error(err) 129 | thrower.Throw(err) 130 | } 131 | return nil 132 | }() 133 | if err != ErrFletcherChecksum { 134 | t.Error("Got", err, "expected", ErrFletcherChecksum) 135 | return 136 | } 137 | 138 | // try again with the old reader 139 | err = func() (err error) { 140 | defer thrower.RecoverError(&err) 141 | r := bytes.NewReader(b) 142 | fl := oldFletcher32Reader(r, uint64(len(b))) 143 | var b2 [nbytes]byte 144 | _, err = fl.Read(b2[:]) 145 | if err != nil { 146 | t.Error(err) 147 | thrower.Throw(err) 148 | } 149 | return nil 150 | }() 151 | if err != ErrFletcherChecksum { 152 | t.Error("Got", err, "expected", ErrFletcherChecksum) 153 | return 154 | } 155 | } 156 | 157 | func TestFletcherSingle(t *testing.T) { 158 | const nbytes = 1 159 | // Create the input reader 160 | b := make([]byte, nbytes+4) 161 | // Put in the checksum 162 | copy(b[nbytes:], []byte{0x00, 0x01, 0x00, 0x01}) 163 | // Then write the data 164 | for i := 0; i < nbytes; i++ { 165 | b[i] = byte(i) + 1 166 | } 167 | r := bytes.NewReader(b) 168 | 169 | // create the Fletcher32 reader and verify we can read from it 170 | fl := newFletcher32Reader(r, uint64(len(b))) 171 | var b2 [nbytes]byte 172 | n, err := fl.Read(b2[:]) 173 | if err != nil { 174 | t.Error(err) 175 | return 176 | } 177 | if n != len(b2) { 178 | t.Error("Got", n, "expected", len(b2)) 179 | return 180 | } 181 | for i := 0; i < nbytes; i++ { 182 | if b2[i] != b[i] { 183 | t.Error("Got", b2[i], "at offset", i, "expected", b[i]) 184 | } 185 | } 186 | 187 | // try again with the old reader 188 | r = bytes.NewReader(b) 189 | fl = oldFletcher32Reader(r, uint64(len(b))) 190 | n, err = fl.Read(b2[:]) 191 | if err != nil { 192 | t.Error(err) 193 | return 194 | } 195 | if n != len(b2) { 196 | t.Error("Got", n, "expected", len(b2)) 197 | return 198 | } 199 | for i := 0; i < nbytes; i++ { 200 | if b2[i] != b[i] { 201 | t.Error("Got", b2[i], "at offset", i, "expected", b[i]) 202 | } 203 | } 204 | } 205 | -------------------------------------------------------------------------------- /netcdf/hdf5/enum.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "strings" 7 | 8 | "github.com/batchatco/go-thrower" 9 | ) 10 | 11 | type enumManagerType struct{} 12 | 13 | type enumerated struct { 14 | values interface{} 15 | } 16 | 17 | var ( 18 | enumManager = enumManagerType{} 19 | _ typeManager = enumManager 20 | ) 21 | 22 | func (enumManagerType) cdlTypeString(sh sigHelper, name string, attr *attribute, origNames map[string]bool) string { 23 | assert(len(attr.children) == 1, "enum should have one child") 24 | enumAttr := attr.children[0] 25 | assert(len(enumAttr.children) == 0, "no recursion") 26 | ty := cdlTypeString(enumAttr.class, sh, name, enumAttr, origNames) 27 | assert(ty != "", "unable to parse enum attr") 28 | list := make([]string, len(enumAttr.enumNames)) 29 | for i, name := range enumAttr.enumNames { 30 | list[i] = fmt.Sprintf("\t%s = %v", name, enumAttr.enumValues[i]) 31 | } 32 | interior := strings.Join(list, ",\n") 33 | signature := fmt.Sprintf("%s enum {\n%s\n}", ty, interior) 34 | namedType := sh.findSignature(signature, name, origNames, cdlTypeString) 35 | if namedType != "" { 36 | return namedType 37 | } 38 | return signature 39 | } 40 | 41 | func (enumManagerType) goTypeString(sh sigHelper, typeName string, attr *attribute, origNames map[string]bool) string { 42 | assert(len(attr.children) == 1, "enum should have one child") 43 | enumAttr := attr.children[0] 44 | assert(len(enumAttr.children) == 0, "no recursion") 45 | ty := goTypeString(enumAttr.class, sh, typeName, enumAttr, origNames) 46 | assert(ty != "", "unable to parse enum attr") 47 | list := make([]string, len(enumAttr.enumNames)) 48 | for i, enumName := range enumAttr.enumNames { 49 | if i == 0 { 50 | list[i] = fmt.Sprintf("\t%s %s = %v", enumName, typeName, enumAttr.enumValues[i]) 51 | } else { 52 | list[i] = fmt.Sprintf("\t%s = %v", enumName, enumAttr.enumValues[i]) 53 | } 54 | } 55 | interior := strings.Join(list, "\n") 56 | signature := fmt.Sprintf("%s\nconst (\n%s\n)\n", ty, interior) 57 | namedType := sh.findSignature(signature, typeName, origNames, goTypeString) 58 | if namedType != "" { 59 | return namedType 60 | } 61 | return signature 62 | } 63 | 64 | func (enumManagerType) alloc(hr heapReader, c caster, bf io.Reader, attr *attribute, 65 | dimensions []uint64) interface{} { 66 | var values interface{} 67 | enumAttr := attr.children[0] 68 | cast := c.cast(*enumAttr) 69 | switch enumAttr.class { 70 | case typeFixedPoint: 71 | switch enumAttr.length { 72 | case 1: 73 | values = allocInt8s(bf, dimensions, enumAttr.signed, cast) 74 | case 2: 75 | values = allocShorts(bf, dimensions, enumAttr.endian, enumAttr.signed, cast) 76 | case 4: 77 | values = allocInts(bf, dimensions, enumAttr.endian, enumAttr.signed, cast) 78 | case 8: 79 | values = allocInt64s(bf, dimensions, enumAttr.endian, enumAttr.signed, cast) 80 | default: 81 | fail(fmt.Sprintf("bad size enum fixed: %d", enumAttr.length)) 82 | } 83 | case typeFloatingPoint: 84 | if floatEnums { 85 | // Floating point enums are not part of NetCDF. 86 | switch enumAttr.length { 87 | case 4: 88 | values = allocFloats(bf, dimensions, enumAttr.endian) 89 | case 8: 90 | values = allocDoubles(bf, dimensions, enumAttr.endian) 91 | default: 92 | fail(fmt.Sprintf("bad size enum float: %d", attr.length)) 93 | } 94 | break 95 | } 96 | fallthrough 97 | default: 98 | fail(fmt.Sprint("can't handle this class: ", enumAttr.class)) 99 | } 100 | if cast != nil { 101 | return values 102 | } 103 | return enumerated{values} 104 | } 105 | 106 | func (enumManagerType) defaultFillValue(obj *object, objFillValue []byte, undefinedFillValue bool) []byte { 107 | return objFillValue 108 | } 109 | 110 | func (enumManagerType) parse(hr heapReader, c caster, attr *attribute, bitFields uint32, bf remReader, df remReader) { 111 | logger.Info("blen begin", bf.Count()) 112 | var enumAttr attribute 113 | printDatatype(hr, c, bf, nil, 0, &enumAttr) 114 | logger.Info("blen now", bf.Count()) 115 | numberOfMembers := bitFields & 0b11111111 116 | logger.Info("number of members=", numberOfMembers) 117 | names := make([]string, numberOfMembers) 118 | padding := 7 119 | switch attr.dtversion { 120 | case dtversionStandard: 121 | case dtversionArray: 122 | break 123 | default: 124 | padding = 0 125 | } 126 | for i := uint32(0); i < numberOfMembers; i++ { 127 | name := readNullTerminatedName(bf, padding) 128 | names[i] = name 129 | } 130 | enumAttr.enumNames = names 131 | logger.Info("enum names:", names) 132 | assert(enumAttr.class == typeFixedPoint, "only fixed-point enums supported") 133 | switch enumAttr.length { 134 | case 1, 2, 4, 8: 135 | break 136 | default: 137 | thrower.Throw(ErrFixedPoint) 138 | } 139 | switch attr.length { 140 | case 1, 2, 4, 8: 141 | break 142 | default: 143 | thrower.Throw(ErrFixedPoint) 144 | } 145 | values := make([]interface{}, numberOfMembers) 146 | enumAttr.enumValues = values 147 | for i := uint32(0); i < numberOfMembers; i++ { 148 | values[i] = getDataAttr(hr, c, bf, enumAttr) 149 | switch values[i].(type) { 150 | case uint64, int64: 151 | case uint32, int32: 152 | case uint16, int16: 153 | case uint8, int8: 154 | default: 155 | // Other enumeration types are not supported in NetCDF 156 | fail("unknown enumeration type") 157 | } 158 | } 159 | enumAttr.enumValues = values 160 | 161 | logger.Info("enum values:", values) 162 | attr.children = []*attribute{&enumAttr} 163 | if df != nil && df.Rem() > 0 { 164 | // Read away some bytes 165 | attr.df = newResetReaderSave(df, df.Rem()) 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /netcdf/util/orderedmap_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestNil(t *testing.T) { 8 | _, err := NewOrderedMap(nil, nil) 9 | if err != nil { 10 | t.Error(err) 11 | return 12 | } 13 | _, err = NewOrderedMap(nil, map[string]interface{}{}) 14 | if err != nil { 15 | t.Error(err) 16 | return 17 | } 18 | _, err = NewOrderedMap([]string{}, nil) 19 | if err != nil { 20 | t.Error(err) 21 | return 22 | } 23 | } 24 | 25 | func TestMismatchedLength(t *testing.T) { 26 | _, err := NewOrderedMap([]string{"a", "b"}, 27 | map[string]interface{}{"a": nil}) 28 | if err != ErrorKeysDontMatchValues { 29 | t.Error("Should have returned an error") 30 | return 31 | } 32 | } 33 | 34 | func TestHidden(t *testing.T) { 35 | om, err := NewOrderedMap([]string{"a", "b"}, 36 | map[string]interface{}{"a": nil, "b": nil}) 37 | if err != nil { 38 | t.Error(err) 39 | return 40 | } 41 | om.Hide("a") 42 | keys := om.Keys() 43 | if len(keys) != 1 || keys[0] != "b" { 44 | t.Error("Hide() failed") 45 | return 46 | } 47 | om.Add("a", 1) 48 | keys = om.Keys() 49 | if len(keys) != 1 || keys[0] != "b" { 50 | t.Error("Hide() failed") 51 | return 52 | } 53 | om.Hide("c") 54 | } 55 | 56 | func TestMismatchedKeys(t *testing.T) { 57 | _, err := NewOrderedMap([]string{"a", "b"}, 58 | map[string]interface{}{"a": nil, "c": nil}) 59 | if err != ErrorKeysDontMatchValues { 60 | t.Error("Should have returned an error") 61 | return 62 | } 63 | } 64 | 65 | func TestAdd(t *testing.T) { 66 | om, err := NewOrderedMap(nil, nil) 67 | if err != nil { 68 | t.Error(err) 69 | return 70 | } 71 | om.Add("a", 1) 72 | val, has := om.Get("a") 73 | if !has { 74 | t.Error("Did not find expected key") 75 | return 76 | } 77 | if val.(int) != 1 { 78 | t.Error("Did not get expected value back") 79 | return 80 | } 81 | } 82 | 83 | var i int32 84 | var f float32 85 | var d float64 86 | var s string 87 | 88 | type sI struct { 89 | s string 90 | I int32 91 | } 92 | 93 | var n sI 94 | 95 | var i1 []int32 96 | var i2 = []int32{0} 97 | var f1 []float32 98 | var f2 = []float32{1, 2} 99 | var d1 []float64 100 | var d2 = []float64{1, 2, 3} 101 | var s1 []string 102 | var s2 = []string{"1", "2", "3", "4"} 103 | 104 | type fI struct { 105 | f float32 106 | I int32 107 | } 108 | 109 | var n1 []fI 110 | 111 | var n2 = []fI{ 112 | {0, 1}, 113 | {2, 3}, 114 | {4, 5}, 115 | {6, 7}, 116 | } 117 | 118 | var i12 [2]int32 119 | var f12 [2]float32 120 | var d12 [2]float64 121 | var s12 [2]string 122 | 123 | type If struct { 124 | I int32 125 | f float32 126 | } 127 | 128 | var n12 [2]If 129 | 130 | type c25i struct{ member [2][5]int32 } 131 | 132 | var i22 c25i 133 | 134 | type c257i struct{ member [2][5][7]int32 } 135 | 136 | var i23 c257i 137 | 138 | var myMap = map[string]interface{}{ 139 | // scalars 140 | "i": i, "f": f, "d": d, "s": s, 141 | "n": n, 142 | // empty slices 143 | "i1": i1, "f1": f1, "d1": d1, "s1": s1, 144 | "n1": n1, 145 | // filled slices 146 | "i2": i2, "f2": f2, "d2": d2, "s2": s2, 147 | "n2": n2, 148 | // arrays can only be defined as compound fields 149 | "i12": i12, "f12": f12, "d12": d12, "s12": s12, 150 | "n12": n12, 151 | // 2-dimensional 152 | "i22": i22, 153 | // 3-dimensional 154 | "i23": i23, 155 | } 156 | var om *OrderedMap 157 | 158 | func initMaps(t *testing.T) { 159 | var err error 160 | om, err = NewOrderedMap([]string{ 161 | "i", "f", "d", "s", 162 | "n", 163 | "i1", "f1", "d1", "s1", 164 | "n1", 165 | "i2", "f2", "d2", "s2", 166 | "n2", 167 | "i12", "f12", "d12", "s12", 168 | "n12", "i22", "i23", 169 | }, myMap) 170 | if err != nil { 171 | t.Error(err) 172 | return 173 | } 174 | } 175 | 176 | func TestOrder(t *testing.T) { 177 | myMap := map[string]interface{}{"a": nil, "b": nil, "c": nil} 178 | om, err := NewOrderedMap([]string{"c", "b", "a"}, myMap) 179 | if err != nil { 180 | t.Error(err) 181 | return 182 | } 183 | keys := om.Keys() 184 | if keys[0] != "c" || keys[1] != "b" || keys[2] != "a" { 185 | t.Error("Incorrect key order:", keys) 186 | } 187 | } 188 | 189 | func TestGoType(t *testing.T) { 190 | initMaps(t) 191 | rightTypes := map[string]string{ 192 | "i": "int32", 193 | "f": "float32", 194 | "d": "float64", 195 | "s": "string", 196 | "n": "struct { s string; I int32 }", 197 | "i1": "[]int32", 198 | "f1": "[]float32", 199 | "d1": "[]float64", 200 | "s1": "[]string", 201 | "n1": "[]struct { f float32; I int32 }", 202 | "i2": "[]int32", 203 | "f2": "[]float32", 204 | "d2": "[]float64", 205 | "s2": "[]string", 206 | "n2": "[]struct { f float32; I int32 }", 207 | "i12": "[2]int32", 208 | "f12": "[2]float32", 209 | "d12": "[2]float64", 210 | "s12": "[2]string", 211 | "n12": "[2]struct { I int32; f float32 }", 212 | "i22": "struct { member [2][5]int32 }", 213 | "i23": "struct { member [2][5][7]int32 }", 214 | } 215 | for v, exp := range rightTypes { 216 | got, has := om.GetGoType(v) 217 | if !has { 218 | t.Errorf("Var %s is missing", v) 219 | continue 220 | } 221 | if got != exp { 222 | t.Errorf("wrong type for %s: got=%s exp=%s", v, got, exp) 223 | continue 224 | } 225 | } 226 | } 227 | 228 | func TestType(t *testing.T) { 229 | initMaps(t) 230 | rightTypes := map[string]string{ 231 | "i": "int", 232 | "f": "float", 233 | "d": "double", 234 | "s": "string", 235 | "n": "compound { string s; int I; }", 236 | "i1": "int(*)", 237 | "f1": "float(*)", 238 | "d1": "double(*)", 239 | "s1": "string(*)", 240 | "n1": "compound { float f; int I; }(*)", 241 | "i2": "int(1)", 242 | "f2": "float(2)", 243 | "d2": "double(3)", 244 | "s2": "string(4)", 245 | "n2": "compound { float f; int I; }(4)", 246 | "i12": "int(2)", 247 | "f12": "float(2)", 248 | "d12": "double(2)", 249 | "s12": "string(2)", 250 | "n12": "compound { int I; float f; }(2)", 251 | "i22": "compound { int(2,5) member; }", 252 | "i23": "compound { int(2,5,7) member; }", 253 | } 254 | for v, exp := range rightTypes { 255 | got, has := om.GetType(v) 256 | if !has { 257 | t.Errorf("Var %s is missing", v) 258 | continue 259 | } 260 | if got != exp { 261 | t.Errorf("wrong type for %s: got=%s exp=%s", v, got, exp) 262 | continue 263 | } 264 | } 265 | } 266 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-native-netcdf 2 | 3 | ## Introduction 4 | 5 | This is a native implementation of NetCDF in the Go language. It supports the CDF 6 | file format fully, and has limited support for the HDF5 format. It's not a wrapper. 7 | There's no C or C++ code underneath this. It is pure Go code. It benefits from the 8 | the sandboxing and garbage collection that Go provides, so is safer to use in a production 9 | environment. 10 | 11 | The API is mainly intended for reading files, though there is support for writing CDF files. 12 | To read files, please use the generic *Open* and *New()* interface, rather than any lower 13 | layer interfaces. 14 | 15 | The goal of the API is to be easy to use, and some functionality may be compromised because 16 | of that. 17 | 18 | ## Mapping of Types 19 | 20 | Most types are what you what you would expect and map one-to-one to Go language types. 21 | The only tricky ones are *bytes, unsigned bytes and strings*. The NetCDF *byte* type is 22 | signed and the Go language *byte* type is unsigned, so the proper mapping is for NetCDF *bytes* 23 | to become Go *int8s*. Conversely, the NetCDF *ubyte* becomes the Go *uint8*. 24 | 25 | The *char* type in NetCDF is meant for strings, but *char* is a scalar in NetCDF and Go has 26 | no scalar character type, just a *string* type to represent character strings. So, the 27 | mapping of NetCDF char is to Go string as the closest fit. Scalar characters in NetCDF 28 | will be returned as strings of length one. Scalar characters cannot be written to NetCDF 29 | with this API; they will always be written as strings of length one. 30 | 31 | Also note, that while it is normal to use the *int* type in Go, this type cannot be written 32 | to NetCDF. *int32* should be used instead. 33 | 34 | 35 | | NetCDF type | Go type | 36 | |------------------|---------| 37 | | byte | int8 | 38 | | ubyte | uint8 | 39 | | char | string | 40 | | short | int16 | 41 | | ushort | uint16 | 42 | | int | int32 | 43 | | uint | uint32 | 44 | | int64 | int64 | 45 | | uint64 | uint64 | 46 | | float | float32 | 47 | | double | float64 | 48 | 49 | 50 | ## Examples 51 | 52 | ### Reading a NetCDF file (CDF format) 53 | ```go 54 | package main 55 | 56 | import ( 57 | "fmt" 58 | "github.com/batchatco/go-native-netcdf/netcdf" 59 | ) 60 | 61 | func main() { 62 | // Open the file 63 | nc, err := netcdf.Open("data.nc") 64 | if err != nil { 65 | panic(err) 66 | } 67 | defer nc.Close() 68 | 69 | // Read the NetCDF variable from the file 70 | vr, _ := nc.GetVariable("latitude") 71 | if vr == nil { 72 | panic("latitude variable not found") 73 | } 74 | 75 | // Cast the data into a Go type we can use 76 | lats, has := vr.Values.([]float32) 77 | if !has { 78 | panic("latitude data not found") 79 | } 80 | for i, lat := range lats { 81 | fmt.Println(i, lat) 82 | } 83 | } 84 | 85 | ``` 86 | 87 | ### Reading a NetCDF file (HDF5 format) 88 | It is similar, but supports subgroups. 89 | 90 | ```go 91 | package main 92 | 93 | import ( 94 | "fmt" 95 | "github.com/batchatco/go-native-netcdf/netcdf" 96 | ) 97 | 98 | func main() { 99 | // Open the file 100 | ncf, err := netcdf.Open("data.nc") 101 | if err != nil { 102 | panic(err) 103 | } 104 | defer ncf.Close() 105 | 106 | // This is the only thing different about HDF5 from CDF 107 | // in this implementation. 108 | nc, err := ncf.GetGroup("/raw") 109 | if err != nil { 110 | panic(err) 111 | } 112 | defer nc.Close() 113 | 114 | // Read the NetCDF variable from the file 115 | vr, _ := nc.GetVariable("latitude") 116 | if vr == nil { 117 | panic("latitude variable not found") 118 | } 119 | 120 | // Cast the data into a Go type we can use 121 | lats, has := vr.Values.([]float32) 122 | if !has { 123 | panic("latitude data not found") 124 | } 125 | for i, lat := range lats { 126 | fmt.Println(i, lat) 127 | } 128 | } 129 | 130 | ``` 131 | 132 | ### Writing a CDF file 133 | ```go 134 | 135 | package main 136 | 137 | import ( 138 | "github.com/batchatco/go-native-netcdf/netcdf/api" 139 | "github.com/batchatco/go-native-netcdf/netcdf/cdf" 140 | "github.com/batchatco/go-native-netcdf/netcdf/util" 141 | ) 142 | 143 | func main() { 144 | cw, err := cdf.OpenWriter("newdata.nc") 145 | if err != nil { 146 | panic(err) 147 | } 148 | 149 | latitude := []float32{32.5, 64.1} 150 | dimensions := []string{"sounding_id"} 151 | attributes, err := util.NewOrderedMap( 152 | []string{"comment"}, 153 | map[string]interface{}{"comment": "Latitude indexed by sounding ID"}) 154 | if err != nil { 155 | panic(err) 156 | } 157 | variable := api.Variable{ 158 | latitude, 159 | dimensions, 160 | attributes} 161 | err = cw.AddVar("latitude", variable) 162 | if err != nil { 163 | panic(err) 164 | } 165 | // Close will write out the data and close the file 166 | err = cw.Close() 167 | if err != nil { 168 | panic(err) 169 | } 170 | } 171 | ``` 172 | 173 | ## Limitations on the CDF writer 174 | Unlimited data types are not supported. The only exception is 175 | that a one dimensional empty slice will be written out as unlimited, but 176 | currently zero length. For writing out variables with dimensions greater than 177 | one to work, extra information would need to be passed in to know the sizes of 178 | the other dimensions, because they cannot be guessed based upon the information 179 | in the slice. This doesn't seem like all that important of a feature though, 180 | and it would clutter the UI, so it is not implemented. 181 | 182 | ## Some notes about the HDF5 code 183 | The HDF5 code is quite hacky, but it has run though several unit tests, with good coverage, 184 | and should be pretty solid. Performance has not been looked at yet though, so it is likely 185 | slower than the alternative. 186 | 187 | It's working well enough that I feel it is okay to publish it now. I'll continue to work 188 | on it. I can clean up the code and make it run faster, for example. Feedback is welcome. 189 | 190 | Some of the exotic HDF5 types are actually implemented, but the interfaces to them 191 | mostly hidden. Variables of these types will get parsed and returned in 192 | an unsupported format. If you want to play with it, fine. If there's enough demand, 193 | I can expose the interfaces. 194 | 195 | If you want to run the HDF5 unit tests, you will need *netcdf* installed and specifically, 196 | the *ncdump* and *ncgen* commands. You will also need the HDF5 package, and specifically the 197 | *h5dump* and *h5repack* commands. These are both available as an Ubuntu packages. 198 | 199 | ```console 200 | $ sudo apt-get install netcdf-bin 201 | $ sudo apt-get install hdf5-tools 202 | ``` 203 | -------------------------------------------------------------------------------- /netcdf/hdf5/floatingPoint.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "fmt" 7 | "io" 8 | "math" 9 | "reflect" 10 | 11 | "github.com/batchatco/go-thrower" 12 | ) 13 | 14 | type floatingPointManagerType struct{} 15 | 16 | var ( 17 | floatingPointManager = floatingPointManagerType{} 18 | _ typeManager = floatingPointManager 19 | ) 20 | 21 | func (floatingPointManagerType) cdlTypeString(sh sigHelper, name string, attr *attribute, origNames map[string]bool) string { 22 | switch attr.length { 23 | case 4: 24 | return "float" 25 | case 8: 26 | return "double" 27 | default: 28 | panic("bad fp length") 29 | } 30 | } 31 | 32 | func (floatingPointManagerType) goTypeString(sh sigHelper, name string, attr *attribute, origNames map[string]bool) string { 33 | switch attr.length { 34 | case 4: 35 | return "float32" 36 | case 8: 37 | return "float64" 38 | default: 39 | panic("bad fp length") 40 | } 41 | } 42 | 43 | func (floatingPointManagerType) alloc(hr heapReader, c caster, bf io.Reader, attr *attribute, 44 | dimensions []uint64) interface{} { 45 | var values interface{} 46 | switch attr.length { 47 | case 4: 48 | values = allocFloats(bf, dimensions, attr.endian) 49 | logger.Info("done alloc floats, rem=", bf.(remReader).Rem()) 50 | case 8: 51 | values = allocDoubles(bf, dimensions, attr.endian) 52 | default: 53 | fail(fmt.Sprintf("bad size float: %d", attr.length)) 54 | } 55 | return values // already converted 56 | } 57 | 58 | func (floatingPointManagerType) defaultFillValue(obj *object, objFillValue []byte, undefinedFillValue bool) []byte { 59 | switch obj.objAttr.length { 60 | case 4: 61 | var fv float32 62 | if undefinedFillValue { 63 | fv = float32(math.NaN()) 64 | } 65 | var buf bytes.Buffer 66 | err := binary.Write(&buf, obj.objAttr.endian, &fv) 67 | thrower.ThrowIfError(err) 68 | objFillValue = buf.Bytes() 69 | logger.Info("fill value encoded", objFillValue) 70 | case 8: 71 | var fv float64 72 | if undefinedFillValue { 73 | fv = math.NaN() 74 | } 75 | var buf bytes.Buffer 76 | err := binary.Write(&buf, obj.objAttr.endian, &fv) 77 | thrower.ThrowIfError(err) 78 | objFillValue = buf.Bytes() 79 | logger.Info("fill value encoded", objFillValue) 80 | default: 81 | thrower.Throw(ErrInternal) 82 | } 83 | return objFillValue 84 | } 85 | 86 | func (floatingPointManagerType) parse(hr heapReader, c caster, attr *attribute, bitFields uint32, bf remReader, df remReader) { 87 | assertError(attr.dtversion == 1, ErrFloatingPoint, "Only support version 1 of float") 88 | logger.Info("* floating-point") 89 | endian := ((bitFields >> 5) & 0b10) | (bitFields & 0b1) 90 | switch endian { 91 | case 0: 92 | attr.endian = binary.LittleEndian 93 | case 1: 94 | attr.endian = binary.BigEndian 95 | default: 96 | fail(fmt.Sprint("unhandled byte order: ", endian)) 97 | } 98 | loPad := (bitFields & 0b10) == 0b10 99 | assertError(!loPad, ErrFloatingPoint, "low pad not supported") 100 | hiPad := (bitFields & 0b100) == 0b100 101 | assertError(!hiPad, ErrFloatingPoint, "high pad not supported") 102 | intPad := (bitFields & 0b1000) == 0b1000 103 | assertError(!intPad, ErrFloatingPoint, "internal pad not supported") 104 | mantissaNormalization := (bitFields >> 4) & 0b11 105 | logger.Info("* mantissa normalization:", mantissaNormalization) 106 | sign := (bitFields >> 8) & 0b11111111 107 | logger.Info("* sign: ", sign) 108 | assert(bf.Rem() >= 12, 109 | fmt.Sprint("Properties need to be at least 12 bytes, was ", bf.Rem())) 110 | bitOffset := read16(bf) 111 | bitPrecision := read16(bf) 112 | exponentLocation := read8(bf) 113 | exponentSize := read8(bf) 114 | mantissaLocation := read8(bf) 115 | mantissaSize := read8(bf) 116 | exponentBias := read32(bf) 117 | 118 | logger.Infof("* bitOffset=%d bitPrecision=%d exponentLocation=%d exponentSize=%d mantissaLocation=%d mantissaSize=%d exponentBias=%d", 119 | bitOffset, 120 | bitPrecision, 121 | exponentLocation, 122 | exponentSize, 123 | mantissaLocation, 124 | mantissaSize, 125 | exponentBias) 126 | assertError(bitOffset == 0, ErrFloatingPoint, "bit offset must be zero") 127 | assertError(mantissaNormalization == 2, ErrFloatingPoint, "mantissa normalization must be 2") 128 | switch attr.length { 129 | case 4: 130 | assertError(sign == 31, ErrFloatingPoint, "float32 sign location must be 31") 131 | assertError(bitPrecision == 32, ErrFloatingPoint, "float32 precision must be 32") 132 | assertError(exponentLocation == 23, ErrFloatingPoint, "float32 exponent location must be 23") 133 | assertError(exponentSize == 8, ErrFloatingPoint, "float32 exponent size must be 8") 134 | assertError(exponentBias == 127, ErrFloatingPoint, "float32 exponent bias must be 127") 135 | case 8: 136 | assertError(sign == 63, ErrFloatingPoint, "float64 sign location must be 63") 137 | assertError(bitPrecision == 64, ErrFloatingPoint, "float64 precision must be 64") 138 | assertError(exponentLocation == 52, ErrFloatingPoint, "float64 exponent location must be 52") 139 | assertError(exponentSize == 11, ErrFloatingPoint, "float64 exponent size must be 11") 140 | assertError(exponentBias == 1023, ErrFloatingPoint, "float64 exponent bias must be 1023") 141 | default: 142 | logger.Error("bad dtlenth for fp", attr.length) 143 | thrower.Throw(ErrFloatingPoint) 144 | } 145 | if df == nil { 146 | logger.Infof("no data") 147 | return 148 | } 149 | logger.Info("data len", df.Rem()) 150 | assert(df.Rem() >= int64(attr.length), "floating-point data short") 151 | attr.df = newResetReaderSave(df, df.Rem()) 152 | } 153 | 154 | func allocFloats(bf io.Reader, dimLengths []uint64, endian binary.ByteOrder) interface{} { 155 | if len(dimLengths) == 0 { 156 | var value float32 157 | err := binary.Read(bf, endian, &value) 158 | thrower.ThrowIfError(err) 159 | return value 160 | } 161 | thisDim := dimLengths[0] 162 | if len(dimLengths) == 1 { 163 | values := make([]float32, thisDim) 164 | err := binary.Read(bf, endian, values) 165 | thrower.ThrowIfError(err) 166 | return values 167 | } 168 | vals := makeSlices(reflect.TypeOf(float32(0)), dimLengths) 169 | for i := uint64(0); i < thisDim; i++ { 170 | vals.Index(int(i)).Set(reflect.ValueOf(allocFloats(bf, dimLengths[1:], endian))) 171 | } 172 | return vals.Interface() 173 | } 174 | 175 | func allocDoubles(bf io.Reader, dimLengths []uint64, endian binary.ByteOrder) interface{} { 176 | if len(dimLengths) == 0 { 177 | var value float64 178 | err := binary.Read(bf, endian, &value) 179 | thrower.ThrowIfError(err) 180 | return value 181 | } 182 | thisDim := dimLengths[0] 183 | if len(dimLengths) == 1 { 184 | values := make([]float64, thisDim) 185 | err := binary.Read(bf, endian, values) 186 | thrower.ThrowIfError(err) 187 | return values 188 | } 189 | vals := makeSlices(reflect.TypeOf(float64(0)), dimLengths) 190 | for i := uint64(0); i < thisDim; i++ { 191 | vals.Index(int(i)).Set(reflect.ValueOf(allocDoubles(bf, dimLengths[1:], endian))) 192 | } 193 | return vals.Interface() 194 | } 195 | -------------------------------------------------------------------------------- /netcdf/hdf5/compound.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "reflect" 7 | "strings" 8 | 9 | "github.com/batchatco/go-thrower" 10 | ) 11 | 12 | type compoundManagerType struct{} 13 | 14 | type compoundField struct { 15 | Name string 16 | Val interface{} 17 | } 18 | type compound []compoundField 19 | 20 | var ( 21 | compoundManager = compoundManagerType{} 22 | _ typeManager = compoundManager 23 | ) 24 | 25 | func (compoundManagerType) cdlTypeString(sh sigHelper, name string, attr *attribute, origNames map[string]bool) string { 26 | members := make([]string, len(attr.children)) 27 | for i, cattr := range attr.children { 28 | ty := cdlTypeString(cattr.class, sh, name, cattr, origNames) 29 | members[i] = fmt.Sprintf("\t%s %s;\n", ty, cattr.name) 30 | } 31 | interior := strings.Join(members, "") 32 | signature := fmt.Sprintf("compound {\n%s}", interior) 33 | namedType := sh.findSignature(signature, name, origNames, cdlTypeString) 34 | if namedType != "" { 35 | return namedType 36 | } 37 | return signature 38 | } 39 | 40 | func (compoundManagerType) goTypeString(sh sigHelper, typeName string, attr *attribute, origNames map[string]bool) string { 41 | members := make([]string, len(attr.children)) 42 | for i, cattr := range attr.children { 43 | ty := goTypeString(cattr.class, sh, typeName, cattr, origNames) 44 | members[i] = fmt.Sprintf("\t%s %s", cattr.name, ty) 45 | } 46 | interior := strings.Join(members, "\n") 47 | signature := fmt.Sprintf("struct {\n%s\n}\n", interior) 48 | namedType := sh.findSignature(signature, typeName, origNames, goTypeString) 49 | if namedType != "" { 50 | return namedType 51 | } 52 | return signature 53 | } 54 | 55 | func (compoundManagerType) alloc(hr heapReader, c caster, bf io.Reader, attr *attribute, 56 | dimensions []uint64) interface{} { 57 | cast := c.cast(*attr) 58 | values := allocCompounds(hr, c, bf, dimensions, *attr, cast) 59 | return values 60 | } 61 | 62 | func (compoundManagerType) defaultFillValue(obj *object, objFillValue []byte, undefinedFillValue bool) []byte { 63 | return objFillValue 64 | } 65 | 66 | func (compoundManagerType) parse(hr heapReader, c caster, attr *attribute, bitFields uint32, bf remReader, df remReader) { 67 | logger.Info("* compound") 68 | logger.Info("attr.dtversion", attr.dtversion) 69 | assert(attr.dtversion >= 1 && attr.dtversion <= maxDTVersion, 70 | fmt.Sprintln("compound datatype version", attr.dtversion, "not supported")) 71 | nmembers := bitFields & 0b11111111 72 | logger.Info("* number of members:", nmembers) 73 | 74 | padding := 7 75 | switch attr.dtversion { 76 | case dtversionStandard, dtversionArray: 77 | break 78 | default: 79 | padding = 0 80 | } 81 | rem := int64(0) 82 | if df != nil { 83 | rem = df.Rem() 84 | } 85 | for i := 0; i < int(nmembers); i++ { 86 | name := readNullTerminatedName(bf, padding) 87 | logger.Info(i, "compound name=", name) 88 | var byteOffset uint32 89 | var nbytes uint8 90 | switch attr.dtversion { 91 | case dtversionStandard, dtversionArray: 92 | nbytes = 4 93 | case dtversionPacked, dtversionV4: 94 | switch { 95 | case attr.length < 256: 96 | nbytes = 1 97 | case attr.length < 65536: 98 | nbytes = 2 99 | case attr.length < 16777216: 100 | nbytes = 3 101 | default: 102 | nbytes = 4 103 | } 104 | } 105 | byteOffset = uint32(readEnc(bf, nbytes)) 106 | logger.Infof("[%d] byteOffset=0x%x", nbytes, byteOffset) 107 | var compoundAttribute attribute 108 | compoundAttribute.name = name 109 | compoundAttribute.byteOffset = byteOffset 110 | if attr.dtversion == dtversionStandard { 111 | dimensionality := read8(bf) 112 | logger.Info("dimensionality", dimensionality) 113 | checkZeroes(bf, 3) 114 | perm := read32(bf) 115 | logger.Info("permutation", perm) 116 | if perm != 0 { 117 | maybeFail( 118 | fmt.Sprint("permutation field should be zero, was ", perm)) 119 | } 120 | reserved := read32(bf) 121 | checkVal(0, reserved, "reserved dt") 122 | compoundAttribute.dimensions = make([]uint64, 4) 123 | for i := 0; i < 4; i++ { 124 | dsize := read32(bf) 125 | logger.Info("dimension", i, "size", dsize) 126 | compoundAttribute.dimensions[i] = uint64(dsize) 127 | } 128 | compoundAttribute.dimensions = compoundAttribute.dimensions[:dimensionality] 129 | } 130 | 131 | logger.Infof("%d compound before: len(prop) = %d len(data) = %d", i, bf.Rem(), rem) 132 | printDatatype(hr, c, bf, nil, 0, &compoundAttribute) 133 | logger.Infof("%d compound after: len(prop) = %d len(data) = %d", i, bf.Rem(), rem) 134 | logger.Infof("%d compound dtlength", compoundAttribute.length) 135 | attr.children = append(attr.children, &compoundAttribute) 136 | } 137 | logger.Info("Compound length is", attr.length) 138 | if rem > 0 { 139 | attrSize := calcAttrSize(attr) 140 | logger.Info("compound alloced", df.Count(), df.Rem()+df.Count(), 141 | "attrSize=", attrSize) 142 | bff := df 143 | if int64(attrSize) > df.Rem() { 144 | logger.Info("Adding fill value reader") 145 | // bff = makeFillValueReader(obj, df, attrSize) 146 | } 147 | attr.df = newResetReaderSave(bff, bff.Rem()) 148 | logger.Info("rem=", df.Rem(), "nread=", bff.Count()) 149 | } 150 | logger.Info("Finished compound", "rem=", bf.Rem()) 151 | } 152 | 153 | func allocCompounds(hr heapReader, cstr caster, bf io.Reader, dimLengths []uint64, attr attribute, 154 | cast reflect.Type) interface{} { 155 | length := int64(attr.length) 156 | class := typeNames[attr.class] 157 | logger.Info(bf.(remReader).Count(), "Alloc compounds", dimLengths, class, 158 | "length=", length, 159 | "nchildren=", len(attr.children), "rem=", bf.(remReader).Rem()) 160 | dtlen := uint64(0) 161 | for i := range attr.children { 162 | clen := uint64(calcAttrSize(attr.children[i])) 163 | dtlen += clen 164 | } 165 | if len(dimLengths) == 0 { 166 | rem := bf.(remReader).Rem() 167 | if length > rem { 168 | logger.Warn("not enough room", length, rem) 169 | } else { 170 | rem = length 171 | } 172 | cbf := newResetReader(bf, rem) 173 | varray := make([]compoundField, len(attr.children)) 174 | for i, c := range attr.children { 175 | byteOffset := c.byteOffset 176 | clen := uint64(calcAttrSize(c)) 177 | if int64(byteOffset) > cbf.Count() { 178 | skipLen := int64(byteOffset) - cbf.Count() 179 | logger.Info("skip to offset", skipLen) 180 | skip(cbf, int64(skipLen)) 181 | } 182 | ccbf := newResetReader(cbf, int64(clen)) 183 | varray[i].Val = getDataAttr(hr, cstr, ccbf, *c) 184 | varray[i].Name = c.name 185 | } 186 | if cbf.Count() < length { 187 | rem := length - cbf.Count() 188 | skip(cbf, int64(rem)) 189 | } 190 | if cast != nil { 191 | fields := make([]reflect.StructField, len(attr.children)) 192 | for i := range fields { 193 | fields[i] = cast.Field(i) 194 | } 195 | stp := reflect.New(reflect.StructOf(fields)) 196 | st := reflect.Indirect(stp) 197 | for i := range varray { 198 | assertError(st.Field(i).CanSet(), ErrNonExportedField, 199 | "can't set non-exported field") 200 | st.Field(i).Set(reflect.ValueOf(varray[i].Val)) 201 | } 202 | return st.Interface() 203 | } 204 | logger.Info(bf.(remReader).Count(), "return compound count=", bf.(remReader).Count()) 205 | return compound(varray) 206 | } 207 | var t reflect.Type 208 | if cast != nil { 209 | t = cast 210 | } else { 211 | var x compound 212 | t = reflect.TypeOf(x) 213 | } 214 | vals2 := makeSlices(t, dimLengths) 215 | thisDim := dimLengths[0] 216 | for i := uint64(0); i < thisDim; i++ { 217 | if !vals2.Index(int(i)).CanSet() { 218 | thrower.Throw(ErrNonExportedField) 219 | } 220 | vals2.Index(int(i)).Set(reflect.ValueOf(allocCompounds(hr, cstr, bf, dimLengths[1:], attr, cast))) 221 | } 222 | return vals2.Interface() 223 | } 224 | -------------------------------------------------------------------------------- /netcdf/util/orderedmap.go: -------------------------------------------------------------------------------- 1 | // Package util is to create AttributeMaps for the NetCDF API 2 | package util 3 | 4 | import ( 5 | "errors" 6 | "fmt" 7 | "reflect" 8 | "sort" 9 | "strconv" 10 | "strings" 11 | ) 12 | 13 | type funcCallback func(name string) (string, bool) 14 | 15 | type OrderedMap struct { 16 | keys []string 17 | values map[string]interface{} 18 | visibleKeys []string 19 | hiddenKeys map[string]bool 20 | regTypeCallback funcCallback 21 | goTypeCallback funcCallback 22 | } 23 | 24 | var ( 25 | ErrorKeysDontMatchValues = errors.New("keys don't match values") 26 | ) 27 | 28 | // NewOrderedMap takes an unordered map (values) and an order (keys) and 29 | // returns an OrderedMap. 30 | func NewOrderedMap(keys []string, values map[string]interface{}) (*OrderedMap, error) { 31 | if len(keys) != len(values) { 32 | return nil, ErrorKeysDontMatchValues 33 | } 34 | mapKeys := []string{} 35 | for k := range values { 36 | mapKeys = append(mapKeys, k) 37 | } 38 | sort.Strings(mapKeys) 39 | 40 | sortedKeys := make([]string, len(keys)) 41 | copy(sortedKeys, keys) 42 | sort.Strings(sortedKeys) 43 | 44 | for i := range sortedKeys { 45 | if mapKeys[i] != sortedKeys[i] { 46 | return nil, ErrorKeysDontMatchValues 47 | } 48 | } 49 | if values == nil { 50 | values = map[string]interface{}{} 51 | } 52 | 53 | visibleKeys := []string{} 54 | visibleKeys = append(visibleKeys, keys...) 55 | 56 | return &OrderedMap{ 57 | keys: keys, 58 | values: values, 59 | visibleKeys: visibleKeys, 60 | hiddenKeys: map[string]bool{}}, nil 61 | } 62 | 63 | // Add adds a key/value pair to an ordered map at the end. 64 | func (om *OrderedMap) Add(name string, val interface{}) { 65 | if !om.hiddenKeys[name] { 66 | om.keys = append(om.keys, name) 67 | om.visibleKeys = append(om.visibleKeys, name) 68 | } 69 | om.values[name] = val 70 | } 71 | 72 | // Get returns the value associated with key and sets has to true if found. 73 | func (om *OrderedMap) Get(key string) (val interface{}, has bool) { 74 | val, has = om.values[key] 75 | return 76 | } 77 | 78 | // SetTypeCallbacks allows one to override the default type behavior. 79 | // The default type calculation doesn't work with user-defined types. 80 | // It's okay for CDF though, but not HDF5. 81 | func (om *OrderedMap) SetTypeCallbacks(regCb funcCallback, goCb funcCallback) { 82 | om.regTypeCallback = regCb 83 | om.goTypeCallback = goCb 84 | } 85 | 86 | // Hide hides the given key from the Keys() method, but 87 | // it is still in the map. 88 | func (om *OrderedMap) Hide(hiddenKey string) { 89 | om.hiddenKeys[hiddenKey] = true 90 | // recompute visible keys 91 | visibleKeys := []string{} 92 | for _, key := range om.keys { 93 | if om.hiddenKeys[key] { 94 | continue 95 | } 96 | visibleKeys = append(visibleKeys, key) 97 | } 98 | om.visibleKeys = visibleKeys 99 | } 100 | 101 | // Keys returns the keys in order for the map. 102 | // It does not return the hidden keys. 103 | func (om *OrderedMap) Keys() []string { 104 | return om.visibleKeys 105 | } 106 | 107 | func (om *OrderedMap) pGoType(val interface{}) (string, bool) { 108 | rVal := reflect.ValueOf(val) 109 | var prelim string 110 | switch rVal.Kind() { 111 | case reflect.Array: 112 | var i interface{} 113 | if rVal.Len() == 0 { 114 | i = reflect.Zero(rVal.Type().Elem()).Interface() 115 | } else { 116 | i = rVal.Index(0).Interface() 117 | } 118 | inner, has := om.pGoType(i) 119 | if !has { 120 | return "", false 121 | } 122 | prelim = fmt.Sprintf("[%d]%s", rVal.Len(), inner) 123 | case reflect.Slice: 124 | var i interface{} 125 | if rVal.Len() == 0 { 126 | i = reflect.Zero(rVal.Type().Elem()).Interface() 127 | } else { 128 | i = rVal.Index(0).Interface() 129 | } 130 | inner, has := om.pGoType(i) 131 | if !has { 132 | return "", false 133 | } 134 | prelim = fmt.Sprintf("[]%s", inner) 135 | case reflect.Struct: 136 | inner := "" 137 | for i := 0; i < rVal.NumField(); i++ { 138 | field := rVal.Type().Field(i) 139 | fName := field.Name 140 | var fType string 141 | var has bool 142 | var fi interface{} 143 | if field.PkgPath != "" { 144 | ft := rVal.Field(i).Type() 145 | fi = reflect.Zero(ft).Interface() 146 | } else { 147 | fi = rVal.Field(i).Interface() 148 | } 149 | fType, has = om.pGoType(fi) 150 | if !has { 151 | return "", false 152 | } 153 | if inner != "" { 154 | inner = inner + ";" 155 | } 156 | inner = inner + fmt.Sprintf(" %s %s", fName, fType) 157 | } 158 | prelim = fmt.Sprintf("struct {%s }", inner) 159 | default: 160 | return rVal.Kind().String(), true 161 | } 162 | return prelim, true 163 | } 164 | 165 | // GetGoType returns the Go description of the given variable and sets the 166 | // bool to true if found. 167 | func (om *OrderedMap) GetGoType(key string) (ty string, has bool) { 168 | val, has := om.Get(key) 169 | if !has { 170 | return "", false 171 | } 172 | if om.goTypeCallback != nil { 173 | return om.goTypeCallback(key) 174 | } 175 | return om.pGoType(val) 176 | } 177 | 178 | // We end up with dimensions not being in the right CDL format 179 | // from pType. 180 | // It can look like this: int(5)(1,2,3) 181 | func (om *OrderedMap) putDim(inner string, dim int, kind reflect.Kind) string { 182 | var sDim string 183 | if dim == 0 && kind == reflect.Slice { 184 | sDim = "*" 185 | } else { 186 | sDim = strconv.Itoa(dim) 187 | } 188 | parenLoc := strings.LastIndex(inner, "(") 189 | if parenLoc == -1 || parenLoc == len(inner)-1 { 190 | return fmt.Sprintf("%s(%s)", inner, sDim) 191 | } 192 | fixed := fmt.Sprintf("%s%s,%s", inner[:parenLoc+1], sDim, inner[parenLoc+1:]) 193 | return fixed 194 | } 195 | 196 | func (om *OrderedMap) pType(val interface{}) (string, bool) { 197 | rVal := reflect.ValueOf(val) 198 | rTyp := reflect.TypeOf(val) 199 | prelim := "" 200 | switch rVal.Kind() { 201 | case reflect.Array: 202 | var i interface{} 203 | if rVal.Len() == 0 { 204 | i = reflect.Zero(rTyp.Elem()).Interface() 205 | } else { 206 | i = rVal.Index(0).Interface() 207 | } 208 | inner, has := om.pType(i) 209 | if !has || inner == "" { 210 | return "", false 211 | } 212 | prelim = om.putDim(inner, rVal.Len(), rVal.Kind()) 213 | case reflect.Slice: 214 | var i interface{} 215 | if rVal.Len() == 0 { 216 | i = reflect.Zero(rTyp.Elem()).Interface() 217 | } else { 218 | i = rVal.Index(0).Interface() 219 | } 220 | inner, has := om.pType(i) 221 | if !has { 222 | return "", false 223 | } 224 | prelim = om.putDim(inner, rVal.Len(), rVal.Kind()) 225 | 226 | case reflect.Struct: 227 | inner := "" 228 | for i := 0; i < rVal.NumField(); i++ { 229 | field := rTyp.Field(i) 230 | fName := field.Name 231 | var fType string 232 | var has bool 233 | var fi interface{} 234 | if field.PkgPath != "" { 235 | ft := rVal.Field(i).Type() 236 | fi = reflect.Zero(ft).Interface() 237 | } else { 238 | fi = rVal.Field(i).Interface() 239 | } 240 | fType, has = om.pType(fi) 241 | if !has { 242 | return "", false 243 | } 244 | inner = inner + fmt.Sprintf(" %s %s;", fType, fName) 245 | } 246 | prelim = fmt.Sprintf("compound {%s }", inner) 247 | default: 248 | kind := rVal.Kind().String() 249 | replacements := map[string]string{ 250 | "float32": "float", 251 | "float64": "double", 252 | "int8": "byte", 253 | "int16": "short", 254 | "int32": "int", 255 | "uint8": "ubyte", 256 | "uint16": "ushort", 257 | "uint32": "uint", 258 | "string": "string", 259 | } 260 | rep, has := replacements[kind] 261 | if !has { 262 | return kind, true // XXX 263 | } 264 | return rep, true 265 | } 266 | return prelim, true 267 | } 268 | 269 | // GetType returns the CDL description of the given variable and sets the 270 | // bool to true if found. 271 | func (om *OrderedMap) GetType(key string) (ty string, has bool) { 272 | val, has := om.Get(key) 273 | if !has { 274 | return "", false 275 | } 276 | if om.regTypeCallback != nil { 277 | return om.regTypeCallback(key) 278 | } 279 | 280 | prelim, has := om.pType(val) 281 | if !has { 282 | return "", false 283 | } 284 | // Put in the commas. 285 | return strings.ReplaceAll(prelim, ")(", ","), true 286 | } 287 | -------------------------------------------------------------------------------- /netcdf/hdf5/fixedPoint.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "fmt" 7 | "io" 8 | "math" 9 | "reflect" 10 | 11 | "github.com/batchatco/go-thrower" 12 | ) 13 | 14 | type fixedPointManagerType struct{} 15 | 16 | var ( 17 | fixedPointManager = fixedPointManagerType{} 18 | _ typeManager = fixedPointManager 19 | ) 20 | 21 | func (fixedPointManagerType) cdlTypeString(sh sigHelper, name string, attr *attribute, origNames map[string]bool) string { 22 | prefix := "" 23 | if !attr.signed { 24 | prefix = "u" 25 | } 26 | switch attr.length { 27 | case 1: 28 | return prefix + "byte" 29 | case 2: 30 | return prefix + "short" 31 | case 4: 32 | return prefix + "int" 33 | case 8: 34 | return prefix + "int64" 35 | default: 36 | panic("bad int length") 37 | } 38 | } 39 | 40 | func (fixedPointManagerType) goTypeString(sh sigHelper, name string, attr *attribute, origNames map[string]bool) string { 41 | prefix := "" 42 | if !attr.signed { 43 | prefix = "u" 44 | } 45 | switch attr.length { 46 | case 1: 47 | return prefix + "int8" 48 | case 2: 49 | return prefix + "int16" 50 | case 4: 51 | return prefix + "int32" 52 | case 8: 53 | return prefix + "int64" 54 | default: 55 | panic("bad int length") 56 | } 57 | } 58 | 59 | func (fixedPointManagerType) alloc(hr heapReader, c caster, bf io.Reader, attr *attribute, 60 | dimensions []uint64) interface{} { 61 | var values interface{} 62 | switch attr.length { 63 | case 1: 64 | values = allocInt8s(bf, dimensions, attr.signed, nil) 65 | case 2: 66 | values = allocShorts(bf, dimensions, attr.endian, attr.signed, nil) 67 | case 4: 68 | values = allocInts(bf, dimensions, attr.endian, attr.signed, nil) 69 | case 8: 70 | values = allocInt64s(bf, dimensions, attr.endian, attr.signed, nil) 71 | default: 72 | fail(fmt.Sprintf("bad size fixed: %d (%v)", attr.length, attr)) 73 | } 74 | return values // already converted 75 | } 76 | 77 | func (fixedPointManagerType) defaultFillValue(obj *object, objFillValue []byte, undefinedFillValue bool) []byte { 78 | switch obj.objAttr.length { 79 | case 1: 80 | if undefinedFillValue { 81 | fv := math.MinInt8 + 1 82 | objFillValue = []byte{byte(fv)} 83 | } 84 | case 2: 85 | if undefinedFillValue { 86 | fv := int16(math.MinInt16 + 1) 87 | var bb bytes.Buffer 88 | err := binary.Write(&bb, obj.objAttr.endian, fv) 89 | thrower.ThrowIfError(err) 90 | objFillValue = bb.Bytes() 91 | } 92 | case 4: 93 | if undefinedFillValue { 94 | fv := int32(math.MinInt32 + 1) 95 | var bb bytes.Buffer 96 | err := binary.Write(&bb, obj.objAttr.endian, fv) 97 | thrower.ThrowIfError(err) 98 | objFillValue = bb.Bytes() 99 | } 100 | case 8: 101 | if undefinedFillValue { 102 | fv := int64(math.MinInt64 + 1) 103 | var bb bytes.Buffer 104 | err := binary.Write(&bb, obj.objAttr.endian, fv) 105 | thrower.ThrowIfError(err) 106 | objFillValue = bb.Bytes() 107 | } 108 | } 109 | return objFillValue 110 | } 111 | 112 | func (fixedPointManagerType) parse(hr heapReader, c caster, attr *attribute, bitFields uint32, bf remReader, df remReader) { 113 | logger.Info("* fixed-point") 114 | // Same structure for all versions, no need to check 115 | byteOrder := bitFields & 0b1 116 | paddingType := (bitFields >> 1) & 0b11 117 | signed := (bitFields >> 3) & 0b1 118 | attr.signed = signed == 0b1 119 | logger.Infof("byteOrder=%d paddingType=%d, signed=%d", byteOrder, paddingType, signed) 120 | if byteOrder != 0 { 121 | attr.endian = binary.BigEndian 122 | } else { 123 | attr.endian = binary.LittleEndian 124 | } 125 | assertError(paddingType == 0, ErrFixedPoint, 126 | fmt.Sprintf("fixed point padding must be zero 0x%x", bitFields)) 127 | logger.Info("len properties", bf.Rem()) 128 | assert(bf.Rem() > 0, "properties should be here") 129 | bitOffset := read16(bf) 130 | bitPrecision := read16(bf) 131 | logger.Infof("bitOffset=%d bitPrecision=%d blen=%d", bitOffset, bitPrecision, 132 | bf.Count()) 133 | assertError(bitOffset == 0, ErrFixedPoint, "bit offset must be zero") 134 | switch attr.length { 135 | case 1, 2, 4, 8: 136 | break 137 | default: 138 | thrower.Throw(ErrFixedPoint) 139 | } 140 | if df == nil { 141 | logger.Infof("no data") 142 | return 143 | } 144 | if df.Rem() >= int64(attr.length) { 145 | attr.df = newResetReaderSave(df, df.Rem()) 146 | } 147 | } 148 | 149 | func allocInt8s(bf io.Reader, dimLengths []uint64, signed bool, cast reflect.Type) interface{} { 150 | if cast == nil { 151 | if signed { 152 | cast = reflect.TypeOf(int8(0)) 153 | } else { 154 | cast = reflect.TypeOf(uint8(0)) 155 | } 156 | } 157 | if len(dimLengths) == 0 { 158 | value := read8(bf) 159 | return reflect.ValueOf(value).Convert(cast).Interface() 160 | } 161 | thisDim := dimLengths[0] 162 | if len(dimLengths) == 1 { 163 | values := reflect.MakeSlice(reflect.SliceOf(cast), int(thisDim), int(thisDim)).Interface() 164 | err := binary.Read(bf, binary.LittleEndian, values) 165 | thrower.ThrowIfError(err) 166 | return values 167 | } 168 | vals := makeSlices(cast, dimLengths) 169 | for i := uint64(0); i < thisDim; i++ { 170 | vals.Index(int(i)).Set(reflect.ValueOf(allocInt8s(bf, dimLengths[1:], signed, cast))) 171 | } 172 | return vals.Interface() 173 | } 174 | 175 | func allocShorts(bf io.Reader, dimLengths []uint64, endian binary.ByteOrder, signed bool, 176 | cast reflect.Type) interface{} { 177 | if cast == nil { 178 | if signed { 179 | cast = reflect.TypeOf(int16(0)) 180 | } else { 181 | cast = reflect.TypeOf(uint16(0)) 182 | } 183 | } 184 | if len(dimLengths) == 0 { 185 | var value uint16 186 | err := binary.Read(bf, endian, &value) 187 | thrower.ThrowIfError(err) 188 | return reflect.ValueOf(value).Convert(cast).Interface() 189 | } 190 | thisDim := dimLengths[0] 191 | if len(dimLengths) == 1 { 192 | values := reflect.MakeSlice(reflect.SliceOf(cast), int(thisDim), int(thisDim)).Interface() 193 | err := binary.Read(bf, endian, values) 194 | thrower.ThrowIfError(err) 195 | return values 196 | } 197 | vals := makeSlices(cast, dimLengths) 198 | for i := uint64(0); i < thisDim; i++ { 199 | vals.Index(int(i)).Set(reflect.ValueOf(allocShorts(bf, dimLengths[1:], endian, signed, 200 | cast))) 201 | } 202 | return vals.Interface() 203 | } 204 | 205 | func allocInts(bf io.Reader, dimLengths []uint64, endian binary.ByteOrder, signed bool, 206 | cast reflect.Type) interface{} { 207 | if cast == nil { 208 | if signed { 209 | cast = reflect.TypeOf(int32(0)) 210 | } else { 211 | cast = reflect.TypeOf(uint32(0)) 212 | } 213 | } 214 | if len(dimLengths) == 0 { 215 | var value uint32 216 | err := binary.Read(bf, endian, &value) 217 | thrower.ThrowIfError(err) 218 | return reflect.ValueOf(value).Convert(cast).Interface() 219 | } 220 | thisDim := dimLengths[0] 221 | if len(dimLengths) == 1 { 222 | values := reflect.MakeSlice(reflect.SliceOf(cast), int(thisDim), int(thisDim)).Interface() 223 | err := binary.Read(bf, endian, values) 224 | thrower.ThrowIfError(err) 225 | return values 226 | } 227 | vals := makeSlices(cast, dimLengths) 228 | for i := uint64(0); i < thisDim; i++ { 229 | vals.Index(int(i)).Set(reflect.ValueOf(allocInts(bf, dimLengths[1:], endian, signed, 230 | cast))) 231 | } 232 | return vals.Interface() 233 | } 234 | 235 | func allocInt64s(bf io.Reader, dimLengths []uint64, endian binary.ByteOrder, signed bool, 236 | cast reflect.Type) interface{} { 237 | if cast == nil { 238 | if signed { 239 | cast = reflect.TypeOf(int64(0)) 240 | } else { 241 | cast = reflect.TypeOf(uint64(0)) 242 | } 243 | } 244 | if len(dimLengths) == 0 { 245 | var value uint64 246 | err := binary.Read(bf, endian, &value) 247 | thrower.ThrowIfError(err) 248 | return reflect.ValueOf(value).Convert(cast).Interface() 249 | } 250 | thisDim := dimLengths[0] 251 | if len(dimLengths) == 1 { 252 | values := reflect.MakeSlice(reflect.SliceOf(cast), int(thisDim), int(thisDim)).Interface() 253 | err := binary.Read(bf, endian, values) 254 | thrower.ThrowIfError(err) 255 | return values 256 | } 257 | vals := makeSlices(cast, dimLengths) 258 | for i := uint64(0); i < thisDim; i++ { 259 | vals.Index(int(i)).Set(reflect.ValueOf(allocInt64s(bf, dimLengths[1:], endian, signed, 260 | cast))) 261 | } 262 | return vals.Interface() 263 | } 264 | -------------------------------------------------------------------------------- /netcdf/hdf5/vlen.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "io" 7 | "reflect" 8 | 9 | "github.com/batchatco/go-thrower" 10 | ) 11 | 12 | type vlenManagerType struct{} 13 | 14 | var ( 15 | vlenManager = vlenManagerType{} 16 | _ typeManager = vlenManager 17 | ) 18 | 19 | func (vlenManagerType) cdlTypeString(sh sigHelper, name string, attr *attribute, origNames map[string]bool) string { 20 | if attr.vtType == 1 { 21 | // It's a string 22 | return "string" 23 | } 24 | vAttr := attr.children[0] 25 | ty := cdlTypeString(vAttr.class, sh, name, vAttr, origNames) 26 | assert(ty != "", "unable to parse vlen attr") 27 | signature := fmt.Sprintf("%s(*)", ty) 28 | namedType := sh.findSignature(signature, name, origNames, cdlTypeString) 29 | if namedType != "" { 30 | return namedType 31 | } 32 | return signature 33 | } 34 | 35 | func (vlenManagerType) goTypeString(sh sigHelper, typeName string, attr *attribute, origNames map[string]bool) string { 36 | if attr.vtType == 1 { 37 | // It's a string 38 | return "string" 39 | } 40 | vAttr := attr.children[0] 41 | ty := goTypeString(vAttr.class, sh, typeName, vAttr, origNames) 42 | assert(ty != "", "unable to parse vlen attr") 43 | signature := fmt.Sprintf("[]%s", ty) 44 | namedType := sh.findSignature(signature, typeName, origNames, goTypeString) 45 | if namedType != "" { 46 | return namedType 47 | } 48 | return signature 49 | } 50 | 51 | func (vlenManagerType) parse(hr heapReader, c caster, attr *attribute, bitFields uint32, bf remReader, df remReader) { 52 | logger.Info("* variable-length, dtlength=", attr.length, 53 | "proplen=", bf.Rem()) 54 | // checkVal(1, dtversion, "Only support version 1 of variable-length") 55 | vtType := uint8(bitFields & 0b1111) // XXX: we will need other bits too for decoding 56 | vtPad := uint8(bitFields>>4) & 0b1111 57 | // The value of pad here may not have anything to do with reading data, just 58 | // writing. So we could accept all of them 59 | assert(vtPad == 0 || vtPad == 1, "only do v0 and v1 versions of VL padding") 60 | vtCset := (bitFields >> 8) & 0b1111 61 | logger.Infof("type=%d paddingtype=%d cset=%d", vtType, vtPad, vtCset) 62 | switch vtType { 63 | case 0: 64 | checkVal(0, vtCset, "cset when not string") 65 | logger.Infof("sequence") 66 | case 1: 67 | if vtCset == 0 { 68 | logger.Infof("string (ascii)") 69 | } else { 70 | logger.Infof("string (utf8)") 71 | } 72 | default: 73 | fail("unknown variable-length type") 74 | } 75 | var variableAttr attribute 76 | printDatatype(hr, c, bf, nil, 0, &variableAttr) 77 | logger.Info("variable class", variableAttr.class, "vtType", vtType) 78 | attr.children = append(attr.children, &variableAttr) 79 | attr.vtType = vtType 80 | rem := int64(0) 81 | if df != nil { 82 | rem = df.Rem() 83 | } 84 | if rem < int64(attr.length) { 85 | logger.Infof("variable-length short data: %d vs. %d", rem, attr.length) 86 | return 87 | } 88 | logger.Info("len data is", rem, "dlen", df.Count()) 89 | 90 | attr.df = newResetReaderSave(df, df.Rem()) 91 | logger.Infof("Type of this vattr: %T", attr.value) 92 | } 93 | 94 | func (vlenManagerType) defaultFillValue(obj *object, objFillValue []byte, undefinedFillValue bool) []byte { 95 | return []byte{0} 96 | } 97 | 98 | func (vlenManagerType) alloc(hr heapReader, c caster, bf io.Reader, attr *attribute, 99 | dimensions []uint64) interface{} { 100 | logger.Info("dimensions=", dimensions) 101 | if attr.vtType == 1 { 102 | // It's a string 103 | // TODO: use the padding and character set information 104 | logger.Info("variable-length string", len(dimensions)) 105 | return allocStrings(hr, bf, dimensions) // already converted 106 | } 107 | logger.Info("variable-length type", typeNames[int(attr.children[0].class)]) 108 | logger.Info("dimensions=", dimensions, "rem=", bf.(remReader).Rem()) 109 | cast := c.cast(*attr) 110 | values := allocVariable(hr, c, bf, dimensions, *attr.children[0], cast) 111 | logger.Infof("vl kind %T", values) 112 | if cast != nil { 113 | return values 114 | } 115 | return convert(values) 116 | } 117 | 118 | func allocStrings(hr heapReader, bf io.Reader, dimLengths []uint64) interface{} { 119 | logger.Info("allocStrings", dimLengths) 120 | if len(dimLengths) == 0 { 121 | // alloc one scalar 122 | var length uint32 123 | var addr uint64 124 | var index uint32 125 | 126 | var err error 127 | err = binary.Read(bf, binary.LittleEndian, &length) 128 | thrower.ThrowIfError(err) 129 | err = binary.Read(bf, binary.LittleEndian, &addr) 130 | thrower.ThrowIfError(err) 131 | err = binary.Read(bf, binary.LittleEndian, &index) 132 | thrower.ThrowIfError(err) 133 | logger.Infof("String length %d (0x%x), addr 0x%x, index %d (0x%x)", 134 | length, length, addr, index, index) 135 | if length == 0 { 136 | return "" 137 | } 138 | bff, sz := hr.readGlobalHeap(addr, index) 139 | s := make([]byte, sz) 140 | read(bff, s) 141 | logger.Info("string=", string(s)) 142 | return getString(s) // TODO: should be s[:length] 143 | } 144 | thisDim := dimLengths[0] 145 | if len(dimLengths) == 1 { 146 | values := make([]string, thisDim) 147 | for i := uint64(0); i < thisDim; i++ { 148 | var length uint32 149 | var addr uint64 150 | var index uint32 151 | 152 | err := binary.Read(bf, binary.LittleEndian, &length) 153 | thrower.ThrowIfError(err) 154 | err = binary.Read(bf, binary.LittleEndian, &addr) 155 | thrower.ThrowIfError(err) 156 | err = binary.Read(bf, binary.LittleEndian, &index) 157 | thrower.ThrowIfError(err) 158 | logger.Infof("String length %d (0x%x), addr 0x%x, index %d (0x%x)", 159 | length, length, addr, index, index) 160 | if length == 0 { 161 | values[i] = "" 162 | continue 163 | } 164 | bff, sz := hr.readGlobalHeap(addr, index) 165 | s := make([]byte, sz) 166 | read(bff, s) 167 | values[i] = getString(s) // TODO: should be s[:length] 168 | } 169 | return values 170 | } 171 | ty := reflect.TypeOf("") 172 | vals := makeSlices(ty, dimLengths) 173 | for i := uint64(0); i < thisDim; i++ { 174 | vals.Index(int(i)).Set(reflect.ValueOf(allocStrings(hr, bf, dimLengths[1:]))) 175 | } 176 | return vals.Interface() 177 | } 178 | 179 | func allocVariable(hr heapReader, c caster, bf io.Reader, dimLengths []uint64, attr attribute, 180 | cast reflect.Type) interface{} { 181 | logger.Info("allocVariable", dimLengths, "count=", bf.(remReader).Count(), 182 | "rem=", bf.(remReader).Rem()) 183 | if len(dimLengths) == 0 { 184 | var length uint32 185 | var addr uint64 186 | var index uint32 187 | err := binary.Read(bf, binary.LittleEndian, &length) 188 | thrower.ThrowIfError(err) 189 | err = binary.Read(bf, binary.LittleEndian, &addr) 190 | thrower.ThrowIfError(err) 191 | err = binary.Read(bf, binary.LittleEndian, &index) 192 | thrower.ThrowIfError(err) 193 | logger.Infof("length %d(0x%x) addr 0x%x index %d(0x%x)\n", 194 | length, length, addr, index, index) 195 | var val0 interface{} 196 | var s []byte 197 | var bff remReader 198 | if length == 0 { 199 | // If there's no value to read, we fake one to get the type. 200 | attr.dimensions = nil 201 | s = make([]byte, attr.length) 202 | bff = newResetReaderFromBytes(s) 203 | } else { 204 | bff, _ = hr.readGlobalHeap(addr, index) 205 | } 206 | var t reflect.Type 207 | val0 = getDataAttr(hr, c, bff, attr) 208 | if cast != nil { 209 | t = cast.Elem() 210 | } else { 211 | t = reflect.ValueOf(val0).Type() 212 | } 213 | sl := reflect.MakeSlice(reflect.SliceOf(t), int(length), int(length)) 214 | if cast != nil { 215 | sl = sl.Convert(cast) 216 | } 217 | if length > 0 { 218 | sl.Index(0).Set(reflect.ValueOf(val0)) 219 | for i := 1; i < int(length); i++ { 220 | val := getDataAttr(hr, c, bff, attr) 221 | sl.Index(i).Set(reflect.ValueOf(val)) 222 | } 223 | } 224 | return sl.Interface() 225 | } 226 | thisDim := dimLengths[0] 227 | if len(dimLengths) == 1 { 228 | // For scalars, this can be faster using binary.Read 229 | vals := make([]interface{}, thisDim) 230 | for i := uint64(0); i < thisDim; i++ { 231 | logger.Info("Alloc inner", i, "of", thisDim) 232 | vals[i] = allocVariable(hr, c, bf, dimLengths[1:], attr, cast) 233 | } 234 | assert(vals[0] != nil, "we never return nil") 235 | t := reflect.ValueOf(vals[0]).Type() 236 | vals2 := reflect.MakeSlice(reflect.SliceOf(t), int(thisDim), int(thisDim)) 237 | for i := 0; i < int(thisDim); i++ { 238 | vals2.Index(i).Set(reflect.ValueOf(vals[i])) 239 | } 240 | return vals2.Interface() 241 | } 242 | 243 | // TODO: we sometimes know the type (float32) and can do something smarter here 244 | 245 | vals := make([]interface{}, thisDim) 246 | for i := uint64(0); i < thisDim; i++ { 247 | logger.Info("Alloc outer", i, "of", thisDim) 248 | vals[i] = allocVariable(hr, c, bf, dimLengths[1:], attr, cast) 249 | } 250 | t := reflect.ValueOf(vals[0]).Type() 251 | vals2 := reflect.MakeSlice(reflect.SliceOf(t), int(thisDim), int(thisDim)) 252 | for i := 0; i < int(thisDim); i++ { 253 | vals2.Index(i).Set(reflect.ValueOf(vals[i])) 254 | } 255 | return vals2.Interface() 256 | } 257 | -------------------------------------------------------------------------------- /netcdf/hdf5/types_test.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | func TestAttrTypes(t *testing.T) { 9 | fileName := "testvlen" 10 | genName := ncGen(t, fileName) 11 | if genName == "" { 12 | t.Error("Error opening", fileName, ":", errorNcGen) 13 | return 14 | } 15 | defer os.Remove(genName) 16 | nc, err := Open(genName) 17 | if err != nil { 18 | t.Error("Error opening", genName, ":", err) 19 | return 20 | } 21 | defer nc.Close() 22 | aName := "Tricky" 23 | at, has := nc.Attributes().GetType(aName) 24 | if !has { 25 | t.Error("Can't find type for attribute", aName) 26 | return 27 | } 28 | tName := "Tricky_t" 29 | if at != tName { 30 | t.Error("wrong type for attribute", aName, "got=", at, "exp=", tName) 31 | } 32 | } 33 | 34 | func TestGlobalAttrTypes(t *testing.T) { 35 | fileName := "testvlen" 36 | genName := ncGen(t, fileName) 37 | if genName == "" { 38 | t.Error("Error opening", fileName, ":", errorNcGen) 39 | return 40 | } 41 | defer os.Remove(genName) 42 | nc, err := Open(genName) 43 | if err != nil { 44 | t.Error("Error opening", genName, ":", err) 45 | return 46 | } 47 | defer nc.Close() 48 | aName := "Tricky" 49 | at, has := nc.Attributes().GetType(aName) 50 | if !has { 51 | t.Error("Can't find type for attribute", aName) 52 | return 53 | } 54 | tName := "Tricky_t" 55 | if at != tName { 56 | t.Error("wrong type for attribute", aName, "got=", at, "exp=", tName) 57 | } 58 | } 59 | 60 | func TestLocalAttrTypes(t *testing.T) { 61 | fileName := "testvlen" 62 | genName := ncGen(t, fileName) 63 | if genName == "" { 64 | t.Error("Error opening", fileName, ":", errorNcGen) 65 | return 66 | } 67 | defer os.Remove(genName) 68 | nc, err := Open(genName) 69 | if err != nil { 70 | t.Error("Error opening", genName, ":", err) 71 | return 72 | } 73 | defer nc.Close() 74 | 75 | type vatt struct { 76 | varName string 77 | attrName string 78 | typeName string 79 | goTypeName string 80 | } 81 | vatts := []vatt{ 82 | {"v", "Tricky", "Tricky_t", "Tricky_t"}, 83 | {"v2", "Vint", "vint(*)", "[]vint"}, 84 | } 85 | for _, v := range vatts { 86 | vg, err := nc.GetVarGetter(v.varName) 87 | if err != nil { 88 | t.Error(err) 89 | continue 90 | } 91 | at := vg.Attributes() 92 | tt, has := at.GetType(v.attrName) 93 | if !has { 94 | t.Error("missing attr", v.attrName) 95 | continue 96 | } 97 | if tt != v.typeName { 98 | t.Error("wrong type for attribute", v.attrName, ",", tt) 99 | } 100 | gt, has := at.GetGoType(v.attrName) 101 | if !has { 102 | t.Error("missing attr", v.attrName) 103 | continue 104 | } 105 | if gt != v.goTypeName { 106 | t.Error("wrong go type for attribute", v.attrName, ",", gt) 107 | } 108 | } 109 | } 110 | 111 | func TestVarTypes(t *testing.T) { 112 | fileName := "testvtypes" 113 | genName := ncGen(t, fileName) 114 | if genName == "" { 115 | t.Error("Error opening", fileName, ":", errorNcGen) 116 | return 117 | } 118 | defer os.Remove(genName) 119 | nc, err := Open(genName) 120 | if err != nil { 121 | t.Error("Error opening", genName, ":", err) 122 | return 123 | } 124 | defer nc.Close() 125 | 126 | type vatt struct { 127 | varName string 128 | typeName string 129 | goTypeName string 130 | } 131 | vatts := []vatt{ 132 | {"cl", "color", "color"}, 133 | {"e", "easy", "easy"}, 134 | {"ev", "easyVlen", "easyVlen"}, 135 | {"t", "tricky_t", "tricky_t"}, 136 | {"c", "string", "string"}, 137 | {"b", "byte", "int8"}, 138 | {"ub", "ubyte", "uint8"}, 139 | {"s", "short", "int16"}, 140 | {"us", "ushort", "uint16"}, 141 | {"i", "int", "int32"}, 142 | {"ui", "uint", "uint32"}, 143 | {"i64", "int64", "int64"}, 144 | {"ui64", "uint64", "uint64"}, 145 | {"f", "float", "float32"}, 146 | {"d", "double", "float64"}, 147 | } 148 | for _, v := range vatts { 149 | vg, err := nc.GetVarGetter(v.varName) 150 | if err != nil { 151 | t.Error(err) 152 | continue 153 | } 154 | tt := vg.Type() 155 | if tt != v.typeName { 156 | t.Error("wrong type for var", v.varName, ",", tt) 157 | } 158 | gt := vg.GoType() 159 | if gt != v.goTypeName { 160 | t.Error("wrong go type for var", v.varName, ",", gt) 161 | } 162 | } 163 | } 164 | 165 | func TestListTypes(t *testing.T) { 166 | type expected map[string]string 167 | expAll := map[string]expected{ 168 | "testarray": { 169 | "comp": "compound {\n\tint(3) iArray;\n\tfloat(2,3) fArray;\n}", 170 | }, 171 | "testattrs": { 172 | "alltypes": "compound {\n\tbyte b;\n\tshort s;\n\tint i;\n\tfloat f;\n\tdouble d;\n}", 173 | "color": "byte enum {\n\tRED = 0,\n\tYELLOW = 1,\n\tGREEN = 2,\n\tCYAN = 3,\n\tBLUE = 4,\n\tMAGENTA = 5\n}", 174 | }, 175 | "testcompounds": { 176 | 177 | "Alltypes": "compound {\n\tbyte B;\n\tshort S;\n\tint I;\n\tfloat F;\n\tdouble D;\n}", 178 | "Sametypes": "compound {\n\tint A;\n\tint B;\n\tint C;\n}", 179 | "Includes": "compound {\n\tAlltypes A;\n\tstring S;\n}", 180 | }, 181 | "testempty": { 182 | "opaque5": "opaque(5)", 183 | "alltypes": "compound {\n\tbyte b;\n\tshort s;\n\tint i;\n\tfloat f;\n\tdouble d;\n}", 184 | }, 185 | "testenum": { 186 | "color": "byte enum {\n\tRED = 0,\n\tYELLOW = 1,\n\tGREEN = 2,\n\tCYAN = 3,\n\tBLUE = 4,\n\tMAGENTA = 5\n}", 187 | "junk": "int64 enum {\n\tFIRST = 1,\n\tSECOND = 2,\n\tTHIRD = 3,\n\tFOURTH = 4,\n\tFIFTH = 5,\n\tSIXTH = 6\n}", 188 | "color2": "ushort enum {\n\tBLACK = 0,\n\tWHITE = 1\n}", 189 | "junk2": "int enum {\n\tSEVENTH = 7,\n\tEIGHTH = 8\n}", 190 | }, 191 | "testgroups": {}, 192 | "testopaque": { 193 | "opaque5": "opaque(5)", 194 | }, 195 | "testsimple": { 196 | "AAA": "compound {\n\tshort s;\n\tint i;\n}", 197 | "BBB": "compound {\n\tfloat x;\n\tdouble y;\n}", 198 | }, 199 | "testvlen": { 200 | "vint": "int(*)", 201 | "Easy": "compound {\n\tint FirstEasy;\n\tint SecondEasy;\n}", 202 | "EasyVlen": "Easy(*)", 203 | "Tricky_t": "compound {\n\tint TrickyInt;\n\tEasyVlen TrickVlen;\n}", 204 | }, 205 | } 206 | for fileName, m := range expAll { 207 | func(m expected) { 208 | genName := ncGen(t, fileName) 209 | if genName == "" { 210 | t.Error(errorNcGen) 211 | return 212 | } 213 | defer os.Remove(genName) 214 | nc, err := Open(genName) 215 | if err != nil { 216 | t.Error(err) 217 | return 218 | } 219 | defer nc.Close() 220 | types := nc.ListTypes() 221 | hasMap := map[string]bool{} 222 | for _, typeName := range types { 223 | tVal, has := m[typeName] 224 | if !has { 225 | t.Error(fileName, "missing", typeName) 226 | continue 227 | } 228 | hasMap[typeName] = true 229 | val, has := nc.GetType(typeName) 230 | if !has || val != tVal { 231 | t.Errorf("%s: type mismatch got=(%s) exp=(%s)", fileName, val, tVal) 232 | } 233 | } 234 | for typeName := range m { 235 | if !hasMap[typeName] { 236 | t.Error(fileName, "has extra type", typeName) 237 | } 238 | } 239 | }(m) 240 | } 241 | } 242 | 243 | func TestGoTypes(t *testing.T) { 244 | type expected map[string]string 245 | expAll := map[string]expected{ 246 | // Precede the type name with "type" 247 | "testarray": { 248 | "comp": "type comp struct {\n\tiArray [3]int32\n\tfArray [2][3]float32\n}\n", 249 | }, 250 | "testattrs": { 251 | "alltypes": "type alltypes struct {\n\tb int8\n\ts int16\n\ti int32\n\tf float32\n\td float64\n}\n", 252 | "color": "type color int8\nconst (\n\tRED color = 0\n\tYELLOW = 1\n\tGREEN = 2\n\tCYAN = 3\n\tBLUE = 4\n\tMAGENTA = 5\n)\n", 253 | }, 254 | "testcompounds": { 255 | 256 | "Alltypes": "type Alltypes struct {\n\tB int8\n\tS int16\n\tI int32\n\tF float32\n\tD float64\n}\n", 257 | "Sametypes": "type Sametypes struct {\n\tA int32\n\tB int32\n\tC int32\n}\n", 258 | "Includes": "type Includes struct {\n\tA Alltypes\n\tS string\n}\n", 259 | }, 260 | "testempty": { 261 | "opaque5": "type opaque5 [5]uint8", 262 | "alltypes": "type alltypes struct {\n\tb int8\n\ts int16\n\ti int32\n\tf float32\n\td float64\n}\n", 263 | }, 264 | "testenum": { 265 | "color": "type color int8\nconst (\n\tRED color = 0\n\tYELLOW = 1\n\tGREEN = 2\n\tCYAN = 3\n\tBLUE = 4\n\tMAGENTA = 5\n)\n", 266 | "junk": "type junk int64\nconst (\n\tFIRST junk = 1\n\tSECOND = 2\n\tTHIRD = 3\n\tFOURTH = 4\n\tFIFTH = 5\n\tSIXTH = 6\n)\n", 267 | "color2": "type color2 uint16\nconst (\n\tBLACK color2 = 0\n\tWHITE = 1\n)\n", 268 | "junk2": "type junk2 int32\nconst (\n\tSEVENTH junk2 = 7\n\tEIGHTH = 8\n)\n", 269 | }, 270 | "testgroups": {}, 271 | "testopaque": { 272 | "opaque5": "type opaque5 [5]uint8", 273 | }, 274 | "testsimple": { 275 | "AAA": "type AAA struct {\n\ts int16\n\ti int32\n}\n", 276 | "BBB": "type BBB struct {\n\tx float32\n\ty float64\n}\n", 277 | }, 278 | "testvlen": { 279 | "vint": "type vint []int32", 280 | "Easy": "type Easy struct {\n\tFirstEasy int32\n\tSecondEasy int32\n}\n", 281 | "EasyVlen": "type EasyVlen []Easy", 282 | "Tricky_t": "type Tricky_t struct {\n\tTrickyInt int32\n\tTrickVlen EasyVlen\n}\n", 283 | }, 284 | } 285 | for fileName, m := range expAll { 286 | func(m expected) { 287 | genName := ncGen(t, fileName) 288 | if genName == "" { 289 | t.Error(errorNcGen) 290 | return 291 | } 292 | defer os.Remove(genName) 293 | nc, err := Open(genName) 294 | if err != nil { 295 | t.Error(err) 296 | return 297 | } 298 | defer nc.Close() 299 | types := nc.ListTypes() 300 | hasMap := map[string]bool{} 301 | for _, typeName := range types { 302 | tVal, has := m[typeName] 303 | if !has { 304 | t.Error(fileName, "missing", typeName) 305 | continue 306 | } 307 | hasMap[typeName] = true 308 | val, has := nc.GetGoType(typeName) 309 | if !has { 310 | t.Error("type", typeName, "not found") 311 | continue 312 | } 313 | if val != tVal { 314 | t.Errorf("%s: type mismatch got=(%s) exp=(%s)", fileName, val, tVal) 315 | continue 316 | } 317 | } 318 | for typeName := range m { 319 | if !hasMap[typeName] { 320 | t.Error(fileName, "has extra type", typeName) 321 | } 322 | } 323 | vars := nc.ListVariables() 324 | for _, varName := range vars { 325 | val, has := nc.GetGoType(varName) 326 | if has { 327 | t.Errorf("%s: variables are not types got=(%s)", fileName, val) 328 | } 329 | } 330 | }(m) 331 | } 332 | } 333 | 334 | func TestBitfieldType(t *testing.T) { 335 | // BitFields are not part of NetCDF. They are in HDF5 though. 336 | fileName := "testdata/bitfield.h5" 337 | defer setBitfields(setBitfields(true)) 338 | nc, err := Open(fileName) 339 | if err != nil { 340 | t.Error(err) 341 | return 342 | } 343 | defer nc.Close() 344 | a := nc.Attributes() 345 | attrName := "bitfield" 346 | _, has := a.Get(attrName) 347 | if !has { 348 | t.Error("attribute", attrName, "not found") 349 | return 350 | } 351 | 352 | expectedCDL := "uchar(*)" 353 | val, has := a.GetType(attrName) 354 | if !has || val != expectedCDL { 355 | t.Errorf("%s: type mismatch got=(%s) exp=(%s)", fileName, val, expectedCDL) 356 | } 357 | expectedGo := "[]uint8" 358 | val, has = a.GetGoType(attrName) 359 | if !has || val != expectedGo { 360 | t.Errorf("%s: type mismatch got=(%s) exp=(%s)", fileName, val, expectedGo) 361 | } 362 | } 363 | 364 | func TestReferenceType(t *testing.T) { 365 | // References are not part of NetCDF. They are in HDF5 though. 366 | fileName := "testdata/reference.h5" 367 | defer setBitfields(setBitfields(true)) 368 | defer setReferences(setReferences(true)) 369 | nc, err := Open(fileName) 370 | if err != nil { 371 | t.Error(err) 372 | return 373 | } 374 | defer nc.Close() 375 | 376 | varName := "Dataset3" 377 | vg, err := nc.GetVarGetter(varName) 378 | if err != nil { 379 | t.Error("attribute", varName, "not found", err) 380 | return 381 | } 382 | 383 | expectedCDL := "uint64" 384 | val := vg.Type() 385 | if val != expectedCDL { 386 | t.Errorf("%s: type mismatch got=(%s) exp=(%s)", fileName, val, expectedCDL) 387 | } 388 | expectedGo := "uint64" 389 | val = vg.GoType() 390 | if val != expectedGo { 391 | t.Errorf("%s: type mismatch got=(%s) exp=(%s)", fileName, val, expectedGo) 392 | } 393 | } 394 | -------------------------------------------------------------------------------- /netcdf/hdf5/readers.go: -------------------------------------------------------------------------------- 1 | package hdf5 2 | 3 | // Various readers here 4 | 5 | import ( 6 | "bufio" 7 | "bytes" 8 | "encoding/binary" 9 | "io" 10 | "sync" 11 | 12 | "github.com/batchatco/go-thrower" 13 | ) 14 | 15 | func skip(r io.Reader, length int64) { 16 | data := make([]byte, length) 17 | err := binary.Read(r, binary.LittleEndian, data) 18 | thrower.ThrowIfError(err) 19 | } 20 | 21 | // remReader remembers the count (used) and remaining bytes 22 | type remReader interface { 23 | io.Reader 24 | Count() int64 25 | Rem() int64 26 | } 27 | 28 | // refCountedFile allows multiple readers on the same file handle 29 | type refCountedFile struct { 30 | file io.ReadSeeker 31 | refCount int 32 | lock sync.Mutex // TODO: do we need this lock? 33 | } 34 | 35 | // Random access file: can do random seeks 36 | type raFile struct { 37 | rcFile *refCountedFile // ref counted file 38 | seekPointer int64 39 | } 40 | 41 | // resetreader takes a file and a size (or a byte array), and 42 | // returns a remReader for it. 43 | type resetReader struct { 44 | remReader 45 | lr *io.LimitedReader 46 | size int64 47 | bytes []byte 48 | } 49 | 50 | // unshuffleReader implements the shuffle algorithm 51 | type unshuffleReader struct { 52 | r io.Reader 53 | b []byte 54 | size uint64 55 | shuffleParam uint32 56 | } 57 | 58 | // Streaming version of a fletcher32 reader 59 | type fletcher struct { 60 | r io.Reader 61 | size uint64 62 | count uint64 63 | sum1 uint32 64 | sum2 uint32 65 | partial uint8 66 | checksum uint32 67 | readChecksum bool 68 | } 69 | 70 | // newFletcher32reader creates a reader implementing the fletcher32 algorithm. 71 | func newFletcher32Reader(r io.Reader, size uint64) remReader { 72 | assert(size >= 4, "bad size for fletcher") 73 | return &fletcher{ 74 | r: r, 75 | size: size, 76 | count: 0, 77 | sum1: 0, 78 | sum2: 0, 79 | partial: 0, 80 | checksum: 0, 81 | readChecksum: false, 82 | } 83 | } 84 | 85 | func (fl *fletcher) Rem() int64 { 86 | return int64((fl.size - 4) - fl.count) 87 | } 88 | 89 | func (fl *fletcher) Size() int64 { 90 | return int64(fl.size) 91 | } 92 | 93 | func (fl *fletcher) Count() int64 { 94 | return int64(fl.count) 95 | } 96 | 97 | func (fl *fletcher) Read(b []byte) (int, error) { 98 | thisLen := uint64(len(b)) 99 | if thisLen+fl.count > fl.size-4 { 100 | thisLen = (fl.size - 4) - fl.count 101 | } 102 | n, err := fl.r.Read(b[:thisLen]) 103 | if err != nil && err != io.EOF { 104 | return 0, err 105 | } 106 | addToSums := func(val uint16) { 107 | fl.sum1 = (fl.sum1 + uint32(val)) % 65535 108 | fl.sum2 = (fl.sum1 + fl.sum2) % 65535 109 | } 110 | for i := 0; i < n; i++ { 111 | var val uint16 112 | switch { 113 | case fl.count%2 == 1: 114 | // Previous read left us with an odd count. 115 | // Recover the partial read and compute val. 116 | val = (uint16(fl.partial) << 8) | uint16(b[i]) 117 | addToSums(val) 118 | case i == n-1: 119 | // Can't complete a read, so save the partial. 120 | fl.partial = b[i] 121 | default: 122 | val = (uint16(b[i]) << 8) | uint16(b[i+1]) 123 | addToSums(val) 124 | i++ 125 | } 126 | } 127 | fl.count += uint64(n) 128 | if fl.count == fl.size-4 { 129 | if fl.size%2 == 1 { 130 | // Retrieve partial and complete sum as if next byte were zero. 131 | val := uint16(fl.partial) << 8 132 | addToSums(val) 133 | } 134 | calcedSum := (uint32(fl.sum2) << 16) | uint32(fl.sum1) 135 | if !fl.readChecksum { 136 | binary.Read(fl.r, binary.LittleEndian, &fl.checksum) 137 | } 138 | if calcedSum != fl.checksum { 139 | logger.Infof("checksum failure: sum=%#x file sum=%#x\n", calcedSum, 140 | fl.checksum) 141 | thrower.Throw(ErrFletcherChecksum) 142 | } 143 | } 144 | return n, nil 145 | } 146 | 147 | // oldFletcher32reader creates a reader implementing the fletcher32 algorithm. 148 | // It is inefficient and is here for testing purposes only. 149 | func oldFletcher32Reader(r io.Reader, size uint64) remReader { 150 | assert(size >= 4, "bad size for fletcher") 151 | b := make([]byte, size-4) 152 | read(r, b) 153 | var checksum uint32 154 | binary.Read(r, binary.LittleEndian, &checksum) 155 | bf := newResetReaderFromBytes(b) 156 | values := make([]uint16, len(b)/2) 157 | binary.Read(bf, binary.BigEndian, values) 158 | if len(b)%2 == 1 { 159 | last := uint16(b[len(b)-1]) 160 | values = append(values, last<<8) 161 | } 162 | calcedSum := fletcher32(values) 163 | if calcedSum != checksum { 164 | logger.Infof("checksum failure: calced sum=%#x filesum=%#x\n", calcedSum, checksum) 165 | thrower.Throw(ErrFletcherChecksum) 166 | } 167 | return newResetReaderFromBytes(b) 168 | } 169 | 170 | func (r *resetReader) Rem() int64 { 171 | return r.lr.N 172 | } 173 | 174 | func (r *resetReader) Count() int64 { 175 | return r.size - r.lr.N 176 | } 177 | 178 | func (r *resetReader) Read(p []byte) (int, error) { 179 | return r.lr.Read(p) 180 | } 181 | 182 | func newResetReaderFromBytes(b []byte) remReader { 183 | file := bytes.NewReader(b) 184 | size := int64(len(b)) 185 | ret := &resetReader{ 186 | lr: &io.LimitedReader{R: file, N: size}, 187 | size: size, 188 | bytes: b, 189 | } 190 | return ret 191 | } 192 | 193 | func newResetReader(file io.Reader, size int64) remReader { 194 | ret := &resetReader{ 195 | lr: &io.LimitedReader{R: file, N: size}, 196 | size: size, 197 | } 198 | return ret 199 | } 200 | 201 | func newResetReaderSave(file io.Reader, size int64) remReader { 202 | b := make([]byte, size) 203 | read(file, b) 204 | return newResetReaderFromBytes(b) 205 | } 206 | 207 | func newResetReaderOffset(file *raFile, size int64, offset uint64) remReader { 208 | ra := file.seekAt(int64(offset)) 209 | bf := bufio.NewReader(ra) 210 | ret := &resetReader{ 211 | lr: &io.LimitedReader{R: bf, N: size}, 212 | size: size, 213 | } 214 | return ret 215 | } 216 | 217 | func newRaFile(file io.ReadSeeker) *raFile { 218 | return &raFile{ 219 | rcFile: newRefCountedFile(file), 220 | seekPointer: 0, 221 | } 222 | } 223 | 224 | func (f *raFile) Close() error { 225 | return f.rcFile.dereference() 226 | } 227 | 228 | func (f *raFile) seekAt(offset int64) *raFile { 229 | return &raFile{ 230 | rcFile: f.rcFile, 231 | seekPointer: offset, 232 | } 233 | } 234 | 235 | func (f *raFile) dup() *raFile { 236 | f.rcFile.reference() 237 | return &raFile{ 238 | rcFile: f.rcFile, 239 | seekPointer: f.seekPointer, 240 | } 241 | } 242 | 243 | func (f *raFile) Read(p []byte) (int, error) { 244 | // Do read 245 | f.rcFile.Lock() 246 | defer f.rcFile.Unlock() 247 | return f.readNoLock(p) 248 | } 249 | 250 | func (f *raFile) readNoLock(p []byte) (int, error) { 251 | thisLen := len(p) 252 | _, err := f.rcFile.file.Seek(f.seekPointer, io.SeekStart) 253 | if err != nil { 254 | logger.Error("Seek error in Read", err, io.SeekStart) 255 | return 0, err 256 | } 257 | n, err := f.rcFile.file.Read(p[:thisLen]) 258 | if err != nil { 259 | logger.Error("Read error in Read", err, io.SeekStart) 260 | return 0, err 261 | } 262 | // Save position 263 | f.seekPointer += int64(n) 264 | // Return read results 265 | return n, err 266 | } 267 | 268 | func (f *raFile) ReadAt(b []byte, offset int64) (int, error) { 269 | f.rcFile.Lock() 270 | defer f.rcFile.Unlock() 271 | save := f.seekPointer 272 | _, err := f.rcFile.file.Seek(offset, io.SeekStart) 273 | if err != nil { 274 | logger.Error("Seek error in ReadAt", err, offset, io.SeekStart) 275 | return 0, err 276 | } 277 | f.seekPointer = offset 278 | n, retErr := f.readNoLock(b) 279 | f.seekPointer = save 280 | return n, retErr 281 | } 282 | 283 | func newRefCountedFile(file io.ReadSeeker) *refCountedFile { 284 | return &refCountedFile{file, 1, sync.Mutex{}} 285 | } 286 | 287 | func (rcf *refCountedFile) reference() { 288 | rcf.refCount++ 289 | } 290 | 291 | func (rcf *refCountedFile) Lock() { 292 | rcf.lock.Lock() 293 | } 294 | 295 | func (rcf *refCountedFile) Unlock() { 296 | rcf.lock.Unlock() 297 | } 298 | 299 | func (rcf *refCountedFile) dereference() error { 300 | rcf.lock.Lock() 301 | defer rcf.lock.Unlock() 302 | var err error 303 | rcf.refCount-- 304 | switch { 305 | case rcf.refCount == 0: 306 | logger.Info("Closing file") 307 | if f, ok := rcf.file.(io.Closer); ok { 308 | f.Close() 309 | } 310 | rcf.file = nil 311 | case rcf.refCount < 0: 312 | err = ErrInternal 313 | } 314 | return err 315 | } 316 | 317 | func newUnshuffleReader(r io.Reader, size uint64, shuffleParam uint32) remReader { 318 | return newResetReader(&unshuffleReader{r, nil, size, shuffleParam}, int64(size)) 319 | } 320 | 321 | func unshuffle(val []byte, n uint32) { 322 | if n == 1 { 323 | return // avoids allocation 324 | } 325 | // shuffle params are never very large in practice, so this isn't wasteful 326 | // of memory. 327 | tmp := make([]byte, len(val)) 328 | nelems := len(val) / int(n) 329 | for i := 0; i < int(n); i++ { 330 | for j := 0; j < nelems; j++ { 331 | tmp[j*int(n)+i] = val[i*nelems+j] 332 | } 333 | } 334 | copy(val, tmp) 335 | } 336 | 337 | func (r *unshuffleReader) Read(p []byte) (int, error) { 338 | if r.size == 0 { 339 | return 0, io.EOF 340 | } 341 | thisLen := uint64(len(p)) 342 | if thisLen > r.size { 343 | thisLen = r.size 344 | } 345 | var err error 346 | if r.b == nil { 347 | r.b = make([]byte, r.size) 348 | tot, err := readAll(r.r, r.b) 349 | unshuffle(r.b[:tot], r.shuffleParam) 350 | if err != nil { 351 | assert(err == io.EOF, err.Error()) 352 | } 353 | } 354 | copy(p, r.b[:thisLen]) 355 | r.b = r.b[thisLen:] 356 | r.size -= thisLen 357 | if r.size == 0 { 358 | return int(thisLen), io.EOF 359 | } 360 | return int(thisLen), err 361 | } 362 | 363 | type layoutReader struct { 364 | r remReader 365 | obj *object 366 | buf []byte 367 | size int64 368 | count uint64 // number bytes read from buffer 369 | total uint64 // total bytes buffered 370 | lastChunkRows uint64 371 | } 372 | 373 | // calcChunkSize calculates the size of a chunk 374 | // Returns 0 if not chunked 375 | func calcChunkSize(attr *attribute) uint64 { 376 | chunkSize := uint64(1) 377 | for _, v := range attr.layout { 378 | chunkSize *= v 379 | } 380 | return chunkSize 381 | } 382 | 383 | // calcAttrSize computes the total size of an object in bytes. 384 | // If it is chunked, it may be larger than the data size. 385 | func calcAttrSize(attr *attribute) uint64 { 386 | dtLength := uint64(attr.length) 387 | lastLayoutIndex := len(attr.layout) - 1 388 | if lastLayoutIndex == -1 { 389 | return calcDataSize(attr) * dtLength // not chunked 390 | } 391 | lastDimIndex := len(attr.dimensions) - 1 392 | lineLen := attr.dimensions[lastDimIndex] 393 | if lineLen == 0 { 394 | logger.Info("zero sized dimension") 395 | return 0 396 | } 397 | chunkLen := attr.layout[lastLayoutIndex] 398 | chunkSize := calcChunkSize(attr) 399 | chunksPerLine := (lineLen + chunkLen - 1) / chunkLen 400 | chunkRows := chunkSize / chunkLen 401 | dataSize := calcDataSize(attr) 402 | chunksPerColumn := ((dataSize / lineLen) + chunkRows - 1) / chunkRows 403 | totSize := chunksPerLine * chunksPerColumn * chunkSize 404 | return totSize * dtLength 405 | } 406 | 407 | // calcDataSize computes the size of the data, not counting chunks. 408 | func calcDataSize(attr *attribute) uint64 { 409 | dataSize := uint64(1) 410 | for _, v := range attr.dimensions { 411 | dataSize *= v 412 | } 413 | return dataSize 414 | } 415 | 416 | func (lr *layoutReader) fillrow() (nrows uint64) { 417 | dtLength := uint64(lr.obj.objAttr.length) 418 | lastDimIndex := len(lr.obj.objAttr.dimensions) - 1 419 | lineLen := lr.obj.objAttr.dimensions[lastDimIndex] 420 | lastLayoutIndex := len(lr.obj.objAttr.layout) - 1 421 | chunkLen := lr.obj.objAttr.layout[lastLayoutIndex] 422 | 423 | chunkSize := calcChunkSize(lr.obj.objAttr) 424 | chunksPerLine := (lineLen + chunkLen - 1) / chunkLen 425 | chunkRows := chunkSize / chunkLen 426 | dataSize := calcDataSize(lr.obj.objAttr) 427 | dataRows := dataSize / lineLen 428 | chunksPerColumn := ((dataSize / lineLen) + chunkRows - 1) / chunkRows 429 | totSize := chunksPerLine * chunksPerColumn 430 | if lr.buf == nil { 431 | sliceSize := chunksPerLine * chunkSize * dtLength 432 | extra := (chunksPerLine * chunkLen) - lineLen 433 | sliceSize -= extra 434 | lr.buf = make([]byte, sliceSize) 435 | } 436 | readBuf := make([]byte, chunkLen*dtLength) 437 | usedRows := (lr.total / lineLen) / dtLength 438 | outer: 439 | for col := uint64(0); col < chunksPerLine; col++ { 440 | for row := uint64(0); row < chunkRows; row++ { 441 | // for each of the rows, read a chunkLen 442 | thisLen := chunkLen 443 | usedLen := chunkLen 444 | if col == (chunksPerLine - 1) { 445 | // last column could be runt 446 | usedLen = lineLen - (chunkLen * col) 447 | } 448 | thisOffset := row*lineLen + col*chunkLen 449 | if usedRows+row >= dataRows { 450 | usedLen = 0 451 | } 452 | thisLen *= dtLength 453 | usedLen *= dtLength 454 | thisOffset *= dtLength 455 | for thisLen > 0 { 456 | // Read one chunk 457 | n, err := lr.r.Read(readBuf[:thisLen]) 458 | if n == 0 { 459 | if err == io.EOF { 460 | return row 461 | } 462 | thrower.ThrowIfError(err) 463 | } 464 | // Only copy usedlen 465 | if usedLen > 0 { 466 | if uint64(n) > usedLen { 467 | copy(lr.buf[thisOffset:thisOffset+usedLen], readBuf[:usedLen]) 468 | } else { 469 | copy(lr.buf[thisOffset:thisOffset+uint64(n)], readBuf[:n]) 470 | } 471 | } 472 | if uint64(n) < thisLen { 473 | logger.Info("short read", lr.total, "size in file", lr.size, "len", n, "thisLen", thisLen) 474 | } 475 | thisOffset += uint64(n) 476 | thisLen -= uint64(n) 477 | if usedLen >= uint64(n) { 478 | usedLen -= uint64(n) 479 | lr.total += uint64(n) 480 | } else { 481 | lr.total += usedLen 482 | usedLen = 0 483 | } 484 | } 485 | if lr.total == totSize { 486 | logger.Infof("size match") 487 | break outer 488 | } 489 | } 490 | } 491 | logger.Info("fillrow full", "chunkRows", chunkRows, "chunksPerLine", chunksPerLine) 492 | return chunkRows 493 | } 494 | 495 | func (lr *layoutReader) Read(b []byte) (int, error) { 496 | // Offset in units 497 | dtLength := uint64(lr.obj.objAttr.length) 498 | curOffset := lr.count / dtLength 499 | lastDimIndex := len(lr.obj.objAttr.dimensions) - 1 500 | lineLen := lr.obj.objAttr.dimensions[lastDimIndex] 501 | assert(lineLen != 0, "linelen can't be zero") 502 | lastLayoutIndex := len(lr.obj.objAttr.layout) - 1 503 | chunkLen := lr.obj.objAttr.layout[lastLayoutIndex] 504 | 505 | chunkSize := calcChunkSize(lr.obj.objAttr) 506 | chunkRows := chunkSize / chunkLen 507 | if lr.lastChunkRows > 0 { 508 | chunkRows = lr.lastChunkRows 509 | } 510 | if curOffset%(chunkRows*lineLen) == 0 { 511 | // TODO: we don't have to fill a whole row necessarily. 512 | thisChunkRows := lr.fillrow() 513 | if thisChunkRows == 0 { 514 | logger.Info("nil buf EOF") 515 | lr.buf = nil 516 | return 0, io.EOF 517 | } 518 | if thisChunkRows == chunkRows { 519 | logger.Info("full chunk", "rows", chunkRows) 520 | } else { 521 | logger.Info("partial chunk", thisChunkRows, "full", chunkRows) 522 | lr.lastChunkRows = thisChunkRows 523 | } 524 | chunkRows = thisChunkRows 525 | } 526 | thisRow := (curOffset / lineLen) % chunkRows 527 | thisLen := uint64(len(b)) 528 | offset := (thisRow*lineLen + curOffset%lineLen) * dtLength 529 | rem := uint64(len(lr.buf)) - offset 530 | if rem < thisLen { 531 | thisLen = rem 532 | } 533 | copy(b, lr.buf[offset:offset+thisLen]) 534 | lr.count += thisLen 535 | curOffset = lr.count / dtLength 536 | if curOffset%(chunkRows*lineLen) == 0 { 537 | logger.Info("nil buf -- last col") 538 | lr.buf = nil 539 | } 540 | logger.Infof("layout reader read %d, rem %d", thisLen, lr.Rem()) 541 | return int(thisLen), nil 542 | } 543 | 544 | func (lr *layoutReader) Rem() int64 { 545 | return lr.size - int64(lr.count) 546 | } 547 | 548 | func (lr *layoutReader) Count() int64 { 549 | return int64(lr.count) 550 | } 551 | 552 | func newLayoutReader(r remReader, obj *object) io.Reader { 553 | return &layoutReader{r, obj, nil, r.Rem(), 0, 0, 0} 554 | } 555 | -------------------------------------------------------------------------------- /netcdf/cdf/cdfwriter.go: -------------------------------------------------------------------------------- 1 | package cdf 2 | 3 | // TODO: write unlimited dimension 4 | // TODO: too many dimensions error 5 | // TODO: api for dimensions in case of unlimited and zero length 6 | import ( 7 | "bufio" 8 | "encoding/binary" 9 | "errors" 10 | "fmt" 11 | "io" 12 | "math" 13 | "os" 14 | "reflect" 15 | 16 | "github.com/batchatco/go-native-netcdf/internal" 17 | "github.com/batchatco/go-native-netcdf/netcdf/api" 18 | "github.com/batchatco/go-native-netcdf/netcdf/util" 19 | "github.com/batchatco/go-thrower" 20 | ) 21 | 22 | type countedWriter struct { 23 | w *bufio.Writer 24 | count int64 25 | } 26 | 27 | type savedVar struct { 28 | name string 29 | val interface{} 30 | ty int 31 | dimLengths []int64 32 | dimNames []string 33 | attrs api.AttributeMap 34 | vsize int64 35 | } 36 | 37 | type CDFWriter struct { 38 | file *os.File 39 | bf *countedWriter 40 | vars []savedVar 41 | globalAttrs api.AttributeMap 42 | dimLengths map[string]int64 43 | dimNames []string 44 | dimIds map[string]int64 45 | nextID int64 46 | version int8 47 | begin int64 48 | } 49 | 50 | var ( 51 | ErrUnlimitedMustBeFirst = errors.New("unlimited dimension must be first") 52 | ErrDimensionSize = errors.New("dimension doesn't match size") 53 | ErrInvalidName = errors.New("invalid name") 54 | ErrAttribute = errors.New("invalid attribute") 55 | ErrEmptySlice = errors.New("empty slice encountered") 56 | ) 57 | 58 | func (c *countedWriter) Count() int64 { 59 | return c.count 60 | } 61 | 62 | func (c *countedWriter) Flush() error { 63 | return c.w.Flush() 64 | } 65 | 66 | func (c *countedWriter) Write(p []byte) (int, error) { 67 | n, err := c.w.Write(p) 68 | c.count += int64(n) 69 | return n, err 70 | } 71 | 72 | func (cw *CDFWriter) storeChars(val reflect.Value, dimLengths []int64) { 73 | if len(dimLengths) == 0 { 74 | // a single character 75 | str := val.String() 76 | if len(str) == 0 { 77 | return 78 | } 79 | write8(cw.bf, int8(str[0])) 80 | return 81 | } 82 | thisDim := dimLengths[0] 83 | if len(dimLengths) == 1 { 84 | // a string which must be padded 85 | s, ok := val.Interface().(string) 86 | if !ok { 87 | thrower.Throw(ErrInternal) 88 | } 89 | // TODO: this is probably wrong because it could be unlimited 90 | writeBytes(cw.bf, []byte(s)) 91 | offset := int64(val.Len()) 92 | for offset < thisDim { 93 | write8(cw.bf, 0) 94 | offset++ 95 | } 96 | return 97 | } 98 | for i := 0; i < val.Len(); i++ { 99 | value := val.Index(int(i)) 100 | cw.storeChars(value, dimLengths[1:]) 101 | } 102 | } 103 | 104 | func (cw *CDFWriter) storeBytes(val reflect.Value, dimLengths []int64) { 105 | if len(dimLengths) == 0 { 106 | write8(cw.bf, int8(val.Int())) 107 | return 108 | } 109 | if len(dimLengths) == 1 { 110 | b := val.Interface().([]int8) 111 | writeAny(cw.bf, b) 112 | return 113 | } 114 | for i := 0; i < val.Len(); i++ { 115 | value := val.Index(int(i)) 116 | cw.storeBytes(value, dimLengths[1:]) 117 | } 118 | } 119 | 120 | func (cw *CDFWriter) storeUBytes(val reflect.Value, dimLengths []int64) { 121 | if len(dimLengths) == 0 { 122 | write8(cw.bf, int8(val.Uint())) 123 | return 124 | } 125 | if len(dimLengths) == 1 { 126 | b := val.Interface().([]uint8) 127 | writeAny(cw.bf, b) 128 | return 129 | } 130 | for i := 0; i < val.Len(); i++ { 131 | value := val.Index(int(i)) 132 | cw.storeUBytes(value, dimLengths[1:]) 133 | } 134 | } 135 | 136 | func (cw *CDFWriter) storeShorts(val reflect.Value, dimLengths []int64) { 137 | if len(dimLengths) == 0 { 138 | write16(cw.bf, int16(val.Int())) 139 | return 140 | } 141 | if len(dimLengths) == 1 { 142 | s := val.Interface().([]int16) 143 | writeAny(cw.bf, s) 144 | return 145 | } 146 | for i := 0; i < val.Len(); i++ { 147 | value := val.Index(int(i)) 148 | cw.storeShorts(value, dimLengths[1:]) 149 | } 150 | } 151 | 152 | func (cw *CDFWriter) storeUShorts(val reflect.Value, dimLengths []int64) { 153 | if len(dimLengths) == 0 { 154 | write16(cw.bf, int16(val.Uint())) 155 | return 156 | } 157 | if len(dimLengths) == 1 { 158 | s := val.Interface().([]uint16) 159 | writeAny(cw.bf, s) 160 | return 161 | } 162 | for i := 0; i < val.Len(); i++ { 163 | value := val.Index(int(i)) 164 | cw.storeUShorts(value, dimLengths[1:]) 165 | } 166 | } 167 | 168 | func (cw *CDFWriter) storeInts(val reflect.Value, dimLengths []int64) { 169 | if len(dimLengths) == 0 { 170 | write32(cw.bf, int32(val.Int())) 171 | return 172 | } 173 | if len(dimLengths) == 1 { 174 | iv := val.Interface().([]int32) 175 | writeAny(cw.bf, iv) 176 | return 177 | } 178 | for i := 0; i < val.Len(); i++ { 179 | value := val.Index(int(i)) 180 | cw.storeInts(value, dimLengths[1:]) 181 | } 182 | } 183 | 184 | func (cw *CDFWriter) storeUInts(val reflect.Value, dimLengths []int64) { 185 | if len(dimLengths) == 0 { 186 | write32(cw.bf, int32(val.Uint())) 187 | return 188 | } 189 | if len(dimLengths) == 1 { 190 | iv := val.Interface().([]uint32) 191 | writeAny(cw.bf, iv) 192 | return 193 | } 194 | for i := 0; i < val.Len(); i++ { 195 | value := val.Index(int(i)) 196 | cw.storeUInts(value, dimLengths[1:]) 197 | } 198 | } 199 | 200 | func (cw *CDFWriter) storeInt64s(val reflect.Value, dimLengths []int64) { 201 | if len(dimLengths) == 0 { 202 | write64(cw.bf, val.Int()) 203 | return 204 | } 205 | if len(dimLengths) == 1 { 206 | iv := val.Interface().([]int64) 207 | writeAny(cw.bf, iv) 208 | return 209 | } 210 | for i := 0; i < val.Len(); i++ { 211 | value := val.Index(int(i)) 212 | cw.storeInt64s(value, dimLengths[1:]) 213 | } 214 | } 215 | 216 | func (cw *CDFWriter) storeUInt64s(val reflect.Value, dimLengths []int64) { 217 | if len(dimLengths) == 0 { 218 | write64(cw.bf, int64(val.Uint())) 219 | return 220 | } 221 | if len(dimLengths) == 1 { 222 | iv := val.Interface().([]uint64) 223 | writeAny(cw.bf, iv) 224 | return 225 | } 226 | for i := 0; i < val.Len(); i++ { 227 | value := val.Index(int(i)) 228 | cw.storeUInt64s(value, dimLengths[1:]) 229 | } 230 | } 231 | 232 | func (cw *CDFWriter) storeFloats(val reflect.Value, dimLengths []int64) { 233 | if len(dimLengths) == 0 { 234 | f := float32(val.Float()) 235 | i := math.Float32bits(f) 236 | write32(cw.bf, int32(i)) 237 | return 238 | } 239 | if len(dimLengths) == 1 { 240 | fv := val.Interface().([]float32) 241 | writeAny(cw.bf, fv) 242 | return 243 | } 244 | for i := 0; i < val.Len(); i++ { 245 | value := val.Index(int(i)) 246 | cw.storeFloats(value, dimLengths[1:]) 247 | } 248 | } 249 | 250 | func (cw *CDFWriter) storeDoubles(val reflect.Value, dimLengths []int64) { 251 | if len(dimLengths) == 0 { 252 | f := val.Float() 253 | i := math.Float64bits(f) 254 | write64(cw.bf, int64(i)) 255 | return 256 | } 257 | if len(dimLengths) == 1 { 258 | dv := val.Interface().([]float64) 259 | writeAny(cw.bf, dv) 260 | return 261 | } 262 | for i := 0; i < val.Len(); i++ { 263 | value := val.Index(int(i)) 264 | cw.storeDoubles(value, dimLengths[1:]) 265 | } 266 | } 267 | 268 | func (cw *CDFWriter) getDimLengthsHelper( 269 | rv reflect.Value, 270 | dims []int64, 271 | dimNames []string) ([]int64, int) { 272 | 273 | t := rv.Type() 274 | kind := cw.scalarKind(t.Kind()) 275 | switch kind { 276 | case typeNone: 277 | break 278 | case typeChar: 279 | if rv.Len() == 1 && len(dimNames) == 0 { 280 | return dims, kind 281 | } 282 | default: 283 | return dims, kind 284 | } 285 | vLen := int64(rv.Len()) 286 | dims = append(dims, vLen) 287 | switch t.Kind() { 288 | case reflect.Array, reflect.Slice: 289 | if t.Elem().Kind() == reflect.String { 290 | // special case to longest string 291 | maxLen := int64(0) 292 | for i := 0; i < int(vLen); i++ { 293 | if int64(rv.Index(i).Len()) > maxLen { 294 | maxLen = int64(rv.Index(i).Len()) 295 | } 296 | } 297 | if maxLen == 0 { 298 | thrower.Throw(ErrEmptySlice) 299 | } 300 | dims = append(dims, maxLen) 301 | return dims, typeChar 302 | } 303 | if vLen == 0 { 304 | // minimal support for unlimited, when it is the only dimension. 305 | kind = cw.scalarKind(t.Elem().Kind()) 306 | for kind == typeNone { 307 | // there are other dimensions and we can't tell the size of them. 308 | thrower.Throw(ErrEmptySlice) 309 | } 310 | return dims, kind 311 | } 312 | return cw.getDimLengthsHelper(rv.Index(0), dims, dimNames) 313 | 314 | case reflect.String: 315 | return dims, typeChar // we will make this an array of characters (bytes) 316 | } 317 | 318 | logger.Info("Unknown type", t.Kind()) 319 | thrower.Throw(ErrUnknownType) 320 | panic("internal error") // should never happen 321 | } 322 | 323 | func (cw *CDFWriter) getDimLengths(val interface{}, dimNames []string) ([]int64, int) { 324 | v := reflect.ValueOf(val) 325 | dims := make([]int64, 0) 326 | return cw.getDimLengthsHelper(v, dims, dimNames) 327 | } 328 | 329 | func (cw *CDFWriter) scalarKind(goKind reflect.Kind) int { 330 | switch goKind { 331 | 332 | case reflect.String: 333 | return typeChar // This may not be a scalar: must do other checks 334 | 335 | case reflect.Int8: 336 | return typeByte 337 | 338 | case reflect.Int16: 339 | return typeShort 340 | 341 | case reflect.Int32: 342 | return typeInt 343 | 344 | case reflect.Float32: 345 | return typeFloat 346 | 347 | case reflect.Float64: 348 | return typeDouble 349 | 350 | // v5 351 | case reflect.Uint8: 352 | cw.version = 5 353 | return typeUByte 354 | 355 | case reflect.Uint16: 356 | cw.version = 5 357 | return typeUShort 358 | 359 | case reflect.Uint32: 360 | cw.version = 5 361 | return typeUInt 362 | 363 | case reflect.Uint64: 364 | cw.version = 5 365 | return typeUInt64 366 | 367 | case reflect.Int64: 368 | cw.version = 5 369 | return typeInt64 370 | 371 | } 372 | // not a scalar 373 | return typeNone 374 | } 375 | 376 | func hasValidNames(am api.AttributeMap) bool { 377 | if am == nil { 378 | return true 379 | } 380 | for _, key := range am.Keys() { 381 | if !internal.IsValidNetCDFName(key) { 382 | return false 383 | } 384 | } 385 | return true 386 | } 387 | 388 | // AddGlobalAttrs adds global attributes to be written out. 389 | // Use util.NewOrderedMap to create attribute maps. 390 | func (cw *CDFWriter) AddGlobalAttrs(attrs api.AttributeMap) error { 391 | if !hasValidNames(attrs) { 392 | return ErrInvalidName 393 | } 394 | cw.globalAttrs = attrs 395 | return nil 396 | } 397 | 398 | // AddVar adds a variable to be written out. 399 | // Use util.NewOrderedMap to create attribute maps for the variable. 400 | func (cw *CDFWriter) AddVar(name string, vr api.Variable) (err error) { 401 | defer thrower.RecoverError(&err) 402 | 403 | if !internal.IsValidNetCDFName(name) { 404 | return ErrInvalidName 405 | } 406 | if !hasValidNames(vr.Attributes) { 407 | return ErrInvalidName 408 | } 409 | // TODO: check name for validity 410 | cw.checkV5Attributes(vr.Attributes) 411 | dimLengths, ty := cw.getDimLengths(vr.Values, vr.Dimensions) 412 | switch ty { 413 | case typeUByte, typeUShort, typeUInt, typeUInt64, typeInt64: 414 | cw.version = 5 415 | } 416 | for i := 0; i < len(dimLengths); i++ { 417 | var dimName string 418 | if i < len(vr.Dimensions) { 419 | dimName = vr.Dimensions[i] 420 | } 421 | if dimName == "" { 422 | if ty == typeChar && i == len(dimLengths)-1 { 423 | dimName = fmt.Sprintf("_stringlen_%s", name) 424 | } else { 425 | dimName = fmt.Sprintf("_dimid_%d", cw.nextID) 426 | } 427 | vr.Dimensions = append(vr.Dimensions, dimName) 428 | } 429 | 430 | dimName = vr.Dimensions[i] 431 | currentLength, has := cw.dimLengths[dimName] 432 | if has { 433 | if dimLengths[i] != currentLength { 434 | thrower.Throw(ErrDimensionSize) 435 | } 436 | } else { 437 | cw.dimLengths[dimName] = dimLengths[i] 438 | cw.dimIds[dimName] = cw.nextID 439 | cw.dimNames = append(cw.dimNames, dimName) 440 | cw.nextID++ 441 | } 442 | } 443 | cw.vars = append(cw.vars, savedVar{name, vr.Values, ty, dimLengths, 444 | vr.Dimensions, vr.Attributes, 0}) 445 | return nil 446 | } 447 | 448 | func (cw *CDFWriter) writeAttributes(attrs api.AttributeMap) { 449 | if attrs == nil || len(attrs.Keys()) == 0 { 450 | write32(cw.bf, 0) // attributes: absent 451 | cw.writeNumber(int64(0)) // attributes: absent 452 | return 453 | } 454 | write32(cw.bf, fieldAttribute) 455 | cw.writeNumber(int64(len(attrs.Keys()))) 456 | for _, k := range attrs.Keys() { 457 | v, _ := attrs.Get(k) 458 | cw.writeName(k) 459 | switch val := v.(type) { 460 | case string: 461 | write32(cw.bf, typeChar) 462 | cw.writeNumber(int64(len([]byte(val)))) 463 | writeBytes(cw.bf, []byte(val)) 464 | cw.pad() 465 | 466 | case int8: 467 | write32(cw.bf, typeByte) 468 | cw.writeNumber(1) 469 | writeAny(cw.bf, val) 470 | cw.pad() 471 | 472 | case []int8: 473 | write32(cw.bf, typeByte) 474 | cw.writeNumber(int64(len(val))) 475 | writeAny(cw.bf, val) 476 | cw.pad() 477 | 478 | case int16: 479 | write32(cw.bf, typeShort) 480 | cw.writeNumber(1) 481 | writeAny(cw.bf, val) 482 | cw.pad() 483 | 484 | case []int16: 485 | write32(cw.bf, typeShort) 486 | cw.writeNumber(int64(len(val))) 487 | writeAny(cw.bf, val) 488 | cw.pad() 489 | 490 | case int32: 491 | write32(cw.bf, typeInt) 492 | cw.writeNumber(1) 493 | writeAny(cw.bf, val) 494 | 495 | case []int32: 496 | write32(cw.bf, typeInt) 497 | cw.writeNumber(int64(len(val))) 498 | writeAny(cw.bf, val) 499 | 500 | case float32: 501 | write32(cw.bf, typeFloat) 502 | cw.writeNumber(1) 503 | writeAny(cw.bf, val) 504 | 505 | case []float32: 506 | write32(cw.bf, typeFloat) 507 | cw.writeNumber(int64(len(val))) 508 | writeAny(cw.bf, val) 509 | 510 | case float64: 511 | write32(cw.bf, typeDouble) 512 | cw.writeNumber(1) 513 | writeAny(cw.bf, val) 514 | 515 | case []float64: 516 | write32(cw.bf, typeDouble) 517 | cw.writeNumber(int64(len(val))) 518 | writeAny(cw.bf, val) 519 | 520 | // v5 521 | case uint8: // []uint8 and []byte are the same thing 522 | write32(cw.bf, typeUByte) 523 | cw.writeNumber(1) 524 | writeAny(cw.bf, val) 525 | cw.pad() 526 | 527 | case []uint8: // []uint8 and []byte are the same thing 528 | write32(cw.bf, typeUByte) 529 | cw.writeNumber(int64(len(val))) 530 | writeAny(cw.bf, val) 531 | cw.pad() 532 | 533 | case uint16: 534 | write32(cw.bf, typeUShort) 535 | cw.writeNumber(1) 536 | writeAny(cw.bf, val) 537 | cw.pad() 538 | 539 | case []uint16: 540 | write32(cw.bf, typeUShort) 541 | cw.writeNumber(int64(len(val))) 542 | writeAny(cw.bf, val) 543 | cw.pad() 544 | 545 | case uint32: 546 | write32(cw.bf, typeUInt) 547 | cw.writeNumber(1) 548 | writeAny(cw.bf, val) 549 | 550 | case []uint32: 551 | write32(cw.bf, typeUInt) 552 | cw.writeNumber(int64(len(val))) 553 | writeAny(cw.bf, val) 554 | 555 | case int64: 556 | write32(cw.bf, typeInt64) 557 | cw.writeNumber(1) 558 | writeAny(cw.bf, val) 559 | 560 | case []int64: 561 | write32(cw.bf, typeInt64) 562 | cw.writeNumber(int64(len(val))) 563 | writeAny(cw.bf, val) 564 | 565 | case uint64: 566 | write32(cw.bf, typeUInt64) 567 | cw.writeNumber(1) 568 | writeAny(cw.bf, val) 569 | 570 | case []uint64: 571 | write32(cw.bf, typeUInt64) 572 | cw.writeNumber(int64(len(val))) 573 | writeAny(cw.bf, val) 574 | 575 | default: 576 | logger.Warnf("Unknown type %T, %#v=%#v", v, k, v) 577 | thrower.Throw(ErrUnknownType) 578 | } 579 | } 580 | } 581 | 582 | func (cw *CDFWriter) checkV5Attributes(attrs api.AttributeMap) { 583 | if attrs == nil { 584 | return 585 | } 586 | for _, k := range attrs.Keys() { 587 | v, _ := attrs.Get(k) 588 | switch v.(type) { 589 | case string, int8, int16, int32, float32, float64, 590 | []int8, []int16, []int32, []float32, []float64: 591 | 592 | case []uint64, uint64, []int64, int64, []uint8, uint8, []uint16, uint16, []uint32, uint32: 593 | cw.version = 5 594 | return 595 | 596 | default: 597 | logger.Errorf("invalid attribute %#v", v) 598 | thrower.Throw(ErrAttribute) 599 | } 600 | } 601 | } 602 | 603 | func (cw *CDFWriter) writeVar(which int) { 604 | saved := &cw.vars[which] 605 | cw.writeName(saved.name) 606 | cw.writeNumber(int64(len(saved.dimLengths))) 607 | for i := range saved.dimLengths { 608 | name := saved.dimNames[i] 609 | dimID := cw.dimIds[name] 610 | cw.writeNumber(int64(dimID)) 611 | } 612 | cw.writeAttributes(saved.attrs) 613 | 614 | write32(cw.bf, int32(saved.ty)) 615 | vsize := int64(0) 616 | switch saved.ty { 617 | case typeDouble, typeInt64, typeUInt64: 618 | vsize = 8 619 | case typeInt, typeFloat, typeUInt: 620 | vsize = 4 621 | case typeShort, typeUShort: 622 | vsize = 2 623 | case typeChar, typeByte, typeUByte: 624 | vsize = 1 625 | default: 626 | thrower.Throw(ErrInternal) 627 | } 628 | for i, v := range saved.dimLengths { 629 | if v != 0 { 630 | vsize *= v 631 | continue 632 | } 633 | if i != 0 { 634 | logger.Error(saved.name, "dimension", i, "name", saved.dimNames[i], 635 | "has length 0") 636 | thrower.Throw(ErrUnlimitedMustBeFirst) 637 | } 638 | } 639 | // pad vsize 640 | vsize = 4 * ((vsize + 3) / 4) 641 | cw.writeNumber(int64(vsize)) 642 | saved.vsize = vsize 643 | 644 | calcOffset := cw.begin 645 | for i := 0; i < which; i++ { 646 | calcOffset += cw.vars[i].vsize 647 | } 648 | write64(cw.bf, calcOffset) 649 | } 650 | 651 | func (cw *CDFWriter) computeAttributeSize(attrs api.AttributeMap) int64 { 652 | count := int64(0) 653 | addLength := func() { 654 | count += 4 655 | if cw.version == 5 { 656 | count += 4 657 | } 658 | } 659 | pad := func() { 660 | count = (count + 3) & ^0x3 661 | } 662 | count += 4 663 | addLength() 664 | if attrs == nil || len(attrs.Keys()) == 0 { 665 | return count 666 | } 667 | for _, k := range attrs.Keys() { 668 | v, _ := attrs.Get(k) 669 | // name 670 | addLength() 671 | count += int64(len(k)) 672 | pad() 673 | 674 | // type 675 | count += 4 676 | 677 | // nelems 678 | addLength() 679 | switch val := v.(type) { 680 | case string: 681 | count += int64(len(val)) 682 | 683 | case int8: 684 | count += 1 685 | 686 | case []int8: 687 | count += int64(len(val)) 688 | 689 | case int16: 690 | count += 2 691 | 692 | case []int16: 693 | count += 2 * int64(len(val)) 694 | 695 | case int32: 696 | count += 4 697 | 698 | case []int32: 699 | count += 4 * int64(len(val)) 700 | 701 | case float32: 702 | count += 4 703 | 704 | case []float32: 705 | count += 4 * int64(len(val)) 706 | 707 | case float64: 708 | count += 8 709 | 710 | case []float64: 711 | count += 8 * int64(len(val)) 712 | 713 | // v5 714 | case uint8: // []uint8 and []byte are the same thing 715 | count += 1 716 | 717 | case []uint8: // []uint8 and []byte are the same thing 718 | count += int64(len(val)) 719 | 720 | case uint16: 721 | count += 2 722 | 723 | case []uint16: 724 | count += 2 * int64(len(val)) 725 | 726 | case uint32: 727 | count += 4 728 | 729 | case []uint32: 730 | count += 4 * int64(len(val)) 731 | 732 | case int64: 733 | count += 8 734 | 735 | case []int64: 736 | count += 8 * int64(len(val)) 737 | 738 | case uint64: 739 | count += 8 740 | 741 | case []uint64: 742 | count += 8 * int64(len(val)) 743 | 744 | default: 745 | logger.Warnf("Unknown type %T, %#v=%#v", v, k, v) 746 | thrower.Throw(ErrUnknownType) 747 | } 748 | pad() 749 | } 750 | return count 751 | } 752 | 753 | func (cw *CDFWriter) computeVarSize(saved *savedVar) int64 { 754 | count := int64(0) 755 | addLength := func() { 756 | count += 4 757 | if cw.version == 5 { 758 | count += 4 759 | } 760 | } 761 | pad := func() { 762 | count = (count + 3) & ^0x3 763 | } 764 | // name 765 | addLength() 766 | count += int64(len(saved.name)) 767 | pad() 768 | 769 | // dims 770 | addLength() 771 | // dims 772 | count += int64(len(saved.dimLengths) * 4) 773 | if cw.version == 5 { 774 | count += int64(len(saved.dimLengths) * 4) 775 | } 776 | 777 | // attributes 778 | count += cw.computeAttributeSize(saved.attrs) 779 | // type 780 | count += 4 781 | // vsize 782 | addLength() 783 | // offset 784 | count += 8 785 | return count 786 | } 787 | 788 | func roundInt64(i int64) int64 { 789 | return (i + 3) & ^0x3 790 | } 791 | 792 | func (cw *CDFWriter) pad() { 793 | offset := cw.bf.Count() 794 | extra := roundInt64(offset) - offset 795 | if extra > 0 { 796 | zero := [3]byte{} 797 | writeBytes(cw.bf, zero[:extra]) 798 | } 799 | } 800 | 801 | func (cw *CDFWriter) writeData(saved savedVar) { 802 | switch saved.ty { 803 | case typeByte: 804 | cw.storeBytes(reflect.ValueOf(saved.val), saved.dimLengths) 805 | cw.pad() 806 | case typeChar: // char in CDF is string/byte in Go 807 | cw.storeChars(reflect.ValueOf(saved.val), saved.dimLengths) 808 | cw.pad() 809 | case typeShort: 810 | cw.storeShorts(reflect.ValueOf(saved.val), saved.dimLengths) 811 | cw.pad() 812 | case typeInt: 813 | cw.storeInts(reflect.ValueOf(saved.val), saved.dimLengths) 814 | case typeFloat: 815 | cw.storeFloats(reflect.ValueOf(saved.val), saved.dimLengths) 816 | case typeDouble: 817 | cw.storeDoubles(reflect.ValueOf(saved.val), saved.dimLengths) 818 | case typeInt64: 819 | cw.storeInt64s(reflect.ValueOf(saved.val), saved.dimLengths) 820 | case typeUInt64: 821 | cw.storeUInt64s(reflect.ValueOf(saved.val), saved.dimLengths) 822 | case typeUInt: 823 | cw.storeUInts(reflect.ValueOf(saved.val), saved.dimLengths) 824 | case typeUShort: 825 | cw.storeUShorts(reflect.ValueOf(saved.val), saved.dimLengths) 826 | cw.pad() 827 | case typeUByte: 828 | cw.storeUBytes(reflect.ValueOf(saved.val), saved.dimLengths) 829 | cw.pad() 830 | default: 831 | thrower.Throw(ErrInternal) 832 | } 833 | } 834 | 835 | // Close writes all the data out and closes the file. 836 | func (cw *CDFWriter) Close() (err error) { 837 | defer thrower.RecoverError(&err) 838 | cw.writeAll() 839 | err = cw.bf.Flush() 840 | err2 := cw.file.Close() 841 | if err == nil { 842 | err = err2 843 | } else { 844 | // return the first error, log the second 845 | logger.Error(err2) 846 | } 847 | cw.file = nil 848 | return err 849 | } 850 | 851 | func writeAny(w io.Writer, any interface{}) { 852 | err := binary.Write(w, binary.BigEndian, any) 853 | thrower.ThrowIfError(err) 854 | } 855 | 856 | func writeBytes(w io.Writer, bytes []byte) { 857 | err := binary.Write(w, binary.BigEndian, bytes) 858 | thrower.ThrowIfError(err) 859 | } 860 | 861 | func write8(w io.Writer, i int8) { 862 | data := byte(i) 863 | err := binary.Write(w, binary.BigEndian, &data) 864 | thrower.ThrowIfError(err) 865 | } 866 | 867 | func write16(w io.Writer, i int16) { 868 | err := binary.Write(w, binary.BigEndian, &i) 869 | thrower.ThrowIfError(err) 870 | } 871 | 872 | func write32(w io.Writer, i int32) { 873 | err := binary.Write(w, binary.BigEndian, &i) 874 | thrower.ThrowIfError(err) 875 | } 876 | 877 | func write64(w io.Writer, i int64) { 878 | err := binary.Write(w, binary.BigEndian, &i) 879 | thrower.ThrowIfError(err) 880 | } 881 | 882 | func (cw *CDFWriter) writeName(name string) { 883 | // namelength 884 | cw.writeNumber(int64(len(name))) 885 | // name 886 | writeBytes(cw.bf, []byte(name)) 887 | cw.pad() 888 | } 889 | 890 | func (cw *CDFWriter) writeNumber(n int64) { 891 | if cw.version < 5 { 892 | write32(cw.bf, int32(n)) 893 | } else { 894 | write64(cw.bf, n) 895 | } 896 | } 897 | 898 | func (cw *CDFWriter) writeAll() { 899 | writeBytes(cw.bf, []byte("CDF")) 900 | write8(cw.bf, cw.version) // version 2 to handle big files 901 | numRecs := int64(0) // 0: not unlimited 902 | cw.writeNumber(numRecs) 903 | if len(cw.dimLengths) > 0 { 904 | write32(cw.bf, fieldDimension) 905 | cw.writeNumber(int64(len(cw.dimLengths))) 906 | for dimid := int64(0); dimid < cw.nextID; dimid++ { 907 | name := cw.dimNames[dimid] 908 | cw.writeName(name) 909 | cw.writeNumber(cw.dimLengths[name]) 910 | } 911 | } else { 912 | write32(cw.bf, 0) // dimensions: absent 913 | cw.writeNumber(int64(0)) // dimensions: absent 914 | } 915 | cw.checkV5Attributes(cw.globalAttrs) 916 | cw.writeAttributes(cw.globalAttrs) 917 | if len(cw.vars) > 0 { 918 | write32(cw.bf, fieldVariable) 919 | cw.writeNumber(int64(len(cw.vars))) 920 | 921 | // Calculate the beginning of where the data is going to be stored. 922 | // The var entries will need that to calculate their offset. 923 | cw.begin = cw.bf.Count() 924 | for i := range cw.vars { 925 | cw.begin += cw.computeVarSize(&cw.vars[i]) 926 | } 927 | 928 | for i := range cw.vars { 929 | cw.writeVar(i) 930 | } 931 | for i := range cw.vars { 932 | cw.writeData(cw.vars[i]) 933 | } 934 | } else { 935 | write32(cw.bf, 0) // variables: absent 936 | cw.writeNumber(int64(0)) // variables: absent 937 | } 938 | } 939 | 940 | // OpenWriter creates the file and make it available for writing 941 | // using AddVar and AddGlobalAttrs. The file must be closed to actually 942 | // write it out. 943 | func OpenWriter(fileName string) (*CDFWriter, error) { 944 | file, err := os.Create(fileName) 945 | if err != nil { 946 | return nil, err 947 | } 948 | bf := bufio.NewWriter(file) 949 | globalAttrs, err := util.NewOrderedMap( 950 | []string{ncpKey}, 951 | map[string]interface{}{ 952 | ncpKey: "version=2,github.com/batchatco/go-native-netcdf=1.0", 953 | }) 954 | thrower.ThrowIfError(err) 955 | cw := &CDFWriter{ 956 | file: file, 957 | bf: &countedWriter{bf, 0}, 958 | vars: nil, 959 | globalAttrs: globalAttrs, 960 | dimLengths: make(map[string]int64), 961 | dimIds: make(map[string]int64), 962 | dimNames: nil, 963 | nextID: 0, 964 | version: 2} 965 | return cw, nil 966 | } 967 | --------------------------------------------------------------------------------