├── test ├── breakfast.jlso ├── .gitignore ├── specimens │ ├── v1_bson.jlso │ ├── v1_serialize.jlso │ ├── v2_bson_gzip.jlso │ ├── v2_bson_none.jlso │ ├── v3_bson_gzip.jlso │ ├── v3_bson_none.jlso │ ├── v2_bson_gzip_fastest.jlso │ ├── v2_bson_gzip_smallest.jlso │ ├── v3_bson_gzip_fastest.jlso │ ├── v3_bson_gzip_smallest.jlso │ ├── v2_julia_serialize_gzip.jlso │ ├── v2_julia_serialize_none.jlso │ ├── v3_julia_serialize_gzip.jlso │ ├── v3_julia_serialize_none.jlso │ ├── v2_julia_serialize_gzip_fastest.jlso │ ├── v3_julia_serialize_gzip_fastest.jlso │ ├── v2_julia_serialize_gzip_smallest.jlso │ └── v3_julia_serialize_gzip_smallest.jlso ├── runtests.jl ├── test_data_setup.jl ├── JLSOFile.jl ├── backwards_compat.jl └── file_io.jl ├── docs ├── src │ ├── index.md │ ├── api.md │ ├── assets │ │ └── invenia.css │ ├── metadata.md │ └── upgrading.md ├── Project.toml ├── make.jl └── Manifest.toml ├── REQUIRE ├── .gitignore ├── .github └── workflows │ └── TagBot.yml ├── .travis.yml ├── .appveyor.yml ├── LICENSE ├── Project.toml ├── src ├── metadata.jl ├── JLSO.jl ├── serialization.jl ├── JLSOFile.jl ├── file_io.jl └── upgrade.jl └── README.md /test/breakfast.jlso: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/.gitignore: -------------------------------------------------------------------------------- 1 | demo 2 | -------------------------------------------------------------------------------- /docs/src/index.md: -------------------------------------------------------------------------------- 1 | ../../README.md -------------------------------------------------------------------------------- /REQUIRE: -------------------------------------------------------------------------------- 1 | julia 0.7 2 | BSON 3 | Memento 0.10.0 4 | -------------------------------------------------------------------------------- /test/specimens/v1_bson.jlso: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/findmyway/JLSO.jl/master/test/specimens/v1_bson.jlso -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | # Documenter generated files 3 | /docs/build/ 4 | /docs/site/ 5 | 6 | /Manifest.toml 7 | -------------------------------------------------------------------------------- /test/specimens/v1_serialize.jlso: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/findmyway/JLSO.jl/master/test/specimens/v1_serialize.jlso -------------------------------------------------------------------------------- /test/specimens/v2_bson_gzip.jlso: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/findmyway/JLSO.jl/master/test/specimens/v2_bson_gzip.jlso -------------------------------------------------------------------------------- /test/specimens/v2_bson_none.jlso: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/findmyway/JLSO.jl/master/test/specimens/v2_bson_none.jlso -------------------------------------------------------------------------------- /test/specimens/v3_bson_gzip.jlso: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/findmyway/JLSO.jl/master/test/specimens/v3_bson_gzip.jlso -------------------------------------------------------------------------------- /test/specimens/v3_bson_none.jlso: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/findmyway/JLSO.jl/master/test/specimens/v3_bson_none.jlso -------------------------------------------------------------------------------- /test/specimens/v2_bson_gzip_fastest.jlso: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/findmyway/JLSO.jl/master/test/specimens/v2_bson_gzip_fastest.jlso -------------------------------------------------------------------------------- /test/specimens/v2_bson_gzip_smallest.jlso: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/findmyway/JLSO.jl/master/test/specimens/v2_bson_gzip_smallest.jlso -------------------------------------------------------------------------------- /test/specimens/v3_bson_gzip_fastest.jlso: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/findmyway/JLSO.jl/master/test/specimens/v3_bson_gzip_fastest.jlso -------------------------------------------------------------------------------- /test/specimens/v3_bson_gzip_smallest.jlso: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/findmyway/JLSO.jl/master/test/specimens/v3_bson_gzip_smallest.jlso -------------------------------------------------------------------------------- /test/specimens/v2_julia_serialize_gzip.jlso: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/findmyway/JLSO.jl/master/test/specimens/v2_julia_serialize_gzip.jlso -------------------------------------------------------------------------------- /test/specimens/v2_julia_serialize_none.jlso: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/findmyway/JLSO.jl/master/test/specimens/v2_julia_serialize_none.jlso -------------------------------------------------------------------------------- /test/specimens/v3_julia_serialize_gzip.jlso: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/findmyway/JLSO.jl/master/test/specimens/v3_julia_serialize_gzip.jlso -------------------------------------------------------------------------------- /test/specimens/v3_julia_serialize_none.jlso: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/findmyway/JLSO.jl/master/test/specimens/v3_julia_serialize_none.jlso -------------------------------------------------------------------------------- /test/specimens/v2_julia_serialize_gzip_fastest.jlso: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/findmyway/JLSO.jl/master/test/specimens/v2_julia_serialize_gzip_fastest.jlso -------------------------------------------------------------------------------- /test/specimens/v3_julia_serialize_gzip_fastest.jlso: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/findmyway/JLSO.jl/master/test/specimens/v3_julia_serialize_gzip_fastest.jlso -------------------------------------------------------------------------------- /test/specimens/v2_julia_serialize_gzip_smallest.jlso: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/findmyway/JLSO.jl/master/test/specimens/v2_julia_serialize_gzip_smallest.jlso -------------------------------------------------------------------------------- /test/specimens/v3_julia_serialize_gzip_smallest.jlso: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/findmyway/JLSO.jl/master/test/specimens/v3_julia_serialize_gzip_smallest.jlso -------------------------------------------------------------------------------- /docs/Project.toml: -------------------------------------------------------------------------------- 1 | [deps] 2 | Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" 3 | JLSO = "9da8a3cd-07a3-59c0-a743-3fdc52c30d11" 4 | 5 | [compat] 6 | Documenter = "~0.21" 7 | -------------------------------------------------------------------------------- /docs/src/api.md: -------------------------------------------------------------------------------- 1 | # API 2 | 3 | ```@autodocs 4 | Modules = [JLSO] 5 | Public = true 6 | Private = true 7 | Pages = ["JLSO.jl", "JLSOFile.jl", "metadata.jl", "file_io.jl", "serialization.jl"] 8 | ``` 9 | -------------------------------------------------------------------------------- /.github/workflows/TagBot.yml: -------------------------------------------------------------------------------- 1 | name: TagBot 2 | on: 3 | schedule: 4 | - cron: 0 * * * * 5 | jobs: 6 | TagBot: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: JuliaRegistries/TagBot@v1 10 | with: 11 | token: ${{ secrets.GITHUB_TOKEN }} 12 | -------------------------------------------------------------------------------- /test/runtests.jl: -------------------------------------------------------------------------------- 1 | using BSON 2 | using Dates 3 | using Distributed 4 | using FilePathsBase: SystemPath 5 | using InteractiveUtils 6 | using Memento 7 | using Pkg 8 | using Random 9 | using Serialization 10 | using Suppressor 11 | using Test 12 | 13 | using JLSO 14 | using JLSO: JLSOFile, LOGGER, upgrade_jlso 15 | 16 | # To test different types from common external packages 17 | using DataFrames 18 | using Distributions 19 | using TimeZones 20 | 21 | include("test_data_setup.jl") 22 | @testset "JLSO" begin 23 | include("backwards_compat.jl") 24 | include("JLSOFile.jl") 25 | include("file_io.jl") 26 | end 27 | -------------------------------------------------------------------------------- /docs/make.jl: -------------------------------------------------------------------------------- 1 | using Documenter, JLSO 2 | 3 | makedocs( 4 | modules=[JLSO], 5 | format=Documenter.HTML(prettyurls = get(ENV, "CI", nothing) == "true"), 6 | pages=[ 7 | "Home" => "index.md", 8 | "Metadata" => "metadata.md", 9 | "Upgrading" => "upgrading.md", 10 | "API" => "api.md", 11 | ], 12 | repo="https://github.com/invenia/JLSO.jl/blob/{commit}{path}#L{line}", 13 | sitename="JLSO.jl", 14 | authors="Rory Finnegan", 15 | assets=[ 16 | "assets/invenia.css", 17 | ], 18 | strict = true, 19 | checkdocs = :none, 20 | ) 21 | 22 | deploydocs(; 23 | repo="github.com/invenia/JLSO.jl", 24 | ) 25 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Documentation: http://docs.travis-ci.com/user/languages/julia/ 2 | language: julia 3 | os: 4 | - linux 5 | - osx 6 | julia: 7 | - 1.0 8 | - 1.1 9 | - 1.2 10 | - 1.3 11 | - nightly 12 | matrix: 13 | allow_failures: 14 | - julia: nightly 15 | fast_finish: true 16 | notifications: 17 | email: false 18 | after_success: 19 | - julia -e 'using Pkg; Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder())' 20 | jobs: 21 | include: 22 | - stage: Documentation 23 | julia: 1 24 | script: julia --project=docs -e ' 25 | using Pkg; 26 | Pkg.develop(PackageSpec(path=pwd())); 27 | Pkg.instantiate(); 28 | include("docs/make.jl");' 29 | after_success: skip 30 | -------------------------------------------------------------------------------- /.appveyor.yml: -------------------------------------------------------------------------------- 1 | # Documentation: https://github.com/JuliaCI/Appveyor.jl 2 | environment: 3 | matrix: 4 | - julia_version: 1.0 5 | - julia_version: 1.1 6 | - julia_version: nightly 7 | platform: 8 | - x86 9 | - x64 10 | matrix: 11 | allow_failures: 12 | - julia_version: nightly 13 | - platform: x86 # see https://github.com/invenia/JLSO.jl/issues/12 14 | branches: 15 | only: 16 | - master 17 | - /release-.*/ 18 | notifications: 19 | - provider: Email 20 | on_build_success: false 21 | on_build_failure: false 22 | on_build_status_changed: false 23 | install: 24 | - ps: iex ((new-object net.webclient).DownloadString("https://raw.githubusercontent.com/JuliaCI/Appveyor.jl/version-1/bin/install.ps1")) 25 | build_script: 26 | - echo "%JL_BUILD_SCRIPT%" 27 | - C:\julia\bin\julia -e "%JL_BUILD_SCRIPT%" 28 | test_script: 29 | - echo "%JL_TEST_SCRIPT%" 30 | - C:\julia\bin\julia -e "%JL_TEST_SCRIPT%" 31 | on_success: 32 | - echo "%JL_CODECOV_SCRIPT%" 33 | - C:\julia\bin\julia -e "%JL_CODECOV_SCRIPT%" 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019 Invenia Technical Computing Corporation 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /Project.toml: -------------------------------------------------------------------------------- 1 | name = "JLSO" 2 | uuid = "9da8a3cd-07a3-59c0-a743-3fdc52c30d11" 3 | license = "MIT" 4 | authors = ["Invenia Technical Computing Corperation"] 5 | version = "2.3.2" 6 | 7 | [deps] 8 | BSON = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0" 9 | CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193" 10 | FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f" 11 | Memento = "f28f55f0-a522-5efc-85c2-fe41dfb9b2d9" 12 | Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" 13 | Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" 14 | 15 | [compat] 16 | BSON = "0.2.4" 17 | CodecZlib = "0.6, 0.7" 18 | FilePathsBase = "0.7, 0.8, 0.9" 19 | Memento = "0.10, 0.11, 0.12, 0.13, 1" 20 | TimeZones = "0.9, 0.10, 0.11, 1" 21 | julia = "0.7, 1.0" 22 | 23 | [extras] 24 | AxisArrays = "39de3d68-74b9-583c-8d2d-e117c070f3a9" 25 | DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" 26 | Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" 27 | Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" 28 | Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" 29 | InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" 30 | Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" 31 | Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb" 32 | Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" 33 | TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53" 34 | 35 | [targets] 36 | test = ["AxisArrays", "DataFrames", "Dates", "Distributed", "Distributions", "InteractiveUtils", "Random", "Suppressor", "Test", "TimeZones"] 37 | -------------------------------------------------------------------------------- /test/test_data_setup.jl: -------------------------------------------------------------------------------- 1 | # Serialize "Hello World!" on julia 0.5.2 (not supported) 2 | const img = JLSO._image() 3 | const pkgs = JLSO._pkgs() 4 | const project, manifest = Pkg.TOML.parse.(JLSO._env()) 5 | const hw_5 = UInt8[ 6 | 0x26, 0x15, 0x87, 0x48, 0x65, 7 | 0x6c, 0x6c, 0x6f, 0x20, 0x57, 8 | 0x6f, 0x72, 0x6c, 0x64, 0x21 9 | ] 10 | 11 | const datas = Dict( 12 | :String => "Hello World!", 13 | :Vector => [0.867244, 0.711437, 0.512452, 0.863122, 0.907903], 14 | :Matrix => [ 15 | 0.400348 0.892196 0.848164; 16 | 0.0183529 0.755449 0.397538; 17 | 0.870458 0.0441878 0.170899 18 | ], 19 | :DateTime => DateTime(2018, 1, 28), 20 | :ZonedDateTime => ZonedDateTime(2018, 1, 28, tz"America/Chicago"), 21 | :DataFrame => DataFrame( 22 | :a => collect(1:5), 23 | :b => [0.867244, 0.711437, 0.512452, 0.863122, 0.907903], 24 | :c => ["a", "b", "c", "d", "e"], 25 | :d => [true, true, false, false, true], 26 | ), 27 | :Distribution => Normal(50.2, 4.3), 28 | ) 29 | 30 | #== 31 | for format in (:bson, :julia_serialize) 32 | for compression in (:none, :gzip, :gzip_fastest, :gzip_smallest) 33 | fn = "specimens/v2_$(format)_$(compression).jlso" 34 | time = @elapsed JLSO.save(fn, datas; format=format, compression=compression); 35 | time = @elapsed JLSO.save(fn, datas; format=format, compression=compression); 36 | @info "" format compression time size=filesize(fn) 37 | end 38 | end 39 | ==# 40 | -------------------------------------------------------------------------------- /docs/src/assets/invenia.css: -------------------------------------------------------------------------------- 1 | /* Links */ 2 | 3 | a { 4 | color: #4595D1; 5 | } 6 | 7 | a:hover, a:focus { 8 | color: #194E82; 9 | } 10 | 11 | /* Navigation */ 12 | 13 | nav.toc ul a:hover, 14 | nav.toc ul.internal a:hover { 15 | color: #FFFFFF; 16 | background-color: #4595D1; 17 | } 18 | 19 | nav.toc ul .toctext { 20 | color: #FFFFFF; 21 | } 22 | 23 | nav.toc { 24 | box-shadow: none; 25 | color: #FFFFFF; 26 | background-color: #194E82; 27 | } 28 | 29 | nav.toc li.current > .toctext { 30 | color: #FFFFFF; 31 | background-color: #4595D1; 32 | border-top-width: 0px; 33 | border-bottom-width: 0px; 34 | } 35 | 36 | nav.toc ul.internal a { 37 | color: #194E82; 38 | background-color: #FFFFFF; 39 | } 40 | 41 | /* Text */ 42 | 43 | article#docs a.nav-anchor { 44 | color: #194E82; 45 | } 46 | 47 | article#docs blockquote { 48 | font-style: italic; 49 | } 50 | 51 | /* Code */ 52 | 53 | code .hljs-meta { 54 | color: #4595D1; 55 | } 56 | 57 | code .hljs-keyword { 58 | color: #194E82; 59 | } 60 | 61 | pre, code { 62 | font-family: "Liberation Mono", "Consolas", "DejaVu Sans Mono", "Ubuntu Mono", "Courier New", "andale mono", "lucida console", monospace; 63 | } 64 | 65 | /* mkdocs (old) */ 66 | 67 | /*.navbar-default { 68 | background-color: #194E82; 69 | } 70 | 71 | .navbar-default .navbar-nav > .active > a, 72 | .navbar-default .navbar-nav > .active > a:hover, 73 | .navbar-default .navbar-nav > .active > a:focus { 74 | background-color: #4595D1; 75 | }*/ 76 | -------------------------------------------------------------------------------- /test/JLSOFile.jl: -------------------------------------------------------------------------------- 1 | @testset "JLSOFile" begin 2 | 3 | withenv("JLSO_IMAGE" => "busybox") do 4 | jlso = JLSOFile(:data => "the image env variable is set") 5 | @test jlso.image == "busybox" 6 | end 7 | 8 | # Reset the cached image for future tests 9 | JLSO._CACHE[:IMAGE] = "" 10 | 11 | @testset "$(fmt): $k" for fmt in (:bson, :julia_serialize), (k, v) in datas 12 | jlso = JLSOFile(k => v; format=fmt, compression=:none) 13 | io = IOBuffer() 14 | bytes = fmt === :bson ? bson(io, Dict("object" => v)) : serialize(io, v) 15 | expected = take!(io) 16 | 17 | @test jlso.objects[k] == expected 18 | end 19 | 20 | @testset "kwarg constructor" begin 21 | jlso = JLSOFile(; a=collect(1:10), b="hello") 22 | @test jlso[:b] == "hello" 23 | @test haskey(jlso.manifest, "BSON") 24 | end 25 | end 26 | 27 | @testset "unknown format" begin 28 | @test_throws( 29 | LOGGER, 30 | MethodError, 31 | JLSOFile("String" => "Hello World!", format=:unknown) 32 | ) 33 | end 34 | 35 | @testset "show" begin 36 | jlso = JLSOFile(:string => datas[:String]) 37 | expected = string( 38 | "JLSOFile([data]; version=v\"2.0.0\", julia=v\"$VERSION\", ", 39 | "format=:julia_serialize, image=\"\")" 40 | ) 41 | @test sprint(show, jlso) == sprint(print, jlso) 42 | end 43 | 44 | @testset "activate" begin 45 | jlso = JLSOFile(:string => datas[:String]) 46 | mktempdir() do d 47 | Pkg.activate(jlso, d) do 48 | @show Base.active_project() 49 | end 50 | 51 | @test ispath(joinpath(d, "Project.toml")) 52 | @test ispath(joinpath(d, "Manifest.toml")) 53 | 54 | @test Pkg.TOML.parsefile(joinpath(d, "Project.toml")) == jlso.project 55 | @test Pkg.TOML.parsefile(joinpath(d, "Manifest.toml")) == jlso.manifest 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /docs/src/metadata.md: -------------------------------------------------------------------------------- 1 | # Metadata 2 | 3 | Manually reading JLSO files can be helpful when addressing issues deserializing objects or to simply to help with reproducibility. 4 | 5 | ``` 6 | julia> jlso = read("breakfast.jlso", JLSOFile) 7 | JLSOFile([cost, food, time]; version="3.0.0", julia="1.0.3", format=:julia_serialize, compression=:gzip, image="") 8 | ``` 9 | 10 | Now we can manually access the serialized objects: 11 | ``` 12 | julia> jlso.objects 13 | Dict{Symbol,Array{UInt8,1}} with 3 entries: 14 | :cost => UInt8[0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13 … 0x0e, 0x00, 0x8b, 0xea, 0xfc, 0xc1, 0x11, 0x00, 0x00, 0x00] 15 | :food => UInt8[0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13 … 0x66, 0x00, 0x10, 0x8c, 0x21, 0xf6, 0x18, 0x00, 0x00, 0x00] 16 | :time => UInt8[0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13 … 0x0c, 0x00, 0xa7, 0xc5, 0x38, 0x8c, 0x5b, 0x00, 0x00, 0x00] 17 | ``` 18 | 19 | Or deserialize individual objects: 20 | ``` 21 | julia> jlso[:food] 22 | "☕️🥓🍳" 23 | ``` 24 | 25 | Maybe you need to figure out what the environment state was when you wrote the file? 26 | ``` 27 | julia> jlso.project 28 | Dict{String,Any} with 9 entries: 29 | "deps" => Dict{String,Any}("CodecZlib"=>"944b1d66-785c-5afd-91f1-9de20f533193","Pkg"=>"44cfe95a-1eb2-52ea-b672-e2afdf69b78f","Serialization"=>"9e88b42a-f829-5b0c-bbe9-9e923198166b","BSON"=>"fbb218c0-5317… 30 | "name" => "JLSO" 31 | ... 32 | 33 | julia> jlso.manifest 34 | Dict{String,Any} with 39 entries: 35 | "Mocking" => Dict{String,Any}[Dict("git-tree-sha1"=>"bd2623f8b728af988d2afec53d611acb621f3bc4","uuid"=>"78c3b35d-d492-501b-9361-3d52fe80e533","version"=>"0.7.0")] 36 | "Pkg" => Dict{String,Any}[Dict("deps"=>["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"], 37 | ... 38 | 39 | ``` 40 | 41 | These `project` and `manifest` fields are just the dictionary representations of the Project.toml and Manifest.toml files found in a Julia Pkg environment. 42 | In the future, we may add some tooling to make it easier to view and compare these dictionaries, but it's currently unclear if that should live here or in Pkg.jl. 43 | -------------------------------------------------------------------------------- /src/metadata.jl: -------------------------------------------------------------------------------- 1 | function _versioncheck(version::VersionNumber, valid_versions) 2 | supported = version ∈ valid_versions 3 | supported || error(LOGGER, ArgumentError( 4 | string( 5 | "Unsupported version ($version). ", 6 | "Expected a value between ($valid_versions)." 7 | ) 8 | )) 9 | end 10 | 11 | function _versioncheck(version::String, valid_versions) 12 | return _versioncheck(VersionNumber(version), valid_versions) 13 | end 14 | 15 | # Cache of the versioninfo and image, so we don't compute these every time. 16 | const _CACHE = Dict( 17 | :MANIFEST => "", 18 | :PROJECT => "", 19 | :PKGS => Dict{String, VersionNumber}(), 20 | :IMAGE => "", 21 | ) 22 | 23 | function _pkgs() 24 | if isempty(_CACHE[:PKGS]) 25 | for (pkg, ver) in Pkg.installed() 26 | # BSON can't handle Void types 27 | if ver !== nothing 28 | global _CACHE[:PKGS][pkg] = ver 29 | end 30 | end 31 | end 32 | 33 | return _CACHE[:PKGS] 34 | end 35 | 36 | function _env() 37 | if isempty(_CACHE[:PROJECT]) && isempty(_CACHE[:MANIFEST]) 38 | _CACHE[:PROJECT] = read(Base.active_project(), String) 39 | _CACHE[:MANIFEST] = read( 40 | joinpath(dirname(Base.active_project()), "Manifest.toml"), 41 | String 42 | ) 43 | end 44 | 45 | return (_CACHE[:PROJECT], _CACHE[:MANIFEST]) 46 | end 47 | 48 | function _image() 49 | if isempty(_CACHE[:IMAGE]) && haskey(ENV, "JLSO_IMAGE") 50 | return ENV["JLSO_IMAGE"] 51 | end 52 | 53 | return _CACHE[:IMAGE] 54 | end 55 | 56 | function Pkg.activate(jlso::JLSOFile, path=tempdir(); kwargs...) 57 | mkpath(path) 58 | open(io -> Pkg.TOML.print(io, jlso.project), joinpath(path, "Project.toml"), "w") 59 | open(io -> Pkg.TOML.print(io, jlso.manifest), joinpath(path, "Manifest.toml"), "w") 60 | Pkg.activate(path; kwargs...) 61 | end 62 | 63 | function Pkg.activate(f::Function, jlso::JLSOFile, path=tempdir(); kwargs...) 64 | curr_env = dirname(Base.active_project()) 65 | try 66 | Pkg.activate(jlso, path; kwargs...) 67 | f() 68 | finally 69 | Pkg.activate(curr_env) 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /src/JLSO.jl: -------------------------------------------------------------------------------- 1 | """ 2 | A julia serialized object (JLSO) file format for storing checkpoint data. 3 | 4 | # Structure 5 | 6 | The .jlso files are BSON files containing the dictionaries with a specific schema. 7 | NOTE: The raw dictionary should be loadable by any BSON library even if serialized objects 8 | themselves aren't reconstructable. 9 | 10 | Example) 11 | ``` 12 | Dict( 13 | "metadata" => Dict( 14 | "version" => v"2.0", 15 | "julia" => v"1.0.4", 16 | "format" => :bson, # Could also be :julia_serialize 17 | "compression" => :gzip_fastest, # could also be: :none, :gzip_smallest, or :gzip 18 | "image" => "xxxxxxxxxxxx.dkr.ecr.us-east-1.amazonaws.com/myrepository:latest" 19 | "project" => Dict{String, Any}(...), 20 | "manifest" => Dict{String, Any}(...), 21 | ), 22 | "objects" => Dict( 23 | "var1" => [0x35, 0x10, 0x01, 0x04, 0x44], 24 | "var2" => [...], 25 | ), 26 | ) 27 | ``` 28 | WARNING: Regardless of serialization `format`, the serialized objects can not be deserialized 29 | into structures with different fields, or if the types have been renamed or removed from the 30 | packages. 31 | Further, the `:julia_serialize` format is not intended for long term storage and is not 32 | portable across julia versions. As a result, we're storing the serialized object data 33 | in a json file which should also be able to load the docker image and versioninfo to allow 34 | reconstruction. 35 | """ 36 | module JLSO 37 | 38 | using BSON 39 | using CodecZlib 40 | using FilePathsBase: AbstractPath 41 | using Memento 42 | using Pkg: Pkg 43 | using Pkg.Types: semver_spec 44 | using Serialization 45 | 46 | @static if VERSION < v"1.3.0" 47 | macro spawn(ex) 48 | esc(ex) 49 | end 50 | else 51 | using Base.Threads: @spawn 52 | end 53 | 54 | # We need to import these cause of a deprecation on object index via strings. 55 | import Base: getindex, setindex! 56 | 57 | export JLSOFile 58 | 59 | const READABLE_VERSIONS = semver_spec("1, 2, 3") 60 | const WRITEABLE_VERSIONS = semver_spec("3") 61 | 62 | const LOGGER = getlogger(@__MODULE__) 63 | __init__() = Memento.register(LOGGER) 64 | 65 | include("JLSOFile.jl") 66 | include("upgrade.jl") 67 | include("file_io.jl") 68 | include("metadata.jl") 69 | include("serialization.jl") 70 | 71 | end # module 72 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # JLSO 2 | 3 | [![Stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://invenia.github.io/JLSO.jl/stable) 4 | [![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://invenia.github.io/JLSO.jl/dev) 5 | [![Build Status](https://travis-ci.com/invenia/JLSO.jl.svg?branch=master)](https://travis-ci.com/invenia/JLSO.jl) 6 | [![Build Status](https://ci.appveyor.com/api/projects/status/github/invenia/JLSO.jl?svg=true)](https://ci.appveyor.com/project/invenia/JLSO-jl) 7 | [![Codecov](https://codecov.io/gh/invenia/JLSO.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/invenia/JLSO.jl) 8 | [![Code Style: Blue](https://img.shields.io/badge/code%20style-blue-4495d1.svg)](https://github.com/invenia/BlueStyle) 9 | 10 | JLSO is a storage container for serialized Julia objects. 11 | Think of it less as a serialization format but as a container, 12 | that employs a serializer, and a compressor, handles all the other concerns including metadata and saving. 13 | Such that the serializer just needs to determine how to turn a julia object into a stream`Vector{UInt8}`, 14 | and the compressor just needs to determine how to turn one stream of `UInt8`s into a smaller one (and the reverse). 15 | 16 | 17 | At the top-level it is a BSON file, 18 | where it stores metadata about the system it was created on as well as a collection of objects (the actual data). 19 | Depending on configuration, those objects may themselves be stored as BSON sub-documents, 20 | or in the native Julia serialization format (default), under various levels of compression (`gzip` default). 21 | It is fast and efficient to load just single objects out of a larger file that contains many objects. 22 | 23 | The metadata includes the Julia version and the versions of all packages installed. 24 | It is always store in plain BSON without julia specific extensions. 25 | This means in the worst case you can install everything again and replicate your system. 26 | (Extreme worst case scenario, using a BSON reader from another programming language). 27 | 28 | Note: If the amount of data you have to store is very small, relative to the metadata about your environment, then JLSO is a pretty suboptimal format. 29 | 30 | 31 | ### Basic Example: 32 | 33 | ``` 34 | julia> using JLSO, Dates 35 | 36 | julia> JLSO.save("breakfast.jlso", :food => "☕️🥓🍳", :cost => 11.95, :time => Time(9, 0)) 37 | 38 | julia> loaded = JLSO.load("breakfast.jlso") 39 | Dict{Symbol,Any} with 3 entries: 40 | :cost => 11.95 41 | :time => 09:00:00 42 | :food => "☕️🥓🍳" 43 | ``` 44 | -------------------------------------------------------------------------------- /src/serialization.jl: -------------------------------------------------------------------------------- 1 | # This is the code that hands the serialiation and deserialization of each object 2 | 3 | struct Formatter{S} end 4 | deserialize(format::Symbol, io) = deserialize(Formatter{format}(), io) 5 | serialize(format::Symbol, io, value) = serialize(Formatter{format}(), io, value) 6 | 7 | deserialize(::Formatter{:bson}, io) = first(values(BSON.load(io))) 8 | serialize(::Formatter{:bson}, io, value) = bson(io, Dict("object" => value)) 9 | 10 | deserialize(::Formatter{:julia_serialize}, io) = Serialization.deserialize(io) 11 | serialize(::Formatter{:julia_serialize}, io, value) = Serialization.serialize(io, value) 12 | 13 | struct Compressor{S} end 14 | compress(compression::Symbol, io) = compress(Compressor{compression}(), io) 15 | decompress(compression::Symbol, io) = decompress(Compressor{compression}(), io) 16 | 17 | compress(::Compressor{:none}, io) = io 18 | decompress(::Compressor{:none}, io) = io 19 | 20 | compress(::Compressor{:gzip}, io) = GzipCompressorStream(io) 21 | decompress(::Compressor{:gzip}, io) = GzipDecompressorStream(io) 22 | 23 | compress(::Compressor{:gzip_fastest}, io) = GzipCompressorStream(io; level=1) 24 | decompress(::Compressor{:gzip_fastest}, io) = GzipDecompressorStream(io) 25 | 26 | compress(::Compressor{:gzip_smallest}, io) = GzipCompressorStream(io; level=9) 27 | decompress(::Compressor{:gzip_smallest}, io) = GzipDecompressorStream(io) 28 | 29 | """ 30 | complete_compression(compressing_buffer) 31 | Writes any end of compression sequence to the compressing buffer; 32 | but does not close the underlying stream. 33 | The compressing_buffer itself should not be used after this operation 34 | """ 35 | complete_compression(::Any) = nothing 36 | function complete_compression(compressing_buffer::CodecZlib.TranscodingStream) 37 | # need to close `compressing_buffer` so any compression can write end of body stuffs. 38 | # But can't use normal `close` without closing `buffer` as well 39 | # see https://github.com/bicycle1885/TranscodingStreams.jl/issues/85 40 | CodecZlib.TranscodingStreams.changemode!(compressing_buffer, :close) 41 | end 42 | 43 | 44 | """ 45 | getindex(jlso, name) 46 | 47 | Returns the deserialized object with the specified name. 48 | """ 49 | function getindex(jlso::JLSOFile, name::Symbol) 50 | try 51 | buffer = IOBuffer(jlso.objects[name]) 52 | decompressing_buffer = decompress(jlso.compression, buffer) 53 | return deserialize(jlso.format, decompressing_buffer) 54 | catch e 55 | warn(LOGGER, e) 56 | return jlso.objects[name] 57 | end 58 | end 59 | 60 | """ 61 | setindex!(jlso, value, name) 62 | 63 | Adds the object to the file and serializes it. 64 | """ 65 | function setindex!(jlso::JLSOFile, value, name::Symbol) 66 | buffer = IOBuffer() 67 | compressing_buffer = compress(jlso.compression, buffer) 68 | serialize(jlso.format, compressing_buffer, value) 69 | complete_compression(compressing_buffer) 70 | result = take!(buffer) 71 | 72 | lock(jlso.lock) do 73 | jlso.objects[name] = result 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /src/JLSOFile.jl: -------------------------------------------------------------------------------- 1 | struct JLSOFile 2 | version::VersionNumber 3 | julia::VersionNumber 4 | format::Symbol 5 | compression::Symbol 6 | image::String 7 | project::Dict{String, Any} 8 | manifest::Dict{String, Any} 9 | objects::Dict{Symbol, Vector{UInt8}} 10 | lock::ReentrantLock 11 | end 12 | 13 | """ 14 | JLSOFile(data; format=:julia_serialize, compression=:gzip, kwargs...) 15 | 16 | Stores the information needed to write a .jlso file. 17 | 18 | # Arguments 19 | 20 | - `data` - The objects to be stored in the file. 21 | 22 | # Keywords 23 | 24 | - `image=""` - The docker image URI that was used to generate the file 25 | - `julia=$VERSION` - The julia version used to write the file 26 | - `version=v"2.0"` - The file schema version 27 | - `format=:julia_serialize` - The format to use for serializing individual objects. While `:bson` is 28 | recommended for longer term object storage, `:julia_serialize` tends to be the faster choice 29 | for adhoc serialization. 30 | - `compression=:gzip`, what form of compression to apply to the objects. 31 | Use :none, to not compress. :gzip_fastest for the fastest gzip compression, 32 | :gzip_smallest for the most compact (but slowest), or :gzip for a generally good compromize. 33 | Due to the time taken for disk IO, :none is not normally as fast as using some compression. 34 | """ 35 | function JLSOFile( 36 | data::Dict{Symbol, <:Any}; 37 | version=v"3", 38 | julia=VERSION, 39 | format=:julia_serialize, 40 | compression=:gzip, 41 | image=_image(), 42 | ) 43 | if format === :serialize 44 | # Deprecation warning 45 | @warn "The `:serialize` format has been renamed to `:julia_serialize`." 46 | format = :julia_serialize 47 | end 48 | 49 | _versioncheck(version, WRITEABLE_VERSIONS) 50 | project_toml, manifest_toml = _env() 51 | jlso = JLSOFile( 52 | version, 53 | julia, 54 | format, 55 | compression, 56 | image, 57 | Pkg.TOML.parse(project_toml), 58 | Pkg.TOML.parse(manifest_toml), 59 | Dict{Symbol, Vector{UInt8}}(), 60 | ReentrantLock() 61 | ) 62 | 63 | @sync for (key, val) in data 64 | @spawn jlso[key] = val 65 | end 66 | 67 | return jlso 68 | end 69 | 70 | function JLSOFile(; 71 | version=v"3", 72 | julia=VERSION, 73 | format=:julia_serialize, 74 | compression=:gzip, 75 | image=_image(), 76 | kwargs... 77 | ) 78 | return JLSOFile( 79 | Dict(kwargs); 80 | version=version, 81 | julia=julia, 82 | format=format, 83 | compression=compression, 84 | image=image, 85 | ) 86 | end 87 | 88 | 89 | JLSOFile(data::Pair{Symbol}...; kwargs...) = JLSOFile(Dict(data); kwargs...) 90 | 91 | function Base.show(io::IO, jlso::JLSOFile) 92 | variables = join(names(jlso), ", ") 93 | kwargs = join( 94 | [ 95 | "version=\"$(jlso.version)\"", 96 | "julia=\"$(jlso.julia)\"", 97 | "format=:$(jlso.format)", 98 | "compression=:$(jlso.compression)", 99 | "image=\"$(jlso.image)\"", 100 | ], 101 | ", " 102 | ) 103 | 104 | print(io, "JLSOFile([$variables]; $kwargs)") 105 | end 106 | 107 | function Base.:(==)(a::JLSOFile, b::JLSOFile) 108 | return ( 109 | a.version == b.version && 110 | a.julia == b.julia && 111 | a.image == b.image && 112 | a.manifest == b.manifest && 113 | a.format == b.format && 114 | a.compression == b.compression && 115 | a.objects == b.objects 116 | ) 117 | end 118 | 119 | Base.names(jlso::JLSOFile) = collect(keys(jlso.objects)) 120 | -------------------------------------------------------------------------------- /src/file_io.jl: -------------------------------------------------------------------------------- 1 | # This is the code that handles getting the JLSO file itself on to and off of the disk 2 | # However, it does not describe how to serialize or deserialize the individual objects 3 | # that is done lazily and the code for that is in serialization.jl 4 | 5 | function Base.write(io::IO, jlso::JLSOFile) 6 | _versioncheck(jlso.version, WRITEABLE_VERSIONS) 7 | 8 | # Setup an IOBuffer for serializing the manifest 9 | project_toml = sprint(Pkg.TOML.print, jlso.project) 10 | # @show jlso.manifest 11 | manifest_toml = read( 12 | compress(jlso.compression, IOBuffer(sprint(Pkg.TOML.print, jlso.manifest))) 13 | ) 14 | 15 | bson( 16 | io, 17 | # We declare the dict types to be more explicit about the output format. 18 | Dict{String, Dict}( 19 | "metadata" => Dict{String, Union{String, Vector{UInt8}}}( 20 | "version" => string(jlso.version), 21 | "julia" => string(jlso.julia), 22 | "format" => string(jlso.format), 23 | "compression" => string(jlso.compression), 24 | "image" => jlso.image, 25 | "project" => project_toml, 26 | "manifest" => manifest_toml, 27 | ), 28 | "objects" => Dict{String, Vector{UInt8}}( 29 | string(k) => v for (k, v) in jlso.objects 30 | ), 31 | ) 32 | ) 33 | end 34 | 35 | # `read`, unlike `load` does not deserialize any of the objects within the JLSO file, 36 | # they will be `deserialized` when they are indexed out of the returned JSLOFile object. 37 | function Base.read(io::IO, ::Type{JLSOFile}) 38 | parsed = BSON.load(io) 39 | _versioncheck(parsed["metadata"]["version"], READABLE_VERSIONS) 40 | d = upgrade_jlso(parsed) 41 | compression = Symbol(d["metadata"]["compression"]) 42 | 43 | # Try decompressing the manifest, otherwise just return a dict with the raw data 44 | manifest = try 45 | Pkg.TOML.parse( 46 | read( 47 | decompress(compression, IOBuffer(d["metadata"]["manifest"])), 48 | String 49 | ) 50 | ) 51 | catch e 52 | warn(LOGGER, e) 53 | Dict("raw" => d["metadata"]["manifest"]) 54 | end 55 | 56 | return JLSOFile( 57 | VersionNumber(d["metadata"]["version"]), 58 | VersionNumber(d["metadata"]["julia"]), 59 | Symbol(d["metadata"]["format"]), 60 | compression, 61 | d["metadata"]["image"], 62 | Pkg.TOML.parse(d["metadata"]["project"]), 63 | manifest, 64 | Dict{Symbol, Vector{UInt8}}(Symbol(k) => v for (k, v) in d["objects"]), 65 | ReentrantLock() 66 | ) 67 | end 68 | 69 | """ 70 | save(io, data) 71 | save(path, data) 72 | 73 | Creates a JLSOFile with the specified data and kwargs and writes it back to the io. 74 | """ 75 | save(io::IO, data; kwargs...) = write(io, JLSOFile(data; kwargs...)) 76 | save(io::IO, data::Pair...; kwargs...) = save(io, Dict(data...); kwargs...) 77 | function save(path::Union{AbstractPath, AbstractString}, args...; kwargs...) 78 | return open(io -> save(io, args...; kwargs...), path, "w") 79 | end 80 | 81 | """ 82 | load(io, objects...) -> Dict{Symbol, Any} 83 | load(path, objects...) -> Dict{Symbol, Any} 84 | 85 | Load the JLSOFile from the io and deserialize the specified objects. 86 | If no object names are specified then all objects in the file are returned. 87 | """ 88 | load(path::Union{AbstractPath, AbstractString}, args...) = open(io -> load(io, args...), path) 89 | function load(io::IO, objects::Symbol...) 90 | jlso = read(io, JLSOFile) 91 | objects = isempty(objects) ? names(jlso) : objects 92 | result = Dict{Symbol, Any}() 93 | 94 | @sync for o in objects 95 | @spawn begin 96 | # Note that calling getindex on the jlso triggers the deserialization of the object 97 | deserialized = jlso[o] 98 | lock(jlso.lock) do 99 | result[o] = deserialized 100 | end 101 | end 102 | end 103 | 104 | return result 105 | end 106 | 107 | # FileIO Interface 108 | fileio_save(f, args...; kwargs...) = save(f.filename, args...; kwargs...) 109 | fileio_load(f, args...) = load(f.filename, args...) 110 | -------------------------------------------------------------------------------- /docs/src/upgrading.md: -------------------------------------------------------------------------------- 1 | # Upgrading 2 | 3 | JLSO.jl will automatically upgrade older versions of the file format when you call `JLSO.load` or `read`. 4 | 5 | ``` 6 | julia> jlso = read("test/specimens/v1_bson.jlso", JLSOFile) 7 | [info | JLSO]: Upgrading JLSO format from v1 to v2 8 | [info | JLSO]: Upgrading JLSO format from v2 to v3 9 | Updating registry at `~/.julia/registries/General` 10 | Updating git-repo `git@github.com:JuliaRegistries/General.git` 11 | Updating registry at `~/.julia/registries/Invenia` 12 | Updating git-repo `git@gitlab.invenia.ca:invenia/PackageRegistry.git` 13 | Resolving package versions... 14 | Updating `/private/var/folders/h_/vkjv56410m7f75g9ffp46mf80000gp/T/tmphxMa6g/Project.toml` 15 | [39de3d68] + AxisArrays v0.3.0 16 | [fbb218c0] + BSON v0.2.3 17 | [b99e7846] + BinaryProvider v0.5.4 18 | [34da2185] + Compat v2.1.0 19 | [a93c6f00] + DataFrames v0.18.2 20 | [31c24e10] + Distributions v0.20.0 21 | [8f5d6c58] + EzXML v0.9.1 22 | [9da8a3cd] + JLSO v1.0.0 23 | [f28f55f0] + Memento v0.12.1 24 | [78c3b35d] + Mocking v0.5.7 25 | [f269a46b] + TimeZones v0.9.1 26 | Updating `/private/var/folders/h_/vkjv56410m7f75g9ffp46mf80000gp/T/tmphxMa6g/Manifest.toml` 27 | [7d9fca2a] + Arpack v0.3.1 28 | [39de3d68] + AxisArrays v0.3.0 29 | [fbb218c0] + BSON v0.2.3 30 | ... 31 | [ Info: activating new environment at ~/repos/JLSO.jl/Project.toml. 32 | JLSOFile([ZonedDateTime, DataFrame, Vector, DateTime, String, Matrix, Distribution]; version="3.0.0", julia="1.1.0", format=:bson, compression=:none, image="") 33 | ``` 34 | 35 | Upgrading to v3 requires generating a new manifest and project fields from the legacy `pkgs` field (as seen above) which can be slow and may require manual intervention to address package name collisions across registries. 36 | `JLSO.upgrade` can be used to mitigate these issues. 37 | 38 | To upgrade a single file: 39 | 40 | ``` 41 | julia> JLSO.upgrade("test/specimens/v1_bson.jlso", "v3_bson.jlso") 42 | [info | JLSO]: Upgrading test/specimens/v1_bson.jlso -> v3_bson.jlso 43 | [info | JLSO]: Upgrading JLSO format from v1 to v2 44 | [info | JLSO]: Upgrading JLSO format from v2 to v3 45 | Resolving package versions... 46 | Updating `/private/var/folders/h_/vkjv56410m7f75g9ffp46mf80000gp/T/tmpTxpaLh/Project.toml` 47 | [39de3d68] + AxisArrays v0.3.0 48 | [fbb218c0] + BSON v0.2.3 49 | [b99e7846] + BinaryProvider v0.5.4 50 | [34da2185] + Compat v2.1.0 51 | [a93c6f00] + DataFrames v0.18.2 52 | [31c24e10] + Distributions v0.20.0 53 | [8f5d6c58] + EzXML v0.9.1 54 | [9da8a3cd] + JLSO v1.0.0 55 | [f28f55f0] + Memento v0.12.1 56 | [78c3b35d] + Mocking v0.5.7 57 | [f269a46b] + TimeZones v0.9.1 58 | Updating `/private/var/folders/h_/vkjv56410m7f75g9ffp46mf80000gp/T/tmpTxpaLh/Manifest.toml` 59 | [7d9fca2a] + Arpack v0.3.1 60 | [39de3d68] + AxisArrays v0.3.0 61 | [fbb218c0] + BSON v0.2.3 62 | ... 63 | [ Info: activating new environment at ~/repos/JLSO.jl/Project.toml. 64 | ``` 65 | 66 | To batch upgrade files created with the same environment: 67 | 68 | ``` 69 | julia> filenames = ["v1_bson.jlso", "v1_serialize.jlso"] 70 | 2-element Array{String,1}: 71 | "v1_bson.jlso" 72 | "v1_serialize.jlso" 73 | 74 | julia> JLSO.upgrade(joinpath.("test/specimens", filenames), filenames) 75 | [info | JLSO]: Upgrading JLSO format from v1 to v2 76 | [info | JLSO]: Upgrading JLSO format from v2 to v3 77 | Resolving package versions... 78 | Updating `/private/var/folders/h_/vkjv56410m7f75g9ffp46mf80000gp/T/tmptUsSQV/Project.toml` 79 | [39de3d68] + AxisArrays v0.3.0 80 | [fbb218c0] + BSON v0.2.3 81 | [b99e7846] + BinaryProvider v0.5.4 82 | [34da2185] + Compat v2.1.0 83 | [a93c6f00] + DataFrames v0.18.2 84 | [31c24e10] + Distributions v0.20.0 85 | [8f5d6c58] + EzXML v0.9.1 86 | [9da8a3cd] + JLSO v1.0.0 87 | [f28f55f0] + Memento v0.12.1 88 | [78c3b35d] + Mocking v0.5.7 89 | [f269a46b] + TimeZones v0.9.1 90 | Updating `/private/var/folders/h_/vkjv56410m7f75g9ffp46mf80000gp/T/tmptUsSQV/Manifest.toml` 91 | [7d9fca2a] + Arpack v0.3.1 92 | [39de3d68] + AxisArrays v0.3.0 93 | [fbb218c0] + BSON v0.2.3 94 | ... 95 | [ Info: activating new environment at ~/repos/JLSO.jl/Project.toml. 96 | [info | JLSO]: Upgrading test/specimens/v1_serialize.jlso -> v1_serialize.jlso 97 | [info | JLSO]: Upgrading JLSO format from v1 to v2 98 | [info | JLSO]: Upgrading JLSO format from v2 to v3 99 | Resolving package versions... 100 | [ Info: activating new environment at ~/repos/JLSO.jl/Project.toml. 101 | 102 | ``` 103 | 104 | In the above case, the project and manifest is only generated for the first file and reused for all subsequent files. 105 | -------------------------------------------------------------------------------- /test/backwards_compat.jl: -------------------------------------------------------------------------------- 1 | @testset "Upgrading" begin 2 | @testset "JLSO.upgrade_jlso" begin 3 | @testset "no change for current version" begin 4 | d = Dict("metadata" => Dict("version" => v"3.0")) 5 | new_d = @suppress_out upgrade_jlso(d) 6 | @test new_d == Dict("metadata" => Dict("version" => v"3.0")) 7 | end 8 | 9 | @testset "upgrade 1.0" begin 10 | d = Dict("metadata" => Dict("version" => v"1.0", "format"=>:serialize)) 11 | new_d = @suppress_out upgrade_jlso(d) 12 | @test new_d["metadata"]["version"] == "3" 13 | @test new_d["metadata"]["format"] == "julia_serialize" 14 | @test new_d["metadata"]["compression"] == "none" 15 | @test isempty(new_d["metadata"]["project"]) 16 | @test isempty(new_d["metadata"]["manifest"]) 17 | @test isempty(new_d["objects"]) 18 | 19 | @testset "Don't rename bson format" begin 20 | d = Dict("metadata" => Dict("version" => v"1.0", "format"=>:bson)) 21 | new_d = @suppress_out upgrade_jlso(d) 22 | @test new_d["metadata"]["version"] == "3" 23 | @test new_d["metadata"]["format"] == "bson" 24 | @test new_d["metadata"]["compression"] == "none" 25 | @test isempty(new_d["metadata"]["project"]) 26 | @test isempty(new_d["metadata"]["manifest"]) 27 | @test isempty(new_d["objects"]) 28 | end 29 | 30 | @testset "Invalid VersionNumber" begin 31 | d = Dict( 32 | "metadata" => Dict( 33 | "version" => v"1.0", "format" => :bson, 34 | # We don't intend to go back an make a 0.3.14 release at any point. 35 | "pkgs" => merge(pkgs, Dict("JLSO" => v"0.3.14")), 36 | ) 37 | ) 38 | 39 | new_d = @suppress_out upgrade_jlso(d) 40 | @test !isempty(new_d["metadata"]["project"]) 41 | @test !isempty(new_d["metadata"]["manifest"]) 42 | 43 | # Test that the project.toml has the JLSO dep, but not a specific version 44 | # (fallback when package versions don't exist in the registry) 45 | _project = Pkg.TOML.parse(new_d["metadata"]["project"]) 46 | @test haskey(_project, "deps") 47 | @test haskey(_project["deps"], "JLSO") 48 | @test haskey(_project, "compat") 49 | # Check that the compat section doesn't have our package version 50 | @test !haskey(_project["compat"], "JLSO") 51 | end 52 | end 53 | end 54 | 55 | # The below is how we saves the specimens for compat testing 56 | # JLSO.save("specimens/v1_serialize.jlso", datas; format=:serialize) 57 | # JLSO.save("specimens/v1_bson.jlso", datas; format=:bson) 58 | 59 | @testset "Can still load old files" begin 60 | dir = joinpath(@__DIR__, "specimens") 61 | @testset "$fn" for fn in readdir(dir) 62 | jlso_data = @suppress_out JLSO.load(joinpath(dir, fn)) 63 | @test jlso_data == datas 64 | end 65 | end 66 | 67 | # Test upgrading files to avoid repeated upgrading 68 | @testset "JLSO.upgrade" begin 69 | src_dir = joinpath(@__DIR__, "specimens") 70 | @testset "upgrade" begin 71 | mktempdir() do dest_dir 72 | @testset "$fn" for fn in readdir(src_dir) 73 | src = joinpath(src_dir, fn) 74 | dest = joinpath(dest_dir, fn) 75 | @suppress_out JLSO.upgrade(src, dest) 76 | jlso_data = JLSO.load(dest) 77 | @test jlso_data == datas 78 | end 79 | end 80 | end 81 | @testset "batch upgrade" begin 82 | mktempdir() do dest_dir 83 | fns = readdir(src_dir) 84 | src = joinpath.(src_dir, fns) 85 | dest = joinpath.(dest_dir, fns) 86 | 87 | @suppress_out JLSO.upgrade(src, dest) 88 | jlso_data = JLSO.load(first(dest)) 89 | @test jlso_data == datas 90 | end 91 | end 92 | @testset "upgrade w/ manifest" begin 93 | src = joinpath(src_dir, "v3_bson_none.jlso") 94 | orig_jlso = read(src, JLSOFile) 95 | 96 | # Extract the project & manifest to modify 97 | new_project = deepcopy(orig_jlso.project) 98 | new_manifest = deepcopy(orig_jlso.manifest) 99 | 100 | # Check the current state is what we expect 101 | @test new_project["compat"]["Memento"] == "0.10, 0.11, 0.12" 102 | @test new_manifest["Memento"][1]["version"] == "0.12.1" 103 | 104 | 105 | # Update those values 106 | new_project["compat"]["Memento"] = "0.11, 0.12" 107 | new_manifest["Memento"][1]["version"] = "0.12.0" 108 | 109 | mktempdir() do dest_dir 110 | dest = joinpath(dest_dir, "v3_bson_none.jlso") 111 | @suppress_out JLSO.upgrade(src, dest, new_project, new_manifest) 112 | new_jlso = read(dest, JLSOFile) 113 | 114 | @test new_project["compat"]["Memento"] == "0.11, 0.12" 115 | @test new_manifest["Memento"][1]["version"] == "0.12.0" 116 | end 117 | end 118 | end 119 | end 120 | -------------------------------------------------------------------------------- /src/upgrade.jl: -------------------------------------------------------------------------------- 1 | """ 2 | upgrade(src, dest) 3 | upgrade(src, dest, project, manifest) 4 | upgrade(src::Vector, dest::Vector) 5 | 6 | Handle upgrading existing JLSO files. This can be useful to: 7 | 8 | 1. Avoid repeatedly reading legacy jlso files (requiring structure updates at runtime) 9 | 2. Allow overriding project and manifest data rather than building it as part of the 10 | upgrade step. Won't be needed once the v2 file format is dropped. 11 | 3. Allows for efficient batch updating of files created with the same environment. 12 | 13 | !!! warning 14 | The manifest generated by this process are best guesses. 15 | JLSO v2 files do not include information on indirect dependencies, so the upgrade process relies on the package manager to work those out. 16 | The project generated is accurate, except for duplicated names (no UUID) and custom branchs. 17 | """ 18 | function upgrade(src, dest) 19 | info(LOGGER, "Upgrading $src -> $dest") 20 | jlso = open(io -> read(io, JLSOFile), src) 21 | write(dest, jlso) 22 | end 23 | 24 | function upgrade(src, dest, project, manifest) 25 | info(LOGGER, "Upgrading $src -> $dest") 26 | parsed = BSON.load(string(src)) 27 | haskey(parsed["metadata"], "pkgs") && empty!(parsed["metadata"]["pkgs"]) 28 | d = upgrade_jlso(parsed) 29 | jlso = JLSOFile( 30 | VersionNumber(d["metadata"]["version"]), 31 | VersionNumber(d["metadata"]["julia"]), 32 | Symbol(d["metadata"]["format"]), 33 | Symbol(d["metadata"]["compression"]), 34 | d["metadata"]["image"], 35 | project, 36 | manifest, 37 | Dict{Symbol, Vector{UInt8}}(Symbol(k) => v for (k, v) in d["objects"]), 38 | ReentrantLock() 39 | ) 40 | write(dest, jlso) 41 | end 42 | 43 | function upgrade(sources::Vector, destinations::Vector) 44 | iter = zip(sources, destinations) 45 | 46 | if !isempty(iter) 47 | (src, dest), s = iterate(iter) 48 | mkpath(dirname(dest)) 49 | jlso = open(io -> read(io, JLSOFile), src) 50 | write(dest, jlso) 51 | 52 | for (src, dest) in Iterators.rest(iter, s) 53 | mkpath(dirname(dest)) 54 | upgrade(src, dest, jlso.project, jlso.manifest) 55 | end 56 | end 57 | end 58 | 59 | function upgrade_jlso(raw_dict::AbstractDict) 60 | result = copy(raw_dict) 61 | version = version_number(result["metadata"]["version"]) 62 | 63 | while version < v"3" 64 | result = upgrade_jlso(result, Val(Int(version.major))) 65 | version = version_number(result["metadata"]["version"]) 66 | end 67 | 68 | return result 69 | end 70 | 71 | # v1 and v2 stored version numbers but v3 stores strings to be bson parser agnostic 72 | version_number(x::VersionNumber) = x 73 | version_number(x::String) = VersionNumber(x) 74 | 75 | # Metadata changes to upgrade file from v1 to v2 76 | function upgrade_jlso(raw_dict::AbstractDict, ::Val{1}) 77 | info(LOGGER, "Upgrading JLSO format from v1 to v2") 78 | metadata = copy(raw_dict["metadata"]) 79 | version = version_number(metadata["version"]) 80 | @assert version ∈ semver_spec("1") 81 | 82 | if metadata["format"] == :serialize 83 | metadata["format"] = :julia_serialize 84 | end 85 | 86 | metadata["compression"] = :none 87 | metadata["version"] = v"2" 88 | 89 | return Dict("metadata" => metadata, "objects" => get(raw_dict, "objects", Dict())) 90 | end 91 | 92 | # Metadata changes to upgrade from v2 to v3 93 | function upgrade_jlso(raw_dict::AbstractDict, ::Val{2}) 94 | info(LOGGER, "Upgrading JLSO format from v2 to v3") 95 | metadata = copy(raw_dict["metadata"]) 96 | version = version_number(metadata["version"]) 97 | @assert version ∈ semver_spec("2") 98 | 99 | # JLSO v3 expects to be loading a compressed manifest 100 | compression = get(metadata, "compression", :none) 101 | project, manifest = _upgrade_env(get(metadata, "pkgs", nothing)) 102 | 103 | return Dict{String, Dict}( 104 | "metadata" => Dict{String, Union{String, Vector{UInt8}}}( 105 | "version" => "3", 106 | "julia" => string(get(metadata, "julia", VERSION)), 107 | "format" => string(get(metadata, "format", :julia_serialize)), 108 | "compression" => string(compression), 109 | "image" => get(metadata, "image", ""), 110 | "project" => project, 111 | "manifest" => read(compress(compression, IOBuffer(manifest))), 112 | ), 113 | "objects" => Dict{String, Vector{UInt8}}( 114 | k => v for (k, v) in get(raw_dict, "objects", Dict()) 115 | ), 116 | ) 117 | end 118 | 119 | # Used to convert the old "PkgName" => VersionNumber metadata to a 120 | # Project.toml and Manifest.toml file. 121 | function _upgrade_env(pkgs::Dict{String, VersionNumber}) 122 | src_env = Base.active_project() 123 | 124 | try 125 | mktempdir() do tmp 126 | Pkg.activate(tmp) 127 | 128 | # We construct an array of PackageSpecs to avoid ordering problems with 129 | # adding each package individually 130 | try 131 | Pkg.add([ 132 | Pkg.PackageSpec(; name=key, version=value) for (key, value) in pkgs 133 | ]) 134 | catch e 135 | # Warn about failure and fallback to simply trying to install the pacakges 136 | # without version constraints. 137 | warn(LOGGER) do 138 | "Failed to construct an environment with the provide package version " * 139 | "($pkgs): $e.\n Falling back to simply adding the packages." 140 | end 141 | Pkg.add([Pkg.PackageSpec(; name=key) for (key, value) in pkgs]) 142 | end 143 | 144 | return _env() 145 | end 146 | finally 147 | Pkg.activate(src_env) 148 | end 149 | end 150 | 151 | _upgrade_env(::Nothing) = ("", "") 152 | -------------------------------------------------------------------------------- /docs/Manifest.toml: -------------------------------------------------------------------------------- 1 | [[BSON]] 2 | deps = ["Profile", "Test"] 3 | git-tree-sha1 = "6453cef4f9cb8ded8e28e4d6d12e11e20eb692ea" 4 | uuid = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0" 5 | version = "0.2.3" 6 | 7 | [[Base64]] 8 | uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" 9 | 10 | [[BinaryProvider]] 11 | deps = ["Libdl", "SHA"] 12 | git-tree-sha1 = "c7361ce8a2129f20b0e05a89f7070820cfed6648" 13 | uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232" 14 | version = "0.5.4" 15 | 16 | [[CodecZlib]] 17 | deps = ["BinaryProvider", "Libdl", "TranscodingStreams"] 18 | git-tree-sha1 = "05916673a2627dd91b4969ff8ba6941bc85a960e" 19 | uuid = "944b1d66-785c-5afd-91f1-9de20f533193" 20 | version = "0.6.0" 21 | 22 | [[Compat]] 23 | deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] 24 | git-tree-sha1 = "84aa74986c5b9b898b0d1acaf3258741ee64754f" 25 | uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" 26 | version = "2.1.0" 27 | 28 | [[Dates]] 29 | deps = ["Printf"] 30 | uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" 31 | 32 | [[DelimitedFiles]] 33 | deps = ["Mmap"] 34 | uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" 35 | 36 | [[Distributed]] 37 | deps = ["LinearAlgebra", "Random", "Serialization", "Sockets"] 38 | uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" 39 | 40 | [[DocStringExtensions]] 41 | deps = ["LibGit2", "Markdown", "Pkg", "Test"] 42 | git-tree-sha1 = "4d30e889c9f106a51ffa4791a88ffd4765bf20c3" 43 | uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" 44 | version = "0.7.0" 45 | 46 | [[Documenter]] 47 | deps = ["Base64", "DocStringExtensions", "InteractiveUtils", "LibGit2", "Logging", "Markdown", "Pkg", "REPL", "Random", "Test", "Unicode"] 48 | git-tree-sha1 = "a8c41ba3d0861240dbec942ee1d0f86c57c37c1c" 49 | uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" 50 | version = "0.21.5" 51 | 52 | [[EzXML]] 53 | deps = ["BinaryProvider", "Libdl", "Pkg", "Printf", "Test"] 54 | git-tree-sha1 = "ad00b79cca4bb3eabb4209217859c553af4401f5" 55 | uuid = "8f5d6c58-4d21-5cfd-889c-e3ad7ee6a615" 56 | version = "0.9.1" 57 | 58 | [[InteractiveUtils]] 59 | deps = ["LinearAlgebra", "Markdown"] 60 | uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" 61 | 62 | [[JLSO]] 63 | deps = ["BSON", "CodecZlib", "Memento", "Pkg", "Serialization"] 64 | path = ".." 65 | uuid = "9da8a3cd-07a3-59c0-a743-3fdc52c30d11" 66 | version = "1.1.0" 67 | 68 | [[JSON]] 69 | deps = ["Dates", "Distributed", "Mmap", "Sockets", "Test", "Unicode"] 70 | git-tree-sha1 = "1f7a25b53ec67f5e9422f1f551ee216503f4a0fa" 71 | uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" 72 | version = "0.20.0" 73 | 74 | [[LibGit2]] 75 | uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" 76 | 77 | [[Libdl]] 78 | uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" 79 | 80 | [[LinearAlgebra]] 81 | deps = ["Libdl"] 82 | uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" 83 | 84 | [[Logging]] 85 | uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" 86 | 87 | [[Markdown]] 88 | deps = ["Base64"] 89 | uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" 90 | 91 | [[Memento]] 92 | deps = ["Dates", "Distributed", "JSON", "Serialization", "Sockets", "Syslogs", "Test", "TimeZones", "UUIDs"] 93 | git-tree-sha1 = "090463b13da88689e5eae6468a6f531a21392175" 94 | uuid = "f28f55f0-a522-5efc-85c2-fe41dfb9b2d9" 95 | version = "0.12.1" 96 | 97 | [[Mmap]] 98 | uuid = "a63ad114-7e13-5084-954f-fe012c677804" 99 | 100 | [[Mocking]] 101 | deps = ["Compat", "Dates"] 102 | git-tree-sha1 = "4bf69aaf823b119b034e091e16b18311aa191663" 103 | uuid = "78c3b35d-d492-501b-9361-3d52fe80e533" 104 | version = "0.5.7" 105 | 106 | [[Nullables]] 107 | deps = ["Compat"] 108 | git-tree-sha1 = "ae1a63457e14554df2159b0b028f48536125092d" 109 | uuid = "4d1e1d77-625e-5b40-9113-a560ec7a8ecd" 110 | version = "0.0.8" 111 | 112 | [[Pkg]] 113 | deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] 114 | uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" 115 | 116 | [[Printf]] 117 | deps = ["Unicode"] 118 | uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" 119 | 120 | [[Profile]] 121 | deps = ["Printf"] 122 | uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" 123 | 124 | [[REPL]] 125 | deps = ["InteractiveUtils", "Markdown", "Sockets"] 126 | uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" 127 | 128 | [[Random]] 129 | deps = ["Serialization"] 130 | uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" 131 | 132 | [[SHA]] 133 | uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" 134 | 135 | [[Serialization]] 136 | uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" 137 | 138 | [[SharedArrays]] 139 | deps = ["Distributed", "Mmap", "Random", "Serialization"] 140 | uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" 141 | 142 | [[Sockets]] 143 | uuid = "6462fe0b-24de-5631-8697-dd941f90decc" 144 | 145 | [[SparseArrays]] 146 | deps = ["LinearAlgebra", "Random"] 147 | uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" 148 | 149 | [[Statistics]] 150 | deps = ["LinearAlgebra", "SparseArrays"] 151 | uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" 152 | 153 | [[Syslogs]] 154 | deps = ["Compat", "Nullables"] 155 | git-tree-sha1 = "d3e512a044cc8873c741d88758f8e1888c7c47d3" 156 | uuid = "cea106d9-e007-5e6c-ad93-58fe2094e9c4" 157 | version = "0.2.0" 158 | 159 | [[Test]] 160 | deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] 161 | uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" 162 | 163 | [[TimeZones]] 164 | deps = ["Dates", "EzXML", "Mocking", "Printf", "Serialization", "Test", "Unicode"] 165 | git-tree-sha1 = "fdf5d2136d16498cb67d648cedd33b83c599e0c5" 166 | uuid = "f269a46b-ccf7-5d73-abea-4c690281aa53" 167 | version = "0.9.0" 168 | 169 | [[TranscodingStreams]] 170 | deps = ["Random", "Test"] 171 | git-tree-sha1 = "7c53c35547de1c5b9d46a4797cf6d8253807108c" 172 | uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" 173 | version = "0.9.5" 174 | 175 | [[UUIDs]] 176 | deps = ["Random"] 177 | uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" 178 | 179 | [[Unicode]] 180 | uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" 181 | -------------------------------------------------------------------------------- /test/file_io.jl: -------------------------------------------------------------------------------- 1 | @testset "reading and writing" begin 2 | @testset "$fmt - $k" for fmt in (:bson, :julia_serialize), (k, v) in datas 3 | io = IOBuffer() 4 | orig = JLSOFile(:data => v; format=fmt) 5 | write(io, orig) 6 | 7 | seekstart(io) 8 | 9 | result = read(io, JLSOFile) 10 | @test result.version == orig.version 11 | @test result.julia == orig.julia 12 | @test result.image == orig.image 13 | @test result.manifest == orig.manifest 14 | @test result.format == orig.format 15 | @test result.compression == orig.compression 16 | @test result.objects == orig.objects 17 | @test result == orig 18 | end 19 | end 20 | 21 | @testset "deserialization" begin 22 | # Test deserialization works 23 | @testset "$fmt - $k" for fmt in (:bson, :julia_serialize), (k, v) in datas 24 | jlso = JLSOFile(:data => v; format=fmt) 25 | @test jlso[:data] == v 26 | end 27 | 28 | @testset "unsupported julia version $jlso_version" for jlso_version in (v"1.0", v"2.0") 29 | jlso = @suppress_out begin 30 | JLSOFile( 31 | jlso_version, 32 | VERSION, 33 | :julia_serialize, 34 | :none, 35 | img, 36 | project, 37 | manifest, 38 | Dict(:data => hw_5), 39 | ReentrantLock() 40 | ) 41 | end 42 | 43 | # Test failing to deserialize data because of incompatible julia versions 44 | # will return the raw bytes 45 | result = if VERSION < v"1.2" 46 | @test_warn(LOGGER, r"MethodError*", jlso[:data]) 47 | else 48 | @test_warn(LOGGER, r"TypeError*", jlso[:data]) 49 | end 50 | 51 | @test result == hw_5 52 | 53 | # TODO: Test that BSON works across julia versions using external files? 54 | end 55 | 56 | @testset "missing module" begin 57 | # We need to load and use AxisArrays on another process to cause the 58 | # deserialization error 59 | pnum = first(addprocs(1)) 60 | 61 | try 62 | # We need to do this separately because there appears to be a race 63 | # condition on AxisArrays being loaded. 64 | f = @spawnat pnum begin 65 | @eval Main using Serialization 66 | @eval Main using BSON 67 | @eval Main using AxisArrays 68 | end 69 | 70 | fetch(f) 71 | 72 | @testset "serialize" begin 73 | f = @spawnat pnum begin 74 | io = IOBuffer() 75 | serialize( 76 | io, 77 | AxisArray( 78 | rand(20, 10), 79 | Axis{:time}(14010:10:14200), 80 | Axis{:id}(1:10) 81 | ) 82 | ) 83 | return io 84 | end 85 | 86 | io = fetch(f) 87 | bytes = take!(io) 88 | 89 | jlso = JLSOFile( 90 | v"3.0", 91 | VERSION, 92 | :julia_serialize, 93 | :none, 94 | img, 95 | project, 96 | manifest, 97 | Dict(:data => bytes), 98 | ReentrantLock() 99 | ) 100 | 101 | # Test failing to deserailize data because of missing modules will 102 | # still return the raw bytes 103 | result = @test_warn(LOGGER, r"KeyError*", jlso[:data]) 104 | @test result == bytes 105 | end 106 | 107 | @testset "bson" begin 108 | f = @spawnat pnum begin 109 | io = IOBuffer() 110 | bson( 111 | io, 112 | Dict( 113 | :data => AxisArray( 114 | rand(20, 10), 115 | Axis{:time}(14010:10:14200), 116 | Axis{:id}(1:10) 117 | ) 118 | ) 119 | ) 120 | return io 121 | end 122 | 123 | io = fetch(f) 124 | bytes = take!(io) 125 | 126 | jlso = JLSOFile( 127 | v"3.0", 128 | VERSION, 129 | :bson, 130 | :none, 131 | img, 132 | project, 133 | manifest, 134 | Dict(:data => bytes), 135 | ReentrantLock() 136 | ) 137 | 138 | # Test failing to deserailize data because of missing modules will 139 | # still return the raw bytes 140 | result = @test_warn(LOGGER, r"UndefVarError*", jlso[:data]) 141 | 142 | @test result == bytes 143 | end 144 | finally 145 | rmprocs(pnum) 146 | end 147 | end 148 | end 149 | 150 | @testset "saving and loading" begin 151 | function test_save_and_load(dirpath) 152 | @testset "$fmt - $k" for fmt in (:bson, :julia_serialize), (k, v) in datas 153 | filepath = joinpath(dirpath, "$fmt-$k.jlso") 154 | JLSO.save(filepath, k => v; format=fmt) 155 | result = JLSO.load(filepath) 156 | single_item = JLSO.load(filepath, k) 157 | @test result[k] == single_item[k] == v 158 | end 159 | end 160 | @testset "String-type paths" begin 161 | mktempdir(test_save_and_load) 162 | end 163 | @testset "Path-type paths" begin 164 | mktempdir(test_save_and_load, SystemPath) 165 | end 166 | @testset "keys are not Symbols" begin 167 | @test_throws( 168 | getlogger(JLSO), 169 | MethodError, 170 | JLSO.save("breakfast.jlso", "food" => "☕️🥓🍳", "time" => Time(9, 0)), 171 | ) 172 | end 173 | @testset "README example" begin 174 | mktempdir() do path 175 | JLSO.save( 176 | joinpath(path, "breakfast.jlso"), 177 | :food => "☕️🥓🍳", 178 | :cost => 11.95, 179 | :time => Time(9, 0), 180 | ) 181 | loaded = JLSO.load("$path/breakfast.jlso") 182 | @test loaded == Dict{Symbol,Any}( 183 | :cost => 11.95, 184 | :time => Time(9, 0), 185 | :food => "☕️🥓🍳", 186 | ) 187 | end 188 | end 189 | end 190 | --------------------------------------------------------------------------------