├── CMakeLists.txt ├── LICENSE ├── README.md ├── example ├── test.json └── test.py └── src └── installer.cpp /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, QuantStack and Mamba Contributors 2 | # 3 | # Distributed under the terms of the BSD 3-Clause License. 4 | # 5 | # The full license is in the file LICENSE, distributed with this software. 6 | 7 | cmake_minimum_required (VERSION 2.8.11) 8 | 9 | project(monstructor) 10 | 11 | set(MONSTRUCTOR_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include) 12 | set(MONSTRUCTOR_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src) 13 | 14 | find_package(Threads REQUIRED) 15 | find_package(nlohmann_json REQUIRED) 16 | 17 | set(MONSTRUCTOR_SOURCES 18 | ${MONSTRUCTOR_SOURCE_DIR}/installer.cpp 19 | ) 20 | 21 | if (WIN32) 22 | include_directories($ENV{CONDA_PREFIX}/Library/include) 23 | endif() 24 | 25 | add_executable(monstructor ${MONSTRUCTOR_SOURCES}) 26 | set_property(TARGET monstructor PROPERTY CXX_STANDARD 17) 27 | if (WIN32) 28 | find_library(MAMBA_LIBRARY NAMES libmamba_static) 29 | target_link_libraries(monstructor ${MAMBA_LIBRARY}) 30 | else() 31 | target_link_libraries(monstructor mamba) 32 | endif() 33 | target_link_libraries(monstructor nlohmann_json) 34 | 35 | find_library(LIBSOLV_LIBRARIES NAMES solv) 36 | find_library(LIBSOLVEXT_LIBRARIES NAMES solvext) 37 | find_package(CURL REQUIRED) 38 | find_package(LibArchive REQUIRED) 39 | find_package(OpenSSL REQUIRED) 40 | find_package(yaml-cpp CONFIG REQUIRED) 41 | 42 | target_link_libraries(monstructor 43 | ${LIBSOLV_LIBRARIES} 44 | ${LIBSOLVEXT_LIBRARIES} 45 | ${LibArchive_LIBRARIES} 46 | ${CURL_LIBRARIES} 47 | ${OPENSSL_LIBRARIES} 48 | ${YAML_CPP_LIBRARIES} 49 | Threads::Threads 50 | ) 51 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2019 QuantStack and the Monstructor contributors. 2 | 3 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 4 | 5 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 6 | 7 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | 9 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 10 | 11 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ### monstructor, a stand-alone installer generator based on mamba 2 | 3 | **Note: very early stage software** 4 | 5 | Monstructor generates standalone installers (similar to `constructor` for conda) that uses `conda` packages and the C++ of `mamba` to install the packages. 6 | 7 | The `monstructor` project is currently a Work-In-Progress. The current idea is to have a installer layout (for Linux) that looks like the following: 8 | 9 | - monstructor binary 10 | - JSON description of payload (list of repodata snippets including channel, URL, SHA256, file size) 11 | - concatenated binary tar.bz2 or .conda files 12 | 13 | ### Usage 14 | 15 | To use the monstructor, you have to have a conda environment. Currently, mamba needs to be installed from source (but with mamba 0.5.4 it will be enough to have `libmamba-static` installed!). 16 | 17 | First, build monstructor: 18 | 19 | - `mkdir build && cd build` 20 | - `cmake .. -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX` 21 | - `make -j4` 22 | 23 | With the monstructor binary in place, you can generate new monstructors. For this, run the python script included in `example`. First, select (or create) an environment, and export it: 24 | 25 | - `mamba create -n xtensor xtensor -c conda-forge` 26 | - `conda activate xtensor` 27 | - `mamba list --json > exported_pkgs.json` 28 | - `python examples/test.py build/monstructor ./exported_pkgs.json` 29 | 30 | This will place a new, bundled executable into the `build` folder (`monstructor.done`). You can execute this file: 31 | 32 | - `./build/monstructor.done -p /some/target/prefix` 33 | 34 | To recreate the environment you previously exported. -------------------------------------------------------------------------------- /example/test.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "base_url": "https://conda.anaconda.org/conda-forge", 4 | "build_number": 0, 5 | "build_string": "conda_forge", 6 | "channel": "conda-forge", 7 | "dist_name": "_libgcc_mutex-0.1-conda_forge", 8 | "name": "_libgcc_mutex", 9 | "platform": "linux-64", 10 | "version": "0.1" 11 | }, 12 | { 13 | "base_url": "https://conda.anaconda.org/conda-forge", 14 | "build_number": 14, 15 | "build_string": "1_gnu", 16 | "channel": "conda-forge", 17 | "dist_name": "_openmp_mutex-4.5-1_gnu", 18 | "name": "_openmp_mutex", 19 | "platform": "linux-64", 20 | "version": "4.5" 21 | }, 22 | { 23 | "base_url": "https://conda.anaconda.org/conda-forge", 24 | "build_number": 16, 25 | "build_string": "h24d8f2e_16", 26 | "channel": "conda-forge", 27 | "dist_name": "libgcc-ng-9.3.0-h24d8f2e_16", 28 | "name": "libgcc-ng", 29 | "platform": "linux-64", 30 | "version": "9.3.0" 31 | }, 32 | { 33 | "base_url": "https://conda.anaconda.org/conda-forge", 34 | "build_number": 16, 35 | "build_string": "h24d8f2e_16", 36 | "channel": "conda-forge", 37 | "dist_name": "libgomp-9.3.0-h24d8f2e_16", 38 | "name": "libgomp", 39 | "platform": "linux-64", 40 | "version": "9.3.0" 41 | }, 42 | { 43 | "base_url": "https://conda.anaconda.org/conda-forge", 44 | "build_number": 16, 45 | "build_string": "hdf63c60_16", 46 | "channel": "conda-forge", 47 | "dist_name": "libstdcxx-ng-9.3.0-hdf63c60_16", 48 | "name": "libstdcxx-ng", 49 | "platform": "linux-64", 50 | "version": "9.3.0" 51 | }, 52 | { 53 | "base_url": "https://conda.anaconda.org/conda-forge", 54 | "build_number": 0, 55 | "build_string": "hc9558a2_0", 56 | "channel": "conda-forge", 57 | "dist_name": "xtl-0.6.18-hc9558a2_0", 58 | "name": "xtl", 59 | "platform": "linux-64", 60 | "version": "0.6.18" 61 | }, 62 | { 63 | "base_url": "https://conda.anaconda.org/conda-forge", 64 | "build_number": 0, 65 | "build_string": "hc9558a2_0", 66 | "channel": "conda-forge", 67 | "dist_name": "xsimd-7.4.8-hc9558a2_0", 68 | "name": "xsimd", 69 | "platform": "linux-64", 70 | "version": "7.4.8" 71 | }, 72 | { 73 | "base_url": "https://conda.anaconda.org/conda-forge", 74 | "build_number": 0, 75 | "build_string": "hc9558a2_0", 76 | "channel": "conda-forge", 77 | "dist_name": "xtensor-0.21.5-hc9558a2_0", 78 | "name": "xtensor", 79 | "platform": "linux-64", 80 | "version": "0.21.5" 81 | } 82 | ] -------------------------------------------------------------------------------- /example/test.py: -------------------------------------------------------------------------------- 1 | # from conda.core.portability import binary_replace 2 | import sys, re, json, os 3 | import stat 4 | 5 | # from conda 6 | def binary_replace(data, a, b): 7 | """ 8 | Perform a binary replacement of `data`, where the placeholder `a` is 9 | replaced with `b` and the remaining string is padded with null characters. 10 | All input arguments are expected to be bytes objects. 11 | """ 12 | def replace(match): 13 | occurances = match.group().count(a) 14 | padding = (len(a) - len(b)) * occurances 15 | if padding < 0: 16 | raise _PaddingError 17 | return match.group().replace(a, b) + b'\0' * padding 18 | 19 | original_data_len = len(data) 20 | pat = re.compile(re.escape(a) + b'([^\0]*?)\0') 21 | data = pat.sub(replace, data) 22 | assert len(data) == original_data_len 23 | 24 | return data 25 | 26 | with open(sys.argv[1], 'rb+') as fi: 27 | data = fi.read() 28 | 29 | with open(sys.argv[2]) as json_fi: 30 | json_files = json.loads(json_fi.read()) 31 | 32 | root_prefix = os.environ["CONDA_PREFIX"] if int(os.environ["CONDA_SHLVL"]) == 1 else os.environ["CONDA_PREFIX_1"] 33 | pkgs_cache = os.path.join(root_prefix, 'pkgs') 34 | 35 | for f in json_files: 36 | with open(os.path.join(pkgs_cache, f['dist_name'], 'info/repodata_record.json')) as repodata_json: 37 | repodata = json.load(repodata_json) 38 | f.update(repodata) 39 | 40 | json_bytes = json.dumps(json_files).encode('utf-8') 41 | self_size = len(data) 42 | json_size = len(json_bytes) 43 | payload_size = 0 44 | 45 | outfile = sys.argv[1] + '.done' 46 | with open(outfile, 'wb') as fo: 47 | 48 | for repodata in json_files: 49 | full_file = os.path.join(pkgs_cache, repodata['fn']) 50 | payload_size += os.path.getsize(full_file) 51 | 52 | print(f"Sizes: {self_size}, {json_size}, {payload_size}") 53 | sizes_str = f"{self_size};{json_size};{payload_size}" 54 | data = binary_replace(data, b'thisisaverylongstringthatshouldbereplacedbythebuildprocessabcdef', sizes_str.encode('utf-8')) 55 | 56 | fo.write(data) 57 | fo.write(json_bytes) 58 | 59 | for repodata in json_files: 60 | print(repodata['dist_name']) 61 | full_file = os.path.join(pkgs_cache, repodata['fn']) 62 | with open(full_file, 'rb') as pkg_in: 63 | fo.write(pkg_in.read()) 64 | 65 | st = os.stat(outfile) 66 | os.chmod(outfile, st.st_mode | stat.S_IEXEC) 67 | -------------------------------------------------------------------------------- /src/installer.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | static std::string magic_numbers = "thisisaverylongstringthatshouldbereplacedbythebuildprocessabcdef"; 7 | 8 | void write_repodata_json(const fs::path& pkg_file, 9 | const fs::path& pkg, 10 | const std::string& url, 11 | const std::string& channel, 12 | const std::string& filename) 13 | { 14 | fs::path repodata_record_path = pkg / "info" / "repodata_record.json"; 15 | fs::path index_path = pkg / "info" / "index.json"; 16 | 17 | nlohmann::json index, solvable_json; 18 | std::ifstream index_file(index_path, std::ios::in | std::ios::binary); 19 | index_file >> index; 20 | 21 | // what's missing?! 22 | // SHA256SUM 23 | // MD5 SUM 24 | // ... 25 | 26 | index["sha256"] = validate::sha256sum(pkg_file); 27 | index["md5"] = validate::md5sum(pkg_file); 28 | index["size"] = fs::file_size(pkg_file); 29 | 30 | // solvable_json = solvable_to_json(m_solv); 31 | // merge those two 32 | // index.insert(solvable_json.cbegin(), solvable_json.cend()); 33 | 34 | index["url"] = url; 35 | index["channel"] = channel; 36 | index["fn"] = filename; 37 | 38 | std::ofstream repodata_record(repodata_record_path, std::ios::out | std::ios::binary); 39 | repodata_record << index.dump(4); 40 | } 41 | 42 | void link_to_prefix(const fs::path& prefix, const std::vector& pkgs, const nlohmann::json& repodata) 43 | { 44 | // new transaction context without Python version! 45 | mamba::TransactionContext tc(prefix, ""); 46 | 47 | // we need to find the root prefix, or use the new prefix as root prefix 48 | // to place & extract the packages at `$ROOT_PREFIX/pkgs`. 49 | fs::path root_prefix = mamba::Context::instance().root_prefix; 50 | if (root_prefix.empty()) 51 | { 52 | root_prefix = prefix; 53 | } 54 | if (!fs::exists(root_prefix / "pkgs")) 55 | { 56 | fs::create_directories(root_prefix / "pkgs"); 57 | } 58 | 59 | std::vector extracted_pkgs; 60 | for (auto& pkg : pkgs) 61 | { 62 | std::cout << "Handling " << pkg << std::endl; 63 | fs::path pkg_name = fs::path(pkg).filename(); 64 | fs::path pkg_extr; 65 | 66 | // TODO check sha256!!! 67 | if (!fs::exists(root_prefix / "pkgs" / pkg_name)) 68 | { 69 | fs::copy(pkg, root_prefix / "pkgs"); 70 | } 71 | 72 | if (!fs::exists(root_prefix / mamba::strip_package_extension(pkg_name))) 73 | { 74 | pkg_extr = mamba::extract(root_prefix / "pkgs" / pkg_name); 75 | } 76 | 77 | extracted_pkgs.push_back(pkg_extr); 78 | std::cout << "Writing repodata_json " << (root_prefix / "pkgs" / pkg_name).string() << std::endl; 79 | write_repodata_json( 80 | root_prefix / "pkgs" / pkg_name, 81 | pkg_extr, 82 | repodata["url"], 83 | repodata["channel"], 84 | repodata["fn"]); 85 | } 86 | 87 | for (auto& pkg : extracted_pkgs) 88 | { 89 | std::ifstream repodata_json(pkg / "info" / "repodata_record.json", std::ios::in | std::ios::binary); 90 | nlohmann::json pkg_info_json; 91 | repodata_json >> pkg_info_json; 92 | mamba::PackageInfo pkg_info(std::move(pkg_info_json)); 93 | mamba::LinkPackage lnk(pkg_info, root_prefix / "pkgs", &tc); 94 | lnk.execute(); 95 | } 96 | } 97 | 98 | int main(int argc, char** argv) 99 | { 100 | CLI::App app{"Monstructor"}; 101 | 102 | fs::path prefix_path = "~/monstructor"; 103 | app.add_option("-p,--prefix", prefix_path, "Path to prefix")->required(); 104 | 105 | CLI11_PARSE(app, argc, argv); 106 | 107 | std::vector sizes = mamba::split(magic_numbers, ";"); 108 | int self_size = std::stoi(sizes[0].c_str()); 109 | int json_size = std::stoi(sizes[1].c_str()); 110 | int payload_size = std::stoi(sizes[2].c_str()); 111 | 112 | std::cout << "Self exe size: " << self_size << " json size: " << json_size << " payload: " << payload_size; 113 | 114 | mamba::Context::instance().verbosity = 3; 115 | 116 | fs::path target_prefix = mamba::env::expand_user(prefix_path); 117 | 118 | auto bin = mamba::get_self_exe_path(); 119 | 120 | // first extract json 121 | std::ifstream self_exe(bin, std::ios::in | std::ios::binary); 122 | self_exe.seekg(self_size); 123 | std::string json_buf(json_size, ' '); 124 | self_exe.read(json_buf.data(), json_size); 125 | 126 | auto payload_meta = nlohmann::json::parse(json_buf); 127 | 128 | for (auto& pkg_meta : payload_meta) 129 | { 130 | const std::size_t BUF_SIZE = 8096; 131 | 132 | std::cout << pkg_meta << "\n\n" << std::endl; 133 | 134 | if (!fs::exists(target_prefix / "pkgs")) 135 | { 136 | fs::create_directories(target_prefix / "pkgs"); 137 | } 138 | fs::path pkg_file = target_prefix / "pkgs" / pkg_meta["fn"]; 139 | std::ofstream tmp_s(pkg_file, std::ios::out | std::ios::binary); 140 | 141 | std::array pkg_buf; 142 | std::ptrdiff_t archive_size = pkg_meta["size"]; 143 | std::ptrdiff_t read_size = archive_size; 144 | 145 | for (; read_size > BUF_SIZE; read_size -= BUF_SIZE) 146 | { 147 | self_exe.read(pkg_buf.data(), BUF_SIZE); 148 | tmp_s.write(pkg_buf.data(), BUF_SIZE); 149 | } 150 | self_exe.read(pkg_buf.data(), read_size); 151 | tmp_s.write(pkg_buf.data(), read_size); 152 | tmp_s.close(); 153 | 154 | link_to_prefix(target_prefix, std::vector{pkg_file.string()}, pkg_meta); 155 | } 156 | return 0; 157 | } --------------------------------------------------------------------------------