├── LICENSE ├── README.md ├── core.lua ├── db.lua ├── init.lua ├── package.lua ├── storage.lua └── tests └── test-core.lua /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Tim Caswell 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # lua-git 2 | 3 | Git implementation in pure lua for luvit. 4 | 5 | ## Get Git 6 | 7 | Install using the [lit](https://github.com/luvit/lit) toolkit. 8 | 9 | ```sh 10 | lit install creationix/git 11 | ``` 12 | 13 | ## git.mount(fs) -> db 14 | 15 | The primary interface to the git module is a mount command that accepts a fs 16 | interface instance and returns a git database. Under the hood it mounts a git 17 | repository using the same filesystem format as the native git tool. This 18 | includes support for reading packed objects and refs. 19 | 20 | The `fs` interface is required to implement the same API as a coro-fs chroot that 21 | is rooted at the database root. 22 | 23 | ```lua 24 | local import = _G.import or require 25 | 26 | local makeChroot = import('coro-fs').chroot 27 | local mount = import('git').mount 28 | 29 | local db = mount(makeChroot("path/to/.git")) 30 | 31 | --[[ 32 | db.has(hash) -> bool - check if db has an object 33 | db.load(hash) -> raw - load raw data, nil if not found 34 | db.loadAny(hash) -> kind, value - pre-decode data, error if not found 35 | db.loadAs(kind, hash) -> value - pre-decode and check type or error 36 | db.save(raw) -> hash - save pre-encoded and framed data 37 | db.saveAs(kind, value) -> hash - encode, frame and save to objects/$ha/$sh 38 | db.hashes() -> iter - Iterate over all hashes 39 | 40 | db.getHead() -> hash - Read the hash via HEAD 41 | db.getRef(ref) -> hash - Read hash of a ref 42 | db.resolve(ref) -> hash - Resolve hash, tag, branch, or "HEAD" to hash 43 | db.nodes(prefix) -> iter - iterate over non-leaf refs 44 | db.leaves(prefix) -> iter - iterate over leaf refs 45 | 46 | db.storage - table containing storage interface. 47 | 48 | storage.write(path, raw) - Write mutable data by path 49 | storage.put(path, raw) - Write immutable data by path 50 | storage.read(path) -> raw - Read mutable data by path (nil if not found) 51 | storage.delete(path) - Delete an entry (removes empty parent dirs) 52 | storage.nodes(path) -> iter - Iterate over node children of path 53 | (empty iter if not found) 54 | storage.leaves(path) -> iter - Iterate over node children of path 55 | (empty iter if not found) 56 | 57 | storage.fs - The fs instance originally passed in. 58 | ]] 59 | ``` 60 | 61 | ## git.modes 62 | 63 | The modes table contains constants and helpers for working with git tree entry modes. 64 | 65 | TODO: show example usage. 66 | 67 | ## git.encoders 68 | 69 | This table contains the internal encoder functions for constructing the binary 70 | representation of git objects. 71 | 72 | ## git.decoders 73 | 74 | This table contains decoders for reading binary git objects into lua. 75 | -------------------------------------------------------------------------------- /core.lua: -------------------------------------------------------------------------------- 1 | 2 | local exports = {} 3 | local modes = { 4 | tree = 16384, -- 040000 5 | blob = 33188, -- 0100644 6 | exec = 33261, -- 0100755 7 | sym = 40960, -- 0120000 8 | commit = 57344, -- 0160000 9 | } 10 | modes.file = modes.blob 11 | exports.modes = modes 12 | local encoders = {} 13 | exports.encoders = encoders 14 | local decoders = {} 15 | exports.decoders = decoders 16 | 17 | -- Given a raw character, return two hex characters 18 | local function binToHex(c) 19 | return string.format("%02x", string.byte(c, 1)) 20 | end 21 | 22 | exports.binToHex = function(bin) 23 | local hex = string.gsub(bin, ".", binToHex) 24 | return hex 25 | end 26 | 27 | -- Given two hex characters, return a single character 28 | local function hexToBin(cc) 29 | return string.char(tonumber(cc, 16)) 30 | end 31 | 32 | exports.hexToBin = function (hex) 33 | local bin = string.gsub(hex, "..", hexToBin) 34 | return bin 35 | end 36 | 37 | function modes.isBlob(mode) 38 | -- (mode & 0140000) == 0100000 39 | return bit.band(mode, 49152) == 32768 40 | end 41 | 42 | function modes.isFile(mode) 43 | -- (mode & 0160000) === 0100000 44 | return bit.band(mode, 57344) == 32768 45 | end 46 | 47 | function modes.toType(mode) 48 | -- 0160000 49 | return mode == 57344 and "commit" 50 | -- 040000 51 | or mode == 16384 and "tree" 52 | -- (mode & 0140000) == 0100000 53 | or bit.band(mode, 49152) == 32768 and "blob" or 54 | "unknown" 55 | end 56 | 57 | local function treeSort(a, b) 58 | return ((a.mode == modes.tree) and (a.name .. "/") or a.name) 59 | < ((b.mode == modes.tree) and (b.name .. "/") or b.name) 60 | end 61 | 62 | 63 | -- Remove illegal characters in things like emails and names 64 | local function safe(text) 65 | return text:gsub("^[%.,:;\"']+", "") 66 | :gsub("[%.,:;\"']+$", "") 67 | :gsub("[%z\n<>]+", "") 68 | end 69 | exports.safe = safe 70 | 71 | local function formatDate(date) 72 | local seconds = date.seconds 73 | local offset = date.offset 74 | assert(type(seconds) == "number", "date.seconds must be number") 75 | assert(type(offset) == "number", "date.offset must be number") 76 | return string.format("%d %+03d%02d", seconds, offset / 60, offset % 60) 77 | end 78 | 79 | local function formatPerson(person) 80 | assert(type(person.name) == "string", "person.name must be string") 81 | assert(type(person.email) == "string", "person.email must be string") 82 | assert(type(person.date) == "table", "person.date must be table") 83 | return safe(person.name) .. 84 | " <" .. safe(person.email) .. "> " .. 85 | formatDate(person.date) 86 | end 87 | 88 | local function parsePerson(raw) 89 | local pattern = "^([^<]+) <([^>]+)> (%d+) ([-+])(%d%d)(%d%d)$" 90 | local name, email, seconds, sign, hours, minutes = string.match(raw, pattern) 91 | local offset = tonumber(hours) * 60 + tonumber(minutes) 92 | if sign == "-" then offset = -offset end 93 | return { 94 | name = name, 95 | email = email, 96 | date = { 97 | seconds = tonumber(seconds), 98 | offset = offset 99 | } 100 | } 101 | end 102 | 103 | function encoders.blob(blob) 104 | assert(type(blob) == "string", "blobs must be strings") 105 | return blob 106 | end 107 | 108 | function decoders.blob(raw) 109 | return raw 110 | end 111 | 112 | function encoders.tree(tree) 113 | assert(type(tree) == "table", "trees must be tables") 114 | local parts = {} 115 | for i = 1, #tree do 116 | local value = tree[i] 117 | assert(type(value.name) == "string", "tree entry name must be string") 118 | assert(type(value.mode) == "number", "tree entry mode must be number") 119 | assert(type(value.hash) == "string", "tree entry hash must be string") 120 | parts[#parts + 1] = { 121 | name = value.name, 122 | mode = value.mode, 123 | hash = value.hash, 124 | } 125 | end 126 | table.sort(parts, treeSort) 127 | for i = 1, #parts do 128 | local entry = parts[i] 129 | parts[i] = string.format("%o %s\0", 130 | entry.mode, 131 | entry.name 132 | ) .. string.gsub(entry.hash, "..", hexToBin) 133 | end 134 | return table.concat(parts) 135 | end 136 | 137 | function decoders.tree(raw) 138 | local start, length = 1, #raw 139 | local tree = {} 140 | while start <= length do 141 | local pattern = "^([0-7]+) ([^%z]+)%z(....................)" 142 | local s, e, mode, name, hash = string.find(raw, pattern, start) 143 | assert(s, "Problem parsing tree") 144 | hash = string.gsub(hash, ".", binToHex) 145 | mode = tonumber(mode, 8) 146 | tree[#tree + 1] = { 147 | name = name, 148 | mode = mode, 149 | hash = hash 150 | } 151 | start = e + 1 152 | end 153 | return tree 154 | end 155 | 156 | function encoders.tag(tag) 157 | assert(type(tag) == "table", "annotated tags must be tables") 158 | assert(type(tag.object) == "string", "tag.object must be hash string") 159 | assert(type(tag.type) == "string", "tag.type must be string") 160 | assert(type(tag.tag) == "string", "tag.tag must be string") 161 | assert(type(tag.tagger) == "table", "tag.tagger must be table") 162 | assert(type(tag.message) == "string", "tag.message must be string") 163 | if tag.message[#tag.message] ~= "\n" then 164 | tag.message = tag.message .. "\n" 165 | end 166 | return string.format( 167 | "object %s\ntype %s\ntag %s\ntagger %s\n\n%s", 168 | tag.object, tag.type, tag.tag, formatPerson(tag.tagger), tag.message) 169 | end 170 | 171 | function decoders.tag(raw) 172 | local s, e, _, message 173 | s, _, message = string.find(raw, "\n\n(.*)$", 1) 174 | raw = string.sub(raw, 1, s) 175 | local start = 1 176 | local pattern = "^([^ ]+) ([^\n]+)\n" 177 | local data = {message=message} 178 | while true do 179 | local name, value 180 | s, e, name, value = string.find(raw, pattern, start) 181 | if not s then break end 182 | if name == "tagger" then 183 | value = parsePerson(value) 184 | end 185 | data[name] = value 186 | start = e + 1 187 | end 188 | return data 189 | end 190 | 191 | 192 | function encoders.commit(commit) 193 | assert(type(commit) == "table", "commits must be tables") 194 | assert(type(commit.tree) == "string", "commit.tree must be hash string") 195 | assert(type(commit.parents) == "table", "commit.parents must be table") 196 | assert(type(commit.author) == "table", "commit.author must be table") 197 | assert(type(commit.committer) == "table", "commit.committer must be table") 198 | assert(type(commit.message) == "string", "commit.message must be string") 199 | local parents = {} 200 | for i = 1, #commit.parents do 201 | local parent = commit.parents[i] 202 | assert(type(parent) == "string", "commit.parents must be hash strings") 203 | parents[i] = string.format("parent %s\n", parent) 204 | end 205 | return string.format( 206 | "tree %s\n%sauthor %s\ncommitter %s\n\n%s", 207 | commit.tree, table.concat(parents), formatPerson(commit.author), 208 | formatPerson(commit.committer), commit.message) 209 | end 210 | 211 | function decoders.commit(raw) 212 | local s, e, _, message 213 | s, _, message = string.find(raw, "\n\n(.*)$", 1) 214 | raw = string.sub(raw, 1, s) 215 | local start = 1 216 | local pattern = "^([^ ]+) ([^\n]+)\n" 217 | local parents = {} 218 | local data = {message=message,parents=parents} 219 | while true do 220 | local name, value 221 | s, e, name, value = string.find(raw, pattern, start) 222 | if not s then break end 223 | if name == "author" or name == "committer" then 224 | value = parsePerson(value) 225 | end 226 | if name == "parent" then 227 | parents[#parents + 1] = value 228 | else 229 | data[name] = value 230 | end 231 | start = e + 1 232 | end 233 | return data 234 | end 235 | 236 | function exports.frame(kind, data) 237 | assert(type(data) == "string", "data must be pre-encoded string") 238 | data = string.format("%s %d\0", kind, #data) .. data 239 | return data 240 | end 241 | 242 | function exports.deframe(raw) 243 | local pattern = "^([^ ]+) (%d+)%z(.*)$" 244 | local kind, size, body = string.match(raw, pattern) 245 | assert(kind, "Problem parsing framed git data") 246 | size = tonumber(size) 247 | assert(size == #body, "Body size mismatch") 248 | return kind, body 249 | end 250 | 251 | function exports.listToMap(list) 252 | local map = {} 253 | for i = 1, #list do 254 | local entry = list[i] 255 | map[entry.name] = {mode = entry.mode, hash = entry.hash} 256 | end 257 | return map 258 | end 259 | 260 | function exports.mapToList(map) 261 | local list = {} 262 | for name, entry in pairs(map) do 263 | list[#list + 1] = {name = name, mode = entry.mode, hash = entry.hash} 264 | end 265 | return list 266 | end 267 | 268 | return exports 269 | -------------------------------------------------------------------------------- /db.lua: -------------------------------------------------------------------------------- 1 | --[[ 2 | 3 | Git Object Database 4 | =================== 5 | 6 | Consumes a storage interface and return a git database interface 7 | 8 | db.has(hash) -> bool - check if db has an object 9 | db.load(hash) -> raw - load raw data, nil if not found 10 | db.loadAny(hash) -> kind, value - pre-decode data, error if not found 11 | db.loadAs(kind, hash) -> value - pre-decode and check type or error 12 | db.save(raw) -> hash - save pre-encoded and framed data 13 | db.saveAs(kind, value) -> hash - encode, frame and save to objects/$ha/$sh 14 | db.hashes() -> iter - Iterate over all hashes 15 | 16 | db.getHead() -> hash - Read the hash via HEAD 17 | db.getRef(ref) -> hash - Read hash of a ref 18 | db.resolve(ref) -> hash - Given a hash, tag, branch, or HEAD, return the hash 19 | db.nodes(prefix) -> iter - iterate over non-leaf refs 20 | db.leaves(prefix) -> iter - iterate over leaf refs 21 | ]] 22 | local import = _G.import or require 23 | 24 | local core = import('./core') 25 | local miniz = require('miniz') 26 | local openssl = require('openssl') 27 | local uv = require('uv') 28 | 29 | local numToType = { 30 | [1] = "commit", 31 | [2] = "tree", 32 | [3] = "blob", 33 | [4] = "tag", 34 | [6] = "ofs-delta", 35 | [7] = "ref-delta", 36 | } 37 | 38 | local encoders = core.encoders 39 | local decoders = core.decoders 40 | local frame = core.frame 41 | local deframe = core.deframe 42 | local binToHex = core.binToHex 43 | local hexToBin = core.hexToBin 44 | local deflate = miniz.deflate 45 | local inflate = miniz.inflate 46 | local digest = openssl.digest.digest 47 | 48 | local band = bit.band 49 | local bor = bit.bor 50 | local lshift = bit.lshift 51 | local rshift = bit.rshift 52 | local byte = string.byte 53 | local sub = string.sub 54 | local match = string.format 55 | local format = string.format 56 | local concat = table.concat 57 | 58 | local quotepattern = '(['..("%^$().[]*+-?"):gsub("(.)", "%%%1")..'])' 59 | local function escape(str) 60 | return str:gsub(quotepattern, "%%%1") 61 | end 62 | 63 | local function applyDelta(base, delta) --> raw 64 | local deltaOffset = 0; 65 | 66 | -- Read a variable length number our of delta and move the offset. 67 | local function readLength() 68 | deltaOffset = deltaOffset + 1 69 | local b = byte(delta, deltaOffset) 70 | local length = band(b, 0x7f) 71 | local shift = 7 72 | while band(b, 0x80) > 0 do 73 | deltaOffset = deltaOffset + 1 74 | b = byte(delta, deltaOffset) 75 | length = bor(length, lshift(band(b, 0x7f), shift)) 76 | shift = shift + 7 77 | end 78 | return length 79 | end 80 | 81 | assert(#base == readLength(), "base length mismatch") 82 | 83 | local outLength = readLength() 84 | local parts = {} 85 | while deltaOffset < #delta do 86 | deltaOffset = deltaOffset + 1 87 | local b = byte(delta, deltaOffset) 88 | 89 | if band(b, 0x80) > 0 then 90 | -- Copy command. Tells us offset in base and length to copy. 91 | local offset = 0 92 | local length = 0 93 | if band(b, 0x01) > 0 then 94 | deltaOffset = deltaOffset + 1 95 | offset = bor(offset, byte(delta, deltaOffset)) 96 | end 97 | if band(b, 0x02) > 0 then 98 | deltaOffset = deltaOffset + 1 99 | offset = bor(offset, lshift(byte(delta, deltaOffset), 8)) 100 | end 101 | if band(b, 0x04) > 0 then 102 | deltaOffset = deltaOffset + 1 103 | offset = bor(offset, lshift(byte(delta, deltaOffset), 16)) 104 | end 105 | if band(b, 0x08) > 0 then 106 | deltaOffset = deltaOffset + 1 107 | offset = bor(offset, lshift(byte(delta, deltaOffset), 24)) 108 | end 109 | if band(b, 0x10) > 0 then 110 | deltaOffset = deltaOffset + 1 111 | length = bor(length, byte(delta, deltaOffset)) 112 | end 113 | if band(b, 0x20) > 0 then 114 | deltaOffset = deltaOffset + 1 115 | length = bor(length, lshift(byte(delta, deltaOffset), 8)) 116 | end 117 | if band(b, 0x40) > 0 then 118 | deltaOffset = deltaOffset + 1 119 | length = bor(length, lshift(byte(delta, deltaOffset), 16)) 120 | end 121 | if length == 0 then length = 0x10000 end 122 | -- copy the data 123 | parts[#parts + 1] = sub(base, offset + 1, offset + length) 124 | elseif b > 0 then 125 | -- Insert command, opcode byte is length itself 126 | parts[#parts + 1] = sub(delta, deltaOffset + 1, deltaOffset + b) 127 | deltaOffset = deltaOffset + b 128 | else 129 | error("Invalid opcode in delta") 130 | end 131 | end 132 | local out = concat(parts) 133 | assert(#out == outLength, "final size mismatch in delta application") 134 | return concat(parts) 135 | end 136 | 137 | local function readUint32(buffer, offset) 138 | offset = offset or 0 139 | assert(#buffer >= offset + 4, "not enough buffer") 140 | return bor( 141 | lshift(byte(buffer, offset + 1), 24), 142 | lshift(byte(buffer, offset + 2), 16), 143 | lshift(byte(buffer, offset + 3), 8), 144 | byte(buffer, offset + 4) 145 | ) 146 | end 147 | 148 | local function readUint64(buffer, offset) 149 | offset = offset or 0 150 | assert(#buffer >= offset + 8, "not enough buffer") 151 | return 152 | (lshift(byte(buffer, offset + 1), 24) + 153 | lshift(byte(buffer, offset + 2), 16) + 154 | lshift(byte(buffer, offset + 3), 8) + 155 | byte(buffer, offset + 4)) * 0x100000000 + 156 | lshift(byte(buffer, offset + 5), 24) + 157 | lshift(byte(buffer, offset + 6), 16) + 158 | lshift(byte(buffer, offset + 7), 8) + 159 | byte(buffer, offset + 8) 160 | end 161 | 162 | local function assertHash(hash) 163 | assert(hash and #hash == 40 and match(hash, "^%x+$"), "Invalid hash") 164 | end 165 | 166 | local function hashPath(hash) 167 | return format("objects/%s/%s", sub(hash, 1, 2), sub(hash, 3)) 168 | end 169 | 170 | return function (storage) 171 | 172 | local db = { storage = storage } 173 | local fs = storage.fs 174 | 175 | -- Initialize the git file storage tree if it does't exist yet 176 | if not fs.access("HEAD") then 177 | assert(fs.mkdirp("objects")) 178 | assert(fs.mkdirp("refs/tags")) 179 | assert(fs.writeFile("HEAD", "ref: refs/heads/master\n")) 180 | assert(fs.writeFile("config", [[ 181 | [core] 182 | repositoryformatversion = 0 183 | filemode = true 184 | bare = true 185 | [gc] 186 | auto = 0 187 | ]])) 188 | end 189 | 190 | local packs = {} 191 | local function makePack(packHash) 192 | local pack = packs[packHash] 193 | if pack then 194 | if pack.waiting then 195 | pack.waiting[#pack.waiting + 1] = coroutine.running() 196 | return coroutine.yield() 197 | end 198 | return pack 199 | end 200 | local waiting = {} 201 | pack = { waiting=waiting } 202 | 203 | local timer, indexFd, packFd, indexLength 204 | local hashOffset, crcOffset 205 | local offsets, lengths, packSize 206 | 207 | local function close() 208 | if pack then 209 | pack.waiting = nil 210 | if packs[packHash] == pack then 211 | packs[packHash] = nil 212 | end 213 | pack = nil 214 | end 215 | if timer then 216 | timer:stop() 217 | timer:close() 218 | timer = nil 219 | end 220 | if indexFd then 221 | fs.close(indexFd) 222 | indexFd = nil 223 | end 224 | if packFd then 225 | fs.close(packFd) 226 | packFd = nil 227 | end 228 | end 229 | 230 | local function timeout() 231 | coroutine.wrap(close)() 232 | end 233 | 234 | 235 | timer = uv.new_timer() 236 | uv.unref(timer) 237 | timer:start(2000, 2000, timeout) 238 | 239 | packFd = assert(fs.open("objects/pack/pack-" .. packHash .. ".pack")) 240 | local stat = assert(fs.fstat(packFd)) 241 | packSize = stat.size 242 | assert(fs.read(packFd, 8, 0) == "PACK\0\0\0\2", "Only v2 pack files supported") 243 | 244 | indexFd = assert(fs.open("objects/pack/pack-" .. packHash .. ".idx")) 245 | assert(fs.read(indexFd, 8, 0) == '\255tOc\0\0\0\2', 'Only pack index v2 supported') 246 | indexLength = readUint32(assert(fs.read(indexFd, 4, 8 + 255 * 4))) 247 | hashOffset = 8 + 255 * 4 + 4 248 | crcOffset = hashOffset + 20 * indexLength 249 | local lengthOffset = crcOffset + 4 * indexLength 250 | local largeOffset = lengthOffset + 4 * indexLength 251 | offsets = {} 252 | lengths = {} 253 | local sorted = {} 254 | local data = assert(fs.read(indexFd, 4 * indexLength, lengthOffset)) 255 | for i = 1, indexLength do 256 | local offset = readUint32(data, (i - 1) * 4) 257 | if band(offset, 0x80000000) > 0 then 258 | error("TODO: Implement large offsets properly") 259 | offset = largeOffset + band(offset, 0x7fffffff) * 8; 260 | offset = readUint64(assert(fs.read(indexFd, 8, offset))) 261 | end 262 | offsets[i] = offset 263 | sorted[i] = offset 264 | end 265 | table.sort(sorted) 266 | for i = 1, indexLength do 267 | local offset = offsets[i] 268 | local length 269 | for j = 1, indexLength - 1 do 270 | if sorted[j] == offset then 271 | length = sorted[j + 1] - offset 272 | break 273 | end 274 | end 275 | lengths[i] = length or (packSize - offset - 20) 276 | end 277 | 278 | local function loadHash(hash) --> offset 279 | 280 | -- Read first fan-out table to get index into offset table 281 | local prefix = hexToBin(hash:sub(1, 2)):byte(1) 282 | local first = prefix == 0 and 0 or readUint32(assert(fs.read(indexFd, 4, 8 + (prefix - 1) * 4))) 283 | local last = readUint32(assert(fs.read(indexFd, 4, 8 + prefix * 4))) 284 | 285 | for index = first, last do 286 | local start = hashOffset + index * 20 287 | local foundHash = binToHex(assert(fs.read(indexFd, 20, start))) 288 | if foundHash == hash then 289 | index = index + 1 290 | return offsets[index], lengths[index] 291 | end 292 | end 293 | end 294 | 295 | local function loadRaw(offset, length) -->raw 296 | -- Shouldn't need more than 32 bytes to read variable length header and 297 | -- optional hash or offset 298 | local chunk = assert(fs.read(packFd, 32, offset)) 299 | local b = byte(chunk, 1) 300 | 301 | -- Parse out the git type 302 | local kind = numToType[band(rshift(b, 4), 0x7)] 303 | 304 | -- Parse out the uncompressed length 305 | local size = band(b, 0xf) 306 | local left = 4 307 | local i = 2 308 | while band(b, 0x80) > 0 do 309 | b = byte(chunk, i) 310 | i = i + 1 311 | size = bor(size, lshift(band(b, 0x7f), left)) 312 | left = left + 7 313 | end 314 | 315 | -- Optionally parse out the hash or offset for deltas 316 | local ref 317 | if kind == "ref-delta" then 318 | ref = binToHex(chunk:sub(i, i + 19)) 319 | i = i + 20 320 | elseif kind == "ofs-delta" then 321 | b = byte(chunk, i) 322 | i = i + 1 323 | ref = band(b, 0x7f) 324 | while band(b, 0x80) > 0 do 325 | b = byte(chunk, i) 326 | i = i + 1 327 | ref = bor(lshift(ref + 1, 7), band(b, 0x7f)) 328 | end 329 | end 330 | 331 | -- Guess an upper bound for the compressed data. 332 | local guessedSize = math.max(100, (length or size) * 2) 333 | local compressed = assert(fs.read(packFd, guessedSize, offset + i - 1)) 334 | local raw = inflate(compressed, 1) 335 | 336 | if #raw ~= size then 337 | error("inflate error or size mismatch at offset " .. offset) 338 | end 339 | 340 | if kind == "ref-delta" then 341 | local base 342 | kind, base = deframe(db.load(ref)) 343 | raw = applyDelta(base, raw) 344 | elseif kind == "ofs-delta" then 345 | local base 346 | kind, base = loadRaw(offset - ref) 347 | raw = applyDelta(base, raw) 348 | end 349 | return kind, raw 350 | end 351 | 352 | function pack.load(hash) --> raw 353 | if not pack then 354 | return makePack(packHash).load(hash) 355 | end 356 | timer:again() 357 | local success, result = pcall(function () 358 | local offset, length = loadHash(hash) 359 | if not offset then return end 360 | local kind, raw = loadRaw(offset, length) 361 | return frame(kind, raw) 362 | end) 363 | if success then return result end 364 | -- close() 365 | error(result) 366 | end 367 | 368 | packs[packHash] = pack 369 | pack.waiting = nil 370 | for i = 1, #waiting do 371 | assert(coroutine.resume(waiting[i], pack)) 372 | end 373 | 374 | return pack 375 | end 376 | 377 | function db.has(hash) 378 | assertHash(hash) 379 | return storage.read(hashPath(hash)) and true or false 380 | end 381 | 382 | function db.load(hash) 383 | assert(hash, "hash required") 384 | hash = db.resolve(hash) 385 | assertHash(hash) 386 | local compressed, err = storage.read(hashPath(hash)) 387 | if not compressed then 388 | for file in storage.leaves("objects/pack") do 389 | local packHash = file:match("^pack%-(%x+)%.idx$") 390 | if packHash then 391 | local raw 392 | raw, err = makePack(packHash).load(hash) 393 | if raw then return raw end 394 | end 395 | end 396 | return nil, err 397 | end 398 | return inflate(compressed, 1) 399 | end 400 | 401 | function db.loadAny(hash) 402 | local raw = assert(db.load(hash), "no such hash") 403 | local kind, value = deframe(raw) 404 | return kind, decoders[kind](value) 405 | end 406 | 407 | function db.loadAs(kind, hash) 408 | local actualKind, value = db.loadAny(hash) 409 | assert(kind == actualKind, "Kind mismatch") 410 | return value 411 | end 412 | 413 | function db.save(raw) 414 | local hash = digest("sha1", raw) 415 | -- 0x1000 = TDEFL_WRITE_ZLIB_HEADER 416 | -- 4095 = Huffman+LZ (slowest/best compression) 417 | storage.put(hashPath(hash), deflate(raw, 0x1000 + 4095)) 418 | return hash 419 | end 420 | 421 | function db.saveAs(kind, value) 422 | if type(value) ~= "string" then 423 | value = encoders[kind](value) 424 | end 425 | return db.save(frame(kind, value)) 426 | end 427 | 428 | function db.hashes() 429 | local groups = storage.nodes("objects") 430 | local prefix, iter 431 | return function () 432 | while true do 433 | if prefix then 434 | local rest = iter() 435 | if rest then return prefix .. rest end 436 | prefix = nil 437 | iter = nil 438 | end 439 | prefix = groups() 440 | if not prefix then return end 441 | iter = storage.leaves("objects/" .. prefix) 442 | end 443 | end 444 | end 445 | 446 | function db.getHead() 447 | local head = storage.read("HEAD") 448 | if not head then return end 449 | local ref = head:match("^ref: *([^\n]+)") 450 | return ref and db.getRef(ref) 451 | end 452 | 453 | function db.getRef(ref) 454 | local hash = storage.read(ref) 455 | if hash then return hash:match("%x+") end 456 | local refs = storage.read("packed-refs") 457 | return refs and refs:match("(%x+) " .. escape(ref)) 458 | end 459 | 460 | function db.resolve(ref) 461 | if ref == "HEAD" then return db.getHead() end 462 | local hash = ref:match("^%x+$") 463 | if hash and #hash == 40 then return hash end 464 | return db.getRef(ref) 465 | or db.getRef("refs/heads/" .. ref) 466 | or db.getRef("refs/tags/" .. ref) 467 | end 468 | 469 | local function makePackedIter(prefix, inner) 470 | local packed = storage.read("packed-refs") 471 | if not packed then return function () end end 472 | if prefix:byte(-1) ~= 47 then 473 | prefix = prefix .. "/" 474 | end 475 | if inner then 476 | return packed:gmatch(escape(prefix) .. "([^/ \r\n]+)/") 477 | else 478 | return packed:gmatch(escape(prefix) .. "([^/ \r\n]+)") 479 | end 480 | end 481 | 482 | local function commonIter(iter1, iter2) 483 | local seen = {} 484 | return function () 485 | if iter1 then 486 | local item =iter1() 487 | if item then 488 | seen[item] = true 489 | return item 490 | end 491 | iter1 = nil 492 | end 493 | while true do 494 | local item = iter2() 495 | if not item then return end 496 | if not seen[item] then 497 | seen[item] = true 498 | return item 499 | end 500 | end 501 | end 502 | end 503 | 504 | function db.nodes(prefix) 505 | return commonIter( 506 | storage.nodes(prefix), 507 | makePackedIter(prefix, true) 508 | ) 509 | end 510 | 511 | function db.leaves(prefix) 512 | return commonIter( 513 | storage.leaves(prefix), 514 | makePackedIter(prefix, false) 515 | ) 516 | end 517 | 518 | return db 519 | end 520 | -------------------------------------------------------------------------------- /init.lua: -------------------------------------------------------------------------------- 1 | local import = _G.import or require 2 | 3 | local core = import('./core') 4 | local exports = {} 5 | for key, value in pairs(core) do 6 | exports[key] = value 7 | end 8 | 9 | function exports.mount(fs) 10 | return import('./db')(import('./storage')(fs)) 11 | end 12 | 13 | return exports 14 | -------------------------------------------------------------------------------- /package.lua: -------------------------------------------------------------------------------- 1 | return { 2 | name = "creationix/git", 3 | version = "2.1.2", 4 | homepage = "https://github.com/creationix/lua-git", 5 | description = "An implementation of git in pure lua.", 6 | tags = {"git","db","codec"}, 7 | license = "MIT", 8 | author = { name = "Tim Caswell" }, 9 | files = { 10 | "*.lua", 11 | }, 12 | } 13 | -------------------------------------------------------------------------------- /storage.lua: -------------------------------------------------------------------------------- 1 | 2 | --[[ 3 | Low Level Storage Commands 4 | ========================== 5 | 6 | These are the filesystem abstractions needed by a git database 7 | 8 | storage.write(path, raw) - Write mutable data by path 9 | storage.put(path, raw) - Write immutable data by path 10 | storage.read(path) -> raw - Read mutable data by path (nil if not found) 11 | storage.delete(path) - Delete an entry (removes empty parent directories) 12 | storage.nodes(path) -> iter - Iterate over node children of path 13 | (empty iter if not found) 14 | storage.leaves(path) -> iter - Iterate over node children of path 15 | (empty iter if not found) 16 | ]] 17 | 18 | return function (fs) 19 | 20 | local storage = { fs = fs } 21 | 22 | local function dirname(path) 23 | return path:match("^(.*)/") or "" 24 | end 25 | 26 | -- Perform an atomic write (with temp file and rename) for mutable data 27 | function storage.write(path, data) 28 | -- Ensure the parent directory exists first. 29 | assert(fs.mkdirp(dirname(path))) 30 | -- Write the data out to a temporary file. 31 | local tempPath = path .. "~" 32 | do 33 | local fd, success, err 34 | fd = assert(fs.open(tempPath, "w", 384)) 35 | success, err = fs.write(fd, data) 36 | fs.close(fd) 37 | assert(success, err) 38 | end 39 | -- Rename the temp file on top of the old file for atomic commit. 40 | assert(fs.rename(tempPath, path)) 41 | end 42 | 43 | -- Write immutable data with an exclusive open. 44 | function storage.put(path, data) 45 | local fd, success, err 46 | assert(fs.mkdirp(dirname(path))) 47 | fd, err = fs.open(path, "wx") 48 | if not fd then 49 | -- If the file already exists, do nothing, it's immutable. 50 | if err:match("^EEXIST:") then return end 51 | error(err) 52 | end 53 | success, err = fs.write(fd, data) 54 | if success then 55 | success, err = fs.fchmod(fd, 256) 56 | end 57 | fs.close(fd) 58 | assert(success, err) 59 | end 60 | 61 | function storage.read(path) 62 | local data, err = fs.readFile(path) 63 | if data then return data end 64 | if err:match("^ENOENT:") then return end 65 | assert(data, err) 66 | end 67 | 68 | function storage.delete(path) 69 | assert(fs.unlink(path)) 70 | local dirPath = path 71 | while true do 72 | dirPath = dirname(dirPath) 73 | local iter = assert(fs.scandir(dirPath)) 74 | if iter() then return end 75 | assert(fs.rmdir(dirPath)) 76 | end 77 | end 78 | 79 | local function iter(path, filter) 80 | local it, err = fs.scandir(path) 81 | if not it then 82 | if err:match("^ENOENT:") then 83 | return function() end 84 | end 85 | assert(it, err) 86 | end 87 | return function () 88 | while true do 89 | local item = it() 90 | if not item then return end 91 | if item.type == filter then 92 | return item.name 93 | end 94 | end 95 | end 96 | end 97 | 98 | function storage.nodes(path) 99 | return iter(path, "directory") 100 | end 101 | 102 | function storage.leaves(path) 103 | return iter(path, "file") 104 | end 105 | 106 | return storage 107 | end 108 | -------------------------------------------------------------------------------- /tests/test-core.lua: -------------------------------------------------------------------------------- 1 | local import = _G.import or require 2 | 3 | local core = import('../core') 4 | local encoders = core.encoders 5 | local decoders = core.decoders 6 | local frame = core.frame 7 | local deframe = core.deframe 8 | local modes = core.modes 9 | local dump = import('pretty-print').dump 10 | 11 | local tests = { 12 | "blob", "Hello World\n", "Hello World\n", 13 | "blob", "\1\0\2\0\3\0\4\0\5\0", "\1\0\2\0\3\0\4\0\5\0", 14 | "blob", "", "", 15 | "tag", { 16 | object = "affe225fdb68803477e0fd9dea2f1cee309faed3", 17 | type = "commit", 18 | tag = "v2.0.9", 19 | tagger = { 20 | name = "Tim Caswell", 21 | email = "tim@creationix.com", 22 | date = { 23 | seconds = 1431710918, 24 | offset = -300, 25 | } 26 | }, 27 | message = "Luvi v2.0.9" 28 | }, "object affe225fdb68803477e0fd9dea2f1cee309faed3\ntype commit\ntag v2.0.9\ntagger Tim Caswell 1431710918 -0500\n\nLuvi v2.0.9\n", 29 | "commit", { 30 | tree = "9b5a82b1cf7306709848d502121c22a50cfa4667", 31 | parents = { "d966b5d6668fb63e0eede457367c4f9d9067e745" }, 32 | author = { 33 | name = "Tim Caswell", 34 | email = "tim@creationix.com", 35 | date = { 36 | seconds = 1431710879, 37 | offset = -300 38 | } 39 | }, 40 | committer = { 41 | name = "Tim Caswell", 42 | email = "tim@creationix.com", 43 | date = { 44 | seconds = 1431710879, 45 | offset = -300 46 | } 47 | }, 48 | message = "Bump version again to get good submodules\n" 49 | }, "tree 9b5a82b1cf7306709848d502121c22a50cfa4667\nparent d966b5d6668fb63e0eede457367c4f9d9067e745\nauthor Tim Caswell 1431710879 -0500\ncommitter Tim Caswell 1431710879 -0500\n\nBump version again to get good submodules\n", 50 | "tree", { 51 | { mode = modes.tree, hash = "07bee11133660674c182cba3a85be87e2308d01e", name = "cmake"}, 52 | { mode = modes.tree, hash = "ce3be9f3a01701602b080ac8eef48a34ca73f051", name = "deps"}, 53 | { mode = modes.tree, hash = "215e61b297bccc78580be6fbe135dc11c4122f09", name = "packaging"}, 54 | { mode = modes.tree, hash = "49f99fd5989f62921e47b7a5332548ee0e6d5226", name = "samples"}, 55 | { mode = modes.tree, hash = "a89d220db1d54e9a68a6477fea288dd316b6a912", name = "src"}, 56 | { mode = modes.blob, hash = "8e9eb228a657ef6eb6acc31cc77d279fa3011c63", name = ".gitignore"}, 57 | { mode = modes.blob, hash = "9bb8db6088ab2fc9b35a2347aa6b69365d3e9066", name = ".gitmodules"}, 58 | { mode = modes.blob, hash = "426d23fa5454ca2ce1b3dd784ce2d6635bc63729", name = ".travis.yml"}, 59 | { mode = modes.blob, hash = "31acbf043bc9fa4db678fbe819af4bf851838ef2", name = "CHANGELOG.md"}, 60 | { mode = modes.blob, hash = "2d0af1fe328fba97544fd07cf38d43e696079695", name = "CMakeLists.txt"}, 61 | { mode = modes.blob, hash = "d645695673349e3947e8e5ae42332d0ac3164cd7", name = "LICENSE.txt"}, 62 | { mode = modes.blob, hash = "ab9413cae513746de3b080c4a5446990cc885d55", name = "Makefile"}, 63 | { mode = modes.blob, hash = "04a8167099345935d558e4e3cd59ba1ebf31e47e", name = "README.md"}, 64 | { mode = modes.blob, hash = "3ba44e645a76e7786e4fe10d1b336de3fd7757a7", name = "appveyor.yml"}, 65 | { mode = modes.blob, hash = "6001cab8388bf5e17903cc440c8e747cf1908a00", name = "make.bat"}, 66 | }, "100644 .gitignore\0\x8e\x9e\xb2\x28\xa6\x57\xef\x6e\xb6\xac\xc3\x1c\xc7\x7d\x27\x9f\xa3\x01\x1c\x63" .. 67 | "100644 .gitmodules\0\x9b\xb8\xdb\x60\x88\xab\x2f\xc9\xb3\x5a\x23\x47\xaa\x6b\x69\x36\x5d\x3e\x90\x66" .. 68 | "100644 .travis.yml\0\x42\x6d\x23\xfa\x54\x54\xca\x2c\xe1\xb3\xdd\x78\x4c\xe2\xd6\x63\x5b\xc6\x37\x29" .. 69 | "100644 CHANGELOG.md\0\x31\xac\xbf\x04\x3b\xc9\xfa\x4d\xb6\x78\xfb\xe8\x19\xaf\x4b\xf8\x51\x83\x8e\xf2" .. 70 | "100644 CMakeLists.txt\0\x2d\x0a\xf1\xfe\x32\x8f\xba\x97\x54\x4f\xd0\x7c\xf3\x8d\x43\xe6\x96\x07\x96\x95" .. 71 | "100644 LICENSE.txt\0\xd6\x45\x69\x56\x73\x34\x9e\x39\x47\xe8\xe5\xae\x42\x33\x2d\x0a\xc3\x16\x4c\xd7" .. 72 | "100644 Makefile\0\xab\x94\x13\xca\xe5\x13\x74\x6d\xe3\xb0\x80\xc4\xa5\x44\x69\x90\xcc\x88\x5d\x55" .. 73 | "100644 README.md\0\x04\xa8\x16\x70\x99\x34\x59\x35\xd5\x58\xe4\xe3\xcd\x59\xba\x1e\xbf\x31\xe4\x7e" .. 74 | "100644 appveyor.yml\0\x3b\xa4\x4e\x64\x5a\x76\xe7\x78\x6e\x4f\xe1\x0d\x1b\x33\x6d\xe3\xfd\x77\x57\xa7" .. 75 | "40000 cmake\0\x07\xbe\xe1\x11\x33\x66\x06\x74\xc1\x82\xcb\xa3\xa8\x5b\xe8\x7e\x23\x08\xd0\x1e" .. 76 | "40000 deps\0\xce\x3b\xe9\xf3\xa0\x17\x01\x60\x2b\x08\x0a\xc8\xee\xf4\x8a\x34\xca\x73\xf0\x51" .. 77 | "100644 make.bat\0\x60\x01\xca\xb8\x38\x8b\xf5\xe1\x79\x03\xcc\x44\x0c\x8e\x74\x7c\xf1\x90\x8a\x00" .. 78 | "40000 packaging\0\x21\x5e\x61\xb2\x97\xbc\xcc\x78\x58\x0b\xe6\xfb\xe1\x35\xdc\x11\xc4\x12\x2f\x09" .. 79 | "40000 samples\0\x49\xf9\x9f\xd5\x98\x9f\x62\x92\x1e\x47\xb7\xa5\x33\x25\x48\xee\x0e\x6d\x52\x26" .. 80 | "40000 src\0\xa8\x9d\x22\x0d\xb1\xd5\x4e\x9a\x68\xa6\x47\x7f\xea\x28\x8d\xd3\x16\xb6\xa9\x12", 81 | } 82 | 83 | for i = 1, #tests, 3 do 84 | local kind = tests[i] 85 | local input = tests[i + 1] 86 | local expected = tests[i + 2] 87 | local ok, actual = pcall(encoders[kind], input) 88 | if not ok then 89 | error("encoders." .. kind .. "(" .. dump(input) .. ") errored: " .. actual) 90 | end 91 | if actual ~= expected then 92 | error("encoders." .. kind .. "(" .. dump(input) .. ") should be " .. dump(expected) .. " but was " .. dump(actual)) 93 | end 94 | end 95 | 96 | -- Test above had wrong sort order on purpose. We need to fix it to do the decode test. 97 | tests[17] = { 98 | { mode = modes.blob, hash = "8e9eb228a657ef6eb6acc31cc77d279fa3011c63", name = ".gitignore"}, 99 | { mode = modes.blob, hash = "9bb8db6088ab2fc9b35a2347aa6b69365d3e9066", name = ".gitmodules"}, 100 | { mode = modes.blob, hash = "426d23fa5454ca2ce1b3dd784ce2d6635bc63729", name = ".travis.yml"}, 101 | { mode = modes.blob, hash = "31acbf043bc9fa4db678fbe819af4bf851838ef2", name = "CHANGELOG.md"}, 102 | { mode = modes.blob, hash = "2d0af1fe328fba97544fd07cf38d43e696079695", name = "CMakeLists.txt"}, 103 | { mode = modes.blob, hash = "d645695673349e3947e8e5ae42332d0ac3164cd7", name = "LICENSE.txt"}, 104 | { mode = modes.blob, hash = "ab9413cae513746de3b080c4a5446990cc885d55", name = "Makefile"}, 105 | { mode = modes.blob, hash = "04a8167099345935d558e4e3cd59ba1ebf31e47e", name = "README.md"}, 106 | { mode = modes.blob, hash = "3ba44e645a76e7786e4fe10d1b336de3fd7757a7", name = "appveyor.yml"}, 107 | { mode = modes.tree, hash = "07bee11133660674c182cba3a85be87e2308d01e", name = "cmake"}, 108 | { mode = modes.tree, hash = "ce3be9f3a01701602b080ac8eef48a34ca73f051", name = "deps"}, 109 | { mode = modes.blob, hash = "6001cab8388bf5e17903cc440c8e747cf1908a00", name = "make.bat"}, 110 | { mode = modes.tree, hash = "215e61b297bccc78580be6fbe135dc11c4122f09", name = "packaging"}, 111 | { mode = modes.tree, hash = "49f99fd5989f62921e47b7a5332548ee0e6d5226", name = "samples"}, 112 | { mode = modes.tree, hash = "a89d220db1d54e9a68a6477fea288dd316b6a912", name = "src"}, 113 | } 114 | 115 | local function deepEqual(a, b) 116 | if a == b then return true end 117 | if type(a) ~= type(b) or type(a) ~= "table" then return false end 118 | if #a ~= #b then return false end 119 | local count = 0 120 | for key, value in pairs(a) do 121 | count = count + 1 122 | if not deepEqual(value, b[key]) then return false end 123 | end 124 | for _ in pairs(b) do 125 | count = count - 1 126 | end 127 | if count ~= 0 then return false end 128 | return true 129 | end 130 | 131 | for i = 1, #tests, 3 do 132 | local kind = tests[i] 133 | local expected = tests[i + 1] 134 | local input = tests[i + 2] 135 | local ok, actual = pcall(decoders[kind], input) 136 | if not ok then 137 | error("decoders." .. kind .. "(" .. dump(input) .. ") errored: " .. actual) 138 | end 139 | if not deepEqual(actual, expected) then 140 | error("decoders." .. kind .. "(" .. dump(input) .. ") should be " .. dump(expected) .. " but was " .. dump(actual)) 141 | end 142 | end 143 | 144 | local prefixes = { 145 | "blob 12\0", 146 | "blob 10\0", 147 | "blob 0\0", 148 | "tag 141\0", 149 | "commit 254\0", 150 | "tree 549\0", 151 | } 152 | 153 | for i = 1, #tests, 3 do 154 | local kind = tests[i] 155 | local encoded = tests[i + 2] 156 | local prefix = prefixes[(i + 2) / 3] 157 | local expected = prefix .. encoded 158 | local ok, actual = pcall(frame, kind, encoded) 159 | if not ok then 160 | error("frame(" .. dump(kind) .. ", " .. dump(encoded) .. ") errored: " .. actual) 161 | end 162 | if actual ~= expected then 163 | error("frame(" .. dump(kind) .. ", " .. dump(encoded) .. ") should be " .. dump(expected) .. " but was " .. dump(actual)) 164 | end 165 | local data 166 | ok, actual, data = pcall(deframe, expected) 167 | if not ok then 168 | error("deframe(" .. dump(expected) .. ") errored: " .. actual) 169 | end 170 | if actual ~= kind or data ~= encoded then 171 | error("deframe(" .. dump(expected) .. ") should be " .. dump(kind) .. ", " .. dump(encoded) .. " but was " .. dump(actual) .. ", " .. dump(data)) 172 | end 173 | end 174 | 175 | for i = 1, #tests, 3 do 176 | local kind = tests[i] 177 | local input = tests[i + 1] 178 | 179 | local framed = frame(kind, encoders[kind](input)) 180 | local outKind, raw = deframe(framed) 181 | assert(outKind == kind, "Kind mismatch in round trip") 182 | assert(deepEqual(decoders[kind](raw), input), "Value mismatch in round trip") 183 | end 184 | --------------------------------------------------------------------------------