├── git_remote_ipfs ├── __init__.py ├── exc.py ├── main.py ├── marks.py ├── helper.py ├── importer.py └── remote.py ├── requirements.txt ├── setup.py └── README.md /git_remote_ipfs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | fastimport 2 | ipfs-api 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | setup(name='git-remote-ipfs', 4 | author='Lars Kellogg-Stedman', 5 | author_email='lars@oddbit.com', 6 | url='https://github.com/larsks/git-remote-ipfs', 7 | version='0.1', 8 | packages=find_packages(), 9 | install_requires=[ 10 | 'ipfs-api', 11 | ], 12 | entry_points={ 13 | 'console_scripts': 14 | ['git-remote-ipfs = git_remote_ipfs.main:main'], 15 | }) 16 | -------------------------------------------------------------------------------- /git_remote_ipfs/exc.py: -------------------------------------------------------------------------------- 1 | class IPFSError(Exception): 2 | pass 3 | 4 | 5 | class CLIError(IPFSError): 6 | pass 7 | 8 | 9 | class FeatureNotImplemented(IPFSError): 10 | def __init__(self, feature): 11 | super(FeatureNotImplemented, self).__init__( 12 | 'Feature not implemented: %s' % feature) 13 | 14 | 15 | class CommandNotImplemented(IPFSError): 16 | def __init__(self, command): 17 | super(CommandNotImplemented, self).__init__( 18 | 'Command not implemented: %s' % command) 19 | 20 | class UnknownReference(IPFSError): 21 | def __init__(self, ref): 22 | super(UnknownReference, self).__init__( 23 | 'Unknown reference: %s' % ref) 24 | 25 | 26 | class InvalidURL(IPFSError): 27 | def __init__(self, url): 28 | super(InvalidURL, self).__init__( 29 | 'Invalid URL: %s' % ref) 30 | 31 | -------------------------------------------------------------------------------- /git_remote_ipfs/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import atexit 4 | import os 5 | import sys 6 | import argparse 7 | import logging 8 | 9 | import git_remote_ipfs.remote 10 | import git_remote_ipfs.helper 11 | from git_remote_ipfs.exc import * 12 | 13 | LOG = logging.getLogger(__name__) 14 | 15 | 16 | def parse_args(): 17 | p = argparse.ArgumentParser() 18 | p.add_argument('--ipfs-gateway', '-g', 19 | default=os.environ.get('GIT_IPFS_GATEWAY')) 20 | p.add_argument('--debug', 21 | action='store_const', 22 | const='DEBUG', 23 | dest='loglevel') 24 | p.add_argument('--verbose', 25 | action='store_const', 26 | const='INFO', 27 | dest='loglevel') 28 | p.add_argument('--git-dir', '-d', 29 | default=os.environ.get('GIT_DIR')) 30 | p.add_argument('alias') 31 | p.add_argument('url') 32 | 33 | p.set_defaults(loglevel=os.environ.get('GIT_IPFS_LOGLEVEL', 'WARN')) 34 | return p.parse_args() 35 | 36 | 37 | def main(): 38 | args = parse_args() 39 | logging.basicConfig(level=args.loglevel) 40 | logging.getLogger('requests').setLevel('WARN') 41 | 42 | try: 43 | if args.git_dir is None: 44 | raise CLIError('GIT_DIR is undefined') 45 | 46 | repo = git_remote_ipfs.remote.IPFSRemote( 47 | args.git_dir, args.alias, args.url, 48 | ipfs_gateway=args.ipfs_gateway) 49 | helper = git_remote_ipfs.helper.Helper(repo=repo) 50 | 51 | atexit.register(repo.cleanup) 52 | helper.run() 53 | except IPFSError as err: 54 | LOG.error(err) 55 | sys.exit(1) 56 | 57 | if __name__ == '__main__': 58 | main() 59 | -------------------------------------------------------------------------------- /git_remote_ipfs/marks.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | 4 | class Marks(object): 5 | def __init__(self, path): 6 | self.path = path 7 | self.refs = {} 8 | self.marks = {} 9 | self.rev_marks = {} 10 | self.last_mark = 0 11 | self.load() 12 | 13 | def load(self): 14 | try: 15 | with open(self.path) as fd: 16 | toc = json.load(fd) 17 | except IOError: 18 | return 19 | 20 | self.refs = toc['refs'] 21 | self.marks = toc['marks'] 22 | self.rev_marks = dict((v, k) for k, v in self.marks.items()) 23 | 24 | if self.marks: 25 | self.last_mark = sorted(int(x[1:]) for x in self.marks.keys())[-1] 26 | 27 | def store(self): 28 | with open(self.path, 'w') as fd: 29 | json.dump(self.to_dict(), fd, indent=2) 30 | 31 | def to_dict(self): 32 | return { 33 | 'refs': self.refs, 34 | 'marks': self.marks, 35 | 'last_mark': self.last_mark, 36 | } 37 | 38 | def next_mark(self): 39 | self.last_mark += 1 40 | return ':%d' % self.last_mark 41 | 42 | def from_rev(self, rev): 43 | return self.rev_marks[rev] 44 | 45 | def from_mark(self, mark): 46 | return self.marks[mark] 47 | 48 | def is_marked(self, rev): 49 | return rev in self.rev_marks 50 | 51 | def add_rev(self, rev): 52 | mark = self.next_mark() 53 | self.marks[mark] = rev 54 | self.rev_marks[rev] = mark 55 | return mark 56 | 57 | def add_mark(self, mark, rev): 58 | self.marks[mark] = rev 59 | self.rev_marks[rev] = mark 60 | self.last_mark = mark 61 | 62 | def get_ref(self, ref): 63 | return self.refs.get(ref) 64 | 65 | def set_ref(self, ref, hash): 66 | self.refs[ref] = hash 67 | -------------------------------------------------------------------------------- /git_remote_ipfs/helper.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import logging 4 | 5 | import fastimport.parser 6 | import git_remote_ipfs.importer 7 | from git_remote_ipfs.exc import * 8 | 9 | LOG = logging.getLogger(__name__) 10 | 11 | 12 | class Helper(object): 13 | def __init__(self, repo, fd=sys.stdin): 14 | self.fd = fd 15 | self.done = False 16 | self.repo = repo 17 | self.importing = False 18 | 19 | def lines(self): 20 | while not self.done: 21 | line = self.fd.readline() 22 | if not line: 23 | break 24 | 25 | line = line.rstrip() 26 | LOG.debug('line = %s', line) 27 | yield line 28 | 29 | def run(self): 30 | for line in self.lines(): 31 | if not line: 32 | if self.importing: 33 | LOG.debug('finishing imports') 34 | print 'done' 35 | sys.stdout.flush() 36 | self.importing = False 37 | continue 38 | 39 | try: 40 | command, args = line.split(' ', 1) 41 | except ValueError: 42 | command, args = line, '' 43 | 44 | try: 45 | handler = getattr(self, 'do_%s' % command) 46 | except AttributeError: 47 | raise CommandNotImplemented(command) 48 | 49 | handler(command, args) 50 | sys.stdout.flush() 51 | 52 | def do_capabilities(self, command, args): 53 | print 'import' 54 | print 'export' 55 | print 'refspec refs/heads/*:%s/heads/*' % self.repo.prefix 56 | print 'refspec refs/tags/*:%s/tags/*' % self.repo.prefix 57 | print '*export-marks %s' % self.repo.markpath 58 | if os.path.isfile(self.repo.markpath): 59 | print '*import-marks %s' % self.repo.markpath 60 | print 61 | 62 | def do_list(self, command, args): 63 | for ref in self.repo.marks.refs: 64 | print '? %s' % ref 65 | print '@refs/heads/master HEAD' 66 | print 67 | 68 | def do_option(self, command, args): 69 | print 'unsupported' 70 | 71 | def do_export(self, command, args): 72 | importer = git_remote_ipfs.importer.ImportProcessor(self.repo) 73 | parser = fastimport.parser.ImportParser(self.fd) 74 | importer.process(parser.iter_commands) 75 | self.done = True 76 | 77 | def do_import(self, command, args): 78 | if not self.importing: 79 | print 'feature done' 80 | print 'feature export-marks=%s' % self.repo.markpath 81 | if os.path.isfile(self.repo.markpath): 82 | print 'feature import-marks=%s' % self.repo.markpath 83 | print 'feature force' 84 | 85 | self.importing = True 86 | exporter = git_remote_ipfs.importer.ExportProcessor(self.repo) 87 | exporter.export(args) 88 | self.repo.commit() 89 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This is a [gitremote helper][] permitting git to clone from and push 2 | to [ipfs][]. Note that while it works in principal, it is only mildly 3 | useful until such time as the ipfs project introduces support for 4 | publishing multiple names via ipns. 5 | 6 | [gitremote helper]: https://www.kernel.org/pub/software/scm/git/docs/gitremote-helpers.html 7 | [ipfs]: http://ipfs.io/ 8 | 9 | ## INSTALLATION 10 | 11 | You can install this module directly using `pip`, like this: 12 | 13 | pip install git+https://github.com/larsks/git-remote-ipfs 14 | 15 | You can of course also clone it and run `setup.py` instead. 16 | 17 | ## SPECIFYING AN IPFS REMOTE 18 | 19 | ### Using IPFS style paths 20 | 21 | Because paths like `/ipfs/HASH` look just like filesystem paths, we 22 | need to explicitly tell git to use an ipfs remote by prefixing the 23 | path with `ipfs::`, like this: 24 | 25 | git clone ipfs::/ipfs/HASH myproject 26 | 27 | The code is able to resolve ipns names, so this will also work: 28 | 29 | git clone ipfs::/ipfs/HASH myproject 30 | 31 | Note that ipns support is effectively useless right now, until it 32 | becomes possible to publish more than a single name per client. 33 | 34 | ### Using IPFS URLs 35 | 36 | This code also supports a URL format for ipfs remotes. For explicit 37 | hashes (the equivalent of `/ipfs/HASH`), the format is: 38 | 39 | ipfs:///HASH 40 | 41 | So: 42 | 43 | git clone ipfs:///HASH 44 | 45 | For ipns names, the format is: 46 | 47 | ipfs://HASH 48 | 49 | Yes, the difference is a single `/`. This will probably changed, based on the discussion in [issue 1678][]. 50 | 51 | [issue 1678]: https://github.com/ipfs/go-ipfs/issues/1678 52 | 53 | ## EXAMPLE USAGE 54 | 55 | ### Pushing to ipfs 56 | 57 | $ git remote add ipfs ipfs:: 58 | $ git push ipfs master 59 | WARNING:git_remote_ipfs.remote:new repository hash = QmctS8mbpdQ1rgvS9SdFfJsoAE8s97FdXDHJQtobewAXKG 60 | To ipfs:: 61 | * [new branch] master -> master 62 | 63 | ### Cloning from ipfs 64 | 65 | To clone from ipfs the repository pushed in the previous example: 66 | 67 | $ git clone ipfs::/ipfs/QmctS8mbpdQ1rgvS9SdFfJsoAE8s97FdXDHJQtobewAXKG myproject 68 | 69 | ## KNOWN BUGS 70 | 71 | The support for ipns references is completely untested at this point. 72 | While an initial clone from an ipns name should work, there is no code 73 | for updating the name with a new HEAD when pushing changes. 74 | 75 | ## DEBUGGING 76 | 77 | You can enable verbose debugging by setting `GIT_IPFS_LOGLEVEL=DEBUG` 78 | in your environment. 79 | 80 | ### LICENSE 81 | 82 | git-remote-ipfs -- a gitremote helper for ipfs 83 | Copyright (C) 2015 Lars Kellogg-Stedman 84 | 85 | This program is free software: you can redistribute it and/or modify 86 | it under the terms of the GNU General Public License as published by 87 | the Free Software Foundation, either version 3 of the License, or 88 | (at your option) any later version. 89 | 90 | This program is distributed in the hope that it will be useful, 91 | but WITHOUT ANY WARRANTY; without even the implied warranty of 92 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 93 | GNU General Public License for more details. 94 | 95 | You should have received a copy of the GNU General Public License 96 | along with this program. If not, see . 97 | 98 | -------------------------------------------------------------------------------- /git_remote_ipfs/importer.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import StringIO 3 | import fastimport.parser 4 | import fastimport.processor 5 | import logging 6 | 7 | from git_remote_ipfs.exc import * 8 | 9 | LOG = logging.getLogger(__name__) 10 | 11 | 12 | class ImportProcessor(fastimport.processor.ImportProcessor): 13 | def __init__(self, repo): 14 | self.repo = repo 15 | self.refs = set() 16 | super(ImportProcessor, self).__init__() 17 | 18 | def blob_handler(self, cmd): 19 | self.repo.add_blob(cmd) 20 | 21 | def commit_handler(self, cmd): 22 | self.repo.add_commit(cmd) 23 | self.refs.add(cmd.ref) 24 | 25 | def feature_handler(self, cmd): 26 | LOG.debug('ignoring feature %s', cmd) 27 | pass 28 | 29 | def reset_handler(self, cmd): 30 | self.repo.set_ref(cmd) 31 | self.refs.add(cmd.ref) 32 | 33 | def post_process(self): 34 | self.repo.export_complete() 35 | 36 | for ref in self.refs: 37 | LOG.debug('confirming %s', ref) 38 | print 'ok %s' % ref 39 | print 40 | 41 | 42 | class ExportProcessor(object): 43 | def __init__(self, repo): 44 | self.repo = repo 45 | self.marks = repo.marks 46 | self.api = self.repo.api 47 | self.exported = set() 48 | 49 | def parse_str(self, str): 50 | str = str.encode('utf-8') 51 | buf = StringIO.StringIO(str) 52 | parser = fastimport.parser.ImportParser(buf) 53 | return parser.iter_commands().next() 54 | 55 | def export_files(self, commit): 56 | for fspec in commit.file_iter: 57 | if hasattr(fspec, 'dataref'): 58 | hash = fspec.dataref 59 | data = self.parse_str(self.api.cat(hash)) 60 | mark = self.marks.add_rev(hash) 61 | fspec.dataref = mark 62 | data.mark = mark[1:] 63 | print data 64 | 65 | def resolve_parents(self, commit): 66 | if commit.from_: 67 | commit.from_ = self.marks.from_rev(commit.from_) 68 | 69 | merges = [] 70 | for merge in commit.merges: 71 | merges.append(self.marks.from_rev(merge)) 72 | 73 | commit.merges = merges 74 | 75 | def export_commit(self, ref, hash): 76 | LOG.debug('export_commit %s %s', ref, hash) 77 | if self.marks.is_marked(hash): 78 | return 79 | 80 | commit = self.parse_str(self.api.cat(hash)) 81 | if commit.from_: 82 | self.export_commit(ref, commit.from_) 83 | 84 | self.export_files(commit) 85 | self.resolve_parents(commit) 86 | 87 | self.marks.set_ref(ref, hash) 88 | mark = self.marks.add_rev(hash) 89 | commit.mark = mark[1:] 90 | 91 | if not ref in self.exported: 92 | print 'reset %s' % self.repo.adjust_ref(ref) 93 | self.exported.add(ref) 94 | 95 | commit.ref = self.repo.adjust_ref(commit.ref) 96 | print commit 97 | 98 | def export(self, ref): 99 | self.repo.refresh() 100 | self.exported = set() 101 | 102 | if ref == 'HEAD': 103 | ref = self.repo.default_branch 104 | 105 | try: 106 | start = self.marks.get_ref(ref) 107 | except KeyError: 108 | raise UnknownReference(ref) 109 | 110 | LOG.debug('exporting %s from %s', ref, start) 111 | self.export_commit(ref, start) 112 | mark = self.marks.from_rev(self.marks.refs[ref]) 113 | print 'reset %s' % self.repo.adjust_ref(ref) 114 | print 'from %s' % mark 115 | print 116 | -------------------------------------------------------------------------------- /git_remote_ipfs/remote.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import urlparse 4 | import logging 5 | import hashlib 6 | import json 7 | 8 | import ipfsApi 9 | import fastimport.commands 10 | 11 | import git_remote_ipfs.marks 12 | from git_remote_ipfs.exc import * 13 | 14 | LOG = logging.getLogger(__name__) 15 | 16 | repo_format_version = 2 17 | 18 | 19 | def quote_filename(f): 20 | return '"%s"' % f.replace('"', '\\"') 21 | 22 | 23 | class IPFSRemote (object): 24 | default_branch = 'refs/heads/master' 25 | 26 | def __init__(self, git_dir, alias, url, 27 | ipfs_gateway=None): 28 | self.git_dir = git_dir 29 | self.alias = alias 30 | self.url = url 31 | self.ipfs_gateway = ipfs_gateway 32 | self.temporary = False 33 | 34 | if self.alias == self.url: 35 | self.alias = hashlib.sha1(self.alias).hexdigest() 36 | self.temporary = True 37 | 38 | self.prefix = 'refs/ipfs/%s' % self.alias 39 | self.pvt_dir = os.path.join(git_dir, 'ipfs', self.alias) 40 | self.tocpath = os.path.join(self.pvt_dir, 'toc') 41 | self.markpath = os.path.join(self.pvt_dir, 'marks') 42 | self.repopath = os.path.join(self.pvt_dir, 'repo') 43 | self.marks = git_remote_ipfs.marks.Marks(self.tocpath) 44 | 45 | self.init_api() 46 | self.init_dir() 47 | self.init_path() 48 | self.init_refs() 49 | 50 | def __repr__(self): 51 | return '' % ( 52 | self.alias, self.url) 53 | 54 | __str__ = __repr__ 55 | 56 | def init_api(self): 57 | if self.ipfs_gateway is not None: 58 | if ':' in self.ipfs_gateway: 59 | host, port = self.ipfs_gateway.split(':') 60 | else: 61 | host, port = self.ipfs_gateway, 5001 62 | 63 | port = int(port) 64 | else: 65 | host = 'localhost' 66 | port = 5001 67 | 68 | self.api = ipfsApi.Client(host=host, port=port) 69 | 70 | # fail quickly if we're not able to contact ipfs 71 | self.id = self.api.id() 72 | LOG.debug('initialized ipfs api') 73 | 74 | def init_dir(self): 75 | if not os.path.isdir(self.pvt_dir): 76 | LOG.debug('creating directory %s', self.pvt_dir) 77 | os.makedirs(self.pvt_dir) 78 | 79 | def get_path_from_url(self): 80 | if not self.url: 81 | path = None 82 | elif self.url.startswith('ipfs://'): 83 | path = self.get_repo_from_url() 84 | elif self.url[:5] in ['/ipfs', '/ipns']: 85 | path = self.url 86 | else: 87 | raise InvalidURL(self.url) 88 | 89 | return path 90 | 91 | def init_path(self): 92 | self.path = self.get_path_from_url() 93 | if self.path: 94 | LOG.debug('found repo ipfs path = %s', self.path) 95 | 96 | def init_refs(self): 97 | # Do nothing if we were able to load refs from a 98 | # toc file earlier. 99 | if self.marks.refs: 100 | LOG.debug('found existing toc') 101 | return 102 | 103 | if not self.path: 104 | LOG.debug('unable to find repository path in ipfs') 105 | return 106 | 107 | self.refresh() 108 | 109 | def refresh(self): 110 | self.repo = self.api.cat(self.path) 111 | self.repo_check_version() 112 | self.repo_discover_refs() 113 | 114 | def repo_check_version(self): 115 | found_version = self.repo.get('version') 116 | if found_version != repo_format_version: 117 | raise IPFSError( 118 | 'incompatible repository format (want %s, found %s)' % ( 119 | repo_format_version, found_version)) 120 | 121 | def repo_discover_refs(self): 122 | for ref, hash in self.repo['refs'].items(): 123 | LOG.debug('found ref %s = %s', ref, hash) 124 | self.marks.set_ref(ref, hash) 125 | 126 | def get_repo_from_url(self): 127 | url = urlparse.urlparse(self.url) 128 | 129 | # resolve via ipns 130 | if url.netloc: 131 | return '/ipns/%s' % url.netloc 132 | else: 133 | path = '/ipfs/%s' % url.path[1:] 134 | 135 | return path 136 | 137 | def update(self): 138 | LOG.debug('updating repository toc in ipfs') 139 | self.repo = self.api.add_json({ 140 | 'version': repo_format_version, 141 | 'refs': self.marks.refs, 142 | }) 143 | 144 | with open(self.repopath, 'w') as fd: 145 | fd.write(self.repo + '\n') 146 | 147 | LOG.warn('new repository hash = %s', self.repo) 148 | 149 | if self.path.startswith('/ipns/%s' % self.id['ID']): 150 | LOG.info('publishing new hash to %s', self.path) 151 | self.api.name_publish(self.repo) 152 | 153 | def commit(self): 154 | LOG.debug('committing repository to disk') 155 | self.marks.store() 156 | 157 | def export_complete(self): 158 | self.commit() 159 | self.update() 160 | 161 | def cleanup(self): 162 | if self.temporary: 163 | if os.path.isdir(self.pvt_dir): 164 | LOG.debug('removing directory %s', self.pvt_dir) 165 | os.removedirs(self.pvt_dir) 166 | 167 | def add_blob(self, obj): 168 | mark = obj.id 169 | obj.mark = None 170 | hash = self.api.add_str(str(obj)) 171 | LOG.debug('added blob %s as %s', mark, hash) 172 | self.marks.add_mark(mark, hash) 173 | 174 | def set_ref(self, obj): 175 | mark = obj.from_ 176 | if mark: 177 | hash = self.marks.from_mark(mark) 178 | else: 179 | hash = None 180 | self.marks.set_ref(obj.ref, hash) 181 | 182 | def resolve_marks(self, commit): 183 | for fspec in commit.file_iter: 184 | if hasattr(fspec, 'dataref'): 185 | fspec.dataref = self.marks.from_mark(fspec.dataref) 186 | 187 | merges = [] 188 | for parent in commit.merges: 189 | parents.append(self.marks.from_mark(parent)) 190 | 191 | commit.merges = merges 192 | 193 | if commit.from_: 194 | commit.from_ = self.marks.from_mark(commit.from_) 195 | 196 | def add_commit(self, obj): 197 | self.resolve_marks(obj) 198 | mark = obj.id 199 | obj.mark = None 200 | hash = self.api.add_str(str(obj) + '\n') 201 | LOG.debug('added commit %s as %s', mark, hash) 202 | self.marks.add_mark(mark, hash) 203 | self.marks.set_ref(obj.ref, hash) 204 | 205 | def adjust_ref(self, ref): 206 | if ref.startswith('refs/heads/'): 207 | return '%s/heads/%s' % ( 208 | self.prefix, 209 | ref[len('refs/heads/'):]) 210 | --------------------------------------------------------------------------------