├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── config.yml ├── package ├── .rpmmacros ├── fpm │ └── deb │ │ └── .keep └── rpmbuild │ ├── RPMS │ └── noarch │ │ └── svn-to-github-1.16.0-0.noarch.rpm │ ├── SOURCES │ └── svn-to-github-n │ │ ├── README.md │ │ ├── config.yml │ │ └── svn-to-github │ ├── SPECS │ └── svn-to-github.spec │ └── re-build.sh ├── re-build.sh ├── svn-to-github ├── svn-to-github.spec └── test ├── bfg_test.sh ├── filter_test.sh ├── submodules_test.sh └── svn-to-github /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | # WebStorm 27 | *.iml 28 | 29 | # Directory-based project format: 30 | .idea/ 31 | .idea/workspace.xml 32 | **/.idea/workspace.xml 33 | 34 | # mac hidden files 35 | .DS_Store 36 | 37 | ##other 38 | #node_modules/ 39 | #bower_components/ 40 | .tmp 41 | #.sass-cache 42 | #builds/**/images/* 43 | #*.ogg 44 | #*.mp3 45 | #*.mp4 46 | #*.png 47 | #*.jpeg 48 | *.psd 49 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## Added Config 2 | * added config file to swap between GHE and GH APIs n downloading external links 3 | 4 | ## Ubuntu Deprecated 5 | * still possible to use with `skip-install` but no longer packaged 6 | 7 | ## Ubuntu release 8 | * Fixed --skip-install 9 | * Added Deb pkg 10 | 11 | ## First release 12 | * Fixed nested submodules of submodules 13 | * Fixed empty modules with nested submodules 14 | * Fixed / Deprecated blob to lfs conversion 15 | * Deprecated NEW_DIR 16 | * Fixed retry 17 | 18 | ## Beta 19 | * Add original Tree function / file to master 20 | * FIX this: 21 | ``` 22 | find: ‘’: No such file or directory 23 | basename: invalid option -- ':' 24 | Try 'basename --help' for more information. 25 | ./svn-to-github.sh: line 893: cd: -:: invalid option 26 | cd: usage: cd [-L|[-P [-e]]] [dir] 27 | dirname: invalid option -- ':' 28 | Try 'dirname --help' for more information. 29 | basename: missing operand 30 | ``` 31 | 32 | * FIX: missing trunk branch results in missing trunk dir and false tree 33 | * FIX RENAME 34 | * Deprecate $NAME.git dir? 35 | * BFG TWICE? WHY? 36 | * FIX: et_al submodules (BAD REGEX, renaming work-dir to svn-to-github caused et_al to be ignored) 37 | * Remove username@ from repo description 38 | * FIX: SVN -> git BRANCH CONVERSION 39 | * FIX: Adding submodules to $subdir/trunk instead of $subdir ???? 40 | * FIX: Fail to et_al_primary after (submodule add fails?) 41 | * FIX: sed replace [[:space:]] with _ in repo names `svn-prj` becomes `svn-prj` 42 | * FIX THIS: ```RA layer request failed: REPORT of '/svn/!svn/vcc/default': Could not read chunk size: Secure connection truncated (https://repo.net) at /usr/share/perl5/vendor_perl/Git/SVN/Ra.pm line 282.``` 43 | - caused by read permissions being set on svn host. `git svn clone` combines git svn init && git svn fetch, git svn init --no-minimize-url will or may resolve the permissions issue. 44 | * fix et_al_primary for primary containing trunk ```/app/svn-to-github/author_default.sh: No such file or directory``` 45 | * fix --new-name $RENAME 46 | * add gzip of $.github as final step before report 47 | * deprecate --no-org by making ntid default 48 | * fix `xrealloc: cannot allocate 18446744071562067968 bytes (667648 bytes allocated)` 49 | * Add true 1 to 1 directory tree conversion 50 | * Fix Retry `CLONE: No such file or directory` 51 | * Fix svn Credentials expect to allow multiple Credentials with same base URL or (each conversion uses a unique user to store creds in their $HOME dir) 52 | * sanitize --ignore and --authors to seek relative dir first so full path isnt necessary 53 | * fix retry function `error no such dir` 54 | 55 | 56 | # Alpha 57 | * fix ignore files for nested submodules to have relative ignores if unique attributes in svn are absent 58 | * Fix LFS loop to allow files with spaces 59 | * fix need to push primary 2x or just push it 2x (for some reason it prompts for creds twice)[is this still a problem?] 60 | * make service stop/start ? 61 | * make resume ? 62 | * Package 63 | * Ansible Playbook 64 | * verify ignore with spaces works, and ignore paths are preserved in submodules 65 | * no nested non repo directory that contains nested repos may begin with "-:-" 66 | * add repo to team option if team exists in org (could suck because team id's are just numeric, so users might not know what a team's id number is) 67 | * could just do a create team function at the same time importing the team members from the commit logs 68 | 69 | 70 | # TODO 71 | * Change default unit of lfs and bfg to Megabytes 72 | * Add default READM.md with useful links and info 73 | * deprecate $NEW_DIR 74 | * daemonize and make job queue with .conf file 75 | * Add sleep or wait to BFG to ensure push for slow bfg jobs 76 | * NEW ISSUE: `real_parent=$(resolve_name "$(get_parent "$module")")` resulting in "Submodule of svn_repo-" 77 | * FIX: 78 | ``` 79 | 80 | ``` 81 | * Unknown Issue: [possible fix here](http://stackoverflow.com/questions/4929674/what-can-i-do-with-git-corruption-due-to-a-missing-object) 82 | * Add conf file for pkg to use in playbook 83 | * Add ntid as default if -z org 84 | * FIX Rename 85 | * FIX nested repo designation chars results in the following error: 86 | ``` 87 | find: ‘’: No such file or directory 88 | basename: invalid option -- ':' 89 | Try 'basename --help' for more information. 90 | ``` 91 | * Add tar & gzip .github dir into "Jobs Archive" 92 | __validate every argument works__ 93 | * validated 94 | - svn 95 | - authors 96 | - org 97 | - ntid 98 | - token 99 | - ignore 100 | - work-dir 101 | - undo 102 | - no-tag-branches 103 | - no-branches 104 | - no-tags 105 | - no-org 106 | - safe 107 | - force 108 | 109 | * unvalidated 110 | + private 111 | + branches 112 | + tags 113 | + trunk 114 | + prefix 115 | + check-size 116 | + no-preserve-trunk 117 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # svn-to-github 2 | SVN to Github Tool for Public and Enterprise 3 | [![GitHub release](https://img.shields.io/github/release/Comcast/svn-to-github.svg)](https://github.com/Comcast/svn-to-github/releases/download/1.16.0/svn-to-github-1.16.0-0.noarch.rpm) [![GitHub issues](https://img.shields.io/github/issues/comcast/svn-to-github.svg)](https://github.com/Comcast/svn-to-github/issues) [![GitHub contributors](https://img.shields.io/github/contributors/comcast/svn-to-github.svg)](https://github.com/Comcast/svn-to-github/graphs/contributors) [![license](https://img.shields.io/github/license/comcast/svn-to-github.svg)](https://github.com/Comcast/svn-to-github/blob/master/LICENSE) 4 | 5 | ## Overview 6 | 7 | This routine will convert an SVN repo to github. If it has nested projects, those projects will become git submodules linked to their nearest ancestor. 8 | The git master branch is created from trunk, and a .gitinore file is added from svn properties. 9 | Trunk is left intact and untouched as a branch, unless `--no-branches` or `--no-preserve-trunk` is used, in which case only master will exist, which will be the original trunk converted 10 | There is a .gitignore file that is created from the original repo, and used as the first commit on the new master branch. 11 | Files and folders above the trunk are considered part of the repository and are added to the new master branch as the final conversion commit. 12 | Trunk directories are created in each master branch and point to the trunk branch to maintain the trunk directory tree for trunks, but can be disabled through option `--no-preserve-trunk`. 13 | Nested repos Under Trunk only exist in the master branch, and not in the trunk branch. 14 | LDAP is required to be configured in order for svn Author lookups to work when building the "SVN Authors File" for full names and email addresses, otherwise svn commits will not be linked to their users in github. 15 | 16 | __Conversion time could very greatly__ 17 | Conversion time factors are, directory size, commit length of branches/tags and trunk(s) and the greatest factor of all is missing branches or tags. When a branch is deleted from svn permanently, the conversion process must start over from the beginning of the commit chain for every occurrence. It is highly advisable one does not attempt to prune out unwanted branches or tags prior to conversion, but rather after because doing so will result in longer conversion time and errors in the logs. 18 | 19 | ## Features 20 | 21 | 1. Preserves the 'svn trunk' directory structure using git submodules [see caveats and limitations] 22 | 2. Handles naming conflicts in both the source repo and in github 23 | * Duplicate named projects in svn will be prefixed with their nearest ancestor's repo name 24 | * If a repo exists in github with the same name in the same namespace for a submodule, that submodule will be suffixed with the first non-conflicting integer 25 | 3. The program will fail immediately if the source repo name already exists in github in the namespace specified 26 | 4. Converts files of a specified size to git-lfs for tracking independently of git compression 27 | 5. Will remove deleted files of a specified size from git history 28 | 6. The original svn tree is saved in the parent repo's master branch in a hidden file called `.svn_tree` 29 | 7. All logs are preserved in the archive directory for each job 30 | 8. Ability remove repos from github that were converted as a batch using the `--undo` flag along with the `--svn`, `--ghuser`, and `--token` flags and also the `--org` flag if writen to a particular Org's namespace. Useful for sandboxing and redoing jobs with different parameters. Only the `*.json` files need to be preserved which are also saved in the archive dir. 31 | 9. Removes the ability to make revisions back to SVN, this is a 1 way conversion, repo syncing is disabled intentionally, revs made during a conversion will not be reflected, so prepare to wait. Conversion time is a function of `revs * branches||tags` 32 | 33 | #### Options 34 | 35 | All options from the command line must be given in the form of `--option=value` including the csv list ie: `--tags=tags,tag,releases` 36 | 37 | * __ghuser__ `[default: prompt ]` [Join Github github.com/join](https://github.com/join) 38 | * __token__ `[default: prompt ]` [github.com/settings/tokens/new](https://github.com/settings/tokens/new) 39 | * __svn-url__ `[default: prompt ]` The SVN Source URL 40 | * __svn-user__ `[default: null ]` The SVN Source Username 41 | * __svn-pass__ `[default: null ]` The SVN Source Password 42 | * __org__ `[default: prompt ]` [Organization Docs help.github.com/articles/about-organizations](https://help.github.com/articles/about-organizations/) 43 | * __no-org__ `[default: false ]` - don't create repos under an org, but rather under a user instead, caveat is repo(s) cannot already exist in user's primary Organization 44 | * __no-branches__ `[default: false ]` - do not convert branches, ignore all branches, should NOT be combined with `--no-preserve-trunk` 45 | * __no-tags__ `[default: false ]` - do not convert tags, ignore all tags 46 | * __no-tag-branches__ `[default: false ]` - do not allow tags to be branches, convert tags to releases only 47 | * __no-preserve-trunk__ `[default: false ]` - do not keep the trunk directory, so trunk becomes the root of master, should NOT be combined with `--no-branches` 48 | * __no-other-repos__ `[default: false ]` - do not create submodule repos, only create the primary repo but nest them if they already exist 49 | * __private__ `[default: false ]` - make all repo(s) private in github 50 | * __check-size__ `[default: false ]` - estimate the size on disk needed for conversion of non-standard-layouts, can be very time consuming 51 | * __safe__ `[default: false ]` - be prompted before all github repository creations or deletions when combined with the undo option 52 | * __install__ `[default: false ]` - dependency installation, Warning: May Fail if dependencies are not met. 53 | * __force__ `[default: false ]` - force the creation, will delete all data from previous run and possibly create duplicate submodule repos with suffixes in github 54 | * __undo__ `[default: false ]` - delete the github repos from a previous run, but preserve all log data 55 | * __new-name__ `[default: null ]` - rename your new repo name in github, required if name already exists in github under user or organization 56 | * __trunk__ `[default: null ]` - a csv list of alternative trunk dir names or specify just one 57 | * __tags__ `[default: null ]` - a csv list of alternative tag dir names or specify just one 58 | * __branches__ `[default: null ]` - a csv list of alternative branch dir names or specify just one 59 | * __svn-prefix__ `[default: null ]` - specify a prefix given to an otherwise standard-layout of svn directories 60 | * __authors__ `[default: null ]` - path of authors file rather than generate one, may not provide correct email address for users who do not conform to first_last@yourdomain.com 61 | * __ignore__ `[default: null ]` - path of import file for gitignore file rather than convert it from the existing svn ignore attributes 62 | * __lfs-limit__ `[default: 150 ]` - specify the large-file-size limit, 2 ~ 2 Megabytes, 100K ~ 100 Kilobytes, maximum in github is 50, [2-5] is optimal 63 | * __blob-limit__ `[default: 150 ]` - specify the max size of blobs allowed, 50 ~ 50 Megabytes, maximum in github is 50, < 50 is optimal 64 | * __work-dir__ `[default: /opt/svn-to-github ]` - to specify a working directory for creating files 65 | 66 | #### Requirements 67 | 68 | * Must be used on an Enterprise Linux 7 system (RHEL/CentOS/Fedora/Oracle) or with the prerequisites met use `--skip-install` __NOTE:__ git > 1.8.2 package on el6 does not exist thus requires src build, which is why EL7 is the requirement for the package install, early builds ran on Debian LTS but have since been discontinued 69 | * Must have account in github with permissions to create repos in organization 70 | * Must have root access or the package `git-svn and `git-lfs` already installed 71 | * __FOR ENTERPRISE ONLY__ without LDAP access, the script cannot query the directory for real names and email addresses of authors found in the svn history, therefore all historical commits will have generic & invalid names and email addresses Unknown_ghuser Missing_ghuser@yourdomain.com 72 | 73 | #### Software 74 | 75 | The software used to perform the tasks are common tools loaded from any bash 4.x shell, the following are installed required on EL systems: 76 | 77 | * git >= 1.8.2 78 | * bash >= 4.0 79 | * subversion 80 | * git-lfs (installed by first run) 81 | * git-svn 82 | * java-1.7.0-openjdk 83 | * expect 84 | * curl 85 | * gawk 86 | * findutils 87 | * coreutils 88 | * sed 89 | * yum 90 | * grep 91 | * procps 92 | * glibc-common 93 | * which 94 | * initscripts 95 | * tree 96 | 97 | ## Setup 98 | 99 | 1. Create a personal access API token with full permissions in github. [github.com/settings/tokens/new](https://github.com/settings/tokens/new) 100 | 101 | __TOKEN IS REQUIRED__ a password is not sufficient when using --org=organization 102 | 103 | * Install the git-lfs 104 | ` 105 | yum -y install https://packagecloud.io/github/git-lfs/packages/el/7/git-lfs-2.1.1-1.el7.x86_64.rpm/download 106 | ` 107 | * Install from RPM on EL7 108 | ` 109 | yum -y install https://github.com/comcast/svn-to-github/releases/download/v1.16.0/svn-to-github-1.16.0.el7.noarch.rpm 110 | ` 111 | 112 | ## Usage 113 | 114 | __USE NOHUP__ because your session could likely timeout before conversion completes. Or Feel free to send a pull request 115 | 116 | __Logs:__ `/opt/svn-to-git/$REPO/$REPO.log` 117 | __Jobs:__ `/opt/svn-to-git/archive` 118 | 119 | __Convert a Single Repo to Org:__ 120 | ` 121 | nohup svn-to-github --no-other-repos --org=svn2git --svn-url=http://svn.yourdomain.net/repos/svn_repo --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 & 122 | ` 123 | 124 | 125 | __Convert a Repo to Org:__ 126 | ` 127 | nohup svn-to-github --org=svn2git --svn-url=http://svn.yourdomain.net/repos/svn_repo --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 & 128 | ` 129 | 130 | __Convert a Repo to User:__ 131 | ` 132 | nohup svn-to-github --no-org --svn-url=http://svn.yourdomain.net/repos/svn_repo --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 & 133 | ` 134 | 135 | __Convert a Secure SVN Repo:__ 136 | ` 137 | nohup svn-to-github --no-org --svn-url=http://svn.yourdomain.net/repos/svn_repo --svn-user=svn2gituser --svn-pass='s3cret!' --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 & 138 | ` 139 | The password must be quoted in single quotes if a standard escape character is part of the string 140 | 141 | __Convert a SVN Repo to a Private Repo:__ 142 | ` 143 | nohup svn-to-github --private --svn-url=http://svn.yourdomain.net/repos/svn_repo --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 --org=aps & 144 | ` 145 | 146 | __Convert with lots of options:__ 147 | ` 148 | nohup svn-to-github --private --svn-url=http://svn.yourdomain.net/repos/svn_repo --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 --org=aps --check-size --lfs-limit=50 --blob-limit=50 --work-dir=/home/swizzley --new-name=myrepo --svn-user=svn2gituser --svn-pass=s3cret --no-tag-branches --no-preserve-trunk --no-other-repos --install --force & 149 | ` 150 | check-size can take a very long time, but a drop in the bucket for large repos with long chains. 151 | forcing will delete a previous checkout 152 | 153 | __Convert but provide pre-formatted Authors file:__ 154 | ` 155 | nohup svn-to-github --private --svn-url=http://svn.yourdomain.net/repos/svn_repo --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 --no-org --blob-limit=50 --work-dir=/home/swizzley --authors=/home/swizzley/svn_authors.txt & 156 | ` 157 | 158 | __Convert but provide Ignore file:__ 159 | ` 160 | nohup svn-to-github --private --svn-url=http://svn.yourdomain.net/repos/svn_repo --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 --no-org --blob-limit=50 --work-dir=/home/swizzley --ignore=/home/swizzley/svn_ignore.txt & 161 | ` 162 | Ignore file is Global to repo and all sub repos, if a conflict exists in dir levels for nested repos, modify the ignore attributes in SVN before hand or provide a file to use instead and resolve the conflicts after the conversion, anything ignored will not be added to the final git repo(s) 163 | 164 | __Convert but provide Unique Names for SVN Dirs:__ 165 | ` 166 | nohup svn-to-github --private --svn-url=http://svn.yourdomain.net/repos/svn_repo --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 --no-org --branches=patches,releases,Patch,Revs --tags=10-14-2003,Jun_2014 --svn-prefix=_svn- & 167 | ` 168 | This is in addition to, not in replacement of svn standard-layout directories (trunk,tags,branches) 169 | 170 | __Convert and manually approve every GHE repo name one by one:__ 171 | ` 172 | nohup svn-to-github --private --svn-url=http://svn.yourdomain.net/repos/svn_repo --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 --no-org --safe & 173 | ` 174 | Requires user to type 'yes' before every repo is created, and requires the checkout to be complete, (NOT Advisable for long commit chains, large repos over ssh due to timeouts) 175 | 176 | __Convert and rename the GHE Repo:__ 177 | ` 178 | nohup svn-to-github --private --svn-url=http://svn.yourdomain.net/repos/svn_repo --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 --no-org --new-name=myrepo & 179 | ` 180 | This will all rename the Prefix of all submodules nested inside 181 | 182 | __Undo Conversion by DELETING ALL REPOS converted in GHE under Org:__ 183 | ` 184 | svn-to-github --svn-url=http://svn.yourdomain.net/repos/svn_repo --org=aps --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 --undo 185 | ` 186 | 187 | __Undo Conversion by DELETING ALL REPOS converted in GHE under User:__ 188 | ` 189 | svn-to-github --svn-url=http://svn.yourdomain.net/repos/svn_repo --no-org --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 --undo 190 | ` 191 | 192 | ## Log-Analysis 193 | 194 | Under each conversion job is a log directory, inside that directory shows the real-time status of any particular stage of the job. These logs along with the json files used to create and delete repos with --undo in GHE are archived in the working-dir's archive folder with the name and date that the job finished. The `report.log` file will show any failures, conversion failures typically occur when the commit chain in svn is broken and cannot be resolved automatically through the git svn clone process. These failures must be resolved by hand afterwards, typically by specifying the Rev number in SVN or by ignoring what was broken. There is a `retry.log` that shows when a process failed for some reason, and that process is given three attempts to work before being logged as a failure. 195 | 196 | ## Post-Conversion-Git-Usage 197 | 198 | __Clone the Full Repo:__ 199 | ` 200 | git clone --recursive $REPO_URL 201 | ` 202 | 203 | __Clone a particular branch Only:__ 204 | ` 205 | git clone -b $BRANCH_NAME $REPO_URL 206 | ` 207 | 208 | __Update all submodules:__ 209 | ` 210 | git submodule update --recursive 211 | ` 212 | 213 | ## Caveats and Limitations for non-standard svn repo layouts 214 | 215 | 1. Branches only get converted if they exist in a case insensitive dir `*/branches` by default 216 | * alternative branch directories can be specified as a csv list using the Argument `--branches=/path/to/file.csv` 217 | * or specify a single alternative with the name of the label, like this: `--branches=some_label` 218 | * skip branches entirely with `--no-branches` any alternative branch label must be specified with `--branches` or the contents will be added as files under master 219 | * original branch directories are __NOT PRESERVED__ 220 | 2. Tags get converted to tags and branches by default 221 | * skip converting tags to branches with `--tags-not-branches` 222 | * skip tags entirely with `--no-tags` 223 | * original tag directories are __NOT PRESERVED__ 224 | 3. Spaces in repo names will be replaced with underscores to avoid ugly unicode in github 225 | 4. Trunks nested under tags or branches are __NOT PRESERVED__ 226 | 5. Empty directories are __NOT PRESERVED__ 227 | 6. Large files are tracked using `git lfs`, which uses patterns, those patterns are based on the file extension of a file equal to or over the `--lfs-limit`, if that file does not have an extension, the full name of the file is used. 228 | 7. Directory tree overall is preserved except directories for tags and branches which become git tags and git branches, and trunk directory is only preserved if it was named trunk, otherwise one can use the `--no-preserve-trunk` option to accept the converted directory structure without any trunk directories 229 | 230 | ### Routine 231 | 232 | 1. Collect svn source repo location; use the svn argument to skip prompt eg: `--svn=https://server/svn/myrepo` 233 | 2. Creates a work dir in `/opt/svn-to-github/$myrepo`; but one can be specified using `--work-dir=/opt/myrepo` 234 | 3. Collect and or validate authentication to github; specify the org, ghuser and token arguments to skip prompt eg: `--org=myorg --ghuser=username --token=password` 235 | 4. Checks to see if svn repo name exists in github; if it does it fails and prompts to run again using the argument `--rename=my-new-repo-name` 236 | 5. Checks for `--undo` flag to see if user wants to reverse the process from a previous run, in which case it will remove any repos that it created in github and exit afterwards 237 | 6. Checks that the user running is `root`; if not it will check if requirements exist; otherwise it will exit 238 | 7. Checks to see if requirements are installed and permissions in work-dir are met 239 | 8. Optionally checks disk space and repo's size to estimate if adequate disk space exists in work-dir mount (off by default because checking a remote location is slow) 240 | 9. Checkout local copy of svn repo URL 241 | 10. Generates authors file from svn log to maintain commit history in github, [optionally] using `---authors=some_file` skips this step 242 | 11. Checks for nested repos 243 | 12. Creates all repo(s) in github, nested repos are created with a prefix of their parent repo's name in github, any duplicates will have a suffix number 244 | 13. Begins cloning any nested repos individually and concurrently 245 | 14. Nested repos that have finished cloning are then converted completely with branches and tags as well as concurrently as soon as they are finished cloning 246 | 15. Any parent repo that has finished must wait for all child repos to finish converting before it can begin converting so that all submodules will be added prior to completion 247 | 16. The top scope/primary parent repo is then created or converted and all nested repos are added to it as submodules 248 | 249 | ## Useful_Links 250 | 251 | * [git svn crash course](http://git.or.cz/course/svn.html) 252 | * [git-svn Manual](https://git-scm.com/docs/git-svn) 253 | * [git blob cleaner](https://rtyley.github.io/bfg-repo-cleaner) 254 | * [git-lfs extension](https://github.com/github/git-lfs) 255 | * [git submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules) 256 | * [github API](https://developer.github.com/v3/) 257 | 258 | 259 | ## TODO 260 | 261 | * Add full directory tree mirroring support for branches and tags 262 | -------------------------------------------------------------------------------- /config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | ## Global Settings 4 | DATA: 5 | DIR: "/opt/svn-to-github/data" 6 | ARCHIVE: "$DATA_DIR/archive" 7 | GITHUB: "github.com" 8 | API: "https://$GITHUB" 9 | # API: "https://$GITHUB/api/v3" #Enterprise Only 10 | BFG: 11 | DOWNLOAD: "http://repo1.maven.org/maven2/com/madgag/bfg/1.12.15/bfg-1.12.15.jar" 12 | LIMIT: "50" 13 | LFS: 14 | DOWNLOAD: "https://packagecloud.io/github/git-lfs/packages/el/7/git-lfs-2.1.1-1.el7.x86_64.rpm/download" 15 | LIMIT: "50" 16 | DEFAULT: 17 | DOMAIN: "yourdomain.com" 18 | -------------------------------------------------------------------------------- /package/.rpmmacros: -------------------------------------------------------------------------------- 1 | %packager Dustin_Morgan 2 | %_topdir %(echo $HOME)/rpmbuild 3 | %_tmppath %(echo $HOME)/rpmbuild/tmp 4 | %_smp_mflags -j3 5 | %__arch_install_post /usr/lib/rpm/check-rpaths /usr/lib/rpm/check-buildroot 6 | -------------------------------------------------------------------------------- /package/fpm/deb/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Comcast/svn-to-github/9a25e251bb99cadc0e3c4108b3442ee2221d91b7/package/fpm/deb/.keep -------------------------------------------------------------------------------- /package/rpmbuild/RPMS/noarch/svn-to-github-1.16.0-0.noarch.rpm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Comcast/svn-to-github/9a25e251bb99cadc0e3c4108b3442ee2221d91b7/package/rpmbuild/RPMS/noarch/svn-to-github-1.16.0-0.noarch.rpm -------------------------------------------------------------------------------- /package/rpmbuild/SOURCES/svn-to-github-n/README.md: -------------------------------------------------------------------------------- 1 | # svn-to-github 2 | SVN to Github Tool for Public and Enterprise 3 | [![GitHub release](https://img.shields.io/github/release/qubyte/rubidium-v1.16.0-blue.svg)](https://github.com/comcast/svn-to-github/releases/tag/v1.16.0) [![GitHub issues](https://img.shields.io/github/issues/badges/shields.svg)](https://github.com/Comcast/svn-to-github/issues) [![GitHub contributors](https://img.shields.io/github/contributors/cdnjs/cdnjs.svg)](https://github.com/Comcast/svn-to-github/graphs/contributors) [![license](https://img.shields.io/github/license/mashape/apistatus.svg)](https://github.com/Comcast/svn-to-github/blob/master/LICENSE) 4 | 5 | ## Overview 6 | 7 | This routine will convert an SVN repo to github. If it has nested projects, those projects will become git submodules linked to their nearest ancestor. 8 | The git master branch is created from trunk, and a .gitinore file is added from svn properties. 9 | Trunk is left intact and untouched as a branch, unless `--no-branches` or `--no-preserve-trunk` is used, in which case only master will exist, which will be the original trunk converted 10 | There is a .gitignore file that is created from the original repo, and used as the first commit on the new master branch. 11 | Files and folders above the trunk are considered part of the repository and are added to the new master branch as the final conversion commit. 12 | Trunk directories are created in each master branch and point to the trunk branch to maintain the trunk directory tree for trunks, but can be disabled through option `--no-preserve-trunk`. 13 | Nested repos Under Trunk only exist in the master branch, and not in the trunk branch. 14 | LDAP is required to be configured in order for svn Author lookups to work when building the "SVN Authors File" for full names and email addresses, otherwise svn commits will not be linked to their users in github. 15 | 16 | __Conversion time could very greatly__ 17 | Conversion time factors are, directory size, commit length of branches/tags and trunk(s) and the greatest factor of all is missing branches or tags. When a branch is deleted from svn permanently, the conversion process must start over from the beginning of the commit chain for every occurrence. It is highly advisable one does not attempt to prune out unwanted branches or tags prior to conversion, but rather after because doing so will result in longer conversion time and errors in the logs. 18 | 19 | ## Features 20 | 21 | 1. Preserves the 'svn trunk' directory structure using git submodules [see caveats and limitations] 22 | 2. Handles naming conflicts in both the source repo and in github 23 | * Duplicate named projects in svn will be prefixed with their nearest ancestor's repo name 24 | * If a repo exists in github with the same name in the same namespace for a submodule, that submodule will be suffixed with the first non-conflicting integer 25 | 3. The program will fail immediately if the source repo name already exists in github in the namespace specified 26 | 4. Converts files of a specified size to git-lfs for tracking independently of git compression 27 | 5. Will remove deleted files of a specified size from git history 28 | 6. The original svn tree is saved in the parent repo's master branch in a hidden file called `.svn_tree` 29 | 7. All logs are preserved in the archive directory for each job 30 | 8. Ability remove repos from github that were converted as a batch using the `--undo` flag along with the `--svn`, `--ghuser`, and `--token` flags and also the `--org` flag if writen to a particular Org's namespace. Useful for sandboxing and redoing jobs with different parameters. Only the `*.json` files need to be preserved which are also saved in the archive dir. 31 | 9. Removes the ability to make revisions back to SVN, this is a 1 way conversion, repo syncing is disabled intentionally, revs made during a conversion will not be reflected, so prepare to wait. Conversion time is a function of `revs * branches||tags` 32 | 33 | #### Options 34 | 35 | All options from the command line must be given in the form of `--option=value` including the csv list ie: `--tags=tags,tag,releases` 36 | 37 | * __ghuser__ `[default: prompt ]` [Join Github github.com/join](https://github.com/join) 38 | * __token__ `[default: prompt ]` [github.com/settings/tokens/new](https://github.com/settings/tokens/new) 39 | * __svn-url__ `[default: prompt ]` The SVN Source URL 40 | * __svn-user__ `[default: null ]` The SVN Source Username 41 | * __svn-pass__ `[default: null ]` The SVN Source Password 42 | * __org__ `[default: prompt ]` [Organization Docs help.github.com/articles/about-organizations](https://help.github.com/articles/about-organizations/) 43 | * __no-org__ `[default: false ]` - don't create repos under an org, but rather under a user instead, caveat is repo(s) cannot already exist in user's primary Organization 44 | * __no-branches__ `[default: false ]` - do not convert branches, ignore all branches, should NOT be combined with `--no-preserve-trunk` 45 | * __no-tags__ `[default: false ]` - do not convert tags, ignore all tags 46 | * __no-tag-branches__ `[default: false ]` - do not allow tags to be branches, convert tags to releases only 47 | * __no-preserve-trunk__ `[default: false ]` - do not keep the trunk directory, so trunk becomes the root of master, should NOT be combined with `--no-branches` 48 | * __no-other-repos__ `[default: false ]` - do not create submodule repos, only create the primary repo but nest them if they already exist 49 | * __private__ `[default: false ]` - make all repo(s) private in github 50 | * __check-size__ `[default: false ]` - estimate the size on disk needed for conversion of non-standard-layouts, can be very time consuming 51 | * __safe__ `[default: false ]` - be prompted before all github repository creations or deletions when combined with the undo option 52 | * __install__ `[default: false ]` - dependency installation, Warning: May Fail if dependencies are not met. 53 | * __force__ `[default: false ]` - force the creation, will delete all data from previous run and possibly create duplicate submodule repos with suffixes in github 54 | * __undo__ `[default: false ]` - delete the github repos from a previous run, but preserve all log data 55 | * __new-name__ `[default: null ]` - rename your new repo name in github, required if name already exists in github under user or organization 56 | * __trunk__ `[default: null ]` - a csv list of alternative trunk dir names or specify just one 57 | * __tags__ `[default: null ]` - a csv list of alternative tag dir names or specify just one 58 | * __branches__ `[default: null ]` - a csv list of alternative branch dir names or specify just one 59 | * __svn-prefix__ `[default: null ]` - specify a prefix given to an otherwise standard-layout of svn directories 60 | * __authors__ `[default: null ]` - path of authors file rather than generate one, may not provide correct email address for users who do not conform to first_last@cable.comcast.com 61 | * __ignore__ `[default: null ]` - path of import file for gitignore file rather than convert it from the existing svn ignore attributes 62 | * __lfs-limit__ `[default: 150 ]` - specify the large-file-size limit, 2 ~ 2 Megabytes, 100K ~ 100 Kilobytes, maximum in github is 50, [2-5] is optimal 63 | * __blob-limit__ `[default: 150 ]` - specify the max size of blobs allowed, 50 ~ 50 Megabytes, maximum in github is 50, < 50 is optimal 64 | * __work-dir__ `[default: /opt/svn-to-github ]` - to specify a working directory for creating files 65 | 66 | #### Requirements 67 | 68 | * Must be used on an Enterprise Linux 7 system (RHEL/CentOS/Fedora/Oracle) or with the prerequisites met use `--skip-install` __NOTE:__ git > 1.8.2 package on el6 does not exist thus requires src build, which is why EL7 is the requirement for the package install, early builds ran on Debian LTS but have since been discontinued 69 | * Must have account in github with permissions to create repos in organization 70 | * Must have root access or the package `git-svn and `git-lfs` already installed 71 | * __FOR ENTERPRISE ONLY__ without LDAP access, the script cannot query the directory for real names and email addresses of authors found in the svn history, therefore all historical commits will have generic & invalid names and email addresses Unknown_ghuser Missing_ghuser@yourdomain.com 72 | 73 | #### Software 74 | 75 | The software used to perform the tasks are common tools loaded from any bash 4.x shell, the following are installed required on EL systems: 76 | 77 | * git >= 1.8.2 78 | * bash >= 4.0 79 | * subversion 80 | * git-lfs (installed by first run) 81 | * git-svn 82 | * java-1.7.0-openjdk 83 | * expect 84 | * curl 85 | * gawk 86 | * findutils 87 | * coreutils 88 | * sed 89 | * yum 90 | * grep 91 | * procps 92 | * glibc-common 93 | * which 94 | * initscripts 95 | * tree 96 | 97 | ## Setup 98 | 99 | 1. Create a personal access API token with full permissions in github. [github.com/settings/tokens/new](https://github.com/settings/tokens/new) 100 | 101 | __TOKEN IS REQUIRED__ a password is not sufficient when using --org=organization 102 | 103 | * Install the git-lfs 104 | ` 105 | yum -y install https://packagecloud.io/github/git-lfs/packages/el/7/git-lfs-2.1.1-1.el7.x86_64.rpm/download 106 | ` 107 | * Install from RPM on EL7 108 | ` 109 | yum -y install https://github.com/comcast/svn-to-github/releases/download/v1.16.0/svn-to-github-1.16.0.el7.noarch.rpm 110 | ` 111 | 112 | ## Usage 113 | 114 | __USE NOHUP__ because your session could likely timeout before conversion completes. Or Feel free to send a pull request 115 | 116 | __Logs:__ `/opt/svn-to-git/$REPO/$REPO.log` 117 | __Jobs:__ `/opt/svn-to-git/archive` 118 | 119 | __Convert a Single Repo to Org:__ 120 | ` 121 | nohup svn-to-github --no-other-repos --org=svn2git --svn-url=http://svn.yourdomain.net/repos/svn_repo --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 & 122 | ` 123 | 124 | 125 | __Convert a Repo to Org:__ 126 | ` 127 | nohup svn-to-github --org=svn2git --svn-url=http://svn.yourdomain.net/repos/svn_repo --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 & 128 | ` 129 | 130 | __Convert a Repo to User:__ 131 | ` 132 | nohup svn-to-github --no-org --svn-url=http://svn.yourdomain.net/repos/svn_repo --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 & 133 | ` 134 | 135 | __Convert a Secure SVN Repo:__ 136 | ` 137 | nohup svn-to-github --no-org --svn-url=http://svn.yourdomain.net/repos/svn_repo --svn-user=svn2gituser --svn-pass='s3cret!' --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 & 138 | ` 139 | The password must be quoted in single quotes if a standard escape character is part of the string 140 | 141 | __Convert a SVN Repo to a Private Repo:__ 142 | ` 143 | nohup svn-to-github --private --svn-url=http://svn.yourdomain.net/repos/svn_repo --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 --org=aps & 144 | ` 145 | 146 | __Convert with lots of options:__ 147 | ` 148 | nohup svn-to-github --private --svn-url=http://svn.yourdomain.net/repos/svn_repo --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 --org=aps --check-size --lfs-limit=50 --blob-limit=50 --work-dir=/home/swizzley --new-name=myrepo --svn-user=svn2gituser --svn-pass=s3cret --no-tag-branches --no-preserve-trunk --no-other-repos --install --force & 149 | ` 150 | check-size can take a very long time, but a drop in the bucket for large repos with long chains. 151 | forcing will delete a previous checkout 152 | 153 | __Convert but provide pre-formatted Authors file:__ 154 | ` 155 | nohup svn-to-github --private --svn-url=http://svn.yourdomain.net/repos/svn_repo --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 --no-org --blob-limit=50 --work-dir=/home/swizzley --authors=/home/swizzley/svn_authors.txt & 156 | ` 157 | 158 | __Convert but provide Ignore file:__ 159 | ` 160 | nohup svn-to-github --private --svn-url=http://svn.yourdomain.net/repos/svn_repo --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 --no-org --blob-limit=50 --work-dir=/home/swizzley --ignore=/home/swizzley/svn_ignore.txt & 161 | ` 162 | Ignore file is Global to repo and all sub repos, if a conflict exists in dir levels for nested repos, modify the ignore attributes in SVN before hand or provide a file to use instead and resolve the conflicts after the conversion, anything ignored will not be added to the final git repo(s) 163 | 164 | __Convert but provide Unique Names for SVN Dirs:__ 165 | ` 166 | nohup svn-to-github --private --svn-url=http://svn.yourdomain.net/repos/svn_repo --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 --no-org --branches=patches,releases,Patch,Revs --tags=10-14-2003,Jun_2014 --svn-prefix=_svn- & 167 | ` 168 | This is in addition to, not in replacement of svn standard-layout directories (trunk,tags,branches) 169 | 170 | __Convert and manually approve every GHE repo name one by one:__ 171 | ` 172 | nohup svn-to-github --private --svn-url=http://svn.yourdomain.net/repos/svn_repo --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 --no-org --safe & 173 | ` 174 | Requires user to type 'yes' before every repo is created, and requires the checkout to be complete, (NOT Advisable for long commit chains, large repos over ssh due to timeouts) 175 | 176 | __Convert and rename the GHE Repo:__ 177 | ` 178 | nohup svn-to-github --private --svn-url=http://svn.yourdomain.net/repos/svn_repo --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 --no-org --new-name=myrepo & 179 | ` 180 | This will all rename the Prefix of all submodules nested inside 181 | 182 | __Undo Conversion by DELETING ALL REPOS converted in GHE under Org:__ 183 | ` 184 | svn-to-github --svn-url=http://svn.yourdomain.net/repos/svn_repo --org=aps --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 --undo 185 | ` 186 | 187 | __Undo Conversion by DELETING ALL REPOS converted in GHE under User:__ 188 | ` 189 | svn-to-github --svn-url=http://svn.yourdomain.net/repos/svn_repo --no-org --ghuser=swizzley --token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 --undo 190 | ` 191 | 192 | ## Log-Analysis 193 | 194 | Under each conversion job is a log directory, inside that directory shows the real-time status of any particular stage of the job. These logs along with the json files used to create and delete repos with --undo in GHE are archived in the working-dir's archive folder with the name and date that the job finished. The `report.log` file will show any failures, conversion failures typically occur when the commit chain in svn is broken and cannot be resolved automatically through the git svn clone process. These failures must be resolved by hand afterwards, typically by specifying the Rev number in SVN or by ignoring what was broken. There is a `retry.log` that shows when a process failed for some reason, and that process is given three attempts to work before being logged as a failure. 195 | 196 | ## Post-Conversion-Git-Usage 197 | 198 | __Clone the Full Repo:__ 199 | ` 200 | git clone --recursive $REPO_URL 201 | ` 202 | 203 | __Clone a particular branch Only:__ 204 | ` 205 | git clone -b $BRANCH_NAME $REPO_URL 206 | ` 207 | 208 | __Update all submodules:__ 209 | ` 210 | git submodule update --recursive 211 | ` 212 | 213 | ## Caveats and Limitations for non-standard svn repo layouts 214 | 215 | 1. Branches only get converted if they exist in a case insensitive dir `*/branches` by default 216 | * alternative branch directories can be specified as a csv list using the Argument `--branches=/path/to/file.csv` 217 | * or specify a single alternative with the name of the label, like this: `--branches=some_label` 218 | * skip branches entirely with `--no-branches` any alternative branch label must be specified with `--branches` or the contents will be added as files under master 219 | * original branch directories are __NOT PRESERVED__ 220 | 2. Tags get converted to tags and branches by default 221 | * skip converting tags to branches with `--tags-not-branches` 222 | * skip tags entirely with `--no-tags` 223 | * original tag directories are __NOT PRESERVED__ 224 | 3. Spaces in repo names will be replaced with underscores to avoid ugly unicode in github 225 | 4. Trunks nested under tags or branches are __NOT PRESERVED__ 226 | 5. Empty directories are __NOT PRESERVED__ 227 | 6. Large files are tracked using `git lfs`, which uses patterns, those patterns are based on the file extension of a file equal to or over the `--lfs-limit`, if that file does not have an extension, the full name of the file is used. 228 | 7. Directory tree overall is preserved except directories for tags and branches which become git tags and git branches, and trunk directory is only preserved if it was named trunk, otherwise one can use the `--no-preserve-trunk` option to accept the converted directory structure without any trunk directories 229 | 230 | ### Routine 231 | 232 | 1. Collect svn source repo location; use the svn argument to skip prompt eg: `--svn=https://server/svn/myrepo` 233 | 2. Creates a work dir in `/opt/svn-to-github/$myrepo`; but one can be specified using `--work-dir=/opt/myrepo` 234 | 3. Collect and or validate authentication to github; specify the org, ghuser and token arguments to skip prompt eg: `--org=myorg --ghuser=username --token=password` 235 | 4. Checks to see if svn repo name exists in github; if it does it fails and prompts to run again using the argument `--rename=my-new-repo-name` 236 | 5. Checks for `--undo` flag to see if user wants to reverse the process from a previous run, in which case it will remove any repos that it created in github and exit afterwards 237 | 6. Checks that the user running is `root`; if not it will check if requirements exist; otherwise it will exit 238 | 7. Checks to see if requirements are installed and permissions in work-dir are met 239 | 8. Optionally checks disk space and repo's size to estimate if adequate disk space exists in work-dir mount (off by default because checking a remote location is slow) 240 | 9. Checkout local copy of svn repo URL 241 | 10. Generates authors file from svn log to maintain commit history in github, [optionally] using `---authors=some_file` skips this step 242 | 11. Checks for nested repos 243 | 12. Creates all repo(s) in github, nested repos are created with a prefix of their parent repo's name in github, any duplicates will have a suffix number 244 | 13. Begins cloning any nested repos individually and concurrently 245 | 14. Nested repos that have finished cloning are then converted completely with branches and tags as well as concurrently as soon as they are finished cloning 246 | 15. Any parent repo that has finished must wait for all child repos to finish converting before it can begin converting so that all submodules will be added prior to completion 247 | 16. The top scope/primary parent repo is then created or converted and all nested repos are added to it as submodules 248 | 249 | ## Useful_Links 250 | 251 | * [git svn crash course](http://git.or.cz/course/svn.html) 252 | * [git-svn Manual](https://git-scm.com/docs/git-svn) 253 | * [git blob cleaner](https://rtyley.github.io/bfg-repo-cleaner) 254 | * [git-lfs extension](https://github.com/github/git-lfs) 255 | * [git submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules) 256 | * [github API](https://developer.github.com/v3/) 257 | 258 | 259 | ## TODO 260 | 261 | * Add full directory tree mirroring support for branches and tags 262 | -------------------------------------------------------------------------------- /package/rpmbuild/SOURCES/svn-to-github-n/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | ## Global Settings 4 | DATA: 5 | DIR: "/opt/svn-to-github/data" 6 | ARCHIVE: "$DATA_DIR/archive" 7 | GITHUB: "github.com" 8 | API: "https://$GITHUB" 9 | # API: "https://$GITHUB/api/v3" #Enterprise Only 10 | BFG: 11 | DOWNLOAD: "http://repo1.maven.org/maven2/com/madgag/bfg/1.12.15/bfg-1.12.15.jar" 12 | LIMIT: "50" 13 | LFS: 14 | DOWNLOAD: "https://packagecloud.io/github/git-lfs/packages/el/7/git-lfs-2.1.1-1.el7.x86_64.rpm/download" 15 | LIMIT: "50" 16 | DEFAULT: 17 | DOMAIN: "yourdomain.com" 18 | -------------------------------------------------------------------------------- /package/rpmbuild/SOURCES/svn-to-github-n/svn-to-github: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Name: SVN2GitHub 4 | # Author: Dustin Morgan 5 | # Date: Jun 13, 2017 6 | # Purpose: Ease the conversion of SVN to Git and use git submodules for 7 | # non-standard svn layouts with nested projects 8 | # 9 | #shellcheck svn-to-github --exclude=SC2143,SC2155,SC2001,SC2162,SC2002,SC2013,SC2116,SC2005,SC2003,SC2035,SC2010,SC2164,SC2054,SC2140,SC2034,SC2068,SC2145,SC2062,SC2063,SC2012,SC2128,SC2188,SC2129,SC2126 10 | 11 | #ARGS 12 | while test $# -gt 0; do 13 | case "$1" in 14 | -h|--help) 15 | echo "Run this with answer flags for [required] options to bypass interactive mode when converting an svn repo to a github repo." 16 | echo "" 17 | echo "options:" 18 | echo "-h, --help" 19 | echo "--ghuser=swizzley [required]: the ghuser used to login to github" 20 | echo "--token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 [required]: the github user authorization token, setup here: https://github.com/settings/tokens" 21 | echo "--svn-url=https://svn.yourdomain.net/my_repo [required]: to specify the source svn repo" 22 | echo "--svn-user=username [optional]: to specify the source svn repo username" 23 | echo "--svn-pass=password [optional]: to specify the source svn repo password, if special characters are used quote password in single-quotes" 24 | echo "--org=my_org [required option A]: the github organization to create the repo(s) in" 25 | echo "--no-org [required option B]: don't create repos under an org, but rather under a user instead, caveat is repo(s) cannot already exist in user's primary Organization" 26 | echo "--no-branches [optional]: do not convert branches, ignore all branches" 27 | echo "--no-tags [optional]: do not convert tags, ignore all tags" 28 | echo "--no-tag-branches [optional]: do not allow tags to be branches, convert tags to releases only" 29 | echo "--no-preserve-trunk [optional]: do not keep the trunk directory, so trunk becomes the root of master" 30 | echo "--no-other-repos [optional]: do not create submodule repos, only create the primary repo but nest them if they already exist" 31 | echo "--new-name=repo_name [required or optional]: rename your new repo name in github, required if name already exists in github under user or organization, whichever is specified by option A or B above" 32 | echo "--lfs-limit=50 [default=50, max=50]: specify the large-file-size limit, --lfs-limit=2 ~ 2 Megabytes, maximum in github is 50, default is 50, 2-5 is optimal" 33 | echo "--blob-limit=50 [default=50, max=50]: specify the max size of blobs allowed, --blob-limit=50 ~ 50 Megabytes, maximum in github.com is 50M, default is 50, < 50 is optimal" 34 | echo "--authors=/path/to/authors_file [optional]: to import an authors file rather than generate one, may not provide correct email address for users who do not conform to first_last@yourdomain.com" 35 | echo "--ignore=/path/to/ignore_file [optional]: to import the gitignore file rather than convert it from the existing svn ignore attributes" #TODO 36 | echo "--work-dir=/path/to/store/files [optional]: to specify a working directory for creating files" 37 | echo "--branches=label1,label2 or --branches=custom_label [optional]: a csv list of alternative branch dir names or specify just one" 38 | echo "--trunk=label1,label2 or --trunk=custom_label [optional]: a csv list of alternative trunk dir names or specify just one" 39 | echo "--tags=label1,label2 or --tags=custom_label [optional]: a csv list of alternative tag dir names or specify just one" 40 | echo "--svn-prefix=svn- [optional]: specify a prefix given to an otherwise standard-layout of svn directories, '--svn-prefix=svn-' == '--branch=svn-branches --trunk=svn-trunk --tags=svn-tags'" 41 | echo "--private [optional]: make all repo(s) private in github" 42 | echo "--check-size [optional]: estimate the size on disk needed for conversion of non-standard-layouts, can be very time consuming" 43 | echo "--safe [optional]: be prompted before all github repository creations or deletions when combined with --undo" 44 | echo "--install [optional]: dependency check & installation, Warning: Will Fail if dependencies are not met. Deps: git-svn, java-7, git-lfs, bfg-repo-cleaner ver. >= 1.12.5 " 45 | echo "--force [warning!]: force the creation, will delete all data from previous run and possibly create duplicate submodule repos with suffixes in github, it will NOT allow bypassing of --new-name for the primary repo" 46 | echo "--undo [warning!]: delete the github repos from a previous run, but preserve all log data" 47 | exit 48 | ;; 49 | --authors*) 50 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 51 | export AUTHORS_FILE=$(echo "$1" | sed -e 's/^[^=]*=//g') 52 | export IMPORT_AUTHORS=false 53 | shift 54 | ;; 55 | --svn-url*) 56 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 57 | export SVN_REPO=$(echo "$1" | sed -e 's/^[^=]*=//g') 58 | shift 59 | ;; 60 | --org*) 61 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 62 | export ORG=$(echo "$1" | sed -e 's/^[^=]*=//g') 63 | shift 64 | ;; 65 | --ghuser*) 66 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 67 | export NTID=$(echo "$1" | sed -e 's/^[^=]*=//g') 68 | shift 69 | ;; 70 | --token*) 71 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 72 | export TOKEN=$(echo "$1" | sed -e 's/^[^=]*=//g') 73 | shift 74 | ;; 75 | --new-name*) 76 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 77 | export RENAME=$(echo "$1" | sed -e 's/^[^=]*=//g') 78 | shift 79 | ;; 80 | --ignore*) 81 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 82 | export IGNORE_FILE=$(echo "$1" | sed -e 's/^[^=]*=//g') 83 | export IMPORT_IGNORE=false 84 | shift 85 | ;; 86 | --work-dir*) 87 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 88 | export DATA_DIR=$(echo "$1" | sed -e 's/^[^=]*=//g') 89 | shift 90 | ;; 91 | --force*) 92 | export FORCE=true 93 | shift 94 | ;; 95 | --undo*) 96 | export UNDO=true 97 | shift 98 | ;; 99 | --branches*) 100 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 101 | export ALT_BRANCH=$(echo "$1" | sed -e 's/^[^=]*=//g') 102 | shift 103 | ;; 104 | --tags*) 105 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 106 | export ALT_TAGS=$(echo "$1" | sed -e 's/^[^=]*=//g') 107 | shift 108 | ;; 109 | --trunk*) 110 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 111 | export ALT_TRUNK=$(echo "$1" | sed -e 's/^[^=]*=//g') 112 | shift 113 | ;; 114 | --svn-prefix*) 115 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 116 | export PREFIX=$(echo "$1" | sed -e 's/^[^=]*=//g') 117 | shift 118 | ;; 119 | --svn-user*) 120 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 121 | export SVN_USER=$(echo "$1" | sed -e 's/^[^=]*=//g') 122 | shift 123 | ;; 124 | --svn-pass*) 125 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 126 | export SVN_PASS=$(echo "$1" | sed -e 's/^[^=]*=//g') 127 | shift 128 | ;; 129 | --no-branches*) 130 | export BRANCHES=false 131 | shift 132 | ;; 133 | --no-tags*) 134 | export TAGS=false 135 | shift 136 | ;; 137 | --no-preserve-trunk*) 138 | export PRESERVE_TRUNK=false 139 | shift 140 | ;; 141 | --no-tag-branches*) 142 | export TAG_BRANCH=false 143 | shift 144 | ;; 145 | --check-size*) 146 | export CHECK_SIZE=true 147 | shift 148 | ;; 149 | --private*) 150 | export PRIVATE='true' 151 | shift 152 | ;; 153 | --install*) 154 | export INSTALL=true 155 | shift 156 | ;; 157 | --no-org*) 158 | export NOORG=true 159 | shift 160 | ;; 161 | --lfs-limit*) 162 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 163 | export LFS=$(echo "$1" | sed -e 's/^[^=]*=//g') 164 | shift 165 | ;; 166 | --blob-limit*) 167 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 168 | export BLOB_LIMIT=$(echo "$1" | sed -e 's/^[^=]*=//g') 169 | shift 170 | ;; 171 | --safe*) 172 | export SAFE=true 173 | shift 174 | ;; 175 | --no-other-repos*) 176 | export DRYRUN=true 177 | shift 178 | ;; 179 | *) 180 | echo "Invalid Argument" 181 | exit 1 182 | ;; 183 | esac 184 | done 185 | 186 | #RUN BEGINS HERE 187 | main () 188 | { 189 | eval $(config /etc/svn-to-github/config.yml) 190 | set_vars 191 | 192 | if [ -z "$ORG" ] || [ -z "$NTID" ] || [ -z "$TOKEN" ]; then 193 | interactive 194 | if ! $NOORG ; then 195 | echo "Checking github organization permissions..." | tee -a "$STATUS" 196 | gh_test_org 197 | fi 198 | if ! $UNDO ; then 199 | echo "Checking for name conflict in github for \"$REPO_NAME\"" | tee -a "$STATUS" 200 | gh_test_repo 201 | fi 202 | else 203 | export AUTH="-u $NTID:$TOKEN" 204 | if ! $NOORG ; then 205 | echo "Checking github organization permissions..." | tee -a "$STATUS" 206 | gh_test_org 207 | fi 208 | if ! $UNDO ; then 209 | echo "Checking for name conflict in github for \"$REPO_NAME\"" | tee -a "$STATUS" 210 | gh_test_repo 211 | fi 212 | fi 213 | 214 | if $UNDO && [ "$SAFE" == false ] ; then 215 | echo "Deleting all repos from github that were created from a previos run..." | tee -a "$STATUS" 216 | undo 217 | report 218 | elif [ "$UNDO" == true ] && [ "$FORCE" == true ] && [ "$SAFE" == true ]; then 219 | echo "Force initiated! Removing all traces of previos run and starting over" | tee -a "$STATUS" 220 | mkdir -p "$TMP_DIR" 221 | checkout 222 | create_submodules 223 | create_primary 224 | create_repos 225 | undo 226 | report 227 | echo "SUCCESS: Conversion complete" | tee -a "$STATUS" 228 | exit 0 229 | else 230 | echo "NOTICE: Getting started..." | tee -a "$STATUS" 231 | clean_logs | tee -a "$STATUS" 232 | 233 | if $INSTALL ; then 234 | if [ "$USER" == 'root' ]; then 235 | echo "Checking prerequisites..." | tee -a "$STATUS" 236 | install 237 | elif [ "$(touch "$WORK_DIR"/.lock && test -f "$WORK_DIR"/.lock)" ] && [ "$(yum list installed|grep git-svn &> /dev/null)" ] && [ "$(git lfs &> /dev/null)" ] && [ "$(which java &> /dev/null)" ]; then 238 | echo "WARNING: Not running as root, but things look safe to proceed." | tee -a "$STATUS" 239 | else 240 | echo "FAILURE: Authentication failed." >> "$STATUS" 241 | fail 242 | fi 243 | else 244 | if [ "$(touch "$WORK_DIR"/.lock && test -f "$WORK_DIR"/.lock)" ] && [ "$(which git svn)" ]; then 245 | echo "WARNING: Install was skipped, but things look safe to proceed." | tee -a "$STATUS" 246 | else 247 | echo "WARNING: Something went wrong, either permission denied on \"$WORK_DIR\" or the package \"git-svn\" is not installed or is missing from \$PATH." >> "$STATUS" 248 | fail 249 | fi 250 | fi 251 | 252 | if $FORCE ; then 253 | echo "Force initiated! Removing all traces of previos run..." | tee -a "$STATUS" 254 | rm -rf "$AUTHORPROG" "$BFGPROG" "$TMP_DIR" "$SVN_DIR" 255 | fi 256 | 257 | if $CHECK_SIZE ; then 258 | echo "Checking size, this could take a while..." | tee -a "$STATUS" 259 | check_size | tee -a "$STATUS" 260 | fi 261 | 262 | if [ ! -d "$SVN_DIR/.svn" ]; then 263 | echo "Checking out \"$SVN_REPO\" locally, this could take a while... see \"$COLOG\" for details." | tee -a "$STATUS" 264 | if [ -n "$SVN_USER" ] && [ -n "$SVN_PASS" ]; then 265 | save_svn_credentials &>> "$STATUS" 266 | fi 267 | checkout 268 | else 269 | echo "WARNING: \"$SVN_DIR\" already exists, if this is not a complete and up-to-date revision this whole process will result in erroneous conversion." | tee -a "$STATUS" 270 | echo "Checking for submodules..." 271 | export MODULES=$(find "$SVN_DIR" -mindepth 1 -type d -regextype posix-extended -regex "^.*/$TRUNK"|grep -Ev "/$TAG/"|grep -Ev "/$BRANCH/"|rev|cut -d / -f 2-|rev|sort -u) 272 | fi 273 | 274 | if [ ! -f "$AUTHORS_FILE" ]; then 275 | echo "Getting list of authors..." | tee -a "$STATUS" 276 | authors | tee -a "$STATUS" 277 | fi 278 | 279 | if [ ! -f "$IGNORE_FILE" ]; then 280 | echo "Gathering svn ignore properties..." | tee -a "$STATUS" 281 | ignore | tee -a "$STATUS" 282 | fi 283 | 284 | if [ ! -d "$TMP_DIR" ]; then 285 | mkdir "$TMP_DIR" 286 | git config --global user.name "SVN Converter" &>> "$CNVLOG" 287 | git config --global user.email svn_converter@devnull.com &>> "$CNVLOG" 288 | fi 289 | 290 | if $DRYRUN ; then 291 | echo "NOTICE: --no-other-repos enabled, no submodules will be created..." 292 | create_submodules | tee -a "$STATUS" 293 | create_primary | tee -a "$STATUS" 294 | else 295 | create_repos | tee -a "$STATUS" 296 | fi 297 | archive_job | tee -a "$STATUS" 298 | #svn_depth 299 | report | tee -a "$STATUS" 300 | echo "SUCCESS: Conversion complete" | tee -a "$STATUS" 301 | exit 0 302 | fi 303 | } 304 | 305 | #DEFAULTS 306 | config () 307 | { 308 | local prefix=$2 309 | local s='[[:space:]]*' w='[a-zA-Z0-9_]*' 310 | fs=$(echo @|tr @ '\034') 311 | sed -ne "s|^\($s\):|\1|" \ 312 | -e "s|^\($s\)\($w\)$s:$s[\"']\(.*\)[\"']$s\$|\1$fs\2$fs\3|p" \ 313 | -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" $1 | 314 | awk -F$fs '{ 315 | indent = length($1)/2; 316 | vname[indent] = $2; 317 | for (i in vname) {if (i > indent) {delete vname[i]}} 318 | if (length($3) > 0) { 319 | vn=""; for (i=0; i> "$STATUS" 666 | else 667 | echo "FAILURE: User \"$NTID\" is not a member of \"$ORG\", or \"$ORG\" does not exist." >> "$STATUS" 668 | fail 669 | fi 670 | } 671 | 672 | gh_test_repo () 673 | { 674 | if $NOORG ; then 675 | if [ ! "$(curl -s "$AUTH" "$GITHUB_API/users/$NTID/repos"|grep -F '"name":'|grep "$REPO_NAME"$)" ]; then 676 | echo "SUCCESS: Repository \"$REPO_NAME\" does not yet exist under \"$NTID\"." >> "$STATUS" 677 | else 678 | echo "FAILURE: Repo \"$REPO_NAME\" already exists under user \"$NTID\" try re-running with --new-name=repo_name" >> "$STATUS" 679 | fail 680 | fi 681 | else 682 | if [ ! "$(curl -s "$AUTH" "$GITHUB_API/orgs/$ORG/repos"|grep -F '"name":'|grep "$REPO_NAME"$)" ]; then 683 | echo "SUCCESS: Repository \"$REPO_NAME\" does not yet exist in organization \"$ORG\"." | tee -a "$STATUS" 684 | else 685 | echo "FAILURE: Repo \"$REPO_NAME\" already exists in \"$ORG\" try re-running with --new-name=repo_name" >> "$STATUS" 686 | fail 687 | fi 688 | fi 689 | } 690 | 691 | check_size () 692 | { 693 | if [ ! "$(echo "$SVN_REPO"| grep -Ei '^file://')" ]; then 694 | repo_file_info=$(svn list -vR "$SVN_REPO"|awk '{if ($3 !="") sum+=$3} END {print sum}') 695 | repo_approx_size=$(expr "$(echo "$repo_file_info")" * 1.25 ) 696 | else 697 | repo_file_info=$(du -s "$SVN_DIR"|awk '{print $1}') 698 | repo_approx_size="$repo_file_info" 699 | fi 700 | 701 | avail=$(expr "$(df "$WORK_DIR"|grep ^/|awk '{print $4}')" - 0 ) 702 | needed=$(expr "$repo_approx_size" * 2.25 ) 703 | 704 | if [ "$avail" -gt "$needed" ]; then 705 | echo "SUCCESS: Disk space is adequate..." | tee -a "$STATUS" 706 | return 0 707 | else 708 | echo "FAILURE: Disk space in work-dir \"$WORK_DIR\" only has $avail Bytes available and $needed are needed." >> "$STATUS" 709 | fail 710 | fi 711 | } 712 | 713 | #### PROMPTS 714 | interactive () 715 | { 716 | if [ -z "$ORG" ] && [ "$NOORG" == false ]; then 717 | echo "Please enter the github Orgnaization, or re-run with --no-org to save repositories under NTID namespace instead of Organization namespace." 718 | read ORG 719 | export ORG 720 | fi 721 | if [ -z "$NTID" ]; then 722 | echo "Please enter the NTID of a member of \"$ORG\"." 723 | read NTID 724 | export NTID 725 | fi 726 | if [ -z "$TOKEN" ]; then 727 | echo "Please enter the Personal Access Token for \"$NTID\"." 728 | read -s TOKEN 729 | export TOKEN 730 | fi 731 | export AUTH="-u $NTID:$TOKEN" 732 | } 733 | 734 | gh_auth () 735 | { 736 | if [ "$(curl -s "$AUTH" "$GITHUB_API/user"|grep login|grep "$NTID")" ]; then 737 | echo "SUCCESS: Authentication worked!" 738 | else 739 | echo "FAILURE: Authentication failed." 740 | export ORG="" 741 | export NTID="" 742 | export TOKEN="" 743 | export RETRY=$(expr $RETRY - 1) 744 | if [ "$RETRY" -gt 0 ]; then 745 | interactive 746 | else 747 | echo "FAILURE: Failed to authenticate user $NTID with token $TOKEN to $GITHUB $RETRY times." >> "$STATUS" 748 | fail 749 | fi 750 | fi 751 | } 752 | 753 | safe_prompt () 754 | { 755 | if ! $FORCE ; then 756 | answer="" 757 | echo "WARNING: SAFE-MODE ENABLED. To create repo $1 in github, type 'yes' (case-sensitive)" 758 | read answer 759 | if [ "$answer" != 'yes' ]; then 760 | return 1 761 | else 762 | return 0 763 | fi 764 | else 765 | return 0 766 | fi 767 | } 768 | 769 | #### TESTS 770 | has_trunk () 771 | { 772 | if [ -n "$(find "$1" -maxdepth 1 -type d -regextype posix-extended -regex "^.*/$TRUNK")" ]; then 773 | return 0 774 | else 775 | return 1 776 | fi 777 | } 778 | 779 | has_tags () 780 | { 781 | if [ "$(ls -l "$1"|grep ^d|grep -qE "$TAG")" ]; then 782 | return 0 783 | else 784 | return 1 785 | fi 786 | } 787 | 788 | has_branches () 789 | { 790 | if [ "$(ls -l "$1"|grep ^d|grep -qE "$BRANCH")" ]; then 791 | return 0 792 | else 793 | return 1 794 | fi 795 | } 796 | 797 | is_stdlayout () 798 | { 799 | if has_trunk "$1" &> /dev/null && has_branches "$1" &> /dev/null && has_tags "$1" &> /dev/null ; then 800 | return 0 801 | else 802 | return 1 803 | fi 804 | } 805 | 806 | is_empty () 807 | { 808 | if [ -z "$(find "$1" -type f -print -quit)" ]; then 809 | return 0 810 | else 811 | return 1 812 | fi 813 | } 814 | 815 | gh_conflict () 816 | { 817 | if $NOORG ; then 818 | check=$(curl -s "$AUTH" "$GITHUB_API/users/$NTID/repos"|grep -F '"name":'|grep "$1") 819 | else 820 | check=$(curl -s "$AUTH" "$GITHUB_API/orgs/$ORG/repos"|grep -F '"name":'|grep "$1") 821 | fi 822 | if [ "$check" -ne 0 ]; then fail ; fi 823 | } 824 | 825 | #### ACTIONS 826 | create_repo () 827 | { 828 | if [ -f "$RPO_DIR/$1.json" ] && [ ! $FORCE ]; then 829 | echo "WARNING: $1.json already exists, overriding with --force, may result in a duplicate repo in github." | tee -a "$STATUS" 830 | elif [ $FORCE ] || [ ! -f "$RPO_DIR/$1.json" ]; then 831 | create=true 832 | desc=$(echo "$2"|sed s/\\/\\/.*@/\\/\\//g) 833 | 834 | if $SAFE ; then 835 | if [ ! "$(safe_prompt "$1")" ] && [ "$UNDO" == false ]; then 836 | echo "WARNING: $1 was not confirmed in the --safe mode prompt, and was skipped from conversion." 837 | create=false 838 | fi 839 | fi 840 | 841 | if $create ; then 842 | cat < "$RPO_DIR"/"$1".json 843 | { 844 | "name": "$1", 845 | "description": "$desc", 846 | "homepage": "", 847 | "private": $PRIVATE, 848 | "has_issues": true, 849 | "has_wiki": true, 850 | "has_downloads": true 851 | } 852 | EOF 853 | if ! $UNDO ; then 854 | if $NOORG ; then 855 | curl -X POST -s "$AUTH" "$GITHUB_API/user/repos" --data "@$RPO_DIR/$1.json" >> "$GHLOG" 2>&1 856 | result=$? 857 | if [ "$result" -eq 0 ]; then 858 | echo "SUCCESS: $1 repository created at https://$GITHUB/$NTID/$1" | tee -a "$CRTLOG" "$STATUS" 859 | else 860 | echo "RETRYING: create_repo $1" >> "$RTYLOG" 861 | retry curl -X POST -s "$AUTH" "$GITHUB_API"/user/repos --data @"$RPO_DIR"/"$1".json 862 | fi 863 | else 864 | curl -X POST -s "$AUTH" "$GITHUB_API/orgs/$ORG/repos" --data "@$RPO_DIR/$1.json" >> "$GHLOG" 2>&1 865 | result=$? 866 | if [ "$result" -eq 0 ]; then 867 | echo "SUCCESS: $1 repository created at https://$GITHUB/$ORG/$1" | tee -a "$CRTLOG" "$STATUS" 868 | else 869 | echo "RETRYING: create_repo $1" >> "$RTYLOG" 870 | retry curl -X POST -s "$AUTH" "$GITHUB_API"/orgs/"$ORG"/repos --data @"$RPO_DIR"/"$1".json 871 | fi 872 | fi 873 | fi 874 | fi 875 | fi 876 | } 877 | 878 | gh_delete () 879 | { 880 | delete=true 881 | 882 | if $SAFE ; then 883 | if [ ! "$(safe_prompt "$1")" ]; then 884 | echo "WARNING: $1 was not confirmed in the --safe mode prompt, and was skipped from deletion." 885 | delete=false 886 | fi 887 | fi 888 | 889 | if $delete ; then 890 | if $NOORG ; then 891 | curl -X DELETE -s "$AUTH" "$GITHUB_API/repos/$NTID/$1" >> "$GHLOG" 2>&1 892 | result=$? 893 | else 894 | curl -X DELETE -s "$AUTH" "$GITHUB_API/repos/$ORG/$1" >> "$GHLOG" 2>&1 895 | result=$? 896 | fi 897 | if [ "$result" -eq 0 ]; then 898 | if $NOORG ; then 899 | echo "SUCCESS: $1 repository removed from github at https://$GITHUB/$NTID/$1" | tee -a "$UNDLOG" "$STATUS" 900 | else 901 | echo "SUCCESS: $1 repository created at https://$GITHUB/$ORG/$1" | tee -a "$UNDLOG" "$STATUS" 902 | fi 903 | else 904 | echo "FAILURE: $1 failed to remove, see \"$GHLOG\" for details." >> "$STATUS" 905 | fail 906 | fi 907 | fi 908 | } 909 | 910 | author_prog () 911 | { 912 | cat < $AUTHORPROG 913 | #!/bin/bash 914 | echo "\$1 <\$1>"; 915 | EOF 916 | chmod +x "$AUTHORPROG" 917 | } 918 | 919 | #TODO - Unused 920 | #svn_depth () 921 | #{ 922 | # cd "$SVN_DIR" 923 | # echo "" > "$FAMILY" 924 | # MAX_DEPTH=$(echo $MODULES|sed s/' '/"_"/g|awk -F "$SVN_DIR" '{print $2}'|awk -F "/" '{print NF - 1}'|sort -nr|head -n1) 925 | # 926 | # if [ "$MAX_DEPTH" -gt 6 ];then 927 | # echo "WARNING: Tree depth too deep to label as family" | tee -a "$STATUS" 928 | # fi 929 | # 930 | # for i in $(seq 1 $MAX_DEPTH) 931 | # do 932 | # COUNT=$(echo $MODULES|sed s/' '/"_"/g|awk -F "$SVN_DIR" '{print $2}'|awk -F "/" '{print NF - 1 }'|grep "$i"|wc -l) 933 | # min_depth=$(expr $i + 1) 934 | # if [ "$i" -eq 1 ]; then 935 | # echo "$COUNT svn siblings found." >> "$FAMILY" 936 | # elif [ "$i" -eq 2 ]; then 937 | # echo "$COUNT svn children found." >> "$FAMILY" 938 | # elif [ "$i" -eq 3 ]; then 939 | # echo "$COUNT svn grandchildren found." >> "$FAMILY" 940 | # elif [ "$i" -eq 4 ]; then 941 | # echo "$COUNT svn great grandchildren found." >> "$FAMILY" 942 | # elif [ "$i" -eq 5 ]; then 943 | # echo "$COUNT svn great great grandchildren found." >> "$FAMILY" 944 | # elif [ "$i" -eq 6 ]; then 945 | # echo "$COUNT svn great great great grandchildren found." >> "$FAMILY" 946 | # else 947 | # echo "$COUNT svn repos found at depth $i" >> "$FAMILY" 948 | # fi 949 | # done 950 | # 951 | # export MOD_COUNT=$(echo $MODULES|wc -w) 952 | #} 953 | 954 | get_parent () 955 | { 956 | if [ -d "$1" ]; then 957 | cd "$1" 958 | local parent="" 959 | 960 | while [ "$(pwd)" != "$SVN_DIR" ] && [ -z "$parent" ]; do 961 | if has_trunk "$(dirname "$(pwd)")" ; then 962 | if [ "$(dirname "$(pwd)"|grep -E /"$TRUNK"$)" ]; then 963 | cd ../ 964 | else 965 | parent="$SUB_STRING$(basename "$(dirname "$(pwd)")")" 966 | fi 967 | else 968 | cd ../ 969 | fi 970 | done 971 | 972 | if [ -z "$parent" ]; then 973 | if [ "$(basename "$(dirname "$1")"|grep ^$SUB_STRING)" ]; then 974 | echo "FAILURE: Cannot process directory that begins with \"$SUB_STRING\"" >> "$STATUS" 975 | fail 976 | else 977 | local nearest_parent="$(basename "$(dirname "$1")")" 978 | if [ ! "$(echo "$nearest_parent"|grep -E "$TRUNK")" ]; then 979 | local parent="$nearest_parent" 980 | else 981 | local parent="$REPO_NAME_lower" 982 | fi 983 | fi 984 | fi 985 | 986 | echo "$parent" 987 | else 988 | echo "FAILURE: $1 is missing, Unknown Error." >> "$STATUS" 989 | fail 990 | fi 991 | } 992 | 993 | #Only called for known submodules with children 994 | get_parent_path () 995 | { 996 | if [ -d "$1" ]; then 997 | cd "$1" 998 | while [ "$1" != "$SVN_DIR" ] && [ -z "$parent_path" ]; do 999 | if has_trunk "$(dirname "$1")" ; then 1000 | if [ "$(echo "$(dirname "$1")"|grep -E "$TRUNK")" ]; then 1001 | parent_path="$(dirname "$(dirname "$1")")" 1002 | else 1003 | parent_path="$(dirname "$1")" 1004 | fi 1005 | else 1006 | cd ../ 1007 | fi 1008 | done 1009 | 1010 | echo "$parent_path" 1011 | else 1012 | echo "FAILURE: $1 is missing, Unknown Error." >> "$STATUS" 1013 | fail 1014 | fi 1015 | } 1016 | 1017 | #Only called for known submodules with children 1018 | get_child_path () 1019 | { 1020 | cd "$1" && cd ../ 1021 | 1022 | until has_trunk "$(pwd)" ; do 1023 | cd ../ 1024 | done 1025 | child_path=$(echo "$1"| awk -F "$(basename "$(pwd)")/" '{print $2}') 1026 | 1027 | echo "$child_path" 1028 | } 1029 | 1030 | resolve_name () 1031 | { 1032 | 1033 | local name=$(basename "$1"|sed s/' '/_/g) 1034 | local prefix=$REPO_NAME_lower 1035 | local parent_name=$(get_parent "$1"|sed s/' '/_/g) 1036 | 1037 | if [ "$parent_name" == "$REPO_NAME_lower" ]; then 1038 | test_name="$prefix-$name" 1039 | elif [ ! "$(echo "$parent_name"|grep ^$SUB_STRING)" ]; then 1040 | test_name="$prefix-$parent_name-$name" 1041 | else 1042 | parent_trunk=$(echo "$parent_name"|sed s/"^$SUB_STRING"/""/g) 1043 | test_name="$prefix--$parent_trunk-$name" 1044 | fi 1045 | if ! $DRYRUN ; then 1046 | if gh_conflict "$test_name" ; then 1047 | suffix=0 1048 | until ! gh_conflict "$test_name-$suffix" 1049 | do 1050 | suffix=$(expr $suffix + 1) 1051 | done 1052 | gh_name="$test_name-$suffix" 1053 | else 1054 | gh_name="$test_name" 1055 | fi 1056 | else 1057 | gh_name="$test_name" 1058 | fi 1059 | echo "$gh_name" 1060 | } 1061 | 1062 | create_submodules () 1063 | { 1064 | if [ -n "$MODULES" ]; then 1065 | echo "Creating submodules in github..." 1066 | declare -gA SubModules 1067 | declare -gA ClonePids 1068 | declare -gA ConvertPids 1069 | declare -gA ModuleNames 1070 | declare -gA ModulePaths 1071 | declare -gA ModuleUrls 1072 | declare -gA ModuleRelatives 1073 | declare -gA ChildPaths 1074 | declare -ga ModuleOrigins 1075 | declare -ga Modules 1076 | 1077 | if [ ! -d "$SUB_DIR" ]; then 1078 | mkdir "$SUB_DIR" 1079 | fi 1080 | #TODO DEBUG 1081 | for module in $MODULES ; do #full run 1082 | local relative_path=$(echo "$module"|awk -F "$SVN_DIR/" '{print $2}') 1083 | local name=$(basename "$module") 1084 | local real_name=$(resolve_name "$module") 1085 | 1086 | if is_empty "$module"; then 1087 | echo "NOTICE: Skipping EMPTY \"$name\"" >> "$STATUS" 1088 | else 1089 | if [ ! -f "$RPO_DIR/$real_name.json" ] || [ "$DRYRUN" == true ] ; then 1090 | #Setup Arrays 1091 | SubModules["$real_name"]="" #for adding submodules 1092 | ClonePids["$real_name"]="" #for tracking Pids of Clone jobs 1093 | ConvertPids["$real_name"]="" #for tracking Pids of Convert jobs 1094 | ModuleNames["$real_name"]="$name" #for name tanslation 1095 | ModuleUrls["$real_name"]="$EVC/$REPO_NAME_ORIG/$relative_path" #for cloning svn 1096 | ModulePaths["$real_name"]="$module" #for location reference 1097 | ModuleRelatives["$real_name"]="$relative_path" #for et_al filter exception 1098 | ModuleOrigins+=("$relative_path",) #for et_al filter 1099 | Modules+=("$real_name") #for multi-threading loops 1100 | 1101 | if [ "$(echo "$real_name"|grep -E ^"$REPO_NAME_lower"--)" ]; then 1102 | #Create list "$SUB_DIR/$real_parent" of nested submodules 1103 | ChildPaths["$real_name"]="$(get_child_path "$module")" #for location reference of nested repo 1104 | real_parent="$(resolve_name "$(get_parent_path "$module")")" 1105 | echo "$real_name" >> "$SUB_DIR/$real_parent" 1106 | else 1107 | echo "$real_name" >> "$SUB_DIR/$REPO_NAME" 1108 | real_parent=$REPO_NAME 1109 | fi 1110 | if ! $DRYRUN ; then 1111 | #Create the repo in github 1112 | submodule_repo "$real_name" "$real_parent" 1113 | fi 1114 | fi 1115 | fi 1116 | done 1117 | fi 1118 | } 1119 | 1120 | add_submodules () 1121 | { 1122 | if [ -f "$SUB_DIR/$1" ] && [ "$UNDO" == false ]; then 1123 | cd "$TMP_DIR/$1.git" 1124 | 1125 | declare -A Children 1126 | 1127 | #Create Array of Children 1128 | for child in $(cat "$SUB_DIR/$1"); do 1129 | Children["$child"]="$child" 1130 | done 1131 | 1132 | #Loop through processing children until all children are done cloning 1133 | while [ "${#Children[@]}" -ne 0 ]; do 1134 | for child in "${Children[@]}" ; do 1135 | #echo "DEBUG child is $child" 1136 | clone_pid=${SubModules["$child"]} #race condition for submodules of submodules 1137 | #echo "DEBUG clonepid $clone_pid" 1138 | convert_pid=${ConvertPids["$child"]} #race condition of submodules of primary repo 1139 | #echo "DEBUG convertpid $convert_pid" 1140 | if [ "$1" == "$REPO_NAME" ]; then 1141 | original_path=${ModuleRelatives["$child"]} 1142 | else 1143 | original_path=${ChildPaths["$child"]} 1144 | fi 1145 | 1146 | if ([ "$(ps aux|awk '{print $2}'|grep ^$clone_pid$)" ] && [ -n "$clone_pid" ]) || ([ "$(ps aux|awk '{print $2}'|grep ^$convert_pid$)" ] && [ -n "$convert_pid" ]); then 1147 | echo "Waiting for child submodule(s) to finish converting. Sleeping $SLEEP_TIME seconds..." | tee -a "$STATUS" 1148 | sleep $SLEEP_TIME 1149 | else 1150 | if $NOORG ; then 1151 | add_url="https://$NTID:$TOKEN@$GITHUB/$NTID/$child.git" 1152 | else 1153 | add_url="https://$NTID:$TOKEN@$GITHUB/$ORG/$child.git" 1154 | fi 1155 | 1156 | git submodule add "$add_url" "$original_path" &>> "$PRNLOG" 1157 | result=$? 1158 | if [ "$result" -ne 0 ]; then 1159 | echo "RETRYING: add_submodules $1" >> "$RTYLOG" 1160 | retry git submodule add "$add_url" "$original_path" 1161 | fi 1162 | unset Children["$child"] 1163 | fi 1164 | done 1165 | done 1166 | unset Children 1167 | 1168 | #Remove NTID & Token from git URL 1169 | sed -i s/https:\\/\\/$NTID:$TOKEN@$GITHUB\\//git@$GITHUB:/g .gitmodules 1170 | 1171 | #Add submodules to repo 1172 | git add .gitmodules 1173 | echo "SUCCESS: All svn-sub-repos for $1 finished converting to git submodules!" 1174 | fi 1175 | } 1176 | 1177 | multi_thread () 1178 | { 1179 | echo "Converting submodules of \"$REPO_NAME\" to git..." | tee -a "$STATUS" 1180 | while [ "${#ClonePids[@]}" -ne 0 ]; do 1181 | for module in "${Modules[@]}" ; do 1182 | pid=${ClonePids["$module"]} 1183 | module_path=${ModulePaths["$module"]} 1184 | 1185 | if [ ! -d "$TMP_DIR/$module.git" ] && [ -z "$pid" ]; then 1186 | git_clone "$module" & 1187 | ClonePids["$module"]=$! 1188 | SubModules["$module"]=${ClonePids["$module"]} #resolve race condition for cloning and adding multiple submodules 1189 | elif [ -n "$pid" ] && [ ! "$(ps aux|awk '{print $2}'|grep ^$pid$)" ] && [ -d "$TMP_DIR/$module.git" ]; then 1190 | unset ClonePids["$module"] 1191 | convert "$module" & 1192 | ConvertPids["$module"]=$! 1193 | fi 1194 | done 1195 | echo "Waiting for submodule clone(s) to complete. Sleeping $SLEEP_TIME seconds..." | tee -a "$STATUS" 1196 | sleep $SLEEP_TIME 1197 | done 1198 | echo "SUCCESS: All submodules finished cloning!" | tee -a "$STATUS" 1199 | } 1200 | 1201 | submodule_repo () 1202 | { 1203 | if $NOORG ; then 1204 | create_repo "$1" "Submodule of $2 at https://$GITHUB/$NTID/$2" 1205 | else 1206 | create_repo "$1" "Submodule of $2 at https://$GITHUB/$ORG/$2" 1207 | fi 1208 | } 1209 | 1210 | 1211 | create_repos () 1212 | { 1213 | if [ -n "$MODULES" ] && [ "$DRYRUN" == false ]; then 1214 | create_submodules 1215 | if $SAFE; then 1216 | single_thread 1217 | else 1218 | multi_thread 1219 | fi 1220 | fi 1221 | create_primary 1222 | } 1223 | 1224 | git_ignore () 1225 | { 1226 | cd "$TMP_DIR/$1.git" 1227 | git svn show-ignore > .gitignore 1228 | if [ -s .gitignore ]; then 1229 | if [ ! -f "$IGNORE_FILE" ]; then 1230 | ignore 1231 | fi 1232 | cat "$IGNORE_FILE" > .gitignore 1233 | fi 1234 | git add .gitignore &>> "$CNVLOG" 1235 | 1236 | #TODO add Sync back option here #SYNC DOES NOT ALLOW FOR BFG 1237 | git config --remove-section svn-remote.svn 1238 | } 1239 | 1240 | git_add_remote () 1241 | { 1242 | if $NOORG ; then 1243 | if [ ! "$(git remote -v|grep origin|grep "https://$NTID:$TOKEN@$GITHUB/$NTID/$1.git")" ]; then 1244 | git remote add origin "https://$NTID:$TOKEN@$GITHUB/$NTID/$1.git" 1245 | fi 1246 | else 1247 | if [ ! "$(git remote -v|grep origin|grep "https://$NTID:$TOKEN@$GITHUB/$ORG/$1.git")" ]; then 1248 | git remote add origin "https://$NTID:$TOKEN@$GITHUB/$ORG/$1.git" 1249 | fi 1250 | fi 1251 | } 1252 | 1253 | retry () 1254 | { 1255 | if [ "$UNDO" == true ]; then 1256 | return 0 1257 | else 1258 | retry=1 1259 | until $@ ; do 1260 | echo "RETRY #$retry : $@" | tee -a "$RTYLOG" 1261 | sleep $SLEEP_TIME 1262 | retry=$(expr $retry + 1) 1263 | if [ "$retry" -eq 3 ]; then 1264 | echo "RETRY #$retry : $@" | tee -a "$RTYLOG" 1265 | echo "$@"|bash 1266 | result=$? 1267 | if [ "$result" -ne 0 ]; then 1268 | echo "FAILURE: $@" | tee -a "$STATUS" "$RTYLOG" 1269 | else 1270 | echo "SUCCESS: $@"| tee -a "$STATUS" "$RTYLOG" 1271 | fi 1272 | break 1273 | fi 1274 | done 1275 | fi 1276 | } 1277 | 1278 | convert () 1279 | { 1280 | cd "$TMP_DIR/$1.git" 1281 | 1282 | git config --global push.default upstream 1283 | 1284 | if [ "$(git branch -a|grep [[:space:]]master$)" ]; then 1285 | if [ ! "$(git branch -a|grep \*[[:space:]]master$)" ]; then 1286 | git checkout master &>> "$CNVLOG" 1287 | fi 1288 | else 1289 | git checkout -b master &>> "$CNVLOG" 1290 | fi 1291 | 1292 | git_ignore "$1" | tee -a "$STATUS" 1293 | git_bfg_lfs "$1" | tee -a "$STATUS" 1294 | git commit -m 'SVN2GitHub Master initialization.' &>> "$CNVLOG" 1295 | git_add_remote "$1" 1296 | 1297 | git push -u origin master >> "$PHLOG" 2&>1 1298 | result=$? 1299 | if [ "$result" -ne 0 ]; then 1300 | echo "RETRYING: convert init $1" >> "$RTYLOG" 1301 | retry git push -u origin master 1302 | fi 1303 | 1304 | git config remote.origin.push 'refs/remotes/*:refs/heads/*' 1305 | 1306 | if ! $DRYRUN ; then 1307 | add_submodules "$1" | tee -a "$STATUS" 1308 | fi 1309 | 1310 | if [ "$1" == "$REPO_NAME" ]; then 1311 | git_et_al_primary "$SVN_DIR" | tee -a "$STATUS" 1312 | else 1313 | git_et_al "$1" | tee -a "$STATUS" 1314 | fi 1315 | 1316 | git commit -m 'SVN2GitHub added other files from svn repo not under version control.' 1317 | git_bfg_lfs "$1" 1318 | git commit -m 'SVN2GitHub Master branch conversion complete.' &>> "$CNVLOG" 1319 | git push -u origin master >> "$PHLOG" 2&>1 1320 | result=$? 1321 | if [ "$result" -ne 0 ]; then 1322 | #TODO cleanup? git_bfg_lfs "$1" 1323 | echo "RETRYING: convert master $1" >> "$RTYLOG" 1324 | retry git push -u origin master 1325 | fi 1326 | 1327 | if $TAGS ; then 1328 | git_tags | tee -a "$STATUS" 1329 | fi 1330 | 1331 | if $BRANCHES ; then 1332 | git_branches | tee -a "$STATUS" 1333 | fi 1334 | 1335 | if $PRESERVE_TRUNK; then 1336 | #git_nest_submodules_in_trunk "$1" 1337 | git_nest_trunk_as_submodule "$1" | tee -a "$STATUS" 1338 | fi 1339 | echo "SUCCESS: Conversion of repo \"$1\" complete." | tee -a "$STATUS" 1340 | } 1341 | 1342 | #TODO - Unused 1343 | #git_nest_submodules_in_trunk () 1344 | #{ 1345 | # git checkout -b trunk &>> "$CNVLOG" 1346 | # git checkout trunk &>> "$CNVLOG" 1347 | # git pull --no-edit origin trunk &>> "$CNVLOG" 1348 | # add_submodules "$1" 1349 | # git commit -m 'SVN2GitHub conversion: Trunk Preservation Enabled.' &>> "$CNVLOG" 1350 | # git push -u origin trunk &>> "$PHLOG" 1351 | # if [ "$?" -ne 0 ]; then 1352 | # echo "RETRYING: nest submodules in trunk $1" >> "$RTYLOG" 1353 | # retry git push -u origin trunk 1354 | # fi 1355 | #} 1356 | 1357 | git_nest_trunk_as_submodule () 1358 | { 1359 | if [ ! "$(git branch -a|grep \*[[:space:]]master$)" ]; then 1360 | git checkout master &>> "$CNVLOG" 1361 | fi 1362 | 1363 | git pull --no-edit origin master &>> "$CNVLOG" 1364 | 1365 | if $NOORG ; then 1366 | trunk_url="https://$NTID:$TOKEN@$GITHUB/$NTID/$(basename "$(pwd)")" 1367 | else 1368 | trunk_url="https://$NTID:$TOKEN@$GITHUB/$ORG/$(basename "$(pwd)")" 1369 | fi 1370 | 1371 | git submodule add -b trunk -f "$trunk_url" ./trunk >> "$CNVLOG" 2>&1 1372 | result=$? 1373 | if [ "$result" -ne 0 ]; then 1374 | echo "RETRYING: git_nest_trunk_as_submodule" >> "$RTYLOG" 1375 | retry git submodule add -b trunk -f "$trunk_url" ./trunk 1376 | fi 1377 | : 1378 | #Remove NTID & Token from git URL 1379 | sed -i s/https:\\/\\/$NTID:$TOKEN@$GITHUB\\//git@$GITHUB:/g .gitmodules 1380 | git add .gitmodules &>> "$CNVLOG" 1381 | git commit -m 'SVN2GitHub conversion: added trunk branch as submodule with path ./trunk' &>> "$CNVLOG" 1382 | git push -u origin master >> "$PHLOG" 2>&1 1383 | result=$? 1384 | if [ "$result" -ne 0 ]; then 1385 | echo "RETRYING: git_nest_trunk_as_submodule" >> "$RTYLOG" 1386 | retry git push -u origin master 1387 | fi 1388 | } 1389 | 1390 | git_clone () 1391 | { 1392 | echo "NOTICE: Cloning $1 from svn to git now..." | tee -a "$STATUS" 1393 | if [ "$1" == "$REPO_NAME" ]; then 1394 | local url=$SVN_REPO 1395 | else 1396 | local url=${ModuleUrls["$1"]} 1397 | fi 1398 | if ! $UNDO ; then 1399 | if [ ! -d "$TMP_DIR/$1.git" ] || [ "$FORCE" == true ]; then 1400 | git svn clone "$url" "$LAYOUT" --authors-file="$AUTHORS_FILE" --authors-prog="$AUTHORPROG" --quiet "$TMP_DIR/$1.git" >> "$CNLOG" 2>&1 1401 | result=$? 1402 | if [ "$result" -ne 0 ]; then 1403 | echo "RETRYING: clone $1" >> "$RTYLOG" 1404 | retry git svn clone "$url" "$LAYOUT" --authors-file="$AUTHORS_FILE" --authors-prog="$AUTHORPROG" --quiet "$TMP_DIR"/"$1".git 1405 | fi 1406 | else 1407 | echo "FAILURE: \"$TMP_DIR/$1.git\" failed to clone completely in a previous run, try again with --force" >> "$STATUS" 1408 | fail 1409 | fi 1410 | echo "SUCCESS: Clone of $1 complete, converting $1 now..." | tee -a "$STATUS" 1411 | fi 1412 | } 1413 | 1414 | svn_tree () 1415 | { 1416 | cd "$TMP_DIR/$REPO_NAME.git" 1417 | if [ ! "$(git branch -a|grep -q \*[[:space:]]master$)" ]; then 1418 | git checkout master &>> "$CNVLOG" 1419 | fi 1420 | git pull --no-edit origin master &>> "$CNVLOG" 1421 | tree "$SVN_DIR" &> .svn_tree 1422 | git add .svn_tree &>> "$CNVLOG" 1423 | git commit -m 'Original SVN Repository Tree' &>> "$CNVLOG" 1424 | git push -u origin master >> "$PHLOG" 2>&1 1425 | result=$? 1426 | if [ "$result" -ne 0 ]; then 1427 | echo "RETRYING: svn_tree" >> "$RTYLOG" 1428 | retry git push -u origin master 1429 | fi 1430 | } 1431 | 1432 | archive_job () 1433 | { 1434 | cp "$RPO_DIR/"* "$LOG_DIR/" 1435 | cp "$AUTHORS_FILE" "$LOG_DIR/" 1436 | cp "$TMP_DIR/$REPO_NAME.git/.svn_tree" "$LOG_DIR/svn_tree" 1437 | cd "$WORK_DIR" 1438 | timestamp=$(date +%s) 1439 | tar czf "$REPO_NAME-$timestamp.tar.gz" "$LOG_DIR" 1440 | mv "$REPO_NAME-$timestamp.tar.gz" "$JOB_DIR" 1441 | } 1442 | 1443 | git_bfg_lfs () 1444 | { 1445 | sleep $SLEEP_TIME #not sure why lfs doesn't have a value sometimes 1446 | lfs=$(find "$TMP_DIR/$1.git" -type f -size +"$LFS"M|grep -Ev '/\.svn|/\.subversion|/\.git') 1447 | echo "DEBUG: LFS = $lfs" >> "$LFSLOG" 1448 | if [ -n "$lfs" ]; then 1449 | #TODO deprecate because it breaks lfs on current branch? 1450 | # if [ ! -d "$LFS_DIR" ]; then 1451 | # mkdir "$LFS_DIR" 1452 | # fi 1453 | # echo "" > "$LFS_DIR/$1" 1454 | # for big_file in $lfs; do 1455 | # if [ "$(basename $big_file|grep .)" ]; then 1456 | # extension=$(echo $big_file|rev|awk -F "." '{print $1}'|rev) 1457 | # echo pattern="'*.$extension'" >> "$LFS_DIR/$1" 1458 | # else 1459 | # echo patten="'$(basename $big_file)'" >> "$LFS_DIR/$1" 1460 | # fi 1461 | # done 1462 | 1463 | 1464 | # for lfs_pattern in $(grep pattern= "$LFS_DIR/$1"|awk -F "pattern=" '{print $2}'|sort -u); do 1465 | # java -jar "$BFGPROG" --convert-to-git-lfs $lfs_pattern --no-blob-protection "$TMP_DIR/$1.git" &>> "$BFGLOG" 1466 | # done 1467 | 1468 | git lfs install &>> "$LFSLOG" 1469 | 1470 | for file in $lfs; do 1471 | track_file=$(echo "$file"|awk -F "$(pwd)/" '{print $2}') 1472 | git lfs track "$track_file" &>> "$LFSLOG" 1473 | git add "$track_file" &>> "$LFSLOG" 1474 | done 1475 | 1476 | git add .gitattributes &>> "$LFSLOG" 1477 | git reflog expire --expire=now --all && git gc --prune=now --aggressive &>> "$BFGLOG" 1478 | fi 1479 | 1480 | java -jar "$BFGPROG" --strip-blobs-bigger-than "$BLOB_LIMIT"M --no-blob-protection &>> "$BFGLOG" 1481 | git reflog expire --expire=now --all && git gc --prune=now --aggressive &>> "$BFGLOG" 1482 | sleep $SLEEP_TIME 1483 | } 1484 | 1485 | git_tags () 1486 | { 1487 | for ref in $(git for-each-ref --format='%(refname)' refs/remotes/tags|cut -d / -f 4) 1488 | do 1489 | tag=$(echo "$ref"|sed s/%20/'_'/g) 1490 | git tag "$tag" "refs/remotes/tags/$ref" &>> "$CNVLOG" 1491 | done 1492 | 1493 | git push --tags >> "$PHLOG" 2>&1 1494 | result=$? 1495 | if [ "$result" -ne 0 ]; then 1496 | echo "RETRYING: git_tags $(pwd)" >> "$RTYLOG" 1497 | retry git push --tags 1498 | fi 1499 | } 1500 | 1501 | git_branches () 1502 | { 1503 | for ref in $(find .git/refs -type f |grep -Ev 'heads/|origin/|/stash'); do 1504 | mv "$ref" .git/refs/remotes/origin/ 1505 | done 1506 | git push -u origin >> "$PHLOG" 2>&1 1507 | result=$? 1508 | if [ "$result" -ne 0 ]; then 1509 | echo "RETRYING: push branches $(pwd)" >> "$RTYLOG" 1510 | retry git push -u origin 1511 | fi 1512 | 1513 | if ! $TAG_BRANCH; then 1514 | for branch in $(git branch -r|grep origin|grep tags/); do 1515 | git push origin --delete "origin/$branch" >> "$CNLOG" 2>&1 1516 | result=$? 1517 | if [ "$result" -ne 0 ]; then 1518 | echo "RETRYING: delete tag branches $(pwd)" >> "$RTYLOG" 1519 | retry git push origin --delete "origin/$branch" 1520 | fi 1521 | done 1522 | fi 1523 | } 1524 | 1525 | undo () 1526 | { 1527 | if [ -n "$(find "$RPO_DIR" -type f -name "*.json")" ]; then 1528 | UNDO=$(ls "$RPO_DIR"/*.json|rev|awk -F "/" '{print $1}'|rev|awk -F ".json" '{print $1}') 1529 | for repo in $UNDO; 1530 | do 1531 | gh_delete "$repo" 1532 | if [ -f "$RPO_DIR/$repo.json" ]; then 1533 | rm -f "$RPO_DIR/$repo.json" 1534 | fi 1535 | if [ -d "$TMP_DIR/$repo.git" ]; then 1536 | rm -rf "$TMP_DIR/$repo.git" 1537 | fi 1538 | done 1539 | elif [ -n "$(find "$JOB_DIR" -type f -name "$REPO_NAME*gz")" ]; then 1540 | zips=$(find "$JOB_DIR" -type f -name "$REPO_NAME*gz") 1541 | for tar in $zips; do 1542 | tar xf "$tar" 1543 | UNDO=$(find "$JOB_DIR" -type f -name "$REPO_NAME_lower*.json"|rev|awk -F "/" '{print $1}'|rev|awk -F ".json" '{print $1}') 1544 | for repo in $UNDO; 1545 | do 1546 | gh_delete "$repo" 1547 | find "$JOB_DIR" -type f -name "$repo.json" -delete 1548 | done 1549 | done 1550 | gh_delete "$REPO_NAME" 1551 | else 1552 | echo "The --undo flag was used, but there appears to be nothing to undo in \"$RPO_DIR\"" | tee -a "$STATUS" 1553 | fi 1554 | } 1555 | 1556 | single_thread () 1557 | { 1558 | for sub in $MODULES; do 1559 | submodule=$(echo "$sub"|awk -F "$SVN_DIR/" '{print $2}'|rev|awk -F "/" '{print $1}'|rev) 1560 | submodules=(${submodules[@]} "$submodule") 1561 | done 1562 | 1563 | for module in $submodules 1564 | do 1565 | if ! gh_conflict "$module" ; then 1566 | if $NOORG; then 1567 | create_repo "$module" "Submodule of https://$GITHUB/$NTID/$REPO_NAME" 1568 | else 1569 | create_repo "$module" "Submodule of https://$GITHUB/$ORG/$REPO_NAME" 1570 | fi 1571 | else 1572 | if $NOORG ; then 1573 | echo "WARNING: $module already exists in $GITHUB under $NTID" | tee -a "$STATUS" 1574 | else 1575 | echo "WARNING: $module already exists in $GITHUB under $ORG" | tee -a "$STATUS" 1576 | fi 1577 | num=0 1578 | until ! gh_conflict "$module-$num" 1579 | do 1580 | num=$(expr $num + 1) 1581 | done 1582 | if $NOORG; then 1583 | create_repo "$module-$num" "Submodule of https://$GITHUB/$NTID/$REPO_NAME" 1584 | else 1585 | create_repo "$module-$num" "Submodule of https://$GITHUB/$ORG/$REPO_NAME" 1586 | fi 1587 | if $NOORG ; then 1588 | echo "WARNING: $module was RENAMED to $module-$num in github because $module already exists under $NTID" | tee -a "$STATUS" 1589 | else 1590 | echo "WARNING: $module was RENAMED to $module-$num in github because $module already exists under $ORG" | tee -a "$STATUS" 1591 | fi 1592 | fi 1593 | git_clone "$module" 1594 | convert "$module" 1595 | done 1596 | } 1597 | 1598 | install () 1599 | { 1600 | yum makecache fast &>> "$STATUS" 1601 | 1602 | if [ ! "$(yum list installed|grep git-svn)" ]; then 1603 | echo "Installing git-svn..." | tee -a "$STATUS" 1604 | yum -qy install git-svn 1605 | fi 1606 | if [ ! "$(yum list installed|grep git-lfs)" ]; then 1607 | echo "Installing git-lfs..." | tee -a "$STATUS" 1608 | #Source from https://packagecloud.io/github/git-lfs 1609 | if [ "$(uname -a|awk '{print $3}'|grep el7)" ]; then 1610 | yum -qy install "$LFS_DOWNLOAD" 1611 | fi 1612 | if [ "$(uname -a|awk '{print $3}'|grep el6)" ]; then 1613 | yum -qy install "$LFS_DOWNLOAD" 1614 | fi 1615 | fi 1616 | if [ ! "$(yum list installed|grep openjdk)" ]; then 1617 | echo "Installing java-1.7.0-openjdk..." | tee -a "$STATUS" 1618 | yum -qy install java-1.7.0-openjdk 1619 | fi 1620 | if [ ! "$(yum list installed|grep expect)" ]; then 1621 | echo "Installing expect..." | tee -a "$STATUS" 1622 | yum -qy install expect 1623 | fi 1624 | if [ ! "$(yum list installed|grep tree)" ]; then 1625 | echo "Installing tree..." | tee -a "$STATUS" 1626 | yum -qy install tree 1627 | fi 1628 | if [ ! -f "$BFGPROG" ]; then 1629 | curl -o "$BFGPROG" -k -s "$BFG_DOWNLOAD" 1630 | fi 1631 | if [ ! -f "$AUTHORPROG" ]; then 1632 | author_prog 1633 | fi 1634 | } 1635 | 1636 | save_svn_credentials () 1637 | { 1638 | if [ ! "$(grep "$EVC" /root/.subversion/auth/svn.ssl.server/*)" ]; then 1639 | expect -c ' 1640 | spawn svn info --username=$::env(SVN_USER) $::env(SVN_REPO) 1641 | expect "Error validating server certificate" 1642 | send "p\r" 1643 | expect eof ' ; &> "$CNLOG" 1644 | fi 1645 | #TODO add rm -rf ~/.svn to --force or fix with individual users, and a check job running for user functions 1646 | #if [ ! "$(grep $SVN_USER /root/.subversion/auth/svn.simple/*)" ]; then 1647 | expect -c ' 1648 | spawn svn info --username=$::env(SVN_USER) $::env(SVN_REPO) 1649 | expect "Password for" 1650 | send "$::env(SVN_PASS)\r" 1651 | expect "Store password" 1652 | send "yes\r" 1653 | expect eof ' ; &> "$CNLOG" 1654 | #fi 1655 | } 1656 | 1657 | checkout () 1658 | { 1659 | cd "$WORK_DIR" 1660 | if [ -n "$SVN_USER" ] && [ -n "$SVN_PASS" ]; then 1661 | svn checkout --trust-server-cert --non-interactive --no-auth-cache --username="$SVN_USER" --password="$SVN_PASS" "$SVN_REPO" > "$COLOG" 2>&1 1662 | result=$? 1663 | else 1664 | svn checkout --trust-server-cert --non-interactive --no-auth-cache "$SVN_REPO" > "$COLOG" 2>&1 1665 | result=$? 1666 | fi 1667 | 1668 | if [ "$result" -eq 0 ]; then 1669 | echo "SUCCESS: \"$REPO_NAME\" was checked out." | tee -a "$STATUS" "$COLOG" 1670 | echo "Checking for submodules..." | tee -a "$STATUS" 1671 | export MODULES=$(find "$SVN_DIR" -mindepth 1 -type d -regextype posix-extended -regex "^.*/$TRUNK"|grep -Ev "/$TAG/"|grep -Ev "/$BRANCH/"|grep -v "$SVN_DIR"$|rev|cut -d / -f 2-|rev|sort -u) 1672 | else 1673 | echo "FAILURE: $REPO_NAME failed to checkout, see \"$COLOG\" for details." | tee -a "$STATUS" 1674 | fail 1675 | fi 1676 | } 1677 | 1678 | authors () 1679 | { 1680 | cd "$SVN_DIR" 1681 | for author in $(svn log -q |grep ^r|awk '{print $3}'|sort -u); do 1682 | if [ -n "$(getent passwd "$author")" ]; then 1683 | name=$(getent passwd "$author"|awk -F ":" '{print $5}') 1684 | email=$(echo "$name"|sed s/' '/'_'/g) 1685 | else 1686 | name="$author" 1687 | email="$author" 1688 | #name="Unknown_$author" 1689 | #email="Missing_$author" 1690 | fi 1691 | echo "$author = $name <$email@$DEFAULT_DOMAIN>" >> "$AUTHORS_FILE" 1692 | done 1693 | } 1694 | 1695 | ignore () 1696 | { 1697 | svn_sed=$(echo "$SVN_DIR"|sed s/'\/'/'\\\/'/g) 1698 | svn pg -R svn:ignore "$SVN_DIR" |sort -u|sed s/^"$svn_sed\/"/''/g|sed s/^"$REPO_NAME\/"/''/g|sed s/^"$REPO_NAME\/"/''/g > "$IGNORE_FILE" 2>&1 1699 | result=$? 1700 | 1701 | if [ "$result" ] && [ -f "$IGNORE_FILE" ]; then 1702 | echo "SUCCESS: gitignore template created from svn properties." | tee -a "$STATUS" 1703 | else 1704 | echo "FAILURE: something went wrong while creating the gitignore template" 1705 | fail 1706 | fi 1707 | } 1708 | 1709 | create_primary () 1710 | { 1711 | echo "Converting primary repository \"$REPO_NAME\" to git..." | tee -a "$STATUS" 1712 | 1713 | create_repo "$REPO_NAME" "Subversion conversion of $SVN_REPO" 1714 | 1715 | if has_trunk "$SVN_DIR" ; then 1716 | echo "Repository \"$REPO_NAME\" was found to have a trunk at root level, this could take a while..." | tee -a "$PRNLOG" 1717 | 1718 | if ! $UNDO ; then 1719 | git_clone "$REPO_NAME" 1720 | convert "$REPO_NAME" 1721 | fi 1722 | 1723 | else 1724 | echo "Creating skeleton repository for \"$REPO_NAME\"" | tee -a "$PRNLOG" 1725 | 1726 | mkdir "$TMP_DIR/$REPO_NAME.git" 1727 | cd "$TMP_DIR/$REPO_NAME.git" 1728 | 1729 | git init &>> "$PRNLOG" 1730 | git checkout -b master &>> "$PRNLOG" 1731 | 1732 | if ! $DRYRUN ; then 1733 | add_submodules "$REPO_NAME" 1734 | fi 1735 | git_et_al_primary "$SVN_DIR" 1736 | git_bfg_lfs "$REPO_NAME" 1737 | 1738 | if $NOORG ; then 1739 | git remote add origin "https://$NTID:$TOKEN@$GITHUB/$NTID/$REPO_NAME.git" &>> "$PRNLOG" 1740 | else 1741 | git remote add origin "https://$NTID:$TOKEN@$GITHUB/$ORG/$REPO_NAME.git" &>> "$PRNLOG" 1742 | fi 1743 | 1744 | git add * &>> "$PRNLOG" 1745 | cat "$IGNORE_FILE" > .gitignore 1746 | git add .gitignore &>> "$PRNLOG" 1747 | git commit -m 'SVN2GitHub conversion Completion.' &>> "$PRNLOG" 1748 | git push -u origin master &>> "$PRNLOG" 1749 | fi 1750 | 1751 | svn_tree 1752 | } 1753 | 1754 | git_et_al_primary () 1755 | { 1756 | local name=$(basename "$1") 1757 | local search_dir="$SVN_DIR" 1758 | local except='$' 1759 | 1760 | nested_projects=$(find "$search_dir" -mindepth 1 -maxdepth 1 -type d|grep -Ev "/$TRUNK/|/$TRUNK$|/$TAG/|/$TAG$|/$BRANCH/|/$BRANCH$|/\.svn|/\.subversion") 1761 | top_files=$(find "$search_dir" -mindepth 1 -maxdepth 1 -type f) 1762 | 1763 | if [ -n "$nested_projects" ]; then 1764 | et_al_dirs=$(find "$search_dir" -type d|grep -Ev "/$TRUNK/|/$TRUNK$|/$TAG/|/$TAG$|/$BRANCH/|/$BRANCH$|/\.svn|/\.subversion|/\.git|$1$") 1765 | filter="($(echo "${ModuleOrigins[@]}"|sed s/', '/'|'/g|sed s/' '/'[[:space:]]'/g|rev|cut -c 2-|rev))" 1766 | other_dirs=$(echo "$et_al_dirs"|grep -Ev "$filter") 1767 | fi 1768 | 1769 | if [ -n "$other_dirs" ] || [ -n "$top_files" ]; then 1770 | cd "$TMP_DIR/$name.git" 1771 | 1772 | if [ -n "$other_dirs" ]; then 1773 | echo "Adding directories and files from \"$SVN_REPO\" to \"$REPO_NAME\" that were not part of any trunk, branch, tag or meta-data..." | tee -a "$STATUS" 1774 | for dir in $other_dirs; do 1775 | et_al_files=$(find "$dir" -mindepth 1 -maxdepth 1 -type f|grep -Ev "/$TRUNK/|/$TRUNK$|/$TAG/|/$TAG$|/$BRANCH/|/$BRANCH$|/\.svn|/\.subversion|/\.git|$1$") 1776 | other_files=$(echo "$et_al_files"|grep -Ev "$filter") 1777 | dir_path=$(echo "$dir"|awk -F "$SVN_DIR/" '{print $2}') 1778 | dir_perm=$(stat -c "%a" "$dir") 1779 | 1780 | if [ ! -d "$TMP_DIR/$name.git/$dir_path" ]; then 1781 | mkdir -p "$TMP_DIR/$name.git/$dir_path" 1782 | fi 1783 | 1784 | chmod "$dir_perm" "$TMP_DIR/$name.git/$dir_path" 1785 | 1786 | for file in $other_files; do 1787 | if [ -n "$dir_path" ]; then 1788 | cp -pv "$file" "$TMP_DIR/$name.git/$dir_path/" &>> "$CPLOG" 1789 | else 1790 | cp -pv "$file" "$TMP_DIR/$name.git/" &>> "$CPLOG" 1791 | fi 1792 | done 1793 | git add "$dir_path" &>> "$CNVLOG" 1794 | done 1795 | fi 1796 | 1797 | if [ -n "$top_files" ]; then 1798 | echo "Adding top level files from \"$SVN_REPO\" to \"$REPO_NAME\" that were not part of a trunk, branch, tag or meta-data..." | tee -a "$STATUS" 1799 | for file in $top_files; do 1800 | cp -pv "$file" "$TMP_DIR/$name.git/" &>> "$CPLOG" 1801 | git add "$TMP_DIR/$name.git/$(basename "$file")" 1802 | done 1803 | fi 1804 | git add * &>> "$CNVLOG" 1805 | fi 1806 | } 1807 | 1808 | git_et_al () 1809 | { 1810 | local search_dir=${ModulePaths["$1"]} 1811 | local original_name=${ModuleNames["$1"]} 1812 | local split="$SVN_DIR/${ModuleRelatives["$1"]}/" 1813 | 1814 | nested_projects=$(find "$search_dir" -mindepth 1 -maxdepth 1 -type d|grep -Ev "/$TRUNK/|/$TRUNK$|/$TAG/|/$TAG$|/$BRANCH/|/$BRANCH$|/\.svn|/\.subversion|/\.git") 1815 | top_files=$(find "$search_dir" -mindepth 1 -maxdepth 1 -type f) 1816 | if [ -n "$nested_projects" ]; then 1817 | other_dirs=$(find "$search_dir" -type d|grep -Ev "/$TRUNK/|/$TRUNK$|/$TAG/|/$TAG$|/$BRANCH/|/$BRANCH$|/\.svn|/\.subversion|$original_name$") 1818 | fi 1819 | if [ -n "$other_dirs" ] || [ -n "$top_files" ]; then 1820 | cd "$TMP_DIR/$1.git" 1821 | if [ -n "$other_dirs" ]; then 1822 | echo "Adding directories and files from to \"$1\" that were not part of any trunk, branch, tag or meta-data..." | tee -a "$STATUS" 1823 | for dir in $other_dirs; do 1824 | dir_path=$(echo "$dir"|awk -F "$split" '{print $2}') 1825 | other_files=$(find "$split/$dir_path" -mindepth 1 -maxdepth 1 -type f|grep -Ev "/$TRUNK/|/$TRUNK$|/$TAG/|/$TAG$|/$BRANCH/|/$BRANCH$|/\.svn|/\.subversion|/\.git") 1826 | dir_perm=$(stat -c "%a" "$dir") 1827 | 1828 | if [ ! -d "$TMP_DIR/$1.git/$dir_path" ]; then 1829 | mkdir -p "$TMP_DIR/$1.git/$dir_path" &>> "$CPLOG" 1830 | fi 1831 | chmod "$dir_perm" "$TMP_DIR/$1.git/$dir_path" &>> "$CPLOG" 1832 | 1833 | for file in $other_files; do 1834 | cp -pv "$file" "$TMP_DIR/$1.git/$dir_path/" &>> "$CPLOG" 1835 | done 1836 | git add "$dir_path" &>> "$CNVLOG" 1837 | done 1838 | fi 1839 | 1840 | if [ -n "$top_files" ]; then 1841 | echo "Adding top level files from to \"$1\" that were not part of a trunk, branch, tag or meta-data..." | tee -a "$STATUS" 1842 | for file in $top_files; do 1843 | cp -pv "$file" "$TMP_DIR/$1.git/" &>> "$CPLOG" 1844 | git add "$TMP_DIR/$1.git/$(basename "$file")" 1845 | done 1846 | fi 1847 | git add * &>> "$CNVLOG" 1848 | fi 1849 | } 1850 | 1851 | clean_logs () 1852 | { 1853 | for log in "$COLOG" "$CNLOG" "$STATUS" "$SVNLOG" "$CNVLOG" "$CRTLOG" "$UNDLOG" "$PHLOG" "$IMPLOG" "$FAMILY" "$GHLOG" "$LFSLOG" "$PRNLOG" "$BFGLOG" "$RPTLOG" "$CPLOG" "$DBGLOG" "$RTYLOG"; do 1854 | if [ -f "$log" ]; then 1855 | echo '' > "$log" 1856 | fi 1857 | done 1858 | 1859 | rm -rf "$SUB_DIR" 1860 | rm -rf "$TMP_DIR"/*bfg-report 1861 | } 1862 | 1863 | report () 1864 | { 1865 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1866 | echo "| -- REPORT -- |" >> "$RPTLOG" 1867 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1868 | if [ -n "$MOD_COUNT" ]; then 1869 | echo "| TOTAL of $MOD_COUNT svn repositories found" >> "$RPTLOG" 1870 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1871 | fi 1872 | 1873 | if [ -f "$STATUS" ]; then 1874 | if [ -n "$(grep EMPTY "$STATUS"|sort -u)" ]; then 1875 | empty_count=$(grep EMPTY "$STATUS"|sort -u|wc -l) 1876 | echo "| $empty_count modules were skipped because they were found to be empty" >> "$RPTLOG" 1877 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1878 | fi 1879 | 1880 | if [ "$(grep RENAMED "$STATUS")" ]; then 1881 | conflict_count=$(grep RENAMED "$STATUS"|sort -u |awk '{print $2}'|sort -u|wc -l) 1882 | echo "| $conflict_count submodules were renamed in github due to naming conflict" >> "$RPTLOG" 1883 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1884 | fi 1885 | fi 1886 | 1887 | if [ -f "$CRTLOG" ]; then 1888 | creation_count=$(grep -F 'SUCCESS' "$CRTLOG"|sort -u|wc -l) 1889 | echo "| $creation_count repositories were created in $GITHUB" >> "$RPTLOG" 1890 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1891 | fi 1892 | 1893 | if [ -f "$UNDLOG" ]; then 1894 | delete_count=$(grep -F 'SUCCESS' "$UNDLOG"|sort -u|wc -l) 1895 | echo "| $delete_count repositories were removed from $GITHUB" >> "$RPTLOG" 1896 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1897 | fi 1898 | 1899 | if [ -f "$AUTHORS_FILE" ]; then 1900 | author_count=$(cat "$AUTHORS_FILE"|sort -u|wc -l) 1901 | echo "| $author_count authors were found in $SVN_REPO" >> "$RPTLOG" 1902 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1903 | fi 1904 | 1905 | if [ -f "$IGNORE_FILE" ]; then 1906 | ignore_count=$(cat "$IGNORE_FILE"|sort -u|wc -l) 1907 | echo "| $ignore_count ignore attriubutes were found in $SVN_REPO" >> "$RPTLOG" 1908 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1909 | fi 1910 | 1911 | if [ -f "$LFSLOG" ]; then 1912 | lfs_count=$(cat "$LFSLOG"|sort -u|wc -l) 1913 | echo "| $lfs_count files larger than $LFS were found" >> "$RPTLOG" 1914 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1915 | fi 1916 | 1917 | if [ -f "$RTYLOG" ]; then 1918 | if [ -n "$(grep 'RETRY #1' "$RTYLOG"|sort -u)" ]; then 1919 | retry_count=$(grep 'RETRY #1' "$RTYLOG"|sort -u|wc -l) 1920 | echo "| $retry_count events occured that failed at first and were retried." >> "$RPTLOG" 1921 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1922 | fi 1923 | 1924 | if [ "$(grep FAILURE: "$RTYLOG")" ]; then 1925 | refail_count=$(grep FAILURE: "$RTYLOG"|sort -u|wc -l) 1926 | echo "| $refail_count events were re-tried and failed completely in this order." >> "$RPTLOG" 1927 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1928 | echo "| -- RETRY FAILURES -- |" >> "$RPTLOG" 1929 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1930 | for refail in $(grep FAILURE: "$RTYLOG"); do 1931 | echo "| $refail " >> "$RPTLOG" 1932 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1933 | done 1934 | fi 1935 | fi 1936 | if [ "$(grep WARNING: "$STATUS")" ]; then 1937 | warning_count=$(grep WARNING: "$STATUS"|wc -l) 1938 | echo "| $warning_count warnings occured" >> "$RPTLOG" 1939 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1940 | echo "| -- WARNINGS -- |" >> "$RPTLOG" 1941 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1942 | for warning in $(grep WARNING: "$STATUS"); do 1943 | echo "| $warning " >> "$RPTLOG" 1944 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1945 | done 1946 | fi 1947 | if [ "$(grep FAILURE: "$STATUS")" ]; then 1948 | failure_count=$(grep FAILURE: "$STATUS"|wc -l) 1949 | echo "| $failure_count failures occured" >> "$RPTLOG" 1950 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1951 | echo "| -- FAILURES -- |" >> "$RPTLOG" 1952 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1953 | for failure in $(grep FAILURE: "$STATUS"); do 1954 | echo "| $failure " >> "$RPTLOG" 1955 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1956 | done 1957 | fi 1958 | 1959 | if [ -f "$FAMILY" ]; then 1960 | cat "$FAMILY" 1961 | fi 1962 | 1963 | if [ -f "$RPTLOG" ]; then 1964 | cat "$RPTLOG" 1965 | fi 1966 | 1967 | } 1968 | 1969 | main 1970 | -------------------------------------------------------------------------------- /package/rpmbuild/SPECS/svn-to-github.spec: -------------------------------------------------------------------------------- 1 | Name: svn-to-github 2 | Version: 1.16.0 3 | Release: 0 4 | Summary: Converts Svn Repos to Github Public or Enterprise in Bulk 5 | URL: https://github.com/Comcast/svn-to-github 6 | Source0: svn-to-github-1.16.0.tar.gz 7 | License: GPL 8 | BuildArch: noarch 9 | BuildRoot: %{_tmppath}/%{name}-buildroot 10 | Provides: svn-to-github 11 | Requires: git >= 1.8.2, subversion, git-svn, java-1.7.0-openjdk, expect, curl, gawk, findutils, coreutils, sed, yum, grep, procps, glibc-common, which, initscripts, tree, bash >= 4.0 12 | %description 13 | This will convert svn repositories into Github repositories, if nested svn projects exist, those will be converted to git submodules preserving the directory tree. 14 | %prep 15 | %setup -q 16 | %build 17 | %install 18 | install --directory $RPM_BUILD_ROOT/opt/svn-to-github 19 | install --directory $RPM_BUILD_ROOT/etc/svn-to-github 20 | install --directory $RPM_BUILD_ROOT/opt/svn-to-github/bin 21 | install -m 0644 README.md $RPM_BUILD_ROOT/opt/svn-to-github/README.md 22 | install -m 0644 README.md $RPM_BUILD_ROOT/opt/svn-to-github/config.yml 23 | install -m 0755 svn-to-github $RPM_BUILD_ROOT/opt/svn-to-github/bin/svn-to-github 24 | %clean 25 | rm -rf $RPM_BUILD_ROOT 26 | %post 27 | ln -s /opt/svn-to-github/bin/svn-to-github /usr/local/bin/svn-to-github 28 | ln -s /opt/svn-to-github/config.yml /etc/svn-to-github/config.yml 29 | echo "svn-to-github installation complete! svn-to-github --help for Instructions" 30 | %dir /opt/svn-to-github 31 | %dir /opt/svn-to-github/bin 32 | %dir /etc/svn-to-github 33 | %files 34 | /opt/svn-to-github/bin/svn-to-github 35 | /opt/svn-to-github/README.md 36 | /opt/svn-to-github/config.yml 37 | -------------------------------------------------------------------------------- /package/rpmbuild/re-build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SETUP BUILD 4 | if [ ! -f ~/.rpmmacros ]; then cp .rpmmacros ~/ ; fi 5 | if [ ! -x /bin/rpmdev-setuptree ]; then yum -y install rpmdevtools ; fi 6 | if [ ! -x /bin/rpmbuild ]; then yum -y install rpm-build ; fi 7 | if [ ! -x /bin/git ]; then yum -y install git ; fi 8 | cd ~/ 9 | rm -rf rpmbuild 10 | 11 | if [ ! -d ~/svn-to-github ]; then 12 | git clone https://github.com/Comcast/svn-to-github.git 13 | else 14 | pushd ~/svn-to-github && git pull && popd 15 | fi 16 | 17 | rpmdev-setuptree 18 | 19 | if [ -z "$1" ] ; then echo "specify a version" ; exit 1 ; fi 20 | if [ -z "$2" ] ; then echo "specify a release" ; exit 1 ; fi 21 | if [ -z "$3" ] ; then echo "specify a build" ; exit 1 ; fi 22 | 23 | cp ~/svn-to-github/svn-to-github.spec ~/rpmbuild/SPECS/svn-to-github.spec 24 | cp -r ~/svn-to-github ~/svn-to-github-$1.$2.$3 25 | tar czf svn-to-github-$1.$2.$3.tar.gz svn-to-github-$1.$2.$3/ 26 | mv ~/svn-to-github-$1.$2.$3 ~/rpmbuild/SOURCES/ 27 | mv svn-to-github-$1.$2.$3.tar.gz ~/rpmbuild/SOURCES/ 28 | sed -i "s/Version: [0-9].*/Version: $1\.$2\.$3/g" ~/rpmbuild/SPECS/svn-to-github.spec 29 | sed -i "s/Release: [0-9].*/Release: $3/g" ~/rpmbuild/SPECS/svn-to-github.spec 30 | sed -i "s/Source0: svn-to-github-[0-9]{1,2}.[0-9]{1,2}.[0-9]{1,2}.tar.gz/Source0: svn-to-github-$1.$2.$3.tar.gz/g" ~/rpmbuild/SPECS/svn-to-github.spec 31 | rpmbuild -ba ~/rpmbuild/SPECS/svn-to-github.spec 32 | -------------------------------------------------------------------------------- /re-build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SETUP BUILD 4 | if [ ! -f ~/.rpmmacros ]; then cp .rpmmacros ~/ ; fi 5 | if [ ! -x /bin/rpmdev-setuptree ]; then yum -y install rpmdevtools ; fi 6 | if [ ! -x /bin/rpmbuild ]; then yum -y install rpm-build ; fi 7 | if [ ! -x /bin/git ]; then yum -y install git ; fi 8 | cd ~/ 9 | rm -rf rpmbuild 10 | 11 | if [ ! -d ~/svn-to-github ]; then 12 | git clone https://github.com/Comcast/svn-to-github.git 13 | else 14 | pushd ~/svn-to-github && git pull && popd 15 | fi 16 | 17 | rpmdev-setuptree 18 | 19 | if [ -z "$1" ] ; then echo "specify a version" ; exit 1 ; fi 20 | if [ -z "$2" ] ; then echo "specify a release" ; exit 1 ; fi 21 | if [ -z "$3" ] ; then echo "specify a build" ; exit 1 ; fi 22 | 23 | cp ~/svn-to-github/svn-to-github.spec ~/rpmbuild/SPECS/svn-to-github.spec 24 | cp -r ~/svn-to-github ~/svn-to-github-$1.$2.$3 25 | tar czf svn-to-github-$1.$2.$3.tar.gz svn-to-github-$1.$2.$3/ 26 | mv ~/svn-to-github-$1.$2.$3 ~/rpmbuild/SOURCES/ 27 | mv svn-to-github-$1.$2.$3.tar.gz ~/rpmbuild/SOURCES/ 28 | sed -i "s/Version: [0-9].*/Version: $1\.$2\.$3/g" ~/rpmbuild/SPECS/svn-to-github.spec 29 | sed -i "s/Release: [0-9].*/Release: $3/g" ~/rpmbuild/SPECS/svn-to-github.spec 30 | sed -i "s/Source0: svn-to-github-[0-9]{1,2}.[0-9]{1,2}.[0-9]{1,2}.tar.gz/Source0: svn-to-github-$1.$2.$3.tar.gz/g" ~/rpmbuild/SPECS/svn-to-github.spec 31 | rpmbuild -ba ~/rpmbuild/SPECS/svn-to-github.spec 32 | -------------------------------------------------------------------------------- /svn-to-github: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Name: SVN2GitHub 4 | # Author: Dustin Morgan 5 | # Date: May 12, 2017 6 | # Purpose: Ease the conversion of SVN to Git and use git submodules for 7 | # non-standard svn layouts with nested projects 8 | # 9 | #shellcheck svn-to-github --exclude=SC2143,SC2155,SC2001,SC2162,SC2002,SC2013,SC2116,SC2005,SC2003,SC2035,SC2010,SC2164,SC2054,SC2140,SC2034,SC2068,SC2145,SC2062,SC2063,SC2012,SC2128,SC2188,SC2129,SC2126 10 | 11 | #ARGS 12 | while test $# -gt 0; do 13 | case "$1" in 14 | -h|--help) 15 | echo "Run this with answer flags for [required] options to bypass interactive mode when converting an svn repo to a github repo." 16 | echo "" 17 | echo "options:" 18 | echo "-h, --help" 19 | echo "--ghuser=swizzley [required]: the ghuser used to login to github" 20 | echo "--token=d938a09236f449c5ea9f6bcfbcb64bacda55e620 [required]: the github user authorization token, setup here: https://github.com/settings/tokens" 21 | echo "--svn-url=https://svn.yourdomain.net/my_repo [required]: to specify the source svn repo" 22 | echo "--svn-user=username [optional]: to specify the source svn repo username" 23 | echo "--svn-pass=password [optional]: to specify the source svn repo password, if special characters are used quote password in single-quotes" 24 | echo "--org=my_org [required option A]: the github organization to create the repo(s) in" 25 | echo "--no-org [required option B]: don't create repos under an org, but rather under a user instead, caveat is repo(s) cannot already exist in user's primary Organization" 26 | echo "--no-branches [optional]: do not convert branches, ignore all branches" 27 | echo "--no-tags [optional]: do not convert tags, ignore all tags" 28 | echo "--no-tag-branches [optional]: do not allow tags to be branches, convert tags to releases only" 29 | echo "--no-preserve-trunk [optional]: do not keep the trunk directory, so trunk becomes the root of master" 30 | echo "--no-other-repos [optional]: do not create submodule repos, only create the primary repo but nest them if they already exist" 31 | echo "--new-name=repo_name [required or optional]: rename your new repo name in github, required if name already exists in github under user or organization, whichever is specified by option A or B above" 32 | echo "--lfs-limit=50 [default=50, max=50]: specify the large-file-size limit, --lfs-limit=2 ~ 2 Megabytes, maximum in github is 50, default is 50, 2-5 is optimal" 33 | echo "--blob-limit=50 [default=50, max=50]: specify the max size of blobs allowed, --blob-limit=50 ~ 50 Megabytes, maximum in github.com is 50M, default is 50, < 50 is optimal" 34 | echo "--authors=/path/to/authors_file [optional]: to import an authors file rather than generate one, may not provide correct email address for users who do not conform to first_last@yourdomain.com" 35 | echo "--ignore=/path/to/ignore_file [optional]: to import the gitignore file rather than convert it from the existing svn ignore attributes" #TODO 36 | echo "--work-dir=/path/to/store/files [optional]: to specify a working directory for creating files" 37 | echo "--branches=label1,label2 or --branches=custom_label [optional]: a csv list of alternative branch dir names or specify just one" 38 | echo "--trunk=label1,label2 or --trunk=custom_label [optional]: a csv list of alternative trunk dir names or specify just one" 39 | echo "--tags=label1,label2 or --tags=custom_label [optional]: a csv list of alternative tag dir names or specify just one" 40 | echo "--svn-prefix=svn- [optional]: specify a prefix given to an otherwise standard-layout of svn directories, '--svn-prefix=svn-' == '--branch=svn-branches --trunk=svn-trunk --tags=svn-tags'" 41 | echo "--private [optional]: make all repo(s) private in github" 42 | echo "--check-size [optional]: estimate the size on disk needed for conversion of non-standard-layouts, can be very time consuming" 43 | echo "--safe [optional]: be prompted before all github repository creations or deletions when combined with --undo" 44 | echo "--install [optional]: dependency check & installation, Warning: Will Fail if dependencies are not met. Deps: git-svn, java-7, git-lfs, bfg-repo-cleaner ver. >= 1.12.5 " 45 | echo "--force [warning!]: force the creation, will delete all data from previous run and possibly create duplicate submodule repos with suffixes in github, it will NOT allow bypassing of --new-name for the primary repo" 46 | echo "--undo [warning!]: delete the github repos from a previous run, but preserve all log data" 47 | exit 48 | ;; 49 | --authors*) 50 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 51 | export AUTHORS_FILE=$(echo "$1" | sed -e 's/^[^=]*=//g') 52 | export IMPORT_AUTHORS=false 53 | shift 54 | ;; 55 | --svn-url*) 56 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 57 | export SVN_REPO=$(echo "$1" | sed -e 's/^[^=]*=//g') 58 | shift 59 | ;; 60 | --org*) 61 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 62 | export ORG=$(echo "$1" | sed -e 's/^[^=]*=//g') 63 | shift 64 | ;; 65 | --ghuser*) 66 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 67 | export NTID=$(echo "$1" | sed -e 's/^[^=]*=//g') 68 | shift 69 | ;; 70 | --token*) 71 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 72 | export TOKEN=$(echo "$1" | sed -e 's/^[^=]*=//g') 73 | shift 74 | ;; 75 | --new-name*) 76 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 77 | export RENAME=$(echo "$1" | sed -e 's/^[^=]*=//g') 78 | shift 79 | ;; 80 | --ignore*) 81 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 82 | export IGNORE_FILE=$(echo "$1" | sed -e 's/^[^=]*=//g') 83 | export IMPORT_IGNORE=false 84 | shift 85 | ;; 86 | --work-dir*) 87 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 88 | export DATA_DIR=$(echo "$1" | sed -e 's/^[^=]*=//g') 89 | shift 90 | ;; 91 | --force*) 92 | export FORCE=true 93 | shift 94 | ;; 95 | --undo*) 96 | export UNDO=true 97 | shift 98 | ;; 99 | --branches*) 100 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 101 | export ALT_BRANCH=$(echo "$1" | sed -e 's/^[^=]*=//g') 102 | shift 103 | ;; 104 | --tags*) 105 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 106 | export ALT_TAGS=$(echo "$1" | sed -e 's/^[^=]*=//g') 107 | shift 108 | ;; 109 | --trunk*) 110 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 111 | export ALT_TRUNK=$(echo "$1" | sed -e 's/^[^=]*=//g') 112 | shift 113 | ;; 114 | --svn-prefix*) 115 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 116 | export PREFIX=$(echo "$1" | sed -e 's/^[^=]*=//g') 117 | shift 118 | ;; 119 | --svn-user*) 120 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 121 | export SVN_USER=$(echo "$1" | sed -e 's/^[^=]*=//g') 122 | shift 123 | ;; 124 | --svn-pass*) 125 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 126 | export SVN_PASS=$(echo "$1" | sed -e 's/^[^=]*=//g') 127 | shift 128 | ;; 129 | --no-branches*) 130 | export BRANCHES=false 131 | shift 132 | ;; 133 | --no-tags*) 134 | export TAGS=false 135 | shift 136 | ;; 137 | --no-preserve-trunk*) 138 | export PRESERVE_TRUNK=false 139 | shift 140 | ;; 141 | --no-tag-branches*) 142 | export TAG_BRANCH=false 143 | shift 144 | ;; 145 | --check-size*) 146 | export CHECK_SIZE=true 147 | shift 148 | ;; 149 | --private*) 150 | export PRIVATE='true' 151 | shift 152 | ;; 153 | --install*) 154 | export INSTALL=true 155 | shift 156 | ;; 157 | --no-org*) 158 | export NOORG=true 159 | shift 160 | ;; 161 | --lfs-limit*) 162 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 163 | export LFS=$(echo "$1" | sed -e 's/^[^=]*=//g') 164 | shift 165 | ;; 166 | --blob-limit*) 167 | if [ ! "$(echo "$1"|grep =)" ]; then echo "--options must be in the form of --option="; exit 1; fi 168 | export BLOB_LIMIT=$(echo "$1" | sed -e 's/^[^=]*=//g') 169 | shift 170 | ;; 171 | --safe*) 172 | export SAFE=true 173 | shift 174 | ;; 175 | --no-other-repos*) 176 | export DRYRUN=true 177 | shift 178 | ;; 179 | *) 180 | echo "Invalid Argument" 181 | exit 1 182 | ;; 183 | esac 184 | done 185 | 186 | #RUN BEGINS HERE 187 | main () 188 | { 189 | eval $(config /etc/svn-to-github/config.yml) 190 | set_vars 191 | 192 | if [ -z "$ORG" ] || [ -z "$NTID" ] || [ -z "$TOKEN" ]; then 193 | interactive 194 | if ! $NOORG ; then 195 | echo "Checking github organization permissions..." | tee -a "$STATUS" 196 | gh_test_org 197 | fi 198 | if ! $UNDO ; then 199 | echo "Checking for name conflict in github for \"$REPO_NAME\"" | tee -a "$STATUS" 200 | gh_test_repo 201 | fi 202 | else 203 | export AUTH="-u $NTID:$TOKEN" 204 | if ! $NOORG ; then 205 | echo "Checking github organization permissions..." | tee -a "$STATUS" 206 | gh_test_org 207 | fi 208 | if ! $UNDO ; then 209 | echo "Checking for name conflict in github for \"$REPO_NAME\"" | tee -a "$STATUS" 210 | gh_test_repo 211 | fi 212 | fi 213 | 214 | if $UNDO && [ "$SAFE" == false ] ; then 215 | echo "Deleting all repos from github that were created from a previos run..." | tee -a "$STATUS" 216 | undo 217 | report 218 | elif [ "$UNDO" == true ] && [ "$FORCE" == true ] && [ "$SAFE" == true ]; then 219 | echo "Force initiated! Removing all traces of previos run and starting over" | tee -a "$STATUS" 220 | mkdir -p "$TMP_DIR" 221 | checkout 222 | create_submodules 223 | create_primary 224 | create_repos 225 | undo 226 | report 227 | echo "SUCCESS: Conversion complete" | tee -a "$STATUS" 228 | exit 0 229 | else 230 | echo "NOTICE: Getting started..." | tee -a "$STATUS" 231 | clean_logs | tee -a "$STATUS" 232 | 233 | if $INSTALL ; then 234 | if [ "$USER" == 'root' ]; then 235 | echo "Checking prerequisites..." | tee -a "$STATUS" 236 | install 237 | elif [ "$(touch "$WORK_DIR"/.lock && test -f "$WORK_DIR"/.lock)" ] && [ "$(yum list installed|grep git-svn &> /dev/null)" ] && [ "$(git lfs &> /dev/null)" ] && [ "$(which java &> /dev/null)" ]; then 238 | echo "WARNING: Not running as root, but things look safe to proceed." | tee -a "$STATUS" 239 | else 240 | echo "FAILURE: Authentication failed." >> "$STATUS" 241 | fail 242 | fi 243 | else 244 | if [ "$(touch "$WORK_DIR"/.lock && test -f "$WORK_DIR"/.lock)" ] && [ "$(which git svn)" ]; then 245 | echo "WARNING: Install was skipped, but things look safe to proceed." | tee -a "$STATUS" 246 | else 247 | echo "WARNING: Something went wrong, either permission denied on \"$WORK_DIR\" or the package \"git-svn\" is not installed or is missing from \$PATH." >> "$STATUS" 248 | fail 249 | fi 250 | fi 251 | 252 | if $FORCE ; then 253 | echo "Force initiated! Removing all traces of previos run..." | tee -a "$STATUS" 254 | rm -rf "$AUTHORPROG" "$BFGPROG" "$TMP_DIR" "$SVN_DIR" 255 | fi 256 | 257 | if $CHECK_SIZE ; then 258 | echo "Checking size, this could take a while..." | tee -a "$STATUS" 259 | check_size | tee -a "$STATUS" 260 | fi 261 | 262 | if [ ! -d "$SVN_DIR/.svn" ]; then 263 | echo "Checking out \"$SVN_REPO\" locally, this could take a while... see \"$COLOG\" for details." | tee -a "$STATUS" 264 | if [ -n "$SVN_USER" ] && [ -n "$SVN_PASS" ]; then 265 | save_svn_credentials &>> "$STATUS" 266 | fi 267 | checkout 268 | else 269 | echo "WARNING: \"$SVN_DIR\" already exists, if this is not a complete and up-to-date revision this whole process will result in erroneous conversion." | tee -a "$STATUS" 270 | echo "Checking for submodules..." 271 | export MODULES=$(find "$SVN_DIR" -mindepth 1 -type d -regextype posix-extended -regex "^.*/$TRUNK"|grep -Ev "/$TAG/"|grep -Ev "/$BRANCH/"|rev|cut -d / -f 2-|rev|sort -u) 272 | fi 273 | 274 | if [ ! -f "$AUTHORS_FILE" ]; then 275 | echo "Getting list of authors..." | tee -a "$STATUS" 276 | authors | tee -a "$STATUS" 277 | fi 278 | 279 | if [ ! -f "$IGNORE_FILE" ]; then 280 | echo "Gathering svn ignore properties..." | tee -a "$STATUS" 281 | ignore | tee -a "$STATUS" 282 | fi 283 | 284 | if [ ! -d "$TMP_DIR" ]; then 285 | mkdir "$TMP_DIR" 286 | git config --global user.name "SVN Converter" &>> "$CNVLOG" 287 | git config --global user.email svn_converter@devnull.com &>> "$CNVLOG" 288 | fi 289 | 290 | if $DRYRUN ; then 291 | echo "NOTICE: --no-other-repos enabled, no submodules will be created..." 292 | create_submodules | tee -a "$STATUS" 293 | create_primary | tee -a "$STATUS" 294 | else 295 | create_repos | tee -a "$STATUS" 296 | fi 297 | archive_job | tee -a "$STATUS" 298 | #svn_depth 299 | report | tee -a "$STATUS" 300 | echo "SUCCESS: Conversion complete" | tee -a "$STATUS" 301 | exit 0 302 | fi 303 | } 304 | 305 | #DEFAULTS 306 | config () 307 | { 308 | local prefix=$2 309 | local s='[[:space:]]*' w='[a-zA-Z0-9_]*' 310 | fs=$(echo @|tr @ '\034') 311 | sed -ne "s|^\($s\):|\1|" \ 312 | -e "s|^\($s\)\($w\)$s:$s[\"']\(.*\)[\"']$s\$|\1$fs\2$fs\3|p" \ 313 | -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" $1 | 314 | awk -F$fs '{ 315 | indent = length($1)/2; 316 | vname[indent] = $2; 317 | for (i in vname) {if (i > indent) {delete vname[i]}} 318 | if (length($3) > 0) { 319 | vn=""; for (i=0; i> "$STATUS" 666 | else 667 | echo "FAILURE: User \"$NTID\" is not a member of \"$ORG\", or \"$ORG\" does not exist." >> "$STATUS" 668 | fail 669 | fi 670 | } 671 | 672 | gh_test_repo () 673 | { 674 | if $NOORG ; then 675 | if [ ! "$(curl -s "$AUTH" "$GITHUB_API/users/$NTID/repos"|grep -F '"name":'|grep "$REPO_NAME"$)" ]; then 676 | echo "SUCCESS: Repository \"$REPO_NAME\" does not yet exist under \"$NTID\"." >> "$STATUS" 677 | else 678 | echo "FAILURE: Repo \"$REPO_NAME\" already exists under user \"$NTID\" try re-running with --new-name=repo_name" >> "$STATUS" 679 | fail 680 | fi 681 | else 682 | if [ ! "$(curl -s "$AUTH" "$GITHUB_API/orgs/$ORG/repos"|grep -F '"name":'|grep "$REPO_NAME"$)" ]; then 683 | echo "SUCCESS: Repository \"$REPO_NAME\" does not yet exist in organization \"$ORG\"." | tee -a "$STATUS" 684 | else 685 | echo "FAILURE: Repo \"$REPO_NAME\" already exists in \"$ORG\" try re-running with --new-name=repo_name" >> "$STATUS" 686 | fail 687 | fi 688 | fi 689 | } 690 | 691 | check_size () 692 | { 693 | if [ ! "$(echo "$SVN_REPO"| grep -Ei '^file://')" ]; then 694 | repo_file_info=$(svn list -vR "$SVN_REPO"|awk '{if ($3 !="") sum+=$3} END {print sum}') 695 | repo_approx_size=$(expr "$(echo "$repo_file_info")" * 1.25 ) 696 | else 697 | repo_file_info=$(du -s "$SVN_DIR"|awk '{print $1}') 698 | repo_approx_size="$repo_file_info" 699 | fi 700 | 701 | avail=$(expr "$(df "$WORK_DIR"|grep ^/|awk '{print $4}')" - 0 ) 702 | needed=$(expr "$repo_approx_size" * 2.25 ) 703 | 704 | if [ "$avail" -gt "$needed" ]; then 705 | echo "SUCCESS: Disk space is adequate..." | tee -a "$STATUS" 706 | return 0 707 | else 708 | echo "FAILURE: Disk space in work-dir \"$WORK_DIR\" only has $avail Bytes available and $needed are needed." >> "$STATUS" 709 | fail 710 | fi 711 | } 712 | 713 | #### PROMPTS 714 | interactive () 715 | { 716 | if [ -z "$ORG" ] && [ "$NOORG" == false ]; then 717 | echo "Please enter the github Orgnaization, or re-run with --no-org to save repositories under NTID namespace instead of Organization namespace." 718 | read ORG 719 | export ORG 720 | fi 721 | if [ -z "$NTID" ]; then 722 | echo "Please enter the NTID of a member of \"$ORG\"." 723 | read NTID 724 | export NTID 725 | fi 726 | if [ -z "$TOKEN" ]; then 727 | echo "Please enter the Personal Access Token for \"$NTID\"." 728 | read -s TOKEN 729 | export TOKEN 730 | fi 731 | export AUTH="-u $NTID:$TOKEN" 732 | } 733 | 734 | gh_auth () 735 | { 736 | if [ "$(curl -s "$AUTH" "$GITHUB_API/user"|grep login|grep "$NTID")" ]; then 737 | echo "SUCCESS: Authentication worked!" 738 | else 739 | echo "FAILURE: Authentication failed." 740 | export ORG="" 741 | export NTID="" 742 | export TOKEN="" 743 | export RETRY=$(expr $RETRY - 1) 744 | if [ "$RETRY" -gt 0 ]; then 745 | interactive 746 | else 747 | echo "FAILURE: Failed to authenticate user $NTID with token $TOKEN to $GITHUB $RETRY times." >> "$STATUS" 748 | fail 749 | fi 750 | fi 751 | } 752 | 753 | safe_prompt () 754 | { 755 | if ! $FORCE ; then 756 | answer="" 757 | echo "WARNING: SAFE-MODE ENABLED. To create repo $1 in github, type 'yes' (case-sensitive)" 758 | read answer 759 | if [ "$answer" != 'yes' ]; then 760 | return 1 761 | else 762 | return 0 763 | fi 764 | else 765 | return 0 766 | fi 767 | } 768 | 769 | #### TESTS 770 | has_trunk () 771 | { 772 | if [ -n "$(find "$1" -maxdepth 1 -type d -regextype posix-extended -regex "^.*/$TRUNK")" ]; then 773 | return 0 774 | else 775 | return 1 776 | fi 777 | } 778 | 779 | has_tags () 780 | { 781 | if [ "$(ls -l "$1"|grep ^d|grep -qE "$TAG")" ]; then 782 | return 0 783 | else 784 | return 1 785 | fi 786 | } 787 | 788 | has_branches () 789 | { 790 | if [ "$(ls -l "$1"|grep ^d|grep -qE "$BRANCH")" ]; then 791 | return 0 792 | else 793 | return 1 794 | fi 795 | } 796 | 797 | is_stdlayout () 798 | { 799 | if has_trunk "$1" &> /dev/null && has_branches "$1" &> /dev/null && has_tags "$1" &> /dev/null ; then 800 | return 0 801 | else 802 | return 1 803 | fi 804 | } 805 | 806 | is_empty () 807 | { 808 | if [ -z "$(find "$1" -type f -print -quit)" ]; then 809 | return 0 810 | else 811 | return 1 812 | fi 813 | } 814 | 815 | gh_conflict () 816 | { 817 | if $NOORG ; then 818 | check=$(curl -s "$AUTH" "$GITHUB_API/users/$NTID/repos"|grep -F '"name":'|grep "$1") 819 | else 820 | check=$(curl -s "$AUTH" "$GITHUB_API/orgs/$ORG/repos"|grep -F '"name":'|grep "$1") 821 | fi 822 | if [ "$check" -ne 0 ]; then fail ; fi 823 | } 824 | 825 | #### ACTIONS 826 | create_repo () 827 | { 828 | if [ -f "$RPO_DIR/$1.json" ] && [ ! $FORCE ]; then 829 | echo "WARNING: $1.json already exists, overriding with --force, may result in a duplicate repo in github." | tee -a "$STATUS" 830 | elif [ $FORCE ] || [ ! -f "$RPO_DIR/$1.json" ]; then 831 | create=true 832 | desc=$(echo "$2"|sed s/\\/\\/.*@/\\/\\//g) 833 | 834 | if $SAFE ; then 835 | if [ ! "$(safe_prompt "$1")" ] && [ "$UNDO" == false ]; then 836 | echo "WARNING: $1 was not confirmed in the --safe mode prompt, and was skipped from conversion." 837 | create=false 838 | fi 839 | fi 840 | 841 | if $create ; then 842 | cat < "$RPO_DIR"/"$1".json 843 | { 844 | "name": "$1", 845 | "description": "$desc", 846 | "homepage": "", 847 | "private": $PRIVATE, 848 | "has_issues": true, 849 | "has_wiki": true, 850 | "has_downloads": true 851 | } 852 | EOF 853 | if ! $UNDO ; then 854 | if $NOORG ; then 855 | curl -X POST -s "$AUTH" "$GITHUB_API/user/repos" --data "@$RPO_DIR/$1.json" >> "$GHLOG" 2>&1 856 | result=$? 857 | if [ "$result" -eq 0 ]; then 858 | echo "SUCCESS: $1 repository created at https://$GITHUB/$NTID/$1" | tee -a "$CRTLOG" "$STATUS" 859 | else 860 | echo "RETRYING: create_repo $1" >> "$RTYLOG" 861 | retry curl -X POST -s "$AUTH" "$GITHUB_API"/user/repos --data @"$RPO_DIR"/"$1".json 862 | fi 863 | else 864 | curl -X POST -s "$AUTH" "$GITHUB_API/orgs/$ORG/repos" --data "@$RPO_DIR/$1.json" >> "$GHLOG" 2>&1 865 | result=$? 866 | if [ "$result" -eq 0 ]; then 867 | echo "SUCCESS: $1 repository created at https://$GITHUB/$ORG/$1" | tee -a "$CRTLOG" "$STATUS" 868 | else 869 | echo "RETRYING: create_repo $1" >> "$RTYLOG" 870 | retry curl -X POST -s "$AUTH" "$GITHUB_API"/orgs/"$ORG"/repos --data @"$RPO_DIR"/"$1".json 871 | fi 872 | fi 873 | fi 874 | fi 875 | fi 876 | } 877 | 878 | gh_delete () 879 | { 880 | delete=true 881 | 882 | if $SAFE ; then 883 | if [ ! "$(safe_prompt "$1")" ]; then 884 | echo "WARNING: $1 was not confirmed in the --safe mode prompt, and was skipped from deletion." 885 | delete=false 886 | fi 887 | fi 888 | 889 | if $delete ; then 890 | if $NOORG ; then 891 | curl -X DELETE -s "$AUTH" "$GITHUB_API/repos/$NTID/$1" >> "$GHLOG" 2>&1 892 | result=$? 893 | else 894 | curl -X DELETE -s "$AUTH" "$GITHUB_API/repos/$ORG/$1" >> "$GHLOG" 2>&1 895 | result=$? 896 | fi 897 | if [ "$result" -eq 0 ]; then 898 | if $NOORG ; then 899 | echo "SUCCESS: $1 repository removed from github at https://$GITHUB/$NTID/$1" | tee -a "$UNDLOG" "$STATUS" 900 | else 901 | echo "SUCCESS: $1 repository created at https://$GITHUB/$ORG/$1" | tee -a "$UNDLOG" "$STATUS" 902 | fi 903 | else 904 | echo "FAILURE: $1 failed to remove, see \"$GHLOG\" for details." >> "$STATUS" 905 | fail 906 | fi 907 | fi 908 | } 909 | 910 | author_prog () 911 | { 912 | cat < $AUTHORPROG 913 | #!/bin/bash 914 | echo "\$1 <\$1>"; 915 | EOF 916 | chmod +x "$AUTHORPROG" 917 | } 918 | 919 | #TODO - Unused 920 | #svn_depth () 921 | #{ 922 | # cd "$SVN_DIR" 923 | # echo "" > "$FAMILY" 924 | # MAX_DEPTH=$(echo $MODULES|sed s/' '/"_"/g|awk -F "$SVN_DIR" '{print $2}'|awk -F "/" '{print NF - 1}'|sort -nr|head -n1) 925 | # 926 | # if [ "$MAX_DEPTH" -gt 6 ];then 927 | # echo "WARNING: Tree depth too deep to label as family" | tee -a "$STATUS" 928 | # fi 929 | # 930 | # for i in $(seq 1 $MAX_DEPTH) 931 | # do 932 | # COUNT=$(echo $MODULES|sed s/' '/"_"/g|awk -F "$SVN_DIR" '{print $2}'|awk -F "/" '{print NF - 1 }'|grep "$i"|wc -l) 933 | # min_depth=$(expr $i + 1) 934 | # if [ "$i" -eq 1 ]; then 935 | # echo "$COUNT svn siblings found." >> "$FAMILY" 936 | # elif [ "$i" -eq 2 ]; then 937 | # echo "$COUNT svn children found." >> "$FAMILY" 938 | # elif [ "$i" -eq 3 ]; then 939 | # echo "$COUNT svn grandchildren found." >> "$FAMILY" 940 | # elif [ "$i" -eq 4 ]; then 941 | # echo "$COUNT svn great grandchildren found." >> "$FAMILY" 942 | # elif [ "$i" -eq 5 ]; then 943 | # echo "$COUNT svn great great grandchildren found." >> "$FAMILY" 944 | # elif [ "$i" -eq 6 ]; then 945 | # echo "$COUNT svn great great great grandchildren found." >> "$FAMILY" 946 | # else 947 | # echo "$COUNT svn repos found at depth $i" >> "$FAMILY" 948 | # fi 949 | # done 950 | # 951 | # export MOD_COUNT=$(echo $MODULES|wc -w) 952 | #} 953 | 954 | get_parent () 955 | { 956 | if [ -d "$1" ]; then 957 | cd "$1" 958 | local parent="" 959 | 960 | while [ "$(pwd)" != "$SVN_DIR" ] && [ -z "$parent" ]; do 961 | if has_trunk "$(dirname "$(pwd)")" ; then 962 | if [ "$(dirname "$(pwd)"|grep -E /"$TRUNK"$)" ]; then 963 | cd ../ 964 | else 965 | parent="$SUB_STRING$(basename "$(dirname "$(pwd)")")" 966 | fi 967 | else 968 | cd ../ 969 | fi 970 | done 971 | 972 | if [ -z "$parent" ]; then 973 | if [ "$(basename "$(dirname "$1")"|grep ^$SUB_STRING)" ]; then 974 | echo "FAILURE: Cannot process directory that begins with \"$SUB_STRING\"" >> "$STATUS" 975 | fail 976 | else 977 | local nearest_parent="$(basename "$(dirname "$1")")" 978 | if [ ! "$(echo "$nearest_parent"|grep -E "$TRUNK")" ]; then 979 | local parent="$nearest_parent" 980 | else 981 | local parent="$REPO_NAME_lower" 982 | fi 983 | fi 984 | fi 985 | 986 | echo "$parent" 987 | else 988 | echo "FAILURE: $1 is missing, Unknown Error." >> "$STATUS" 989 | fail 990 | fi 991 | } 992 | 993 | #Only called for known submodules with children 994 | get_parent_path () 995 | { 996 | if [ -d "$1" ]; then 997 | cd "$1" 998 | while [ "$1" != "$SVN_DIR" ] && [ -z "$parent_path" ]; do 999 | if has_trunk "$(dirname "$1")" ; then 1000 | if [ "$(echo "$(dirname "$1")"|grep -E "$TRUNK")" ]; then 1001 | parent_path="$(dirname "$(dirname "$1")")" 1002 | else 1003 | parent_path="$(dirname "$1")" 1004 | fi 1005 | else 1006 | cd ../ 1007 | fi 1008 | done 1009 | 1010 | echo "$parent_path" 1011 | else 1012 | echo "FAILURE: $1 is missing, Unknown Error." >> "$STATUS" 1013 | fail 1014 | fi 1015 | } 1016 | 1017 | #Only called for known submodules with children 1018 | get_child_path () 1019 | { 1020 | cd "$1" && cd ../ 1021 | 1022 | until has_trunk "$(pwd)" ; do 1023 | cd ../ 1024 | done 1025 | child_path=$(echo "$1"| awk -F "$(basename "$(pwd)")/" '{print $2}') 1026 | 1027 | echo "$child_path" 1028 | } 1029 | 1030 | resolve_name () 1031 | { 1032 | 1033 | local name=$(basename "$1"|sed s/' '/_/g) 1034 | local prefix=$REPO_NAME_lower 1035 | local parent_name=$(get_parent "$1"|sed s/' '/_/g) 1036 | 1037 | if [ "$parent_name" == "$REPO_NAME_lower" ]; then 1038 | test_name="$prefix-$name" 1039 | elif [ ! "$(echo "$parent_name"|grep ^$SUB_STRING)" ]; then 1040 | test_name="$prefix-$parent_name-$name" 1041 | else 1042 | parent_trunk=$(echo "$parent_name"|sed s/"^$SUB_STRING"/""/g) 1043 | test_name="$prefix--$parent_trunk-$name" 1044 | fi 1045 | if ! $DRYRUN ; then 1046 | if gh_conflict "$test_name" ; then 1047 | suffix=0 1048 | until ! gh_conflict "$test_name-$suffix" 1049 | do 1050 | suffix=$(expr $suffix + 1) 1051 | done 1052 | gh_name="$test_name-$suffix" 1053 | else 1054 | gh_name="$test_name" 1055 | fi 1056 | else 1057 | gh_name="$test_name" 1058 | fi 1059 | echo "$gh_name" 1060 | } 1061 | 1062 | create_submodules () 1063 | { 1064 | if [ -n "$MODULES" ]; then 1065 | echo "Creating submodules in github..." 1066 | declare -gA SubModules 1067 | declare -gA ClonePids 1068 | declare -gA ConvertPids 1069 | declare -gA ModuleNames 1070 | declare -gA ModulePaths 1071 | declare -gA ModuleUrls 1072 | declare -gA ModuleRelatives 1073 | declare -gA ChildPaths 1074 | declare -ga ModuleOrigins 1075 | declare -ga Modules 1076 | 1077 | if [ ! -d "$SUB_DIR" ]; then 1078 | mkdir "$SUB_DIR" 1079 | fi 1080 | #TODO DEBUG 1081 | for module in $MODULES ; do #full run 1082 | local relative_path=$(echo "$module"|awk -F "$SVN_DIR/" '{print $2}') 1083 | local name=$(basename "$module") 1084 | local real_name=$(resolve_name "$module") 1085 | 1086 | if is_empty "$module"; then 1087 | echo "NOTICE: Skipping EMPTY \"$name\"" >> "$STATUS" 1088 | else 1089 | if [ ! -f "$RPO_DIR/$real_name.json" ] || [ "$DRYRUN" == true ] ; then 1090 | #Setup Arrays 1091 | SubModules["$real_name"]="" #for adding submodules 1092 | ClonePids["$real_name"]="" #for tracking Pids of Clone jobs 1093 | ConvertPids["$real_name"]="" #for tracking Pids of Convert jobs 1094 | ModuleNames["$real_name"]="$name" #for name tanslation 1095 | ModuleUrls["$real_name"]="$EVC/$REPO_NAME_ORIG/$relative_path" #for cloning svn 1096 | ModulePaths["$real_name"]="$module" #for location reference 1097 | ModuleRelatives["$real_name"]="$relative_path" #for et_al filter exception 1098 | ModuleOrigins+=("$relative_path",) #for et_al filter 1099 | Modules+=("$real_name") #for multi-threading loops 1100 | 1101 | if [ "$(echo "$real_name"|grep -E ^"$REPO_NAME_lower"--)" ]; then 1102 | #Create list "$SUB_DIR/$real_parent" of nested submodules 1103 | ChildPaths["$real_name"]="$(get_child_path "$module")" #for location reference of nested repo 1104 | real_parent="$(resolve_name "$(get_parent_path "$module")")" 1105 | echo "$real_name" >> "$SUB_DIR/$real_parent" 1106 | else 1107 | echo "$real_name" >> "$SUB_DIR/$REPO_NAME" 1108 | real_parent=$REPO_NAME 1109 | fi 1110 | if ! $DRYRUN ; then 1111 | #Create the repo in github 1112 | submodule_repo "$real_name" "$real_parent" 1113 | fi 1114 | fi 1115 | fi 1116 | done 1117 | fi 1118 | } 1119 | 1120 | add_submodules () 1121 | { 1122 | if [ -f "$SUB_DIR/$1" ] && [ "$UNDO" == false ]; then 1123 | cd "$TMP_DIR/$1.git" 1124 | 1125 | declare -A Children 1126 | 1127 | #Create Array of Children 1128 | for child in $(cat "$SUB_DIR/$1"); do 1129 | Children["$child"]="$child" 1130 | done 1131 | 1132 | #Loop through processing children until all children are done cloning 1133 | while [ "${#Children[@]}" -ne 0 ]; do 1134 | for child in "${Children[@]}" ; do 1135 | #echo "DEBUG child is $child" 1136 | clone_pid=${SubModules["$child"]} #race condition for submodules of submodules 1137 | #echo "DEBUG clonepid $clone_pid" 1138 | convert_pid=${ConvertPids["$child"]} #race condition of submodules of primary repo 1139 | #echo "DEBUG convertpid $convert_pid" 1140 | if [ "$1" == "$REPO_NAME" ]; then 1141 | original_path=${ModuleRelatives["$child"]} 1142 | else 1143 | original_path=${ChildPaths["$child"]} 1144 | fi 1145 | 1146 | if ([ "$(ps aux|awk '{print $2}'|grep ^$clone_pid$)" ] && [ -n "$clone_pid" ]) || ([ "$(ps aux|awk '{print $2}'|grep ^$convert_pid$)" ] && [ -n "$convert_pid" ]); then 1147 | echo "Waiting for child submodule(s) to finish converting. Sleeping $SLEEP_TIME seconds..." | tee -a "$STATUS" 1148 | sleep $SLEEP_TIME 1149 | else 1150 | if $NOORG ; then 1151 | add_url="https://$NTID:$TOKEN@$GITHUB/$NTID/$child.git" 1152 | else 1153 | add_url="https://$NTID:$TOKEN@$GITHUB/$ORG/$child.git" 1154 | fi 1155 | 1156 | git submodule add "$add_url" "$original_path" &>> "$PRNLOG" 1157 | result=$? 1158 | if [ "$result" -ne 0 ]; then 1159 | echo "RETRYING: add_submodules $1" >> "$RTYLOG" 1160 | retry git submodule add "$add_url" "$original_path" 1161 | fi 1162 | unset Children["$child"] 1163 | fi 1164 | done 1165 | done 1166 | unset Children 1167 | 1168 | #Remove NTID & Token from git URL 1169 | sed -i s/https:\\/\\/$NTID:$TOKEN@$GITHUB\\//git@$GITHUB:/g .gitmodules 1170 | 1171 | #Add submodules to repo 1172 | git add .gitmodules 1173 | echo "SUCCESS: All svn-sub-repos for $1 finished converting to git submodules!" 1174 | fi 1175 | } 1176 | 1177 | multi_thread () 1178 | { 1179 | echo "Converting submodules of \"$REPO_NAME\" to git..." | tee -a "$STATUS" 1180 | while [ "${#ClonePids[@]}" -ne 0 ]; do 1181 | for module in "${Modules[@]}" ; do 1182 | pid=${ClonePids["$module"]} 1183 | module_path=${ModulePaths["$module"]} 1184 | 1185 | if [ ! -d "$TMP_DIR/$module.git" ] && [ -z "$pid" ]; then 1186 | git_clone "$module" & 1187 | ClonePids["$module"]=$! 1188 | SubModules["$module"]=${ClonePids["$module"]} #resolve race condition for cloning and adding multiple submodules 1189 | elif [ -n "$pid" ] && [ ! "$(ps aux|awk '{print $2}'|grep ^$pid$)" ] && [ -d "$TMP_DIR/$module.git" ]; then 1190 | unset ClonePids["$module"] 1191 | convert "$module" & 1192 | ConvertPids["$module"]=$! 1193 | fi 1194 | done 1195 | echo "Waiting for submodule clone(s) to complete. Sleeping $SLEEP_TIME seconds..." | tee -a "$STATUS" 1196 | sleep $SLEEP_TIME 1197 | done 1198 | echo "SUCCESS: All submodules finished cloning!" | tee -a "$STATUS" 1199 | } 1200 | 1201 | submodule_repo () 1202 | { 1203 | if $NOORG ; then 1204 | create_repo "$1" "Submodule of $2 at https://$GITHUB/$NTID/$2" 1205 | else 1206 | create_repo "$1" "Submodule of $2 at https://$GITHUB/$ORG/$2" 1207 | fi 1208 | } 1209 | 1210 | 1211 | create_repos () 1212 | { 1213 | if [ -n "$MODULES" ] && [ "$DRYRUN" == false ]; then 1214 | create_submodules 1215 | if $SAFE; then 1216 | single_thread 1217 | else 1218 | multi_thread 1219 | fi 1220 | fi 1221 | create_primary 1222 | } 1223 | 1224 | git_ignore () 1225 | { 1226 | cd "$TMP_DIR/$1.git" 1227 | git svn show-ignore > .gitignore 1228 | if [ -s .gitignore ]; then 1229 | if [ ! -f "$IGNORE_FILE" ]; then 1230 | ignore 1231 | fi 1232 | cat "$IGNORE_FILE" > .gitignore 1233 | fi 1234 | git add .gitignore &>> "$CNVLOG" 1235 | 1236 | #TODO add Sync back option here #SYNC DOES NOT ALLOW FOR BFG 1237 | git config --remove-section svn-remote.svn 1238 | } 1239 | 1240 | git_add_remote () 1241 | { 1242 | if $NOORG ; then 1243 | if [ ! "$(git remote -v|grep origin|grep "https://$NTID:$TOKEN@$GITHUB/$NTID/$1.git")" ]; then 1244 | git remote add origin "https://$NTID:$TOKEN@$GITHUB/$NTID/$1.git" 1245 | fi 1246 | else 1247 | if [ ! "$(git remote -v|grep origin|grep "https://$NTID:$TOKEN@$GITHUB/$ORG/$1.git")" ]; then 1248 | git remote add origin "https://$NTID:$TOKEN@$GITHUB/$ORG/$1.git" 1249 | fi 1250 | fi 1251 | } 1252 | 1253 | retry () 1254 | { 1255 | if [ "$UNDO" == true ]; then 1256 | return 0 1257 | else 1258 | retry=1 1259 | until $@ ; do 1260 | echo "RETRY #$retry : $@" | tee -a "$RTYLOG" 1261 | sleep $SLEEP_TIME 1262 | retry=$(expr $retry + 1) 1263 | if [ "$retry" -eq 3 ]; then 1264 | echo "RETRY #$retry : $@" | tee -a "$RTYLOG" 1265 | echo "$@"|bash 1266 | result=$? 1267 | if [ "$result" -ne 0 ]; then 1268 | echo "FAILURE: $@" | tee -a "$STATUS" "$RTYLOG" 1269 | else 1270 | echo "SUCCESS: $@"| tee -a "$STATUS" "$RTYLOG" 1271 | fi 1272 | break 1273 | fi 1274 | done 1275 | fi 1276 | } 1277 | 1278 | convert () 1279 | { 1280 | cd "$TMP_DIR/$1.git" 1281 | 1282 | git config --global push.default upstream 1283 | 1284 | if [ "$(git branch -a|grep [[:space:]]master$)" ]; then 1285 | if [ ! "$(git branch -a|grep \*[[:space:]]master$)" ]; then 1286 | git checkout master &>> "$CNVLOG" 1287 | fi 1288 | else 1289 | git checkout -b master &>> "$CNVLOG" 1290 | fi 1291 | 1292 | git_ignore "$1" | tee -a "$STATUS" 1293 | git_bfg_lfs "$1" | tee -a "$STATUS" 1294 | git commit -m 'SVN2GitHub Master initialization.' &>> "$CNVLOG" 1295 | git_add_remote "$1" 1296 | 1297 | git push -u origin master >> "$PHLOG" 2&>1 1298 | result=$? 1299 | if [ "$result" -ne 0 ]; then 1300 | echo "RETRYING: convert init $1" >> "$RTYLOG" 1301 | retry git push -u origin master 1302 | fi 1303 | 1304 | git config remote.origin.push 'refs/remotes/*:refs/heads/*' 1305 | 1306 | if ! $DRYRUN ; then 1307 | add_submodules "$1" | tee -a "$STATUS" 1308 | fi 1309 | 1310 | if [ "$1" == "$REPO_NAME" ]; then 1311 | git_et_al_primary "$SVN_DIR" | tee -a "$STATUS" 1312 | else 1313 | git_et_al "$1" | tee -a "$STATUS" 1314 | fi 1315 | 1316 | git commit -m 'SVN2GitHub added other files from svn repo not under version control.' 1317 | git_bfg_lfs "$1" 1318 | git commit -m 'SVN2GitHub Master branch conversion complete.' &>> "$CNVLOG" 1319 | git push -u origin master >> "$PHLOG" 2&>1 1320 | result=$? 1321 | if [ "$result" -ne 0 ]; then 1322 | #TODO cleanup? git_bfg_lfs "$1" 1323 | echo "RETRYING: convert master $1" >> "$RTYLOG" 1324 | retry git push -u origin master 1325 | fi 1326 | 1327 | if $TAGS ; then 1328 | git_tags | tee -a "$STATUS" 1329 | fi 1330 | 1331 | if $BRANCHES ; then 1332 | git_branches | tee -a "$STATUS" 1333 | fi 1334 | 1335 | if $PRESERVE_TRUNK; then 1336 | #git_nest_submodules_in_trunk "$1" 1337 | git_nest_trunk_as_submodule "$1" | tee -a "$STATUS" 1338 | fi 1339 | echo "SUCCESS: Conversion of repo \"$1\" complete." | tee -a "$STATUS" 1340 | } 1341 | 1342 | #TODO - Unused 1343 | #git_nest_submodules_in_trunk () 1344 | #{ 1345 | # git checkout -b trunk &>> "$CNVLOG" 1346 | # git checkout trunk &>> "$CNVLOG" 1347 | # git pull --no-edit origin trunk &>> "$CNVLOG" 1348 | # add_submodules "$1" 1349 | # git commit -m 'SVN2GitHub conversion: Trunk Preservation Enabled.' &>> "$CNVLOG" 1350 | # git push -u origin trunk &>> "$PHLOG" 1351 | # if [ "$?" -ne 0 ]; then 1352 | # echo "RETRYING: nest submodules in trunk $1" >> "$RTYLOG" 1353 | # retry git push -u origin trunk 1354 | # fi 1355 | #} 1356 | 1357 | git_nest_trunk_as_submodule () 1358 | { 1359 | if [ ! "$(git branch -a|grep \*[[:space:]]master$)" ]; then 1360 | git checkout master &>> "$CNVLOG" 1361 | fi 1362 | 1363 | git pull --no-edit origin master &>> "$CNVLOG" 1364 | 1365 | if $NOORG ; then 1366 | trunk_url="https://$NTID:$TOKEN@$GITHUB/$NTID/$(basename "$(pwd)")" 1367 | else 1368 | trunk_url="https://$NTID:$TOKEN@$GITHUB/$ORG/$(basename "$(pwd)")" 1369 | fi 1370 | 1371 | git submodule add -b trunk -f "$trunk_url" ./trunk >> "$CNVLOG" 2>&1 1372 | result=$? 1373 | if [ "$result" -ne 0 ]; then 1374 | echo "RETRYING: git_nest_trunk_as_submodule" >> "$RTYLOG" 1375 | retry git submodule add -b trunk -f "$trunk_url" ./trunk 1376 | fi 1377 | : 1378 | #Remove NTID & Token from git URL 1379 | sed -i s/https:\\/\\/$NTID:$TOKEN@$GITHUB\\//git@$GITHUB:/g .gitmodules 1380 | git add .gitmodules &>> "$CNVLOG" 1381 | git commit -m 'SVN2GitHub conversion: added trunk branch as submodule with path ./trunk' &>> "$CNVLOG" 1382 | git push -u origin master >> "$PHLOG" 2>&1 1383 | result=$? 1384 | if [ "$result" -ne 0 ]; then 1385 | echo "RETRYING: git_nest_trunk_as_submodule" >> "$RTYLOG" 1386 | retry git push -u origin master 1387 | fi 1388 | } 1389 | 1390 | git_clone () 1391 | { 1392 | echo "NOTICE: Cloning $1 from svn to git now..." | tee -a "$STATUS" 1393 | if [ "$1" == "$REPO_NAME" ]; then 1394 | local url=$SVN_REPO 1395 | else 1396 | local url=${ModuleUrls["$1"]} 1397 | fi 1398 | if ! $UNDO ; then 1399 | if [ ! -d "$TMP_DIR/$1.git" ] || [ "$FORCE" == true ]; then 1400 | git svn clone "$url" "$LAYOUT" --authors-file="$AUTHORS_FILE" --authors-prog="$AUTHORPROG" --quiet "$TMP_DIR/$1.git" >> "$CNLOG" 2>&1 1401 | result=$? 1402 | if [ "$result" -ne 0 ]; then 1403 | echo "RETRYING: clone $1" >> "$RTYLOG" 1404 | retry git svn clone "$url" "$LAYOUT" --authors-file="$AUTHORS_FILE" --authors-prog="$AUTHORPROG" --quiet "$TMP_DIR"/"$1".git 1405 | fi 1406 | else 1407 | echo "FAILURE: \"$TMP_DIR/$1.git\" failed to clone completely in a previous run, try again with --force" >> "$STATUS" 1408 | fail 1409 | fi 1410 | echo "SUCCESS: Clone of $1 complete, converting $1 now..." | tee -a "$STATUS" 1411 | fi 1412 | } 1413 | 1414 | svn_tree () 1415 | { 1416 | cd "$TMP_DIR/$REPO_NAME.git" 1417 | if [ ! "$(git branch -a|grep -q \*[[:space:]]master$)" ]; then 1418 | git checkout master &>> "$CNVLOG" 1419 | fi 1420 | git pull --no-edit origin master &>> "$CNVLOG" 1421 | tree "$SVN_DIR" &> .svn_tree 1422 | git add .svn_tree &>> "$CNVLOG" 1423 | git commit -m 'Original SVN Repository Tree' &>> "$CNVLOG" 1424 | git push -u origin master >> "$PHLOG" 2>&1 1425 | result=$? 1426 | if [ "$result" -ne 0 ]; then 1427 | echo "RETRYING: svn_tree" >> "$RTYLOG" 1428 | retry git push -u origin master 1429 | fi 1430 | } 1431 | 1432 | archive_job () 1433 | { 1434 | cp "$RPO_DIR/"* "$LOG_DIR/" 1435 | cp "$AUTHORS_FILE" "$LOG_DIR/" 1436 | cp "$TMP_DIR/$REPO_NAME.git/.svn_tree" "$LOG_DIR/svn_tree" 1437 | cd "$WORK_DIR" 1438 | timestamp=$(date +%s) 1439 | tar czf "$REPO_NAME-$timestamp.tar.gz" "$LOG_DIR" 1440 | mv "$REPO_NAME-$timestamp.tar.gz" "$JOB_DIR" 1441 | } 1442 | 1443 | git_bfg_lfs () 1444 | { 1445 | sleep $SLEEP_TIME #not sure why lfs doesn't have a value sometimes 1446 | lfs=$(find "$TMP_DIR/$1.git" -type f -size +"$LFS"M|grep -Ev '/\.svn|/\.subversion|/\.git') 1447 | echo "DEBUG: LFS = $lfs" >> "$LFSLOG" 1448 | if [ -n "$lfs" ]; then 1449 | #TODO deprecate because it breaks lfs on current branch? 1450 | # if [ ! -d "$LFS_DIR" ]; then 1451 | # mkdir "$LFS_DIR" 1452 | # fi 1453 | # echo "" > "$LFS_DIR/$1" 1454 | # for big_file in $lfs; do 1455 | # if [ "$(basename $big_file|grep .)" ]; then 1456 | # extension=$(echo $big_file|rev|awk -F "." '{print $1}'|rev) 1457 | # echo pattern="'*.$extension'" >> "$LFS_DIR/$1" 1458 | # else 1459 | # echo patten="'$(basename $big_file)'" >> "$LFS_DIR/$1" 1460 | # fi 1461 | # done 1462 | 1463 | 1464 | # for lfs_pattern in $(grep pattern= "$LFS_DIR/$1"|awk -F "pattern=" '{print $2}'|sort -u); do 1465 | # java -jar "$BFGPROG" --convert-to-git-lfs $lfs_pattern --no-blob-protection "$TMP_DIR/$1.git" &>> "$BFGLOG" 1466 | # done 1467 | 1468 | git lfs install &>> "$LFSLOG" 1469 | 1470 | for file in $lfs; do 1471 | track_file=$(echo "$file"|awk -F "$(pwd)/" '{print $2}') 1472 | git lfs track "$track_file" &>> "$LFSLOG" 1473 | git add "$track_file" &>> "$LFSLOG" 1474 | done 1475 | 1476 | git add .gitattributes &>> "$LFSLOG" 1477 | git reflog expire --expire=now --all && git gc --prune=now --aggressive &>> "$BFGLOG" 1478 | fi 1479 | 1480 | java -jar "$BFGPROG" --strip-blobs-bigger-than "$BLOB_LIMIT"M --no-blob-protection &>> "$BFGLOG" 1481 | git reflog expire --expire=now --all && git gc --prune=now --aggressive &>> "$BFGLOG" 1482 | sleep $SLEEP_TIME 1483 | } 1484 | 1485 | git_tags () 1486 | { 1487 | for ref in $(git for-each-ref --format='%(refname)' refs/remotes/tags|cut -d / -f 4) 1488 | do 1489 | tag=$(echo "$ref"|sed s/%20/'_'/g) 1490 | git tag "$tag" "refs/remotes/tags/$ref" &>> "$CNVLOG" 1491 | done 1492 | 1493 | git push --tags >> "$PHLOG" 2>&1 1494 | result=$? 1495 | if [ "$result" -ne 0 ]; then 1496 | echo "RETRYING: git_tags $(pwd)" >> "$RTYLOG" 1497 | retry git push --tags 1498 | fi 1499 | } 1500 | 1501 | git_branches () 1502 | { 1503 | for ref in $(find .git/refs -type f |grep -Ev 'heads/|origin/|/stash'); do 1504 | mv "$ref" .git/refs/remotes/origin/ 1505 | done 1506 | git push -u origin >> "$PHLOG" 2>&1 1507 | result=$? 1508 | if [ "$result" -ne 0 ]; then 1509 | echo "RETRYING: push branches $(pwd)" >> "$RTYLOG" 1510 | retry git push -u origin 1511 | fi 1512 | 1513 | if ! $TAG_BRANCH; then 1514 | for branch in $(git branch -r|grep origin|grep tags/); do 1515 | git push origin --delete "origin/$branch" >> "$CNLOG" 2>&1 1516 | result=$? 1517 | if [ "$result" -ne 0 ]; then 1518 | echo "RETRYING: delete tag branches $(pwd)" >> "$RTYLOG" 1519 | retry git push origin --delete "origin/$branch" 1520 | fi 1521 | done 1522 | fi 1523 | } 1524 | 1525 | undo () 1526 | { 1527 | if [ -n "$(find "$RPO_DIR" -type f -name "*.json")" ]; then 1528 | UNDO=$(ls "$RPO_DIR"/*.json|rev|awk -F "/" '{print $1}'|rev|awk -F ".json" '{print $1}') 1529 | for repo in $UNDO; 1530 | do 1531 | gh_delete "$repo" 1532 | if [ -f "$RPO_DIR/$repo.json" ]; then 1533 | rm -f "$RPO_DIR/$repo.json" 1534 | fi 1535 | if [ -d "$TMP_DIR/$repo.git" ]; then 1536 | rm -rf "$TMP_DIR/$repo.git" 1537 | fi 1538 | done 1539 | elif [ -n "$(find "$JOB_DIR" -type f -name "$REPO_NAME*gz")" ]; then 1540 | zips=$(find "$JOB_DIR" -type f -name "$REPO_NAME*gz") 1541 | for tar in $zips; do 1542 | tar xf "$tar" 1543 | UNDO=$(find "$JOB_DIR" -type f -name "$REPO_NAME_lower*.json"|rev|awk -F "/" '{print $1}'|rev|awk -F ".json" '{print $1}') 1544 | for repo in $UNDO; 1545 | do 1546 | gh_delete "$repo" 1547 | find "$JOB_DIR" -type f -name "$repo.json" -delete 1548 | done 1549 | done 1550 | gh_delete "$REPO_NAME" 1551 | else 1552 | echo "The --undo flag was used, but there appears to be nothing to undo in \"$RPO_DIR\"" | tee -a "$STATUS" 1553 | fi 1554 | } 1555 | 1556 | single_thread () 1557 | { 1558 | for sub in $MODULES; do 1559 | submodule=$(echo "$sub"|awk -F "$SVN_DIR/" '{print $2}'|rev|awk -F "/" '{print $1}'|rev) 1560 | submodules=(${submodules[@]} "$submodule") 1561 | done 1562 | 1563 | for module in $submodules 1564 | do 1565 | if ! gh_conflict "$module" ; then 1566 | if $NOORG; then 1567 | create_repo "$module" "Submodule of https://$GITHUB/$NTID/$REPO_NAME" 1568 | else 1569 | create_repo "$module" "Submodule of https://$GITHUB/$ORG/$REPO_NAME" 1570 | fi 1571 | else 1572 | if $NOORG ; then 1573 | echo "WARNING: $module already exists in $GITHUB under $NTID" | tee -a "$STATUS" 1574 | else 1575 | echo "WARNING: $module already exists in $GITHUB under $ORG" | tee -a "$STATUS" 1576 | fi 1577 | num=0 1578 | until ! gh_conflict "$module-$num" 1579 | do 1580 | num=$(expr $num + 1) 1581 | done 1582 | if $NOORG; then 1583 | create_repo "$module-$num" "Submodule of https://$GITHUB/$NTID/$REPO_NAME" 1584 | else 1585 | create_repo "$module-$num" "Submodule of https://$GITHUB/$ORG/$REPO_NAME" 1586 | fi 1587 | if $NOORG ; then 1588 | echo "WARNING: $module was RENAMED to $module-$num in github because $module already exists under $NTID" | tee -a "$STATUS" 1589 | else 1590 | echo "WARNING: $module was RENAMED to $module-$num in github because $module already exists under $ORG" | tee -a "$STATUS" 1591 | fi 1592 | fi 1593 | git_clone "$module" 1594 | convert "$module" 1595 | done 1596 | } 1597 | 1598 | install () 1599 | { 1600 | yum makecache fast &>> "$STATUS" 1601 | 1602 | if [ ! "$(yum list installed|grep git-svn)" ]; then 1603 | echo "Installing git-svn..." | tee -a "$STATUS" 1604 | yum -qy install git-svn 1605 | fi 1606 | if [ ! "$(yum list installed|grep git-lfs)" ]; then 1607 | echo "Installing git-lfs..." | tee -a "$STATUS" 1608 | #Source from https://packagecloud.io/github/git-lfs 1609 | if [ "$(uname -a|awk '{print $3}'|grep el7)" ]; then 1610 | yum -qy install "$LFS_DOWNLOAD" 1611 | fi 1612 | if [ "$(uname -a|awk '{print $3}'|grep el6)" ]; then 1613 | yum -qy install "$LFS_DOWNLOAD" 1614 | fi 1615 | fi 1616 | if [ ! "$(yum list installed|grep openjdk)" ]; then 1617 | echo "Installing java-1.7.0-openjdk..." | tee -a "$STATUS" 1618 | yum -qy install java-1.7.0-openjdk 1619 | fi 1620 | if [ ! "$(yum list installed|grep expect)" ]; then 1621 | echo "Installing expect..." | tee -a "$STATUS" 1622 | yum -qy install expect 1623 | fi 1624 | if [ ! "$(yum list installed|grep tree)" ]; then 1625 | echo "Installing tree..." | tee -a "$STATUS" 1626 | yum -qy install tree 1627 | fi 1628 | if [ ! -f "$BFGPROG" ]; then 1629 | curl -o "$BFGPROG" -k -s "$BFG_DOWNLOAD" 1630 | fi 1631 | if [ ! -f "$AUTHORPROG" ]; then 1632 | author_prog 1633 | fi 1634 | } 1635 | 1636 | save_svn_credentials () 1637 | { 1638 | if [ ! "$(grep "$EVC" /root/.subversion/auth/svn.ssl.server/*)" ]; then 1639 | expect -c ' 1640 | spawn svn info --username=$::env(SVN_USER) $::env(SVN_REPO) 1641 | expect "Error validating server certificate" 1642 | send "p\r" 1643 | expect eof ' ; &> "$CNLOG" 1644 | fi 1645 | #TODO add rm -rf ~/.svn to --force or fix with individual users, and a check job running for user functions 1646 | #if [ ! "$(grep $SVN_USER /root/.subversion/auth/svn.simple/*)" ]; then 1647 | expect -c ' 1648 | spawn svn info --username=$::env(SVN_USER) $::env(SVN_REPO) 1649 | expect "Password for" 1650 | send "$::env(SVN_PASS)\r" 1651 | expect "Store password" 1652 | send "yes\r" 1653 | expect eof ' ; &> "$CNLOG" 1654 | #fi 1655 | } 1656 | 1657 | checkout () 1658 | { 1659 | cd "$WORK_DIR" 1660 | if [ -n "$SVN_USER" ] && [ -n "$SVN_PASS" ]; then 1661 | svn checkout --trust-server-cert --non-interactive --no-auth-cache --username="$SVN_USER" --password="$SVN_PASS" "$SVN_REPO" > "$COLOG" 2>&1 1662 | result=$? 1663 | else 1664 | svn checkout --trust-server-cert --non-interactive --no-auth-cache "$SVN_REPO" > "$COLOG" 2>&1 1665 | result=$? 1666 | fi 1667 | 1668 | if [ "$result" -eq 0 ]; then 1669 | echo "SUCCESS: \"$REPO_NAME\" was checked out." | tee -a "$STATUS" "$COLOG" 1670 | echo "Checking for submodules..." | tee -a "$STATUS" 1671 | export MODULES=$(find "$SVN_DIR" -mindepth 1 -type d -regextype posix-extended -regex "^.*/$TRUNK"|grep -Ev "/$TAG/"|grep -Ev "/$BRANCH/"|grep -v "$SVN_DIR"$|rev|cut -d / -f 2-|rev|sort -u) 1672 | else 1673 | echo "FAILURE: $REPO_NAME failed to checkout, see \"$COLOG\" for details." | tee -a "$STATUS" 1674 | fail 1675 | fi 1676 | } 1677 | 1678 | authors () 1679 | { 1680 | cd "$SVN_DIR" 1681 | for author in $(svn log -q |grep ^r|awk '{print $3}'|sort -u); do 1682 | if [ -n "$(getent passwd "$author")" ]; then 1683 | name=$(getent passwd "$author"|awk -F ":" '{print $5}') 1684 | email=$(echo "$name"|sed s/' '/'_'/g) 1685 | else 1686 | name="$author" 1687 | email="$author" 1688 | #name="Unknown_$author" 1689 | #email="Missing_$author" 1690 | fi 1691 | echo "$author = $name <$email@$DEFAULT_DOMAIN>" >> "$AUTHORS_FILE" 1692 | done 1693 | } 1694 | 1695 | ignore () 1696 | { 1697 | svn_sed=$(echo "$SVN_DIR"|sed s/'\/'/'\\\/'/g) 1698 | svn pg -R svn:ignore "$SVN_DIR" |sort -u|sed s/^"$svn_sed\/"/''/g|sed s/^"$REPO_NAME\/"/''/g|sed s/^"$REPO_NAME\/"/''/g > "$IGNORE_FILE" 2>&1 1699 | result=$? 1700 | 1701 | if [ "$result" ] && [ -f "$IGNORE_FILE" ]; then 1702 | echo "SUCCESS: gitignore template created from svn properties." | tee -a "$STATUS" 1703 | else 1704 | echo "FAILURE: something went wrong while creating the gitignore template" 1705 | fail 1706 | fi 1707 | } 1708 | 1709 | create_primary () 1710 | { 1711 | echo "Converting primary repository \"$REPO_NAME\" to git..." | tee -a "$STATUS" 1712 | 1713 | create_repo "$REPO_NAME" "Subversion conversion of $SVN_REPO" 1714 | 1715 | if has_trunk "$SVN_DIR" ; then 1716 | echo "Repository \"$REPO_NAME\" was found to have a trunk at root level, this could take a while..." | tee -a "$PRNLOG" 1717 | 1718 | if ! $UNDO ; then 1719 | git_clone "$REPO_NAME" 1720 | convert "$REPO_NAME" 1721 | fi 1722 | 1723 | else 1724 | echo "Creating skeleton repository for \"$REPO_NAME\"" | tee -a "$PRNLOG" 1725 | 1726 | mkdir "$TMP_DIR/$REPO_NAME.git" 1727 | cd "$TMP_DIR/$REPO_NAME.git" 1728 | 1729 | git init &>> "$PRNLOG" 1730 | git checkout -b master &>> "$PRNLOG" 1731 | 1732 | if ! $DRYRUN ; then 1733 | add_submodules "$REPO_NAME" 1734 | fi 1735 | git_et_al_primary "$SVN_DIR" 1736 | git_bfg_lfs "$REPO_NAME" 1737 | 1738 | if $NOORG ; then 1739 | git remote add origin "https://$NTID:$TOKEN@$GITHUB/$NTID/$REPO_NAME.git" &>> "$PRNLOG" 1740 | else 1741 | git remote add origin "https://$NTID:$TOKEN@$GITHUB/$ORG/$REPO_NAME.git" &>> "$PRNLOG" 1742 | fi 1743 | 1744 | git add * &>> "$PRNLOG" 1745 | cat "$IGNORE_FILE" > .gitignore 1746 | git add .gitignore &>> "$PRNLOG" 1747 | git commit -m 'SVN2GitHub conversion Completion.' &>> "$PRNLOG" 1748 | git push -u origin master &>> "$PRNLOG" 1749 | fi 1750 | 1751 | svn_tree 1752 | } 1753 | 1754 | git_et_al_primary () 1755 | { 1756 | local name=$(basename "$1") 1757 | local search_dir="$SVN_DIR" 1758 | local except='$' 1759 | 1760 | nested_projects=$(find "$search_dir" -mindepth 1 -maxdepth 1 -type d|grep -Ev "/$TRUNK/|/$TRUNK$|/$TAG/|/$TAG$|/$BRANCH/|/$BRANCH$|/\.svn|/\.subversion") 1761 | top_files=$(find "$search_dir" -mindepth 1 -maxdepth 1 -type f) 1762 | 1763 | if [ -n "$nested_projects" ]; then 1764 | et_al_dirs=$(find "$search_dir" -type d|grep -Ev "/$TRUNK/|/$TRUNK$|/$TAG/|/$TAG$|/$BRANCH/|/$BRANCH$|/\.svn|/\.subversion|/\.git|$1$") 1765 | filter="($(echo "${ModuleOrigins[@]}"|sed s/', '/'|'/g|sed s/' '/'[[:space:]]'/g|rev|cut -c 2-|rev))" 1766 | other_dirs=$(echo "$et_al_dirs"|grep -Ev "$filter") 1767 | fi 1768 | 1769 | if [ -n "$other_dirs" ] || [ -n "$top_files" ]; then 1770 | cd "$TMP_DIR/$name.git" 1771 | 1772 | if [ -n "$other_dirs" ]; then 1773 | echo "Adding directories and files from \"$SVN_REPO\" to \"$REPO_NAME\" that were not part of any trunk, branch, tag or meta-data..." | tee -a "$STATUS" 1774 | for dir in $other_dirs; do 1775 | et_al_files=$(find "$dir" -mindepth 1 -maxdepth 1 -type f|grep -Ev "/$TRUNK/|/$TRUNK$|/$TAG/|/$TAG$|/$BRANCH/|/$BRANCH$|/\.svn|/\.subversion|/\.git|$1$") 1776 | other_files=$(echo "$et_al_files"|grep -Ev "$filter") 1777 | dir_path=$(echo "$dir"|awk -F "$SVN_DIR/" '{print $2}') 1778 | dir_perm=$(stat -c "%a" "$dir") 1779 | 1780 | if [ ! -d "$TMP_DIR/$name.git/$dir_path" ]; then 1781 | mkdir -p "$TMP_DIR/$name.git/$dir_path" 1782 | fi 1783 | 1784 | chmod "$dir_perm" "$TMP_DIR/$name.git/$dir_path" 1785 | 1786 | for file in $other_files; do 1787 | if [ -n "$dir_path" ]; then 1788 | cp -pv "$file" "$TMP_DIR/$name.git/$dir_path/" &>> "$CPLOG" 1789 | else 1790 | cp -pv "$file" "$TMP_DIR/$name.git/" &>> "$CPLOG" 1791 | fi 1792 | done 1793 | git add "$dir_path" &>> "$CNVLOG" 1794 | done 1795 | fi 1796 | 1797 | if [ -n "$top_files" ]; then 1798 | echo "Adding top level files from \"$SVN_REPO\" to \"$REPO_NAME\" that were not part of a trunk, branch, tag or meta-data..." | tee -a "$STATUS" 1799 | for file in $top_files; do 1800 | cp -pv "$file" "$TMP_DIR/$name.git/" &>> "$CPLOG" 1801 | git add "$TMP_DIR/$name.git/$(basename "$file")" 1802 | done 1803 | fi 1804 | git add * &>> "$CNVLOG" 1805 | fi 1806 | } 1807 | 1808 | git_et_al () 1809 | { 1810 | local search_dir=${ModulePaths["$1"]} 1811 | local original_name=${ModuleNames["$1"]} 1812 | local split="$SVN_DIR/${ModuleRelatives["$1"]}/" 1813 | 1814 | nested_projects=$(find "$search_dir" -mindepth 1 -maxdepth 1 -type d|grep -Ev "/$TRUNK/|/$TRUNK$|/$TAG/|/$TAG$|/$BRANCH/|/$BRANCH$|/\.svn|/\.subversion|/\.git") 1815 | top_files=$(find "$search_dir" -mindepth 1 -maxdepth 1 -type f) 1816 | if [ -n "$nested_projects" ]; then 1817 | other_dirs=$(find "$search_dir" -type d|grep -Ev "/$TRUNK/|/$TRUNK$|/$TAG/|/$TAG$|/$BRANCH/|/$BRANCH$|/\.svn|/\.subversion|$original_name$") 1818 | fi 1819 | if [ -n "$other_dirs" ] || [ -n "$top_files" ]; then 1820 | cd "$TMP_DIR/$1.git" 1821 | if [ -n "$other_dirs" ]; then 1822 | echo "Adding directories and files from to \"$1\" that were not part of any trunk, branch, tag or meta-data..." | tee -a "$STATUS" 1823 | for dir in $other_dirs; do 1824 | dir_path=$(echo "$dir"|awk -F "$split" '{print $2}') 1825 | other_files=$(find "$split/$dir_path" -mindepth 1 -maxdepth 1 -type f|grep -Ev "/$TRUNK/|/$TRUNK$|/$TAG/|/$TAG$|/$BRANCH/|/$BRANCH$|/\.svn|/\.subversion|/\.git") 1826 | dir_perm=$(stat -c "%a" "$dir") 1827 | 1828 | if [ ! -d "$TMP_DIR/$1.git/$dir_path" ]; then 1829 | mkdir -p "$TMP_DIR/$1.git/$dir_path" &>> "$CPLOG" 1830 | fi 1831 | chmod "$dir_perm" "$TMP_DIR/$1.git/$dir_path" &>> "$CPLOG" 1832 | 1833 | for file in $other_files; do 1834 | cp -pv "$file" "$TMP_DIR/$1.git/$dir_path/" &>> "$CPLOG" 1835 | done 1836 | git add "$dir_path" &>> "$CNVLOG" 1837 | done 1838 | fi 1839 | 1840 | if [ -n "$top_files" ]; then 1841 | echo "Adding top level files from to \"$1\" that were not part of a trunk, branch, tag or meta-data..." | tee -a "$STATUS" 1842 | for file in $top_files; do 1843 | cp -pv "$file" "$TMP_DIR/$1.git/" &>> "$CPLOG" 1844 | git add "$TMP_DIR/$1.git/$(basename "$file")" 1845 | done 1846 | fi 1847 | git add * &>> "$CNVLOG" 1848 | fi 1849 | } 1850 | 1851 | clean_logs () 1852 | { 1853 | for log in "$COLOG" "$CNLOG" "$STATUS" "$SVNLOG" "$CNVLOG" "$CRTLOG" "$UNDLOG" "$PHLOG" "$IMPLOG" "$FAMILY" "$GHLOG" "$LFSLOG" "$PRNLOG" "$BFGLOG" "$RPTLOG" "$CPLOG" "$DBGLOG" "$RTYLOG"; do 1854 | if [ -f "$log" ]; then 1855 | echo '' > "$log" 1856 | fi 1857 | done 1858 | 1859 | rm -rf "$SUB_DIR" 1860 | rm -rf "$TMP_DIR"/*bfg-report 1861 | } 1862 | 1863 | report () 1864 | { 1865 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1866 | echo "| -- REPORT -- |" >> "$RPTLOG" 1867 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1868 | if [ -n "$MOD_COUNT" ]; then 1869 | echo "| TOTAL of $MOD_COUNT svn repositories found" >> "$RPTLOG" 1870 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1871 | fi 1872 | 1873 | if [ -f "$STATUS" ]; then 1874 | if [ -n "$(grep EMPTY "$STATUS"|sort -u)" ]; then 1875 | empty_count=$(grep EMPTY "$STATUS"|sort -u|wc -l) 1876 | echo "| $empty_count modules were skipped because they were found to be empty" >> "$RPTLOG" 1877 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1878 | fi 1879 | 1880 | if [ "$(grep RENAMED "$STATUS")" ]; then 1881 | conflict_count=$(grep RENAMED "$STATUS"|sort -u |awk '{print $2}'|sort -u|wc -l) 1882 | echo "| $conflict_count submodules were renamed in github due to naming conflict" >> "$RPTLOG" 1883 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1884 | fi 1885 | fi 1886 | 1887 | if [ -f "$CRTLOG" ]; then 1888 | creation_count=$(grep -F 'SUCCESS' "$CRTLOG"|sort -u|wc -l) 1889 | echo "| $creation_count repositories were created in $GITHUB" >> "$RPTLOG" 1890 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1891 | fi 1892 | 1893 | if [ -f "$UNDLOG" ]; then 1894 | delete_count=$(grep -F 'SUCCESS' "$UNDLOG"|sort -u|wc -l) 1895 | echo "| $delete_count repositories were removed from $GITHUB" >> "$RPTLOG" 1896 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1897 | fi 1898 | 1899 | if [ -f "$AUTHORS_FILE" ]; then 1900 | author_count=$(cat "$AUTHORS_FILE"|sort -u|wc -l) 1901 | echo "| $author_count authors were found in $SVN_REPO" >> "$RPTLOG" 1902 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1903 | fi 1904 | 1905 | if [ -f "$IGNORE_FILE" ]; then 1906 | ignore_count=$(cat "$IGNORE_FILE"|sort -u|wc -l) 1907 | echo "| $ignore_count ignore attriubutes were found in $SVN_REPO" >> "$RPTLOG" 1908 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1909 | fi 1910 | 1911 | if [ -f "$LFSLOG" ]; then 1912 | lfs_count=$(cat "$LFSLOG"|sort -u|wc -l) 1913 | echo "| $lfs_count files larger than $LFS were found" >> "$RPTLOG" 1914 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1915 | fi 1916 | 1917 | if [ -f "$RTYLOG" ]; then 1918 | if [ -n "$(grep 'RETRY #1' "$RTYLOG"|sort -u)" ]; then 1919 | retry_count=$(grep 'RETRY #1' "$RTYLOG"|sort -u|wc -l) 1920 | echo "| $retry_count events occured that failed at first and were retried." >> "$RPTLOG" 1921 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1922 | fi 1923 | 1924 | if [ "$(grep FAILURE: "$RTYLOG")" ]; then 1925 | refail_count=$(grep FAILURE: "$RTYLOG"|sort -u|wc -l) 1926 | echo "| $refail_count events were re-tried and failed completely in this order." >> "$RPTLOG" 1927 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1928 | echo "| -- RETRY FAILURES -- |" >> "$RPTLOG" 1929 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1930 | for refail in $(grep FAILURE: "$RTYLOG"); do 1931 | echo "| $refail " >> "$RPTLOG" 1932 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1933 | done 1934 | fi 1935 | fi 1936 | if [ "$(grep WARNING: "$STATUS")" ]; then 1937 | warning_count=$(grep WARNING: "$STATUS"|wc -l) 1938 | echo "| $warning_count warnings occured" >> "$RPTLOG" 1939 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1940 | echo "| -- WARNINGS -- |" >> "$RPTLOG" 1941 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1942 | for warning in $(grep WARNING: "$STATUS"); do 1943 | echo "| $warning " >> "$RPTLOG" 1944 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1945 | done 1946 | fi 1947 | if [ "$(grep FAILURE: "$STATUS")" ]; then 1948 | failure_count=$(grep FAILURE: "$STATUS"|wc -l) 1949 | echo "| $failure_count failures occured" >> "$RPTLOG" 1950 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1951 | echo "| -- FAILURES -- |" >> "$RPTLOG" 1952 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1953 | for failure in $(grep FAILURE: "$STATUS"); do 1954 | echo "| $failure " >> "$RPTLOG" 1955 | echo "|---------------------------------------------------------------------------------------------------------------|" >> "$RPTLOG" 1956 | done 1957 | fi 1958 | 1959 | if [ -f "$FAMILY" ]; then 1960 | cat "$FAMILY" 1961 | fi 1962 | 1963 | if [ -f "$RPTLOG" ]; then 1964 | cat "$RPTLOG" 1965 | fi 1966 | 1967 | } 1968 | 1969 | main 1970 | -------------------------------------------------------------------------------- /svn-to-github.spec: -------------------------------------------------------------------------------- 1 | Name: svn-to-github 2 | Version: 1.16.0 3 | Release: 0 4 | Summary: Converts Svn Repos to Github Public or Enterprise in Bulk 5 | URL: https://github.com/Comcast/svn-to-github 6 | Source0: svn-to-github-1.16.0.tar.gz 7 | License: GPL 8 | BuildArch: noarch 9 | BuildRoot: %{_tmppath}/%{name}-buildroot 10 | Provides: svn-to-github 11 | Requires: git >= 1.8.2, subversion, git-svn, java-1.7.0-openjdk, expect, curl, gawk, findutils, coreutils, sed, yum, grep, procps, glibc-common, which, initscripts, tree, bash >= 4.0 12 | %description 13 | This will convert svn repositories into Github repositories, if nested svn projects exist, those will be converted to git submodules preserving the directory tree. 14 | %prep 15 | %setup -q 16 | %build 17 | %install 18 | install --directory $RPM_BUILD_ROOT/opt/svn-to-github 19 | install --directory $RPM_BUILD_ROOT/etc/svn-to-github 20 | install --directory $RPM_BUILD_ROOT/opt/svn-to-github/bin 21 | install -m 0644 README.md $RPM_BUILD_ROOT/opt/svn-to-github/README.md 22 | install -m 0644 README.md $RPM_BUILD_ROOT/opt/svn-to-github/config.yml 23 | install -m 0755 svn-to-github $RPM_BUILD_ROOT/opt/svn-to-github/bin/svn-to-github 24 | %clean 25 | rm -rf $RPM_BUILD_ROOT 26 | %post 27 | ln -s /opt/svn-to-github/bin/svn-to-github /usr/local/bin/svn-to-github 28 | ln -s /opt/svn-to-github/config.yml /etc/svn-to-github/config.yml 29 | echo "svn-to-github installation complete! svn-to-github --help for Instructions" 30 | %dir /opt/svn-to-github 31 | %dir /opt/svn-to-github/bin 32 | %dir /etc/svn-to-github 33 | %files 34 | /opt/svn-to-github/bin/svn-to-github 35 | /opt/svn-to-github/README.md 36 | /opt/svn-to-github/config.yml 37 | -------------------------------------------------------------------------------- /test/bfg_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | git_bfg_lfs () 4 | { 5 | exportWORK_DIR="/app/foo/var/test/recover/var" 6 | BFGPROG="/app/foo/var/test/recover/var/bfg-1.12.12.jar" 7 | TMP_DIR="/app/foo/var/test/recover/var/svn_repo.clone" 8 | BFGLOG=../bfg.log 9 | LFSLOG=../lfs.log 10 | LFS="50M" 11 | BLOB_LIMIT="50M" 12 | IFS=$(echo -en "\n\b") 13 | 14 | if [ ! -f "$BFGPROG" ]; then 15 | cd "$WORK_DIR" 16 | curl -O -k -s "$BFG_DOWNLOAD" 17 | fi 18 | 19 | cd "$TMP_DIR/$1.git" 20 | lfs=$(find "$TMP_DIR/$1.git" -type f -size +"$LFS"|grep -Ev '.svn|.subversion|/.git') 21 | 22 | echo "" > "$BFGLOG" 23 | for big_file in $lfs; do 24 | if [ "$(basename $big_file|grep .)" ]; then 25 | extension=$(echo $big_file|rev|awk -F "." '{print $1}'|rev) 26 | echo pattern="'*.$extension'" >> "$BFGLOG" 27 | else 28 | echo patten="'$(basename $big_file)'" >> "$BFGLOG" 29 | fi 30 | done 31 | 32 | for lfs_pattern in $(cat "$BFGLOG"|grep pattern|awk -F "pattern=" '{print $2}'|sort -u); do 33 | echo "DEBUG: $BFGPROG pattern $lfs_pattern" 34 | java -jar "$BFGPROG" --convert-to-git-lfs $lfs_pattern --no-blob-protection 35 | done 36 | 37 | #TODO 38 | # git reflog expire --expire=now --all && git gc --prune=now &>> "$BFGLOG" 39 | git lfs install &>> "$LFSLOG" 40 | 41 | for file in $lfs; do 42 | track_file=$(echo "$file"|awk -F "$(pwd)/" '{print $2}') 43 | echo "DEBUG: track file is $track_file" 44 | git lfs track "$track_file" &>> "$LFSLOG" 45 | git add "$track_file" &>> "$LFSLOG" 46 | git commit -m "Tracking large file \"$track_file\" with lfs" &>> "$LFSLOG" 47 | git lfs push origin master &>> "$LFSLOG" 48 | git rm "$track_file" &>> "$LFSLOG" 49 | done 50 | 51 | java -jar "$BFGPROG" --strip-blobs-bigger-than "$BLOB_LIMIT" 52 | git reflog expire --expire=now --all && git gc --prune=now &>> "$BFGLOG" 53 | 54 | git add .gitattributes &>> "$LFSLOG" 55 | git commit -m "Large file(s) converted to LFS pointers" &>> "$LFSLOG" 56 | 57 | } 58 | 59 | git_bfg_lfs "svn_repo" 60 | -------------------------------------------------------------------------------- /test/filter_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export SAVEIFS=$IFS 4 | export IFS=$(echo -en "\n\b") 5 | export SVN_DIR="/app/foor/var/test/recover/var/bar" 6 | export TAG='((T|t)(A|a)(G|g)(S|s)|(T|t)(A|a)(G|g))' 7 | export BRANCH='((B|b)(R|r)(A|a)(N|n)(C|c)(H|h)(E|e)(S|s)|(B|b)(R|r)(A|a)(N|n)(C|c)(H|h))' 8 | export TRUNK='(T|t)(R|r)(U|u)(N|n)(K|k)' 9 | export MODULESS=$(find "$SVN_DIR" -mindepth 1 -type d -regextype posix-extended -regex "^.*/$TRUNK"|grep -Ev "/$TAG/"|grep -Ev "/$BRANCH/"|rev|cut -d / -f 2-|rev|sort -u) 10 | 11 | #echo $MODULESS 12 | create_submodules () 13 | { 14 | if [ -n "$MODULESS" ]; then 15 | #echo "Creating submodules in github..." 16 | # declare -gA SubModules 17 | # declare -gA ClonePids 18 | # declare -gA ConvertPids 19 | # declare -gA ModuleNames 20 | # declare -gA ModulePaths 21 | # declare -gA ModuleUrls 22 | # declare -gA ModuleRelatives 23 | # declare -gA ChildPaths 24 | declare -ga ModuleOriginss 25 | # declare -ga Modules 26 | 27 | # if [ ! -d "$SUB_DIR" ]; then 28 | # mkdir "$SUB_DIR" 29 | # fi 30 | #TODO revert 31 | #for module in $(echo "$MODULES"|tail -1) ; do 32 | # echo "FOR LOOP HERE" 33 | for modulew in $MODULESS ; do 34 | # echo $modulew 35 | local relative_pathh=$(echo $modulew|awk -F "$SVN_DIR/" '{print $2}') 36 | # local name=$(basename $module) 37 | # local real_name=$(resolve_name "$module") 38 | 39 | # if is_empty "$module"; then 40 | # echo "NOTICE: Skipping empty \"$name\"" >> "$TREELOG" 41 | # else 42 | # if [ ! -f "$RPO_DIR/$real_name.json" ]; then 43 | #Setup Arrays 44 | # SubModules["$real_name"]="" #for adding submodules 45 | # ClonePids["$real_name"]="" #for tracking Pids of Clone jobs 46 | # ConvertPids["$real_name"]="" #for tracking Pids of Convert jobs 47 | # ModuleNames["$real_name"]="$name" #for name tanslation 48 | # ModuleUrls["$real_name"]="$EVC/$REPO_NAME/$relative_path" #for cloning svn 49 | # ModulePaths["$real_name"]="$module" #for location reference 50 | # ModuleRelatives["$real_name"]="$relative_path" #for et_al filter exception 51 | #echo $relative_pathh 52 | ModuleOriginss+=("$relative_pathh",) #for et_al filter 53 | # Modules+=("$real_name") #for multi-threading loops 54 | 55 | # if [ $(echo "$real_name"|grep ^$REPO_NAME_lower--) ]; then 56 | # #Create list "$SUB_DIR/$real_parent" of nested submodules 57 | # ChildPaths["$real_name"]="$(get_child_path "$module")" #for location reference of nested repo 58 | # real_parent=$(resolve_name "$(get_parent "$module")") 59 | # echo "$real_name " >> "$SUB_DIR/$real_parent" 60 | # else 61 | # echo "$real_name" >> "$SUB_DIR/$REPO_NAME" 62 | # real_parent=$REPO_NAME 63 | # fi 64 | # if ! $DRYRUN ; then 65 | # #Create the repo in github 66 | # submodule_repo "$real_name" "$real_parent" 67 | # fi 68 | # fi 69 | # fi 70 | # echo "arry = ${ModuleOriginss[@]}" 71 | done 72 | 73 | fi 74 | } 75 | 76 | git_et_al_primary () 77 | { 78 | local search_dirr="$SVN_DIR" 79 | 80 | nested_projectss=$(find "$search_dirr" -mindepth 1 -maxdepth 1 -type d|grep -Ev "/$TRUNK/|/$TRUNK$|/$TAG/|/$TAG$|/$BRANCH/|/$BRANCH$|.svn|.subversion") 81 | #echo $nested_projectss 82 | #if [ -n "$nested_projectss" ]; then 83 | et_al_dirss=$(find "$search_dirr" -type d|grep -Ev "/$TRUNK/|/$TRUNK$|/$TAG/|/$TAG$|/$BRANCH/|/$BRANCH$|.svn|.subversion|/.git|$search_dirr$") 84 | #echo $et_al_dirss 85 | #filterr="($(echo "${ModuleOriginss[@]}"|sed s/',,'/'|'/g|sed s/' '/'[[:space:]]'/g|cut -c 2-|rev|cut -c 2-|rev))" 86 | filterr="($(echo "${ModuleOriginss[@]}"|sed s/', '/'|'/g|sed s/' '/'[[:space:]]'/g|rev|cut -c 2-|rev))" 87 | echo $filterr 88 | other_dirss=$(echo "$et_al_dirss"|grep -Ev "$filterr") 89 | #echo $(echo "$et_al_dirss"|wc -l) 90 | #echo $(echo "$other_dirss"|wc -l) 91 | #echo "$other_dirss" 92 | #fi 93 | 94 | #echo "DEBUG OTHER_DIRS = $other_dirss" 95 | 96 | # if [ -n "$other_dirs" ] || [ -n "$top_files" ]; then 97 | # cd "$TMP_DIR/$name.git" 98 | # 99 | # if [ -n "$other_dirs" ]; then 100 | # echo "Adding directories and files from \"$SVN_REPO\" to \"$REPO_NAME\" that were not part of any trunk, branch, tag or meta-data..." | tee -a "$STATUS" 101 | # for dir in $other_dirs; do 102 | # et_al_files=$(find "$dir" -mindepth 1 -maxdepth 1 -type f|grep -Ev "/$TRUNK/|/$TRUNK$|/$TAG/|/$TAG$|/$BRANCH/|/$BRANCH$|.svn|.subversion|/.git|$1$") 103 | # other_files=$(echo "$et_al_files"|grep -Ev "$filter") 104 | # dir_path=$(echo $dir|awk -F "$SVN_DIR/" '{print $2}') 105 | # dir_perm=$(stat -c "%a" $dir) 106 | # 107 | # if [ ! -d "$TMP_DIR/$name.git/$dir_path" ]; then 108 | # mkdir -p "$TMP_DIR/$name.git/$dir_path" 109 | # fi 110 | # 111 | # chmod $dir_perm "$TMP_DIR/$name.git/$dir_path" 112 | # 113 | # for file in $other_files; do 114 | # if [ -n "$dir_path" ]; then 115 | # cp -pv "$file" "$TMP_DIR/$name.git/$dir_path/" &>> "$CPLOG" 116 | # else 117 | # cp -pv "$file" "$TMP_DIR/$name.git/" &>> "$CPLOG" 118 | # fi 119 | # done 120 | # done 121 | # fi 122 | # 123 | # if [ -n "$top_files" ]; then 124 | # echo "Adding top level files from \"$SVN_REPO\" to \"$REPO_NAME\" that were not part of a trunk, branch, tag or meta-data..." | tee -a "$STATUS" 125 | # for file in $top_files; do 126 | # cp -pv "$file" "$TMP_DIR/$name.git/" &>> "$CPLOG" 127 | # done 128 | # fi 129 | # fi 130 | } 131 | 132 | create_submodules 133 | git_et_al_primary 134 | -------------------------------------------------------------------------------- /test/submodules_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #misc 4 | 5 | export SVN_DIR="/app/var/foo/test/recover/svn_repo/svn_repo" 6 | export TRUNK='(T|t)(R|r)(U|u)(N|n)(K|k)' 7 | export BRANCH='((B|b)(R|r)(A|a)(N|n)(C|c)(H|h)(E|e)(S|s)|(B|b)(R|r)(A|a)(N|n)(C|c)(H|h))' 8 | export TAG='((T|t)(A|a)(G|g)(S|s)|(T|t)(A|a)(G|g))' 9 | export SAVEIFS=$IFS 10 | export IFS=$(echo -en "\n\b") 11 | export WORK_DIR="/app/var/foo/test/recover/svn_repo" 12 | export NOORG=true 13 | export NTID='swizzley' 14 | export TOKEN='d938a09236f449cas5ea9f6bcfbcb64basadscda55e620' 15 | export GITHUB="github.com" 16 | export GITHUB_API="https://$GITHUB" 17 | export REPO_NAME="foo" 18 | 19 | # Dirs 20 | export SVN_DIR="$WORK_DIR/$REPO_NAME" 21 | export NEW_DIR="$WORK_DIR/$REPO_NAME.git" 22 | export TMP_DIR="$WORK_DIR/$REPO_NAME.clone" 23 | export LOG_DIR="$WORK_DIR/$REPO_NAME.log" 24 | export RPO_DIR="$WORK_DIR/$REPO_NAME.github" 25 | export SUB_DIR="$WORK_DIR/$REPO_NAME.submodules" 26 | export CLN_DIR="$LOG_DIR/clone" 27 | export CVT_DIR="$LOG_DIR/convert" 28 | export PSH_DIR="$LOG_DIR/push" 29 | # Logs 30 | export TREELOG="$LOG_DIR/tree.log" 31 | export COLOG="$LOG_DIR/checkout.log" 32 | export CNLOG="$LOG_DIR/clone.log" 33 | export STATUS="$LOG_DIR/status.log" 34 | export SVNLOG="$LOG_DIR/svnserve.log" 35 | export CNVLOG="$LOG_DIR/convert.log" 36 | export CRTLOG="$LOG_DIR/create.log" 37 | export UNDLOG="$LOG_DIR/undo.log" 38 | export PHLOG="$LOG_DIR/push.log" 39 | export IMPLOG="$LOG_DIR/import.log" 40 | export FAMILY="$LOG_DIR/family.log" 41 | export GHLOG="$LOG_DIR/github.log" 42 | export LFSLOG="$LOG_DIR/lfs.log" 43 | export PRNLOG="$LOG_DIR/parent.log" 44 | export BFGLOG="$LOG_DIR/bfg.log" 45 | export RPTLOG="$LOG_DIR/report.log" 46 | export CPLOG="$LOG_DIR/copy.log" 47 | export DBGLOG="$LOG_DIR/debug.log" 48 | 49 | 50 | export MODULES=$(find "$SVN_DIR" -mindepth 1 -type d -regextype posix-extended -regex "^.*/$TRUNK"|grep -Ev "/$TAG/"|grep -Ev "/$BRANCH/"|rev|cut -d / -f 2-|rev|sort -u) 51 | 52 | 53 | 54 | create_submodules () 55 | { 56 | #echo "$MODULES" 57 | "$TMP_DIR/svn_repo.git" 58 | if [ -n "$MODULES" ]; then 59 | # echo "Creating submodules in github..." 60 | declare -gA SubModules 61 | declare -gA ClonePids 62 | declare -gA ModuleNames 63 | declare -gA ModulePaths 64 | declare -gA ModuleUrls 65 | declare -gA ModuleRelatives 66 | declare -gA ChildPaths 67 | declare -ga ModuleOrigins 68 | declare -ga Modules 69 | #echo "RELATIVES ARRAY = ${ModuleRelatives[@]}" 70 | if [ ! -d "$SUB_DIR" ]; then 71 | mkdir "$SUB_DIR" 72 | fi 73 | #TODO cleanup 74 | #for module in $(echo "$MODULES"|tail -2) ; do 75 | for module in $MODULES ; do 76 | #echo "DEBUG Adding $module to array" 77 | relative_path=$(echo $module|awk -F "$SVN_DIR/" '{print $2}') 78 | #echo "relative_path = $relative_path" 79 | name=$(basename $module) 80 | #echo "subdir is $SUB_DIR" 81 | real_name=$(grep "$name" $SUB_DIR/svn_repo|head -1) 82 | #echo "DEBUG Adding real name $real_name to array" 83 | # if is_empty "$module"; then 84 | # echo "NOTICE: Skipping empty \"$name\"" >> "$TREELOG" 85 | # else 86 | # if [ ! -f "$RPO_DIR/$real_name.json" ]; then 87 | # #Setup Arrays 88 | # if [ -n $real_name ]; then 89 | # # SubModules["$real_name"]="wtf" #for adding submodules 90 | # # ClonePids["$real_name"]="" #for Pids 91 | # # ModuleNames["$real_name"]="$name" #for name tanslation 92 | # # ModuleUrls["$real_name"]="$EVC/$REPO_NAME/$relative_path" #for cloning svn 93 | # # ModulePaths["$real_name"]="$module" #for location reference 94 | # echo "DEBUG adding realname $real_name with relative path $relative_path to ModuleRelatives" 95 | #ModuleRelatives["$real_name"]="$relative_path" #for et_al filter exception 96 | git submodule add "https://$NTID:$TOKEN@$GITHUB/$NTID/$real_name" "$relative_path" 97 | #echo ${ModuleRelatives["$real_name"]} 98 | # fi 99 | # # #TODO FIX? 100 | # #ModuleOrigins+=("$module",) #for et_al filter 101 | # ModuleOrigins+=("$relative_path",) #for et_al filter 102 | # Modules+=("$real_name") #for multi-threading loops 103 | # 104 | # # if [ $(echo "$real_name"|grep ^$REPO_NAME_lower--) ]; then 105 | # # #Create list "$SUB_DIR/$real_parent" of nested submodules 106 | # # ChildPaths["$real_name"]="$(get_child_path "$module")" #for location reference of nested repo 107 | # # real_parent=$(resolve_name "$(get_parent "$module")") 108 | # # echo "$real_name " >> "$SUB_DIR/$real_parent" 109 | # # # else 110 | # # # echo "$real_name" >> "$SUB_DIR/$REPO_NAME" 111 | # # # real_parent=$REPO_NAME 112 | # # fi 113 | # # if ! $DRYRUN ; then 114 | # # #Create the repo in github 115 | # # submodule_repo "$real_name" "$real_parent" 116 | # # fi 117 | # fi 118 | #fi 119 | done 120 | 121 | fi 122 | 123 | 124 | } 125 | add_submodules () 126 | { 127 | echo "DEBUG: ADD_SUBMODULE" 128 | if [ -f "$SUB_DIR/$1" ]; then 129 | cd "$TMP_DIR/$1.git" 130 | git checkout master 131 | 132 | declare -A Children 133 | 134 | #Create Array of Children 135 | for child in $(cat "$SUB_DIR/$1"); do 136 | Children["$child"]="$child" 137 | done 138 | 139 | #echo "CHILDREN = ${Children[@]}" 140 | #exit 141 | #Loop through processing children until all children are done cloning 142 | while [ "${#Children[@]}" -ne 0 ]; do 143 | for child in "${Children[@]}" ; do 144 | #child_pid=${SubModules["$child"]} 145 | #if [ "$1" == "$REPO_NAME" ]; then 146 | original_path=${ModuleRelatives["$child"]} 147 | #else 148 | # original_path=${ChildPaths["$child"]} 149 | #fi 150 | echo "DEBUG: ADD_SUBMODULES child = $child original path = $original_path" 151 | exit 152 | # if [ ! "$(ps aux|awk '{print $2}'|grep ^$child_pid$)" ] && [ -n "$child_pid" ]; then 153 | #if [ -f .gitmodules] && [ ! "$(grep $original_path .gitmodules)" ] 154 | if $NOORG ; then 155 | git submodule add "https://$NTID:$TOKEN@$GITHUB/$NTID/$child" "$original_path" &>> "$PRNLOG" 156 | unset Children["$child"] 157 | else 158 | #git submodule add "https://$NTID:$TOKEN@$GITHUB/$ORG/$child" "$original_path" &>> "$PRNLOG" 159 | unset Children["$child"] 160 | fi 161 | # fi 162 | done 163 | echo "Waiting for children of submodule ${ModuleNames["$1"]} to finish cloning. Sleeping $sleep_time seconds..." | tee -a "$STATUS" 164 | sleep $sleep_time 165 | done 166 | unset Children 167 | 168 | #Remove NTID & Token from git URL 169 | sed -i s/"https://$NTID:$TOKEN@$GITHUB/"/"git@$GITHUB:"/g .gitmodules 170 | 171 | #Add submodules to repo 172 | git add .gitmodules 173 | fi 174 | } 175 | create_submodules 176 | #add_submodules "svn_repo" 177 | --------------------------------------------------------------------------------