├── .dockerignore ├── .gitignore ├── .prettierrc ├── .vscode └── launch.json ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── package-lock.json ├── package.json ├── sample_settings.ts ├── src ├── githubHelper.ts ├── gitlabHelper.ts ├── index.ts ├── settings.ts └── utils.ts ├── tsconfig.dev.json └── tsconfig.json /.dockerignore: -------------------------------------------------------------------------------- 1 | .husky/ 2 | .vscode/ 3 | node_modules/ 4 | .gitignore 5 | LICENSE 6 | README.md -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage 15 | 16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 17 | .grunt 18 | 19 | # node-waf configuration 20 | .lock-wscript 21 | 22 | # Compiled binary addons (http://nodejs.org/api/addons.html) 23 | build/Release 24 | 25 | # Dependency directory 26 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git 27 | node_modules 28 | 29 | # ignore settings.js, settings.ts 30 | settings.js 31 | settings.ts 32 | 33 | # ignore the log file 34 | merge-requests.json 35 | 36 | # ignore the lock 37 | #package-lock.json 38 | 39 | 40 | # Created by https://www.gitignore.io/api/visualstudiocode 41 | # Edit at https://www.gitignore.io/?templates=visualstudiocode 42 | 43 | ### VisualStudioCode ### 44 | .vscode/* 45 | !.vscode/settings.json 46 | !.vscode/tasks.json 47 | !.vscode/launch.json 48 | !.vscode/extensions.json 49 | 50 | ### VisualStudioCode Patch ### 51 | # Ignore all local history of files 52 | .history 53 | 54 | # End of https://www.gitignore.io/api/visualstudiocode 55 | 56 | # Created by https://www.gitignore.io/api/webstorm+iml 57 | # Edit at https://www.gitignore.io/?templates=webstorm+iml 58 | 59 | ### WebStorm+iml ### 60 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 61 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 62 | 63 | # User-specific stuff 64 | .idea/**/workspace.xml 65 | .idea/**/tasks.xml 66 | .idea/**/usage.statistics.xml 67 | .idea/**/dictionaries 68 | .idea/**/shelf 69 | 70 | # Generated files 71 | .idea/**/contentModel.xml 72 | 73 | # Sensitive or high-churn files 74 | .idea/**/dataSources/ 75 | .idea/**/dataSources.ids 76 | .idea/**/dataSources.local.xml 77 | .idea/**/sqlDataSources.xml 78 | .idea/**/dynamic.xml 79 | .idea/**/uiDesigner.xml 80 | .idea/**/dbnavigator.xml 81 | 82 | # Gradle 83 | .idea/**/gradle.xml 84 | .idea/**/libraries 85 | 86 | # Gradle and Maven with auto-import 87 | # When using Gradle or Maven with auto-import, you should exclude module files, 88 | # since they will be recreated, and may cause churn. Uncomment if using 89 | # auto-import. 90 | # .idea/modules.xml 91 | # .idea/*.iml 92 | # .idea/modules 93 | # *.iml 94 | # *.ipr 95 | 96 | # CMake 97 | cmake-build-*/ 98 | 99 | # Mongo Explorer plugin 100 | .idea/**/mongoSettings.xml 101 | 102 | # File-based project format 103 | *.iws 104 | 105 | # IntelliJ 106 | out/ 107 | 108 | # mpeltonen/sbt-idea plugin 109 | .idea_modules/ 110 | 111 | # JIRA plugin 112 | atlassian-ide-plugin.xml 113 | 114 | # Cursive Clojure plugin 115 | .idea/replstate.xml 116 | 117 | # Crashlytics plugin (for Android Studio and IntelliJ) 118 | com_crashlytics_export_strings.xml 119 | crashlytics.properties 120 | crashlytics-build.properties 121 | fabric.properties 122 | 123 | # Editor-based Rest Client 124 | .idea/httpRequests 125 | 126 | # Android studio 3.1+ serialized cache file 127 | .idea/caches/build_file_checksums.ser 128 | 129 | ### WebStorm+iml Patch ### 130 | # Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 131 | 132 | *.iml 133 | modules.xml 134 | .idea/misc.xml 135 | *.ipr 136 | 137 | # End of https://www.gitignore.io/api/webstorm+iml 138 | 139 | # Ignore whole .idea folder: 140 | .idea/ 141 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "trailingComma": "es5", 3 | "tabWidth": 2, 4 | "semi": true, 5 | "singleQuote": true, 6 | "arrowParens": "avoid" 7 | } 8 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "node", 9 | "request": "launch", 10 | "name": "Node: Launch", 11 | "env": { "TS_NODE_PROJECT": "tsconfig.dev.json" }, 12 | "runtimeExecutable": "node", 13 | "args": [ 14 | "--inspect", 15 | "-r", 16 | "ts-node/register/type-check", 17 | "-r", 18 | "tsconfig-paths/register", 19 | "src/index.ts" 20 | ], 21 | "protocol": "inspector", 22 | "sourceMaps": true, 23 | "console": "integratedTerminal" 24 | } 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # This Dockerfile is used to set up a container environment with all the required 2 | # tools to use this project. You only need to provide the necessary environment 3 | # variables, as described in the README. 4 | # 5 | # Docker version that I used: 20.10.17 6 | # 7 | # If you're interested in testing other base images, take a look at this reference: 8 | # https://github.com/BretFisher/nodejs-rocks-in-docker 9 | FROM node:22-bullseye-slim 10 | 11 | ARG USERNAME=migrator 12 | ARG USER_UID=2000 13 | ARG USER_GID=$USER_UID 14 | 15 | LABEL version="0.1.5" 16 | LABEL description="Migrate Issues, Wiki from gitlab to github." 17 | 18 | WORKDIR /app 19 | 20 | # Add a non-root user, so later we can explore methods to scale 21 | # privileges within this container. 22 | # https://code.visualstudio.com/remote/advancedcontainers/add-nonroot-user#_creating-a-nonroot-user 23 | RUN groupadd --gid $USER_GID $USERNAME 24 | RUN useradd --uid $USER_UID --gid $USER_GID -m $USERNAME 25 | RUN chown -R $USERNAME /app 26 | 27 | # Copy the project contents to the container 28 | COPY --chown=$USERNAME . /app 29 | 30 | USER $USERNAME 31 | 32 | # Install dependencies 33 | RUN npm i 34 | 35 | # Start the process 36 | ENTRYPOINT ["/bin/bash", "-c", "npm run start"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 piceaTech 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CONTAINER_IMAGE ?= node-gitlab-2-github 2 | CONTAINER_TAG ?= latest 3 | LOCAL_PWD = $(shell pwd) 4 | 5 | .PHONY: build-image 6 | build-image: ##@docker Build the Docker image 7 | docker build -t $(CONTAINER_IMAGE):$(CONTAINER_TAG) . 8 | 9 | .PHONY: docker-run 10 | docker-run: 11 | docker run $(CONTAINER_IMAGE):$(CONTAINER_TAG) 12 | 13 | .PHONY: docker-run-bind 14 | docker-run-bind: 15 | docker run \ 16 | --mount type=bind,source="$(LOCAL_PWD)/settings.ts",target="/app/settings.ts",readonly \ 17 | $(CONTAINER_IMAGE):$(CONTAINER_TAG) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # node-gitlab-2-github 2 | 3 | ## Install 4 | 5 | 1. You need nodejs and npm installed 6 | 1. Clone this repo with `git clone https://github.com/piceaTech/node-gitlab-2-github.git` 7 | 1. `cd node-gitlab-2-github` 8 | 1. `npm i` 9 | 10 | ## Preliminaries 11 | 12 | Before using this script, you must mirror your GitLab repo to your new GitHub repo. This can be done with the following steps: 13 | 14 | ```bash 15 | # Clone the repo from GitLab using the `--mirror` option. This is like 16 | # `--bare` but also copies all refs as-is. Useful for a full backup/move. 17 | git clone --mirror git@your-gitlab-site.com:username/repo.git 18 | 19 | # Change into newly created repo directory 20 | cd repo 21 | 22 | # Push to GitHub using the `--mirror` option. The `--no-verify` option skips any hooks. 23 | git push --no-verify --mirror git@github.com:username/repo.git 24 | 25 | # Set push URL to the mirror location 26 | git remote set-url --push origin git@github.com:username/repo.git 27 | 28 | # To periodically update the repo on GitHub with what you have in GitLab 29 | git fetch -p origin 30 | git push --no-verify --mirror 31 | ``` 32 | 33 | After doing this, the autolinking of issues, commits, and branches will work. See **Usage** for next steps. 34 | 35 | ## Usage 36 | 37 | The user must be a member of the project you want to copy. This user must be the one 38 | 39 | 1. `cp sample_settings.ts settings.ts` 40 | 1. edit settings.ts 41 | 1. run `npm run start` 42 | 43 | ### Docker 44 | 45 | If you don't have Node.js installed in your local environment and don't want to install it you can use the Dockerized approach. 46 | 47 | 1. Make sure that you have [Docker](https://docs.docker.com/engine/install/) installed in your computer. You can test running `docker version` in the terminal. 48 | 1. `cp sample_settings.ts settings.ts` 49 | 1. edit settings.ts 50 | 1. `docker build -t node-gitlab-2-github:latest .`, or, you can use `make build-image` 51 | 1. `docker run node-gitlab-2-github:latest`, or, you can use `make docker-run` 52 | 53 | If you want to let it run in the background (detached mode), just use the following command: 54 | 55 | 1. `docker run -d node-gitlab-2-github:latest` 56 | 57 | ### Docker with bind mounts 58 | 59 | In order to optimize the usage of the dockerized application, one can use the `bind mounts` feature of Docker ([Docker docs](https://docs.docker.com/storage/bind-mounts/)). This way, whenever you change the `settings.ts` file in the host environment it will change in the container filesystem as well. 60 | 61 | The process to use this trick is pretty much the same we presented before, the only different is the addition of a flag in the docker command to tell it what is the directory/file to be bound. 62 | 63 | 1. Make sure that you have [Docker](https://docs.docker.com/engine/install/) installed in your computer. You can test running `docker version` in the terminal. 64 | 1. `cp sample_settings.ts settings.ts` 65 | 1. edit settings.ts 66 | 1. `docker build -t node-gitlab-2-github:latest .`, or, you can use `make build-image` 67 | 1. This command must work for **Linux** or **Mac**: `docker run --mount type=bind,source="$(pwd)/settings.ts",target="/app/settings.ts",readonly node-gitlab-2-github:latest`, or, you can use `make docker-run-bind` 68 | 69 | * If you want to run this last command in the Windows environment, please consult the Docker documentation on how to solve the problem of the pwd command expanding incorrectly there - [Docker documentation - Topics for windows](https://docs.docker.com/desktop/troubleshoot/topics/#topics-for-windows). 70 | 71 | ## Where to find info for the `settings.ts` 72 | 73 | ### gitlab 74 | 75 | #### gitlab.url 76 | 77 | The URL under which your gitlab instance is hosted. Default is the official `http://gitlab.com` domain. 78 | 79 | #### gitlab.token 80 | 81 | Go to [Settings / Access Tokens](https://gitlab.com/-/profile/personal_access_tokens). Create a new Access Token with `api` and `read_repository` scopes and copy that into the `settings.ts` 82 | 83 | #### gitlab.projectID 84 | 85 | Leave it null for the first run of the script. Then the script will show you which projects there are. Can be either string or number. 86 | 87 | #### gitlab.listArchivedProjects 88 | 89 | When listing projects on the first run (projectID = null), include archived ones too. The default is `true`. 90 | 91 | #### gitlab.sessionCookie 92 | 93 | GitLab's API [does not allow downloading of attachments](https://gitlab.com/gitlab-org/gitlab/-/issues/24155) and only images can be downloaded using HTTP. To work around this limitation and enable binary attachments to be migrated one can use the session cookie set in the browser after logging in to the gitlab instance. The cookie is named `_gitlab_session`. 94 | 95 | ### github 96 | 97 | #### github.baseUrl 98 | 99 | Where is the github instance hosted? The default is the official `github.com` domain 100 | 101 | #### github.apiUrl 102 | 103 | Point this to the api. The default is `api.github.com`. 104 | 105 | #### github.owner 106 | 107 | Under which organisation or user will the new project be hosted 108 | 109 | #### github.ownerIsOrg 110 | 111 | A boolean indicator (default is `false`) to specify that the owner of this repo is an Organisation. 112 | 113 | #### github.token 114 | 115 | Go to [Settings / Developer settings / Personal access tokens](https://github.com/settings/tokens). Generate a new token with `repo` scope and copy that into the `settings.ts` 116 | 117 | #### github.token_owner 118 | 119 | Set to the user name of the user whose token is used (see above). This is required to determine whether the user running the migration is also the creator of comments and issues. If this is the case and `useIssueCreationAPI` is `true` (see below), the extra line specifying who created a comment or issue will not be added. 120 | 121 | #### github.repo 122 | 123 | What is the name of the new repo 124 | 125 | #### github.recreateRepo 126 | 127 | If `true` (default is `false`), we will try to delete the destination github repository if present, and (re)create it. The github token must be granted `delete_repo` scope. The newly created repository will be made private by default. 128 | 129 | If you've set `github.recreateRepo` to `true` and the repo belongs to an Organisation, the `github.ownerIsOrg` flag **must** be set as `true`. 130 | 131 | This is useful when debugging this tool or a specific migration. You will always be prompted for confirmation. 132 | 133 | ### s3 (optional) 134 | 135 | S3 can be used to store attachments from issues. If omitted, `has attachment` label will be added to GitHub issue. 136 | 137 | #### s3.accessKeyId and s3.secretAccessKey 138 | 139 | AWS [credentials](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) that are used to copy attachments from GitLab into the S3 bucket. 140 | 141 | IAM User who owns these credential must have [write permissions](https://docs.aws.amazon.com/AmazonS3/latest/dev/example-policies-s3.html#iam-policy-ex0) to the bucket. 142 | 143 | #### s3.bucket 144 | 145 | Existing bucket, with an appropriate security policy. One possible policy is to allow [public access](https://docs.aws.amazon.com/AmazonS3/latest/dev/WebsiteAccessPermissionsReqd.html). 146 | 147 | #### s3.region 148 | 149 | Specify Region (example: us-west-1) of bucket [list of regions](https://docs.aws.amazon.com/general/latest/gr/s3.html) 150 | 151 | ### usermap 152 | 153 | Maps the usernames from gitlab to github. If the assinee of the gitlab issue is equal to the one currently logged in github it will also get assigned without a usermap. The Mentions in issues will also be translated to the new github name. 154 | 155 | ### projectmap 156 | 157 | When one renames the project while transfering so that the projects don't loose there links to the mentioned issues. 158 | 159 | ### conversion 160 | 161 | #### conversion.useLowerCaseLabels 162 | 163 | If this is set to `true` (default) then labels from GitLab will be converted to lowercase in GitHub. 164 | 165 | #### conversion.addIssueInformation 166 | 167 | If this is set to `true` (default) then issues and pull requests will get information about assignees (both), reviewers and approvers (PR only) added to their description. 168 | 169 | ### transfer 170 | 171 | #### transfer.milestones 172 | 173 | If this is set to `true` (default) then the migration process will transfer milestones. 174 | 175 | #### transfer.labels 176 | 177 | If this is set to `true` (default) then the migration process will transfer labels. 178 | 179 | #### transfer.issues 180 | 181 | If this is set to `true` (default) then the migration process will transfer issues. 182 | 183 | #### transfer.mergeRequests 184 | 185 | If this is set to `true` (default) then the migration process will transfer merge requests. 186 | 187 | #### transfer.releases 188 | 189 | If this is set to `true` (default) then the migration process will transfer releases. 190 | Note that github api for releases is limited and hence this will only transfer the title and description of the releases 191 | and add them to github in chronological order, but it would not preserve the original release dates, nor transfer artefacts or assets. 192 | 193 | ### dryRun 194 | 195 | As default it is set to `false`. Doesn't fire the requests to github api and only does the work on the gitlab side to test for wonky cases before using up api-calls 196 | 197 | ### exportUsers 198 | 199 | If this is set to `true` (default is `false`) then a file called "users.txt" wil be created containing all 200 | usernames that contributed to the repository. You can use this with dryRun when you need to map users 201 | for the migration, but you do not know all the source usernames. 202 | 203 | ### useIssueImportAPI 204 | 205 | Set to `true` (default) to enable using the [GitHub preview API for importing issues](https://gist.github.com/jonmagic/5282384165e0f86ef105). This allows setting the date for issues and comments instead of inserting an additional line in the body. 206 | 207 | ### usePlaceholderIssuesForMissingIssues 208 | 209 | If this is set to `true` (default) then the migration process will automatically create empty dummy issues for every 'missing' GitLab issue (if you deleted a GitLab issue for example). Those issues will be closed on Github and they ensure that the issue ids stay the same on both GitLab and Github. 210 | 211 | #### usePlaceholderMilestonesForMissingMilestones 212 | 213 | If this is set to `true` (default) then the migration process will automatically create empty dummy milestones for every 'missing' GitLab milestone (if you deleted a GitLab milestone for example). Those milestones will be closed on Github and they ensure that the milestone ids stay the same on both GitLab and Github. 214 | 215 | #### useReplacementIssuesForCreationFails 216 | 217 | If this is set to `true` (default) then the migration process will automatically create so called "replacement-issues" for every issue where the migration fails. This replacement issue will be exactly the same, but the original description will be lost. In the future, the description of the replacement issue will also contain a link to the original issue on GitLab. This way, users who still have access to the GitLab repository can still view its content. However, this is still an open task. (TODO) 218 | 219 | It would of course be better to find the cause for migration fails, so that no replacement issues would be needed. Finding the cause together with a retry-mechanism would be optimal, and will maybe come in the future - currently the replacement-issue-mechanism helps to keep things in order. 220 | 221 | ### useIssuesForAllMergeRequests 222 | 223 | If this is set to `true` (default is `false`) then all merge requests will be migrated as GitHub issues (rather than pull requests). This can be 224 | used to sidestep the problem where pull requests are rejected by GitHub if the feature branch no longer exists or has been merged. 225 | 226 | ### filterByLabel 227 | 228 | Filters all merge requests and issues by these labels. The applicable values can be found in the Gitlab API documentation for [issues](https://docs.gitlab.com/ee/api/issues.html#list-project-issues) and [merge requests](https://docs.gitlab.com/ee/api/merge_requests.html#list-merge-requests) respectively. Default is `null` which returns all issues/merge requests. 229 | 230 | ### skipMergeRequestStates 231 | 232 | Merge requests in GitLab with any of the states listed in this array will not be transferred to GitHub (e.g. set to `['merged', 'closed']` to avoid creating issues for closed MRs whose branches have been deleted). 233 | 234 | ### skipMatchingComments 235 | 236 | This is an array (empty per default) that may contain string values. Any note/comment in any issue, that contains one or more of those string values, will be skipped (meaining not migrated). Note that this is case insensitive, therefore the string value `foo` would also lead to skipping notes containing a (sub)string `FOO`. 237 | 238 | Suggested values: 239 | 240 | - `time spent`, since those kind of terms can be used in GitLab to track time, they are rather meaningless in Github though 241 | - action entries, such as `changed the description`, `added 1 commit`, `mentioned in merge request`, etc as they are interpreted as comments 242 | 243 | ### mergeRequests 244 | 245 | Object consisting of `logfile` and `log`. If `log` is set to `true`, then the merge requests are logged in the specified file and not migrated. Conversely, if `log` is set to `false`, then the merge requests are migrated to GitHub and not logged. If the source or target branches linked to the merge request have been deleted, the merge request cannot be migrated to a pull request; instead, an issue with a custom "gitlab merge request" tag is created with the full comment history of the merge request. 246 | 247 | ### usermap 248 | 249 | Maps gitlab user names to github users. This is used to properly set assignees in issues and PRs and to translate mentions in issues. 250 | 251 | ### projectmap 252 | 253 | This is useful when migrating multiple projects if they are renamed at destination. Provide a map from gitlab names to github names so that any cross-project references (e.g. issues) are not lost. 254 | 255 | ## Import limit 256 | 257 | Because Github has a limit of 5000 Api requests per hour one has to be careful not to go over this limit. I transferred one of my project with it ~ 300 issues with ~ 200 notes. This totals to some 500 objects excluding commits which are imported through githubs importer. I never got under 3800 remaining requests (while testing it two times in one hour). 258 | 259 | So the rule of thumb should be that one can import a repo with ~ 2500 issues without a problem. 260 | 261 | ## Bugs 262 | 263 | ### Issue migration fail 264 | 265 | See section 'useReplacementIssuesForCreationFails' above for more infos! 266 | One reason seems to be some error with `Octokit` (error message snippet: https://pastebin.com/3VNUNYLh) 267 | 268 | ### Milestone, MR and issue references 269 | 270 | This is WIP 271 | 272 | the milestone refs and issue refs do not seem to be rewritten properly at the 273 | moment. specifically, milestones show up like `%4` in comments 274 | and issue refs like `#42` do not remap to the `#42` from gitlab under the new 275 | issue number in github. @ references are remapped properly (yay). If this is a 276 | deal breaker, a large amount of the code to do this has been written it just 277 | appears to no longer work in current form :( 278 | 279 | ## Feature suggestions / ideas 280 | 281 | ### Throttling mechanism 282 | 283 | A throttling mechanism could maybe help to avoid api rate limit errors. 284 | In some scenarios the ability to migrate is probably more important than the total 285 | duration of the migration process. Some users may even be willing to accept a very long duration (> 1 day if necessary?), if they can get the migration done at all, in return. 286 | 287 | ### Make requests run in parallel 288 | 289 | Some requests could be run in parallel, to shorten the total duration. Currently all GitLab- and Github-Api-Requests are run sequentially. 290 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gitlab-2-github", 3 | "version": "0.1.5", 4 | "description": "Migrate Issues, Wiki from gitlab to github", 5 | "main": "index.js", 6 | "scripts": { 7 | "prepare": "husky install", 8 | "start": "node node_modules/ts-node/dist/bin.js ./src/index.ts" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "git+https://github.com/piceaTech/node-gitlab-2-github.git" 13 | }, 14 | "keywords": [ 15 | "github", 16 | "gitlab", 17 | "import", 18 | "export", 19 | "issue", 20 | "issues" 21 | ], 22 | "author": "Spruce, ", 23 | "license": "MIT", 24 | "bugs": { 25 | "url": "https://github.com/piceaTech/node-gitlab-2-github/issues" 26 | }, 27 | "homepage": "https://github.com/piceaTech/node-gitlab-2-github#readme", 28 | "dependencies": { 29 | "@gitbeaker/node": "^35.1.0", 30 | "@octokit/plugin-throttling": "^11.0.1", 31 | "@octokit/rest": "^21.1.1", 32 | "@octokit/types": "^13.8.0", 33 | "aws-sdk": "^2.1053.0", 34 | "axios": "^1.6.0", 35 | "mime-types": "^2.1.34", 36 | "readline-sync": "^1.4.10", 37 | "ts-node": "^10.4.0" 38 | }, 39 | "devDependencies": { 40 | "@types/mime-types": "^2.1.1", 41 | "@types/node": "^14.18.5", 42 | "@types/readline-sync": "^1.4.4", 43 | "husky": "^7.0.4", 44 | "lint-staged": "^13.2.1", 45 | "prettier": "^2.5.1", 46 | "tsconfig-paths": "^3.12.0", 47 | "typescript": "^3.9.6" 48 | }, 49 | "husky": { 50 | "hooks": { 51 | "pre-commit": "lint-staged" 52 | } 53 | }, 54 | "lint-staged": { 55 | "*.{js,css,json,md}": [ 56 | "prettier --write", 57 | "git add" 58 | ] 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /sample_settings.ts: -------------------------------------------------------------------------------- 1 | import Settings from './src/settings'; 2 | 3 | export default { 4 | gitlab: { 5 | // url: 'https://gitlab.mycompany.com', 6 | token: '{{gitlab private token}}', 7 | projectId: 0, 8 | listArchivedProjects: true, 9 | sessionCookie: "", 10 | }, 11 | github: { 12 | // baseUrl: 'https://github.mycompany.com:123/etc', 13 | // apiUrl: 'https://api.github.mycompany.com', 14 | owner: '{{repository owner (user or organization)}}', 15 | ownerIsOrg: false, 16 | token: '{{token}}', 17 | token_owner: '{{token_owner}}', 18 | repo: '{{repo}}', 19 | recreateRepo: false, 20 | }, 21 | s3: { 22 | accessKeyId: '{{accessKeyId}}', 23 | secretAccessKey: '{{secretAccessKey}}', 24 | bucket: 'my-gitlab-bucket', 25 | region: 'us-west-1', 26 | }, 27 | usermap: { 28 | 'username.gitlab.1': 'username.github.1', 29 | 'username.gitlab.2': 'username.github.2', 30 | }, 31 | projectmap: { 32 | 'gitlabgroup/projectname.1': 'GitHubOrg/projectname.1', 33 | 'gitlabgroup/projectname.2': 'GitHubOrg/projectname.2', 34 | }, 35 | conversion: { 36 | useLowerCaseLabels: true, 37 | addIssueInformation: true, 38 | }, 39 | transfer: { 40 | description: true, 41 | milestones: true, 42 | labels: true, 43 | issues: true, 44 | mergeRequests: true, 45 | releases: true, 46 | }, 47 | dryRun: false, 48 | exportUsers: false, 49 | useIssueImportAPI: true, 50 | usePlaceholderMilestonesForMissingMilestones: true, 51 | usePlaceholderIssuesForMissingIssues: true, 52 | useReplacementIssuesForCreationFails: true, 53 | useIssuesForAllMergeRequests: false, 54 | filterByLabel: undefined, 55 | trimOversizedLabelDescriptions: false, 56 | skipMergeRequestStates: [], 57 | skipMatchingComments: [], 58 | mergeRequests: { 59 | logFile: './merge-requests.json', 60 | log: false, 61 | }, 62 | } as Settings; 63 | -------------------------------------------------------------------------------- /src/githubHelper.ts: -------------------------------------------------------------------------------- 1 | import settings from '../settings'; 2 | import { GithubSettings } from './settings'; 3 | import * as utils from './utils'; 4 | import { Octokit as GitHubApi, RestEndpointMethodTypes } from '@octokit/rest'; 5 | import { Endpoints } from '@octokit/types'; 6 | import { 7 | GitLabDiscussion, 8 | GitLabDiscussionNote, 9 | GitlabHelper, 10 | GitLabIssue, 11 | GitLabMergeRequest, 12 | GitLabNote, 13 | GitLabUser, 14 | } from './gitlabHelper'; 15 | 16 | type IssuesListForRepoResponseData = 17 | Endpoints['GET /repos/{owner}/{repo}/issues']['response']['data']; 18 | type PullsListResponseData = 19 | Endpoints['GET /repos/{owner}/{repo}/pulls']['response']['data']; 20 | 21 | type GitHubIssue = IssuesListForRepoResponseData[0]; 22 | type GitHubPullRequest = PullsListResponseData[0]; 23 | 24 | const gitHubLocation = 'https://github.com'; 25 | 26 | // Source for regex: https://stackoverflow.com/a/30281147 27 | const usernameRegex = new RegExp('\\B@([a-z0-9](?:-(?=[a-z0-9])|[a-z0-9]){0,38}(?<=[a-z0-9]))', 'gi') 28 | 29 | interface CommentImport { 30 | created_at?: string; 31 | body: string; 32 | } 33 | 34 | interface IssueImport { 35 | title: string; 36 | body: string; 37 | closed: boolean; 38 | assignee?: string; 39 | created_at?: string; 40 | updated_at?: string; 41 | closed_at?: string; 42 | milestone?: number; 43 | labels?: string[]; 44 | } 45 | 46 | export interface MilestoneImport { 47 | id: number; // GitHub internal identifier 48 | iid: number; // GitLab external number 49 | title: string; 50 | description: string; 51 | state: string; 52 | due_date?: string; 53 | } 54 | 55 | export interface SimpleLabel { 56 | name: string; 57 | color: string; 58 | description: string; 59 | } 60 | 61 | export interface SimpleMilestone { 62 | number: number; 63 | title: string; 64 | } 65 | 66 | export class GithubHelper { 67 | githubApi: GitHubApi; 68 | githubUrl: string; 69 | githubOwner: string; 70 | githubOwnerIsOrg: boolean; 71 | githubToken: string; 72 | githubTokenOwner: string; 73 | githubRepo: string; 74 | githubTimeout?: number; 75 | gitlabHelper: GitlabHelper; 76 | repoId?: number; 77 | delayInMs: number; 78 | useIssuesForAllMergeRequests: boolean; 79 | milestoneMap?: Map; 80 | users: Set; 81 | 82 | constructor( 83 | githubApi: GitHubApi, 84 | githubSettings: GithubSettings, 85 | gitlabHelper: GitlabHelper, 86 | useIssuesForAllMergeRequests: boolean 87 | ) { 88 | this.githubApi = githubApi; 89 | this.githubUrl = githubSettings.baseUrl 90 | ? githubSettings.baseUrl 91 | : gitHubLocation; 92 | this.githubOwner = githubSettings.owner; 93 | this.githubOwnerIsOrg = githubSettings.ownerIsOrg ?? false; 94 | this.githubToken = githubSettings.token; 95 | this.githubTokenOwner = githubSettings.token_owner; 96 | this.githubRepo = githubSettings.repo; 97 | this.githubTimeout = githubSettings.timeout; 98 | this.gitlabHelper = gitlabHelper; 99 | this.delayInMs = 2000; 100 | this.useIssuesForAllMergeRequests = useIssuesForAllMergeRequests; 101 | this.users = new Set(); 102 | } 103 | 104 | /* 105 | ****************************************************************************** 106 | ******************************** GET METHODS ********************************* 107 | ****************************************************************************** 108 | */ 109 | 110 | /** 111 | * Store the new repo id 112 | */ 113 | async registerRepoId() { 114 | try { 115 | await utils.sleep(this.delayInMs); 116 | let result = await this.githubApi.repos.get({ 117 | owner: this.githubOwner, 118 | repo: this.githubRepo, 119 | }); 120 | 121 | this.repoId = result.data.id; 122 | } catch (err) { 123 | console.error('Could not access GitHub repo'); 124 | console.error(err); 125 | process.exit(1); 126 | } 127 | } 128 | 129 | // ---------------------------------------------------------------------------- 130 | 131 | /** 132 | * Get a list of all GitHub milestones currently in new repo 133 | */ 134 | async getAllGithubMilestones(): Promise { 135 | try { 136 | await utils.sleep(this.delayInMs); 137 | // get an array of GitHub milestones for the new repo 138 | let result = await this.githubApi.issues.listMilestones({ 139 | owner: this.githubOwner, 140 | repo: this.githubRepo, 141 | state: 'all', 142 | }); 143 | 144 | return result.data.map(x => ({ number: x.number, title: x.title })); 145 | } catch (err) { 146 | console.error('Could not access all GitHub milestones'); 147 | console.error(err); 148 | process.exit(1); 149 | } 150 | } 151 | 152 | // ---------------------------------------------------------------------------- 153 | 154 | /** 155 | * Get a list of all the current GitHub issues. 156 | * This uses a while loop to make sure that each page of issues is received. 157 | */ 158 | async getAllGithubIssues() { 159 | let allIssues: IssuesListForRepoResponseData = []; 160 | let page = 1; 161 | const perPage = 100; 162 | 163 | while (true) { 164 | await utils.sleep(this.delayInMs); 165 | // get a paginated list of issues 166 | const issues = await this.githubApi.issues.listForRepo({ 167 | owner: this.githubOwner, 168 | repo: this.githubRepo, 169 | state: 'all', 170 | labels: 'gitlab merge request', 171 | per_page: perPage, 172 | page: page, 173 | }); 174 | 175 | // if this page has zero issues then we are done! 176 | if (issues.data.length === 0) break; 177 | 178 | // join this list of issues with the master list 179 | allIssues = allIssues.concat(issues.data); 180 | 181 | // if there are strictly less issues on this page than the maximum number per page 182 | // then we can be sure that this is all the issues. No use querying again. 183 | if (issues.data.length < perPage) break; 184 | 185 | // query for the next page of issues next iteration 186 | page++; 187 | } 188 | 189 | return allIssues; 190 | } 191 | 192 | // ---------------------------------------------------------------------------- 193 | 194 | /** 195 | * Get a list of all GitHub label names currently in new repo 196 | */ 197 | async getAllGithubLabelNames() { 198 | try { 199 | await utils.sleep(this.delayInMs); 200 | // get an array of GitHub labels for the new repo 201 | let result = await this.githubApi.issues.listLabelsForRepo({ 202 | owner: this.githubOwner, 203 | repo: this.githubRepo, 204 | }); 205 | 206 | // extract the label name and put into a new array 207 | let labels = result.data.map(x => x.name); 208 | 209 | return labels; 210 | } catch (err) { 211 | console.error('Could not access all GitHub label names'); 212 | console.error(err); 213 | process.exit(1); 214 | } 215 | } 216 | 217 | // ---------------------------------------------------------------------------- 218 | 219 | /** 220 | * Gets a release by tag name 221 | * @param tag {string} - the tag name to search a release for 222 | * @returns 223 | */ 224 | async getReleaseByTag(tag: string) { 225 | try { 226 | await utils.sleep(this.delayInMs); 227 | // get an existing release by tag name in github 228 | let result = await this.githubApi.repos.getReleaseByTag({ 229 | owner: this.githubOwner, 230 | repo: this.githubRepo, 231 | tag: tag, 232 | }); 233 | 234 | return result; 235 | } catch (err) { 236 | console.error('No existing release for this tag on github'); 237 | return null; 238 | } 239 | } 240 | 241 | // ---------------------------------------------------------------------------- 242 | 243 | /** 244 | * Creates a new release on github 245 | * @param tag_name {string} - the tag name 246 | * @param name {string} - title of the release 247 | * @param body {string} - description for the release 248 | */ 249 | async createRelease(tag_name: string, name: string, body: string) { 250 | try { 251 | await utils.sleep(this.delayInMs); 252 | // get an array of GitHub labels for the new repo 253 | let result = await this.githubApi.repos.createRelease({ 254 | owner: this.githubOwner, 255 | repo: this.githubRepo, 256 | tag_name, 257 | name, 258 | body, 259 | }); 260 | 261 | return result; 262 | } catch (err) { 263 | console.error('Could not create release on github'); 264 | console.error(err); 265 | return null; 266 | } 267 | } 268 | 269 | // ---------------------------------------------------------------------------- 270 | 271 | /** 272 | * Get a list of all the current GitHub pull requests. 273 | * This uses a while loop to make sure that each page of issues is received. 274 | */ 275 | async getAllGithubPullRequests() { 276 | let allPullRequests: PullsListResponseData = []; 277 | let page = 1; 278 | const perPage = 100; 279 | 280 | while (true) { 281 | await utils.sleep(this.delayInMs); 282 | // get a paginated list of pull requests 283 | const pullRequests = await this.githubApi.pulls.list({ 284 | owner: this.githubOwner, 285 | repo: this.githubRepo, 286 | state: 'all', 287 | per_page: perPage, 288 | page: page, 289 | }); 290 | 291 | // if this page has zero PRs then we are done! 292 | if (pullRequests.data.length === 0) break; 293 | 294 | // join this list of PRs with the master list 295 | allPullRequests = allPullRequests.concat(pullRequests.data); 296 | 297 | // if there are strictly less PRs on this page than the maximum number per page 298 | // then we can be sure that this is all the PRs. No use querying again. 299 | if (pullRequests.data.length < perPage) break; 300 | 301 | // query for the next page of PRs next iteration 302 | page++; 303 | } 304 | 305 | return allPullRequests; 306 | } 307 | 308 | // ---------------------------------------------------------------------------- 309 | 310 | /* 311 | ****************************************************************************** 312 | ******************************** POST METHODS ******************************** 313 | ****************************************************************************** 314 | */ 315 | userIsCreator(author: GitLabUser) { 316 | this.users.add(author.username as string); 317 | return ( 318 | author && 319 | ((settings.usermap && 320 | settings.usermap[author.username as string] === 321 | settings.github.token_owner) || 322 | author.username === settings.github.token_owner) 323 | ); 324 | } 325 | 326 | /** 327 | * Update the description of the repository on GitHub. 328 | * Replaces newlines and tabs with spaces. No attempt is made to remove e.g. Markdown 329 | * links or other special formatting. 330 | */ 331 | async updateRepositoryDescription(description: string) { 332 | let props: RestEndpointMethodTypes['repos']['update']['parameters'] = { 333 | owner: this.githubOwner, 334 | repo: this.githubRepo, 335 | description: description?.replace(/\s+/g, ' ') || '', 336 | }; 337 | return this.githubApi.repos.update(props); 338 | } 339 | 340 | /** 341 | * TODO description 342 | * @param milestones All GitHub milestones 343 | * @param issue The GitLab issue object 344 | */ 345 | async createIssue(issue: GitLabIssue) { 346 | let bodyConverted = await this.convertIssuesAndComments( 347 | issue.description ?? '', 348 | issue, 349 | !this.userIsCreator(issue.author) || !issue.description, 350 | true, 351 | true, 352 | ); 353 | 354 | let props: RestEndpointMethodTypes['issues']['create']['parameters'] = { 355 | owner: this.githubOwner, 356 | repo: this.githubRepo, 357 | title: issue.title ? issue.title.trim() : '', 358 | body: bodyConverted, 359 | }; 360 | 361 | props.assignees = this.convertAssignees(issue); 362 | props.milestone = this.convertMilestone(issue); 363 | props.labels = this.convertLabels(issue); 364 | 365 | await utils.sleep(this.delayInMs); 366 | 367 | if (settings.dryRun) return Promise.resolve({ data: issue }); 368 | 369 | return this.githubApi.issues.create(props); 370 | } 371 | 372 | /** 373 | * Converts GitLab assignees to GitHub usernames, using settings.usermap 374 | */ 375 | convertAssignees(item: GitLabIssue | GitLabMergeRequest): string[] { 376 | if (!item.assignees) return []; 377 | let assignees: string[] = []; 378 | for (let assignee of item.assignees) { 379 | let username: string = assignee.username as string; 380 | this.users.add(username); 381 | if (username === settings.github.username) { 382 | assignees.push(settings.github.username); 383 | } else if (settings.usermap && settings.usermap[username]) { 384 | assignees.push(settings.usermap[username]); 385 | } 386 | } 387 | return assignees; 388 | } 389 | 390 | /** 391 | * Returns the GitHub milestone id for a milestone GitLab property of an issue or MR 392 | * 393 | * Note that this requires milestoneMap to be built, either during migration 394 | * or read from GitHub using registerMilestoneMap() 395 | */ 396 | convertMilestone(item: GitLabIssue | GitLabMergeRequest): number | undefined { 397 | if (!this.milestoneMap) throw Error('this.milestoneMap not initialised'); 398 | if (!item.milestone) return undefined; 399 | 400 | for (let m of this.milestoneMap.values()) 401 | if (m.title == item.milestone.title) return m.number; 402 | 403 | return undefined; 404 | } 405 | 406 | /** 407 | * Converts GitLab labels to GitHub labels. 408 | * 409 | * This also adds "has attachment" if the issue links to data. 410 | */ 411 | convertLabels(item: GitLabIssue | GitLabMergeRequest): string[] { 412 | let labels: string[] = []; 413 | if (item.labels) { 414 | labels = item.labels.filter(l => { 415 | if (item.state !== 'closed') return true; 416 | 417 | let lower = l.toLowerCase(); 418 | // ignore any labels that should have been removed when the issue was closed 419 | return lower !== 'doing' && lower !== 'to do'; 420 | }); 421 | if (settings.conversion.useLowerCaseLabels) { 422 | labels = labels.map((el: string) => el.toLowerCase()); 423 | } 424 | } 425 | 426 | // If the item's description contains a url that contains "/uploads/", 427 | // it is likely to have an attachment 428 | if ( 429 | item.description && 430 | item.description.indexOf('/uploads/') > -1 && 431 | !settings.s3 432 | ) { 433 | labels.push('has attachment'); 434 | } 435 | 436 | // Differentiate between issue and merge request 437 | // Note that it needs to apply to placeholders as well 438 | if ('merge_requests_count' in item) { 439 | labels.push('gitlab issue'); 440 | } else { 441 | labels.push('gitlab merge request'); 442 | } 443 | 444 | if (item.state === 'merged') { 445 | labels.push('merged'); 446 | } 447 | 448 | return labels; 449 | } 450 | 451 | /** 452 | * Uses the preview issue import API to set creation date on issues and comments. 453 | * Also it does not notify assignees. 454 | * 455 | * See https://gist.github.com/jonmagic/5282384165e0f86ef105 456 | * @param milestones All GitHub milestones 457 | * @param issue The GitLab issue object 458 | */ 459 | async importIssueAndComments(issue: GitLabIssue) { 460 | let bodyConverted = issue.isPlaceholder 461 | ? issue.description ?? '' 462 | : await this.convertIssuesAndComments( 463 | issue.description ?? '', 464 | issue, 465 | !this.userIsCreator(issue.author) || !issue.description, 466 | true, 467 | true, 468 | ); 469 | 470 | let props: IssueImport = { 471 | title: issue.title ? issue.title.trim() : '', 472 | body: bodyConverted, 473 | created_at: issue.created_at, 474 | updated_at: issue.updated_at, 475 | closed: issue.state === 'closed', 476 | }; 477 | 478 | if (issue.state === 'closed') { 479 | props.closed_at = issue.closed_at; 480 | } 481 | 482 | let assignees = this.convertAssignees(issue); 483 | props.assignee = assignees.length == 1 ? assignees[0] : undefined; 484 | props.milestone = this.convertMilestone(issue); 485 | props.labels = this.convertLabels(issue); 486 | 487 | if (settings.dryRun) return Promise.resolve({ data: issue }); 488 | 489 | // 490 | // Issue comments 491 | // 492 | 493 | console.log('\tMigrating issue comments...'); 494 | 495 | let comments: CommentImport[] = []; 496 | 497 | if (issue.isPlaceholder) { 498 | console.log( 499 | '\t...this is a placeholder issue, no comments are migrated.' 500 | ); 501 | } else { 502 | let notes = await this.gitlabHelper.getIssueNotes(issue.iid); 503 | comments = await this.processNotesIntoComments(notes); 504 | } 505 | 506 | const issue_number = await this.requestImportIssue(props, comments); 507 | 508 | if (assignees.length > 1 && issue_number) { 509 | if (assignees.length > 10) { 510 | console.error( 511 | `Cannot add more than 10 assignees to GitHub issue #${issue_number}.` 512 | ); 513 | } else { 514 | console.log( 515 | `Importing ${assignees.length} assignees for GitHub issue #${issue_number}` 516 | ); 517 | } 518 | this.githubApi.issues.addAssignees({ 519 | owner: this.githubOwner, 520 | repo: this.githubRepo, 521 | issue_number: issue_number, 522 | assignees: assignees, 523 | }); 524 | } 525 | } 526 | 527 | /** 528 | * 529 | * @param notes 530 | * @returns Comments ready for requestImportIssue() 531 | */ 532 | async processNotesIntoComments( 533 | notes: GitLabNote[] 534 | ): Promise { 535 | if (!notes || !notes.length) { 536 | console.log(`\t...no comments available, nothing to migrate.`); 537 | return []; 538 | } 539 | 540 | let comments: CommentImport[] = []; 541 | 542 | // sort notes in ascending order of when they were created (by id) 543 | notes = notes.sort((a, b) => a.id - b.id); 544 | 545 | let nrOfMigratedNotes = 0; 546 | for (let note of notes) { 547 | if (this.checkIfNoteCanBeSkipped(note.body)) continue; 548 | 549 | this.users.add(note.author.username); 550 | let userIsPoster = 551 | (settings.usermap && 552 | settings.usermap[note.author.username] === 553 | settings.github.token_owner) || 554 | note.author.username === settings.github.token_owner; 555 | 556 | comments.push({ 557 | created_at: note.created_at, 558 | body: await this.convertIssuesAndComments( 559 | note.body, 560 | note, 561 | !userIsPoster || !note.body 562 | ), 563 | }); 564 | 565 | nrOfMigratedNotes++; 566 | } 567 | 568 | console.log( 569 | `\t...Done creating comments (migrated ${nrOfMigratedNotes} comments, skipped ${ 570 | notes.length - nrOfMigratedNotes 571 | } comments)` 572 | ); 573 | return comments; 574 | } 575 | 576 | /** 577 | * 578 | * @param discussions 579 | * @returns Comments ready for requestImportIssue() 580 | */ 581 | async processDiscussionsIntoComments( 582 | discussions: GitLabDiscussion[] 583 | ): Promise { 584 | if (!discussions || !discussions.length) { 585 | console.log(`\t...no comments available, nothing to migrate.`); 586 | return []; 587 | } 588 | 589 | let comments: CommentImport[] = []; 590 | 591 | // sort notes in ascending order of when they were created (by id) 592 | discussions = discussions.sort((a, b) => Number.parseInt(a.id) - Number.parseInt(b.id)); 593 | 594 | let nrOfMigratedNotes = 0; 595 | let nrOfSkippedNotes = 0; 596 | for (let discussion of discussions) { 597 | let discussionComments = []; 598 | 599 | for (let note of discussion.notes) { 600 | if (this.checkIfNoteCanBeSkipped(note.body)) { 601 | nrOfSkippedNotes++; 602 | continue; 603 | } 604 | 605 | let username = note.author.username as string; 606 | this.users.add(username); 607 | let userIsPoster = 608 | (settings.usermap && 609 | settings.usermap[username] === 610 | settings.github.token_owner) || 611 | username === settings.github.token_owner; 612 | 613 | // only add line ref for first note of discussion 614 | const add_line_ref = discussion.notes.indexOf(note) === 0; 615 | 616 | discussionComments.push({ 617 | created_at: note.created_at, 618 | body: await this.convertIssuesAndComments( 619 | note.body, 620 | note, 621 | !userIsPoster || !note.body, 622 | add_line_ref, 623 | ), 624 | }); 625 | 626 | nrOfMigratedNotes++; 627 | } 628 | 629 | // Combine notes for discussion into one comment 630 | if (discussionComments.length == 1) { 631 | comments.push(discussionComments[0]); 632 | } 633 | else if (discussionComments.length > 1) { 634 | let combinedBody = '**Discussion in GitLab:**\n\n'; 635 | let first_created_at = discussionComments[0].created_at; 636 | 637 | combinedBody += discussionComments.map(comment => comment.body).join('\n\n'); 638 | 639 | comments.push({ 640 | created_at: first_created_at, 641 | body: combinedBody, 642 | }); 643 | } 644 | } 645 | 646 | console.log( 647 | `\t...Done creating discussion comments (migrated ${nrOfMigratedNotes} comments, skipped ${ 648 | nrOfSkippedNotes 649 | } comments)` 650 | ); 651 | return comments; 652 | } 653 | 654 | /** 655 | * Calls the preview API for issue importing 656 | * 657 | * @param issue Props for the issue 658 | * @param comments Comments 659 | * @returns GitHub issue number or null if import failed 660 | */ 661 | async requestImportIssue( 662 | issue: IssueImport, 663 | comments: CommentImport[] 664 | ): Promise { 665 | // see: https://github.com/orgs/community/discussions/27190 666 | if (issue.body.length > 65536) { 667 | throw `${issue.title} has a body longer than 65536 characters. Please shorten it.`; 668 | } 669 | // create the GitHub issue from the GitLab issue 670 | let pending = await this.githubApi.request( 671 | `POST /repos/${settings.github.owner}/${settings.github.repo}/import/issues`, 672 | { 673 | issue: issue, 674 | comments: comments, 675 | } 676 | ); 677 | 678 | let result = null; 679 | while (true) { 680 | await utils.sleep(this.delayInMs); 681 | result = await this.githubApi.request( 682 | `GET /repos/${settings.github.owner}/${settings.github.repo}/import/issues/${pending.data.id}` 683 | ); 684 | if ( 685 | result.data.status === 'imported' || 686 | result.data.status === 'failed' 687 | ) { 688 | break; 689 | } 690 | } 691 | if (result.data.status === 'failed') { 692 | console.log('\tFAILED: '); 693 | console.log(result); 694 | console.log('\tERRORS:'); 695 | console.log(result.data.errors); 696 | return null; 697 | } 698 | 699 | let issue_number = result.data.issue_url.split('/').splice(-1)[0]; 700 | return issue_number; 701 | } 702 | 703 | // ---------------------------------------------------------------------------- 704 | 705 | /** 706 | * TODO description 707 | */ 708 | async createIssueComments(githubIssue: GitHubIssue, issue: GitLabIssue) { 709 | console.log('\tMigrating issue comments...'); 710 | 711 | // retrieve any notes/comments associated with this issue 712 | if (issue.isPlaceholder) { 713 | console.log( 714 | '\t...this is a placeholder issue, no comments are migrated.' 715 | ); 716 | return; 717 | } 718 | 719 | let notes = await this.gitlabHelper.getIssueNotes(issue.iid); 720 | 721 | // if there are no notes, then there is nothing to do! 722 | if (notes.length === 0) { 723 | console.log(`\t...no issue comments available, nothing to migrate.`); 724 | return; 725 | } 726 | 727 | // sort notes in ascending order of when they were created (by id) 728 | notes = notes.sort((a, b) => a.id - b.id); 729 | 730 | let nrOfMigratedNotes = 0; 731 | for (let note of notes) { 732 | const gotMigrated = await this.processNote(note, githubIssue); 733 | if (gotMigrated) nrOfMigratedNotes++; 734 | } 735 | 736 | console.log( 737 | `\t...Done creating issue comments (migrated ${nrOfMigratedNotes} comments, skipped ${ 738 | notes.length - nrOfMigratedNotes 739 | } comments)` 740 | ); 741 | } 742 | 743 | // ---------------------------------------------------------------------------- 744 | 745 | /** 746 | * This function checks if a note needs to be processed or if it can be skipped. 747 | * A note can be skipped if it contains predefined terms (like 'Status changed to...') 748 | * or if it contains any value from settings.skipMatchingComments -> 749 | * Note that this is case insensitive! 750 | * 751 | */ 752 | checkIfNoteCanBeSkipped(noteBody: string) { 753 | const stateChange = 754 | (/Status changed to .*/i.test(noteBody) && 755 | !/Status changed to closed by commit.*/i.test(noteBody)) || 756 | /^changed milestone to .*/i.test(noteBody) || 757 | /^Milestone changed to .*/i.test(noteBody) || 758 | /^(Re)*assigned to /i.test(noteBody) || 759 | /^added .* labels/i.test(noteBody) || 760 | /^Added ~.* label/i.test(noteBody) || 761 | /^removed ~.* label/i.test(noteBody) || 762 | /^mentioned in issue #\d+.*/i.test(noteBody) || 763 | // /^marked this issue as related to #\d+/i.test(noteBody) || 764 | /^mentioned in merge request !\d+/i.test(noteBody) || 765 | /^changed the description.*/i.test(noteBody) || 766 | /^changed title from.*to.*/i.test(noteBody); 767 | 768 | const matchingComment = settings.skipMatchingComments.reduce( 769 | (a, b) => a || new RegExp(b, 'i').test(noteBody), 770 | false 771 | ); 772 | 773 | return stateChange || matchingComment; 774 | } 775 | 776 | // ---------------------------------------------------------------------------- 777 | 778 | /* 779 | * Processes the current note. 780 | * This means, it either creates a comment in the github issue, or it gets skipped. 781 | * Return false when it got skipped, otherwise true. 782 | */ 783 | async processNote( 784 | note: GitLabNote | GitLabDiscussionNote, 785 | githubIssue: Pick 786 | ) { 787 | if (this.checkIfNoteCanBeSkipped(note.body)) return false; 788 | 789 | let bodyConverted = await this.convertIssuesAndComments(note.body, note); 790 | 791 | await utils.sleep(this.delayInMs); 792 | 793 | if (settings.dryRun) return true; 794 | 795 | await this.githubApi.issues 796 | .createComment({ 797 | owner: this.githubOwner, 798 | repo: this.githubRepo, 799 | issue_number: githubIssue.number, 800 | body: bodyConverted, 801 | }) 802 | .catch(x => { 803 | console.error('could not create GitHub issue comment!'); 804 | console.error(x); 805 | process.exit(1); 806 | }); 807 | return true; 808 | } 809 | 810 | // ---------------------------------------------------------------------------- 811 | 812 | /** 813 | * Update the issue state (i.e., closed or open). 814 | */ 815 | async updateIssueState(githubIssue: GitHubIssue, issue: GitLabIssue) { 816 | // default state is open so we don't have to update if the issue is closed. 817 | if (issue.state !== 'closed' || githubIssue.state === 'closed') return; 818 | 819 | let props: RestEndpointMethodTypes['issues']['update']['parameters'] = { 820 | owner: this.githubOwner, 821 | repo: this.githubRepo, 822 | issue_number: githubIssue.number, 823 | state: issue.state, 824 | }; 825 | 826 | await utils.sleep(this.delayInMs); 827 | 828 | if (settings.dryRun) return Promise.resolve(); 829 | 830 | return await this.githubApi.issues.update(props); 831 | } 832 | 833 | // ---------------------------------------------------------------------------- 834 | 835 | /** 836 | * Create a GitHub milestone from a GitLab milestone 837 | * @param milestone GitLab milestone data 838 | * @return Created milestone data (or void if debugging => nothing created) 839 | */ 840 | async createMilestone(milestone: MilestoneImport): Promise { 841 | // convert from GitLab to GitHub 842 | let bodyConverted = await this.convertIssuesAndComments( 843 | milestone.description, 844 | milestone, 845 | false 846 | ); 847 | 848 | let githubMilestone: RestEndpointMethodTypes['issues']['createMilestone']['parameters'] = 849 | { 850 | owner: this.githubOwner, 851 | repo: this.githubRepo, 852 | title: milestone.title, 853 | description: bodyConverted, 854 | state: milestone.state === 'active' ? 'open' : 'closed', 855 | }; 856 | 857 | if (milestone.due_date) { 858 | githubMilestone.due_on = milestone.due_date + 'T00:00:00Z'; 859 | } 860 | 861 | await utils.sleep(this.delayInMs); 862 | 863 | if (settings.dryRun) return Promise.resolve({ number: -1, title: 'DEBUG' }); 864 | 865 | const created = await this.githubApi.issues.createMilestone( 866 | githubMilestone 867 | ); 868 | 869 | return { number: created.data.number, title: created.data.title }; 870 | } 871 | 872 | // ---------------------------------------------------------------------------- 873 | 874 | /** 875 | * Create a GitHub label from a GitLab label 876 | */ 877 | async createLabel(label: SimpleLabel) { 878 | // convert from GitLab to GitHub 879 | let githubLabel = { 880 | owner: this.githubOwner, 881 | repo: this.githubRepo, 882 | name: label.name, 883 | color: label.color.substring(1), // remove leading "#" because gitlab returns it but github wants the color without it 884 | description: label.description, 885 | }; 886 | 887 | await utils.sleep(this.delayInMs); 888 | 889 | if (settings.dryRun) return Promise.resolve(); 890 | // create the GitHub label 891 | return await this.githubApi.issues.createLabel(githubLabel); 892 | } 893 | 894 | // ---------------------------------------------------------------------------- 895 | 896 | /** 897 | * Create a pull request, set its data, and set its comments 898 | * @param mergeRequest the GitLab merge request that we want to migrate 899 | */ 900 | async createPullRequestAndComments( 901 | mergeRequest: GitLabMergeRequest 902 | ): Promise { 903 | let pullRequestData = await this.createPullRequest(mergeRequest); 904 | 905 | // createPullRequest() returns an issue number if a PR could not be created and 906 | // an issue was created instead, and settings.useIssueImportAPI is true. In that 907 | // case comments were already added and the state is already properly set 908 | if (typeof pullRequestData === 'number' || !pullRequestData) return; 909 | 910 | let pullRequest = pullRequestData.data; 911 | 912 | // data is set to null if one of the branches does not exist and the pull request cannot be created 913 | if (pullRequest) { 914 | // Add milestones, labels, and other attributes from the Issues API 915 | await this.updatePullRequestData(pullRequest, mergeRequest); 916 | 917 | // add any comments/nodes associated with this pull request 918 | await this.createPullRequestComments(pullRequest, mergeRequest); 919 | 920 | // Make sure to close the GitHub pull request if it is closed or merged in GitLab 921 | await this.updatePullRequestState(pullRequest, mergeRequest); 922 | } 923 | } 924 | 925 | // ---------------------------------------------------------------------------- 926 | 927 | /** 928 | * Create a pull request. A pull request can only be created if both the target and source branches exist on the GitHub 929 | * repository. In many cases, the source branch is deleted when the merge occurs, and the merge request may not be able 930 | * to be migrated. In this case, an issue is created instead with a 'gitlab merge request' label. 931 | * @param mergeRequest the GitLab merge request object that we want to duplicate 932 | * @returns {Promise|Promise>|Promise<{data: *}>>} 933 | */ 934 | async createPullRequest(mergeRequest: GitLabMergeRequest) { 935 | let canCreate = !this.useIssuesForAllMergeRequests; 936 | 937 | if (canCreate) { 938 | // Check to see if the target branch exists in GitHub - if it does not exist, we cannot create a pull request 939 | try { 940 | await this.githubApi.repos.getBranch({ 941 | owner: this.githubOwner, 942 | repo: this.githubRepo, 943 | branch: mergeRequest.target_branch, 944 | }); 945 | } catch (err) { 946 | let gitlabBranches = await this.gitlabHelper.getAllBranches(); 947 | if (gitlabBranches.find(m => m.name === mergeRequest.target_branch)) { 948 | // Need to move that branch over to GitHub! 949 | console.error( 950 | `The '${mergeRequest.target_branch}' branch exists on GitLab but has not been migrated to GitHub. Please migrate the branch before migrating pull request #${mergeRequest.iid}.` 951 | ); 952 | return Promise.resolve({ data: null }); 953 | } else { 954 | console.error( 955 | `Merge request ${mergeRequest.iid} (target branch '${mergeRequest.target_branch}' does not exist => cannot migrate pull request, creating an issue instead.` 956 | ); 957 | canCreate = false; 958 | } 959 | } 960 | } 961 | 962 | if (canCreate) { 963 | // Check to see if the source branch exists in GitHub - if it does not exist, we cannot create a pull request 964 | try { 965 | await this.githubApi.repos.getBranch({ 966 | owner: this.githubOwner, 967 | repo: this.githubRepo, 968 | branch: mergeRequest.source_branch, 969 | }); 970 | } catch (err) { 971 | let gitlabBranches = await this.gitlabHelper.getAllBranches(); 972 | if (gitlabBranches.find(m => m.name === mergeRequest.source_branch)) { 973 | // Need to move that branch over to GitHub! 974 | console.error( 975 | `The '${mergeRequest.source_branch}' branch exists on GitLab but has not been migrated to GitHub. Please migrate the branch before migrating pull request #${mergeRequest.iid}.` 976 | ); 977 | return Promise.resolve({ data: null }); 978 | } else { 979 | console.error( 980 | `Pull request #${mergeRequest.iid} (source branch '${mergeRequest.source_branch}' does not exist => cannot migrate pull request, creating an issue instead.` 981 | ); 982 | canCreate = false; 983 | } 984 | } 985 | } 986 | 987 | if (settings.dryRun) return Promise.resolve({ data: mergeRequest }); 988 | 989 | if (canCreate) { 990 | let bodyConverted = await this.convertIssuesAndComments( 991 | mergeRequest.description, 992 | mergeRequest, 993 | true, 994 | true, 995 | true, 996 | ); 997 | 998 | // GitHub API Documentation to create a pull request: https://developer.github.com/v3/pulls/#create-a-pull-request 999 | let props = { 1000 | owner: this.githubOwner, 1001 | repo: this.githubRepo, 1002 | title: mergeRequest.title.trim(), 1003 | body: bodyConverted, 1004 | head: mergeRequest.source_branch, 1005 | base: mergeRequest.target_branch, 1006 | }; 1007 | 1008 | await utils.sleep(this.delayInMs); 1009 | 1010 | try { 1011 | // try to create the GitHub pull request from the GitLab issue 1012 | const response = await this.githubApi.pulls.create(props); 1013 | return Promise.resolve(response); 1014 | } catch (err) { 1015 | if (err.status === 422) { 1016 | console.error( 1017 | `Pull request #${mergeRequest.iid} - attempt to create has failed, assume '${mergeRequest.source_branch}' has already been merged => cannot migrate pull request, creating an issue instead.` 1018 | ); 1019 | // fall through to next section 1020 | } else { 1021 | throw err; 1022 | } 1023 | } 1024 | } 1025 | 1026 | // Failing all else, create an issue with a descriptive title 1027 | 1028 | let mergeStr = 1029 | '_Merges ' + 1030 | mergeRequest.source_branch + 1031 | ' -> ' + 1032 | mergeRequest.target_branch + 1033 | '_\n\n'; 1034 | let bodyConverted = await this.convertIssuesAndComments( 1035 | mergeStr + mergeRequest.description, 1036 | mergeRequest, 1037 | !this.userIsCreator(mergeRequest.author) || !settings.useIssueImportAPI, 1038 | true, 1039 | true, 1040 | ); 1041 | 1042 | if (settings.useIssueImportAPI) { 1043 | let assignees = this.convertAssignees(mergeRequest); 1044 | 1045 | let props: IssueImport = { 1046 | title: mergeRequest.title.trim() + ' - [' + mergeRequest.state + ']', 1047 | body: bodyConverted, 1048 | assignee: assignees.length > 0 ? assignees[0] : undefined, 1049 | created_at: mergeRequest.created_at, 1050 | updated_at: mergeRequest.updated_at, 1051 | closed: 1052 | mergeRequest.state === 'merged' || mergeRequest.state === 'closed', 1053 | labels: ['gitlab merge request'], 1054 | }; 1055 | 1056 | console.log('\tMigrating pull request comments...'); 1057 | let comments: CommentImport[] = []; 1058 | 1059 | if (!mergeRequest.iid) { 1060 | console.log( 1061 | '\t...this is a placeholder for a deleted GitLab merge request, no comments are created.' 1062 | ); 1063 | } else { 1064 | let discussions = await this.gitlabHelper.getAllMergeRequestDiscussions( 1065 | mergeRequest.iid 1066 | ); 1067 | comments = await this.processDiscussionsIntoComments(discussions); 1068 | } 1069 | 1070 | return this.requestImportIssue(props, comments); 1071 | } else { 1072 | let props = { 1073 | owner: this.githubOwner, 1074 | repo: this.githubRepo, 1075 | assignees: this.convertAssignees(mergeRequest), 1076 | title: mergeRequest.title.trim() + ' - [' + mergeRequest.state + ']', 1077 | body: bodyConverted, 1078 | }; 1079 | 1080 | // Add a label to indicate the issue is a merge request 1081 | if (!mergeRequest.labels) mergeRequest.labels = []; 1082 | mergeRequest.labels.push('gitlab merge request'); 1083 | 1084 | return this.githubApi.issues.create(props); 1085 | } 1086 | } 1087 | 1088 | // ---------------------------------------------------------------------------- 1089 | 1090 | /** 1091 | * Create comments for the pull request 1092 | * @param pullRequest the GitHub pull request object 1093 | * @param mergeRequest the GitLab merge request object 1094 | */ 1095 | async createPullRequestComments( 1096 | pullRequest: Pick, 1097 | mergeRequest: GitLabMergeRequest 1098 | ): Promise { 1099 | console.log('\tMigrating pull request comments...'); 1100 | 1101 | if (!mergeRequest.iid) { 1102 | console.log( 1103 | '\t...this is a placeholder for a deleted GitLab merge request, no comments are created.' 1104 | ); 1105 | return Promise.resolve(); 1106 | } 1107 | 1108 | let discussions = await this.gitlabHelper.getAllMergeRequestDiscussions( 1109 | mergeRequest.iid 1110 | ); 1111 | 1112 | // if there are no notes, then there is nothing to do! 1113 | if (discussions.length === 0) { 1114 | console.log( 1115 | `\t...no pull request comments available, nothing to migrate.` 1116 | ); 1117 | return; 1118 | } 1119 | 1120 | // Sort notes in ascending order of when they were created (by id) 1121 | discussions = discussions.sort((a, b) => a.notes[0].id - b.notes[0].id); 1122 | 1123 | let nrOfMigratedNotes = 0; 1124 | let nrOfSkippedNotes = 0; 1125 | for (let discussion of discussions) { 1126 | if (discussion.individual_note) { 1127 | const gotMigrated = await this.processNote(discussion.notes[0], pullRequest); 1128 | if (gotMigrated) { 1129 | nrOfMigratedNotes++; 1130 | } 1131 | else { 1132 | nrOfSkippedNotes++; 1133 | } 1134 | } 1135 | else { 1136 | // console.log('Processing discussion:'); 1137 | let discussionBody = '**Discussion in GitLab:**\n\n'; 1138 | 1139 | for (let note of discussion.notes) { 1140 | if (this.checkIfNoteCanBeSkipped(note.body)) { 1141 | nrOfSkippedNotes++; 1142 | continue; 1143 | } 1144 | 1145 | const add_line_ref = discussion.notes.indexOf(note) === 0; 1146 | let bodyConverted = await this.convertIssuesAndComments(note.body, note, true, add_line_ref); 1147 | discussionBody += bodyConverted; 1148 | discussionBody += '\n\n'; 1149 | nrOfMigratedNotes++; 1150 | } 1151 | 1152 | await utils.sleep(this.delayInMs); 1153 | 1154 | if (!settings.dryRun) { 1155 | await this.githubApi.issues 1156 | .createComment({ 1157 | owner: this.githubOwner, 1158 | repo: this.githubRepo, 1159 | issue_number: pullRequest.number, 1160 | body: discussionBody, 1161 | }) 1162 | .catch(x => { 1163 | console.error('could not create GitHub issue comment!'); 1164 | console.error(x); 1165 | process.exit(1); 1166 | }); 1167 | } 1168 | } 1169 | } 1170 | 1171 | console.log( 1172 | `\t...Done creating pull request comments (migrated ${nrOfMigratedNotes} pull request comments, skipped ${ 1173 | nrOfSkippedNotes 1174 | } pull request comments)` 1175 | ); 1176 | } 1177 | 1178 | // ---------------------------------------------------------------------------- 1179 | 1180 | /** 1181 | * Update the pull request data. The GitHub Pull Request API does not supply mechanisms to set the milestone, assignee, 1182 | * or labels; these data are set via the Issues API in this function 1183 | * @param pullRequest the GitHub pull request object 1184 | * @param mergeRequest the GitLab pull request object 1185 | * @returns {Promise>} 1186 | */ 1187 | async updatePullRequestData( 1188 | pullRequest: Pick, 1189 | mergeRequest: GitLabMergeRequest 1190 | ) { 1191 | let props: RestEndpointMethodTypes['issues']['update']['parameters'] = { 1192 | owner: this.githubOwner, 1193 | repo: this.githubRepo, 1194 | issue_number: pullRequest.number, 1195 | }; 1196 | 1197 | props.assignees = this.convertAssignees(mergeRequest); 1198 | props.milestone = this.convertMilestone(mergeRequest); 1199 | props.labels = this.convertLabels(mergeRequest); 1200 | 1201 | return await this.githubApi.issues.update(props); 1202 | } 1203 | 1204 | // ---------------------------------------------------------------------------- 1205 | 1206 | /** 1207 | * Update the pull request state 1208 | * @param pullRequest GitHub pull request object 1209 | * @param mergeRequest GitLab pull request object 1210 | * @returns {Promise|Github.Response|Promise>} 1211 | */ 1212 | async updatePullRequestState( 1213 | pullRequest: Pick, 1214 | mergeRequest: GitLabMergeRequest 1215 | ) { 1216 | if ( 1217 | mergeRequest.state === 'merged' && 1218 | pullRequest.state !== 'closed' && 1219 | !settings.dryRun 1220 | ) { 1221 | // Merging the pull request adds new commits to the tree; to avoid that, just close the merge requests 1222 | mergeRequest.state = 'closed'; 1223 | } 1224 | 1225 | // Default state is open so we don't have to update if the request is closed 1226 | if (mergeRequest.state !== 'closed' || pullRequest.state === 'closed') 1227 | return; 1228 | 1229 | let props: RestEndpointMethodTypes['issues']['update']['parameters'] = { 1230 | owner: this.githubOwner, 1231 | repo: this.githubRepo, 1232 | issue_number: pullRequest.number, 1233 | state: mergeRequest.state, 1234 | }; 1235 | 1236 | await utils.sleep(this.delayInMs); 1237 | 1238 | if (settings.dryRun) { 1239 | return Promise.resolve(); 1240 | } 1241 | 1242 | // Use the Issues API; all pull requests are issues, and we're not modifying any pull request-sepecific fields. This 1243 | // then works for merge requests that cannot be created and are migrated as issues. 1244 | return await this.githubApi.issues.update(props); 1245 | } 1246 | 1247 | // ---------------------------------------------------------------------------- 1248 | 1249 | /** 1250 | * Creates issues extracting commments from gitlab notes 1251 | * @param milestones GitHub milestones 1252 | * @param issue GitLab issue 1253 | */ 1254 | async createIssueAndComments(issue: GitLabIssue) { 1255 | if (settings.useIssueImportAPI) { 1256 | await this.importIssueAndComments(issue); 1257 | } else { 1258 | const githubIssueData = await this.createIssue(issue); 1259 | const githubIssue = githubIssueData.data; 1260 | // add any comments/notes associated with this issue 1261 | await this.createIssueComments( 1262 | githubIssue as GitHubIssue, 1263 | issue as GitLabIssue 1264 | ); 1265 | // make sure to close the GitHub issue if it is closed in GitLab 1266 | await this.updateIssueState( 1267 | githubIssue as GitHubIssue, 1268 | issue as GitLabIssue 1269 | ); 1270 | } 1271 | } 1272 | 1273 | // ---------------------------------------------------------------------------- 1274 | 1275 | // TODO fix unexpected type coercion risk 1276 | /** 1277 | * Converts issue body or issue comments from GitLab to GitHub. That means: 1278 | * - (optionally) Adds a line at the beginning indicating which original user created the 1279 | * issue or the comment and when - because the GitHub API creates everything 1280 | * as the API user 1281 | * - Changes username from GitLab to GitHub in "mentions" (@username) 1282 | * - Changes milestone references to links 1283 | * - Changes MR references to PR references, taking into account the changes 1284 | * in indexing due to GitHub PRs using following the same numbering as 1285 | * issues 1286 | * - Changes issue numbers (necessary e.g. if dummy GH issues were not 1287 | * created for deleted GL issues). 1288 | * 1289 | * FIXME: conversion should be deactivated depending on the context in the 1290 | * markdown, e.g. strike-through text for labels, or code blocks for all 1291 | * references. 1292 | * 1293 | * @param str Body of the GitLab note 1294 | * @param item GitLab item to which the note belongs 1295 | * @param add_line Set to true to add the line with author and creation date 1296 | * @param add_line_ref Set to true to add the line ref to the comment 1297 | * @param add_issue_information Set to true to add assignees, reviewers, and approvers 1298 | */ 1299 | async convertIssuesAndComments( 1300 | str: string, 1301 | item: GitLabIssue | GitLabMergeRequest | GitLabNote | MilestoneImport | GitLabDiscussionNote, 1302 | add_line: boolean = true, 1303 | add_line_ref: boolean = true, 1304 | add_issue_information: boolean = false, 1305 | ): Promise { 1306 | // A note on implementation: 1307 | // We don't convert project names once at the beginning because otherwise 1308 | // we would have to check whether "text#23" refers to issue 23 or not, and 1309 | // so on for MRs, milestones, etc. 1310 | // Instead we consider either project#issue or " #issue" with non-word char 1311 | // before the #, and we do the same for MRs, labels and milestones. 1312 | 1313 | const repoLink = `${this.githubUrl}/${this.githubOwner}/${this.githubRepo}`; 1314 | const hasUsermap = 1315 | settings.usermap !== null && Object.keys(settings.usermap).length > 0; 1316 | const hasProjectmap = 1317 | settings.projectmap !== null && 1318 | Object.keys(settings.projectmap).length > 0; 1319 | 1320 | if (add_line) str = GithubHelper.addMigrationLine(str, item, repoLink, add_line_ref); 1321 | let reString = ''; 1322 | 1323 | // Store usernames found in the text 1324 | const matches: Array = str.match(usernameRegex); 1325 | if (matches && matches.length > 0) { 1326 | for (const username of matches) { 1327 | this.users.add(username.substring(1)); 1328 | } 1329 | } 1330 | 1331 | // 1332 | // User name conversion 1333 | // 1334 | 1335 | if (hasUsermap) { 1336 | reString = '@' + Object.keys(settings.usermap).join('|@'); 1337 | str = str.replace( 1338 | new RegExp(reString, 'g'), 1339 | match => '@' + settings.usermap[match.substring(1)] 1340 | ); 1341 | } 1342 | 1343 | // 1344 | // Issue reference conversion 1345 | // 1346 | 1347 | let issueReplacer = (match: string) => { 1348 | // TODO: issueMap 1349 | return '#' + match; 1350 | }; 1351 | 1352 | if (hasProjectmap) { 1353 | reString = 1354 | '(' + Object.keys(settings.projectmap).join(')#(\\d+)|(') + ')#(\\d+)'; 1355 | str = str.replace( 1356 | new RegExp(reString, 'g'), 1357 | (_, p1, p2) => settings.projectmap[p1] + '#' + issueReplacer(p2) 1358 | ); 1359 | } 1360 | reString = '(?<=\\W)#(\\d+)'; 1361 | str = str.replace(new RegExp(reString, 'g'), (_, p1) => issueReplacer(p1)); 1362 | 1363 | // 1364 | // Milestone reference replacement 1365 | // 1366 | 1367 | let milestoneReplacer = ( 1368 | number: string = '', 1369 | title: string = '', 1370 | repo: string = '' 1371 | ) => { 1372 | let milestone: SimpleMilestone; 1373 | if (this.milestoneMap) { 1374 | if (number) { 1375 | milestone = this.milestoneMap.get(parseInt(number)); 1376 | } else if (title) { 1377 | for (let m of this.milestoneMap.values()) { 1378 | if (m.title === title) { 1379 | milestone = m; 1380 | break; 1381 | } 1382 | } 1383 | } 1384 | } 1385 | if (milestone) { 1386 | const repoLink = `${this.githubUrl}/${this.githubOwner}/${ 1387 | repo || this.githubRepo 1388 | }`; 1389 | return `[${milestone.title}](${repoLink}/milestone/${milestone.number})`; 1390 | } 1391 | console.log( 1392 | `\tMilestone '${number || title}' not found in milestone map.` 1393 | ); 1394 | return `'Reference to deleted milestone ${number || title}'`; 1395 | }; 1396 | 1397 | if (hasProjectmap) { 1398 | // Replace: project%"Milestone" 1399 | reString = 1400 | '(' + 1401 | Object.keys(settings.projectmap).join(')%(".*?")|(') + 1402 | ')%(".*?")'; 1403 | str = str.replace( 1404 | new RegExp(reString, 'g'), 1405 | (_, p1, p2) => `Milestone ${p2} in ${settings.projectmap[p1]}` 1406 | ); 1407 | 1408 | // Replace: project%nn 1409 | reString = 1410 | '(' + Object.keys(settings.projectmap).join(')%(\\d+)|(') + ')%(\\d+)'; 1411 | str = str.replace( 1412 | new RegExp(reString, 'g'), 1413 | (_, p1, p2) => 1414 | `[Milestone ${p2} in ${settings.projectmap[p1]}](${this.githubUrl}/${this.githubOwner}/${settings.projectmap[p1]})` 1415 | ); 1416 | } 1417 | // Replace: %"Milestone" 1418 | reString = '(?<=\\W)%"(.*?)"'; 1419 | str = str.replace(new RegExp(reString, 'g'), (_, p1) => 1420 | milestoneReplacer('', p1) 1421 | ); 1422 | 1423 | // Replace: %nn 1424 | reString = '(?<=\\W)%(\\d+)'; 1425 | str = str.replace(new RegExp(reString, 'g'), (_, p1) => 1426 | milestoneReplacer(p1, '') 1427 | ); 1428 | 1429 | // Replace github.com URLs to other organizations with redirect.github.com to avoid backlink spam 1430 | let urlMassager = (str: string) => { 1431 | return 'redirect.github.com'; 1432 | } 1433 | 1434 | // Keep owner internal links/back links intact 1435 | reString = `((?:to|redirect\\.|www\\.)?github\\.com)(?!\\/${settings.github.owner}\\/)`; 1436 | str = str.replace(new RegExp(reString, 'g'), (_, p1) => urlMassager(p1)); 1437 | 1438 | // 1439 | // Label reference conversion 1440 | // 1441 | 1442 | // FIXME: strike through in markdown is done as in: ~this text~ 1443 | // These regexes will capture ~this as a label. If it is among the migrated 1444 | // labels, then it will be linked. 1445 | 1446 | let labelReplacer = (label: string) => {}; 1447 | 1448 | // // Single word named label 1449 | // if (hasProjectmap) { 1450 | // const reChunk = '~([^~\\s\\.,;:\'"!@()\\\\\\[\\]])+(?=[^~\\w])'; 1451 | // reString = 1452 | // '(' 1453 | // + Object.keys(settings.projectmap).join(')' + reChunk + '|(') 1454 | // + ')' 1455 | // + reChunk; 1456 | // str = str.replace(new RegExp(reString, 'g'), 1457 | // (_, p1, p2) => ) 1458 | 1459 | // TODO 1460 | // } else { 1461 | // ... 1462 | // } 1463 | 1464 | // // Quoted named label 1465 | // reString = '~"([^~"]|\\w)+"(?=[^~\\w])'; 1466 | 1467 | // 1468 | // MR reference conversion 1469 | // 1470 | // TODO 1471 | 1472 | str = await utils.migrateAttachments( 1473 | str, 1474 | this.repoId, 1475 | settings.s3, 1476 | this.gitlabHelper 1477 | ); 1478 | 1479 | if (add_issue_information && settings.conversion.addIssueInformation) { 1480 | let issue = item as GitLabIssue; 1481 | str = await this.addIssueInformation(issue, str) 1482 | } 1483 | 1484 | if ('web_url' in item) { 1485 | str += '\n\n*Migrated from GitLab: ' + item.web_url + '*'; 1486 | } 1487 | 1488 | return str; 1489 | } 1490 | 1491 | // ---------------------------------------------------------------------------- 1492 | 1493 | /** 1494 | * Adds a line of text at the beginning of a comment that indicates who, when 1495 | * and from GitLab. 1496 | */ 1497 | static addMigrationLine(str: string, item: any, repoLink: string, add_line_ref: boolean = true): string { 1498 | if (!item || !item.author || !item.author.username || !item.created_at) { 1499 | return str; 1500 | } 1501 | 1502 | const dateformatOptions: Intl.DateTimeFormatOptions = { 1503 | month: 'short', 1504 | day: 'numeric', 1505 | year: 'numeric', 1506 | hour: 'numeric', 1507 | minute: 'numeric', 1508 | hour12: false, 1509 | timeZoneName: 'short', 1510 | }; 1511 | 1512 | const formattedDate = new Date(item.created_at).toLocaleString( 1513 | 'en-US', 1514 | dateformatOptions 1515 | ); 1516 | 1517 | const attribution = `***In GitLab by @${item.author.username} on ${formattedDate}:***`; 1518 | const lineRef = 1519 | item && item.position && add_line_ref 1520 | ? GithubHelper.createLineRef(item.position, repoLink) 1521 | : ''; 1522 | 1523 | const summary = attribution + (lineRef ? `\n\n${lineRef}` : ''); 1524 | 1525 | return `${summary}\n\n${str}`; 1526 | } 1527 | 1528 | /** 1529 | * When migrating in-line comments to GitHub then creates a link to the 1530 | * appropriate line of the diff. 1531 | */ 1532 | static createLineRef(position, repoLink: string): string { 1533 | if ( 1534 | !repoLink || 1535 | !repoLink.startsWith(gitHubLocation) || 1536 | !position || 1537 | !position.head_sha 1538 | ) { 1539 | return ''; 1540 | } 1541 | const base_sha = position.base_sha; 1542 | let head_sha = position.head_sha; 1543 | var path = ''; 1544 | var line = ''; 1545 | var slug = ''; 1546 | if ( 1547 | (position.new_line && position.new_path) || 1548 | (position.old_line && position.old_path) 1549 | ) { 1550 | var side; 1551 | if (!position.old_line || !position.old_path) { 1552 | side = 'R'; 1553 | path = position.new_path; 1554 | line = position.new_line; 1555 | } else { 1556 | side = 'L'; 1557 | path = position.old_path; 1558 | line = position.old_line; 1559 | } 1560 | const crypto = require('crypto'); 1561 | const hash = crypto.createHash('sha256').update(path).digest('hex'); 1562 | slug = `#diff-${hash}${side}${line}`; 1563 | } 1564 | // Mention the file and line number. If we can't get this for some reason then use the commit id instead. 1565 | const ref = path && line ? `${path} line ${line}` : `${head_sha}`; 1566 | let lineRef = `Commented on [${ref}](${repoLink}/compare/${base_sha}..${head_sha}${slug})\n\n`; 1567 | 1568 | if (position.line_range.start.type === 'new') { 1569 | const startLine = position.line_range.start.new_line; 1570 | const endLine = position.line_range.end.new_line; 1571 | const lineRange = (startLine !== endLine) ? `L${startLine}-L${endLine}` : `L${startLine}`; 1572 | lineRef += `${repoLink}/blob/${head_sha}/${path}#${lineRange}\n\n`; 1573 | } 1574 | else { 1575 | const startLine = position.line_range.start.old_line; 1576 | const endLine = position.line_range.end.old_line; 1577 | const lineRange = (startLine !== endLine) ? `L${startLine}-L${endLine}` : `L${startLine}`; 1578 | lineRef += `${repoLink}/blob/${head_sha}/${path}#${lineRange}\n\n`; 1579 | } 1580 | 1581 | return lineRef; 1582 | } 1583 | 1584 | async addIssueInformation(issue: GitLabIssue | GitLabMergeRequest, description: string): Promise { 1585 | let bodyConverted = description; 1586 | 1587 | let assignees = issue.assignees.map(a => a.username) as string[]; 1588 | bodyConverted += utils.organizationUsersString(assignees, "Assignees"); 1589 | 1590 | // check whether issue is of type GitLabMergeRequest 1591 | if (issue.reviewers) { 1592 | let mergeRequest = issue as GitLabMergeRequest; 1593 | 1594 | let mrReviewers = mergeRequest.reviewers.map(a => a.username) as string[]; 1595 | bodyConverted += utils.organizationUsersString(mrReviewers, 'Reviewers'); 1596 | 1597 | let approvals = await this.gitlabHelper.getMergeRequestApprovals(mergeRequest.iid); 1598 | bodyConverted += utils.organizationUsersString(approvals, 'Approved by'); 1599 | } 1600 | 1601 | return bodyConverted; 1602 | } 1603 | 1604 | /** 1605 | * Meh... 1606 | * @param milestoneMap 1607 | */ 1608 | async registerMilestoneMap(milestoneMap?: Map) { 1609 | if (milestoneMap) { 1610 | this.milestoneMap = milestoneMap; 1611 | } else if (!milestoneMap && !this.milestoneMap) { 1612 | let milestoneData = await this.getAllGithubMilestones(); 1613 | this.milestoneMap = new Map(); 1614 | milestoneData.forEach(m => this.milestoneMap.set(m.number, m)); 1615 | } 1616 | } 1617 | 1618 | /** 1619 | * Deletes the GH repository, then creates it again. 1620 | */ 1621 | async recreateRepo() { 1622 | let params = { 1623 | owner: this.githubOwner, 1624 | repo: this.githubRepo, 1625 | }; 1626 | 1627 | try { 1628 | console.log(`Deleting repo ${params.owner}/${params.repo}...`); 1629 | await this.githubApi.repos.delete(params); 1630 | console.log('\t...done.'); 1631 | } catch (err) { 1632 | if (err.status == 404) console.log(' not found.'); 1633 | else console.error(`\n\tSomething went wrong: ${err}.`); 1634 | } 1635 | try { 1636 | if (this.githubOwnerIsOrg) { 1637 | console.log(`Creating repo in organisation ${this.githubOwner}/${this.githubRepo}...`); 1638 | await this.githubApi.repos.createInOrg({ 1639 | org: this.githubOwner, 1640 | name: this.githubRepo, 1641 | private: true, 1642 | }); 1643 | } else { 1644 | console.log(`Creating repo ${this.githubTokenOwner}/${this.githubRepo}...`); 1645 | await this.githubApi.repos.createForAuthenticatedUser({ 1646 | name: this.githubRepo, 1647 | private: true, 1648 | }); 1649 | } 1650 | console.log('\t...done.'); 1651 | } catch (err) { 1652 | console.error(`\n\tSomething went wrong: ${err}.`); 1653 | } 1654 | await utils.sleep(this.delayInMs); 1655 | } 1656 | } 1657 | -------------------------------------------------------------------------------- /src/gitlabHelper.ts: -------------------------------------------------------------------------------- 1 | import { Gitlab } from '@gitbeaker/node'; 2 | import { 3 | DiscussionNote, 4 | DiscussionSchema, 5 | IssueSchema, 6 | MergeRequestSchema, 7 | MilestoneSchema, 8 | NoteSchema, 9 | UserSchema, 10 | } from '@gitbeaker/core/dist/types/types'; 11 | import { GitlabSettings } from './settings'; 12 | import axios from 'axios'; 13 | 14 | export type GitLabDiscussion = DiscussionSchema; 15 | export type GitLabDiscussionNote = DiscussionNote; 16 | export type GitLabIssue = IssueSchema; 17 | export type GitLabNote = NoteSchema; 18 | export type GitLabUser = Omit; 19 | export type GitLabMilestone = MilestoneSchema; 20 | export type GitLabMergeRequest = MergeRequestSchema; 21 | 22 | export class GitlabHelper { 23 | // Wait for this issue to be resolved 24 | // https://github.com/jdalrymple/gitbeaker/issues/793 25 | gitlabApi: InstanceType; 26 | 27 | gitlabUrl?: string; 28 | gitlabToken: string; 29 | gitlabProjectId: number; 30 | archived?: boolean; 31 | sessionCookie: string; 32 | 33 | host: string; 34 | projectPath?: string; 35 | allBranches: any; 36 | 37 | constructor( 38 | gitlabApi: InstanceType, 39 | gitlabSettings: GitlabSettings 40 | ) { 41 | this.gitlabApi = gitlabApi; 42 | this.gitlabUrl = gitlabSettings.url; 43 | this.gitlabToken = gitlabSettings.token; 44 | this.gitlabProjectId = gitlabSettings.projectId; 45 | this.host = gitlabSettings.url ? gitlabSettings.url : 'https://gitlab.com'; 46 | this.host = this.host.endsWith('/') 47 | ? this.host.substring(0, this.host.length - 1) 48 | : this.host; 49 | this.archived = gitlabSettings.listArchivedProjects ?? true; 50 | this.sessionCookie = gitlabSettings.sessionCookie; 51 | this.allBranches = null; 52 | } 53 | 54 | /** 55 | * List all projects that the GitLab user is associated with. 56 | */ 57 | async listProjects() { 58 | try { 59 | let projects; 60 | if (this.archived) { 61 | projects = await this.gitlabApi.Projects.all({ membership: true }); 62 | } else { 63 | projects = await this.gitlabApi.Projects.all({ membership: true, archived: this.archived }); 64 | } 65 | 66 | // print each project with info 67 | for (let project of projects) { 68 | console.log( 69 | project.id.toString(), 70 | '\t', 71 | project.name, 72 | '\t--\t', 73 | project['description'] 74 | ); 75 | } 76 | 77 | // instructions for user 78 | console.log('\n\n'); 79 | console.log( 80 | 'Select which project ID should be transported to github. Edit the settings.js accordingly. (gitlab.projectID)' 81 | ); 82 | console.log('\n\n'); 83 | } catch (err) { 84 | console.error('An Error occured while fetching all GitLab projects:'); 85 | console.error(err); 86 | } 87 | } 88 | 89 | /** 90 | * Stores project path in a field 91 | */ 92 | async registerProjectPath(project_d: number) { 93 | try { 94 | const project = await this.gitlabApi.Projects.show(project_d); 95 | this.projectPath = project['path_with_namespace']; 96 | } catch (err) { 97 | console.error('An Error occured while fetching all GitLab projects:'); 98 | console.error(err); 99 | } 100 | } 101 | 102 | /** 103 | * Gets all notes for a given issue. 104 | */ 105 | async getIssueNotes(issueIid: number): Promise { 106 | try { 107 | return await this.gitlabApi.IssueNotes.all( 108 | this.gitlabProjectId, 109 | issueIid, 110 | {} 111 | ); 112 | } catch (err) { 113 | console.error(`Could not fetch notes for GitLab issue #${issueIid}.`); 114 | return []; 115 | } 116 | } 117 | 118 | /** 119 | * Gets attachment using http get 120 | */ 121 | async getAttachment(relurl: string) { 122 | try { 123 | const attachmentUrl = this.host + '/-/project/' + this.gitlabProjectId + relurl; 124 | const data = ( 125 | await axios.get(attachmentUrl, { 126 | responseType: 'arraybuffer', 127 | headers: { 128 | // HACK: work around GitLab's API lack of GET for attachments 129 | // See https://gitlab.com/gitlab-org/gitlab/-/issues/24155 130 | Cookie: `_gitlab_session=${this.sessionCookie}`, 131 | }, 132 | }) 133 | ).data; 134 | return Buffer.from(data, 'binary'); 135 | } catch (err) { 136 | console.error(`Could not download attachment ${relurl}: ${err.response.statusText}`); 137 | console.error('Is your session cookie still valid?'); 138 | return null; 139 | } 140 | } 141 | 142 | /** 143 | * Gets all branches. 144 | */ 145 | async getAllBranches() { 146 | if (!this.allBranches) { 147 | this.allBranches = await this.gitlabApi.Branches.all( 148 | this.gitlabProjectId 149 | ); 150 | } 151 | return this.allBranches as any[]; 152 | } 153 | 154 | async getMergeRequestApprovals(pullRequestIid: number): Promise { 155 | try { 156 | let approvals = await this.gitlabApi.MergeRequestApprovals.showConfiguration( 157 | this.gitlabProjectId, 158 | { 159 | mergerequestIId: pullRequestIid, 160 | }, 161 | ); 162 | 163 | if (approvals.rules[0]) { 164 | return approvals.rules[0].approved_by.map(user => user.username); 165 | } 166 | 167 | console.log(`No approvals found for GitLab merge request !${pullRequestIid}.`) 168 | } catch (err) { 169 | console.error( 170 | `Could not fetch approvals for GitLab merge request !${pullRequestIid}: ${err}` 171 | ); 172 | } 173 | return []; 174 | } 175 | 176 | /** 177 | * Gets all notes for a given merge request. 178 | */ 179 | async getAllMergeRequestNotes(pullRequestIid: number): Promise { 180 | try { 181 | return this.gitlabApi.MergeRequestNotes.all( 182 | this.gitlabProjectId, 183 | pullRequestIid, 184 | {} 185 | ); 186 | } catch (err) { 187 | console.error( 188 | `Could not fetch notes for GitLab merge request #${pullRequestIid}.` 189 | ); 190 | return []; 191 | } 192 | } 193 | 194 | /** 195 | * Gets all notes for a given merge request. 196 | */ 197 | async getAllMergeRequestDiscussions(pullRequestIid: number): Promise { 198 | try { 199 | return this.gitlabApi.MergeRequestDiscussions.all( 200 | this.gitlabProjectId, 201 | pullRequestIid, 202 | {} 203 | ); 204 | } catch (err) { 205 | console.error( 206 | `Could not fetch notes for GitLab merge request #${pullRequestIid}.` 207 | ); 208 | return []; 209 | } 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { 2 | GithubHelper, 3 | MilestoneImport, 4 | SimpleLabel, 5 | SimpleMilestone, 6 | } from './githubHelper'; 7 | import { GitlabHelper, GitLabIssue, GitLabMilestone } from './gitlabHelper'; 8 | import settings from '../settings'; 9 | 10 | import { Octokit as GitHubApi } from '@octokit/rest'; 11 | import { throttling } from '@octokit/plugin-throttling'; 12 | import { Gitlab } from '@gitbeaker/node'; 13 | 14 | import { default as readlineSync } from 'readline-sync'; 15 | import * as fs from 'fs'; 16 | 17 | import AWS from 'aws-sdk'; 18 | 19 | const CCERROR = '\x1b[31m%s\x1b[0m'; // red 20 | const CCWARN = '\x1b[33m%s\x1b[0m'; // yellow 21 | const CCINFO = '\x1b[36m%s\x1b[0m'; // cyan 22 | const CCSUCCESS = '\x1b[32m%s\x1b[0m'; // green 23 | 24 | const counters = { 25 | nrOfPlaceholderIssues: 0, 26 | nrOfReplacementIssues: 0, 27 | nrOfFailedIssues: 0, 28 | nrOfPlaceholderMilestones: 0, 29 | }; 30 | 31 | if (settings.s3) { 32 | AWS.config.credentials = new AWS.Credentials({ 33 | accessKeyId: settings.s3.accessKeyId, 34 | secretAccessKey: settings.s3.secretAccessKey, 35 | }); 36 | } 37 | 38 | // Ensure that the GitLab token has been set in settings.js 39 | if ( 40 | !settings.gitlab.token || 41 | settings.gitlab.token === '{{gitlab private token}}' 42 | ) { 43 | console.log( 44 | '\n\nYou have to enter your GitLab private token in the settings.js file.' 45 | ); 46 | process.exit(1); 47 | } 48 | 49 | // Create a GitLab API object 50 | const gitlabApi = new Gitlab({ 51 | host: settings.gitlab.url ? settings.gitlab.url : 'http://gitlab.com', 52 | token: settings.gitlab.token, 53 | }); 54 | 55 | const MyOctokit = GitHubApi.plugin(throttling); 56 | 57 | // Create a GitHub API object 58 | const githubApi = new MyOctokit({ 59 | previews: settings.useIssueImportAPI ? ['golden-comet'] : [], 60 | debug: false, 61 | baseUrl: settings.github.apiUrl 62 | ? settings.github.apiUrl 63 | : 'https://api.github.com', 64 | timeout: 5000, 65 | headers: { 66 | 'user-agent': 'node-gitlab-2-github', // GitHub is happy with a unique user agent 67 | accept: 'application/vnd.github.v3+json', 68 | }, 69 | auth: 'token ' + settings.github.token, 70 | throttle: { 71 | onRateLimit: async (retryAfter, options, octokit, retryCount) => { 72 | console.log( 73 | `Request quota exhausted for request ${options.method} ${options.url}` 74 | ); 75 | console.log(`Retrying after ${retryAfter} seconds!`); 76 | return true; 77 | }, 78 | onSecondaryRateLimit: async (retryAfter, options, octokit, retryCount) => { 79 | console.log( 80 | `SecondaryRateLimit detected for request ${options.method} ${options.url}` 81 | ) 82 | console.log(`Retrying after ${retryAfter} seconds!`); 83 | return true; 84 | }, 85 | fallbackSecondaryRateRetryAfter: 600, 86 | }, 87 | }); 88 | 89 | const gitlabHelper = new GitlabHelper(gitlabApi, settings.gitlab); 90 | const githubHelper = new GithubHelper( 91 | githubApi, 92 | settings.github, 93 | gitlabHelper, 94 | settings.useIssuesForAllMergeRequests 95 | ); 96 | 97 | // If no project id is given in settings.js, just return 98 | // all of the projects that this user is associated with. 99 | if (!settings.gitlab.projectId) { 100 | gitlabHelper.listProjects(); 101 | } else { 102 | // user has chosen a project 103 | if (settings.github.recreateRepo === true) { 104 | recreate(); 105 | } 106 | migrate(); 107 | } 108 | 109 | // ---------------------------------------------------------------------------- 110 | 111 | /** 112 | * Asks for confirmation and maybe recreates the GitHub repository. 113 | */ 114 | async function recreate() { 115 | readlineSync.setDefaultOptions({ 116 | limit: ['no', 'yes'], 117 | limitMessage: 'Please enter yes or no', 118 | defaultInput: 'no', 119 | }); 120 | const ans = readlineSync.question('Delete and recreate? [yes/no] '); 121 | if (ans == 'yes') await githubHelper.recreateRepo(); 122 | else console.log("OK, I won't delete anything then."); 123 | } 124 | 125 | /** 126 | * Creates dummy data for a placeholder milestone 127 | * 128 | * @param expectedIdx Number of the GitLab milestone 129 | * @returns Data for the milestone 130 | */ 131 | function createPlaceholderMilestone(expectedIdx: number): MilestoneImport { 132 | return { 133 | id: -1, // dummy 134 | iid: expectedIdx, 135 | title: `[PLACEHOLDER] - for milestone #${expectedIdx}`, 136 | description: 137 | 'This is to ensure that milestone numbers in GitLab and GitHub are the same', 138 | state: 'closed', 139 | }; 140 | } 141 | 142 | /** 143 | * Creates dummy data for a placeholder issue 144 | * 145 | * @param expectedIdx Number of the GitLab issue 146 | * @returns Data for the issue 147 | */ 148 | function createPlaceholderIssue(expectedIdx: number): Partial { 149 | return { 150 | iid: expectedIdx, 151 | title: `[PLACEHOLDER] - for issue #${expectedIdx}`, 152 | description: 153 | 'This is to ensure that issue numbers in GitLab and GitHub are the same', 154 | state: 'closed', 155 | isPlaceholder: true, 156 | }; 157 | } 158 | 159 | // ---------------------------------------------------------------------------- 160 | 161 | /** 162 | * Creates a so-called "replacement-issue". 163 | * 164 | * This is used for issues where the migration fails. The replacement issue will 165 | * have the same number and title, but the original description will be lost. 166 | */ 167 | function createReplacementIssue(issue: GitLabIssue) { 168 | let description = `The original issue\n\n\tId: ${issue.iid}\n\tTitle: ${issue.title}\n\ncould not be created.\nThis is a dummy issue, replacing the original one.`; 169 | 170 | if (issue.web_url) { 171 | description += `In case the gitlab repository still exists, visit the following link to see the original issue:\n\n${issue.web_url}`; 172 | } 173 | 174 | return { 175 | iid: issue.iid, 176 | title: `${issue.title} [REPLACEMENT ISSUE]`, 177 | description, 178 | state: issue.state, 179 | created_at: issue.created_at, 180 | }; 181 | } 182 | 183 | // ---------------------------------------------------------------------------- 184 | 185 | /** 186 | * Performs all of the migration tasks to move a GitLab repo to GitHub 187 | */ 188 | async function migrate() { 189 | // 190 | // Sequentially transfer repo things 191 | // 192 | 193 | try { 194 | await githubHelper.registerRepoId(); 195 | await gitlabHelper.registerProjectPath(settings.gitlab.projectId); 196 | 197 | if (settings.transfer.description) { 198 | await transferDescription(); 199 | } 200 | 201 | if (settings.transfer.milestones) { 202 | await transferMilestones( 203 | settings.usePlaceholderMilestonesForMissingMilestones 204 | ); 205 | } 206 | 207 | if (settings.transfer.labels) { 208 | await transferLabels(true, settings.conversion.useLowerCaseLabels); 209 | } 210 | 211 | if (settings.transfer.releases) { 212 | await transferReleases(); 213 | } 214 | 215 | // Important: do this before transferring the merge requests 216 | if (settings.transfer.issues) { 217 | await transferIssues(); 218 | } 219 | 220 | if (settings.transfer.mergeRequests) { 221 | if (settings.mergeRequests.log) { 222 | await logMergeRequests(settings.mergeRequests.logFile); 223 | } else { 224 | await transferMergeRequests(); 225 | } 226 | } 227 | 228 | if (settings.exportUsers) { 229 | const users = Array.from(githubHelper.users.values()).join("\n"); 230 | fs.writeFileSync('users.txt', users); 231 | } 232 | } catch (err) { 233 | console.error('Error during transfer:'); 234 | console.error(err); 235 | } 236 | 237 | console.log('\n\nTransfer complete!\n\n'); 238 | } 239 | 240 | // ---------------------------------------------------------------------------- 241 | 242 | /** 243 | * Transfer the description of the repository. 244 | */ 245 | async function transferDescription() { 246 | inform('Transferring Description'); 247 | 248 | let project = await gitlabApi.Projects.show(settings.gitlab.projectId); 249 | 250 | if (project.description) { 251 | await githubHelper.updateRepositoryDescription(project.description); 252 | console.log('Done.'); 253 | } else { 254 | console.log('Description is empty, nothing to transfer.') 255 | } 256 | } 257 | 258 | // ---------------------------------------------------------------------------- 259 | 260 | /** 261 | * Transfer any milestones that exist in GitLab that do not exist in GitHub. 262 | */ 263 | async function transferMilestones(usePlaceholders: boolean) { 264 | inform('Transferring Milestones'); 265 | 266 | // Get a list of all milestones associated with this project 267 | // FIXME: don't use type join but ensure everything is milestoneImport 268 | let milestones: (GitLabMilestone | MilestoneImport)[] = 269 | await gitlabApi.ProjectMilestones.all(settings.gitlab.projectId); 270 | 271 | // sort milestones in ascending order of when they were created (by id) 272 | milestones = milestones.sort((a, b) => a.id - b.id); 273 | 274 | // get a list of the current milestones in the new GitHub repo (likely to be empty) 275 | const githubMilestones = await githubHelper.getAllGithubMilestones(); 276 | let lastMilestoneId = 0; 277 | milestones.forEach(milestone => { 278 | lastMilestoneId = Math.max(lastMilestoneId, milestone.iid); 279 | }); 280 | 281 | let milestoneMap = new Map(); 282 | for (let i = 0; i < milestones.length; i++) { 283 | let milestone = milestones[i]; 284 | let expectedIdx = i + 1; // GitLab internal Id (iid) 285 | 286 | // Create placeholder milestones so that new GitHub milestones will have 287 | // the same milestone number as in GitLab. Gaps are caused by deleted 288 | // milestones 289 | if (usePlaceholders && milestone.iid !== expectedIdx) { 290 | let placeholder = createPlaceholderMilestone(expectedIdx); 291 | milestones.splice(i, 0, placeholder); 292 | counters.nrOfPlaceholderMilestones++; 293 | console.log( 294 | `Added placeholder milestone for GitLab milestone %${expectedIdx}.` 295 | ); 296 | milestoneMap.set(expectedIdx, { 297 | number: expectedIdx, 298 | title: placeholder.title, 299 | }); 300 | } else { 301 | milestoneMap.set(milestone.iid, { 302 | number: expectedIdx, 303 | title: milestone.title, 304 | }); 305 | } 306 | } 307 | await githubHelper.registerMilestoneMap(milestoneMap); 308 | 309 | // if a GitLab milestone does not exist in GitHub repo, create it. 310 | 311 | for (let milestone of milestones) { 312 | let foundMilestone = githubMilestones.find( 313 | m => m.title === milestone.title 314 | ); 315 | if (!foundMilestone) { 316 | console.log('Creating: ' + milestone.title); 317 | await githubHelper 318 | .createMilestone(milestone) 319 | .then(created => { 320 | let m = milestoneMap.get(milestone.iid); 321 | if (m && m.number != created.number) { 322 | throw new Error( 323 | `Mismatch between milestone ${m.number}: '${m.title}' in map and created ${created.number}: '${created.title}'` 324 | ); 325 | } 326 | }) 327 | .catch(err => { 328 | console.error(`Error creating milestone '${milestone.title}'.`); 329 | console.error(err); 330 | }); 331 | } else { 332 | console.log('Already exists: ' + milestone.title); 333 | } 334 | } 335 | } 336 | 337 | // ---------------------------------------------------------------------------- 338 | 339 | /** 340 | * Transfer any labels that exist in GitLab that do not exist in GitHub. 341 | */ 342 | async function transferLabels(attachmentLabel = true, useLowerCase = true) { 343 | inform('Transferring Labels'); 344 | console.warn(CCWARN,'NOTE (2022): GitHub descriptions are limited to 100 characters, and do not accept 4-byte Unicode'); 345 | 346 | const invalidUnicode = /[\u{10000}-\u{10FFFF}]|(?![*#0-9]+)[\p{Emoji}\p{Emoji_Modifier}\p{Emoji_Component}\p{Emoji_Modifier_Base}\p{Emoji_Presentation}]/gu; 347 | 348 | // Get a list of all labels associated with this project 349 | let labels: SimpleLabel[] = await gitlabApi.Labels.all( 350 | settings.gitlab.projectId 351 | ); 352 | 353 | // get a list of the current label names in the new GitHub repo (likely to be just the defaults) 354 | let githubLabels: string[] = await githubHelper.getAllGithubLabelNames(); 355 | 356 | // create a hasAttachment label for manual attachment migration 357 | if (attachmentLabel) { 358 | const hasAttachmentLabel = { 359 | name: 'has attachment', 360 | color: '#fbca04', 361 | description: 'Attachment was not transfered from GitLab', 362 | }; 363 | labels.push(hasAttachmentLabel); 364 | } 365 | 366 | const gitlabMergeRequestLabel = { 367 | name: 'gitlab merge request', 368 | color: '#b36b00', 369 | description: '', 370 | }; 371 | labels.push(gitlabMergeRequestLabel); 372 | 373 | // if a GitLab label does not exist in GitHub repo, create it. 374 | for (let label of labels) { 375 | // GitHub prefers lowercase label names 376 | if (useLowerCase) label.name = label.name.toLowerCase(); 377 | 378 | if (!githubLabels.find(l => l === label.name)) { 379 | console.log('Creating: ' + label.name); 380 | 381 | if (label.description) { 382 | if (label.description.match(invalidUnicode)) { 383 | console.warn(CCWARN,`⚠️ Removed invalid unicode characters from description.`); 384 | const cleanedDescription = label.description.replace(invalidUnicode, '').trim(); 385 | console.debug(` "${label.description}"\n\t to\n "${cleanedDescription}"`); 386 | label.description = cleanedDescription; 387 | } 388 | if (label.description.length > 100) { 389 | const trimmedDescription = label.description.slice(0,100).trim(); 390 | if (settings.trimOversizedLabelDescriptions) { 391 | console.warn(CCWARN,`⚠️ Description too long (${label.description.length}), it was trimmed:`); 392 | console.debug(` "${label.description}"\n\t to\n "${trimmedDescription}"`); 393 | label.description = trimmedDescription; 394 | } else { 395 | console.warn(CCWARN,`⚠️ Description too long (${label.description.length}), it was excluded.`); 396 | console.debug(` "${label.description}"`); 397 | label.description = ''; 398 | } 399 | } 400 | } 401 | 402 | try { 403 | // process asynchronous code in sequence 404 | await githubHelper.createLabel(label).catch(x => {}); 405 | } catch (err) { 406 | console.error('Could not create label', label.name); 407 | console.error(err); 408 | } 409 | } else { 410 | console.log('Already exists: ' + label.name); 411 | } 412 | } 413 | } 414 | 415 | // ---------------------------------------------------------------------------- 416 | 417 | /** 418 | * Transfer any issues and their comments that exist in GitLab that do not exist in GitHub. 419 | */ 420 | async function transferIssues() { 421 | inform('Transferring Issues'); 422 | 423 | await githubHelper.registerMilestoneMap(); 424 | 425 | // get a list of all GitLab issues associated with this project 426 | // TODO return all issues via pagination 427 | let issues = (await gitlabApi.Issues.all({ 428 | projectId: settings.gitlab.projectId, 429 | labels: settings.filterByLabel, 430 | })) as GitLabIssue[]; 431 | 432 | // sort issues in ascending order of their issue number (by iid) 433 | issues = issues.sort((a, b) => a.iid - b.iid); 434 | 435 | // get a list of the current issues in the new GitHub repo (likely to be empty) 436 | let githubIssues = await githubHelper.getAllGithubIssues(); 437 | 438 | console.log(`Transferring ${issues.length} issues.`); 439 | 440 | if (settings.usePlaceholderIssuesForMissingIssues) { 441 | for (let i = 0; i < issues.length; i++) { 442 | // GitLab issue internal Id (iid) 443 | let expectedIdx = i + 1; 444 | 445 | // is there a gap in the GitLab issues? 446 | // Create placeholder issues so that new GitHub issues will have the same 447 | // issue number as in GitLab. If a placeholder is used it is because there 448 | // was a gap in GitLab issues -- likely caused by a deleted GitLab issue. 449 | if (issues[i].iid !== expectedIdx) { 450 | issues.splice(i, 0, createPlaceholderIssue(expectedIdx) as GitLabIssue); // HACK: remove type coercion 451 | counters.nrOfPlaceholderIssues++; 452 | console.log( 453 | `Added placeholder issue for GitLab issue #${expectedIdx}.` 454 | ); 455 | } 456 | } 457 | } 458 | 459 | // 460 | // Create GitHub issues for each GitLab issue 461 | // 462 | 463 | // if a GitLab issue does not exist in GitHub repo, create it -- along with comments. 464 | for (let issue of issues) { 465 | // try to find a GitHub issue that already exists for this GitLab issue 466 | let githubIssue = githubIssues.find( 467 | i => i.title.trim() === issue.title.trim() && i.body.includes(issue.web_url) 468 | ); 469 | if (!githubIssue) { 470 | console.log(`\nMigrating issue #${issue.iid} ('${issue.title}')...`); 471 | try { 472 | // process asynchronous code in sequence -- treats the code sort of like blocking 473 | await githubHelper.createIssueAndComments(issue); 474 | console.log(`\t...DONE migrating issue #${issue.iid}.`); 475 | } catch (err) { 476 | console.log(`\t...ERROR while migrating issue #${issue.iid}.`); 477 | 478 | console.error('DEBUG:\n', err); // TODO delete this after issue-migration-fails have been fixed 479 | 480 | if (settings.useReplacementIssuesForCreationFails) { 481 | console.log('\t-> creating a replacement issue...'); 482 | const replacementIssue = createReplacementIssue(issue); 483 | try { 484 | await githubHelper.createIssueAndComments( 485 | replacementIssue as GitLabIssue 486 | ); // HACK: remove type coercion 487 | 488 | counters.nrOfReplacementIssues++; 489 | console.error('\t...DONE.'); 490 | } catch (err) { 491 | counters.nrOfFailedIssues++; 492 | console.error( 493 | '\t...ERROR: Could not create replacement issue either!' 494 | ); 495 | } 496 | } 497 | } 498 | } else { 499 | console.log(`Updating issue #${issue.iid} - ${issue.title}...`); 500 | try { 501 | await githubHelper.updateIssueState(githubIssue, issue); 502 | console.log(`...Done updating issue #${issue.iid}.`); 503 | } catch (err) { 504 | console.log(`...ERROR while updating issue #${issue.iid}.`); 505 | } 506 | } 507 | } 508 | 509 | // print statistics about issue migration: 510 | console.log(`DONE creating issues.`); 511 | console.log(`\n\tStatistics:`); 512 | console.log(`\tTotal nr. of issues: ${issues.length}`); 513 | console.log( 514 | `\tNr. of used placeholder issues: ${counters.nrOfPlaceholderIssues}` 515 | ); 516 | console.log( 517 | `\tNr. of used replacement issues: ${counters.nrOfReplacementIssues}` 518 | ); 519 | console.log(`\tNr. of issue migration fails: ${counters.nrOfFailedIssues}`); 520 | } 521 | // ---------------------------------------------------------------------------- 522 | 523 | /** 524 | * Transfer any merge requests that exist in GitLab that do not exist in GitHub 525 | * TODO - Update all text references to use the new issue numbers; 526 | * GitHub treats pull requests as issues, therefore their numbers are changed 527 | * @returns {Promise} 528 | */ 529 | async function transferMergeRequests() { 530 | inform('Transferring Merge Requests'); 531 | 532 | await githubHelper.registerMilestoneMap(); 533 | 534 | // Get a list of all pull requests (merge request equivalent) associated with 535 | // this project 536 | let mergeRequests = await gitlabApi.MergeRequests.all({ 537 | projectId: settings.gitlab.projectId, 538 | labels: settings.filterByLabel, 539 | }); 540 | 541 | // Sort merge requests in ascending order of their number (by iid) 542 | mergeRequests = mergeRequests.sort((a, b) => a.iid - b.iid); 543 | 544 | // Get a list of the current pull requests in the new GitHub repo (likely to 545 | // be empty) 546 | let githubPullRequests = await githubHelper.getAllGithubPullRequests(); 547 | 548 | // get a list of the current issues in the new GitHub repo (likely to be empty) 549 | // Issues are sometimes created from Gitlab merge requests. Avoid creating duplicates. 550 | let githubIssues = await githubHelper.getAllGithubIssues(); 551 | 552 | console.log( 553 | 'Transferring ' + mergeRequests.length.toString() + ' merge requests' 554 | ); 555 | 556 | // 557 | // Create GitHub pull request for each GitLab merge request 558 | // 559 | 560 | // if a GitLab merge request does not exist in GitHub repo, create it -- along 561 | // with comments 562 | for (let mr of mergeRequests) { 563 | // Try to find a GitHub pull request that already exists for this GitLab 564 | // merge request 565 | let githubRequest = githubPullRequests.find( 566 | i => i.title.trim() === mr.title.trim() && i.body.includes(mr.web_url) 567 | ); 568 | let githubIssue = githubIssues.find( 569 | // allow for issues titled "Original Issue Name - [merged|closed]" 570 | i => { 571 | // regex needs escaping in case merge request title contains special characters 572 | const regex = new RegExp(escapeRegExp(mr.title.trim()) + ' - \\[(merged|closed)\\]'); 573 | return regex.test(i.title.trim()) && i.body.includes(mr.web_url); 574 | } 575 | ); 576 | if (!githubRequest && !githubIssue) { 577 | if (settings.skipMergeRequestStates.includes(mr.state)) { 578 | console.log( 579 | `Skipping MR ${mr.iid} in "${mr.state}" state: ${mr.title}` 580 | ); 581 | continue; 582 | } 583 | console.log('Creating pull request: !' + mr.iid + ' - ' + mr.title); 584 | try { 585 | // process asynchronous code in sequence 586 | await githubHelper.createPullRequestAndComments(mr); 587 | } catch (err) { 588 | console.error( 589 | 'Could not create pull request: !' + mr.iid + ' - ' + mr.title 590 | ); 591 | console.error(err); 592 | // Stop execution to be able to correct issue 593 | process.exit(1); 594 | } 595 | } else { 596 | if (githubRequest) { 597 | console.log( 598 | 'Gitlab merge request already exists (as github pull request): ' + 599 | mr.iid + 600 | ' - ' + 601 | mr.title 602 | ); 603 | githubHelper.updatePullRequestState(githubRequest, mr); 604 | } else { 605 | console.log( 606 | 'Gitlab merge request already exists (as github issue): ' + 607 | mr.iid + 608 | ' - ' + 609 | mr.title 610 | ); 611 | } 612 | } 613 | } 614 | } 615 | 616 | /** 617 | * Transfer any releases that exist in GitLab that do not exist in GitHub 618 | * Please note that due to github api restrictions, this only transfers the 619 | * name, description and tag name of the release. It sorts the releases chronologically 620 | * and creates them on github one by one 621 | * @returns {Promise} 622 | */ 623 | async function transferReleases() { 624 | inform('Transferring Releases'); 625 | 626 | // Get a list of all releases associated with this project 627 | let releases = await gitlabApi.Releases.all(settings.gitlab.projectId); 628 | 629 | // Sort releases in ascending order of their release date 630 | releases = releases.sort((a, b) => { 631 | return (new Date(a.released_at) as any) - (new Date(b.released_at) as any); 632 | }); 633 | 634 | console.log('Transferring ' + releases.length.toString() + ' releases'); 635 | 636 | // 637 | // Create GitHub release for each GitLab release 638 | // 639 | 640 | // if a GitLab release does not exist in GitHub repo, create it 641 | for (let release of releases) { 642 | // Try to find an existing github release that already exists for this GitLab 643 | // release 644 | let githubRelease = await githubHelper.getReleaseByTag(release.tag_name); 645 | 646 | if (!githubRelease) { 647 | console.log( 648 | 'Creating release: !' + release.name + ' - ' + release.tag_name 649 | ); 650 | try { 651 | // process asynchronous code in sequence 652 | await githubHelper.createRelease( 653 | release.tag_name, 654 | release.name, 655 | release.description 656 | ); 657 | } catch (err) { 658 | console.error( 659 | 'Could not create release: !' + 660 | release.name + 661 | ' - ' + 662 | release.tag_name 663 | ); 664 | console.error(err); 665 | } 666 | } else { 667 | console.log( 668 | 'Gitlab release already exists (as github release): ' + 669 | githubRelease.data.name + 670 | ' - ' + 671 | githubRelease.data.tag_name 672 | ); 673 | } 674 | } 675 | } 676 | 677 | //----------------------------------------------------------------------------- 678 | 679 | /** 680 | * logs merge requests that exist in GitLab to a file. 681 | */ 682 | async function logMergeRequests(logFile: string) { 683 | inform('Logging Merge Requests'); 684 | 685 | // get a list of all GitLab merge requests associated with this project 686 | // TODO return all MRs via pagination 687 | let mergeRequests = await gitlabApi.MergeRequests.all({ 688 | projectId: settings.gitlab.projectId, 689 | labels: settings.filterByLabel, 690 | }); 691 | 692 | // sort MRs in ascending order of when they were created (by id) 693 | mergeRequests = mergeRequests.sort((a, b) => a.id - b.id); 694 | 695 | console.log('Logging ' + mergeRequests.length.toString() + ' merge requests'); 696 | 697 | for (let mr of mergeRequests) { 698 | let mergeRequestDiscussions = await gitlabApi.MergeRequestDiscussions.all( 699 | settings.gitlab.projectId, 700 | mr.iid 701 | ); 702 | let mergeRequestNotes = await gitlabApi.MergeRequestNotes.all( 703 | settings.gitlab.projectId, 704 | mr.iid, 705 | {} 706 | ); 707 | 708 | mr.discussions = mergeRequestDiscussions ? mergeRequestDiscussions : []; 709 | mr.notes = mergeRequestNotes ? mergeRequestNotes : []; 710 | } 711 | 712 | // 713 | // Log the merge requests to a file 714 | // 715 | const output = { mergeRequests: mergeRequests }; 716 | 717 | fs.writeFileSync(logFile, JSON.stringify(output, null, 2)); 718 | } 719 | 720 | // ---------------------------------------------------------------------------- 721 | 722 | /** 723 | * Print out a section heading to let the user know what is happening 724 | */ 725 | function inform(msg: string) { 726 | console.log('=================================='); 727 | console.log(msg); 728 | console.log('=================================='); 729 | } 730 | 731 | function escapeRegExp(string) { 732 | return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string 733 | } 734 | -------------------------------------------------------------------------------- /src/settings.ts: -------------------------------------------------------------------------------- 1 | export default interface Settings { 2 | dryRun: boolean; 3 | exportUsers: boolean; 4 | gitlab: GitlabSettings; 5 | github: GithubSettings; 6 | usermap: { 7 | [key: string]: string; 8 | }; 9 | projectmap: { 10 | [key: string]: string; 11 | }; 12 | conversion: { 13 | useLowerCaseLabels: boolean; 14 | addIssueInformation: boolean; 15 | }; 16 | transfer: { 17 | description: boolean; 18 | milestones: boolean; 19 | labels: boolean; 20 | issues: boolean; 21 | mergeRequests: boolean; 22 | releases: boolean; 23 | }; 24 | useIssueImportAPI: boolean; 25 | usePlaceholderMilestonesForMissingMilestones: boolean; 26 | usePlaceholderIssuesForMissingIssues: boolean; 27 | useReplacementIssuesForCreationFails: boolean; 28 | useIssuesForAllMergeRequests: boolean; 29 | filterByLabel?: string; 30 | trimOversizedLabelDescriptions: boolean; 31 | skipMergeRequestStates: string[]; 32 | skipMatchingComments: string[]; 33 | mergeRequests: { 34 | logFile: string; 35 | log: boolean; 36 | }; 37 | s3?: S3Settings; 38 | } 39 | 40 | export interface GithubSettings { 41 | baseUrl?: string; 42 | apiUrl?: string; 43 | owner: string; 44 | ownerIsOrg?: boolean; 45 | token: string; 46 | token_owner: string; 47 | repo: string; 48 | timeout?: number; 49 | username?: string; // when is this set??? 50 | recreateRepo?: boolean; 51 | } 52 | 53 | export interface GitlabSettings { 54 | url?: string; 55 | token: string; 56 | projectId: number; 57 | listArchivedProjects?: boolean; 58 | sessionCookie: string; 59 | } 60 | 61 | export interface S3Settings { 62 | accessKeyId: string; 63 | secretAccessKey: string; 64 | bucket: string; 65 | region: string; 66 | } 67 | -------------------------------------------------------------------------------- /src/utils.ts: -------------------------------------------------------------------------------- 1 | import { S3Settings } from './settings'; 2 | import settings from '../settings'; 3 | import * as mime from 'mime-types'; 4 | import * as path from 'path'; 5 | import * as crypto from 'crypto'; 6 | import S3 from 'aws-sdk/clients/s3'; 7 | import { GitlabHelper } from './gitlabHelper'; 8 | 9 | export const sleep = (milliseconds: number) => { 10 | return new Promise(resolve => setTimeout(resolve, milliseconds)); 11 | }; 12 | 13 | // Creates new attachments and replaces old links 14 | export const migrateAttachments = async ( 15 | body: string, 16 | githubRepoId: number | undefined, 17 | s3: S3Settings | undefined, 18 | gitlabHelper: GitlabHelper 19 | ) => { 20 | const regexp = /(!?)\[([^\]]+)\]\((\/uploads[^)]+)\)/g; 21 | 22 | // Maps link offset to a new name in S3 23 | const offsetToAttachment: { 24 | [key: number]: string; 25 | } = {}; 26 | 27 | // Find all local links 28 | const matches = body.matchAll(regexp); 29 | 30 | for (const match of matches) { 31 | const prefix = match[1] || ''; 32 | const name = match[2]; 33 | const url = match[3]; 34 | 35 | if (s3 && s3.bucket) { 36 | const basename = path.basename(url); 37 | const mimeType = mime.lookup(basename); 38 | const attachmentBuffer = await gitlabHelper.getAttachment(url); 39 | if (!attachmentBuffer) { 40 | continue; 41 | } 42 | 43 | // // Generate file name for S3 bucket from URL 44 | const hash = crypto.createHash('sha256'); 45 | hash.update(url); 46 | const newFileName = hash.digest('hex') + '/' + basename; 47 | const relativePath = githubRepoId 48 | ? `${githubRepoId}/${newFileName}` 49 | : newFileName; 50 | // Doesn't seem like it is easy to upload an issue to github, so upload to S3 51 | //https://stackoverflow.com/questions/41581151/how-to-upload-an-image-to-use-in-issue-comments-via-github-api 52 | 53 | // Attempt to fix issue #140 54 | //const s3url = `https://${s3.bucket}.s3.amazonaws.com/${relativePath}`; 55 | let hostname = `${s3.bucket}.s3.amazonaws.com`; 56 | if (s3.region) { 57 | hostname = `s3.${s3.region}.amazonaws.com/${s3.bucket}`; 58 | } 59 | const s3url = `https://${hostname}/${relativePath}`; 60 | 61 | const s3bucket = new S3(); 62 | s3bucket.createBucket(() => { 63 | const params: S3.PutObjectRequest = { 64 | Key: relativePath, 65 | Body: attachmentBuffer, 66 | ContentType: mimeType === false ? undefined : mimeType, 67 | Bucket: s3.bucket, 68 | }; 69 | 70 | s3bucket.upload(params, function (err, data) { 71 | console.log(`\tUploading ${basename} to ${s3url}... `); 72 | if (err) { 73 | console.log('ERROR: ', err); 74 | } else { 75 | console.log(`\t...Done uploading`); 76 | } 77 | }); 78 | }); 79 | 80 | // Add the new URL to the map 81 | offsetToAttachment[ 82 | match.index as number 83 | ] = `${prefix}[${name}](${s3url})`; 84 | } else { 85 | // Not using S3: default to old URL, adding absolute path 86 | const host = gitlabHelper.host.endsWith('/') 87 | ? gitlabHelper.host 88 | : gitlabHelper.host + '/'; 89 | const attachmentUrl = host + gitlabHelper.projectPath + url; 90 | offsetToAttachment[ 91 | match.index as number 92 | ] = `${prefix}[${name}](${attachmentUrl})`; 93 | } 94 | } 95 | 96 | return body.replace( 97 | regexp, 98 | ({}, {}, {}, {}, offset, {}) => offsetToAttachment[offset] 99 | ); 100 | }; 101 | 102 | export const organizationUsersString = (users: string[], prefix: string): string => { 103 | let organizationUsers = []; 104 | for (let assignee of users) { 105 | let githubUser = settings.usermap[assignee as string]; 106 | if (githubUser) { 107 | githubUser = '@' + githubUser; 108 | } else { 109 | githubUser = assignee as string; 110 | } 111 | organizationUsers.push(githubUser); 112 | } 113 | 114 | if (organizationUsers.length > 0) { 115 | return `\n\n**${prefix}:** ` + organizationUsers.join(', '); 116 | } 117 | 118 | return ''; 119 | } 120 | -------------------------------------------------------------------------------- /tsconfig.dev.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "module": "commonjs", 5 | "baseUrl": "." 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "allowJs": false, 4 | "lib": ["es2020.string"], 5 | "target": "es5", 6 | "downlevelIteration": true, 7 | //"strict": true, 8 | "esModuleInterop": true 9 | }, 10 | "include": ["./src/**/*", "settings.ts"] 11 | } 12 | --------------------------------------------------------------------------------