├── .gitignore ├── .prout.json ├── .travis.yml ├── BUILD.md ├── README.md ├── app.json ├── app ├── Filters.scala ├── controllers │ ├── Api.scala │ ├── Application.scala │ ├── Auth.scala │ ├── Binders.scala │ └── EmailRegistration.scala ├── lib │ ├── AnalyticsConfig.scala │ ├── AppUser.scala │ ├── Bot.scala │ ├── Dates.scala │ ├── Email.scala │ ├── GithubAppConfig.scala │ ├── Links.scala │ ├── MailArchive.scala │ ├── MailType.scala │ ├── MarkdownParsing.scala │ ├── PRMailSettings.scala │ ├── PreviewSignatures.scala │ ├── ProjectRepo.scala │ ├── ProposedMail.scala │ ├── actions │ │ ├── Actions.scala │ │ └── requests.scala │ ├── aws │ │ ├── SES.scala │ │ └── SesAsyncHelpers.scala │ ├── checks │ │ ├── Check.scala │ │ ├── Checks.scala │ │ └── PRChecks.scala │ └── model │ │ ├── MessageSummary.scala │ │ ├── PRMessageIdFinder.scala │ │ ├── Patch.scala │ │ ├── PatchBomb.scala │ │ ├── PatchCommit.scala │ │ └── SubjectPrefix.scala └── views │ ├── fragments │ ├── emailAddresses.scala.html │ ├── footer.scala.html │ ├── head.scala.html │ ├── linkedBreadcrumb.scala.html │ ├── listPullRequests.scala.html │ ├── mailIdent.scala.html │ ├── patchBaseGuidelines.scala.html │ ├── sendTab.scala.html │ ├── sendTabContent.scala.html │ ├── sendTabs.scala.html │ └── user.scala.html │ ├── index.scala.html │ ├── listPullRequests.scala.html │ ├── main.scala.html │ ├── pullRequestSent.scala.html │ └── reviewPullRequest.scala.html ├── build.sbt ├── conf ├── application.conf └── routes ├── heroku.md ├── notes.md ├── project ├── build.properties └── plugins.sbt ├── public ├── images │ ├── favicon.png │ └── git-logo.png ├── javascript │ ├── changeHighlighter.js │ └── messageIdPicker.js └── stylesheets │ └── main.css ├── system.properties └── test ├── lib ├── MailArchiveSpec.scala ├── MailTypeSpec.scala ├── MarkdownParsingSpec.scala ├── PatchParsingSpec.scala └── model │ ├── MessageSummarySpec.scala │ ├── PatchBombSpec.scala │ └── SubjectPrefixSpec.scala └── resources └── samples ├── bfg └── pull87 │ ├── 0001-WIP-Support-converting-repos-to-Git-LFS.patch │ ├── 0002-Fix-Git-LFS-filepath-sharding.patch │ └── github.patch ├── git └── pull145 │ └── github.patch ├── mailarchives └── marc │ └── m.143228360912708.html └── raw.posts ├── raw.posted-by-aws-ses.txt ├── raw.posted-by-google-groups.txt └── raw.public-inbox.txt /.gitignore: -------------------------------------------------------------------------------- 1 | working-dir 2 | 3 | logs 4 | project/project 5 | project/target 6 | target 7 | tmp 8 | .history 9 | dist 10 | /.idea 11 | /*.iml 12 | /out 13 | /.idea_modules 14 | /.classpath 15 | /.project 16 | /RUNNING_PID 17 | /.settings 18 | # Created by http://www.gitignore.io 19 | 20 | ### Scala ### 21 | *.class 22 | *.log 23 | 24 | # sbt specific 25 | .cache/ 26 | .history/ 27 | .lib/ 28 | dist/* 29 | target/ 30 | lib_managed/ 31 | src_managed/ 32 | project/boot/ 33 | project/plugins/project/ 34 | 35 | # Scala-IDE specific 36 | .scala_dependencies 37 | .worksheet 38 | 39 | 40 | ### Intellij ### 41 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode 42 | 43 | ## Directory-based project format 44 | .idea/ 45 | # if you remove the above rule, at least ignore user-specific stuff: 46 | # .idea/workspace.xml 47 | # .idea/tasks.xml 48 | # and these sensitive or high-churn files: 49 | # .idea/dataSources.ids 50 | # .idea/dataSources.xml 51 | # .idea/sqlDataSources.xml 52 | # .idea/dynamic.xml 53 | 54 | ## File-based project format 55 | *.ipr 56 | *.iws 57 | *.iml 58 | 59 | ## Additional for IntelliJ 60 | out/ 61 | 62 | # generated by mpeltonen/sbt-idea plugin 63 | .idea_modules/ 64 | 65 | -------------------------------------------------------------------------------- /.prout.json: -------------------------------------------------------------------------------- 1 | { 2 | "checkpoints": { 3 | "PROD": { "url": "https://submitgit.herokuapp.com/", "overdue": "12M" } 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | scala: 3 | - 2.11.8 4 | jdk: 5 | - oraclejdk8 6 | sudo: false 7 | cache: 8 | directories: 9 | - $HOME/.sbt 10 | - $HOME/.ivy2 11 | before_cache: 12 | - find $HOME/.sbt -name "*.lock" -type f -delete 13 | - find $HOME/.ivy2/cache -name "ivydata-*.properties" -type f -delete 14 | -------------------------------------------------------------------------------- /BUILD.md: -------------------------------------------------------------------------------- 1 | For possible contributors looking to build & run _submitGit_ locally... 2 | 3 | # Requirements 4 | 5 | ### System Requirements 6 | 7 | _submitGit_ is written in Scala, a modern functional language that runs on the JVM - so it 8 | can run anywhere Java can. 9 | 10 | * Install JDK 8 or above 11 | * Install [sbt](http://www.scala-sbt.org/release/tutorial/Setup.html), the Scala build tool 12 | 13 | ### Service Credentials 14 | 15 | ##### Amazon Simple Email Service (SES) 16 | 17 | [Amazon SES](http://docs.aws.amazon.com/ses/latest/DeveloperGuide/Welcome.html) is used to 18 | send email for preview, and to the mailing list. Part of the reason for choosing it 19 | is that it's an SMTP service in good standing (not associated with spam) which allows 20 | you to send emails with arbitrary 'From:' addresses - so long as those emails are validated 21 | with Amazon SES 22 | 23 | You'll need an Amazon AWS account. Create an IAM User, and attach a policy to it allowing it to 24 | perform these actions: 25 | 26 | * `ses:SendRawEmail` 27 | * `ses:VerifyEmailIdentity` 28 | * `ses:GetIdentityVerificationAttributes` 29 | 30 | The AWS credentials associated with this user should be stored as an AWS CLI profile 31 | named `submitgit`. 32 | 33 | ##### GitHub 34 | 35 | Register a new OAuth application with GitHub on https://github.com/settings/applications/new : 36 | 37 | * Homepage URL : http://localhost:9000/ 38 | * Authorization callback URL : http://localhost:9000/oauth/callback 39 | 40 | Also generate a GitHub 'personal access token' with the `public_repo` scope at 41 | https://github.com/settings/tokens/new 42 | 43 | Tell _submitGit_ about these credentials by setting the following environment variables: 44 | 45 | * `GITHUB_APP_CLIENT_ID` 46 | * `GITHUB_APP_CLIENT_SECRET` 47 | * `SUBMITGIT_GITHUB_ACCESS_TOKEN` 48 | 49 | 50 | # Finally, build and run... 51 | 52 | * `git clone https://github.com/rtyley/submitgit.git` 53 | * `cd submitgit` 54 | * `sbt`<- start the sbt console 55 | * `run` <- run the server 56 | * Visit http://localhost:9000/ 57 | 58 | If you're going to make changes to the Scala code, you may want to use IntelliJ and it's Scala 59 | plugin to help with the Scala syntax...! 60 | 61 | I personally found Coursera's [online Scale course](https://www.coursera.org/course/progfun) 62 | very helpful in learning Scala, YMMV. 63 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | submitGit (beta) 2 | ========= 3 | 4 | The Git project has a [patch submission process](https://github.com/git/git/blob/1eb0545c/Documentation/SubmittingPatches#L120-L464) 5 | that's _unfamilar to the majority of Git users_. Any potential 6 | contributor—wanting to fix a small bug, or tweak documentation 7 | to make it more friendly—will learn that mailing a patch is 8 | [not as simple as you might expect](http://git-scm.com/docs/git-format-patch#_mua_specific_hints), 9 | especially if they're using a webmail service, like Gmail. The 10 | recommended approach is to forgo regular email clients, and 11 | use [`git-format-patch`](http://git-scm.com/docs/git-format-patch) 12 | and [`git-send-email`](http://git-scm.com/docs/git-send-email) 13 | on the command-line to ensure nothing gets lost or auto-bounced 14 | (for containing HTML, for instance, which Gmail does by default). 15 | 16 | This might change in the future, but for the moment, the core Git 17 | contributors have decided to keep patch review where it is, on 18 | the mailing list, and won't be reviewing, for instance, GitHub or 19 | Bitbucket pull requests. 20 | 21 | _submitGit_ is a small step in trying to make patch submission 22 | easier. If you create a pull request on [github.com/git/git/](https://github.com/git/git/pulls), 23 | _submitGit_ can send it to the mailing list for you, correctly 24 | formatting the patches. The discussion stays where it is—on the 25 | list—but at least that initial step is a little easier. 26 | -------------------------------------------------------------------------------- /app.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "submitGit", 3 | "description": "A bridge for submitting PRs to the GitHub mailing list", 4 | "repository": "https://github.com/rtyley/submitgit", 5 | "logo": "https://cloud.githubusercontent.com/assets/52038/5842786/aed6db4a-a19f-11e4-820b-48bce30a8eee.png", 6 | "success_url": "/", 7 | "keywords": ["github", "pull request", "git", "patch", "email"], 8 | "env": { 9 | "GITHUB_APP_CLIENT_ID": { 10 | "description": "register your instance of submitGit as an application in your GitHub settings.", 11 | "required": true 12 | }, 13 | "GITHUB_APP_CLIENT_SECRET": { 14 | "description": "The Client Secret credentials for GitHub OAuth V2.", 15 | "required": true 16 | }, 17 | "AWS_ACCESS_KEY_ID": { 18 | "description": " AWS access key ID", 19 | "required": true 20 | }, 21 | "AWS_SECRET_KEY": { 22 | "description": "AWS secret key", 23 | "required": true 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /app/Filters.scala: -------------------------------------------------------------------------------- 1 | import javax.inject.Inject 2 | 3 | import play.api.http.HttpFilters 4 | import play.filters.csrf.CSRFFilter 5 | 6 | class Filters @Inject() (csrfFilter: CSRFFilter) extends HttpFilters { 7 | def filters = Seq(csrfFilter) 8 | } -------------------------------------------------------------------------------- /app/controllers/Api.scala: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import javax.inject.Inject 4 | 5 | import com.madgag.scalagithub.model.RepoId 6 | import lib._ 7 | import lib.model.MessageSummary 8 | import play.api.cache.Cached 9 | import play.api.libs.json.Json._ 10 | import play.api.mvc._ 11 | 12 | import scala.Function.unlift 13 | import scala.concurrent.ExecutionContext.Implicits.global 14 | import scala.concurrent.duration._ 15 | import scala.language.postfixOps 16 | 17 | class Api @Inject() (cached: Cached) extends Controller { 18 | 19 | val MessageFoundHeader = "X-submitGit-MessageFound" 20 | 21 | val CacheForLongerIfMessageFound = 22 | unlift[ResponseHeader, Duration](_.headers.get(MessageFoundHeader).map(_ => 100 days)) 23 | 24 | def messageLookup(repoId: RepoId, query: String) = { 25 | cached.apply((_ => s"$repoId $query"): (RequestHeader => String), CacheForLongerIfMessageFound).default(3 seconds) { 26 | Action.async { 27 | for { 28 | messagesOpt <- Project.byRepoId(repoId).mailingList.lookupMessage(query) 29 | } yield { 30 | Ok(toJson(messagesOpt: Seq[MessageSummary])).withHeaders(messagesOpt.map(_ => MessageFoundHeader -> "true") : _*) 31 | } 32 | } 33 | } 34 | } 35 | 36 | } 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /app/controllers/Application.scala: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import com.madgag.scalagithub.GitHub._ 4 | import com.madgag.scalagithub.model.{PullRequestId, RepoId, User} 5 | import lib.MailType.proposedMailByTypeFor 6 | import lib._ 7 | import lib.actions.Actions._ 8 | import lib.actions.Requests._ 9 | import lib.aws.SES._ 10 | import lib.aws.SesAsyncHelpers._ 11 | import lib.model.PRMessageIdFinder.messageIdsByMostRecentUsageIn 12 | import lib.model.PatchBomb 13 | import org.eclipse.jgit.lib.ObjectId 14 | import play.api.data.Form 15 | import play.api.data.Forms._ 16 | import play.api.i18n.Messages.Implicits._ 17 | import play.api.libs.json.Json 18 | import play.api.libs.json.Json._ 19 | import play.api.mvc._ 20 | import views.html.pullRequestSent 21 | 22 | import scala.concurrent.ExecutionContext.Implicits.global 23 | import scala.concurrent.Future 24 | 25 | object Application extends Controller { 26 | 27 | import play.api.Play.current 28 | 29 | def index = Action { implicit req => 30 | Ok(views.html.index()) 31 | } 32 | 33 | def listPullRequests(repoId: RepoId) = githubRepoAction(repoId).async { implicit req => 34 | implicit val g = req.gitHub 35 | for { 36 | openPRs <- req.repo.pullRequests.list(Map("state"->"open")).all() // req.repo.getPullRequests(GHIssueState.OPEN) 37 | closedPRs <- req.repo.pullRequests.list(Map("state"->"closed")).all() 38 | myself: User <- req.userF 39 | } yield { 40 | val (userPRs, otherPRs) = openPRs.partition(_.user.id.equals(myself.id)) 41 | 42 | val alternativePRs = otherPRs ++ closedPRs 43 | 44 | Ok(views.html.listPullRequests(userPRs, alternativePRs.take(3))) 45 | } 46 | } 47 | 48 | def reviewPullRequest(prId: PullRequestId) = githubPRAction(prId).async { implicit req => 49 | implicit val g = req.gitHub 50 | for { 51 | messageIds <- messageIdsByMostRecentUsageIn(req.pr) 52 | commits <- req.pr.commits.list().all() 53 | proposedMailByType <- proposedMailByTypeFor(req) 54 | } yield { 55 | implicit val form = mailSettingsForm.fill(settingsFor(req, messageIds)) 56 | Ok(views.html.reviewPullRequest(commits, proposedMailByType, messageIds)) 57 | } 58 | } 59 | 60 | def settingsFor(req: GHPRRequest[_], messageIds: Seq[String]): PRMailSettings = (for { 61 | data <- req.session.get(req.pr.prId.slug) 62 | s <- Json.parse(data).validate[PRMailSettings].asOpt 63 | } yield s).getOrElse(PRMailSettings("PATCH", messageIds.headOption)) 64 | 65 | def acknowledgePreview(prId: PullRequestId, headCommit: ObjectId, signature: String) = 66 | (GitHubAuthenticatedAction andThen verifyCommitSignature(headCommit, Some(signature))).async { 67 | implicit req => 68 | def whatDoWeTellTheUser(userEmail: String, verificationStatusOpt: Option[VerificationStatus]): Future[Option[(String, String)]] = { 69 | verificationStatusOpt match { 70 | case Some(VerificationStatus.Success) => // Nothing to do 71 | Future.successful(None) 72 | case Some(VerificationStatus.Pending) => // Remind user to click the link in their email 73 | Future.successful(Some("notifyEmailVerification" -> userEmail)) 74 | case _ => // send verification email, tell user to click on it 75 | ses.sendVerificationEmailTo(userEmail).map(_ => Some("notifyEmailVerification" -> userEmail)) 76 | } 77 | } 78 | 79 | for { 80 | userPrimaryEmail <- req.userPrimaryEmailF 81 | userEmail = userPrimaryEmail.email 82 | verificationStatusOpt <- ses.getIdentityVerificationStatusFor(userEmail) 83 | flashOpt <- whatDoWeTellTheUser(userEmail, verificationStatusOpt) 84 | } yield { 85 | Redirect(routes.Application.reviewPullRequest(prId)).addingToSession(PreviewSignatures.keyFor(headCommit) -> signature).flashing(flashOpt.toSeq: _*) 86 | } 87 | } 88 | 89 | val mailSettingsForm = Form( 90 | mapping( 91 | "subjectPrefix" -> default(text(maxLength = 20), "PATCH"), 92 | "inReplyTo" -> optional(text) 93 | )(PRMailSettings.apply)(PRMailSettings.unapply) 94 | ) 95 | 96 | def mailPullRequest(prId: PullRequestId, mailType: MailType) = (githubPRAction(prId) andThen mailChecks(mailType)).async(parse.form(mailSettingsForm)) { 97 | implicit req => 98 | implicit val g = req.gitHub 99 | val mailingList = Project.byRepoId(req.repo.repoId).mailingList 100 | 101 | val addresses = mailType.addressing(mailingList, req.user.address) 102 | 103 | val settings = req.body 104 | 105 | for { 106 | patchCommits <- req.patchCommitsF 107 | patchBomb = PatchBomb(patchCommits, addresses, settings.subjectPrefix, mailType.subjectPrefix, mailType.footer(req.pr)) 108 | initialEmail = patchBomb.emails.head 109 | initialMessageId <- ses.send(settings.inReplyTo.fold(initialEmail)(initialEmail.inReplyTo)) 110 | } yield { 111 | for (email <- patchBomb.emails.tail) { 112 | ses.send(email.inReplyTo(initialMessageId)) 113 | } 114 | val updatedSettings = mailType.afterSending(req.pr, patchBomb, initialMessageId, settings) 115 | Ok(pullRequestSent(req.user, mailType)).addingToSession(prId.slug -> toJson(updatedSettings).toString) 116 | } 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /app/controllers/Auth.scala: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import com.madgag.playgithub.auth.{AuthController, Client} 4 | import lib.GithubAppConfig 5 | 6 | object Auth extends AuthController { 7 | override val authClient: Client = GithubAppConfig.authClient 8 | } 9 | -------------------------------------------------------------------------------- /app/controllers/Binders.scala: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import com.madgag.scalagithub.model.{PullRequestId, RepoId} 4 | import lib.MailType 5 | import org.eclipse.jgit.lib.ObjectId 6 | import play.api.mvc.PathBindable.Parsing 7 | 8 | object Binders { 9 | 10 | implicit object bindableObjectId extends Parsing[ObjectId]( 11 | ObjectId.fromString, _.name, (key: String, e: Exception) => s"Cannot parse parameter '$key' as a commit id: ${e.getMessage}" 12 | ) 13 | 14 | implicit object bindableMailType extends Parsing[MailType]( 15 | MailType.bySlug, _.slug, (key: String, e: Exception) => s"Cannot parse parameter '$key' as a commit id: ${e.getMessage}" 16 | ) 17 | 18 | implicit object bindableRepoId extends Parsing[RepoId]( 19 | RepoId.from, _.fullName, (key: String, e: Exception) => s"Cannot parse repo name '$key'" 20 | ) 21 | 22 | implicit object bindablePullRequestId extends Parsing[PullRequestId]( 23 | PullRequestId.from, _.slug, (key: String, e: Exception) => s"Cannot parse pull request '$key'" 24 | ) 25 | } 26 | -------------------------------------------------------------------------------- /app/controllers/EmailRegistration.scala: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import com.madgag.playgithub.auth.GHRequest 4 | import lib.AppUser 5 | import lib.actions.Actions._ 6 | import lib.aws.SES._ 7 | import lib.aws.SesAsyncHelpers._ 8 | import play.api.mvc._ 9 | 10 | import scala.concurrent.ExecutionContext.Implicits.global 11 | 12 | object EmailRegistration extends Controller { 13 | 14 | val GitHubUserWithVerifiedEmail: ActionBuilder[GHRequest] = 15 | GitHubAuthenticatedAction andThen EnsureGitHubVerifiedEmail 16 | 17 | def isRegisteredEmail(email: String) = GitHubUserWithVerifiedEmail.async { 18 | for (status <- ses.getIdentityVerificationStatusFor(email)) yield Ok(status.map(_.string).getOrElse("Unknown")) 19 | } 20 | 21 | def registerEmail = GitHubUserWithVerifiedEmail.async { req => 22 | for { 23 | appUser <- AppUser.from(req) 24 | res <- ses.sendVerificationEmailTo(appUser.address.getAddress) 25 | } yield Ok("Registration email sent") 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /app/lib/AnalyticsConfig.scala: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | 4 | object AnalyticsConfig { 5 | 6 | import play.api.Play.current 7 | val config = play.api.Play.configuration 8 | 9 | val googleAnalyticsIdOpt = config.getString("google.analytics.id") 10 | } 11 | -------------------------------------------------------------------------------- /app/lib/AppUser.scala: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import javax.mail.internet.InternetAddress 4 | 5 | import com.madgag.playgithub.auth.GHRequest 6 | import com.madgag.scalagithub 7 | 8 | import scala.concurrent.ExecutionContext.Implicits.global 9 | 10 | case class AppUser( 11 | ghUser: scalagithub.model.User, 12 | primaryEmail: scalagithub.model.Email 13 | ) { 14 | lazy val address = new InternetAddress(primaryEmail.email, ghUser.displayName) 15 | } 16 | 17 | object AppUser { 18 | implicit def oGHUser(appUser: AppUser): scalagithub.model.User = appUser.ghUser 19 | 20 | def from(ghRequest: GHRequest[_]) = for { 21 | user <- ghRequest.userF 22 | userEmails <- ghRequest.userEmailsF 23 | } yield AppUser(user, userEmails.find(_.primary).get) 24 | } -------------------------------------------------------------------------------- /app/lib/Bot.scala: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import com.madgag.scalagithub.{GitHub, GitHubCredentials} 4 | 5 | import scalax.file.ImplicitConversions._ 6 | import scalax.file.Path 7 | 8 | object Bot { 9 | 10 | val workingDir = Path.fromString("/tmp") / "bot" / "working-dir" 11 | 12 | import play.api.Play.current 13 | val config = play.api.Play.configuration 14 | 15 | val accessToken = config.getString("github.botAccessToken").get 16 | 17 | val ghCreds = GitHubCredentials.forAccessKey(accessToken, workingDir.toPath).get 18 | 19 | def conn() = new GitHub(ghCreds) 20 | } 21 | -------------------------------------------------------------------------------- /app/lib/Dates.scala: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import org.joda.time.Period 4 | import org.joda.time.format.PeriodFormat 5 | 6 | object Dates { 7 | 8 | val humanPeriodFormat = PeriodFormat.getDefault 9 | 10 | implicit class RichPeriod(period: Period) { 11 | lazy val pretty = period.withMillis(0).toString(humanPeriodFormat) 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /app/lib/Email.scala: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import java.io.ByteArrayOutputStream 4 | import java.nio.ByteBuffer 5 | import java.util.Properties 6 | import javax.mail.Message.RecipientType 7 | import javax.mail.internet.{InternetAddress, MimeMessage} 8 | import javax.mail.{Address, Message} 9 | 10 | import lib.Email.Addresses 11 | 12 | object Email { 13 | case class Addresses( 14 | from: InternetAddress, 15 | to: Seq[InternetAddress] = Seq.empty, 16 | cc: Seq[InternetAddress] = Seq.empty, 17 | bcc: Seq[InternetAddress] = Seq.empty, 18 | replyTo: Option[InternetAddress] = None 19 | ) 20 | } 21 | 22 | case class Email( 23 | addresses: Addresses, 24 | subject: String, 25 | bodyText: String, 26 | charset: Option[String] = None, 27 | headers: Seq[(String, String)] = Seq.empty 28 | ) { 29 | 30 | def inReplyTo(unenclosedMessageId: String): Email = { 31 | val encloseMessageId = s"<$unenclosedMessageId>" 32 | copy(headers = headers ++ Seq("References" -> encloseMessageId, "In-Reply-To" -> encloseMessageId)) 33 | } 34 | 35 | lazy val toMimeMessage: ByteBuffer = { 36 | val s = javax.mail.Session.getInstance(new Properties(), null) 37 | val msg = new MimeMessage(s) 38 | 39 | // Sender and recipient 40 | msg.setFrom(addresses.from) 41 | def setRecipients(typ: Message.RecipientType, recipients: Seq[InternetAddress]) { 42 | if (recipients.nonEmpty) msg.setRecipients(typ, recipients.toArray[Address]) 43 | } 44 | 45 | setRecipients(RecipientType.TO, addresses.to) 46 | setRecipients(RecipientType.CC, addresses.cc) 47 | setRecipients(RecipientType.BCC, addresses.bcc) 48 | 49 | msg.setSubject(subject) 50 | headers.foreach(t => msg.addHeader(t._1, t._2)) 51 | addresses.replyTo.foreach(r => msg.setReplyTo(Array(r))) 52 | 53 | msg.setText(bodyText) 54 | 55 | val b = new ByteArrayOutputStream() 56 | 57 | msg.writeTo(b) 58 | 59 | ByteBuffer.wrap(b.toByteArray) 60 | 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /app/lib/GithubAppConfig.scala: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import com.madgag.playgithub.auth.Client 4 | 5 | object GithubAppConfig { 6 | 7 | import play.api.Play.current 8 | val config = play.api.Play.configuration 9 | 10 | val authClient = { 11 | val clientId = config.getString("github.clientId").getOrElse("blah") 12 | val clientSecret = config.getString("github.clientSecret").getOrElse("blah") 13 | 14 | Client(clientId, clientSecret) 15 | } 16 | 17 | } 18 | 19 | -------------------------------------------------------------------------------- /app/lib/Links.scala: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | object Links { 4 | val ChoosePatchBase = "https://github.com/git/git/blob/master/Documentation/SubmittingPatches#L4-7" 5 | } 6 | -------------------------------------------------------------------------------- /app/lib/MailArchive.scala: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder} 4 | import java.time.temporal.ChronoField._ 5 | import java.time.{LocalDateTime, ZoneId} 6 | import javax.mail.internet.InternetAddress 7 | 8 | import cats.data.OptionT 9 | import cats.implicits._ 10 | import com.madgag.okhttpscala._ 11 | import com.netaporter.uri.Uri 12 | import com.netaporter.uri.config.UriConfig 13 | import com.netaporter.uri.decoding.NoopDecoder 14 | import com.netaporter.uri.dsl._ 15 | import controllers.Application._ 16 | import lib.Email.Addresses 17 | import lib.MailArchive.okClient 18 | import lib.model.MessageSummary 19 | import okhttp3.{OkHttpClient, Request} 20 | import org.jsoup.Jsoup 21 | import org.jsoup.nodes.Element 22 | 23 | import scala.collection.convert.wrapAsScala._ 24 | import scala.concurrent.{ExecutionContext, Future} 25 | 26 | object RedirectCapturer { 27 | val okClient = new OkHttpClient.Builder().followRedirects(false).build() 28 | 29 | def redirectFor(url: Uri)(implicit ec: ExecutionContext): OptionT[Future, Uri] = 30 | OptionT(okClient.execute(new Request.Builder().url(url).build()) { resp => 31 | resp.code match { 32 | case FOUND => 33 | implicit val c = UriConfig(decoder = NoopDecoder) 34 | val redirectUri = Uri.parse(resp.header(LOCATION)) 35 | Some(redirectUri.copy( 36 | host = redirectUri.host.orElse(url.host), 37 | scheme = redirectUri.scheme.orElse(url.scheme) 38 | )) 39 | case _ => None 40 | } 41 | }) 42 | } 43 | 44 | 45 | trait MailArchive { 46 | val providerName: String 47 | 48 | val url: String 49 | 50 | /** 51 | * @return a url that's derived from the message-id - it may not be the 'canonical' url for the message, so hitting 52 | * this url may yield a redirect 53 | */ 54 | def linkFor(messageId: String): Uri 55 | 56 | /** 57 | * @return the canonical url for a message, if found- to find this out may require a network request to the mail archive. 58 | * The url may contain a mail-archive-specific id, eg: 59 | * https://marc.info/?l=git&m=143387261528519 60 | * https://groups.google.com/forum/#!msg/submitgit-test/-cq4q1w7jyY/fLlC47tosH4J 61 | */ 62 | def canonicalUrlFor(messageId: String)(implicit ec: ExecutionContext): OptionT[Future, Uri] 63 | 64 | def messageSummaryBasedOn(canonicalMessageUrl: Uri)(implicit ec: ExecutionContext): Future[MessageSummary] 65 | 66 | def lookupMessage(query: String)(implicit ec: ExecutionContext): Future[Seq[MessageSummary]] = { 67 | for { 68 | canonicalMessageUrl <- canonicalUrlFor(query) 69 | articleMessageSummary <- OptionT.liftF(messageSummaryBasedOn(canonicalMessageUrl)) 70 | } yield articleMessageSummary.copy(id = query) 71 | }.value.map(_.toSeq) 72 | 73 | } 74 | 75 | object MailArchive { 76 | val okClient = new OkHttpClient() 77 | } 78 | 79 | 80 | trait OffersRawMessage extends MailArchive { 81 | def rawUrlFor(canonicalMessageUrl: Uri): Uri 82 | 83 | override def messageSummaryBasedOn(canonicalMessageUrl: Uri)(implicit ec: ExecutionContext): Future[MessageSummary] = for { 84 | raw <- okClient.execute(new Request.Builder().url(rawUrlFor(canonicalMessageUrl)).build())(_.body.string) 85 | } yield MessageSummary.fromRawMessage(raw, canonicalMessageUrl) 86 | } 87 | 88 | 89 | case class PublicInbox(groupName: String) extends MailArchive with OffersRawMessage { 90 | val providerName = "public-inbox" 91 | 92 | val url = s"https://public-inbox.org/$groupName/" 93 | 94 | def linkFor(messageId: String) = s"https://public-inbox.org/$groupName/$messageId/" 95 | 96 | override def canonicalUrlFor(messageId: String)(implicit ec: ExecutionContext) = OptionT.some(linkFor(messageId)) 97 | 98 | override def rawUrlFor(canonicalMessageUrl: Uri) = canonicalMessageUrl + "raw" 99 | 100 | } 101 | 102 | object PublicInbox { 103 | val Git = PublicInbox("git") 104 | } 105 | 106 | object Marc { 107 | val Git = Marc("git") 108 | 109 | def deobfuscate(emailOrMessageId: String) = emailOrMessageId 110 | .replace(" () ","@") 111 | .replace(" ! ", ".") 112 | .replace("<","<") 113 | .replace(">",">") 114 | 115 | val DateTimeFormat = new DateTimeFormatterBuilder().parseCaseInsensitive 116 | .append(DateTimeFormatter.ISO_LOCAL_DATE).appendLiteral(' ') 117 | .appendValue(HOUR_OF_DAY).appendLiteral(':').appendValue(MINUTE_OF_HOUR).optionalStart.appendLiteral(':').appendValue(SECOND_OF_MINUTE) 118 | .toFormatter 119 | 120 | /* 121 | * Hacks EVERYWHERE - MARC unfortunately doesn't expose this data in a nice format for us 122 | */ 123 | def messageSummaryFor(articleHtml: String): MessageSummary = { 124 | val elements = Jsoup.parse(articleHtml).select("""pre b font[size="+1"]""") 125 | val nodes = elements.get(0).childNodes().toList 126 | val headerMap = (for { header :: value :: Nil <- nodes.grouped(2) } yield { 127 | header.outerHtml.trim.stripSuffix(":") -> value.asInstanceOf[Element].html() 128 | }).toMap 129 | val messageId = deobfuscate(headerMap("Message-ID")) 130 | val from = new InternetAddress(deobfuscate(headerMap("From"))) 131 | 132 | val date = LocalDateTime.parse(headerMap("Date"), DateTimeFormat).atZone(ZoneId.of("UTC")) 133 | MessageSummary(messageId, headerMap("Subject"), date, Addresses(from), "") 134 | } 135 | } 136 | 137 | case class Marc(groupName: String) extends MailArchive { 138 | val providerName = "MARC" 139 | 140 | val url = s"http://marc.info/?l=$groupName" 141 | 142 | def linkFor(messageId: String) = s"http://marc.info/?i=$messageId" 143 | 144 | override def canonicalUrlFor(messageId: String)(implicit ec: ExecutionContext) = 145 | RedirectCapturer.redirectFor(linkFor(messageId)) 146 | 147 | override def messageSummaryBasedOn(canonicalMessageUrl: Uri)(implicit ec: ExecutionContext): Future[MessageSummary] = for { 148 | raw <- okClient.execute(new Request.Builder().url(canonicalMessageUrl).build())(_.body.string) 149 | } yield Marc.messageSummaryFor(raw).copy(groupLink = canonicalMessageUrl) 150 | } 151 | 152 | case class GoogleGroup(groupName: String) extends MailArchive with OffersRawMessage { 153 | 154 | val providerName = "Google Groups" 155 | 156 | val url = s"https://groups.google.com/forum/#!forum/$groupName" 157 | 158 | // submitgit-test@googlegroups.com 159 | val emailAddress = new InternetAddress(s"$groupName@googlegroups.com") 160 | 161 | // https://groups.google.com/d/msgid/submitgit-test/0000014d9a92ef17-abf8da02-8ed6-4f1e-b959-db3d4617e750-000000@eu-west-1.amazonses.com 162 | def linkFor(messageId: String) = s"https://groups.google.com/d/msgid/$groupName/$messageId" 163 | 164 | override def canonicalUrlFor(messageId: String)(implicit ec: ExecutionContext) = RedirectCapturer.redirectFor(linkFor(messageId)) 165 | 166 | /* redirect: /forum/#!msg/submitgit-test/-cq4q1w7jyY/fLlC47tosH4J 167 | * raw resource: https://groups.google.com/forum/message/raw?msg=submitgit-test/-cq4q1w7jyY/fLlC47tosH4J 168 | */ 169 | override def rawUrlFor(canonicalMessageUrl: Uri) = 170 | "https://groups.google.com/forum/message/raw?msg="+canonicalMessageUrl.fragment.get.stripPrefix("!msg/") 171 | 172 | val mailingList = MailingList(emailAddress, Seq(this)) 173 | 174 | } 175 | -------------------------------------------------------------------------------- /app/lib/MailType.scala: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import javax.mail.internet.InternetAddress 4 | 5 | import com.github.nscala_time.time.Imports._ 6 | import com.madgag.scalagithub.GitHub 7 | import com.madgag.scalagithub.commands.CreateComment 8 | import com.madgag.scalagithub.model.PullRequest 9 | import controllers.routes 10 | import lib.actions.Requests._ 11 | import lib.checks.{Check, GHChecks, PRChecks} 12 | import lib.model.PatchBomb 13 | import play.api.mvc.RequestHeader 14 | 15 | import scala.concurrent.ExecutionContext.Implicits.global 16 | import scala.concurrent.Future 17 | 18 | sealed trait MailType { 19 | def afterSending(pr: PullRequest, patchBomb: PatchBomb, messageId: String, prMailSettings: PRMailSettings): PRMailSettings 20 | 21 | val slug: String 22 | 23 | val subjectPrefix: Option[String] 24 | 25 | def addressing(mailingList: MailingList, userEmailAddress: InternetAddress): Email.Addresses 26 | 27 | def footer(pr: PullRequest)(implicit request: RequestHeader): String 28 | 29 | val checks: Seq[Check[GHPRRequest[_]]] 30 | } 31 | 32 | object MailType { 33 | 34 | def internetAddressFor(emailAddress: String, displayName: String) = 35 | new InternetAddress(emailAddress, displayName) 36 | 37 | val all = Seq(Preview, Live) 38 | 39 | val bySlug = all.map(mt => mt.slug -> mt).toMap 40 | 41 | def proposedMailFor(mt: MailType)(implicit req: GHPRRequest[_]): Future[ProposedMail] = for { 42 | errors <- Check.all(req, mt.checks) 43 | } yield ProposedMail(mt.addressing(Project.byRepoId(req.repo.repoId).mailingList, req.user.address), errors) 44 | 45 | def proposedMailByTypeFor(implicit req: GHPRRequest[_]): Future[Map[MailType, ProposedMail]] = 46 | Future.traverse(MailType.all)(mt => proposedMailFor(mt).map(mt -> _)).map(_.toMap) 47 | 48 | object Preview extends MailType { 49 | override val slug = "submit-preview" 50 | 51 | def footer(pr: PullRequest)(implicit request: RequestHeader): String = { 52 | val headCommitId = pr.head.sha 53 | val ackUrl = routes.Application.acknowledgePreview(pr.prId, headCommitId, PreviewSignatures.signatureFor(headCommitId)).absoluteURL 54 | s"This is a preview - if you want to send this message for real, click here:\n$ackUrl" 55 | } 56 | 57 | override def addressing(mailingList: MailingList, userEmail: InternetAddress) = Email.Addresses( 58 | from = new InternetAddress("submitGit "), 59 | to = Seq(userEmail) 60 | ) 61 | 62 | override val subjectPrefix = Some("TEST") 63 | 64 | import GHChecks._ 65 | import PRChecks._ 66 | 67 | val checks = Seq( 68 | accountIsOlderThan(1.hour), 69 | EmailVerified, 70 | HasAReasonableNumberOfCommits 71 | ) 72 | 73 | override def afterSending(request: PullRequest, patchBomb: PatchBomb, messageId: String, prMailSettings: PRMailSettings) = { 74 | // Nothing for preview 75 | prMailSettings 76 | } 77 | } 78 | 79 | object Live extends MailType { 80 | override def footer(pr: PullRequest)(implicit request: RequestHeader): String = pr.html_url 81 | 82 | override def addressing(mailingList: MailingList, userEmail: InternetAddress) = Email.Addresses( 83 | from = userEmail, 84 | to = Seq(mailingList.emailAddress), 85 | bcc = Seq(userEmail) 86 | ) 87 | 88 | override val subjectPrefix = None 89 | override val slug = "submit" 90 | 91 | import GHChecks._ 92 | import PRChecks._ 93 | 94 | val AccountAgeThreshold = 1.day 95 | 96 | val generalChecks = Seq( 97 | EmailVerified, 98 | accountIsOlderThan(AccountAgeThreshold), 99 | UserHasNameSetInProfile, 100 | RegisteredEmailWithSES 101 | ) 102 | 103 | val checks = generalChecks ++ Seq( 104 | UserOwnsPR, 105 | PRIsOpen, 106 | HasBeenPreviewed, 107 | HasAReasonableNumberOfCommits 108 | ) 109 | 110 | override def afterSending(pr: PullRequest, patchBomb: PatchBomb, messageId: String, prMailSettings: PRMailSettings) = { 111 | val mailingListLinks = Project.byRepoId(pr.baseRepo.repoId).mailingList.archives.map(a => s"[${a.providerName}](${a.linkFor(messageId)})").mkString(", ") 112 | 113 | val numCommits = patchBomb.patchCommits.size 114 | val patchBombDesc = if (numCommits == 1) { 115 | s"this commit (${pr.compareUrl}) as a patch" 116 | } else { 117 | s"these $numCommits commits (${pr.compareUrl}) as a set of patches" 118 | } 119 | 120 | implicit val github = Bot.conn() 121 | 122 | pr.comments2.create(CreateComment( 123 | s"${pr.user.atLogin} sent $patchBombDesc to the mailing list with [_submitGit_](https://github.com/rtyley/submitgit) - " + 124 | s"here on $mailingListLinks []($messageId)" 125 | )) 126 | prMailSettings.afterBeingUsedToSend(messageId) 127 | } 128 | } 129 | } 130 | 131 | 132 | 133 | 134 | 135 | 136 | -------------------------------------------------------------------------------- /app/lib/MarkdownParsing.scala: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | object MarkdownParsing { 4 | 5 | val hiddenLinkRegex = """\[\]\((.+?)\)""".r 6 | 7 | def parseHiddenLinksOutOf(markdown: String): Seq[String] = 8 | (for (m <- hiddenLinkRegex.findAllMatchIn(markdown)) yield m.group(1)).toSeq 9 | } 10 | -------------------------------------------------------------------------------- /app/lib/PRMailSettings.scala: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import lib.model.SubjectPrefixParsing 4 | import play.api.libs.json.Json 5 | 6 | case class PRMailSettings(subjectPrefix: String, inReplyTo: Option[String] = None) { 7 | def afterBeingUsedToSend(messageId: String) = copy( 8 | subjectPrefix = 9 | SubjectPrefixParsing.patchPrefixContent.parse(subjectPrefix).get.value.suggestsNext.toString, 10 | inReplyTo = Some(messageId) 11 | ) 12 | } 13 | 14 | object PRMailSettings { 15 | implicit val formats = Json.format[PRMailSettings] 16 | } 17 | 18 | -------------------------------------------------------------------------------- /app/lib/PreviewSignatures.scala: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import com.madgag.scalagithub.model.PullRequest 4 | import org.eclipse.jgit.lib.ObjectId 5 | import play.api.libs.Crypto 6 | import play.api.mvc.RequestHeader 7 | 8 | 9 | object PreviewSignatures { 10 | def keyFor(headCommit: ObjectId) = s"previewed_${headCommit.name}" 11 | 12 | def signatureFor(headCommit: ObjectId) = Crypto.sign(keyFor(headCommit)) 13 | 14 | def hasPreviewed(pr: PullRequest)(implicit req: RequestHeader): Boolean = { 15 | val headCommit = pr.head.sha 16 | req.session.get(keyFor(headCommit)).exists(sig => Crypto.constantTimeEquals(sig, signatureFor(headCommit))) 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /app/lib/ProjectRepo.scala: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import javax.mail.internet.InternetAddress 4 | 5 | import com.madgag.scalagithub.model.RepoId 6 | import lib.model.MessageSummary 7 | 8 | import scala.concurrent.{ExecutionContext, Future} 9 | 10 | case class MailingList(emailAddress: InternetAddress, archives: Seq[MailArchive]) { 11 | def lookupMessage(query: String)(implicit ec: ExecutionContext): Future[Seq[MessageSummary]] = 12 | Future.find(archives.map(_.lookupMessage(query)))(_.nonEmpty).map(_.toSeq.flatten) 13 | } 14 | 15 | object Project { 16 | val Git = Project( 17 | RepoId("git", "git"), 18 | MailingList(new InternetAddress("git@vger.kernel.org"), Seq( 19 | PublicInbox.Git, 20 | Marc.Git 21 | )) 22 | ) 23 | val PretendGit = Project( 24 | RepoId("submitgit", "pretend-git"), 25 | GoogleGroup("submitgit-test").mailingList 26 | ) 27 | 28 | val all = Set(Git, PretendGit) 29 | 30 | val byRepoId = all.map(p => p.repoId -> p).toMap 31 | } 32 | 33 | case class Project(repoId: RepoId, mailingList: MailingList) 34 | -------------------------------------------------------------------------------- /app/lib/ProposedMail.scala: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | case class ProposedMail(addresses: Email.Addresses, errors: Seq[String]) 4 | -------------------------------------------------------------------------------- /app/lib/actions/Actions.scala: -------------------------------------------------------------------------------- 1 | package lib.actions 2 | 3 | import com.madgag.playgithub.auth.AuthenticatedSessions.AccessToken 4 | import com.madgag.playgithub.auth.{Client, GHRequest} 5 | import com.madgag.scalagithub.model.{PullRequestId, RepoId} 6 | import controllers.Application._ 7 | import controllers.Auth 8 | import lib._ 9 | import lib.actions.Requests._ 10 | import lib.checks.Check 11 | import org.eclipse.jgit.lib.ObjectId 12 | import play.api.libs.Crypto 13 | import play.api.mvc._ 14 | 15 | import scala.concurrent.ExecutionContext.Implicits.global 16 | import scala.concurrent.Future 17 | import scalax.file.ImplicitConversions._ 18 | 19 | 20 | object Actions { 21 | private val authScopes = Seq("user:email") 22 | 23 | implicit val authClient: Client = Auth.authClient 24 | 25 | implicit val provider = AccessToken.FromSession 26 | 27 | val GitHubAuthenticatedAction = com.madgag.playgithub.auth.Actions.gitHubAction(authScopes, Bot.workingDir.toPath) 28 | 29 | def githubRepoAction(repoId: RepoId) = GitHubAuthenticatedAction andThen new ActionRefiner[GHRequest, GHRepoRequest] { 30 | override protected def refine[A](request: GHRequest[A]): Future[Either[Result, GHRepoRequest[A]]] = 31 | if (Project.byRepoId.contains(repoId)) for (repo <- Bot.conn().getRepo(repoId)) yield { 32 | Right(new GHRepoRequest(request.gitHubCredentials, repo, request)) 33 | } else Future.successful(Left(Forbidden("Not a supported repo"))) 34 | } 35 | 36 | def githubPRAction(prId: PullRequestId) = githubRepoAction(prId.repo) andThen new ActionRefiner[GHRepoRequest, GHPRRequest] { 37 | override protected def refine[A](request: GHRepoRequest[A]): Future[Either[Result, GHPRRequest[A]]] = { 38 | implicit val g = request.gitHub 39 | val prF = request.repo.pullRequests.get(prId.num) 40 | for { 41 | appUser <- AppUser.from(request) 42 | pr <- prF 43 | } yield Right(new GHPRRequest[A](request.gitHubCredentials, appUser, pr, request)) 44 | }.recover { 45 | case _ => Left(NotFound(s"${request.repo.full_name} doesn't seem to have PR #${prId.num}")) 46 | } 47 | } 48 | 49 | val EnsureGitHubVerifiedEmail = new ActionFilter[GHRequest] { 50 | override protected def filter[A](request: GHRequest[A]): Future[Option[Result]] = for { 51 | email <- request.userPrimaryEmailF 52 | } yield { 53 | if (email.verified) None 54 | else Some(Forbidden(s"User primary email ${email.email} has not been verified with GitHub")) 55 | } 56 | } 57 | 58 | def verifyCommitSignature[G[X] <: Request[X]](headCommit: ObjectId, signature: Option[String] = None) = new ActionFilter[G] { 59 | override protected def filter[A](request: G[A]): Future[Option[Result]] = Future { 60 | val sig = signature.getOrElse(request.session(PreviewSignatures.keyFor(headCommit))) 61 | if (Crypto.constantTimeEquals(sig, PreviewSignatures.signatureFor(headCommit))) None else 62 | Some(Unauthorized("No valid Preview signature")) 63 | } 64 | } 65 | 66 | def mailChecks(mailType: MailType) = new ActionFilter[GHPRRequest] { 67 | override protected def filter[A](req: GHPRRequest[A]): Future[Option[Result]] = for { 68 | errors <- Check.all(req, mailType.checks) 69 | } yield if (errors.isEmpty) None else Some(Forbidden(errors.mkString("\n"))) 70 | } 71 | 72 | } -------------------------------------------------------------------------------- /app/lib/actions/requests.scala: -------------------------------------------------------------------------------- 1 | package lib.actions 2 | 3 | import com.madgag.okhttpscala._ 4 | import com.madgag.playgithub.auth.GHRequest 5 | import com.madgag.scalagithub.GitHub._ 6 | import com.madgag.scalagithub.GitHubCredentials 7 | import com.madgag.scalagithub.model.{PullRequest, Repo} 8 | import lib.MailType.{Live, Preview} 9 | import lib._ 10 | import lib.model.PatchCommit 11 | import okhttp3.OkHttpClient 12 | import play.api.mvc.Request 13 | 14 | import scala.concurrent.ExecutionContext.Implicits.global 15 | import scala.concurrent.Future 16 | 17 | object Requests { 18 | 19 | val okHttpClient = new OkHttpClient() 20 | 21 | class GHRepoRequest[A](gitHubCredentials: GitHubCredentials, val repo: Repo, request: Request[A]) extends GHRequest[A](gitHubCredentials, request) 22 | 23 | class GHPRRequest[A](gitHubCredentials: GitHubCredentials, val user: AppUser, val pr: PullRequest, request: Request[A]) extends GHRepoRequest[A](gitHubCredentials, pr.baseRepo, request) { 24 | implicit val g = gitHub 25 | 26 | lazy val userOwnsPR = user.id == pr.user.id 27 | 28 | lazy val patchCommitsF: Future[Seq[PatchCommit]] = { 29 | for { 30 | ghCommits <- pr.commits.list().all() 31 | patch <- okHttpClient.execute(new okhttp3.Request.Builder().url(pr.patch_url).build())(_.body().string) 32 | } yield { 33 | PatchCommit.from(ghCommits, patch) 34 | } 35 | } 36 | 37 | lazy val hasBeenPreviewed = PreviewSignatures.hasPreviewed(pr)(request) 38 | 39 | lazy val defaultMailType = if (hasBeenPreviewed) Live else Preview 40 | } 41 | 42 | } -------------------------------------------------------------------------------- /app/lib/aws/SES.scala: -------------------------------------------------------------------------------- 1 | package lib.aws 2 | 3 | import com.amazonaws.auth.profile.ProfileCredentialsProvider 4 | import com.amazonaws.auth.{AWSCredentialsProviderChain, EnvironmentVariableCredentialsProvider} 5 | import com.amazonaws.regions.Regions._ 6 | import com.amazonaws.services.simpleemail.model.IdentityVerificationAttributes 7 | import com.amazonaws.services.simpleemail.{AmazonSimpleEmailServiceAsync, AmazonSimpleEmailServiceAsyncClient} 8 | 9 | object SES { 10 | 11 | sealed abstract class VerificationStatus(val string: String) 12 | 13 | object VerificationStatus { 14 | object Pending extends VerificationStatus("Pending") 15 | object Success extends VerificationStatus("Success") 16 | object Failed extends VerificationStatus("Failed") 17 | object TemporaryFailure extends VerificationStatus("TemporaryFailure") 18 | object NotStarted extends VerificationStatus("NotStarted") 19 | 20 | val all = Set(Pending, Success, Failed, TemporaryFailure, NotStarted) 21 | 22 | def from(string: String): Option[VerificationStatus] = all.find(_.string == string) 23 | } 24 | 25 | implicit class RichIdentityVerificationAttributes(attrs: IdentityVerificationAttributes) { 26 | lazy val status: Option[VerificationStatus] = VerificationStatus.from(attrs.getVerificationStatus) 27 | } 28 | 29 | val AwsCredentials = new AWSCredentialsProviderChain( 30 | new ProfileCredentialsProvider("submitgit"), 31 | new EnvironmentVariableCredentialsProvider() 32 | ) 33 | 34 | val ses: AmazonSimpleEmailServiceAsync = AmazonSimpleEmailServiceAsyncClient.asyncBuilder 35 | .withCredentials(AwsCredentials) 36 | .withRegion(EU_WEST_1) 37 | .build() 38 | 39 | } 40 | -------------------------------------------------------------------------------- /app/lib/aws/SesAsyncHelpers.scala: -------------------------------------------------------------------------------- 1 | package lib.aws 2 | 3 | import com.amazonaws.AmazonWebServiceRequest 4 | import com.amazonaws.handlers.AsyncHandler 5 | import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceAsync 6 | import com.amazonaws.services.simpleemail.model._ 7 | import lib.Email 8 | 9 | import scala.collection.convert.wrapAll._ 10 | import scala.concurrent.{ExecutionContext, Future, Promise} 11 | 12 | object SesAsyncHelpers { 13 | 14 | class AsyncHandlerToFuture[REQUEST <: AmazonWebServiceRequest, RESULT] extends AsyncHandler[REQUEST, RESULT] { 15 | val promise = Promise[RESULT]() 16 | 17 | def future = promise.future 18 | 19 | override def onError(exception: Exception): Unit = promise.failure(exception) 20 | 21 | override def onSuccess(request: REQUEST, result: RESULT): Unit = promise.success(result) 22 | } 23 | 24 | implicit class SesAsync(ses: AmazonSimpleEmailServiceAsync) { 25 | import SES._ 26 | 27 | private def invoke[REQUEST <: AmazonWebServiceRequest, RESULT] 28 | ( 29 | method: (REQUEST, AsyncHandler[REQUEST, RESULT]) => java.util.concurrent.Future[RESULT], 30 | req: REQUEST 31 | ): Future[RESULT] = { 32 | val handler = new AsyncHandlerToFuture[REQUEST, RESULT] 33 | method(req, handler) 34 | handler.future 35 | } 36 | 37 | // ignore the red in IntelliJ here, the scala compiler understands this :) 38 | def verifyEmailIdentityFuture(req: VerifyEmailIdentityRequest): Future[VerifyEmailIdentityResult] = 39 | invoke(ses.verifyEmailIdentityAsync, req) 40 | 41 | def sendVerificationEmailTo(email: String)(implicit ec: ExecutionContext) = 42 | verifyEmailIdentityFuture(new VerifyEmailIdentityRequest().withEmailAddress(email)) 43 | 44 | def getIdentityVerificationAttributesFuture(req: GetIdentityVerificationAttributesRequest): Future[GetIdentityVerificationAttributesResult] = 45 | invoke(ses.getIdentityVerificationAttributesAsync, req) 46 | 47 | def getIdentityVerificationStatusFor(email: String)(implicit ec: ExecutionContext): Future[Option[VerificationStatus]] = { 48 | val idReq = new GetIdentityVerificationAttributesRequest().withIdentities(email) 49 | for (res <- ses.getIdentityVerificationAttributesFuture(idReq)) yield 50 | res.getVerificationAttributes.toMap.get(email).flatMap(_.status) 51 | } 52 | 53 | def sendRawEmailFuture(req: SendRawEmailRequest): Future[SendRawEmailResult] = 54 | invoke(ses.sendRawEmailAsync, req) 55 | 56 | def send(email: Email)(implicit ec: ExecutionContext): Future[String] = { 57 | val rawEmailRequest = new SendRawEmailRequest(new RawMessage(email.toMimeMessage)) 58 | rawEmailRequest.setDestinations(email.addresses.to.map(_.toString)) 59 | rawEmailRequest.setSource(email.addresses.from.toString) 60 | sendRawEmailFuture(rawEmailRequest).map(resp => s"${resp.getMessageId}@eu-west-1.amazonses.com") 61 | } 62 | } 63 | 64 | } -------------------------------------------------------------------------------- /app/lib/checks/Check.scala: -------------------------------------------------------------------------------- 1 | package lib.checks 2 | 3 | import scala.concurrent.{ExecutionContext, Future} 4 | 5 | object Check { 6 | def all[T](t: T, checks: Seq[Check[T]])(implicit ec: ExecutionContext): Future[Seq[String]] = 7 | Future.traverse(checks)(_.check(t)).map(_.flatten.toList) 8 | } 9 | 10 | trait Check[-T] { 11 | def check(req: T): Future[Option[String]] 12 | } 13 | -------------------------------------------------------------------------------- /app/lib/checks/Checks.scala: -------------------------------------------------------------------------------- 1 | package lib.checks 2 | 3 | import scala.concurrent.{ExecutionContext, Future} 4 | 5 | trait Checks[T] { 6 | class CheckBuilder(p: T => Future[Boolean])(implicit ec: ExecutionContext) { 7 | final def or(error: String): Check[T] = or(_ => error) 8 | 9 | final def or(error: T => String)(implicit ec: ExecutionContext): Check[T] = new Check[T] { 10 | override def check(v: T): Future[Option[String]] = p(v).map(if (_) None else Some(error(v))) 11 | } 12 | } 13 | 14 | final def check(p: T => Boolean)(implicit ec: ExecutionContext) = new CheckBuilder(p.andThen(Future.successful)) 15 | 16 | final def checkAsync(p: T => Future[Boolean])(implicit ec: ExecutionContext) = new CheckBuilder(p) 17 | } 18 | -------------------------------------------------------------------------------- /app/lib/checks/PRChecks.scala: -------------------------------------------------------------------------------- 1 | package lib.checks 2 | 3 | import com.github.nscala_time.time.Imports._ 4 | import com.madgag.time.Implicits._ 5 | import lib.Dates._ 6 | import lib.actions.Requests._ 7 | import lib.aws.SES._ 8 | import lib.aws.SesAsyncHelpers._ 9 | import org.joda.time.Period 10 | 11 | import scala.concurrent.ExecutionContext.Implicits.global 12 | 13 | object GHChecks extends Checks[GHPRRequest[_]] { 14 | 15 | val EmailVerified = check(_.user.primaryEmail.verified) or 16 | (req => s"Verify your email address (${req.user.primaryEmail.email}) with GitHub") 17 | 18 | val UserHasNameSetInProfile = check(_.user.name.exists(_.length > 1)) or 19 | "Set a Name in your GitHub profile - it'll be used as a display name next to your email" 20 | 21 | def accountIsOlderThan(period: Period) = check(_.user.created_at.exists(_.isBefore(DateTime.now - period))) or 22 | s"To prevent spam, we don't currently allow GitHub accounts less than ${period.pretty} old - get in touch if that's a problem for you!" 23 | 24 | val RegisteredEmailWithSES = checkAsync(req => ses.getIdentityVerificationStatusFor(req.user.primaryEmail.email).map(_.contains(VerificationStatus.Success))) or 25 | (req => s"Register your email address (${req.user.primaryEmail.email}) with submitGit's Amazon SES account in order for it to send emails from you.") 26 | 27 | } 28 | 29 | object PRChecks extends Checks[GHPRRequest[_]] { 30 | 31 | val MaxCommits = 30 32 | 33 | val UserOwnsPR = check(_.userOwnsPR) or (req => s"This PR was raised by ${req.pr.user.atLogin} - you can't submit it for them") // fatal 34 | 35 | val PRIsOpen = check(_.pr.state == "open") or "Can't submit a closed pull request - reopen it if you're sure" 36 | 37 | val HasBeenPreviewed = check(_.hasBeenPreviewed) or (req => s"You need to preview these commits in a test email to yourself before they can be sent for real - click the link at the bottom of the preview email!") 38 | 39 | val HasAReasonableNumberOfCommits = checkAsync(_.patchCommitsF.map(_.length <= MaxCommits)) or (req => s"For now, submitGit won't submit PRs with more than $MaxCommits commits") 40 | 41 | val CommitSubjectsAreNotTooLong = checkAsync(_.patchCommitsF.map(_.exists(_.commit.subject.size > 72))) or (req => s"Your commit subject lines are too long...") 42 | 43 | } 44 | -------------------------------------------------------------------------------- /app/lib/model/MessageSummary.scala: -------------------------------------------------------------------------------- 1 | package lib.model 2 | 3 | import java.time.{ZoneId, ZonedDateTime} 4 | import javax.mail.internet.{InternetAddress, MailDateFormat} 5 | 6 | import fastparse.core.Parsed 7 | import lib.Email.Addresses 8 | import play.api.libs.json._ 9 | import play.api.mvc.Headers 10 | 11 | object MessageSummary { 12 | 13 | implicit val writesInternetAddress = new Writes[InternetAddress] { 14 | override def writes(ia: InternetAddress) = JsString(ia.toUnicodeString) 15 | } 16 | implicit val writesAddresses = Json.writes[Addresses] 17 | implicit val writesMessageSummary = new Writes[MessageSummary] { 18 | override def writes(o: MessageSummary): JsValue = { 19 | val suggestion = for { 20 | prefix <- o.suggestedPrefixForNextPatchBombOpt 21 | } yield "suggestsPrefix" -> JsString(prefix) 22 | 23 | Json.writes[MessageSummary].writes(o).asInstanceOf[JsObject] ++ JsObject(suggestion.toSeq) 24 | } 25 | } 26 | 27 | def fromRawMessage(rawMessage: String, articleUrl: String): MessageSummary = { 28 | val Parsed.Success(headerTuples, _) = PatchParsing.messageHeaders.parse(rawMessage) 29 | val headers = Headers(headerTuples: _*) 30 | val messageId = headers("Message-Id").stripPrefix("<").stripSuffix(">") 31 | val from = new InternetAddress(headers("From")) 32 | val date = new MailDateFormat().parse(headers("Date")).toInstant.atZone(ZoneId.of("UTC")) 33 | MessageSummary(messageId, headers("Subject"), date, Addresses(from), articleUrl) 34 | } 35 | } 36 | 37 | 38 | case class MessageSummary( 39 | id: String, 40 | subject: String, 41 | date: ZonedDateTime, 42 | addresses: Addresses, 43 | groupLink: String 44 | ) { 45 | lazy val suggestedPrefixForNextPatchBombOpt: Option[String] = 46 | SubjectPrefixParsing.parse(subject).map(_.suggestsNext.toString) 47 | } 48 | -------------------------------------------------------------------------------- /app/lib/model/PRMessageIdFinder.scala: -------------------------------------------------------------------------------- 1 | package lib.model 2 | 3 | import java.time.ZonedDateTime 4 | 5 | import com.madgag.scalagithub.GitHub 6 | import com.madgag.scalagithub.GitHub._ 7 | import com.madgag.scalagithub.model._ 8 | import lib.MarkdownParsing 9 | 10 | import scala.concurrent.ExecutionContext.Implicits.global 11 | import scala.concurrent.Future 12 | 13 | object PRMessageIdFinder { 14 | 15 | def messageIdsByMostRecentUsageIn(pr: PullRequest)(implicit g: GitHub): Future[Seq[String]] = { 16 | for { 17 | comments <- pr.comments2.list().all() 18 | } yield { 19 | val messageIdAndCreationTime = for { 20 | comment <- comments 21 | messageId <- MarkdownParsing.parseHiddenLinksOutOf(comment.body) 22 | } yield { 23 | messageId -> comment.created_at 24 | } 25 | 26 | implicit def dateTimeOrdering: Ordering[ZonedDateTime] = Ordering.fromLessThan(_ isBefore _) 27 | 28 | val messageIdsWithMostRecentUsageTime: Map[String, ZonedDateTime] = 29 | messageIdAndCreationTime.groupBy(_._1).mapValues(_.map(_._2).max) 30 | 31 | messageIdsWithMostRecentUsageTime.toSeq.sortBy(_._2).map(_._1).reverse 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /app/lib/model/Patch.scala: -------------------------------------------------------------------------------- 1 | package lib.model 2 | 3 | import fastparse.all._ 4 | import org.eclipse.jgit.lib.ObjectId 5 | 6 | case class Patch(commitId: ObjectId, body: String) 7 | 8 | object PatchParsing { 9 | 10 | val line = P(CharsWhile(_ != '\n', min = 0) ~ "\n") 11 | 12 | val nonEmptyLine = P(CharsWhile(_ != '\n', min = 1) ~ "\n") 13 | 14 | val hexDigit = P( CharIn('0'to'9', 'a'to'f', 'A'to'F') ) 15 | 16 | val objectId: P[ObjectId] = P(hexDigit.rep(40).!).map(ObjectId.fromString) 17 | 18 | val patchFromHeader = P("From " ~ objectId ~ " Mon Sep 17 00:00:00 2001\n") 19 | 20 | val patchHeaderRegion: P[ObjectId] = P(patchFromHeader ~/ nonEmptyLine.rep) 21 | 22 | val headerKey = P(CharsWhile(_ != ':', min = 1).! ~ ": ") 23 | 24 | val newlineChars = Set('\n', '\r') 25 | 26 | val newline = "\r\n" | "\n" 27 | 28 | val whitespace = P(CharIn(" \t")) 29 | 30 | val nonNewlineChars = CharsWhile(!newlineChars(_), min = 1) 31 | 32 | val headerValue = P((nonNewlineChars ~ (newline ~ whitespace.rep(min = 1) ~ nonNewlineChars).rep).!) 33 | 34 | val header: P[(String, String)] = P(headerKey ~ headerValue) 35 | 36 | val headers: P[Seq[(String, String)]] = P(header.rep(sep = newline)) 37 | 38 | val nonHeaderLines = (!header ~ line).rep(sep = newline) 39 | 40 | /* For some reason the 'raw' messages at public-inbox.org have an initial line which isn't a standard 41 | * message header - eg: "From mboxrd@z Thu Jan 1 00:00:00 1970" does not have a colon (':') 42 | * 43 | * https://public-inbox.org/git/6a72bfd4-5032-5e40-5c6d-8b77ca5ae775@gmail.com/raw 44 | */ 45 | val messageHeaders: P[Seq[(String, String)]] = nonHeaderLines ~ headers 46 | 47 | val patchBodyRegion = P((line ~ !patchFromHeader).rep.!) 48 | 49 | val patch: P[Patch] = P(patchHeaderRegion ~ "\n" ~/ patchBodyRegion).map(Patch.tupled) 50 | 51 | val patches: P[Seq[Patch]] = P(patch.rep(sep = "\n")) 52 | 53 | } -------------------------------------------------------------------------------- /app/lib/model/PatchBomb.scala: -------------------------------------------------------------------------------- 1 | package lib.model 2 | 3 | import java.text.NumberFormat 4 | 5 | import lib.Email 6 | import lib.Email.Addresses 7 | import lib.model.PatchBomb.{countTextFor, fromBodyPrefixOpt} 8 | 9 | case class PatchBomb( 10 | patchCommits: Seq[PatchCommit], 11 | addresses: Addresses, 12 | shortDescription: String = "PATCH", 13 | additionalPrefix: Option[String] = None, 14 | footer: String 15 | ) { 16 | 17 | lazy val emails: Seq[Email] = { 18 | for ((patchCommit, index) <- patchCommits.zipWithIndex) yield { 19 | val patchDescriptionAndIndex = (Seq(shortDescription) ++ countTextFor(index + 1, patchCommits.size)).mkString(" ") 20 | val prefixes = (additionalPrefix.toSeq :+ patchDescriptionAndIndex).map("["+_+"]") 21 | 22 | Email( 23 | addresses, 24 | subject = (prefixes :+ patchCommit.commit.subject).mkString(" "), 25 | bodyText = 26 | (fromBodyPrefixOpt(patchCommit, addresses).toSeq :+ s"${patchCommit.patch.body}\n--\n$footer").mkString("\n") 27 | ) 28 | } 29 | } 30 | } 31 | 32 | object PatchBomb { 33 | 34 | def fromBodyPrefixOpt(patchCommit: PatchCommit, addresses: Addresses): Option[String] = { 35 | val author = patchCommit.commit.author 36 | 37 | val shouldAddFromBodyPrefix = 38 | author.email == addresses.from.getAddress || author.email.contains("noreply") 39 | 40 | if (shouldAddFromBodyPrefix) None else Some(s"From: ${author.addressString}\n") 41 | } 42 | 43 | def numberFormatFor(size: Int) = { 44 | val nf = NumberFormat.getIntegerInstance 45 | nf.setMinimumIntegerDigits(size.toString.length) 46 | nf.setGroupingUsed(false) 47 | nf 48 | } 49 | 50 | def countTextFor(index: Int, size: Int): Option[String] = { 51 | if (size <= 1) None else { 52 | val formattedIndex = numberFormatFor(size).format(index) 53 | Some(f"$formattedIndex/$size") 54 | } 55 | } 56 | } -------------------------------------------------------------------------------- /app/lib/model/PatchCommit.scala: -------------------------------------------------------------------------------- 1 | package lib.model 2 | 3 | import com.madgag.scalagithub.model.PullRequest.CommitOverview 4 | import fastparse.all._ 5 | 6 | case class PatchCommit(patch: Patch, commit: CommitOverview) 7 | 8 | object PatchCommit { 9 | def from(commits: Seq[CommitOverview], githubPatch: String): Seq[PatchCommit] = { 10 | 11 | val Parsed.Success(patches, s) = PatchParsing.patches.parse(githubPatch) 12 | 13 | val commitsById = commits.map(c => c.sha -> c).toMap 14 | 15 | patches.map(patch => PatchCommit(patch, commitsById(patch.commitId))) 16 | } 17 | } -------------------------------------------------------------------------------- /app/lib/model/SubjectPrefix.scala: -------------------------------------------------------------------------------- 1 | package lib.model 2 | 3 | import fastparse.all._ 4 | 5 | case class SubjectPrefix(descriptor: String, version: Option[Int] = None) { 6 | lazy val suggestsNext = copy(version = Some(version.fold(2)(_ + 1))) 7 | 8 | override lazy val toString = (descriptor +: version.map("v"+_).toSeq).mkString(" ") 9 | } 10 | 11 | object SubjectPrefixParsing { 12 | val number: P[Int] = P( CharIn('0'to'9').rep(1).!.map(_.toInt) ) 13 | 14 | val version: P[Int] = P( CharIn("vV") ~ number ) 15 | 16 | val patchIndexAndBombSize = P(number ~"/" ~ number) 17 | 18 | val word: P[String] = P(CharsWhile(!Set('\n',' ', ']').contains(_)).!) 19 | 20 | val nonDescriptorTerms = version | patchIndexAndBombSize 21 | 22 | val descriptorWord = word.filter(!nonDescriptorTerms.parse(_).isInstanceOf[Parsed.Success[_]]) 23 | 24 | val descriptor: P[String] = P( descriptorWord.rep(1, sep = " ").! ) 25 | 26 | val patchPrefixContent: P[SubjectPrefix] = 27 | P( descriptor ~ (" " ~ version).? ~ (" " ~ patchIndexAndBombSize).map(_ => ()).? ).map(SubjectPrefix.tupled) 28 | 29 | val subjectLine: P[SubjectPrefix] = 30 | P( "[" ~ patchPrefixContent ~ "]") 31 | 32 | def parse(subjectLineText: String): Option[SubjectPrefix] = { 33 | subjectLine.parse(subjectLineText) match { 34 | case Parsed.Success(subjectPrefix, _) => Some(subjectPrefix) 35 | case _ => None 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /app/views/fragments/emailAddresses.scala.html: -------------------------------------------------------------------------------- 1 | @(addresses: lib.Email.Addresses)(implicit fc : views.html.b3.B3FieldConstructor) 2 | @import javax.mail.internet.InternetAddress 3 | 4 | @display(label: String, ids: Seq[InternetAddress]) = { 5 | @if(ids.nonEmpty) { 6 | @b3.static(label) { @ids.mkString(", ")} 7 | 8 | } 9 | } 10 | 11 | @display("From", Seq(addresses.from)) 12 | @display("To", addresses.to) 13 | @display("Cc", addresses.cc) 14 | @display("Bcc", addresses.bcc) 15 | @display("Reply-To", addresses.replyTo.toSeq) 16 | -------------------------------------------------------------------------------- /app/views/fragments/footer.scala.html: -------------------------------------------------------------------------------- 1 | 13 | -------------------------------------------------------------------------------- /app/views/fragments/head.scala.html: -------------------------------------------------------------------------------- 1 | 2 | submitGit 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | @for(googleAnalyticsId <- lib.AnalyticsConfig.googleAnalyticsIdOpt) { 18 | 26 | } 27 | -------------------------------------------------------------------------------- /app/views/fragments/linkedBreadcrumb.scala.html: -------------------------------------------------------------------------------- 1 | @(link: Call, octicon: String)(linkText: Html)(implicit req: RequestHeader) 2 |
  • 3 | 4 | @linkText 5 |
    • -------------------------------------------------------------------------------- /app/views/fragments/listPullRequests.scala.html: -------------------------------------------------------------------------------- 1 | @(prs: Seq[PullRequest]) 2 | 3 |
    4 | @for(pullRequest <- prs) { 5 | 6 |

    @pullRequest.title

    7 |

    #@pullRequest.number opened on @pullRequest.created_at

    8 |     9 | } 10 |
    11 | -------------------------------------------------------------------------------- /app/views/fragments/mailIdent.scala.html: -------------------------------------------------------------------------------- 1 | @(messageSummary: lib.model.MessageSummary) 2 | 3 |
    @messageSummary.subject
    5 | @messageSummary.addresses.from - 6 | 7 |     8 | -------------------------------------------------------------------------------- /app/views/fragments/patchBaseGuidelines.scala.html: -------------------------------------------------------------------------------- 1 | @(baseBranch: String) 2 | @import lib.Links 3 | @link(html: Html) = {@html} 4 | 5 | This PR is based off @baseBranch, @baseBranch match { 6 | case "master" => { so @link{in general it should be a new feature, not a bugfix}. } 7 | case "maint" => { so @link{make sure it's a bugfix, not a new feature}. } 8 | case _ => { is that right?! @link{Normally bugfixes go on maint, and new features go on master}. } 9 | } 10 | -------------------------------------------------------------------------------- /app/views/fragments/sendTab.scala.html: -------------------------------------------------------------------------------- 1 | @(mailType: lib.MailType, isDefault: Boolean)(content: Html) 2 | 3 | 5 | -------------------------------------------------------------------------------- /app/views/fragments/sendTabContent.scala.html: -------------------------------------------------------------------------------- 1 | @import lib.Email.Addresses 2 | @import lib.model.MessageSummary 3 | @import play.api.libs.json.JsString 4 | @( 5 | mailType: lib.MailType, 6 | proposedMail: lib.ProposedMail, 7 | messageIds: Seq[String], 8 | isDefault: Boolean, btnClass: String 9 | )(implicit req: GHPRRequest[_], prMailSettings: Form[lib.PRMailSettings], messages: Messages) 10 | @import helper._ 11 | @implicitFieldConstructor = @{ b3.vertical.fieldConstructor } 12 | 13 |
    14 | 15 | @if(proposedMail.errors.nonEmpty) { 16 | 23 | } 24 | 25 |
    26 | @fragments.emailAddresses(proposedMail.addresses)(b3.horizontal.fieldConstructor("col-md-2", "col-md-10")) 27 |
    28 | 29 | @b3.form(routes.Application.mailPullRequest(req.pr.prId, mailType)) { 30 | @CSRF.formField 31 | 32 | @b3.inputWrapped("text", prMailSettings("subjectPrefix"), '_label -> "Subject prefix", 'placeholder -> "PATCH", 'maxlength -> "20") { input => 33 |
    34 | [ 35 | @input 36 | ] 37 |
    38 | } 39 | 40 | @b3.inputWrapped("text", prMailSettings("inReplyTo"), '_label -> "In-reply-to", 'placeholder -> "message-id (optional)", 'class -> "form-control typeahead") { input => 41 |
    42 |
    43 | < 44 | @input 45 | > 46 |
    47 | 51 |
    52 | } 53 | 54 | 57 | 93 | 94 | @b3.submit('class -> s"btn $btnClass btn-lg", 'disabled -> proposedMail.errors.nonEmpty) { 95 | Send 96 | } 97 | } 98 |
    -------------------------------------------------------------------------------- /app/views/fragments/sendTabs.scala.html: -------------------------------------------------------------------------------- 1 | @( 2 | defaultMailTab: lib.MailType, 3 | proposedMailByType: Map[lib.MailType,lib.ProposedMail], 4 | messageIds: Seq[String] 5 | )(implicit req: GHPRRequest[_], prMailSettings: Form[lib.PRMailSettings], messages: Messages) 6 | @import lib.MailType 7 | @import MailType.{Live, Preview} 8 | 9 |

    Send this Pull Request to...

    10 |
    11 | 15 |
    16 | @fragments.sendTabContent(Preview, proposedMailByType(Preview),messageIds, Preview == defaultMailTab,"btn-primary") 17 | @fragments.sendTabContent(Live, proposedMailByType(Live),messageIds, Live == defaultMailTab,"btn-danger") 18 |
    19 |
    20 | 21 | -------------------------------------------------------------------------------- /app/views/fragments/user.scala.html: -------------------------------------------------------------------------------- 1 | @(user: User) 2 | @user.atLogin -------------------------------------------------------------------------------- /app/views/index.scala.html: -------------------------------------------------------------------------------- 1 | @()(implicit req: Request[AnyContent]) 2 | @import com.madgag.scalagithub.model.RepoId 3 | @import lib.Dates._ 4 | @main { 5 |
    6 |

    Contribute to Git!

    7 |

    Patches for the mailing list... pull requests for you

    8 |

    Get started 9 | ...or experiment on our test repo!

    10 |
    11 | 12 |
    13 |
    14 |

    What is submitGit?

    15 |

    The Git project has a patch submission process 16 | that's unfamilar for the majority of Git users: hitting the mailing list with 17 | git-format-patch 18 | and git-send-email.

    19 |

    submitGit is a step in trying to make the process easier. If you create a pull request on 20 | github.com/git/git, 21 | submitGit can close it for you by correctly formatting it as 22 | a series of patches, and sending it to the mailing list.

    23 | 24 |

    The patch review stays on the list - but at least you no longer need to learn a whole new method of email 25 | to submit your improvement to Git.

    26 |
    27 |
    28 |

    Prerequisites

    29 |

    submitGit is set up to allow you to send test messages to yourself - all that requires is that 30 | your email is verified on GitHub - but when you want to mail the Git mailing list for real, there are 31 | a few extra requirements:

    32 | 33 |

      34 |
    • Your GitHub account must be older than @lib.MailType.Live.AccountAgeThreshold.pretty
      • 35 |
    • As well as being verified in GitHub, your email must be registered with submitGit, 36 | to allow submitGit to send emails with your address as the 'From:' field.
      • 37 |
    38 |

    Recommended reading

    39 | 45 |
    46 |
    47 | } -------------------------------------------------------------------------------- /app/views/listPullRequests.scala.html: -------------------------------------------------------------------------------- 1 | @(userPRs: Seq[PullRequest], otherPRs: Seq[PullRequest])(implicit req: GHRepoRequest[_]) 2 | @main { 3 | 4 |
    5 |
      6 |
    1. @req.repo.full_name
      2. 7 |
    8 |
    9 |
    10 |
    11 |
    12 | @if(userPRs.nonEmpty) { 13 |

    Choose one of your pull requests against @req.repo.full_name

    14 | @fragments.listPullRequests(userPRs) 15 | } else { 16 |

    You don't have any pull requests against @req.repo.full_name. 17 | Try a preview with one of these:

    18 | @fragments.listPullRequests(otherPRs) 19 | } 20 |
    21 |
    22 | } -------------------------------------------------------------------------------- /app/views/main.scala.html: -------------------------------------------------------------------------------- 1 | @(content: Html)(implicit req: RequestHeader) 2 | @import com.madgag.playgithub.auth.MinimalGHPerson 3 | 4 | 5 | @fragments.head() 6 | 7 | 35 | 36 |
    @content
    37 | @fragments.footer() 38 | 39 | 40 | -------------------------------------------------------------------------------- /app/views/pullRequestSent.scala.html: -------------------------------------------------------------------------------- 1 | @(user: User, mailType: lib.MailType)(implicit req: GHPRRequest[_]) 2 | @import lib.MailType 3 | @main { 4 |
    5 |
      6 | @fragments.linkedBreadcrumb(routes.Application.listPullRequests(req.repo.repoId), "repo") { @req.repo.full_name } 7 | @fragments.linkedBreadcrumb(routes.Application.reviewPullRequest(req.pr.prId), "git-pull-request") { #@req.pr.number } 8 |
    1. @if(mailType==MailType.Preview){preview }email sent
      2. 9 |
    10 |
    11 |
    12 |
    13 |

    14 | 15 | #@req.pr.number @req.pr.title 16 |

    17 | 18 | 21 |
    22 |
    23 | } -------------------------------------------------------------------------------- /app/views/reviewPullRequest.scala.html: -------------------------------------------------------------------------------- 1 | @( 2 | commits: Seq[PullRequest.CommitOverview], 3 | proposedMailByType: Map[lib.MailType,lib.ProposedMail], 4 | messageIds: Seq[String] 5 | )(implicit req: GHPRRequest[_], prMailSettings: Form[lib.PRMailSettings], messages: Messages) 6 | @main { 7 |
    8 |
      9 | @fragments.linkedBreadcrumb(routes.Application.listPullRequests(req.repo.repoId), "repo") { @req.repo.full_name } 10 |
    1. #@req.pr.number
      2. 11 |
    12 | @for(notifyEmailVerification <- req.flash.get("notifyEmailVerification")) { 13 | 19 | } 20 |
    21 |
    22 |
    23 |

    24 | 25 | #@req.pr.number @req.pr.title 26 |

    27 | 28 | @fragments.patchBaseGuidelines(req.pr.base.ref) 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | @for(commit <- commits) { 40 | 41 | 42 | 43 | 44 | } 45 | 46 |
    Commits
    IdSubject
    @commit.sha.name.take(8)@commit.subject
    47 | 
    @for(line <- req.pr.body.getOrElse("").lines; (allowed, excess) = line.splitAt(72)) {@allowed@if(excess.nonEmpty) {@excess}
    }
    48 | 49 |
    50 |
    51 | @fragments.sendTabs(req.defaultMailType, proposedMailByType, messageIds) 52 |
    53 |
    54 | } -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | name := "submitgit" 2 | 3 | version := "1.0-SNAPSHOT" 4 | 5 | scalaVersion := "2.11.8" 6 | 7 | updateOptions := updateOptions.value.withCachedResolution(true) 8 | 9 | lazy val root = (project in file(".")).enablePlugins( 10 | PlayScala, 11 | BuildInfoPlugin 12 | ).settings( 13 | buildInfoKeys := Seq[BuildInfoKey]( 14 | name, 15 | BuildInfoKey.constant("gitCommitId", Option(System.getenv("SOURCE_VERSION")) getOrElse(try { 16 | "git rev-parse HEAD".!!.trim 17 | } catch { case e: Exception => "unknown" })) 18 | ), 19 | buildInfoPackage := "app" 20 | ) 21 | 22 | TwirlKeys.templateImports ++= Seq( 23 | "com.madgag.github.Implicits._", 24 | "com.madgag.scalagithub.model._", 25 | "lib.actions.Requests._" 26 | ) 27 | 28 | routesImport ++= Seq("lib._","com.madgag.scalagithub.model._","controllers.Binders._","org.eclipse.jgit.lib.ObjectId") 29 | 30 | resolvers ++= Seq( 31 | Resolver.sonatypeRepo("releases") 32 | ) 33 | 34 | libraryDependencies ++= Seq( 35 | cache, 36 | filters, 37 | "org.typelevel" %% "cats-core" % "0.9.0", 38 | "com.madgag" %% "play-git-hub" % "4.0", 39 | "com.typesafe.akka" %% "akka-agent" % "2.3.2", 40 | "org.webjars" % "bootstrap" % "3.3.7", 41 | "com.adrianhurt" %% "play-bootstrap3" % "0.4.5-P24", 42 | "org.webjars.bower" % "octicons" % "4.3.0", 43 | "org.webjars.bower" % "timeago" % "1.5.3", 44 | "org.webjars.bower" % "typeahead.js" % "0.11.1", 45 | "org.webjars.bower" % "typeahead.js-bootstrap3.less" % "0.2.3", 46 | "org.webjars.npm" % "handlebars" % "4.0.6", 47 | "org.jsoup" % "jsoup" % "1.10.2", 48 | "com.github.nscala-time" %% "nscala-time" % "2.16.0", 49 | "com.netaporter" %% "scala-uri" % "0.4.16", 50 | "com.github.scala-incubator.io" %% "scala-io-file" % "0.4.3-1", 51 | "org.scalatestplus" %% "play" % "1.4.0" % "test", 52 | "com.amazonaws" % "aws-java-sdk-ses" % "1.11.102", 53 | "com.sun.mail" % "javax.mail" % "1.5.6" 54 | ) 55 | 56 | sources in (Compile,doc) := Seq.empty 57 | 58 | publishArtifact in (Compile, packageDoc) := false 59 | -------------------------------------------------------------------------------- /conf/application.conf: -------------------------------------------------------------------------------- 1 | # This is the main configuration file for the application. 2 | # ~~~~~ 3 | 4 | # Secret key 5 | # ~~~~~ 6 | # The secret key is used to secure cryptographics functions. 7 | # If you deploy your application to several instances be sure to use the same key! 8 | play.crypto.secret=${?APPLICATION_SECRET} 9 | 10 | # The application languages 11 | # ~~~~~ 12 | play.i18n.langs=["en"] 13 | 14 | google.analytics.id=${?GOOGLE_ANALYTICS_ID} 15 | 16 | github { 17 | botAccessToken=${SUBMITGIT_GITHUB_ACCESS_TOKEN} # Needs 'public_repo' scop in order to comment on PR 18 | 19 | clientId=${GITHUB_APP_CLIENT_ID} 20 | clientSecret=${GITHUB_APP_CLIENT_SECRET} 21 | } -------------------------------------------------------------------------------- /conf/routes: -------------------------------------------------------------------------------- 1 | # Routes 2 | # This file defines all application routes (Higher priority routes first) 3 | # ~~~~ 4 | 5 | GET / controllers.Application.index() 6 | GET /$repo<[^/]+/[^/]+>/pulls controllers.Application.listPullRequests(repo: RepoId) 7 | GET /$pr<[^/]+/[^/]+/pull/[\d]+> controllers.Application.reviewPullRequest(pr: PullRequestId) 8 | POST /$pr<[^/]+/[^/]+/pull/[\d]+>/:submitType controllers.Application.mailPullRequest(pr: PullRequestId, submitType: MailType) 9 | GET /$pr<[^/]+/[^/]+/pull/[\d]+>/:headCommit/:sig controllers.Application.acknowledgePreview(pr: PullRequestId, headCommit: ObjectId, sig: String) 10 | 11 | GET /oauth/callback controllers.Auth.oauthCallback(code: String) 12 | GET /logout controllers.Auth.logout() 13 | 14 | GET /api/$repo<[^/]+/[^/]+>/message-lookup @controllers.Api.messageLookup(repo: RepoId, query: String) 15 | 16 | 17 | # Map static resources from the /public folder to the /assets URL path 18 | GET /assets/*file controllers.Assets.at(path="/public", file) 19 | -------------------------------------------------------------------------------- /heroku.md: -------------------------------------------------------------------------------- 1 | Setting up _submitGit_ on Heroku 2 | ================================ 3 | 4 | Follow the [Heroku Quick-Start guide](https://devcenter.heroku.com/articles/quickstart) to get your developement environment setup with Heroku. 5 | 6 | [![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy?template=https://github.com/rtyley/submitgit) 7 | -------------------------------------------------------------------------------- /notes.md: -------------------------------------------------------------------------------- 1 | In http://git-scm.com/docs/git-format-patch : 2 | 3 | --subject-prefix= 4 | Instead of the standard [PATCH] prefix in the subject line, instead use []. This allows for useful naming of a patch series, and can be combined with the --numbered option. 5 | 6 | [PATCH v2 ...] 7 | [RFC/PATCH ...] 8 | [WIP/PATCH v4 6/8] 9 | [PATCH] 10 | [RFC/WIP PATCH 06/11] 11 | [PATCH/WIP 4/8] 12 | 13 | Matthieu Moy said: I found no way to specify a version like [PATCH v2 ...]. Similarly, there could be a way to say [RFC/PATCH ...]. 14 | 15 | 16 | 17 | Potential features TODO: 18 | 19 | * When closing a PR 'cos it's been posted, link to http://mid.gmane.org/1234567890.1234567890@example.com so we can see the discussion. 20 | * When RE-posting a PR, pull out the OLD message id so we can say it's in-reply to the 21 | previous message. 22 | 23 | Reasons why you can't send to the mailing-list, only preview: 24 | 25 | * You did not create the PR 26 | * The PR is not Open? 27 | * You have not registered your email with Amazon SES 28 | * You have not sent a preview email? 29 | * We're out of SES quota 30 | * Signed off missing - need a full name? 31 | * The body of the commit messages is too short??? There are some commits in Git with just a subject line and "Signed-off-by"s 32 | * The subject line is too long (can't do for whole body, some things can go long!) 33 | 34 | Commit warnings: 35 | 36 | * Message lines too long 37 | * Commit content lines too long 38 | 39 | Sending emails with the correct From header is hard 40 | 41 | 1. Don't use correct From - just send as submitgit@gmail.com - update git-mailinfo in order to parse a new Authored-By header. Ensure that the author 42 | is cc'd on the message, so they at least get the reply when people reply-all 43 | 2. Get the user to forward the message? What if their mail agent corrupts it...? 44 | 3. Use a service like Amazon SES, which has an API for adding verified 45 | email addresses : http://docs.aws.amazon.com/ses/latest/APIReference/API_VerifyEmailIdentity.html 46 | 47 | Need to go *raw* with SES in order to send 'In-Reply-To' & 'References' 48 | headers: 49 | http://docs.aws.amazon.com/ses/latest/DeveloperGuide/send-email-raw.html 50 | 51 | The From address should be the authenticated GitHub user address, not 52 | the email address on the commit - the GitHub user is the person 53 | submitting this patch. 54 | 55 | By default, the subject of a single patch is "[PATCH] " followed by the concatenation of lines from the commit message up to the first blank line (see the DISCUSSION section of git-commit(1)). 56 | 57 | When multiple patches are output, the subject prefix will instead be "[PATCH n/m] ". To force 1/1 to be added for a single patch, use -n. To omit patch numbers from the subject, use -N. 58 | 59 | If given --thread, git-format-patch will generate In-Reply-To and References headers to make the second and subsequent patch mails appear as replies to the first mail; this also generates a Message-Id header to reference. 60 | 61 | http://git-scm.com/docs/git-format-patch 62 | https://github.com/torvalds/linux/pull/17#issuecomment-5654674 63 | https://github.com/torvalds/linux/pull/17#issuecomment-5663780 64 | 65 | 66 | From: 51ac58c9e7bce595f6652513dd6ef2e9f92dd2b1 67 | 68 | Would make sense to link off to the mailing arcive: 69 | 70 | http://dir.gmane.org/gmane.comp.version-control.git 71 | http://marc.info/?l=git 72 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.13 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | // Comment to get more information during initialization 2 | logLevel := Level.Warn 3 | 4 | resolvers += Resolver.typesafeRepo("releases") // Play seems to require quite a few things from here 5 | 6 | // Use the Play sbt plugin for Play projects 7 | addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.4.10") 8 | 9 | addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.6.1") 10 | -------------------------------------------------------------------------------- /public/images/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rtyley/submitgit/353b9aec72829ea392b97eefcb97a7e2a3464022/public/images/favicon.png -------------------------------------------------------------------------------- /public/images/git-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rtyley/submitgit/353b9aec72829ea392b97eefcb97a7e2a3464022/public/images/git-logo.png -------------------------------------------------------------------------------- /public/javascript/changeHighlighter.js: -------------------------------------------------------------------------------- 1 | (function ( jQ ) { 2 | jQ.fn.changeVal = function(newVal) { 3 | var highlighterClass = "highlighter"; 4 | 5 | return this.each(function(index, elem) { 6 | var jqElem = jQ(elem); 7 | 8 | if (newVal && jqElem.val() != newVal) { 9 | jqElem.val(newVal); 10 | jqElem.bind('animationend', function(){ jqElem.removeClass(highlighterClass); }); 11 | jqElem.addClass(highlighterClass); 12 | } 13 | }); 14 | }; 15 | 16 | }( jQuery )); 17 | -------------------------------------------------------------------------------- /public/javascript/messageIdPicker.js: -------------------------------------------------------------------------------- 1 | (function ( jQ ) { 2 | 3 | jQ.fn.messageIdPicker = function(options, datasets, onSelect) { 4 | return this.each(function(index, elem) { 5 | var inputGroup = jQ(elem).find(".input-group"); 6 | var messageIdent = jQ(elem).find(".message-ident"); 7 | 8 | var input = inputGroup.find(".typeahead"); 9 | var messageIdentContent = messageIdent.find(".message-ident-content"); 10 | var messageIdentDelete = messageIdent.find(".close"); 11 | var suggestionTemplate = datasets.templates.suggestion; 12 | 13 | messageIdent.click(function () { 14 | inputGroup.show(); 15 | messageIdent.hide(); 16 | }); 17 | messageIdentDelete.click(function () { 18 | input.val(""); 19 | }); 20 | 21 | input.bind('typeahead:render', function () { 22 | inputGroup.find("time.timeago").timeago(); 23 | }); 24 | var showIdent = function (suggestion) { 25 | messageIdentContent.html(suggestionTemplate(suggestion)); 26 | messageIdentContent.find("time.timeago").timeago(); 27 | inputGroup.hide(); 28 | messageIdent.show(); 29 | onSelect(suggestion); 30 | }; 31 | input.bind('typeahead:select', function (ev, suggestion) { showIdent(suggestion) }); 32 | 33 | var initialMessageId = input.val(); 34 | if (initialMessageId !== "") { 35 | function showIdentIfSingleResult(datums) { 36 | if (datums.length == 1) { showIdent(datums[0]); } 37 | } 38 | datasets.source(initialMessageId, showIdentIfSingleResult, showIdentIfSingleResult); 39 | } 40 | 41 | input.typeahead(options, datasets); 42 | }); 43 | }; 44 | 45 | }( jQuery )); 46 | 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /public/stylesheets/main.css: -------------------------------------------------------------------------------- 1 | pre.non-wrapping code { 2 | overflow: auto; 3 | word-break: normal !important; 4 | word-wrap: normal !important; 5 | white-space: pre !important; 6 | } 7 | pre .long { 8 | border-left: 1px dotted red; 9 | color: red 10 | } 11 | 12 | .gw-footer { 13 | padding-top: 10px; 14 | padding-bottom: 10px; 15 | margin-top: 20px; 16 | color: #777; 17 | text-align: center; 18 | border-top: 1px solid #e5e5e5; 19 | } 20 | .gw-footer-links { 21 | margin-top: 20px; 22 | padding-left: 0; 23 | color: #999; 24 | } 25 | .gw-footer-links li { 26 | display: inline; 27 | padding: 0 2px; 28 | } 29 | .gw-footer-links li:first-child { 30 | padding-left: 0; 31 | } 32 | 33 | .send-tabs .tab-content > .active { 34 | padding: 15px; 35 | border: 1px solid #ddd; 36 | border-bottom-left-radius: 4px; 37 | border-bottom-right-radius: 4px; 38 | border-top: none; 39 | } 40 | 41 | .form-horizontal.addresses .form-group { 42 | margin-bottom: 0px; 43 | } 44 | 45 | 46 | .tt-dropdown-menu .tt-suggestion { 47 | white-space: normal; 48 | } 49 | 50 | @keyframes highlight { 51 | 0% { 52 | background: yellow; 53 | } 54 | 100% { 55 | background: none; 56 | } 57 | } 58 | 59 | .highlighter { 60 | animation: highlight 1s; 61 | } -------------------------------------------------------------------------------- /system.properties: -------------------------------------------------------------------------------- 1 | java.runtime.version=1.8 2 | 3 | -------------------------------------------------------------------------------- /test/lib/MailArchiveSpec.scala: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import javax.mail.internet.MailDateFormat 4 | 5 | import com.madgag.okhttpscala._ 6 | import com.netaporter.uri.dsl._ 7 | import okhttp3.{OkHttpClient, Request} 8 | import org.scalatest.concurrent.{IntegrationPatience, ScalaFutures} 9 | import org.scalatestplus.play.PlaySpec 10 | 11 | import scala.concurrent.Await 12 | import scala.concurrent.ExecutionContext.Implicits.global 13 | import scala.util.Try 14 | import scalax.io.Resource 15 | 16 | object Network { 17 | import scala.concurrent.duration._ 18 | 19 | val okClient = new OkHttpClient() 20 | val secureGoogleRequest = new Request.Builder().url("https://www.google.com/").build() 21 | 22 | def isAvailable: Boolean = 23 | Try(Await.result(okClient.execute(secureGoogleRequest)(_.isSuccessful), 2 second)).getOrElse(false) 24 | } 25 | 26 | class MailArchiveSpec extends PlaySpec with ScalaFutures with IntegrationPatience { 27 | 28 | "Google Groups" should { 29 | val submitGitGoogleGroup = GoogleGroup("submitgit-test") 30 | 31 | val ExampleMessageId = "0000014d9a92ef17-abf8da02-8ed6-4f1e-b959-db3d4617e750-000000@eu-west-1.amazonses.com" 32 | 33 | "have a proper link for a message-id" in { 34 | GoogleGroup("foo").linkFor("bar@baz.com").toString mustEqual 35 | "https://groups.google.com/d/msgid/foo/bar@baz.com" 36 | 37 | submitGitGoogleGroup.linkFor(ExampleMessageId).toString mustEqual 38 | "https://groups.google.com/d/msgid/submitgit-test/0000014d9a92ef17-abf8da02-8ed6-4f1e-b959-db3d4617e750-000000@eu-west-1.amazonses.com" 39 | } 40 | "link to google group" in { 41 | submitGitGoogleGroup.url mustEqual "https://groups.google.com/forum/#!forum/submitgit-test" 42 | } 43 | "derive correct email address" in { 44 | submitGitGoogleGroup.emailAddress.toString mustEqual "submitgit-test@googlegroups.com" 45 | } 46 | "give correct raw article url" in { 47 | submitGitGoogleGroup.rawUrlFor("/forum/#!msg/submitgit-test/-cq4q1w7jyY/A-EH61BAZaUJ").toString mustEqual 48 | "https://groups.google.com/forum/message/raw?msg=submitgit-test/-cq4q1w7jyY/A-EH61BAZaUJ" 49 | } 50 | "get canonical url for message" in { 51 | assume(Network.isAvailable) 52 | whenReady(submitGitGoogleGroup.canonicalUrlFor(ExampleMessageId).value) { 53 | _.value.toString mustEqual "https://groups.google.com/forum/#!msg/submitgit-test/E3FadZAKXU0/7htJeneMBP8J" 54 | } 55 | } 56 | "get message data" in { 57 | assume(Network.isAvailable) 58 | whenReady(submitGitGoogleGroup.lookupMessage(ExampleMessageId)) { s => 59 | val messageSummary = s.head 60 | messageSummary.id mustEqual ExampleMessageId 61 | messageSummary.subject mustEqual "[PATCH] Update gc.c" 62 | } 63 | } 64 | "get '[PATCH/RFC v245] Update gc.c' message data posted by submitGit using AWS SES" in { 65 | assume(Network.isAvailable) 66 | val messageId = "0000014e69391015-5de3c8c6-458f-4eb6-b222-14cfc8a6b055-000000@eu-west-1.amazonses.com" 67 | whenReady(submitGitGoogleGroup.lookupMessage(messageId)) { s => 68 | val messageSummary = s.head 69 | messageSummary.id mustEqual messageId 70 | messageSummary.subject mustEqual "[PATCH/RFC v245] Update gc.c" 71 | } 72 | } 73 | } 74 | "PublicInbox" should { 75 | 76 | val ChurchOfGitMessageId = "1431830650-111684-1-git-send-email-shawn@churchofgit.com" 77 | 78 | "have a proper link for a message-id" in { 79 | PublicInbox.Git.linkFor(ChurchOfGitMessageId).toString mustEqual 80 | "https://public-inbox.org/git/1431830650-111684-1-git-send-email-shawn@churchofgit.com/" 81 | } 82 | "give correct canonical url for message" in { 83 | whenReady(PublicInbox.Git.canonicalUrlFor(ChurchOfGitMessageId).value) { 84 | _.value.toString mustEqual "https://public-inbox.org/git/1431830650-111684-1-git-send-email-shawn@churchofgit.com/" 85 | } 86 | } 87 | "give correct raw article url" in { 88 | PublicInbox.Git.rawUrlFor("https://public-inbox.org/git/1431830650-111684-1-git-send-email-shawn@churchofgit.com/").toString mustEqual 89 | "https://public-inbox.org/git/1431830650-111684-1-git-send-email-shawn@churchofgit.com/raw" 90 | } 91 | "get message data" in { 92 | assume(Network.isAvailable) 93 | whenReady(PublicInbox.Git.lookupMessage(ChurchOfGitMessageId)) { s => 94 | val messageSummary = s.head 95 | messageSummary.id mustEqual ChurchOfGitMessageId 96 | messageSummary.subject mustEqual "[PATCH] daemon: add systemd support" 97 | } 98 | } 99 | } 100 | 101 | "MARC" should { 102 | "deobfuscate this crazy stuff" in { 103 | Marc.deobfuscate("Roberto Tyley <roberto.tyley () gmail ! com>") mustEqual 104 | "Roberto Tyley " 105 | } 106 | "have MessageSummary parsed (totally hackishly) from html - because the alternate raw version excludes headers" in { 107 | val message = 108 | Marc.messageSummaryFor(Resource.fromClasspath("samples/mailarchives/marc/m.143228360912708.html").string) 109 | 110 | message.id mustEqual "CAFY1edY3+Wt-p2iQ5k64Fg-nMk2PmRSvhVkQSVNw94R18uPV2Q@mail.gmail.com" 111 | message.date.toInstant mustEqual new MailDateFormat().parse("Fri, 22 May 2015 09:33:20 +0100").toInstant 112 | message.subject mustEqual 113 | "[Announce] submitGit for patch submission (was \"Diffing submodule does not yield complete logs\")" 114 | } 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /test/lib/MailTypeSpec.scala: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import org.scalatestplus.play.PlaySpec 4 | 5 | class MailTypeSpec extends PlaySpec { 6 | "Email address construction" should { 7 | MailType.internetAddressFor("c@m.com", "Cédric Malard").toString mustEqual 8 | "=?UTF-8?Q?C=C3=A9dric_Malard?= " 9 | } 10 | } -------------------------------------------------------------------------------- /test/lib/MarkdownParsingSpec.scala: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import org.scalatestplus.play.PlaySpec 4 | 5 | class MarkdownParsingSpec extends PlaySpec { 6 | 7 | val prComment = 8 | """ 9 | |@rtyley sent this to the mailing list with [_submitGit_](https://github.com/rtyley/submitgit) - here on [Google Groups](https://groups.google.com/d/msgid/submitgit-test/0000014e6f0b65e9-e3f67cd3-6679-41ed-b2e2-819766dbe2a0-000000@eu-west-1.amazonses.com) [](0000014e6f0b65e9-e3f67cd3-6679-41ed-b2e2-819766dbe2a0-000000@eu-west-1.amazonses.com) 10 | """.stripMargin 11 | 12 | "Markdown parsing" should { 13 | "get hidden message ids" in { 14 | val links = MarkdownParsing.parseHiddenLinksOutOf( 15 | "[](0000014e69391015-5de3c8c6-458f-4eb6-b222-14cfc8a6b055-000000@eu-west-1.amazonses.com)" 16 | ) 17 | 18 | links mustEqual Seq("0000014e69391015-5de3c8c6-458f-4eb6-b222-14cfc8a6b055-000000@eu-west-1.amazonses.com") 19 | } 20 | "get hidden message id in real message" in { 21 | val links = MarkdownParsing.parseHiddenLinksOutOf(prComment) 22 | 23 | links mustEqual Seq("0000014e6f0b65e9-e3f67cd3-6679-41ed-b2e2-819766dbe2a0-000000@eu-west-1.amazonses.com") 24 | } 25 | } 26 | } 27 | 28 | -------------------------------------------------------------------------------- /test/lib/PatchParsingSpec.scala: -------------------------------------------------------------------------------- 1 | package lib 2 | 3 | import com.madgag.git._ 4 | import fastparse.core.Parsed 5 | import lib.model.PatchParsing 6 | import org.scalatestplus.play.PlaySpec 7 | 8 | import scalax.io.Resource 9 | 10 | class PatchParsingSpec extends PlaySpec { 11 | 12 | val pull187PatchesText = Resource.fromClasspath("samples/bfg/pull87/github.patch").string 13 | 14 | "FastParse" should { 15 | "parse the from header" in { 16 | // http://stackoverflow.com/q/15790120/438886 17 | val Parsed.Success(value, successIndex) = 18 | PatchParsing.patchFromHeader.parse("From 21c2ef53cd0809149cdc158a6e5335ab0af77e7f Mon Sep 17 00:00:00 2001\n") 19 | value mustEqual "21c2ef53cd0809149cdc158a6e5335ab0af77e7f".asObjectId 20 | } 21 | 22 | "parse a patch header" in { 23 | val githubPatch = Resource.fromClasspath("samples/git/pull145/github.patch").string 24 | 25 | val Parsed.Success(value, s) = PatchParsing.patchHeaderRegion.parse(githubPatch) 26 | value mustEqual "af7333c176401601d67ea67cb961332ee4ef3574".asObjectId 27 | } 28 | 29 | "parse a patch" in { 30 | val Parsed.Success(patch, successIndex) = PatchParsing.patch.parse(pull187PatchesText) 31 | patch.commitId mustEqual "72428763eee19a2d83cc05a0ae4d55ab76930762".asObjectId 32 | patch.body must endWith(" run(\"--delete-folders .git --no-blob-protection\")\n") 33 | } 34 | 35 | "parse a sequence of patches" in { 36 | val Parsed.Success(patches, successIndex) = PatchParsing.patches.parse(pull187PatchesText) 37 | patches must have size 2 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /test/lib/model/MessageSummarySpec.scala: -------------------------------------------------------------------------------- 1 | package lib.model 2 | 3 | import org.scalatestplus.play.PlaySpec 4 | 5 | import scalax.io.Resource 6 | 7 | class MessageSummarySpec extends PlaySpec { 8 | 9 | val awsSesPost = Resource.fromClasspath("samples/raw.posts/raw.posted-by-aws-ses.txt").string 10 | val googleGroupsPost = Resource.fromClasspath("samples/raw.posts/raw.posted-by-google-groups.txt").string 11 | val publicInbox = Resource.fromClasspath("samples/raw.posts/raw.public-inbox.txt").string 12 | 13 | "Message summary from raw" should { 14 | "parse a message posted by AWS SES" in { 15 | val messageSummary = MessageSummary.fromRawMessage(awsSesPost, "") 16 | messageSummary.subject mustEqual "[PATCH/RFC v245] Update gc.c" 17 | } 18 | 19 | "parse a message posted by the Google Groups interface" in { 20 | val messageSummary = MessageSummary.fromRawMessage(googleGroupsPost, "") 21 | messageSummary.subject mustEqual "Scary fudge" 22 | } 23 | 24 | "parse a message stored on public-inbox.org" in { 25 | val messageSummary = MessageSummary.fromRawMessage(publicInbox, "") 26 | messageSummary.subject mustEqual "[PATCH] daemon: add systemd support" 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /test/lib/model/PatchBombSpec.scala: -------------------------------------------------------------------------------- 1 | package lib.model 2 | 3 | import java.time.ZonedDateTime 4 | import javax.mail.internet.InternetAddress 5 | 6 | import com.madgag.scalagithub.model.CommitIdent 7 | import com.madgag.scalagithub.model.PullRequest.CommitOverview 8 | import lib.Email.Addresses 9 | import org.eclipse.jgit.lib.ObjectId 10 | import org.scalatestplus.play.PlaySpec 11 | 12 | class PatchBombSpec extends PlaySpec { 13 | 14 | val bob = CommitIdent("bob", "bob@x.com", ZonedDateTime.now) 15 | val fred = CommitIdent("fred", "fred@y.com", ZonedDateTime.now) 16 | 17 | def patchCommitAuthoredBy(author: CommitIdent) = 18 | PatchCommit(Patch(ObjectId.zeroId, "PATCHBODY"), 19 | CommitOverview("", ObjectId.zeroId, "", CommitOverview.Commit( 20 | "", author, author, "Commit message", 0 21 | )) 22 | ) 23 | 24 | 25 | def patchBombFrom(user: CommitIdent, patchCommit: PatchCommit) = PatchBomb( 26 | Seq(patchCommit), 27 | Addresses(from = new InternetAddress(user.addressString)), 28 | footer = "FOOTER" 29 | ) 30 | 31 | "Patch bomb" should { 32 | "add a in-body 'From' header when commit author differs from email sender" in { 33 | val patchCommit = patchCommitAuthoredBy(bob) 34 | val patchBomb = patchBombFrom(fred, patchCommit) 35 | 36 | patchBomb.emails.head.bodyText must startWith(s"From: bob \n\n${patchCommit.patch.body}") 37 | } 38 | 39 | "not add an in-body 'From' header when the commit author matches the email sender" in { 40 | val patchBomb = patchBombFrom(bob, patchCommitAuthoredBy(bob)) 41 | 42 | patchBomb.emails.head.bodyText must not include "From:" 43 | } 44 | 45 | "not add an in-body 'From' header when the commit author used a 'noreply' address" in { 46 | // http://article.gmane.org/gmane.comp.version-control.git/286879 47 | val mrNoReply = CommitIdent("Peter Dave Hello", "peterdavehello@users.noreply.github.com", ZonedDateTime.now) 48 | val patchBomb = patchBombFrom(bob, patchCommitAuthoredBy(mrNoReply)) 49 | 50 | val text = patchBomb.emails.head.bodyText 51 | 52 | text must not include "From:" 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /test/lib/model/SubjectPrefixSpec.scala: -------------------------------------------------------------------------------- 1 | package lib.model 2 | 3 | import lib.model.SubjectPrefixParsing.parse 4 | import org.scalatestplus.play.PlaySpec 5 | 6 | class SubjectPrefixSpec extends PlaySpec { 7 | "SubjectPrefixParsing" should { 8 | "be happy if there is no patch version" in { 9 | parse("[PATCH] send-email") mustBe Some(SubjectPrefix("PATCH")) 10 | } 11 | "extract version" in { 12 | parse("[PATCH v3] send-email") mustBe Some(SubjectPrefix("PATCH", Some(3))) 13 | } 14 | "extract version when patch index is present" in { 15 | parse("[PATCH v3 5/7] send-email") mustBe Some(SubjectPrefix("PATCH", Some(3))) 16 | } 17 | "work for prefix text that isn't just 'PATCH'" in { 18 | parse("[RFC/PATCH] send-email") mustBe Some(SubjectPrefix("RFC/PATCH")) 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /test/resources/samples/bfg/pull87/0001-WIP-Support-converting-repos-to-Git-LFS.patch: -------------------------------------------------------------------------------- 1 | From 72428763eee19a2d83cc05a0ae4d55ab76930762 Mon Sep 17 00:00:00 2001 2 | From: Roberto Tyley 3 | Date: Thu, 9 Apr 2015 12:08:55 +0200 4 | Subject: [PATCH 1/2] WIP: Support converting repos to Git LFS 5 | 6 | Git LFS allows uses to commit new files to the LFS store, but replacing 7 | _old_ files requires rewriting history, which is something the BFG is 8 | pretty good at. This rough cut allows replacing blobs with pointer files 9 | throughout repo history. 10 | 11 | Some caveats with this initial implementation: 12 | 13 | * the BFG cleans concurrently, files may unnecessarily be hashed more than once 14 | * the working directory isn't updated 15 | * specifying `-fi *.png` should be unnecessary, should use gitattributes 16 | * need for `--no-blob-protection` is a hangover from normal BFG behaviour 17 | 18 | Example invocation: 19 | 20 | ``` 21 | $ git clone https://github.com/guardian/membership-frontend.git 22 | $ cd membership-frontend 23 | $ java -jar bfg.jar --convert-to-git-lfs -fi *.png --no-blob-protection 24 | ... 25 | $ ls .git/lfs/objects/ | head -2 26 | 0145f7c304ef33a43cc946e0a57b2213d24dcaf8462f3d3b332407a8b258369c 27 | 07010d5ddea536da56ebdbbb28386921c94abd476046a245b35cd47e8eb6e426 28 | $ git reset --hard 29 | $ cat frontend/assets/images/favicons/152x152.png 30 | version https://git-lfs.github.com/spec/v1 31 | oid sha256:0145f7c304ef33a43cc946e0a57b2213d24dcaf8462f3d3b332407a8b258369c 32 | size 1935 33 | $ 34 | ``` 35 | 36 | https://git-lfs.github.com/ 37 | https://github.com/github/git-lfs/blob/5eb9bb01/docs/spec.md#the-pointer 38 | --- 39 | .../madgag/git/bfg/cleaner/LfsBlobConverter.scala | 88 +++++++++++++++++++++ 40 | .../scala/com/madgag/git/bfg/cli/CLIConfig.scala | 16 +++- 41 | .../sample-repos/repoWithBigBlobs.git.zip | Bin 0 -> 24252 bytes 42 | .../scala/com/madgag/git/bfg/cli/MainSpec.scala | 16 +++- 43 | 4 files changed, 117 insertions(+), 3 deletions(-) 44 | create mode 100644 bfg-library/src/main/scala/com/madgag/git/bfg/cleaner/LfsBlobConverter.scala 45 | create mode 100644 bfg/src/test/resources/sample-repos/repoWithBigBlobs.git.zip 46 | 47 | diff --git a/bfg-library/src/main/scala/com/madgag/git/bfg/cleaner/LfsBlobConverter.scala b/bfg-library/src/main/scala/com/madgag/git/bfg/cleaner/LfsBlobConverter.scala 48 | new file mode 100644 49 | index 0000000..ce07008 50 | --- /dev/null 51 | +++ b/bfg-library/src/main/scala/com/madgag/git/bfg/cleaner/LfsBlobConverter.scala 52 | @@ -0,0 +1,88 @@ 53 | +/* 54 | + * Copyright (c) 2015 Roberto Tyley 55 | + * 56 | + * This file is part of 'BFG Repo-Cleaner' - a tool for removing large 57 | + * or troublesome blobs from Git repositories. 58 | + * 59 | + * BFG Repo-Cleaner is free software: you can redistribute it and/or modify 60 | + * it under the terms of the GNU General Public License as published by 61 | + * the Free Software Foundation, either version 3 of the License, or 62 | + * (at your option) any later version. 63 | + * 64 | + * BFG Repo-Cleaner is distributed in the hope that it will be useful, 65 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of 66 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 67 | + * GNU General Public License for more details. 68 | + * 69 | + * You should have received a copy of the GNU General Public License 70 | + * along with this program. If not, see http://www.gnu.org/licenses/ . 71 | + */ 72 | + 73 | +package com.madgag.git.bfg.cleaner 74 | + 75 | +import java.nio.charset.Charset 76 | +import java.security.{DigestInputStream, MessageDigest} 77 | + 78 | +import com.google.common.io.ByteStreams 79 | +import com.madgag.git.ThreadLocalObjectDatabaseResources 80 | +import com.madgag.git.bfg.model.{FileName, TreeBlobEntry} 81 | +import org.apache.commons.codec.binary.Hex.encodeHexString 82 | +import org.eclipse.jgit.lib.Constants.OBJ_BLOB 83 | +import org.eclipse.jgit.lib.ObjectLoader 84 | + 85 | +import scala.util.Try 86 | +import scalax.file.Path 87 | +import scalax.file.Path.createTempFile 88 | +import scalax.io.Resource 89 | + 90 | +trait LfsBlobConverter extends TreeBlobModifier { 91 | + 92 | + val threadLocalObjectDBResources: ThreadLocalObjectDatabaseResources 93 | + 94 | + val lfsSuitableFiles: (FileName => Boolean) 95 | + 96 | + val charset = Charset.forName("UTF-8") 97 | + 98 | + val lfsObjectsDir: Path 99 | + 100 | + override def fix(entry: TreeBlobEntry) = { 101 | + val oid = (for { 102 | + _ <- Some(entry.filename) filter lfsSuitableFiles 103 | + loader = threadLocalObjectDBResources.reader().open(entry.objectId) 104 | + (shaHex, lfsPath) <- buildLfsFileFrom(loader) 105 | + } yield { 106 | + val pointer = 107 | + s"""|version https://git-lfs.github.com/spec/v1 108 | + |oid sha256:$shaHex 109 | + |size ${loader.getSize} 110 | + |""".stripMargin 111 | + 112 | + threadLocalObjectDBResources.inserter().insert(OBJ_BLOB, pointer.getBytes(charset)) 113 | + }).getOrElse(entry.objectId) 114 | + 115 | + (entry.mode, oid) 116 | + } 117 | + 118 | + def buildLfsFileFrom(loader: ObjectLoader): Option[(String, Path)] = { 119 | + val tmpFile = createTempFile() 120 | + 121 | + val digest = MessageDigest.getInstance("SHA-256") 122 | + 123 | + for { 124 | + inStream <- Resource.fromInputStream(new DigestInputStream(loader.openStream(), digest)) 125 | + outStream <- tmpFile.outputStream() 126 | + } ByteStreams.copy(inStream, outStream) 127 | + 128 | + val shaHex = encodeHexString(digest.digest()) 129 | + 130 | + val lfsPath = lfsObjectsDir / shaHex 131 | + 132 | + val ensureLfsFile = Try(if (!lfsPath.exists) tmpFile moveTo lfsPath).recover { 133 | + case _ => lfsPath.size.contains(loader.getSize) 134 | + } 135 | + 136 | + Try(tmpFile.delete(force = true)) 137 | + 138 | + for (_ <- ensureLfsFile.toOption) yield shaHex -> lfsPath 139 | + } 140 | +} 141 | \ No newline at end of file 142 | diff --git a/bfg/src/main/scala/com/madgag/git/bfg/cli/CLIConfig.scala b/bfg/src/main/scala/com/madgag/git/bfg/cli/CLIConfig.scala 143 | index abbdb08..8976a15 100644 144 | --- a/bfg/src/main/scala/com/madgag/git/bfg/cli/CLIConfig.scala 145 | +++ b/bfg/src/main/scala/com/madgag/git/bfg/cli/CLIConfig.scala 146 | @@ -74,6 +74,9 @@ object CLIConfig { 147 | fileMatcher("delete-folders").text("delete folders with the specified names (eg '.svn', '*-tmp' - matches on folder name, not path within repo)").action { 148 | (v, c) => c.copy(deleteFolders = Some(v)) 149 | } 150 | + opt[Unit]("convert-to-git-lfs").text("experimental support for Git LFS, use with '-fi' to specify files").hidden().action { 151 | + (_, c) => c.copy(lfsConversion = true) 152 | + } 153 | opt[File]("replace-text").abbr("rt").valueName("").text("filter content of files, replacing matched text. Match expressions should be listed in the file, one expression per line - " + 154 | "by default, each expression is treated as a literal, but 'regex:' & 'glob:' prefixes are supported, with '==>' to specify a replacement " + 155 | "string other than the default of '***REMOVED***'.").action { 156 | @@ -129,6 +132,7 @@ case class CLIConfig(stripBiggestBlobs: Option[Int] = None, 157 | filterSizeThreshold: Int = BlobTextModifier.DefaultSizeThreshold, 158 | textReplacementExpressions: Traversable[String] = List.empty, 159 | stripBlobsWithIds: Option[Set[ObjectId]] = None, 160 | + lfsConversion: Boolean = false, 161 | strictObjectChecking: Boolean = false, 162 | sensitiveData: Option[Boolean] = None, 163 | massiveNonFileObjects: Option[Int] = None, 164 | @@ -172,6 +176,16 @@ case class CLIConfig(stripBiggestBlobs: Option[Int] = None, 165 | } 166 | } 167 | 168 | + lazy val lfsBlobConverter: Option[LfsBlobConverter] = if (lfsConversion) Some { 169 | + new LfsBlobConverter { 170 | + val lfsObjectsDir = repo.getDirectory / "lfs" / "objects" 171 | + 172 | + val lfsSuitableFiles = filterContentPredicate 173 | + 174 | + val threadLocalObjectDBResources = repo.getObjectDatabase.threadLocalResources 175 | + } 176 | + } else None 177 | + 178 | lazy val privateDataRemoval = sensitiveData.getOrElse(Seq(fileDeletion, folderDeletion, blobTextModifier).flatten.nonEmpty) 179 | 180 | lazy val objectIdSubstitutor = if (privateDataRemoval) ObjectIdSubstitutor.OldIdsPrivate else ObjectIdSubstitutor.OldIdsPublic 181 | @@ -209,7 +223,7 @@ case class CLIConfig(stripBiggestBlobs: Option[Int] = None, 182 | } 183 | } 184 | 185 | - Seq(blobsByIdRemover, blobRemover, fileDeletion, blobTextModifier).flatten 186 | + Seq(blobsByIdRemover, blobRemover, fileDeletion, blobTextModifier, lfsBlobConverter).flatten 187 | } 188 | 189 | lazy val definesNoWork = treeBlobCleaners.isEmpty && folderDeletion.isEmpty && treeEntryListCleaners.isEmpty 190 | diff --git a/bfg/src/test/resources/sample-repos/repoWithBigBlobs.git.zip b/bfg/src/test/resources/sample-repos/repoWithBigBlobs.git.zip 191 | new file mode 100644 192 | index 0000000000000000000000000000000000000000..5648182cb2b5f93d2275d445a370f64fe82b3cf9 193 | GIT binary patch 194 | literal 24252 195 | zcmbrl1FSIJ(&xL4XWO=I+qP}nwr$(CZQI_@wr%hGo^vO2CSNi$-_7l$R=Sg3|E@}^ 196 | z*6L2zPhJWbWM0Q0{n*c%vG{ar9*Ul(0{}Sq$N2uM%)hi~jEzh=Oz91n 198 | z4UG(pn2b40ObwV>4Oto484WlXnAnXN=#34T4UJ8WX#aKh|H=QDOd)#`=;j(Z&eoh* 199 | zo2}NQSR1aKVxuk9vDSaC91dn9oPODSRoZT9H-mUIT3lB!s|U)c2}!{L0DQmN>^sK$ 200 | zoFq$8tV7ckg5f-oHFksSO~#!EDCoO`(DvtOt_Qwb%wX36W_IWB7UMv$)GPyK0#2b(S}E 202 | zcW78TH~zk#<$z54;QXjO7|kEEKsfz4{hARZ4c}_BVOW3n-3n_C2)X4E!i;fV{mkA}dZju+z 204 | z;_-^j5)(k%BSUQp6E}rEQ!)0?)gON7$hX 206 | ze$5&f0e(Tyi{3oqG3(mFseE&meP=cXTV#I;&iY2O95S{?0DwXkzF 210 | z;N&a58LXw<)~$6blLG?-Vf>-IjKgcA;MS4c+;hvH@O4nEA(88Alu5w}q4_u*x4 214 | zM@xn7|7ku=6mV0`vHg6vx4gE^VAFpuU9*G+0 215 | z7|ygiAR-wU2kA@H_wXurP2TmAIR=CzrxPECtwU`RLeQVwmWHmelcigWg3Wp1fNw-h+}8)x0!3K+SzMorL4vxP 217 | zCpNG^vDX+IO*7hIfD4)iHb=C 218 | zHU1Opt!Mu`hPCMRmk{x2X%r 221 | zM#ie93z$A+UsLDa4+RP4A0Hc9^T7By@wB^=!=*$y{>roflh+M}!1q|I3> 224 | zHHb8W4<6iqL6JO%9maA828_caHxa6+|4v<`QI+2-3DU6L>=1KLI)N&LpU5^+wMQNl 225 | z67Q)c5L1a!!24KF%SqQJFWzjw6nHw6@baZFQaIiVd2RvQ*lHWMw_f7*M8Q~p(O+PT 226 | zi~Y`FF@0PzWHxl%h&j4y#F3B=eH!u5BS$?@0gMMkn4TrnQ86vO+nbRw4QTPXj?WhS 227 | zXAFXPY#PBYM2A-UZ#dJ%PEdN!!8FbEb1~3n#Lu)3s?J7d!OmABYV6;6uYTHeh^G{uJ3N)6&wcb}FgQo}62J1z$MWiMaJxRsM6uZ3Ha!K+$s=v6mkS7&&;G_YnpnpUejnIIpipA^>| 235 | zGSiP-Do-EqnD?nq;a&1vpy~-g-42!tP#!!5E7IWV64A!2M+ob`0_$WSEI$IWnw1*A 236 | z*D)&y79VS;pdB$_N+V}`XA^I05n@o*Nz;G53*wp&71ge8q(m<;55}}}+KTj0n*xnF 237 | z0Mb6+sL$~)T4J;{u1fsm^+KBu%ukJH1L4m`eboncb*foqR<|ow87_?iRvWyPkr4}Z 238 | z8wX=GrGX2hL4pAi&3}Y1EcMQS!Uhz;(;5WCO 240 | z0)tq1nYj%hn6}oTrM#=0{>|H^jUC^r!18&hz3~ibbuJ3-rFT`v<@Z&$;bp!z68u}|ONt55i$M9VW@)+TBe!#rgG6rMzHMpKp|(U*Oj2OAH1TB*kyWg(cwW;jwQjXlI7& 244 | zc1*qtF(vKItPUQ_J8MR)4wF)l>c&W^6kt7?K0!M)h!7I48cpPNar28`SOt~T3dT9j 245 | zsxe3LsYMHf<#}?WH}A@0(`-x!VDa4&fL0CbGwMap)ZELHeIMs~JN`&g7Bc4ROj%xD 246 | zK6|weQ8s3)mOw`_?blFAwENerLkwoud(go-J`2K?ld0+-VF 248 | zH4B*4Ops;+^i#wBd|U%N4%5npEMK2(>cvs}B+!ryPV0Cr$Vswqy*NRVwn_Ry_e79^ 249 | z%Jq{ZdS0Ro6Csu&r1xL$S;#%NyTd0Q0f>bknswdc%dk1Nt`Tautu7&iO%nZ!Zk7IS 250 | z+=)F(NGKS$;K0|q@ggbt@_=zMQT}z-ZUHE!*ix>@KV#od49m;Ki>=r06IjxwP7R~N 251 | zZWHgvjQaYKaWS`SRW7@-sO+!KpZTF#rxF#bYi_rKP-lVwv}6 254 | zEtCTSFaysj&9P||J*27eZCbo;ew+RsgugI}Iyj#8zhN6}V59a+J?|i7$i11_lhXo{ 255 | zHp$gcYrlV9QIW5_gLXLq-&f+__Z*r;8qZn>y9sawt_;iT%$!=2oHsSI`^Ht-4geoU 256 | zAGm7VR@tF8v_uZwUG^kyrtBD%)z6L;FF!RcCwkQarmp~X$QC+U~U!+HhMF7_86|8~FYMaqK^V4}d({yY;BG>5#Ws(E$ 258 | z-9Mh##Ncr|Gh=p4{AGTq?5O%X1F>g3B8LwgSbDb*ciKqqnubSPm@_kE^7;##L3~t4(S4yb+8cgRU+`DwkdM90nIxpYR^^LEMQsg 260 | zTjveYXe|Nb!ADw`x};DJFul>`<+3xE__~;DeaHk-y)G*rD|M3jFOrw|-S?e14_EU# 261 | z)@V{0sxaizx~`v+Uuam&+?kWHQjz{uMJA@)_l?ofEl{T^Hg`6=GOobaqIZw;lX9A< 262 | z#{L@v+U)h?&~*E-iCBYNMx-UJ@uSr@rm5bpRo?_Dmfu6;hb!P{3cRRbhr5HtA7gE% 263 | zYy=m%I}$rFmti#7Tl-=j8I#fslo>qeQ1Bx+K=H$x)V3tg7NTWuo~1>-7B(IJ&di>3sD5DDtl#T<7Z8hD;-P}=s3gErPl&36?Hy&VUvD`wD~M<)E#!FqwtoFM 270 | zhV*mYSQm3QZ?{TbOR9%msi|z7IEaDkczya2O}K6z@6+5J@!|M1HZ!ftxQ&-EVq!`X 271 | zX(N;>d<<%giu!NPi7p$KmI`g>C++gmSl3lnjCmk+;uKD|m+_N*saw#v%y%?zCijs8 272 | z1?nk8p3birtuRNoH)HikDO~E`f0}**<2)iPTY}>Cxi{{o#)Dp~#f=1%?bEK!y920? 273 | zRfjB_4kSE&lfuCf==(O3{GC8m2Me}m*DXaJmCspp$NSbt3>rqoC}jXCZef^)i4 274 | zn|k%)dxfif_r}HY);|!Ui3gt13)o~8pBOhR=|7`Fc2UzdH_5YZwgJJu;>+sBgiLVl 275 | zDPpR68YmgfQz45N_U5%zN3)qmJ;Jv%5Cd~g?=B;V+4m3c&Q6apOR`ta`&5dqc2V^Y 276 | znOrirNRG9L=0bB{Ncvu9@9)J`Cy2D6 278 | zw@#uN8nRZrm%@#4;Du%psUx_DH8w3rGPd1W= 279 | zLIWC#kOK2!;hb231mLFEv;z@-8W&{nPD*PhOK>8d~Pe0+WX?x#~&N^P> 283 | zn?R-}@%|>~TD~=o?7+7dt08>(6@P|5NwYv+-0Plu%x|lqgivzd_TVAX0S*(g@3jJ_ 284 | zgtX+w7CtsM26|gJdSu)80wT7)tdln0B?$5Xdu-oWX(&v1&C`emu^YiZ%0 286 | zFDy>cnyQ~qWH_-|1eb`tM+tGbxr0C)A|s52j&&N4 289 | zOm);V@2wLn7%hlAKf%Tv5%O2gb8o>-tE9tgt*b;Is*!p^6{RNFFy^|FbUa$1qaHimYG!R2FEts+Y&UdoGit 293 | z%gxpBdC;c8g5LTYgtWO8jbpKy>tQMxMJI0gLXj91Gk?|x2!0>zyWBEIZfo>PW5XjU2Li!5GmKNMb>)m;3{jf*n&j1>9e?JH~TK(R;2Z3~RY91++2(p`y-}Xfm{y6!AEJ88p@Exej{(N(hoI@7_F8b;)hJGgsk* 297 | zmt4)TwTqmNSl?^(cn~a 298 | zea%?aJe(=QNC`p)Fjp0)MXz%`Z_yW>brFNRhK$Y!y-Qwhyf_khws&V@iGp>$k_fQ~dUX==g 303 | zt)q`qwI9`DQ{#@tg}+GPG*O~^d*}+B>DVnzw@9b5#wsc*ad2?9W8qx&>e4m)+A5KV 304 | zbPjS-E2MC=x_FE*zJ9dZ5vzSd^X<3Ip!kCkV?!Dbj`HlOz

    &0U)m|$nqeNPhu&J 305 | z5|S9CZc}EqS;l_zxWueD3c^ti`FHX_QBhmZMbeHGxAG``!<3*>@u4}N$+l-VKd$zH 306 | zkFkY^As20reKqHhCf0_&oF2v{RMj7ZsCMKO;PnTmWIXr@2uDwQuMcWZuuPxvW=i}w 307 | zjBeuqK*B`Y$sX?T;ai3WZked|NWyf*$mYl|3?qPuiCDT*nL&8g`zZ;zzU5|a2;ZV; 308 | z7DkK*#vjE;Q2N<4xxyrmklstn4c%d$QF4wUuELI2^zAI6o|{W>!ey}zL7Ok=OK4hl 309 | zuX4&3N}B88r)v8puYSqXu(&(2sK%(TtlS)y5oswYTRfYZPD2*es6)3vD~Rga(r@Ch 310 | z%(taXA*M{;ktq3gA+n5lt}-;!RU^MJ%DpWIYBS8U)6?>8$wRe+o#|^Z4qw&aE}j*I 311 | z^Kyi6lC0_hWeAe*3L^qymU?XB*AePY_^dDohwYH78=jK0)HY>g)Ad`y8Ys78RD7;Q 312 | zw(w=&DSWR>rBSsIg55%AEX%Krkr@$B*B$qQ+vmQ>4}|tOTxn%;X$BgL>7MJukEUdq 313 | zi0O(HG`5Bvm&Y$MI=kH#m|T}r(6LJnOQ@=9p94f{xV~27@(GlYUc&$p*q@ZG9b41^ 314 | z?khvj2jCWhSTbUx>QI}51Ok*flYFox3vy;QPw3%$Lxt#kKxR%eWJStSSGJH%z2v3L 315 | z&swvwsRP5>Yeb9HU^r#YXTD?p9qJEUx{S=#gJPo;7~fI5pn>T|D^!bt;)sQtgzdPT 316 | zrIGRD5$ic3b@EGMP=v+z#1tq2JbS@6=nzKv=iURAa1CjBNVx>7RPXm)5V`g!ki7fR 317 | zPPiOdJ}p`*ktsbMJUuI~Yu|pq2dhzWG4sfoaU-xG37koev;(W~Q_C~U5zIaf8ZRcB 318 | z+{1#tPu5I#ZED@^Hh+{=iKy%&tzU0!RLOh-8rMBAJELhuYKVWtVnSXXL# 319 | zmNC8-`3k8=Voxoo!ihS0-&?dQshMQLd{M*H2K*EQQyoEHq3*ZE@WI|r-bQ)IM0qqD 320 | z4&U^3?Rni!!kUZ)u>#mg{}5`8%2%_OT3!{P9T_K6^7{%*`2pNmKX9FF%*Nc;Zp$Mb 321 | z^umrlv2+w3aCS8&7XCdOO_|2OzqfOqI`Qa(Ui6XAqMm#IaWaHMObF{E_Z$%IX5cwD 322 | zpZzw75WhMW1Fmw@n=yx3a^s5c^qhcebzn=*eIL-l10H*%!dqcFbdVmd>W%4wK(VF} 323 | zsLBe-Gk;-1A|=cVS9aXH8DS0Al0MS+O$9`A6EDd7ibBk&prwJLi 324 | zRk@+q3S)iojq(cF&ssvpe;!B^F4R+^c~6@Bq7#`cys;rwH+6+XcB38=PPl6)8b}}o 325 | z7yS$Ja$H8r#is5|)FfA4p2#gp1V0o_z!e@d22>7X>RbE%6ee(Ofnqp9G|6)TH)JkP 326 | zF0iO|FR+A-rORb;$e>N^yZF8%?A-#R7QDT-(KF2BM-}Ara&MG?hjL_A(Uh(>!^PRV 327 | zS*h7P!sh!`-)pSfxQp}S)B4++Z3#v9cGNi$EtH1D=?o?3yL{1REu=TwQD$N*B!Y+o 328 | z@l)h8yQd7yL~VrEdZV!UY2DMs3dgou0EKlZkWs&edL}IqwWLzT+@#&Rt_nOTsA5Wq 330 | zsolvX5=imeVfZ}1ky)JJ`xVobHd6A(rj}Nod42d4t|dCzWAH5kXl-?|U51?qcS~P& 331 | zeRxB4S&-@4eSaZp54WqMzNYZvB!0}ITE6ww#W;TZetGCnB_ws@`AN`rmHrvMK1=ZC 332 | zX!&ATNQl9=4$zEsMM5^tPP 333 | zRLQfDT#227p`Rt1HACkt)@#SS6@1Lt(@aL?701!$l)VPC>lKzs$Z&-w$#MLB7jVb| 334 | zt0S7_?dqPnt3(%W?CLMFXC_VKb-n`N`(y2OXBh>=3%k3dT}Z%;E-WL)uDvddPv_@t 335 | zs&8EE_s_m_**bClM5d2k;i)R>mlD7GHU4WB(J|8L)t%^qArUaGqQ8cDx^d-K 336 | zP|r--BApATj;Yk}1SrE!-0TC*4`P}M2^NOLT3&Fr1!4hffzJ~-A{<9rZp*e1^Bh9B 337 | z)O8-8aQRv46C;*m9NZ@K1&0dzR&4KO6@U^|^5skMnDczl&L!&$6V=8gnn9f_ZS)O5 338 | zLGNzoc|eN5VWKz7p@|n&R${5QjEl3CkY`MKqAVfzcpY+PzC7BJ@1U9lyMFI){$ql9 339 | z(F{cApF{P5Wx)7IvdXi~xW|$Y`~hTFt$Q`2npT0waIJv{vuKU1N6xZRy36?vw>U~0 340 | zc*NnPx;#lx2&xzPn6^yTwOt8QBu|IeLFyc&r(p)3PjY?CFX@mH$}+;?LEc-1^*^7u 341 | zBwG^`3AO9572aBc<&pPUW6Q34{dVf7V_GUM`dUr~vHdMQ_{Z!jNM}#utNB-jMIjY+-(}G!WI?r7!mO;Y1z}YXmpG 343 | z{n;-Y+fA>@Ufy|8lOV`hP|&04#0uh+;qZ%GhjURlO3-cJELfn1e{vF(WlUQFa|K0# 344 | zAB!2`s^L83eg=(AG%I}0<@A(vaJYRZib215K&f7B?9+F3!OZiyMVd*Qo9++zIDO^~ 345 | zzqDeAaSWm0W@r)^Bp*7bFCuAvK+Rk{^Kdb)R5Nd$L`51caSSMVcy`hHa}6+pe4Zgg 346 | zQGLWe6t~K><59zIuBn!EwSPE7GVLetIECn_mRiPN(*(3@(%P)8Z`}or!%BoB{9x=~ 347 | zj5`ZPort~qSH$Uf19*Mie(BVB**CXBWNPCpuxM3>!Q%ev-Gc0BXQh-4IJkIrG)Su~ 348 | zq7a@BvNE6OYA?TmVD!eD9Y-M1!eY1x;OzU&YO_SKNJVXo&PwNr%jMFUviP_Ax_4Bh?K0ql-=7t1p{{x)H%YVTc*W{tPVOfEkzK$I^ua3NmFa6_=~+2P 350 | zb)7|8(L0}$9ioJSJ?N&=M#o)w^%$&lqQTC!vgFYr)_T!on!3)rqzXqdljuqD5CLm= 351 | zMzU__x*;zA?NyOkC3sctY~GSx8fmLfe>qxE!-q=RN5w~ 352 | zVr+l`IfvCX#(5{=J>mKp^*kmaIK(N0emvM`A}V+IphtP{AS3gtU0stt+O$;&wh$-9 353 | zbh_~eMWjB0^oe^^!Ea8}q5(9o)t@WnBsglo&Uw=^mbl;};BCMOSfxUD9m#J+ 357 | zk>q1>*I60OvxW)=Hf~xS+}t1SQ>?Y(O|6ve(0*p}{&){EKHc1>NHbAQWjvXag5~l+ 358 | z<2g#PVURdzzFE4sdP6aMQOWI|wUgw@+d*+QhoZ%WCGkB*6mQs8B2$JovE4+pUKx}I 359 | z98bskqNl9%?U}YPtEn+$^PAmar=^CCRUh=vjvL5M=Qy$W@O1G!(H|^{&G7Did~99@ 360 | z)M6=o8B?cug_BIj)j>t!-K)OR$yS@fzpBb%OhDvYIn2d}2Zu|(mB5s9sph#U{F33b 361 | zVceC6<2JNN`%;~xE8N+o=fG^2e0%WJW^Ev1rH6v}=VHg#lUm!bx?TNB65^hSpvN`P 362 | ztP5{`x5(8k_|hW!V3)mqWbW&|JS~A4DdOse?7+=E7l5WJ*AqBT{G2Ei67&-(z|dGBVd9PG=Qk#LQqur0}ih+FQ>i8OJ6rMAZ!7?}Dq#D?tQ 364 | znw<#2eYymge!`7`Il*Pb&bcFW%}+emPSu#i8LZyIe!`RyTF=2)xn9DF%QOji5&=_* 365 | zEa7Gal+O9(`AE)~TA%5%`mX*=T518|Ime)dQps1L{@+>GqAf}&w>KHU+3+|%AWzaMseiYa6_YXo&tK_O_^%cV2|}iXYT*2b|`RS{Spc 369 | zjL*i%oF(|UntzbUNVp{DjzM2$W1HmY5MPMR`JHPrQMGIzT-EG=*;}$hK?3LBbP&Q7JTI@fB27nhnY}6?!V7A;^3Vvvg;<1n-F 372 | zf14WY<611YZBNtlkPe@3o*_ggw3*+xp4SOLFR@MoLCWM;R#%_qgo;{~?Tv3fY|NuE 373 | z{s7eD%*^AJTW?2~X7s#iYHE*HPH5xxeZpiNZDD4A9spn$+rA90`Z$0rt?%~0_J!2L 374 | z0}uyly&L33LsEVcG^*Vju@`ks)ZuiM473;dd0xgirg~HP5A!A=7%C$HnIR|MMUjhw 375 | zMihBj&i614o)^m#9JOzPUp+{+nXT^&A}j)!l3N{a4pL88l#JWzhtkZV6UxH;xT!7d 376 | z5Xtr$e47J;KQZxMXTpJVY~|h3)3L@vBi-3++V$HlxrveTlfWw}jtuB3I;16sgj&L- 377 | zxUh*j>))6Z#q5S2h*h=i>6;83RqfwAkOe775ujU8s~~_6f5i4*WSz 378 | znbrA{tjQm~vk(g^VTfd%;;Jr3HdG(II;XA$KEmaGb5?(J*cP+8cdq@9Jf8{JN|bTo&=aTAHf-D%@jF 380 | z9Ehh9T{?7{J|wJECLd}+XQB+Xtx5)q8qAWZSJq#8Bj?hp4@T{@V7S2F5b(B~s>|+K 381 | z8#|Fos2*T#X~VIH+Ssrzd?gTn?p@R1qrF>m(%{&;!?xg_PN!!#yt>xr? 383 | zEpHJjT-w0;OGhfVeXpH|+K;nVKaq-NK+EGwJFEKt^JA%)2KUvBx$IGp&14GaDR=p} 384 | zD$Ak}%@J_6@`uvj-?Wd$4i!%#`8|XU?s_LP8AT2ptp0ZKNzX9KK$5jB>^qbTd^EQo 385 | z9;ij)x@(*M91oRA<#hGFtZ}uDvKelm)_vQck3oqGT86{~e!$)q`cIh^N5!Kn=-vVc 386 | z(t?gobJ=M5mLxk~rw#A~Jfx*fr#T=sl);SOd{52@Qr17)$%87NVQ8h0*9L9FLIiZba~V&|Qf51^Aer$p>y 388 | zT$G~uoYAJNcuIx>Y*XO!>(z9)Bfy<(nC}?f!ZR}J*tcMi5oDF`nzc%e85nem@J5#f^1}zpA>e`r#iIA5Z%kMC;bJH}|c< 390 | z*p+MY@%7k`qmhFO 393 | z9ZLNw8Fb7U!3KnmC5P1R!&oi26Wl3?J0e7`5#O211_IJ01rHhPa{i!9pW&tOsNtv6 394 | ze)UtAdsX?WH6|IVLcho+8!KB3J*ZdC=F`#Fb)}FuwSpXz0;9_}W*nWKoIUNhmMqS`{lXg~3uex> 397 | zJdlCeE6cp1SIGTPbssCM$B60yBQ2&=CLEZyt4faNz;2db#HaohhgV#Sx 403 | z#5UPPLZda{d*C>wQ6_BfT<9mkjRc<^&1$C&&e(kuet-SL>z(y8LR_RCu&1w|VDdR2 404 | zBN>A3HCaj}%F0}DA7PXzoR6FrD=9Ec;J1_-WU}B0xU8&ytzZ`PA-wFhN@ 405 | zx<>H7g^ulzyO_7R-8A2I`VmMDFnjW(4EVkdwu9v7rNQB)kXMU<+4h=FiifU`&&K%UOyWjo_X6y6Cgg*tcyqJi 409 | zP#l*4Q_)dquQu(mTUr8{6UX2*JycfNz)z%cp9c>h;UP36(JE85$x7Ibe;yr96-(7y2)wSAIE5Fw`=sc0+V^);1?9*2O1W6UWlIni0 414 | zhC@V2tq!8j>3%(-9<=M~$BJ7E6i{>V7~ZMwxp*K!+zQo-nk@Wqw|DaeBfByAbfq=g&NJK) 416 | zQjb>3e9g-!0Xr)vherx%>8cva$SLu%RxI4pKw!sz)&Mi4C0e}0F5Ru9Oe0NcZVj*@ 417 | zLS-sWvUYuX8#Zni9@t3m^qx*kTE8(1ES8kYX454!bsw-5#}^}lI@JGgzV+ut!;lqd|H{*-Y;aV~>3Ku)vTpW{ur~7n 420 | z@*`F7mr>=}P~@%OuUb7lKUXtExIfK-3R93@?du+O;jSIqDD7$zVPvkKVg~hFfd05L 421 | zhu%ZY9(^qc>Ia?$^?ZJ>D=g~`3a-NqwNhM#b(!VHzBrdP`aZ=1FJ~QCET?d3nLczZ 422 | zbmTxUhGBcGljg&u^mi0*8%nmZM)QaV6!{hhdNB1yi0oN>3%>;mKrO@Xk%x`pe$j3MPu@+5?>a&2?L 424 | z6o(X-(Q9V!$#p&U{`10mY^wp^S3kHNwLsZ;iW_`km5;k<}0rh3S_*ZNld1K 425 | z%uF-j8XA%(gH1-JX(Cd0l%?fe5BV#YP1u9mR{4VpnWQ-3W}`Xxpr5RV(g?jVGGn)T 426 | zFlH9)PeZ4CMQg&lQ~YBUnu^|-MV>8aJc=6<*^<^2HYD67D$-W+v@FWh)_~(a 427 | zP+}P&A?iC}^>y@IlvZdCQC0S=utnkIGtTG_*K!SGfyMp_Du2;i_KDN= 428 | zGLd_rQjyX$D1t!pu@A4@1+ulKX=*%3h){7aRls5x<5^ 430 | zC$qMPh`)>wt|+@hFeXI_A`7#A~tNq+J9kVWS>zzzxPn<@dL 431 | z!t6;aiVX<=?&-oQNKW@T^#lIGtpCb6q}_MH6Lg4U?!HVTkh?L2`cPuc7DWUEH5C+v 432 | zvHE>MHaU}IZn$*KSw2{EU3-&{!OeLE$_E0Nw;~#U(IK}jqzQRcY56`vJ(0JY!r#f^ 433 | zloE9#uKk7_xDF69Zh>zlf8`i;*|p>eaLezu+DAOPOVd#R=I@4_ZTj|5-PEFvdnH+y 434 | zl*%08@>}1$zv2ChMfeXykdtk6>j(${APx!u!2ACXBCs%a|35JVpLz#*)BXMZ!~TEJ 435 | z#Tc-e6cCu==%)q2OmPg;V+5yT!JPmY$S~5Ql&8OT!K*>G-gT*VQFCur 436 | zYX_G2^xR&qoD}%j(A|Hvk@wzc8BgPGxu2n?s&3NUS8jN6dp5nNCM&4P+9KO!wC{J? 437 | z9A=Nccy2O#y>MwC*1h@{KhF4~PKaV~-P$ViK5HKzvByrvj}^OXJlsBob9UNo$XInU 438 | zy>E+Z<3a;`jYdCwVqNhHWARMZk)D&qZiWPA&n|mjGk`7EM(Xh{pfn$>Z;CduXmy=l 439 | zZdSF`a9s}PlXCYW@<3x^`%Zo9t9QBt#ORFr#)4D^!j9MMy;WT=Z__9sn58yGyldUJ 440 | zNI0lgI6S~q2ilm9%#Myz7|eAyi?hf2^ZOvr>~QrNdEll`Z25WPXX+9~hd3-gcN{vxKfu8nj#Sp)v}s1plY 442 | zQ@j6AC({2nc*RUWsFd{|2IXId_`f8~zh(Z#kvRRIl0Y8+|2vrS=g)rvk^e6M4%j~) 443 | znA_P|{SW+x>>uy{Hxc~5DgUqeXh1!^X+Qt~Yajpsc>i<0y`zb}fujkHk)4f=g)@zf 444 | zlNqg(fsMVj$-l$-m+-&rzi>S+N;*zU^zh!KeFyPOGJMoogIZkI(iG57A+z&$sTNi1 445 | zW?IS#+7hM76LF`+90WM{N#MuHJ6l^kCUsLK%O#hj8^l3SFmFoLo+0xXj{voejjww! 446 | zPpDo72;O1x87q>W@}(gtoMf8R5MC6?gP5sv{hFOJeuj6%K4=P>)$r0tb1}*RlI~~0 447 | zP$+|O)kH?HF$$C1B%}Tn3>68YcIJq|^+e_;{TTzQ>HyPS%)l!j123Zl;%KA;nVnxy 448 | zI4TGalnG+Mm;@~vpyj)6Dp|8yBzRF8Fr>&{lBpiZ%q?|^n=de-IQJmbO@9?r}%=CxQ3V5Opszivd_Kee%W^ldzg@ 451 | zXN1e4vo3;F1=$$A6!Qefq@W9`F3;pIHPNfbQ2@H+OzVu;MM@qMwPe&$O4(xcGEDCJLL<3X=J3}W0Bjn 454 | z?Wf2i^B*gDh8#UbXJRw4^T2Q=*6}O`4~Fq>_|U?AJ~C0UVBGCtP% 455 | z-;!-q6^AEZUk@R=_?)?%_dv|93)$z{q>;3U` 456 | zd)_}xU*Z 458 | zhUf(_xr5We-$W3WV#p!pC*8nT6Ut5Y%FZb7I`stWIXDD^_IFR2^oCFXpzU}m)X{nH(V#GLYM 460 | zOUJ~)<^$n<7cA!ZRXMAEDU;@yeE07~?LW0lLSL?3^bcic2?hXw_Mh1PXC41HLbFje 461 | zu-jsV|Hl4~tU@A&(cg&O2a{L@q3=e9g17F+)W_R2%pWcq5g}AbY&>@hvq(A@6H&4<(J*hzDi8p#HX5BZ;1Pb9H0Y0wEpqMYlSmSui$TAfF%PG{}ZR*(*G0_mMF_s?m@gE#zf+vjneGh0e`MrLiTg|C}u 466 | zLP7OJ?69W!;gAjVMEgrN4$e~0f(WnfTD>63I#sMDfP#kyEf#r{E 467 | zYFdgv6!Gb-%HiXHNCDta&}?yq1;~0v!s4QMd%zabUTOcx3y*{(t)ZfKz{Ry{RPE 471 | zD-dDl_QU~$`Om%9z;vX@!OFY~7bd6#M4wJlMHLnhd*T`#BR7LDDK;E3^sMYpj}a&Ez4=_@w_qb77q4>C&$qL 473 | zoa$u0?ahuIwNNCHAhp%y*mr~bBf!#&h6o4&7Ax!tZ5GQ<=+?oQ%Dv2!ZXg5#Ki`P^ 474 | z&ORz8qX0c{zfiW7?(TE#9-HG2cj+kp#K^vk{^sH!@YMe3^78K15}s=>Zjep>em{7) 475 | zivG3GQq;{A!!R4NPm#a*{+xUD^B%v+X}SqbRTq-!tDpBXf8At8PS8)zD_oM2)}$!B 476 | zpvV5$ai@+~J8W#Z>9Dh&V!aWz8ZT_-a$gxUwt#%eBocl7RIqLSJGWraVJ{fKHl)pq 477 | z6>X$hagRKb#_GwIOUT_ZmvE=5%{xQ_(LTXg_&c*N2SyKLhg!lRsr7s?CK?XSuCk?y 478 | zX(N?-LY|NJ`XVP^qiF9M>=Zvs)<;!vX%P6jGux8zh3J*bkL&4YZkucN`(J9_vwIln 479 | z?>}TIA<}=Xc}EjN1E>Gaa+afRWw#}Y`c3^4sU=cVQjOcni@enyyT2}wb!`h|9D^D) 480 | z+#q%&Zz$u~&NM`BdG)*NWEM(DLv?NlI(~B4d)pDWkYr~XRh^xo@2Sppo6mQdx-L2qI$s}sZu7YRD)E|aEx7`)Zrm|dz 485 | z<#ZUY81hPNO*KoKz2VGi%frG-YFKXiiP~V20Q@NIRZ^^Q@?u><7I#?5_X%9))DCjm 486 | zk`^4`v3fSt7K0!7#A(4%%@q$ILZ#Ex#@KxU7?o$jx#>Ti`t>TFOfiLbSHUf!)Vox? 487 | z=Wwwf3aN=H{lb2C#d2iKaqxkR7aVS~ADX46t%1f&Sks>g(=pb&JM^B@rRwF}Q2{N< 488 | zzP=u=4nl5j3ULAaVb0J&B>14j^)2FMwNzV}IpS1?ttlKu%u$8ts{_bo1#qe5M*ZjK 489 | zSLfsxICx@Tx0jU2-Shm|N>W4z5Di{2!2zjBiQ)-t{3Q4IZg2rWtSS#)iWDe={6EQO 490 | zDFUo#Gj!Qfa^as!s}2R)Hzx2A*+(5!P(iP>|GWx8tseE&BfQZ+V!}PCi&O}g>01=!A 492 | zk!@uUo=Lf0&@bNOl$3F8qVJW7%SnnZh^}DUpiZr;kE6QH<5e*J)n*!cUXZnKZFrkM 494 | zGP~nQp6A&SXl`Z0d3i@$-MRXfUKy^GAgq2mULfvy^&_c1)eqidoEewA$RtfcPFXhVH)qoZGJD^WG?t_*kle5 500 | zFMjXGUeWAk>-2hi!wYZZUAaOKDzsMx$xLs#bs?GaLWCikjTUA8mYy1Qr%oh;0)MHK 501 | zD~h(!m(sfDd)o6w5qe76O{(9>!s-e{qpkl##X!l2!M@rS$zd2>P=MhLqW|B!`MT5%Erz;wir>mo@*X_ymXs1v0 507 | znTT;Uv|#5u5?TzlYJ~QuPp&pZOaHu8*{a{mow@MbS<)PYOZ+hQd~jT3`8;J;zxa-i 508 | zJf9@Yy6xu;Vt;aOy>l{=#+y^seBjP)+JRO;Pi;nmB>2z$KOIa{X*-24NVp*~Tr0<@ 509 | z17W84{m@wYRU9yP*-E(D#EZz{r_&y*l9Zwd3sA&D4$tq=)?DIb@v;C}iy+82peR=_ 510 | zG3M;Qn#wx-Zf#{YHtYjw-iih~SHX1psnSn$un7IJGeDk?Gwx=u4%=RTu)vFJxi 511 | zxxj)X4QPq-+?7RqC8>^RW7d5$_3yww!8V%H- 512 | zCp$cxkzp9F)Cw7|3lP5bIu9@i+}JTffKhZkus>9nLK#2Gn!!UIbI}Gr3L#MbT)lek 513 | zDI`rC(e(j7#Tucq_MH3{S(P4u17&~d%x~xxZp=3EFz1wg#SqwhmY~H^nB~}gdi+0? 514 | zoOe8x?;FRDl^M#;p4lVcZ0g9~dzF=W%!5!wSvknap2v(t$IPb4IzmNC9FdH&DP;eS 515 | zuf*{yU!B+MdHL&k-`9OT_j5g;>%K1c4y0JQ63hm@ltFMqaJ{BIDQtb`>~o>0uGGk^ 516 | zGJ$I&c$YjU%Y#*$E0+ZI`T9?DbV+5xgQLfW0n+0c&gGKu#X7JANUtVzJm2P-hPz)S 517 | z)qsQ2m)04I;Uyvtn9S`WC${`vx49fw?Ecwsf{WZQ5uxL1@0Z2D--@c-LIVWLNbRJr 518 | zMVwdWMi_BheII==s!aD2Cg6QnEjPaM=xIZ6SNIG~s+@_oFOfcp>rh2c<{L7j 522 | zQazpWF?eMb=+W~};-jJJ!MT)?J$SDm=S(D{|v 524 | z&@HQ6f`Q58=p@x_`VaJ#T8#m(5(uQ?8C!USIb24Fjl0UAC3iE;#B^>K$hTlHIXaT@ 525 | zF+=Ng8=3^L{4h!$&6CKY`B58kaB3OHbF?88$S4~YXXu0mQV2Pt(=Jgx2a-l%;hGjH 526 | zJUG*Z9_$-6Md03~v0hGGrdIYu2ve>1G#Xf>aiVKEB@F&{ctziQVcX8z^H%K^cSSkAW<7hyZ`Y^rOvcRcQ5Th5wKBe7ajk`m!KWCqN+Q<=IbT58#>`$M 528 | z`Z%+~B)IgI-Lxds#GaStnMPRF{JEr!Fkfi=IE2i6*dzoNzT?Pt3#G-V={R_;V(V%iknhxEP-Qef 535 | zr=JE*Es7dT(6#T(eUp%&)gB6gV%r+T|3lAuj%Qc^8BG6zKz0z!xxsjh%nzW(GAT3YS4*#5r9f}>z4 539 | zt{jW*y_pQ-P^z(>|983${HM96W30#YN{j~Ajf?bf)IqPQ%GOhw-iqfDfv4|D-Mmx0 540 | z(2XE?oMH-03u8C~>?InJqXy$o_GXsR6>%>RsFhaA&t|_XB1@qR* 542 | z6vZsKw$)-`QJJBc?bMdKPH|_|lOXNwTyOw`b@Jz8Hp;9={tIR05{d6QxoCQxkxJ*N 543 | ztbs*xhU*My1JWrudK-wq`X9SyJYhu`FvGg>yKZfyT(A<13-R1#ao!Q?q_Qi+IQ6hd 544 | zBM(_-MEm!}W^x4ztl^um+)rzQmooFVB6_q7pM;*P-g*-dB(z+C4h4ObVApQz?5~y* 545 | zBHQ-+e2ePS<5Iae;Q1Ng<{;`M~N$AADiCCztb0${C@LN 551 | z2l@pKkoZLt%>kx%0hXzaO9Qn}C&qjcRes(<+pFw-ay5Q3oQyj!+k{wUK>9aCT!fQn 552 | z5!r+O_Nq1ML8M|WgMG##MBCV?pF$eetD;-TZNLcR=aBkW2t#gt^O%Fd4!6MOH!N&V 553 | zuw~{GDWLv>pE$^7lZy`2ht&g#!)0N4OXe=G=9RA=?i25m$pP7k&^9V5X{g<3@D3Zy 554 | z9e-_ULvHOi2}mxC#ApnmiK=bZ%=9NxLZpLl=rvMR#4V;)2CCKB=njxw_Vy2V5s@uO 555 | zyOQ4gg4}wRhiD|_(NM@o 556 | zC-!q1=a0|{HlFJGW?{%GJ+xdOrgOQm;vuxA=laSpb6mS=G;}g4BQ#{{8C%~#;>DFh 557 | zcMo?q+HB2^P%r7Hw}%{tO@*AhR9<%7{!vg^Y?MTu{Aj}yw6+Y^=ShF;{LR)Efa=lQ 558 | zffl0-WHDj4eny7kQLww)NqiZuGjC5SK)iK(jjOdjK>LfacZ+%pNsfj@6qDUzz@x6Y 559 | zZWHM;c(P@!J|lmV;E7;|SWoj%i-$ 560 | z8{kM2;YfaVDj;^$!|7h%kk`vV^Losx*jBUClXe@uE~^h{XJ((Kv(pXt892 561 | zMoQezFv?JEfYDhQ1dLW$H?^9>y;ps)n^G8TcfQ`?0ZU+5tkBiQ>uB%Gv0m|P^a&F(eDHL% 564 | z$RwHD-2(njPi}Sw=B`ue>X=fetirMerLv4k)cjK0ZEK`WeZPVdBGTw`fy%-DD@UhC 565 | zVBV~4igTGN!1+G2eu&jKk4i!%*cS^HsJe&tbNS$gK#3q=wO`6V#TIF8-dc%UO!GvW 566 | zZ!9>S5#&Y$KqnF)J;tYpP;ygYdiUOshVOB3s 569 | ztxJYpqbR*6sisu;B<4dAm*|I&J@JNhT#2S@yIAZN#%RG{t 570 | zmV;YO8pZ|&BVh{0GiKWXqmzZ{%4C(|SJ-nC$7e>9&KPiDBmYaq(MKl~+oHmG5cd*Y 571 | zEww=ow-?APZf%KgNaumat7d1^!JepAg$U*QpVcpCpN>csBWrBM)bNY$p 572 | zQ|OA#9XfjfbKobiX!nsadGeU#g&wuxdwIlS4|$a#!j9Bc 575 | z=R)wC2EfmKLPn+dKGF!0c$d0;pjq-tNN2kDL*-T=vMlVWm4&5iTvS9$q?@gSfuY!y 576 | zYaPqM2_%0MslhWWG#L55%loP=!UQqNYVhPOEnUfmaQUW}*Q5C9sjHF&N2>7#^p@|9 577 | zkn#CC^1??(ligyel_ig>mG!YzRh~{DULZdK*6aE_oGRXTYJIJPIkQBBdc*|;A58)c 578 | z8@lX43FXNcMqgJY6OueK7-CEwA|@hRBh6hKb2AEg-Nq96A+F{0)7hm}jq7w7!c*i5 579 | zupzeP<~I_kPhU6;oBk5gG{v9hA@?XOTtVO)AZMrJs-*Z`FO@r0RW;d_c-r~8x#4_| 580 | z%k>mg;oO?+@OExJg_oU*x*F<=JZiih-0bkqSK9jQaDBj6Kt~=EsOK~KzX5me_{F2u 581 | zUo51Hc73x~zWzgvc6SQDzkS!k|K4tEWo~&0a_j+)M{U5{huko?`5Oio$Ssco>Brp* 582 | z6Uk7J1<)3<;-BZQ1n>BSa5&1jd0xDy|%g} 584 | zyzDdm{k9eySs(*8Jo^lF=ibZVgfqO3Aoo`LIib1&uy6zA1r8u78Rm!fz?#RI=$ST? 585 | z?aB_y-&)b|+(v|#(c3g<&_BCKlBH#QFzRlNY)_rJOA)uyF2N9e*WIw+M1boIp49d9 586 | z&6Q#`5z6o&l2RiyX4&;`FRI(yRvM?O@6FF0QbfBR!}bIG5JOH^9C_jHPkGnP$8|Aa 587 | zH8!5P%a4Qkyi$lnCQE~mrR~Qe$;=%@oU*gtx#V~W9_~{$d01&I4<-DnS34l;?_yJjSX 589 | zGg=VYVuewY(x{jBs>6w&eZh1%@`1ku 590 | z2WyM;yYkc9nR3^a40%VOc@p6~lqBxwCRBs3`y}YP)FZM6o#qO!zj(I0{op{~V+{wb$>d@=(+olXObUKVbovTWTbjj|I7_bH0{d;iA 592 | zD%Rd9t4BI9G?f2LJB=)R@BMaZ(uZif@f8_7jzb=_N}-;&x{pKyu=s@>MgD!0%Snwu 593 | z?bkYNL^uC|m;RmC-x=|5_ttTa`2CYY^=;LK0~>zEo|7R_yNdRvg_Ib8a=)FU$lYln 594 | z-_HN$9*xweP?mw8AmCT$5R&zq=AOMJbr2m<`-To|-12Mu#X6>o=wySSHftOnqy*V< 595 | zwb$wd{!Qz~$>^x<5{K#G!oTSM-#l?LIBH$?Fx(FrU4NR6gdeHWo(zt 596 | zn=Rdw!BKbi4#U@E_u)tO`A!B#&9@JO>y-Avf5^U1HUMh+d3XRqmBRy|B%>!oqe}h5 597 | z=qFtJ 3 | Date: Tue, 14 Apr 2015 10:00:05 +0100 4 | Subject: [PATCH 2/2] Fix Git-LFS filepath sharding 5 | 6 | See also: 7 | 8 | https://github.com/github/git-lfs/pull/212 9 | --- 10 | .../src/main/scala/com/madgag/git/bfg/cleaner/LfsBlobConverter.scala | 2 +- 11 | 1 file changed, 1 insertion(+), 1 deletion(-) 12 | 13 | diff --git a/bfg-library/src/main/scala/com/madgag/git/bfg/cleaner/LfsBlobConverter.scala b/bfg-library/src/main/scala/com/madgag/git/bfg/cleaner/LfsBlobConverter.scala 14 | index ce07008..e99496d 100644 15 | --- a/bfg-library/src/main/scala/com/madgag/git/bfg/cleaner/LfsBlobConverter.scala 16 | +++ b/bfg-library/src/main/scala/com/madgag/git/bfg/cleaner/LfsBlobConverter.scala 17 | @@ -75,7 +75,7 @@ trait LfsBlobConverter extends TreeBlobModifier { 18 | 19 | val shaHex = encodeHexString(digest.digest()) 20 | 21 | - val lfsPath = lfsObjectsDir / shaHex 22 | + val lfsPath = lfsObjectsDir / shaHex.substring(0, 2) / shaHex.substring(2, 4) / shaHex 23 | 24 | val ensureLfsFile = Try(if (!lfsPath.exists) tmpFile moveTo lfsPath).recover { 25 | case _ => lfsPath.size.contains(loader.getSize) 26 | -- 27 | 2.1.0 28 | 29 | -------------------------------------------------------------------------------- /test/resources/samples/bfg/pull87/github.patch: -------------------------------------------------------------------------------- 1 | From 72428763eee19a2d83cc05a0ae4d55ab76930762 Mon Sep 17 00:00:00 2001 2 | From: Roberto Tyley 3 | Date: Thu, 9 Apr 2015 12:08:55 +0200 4 | Subject: [PATCH 1/2] WIP: Support converting repos to Git LFS 5 | 6 | Git LFS allows uses to commit new files to the LFS store, but replacing 7 | _old_ files requires rewriting history, which is something the BFG is 8 | pretty good at. This rough cut allows replacing blobs with pointer files 9 | throughout repo history. 10 | 11 | Some caveats with this initial implementation: 12 | 13 | * the BFG cleans concurrently, files may unnecessarily be hashed more than once 14 | * the working directory isn't updated 15 | * specifying `-fi *.png` should be unnecessary, should use gitattributes 16 | * need for `--no-blob-protection` is a hangover from normal BFG behaviour 17 | 18 | Example invocation: 19 | 20 | ``` 21 | $ git clone https://github.com/guardian/membership-frontend.git 22 | $ cd membership-frontend 23 | $ java -jar bfg.jar --convert-to-git-lfs -fi *.png --no-blob-protection 24 | ... 25 | $ ls .git/lfs/objects/ | head -2 26 | 0145f7c304ef33a43cc946e0a57b2213d24dcaf8462f3d3b332407a8b258369c 27 | 07010d5ddea536da56ebdbbb28386921c94abd476046a245b35cd47e8eb6e426 28 | $ git reset --hard 29 | $ cat frontend/assets/images/favicons/152x152.png 30 | version https://git-lfs.github.com/spec/v1 31 | oid sha256:0145f7c304ef33a43cc946e0a57b2213d24dcaf8462f3d3b332407a8b258369c 32 | size 1935 33 | $ 34 | ``` 35 | 36 | https://git-lfs.github.com/ 37 | https://github.com/github/git-lfs/blob/5eb9bb01/docs/spec.md#the-pointer 38 | --- 39 | .../madgag/git/bfg/cleaner/LfsBlobConverter.scala | 88 +++++++++++++++++++++ 40 | .../scala/com/madgag/git/bfg/cli/CLIConfig.scala | 16 +++- 41 | .../sample-repos/repoWithBigBlobs.git.zip | Bin 0 -> 24252 bytes 42 | .../scala/com/madgag/git/bfg/cli/MainSpec.scala | 16 +++- 43 | 4 files changed, 117 insertions(+), 3 deletions(-) 44 | create mode 100644 bfg-library/src/main/scala/com/madgag/git/bfg/cleaner/LfsBlobConverter.scala 45 | create mode 100644 bfg/src/test/resources/sample-repos/repoWithBigBlobs.git.zip 46 | 47 | diff --git a/bfg-library/src/main/scala/com/madgag/git/bfg/cleaner/LfsBlobConverter.scala b/bfg-library/src/main/scala/com/madgag/git/bfg/cleaner/LfsBlobConverter.scala 48 | new file mode 100644 49 | index 0000000..ce07008 50 | --- /dev/null 51 | +++ b/bfg-library/src/main/scala/com/madgag/git/bfg/cleaner/LfsBlobConverter.scala 52 | @@ -0,0 +1,88 @@ 53 | +/* 54 | + * Copyright (c) 2015 Roberto Tyley 55 | + * 56 | + * This file is part of 'BFG Repo-Cleaner' - a tool for removing large 57 | + * or troublesome blobs from Git repositories. 58 | + * 59 | + * BFG Repo-Cleaner is free software: you can redistribute it and/or modify 60 | + * it under the terms of the GNU General Public License as published by 61 | + * the Free Software Foundation, either version 3 of the License, or 62 | + * (at your option) any later version. 63 | + * 64 | + * BFG Repo-Cleaner is distributed in the hope that it will be useful, 65 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of 66 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 67 | + * GNU General Public License for more details. 68 | + * 69 | + * You should have received a copy of the GNU General Public License 70 | + * along with this program. If not, see http://www.gnu.org/licenses/ . 71 | + */ 72 | + 73 | +package com.madgag.git.bfg.cleaner 74 | + 75 | +import java.nio.charset.Charset 76 | +import java.security.{DigestInputStream, MessageDigest} 77 | + 78 | +import com.google.common.io.ByteStreams 79 | +import com.madgag.git.ThreadLocalObjectDatabaseResources 80 | +import com.madgag.git.bfg.model.{FileName, TreeBlobEntry} 81 | +import org.apache.commons.codec.binary.Hex.encodeHexString 82 | +import org.eclipse.jgit.lib.Constants.OBJ_BLOB 83 | +import org.eclipse.jgit.lib.ObjectLoader 84 | + 85 | +import scala.util.Try 86 | +import scalax.file.Path 87 | +import scalax.file.Path.createTempFile 88 | +import scalax.io.Resource 89 | + 90 | +trait LfsBlobConverter extends TreeBlobModifier { 91 | + 92 | + val threadLocalObjectDBResources: ThreadLocalObjectDatabaseResources 93 | + 94 | + val lfsSuitableFiles: (FileName => Boolean) 95 | + 96 | + val charset = Charset.forName("UTF-8") 97 | + 98 | + val lfsObjectsDir: Path 99 | + 100 | + override def fix(entry: TreeBlobEntry) = { 101 | + val oid = (for { 102 | + _ <- Some(entry.filename) filter lfsSuitableFiles 103 | + loader = threadLocalObjectDBResources.reader().open(entry.objectId) 104 | + (shaHex, lfsPath) <- buildLfsFileFrom(loader) 105 | + } yield { 106 | + val pointer = 107 | + s"""|version https://git-lfs.github.com/spec/v1 108 | + |oid sha256:$shaHex 109 | + |size ${loader.getSize} 110 | + |""".stripMargin 111 | + 112 | + threadLocalObjectDBResources.inserter().insert(OBJ_BLOB, pointer.getBytes(charset)) 113 | + }).getOrElse(entry.objectId) 114 | + 115 | + (entry.mode, oid) 116 | + } 117 | + 118 | + def buildLfsFileFrom(loader: ObjectLoader): Option[(String, Path)] = { 119 | + val tmpFile = createTempFile() 120 | + 121 | + val digest = MessageDigest.getInstance("SHA-256") 122 | + 123 | + for { 124 | + inStream <- Resource.fromInputStream(new DigestInputStream(loader.openStream(), digest)) 125 | + outStream <- tmpFile.outputStream() 126 | + } ByteStreams.copy(inStream, outStream) 127 | + 128 | + val shaHex = encodeHexString(digest.digest()) 129 | + 130 | + val lfsPath = lfsObjectsDir / shaHex 131 | + 132 | + val ensureLfsFile = Try(if (!lfsPath.exists) tmpFile moveTo lfsPath).recover { 133 | + case _ => lfsPath.size.contains(loader.getSize) 134 | + } 135 | + 136 | + Try(tmpFile.delete(force = true)) 137 | + 138 | + for (_ <- ensureLfsFile.toOption) yield shaHex -> lfsPath 139 | + } 140 | +} 141 | \ No newline at end of file 142 | diff --git a/bfg/src/main/scala/com/madgag/git/bfg/cli/CLIConfig.scala b/bfg/src/main/scala/com/madgag/git/bfg/cli/CLIConfig.scala 143 | index abbdb08..8976a15 100644 144 | --- a/bfg/src/main/scala/com/madgag/git/bfg/cli/CLIConfig.scala 145 | +++ b/bfg/src/main/scala/com/madgag/git/bfg/cli/CLIConfig.scala 146 | @@ -74,6 +74,9 @@ object CLIConfig { 147 | fileMatcher("delete-folders").text("delete folders with the specified names (eg '.svn', '*-tmp' - matches on folder name, not path within repo)").action { 148 | (v, c) => c.copy(deleteFolders = Some(v)) 149 | } 150 | + opt[Unit]("convert-to-git-lfs").text("experimental support for Git LFS, use with '-fi' to specify files").hidden().action { 151 | + (_, c) => c.copy(lfsConversion = true) 152 | + } 153 | opt[File]("replace-text").abbr("rt").valueName("").text("filter content of files, replacing matched text. Match expressions should be listed in the file, one expression per line - " + 154 | "by default, each expression is treated as a literal, but 'regex:' & 'glob:' prefixes are supported, with '==>' to specify a replacement " + 155 | "string other than the default of '***REMOVED***'.").action { 156 | @@ -129,6 +132,7 @@ case class CLIConfig(stripBiggestBlobs: Option[Int] = None, 157 | filterSizeThreshold: Int = BlobTextModifier.DefaultSizeThreshold, 158 | textReplacementExpressions: Traversable[String] = List.empty, 159 | stripBlobsWithIds: Option[Set[ObjectId]] = None, 160 | + lfsConversion: Boolean = false, 161 | strictObjectChecking: Boolean = false, 162 | sensitiveData: Option[Boolean] = None, 163 | massiveNonFileObjects: Option[Int] = None, 164 | @@ -172,6 +176,16 @@ case class CLIConfig(stripBiggestBlobs: Option[Int] = None, 165 | } 166 | } 167 | 168 | + lazy val lfsBlobConverter: Option[LfsBlobConverter] = if (lfsConversion) Some { 169 | + new LfsBlobConverter { 170 | + val lfsObjectsDir = repo.getDirectory / "lfs" / "objects" 171 | + 172 | + val lfsSuitableFiles = filterContentPredicate 173 | + 174 | + val threadLocalObjectDBResources = repo.getObjectDatabase.threadLocalResources 175 | + } 176 | + } else None 177 | + 178 | lazy val privateDataRemoval = sensitiveData.getOrElse(Seq(fileDeletion, folderDeletion, blobTextModifier).flatten.nonEmpty) 179 | 180 | lazy val objectIdSubstitutor = if (privateDataRemoval) ObjectIdSubstitutor.OldIdsPrivate else ObjectIdSubstitutor.OldIdsPublic 181 | @@ -209,7 +223,7 @@ case class CLIConfig(stripBiggestBlobs: Option[Int] = None, 182 | } 183 | } 184 | 185 | - Seq(blobsByIdRemover, blobRemover, fileDeletion, blobTextModifier).flatten 186 | + Seq(blobsByIdRemover, blobRemover, fileDeletion, blobTextModifier, lfsBlobConverter).flatten 187 | } 188 | 189 | lazy val definesNoWork = treeBlobCleaners.isEmpty && folderDeletion.isEmpty && treeEntryListCleaners.isEmpty 190 | diff --git a/bfg/src/test/resources/sample-repos/repoWithBigBlobs.git.zip b/bfg/src/test/resources/sample-repos/repoWithBigBlobs.git.zip 191 | new file mode 100644 192 | index 0000000000000000000000000000000000000000..5648182cb2b5f93d2275d445a370f64fe82b3cf9 193 | GIT binary patch 194 | literal 24252 195 | zcmbrl1FSIJ(&xL4XWO=I+qP}nwr$(CZQI_@wr%hGo^vO2CSNi$-_7l$R=Sg3|E@}^ 196 | z*6L2zPhJWbWM0Q0{n*c%vG{ar9*Ul(0{}Sq$N2uM%)hi~jEzh=Oz91n 198 | z4UG(pn2b40ObwV>4Oto484WlXnAnXN=#34T4UJ8WX#aKh|H=QDOd)#`=;j(Z&eoh* 199 | zo2}NQSR1aKVxuk9vDSaC91dn9oPODSRoZT9H-mUIT3lB!s|U)c2}!{L0DQmN>^sK$ 200 | zoFq$8tV7ckg5f-oHFksSO~#!EDCoO`(DvtOt_Qwb%wX36W_IWB7UMv$)GPyK0#2b(S}E 202 | zcW78TH~zk#<$z54;QXjO7|kEEKsfz4{hARZ4c}_BVOW3n-3n_C2)X4E!i;fV{mkA}dZju+z 204 | z;_-^j5)(k%BSUQp6E}rEQ!)0?)gON7$hX 206 | ze$5&f0e(Tyi{3oqG3(mFseE&meP=cXTV#I;&iY2O95S{?0DwXkzF 210 | z;N&a58LXw<)~$6blLG?-Vf>-IjKgcA;MS4c+;hvH@O4nEA(88Alu5w}q4_u*x4 214 | zM@xn7|7ku=6mV0`vHg6vx4gE^VAFpuU9*G+0 215 | z7|ygiAR-wU2kA@H_wXurP2TmAIR=CzrxPECtwU`RLeQVwmWHmelcigWg3Wp1fNw-h+}8)x0!3K+SzMorL4vxP 217 | zCpNG^vDX+IO*7hIfD4)iHb=C 218 | zHU1Opt!Mu`hPCMRmk{x2X%r 221 | zM#ie93z$A+UsLDa4+RP4A0Hc9^T7By@wB^=!=*$y{>roflh+M}!1q|I3> 224 | zHHb8W4<6iqL6JO%9maA828_caHxa6+|4v<`QI+2-3DU6L>=1KLI)N&LpU5^+wMQNl 225 | z67Q)c5L1a!!24KF%SqQJFWzjw6nHw6@baZFQaIiVd2RvQ*lHWMw_f7*M8Q~p(O+PT 226 | zi~Y`FF@0PzWHxl%h&j4y#F3B=eH!u5BS$?@0gMMkn4TrnQ86vO+nbRw4QTPXj?WhS 227 | zXAFXPY#PBYM2A-UZ#dJ%PEdN!!8FbEb1~3n#Lu)3s?J7d!OmABYV6;6uYTHeh^G{uJ3N)6&wcb}FgQo}62J1z$MWiMaJxRsM6uZ3Ha!K+$s=v6mkS7&&;G_YnpnpUejnIIpipA^>| 235 | zGSiP-Do-EqnD?nq;a&1vpy~-g-42!tP#!!5E7IWV64A!2M+ob`0_$WSEI$IWnw1*A 236 | z*D)&y79VS;pdB$_N+V}`XA^I05n@o*Nz;G53*wp&71ge8q(m<;55}}}+KTj0n*xnF 237 | z0Mb6+sL$~)T4J;{u1fsm^+KBu%ukJH1L4m`eboncb*foqR<|ow87_?iRvWyPkr4}Z 238 | z8wX=GrGX2hL4pAi&3}Y1EcMQS!Uhz;(;5WCO 240 | z0)tq1nYj%hn6}oTrM#=0{>|H^jUC^r!18&hz3~ibbuJ3-rFT`v<@Z&$;bp!z68u}|ONt55i$M9VW@)+TBe!#rgG6rMzHMpKp|(U*Oj2OAH1TB*kyWg(cwW;jwQjXlI7& 244 | zc1*qtF(vKItPUQ_J8MR)4wF)l>c&W^6kt7?K0!M)h!7I48cpPNar28`SOt~T3dT9j 245 | zsxe3LsYMHf<#}?WH}A@0(`-x!VDa4&fL0CbGwMap)ZELHeIMs~JN`&g7Bc4ROj%xD 246 | zK6|weQ8s3)mOw`_?blFAwENerLkwoud(go-J`2K?ld0+-VF 248 | zH4B*4Ops;+^i#wBd|U%N4%5npEMK2(>cvs}B+!ryPV0Cr$Vswqy*NRVwn_Ry_e79^ 249 | z%Jq{ZdS0Ro6Csu&r1xL$S;#%NyTd0Q0f>bknswdc%dk1Nt`Tautu7&iO%nZ!Zk7IS 250 | z+=)F(NGKS$;K0|q@ggbt@_=zMQT}z-ZUHE!*ix>@KV#od49m;Ki>=r06IjxwP7R~N 251 | zZWHgvjQaYKaWS`SRW7@-sO+!KpZTF#rxF#bYi_rKP-lVwv}6 254 | zEtCTSFaysj&9P||J*27eZCbo;ew+RsgugI}Iyj#8zhN6}V59a+J?|i7$i11_lhXo{ 255 | zHp$gcYrlV9QIW5_gLXLq-&f+__Z*r;8qZn>y9sawt_;iT%$!=2oHsSI`^Ht-4geoU 256 | zAGm7VR@tF8v_uZwUG^kyrtBD%)z6L;FF!RcCwkQarmp~X$QC+U~U!+HhMF7_86|8~FYMaqK^V4}d({yY;BG>5#Ws(E$ 258 | z-9Mh##Ncr|Gh=p4{AGTq?5O%X1F>g3B8LwgSbDb*ciKqqnubSPm@_kE^7;##L3~t4(S4yb+8cgRU+`DwkdM90nIxpYR^^LEMQsg 260 | zTjveYXe|Nb!ADw`x};DJFul>`<+3xE__~;DeaHk-y)G*rD|M3jFOrw|-S?e14_EU# 261 | z)@V{0sxaizx~`v+Uuam&+?kWHQjz{uMJA@)_l?ofEl{T^Hg`6=GOobaqIZw;lX9A< 262 | z#{L@v+U)h?&~*E-iCBYNMx-UJ@uSr@rm5bpRo?_Dmfu6;hb!P{3cRRbhr5HtA7gE% 263 | zYy=m%I}$rFmti#7Tl-=j8I#fslo>qeQ1Bx+K=H$x)V3tg7NTWuo~1>-7B(IJ&di>3sD5DDtl#T<7Z8hD;-P}=s3gErPl&36?Hy&VUvD`wD~M<)E#!FqwtoFM 270 | zhV*mYSQm3QZ?{TbOR9%msi|z7IEaDkczya2O}K6z@6+5J@!|M1HZ!ftxQ&-EVq!`X 271 | zX(N;>d<<%giu!NPi7p$KmI`g>C++gmSl3lnjCmk+;uKD|m+_N*saw#v%y%?zCijs8 272 | z1?nk8p3birtuRNoH)HikDO~E`f0}**<2)iPTY}>Cxi{{o#)Dp~#f=1%?bEK!y920? 273 | zRfjB_4kSE&lfuCf==(O3{GC8m2Me}m*DXaJmCspp$NSbt3>rqoC}jXCZef^)i4 274 | zn|k%)dxfif_r}HY);|!Ui3gt13)o~8pBOhR=|7`Fc2UzdH_5YZwgJJu;>+sBgiLVl 275 | zDPpR68YmgfQz45N_U5%zN3)qmJ;Jv%5Cd~g?=B;V+4m3c&Q6apOR`ta`&5dqc2V^Y 276 | znOrirNRG9L=0bB{Ncvu9@9)J`Cy2D6 278 | zw@#uN8nRZrm%@#4;Du%psUx_DH8w3rGPd1W= 279 | zLIWC#kOK2!;hb231mLFEv;z@-8W&{nPD*PhOK>8d~Pe0+WX?x#~&N^P> 283 | zn?R-}@%|>~TD~=o?7+7dt08>(6@P|5NwYv+-0Plu%x|lqgivzd_TVAX0S*(g@3jJ_ 284 | zgtX+w7CtsM26|gJdSu)80wT7)tdln0B?$5Xdu-oWX(&v1&C`emu^YiZ%0 286 | zFDy>cnyQ~qWH_-|1eb`tM+tGbxr0C)A|s52j&&N4 289 | zOm);V@2wLn7%hlAKf%Tv5%O2gb8o>-tE9tgt*b;Is*!p^6{RNFFy^|FbUa$1qaHimYG!R2FEts+Y&UdoGit 293 | z%gxpBdC;c8g5LTYgtWO8jbpKy>tQMxMJI0gLXj91Gk?|x2!0>zyWBEIZfo>PW5XjU2Li!5GmKNMb>)m;3{jf*n&j1>9e?JH~TK(R;2Z3~RY91++2(p`y-}Xfm{y6!AEJ88p@Exej{(N(hoI@7_F8b;)hJGgsk* 297 | zmt4)TwTqmNSl?^(cn~a 298 | zea%?aJe(=QNC`p)Fjp0)MXz%`Z_yW>brFNRhK$Y!y-Qwhyf_khws&V@iGp>$k_fQ~dUX==g 303 | zt)q`qwI9`DQ{#@tg}+GPG*O~^d*}+B>DVnzw@9b5#wsc*ad2?9W8qx&>e4m)+A5KV 304 | zbPjS-E2MC=x_FE*zJ9dZ5vzSd^X<3Ip!kCkV?!Dbj`HlOz

    &0U)m|$nqeNPhu&J 305 | z5|S9CZc}EqS;l_zxWueD3c^ti`FHX_QBhmZMbeHGxAG``!<3*>@u4}N$+l-VKd$zH 306 | zkFkY^As20reKqHhCf0_&oF2v{RMj7ZsCMKO;PnTmWIXr@2uDwQuMcWZuuPxvW=i}w 307 | zjBeuqK*B`Y$sX?T;ai3WZked|NWyf*$mYl|3?qPuiCDT*nL&8g`zZ;zzU5|a2;ZV; 308 | z7DkK*#vjE;Q2N<4xxyrmklstn4c%d$QF4wUuELI2^zAI6o|{W>!ey}zL7Ok=OK4hl 309 | zuX4&3N}B88r)v8puYSqXu(&(2sK%(TtlS)y5oswYTRfYZPD2*es6)3vD~Rga(r@Ch 310 | z%(taXA*M{;ktq3gA+n5lt}-;!RU^MJ%DpWIYBS8U)6?>8$wRe+o#|^Z4qw&aE}j*I 311 | z^Kyi6lC0_hWeAe*3L^qymU?XB*AePY_^dDohwYH78=jK0)HY>g)Ad`y8Ys78RD7;Q 312 | zw(w=&DSWR>rBSsIg55%AEX%Krkr@$B*B$qQ+vmQ>4}|tOTxn%;X$BgL>7MJukEUdq 313 | zi0O(HG`5Bvm&Y$MI=kH#m|T}r(6LJnOQ@=9p94f{xV~27@(GlYUc&$p*q@ZG9b41^ 314 | z?khvj2jCWhSTbUx>QI}51Ok*flYFox3vy;QPw3%$Lxt#kKxR%eWJStSSGJH%z2v3L 315 | z&swvwsRP5>Yeb9HU^r#YXTD?p9qJEUx{S=#gJPo;7~fI5pn>T|D^!bt;)sQtgzdPT 316 | zrIGRD5$ic3b@EGMP=v+z#1tq2JbS@6=nzKv=iURAa1CjBNVx>7RPXm)5V`g!ki7fR 317 | zPPiOdJ}p`*ktsbMJUuI~Yu|pq2dhzWG4sfoaU-xG37koev;(W~Q_C~U5zIaf8ZRcB 318 | z+{1#tPu5I#ZED@^Hh+{=iKy%&tzU0!RLOh-8rMBAJELhuYKVWtVnSXXL# 319 | zmNC8-`3k8=Voxoo!ihS0-&?dQshMQLd{M*H2K*EQQyoEHq3*ZE@WI|r-bQ)IM0qqD 320 | z4&U^3?Rni!!kUZ)u>#mg{}5`8%2%_OT3!{P9T_K6^7{%*`2pNmKX9FF%*Nc;Zp$Mb 321 | z^umrlv2+w3aCS8&7XCdOO_|2OzqfOqI`Qa(Ui6XAqMm#IaWaHMObF{E_Z$%IX5cwD 322 | zpZzw75WhMW1Fmw@n=yx3a^s5c^qhcebzn=*eIL-l10H*%!dqcFbdVmd>W%4wK(VF} 323 | zsLBe-Gk;-1A|=cVS9aXH8DS0Al0MS+O$9`A6EDd7ibBk&prwJLi 324 | zRk@+q3S)iojq(cF&ssvpe;!B^F4R+^c~6@Bq7#`cys;rwH+6+XcB38=PPl6)8b}}o 325 | z7yS$Ja$H8r#is5|)FfA4p2#gp1V0o_z!e@d22>7X>RbE%6ee(Ofnqp9G|6)TH)JkP 326 | zF0iO|FR+A-rORb;$e>N^yZF8%?A-#R7QDT-(KF2BM-}Ara&MG?hjL_A(Uh(>!^PRV 327 | zS*h7P!sh!`-)pSfxQp}S)B4++Z3#v9cGNi$EtH1D=?o?3yL{1REu=TwQD$N*B!Y+o 328 | z@l)h8yQd7yL~VrEdZV!UY2DMs3dgou0EKlZkWs&edL}IqwWLzT+@#&Rt_nOTsA5Wq 330 | zsolvX5=imeVfZ}1ky)JJ`xVobHd6A(rj}Nod42d4t|dCzWAH5kXl-?|U51?qcS~P& 331 | zeRxB4S&-@4eSaZp54WqMzNYZvB!0}ITE6ww#W;TZetGCnB_ws@`AN`rmHrvMK1=ZC 332 | zX!&ATNQl9=4$zEsMM5^tPP 333 | zRLQfDT#227p`Rt1HACkt)@#SS6@1Lt(@aL?701!$l)VPC>lKzs$Z&-w$#MLB7jVb| 334 | zt0S7_?dqPnt3(%W?CLMFXC_VKb-n`N`(y2OXBh>=3%k3dT}Z%;E-WL)uDvddPv_@t 335 | zs&8EE_s_m_**bClM5d2k;i)R>mlD7GHU4WB(J|8L)t%^qArUaGqQ8cDx^d-K 336 | zP|r--BApATj;Yk}1SrE!-0TC*4`P}M2^NOLT3&Fr1!4hffzJ~-A{<9rZp*e1^Bh9B 337 | z)O8-8aQRv46C;*m9NZ@K1&0dzR&4KO6@U^|^5skMnDczl&L!&$6V=8gnn9f_ZS)O5 338 | zLGNzoc|eN5VWKz7p@|n&R${5QjEl3CkY`MKqAVfzcpY+PzC7BJ@1U9lyMFI){$ql9 339 | z(F{cApF{P5Wx)7IvdXi~xW|$Y`~hTFt$Q`2npT0waIJv{vuKU1N6xZRy36?vw>U~0 340 | zc*NnPx;#lx2&xzPn6^yTwOt8QBu|IeLFyc&r(p)3PjY?CFX@mH$}+;?LEc-1^*^7u 341 | zBwG^`3AO9572aBc<&pPUW6Q34{dVf7V_GUM`dUr~vHdMQ_{Z!jNM}#utNB-jMIjY+-(}G!WI?r7!mO;Y1z}YXmpG 343 | z{n;-Y+fA>@Ufy|8lOV`hP|&04#0uh+;qZ%GhjURlO3-cJELfn1e{vF(WlUQFa|K0# 344 | zAB!2`s^L83eg=(AG%I}0<@A(vaJYRZib215K&f7B?9+F3!OZiyMVd*Qo9++zIDO^~ 345 | zzqDeAaSWm0W@r)^Bp*7bFCuAvK+Rk{^Kdb)R5Nd$L`51caSSMVcy`hHa}6+pe4Zgg 346 | zQGLWe6t~K><59zIuBn!EwSPE7GVLetIECn_mRiPN(*(3@(%P)8Z`}or!%BoB{9x=~ 347 | zj5`ZPort~qSH$Uf19*Mie(BVB**CXBWNPCpuxM3>!Q%ev-Gc0BXQh-4IJkIrG)Su~ 348 | zq7a@BvNE6OYA?TmVD!eD9Y-M1!eY1x;OzU&YO_SKNJVXo&PwNr%jMFUviP_Ax_4Bh?K0ql-=7t1p{{x)H%YVTc*W{tPVOfEkzK$I^ua3NmFa6_=~+2P 350 | zb)7|8(L0}$9ioJSJ?N&=M#o)w^%$&lqQTC!vgFYr)_T!on!3)rqzXqdljuqD5CLm= 351 | zMzU__x*;zA?NyOkC3sctY~GSx8fmLfe>qxE!-q=RN5w~ 352 | zVr+l`IfvCX#(5{=J>mKp^*kmaIK(N0emvM`A}V+IphtP{AS3gtU0stt+O$;&wh$-9 353 | zbh_~eMWjB0^oe^^!Ea8}q5(9o)t@WnBsglo&Uw=^mbl;};BCMOSfxUD9m#J+ 357 | zk>q1>*I60OvxW)=Hf~xS+}t1SQ>?Y(O|6ve(0*p}{&){EKHc1>NHbAQWjvXag5~l+ 358 | z<2g#PVURdzzFE4sdP6aMQOWI|wUgw@+d*+QhoZ%WCGkB*6mQs8B2$JovE4+pUKx}I 359 | z98bskqNl9%?U}YPtEn+$^PAmar=^CCRUh=vjvL5M=Qy$W@O1G!(H|^{&G7Did~99@ 360 | z)M6=o8B?cug_BIj)j>t!-K)OR$yS@fzpBb%OhDvYIn2d}2Zu|(mB5s9sph#U{F33b 361 | zVceC6<2JNN`%;~xE8N+o=fG^2e0%WJW^Ev1rH6v}=VHg#lUm!bx?TNB65^hSpvN`P 362 | ztP5{`x5(8k_|hW!V3)mqWbW&|JS~A4DdOse?7+=E7l5WJ*AqBT{G2Ei67&-(z|dGBVd9PG=Qk#LQqur0}ih+FQ>i8OJ6rMAZ!7?}Dq#D?tQ 364 | znw<#2eYymge!`7`Il*Pb&bcFW%}+emPSu#i8LZyIe!`RyTF=2)xn9DF%QOji5&=_* 365 | zEa7Gal+O9(`AE)~TA%5%`mX*=T518|Ime)dQps1L{@+>GqAf}&w>KHU+3+|%AWzaMseiYa6_YXo&tK_O_^%cV2|}iXYT*2b|`RS{Spc 369 | zjL*i%oF(|UntzbUNVp{DjzM2$W1HmY5MPMR`JHPrQMGIzT-EG=*;}$hK?3LBbP&Q7JTI@fB27nhnY}6?!V7A;^3Vvvg;<1n-F 372 | zf14WY<611YZBNtlkPe@3o*_ggw3*+xp4SOLFR@MoLCWM;R#%_qgo;{~?Tv3fY|NuE 373 | z{s7eD%*^AJTW?2~X7s#iYHE*HPH5xxeZpiNZDD4A9spn$+rA90`Z$0rt?%~0_J!2L 374 | z0}uyly&L33LsEVcG^*Vju@`ks)ZuiM473;dd0xgirg~HP5A!A=7%C$HnIR|MMUjhw 375 | zMihBj&i614o)^m#9JOzPUp+{+nXT^&A}j)!l3N{a4pL88l#JWzhtkZV6UxH;xT!7d 376 | z5Xtr$e47J;KQZxMXTpJVY~|h3)3L@vBi-3++V$HlxrveTlfWw}jtuB3I;16sgj&L- 377 | zxUh*j>))6Z#q5S2h*h=i>6;83RqfwAkOe775ujU8s~~_6f5i4*WSz 378 | znbrA{tjQm~vk(g^VTfd%;;Jr3HdG(II;XA$KEmaGb5?(J*cP+8cdq@9Jf8{JN|bTo&=aTAHf-D%@jF 380 | z9Ehh9T{?7{J|wJECLd}+XQB+Xtx5)q8qAWZSJq#8Bj?hp4@T{@V7S2F5b(B~s>|+K 381 | z8#|Fos2*T#X~VIH+Ssrzd?gTn?p@R1qrF>m(%{&;!?xg_PN!!#yt>xr? 383 | zEpHJjT-w0;OGhfVeXpH|+K;nVKaq-NK+EGwJFEKt^JA%)2KUvBx$IGp&14GaDR=p} 384 | zD$Ak}%@J_6@`uvj-?Wd$4i!%#`8|XU?s_LP8AT2ptp0ZKNzX9KK$5jB>^qbTd^EQo 385 | z9;ij)x@(*M91oRA<#hGFtZ}uDvKelm)_vQck3oqGT86{~e!$)q`cIh^N5!Kn=-vVc 386 | z(t?gobJ=M5mLxk~rw#A~Jfx*fr#T=sl);SOd{52@Qr17)$%87NVQ8h0*9L9FLIiZba~V&|Qf51^Aer$p>y 388 | zT$G~uoYAJNcuIx>Y*XO!>(z9)Bfy<(nC}?f!ZR}J*tcMi5oDF`nzc%e85nem@J5#f^1}zpA>e`r#iIA5Z%kMC;bJH}|c< 390 | z*p+MY@%7k`qmhFO 393 | z9ZLNw8Fb7U!3KnmC5P1R!&oi26Wl3?J0e7`5#O211_IJ01rHhPa{i!9pW&tOsNtv6 394 | ze)UtAdsX?WH6|IVLcho+8!KB3J*ZdC=F`#Fb)}FuwSpXz0;9_}W*nWKoIUNhmMqS`{lXg~3uex> 397 | zJdlCeE6cp1SIGTPbssCM$B60yBQ2&=CLEZyt4faNz;2db#HaohhgV#Sx 403 | z#5UPPLZda{d*C>wQ6_BfT<9mkjRc<^&1$C&&e(kuet-SL>z(y8LR_RCu&1w|VDdR2 404 | zBN>A3HCaj}%F0}DA7PXzoR6FrD=9Ec;J1_-WU}B0xU8&ytzZ`PA-wFhN@ 405 | zx<>H7g^ulzyO_7R-8A2I`VmMDFnjW(4EVkdwu9v7rNQB)kXMU<+4h=FiifU`&&K%UOyWjo_X6y6Cgg*tcyqJi 409 | zP#l*4Q_)dquQu(mTUr8{6UX2*JycfNz)z%cp9c>h;UP36(JE85$x7Ibe;yr96-(7y2)wSAIE5Fw`=sc0+V^);1?9*2O1W6UWlIni0 414 | zhC@V2tq!8j>3%(-9<=M~$BJ7E6i{>V7~ZMwxp*K!+zQo-nk@Wqw|DaeBfByAbfq=g&NJK) 416 | zQjb>3e9g-!0Xr)vherx%>8cva$SLu%RxI4pKw!sz)&Mi4C0e}0F5Ru9Oe0NcZVj*@ 417 | zLS-sWvUYuX8#Zni9@t3m^qx*kTE8(1ES8kYX454!bsw-5#}^}lI@JGgzV+ut!;lqd|H{*-Y;aV~>3Ku)vTpW{ur~7n 420 | z@*`F7mr>=}P~@%OuUb7lKUXtExIfK-3R93@?du+O;jSIqDD7$zVPvkKVg~hFfd05L 421 | zhu%ZY9(^qc>Ia?$^?ZJ>D=g~`3a-NqwNhM#b(!VHzBrdP`aZ=1FJ~QCET?d3nLczZ 422 | zbmTxUhGBcGljg&u^mi0*8%nmZM)QaV6!{hhdNB1yi0oN>3%>;mKrO@Xk%x`pe$j3MPu@+5?>a&2?L 424 | z6o(X-(Q9V!$#p&U{`10mY^wp^S3kHNwLsZ;iW_`km5;k<}0rh3S_*ZNld1K 425 | z%uF-j8XA%(gH1-JX(Cd0l%?fe5BV#YP1u9mR{4VpnWQ-3W}`Xxpr5RV(g?jVGGn)T 426 | zFlH9)PeZ4CMQg&lQ~YBUnu^|-MV>8aJc=6<*^<^2HYD67D$-W+v@FWh)_~(a 427 | zP+}P&A?iC}^>y@IlvZdCQC0S=utnkIGtTG_*K!SGfyMp_Du2;i_KDN= 428 | zGLd_rQjyX$D1t!pu@A4@1+ulKX=*%3h){7aRls5x<5^ 430 | zC$qMPh`)>wt|+@hFeXI_A`7#A~tNq+J9kVWS>zzzxPn<@dL 431 | z!t6;aiVX<=?&-oQNKW@T^#lIGtpCb6q}_MH6Lg4U?!HVTkh?L2`cPuc7DWUEH5C+v 432 | zvHE>MHaU}IZn$*KSw2{EU3-&{!OeLE$_E0Nw;~#U(IK}jqzQRcY56`vJ(0JY!r#f^ 433 | zloE9#uKk7_xDF69Zh>zlf8`i;*|p>eaLezu+DAOPOVd#R=I@4_ZTj|5-PEFvdnH+y 434 | zl*%08@>}1$zv2ChMfeXykdtk6>j(${APx!u!2ACXBCs%a|35JVpLz#*)BXMZ!~TEJ 435 | z#Tc-e6cCu==%)q2OmPg;V+5yT!JPmY$S~5Ql&8OT!K*>G-gT*VQFCur 436 | zYX_G2^xR&qoD}%j(A|Hvk@wzc8BgPGxu2n?s&3NUS8jN6dp5nNCM&4P+9KO!wC{J? 437 | z9A=Nccy2O#y>MwC*1h@{KhF4~PKaV~-P$ViK5HKzvByrvj}^OXJlsBob9UNo$XInU 438 | zy>E+Z<3a;`jYdCwVqNhHWARMZk)D&qZiWPA&n|mjGk`7EM(Xh{pfn$>Z;CduXmy=l 439 | zZdSF`a9s}PlXCYW@<3x^`%Zo9t9QBt#ORFr#)4D^!j9MMy;WT=Z__9sn58yGyldUJ 440 | zNI0lgI6S~q2ilm9%#Myz7|eAyi?hf2^ZOvr>~QrNdEll`Z25WPXX+9~hd3-gcN{vxKfu8nj#Sp)v}s1plY 442 | zQ@j6AC({2nc*RUWsFd{|2IXId_`f8~zh(Z#kvRRIl0Y8+|2vrS=g)rvk^e6M4%j~) 443 | znA_P|{SW+x>>uy{Hxc~5DgUqeXh1!^X+Qt~Yajpsc>i<0y`zb}fujkHk)4f=g)@zf 444 | zlNqg(fsMVj$-l$-m+-&rzi>S+N;*zU^zh!KeFyPOGJMoogIZkI(iG57A+z&$sTNi1 445 | zW?IS#+7hM76LF`+90WM{N#MuHJ6l^kCUsLK%O#hj8^l3SFmFoLo+0xXj{voejjww! 446 | zPpDo72;O1x87q>W@}(gtoMf8R5MC6?gP5sv{hFOJeuj6%K4=P>)$r0tb1}*RlI~~0 447 | zP$+|O)kH?HF$$C1B%}Tn3>68YcIJq|^+e_;{TTzQ>HyPS%)l!j123Zl;%KA;nVnxy 448 | zI4TGalnG+Mm;@~vpyj)6Dp|8yBzRF8Fr>&{lBpiZ%q?|^n=de-IQJmbO@9?r}%=CxQ3V5Opszivd_Kee%W^ldzg@ 451 | zXN1e4vo3;F1=$$A6!Qefq@W9`F3;pIHPNfbQ2@H+OzVu;MM@qMwPe&$O4(xcGEDCJLL<3X=J3}W0Bjn 454 | z?Wf2i^B*gDh8#UbXJRw4^T2Q=*6}O`4~Fq>_|U?AJ~C0UVBGCtP% 455 | z-;!-q6^AEZUk@R=_?)?%_dv|93)$z{q>;3U` 456 | zd)_}xU*Z 458 | zhUf(_xr5We-$W3WV#p!pC*8nT6Ut5Y%FZb7I`stWIXDD^_IFR2^oCFXpzU}m)X{nH(V#GLYM 460 | zOUJ~)<^$n<7cA!ZRXMAEDU;@yeE07~?LW0lLSL?3^bcic2?hXw_Mh1PXC41HLbFje 461 | zu-jsV|Hl4~tU@A&(cg&O2a{L@q3=e9g17F+)W_R2%pWcq5g}AbY&>@hvq(A@6H&4<(J*hzDi8p#HX5BZ;1Pb9H0Y0wEpqMYlSmSui$TAfF%PG{}ZR*(*G0_mMF_s?m@gE#zf+vjneGh0e`MrLiTg|C}u 466 | zLP7OJ?69W!;gAjVMEgrN4$e~0f(WnfTD>63I#sMDfP#kyEf#r{E 467 | zYFdgv6!Gb-%HiXHNCDta&}?yq1;~0v!s4QMd%zabUTOcx3y*{(t)ZfKz{Ry{RPE 471 | zD-dDl_QU~$`Om%9z;vX@!OFY~7bd6#M4wJlMHLnhd*T`#BR7LDDK;E3^sMYpj}a&Ez4=_@w_qb77q4>C&$qL 473 | zoa$u0?ahuIwNNCHAhp%y*mr~bBf!#&h6o4&7Ax!tZ5GQ<=+?oQ%Dv2!ZXg5#Ki`P^ 474 | z&ORz8qX0c{zfiW7?(TE#9-HG2cj+kp#K^vk{^sH!@YMe3^78K15}s=>Zjep>em{7) 475 | zivG3GQq;{A!!R4NPm#a*{+xUD^B%v+X}SqbRTq-!tDpBXf8At8PS8)zD_oM2)}$!B 476 | zpvV5$ai@+~J8W#Z>9Dh&V!aWz8ZT_-a$gxUwt#%eBocl7RIqLSJGWraVJ{fKHl)pq 477 | z6>X$hagRKb#_GwIOUT_ZmvE=5%{xQ_(LTXg_&c*N2SyKLhg!lRsr7s?CK?XSuCk?y 478 | zX(N?-LY|NJ`XVP^qiF9M>=Zvs)<;!vX%P6jGux8zh3J*bkL&4YZkucN`(J9_vwIln 479 | z?>}TIA<}=Xc}EjN1E>Gaa+afRWw#}Y`c3^4sU=cVQjOcni@enyyT2}wb!`h|9D^D) 480 | z+#q%&Zz$u~&NM`BdG)*NWEM(DLv?NlI(~B4d)pDWkYr~XRh^xo@2Sppo6mQdx-L2qI$s}sZu7YRD)E|aEx7`)Zrm|dz 485 | z<#ZUY81hPNO*KoKz2VGi%frG-YFKXiiP~V20Q@NIRZ^^Q@?u><7I#?5_X%9))DCjm 486 | zk`^4`v3fSt7K0!7#A(4%%@q$ILZ#Ex#@KxU7?o$jx#>Ti`t>TFOfiLbSHUf!)Vox? 487 | z=Wwwf3aN=H{lb2C#d2iKaqxkR7aVS~ADX46t%1f&Sks>g(=pb&JM^B@rRwF}Q2{N< 488 | zzP=u=4nl5j3ULAaVb0J&B>14j^)2FMwNzV}IpS1?ttlKu%u$8ts{_bo1#qe5M*ZjK 489 | zSLfsxICx@Tx0jU2-Shm|N>W4z5Di{2!2zjBiQ)-t{3Q4IZg2rWtSS#)iWDe={6EQO 490 | zDFUo#Gj!Qfa^as!s}2R)Hzx2A*+(5!P(iP>|GWx8tseE&BfQZ+V!}PCi&O}g>01=!A 492 | zk!@uUo=Lf0&@bNOl$3F8qVJW7%SnnZh^}DUpiZr;kE6QH<5e*J)n*!cUXZnKZFrkM 494 | zGP~nQp6A&SXl`Z0d3i@$-MRXfUKy^GAgq2mULfvy^&_c1)eqidoEewA$RtfcPFXhVH)qoZGJD^WG?t_*kle5 500 | zFMjXGUeWAk>-2hi!wYZZUAaOKDzsMx$xLs#bs?GaLWCikjTUA8mYy1Qr%oh;0)MHK 501 | zD~h(!m(sfDd)o6w5qe76O{(9>!s-e{qpkl##X!l2!M@rS$zd2>P=MhLqW|B!`MT5%Erz;wir>mo@*X_ymXs1v0 507 | znTT;Uv|#5u5?TzlYJ~QuPp&pZOaHu8*{a{mow@MbS<)PYOZ+hQd~jT3`8;J;zxa-i 508 | zJf9@Yy6xu;Vt;aOy>l{=#+y^seBjP)+JRO;Pi;nmB>2z$KOIa{X*-24NVp*~Tr0<@ 509 | z17W84{m@wYRU9yP*-E(D#EZz{r_&y*l9Zwd3sA&D4$tq=)?DIb@v;C}iy+82peR=_ 510 | zG3M;Qn#wx-Zf#{YHtYjw-iih~SHX1psnSn$un7IJGeDk?Gwx=u4%=RTu)vFJxi 511 | zxxj)X4QPq-+?7RqC8>^RW7d5$_3yww!8V%H- 512 | zCp$cxkzp9F)Cw7|3lP5bIu9@i+}JTffKhZkus>9nLK#2Gn!!UIbI}Gr3L#MbT)lek 513 | zDI`rC(e(j7#Tucq_MH3{S(P4u17&~d%x~xxZp=3EFz1wg#SqwhmY~H^nB~}gdi+0? 514 | zoOe8x?;FRDl^M#;p4lVcZ0g9~dzF=W%!5!wSvknap2v(t$IPb4IzmNC9FdH&DP;eS 515 | zuf*{yU!B+MdHL&k-`9OT_j5g;>%K1c4y0JQ63hm@ltFMqaJ{BIDQtb`>~o>0uGGk^ 516 | zGJ$I&c$YjU%Y#*$E0+ZI`T9?DbV+5xgQLfW0n+0c&gGKu#X7JANUtVzJm2P-hPz)S 517 | z)qsQ2m)04I;Uyvtn9S`WC${`vx49fw?Ecwsf{WZQ5uxL1@0Z2D--@c-LIVWLNbRJr 518 | zMVwdWMi_BheII==s!aD2Cg6QnEjPaM=xIZ6SNIG~s+@_oFOfcp>rh2c<{L7j 522 | zQazpWF?eMb=+W~};-jJJ!MT)?J$SDm=S(D{|v 524 | z&@HQ6f`Q58=p@x_`VaJ#T8#m(5(uQ?8C!USIb24Fjl0UAC3iE;#B^>K$hTlHIXaT@ 525 | zF+=Ng8=3^L{4h!$&6CKY`B58kaB3OHbF?88$S4~YXXu0mQV2Pt(=Jgx2a-l%;hGjH 526 | zJUG*Z9_$-6Md03~v0hGGrdIYu2ve>1G#Xf>aiVKEB@F&{ctziQVcX8z^H%K^cSSkAW<7hyZ`Y^rOvcRcQ5Th5wKBe7ajk`m!KWCqN+Q<=IbT58#>`$M 528 | z`Z%+~B)IgI-Lxds#GaStnMPRF{JEr!Fkfi=IE2i6*dzoNzT?Pt3#G-V={R_;V(V%iknhxEP-Qef 535 | zr=JE*Es7dT(6#T(eUp%&)gB6gV%r+T|3lAuj%Qc^8BG6zKz0z!xxsjh%nzW(GAT3YS4*#5r9f}>z4 539 | zt{jW*y_pQ-P^z(>|983${HM96W30#YN{j~Ajf?bf)IqPQ%GOhw-iqfDfv4|D-Mmx0 540 | z(2XE?oMH-03u8C~>?InJqXy$o_GXsR6>%>RsFhaA&t|_XB1@qR* 542 | z6vZsKw$)-`QJJBc?bMdKPH|_|lOXNwTyOw`b@Jz8Hp;9={tIR05{d6QxoCQxkxJ*N 543 | ztbs*xhU*My1JWrudK-wq`X9SyJYhu`FvGg>yKZfyT(A<13-R1#ao!Q?q_Qi+IQ6hd 544 | zBM(_-MEm!}W^x4ztl^um+)rzQmooFVB6_q7pM;*P-g*-dB(z+C4h4ObVApQz?5~y* 545 | zBHQ-+e2ePS<5Iae;Q1Ng<{;`M~N$AADiCCztb0${C@LN 551 | z2l@pKkoZLt%>kx%0hXzaO9Qn}C&qjcRes(<+pFw-ay5Q3oQyj!+k{wUK>9aCT!fQn 552 | z5!r+O_Nq1ML8M|WgMG##MBCV?pF$eetD;-TZNLcR=aBkW2t#gt^O%Fd4!6MOH!N&V 553 | zuw~{GDWLv>pE$^7lZy`2ht&g#!)0N4OXe=G=9RA=?i25m$pP7k&^9V5X{g<3@D3Zy 554 | z9e-_ULvHOi2}mxC#ApnmiK=bZ%=9NxLZpLl=rvMR#4V;)2CCKB=njxw_Vy2V5s@uO 555 | zyOQ4gg4}wRhiD|_(NM@o 556 | zC-!q1=a0|{HlFJGW?{%GJ+xdOrgOQm;vuxA=laSpb6mS=G;}g4BQ#{{8C%~#;>DFh 557 | zcMo?q+HB2^P%r7Hw}%{tO@*AhR9<%7{!vg^Y?MTu{Aj}yw6+Y^=ShF;{LR)Efa=lQ 558 | zffl0-WHDj4eny7kQLww)NqiZuGjC5SK)iK(jjOdjK>LfacZ+%pNsfj@6qDUzz@x6Y 559 | zZWHM;c(P@!J|lmV;E7;|SWoj%i-$ 560 | z8{kM2;YfaVDj;^$!|7h%kk`vV^Losx*jBUClXe@uE~^h{XJ((Kv(pXt892 561 | zMoQezFv?JEfYDhQ1dLW$H?^9>y;ps)n^G8TcfQ`?0ZU+5tkBiQ>uB%Gv0m|P^a&F(eDHL% 564 | z$RwHD-2(njPi}Sw=B`ue>X=fetirMerLv4k)cjK0ZEK`WeZPVdBGTw`fy%-DD@UhC 565 | zVBV~4igTGN!1+G2eu&jKk4i!%*cS^HsJe&tbNS$gK#3q=wO`6V#TIF8-dc%UO!GvW 566 | zZ!9>S5#&Y$KqnF)J;tYpP;ygYdiUOshVOB3s 569 | ztxJYpqbR*6sisu;B<4dAm*|I&J@JNhT#2S@yIAZN#%RG{t 570 | zmV;YO8pZ|&BVh{0GiKWXqmzZ{%4C(|SJ-nC$7e>9&KPiDBmYaq(MKl~+oHmG5cd*Y 571 | zEww=ow-?APZf%KgNaumat7d1^!JepAg$U*QpVcpCpN>csBWrBM)bNY$p 572 | zQ|OA#9XfjfbKobiX!nsadGeU#g&wuxdwIlS4|$a#!j9Bc 575 | z=R)wC2EfmKLPn+dKGF!0c$d0;pjq-tNN2kDL*-T=vMlVWm4&5iTvS9$q?@gSfuY!y 576 | zYaPqM2_%0MslhWWG#L55%loP=!UQqNYVhPOEnUfmaQUW}*Q5C9sjHF&N2>7#^p@|9 577 | zkn#CC^1??(ligyel_ig>mG!YzRh~{DULZdK*6aE_oGRXTYJIJPIkQBBdc*|;A58)c 578 | z8@lX43FXNcMqgJY6OueK7-CEwA|@hRBh6hKb2AEg-Nq96A+F{0)7hm}jq7w7!c*i5 579 | zupzeP<~I_kPhU6;oBk5gG{v9hA@?XOTtVO)AZMrJs-*Z`FO@r0RW;d_c-r~8x#4_| 580 | z%k>mg;oO?+@OExJg_oU*x*F<=JZiih-0bkqSK9jQaDBj6Kt~=EsOK~KzX5me_{F2u 581 | zUo51Hc73x~zWzgvc6SQDzkS!k|K4tEWo~&0a_j+)M{U5{huko?`5Oio$Ssco>Brp* 582 | z6Uk7J1<)3<;-BZQ1n>BSa5&1jd0xDy|%g} 584 | zyzDdm{k9eySs(*8Jo^lF=ibZVgfqO3Aoo`LIib1&uy6zA1r8u78Rm!fz?#RI=$ST? 585 | z?aB_y-&)b|+(v|#(c3g<&_BCKlBH#QFzRlNY)_rJOA)uyF2N9e*WIw+M1boIp49d9 586 | z&6Q#`5z6o&l2RiyX4&;`FRI(yRvM?O@6FF0QbfBR!}bIG5JOH^9C_jHPkGnP$8|Aa 587 | zH8!5P%a4Qkyi$lnCQE~mrR~Qe$;=%@oU*gtx#V~W9_~{$d01&I4<-DnS34l;?_yJjSX 589 | zGg=VYVuewY(x{jBs>6w&eZh1%@`1ku 590 | z2WyM;yYkc9nR3^a40%VOc@p6~lqBxwCRBs3`y}YP)FZM6o#qO!zj(I0{op{~V+{wb$>d@=(+olXObUKVbovTWTbjj|I7_bH0{d;iA 592 | zD%Rd9t4BI9G?f2LJB=)R@BMaZ(uZif@f8_7jzb=_N}-;&x{pKyu=s@>MgD!0%Snwu 593 | z?bkYNL^uC|m;RmC-x=|5_ttTa`2CYY^=;LK0~>zEo|7R_yNdRvg_Ib8a=)FU$lYln 594 | z-_HN$9*xweP?mw8AmCT$5R&zq=AOMJbr2m<`-To|-12Mu#X6>o=wySSHftOnqy*V< 595 | zwb$wd{!Qz~$>^x<5{K#G!oTSM-#l?LIBH$?Fx(FrU4NR6gdeHWo(zt 596 | zn=Rdw!BKbi4#U@E_u)tO`A!B#&9@JO>y-Avf5^U1HUMh+d3XRqmBRy|B%>!oqe}h5 597 | z=qFtJ 644 | Date: Tue, 14 Apr 2015 10:00:05 +0100 645 | Subject: [PATCH 2/2] Fix Git-LFS filepath sharding 646 | 647 | See also: 648 | 649 | https://github.com/github/git-lfs/pull/212 650 | --- 651 | .../src/main/scala/com/madgag/git/bfg/cleaner/LfsBlobConverter.scala | 2 +- 652 | 1 file changed, 1 insertion(+), 1 deletion(-) 653 | 654 | diff --git a/bfg-library/src/main/scala/com/madgag/git/bfg/cleaner/LfsBlobConverter.scala b/bfg-library/src/main/scala/com/madgag/git/bfg/cleaner/LfsBlobConverter.scala 655 | index ce07008..e99496d 100644 656 | --- a/bfg-library/src/main/scala/com/madgag/git/bfg/cleaner/LfsBlobConverter.scala 657 | +++ b/bfg-library/src/main/scala/com/madgag/git/bfg/cleaner/LfsBlobConverter.scala 658 | @@ -75,7 +75,7 @@ trait LfsBlobConverter extends TreeBlobModifier { 659 | 660 | val shaHex = encodeHexString(digest.digest()) 661 | 662 | - val lfsPath = lfsObjectsDir / shaHex 663 | + val lfsPath = lfsObjectsDir / shaHex.substring(0, 2) / shaHex.substring(2, 4) / shaHex 664 | 665 | val ensureLfsFile = Try(if (!lfsPath.exists) tmpFile moveTo lfsPath).recover { 666 | case _ => lfsPath.size.contains(loader.getSize) 667 | -------------------------------------------------------------------------------- /test/resources/samples/git/pull145/github.patch: -------------------------------------------------------------------------------- 1 | From af7333c176401601d67ea67cb961332ee4ef3574 Mon Sep 17 00:00:00 2001 2 | From: Ariel Faigon 3 | Date: Fri, 5 Jun 2015 16:47:45 -0700 4 | Subject: [PATCH] Avoid failing to create ${__git_tcsh_completion_script} when 5 | 'set noclobber' is in effect 6 | 7 | tcsh users who happen to have 'set noclobber' elsewhere in their ~/.tcshrc or ~/.cshrc startup files get a 'File exist' error, and the tcsh completion file doesn't get generated/updated. Adding a `!` in the redirect works correctly for both clobber and noclobber users. 8 | --- 9 | contrib/completion/git-completion.tcsh | 2 +- 10 | 1 file changed, 1 insertion(+), 1 deletion(-) 11 | 12 | diff --git a/contrib/completion/git-completion.tcsh b/contrib/completion/git-completion.tcsh 13 | index 6104a42..4a790d8 100644 14 | --- a/contrib/completion/git-completion.tcsh 15 | +++ b/contrib/completion/git-completion.tcsh 16 | @@ -41,7 +41,7 @@ if ( ! -e ${__git_tcsh_completion_original_script} ) then 17 | exit 18 | endif 19 | 20 | -cat << EOF > ${__git_tcsh_completion_script} 21 | +cat << EOF >! ${__git_tcsh_completion_script} 22 | #!bash 23 | # 24 | # This script is GENERATED and will be overwritten automatically. 25 | -------------------------------------------------------------------------------- /test/resources/samples/mailarchives/marc/m.143228360912708.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | '[Announce] submitGit for patch submission (was "Diffing submodule does not yield complete logs")' - MARC 4 | 5 | 6 | 7 | 8 | 

    [prev in list] [next in list] [prev in thread] [next in thread] 
  9 | 
 10 | List:       git
 11 | Subject:    [Announce] submitGit for patch submission (was "Diffing submodule does not yield complete logs")
 12 | From:       Roberto Tyley <roberto.tyley () gmail ! com>
 13 | Date:       2015-05-22 8:33:20
 14 | Message-ID: CAFY1edY3+Wt-p2iQ5k64Fg-nMk2PmRSvhVkQSVNw94R18uPV2Q () mail ! gmail ! com
 15 | [Download message RAW]
 16 | 
 17 | On Tuesday, 19 May 2015, Stefan Beller <sbeller@google.com> wrote:
 18 | > On Tue, May 19, 2015 at 12:29 PM, Robert Dailey
 19 | > <rcdailey.lists@gmail.com> wrote:
 20 | > > How do you send your patches inline?
 21 | [snip]
 22 | > This workflow discussion was a topic at the GitMerge2015 conference,
 23 | > and there are essentially 2 groups, those who know how to send email
 24 | > and those who complain about it. A solution was agreed on by nearly all
 25 | > of the contributors. It would be awesome to have a git-to-email proxy,
 26 | > such that you could do a git push <proxy> master:refs/for/mailinglist
 27 | > and this proxy would convert the push into sending patch series to the
 28 | > mailing list. It could even convert the following discussion back into
 29 | > comments (on Github?) but as a first step we'd want to try out a one
 30 | > way proxy.
 31 | >
 32 | > Unfortunately nobody stepped up to actually do the work, yet :(
 33 | 
 34 | 
 35 | Hello, I'm stepping up to do that work :) Or at least, I'm implementing a
 36 | one-way GitHub PR -> Mailing list tool, called submitGit:
 37 | 
 38 | https://submitgit.herokuapp.com/
 39 | 
 40 | Here's what a user does:
 41 | 
 42 | * create a PR on https://github.com/git/git
 43 | * logs into https://submitgit.herokuapp.com/ with GitHub auth
 44 | * selects their PR on https://submitgit.herokuapp.com/git/git/pulls
 45 | * gets submitGit to email the PR as patches to themselves, in order to
 46 | check it looks ok
 47 | * when they're ready, get submitGit to send it to the mailing list on
 48 | their behalf
 49 | 
 50 | All discussion of the patch *stays* on the mailing list - I'm not
 51 | attempting to change
 52 | anything about the Git community process, other than make it easier
 53 | for a wider group
 54 | people to submit patches to the list.
 55 | 
 56 | For hard-core contributors to Git, I'd imagine that git format-patch &
 57 | send-email
 58 | remain the fastest way to do their work. But those tools are _unfamiliar to the
 59 | majority of Git users_ - so submitGit aims to cater to those users, because they
 60 | definitely have valuable contributions to make, which would be tragic
 61 | to throw away.
 62 | 
 63 | I've been working on submitGit in my spare time for the past few
 64 | weeks, and there
 65 | are still features I plan to add (like guiding the user to more
 66 | 'correct' word wrapping,
 67 | sign-off, etc), but given this discussion, I wanted to chime in and
 68 | let people know
 69 | what's here so far. It would be great if people could take the time to
 70 | explore the tool
 71 | (you don't have to raise a git/git PR in order to try sending one *to
 72 | yourself*, for
 73 | instance) and give feedback on list, or in GitHub issues:
 74 | 
 75 | https://github.com/rtyley/submitgit/issues
 76 | 
 77 | I've been lucky enough to discuss the ideas around submitGit with a
 78 | few people at
 79 | the Git-Merge conf, so thanks to Peff, Thomas Ferris Nicolaisen, and Emma Jane
 80 | Hogbin Westby for listening to me (not to imply their endorsement of
 81 | what I've done,
 82 | just thanks for talking about it!).
 83 | 
 84 | Roberto
 85 | --
 86 | To unsubscribe from this list: send the line "unsubscribe git" in
 87 | the body of a message to majordomo@vger.kernel.org
 88 | More majordomo info at  http://vger.kernel.org/majordomo-info.html
 89 | [prev in list] [next in list] [prev in thread] [next in thread] 
 90 |
    91 |
    92 | Configure | 93 | 94 | About | 95 | News | 96 | Add a list | 97 | Sponsored by KoreLogic 98 |
    99 | 100 | 101 | -------------------------------------------------------------------------------- /test/resources/samples/raw.posts/raw.posted-by-aws-ses.txt: -------------------------------------------------------------------------------- 1 | X-Received: by 10.180.77.225 with SMTP id v1mr6964442wiw.5.1436284425893; 2 | Tue, 07 Jul 2015 08:53:45 -0700 (PDT) 3 | X-BeenThere: submitgit-test@googlegroups.com 4 | Received: by 10.152.179.39 with SMTP id dd7ls837412lac.60.gmail; Tue, 07 Jul 5 | 2015 08:53:45 -0700 (PDT) 6 | X-Received: by 10.112.26.5 with SMTP id h5mr2431122lbg.4.1436284425520; 7 | Tue, 07 Jul 2015 08:53:45 -0700 (PDT) 8 | Return-Path: <0000014e69391015-5de3c8c6-458f...@eu-west-1.amazonses.com> 9 | Received: from a7-11.smtp-out.eu-west-1.amazonses.com (a7-11.smtp-out.eu-west-1.amazonses.com. [54.240.7.11]) 10 | by gmr-mx.google.com with ESMTPS id eo3si1863304wib.0.2015.07.07.08.53.45 11 | for 12 | (version=TLSv1 cipher=ECDHE-RSA-RC4-SHA bits=128/128); 13 | Tue, 07 Jul 2015 08:53:45 -0700 (PDT) 14 | Received-SPF: pass (google.com: domain of 0000014e69391015-5de3c8c6-458f...@eu-west-1.amazonses.com designates 54.240.7.11 as permitted sender) client-ip=54.240.7.11; 15 | Authentication-Results: gmr-mx.google.com; 16 | spf=pass (google.com: domain of 0000014e69391015-5de3c8c6-458f...@eu-west-1.amazonses.com designates 54.240.7.11 as permitted sender) smtp.mail=0000014e69391015-5de3c8c6-458f...@eu-west-1.amazonses.com; 17 | dmarc=fail (p=NONE dis=NONE) header.from=gmail.com 18 | Authentication-Results: mx.google.com; 19 | dkim=pass head...@amazonses.com 20 | DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/simple; 21 | s=uku4taia5b5tsbglxyj6zym32efj7xqv; d=amazonses.com; t=1436284424; 22 | h=From:To:Message-ID:In-Reply-To:References:Subject:MIME-Version:Content-Type:Date:Feedback-ID; 23 | bh=H19iXkkHtU+AKPoMRaIOnpNNFD88xZfTWt27yI8MRTw=; 24 | b=XX0Ket/oYqGQqTELToem77+X/6xi0xwCsktgFb2fN5h2zwfB7PLoUyt5KGQzTq/P 25 | WhGiRXBpKyMbDfhyODOVPpZgHw1IKNZVfGTF1pesar74iz20pWXWJOaFVBC/f4uTbEt 26 | 1z29Du+hokN8ldYiwD4P7HFEUO7n2Nr40pglPL1w= 27 | From: Roberto Tyley 28 | To: submitgit-test@googlegroups.com 29 | Message-ID: <0000014e69391015-5de3c8c6-458f-4eb6-b222-14cfc8a6b055-000000@eu-west-1.amazonses.com> 30 | In-Reply-To: <349d78e1-3c4f-4415-908d-599a57d15008@googlegroups.com> 31 | References: <349d78e1-3c4f-4415-908d-599a57d15008@googlegroups.com> 32 | Subject: [PATCH/RFC v245] Update gc.c 33 | MIME-Version: 1.0 34 | Content-Type: multipart/mixed; 35 | boundary="----=_Part_4_1512894794.1436284424168" 36 | Date: Tue, 7 Jul 2015 15:53:44 +0000 37 | X-SES-Outgoing: 2015.07.07-54.240.7.11 38 | Feedback-ID: 1.eu-west-1.YYPRFFOog89kHDDPKvTu4MK67j4wW0z7cAgZtFqQH58=:AmazonSES 39 | 40 | ------=_Part_4_1512894794.1436284424168 41 | Content-Type: text/plain; charset=us-ascii 42 | Content-Transfer-Encoding: 7bit 43 | 44 | Write something helpful here 45 | --- 46 | builtin/gc.c | 2 ++ 47 | 1 file changed, 2 insertions(+) 48 | 49 | diff --git a/builtin/gc.c b/builtin/gc.c 50 | index 5c634af..c8fcb4f 100644 51 | --- a/builtin/gc.c 52 | +++ b/builtin/gc.c 53 | @@ -10,6 +10,8 @@ 54 | * Copyright (c) 2006 Shawn O. Pearce 55 | */ 56 | 57 | +// some random test text - we should support a purge option, etc... 58 | + 59 | #include "builtin.h" 60 | #include "lockfile.h" 61 | #include "parse-options.h" 62 | 63 | --- 64 | https://github.com/submitgit/pretend-git/pull/1 65 | ------=_Part_4_1512894794.1436284424168-- 66 | -------------------------------------------------------------------------------- /test/resources/samples/raw.posts/raw.posted-by-google-groups.txt: -------------------------------------------------------------------------------- 1 | Date: Tue, 30 Jun 2015 14:01:44 -0700 (PDT) 2 | From: Roberto Tyley 3 | To: submitgit-test@googlegroups.com 4 | Message-Id: 5 | Subject: Scary fudge 6 | MIME-Version: 1.0 7 | Content-Type: multipart/mixed; 8 | boundary="----=_Part_112_1073874982.1435698104974" 9 | 10 | ------=_Part_112_1073874982.1435698104974 11 | Content-Type: multipart/alternative; 12 | boundary="----=_Part_113_1164253086.1435698104974" 13 | 14 | ------=_Part_113_1164253086.1435698104974 15 | Content-Type: text/plain; charset=UTF-8 16 | Content-Transfer-Encoding: 7bit 17 | 18 | Does this make sense? 19 | 20 | ------=_Part_113_1164253086.1435698104974 21 | Content-Type: text/html; charset=utf-8 22 | Content-Transfer-Encoding: 7bit 23 | 24 |
    Does this make sense?
    25 | ------=_Part_113_1164253086.1435698104974-- 26 | 27 | ------=_Part_112_1073874982.1435698104974-- 28 | -------------------------------------------------------------------------------- /test/resources/samples/raw.posts/raw.public-inbox.txt: -------------------------------------------------------------------------------- 1 | From mboxrd@z Thu Jan 1 00:00:00 1970 2 | From: Shawn Landden 3 | Subject: [PATCH] daemon: add systemd support 4 | Date: Sat, 16 May 2015 19:44:10 -0700 5 | Message-ID: <1431830650-111684-1-git-send-email-shawn@churchofgit.com> 6 | Cc: Shawn Landden 7 | To: git@vger.kernel.org 8 | X-From: git-owner@vger.kernel.org Sun May 17 04:44:39 2015 9 | Return-path: 10 | Envelope-to: gcvg-git-2@plane.gmane.org 11 | Received: from vger.kernel.org ([209.132.180.67]) 12 | by plane.gmane.org with esmtp (Exim 4.69) 13 | (envelope-from ) 14 | id 1YtoZB-0002LJ-BW 15 | for gcvg-git-2@plane.gmane.org; Sun, 17 May 2015 04:44:37 +0200 16 | Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand 17 | id S1750940AbbEQCoV (ORCPT ); 18 | Sat, 16 May 2015 22:44:21 -0400 19 | Received: from mail-pd0-f169.google.com ([209.85.192.169]:32773 "EHLO 20 | mail-pd0-f169.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org 21 | with ESMTP id S1750774AbbEQCoT (ORCPT ); 22 | Sat, 16 May 2015 22:44:19 -0400 23 | Received: by pdbqa5 with SMTP id qa5so89595529pdb.0 24 | for ; Sat, 16 May 2015 19:44:19 -0700 (PDT) 25 | DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; 26 | d=gmail.com; s=20120113; 27 | h=sender:from:to:cc:subject:date:message-id; 28 | bh=rT2ZlqT3euHQwt2R6T3jZ5AQ4VG5GmJUGcxMyz8EihQ=; 29 | b=JvsrGbxmIqIxuRDQwe8O2CQ3KCRVgKSNpFqxPBOwZjVObge/lVM97Ncfm/Wv82PEuI 30 | 0eQpF/V1/4PpSpQlsFaDY2Zrt3CUzKp8q8GLoD1esixgVs0GfE+m37MmGBRuroD2o0Tx 31 | C812EEEGgIsFba+B6CW0Ml8FPWcyyk6vd6rXGDaQgHJ1BX8KJXcmIfeDRxhyvxkmSGyL 32 | nDe9nK85PEWYGgCXmIgbWNfXQxC/207lre/vcC9esU2RCWtB7bUIH1aiJjTNAcq19fjX 33 | ORh4nsBqIxFTdkUZqoMBfAQbz6oGw5U3tKi4eH8FQ0HwPuCyZLW/NmcJ6Z3DtB22lQ+n 34 | L8Ag== 35 | X-Received: by 10.70.29.4 with SMTP id f4mr32717278pdh.23.1431830658930; 36 | Sat, 16 May 2015 19:44:18 -0700 (PDT) 37 | Received: from zephyr.hsd1.wa.comcast.net. (c-24-19-147-81.hsd1.wa.comcast.net. [24.19.147.81]) 38 | by mx.google.com with ESMTPSA id p5sm5962456pdi.2.2015.05.16.19.44.15 39 | (version=TLSv1.2 cipher=ECDHE-RSA-AES128-SHA bits=128/128); 40 | Sat, 16 May 2015 19:44:16 -0700 (PDT) 41 | X-Mailer: git-send-email 2.2.1.209.g41e5f3a 42 | Sender: git-owner@vger.kernel.org 43 | Precedence: bulk 44 | List-ID: 45 | X-Mailing-List: git@vger.kernel.org 46 | Archived-At: 47 | Archived-At: 48 | List-Archive: 49 | List-Post: 50 | 51 | git-daemon's --systemd mode allows git-daemon to be connect-activated 52 | on one or more addresses or ports. Unlike --inetd[1], git-daemon is 53 | not spawned for every connection. 54 | 55 | [1]which systemd is compatible with using its Accept=yes mode 56 | 57 | Signed-off-by: Shawn Landden 58 | --- 59 | Documentation/git-daemon.txt | 49 ++++++++++++++++++++++++++++++---- 60 | Makefile | 10 +++++++ 61 | daemon.c | 62 +++++++++++++++++++++++++++++++++++++++----- 62 | 3 files changed, 110 insertions(+), 11 deletions(-) 63 | 64 | diff --git a/Documentation/git-daemon.txt b/Documentation/git-daemon.txt 65 | index a69b361..0eab51b 100644 66 | --- a/Documentation/git-daemon.txt 67 | +++ b/Documentation/git-daemon.txt 68 | @@ -19,7 +19,8 @@ SYNOPSIS 69 | [--access-hook=] [--[no-]informative-errors] 70 | [--inetd | 71 | [--listen=] [--port=] 72 | - [--user= [--group=]]] 73 | + [--systemd | 74 | + [--user= [--group=]]] 75 | [...] 76 | 77 | DESCRIPTION 78 | @@ -81,8 +82,8 @@ OPTIONS 79 | 80 | --inetd:: 81 | Have the server run as an inetd service. Implies --syslog. 82 | - Incompatible with --detach, --port, --listen, --user and --group 83 | - options. 84 | + Incompatible with --systemd, --detach, --port, --listen, --user and 85 | + --group options. 86 | 87 | --listen=:: 88 | Listen on a specific IP address or hostname. IP addresses can 89 | @@ -146,8 +147,8 @@ OPTIONS 90 | the option are given to `getpwnam(3)` and `getgrnam(3)` 91 | and numeric IDs are not supported. 92 | + 93 | -Giving these options is an error when used with `--inetd`; use 94 | -the facility of inet daemon to achieve the same before spawning 95 | +Giving these options is an error when used with `--inetd` or `--systemd`; use 96 | +the facility of systemd or the inet daemon to achieve the same before spawning 97 | 'git daemon' if needed. 98 | + 99 | Like many programs that switch user id, the daemon does not reset 100 | @@ -180,6 +181,16 @@ Git configuration files in that directory are readable by ``. 101 | errors are not enabled, all errors report "access denied" to the 102 | client. The default is --no-informative-errors. 103 | 104 | +--systemd:: 105 | + For running git-daemon under systemd(1) which will pass 106 | + an open connection. This is similar to --inetd, except 107 | + that more than one address/port can be listened to at once 108 | + both through systemd and through --listen/--port, and git-daemon 109 | + doesn't get invoked for every connection, but only the first. 110 | + For more details see systemd.socket(5). Incompatible with 111 | + --inetd, --detach, --user and --group options. 112 | + Works with the session manager (systemd --user) too. 113 | + 114 | --access-hook=:: 115 | Every time a client connects, first run an external command 116 | specified by the with service name (e.g. "upload-pack"), 117 | @@ -305,6 +316,34 @@ selectively enable/disable services per repository:: 118 | uploadarch = true 119 | ---------------------------------------------------------------- 120 | 121 | +systemd configuration example:: 122 | +Example systemd configuration files, typically placed in `/etc/systemd/system` 123 | +or `$HOME/.config/systemd/user`. 124 | ++ 125 | +`git-daemon.socket` 126 | ++ 127 | +---------------------------------------------------------------- 128 | +[Unit] 129 | +Description=Git Daemon socket 130 | + 131 | +[Socket] 132 | +ListenStream=9418 133 | + 134 | +[Install] 135 | +WantedBy=sockets.target 136 | +---------------------------------------------------------------- 137 | ++ 138 | +`git-daemon.service` 139 | ++ 140 | +---------------------------------------------------------------- 141 | +[Unit] 142 | +Description=Git Daemon 143 | + 144 | +[Service] 145 | +ExecStart=/usr/lib/git-core/git-daemon --systemd --reuseaddr --base-path=/var/lib /var/lib/git 146 | +User=git-daemon 147 | +StandardError=null 148 | +---------------------------------------------------------------- 149 | 150 | ENVIRONMENT 151 | ----------- 152 | diff --git a/Makefile b/Makefile 153 | index 36655d5..54986a0 100644 154 | --- a/Makefile 155 | +++ b/Makefile 156 | @@ -42,6 +42,9 @@ all:: 157 | # Define NO_EXPAT if you do not have expat installed. git-http-push is 158 | # not built, and you cannot push using http:// and https:// transports (dumb). 159 | # 160 | +# Define NO_SYSTEMD to prevent systemd socket activation support from being 161 | +# built into git-daemon. 162 | +# 163 | # Define EXPATDIR=/foo/bar if your expat header and library files are in 164 | # /foo/bar/include and /foo/bar/lib directories. 165 | # 166 | @@ -997,6 +1000,13 @@ ifeq ($(uname_S),Darwin) 167 | PTHREAD_LIBS = 168 | endif 169 | 170 | +ifndef NO_SYSTEMD 171 | + ifeq ($(shell echo "\#include " | $(CC) -E - -o /dev/null 2>/dev/null && echo y),y) 172 | + BASIC_CFLAGS += -DHAVE_SYSTEMD 173 | + EXTLIBS += -lsystemd 174 | + endif 175 | +endif 176 | + 177 | ifndef CC_LD_DYNPATH 178 | ifdef NO_R_TO_GCC_LINKER 179 | # Some gcc does not accept and pass -R to the linker to specify 180 | diff --git a/daemon.c b/daemon.c 181 | index d3d3e43..42e1441 100644 182 | --- a/daemon.c 183 | +++ b/daemon.c 184 | @@ -1,3 +1,7 @@ 185 | +#ifdef HAVE_SYSTEMD 186 | +# include 187 | +#endif 188 | + 189 | #include "cache.h" 190 | #include "pkt-line.h" 191 | #include "exec_cmd.h" 192 | @@ -28,7 +32,11 @@ static const char daemon_usage[] = 193 | " [--(enable|disable|allow-override|forbid-override)=]\n" 194 | " [--access-hook=]\n" 195 | " [--inetd | [--listen=] [--port=]\n" 196 | +#ifdef HAVE_SYSTEMD 197 | +" [--systemd | [--detach] [--user= [--group=]]]\n" /* exactly 80 characters */ 198 | +#else 199 | " [--detach] [--user= [--group=]]\n" 200 | +#endif 201 | " [...]"; 202 | 203 | /* List of acceptable pathname prefixes */ 204 | @@ -1166,12 +1174,40 @@ static struct credentials *prepare_credentials(const char *user_name, 205 | } 206 | #endif 207 | 208 | +#ifdef HAVE_SYSTEMD 209 | +static int enumerate_sockets(struct socketlist *socklist, struct string_list *listen_addr, int listen_port, int systemd_mode) 210 | +{ 211 | + if (systemd_mode) { 212 | + int i, n; 213 | + 214 | + n = sd_listen_fds(0); 215 | + if (n <= 0) 216 | + die("--systemd mode specified and no file descriptors recieved"); 217 | + ALLOC_GROW(socklist->list, socklist->nr + n, socklist->alloc); 218 | + for (i = 0; i < n; i++) 219 | + socklist->list[socklist->nr++] = SD_LISTEN_FDS_START + i; 220 | + } 221 | + 222 | + if (listen_addr->nr > 0 || !systemd_mode) 223 | + socksetup(listen_addr, listen_port, socklist); 224 | + 225 | + return 0; 226 | +} 227 | +#else 228 | +static int enumerate_sockets(struct socketlist *socklist, struct string_list *listen_addr, int listen_port, int systemd_mode) 229 | +{ 230 | + socksetup(listen_addr, listen_port, socklist); 231 | + 232 | + return 0; 233 | +} 234 | +#endif 235 | + 236 | static int serve(struct string_list *listen_addr, int listen_port, 237 | - struct credentials *cred) 238 | + struct credentials *cred, int systemd_mode) 239 | { 240 | struct socketlist socklist = { NULL, 0, 0 }; 241 | 242 | - socksetup(listen_addr, listen_port, &socklist); 243 | + enumerate_sockets(&socklist, listen_addr, listen_port, systemd_mode); 244 | if (socklist.nr == 0) 245 | die("unable to allocate any listen sockets on port %u", 246 | listen_port); 247 | @@ -1187,7 +1223,7 @@ int main(int argc, char **argv) 248 | { 249 | int listen_port = 0; 250 | struct string_list listen_addr = STRING_LIST_INIT_NODUP; 251 | - int serve_mode = 0, inetd_mode = 0; 252 | + int serve_mode = 0, inetd_mode = 0, systemd_mode = 0; 253 | const char *pid_file = NULL, *user_name = NULL, *group_name = NULL; 254 | int detach = 0; 255 | struct credentials *cred = NULL; 256 | @@ -1322,6 +1358,12 @@ int main(int argc, char **argv) 257 | informative_errors = 0; 258 | continue; 259 | } 260 | +#ifdef HAVE_SYSTEMD 261 | + if (!strcmp(arg, "--systemd")) { 262 | + systemd_mode = 1; 263 | + continue; 264 | + } 265 | +#endif 266 | if (!strcmp(arg, "--")) { 267 | ok_paths = &argv[i+1]; 268 | break; 269 | @@ -1340,8 +1382,16 @@ int main(int argc, char **argv) 270 | /* avoid splitting a message in the middle */ 271 | setvbuf(stderr, NULL, _IOFBF, 4096); 272 | 273 | - if (inetd_mode && (detach || group_name || user_name)) 274 | - die("--detach, --user and --group are incompatible with --inetd"); 275 | + if ((inetd_mode || systemd_mode) && (detach || group_name || user_name)) 276 | + die("--detach, --user and --group are incompatible with --inetd and --systemd"); 277 | + 278 | +#ifdef HAVE_SYSTEMD 279 | + if (systemd_mode && inetd_mode) 280 | + die("--inetd is incompatible with --systemd"); 281 | + 282 | + if (systemd_mode && !sd_booted()) 283 | + die("--systemd passed and not invoked from systemd"); 284 | +#endif 285 | 286 | if (inetd_mode && (listen_port || (listen_addr.nr > 0))) 287 | die("--listen= and --port= are incompatible with --inetd"); 288 | @@ -1386,5 +1436,5 @@ int main(int argc, char **argv) 289 | cld_argv[i+1] = argv[i]; 290 | cld_argv[argc+1] = NULL; 291 | 292 | - return serve(&listen_addr, listen_port, cred); 293 | + return serve(&listen_addr, listen_port, cred, systemd_mode); 294 | } 295 | -- 296 | 2.2.1.209.g41e5f3a 297 | 298 | --------------------------------------------------------------------------------