├── .gitattributes ├── .github ├── dependabot.yml └── workflows │ └── main.yml ├── .gitignore ├── .nuke ├── build.schema.json └── parameters.json ├── LICENSE.txt ├── NuGet.Config ├── README.md ├── build.cmd ├── build.ps1 ├── build.sh ├── build ├── .editorconfig ├── Build.cs ├── Configuration.cs ├── Directory.Build.props ├── Directory.Build.targets ├── _build.csproj └── _build.csproj.DotSettings ├── global.json ├── octoversion.json └── source ├── .editorconfig ├── Octodiff.Benchmarks ├── DeltaApplierBenchmarks.cs ├── Octodiff.Benchmarks.csproj ├── Program.cs └── RandomDataGeneratorStream.cs ├── Octodiff.Tests ├── Core │ ├── Adler32RollingChecksumFixture.cs │ ├── Adler32RollingChecksumV2Fixture.cs │ ├── BinaryDeltaReaderFixture.cs │ ├── BinaryDeltaWriterFixture.cs │ ├── DeltaBuilderFixture.cs │ ├── SignatureBuilderFixture.cs │ └── SignatureReaderFixture.cs ├── DeltaFixture.cs ├── HelpFixture.cs ├── Octodiff.Tests.csproj ├── PackageGenerator.cs ├── PatchFixture.cs ├── Program.cs ├── Properties │ └── AssemblyInfo.cs ├── SignatureFixture.cs ├── Timings.cs └── Util │ ├── CommandLineFixture.cs │ ├── Helpers.cs │ └── SilentProcessRunner.cs ├── Octodiff.sln ├── Octodiff.sln.DotSettings ├── Octodiff.vssettings ├── Octodiff ├── CommandLine │ ├── DeltaCommand.cs │ ├── ExplainDeltaCommand.cs │ ├── HelpCommand.cs │ ├── PatchCommand.cs │ ├── SignatureCommand.cs │ └── Support │ │ ├── CommandAttribute.cs │ │ ├── CommandException.cs │ │ ├── CommandLocator.cs │ │ ├── ICommand.cs │ │ ├── ICommandLocator.cs │ │ ├── ICommandMetadata.cs │ │ └── NDesk.Options.cs ├── Core │ ├── Adler32RollingChecksum.cs │ ├── Adler32RollingChecksumV2.cs │ ├── AggregateCopyOperationsDecorator.cs │ ├── BinaryDeltaReader.cs │ ├── BinaryDeltaWriter.cs │ ├── BinaryFormat.cs │ ├── ChunkSignature.cs │ ├── ChunkSignatureChecksumComparer.cs │ ├── CompatibilityException.cs │ ├── CorruptFileFormatException.cs │ ├── DataRange.cs │ ├── DeltaApplier.cs │ ├── DeltaBuilder.cs │ ├── DeltaStatistics.cs │ ├── HashAlgorithmWrapper.cs │ ├── IDeltaReader.cs │ ├── IDeltaWriter.cs │ ├── IHashAlgorithm.cs │ ├── IRollingChecksum.cs │ ├── ISignatureReader.cs │ ├── ISignatureWriter.cs │ ├── Signature.cs │ ├── SignatureBuilder.cs │ ├── SignatureReader.cs │ ├── SignatureWriter.cs │ ├── SupportedAlgorithms.cs │ └── UsageException.cs ├── Diagnostics │ ├── ConsoleProgressReporter.cs │ ├── IProgressReporter.cs │ └── NullProgressReporter.cs ├── Octodiff.csproj ├── Program.cs └── Properties │ └── AssemblyInfo.cs └── Tests.ps1 /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | # Check for updates to GitHub Actions every week 8 | interval: "weekly" 9 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Build, Test, Package and Push 2 | 3 | # Controls when the action will run. 4 | on: 5 | push: 6 | # Triggers the workflow on pull request events and merges/pushes to master 7 | branches: 8 | - master 9 | - release/* 10 | tags-ignore: 11 | - '**' 12 | 13 | pull_request: 14 | types: [opened, synchronize, reopened] 15 | 16 | schedule: 17 | # Daily 5am australian/brisbane time 18 | - cron: '0 19 * * *' 19 | 20 | # Allows you to run this workflow manually from the Actions tab 21 | workflow_dispatch: 22 | 23 | env: 24 | OCTOVERSION_CurrentBranch: ${{ github.head_ref || github.ref }} 25 | OCTOPUS_SPACE: "Core Platform" 26 | 27 | jobs: 28 | test-linux: 29 | runs-on: ubuntu-latest 30 | steps: 31 | - uses: actions/checkout@v4 32 | with: 33 | fetch-depth: 0 # all 34 | 35 | - name: Setup NET 8.0 36 | uses: actions/setup-dotnet@v4 37 | with: 38 | dotnet-version: '8.0.x' 39 | 40 | - name: Test NET 8 41 | run: ./build.sh -target Test 42 | 43 | - name: Linux unit test report 44 | uses: dorny/test-reporter@6e6a65b7a0bd2c9197df7d0ae36ac5cee784230c # v2.0.0 45 | if: success() || failure() # run this step even if previous step failed 46 | with: 47 | name: Linux unit test results 48 | path: ./TestResults/*.trx 49 | reporter: dotnet-trx 50 | fail-on-error: true 51 | 52 | build-release-windows: 53 | needs: test-linux 54 | runs-on: windows-latest 55 | permissions: 56 | id-token: write # Required to obtain the ID token from GitHub Actions 57 | contents: write # Read Required to check out code, Write to create Git Tags 58 | checks: write # Required for test-reporter 59 | steps: 60 | # Must clone the entire history for OctoVersion to work 61 | - name: Checkout 62 | uses: actions/checkout@v4 63 | with: 64 | fetch-depth: 0 65 | 66 | - name: Setup .NET 8 67 | uses: actions/setup-dotnet@v4 68 | with: 69 | dotnet-version: 8.0.x 70 | 71 | - name: Append OCTOVERSION_CurrentBranch with -nightly- (for scheduled) 72 | if: github.event_name == 'schedule' 73 | run: echo "OCTOVERSION_CurrentBranch=${{ env.OCTOVERSION_CurrentBranch }}-nightly-$(Get-Date -Format 'yyyyMMddHHmmss')" >> $env:GITHUB_ENV 74 | 75 | - name: Nuke Build 🏗 76 | id: build 77 | run: ./build.ps1 --verbosity verbose 78 | 79 | - name: Windows unit test report 80 | uses: dorny/test-reporter@6e6a65b7a0bd2c9197df7d0ae36ac5cee784230c # v2.0.0 81 | if: success() || failure() # run this step even if previous step failed 82 | with: 83 | name: Windows unit test results 84 | path: ./TestResults/*.trx 85 | reporter: dotnet-trx 86 | fail-on-error: true 87 | 88 | - name: Tag release (when not pre-release) 🏷️ 89 | if: ${{ !contains( steps.build.outputs.octoversion_fullsemver, '-' ) }} 90 | uses: actions/github-script@v7 91 | with: 92 | github-token: ${{ github.token }} 93 | script: | 94 | github.rest.git.createRef({ 95 | owner: context.repo.owner, 96 | repo: context.repo.repo, 97 | ref: "refs/tags/${{ steps.build.outputs.octoversion_fullsemver }}", 98 | sha: context.sha 99 | }) 100 | 101 | - name: Login to Octopus Deploy 🐙 102 | if: (! contains(github.ref, '/merge')) && (! contains(github.ref, '/dependabot/')) && (! contains(github.ref, 'prettybot/')) 103 | uses: OctopusDeploy/login@v1 104 | with: 105 | server: ${{ secrets.OCTOPUS_URL }} 106 | service_account_id: 44daf805-da46-4efb-a403-a3b656dc31fc 107 | 108 | - name: Push to Octopus 🐙 109 | uses: OctopusDeploy/push-package-action@v3 110 | if: (! contains(github.ref, '/merge')) && (! contains(github.ref, '/dependabot/')) && (! contains(github.ref, 'prettybot/')) 111 | with: 112 | packages: | 113 | ./artifacts/Octopus.Octodiff.${{ steps.build.outputs.octoversion_fullsemver }}.nupkg 114 | 115 | - name: Create Release in Octopus 🐙 116 | uses: OctopusDeploy/create-release-action@v3 117 | if: (! contains(github.ref, '/merge')) && (! contains(github.ref, '/dependabot/')) && (! contains(github.ref, 'prettybot/')) 118 | with: 119 | project: Octodiff 120 | packages: | 121 | Octopus.Octodiff:${{ steps.build.outputs.octoversion_fullsemver }} 122 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.sln.docstates 8 | .vs/ 9 | .vscode/ 10 | .idea/ 11 | 12 | # Build results 13 | [Dd]ebug/ 14 | [Dd]ebugPublic/ 15 | [Rr]elease/ 16 | x64/ 17 | bld/ 18 | [Bb]in/ 19 | [Oo]bj/ 20 | 21 | # MSTest test Results 22 | [Tt]est[Rr]esult*/ 23 | [Bb]uild[Ll]og.* 24 | 25 | #NUNIT 26 | *.VisualState.xml 27 | TestResult.xml 28 | 29 | # Build Results of an ATL Project 30 | [Dd]ebugPS/ 31 | [Rr]eleasePS/ 32 | dlldata.c 33 | 34 | *_i.c 35 | *_p.c 36 | *_i.h 37 | *.ilk 38 | *.meta 39 | *.obj 40 | *.pch 41 | *.pdb 42 | *.pgc 43 | *.pgd 44 | *.rsp 45 | *.sbr 46 | *.tlb 47 | *.tli 48 | *.tlh 49 | *.tmp 50 | *.tmp_proj 51 | *.log 52 | *.vspscc 53 | *.vssscc 54 | .builds 55 | *.pidb 56 | *.svclog 57 | *.scc 58 | 59 | # Chutzpah Test files 60 | _Chutzpah* 61 | 62 | # Visual C++ cache files 63 | ipch/ 64 | *.aps 65 | *.ncb 66 | *.opensdf 67 | *.sdf 68 | *.cachefile 69 | 70 | # Visual Studio profiler 71 | *.psess 72 | *.vsp 73 | *.vspx 74 | 75 | # TFS 2012 Local Workspace 76 | $tf/ 77 | 78 | # Guidance Automation Toolkit 79 | *.gpState 80 | 81 | # ReSharper is a .NET coding add-in 82 | _ReSharper*/ 83 | *.[Rr]e[Ss]harper 84 | *.DotSettings.user 85 | 86 | # JustCode is a .NET coding addin-in 87 | .JustCode 88 | 89 | # TeamCity is a build add-in 90 | _TeamCity* 91 | 92 | # DotCover is a Code Coverage Tool 93 | *.dotCover 94 | 95 | # NCrunch 96 | *.ncrunch* 97 | _NCrunch_* 98 | .*crunch*.local.xml 99 | 100 | # MightyMoose 101 | *.mm.* 102 | AutoTest.Net/ 103 | 104 | # Web workbench (sass) 105 | .sass-cache/ 106 | 107 | # Installshield output folder 108 | [Ee]xpress/ 109 | 110 | # DocProject is a documentation generator add-in 111 | DocProject/buildhelp/ 112 | DocProject/Help/*.HxT 113 | DocProject/Help/*.HxC 114 | DocProject/Help/*.hhc 115 | DocProject/Help/*.hhk 116 | DocProject/Help/*.hhp 117 | DocProject/Help/Html2 118 | DocProject/Help/html 119 | 120 | # Click-Once directory 121 | publish/ 122 | 123 | # Publish Web Output 124 | *.[Pp]ublish.xml 125 | *.azurePubxml 126 | 127 | # NuGet Packages Directory 128 | packages/ 129 | ## TODO: If the tool you use requires repositories.config uncomment the next line 130 | #!packages/repositories.config 131 | 132 | # Enable "build/" folder in the NuGet Packages folder since NuGet packages use it for MSBuild targets 133 | # This line needs to be after the ignore of the build folder (and the packages folder if the line above has been uncommented) 134 | !packages/build/ 135 | 136 | # Windows Azure Build Output 137 | csx/ 138 | *.build.csdef 139 | 140 | # Windows Store app package directory 141 | AppPackages/ 142 | 143 | # Others 144 | sql/ 145 | *.Cache 146 | ClientBin/ 147 | [Ss]tyle[Cc]op.* 148 | ~$* 149 | *~ 150 | *.dbmdl 151 | *.dbproj.schemaview 152 | *.pfx 153 | *.publishsettings 154 | node_modules/ 155 | 156 | # RIA/Silverlight projects 157 | Generated_Code/ 158 | 159 | # Backup & report files from converting an old project file to a newer 160 | # Visual Studio version. Backup files are not needed, because we have git ;-) 161 | _UpgradeReport_Files/ 162 | Backup*/ 163 | UpgradeLog*.XML 164 | UpgradeLog*.htm 165 | 166 | # SQL Server files 167 | *.mdf 168 | *.ldf 169 | 170 | # Business Intelligence projects 171 | *.rdl.data 172 | *.bim.layout 173 | *.bim_*.settings 174 | 175 | # Microsoft Fakes 176 | FakesAssemblies/ 177 | project.lock.json 178 | source/SmallPackage* 179 | SmallPackage* 180 | tools/ 181 | artifacts/ 182 | -------------------------------------------------------------------------------- /.nuke/build.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-04/schema#", 3 | "definitions": { 4 | "Host": { 5 | "type": "string", 6 | "enum": [ 7 | "AppVeyor", 8 | "AzurePipelines", 9 | "Bamboo", 10 | "Bitbucket", 11 | "Bitrise", 12 | "GitHubActions", 13 | "GitLab", 14 | "Jenkins", 15 | "Rider", 16 | "SpaceAutomation", 17 | "TeamCity", 18 | "Terminal", 19 | "TravisCI", 20 | "VisualStudio", 21 | "VSCode" 22 | ] 23 | }, 24 | "ExecutableTarget": { 25 | "type": "string", 26 | "enum": [ 27 | "CalculateVersion", 28 | "Clean", 29 | "Compile", 30 | "Pack", 31 | "Restore", 32 | "Test" 33 | ] 34 | }, 35 | "Verbosity": { 36 | "type": "string", 37 | "description": "", 38 | "enum": [ 39 | "Verbose", 40 | "Normal", 41 | "Minimal", 42 | "Quiet" 43 | ] 44 | }, 45 | "NukeBuild": { 46 | "properties": { 47 | "Continue": { 48 | "type": "boolean", 49 | "description": "Indicates to continue a previously failed build attempt" 50 | }, 51 | "Help": { 52 | "type": "boolean", 53 | "description": "Shows the help text for this build assembly" 54 | }, 55 | "Host": { 56 | "description": "Host for execution. Default is 'automatic'", 57 | "$ref": "#/definitions/Host" 58 | }, 59 | "NoLogo": { 60 | "type": "boolean", 61 | "description": "Disables displaying the NUKE logo" 62 | }, 63 | "Partition": { 64 | "type": "string", 65 | "description": "Partition to use on CI" 66 | }, 67 | "Plan": { 68 | "type": "boolean", 69 | "description": "Shows the execution plan (HTML)" 70 | }, 71 | "Profile": { 72 | "type": "array", 73 | "description": "Defines the profiles to load", 74 | "items": { 75 | "type": "string" 76 | } 77 | }, 78 | "Root": { 79 | "type": "string", 80 | "description": "Root directory during build execution" 81 | }, 82 | "Skip": { 83 | "type": "array", 84 | "description": "List of targets to be skipped. Empty list skips all dependencies", 85 | "items": { 86 | "$ref": "#/definitions/ExecutableTarget" 87 | } 88 | }, 89 | "Target": { 90 | "type": "array", 91 | "description": "List of targets to be invoked. Default is '{default_target}'", 92 | "items": { 93 | "$ref": "#/definitions/ExecutableTarget" 94 | } 95 | }, 96 | "Verbosity": { 97 | "description": "Logging verbosity during build execution. Default is 'Normal'", 98 | "$ref": "#/definitions/Verbosity" 99 | } 100 | } 101 | } 102 | }, 103 | "allOf": [ 104 | { 105 | "properties": { 106 | "AutoDetectBranch": { 107 | "type": "boolean", 108 | "description": "Whether to auto-detect the branch name - this is okay for a local build, but should not be used under CI" 109 | }, 110 | "OCTOVERSION_CurrentBranch": { 111 | "type": "string", 112 | "description": "Branch name for OctoVersion to use to calculate the version number. Can be set via the environment variable OCTOVERSION_CurrentBranch" 113 | }, 114 | "Solution": { 115 | "type": "string", 116 | "description": "Path to a solution file that is automatically loaded" 117 | }, 118 | "where": { 119 | "type": "string", 120 | "description": "Test filter expression" 121 | } 122 | } 123 | }, 124 | { 125 | "$ref": "#/definitions/NukeBuild" 126 | } 127 | ] 128 | } 129 | -------------------------------------------------------------------------------- /.nuke/parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "./build.schema.json", 3 | "Solution": "source/Octodiff.sln" 4 | } -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) Octopus Deploy and contributors. All rights reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use 4 | these files except in compliance with the License. You may obtain a copy of the 5 | License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software distributed 10 | under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 11 | CONDITIONS OF ANY KIND, either express or implied. See the License for the 12 | specific language governing permissions and limitations under the License. 13 | -------------------------------------------------------------------------------- /NuGet.Config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Octodiff is a 100% managed implementation of remote delta compression. Usage is inspired by [rdiff](https://librsync.github.io/rdiff.html), and like rdiff the [algorithm is based on rsync](http://rsync.samba.org/tech_report/tech_report.html). Octodiff was designed to be used in [Octopus Deploy](http://octopusdeploy.com), an automated deployment tool for .NET developers. 2 | 3 | Octodiff can make deltas of files by comparing a remote file with a local file, without needing both files to exist on the same machine. It does this in three phases: 4 | 5 | 1. Machine A reads the *basis file*, and computes a *signature* of all the chunks in the file 6 | 2. Machine B uses the *signature* and the *new file*, to produce a *delta file*, specifying what changes need to be made 7 | 3. Machine A applies the *delta file* to the *basis file*, which produces an exact copy of the *new file* 8 | 9 | Of course, the benefit is that instead of transferring the entire *new file* to machine A, we just transfer a small signature, and a delta file containing the differences. We're trading off CPU usage for potentially large bandwidth savings. 10 | 11 | Octodiff is an executable, but can also be referenced and used as any .NET assembly. 12 | 13 | ## Signatures 14 | 15 | ``` 16 | Usage: Octodiff signature [] [] 17 | 18 | Arguments: 19 | 20 | basis-file The file to read and create a signature from. 21 | signature-file The file to write the signature to. 22 | 23 | Options: 24 | 25 | --chunk-size=VALUE Maximum bytes per chunk. Defaults to 2048. Min of 26 | 128, max of 31744. 27 | --progress Whether progress should be written to stdout 28 | ``` 29 | 30 | Example: 31 | 32 | ``` 33 | octodiff signature MyApp.1.0.nupkg MyApp.1.0.nupkg.octosig --progress 34 | ``` 35 | 36 | This command calculates the signature of a given file. As per the rsync algorithm, the signature is calculated by reading the file into fixed-size chunks, and then calculating a signature of each chunk. The resulting signature file contains: 37 | 38 | - Metadata about the signature file and algorithms used 39 | - Hash of the file that the signature was created from (*basis file hash*) 40 | - A list of chunk signatures, each 26 bytes long, consisting of: 41 | - The length of the chunk (short) 42 | - Rolling checksum (uint) - calculated using Adler32 43 | - Hash (20 bytes) - calculated using SHA1 44 | 45 | Given that the default chunk size is 2048 bytes, and this is turned into a 26 byte signature, the resulting file is about 1.3% of the size of the original. For example, a 306MB file creates a 3.9MB signature file. The signature of a 300mb file can be calculated in ~3 seconds using ~6mb of memory on a 2013 Macbook Pro. Memory usage during signature calculation should remain constant no matter the size of the file. 46 | 47 | ## Deltas 48 | 49 | ``` 50 | Usage: Octodiff delta [] [] 51 | 52 | Arguments: 53 | 54 | signature-file The file containing the signature from the basis 55 | file. 56 | new-file The file to create the delta from. 57 | delta-file The file to write the delta to. 58 | 59 | Options: 60 | 61 | --progress Whether progress should be written to stdout 62 | ``` 63 | 64 | Example: 65 | 66 | ``` 67 | octodiff delta MyApp.1.0.nupkg.octosig MyApp.1.1.nupkg MyApp.1.0_to_1.1.octodelta --progress 68 | ``` 69 | 70 | This command creates a delta, that specfies how the *basis-file* (using just the information in its *signature file*) can be turned into the *new-file*. First, the signature file is read into memory. Then we scan the new file, looking for chunks that we find in the signature. You can learn more about the process in [the rsync algorithm](http://rsync.samba.org/tech_report/node4.html). 71 | 72 | The delta file contains: 73 | 74 | - Metadata about the signature file and algorithms used 75 | - Hash of the file that the original signature was created from (*basis file hash*) 76 | - A series of instructions to re-create the *new-file* which reference the *basis file*. 77 | 78 | Instructions are either copy commands (read offset X, length Y from the *basis file*) or data commands (add this data). Example: 79 | 80 | 1. Copy 0x0000 to 0x8C00 81 | 2. Data: 5C 9F D9 C7... 82 | 3. Copy 0x8C31 to 0x93C0 83 | 84 | The delta file uses a binary file format to keep encoding overhead to a minimum - copy instructions start with 0x60 and then the start offset and length; data commands are 0x80 followed by the length of the data and then the data to copy. 85 | 86 | For debugging, you can use the following command to print an explanation of what is in a given delta file: 87 | 88 | ``` 89 | octodiff explain-delta MyApp.1.0_to_1.1.octodelta 90 | ``` 91 | 92 | ## Patching 93 | 94 | ``` 95 | Usage: Octodiff patch [] 96 | 97 | Arguments: 98 | 99 | basis-file The file that the delta was created for. 100 | delta-file The delta to apply to the basis file 101 | new-file The file to write the result to. 102 | 103 | Options: 104 | 105 | --progress Whether progress should be written to stdout 106 | --skip-verification Skip checking whether the basis file is the same 107 | as the file used to produce the signature that 108 | created the delta. 109 | ``` 110 | 111 | Example: 112 | 113 | ``` 114 | octodiff patch MyApp.1.0.nupkg MyApp.1.0_to_1.1.octodelta MyApp.1.1.nupkg --progress 115 | ``` 116 | 117 | This command recreates the *new-file* using simply the *basis-file* and the *delta-file*. 118 | 119 | Applying the delta is the easiest part of the process. We simply open the *delta-file*, and follow the instructions. When there's a copy instruction, we seek to that offset in the *basis-file* and copy until we hit the length. When we encounter a data instruction, we append that data. At the end of the process, we have the *new-file*. 120 | 121 | Octodiff embeds a SHA1 hash of the *new-file* in the *delta-file*. After patching, Octodiff compares this hash to the SHA1 hash of the resulting patched file. If they don't match, Octodiff returns a non-zero exit code. 122 | 123 | ## Performance 124 | 125 | **The following section isn't meant to be mathematically accurate, but to give you a rough idea of real-world performance to expect from Octodiff. The tests were done on a Windows 8 VM, running in a 2013 Macbook Pro, with 4 cores and 8GB of memory assigned to the VM. The machine uses an SSD which mean the I/O bound tasks could run significantly slower on non-SSD drives. All measurements were done using simply Windows Task Manager.** 126 | 127 | Signature creation is relatively easy - we're reading the file in fixed-size chunks and computing a checksum. Memory usage should be constant no matter how big the file is - around 8.2 MB. 128 | 129 | - The signature for an 85 MB file can be calculated in ~832 ms 130 | - The signature for a 4.4 GB file can be calculated in ~36 seconds 131 | 132 | We also compute a SHA1 hash of the entire basis file (this takes around 1/3 of the total time above). The resulting signature file size is always ~1.3% of the basis file size. 133 | 134 | Delta creation is the most CPU and memory-intensive aspect of the whole process. First, we assume that we can fit all signatures into memory, which means at a minimum we'll consume at least ~1.3% of the *basis file* in memory, plus extra to store a dictionary of the chunks and buffers as we read data. Budget for about 5x the signature file size in memory (e.g., for a 57 MB signature file (a 4.4 GB basis file), expect to use 250mb of memory). 135 | 136 | - Delta from a 85 MB file took 5 seconds 137 | - Delta from a 4.3 GB ISO took 170 seconds 138 | 139 | Delta creation takes roughly the same amount of time whether there are many differences or none at all. If there are many differences, the resulting delta file will be much larger, so additional I/O producing it may have an impact. 140 | 141 | Patching is the fastest part of the algorithm. 142 | 143 | ## Output and exit codes 144 | 145 | If all goes well, Octodiff produces no output. You can use the `--progress` switch to write progress messages to stdout. 146 | 147 | Octodiff uses the following exit codes: 148 | 149 | - `0` - success 150 | - `1` - environmental problems 151 | - `2` - corrupt signature or delta file 152 | - `3` - internal error or unhandled situation 153 | - `4` - usage problem (you did something wrong, maybe passing the wrong file) 154 | 155 | ## Using OctoDiff classes within your own application 156 | 157 | To use the OctoDiff classes to create signature/delta/final files from within your own application, you can use the below example which creates the signature and delta file and then applies the delta file to create the new file. 158 | 159 | ```csharp 160 | // Create signature file 161 | var signatureBaseFilePath = @"C:\OctoDiffExample\MyPackage.1.0.0.zip"; 162 | var signatureFilePath = @"C:\OctoDiffExample\Output\MyPackage.1.0.0.zip.octosig"; 163 | var signatureOutputDirectory = Path.GetDirectoryName(signatureFilePath); 164 | if(!Directory.Exists(signatureOutputDirectory)) 165 | Directory.CreateDirectory(signatureOutputDirectory); 166 | var signatureBuilder = new SignatureBuilder(); 167 | using (var basisStream = new FileStream(signatureBaseFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) 168 | using (var signatureStream = new FileStream(signatureFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) 169 | { 170 | signatureBuilder.Build(basisStream, new SignatureWriter(signatureStream)); 171 | } 172 | 173 | // Create delta file 174 | var newFilePath = @"C:\OctoDiffExample\MyPackage.1.0.1.zip"; 175 | var deltaFilePath = @"C:\OctoDiffExample\Output\MyPackage.1.0.1.zip.octodelta"; 176 | var deltaOutputDirectory = Path.GetDirectoryName(deltaFilePath); 177 | if(!Directory.Exists(deltaOutputDirectory)) 178 | Directory.CreateDirectory(deltaOutputDirectory); 179 | var deltaBuilder = new DeltaBuilder(); 180 | using(var newFileStream = new FileStream(newFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) 181 | using(var signatureFileStream = new FileStream(signatureFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) 182 | using(var deltaStream = new FileStream(deltaFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) 183 | { 184 | deltaBuilder.BuildDelta(newFileStream, new SignatureReader(signatureFileStream, new ConsoleProgressReporter()), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))); 185 | } 186 | 187 | // Apply delta file to create new file 188 | var newFilePath2 = @"C:\OctoDiffExample\Output\MyPackage.1.0.1.zip"; 189 | var newFileOutputDirectory = Path.GetDirectoryName(newFilePath2); 190 | if(!Directory.Exists(newFileOutputDirectory)) 191 | Directory.CreateDirectory(newFileOutputDirectory); 192 | var deltaApplier = new DeltaApplier { SkipHashCheck = false }; 193 | using(var basisStream = new FileStream(signatureBaseFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) 194 | using(var deltaStream = new FileStream(deltaFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) 195 | using(var newFileStream = new FileStream(newFilePath2, FileMode.Create, FileAccess.ReadWrite, FileShare.Read)) 196 | { 197 | deltaApplier.Apply(basisStream, new BinaryDeltaReader(deltaStream, new ConsoleProgressReporter()), newFileStream); 198 | } 199 | ``` 200 | 201 | ## Development 202 | You need: 203 | - VSCode or Visual Studio 15.3 to compile the solution 204 | - .NET Core 2.0 SDK (https://download.microsoft.com/download/0/F/D/0FD852A4-7EA1-4E2A-983A-0484AC19B92C/dotnet-sdk-2.0.0-win-x64.exe) 205 | 206 | Run `Build.cmd` to build, test and package the project. 207 | 208 | To release to Nuget, tag `master` with the next major, minor or patch number, [TeamCity](https://build.octopushq.com/viewType.html?buildTypeId=OctopusDeploy_LIbraries_Octodiff) will do the rest. 209 | 210 | Every successful TeamCity build for all branches will be pushed to MyGet. 211 | -------------------------------------------------------------------------------- /build.cmd: -------------------------------------------------------------------------------- 1 | :; set -eo pipefail 2 | :; SCRIPT_DIR=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd) 3 | :; ${SCRIPT_DIR}/build.sh "$@" 4 | :; exit $? 5 | 6 | @ECHO OFF 7 | powershell -ExecutionPolicy ByPass -NoProfile -File "%~dp0build.ps1" %* 8 | -------------------------------------------------------------------------------- /build.ps1: -------------------------------------------------------------------------------- 1 | [CmdletBinding()] 2 | Param( 3 | [Parameter(Position=0,Mandatory=$false,ValueFromRemainingArguments=$true)] 4 | [string[]]$BuildArguments 5 | ) 6 | 7 | Write-Output "PowerShell $($PSVersionTable.PSEdition) version $($PSVersionTable.PSVersion)" 8 | 9 | Set-StrictMode -Version 2.0; $ErrorActionPreference = "Stop"; $ConfirmPreference = "None"; trap { Write-Error $_ -ErrorAction Continue; exit 1 } 10 | $PSScriptRoot = Split-Path $MyInvocation.MyCommand.Path -Parent 11 | 12 | ########################################################################### 13 | # CONFIGURATION 14 | ########################################################################### 15 | 16 | $BuildProjectFile = "$PSScriptRoot\build\_build.csproj" 17 | $TempDirectory = "$PSScriptRoot\\.nuke\temp" 18 | 19 | $DotNetGlobalFile = "$PSScriptRoot\\global.json" 20 | $DotNetInstallUrl = "https://dot.net/v1/dotnet-install.ps1" 21 | $DotNetChannel = "STS" 22 | 23 | $env:DOTNET_CLI_TELEMETRY_OPTOUT = 1 24 | $env:DOTNET_NOLOGO = 1 25 | 26 | ########################################################################### 27 | # EXECUTION 28 | ########################################################################### 29 | 30 | function ExecSafe([scriptblock] $cmd) { 31 | & $cmd 32 | if ($LASTEXITCODE) { exit $LASTEXITCODE } 33 | } 34 | 35 | # If dotnet CLI is installed globally and it matches requested version, use for execution 36 | if ($null -ne (Get-Command "dotnet" -ErrorAction SilentlyContinue) -and ` 37 | $(dotnet --version) -and $LASTEXITCODE -eq 0) { 38 | $env:DOTNET_EXE = (Get-Command "dotnet").Path 39 | } 40 | else { 41 | # Download install script 42 | $DotNetInstallFile = "$TempDirectory\dotnet-install.ps1" 43 | New-Item -ItemType Directory -Path $TempDirectory -Force | Out-Null 44 | [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 45 | (New-Object System.Net.WebClient).DownloadFile($DotNetInstallUrl, $DotNetInstallFile) 46 | 47 | # If global.json exists, load expected version 48 | if (Test-Path $DotNetGlobalFile) { 49 | $DotNetGlobal = $(Get-Content $DotNetGlobalFile | Out-String | ConvertFrom-Json) 50 | if ($DotNetGlobal.PSObject.Properties["sdk"] -and $DotNetGlobal.sdk.PSObject.Properties["version"]) { 51 | $DotNetVersion = $DotNetGlobal.sdk.version 52 | } 53 | } 54 | 55 | # Install by channel or version 56 | $DotNetDirectory = "$TempDirectory\dotnet-win" 57 | if (!(Test-Path variable:DotNetVersion)) { 58 | ExecSafe { & powershell $DotNetInstallFile -InstallDir $DotNetDirectory -Channel $DotNetChannel -NoPath } 59 | } else { 60 | ExecSafe { & powershell $DotNetInstallFile -InstallDir $DotNetDirectory -Version $DotNetVersion -NoPath } 61 | } 62 | $env:DOTNET_EXE = "$DotNetDirectory\dotnet.exe" 63 | $env:PATH = "$DotNetDirectory;$env:PATH" 64 | } 65 | 66 | Write-Output "Microsoft (R) .NET SDK version $(& $env:DOTNET_EXE --version)" 67 | 68 | if (Test-Path env:NUKE_ENTERPRISE_TOKEN) { 69 | & $env:DOTNET_EXE nuget remove source "nuke-enterprise" > $null 70 | & $env:DOTNET_EXE nuget add source "https://f.feedz.io/nuke/enterprise/nuget" --name "nuke-enterprise" --username "PAT" --password $env:NUKE_ENTERPRISE_TOKEN > $null 71 | } 72 | 73 | ExecSafe { & $env:DOTNET_EXE build $BuildProjectFile /nodeReuse:false /p:UseSharedCompilation=false -nologo -clp:NoSummary --verbosity quiet } 74 | ExecSafe { & $env:DOTNET_EXE run --project $BuildProjectFile --no-build -- $BuildArguments } 75 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | bash --version 2>&1 | head -n 1 4 | 5 | set -eo pipefail 6 | SCRIPT_DIR=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd) 7 | 8 | ########################################################################### 9 | # CONFIGURATION 10 | ########################################################################### 11 | 12 | BUILD_PROJECT_FILE="$SCRIPT_DIR/build/_build.csproj" 13 | TEMP_DIRECTORY="$SCRIPT_DIR//.nuke/temp" 14 | 15 | DOTNET_GLOBAL_FILE="$SCRIPT_DIR//global.json" 16 | DOTNET_INSTALL_URL="https://dot.net/v1/dotnet-install.sh" 17 | DOTNET_CHANNEL="STS" 18 | 19 | export DOTNET_CLI_TELEMETRY_OPTOUT=1 20 | export DOTNET_NOLOGO=1 21 | 22 | ########################################################################### 23 | # EXECUTION 24 | ########################################################################### 25 | 26 | function FirstJsonValue { 27 | perl -nle 'print $1 if m{"'"$1"'": "([^"]+)",?}' <<< "${@:2}" 28 | } 29 | 30 | # If dotnet CLI is installed globally and it matches requested version, use for execution 31 | if [ -x "$(command -v dotnet)" ] && dotnet --version &>/dev/null; then 32 | export DOTNET_EXE="$(command -v dotnet)" 33 | else 34 | # Download install script 35 | DOTNET_INSTALL_FILE="$TEMP_DIRECTORY/dotnet-install.sh" 36 | mkdir -p "$TEMP_DIRECTORY" 37 | curl -Lsfo "$DOTNET_INSTALL_FILE" "$DOTNET_INSTALL_URL" 38 | chmod +x "$DOTNET_INSTALL_FILE" 39 | 40 | # If global.json exists, load expected version 41 | if [[ -f "$DOTNET_GLOBAL_FILE" ]]; then 42 | DOTNET_VERSION=$(FirstJsonValue "version" "$(cat "$DOTNET_GLOBAL_FILE")") 43 | if [[ "$DOTNET_VERSION" == "" ]]; then 44 | unset DOTNET_VERSION 45 | fi 46 | fi 47 | 48 | # Install by channel or version 49 | DOTNET_DIRECTORY="$TEMP_DIRECTORY/dotnet-unix" 50 | if [[ -z ${DOTNET_VERSION+x} ]]; then 51 | "$DOTNET_INSTALL_FILE" --install-dir "$DOTNET_DIRECTORY" --channel "$DOTNET_CHANNEL" --no-path 52 | else 53 | "$DOTNET_INSTALL_FILE" --install-dir "$DOTNET_DIRECTORY" --version "$DOTNET_VERSION" --no-path 54 | fi 55 | export DOTNET_EXE="$DOTNET_DIRECTORY/dotnet" 56 | export PATH="$DOTNET_DIRECTORY:$PATH" 57 | fi 58 | 59 | echo "Microsoft (R) .NET SDK version $("$DOTNET_EXE" --version)" 60 | 61 | if [[ ! -z ${NUKE_ENTERPRISE_TOKEN+x} && "$NUKE_ENTERPRISE_TOKEN" != "" ]]; then 62 | "$DOTNET_EXE" nuget remove source "nuke-enterprise" &>/dev/null || true 63 | "$DOTNET_EXE" nuget add source "https://f.feedz.io/nuke/enterprise/nuget" --name "nuke-enterprise" --username "PAT" --password "$NUKE_ENTERPRISE_TOKEN" --store-password-in-clear-text &>/dev/null || true 64 | fi 65 | 66 | "$DOTNET_EXE" build "$BUILD_PROJECT_FILE" /nodeReuse:false /p:UseSharedCompilation=false -nologo -clp:NoSummary --verbosity quiet 67 | "$DOTNET_EXE" run --project "$BUILD_PROJECT_FILE" --no-build -- "$@" 68 | -------------------------------------------------------------------------------- /build/.editorconfig: -------------------------------------------------------------------------------- 1 | [*.cs] 2 | dotnet_style_qualification_for_field = false:warning 3 | dotnet_style_qualification_for_property = false:warning 4 | dotnet_style_qualification_for_method = false:warning 5 | dotnet_style_qualification_for_event = false:warning 6 | dotnet_style_require_accessibility_modifiers = never:warning 7 | 8 | csharp_style_expression_bodied_methods = true:silent 9 | csharp_style_expression_bodied_properties = true:warning 10 | csharp_style_expression_bodied_indexers = true:warning 11 | csharp_style_expression_bodied_accessors = true:warning 12 | -------------------------------------------------------------------------------- /build/Build.cs: -------------------------------------------------------------------------------- 1 | // ReSharper disable RedundantUsingDirective 2 | using Nuke.Common; 3 | using Nuke.Common.CI; 4 | using Nuke.Common.Execution; 5 | using Nuke.Common.IO; 6 | using Nuke.Common.ProjectModel; 7 | using Nuke.Common.Tools.DotNet; 8 | using Nuke.Common.Tools.NuGet; 9 | using Nuke.Common.Tools.OctoVersion; 10 | using Nuke.Common.Utilities.Collections; 11 | using Serilog; 12 | using static Nuke.Common.Tools.DotNet.DotNetTasks; 13 | 14 | [ShutdownDotNetAfterServerBuild] 15 | class Build : NukeBuild 16 | { 17 | const string CiBranchNameEnvVariable = "OCTOVERSION_CurrentBranch"; 18 | 19 | readonly Configuration Configuration = Configuration.Release; 20 | 21 | [Solution] readonly Solution Solution; 22 | 23 | [Parameter("Whether to auto-detect the branch name - this is okay for a local build, but should not be used under CI.")] 24 | readonly bool AutoDetectBranch = IsLocalBuild; 25 | 26 | [Parameter("Branch name for OctoVersion to use to calculate the version number. Can be set via the environment variable " + CiBranchNameEnvVariable + ".", Name = CiBranchNameEnvVariable)] 27 | string BranchName { get; set; } 28 | 29 | [OctoVersion(BranchMember = nameof(BranchName), AutoDetectBranchMember = nameof(AutoDetectBranch), Framework = "net8.0")] 30 | public OctoVersionInfo OctoVersionInfo; 31 | 32 | [Parameter("Test filter expression", Name = "where")] readonly string TestFilter = string.Empty; 33 | 34 | AbsolutePath SourceDirectory => RootDirectory / "source"; 35 | AbsolutePath ArtifactsDirectory => RootDirectory / "artifacts"; 36 | AbsolutePath TestResultsDirectory => RootDirectory / "TestResults"; 37 | 38 | Target Clean => _ => _ 39 | .Before(Restore) 40 | .Executes(() => 41 | { 42 | SourceDirectory.GlobDirectories("**/bin", "**/obj").ForEach(d => d.DeleteDirectory()); 43 | TestResultsDirectory.DeleteDirectory(); 44 | ArtifactsDirectory.CreateOrCleanDirectory(); 45 | }); 46 | 47 | Target Restore => _ => _ 48 | .DependsOn(Clean) 49 | .Executes(() => 50 | { 51 | DotNetRestore(s => s 52 | .SetProjectFile(Solution)); 53 | }); 54 | 55 | Target CalculateVersion => _ => _ 56 | .Executes(() => 57 | { 58 | //all the magic happens inside `[OctoVersionInfo]` above. 59 | }); 60 | 61 | Target Compile => _ => _ 62 | .DependsOn(Clean) 63 | .DependsOn(Restore) 64 | .Executes(() => 65 | { 66 | Log.Information("Building Octodiff v{Version}", OctoVersionInfo.FullSemVer); 67 | DotNetBuild(s => s 68 | .SetProjectFile(Solution) 69 | .SetConfiguration(Configuration) 70 | .SetAssemblyVersion(OctoVersionInfo.MajorMinorPatch) 71 | .SetFileVersion(OctoVersionInfo.MajorMinorPatch) 72 | .SetInformationalVersion(OctoVersionInfo.InformationalVersion) 73 | .EnableNoRestore()); 74 | }); 75 | 76 | Target Test => _ => _ 77 | .DependsOn(Compile) 78 | .Executes(() => 79 | { 80 | DotNetTest(_ => _ 81 | .SetProjectFile(Solution) 82 | .SetConfiguration(Configuration) 83 | .SetLoggers("trx") 84 | .SetVerbosity(DotNetVerbosity.normal) 85 | .SetFilter(TestFilter) 86 | .EnableNoBuild() 87 | .EnableNoRestore() 88 | .SetResultsDirectory(TestResultsDirectory)); 89 | }); 90 | 91 | Target Pack => _ => _ 92 | .DependsOn(CalculateVersion) 93 | .DependsOn(Compile) 94 | .DependsOn(Test) 95 | .Executes(() => 96 | { 97 | DotNetPack(_ => _ 98 | .SetProject(Solution) 99 | .SetConfiguration(Configuration) 100 | .SetOutputDirectory(ArtifactsDirectory) 101 | .SetNoBuild(true) 102 | .AddProperty("Version", OctoVersionInfo.FullSemVer) 103 | ); 104 | }); 105 | 106 | /// Support plugins are available for: 107 | /// - JetBrains ReSharper https://nuke.build/resharper 108 | /// - JetBrains Rider https://nuke.build/rider 109 | /// - Microsoft VisualStudio https://nuke.build/visualstudio 110 | /// - Microsoft VSCode https://nuke.build/vscode 111 | public static int Main() => Execute(x => x.Pack); 112 | } 113 | -------------------------------------------------------------------------------- /build/Configuration.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.ComponentModel; 3 | using System.Linq; 4 | using Nuke.Common.Tooling; 5 | 6 | [TypeConverter(typeof(TypeConverter))] 7 | public class Configuration : Enumeration 8 | { 9 | public static Configuration Debug = new Configuration { Value = nameof(Debug) }; 10 | public static Configuration Release = new Configuration { Value = nameof(Release) }; 11 | 12 | public static implicit operator string(Configuration configuration) 13 | { 14 | return configuration.Value; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /build/Directory.Build.props: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /build/Directory.Build.targets: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /build/_build.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | net8.0 6 | 7 | CS0649;CS0169 8 | .. 9 | .. 10 | 1 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /build/_build.csproj.DotSettings: -------------------------------------------------------------------------------- 1 |  2 | DO_NOT_SHOW 3 | DO_NOT_SHOW 4 | DO_NOT_SHOW 5 | DO_NOT_SHOW 6 | Implicit 7 | Implicit 8 | ExpressionBody 9 | 0 10 | NEXT_LINE 11 | True 12 | False 13 | 120 14 | IF_OWNER_IS_SINGLE_LINE 15 | WRAP_IF_LONG 16 | False 17 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 18 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 19 | <Policy><Descriptor Staticness="Instance" AccessRightKinds="Private" Description="Instance fields (private)"><ElementKinds><Kind Name="FIELD" /><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /></Policy> 20 | <Policy><Descriptor Staticness="Static" AccessRightKinds="Private" Description="Static fields (private)"><ElementKinds><Kind Name="FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /></Policy> 21 | True 22 | True 23 | True 24 | True 25 | True 26 | True 27 | True 28 | True 29 | True 30 | True 31 | -------------------------------------------------------------------------------- /global.json: -------------------------------------------------------------------------------- 1 | { 2 | "sdk": { 3 | "version": "8.0.100", 4 | "rollForward": "latestFeature" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /octoversion.json: -------------------------------------------------------------------------------- 1 | { 2 | "NonPreReleaseTags": ["refs/heads/main", "refs/heads/master"] 3 | } -------------------------------------------------------------------------------- /source/.editorconfig: -------------------------------------------------------------------------------- 1 | ; EditorConfig to support per-solution formatting. 2 | ; Use the EditorConfig VS add-in to make this work. 3 | ; http://editorconfig.org/ 4 | 5 | ; This is the default for the codeline. 6 | root = true 7 | 8 | [*] 9 | end_of_line = CRLF 10 | 11 | [*.{cs,props,targets,xml,nuspec}] 12 | indent_style = space 13 | indent_size = 4 14 | 15 | [*.{config}] 16 | indent_style = space 17 | indent_size = 2 18 | 19 | [*.{csproj,resx}] 20 | indent_style = space 21 | indent_size = 2 -------------------------------------------------------------------------------- /source/Octodiff.Benchmarks/DeltaApplierBenchmarks.cs: -------------------------------------------------------------------------------- 1 | using BenchmarkDotNet.Attributes; 2 | using Octodiff.Core; 3 | using Octodiff.Diagnostics; 4 | 5 | namespace Octodiff.Benchmarks; 6 | 7 | [MemoryDiagnoser] 8 | public class DeltaApplierBenchmarks 9 | { 10 | private const int _500MB = 0x320_0000; 11 | private Stream? deltaStream; 12 | private Stream? otherDeltaStream; 13 | 14 | [GlobalSetup] 15 | public void GlobalSetup() 16 | { 17 | var originalStream = new RandomDataGeneratorStream(_500MB, 100); 18 | var newStream = new RandomDataGeneratorStream(_500MB, 200); 19 | var signatureStream = new MemoryStream(); 20 | var signatureBuilder = new SignatureBuilder(); 21 | signatureBuilder.Build(originalStream, new SignatureWriter(signatureStream)); 22 | 23 | signatureStream.Seek(0, SeekOrigin.Begin); 24 | var deltaBuilder = new DeltaBuilder(); 25 | deltaStream = new MemoryStream(); 26 | deltaBuilder.BuildDelta(newStream, new SignatureReader(signatureStream, NullProgressReporter.Instance), new BinaryDeltaWriter(deltaStream)); 27 | deltaStream.Seek(0, SeekOrigin.Begin); 28 | 29 | originalStream.Seek(0, SeekOrigin.Begin); 30 | signatureStream.Seek(0, SeekOrigin.Begin); 31 | otherDeltaStream = new MemoryStream(); 32 | deltaBuilder.BuildDelta(originalStream, new SignatureReader(signatureStream, NullProgressReporter.Instance), new BinaryDeltaWriter(otherDeltaStream)); 33 | otherDeltaStream.Seek(0, SeekOrigin.Begin); 34 | } 35 | 36 | [Benchmark] 37 | public void ApplyBigDelta_Different() 38 | { 39 | var originalStream = new RandomDataGeneratorStream(_500MB, 100); 40 | var deltaApplier = new DeltaApplier { SkipHashCheck = true }; 41 | deltaApplier.Apply(originalStream, new BinaryDeltaReader(deltaStream, NullProgressReporter.Instance), Stream.Null); 42 | } 43 | 44 | [Benchmark] 45 | public void ApplyBigDelta_Identical() 46 | { 47 | var originalStream = new RandomDataGeneratorStream(_500MB, 100); 48 | var deltaApplier = new DeltaApplier { SkipHashCheck = true }; 49 | deltaApplier.Apply(originalStream, new BinaryDeltaReader(otherDeltaStream, NullProgressReporter.Instance), Stream.Null); 50 | } 51 | } -------------------------------------------------------------------------------- /source/Octodiff.Benchmarks/Octodiff.Benchmarks.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | net8.0 6 | enable 7 | enable 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /source/Octodiff.Benchmarks/Program.cs: -------------------------------------------------------------------------------- 1 | using BenchmarkDotNet.Running; 2 | using Octodiff.Benchmarks; 3 | 4 | BenchmarkRunner.Run(); -------------------------------------------------------------------------------- /source/Octodiff.Benchmarks/RandomDataGeneratorStream.cs: -------------------------------------------------------------------------------- 1 | namespace Octodiff.Benchmarks; 2 | 3 | public class RandomDataGeneratorStream : Stream 4 | { 5 | private readonly byte[] masterBuffer = new byte[128]; 6 | private long position; 7 | 8 | public RandomDataGeneratorStream(long desiredLength, int? seed = null) 9 | { 10 | Length = desiredLength; 11 | var random = new Random(seed ?? (int)DateTime.Now.Ticks); 12 | random.NextBytes(masterBuffer); 13 | } 14 | 15 | public override void Flush() 16 | { 17 | throw new NotSupportedException(); 18 | } 19 | 20 | public override int Read(byte[] buffer, int offset, int count) 21 | { 22 | var toCopy = Math.Min(count, Length - position); 23 | var copied = 0; 24 | while (copied < toCopy) 25 | { 26 | var startingPos = (int)(position % masterBuffer.Length); 27 | var copyThisTime = (int)Math.Min(masterBuffer.Length - startingPos, toCopy - copied); 28 | Buffer.BlockCopy(masterBuffer, startingPos, buffer, offset, copyThisTime); 29 | offset += copyThisTime; 30 | position += copyThisTime; 31 | copied += copyThisTime; 32 | } 33 | return (int)toCopy; 34 | } 35 | 36 | public override long Seek(long offset, SeekOrigin origin) 37 | { 38 | position = origin switch 39 | { 40 | SeekOrigin.Begin => offset, 41 | SeekOrigin.Current => position + offset, 42 | SeekOrigin.End => Length - offset, 43 | _ => throw new ArgumentOutOfRangeException(nameof(origin), origin, null) 44 | }; 45 | return position; 46 | } 47 | 48 | public override void SetLength(long value) 49 | { 50 | throw new NotSupportedException(); 51 | } 52 | 53 | public override void Write(byte[] buffer, int offset, int count) 54 | { 55 | throw new NotSupportedException(); 56 | } 57 | 58 | public override bool CanRead => true; 59 | public override bool CanSeek => true; 60 | public override bool CanWrite => false; 61 | public override long Length { get; } 62 | 63 | public override long Position 64 | { 65 | get => position; 66 | set => position = value; 67 | } 68 | } -------------------------------------------------------------------------------- /source/Octodiff.Tests/Core/Adler32RollingChecksumFixture.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using NUnit.Framework.Legacy; 3 | using Octodiff.Core; 4 | using Octodiff.Tests.Util; 5 | 6 | namespace Octodiff.Tests.Core 7 | { 8 | [TestFixture] 9 | public class Adler32RollingChecksumFixture 10 | { 11 | readonly Adler32RollingChecksum c = new Adler32RollingChecksum(); 12 | 13 | [Test] 14 | public void Name() 15 | { 16 | ClassicAssert.AreEqual("Adler32", c.Name); 17 | } 18 | 19 | [Test] 20 | public void Calculate() 21 | { 22 | var block = Helpers.TestData(); 23 | 24 | ClassicAssert.AreEqual(2755533412, c.Calculate(block, 0, 100)); 25 | ClassicAssert.AreEqual(2888047271, c.Calculate(block, 1, 100)); 26 | ClassicAssert.AreEqual(2476743237, c.Calculate(block, 2, 100)); 27 | ClassicAssert.AreEqual(591925890, c.Calculate(block, 93, 100)); 28 | ClassicAssert.AreEqual(4037189623, c.Calculate(block, 0, block.Length)); 29 | 30 | var largeBlock = Helpers.GenerateTestData(100 * 1024); 31 | ClassicAssert.AreEqual(2928347280, c.Calculate(largeBlock, 0, largeBlock.Length)); 32 | } 33 | 34 | [Test] 35 | public void Rotate() 36 | { 37 | ClassicAssert.AreEqual(3209698067, c.Rotate(2755533412, 0, 0xAF, 8)); 38 | ClassicAssert.AreEqual(3209698067, c.Rotate(2755533412, 0, 0xAF, 16)); 39 | ClassicAssert.AreEqual(3209698067, c.Rotate(2755533412, 0, 0xAF, 24)); 40 | ClassicAssert.AreEqual(3209698067, c.Rotate(2755533412, 0, 0xAF, 32)); 41 | 42 | ClassicAssert.AreEqual(3577289570, c.Rotate(3209698067, 0xAF, 0xFE, 8)); 43 | ClassicAssert.AreEqual(3485539170, c.Rotate(3209698067, 0xAF, 0xFE, 16)); 44 | ClassicAssert.AreEqual(3393788770, c.Rotate(3209698067, 0xAF, 0xFE, 24)); 45 | ClassicAssert.AreEqual(3302038370, c.Rotate(3209698067, 0xAF, 0xFE, 32)); 46 | } 47 | } 48 | 49 | } -------------------------------------------------------------------------------- /source/Octodiff.Tests/Core/Adler32RollingChecksumV2Fixture.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using NUnit.Framework.Legacy; 3 | using Octodiff.Core; 4 | using Octodiff.Tests.Util; 5 | 6 | namespace Octodiff.Tests.Core 7 | { 8 | [TestFixture] 9 | public class Adler32RollingChecksumV2Fixture 10 | { 11 | readonly Adler32RollingChecksumV2 c = new Adler32RollingChecksumV2(); 12 | 13 | [Test] 14 | public void Name() 15 | { 16 | ClassicAssert.AreEqual("Adler32V2", c.Name); 17 | } 18 | 19 | [Test] 20 | public void Calculate() 21 | { 22 | var block = Helpers.TestData(); 23 | 24 | ClassicAssert.AreEqual(2760448612, c.Calculate(block, 0, 100)); 25 | ClassicAssert.AreEqual(2892962471, c.Calculate(block, 1, 100)); 26 | ClassicAssert.AreEqual(2481658437, c.Calculate(block, 2, 100)); 27 | ClassicAssert.AreEqual(595858050, c.Calculate(block, 93, 100)); 28 | ClassicAssert.AreEqual(4175798263, c.Calculate(block, 0, block.Length)); 29 | 30 | var largeBlock = Helpers.GenerateTestData(100 * 1024); 31 | ClassicAssert.AreEqual(180621253, c.Calculate(largeBlock, 0, largeBlock.Length)); 32 | } 33 | 34 | [Test] 35 | public void Rotate() 36 | { 37 | ClassicAssert.AreEqual(3209698067, c.Rotate(2755533412, 0, 0xAF, 8)); 38 | ClassicAssert.AreEqual(3209698067, c.Rotate(2755533412, 0, 0xAF, 16)); 39 | ClassicAssert.AreEqual(3209698067, c.Rotate(2755533412, 0, 0xAF, 24)); 40 | ClassicAssert.AreEqual(3209698067, c.Rotate(2755533412, 0, 0xAF, 32)); 41 | 42 | ClassicAssert.AreEqual(3577289570, c.Rotate(3209698067, 0xAF, 0xFE, 8)); 43 | ClassicAssert.AreEqual(3485539170, c.Rotate(3209698067, 0xAF, 0xFE, 16)); 44 | ClassicAssert.AreEqual(3393788770, c.Rotate(3209698067, 0xAF, 0xFE, 24)); 45 | ClassicAssert.AreEqual(3302038370, c.Rotate(3209698067, 0xAF, 0xFE, 32)); 46 | } 47 | } 48 | 49 | } -------------------------------------------------------------------------------- /source/Octodiff.Tests/Core/BinaryDeltaWriterFixture.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using NUnit.Framework; 4 | using NUnit.Framework.Legacy; 5 | using Octodiff.Core; 6 | using Octodiff.Tests.Util; 7 | 8 | namespace Octodiff.Tests.Core 9 | { 10 | public class BinaryDeltaWriterFixture 11 | { 12 | static byte[] WithDeltaWriter(Action action) 13 | { 14 | using (var ms = new MemoryStream()) 15 | { 16 | var b = new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(ms)); 17 | action(b); 18 | b.Finish(); 19 | return ms.ToArray(); 20 | } 21 | } 22 | 23 | [Test] 24 | public void WritesHeader() 25 | { 26 | var output = WithDeltaWriter(b => 27 | { 28 | // we only support sha1 so there's only one thing to test really 29 | b.WriteMetadata(SupportedAlgorithms.Hashing.Create("SHA1"), Helpers.GenerateTestData(20)); 30 | }); 31 | 32 | ClassicAssert.AreEqual("4f43544f44454c54410104534841311400000030820204308201aba003020102021418d83f07713e3e3e", output.ToHexString()); 33 | } 34 | 35 | [Test] 36 | public void WritesCopyCommand() 37 | { 38 | var output = WithDeltaWriter(b => 39 | { 40 | b.WriteCopyCommand(new DataRange(startOffset: 315412, length: 9874563)); // deliberately > 255 so we cross a single byte boundary and detect endianness problems 41 | }); 42 | 43 | ClassicAssert.AreEqual("6014d004000000000083ac960000000000", output.ToHexString()); 44 | } 45 | 46 | [Test] 47 | public void MergesSequentialCopyCommands() 48 | { 49 | var output = WithDeltaWriter(b => 50 | { 51 | // these 3 get merged 52 | b.WriteCopyCommand(new DataRange(startOffset: 0, length: 128)); 53 | b.WriteCopyCommand(new DataRange(startOffset: 128, length: 128)); 54 | b.WriteCopyCommand(new DataRange(startOffset: 256, length: 128)); 55 | 56 | // this one doesn't because there's a 1-byte gap 57 | b.WriteCopyCommand(new DataRange(startOffset: 385, length: 128)); 58 | }); 59 | 60 | // 0x60 signifies a copy command, we can see there's only two here 61 | ClassicAssert.AreEqual("60000000000000000080010000000000006081010000000000008000000000000000", output.ToHexString()); 62 | } 63 | 64 | [Test] 65 | public void DataCommandFlushesCopyCommand() 66 | { 67 | var output = WithDeltaWriter(b => 68 | { 69 | var sourceFile = new MemoryStream(Helpers.GenerateTestData(1024)); 70 | 71 | // these would get merged but they won't because there's a Data command in the middle 72 | b.WriteCopyCommand(new DataRange(startOffset: 0, length: 128)); 73 | b.WriteDataCommand(sourceFile, 500, 128); 74 | b.WriteCopyCommand(new DataRange(startOffset: 128, length: 128)); 75 | }); 76 | 77 | ClassicAssert.AreEqual("6000000000000000008000000000000000808000000000000000bd7ce51a34612015a74648787c7a7e032645377030820204308201aba003020102021418d83f07718be4121df0a18d7610faf8d7a3bec4300a06082a8648ce3d0403023058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f796080000000000000008000000000000000", output.ToHexString()); 78 | } 79 | 80 | [Test] 81 | public void WritesDataCommand() 82 | { 83 | var output = WithDeltaWriter(b => 84 | { 85 | var sourceFile = new MemoryStream(Helpers.GenerateTestData(1024)); 86 | 87 | b.WriteDataCommand(sourceFile, 337, 515); // deliberately > 255 so we cross a single byte boundary and detect endianness problems 88 | }); 89 | 90 | ClassicAssert.AreEqual("8003020000000000000931ecf7f3bd4bce212cfd2cbaa3533051301d0603551d0e04160414badd278a31e012776afbfda4ead8fdce904f0efc301f0603551d23041830168014badd278a31e012776afbfda4ead8fdce904f0efc300f0603551d130101ff040530030101ff300a06082a8648ce3d04030203470030440220599cef920115b64a7d0bc7de55a84bba7f05ee78b9e903af7cb52b4a5dcc8ea2022006575445dab9c21325a48de3bd7ce51a34612015a74648787c7a7e032645377030820204308201aba003020102021418d83f07718be4121df0a18d7610faf8d7a3bec4300a06082a8648ce3d0403023058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c0454455354301e170d3233303332303039343834325a170d3234303331393039343834325a3058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c04544553543059301306072a8648ce3d020106082a8648ce3d03010703420004504b77248d83e2e3e209bbb2297a0e4d24ff45e79eff88dd165e6419ae98512dabd2219da46e93d7ff98d5a1cb80", output.ToHexString()); 91 | } 92 | } 93 | 94 | } -------------------------------------------------------------------------------- /source/Octodiff.Tests/Core/DeltaBuilderFixture.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Linq; 4 | using NUnit.Framework; 5 | using NUnit.Framework.Legacy; 6 | using Octodiff.Core; 7 | using Octodiff.Diagnostics; 8 | using Octodiff.Tests.Util; 9 | 10 | namespace Octodiff.Tests.Core 11 | { 12 | public class DeltaBuilderFixture 13 | { 14 | static byte[] BuildDelta(byte[] newFile, byte[] signatureFile) 15 | { 16 | var d = new DeltaBuilder(); 17 | 18 | using (var inputNewFileStream = new MemoryStream(newFile)) 19 | using (var inputSigFileStream = new MemoryStream(signatureFile)) 20 | using (var outputDeltaFileStream = new MemoryStream()) 21 | { 22 | var sigReader = new SignatureReader(inputSigFileStream, NullProgressReporter.Instance); 23 | // there's not a great deal of value in testing AggregateCopyOperationsDecorator separately from BinaryDeltaWriter 24 | // and we want test output which contains the aggregate so we can replicate on other platforms 25 | var deltaWriter = new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(outputDeltaFileStream)); 26 | 27 | d.BuildDelta(inputNewFileStream, sigReader, deltaWriter); 28 | 29 | return outputDeltaFileStream.ToArray(); 30 | } 31 | } 32 | 33 | // just helps us get test input data 34 | static byte[] BuildSignature(byte[] input, short signatureChunkSize = SignatureBuilder.DefaultChunkSize) 35 | { 36 | using (var inStream = new MemoryStream(input)) 37 | using (var outStream = new MemoryStream()) 38 | { 39 | new SignatureBuilder { ChunkSize = signatureChunkSize }.Build(inStream, new SignatureWriter(outStream)); 40 | 41 | return outStream.ToArray(); 42 | } 43 | } 44 | 45 | [Test] 46 | public void BuildsNoOpDeltaForSameInput() 47 | { 48 | var signature = BuildSignature(Helpers.TestData()); 49 | 50 | var deltaFile = BuildDelta(Helpers.TestData(), signature); 51 | ClassicAssert.AreEqual("4f43544f44454c544101045348413114000000330bd06982d3b5dbda6c1a6ad16687a0cdb03c0d3e3e3e6000000000000000000802000000000000", deltaFile.ToHexString()); 52 | } 53 | 54 | [Test] 55 | public void BuildsFullDeltaForNoInput() 56 | { 57 | var signature = BuildSignature(Array.Empty()); 58 | 59 | var deltaFile = BuildDelta(Helpers.TestData(), signature); 60 | ClassicAssert.AreEqual("4f43544f44454c544101045348413114000000330bd06982d3b5dbda6c1a6ad16687a0cdb03c0d3e3e3e80080200000000000030820204308201aba003020102021418d83f07718be4121df0a18d7610faf8d7a3bec4300a06082a8648ce3d0403023058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c0454455354301e170d3233303332303039343834325a170d3234303331393039343834325a3058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c04544553543059301306072a8648ce3d020106082a8648ce3d03010703420004504b77248d83e2e3e209bbb2297a0e4d24ff45e79eff88dd165e6419ae98512dabd2219da46e93d7ff98d5a1cb80314a57f37d0931ecf7f3bd4bce212cfd2cbaa3533051301d0603551d0e04160414badd278a31e012776afbfda4ead8fdce904f0efc301f0603551d23041830168014badd278a31e012776afbfda4ead8fdce904f0efc300f0603551d130101ff040530030101ff300a06082a8648ce3d04030203470030440220599cef920115b64a7d0bc7de55a84bba7f05ee78b9e903af7cb52b4a5dcc8ea2022006575445dab9c21325a48de3bd7ce51a34612015a74648787c7a7e0326453770", deltaFile.ToHexString()); 61 | } 62 | 63 | [Test] 64 | public void BuildsFullIfInputIsSmallerThanWholeFile() 65 | { 66 | var signature = BuildSignature(Helpers.TestData()); 67 | 68 | var newFile = Helpers.TestData(); 69 | 70 | newFile[32] = 0xaa; 71 | newFile[33] = 0xab; 72 | newFile[34] = 0xac; 73 | 74 | var deltaFile = BuildDelta(newFile, signature); 75 | ClassicAssert.AreEqual("4f43544f44454c54410104534841311400000014ba64eeabad295dd60cfabd648ff176b64890773e3e3e80080200000000000030820204308201aba003020102021418d83f07718be4121df0a18d7610faf8d7aaabac300a06082a8648ce3d0403023058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c0454455354301e170d3233303332303039343834325a170d3234303331393039343834325a3058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c04544553543059301306072a8648ce3d020106082a8648ce3d03010703420004504b77248d83e2e3e209bbb2297a0e4d24ff45e79eff88dd165e6419ae98512dabd2219da46e93d7ff98d5a1cb80314a57f37d0931ecf7f3bd4bce212cfd2cbaa3533051301d0603551d0e04160414badd278a31e012776afbfda4ead8fdce904f0efc301f0603551d23041830168014badd278a31e012776afbfda4ead8fdce904f0efc300f0603551d130101ff040530030101ff300a06082a8648ce3d04030203470030440220599cef920115b64a7d0bc7de55a84bba7f05ee78b9e903af7cb52b4a5dcc8ea2022006575445dab9c21325a48de3bd7ce51a34612015a74648787c7a7e0326453770", deltaFile.ToHexString()); 76 | } 77 | 78 | [Test] 79 | public void BuildsDeltaForSmallChangesInFile() 80 | { 81 | // deliberately use small chunk size because our input is smaller than the default 2k chunk size and otherwise it just re-delta's the whole file 82 | var signature = BuildSignature(Helpers.TestData(), SignatureBuilder.MinimumChunkSize); 83 | 84 | var newFile = Helpers.TestData(); 85 | 86 | newFile[32] = 0xaa; 87 | newFile[33] = 0xab; 88 | newFile[34] = 0xac; 89 | 90 | // Note: our newFile is 520 bytes, but our deltaFile is only 247 91 | 92 | var deltaFile = BuildDelta(newFile, signature); 93 | ClassicAssert.AreEqual("4f43544f44454c54410104534841311400000014ba64eeabad295dd60cfabd648ff176b64890773e3e3e80800000000000000030820204308201aba003020102021418d83f07718be4121df0a18d7610faf8d7aaabac300a06082a8648ce3d0403023058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06036080000000000000008801000000000000", deltaFile.ToHexString()); 94 | } 95 | 96 | [Test] 97 | public void BuildsDeltaForSmallChangesInFile_Prepend() 98 | { 99 | // deliberately use small chunk size because our input is smaller than the default 2k chunk size and otherwise it just re-delta's the whole file 100 | var signature = BuildSignature(Helpers.TestData(), SignatureBuilder.MinimumChunkSize); 101 | 102 | // prepend a byte to the front of the file as an alternate logic test 103 | var newFile = new[] { (byte)0xaa }.Concat(Helpers.TestData()).ToArray(); 104 | 105 | // Note: our newFile is 521 bytes, but our deltaFile is only 137 106 | 107 | var deltaFile = BuildDelta(newFile, signature); 108 | ClassicAssert.AreEqual("4f43544f44454c544101045348413114000000e5ca5051b8cf462ed567a8f88802fd9e62a0f0e83e3e3e800100000000000000aa6000000000000000000802000000000000", deltaFile.ToHexString()); 109 | } 110 | 111 | [Test] 112 | public void BuildsDeltaForSmallChangesInFileEvenWhenFileIsLarger() 113 | { 114 | var original = Helpers.GenerateTestData(128 * 1024); // 128k file 115 | var signature = BuildSignature(original); 116 | 117 | var newFile = original.ToArray(); 118 | 119 | newFile[32] = 0xaa; 120 | newFile[33] = 0xab; 121 | newFile[34] = 0xac; 122 | 123 | var deltaFile = BuildDelta(newFile, signature); 124 | // our delta file is roughly 3k. Still large but much smaller than the entire 128k newFile 125 | ClassicAssert.AreEqual("4f43544f44454c54410104534841311400000050f7ec0e6d4fe4ab8400b759e2b000f1d0aced8e3e3e3e80280000000000000030820204308201aba003020102021418d83f07718be4121df0a18d7610faf8d7aaabac300a06082a6000780000000000000088010000000000800800000000000000a3bec4300a06082a600078000000000000007000000000000080d007000000000000060355040b0c03522644310d300b06035504030c0454455354301e170d3233303332303039343834325a170d3234303331393039343834325a3058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c04544553543059301306072a8648ce3d020106082a8648ce3d03010703420004504b77248d83e2e3e209bbb2297a0e4d24ff45e79eff88dd165e6419ae98512dabd2219da46e93d7ff98d5a1cb80314a57f37d0931ecf7f3bd4bce212cfd2cbaa3533051301d0603551d0e04160414badd278a31e012776afbfda4ead8fdce904f0efc301f0603551d23041830168014badd278a31e012776afbfda4ead8fdce904f0efc300f0603551d130101ff040530030101ff300a06082a8648ce3d04030203470030440220599cef920115b64a7d0bc7de55a84bba7f05ee78b9e903af7cb52b4a5dcc8ea2022006575445dab9c21325a48de3bd7ce51a34612015a74648787c7a7e032645377030820204308201aba003020102021418d83f07718be4121df0a18d7610faf8d7a3bec4300a06082a8648ce3d0403023058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c0454455354301e170d3233303332303039343834325a170d3234303331393039343834325a3058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c04544553543059301306072a8648ce3d020106082a8648ce3d03010703420004504b77248d83e2e3e209bbb2297a0e4d24ff45e79eff88dd165e6419ae98512dabd2219da46e93d7ff98d5a1cb80314a57f37d0931ecf7f3bd4bce212cfd2cbaa3533051301d0603551d0e04160414badd278a31e012776afbfda4ead8fdce904f0efc301f0603551d23041830168014badd278a31e012776afbfda4ead8fdce904f0efc300f0603551d130101ff040530030101ff300a06082a8648ce3d04030203470030440220599cef920115b64a7d0bc7de55a84bba7f05ee78b9e903af7cb52b4a5dcc8ea2022006575445dab9c21325a48de3bd7ce51a34612015a74648787c7a7e032645377030820204308201aba003020102021418d83f07718be4121df0a18d7610faf8d7a3bec4300a06082a8648ce3d0403023058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c0454455354301e170d3233303332303039343834325a170d3234303331393039343834325a3058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c04544553543059301306072a8648ce3d020106082a8648ce3d03010703420004504b77248d83e2e3e209bbb2297a0e4d24ff45e79eff88dd165e6419ae98512dabd2219da46e93d7ff98d5a1cb80314a57f37d0931ecf7f3bd4bce212cfd2cbaa3533051301d0603551d0e04160414badd278a31e012776afbfda4ead8fdce904f0efc301f0603551d23041830168014badd278a31e012776afbfda4ead8fdce904f0efc300f0603551d130101ff040530030101ff300a06082a8648ce3d04030203470030440220599cef920115b64a7d0bc7de55a84bba7f05ee78b9e903af7cb52b4a5dcc8ea2022006575445dab9c21325a48de3bd7ce51a34612015a74648787c7a7e032645377030820204308201aba003020102021418d83f07718be4121df0a18d7610faf8d7a3bec4300a06082a8648ce3d0403023058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c0454455354301e170d3233303332303039343834325a170d3234303331393039343834325a3058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c04544553543059301306072a8648ce3d020106082a8648ce3d03010703420004504b77248d83e2e3e209bbb2297a0e4d24ff45e79eff88dd165e6419ae98512dabd2219da46e93d7ff98d5a1cb80314a57f37d0931ecf7f3bd4bce212cfd2cbaa3533051301d0603551d0e04160414badd278a31e012776afbfda4ead8fdce904f0efc301f0603551d23041830168014badd278a31e012776afbfda4ead8fdce904f0efc300f0603551d130101ff040530030101ff300a06082a8648ce3d04030203470030440220599cef920115b64a7d0bc7de55a84bba7f05ee78b9e903af7cb52b4a5dcc8ea2022006575445dab9c21325a48de3bd7ce51a34612015a74648787c7a7e032645377030820204308201aba003020102021418d83f07718be4121df0a18d7610faf8d7", deltaFile.ToHexString()); 126 | } 127 | 128 | [Test] 129 | public void BuildsDeltaForSmallChangesInFileEvenWhenFileIsLarger_MultipleDisjointChanges() 130 | { 131 | var original = Helpers.GenerateTestData(128 * 1024); // 128k file 132 | var signature = BuildSignature(original); 133 | 134 | var newFile = original.ToArray(); 135 | 136 | newFile[32] = 0xaa; 137 | newFile[33] = 0xab; 138 | newFile[34] = 0xac; 139 | 140 | newFile[32000] = 0xaa; 141 | newFile[33000] = 0xab; 142 | newFile[34000] = 0xac; 143 | 144 | var deltaFile = BuildDelta(newFile, signature); 145 | // our delta file is roughly 6k (two chunks changed). Still large but much smaller than the entire 128k newFile 146 | ClassicAssert.AreEqual("4f43544f44454c544101045348413114000000645a41cab32226e8e9212c54db711c22653c00513e3e3e80280000000000000030820204308201aba003020102021418d83f07718be4121df0a18d7610faf8d7aaabac300a06082a600078000000000000007800000000000080b00c00000000000061746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c0454455354301e170d3233303332303039343834325a170d3234303331393039343834325a3058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c04544553543059301306072a8648ce3d020106082a8648ce3d03010703420004504b77248d83e2e3e209bbb2297a0e4d24ff45e79eff88dd165e6419ae98512dabd2219da46e93d7ff98d5a1cb80314a57f37d0931ecf7f3bd4bce212cfd2cbaa3533051301d0603551d0e04160414badd278a31e012776afbfda4ead8fdce904f0efc301f0603551d23041830168014badd278a31e012776afbfda4ead8fdce904f0efc300f0603551d130101ff040530030101ff300a06082a8648ce3d04030203470030440220599cef920115b64a7d0bc7de55a84bba7f05ee78b9e903af7cb52b4a5dcc8ea2022006575445dab9c21325a48de3bd7ce51a34612015a74648787c7a7e032645377030820204308201aba003020102021418d83f07718be4121df0a18d7610faf8d7a3bec4300a06082a8648ce3d0403023058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c0454455354301e170d3233303332303039343834325a170d3234303331393039343834325a3058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c04544553543059301306072a8648ce3d020106082a8648ce3d03010703420004504b77248d83e2e3e209bbb2297a0e4d24ff45e79eff88dd165e6419ae98512dabd2219da46e93d7ff98d5a1cb80314a57f37d0931ecf7f3bd4bce212cfd2cbaa3533051301d0603551d0e04160414badd278a31e012776afbfda4ead8fdce904f0efc301f0603551d23041830168014badd278a31e012776afbfda4ead8fdce904f0efc300f0603551d130101ff040530030101ff300a06082a8648ce3d04030203470030440220599cef920115b64a7d0bc7de55a84bba7f05ee78b9e903af7cb52b4a5dcc8ea2022006575445dab9c21325a48de3bd7ce51a34612015a74648787c7a7e032645377030820204308201aba003020102021418d83f07718be4121df0a18d7610faf8d7a3bec4300a06082a8648ce3d0403023058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c0454455354301e170d3233303332303039343834325a170d3234303331393039343834325a3058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c04544553543059301306072a8648ce3d020106082a8648ce3d03aa0703420004504b77248d83e2e3e209bbb2297a0e4d24ff45e79eff88dd165e6419ae98512dabd2219da46e93d7ff98d5a1cb80314a57f37d0931ecf7f3bd4bce212cfd2cbaa3533051301d0603551d0e04160414badd278a31e012776afbfda4ead8fdce904f0efc301f0603551d23041830168014badd278a31e012776afbfda4ead8fdce904f0efc300f0603551d130101ff040530030101ff300a06082a8648ce3d04030203470030440220599cef920115b64a7d0bc7de55a84bba7f05ee78b9e903af7cb52b4a5dcc8ea2022006575445dab9c21325a48de3bd7ce51a34612015a74648787c7a7e032645377030820204308201aba003020102021418d83f07718be4121df0a18d7610faf8d7a3bec4300a06082a8648ce3d0403023058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c0454455354301e170d3233303332303039343834325a170d3234303331393039343834325a3058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c04544553543059301306072a8648ce3d020106082a8648ce3d03010703420004504b77248d83e2e3e209bbb2297a0e4d24ff45e79eff88dd165e6419ae98512dabd2219da46e93d7ff98d5a1cb80314a57f37d0931ecf7f3bd4bce212cfd2cbaa3533051301d0603551d0e04160414badd278a31e012776afbfda4ead8fdce904f0efc301f0603551d23041830168014badd278a31e012776afbfda4ead8fdce904f0efc300f0603551d130101ff040530030101ff300a06082a8648ce3d04030203470030440220599cef920115b64a7d0bc7de55a84bba7f05ee78b9e903af7cb52b4a5dcc8ea2022006575445dab9c21325a48de3bd7ce51a34612015a74648787c7a7e032645377030820204308201aba003020102021418d83f07718be4121df0a18d7610faf8d7a3bec4300a06082a8648ce3d0403023058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c0454455354301e170d3233303332303039343834325a170d3234303331393039343834325a3058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0cab522644310d300b06035504030c04544553543059301306072a8648ce3d020106082a8648ce3d03010703420004504b77248d83e2e3e209bbb2297a0e4d24ff45e79eff88dd165e6419ae98512dabd2219da46e93d7ff98d5a1cb80314a57f37d0931ecf7f3bd4bce212cfd2cbaa3533051301d0603551d0e04160414badd278a31e012776afbfda4ead8fdce904f0efc301f0603551d23041830168014badd278a31e012776afbfda4ead8fdce904f0efc300f0603551d130101ff040530030101ff300a06082a8648ce3d04030203470030440220599cef920115b64a7d0bc7de55a84bba7f05ee78b9e903af7cb52b4a5dcc8ea2022006575445dab9c21325a48de3bd7ce51a34612015a74648787c7a7e032645377030820204308201aba003020102021418d83f07718be4121df0a18d7610faf8d7a3bec4300a06082a8648ce3d0403023058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c0454455354301e170d3233303332303039343834325a170d3234303331393039343834325a3058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c04544553543059301306072a8648ce3d020106082a8648ce3d03010703420004504b77248d83e2e3e209bbb2297a0e4d24ff45e79eff88dd165e6419ae98512dabd2219da46e93d7ff98d5a1cb80314a57f37d0931ecf7f3bd4bce212cfd2cbaa3533051301d0603551d0e04160414badd278a31e012776afbfda4ead8fdce904f0efc301f0603551d23041830168014badd278a31e012776afbfda4ead8fdce904f0efc300f0603551d130101ff040530030101ff300a06082a8648ce3d04030203470030440220599cef920115b64a7d0bc7de55a84bba7f05ee78b9e903af7cb52b4a5dcc8ea2022006575445dab9c21325a48de3bd7ce51a34612015a74648787c7a7e032645377030820204308201aba003020102021418d83f07718be4121df0a18d7610faf8d7a3bec4300a06082a8648ce3d0403023058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c0454455354301e170d3233303332303039343834325a170d3234303331393039343834325a3058310b30090603550406130241553113301106035504080c0a536f6d652dac746174653117306000d00000000000000030010000000000800800000000000000a3bec4300a06082a600078000000000000004800000000000080200300000000000006082a8648ce3d03010703420004504b77248d83e2e3e209bbb2297a0e4d24ff45e79eff88dd165e6419ae98512dabd2219da46e93d7ff98d5a1cb80314a57f37d0931ecf7f3bd4bce212cfd2cbaa3533051301d0603551d0e04160414badd278a31e012776afbfda4ead8fdce904f0efc301f0603551d23041830168014badd278a31e012776afbfda4ead8fdce904f0efc300f0603551d130101ff040530030101ff300a06082a8648ce3d04030203470030440220599cef920115b64a7d0bc7de55a84bba7f05ee78b9e903af7cb52b4a5dcc8ea2022006575445dab9c21325a48de3bd7ce51a34612015a74648787c7a7e032645377030820204308201aba003020102021418d83f07718be4121df0a18d7610faf8d7a3bec4300a06082a8648ce3d0403023058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c0454455354301e170d3233303332303039343834325a170d3234303331393039343834325a3058310b30090603550406130241553113301106035504080c0a536f6d652d537461746531173015060355040a0c0e4f63746f707573204465706c6f79310c300a060355040b0c03522644310d300b06035504030c04544553543059301306072a8648ce3d020106082a8648ce3d03010703420004504b77248d83e2e3e209bbb2297a0e4d24ff45e79eff88dd165e6419ae98512dabd2219da46e93d7ff98d5a1cb80314a57f37d0931ecf7f3bd4bce212cfd2cbaa3533051301d0603551d0e04160414badd278a31e012776afbfda4ead8fdce904f0efc301f0603551d23041830168014badd278a31e012776afbfda4ead8fdce904f0efc300f0603551d130101ff040530030101ff300a06082a8648ce3d04030203470030440220599cef920115b64a7d0bc7de55a84bba7f05ee78b9e903af7cb52b4a5dcc8ea2022006575445dab9c21325a48de3bd7ce51a34612015a74648787c7a7e032645377030820204308201aba003020102021418d83f07718be4121df0a18d7610faf8d7", deltaFile.ToHexString()); 147 | } 148 | } 149 | } -------------------------------------------------------------------------------- /source/Octodiff.Tests/Core/SignatureBuilderFixture.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Legacy; 4 | using Octodiff.Core; 5 | using Octodiff.Tests.Util; 6 | 7 | namespace Octodiff.Tests.Core 8 | { 9 | public class SignatureBuilderFixture 10 | { 11 | static byte[] BuildSignature(SignatureBuilder builder, byte[] input) 12 | { 13 | using (var inStream = new MemoryStream(input)) 14 | using (var outStream = new MemoryStream()) 15 | { 16 | builder.Build(inStream, new SignatureWriter(outStream)); 17 | 18 | return outStream.ToArray(); 19 | } 20 | } 21 | 22 | [Test] 23 | public void BuildSignatureWithDefaultSettingsAndTestData() 24 | { 25 | var builder = new SignatureBuilder(); 26 | 27 | var result = BuildSignature(builder, Helpers.TestData()); 28 | 29 | ClassicAssert.AreEqual("4f43544f5349470104534841310741646c657233323e3e3e0802f79fa2f0330bd06982d3b5dbda6c1a6ad16687a0cdb03c0d", result.ToHexString()); 30 | } 31 | 32 | [Test] 33 | public void BuildSignatureWithSmallChunkSize() 34 | { 35 | var builder = new SignatureBuilder 36 | { 37 | ChunkSize = SignatureBuilder.MinimumChunkSize // the smaller the chunk size, the larger the signature file needs to be 38 | }; 39 | 40 | var result = BuildSignature(builder, Helpers.TestData()); 41 | 42 | ClassicAssert.AreEqual("4f43544f5349470104534841310741646c657233323e3e3e8000951f26e719f3978cb607e80a9aab3abbcac8bb1ecbcecf3e80001f18260f0f73196c2aa57877ee5e31291a59b5afca4493658000e035f42a42c4a73471dea3b9746e22dd93893fd8549f11bd8000dd2ff46b72e00e30ecae4c70ee07721d221a3b8a6d1847fa08008a02860c21d4023a8ba580ecdba742e7400aa40b6e449bb3", result.ToHexString()); 43 | } 44 | 45 | [Test] 46 | public void BuildSignatureWithLargeChunkSize() 47 | { 48 | var builder = new SignatureBuilder 49 | { 50 | ChunkSize = SignatureBuilder.MaximumChunkSize 51 | }; 52 | 53 | // our input file is small so larger chunk size doesn't help here 54 | var result = BuildSignature(builder, Helpers.TestData()); 55 | 56 | ClassicAssert.AreEqual("4f43544f5349470104534841310741646c657233323e3e3e0802f79fa2f0330bd06982d3b5dbda6c1a6ad16687a0cdb03c0d", result.ToHexString()); 57 | } 58 | 59 | [Test] 60 | public void BuildSignatureWithDefaultSettingsOverLargeFile() 61 | { 62 | var builder = new SignatureBuilder(); 63 | 64 | var testData = Helpers.GenerateTestData(100 * 1024); 65 | File.WriteAllBytes("c:\\temp\\100ktestdata-csharp.bin", testData); 66 | 67 | var result = BuildSignature(builder, Helpers.GenerateTestData(100*1024)); 68 | 69 | ClassicAssert.AreEqual("4f43544f5349470104534841310741646c657233323e3e3e00080a73b304debbdddd84c899bd9e2cfc5bc2a8bd2adf4d10430008b470604864abf8d34217d6104c6c3e0a173283bce0cc4e8f00085f7932dfd1b95cb3b7c7e3055c90e65467f45497b5c4aadd0008cb71f9f6035eaa709a578e9ac330900410d2753f6ea13d6e00084e72ec239713299d78ea5ea7acd1c84e994ce36f2ee417bb00088171da145be16aacb6ec6e06f0918eac4071760941489da200081a6f85016ccac9c6156923281c2243d40ae6e9bf4b816125000871766eb03307131bc3fb5564d1af5473a7d7b6114bc8a0790008677b41c43d9eed5159c91e7838a90267801ccb5de24ac8b700080e775b2aa6136b72d6dea02464242c5b62eb718d8cd03d6e0008a97aab6682fb7203687b4ca2e7c439f08358a837b95b2d020008c279c99b791abccfb1995cd9a4a66ac3569f9c582e2e7c5d00088c7a86007960c8e890409c8963521738da72fa0d4610744b000844771558315ec5849209629adcd8c94b7e1f2d61880077880008ec7a48c9178db9ca004206a3c26ff417c612b2776cef74d50008d2731d6d1820d2c9fb133a29d0f52d2e82a03a1c59e20cd800085274e6f1ba27e3bc91511330bb5fce4989e5690926e69aea000875708ce1819797a0719d79c675b7f80613cebedfb1441ad500089077a4a7386ed9382e28bb43c082147052f2528953d73946000808730a06145d4895a5653d391fec9657b4742b98d2cba8af0008bf71f44ca07a1551f4a0bc897332dbd57e590853c585216600080974ebc01c988ca3f5eed1bee8dfe37d0b5025ea07b043250008226e2d45b34d4a7ca0763002b12713ce7e2aabf8bf7460e70008f07458ed5bd2b4958d78ce2ca58ef3e07c6d9d5b1544d59800080d7a0e31b4b1028185ae1a971ce79eda4a548ef1e49574b40008d4781b544c5738d6a486f9aa4ca3ca4a349a554262a54dee00088d79116fed45d69653de7d74a40ee2cb286922a990746a320008c0794f80c1323f60cdeb2a2003a6f0abde692c7910ecbb5900086c7b240323b5dee30fdc4223fdcab87aa78b6ac0a24b0ccb00089b77f53777a5e98ea8d6bf11af7168beeb48516871ab666a00086c7a951224931f3a42d2cb069297d41557d233e6b3c943a9000896729f749a2cc3a1306c8d0f94f4f8559a695ba6528ab76d0008ed77a7f2b51cdb799df0705fd44efd0801f93c26c8a8482400082f706cdf8e5a52547d3d003328a464df803e249dd749d60c0008d8749cfddc13b07bb547e1f29943afb03af7c50f4719ba7300084575d50c706878b92b9234309dc507c3e590fea594f2e6f200089572325e9b64675ce94e24956ad2b74ec2cd6e787e21ff2a000818744032610643f7d90c9e16f354e1a705fcb8eaeec01baa00082f6e3b1214540a6e65bb259491ca2084b073862889dfcd7b000862727076186090ba6594f0f1de9f1221dfc2ee1f757a6e1300083179495f8aba759a17c34528009fbdef9e568187efb7812b00083778b057656980acb3ff099c9441d0685f1ecb44b9a7d02c0008ff795a64c70954144a1999ebf8e6538111f1c8a763b5c6d40008517aa7e935f9818d49a3f9aae5bc35b4087dab04849f432600089c7a51ea69d5f2c63f47d912b7b0c1df967f2503d00969b500089a78698c6d644eb1225e35ed4d482c2165fc194569a1d79d0008ca7924791fabfd7725fe1ad8f59719246c06e56405188ca2000898745ebbece121d59349ac31a12ee9a7f61579307008b58500083777fc439b3d465debb1afd76647664939b10beb030c90cf0008a37257b6b454502c707c573fa7fc80c1685273e54b2a4f76", result.ToHexString()); 70 | } 71 | 72 | [Test] 73 | public void BuildSignatureWithLargeChunkSizeOverLargeFile() 74 | { 75 | var builder = new SignatureBuilder 76 | { 77 | ChunkSize = SignatureBuilder.MaximumChunkSize 78 | }; 79 | 80 | var result = BuildSignature(builder, Helpers.GenerateTestData(100*1024)); 81 | 82 | ClassicAssert.AreEqual("4f43544f5349470104534841310741646c657233323e3e3e007cb823382f5470f51bab46eeb3913379e7b70a0d7329a9afce007cb5278ac69c31becd9bcd36f9afbd350ec15f4c437fd0cb67007c7a20e05ec605af9c2fd5a61b60f65600f5849f6ce1c53cf1001cac9ce9f194d25de18f219fa7832df14593cade50d8b0d2a2", result.ToHexString()); 83 | } 84 | 85 | [Test] 86 | public void BuildSignatureWithAdlerV2() 87 | { 88 | var builder = new SignatureBuilder 89 | { 90 | RollingChecksumAlgorithm = new Adler32RollingChecksumV2() 91 | }; 92 | 93 | var result = BuildSignature(builder, Helpers.TestData()); 94 | 95 | ClassicAssert.AreEqual("4f43544f5349470104534841310941646c6572333256323e3e3e0802f79fe5f8330bd06982d3b5dbda6c1a6ad16687a0cdb03c0d", result.ToHexString()); 96 | } 97 | } 98 | } -------------------------------------------------------------------------------- /source/Octodiff.Tests/Core/SignatureReaderFixture.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using NUnit.Framework; 4 | using NUnit.Framework.Legacy; 5 | using Octodiff.Core; 6 | using Octodiff.Diagnostics; 7 | using Octodiff.Tests.Util; 8 | 9 | namespace Octodiff.Tests.Core 10 | { 11 | public class SignatureReaderFixture 12 | { 13 | static Signature ReadSignature(byte[] input) 14 | { 15 | using (var inStream = new MemoryStream(input)) 16 | { 17 | var reader = new SignatureReader(inStream, NullProgressReporter.Instance); 18 | return reader.ReadSignature(); 19 | } 20 | } 21 | 22 | static void AssertChunk(ChunkSignature chunk, long startOffset, UInt32 checksum, short length, string hashAsHexString) 23 | { 24 | ClassicAssert.AreEqual(startOffset, chunk.StartOffset); 25 | ClassicAssert.AreEqual(checksum, chunk.RollingChecksum); 26 | ClassicAssert.AreEqual(length, chunk.Length); 27 | ClassicAssert.AreEqual(hashAsHexString, chunk.Hash.ToHexString()); 28 | } 29 | 30 | [Test] 31 | public void ReadsStandardSignature() 32 | { 33 | var input = 34 | "4f43544f5349470104534841310741646c657233323e3e3e0802f79fa2f0330bd06982d3b5dbda6c1a6ad16687a0cdb03c0d" 35 | .HexStringToByteArray(); 36 | 37 | var s = ReadSignature(input); 38 | ClassicAssert.AreEqual("SHA1", s.HashAlgorithm.Name); 39 | ClassicAssert.AreEqual("Adler32", s.RollingChecksumAlgorithm.Name); 40 | 41 | ClassicAssert.AreEqual(1, s.Chunks.Count); 42 | AssertChunk(s.Chunks[0], 0, 4037189623, 520, "330bd06982d3b5dbda6c1a6ad16687a0cdb03c0d"); 43 | } 44 | 45 | [Test] 46 | public void ReadsStandardSignatureAdlerV2() 47 | { 48 | var input = 49 | "4f43544f5349470104534841310941646c6572333256323e3e3e0802f79fe5f8330bd06982d3b5dbda6c1a6ad16687a0cdb03c0d" 50 | .HexStringToByteArray(); 51 | 52 | var s = ReadSignature(input); 53 | ClassicAssert.AreEqual("SHA1", s.HashAlgorithm.Name); 54 | ClassicAssert.AreEqual("Adler32V2", s.RollingChecksumAlgorithm.Name); 55 | 56 | ClassicAssert.AreEqual(1, s.Chunks.Count); 57 | AssertChunk(s.Chunks[0], 0, 4175798263, 520, "330bd06982d3b5dbda6c1a6ad16687a0cdb03c0d"); 58 | } 59 | 60 | [Test] 61 | public void ReadsSmallChunkSizeSignature() 62 | { 63 | var input = 64 | "4f43544f5349470104534841310741646c657233323e3e3e8000951f26e719f3978cb607e80a9aab3abbcac8bb1ecbcecf3e80001f18260f0f73196c2aa57877ee5e31291a59b5afca4493658000e035f42a42c4a73471dea3b9746e22dd93893fd8549f11bd8000dd2ff46b72e00e30ecae4c70ee07721d221a3b8a6d1847fa08008a02860c21d4023a8ba580ecdba742e7400aa40b6e449bb3" 65 | .HexStringToByteArray(); 66 | 67 | var s = ReadSignature(input); 68 | ClassicAssert.AreEqual("SHA1", s.HashAlgorithm.Name); 69 | ClassicAssert.AreEqual("Adler32", s.RollingChecksumAlgorithm.Name); 70 | 71 | ClassicAssert.AreEqual(5, s.Chunks.Count); 72 | AssertChunk(s.Chunks[0], 0, 3878035349, 128, "19f3978cb607e80a9aab3abbcac8bb1ecbcecf3e"); 73 | AssertChunk(s.Chunks[1], 128, 254154783, 128, "0f73196c2aa57877ee5e31291a59b5afca449365"); 74 | AssertChunk(s.Chunks[2], 256, 720647648, 128, "42c4a73471dea3b9746e22dd93893fd8549f11bd"); 75 | AssertChunk(s.Chunks[3], 384, 1811165149, 128, "72e00e30ecae4c70ee07721d221a3b8a6d1847fa"); 76 | AssertChunk(s.Chunks[4], 512, 210109066, 8, "21d4023a8ba580ecdba742e7400aa40b6e449bb3"); 77 | } 78 | 79 | [Test] 80 | public void ReadsLargeChunkSizeSignatureOverLargeFile() 81 | { 82 | var input = 83 | "4f43544f5349470104534841310741646c657233323e3e3e007cb823382f5470f51bab46eeb3913379e7b70a0d7329a9afce007cb5278ac69c31becd9bcd36f9afbd350ec15f4c437fd0cb67007c7a20e05ec605af9c2fd5a61b60f65600f5849f6ce1c53cf1001cac9ce9f194d25de18f219fa7832df14593cade50d8b0d2a2" 84 | .HexStringToByteArray(); 85 | 86 | var s = ReadSignature(input); 87 | ClassicAssert.AreEqual("SHA1", s.HashAlgorithm.Name); 88 | ClassicAssert.AreEqual("Adler32", s.RollingChecksumAlgorithm.Name); 89 | 90 | ClassicAssert.AreEqual(4, s.Chunks.Count); 91 | AssertChunk(s.Chunks[0], 0, 792208312, 31744, "5470f51bab46eeb3913379e7b70a0d7329a9afce"); 92 | AssertChunk(s.Chunks[1], 31744, 3330942901, 31744, "9c31becd9bcd36f9afbd350ec15f4c437fd0cb67"); 93 | AssertChunk(s.Chunks[2], 63488, 1591746682, 31744, "c605af9c2fd5a61b60f65600f5849f6ce1c53cf1"); 94 | AssertChunk(s.Chunks[3], 95232, 4058619052, 7168, "94d25de18f219fa7832df14593cade50d8b0d2a2"); 95 | } 96 | } 97 | } -------------------------------------------------------------------------------- /source/Octodiff.Tests/DeltaFixture.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | using System.IO; 4 | using System.Text.RegularExpressions; 5 | using NUnit.Framework; 6 | using NUnit.Framework.Legacy; 7 | using Octodiff.Tests.Util; 8 | 9 | namespace Octodiff.Tests 10 | { 11 | [TestFixture] 12 | public class DeltaFixture : CommandLineFixture 13 | { 14 | [Test] 15 | [TestCase("SmallPackage1mb.zip", 10)] 16 | [TestCase("SmallPackage10mb.zip", 100)] 17 | [TestCase("SmallPackage100mb.zip", 1000)] 18 | public void DeltaOfUnchangedFileShouldResultInJustCopySegment(string name, int numberOfFiles) 19 | { 20 | PackageGenerator.GeneratePackage(name, numberOfFiles); 21 | 22 | Run("signature " + name + " " + name + ".sig"); 23 | Assert.That(ExitCode, Is.EqualTo(0)); 24 | 25 | Run("delta " + name + ".sig " + name + " " + name + ".delta"); 26 | Assert.That(ExitCode, Is.EqualTo(0)); 27 | 28 | Run("explain-delta " + name + ".delta"); 29 | Assert.That(Regex.IsMatch(Output, $"^Copy: 0 to ([0-9A-F]+){Environment.NewLine}$")); 30 | Assert.That(Output, Does.Not.Contain("Data:")); 31 | } 32 | 33 | [Test] 34 | [TestCase("SmallPackage1mb.zip", 10)] 35 | [TestCase("SmallPackage10mb.zip", 100)] 36 | [TestCase("SmallPackage100mb.zip", 1000)] 37 | public void DeltaOfChangedFileShouldResultInNewDataSegments(string name, int numberOfFiles) 38 | { 39 | PackageGenerator.GeneratePackage(name, numberOfFiles); 40 | 41 | Run("signature " + name + " " + name + ".sig"); 42 | Assert.That(ExitCode, Is.EqualTo(0)); 43 | 44 | var newName = Path.ChangeExtension(name, "2.zip"); 45 | PackageGenerator.ModifyPackage(name, newName, (int) (0.33*numberOfFiles), (int) (0.10*numberOfFiles)); 46 | 47 | Run("delta " + name + ".sig " + newName + " " + name + ".delta"); 48 | Assert.That(ExitCode, Is.EqualTo(0)); 49 | 50 | Run("explain-delta " + name + ".delta"); 51 | Assert.That(Regex.IsMatch(Output, $"Copy: ([0-9A-F]+) to ([0-9A-F]+){Environment.NewLine}")); 52 | Assert.That(Regex.IsMatch(Output, "Data: \\(([0-9]+) bytes\\)")); 53 | 54 | var originalSize = new FileInfo(name).Length; 55 | var newSize = new FileInfo(newName).Length; 56 | var deltaSize = new FileInfo(name + ".delta").Length; 57 | var actualDifference = Math.Abs(newSize - originalSize); 58 | var deltaToActualRatio = (double) deltaSize/actualDifference; 59 | Trace.WriteLine(string.Format("Delta ratio: {0:n3}", deltaToActualRatio)); 60 | ClassicAssert.IsTrue(deltaSize * 2 < newSize, "Delta should be at least half the new file size"); 61 | ClassicAssert.IsTrue(0.80 <= deltaToActualRatio && deltaToActualRatio <= 1.60, "Delta should be pretty close to the actual file differences"); 62 | } 63 | } 64 | } -------------------------------------------------------------------------------- /source/Octodiff.Tests/HelpFixture.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.ExceptionServices; 2 | using NUnit.Framework; 3 | using Octodiff.Tests.Util; 4 | 5 | namespace Octodiff.Tests 6 | { 7 | [TestFixture] 8 | public class HelpFixture : CommandLineFixture 9 | { 10 | [Test] 11 | [TestCase(4, "")] 12 | [TestCase(0, "help")] 13 | [TestCase(4, "foo")] 14 | public void ShouldPrintGeneralHelp(int exitCode, string args) 15 | { 16 | Run(args); 17 | Assert.That(ExitCode, Is.EqualTo(exitCode)); 18 | Assert.That(Output, Does.Contain("Usage: Octodiff ")); 19 | Assert.That(Output, Does.Not.Contain("Error")); 20 | } 21 | 22 | [Test] 23 | [TestCase(0, "help signature", "signature")] 24 | [TestCase(0, "help delta", "delta")] 25 | [TestCase(0, "help patch", "patch")] 26 | [TestCase(0, "help explain-delta", "explain-delta")] 27 | public void ShouldPrintCommandHelp(int exitCode, string args, string commandName) 28 | { 29 | Run(args); 30 | Assert.That(ExitCode, Is.EqualTo(exitCode)); 31 | Assert.That(Output, Does.Contain("Usage: Octodiff " + commandName)); 32 | Assert.That(Output, Does.Contain("Usage: Octodiff " + commandName)); 33 | Assert.That(Output, Does.Not.Contain("Error")); 34 | } 35 | 36 | [Test] 37 | [TestCase(4, "signature", "No basis file was specified")] 38 | [TestCase(4, "delta", "No signature file was specified")] 39 | [TestCase(4, "delta foo.sig", "No new file was specified")] 40 | [TestCase(4, "patch", "No basis file was specified")] 41 | [TestCase(4, "patch foo.nupkg", "No delta file was specified")] 42 | [TestCase(4, "patch foo.nupkg foo.delta", "No new file was specified")] 43 | public void ShouldPrintHelpWhenAllArgumentsAreNotSpecified(int exitCode, string args, string text) 44 | { 45 | Run(args); 46 | Assert.That(ExitCode, Is.EqualTo(exitCode)); 47 | Assert.That(Output, Does.Contain("Usage")); 48 | Assert.That(Output, Does.Contain("Error: " + text)); 49 | } 50 | } 51 | } -------------------------------------------------------------------------------- /source/Octodiff.Tests/Octodiff.Tests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Octodiff.Tests 5 | false 6 | true 7 | false 8 | 9 | 10 | 11 | net8.0;net462 12 | 13 | 14 | net8.0 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | Exe 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /source/Octodiff.Tests/PackageGenerator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | using System.IO; 4 | using System.IO.Packaging; 5 | using System.Linq; 6 | 7 | namespace Octodiff.Tests 8 | { 9 | public class PackageGenerator 10 | { 11 | static Random r = new Random(); 12 | 13 | public static void GeneratePackage(string fileName, int numberOfFiles = 10, int averageFileSize = 100*1024) 14 | { 15 | var fullPath = Path.GetFullPath(fileName); 16 | using (var package = ZipPackage.Open(fullPath, FileMode.Create)) 17 | { 18 | for (int i = 0; i < numberOfFiles; i++) 19 | { 20 | var buffer = new byte[averageFileSize]; 21 | var part = package.CreatePart(new Uri("/" + Guid.NewGuid(), UriKind.Relative), "text/plain"); 22 | using (var partStream = part.GetStream(FileMode.Create)) 23 | { 24 | r.NextBytes(buffer); 25 | partStream.Write(buffer, 0, buffer.Length); 26 | } 27 | } 28 | } 29 | } 30 | 31 | public static void ModifyPackage(string fileName, string newFileName, int filesToAdd, int filesToRemove, int averageFileSize = 100*1024) 32 | { 33 | File.Copy(fileName, newFileName, true); 34 | var fullPath = Path.GetFullPath(newFileName); 35 | using (var package = ZipPackage.Open(fullPath, FileMode.Open)) 36 | { 37 | for (int i = 0; i < filesToAdd; i++) 38 | { 39 | var buffer = new byte[averageFileSize]; 40 | var part = package.CreatePart(new Uri("/" + Guid.NewGuid(), UriKind.Relative), "text/plain"); 41 | using (var partStream = part.GetStream(FileMode.Create)) 42 | { 43 | r.NextBytes(buffer); 44 | partStream.Write(buffer, 0, buffer.Length); 45 | } 46 | } 47 | var parts = package.GetParts().OrderBy(o => r.Next(0, 10)).Take(filesToRemove).ToArray(); 48 | foreach (var part in parts) 49 | { 50 | package.DeletePart(part.Uri); 51 | } 52 | } 53 | } 54 | } 55 | } -------------------------------------------------------------------------------- /source/Octodiff.Tests/PatchFixture.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Security.Cryptography; 4 | using NUnit.Framework; 5 | using Octodiff.Tests.Util; 6 | 7 | namespace Octodiff.Tests 8 | { 9 | [TestFixture] 10 | public class PatchFixture : CommandLineFixture 11 | { 12 | [Test] 13 | [TestCase("SmallPackage1mb.zip", 10)] 14 | [TestCase("SmallPackage10mb.zip", 100)] 15 | [TestCase("SmallPackage100mb.zip", 1000)] 16 | public void PatchingShouldResultInPerfectCopy(string name, int numberOfFiles) 17 | { 18 | var newName = Path.ChangeExtension(name, "2.zip"); 19 | var copyName = Path.ChangeExtension(name, "2_out.zip"); 20 | PackageGenerator.GeneratePackage(name, numberOfFiles); 21 | PackageGenerator.ModifyPackage(name, newName, (int)(0.33 * numberOfFiles), (int)(0.10 * numberOfFiles)); 22 | 23 | Run("signature " + name + " " + name + ".sig"); 24 | Run("delta " + name + ".sig " + newName + " " + name + ".delta"); 25 | Run("patch " + name + " " + name + ".delta" + " " + copyName); 26 | Assert.That(ExitCode, Is.EqualTo(0)); 27 | 28 | Assert.That(Sha1(newName), Is.EqualTo(Sha1(copyName))); 29 | } 30 | 31 | [Test] 32 | //[TestCase("SmallPackage1mb.zip", 10)] temp disable this passes locally but fails in appveyor? 33 | [TestCase("SmallPackage10mb.zip", 100)] 34 | public void PatchVerificationShouldFailWhenFilesModified(string name, int numberOfFiles) 35 | { 36 | var newBasis = Path.ChangeExtension(name, "1.zip"); 37 | var newName = Path.ChangeExtension(name, "2.zip"); 38 | var copyName = Path.ChangeExtension(name, "2_out.zip"); 39 | PackageGenerator.GeneratePackage(name, numberOfFiles); 40 | PackageGenerator.ModifyPackage(name, newBasis, (int)(0.33 * numberOfFiles), (int)(0.10 * numberOfFiles)); 41 | PackageGenerator.ModifyPackage(name, newName, (int)(0.33 * numberOfFiles), (int)(0.10 * numberOfFiles)); 42 | 43 | Run("signature " + name + " " + name + ".sig"); 44 | Run("delta " + name + ".sig " + newName + " " + name + ".delta"); 45 | Run("patch " + newBasis + " " + name + ".delta" + " " + copyName); 46 | Assert.That(ExitCode, Is.EqualTo(4)); 47 | Assert.That(Output, Does.Contain("Error: Verification of the patched file failed")); 48 | } 49 | 50 | [Test] 51 | [TestCase("SmallPackage10mb.zip", 100)] 52 | public void PatchVerificationCanBeSkipped(string name, int numberOfFiles) 53 | { 54 | var newBasis = Path.ChangeExtension(name, "1.zip"); 55 | var newName = Path.ChangeExtension(name, "2.zip"); 56 | var copyName = Path.ChangeExtension(name, "2_out.zip"); 57 | PackageGenerator.GeneratePackage(name, numberOfFiles); 58 | PackageGenerator.ModifyPackage(name, newBasis, (int)(0.33 * numberOfFiles), (int)(0.10 * numberOfFiles)); 59 | PackageGenerator.ModifyPackage(name, newName, (int)(0.33 * numberOfFiles), (int)(0.10 * numberOfFiles)); 60 | 61 | Run("signature " + name + " " + name + ".sig"); 62 | Run("delta " + name + ".sig " + newName + " " + name + ".delta"); 63 | Run("patch " + newBasis + " " + name + ".delta" + " " + copyName + " --skip-verification"); 64 | Assert.That(ExitCode, Is.EqualTo(0)); 65 | Assert.That(Sha1(newName), Is.Not.EqualTo(Sha1(copyName))); 66 | } 67 | 68 | static string Sha1(string fileName) 69 | { 70 | using (var s = new FileStream(fileName, FileMode.Open)) 71 | { 72 | return BitConverter.ToString(SHA1.Create().ComputeHash(s)).Replace("-", ""); 73 | } 74 | } 75 | } 76 | } -------------------------------------------------------------------------------- /source/Octodiff.Tests/Program.cs: -------------------------------------------------------------------------------- 1 | namespace Octodiff.Tests 2 | { 3 | public class Program 4 | { 5 | public static int Main(string[] args) 6 | { 7 | return Octodiff.Program.Main(args); 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /source/Octodiff.Tests/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | 3 | [assembly: AssemblyTitle("Octodiff.Tests")] -------------------------------------------------------------------------------- /source/Octodiff.Tests/SignatureFixture.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics; 2 | using System.IO; 3 | using NUnit.Framework; 4 | using Octodiff.Tests.Util; 5 | 6 | namespace Octodiff.Tests 7 | { 8 | [TestFixture] 9 | public class SignatureFixture : CommandLineFixture 10 | { 11 | [Test] 12 | [TestCase("SmallPackage1mb.zip", 10)] 13 | [TestCase("SmallPackage10mb.zip", 100)] 14 | [TestCase("SmallPackage100mb.zip", 1000)] 15 | public void ShouldCreateSignature(string name, int numberOfFiles) 16 | { 17 | PackageGenerator.GeneratePackage(name, numberOfFiles); 18 | 19 | Run("signature " + name + " " + name + ".sig"); 20 | Assert.That(ExitCode, Is.EqualTo(0)); 21 | 22 | var basisSize = new FileInfo(name).Length; 23 | var signatureSize = new FileInfo(name + ".sig").Length; 24 | var signatureSizePercentageOfBasis = signatureSize/(double) basisSize; 25 | 26 | Trace.WriteLine(string.Format("Basis size: {0:n0}", basisSize)); 27 | Trace.WriteLine(string.Format("Signature size: {0:n0}", signatureSize)); 28 | Trace.WriteLine(string.Format("Signature ratio: {0:n3}", signatureSizePercentageOfBasis)); 29 | Assert.That(signatureSizePercentageOfBasis, Is.GreaterThanOrEqualTo(0.012).And.LessThanOrEqualTo(0.014)); 30 | } 31 | [Test] 32 | [TestCase("SmallPackage1mb.zip", 10)] 33 | [TestCase("SmallPackage10mb.zip", 100)] 34 | [TestCase("SmallPackage100mb.zip", 1000)] 35 | public void ShouldCreateDifferentSignaturesBasedOnChunkSize(string name, int numberOfFiles) 36 | { 37 | PackageGenerator.GeneratePackage(name, numberOfFiles); 38 | 39 | Run("signature " + name + " " + name + ".sig.1 --chunk-size=128"); 40 | Run("signature " + name + " " + name + ".sig.2 --chunk-size=256"); 41 | Run("signature " + name + " " + name + ".sig.3 --chunk-size=1024"); 42 | Run("signature " + name + " " + name + ".sig.4 --chunk-size=2048"); 43 | Run("signature " + name + " " + name + ".sig.5 --chunk-size=31744"); 44 | 45 | Assert.That(Length(name + ".sig.1") > Length(name + ".sig.2")); 46 | Assert.That(Length(name + ".sig.2") > Length(name + ".sig.3")); 47 | Assert.That(Length(name + ".sig.3") > Length(name + ".sig.4")); 48 | Assert.That(Length(name + ".sig.4") > Length(name + ".sig.5")); 49 | } 50 | 51 | static long Length(string fileName) 52 | { 53 | return new FileInfo(fileName).Length; 54 | } 55 | } 56 | } -------------------------------------------------------------------------------- /source/Octodiff.Tests/Timings.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | using System.IO; 4 | using NUnit.Framework; 5 | using Octodiff.Tests.Util; 6 | 7 | namespace Octodiff.Tests 8 | { 9 | [TestFixture] 10 | public class TimingsFixture : CommandLineFixture 11 | { 12 | [Test] 13 | [TestCase("SmallPackage1mb.zip", 10)] 14 | [TestCase("SmallPackage10mb.zip", 100)] 15 | [TestCase("SmallPackage100mb.zip", 1000)] 16 | public void ExecuteWithTimings(string name, int numberOfFiles) 17 | { 18 | var newName = Path.ChangeExtension(name, "2.zip"); 19 | var copyName = Path.ChangeExtension(name, "2_out.zip"); 20 | 21 | Time("Package creation", () => PackageGenerator.GeneratePackage(name, numberOfFiles)); 22 | Time("Package modification", () => PackageGenerator.ModifyPackage(name, newName, (int)(0.33 * numberOfFiles), (int)(0.10 * numberOfFiles))); 23 | Time("Signature creation", () => Run("signature " + name + " " + name + ".sig")); 24 | Time("Delta creation", () => Run("delta " + name + ".sig " + newName + " " + name + ".delta")); 25 | Time("Patch application", () => Run("patch " + name + " " + name + ".delta" + " " + copyName)); 26 | Time("Patch application (no verify)", () => Run("patch " + name + " " + name + ".delta" + " " + copyName + " --skip-verification")); 27 | } 28 | 29 | static void Time(string task, Action callback) 30 | { 31 | var watch = Stopwatch.StartNew(); 32 | callback(); 33 | Trace.WriteLine(task.PadRight(30, ' ') + ": " + watch.ElapsedMilliseconds + "ms"); 34 | } 35 | } 36 | } -------------------------------------------------------------------------------- /source/Octodiff.Tests/Util/CommandLineFixture.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | using System.IO; 4 | using System.Reflection; 5 | using System.Text; 6 | using Octodiff.Core; 7 | using Octopus.Platform.Util; 8 | 9 | namespace Octodiff.Tests.Util 10 | { 11 | public abstract class CommandLineFixture 12 | { 13 | protected string StdErr { get; private set; } 14 | protected string StdOut { get; private set; } 15 | protected string Output { get; private set; } 16 | protected int ExitCode { get; set; } 17 | 18 | public void Run(string args) 19 | { 20 | var stdErrBuilder = new StringBuilder(); 21 | var stdOutBuilder = new StringBuilder(); 22 | var outputBuilder = new StringBuilder(); 23 | var path = GetExePath(); 24 | #if !NET462 25 | args = $"{path} {args}"; 26 | path = "dotnet"; 27 | #endif 28 | var exit = SilentProcessRunner.ExecuteCommand(path, 29 | args, 30 | GetCurrentDirectory(), 31 | output => 32 | { 33 | stdOutBuilder.AppendLine(output); 34 | outputBuilder.AppendLine(output); 35 | Trace.WriteLine(output); 36 | }, 37 | output => 38 | { 39 | stdErrBuilder.AppendLine(output); 40 | outputBuilder.AppendLine(output); 41 | Trace.WriteLine(output); 42 | }); 43 | 44 | StdErr = stdErrBuilder.ToString(); 45 | StdOut = stdOutBuilder.ToString(); 46 | Output = outputBuilder.ToString(); 47 | ExitCode = exit; 48 | } 49 | 50 | string GetExePath() 51 | { 52 | #if NET462 53 | return new Uri(typeof(DeltaBuilder).Assembly.CodeBase).LocalPath; 54 | #else 55 | return Path.Combine(Path.GetDirectoryName(new Uri(typeof(CommandLineFixture).GetTypeInfo().Assembly.CodeBase).LocalPath), "Octodiff.Tests.dll"); 56 | #endif 57 | } 58 | 59 | string GetCurrentDirectory() 60 | { 61 | #if NET462 62 | return Environment.CurrentDirectory; 63 | #else 64 | return System.IO.Directory.GetCurrentDirectory(); 65 | #endif 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /source/Octodiff.Tests/Util/Helpers.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Text; 3 | 4 | namespace Octodiff.Tests.Util 5 | { 6 | public static class Helpers 7 | { 8 | // this is a self-signed x509 cert generated by OpenSSL. Ingore the fact that it's a certificate; Used as test data because x509 is full of weird binary data values, increasing our likelihood of catching signed/unsigned mismatch bugs. 9 | private static readonly byte[] testData = Convert.FromBase64String( 10 | "MIICBDCCAaugAwIBAgIUGNg/B3GL5BId8KGNdhD6+NejvsQwCgYIKoZIzj0EAwIwWDELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxFzAVBgNVBAoMDk9jdG9wdXMgRGVwbG95MQwwCgYDVQQLDANSJkQxDTALBgNVBAMMBFRFU1QwHhcNMjMwMzIwMDk0ODQyWhcNMjQwMzE5MDk0ODQyWjBYMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEXMBUGA1UECgwOT2N0b3B1cyBEZXBsb3kxDDAKBgNVBAsMA1ImRDENMAsGA1UEAwwEVEVTVDBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABFBLdySNg+Lj4gm7sil6Dk0k/0Xnnv+I3RZeZBmumFEtq9IhnaRuk9f/mNWhy4AxSlfzfQkx7PfzvUvOISz9LLqjUzBRMB0GA1UdDgQWBBS63SeKMeASd2r7/aTq2P3OkE8O/DAfBgNVHSMEGDAWgBS63SeKMeASd2r7/aTq2P3OkE8O/DAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMCA0cAMEQCIFmc75IBFbZKfQvH3lWoS7p/Be54uekDr3y1K0pdzI6iAiAGV1RF2rnCEyWkjeO9fOUaNGEgFadGSHh8en4DJkU3cA=="); 11 | 12 | // returns our dummy x509 certificate. Copies buffer to ensure the underlying source isn't mutated 13 | // we can't use nice things like ReadOnlyMemory or because this cross-compiles for dotnet 4.x, not just core 14 | public static byte[] TestData() => GenerateTestData(testData.Length); 15 | 16 | // generates arbitrary length test data by repeating copies of our dummy x509 certificate 17 | public static byte[] GenerateTestData(int byteCount) 18 | { 19 | var testDataLen = testData.Length; 20 | var result = new byte[byteCount]; 21 | for (var offset = 0; offset <= byteCount; offset += testDataLen) 22 | { 23 | Buffer.BlockCopy(testData, 0, result, offset, Math.Min(testDataLen, byteCount - offset)); 24 | } 25 | 26 | return result; 27 | } 28 | 29 | // courtesy https://stackoverflow.com/a/9995303/234 30 | public static byte[] HexStringToByteArray(this string hex) 31 | { 32 | int GetHexVal(int val) 33 | { 34 | //For uppercase A-F letters: 35 | //return val - (val < 58 ? 48 : 55); 36 | //For lowercase a-f letters: 37 | //return val - (val < 58 ? 48 : 87); 38 | //Or the two combined, but a bit slower: 39 | return val - (val < 58 ? 48 : (val < 97 ? 55 : 87)); 40 | } 41 | 42 | if (hex.Length % 2 == 1) 43 | throw new Exception("The binary key cannot have an odd number of digits"); 44 | 45 | byte[] arr = new byte[hex.Length >> 1]; 46 | 47 | for (int i = 0; i < hex.Length >> 1; ++i) 48 | { 49 | arr[i] = (byte)((GetHexVal(hex[i << 1]) << 4) + (GetHexVal(hex[(i << 1) + 1]))); 50 | } 51 | 52 | return arr; 53 | } 54 | 55 | public static string ToHexString(this byte[] byteArray) 56 | { 57 | var hex = new StringBuilder(byteArray.Length * 2); 58 | foreach (var b in byteArray) 59 | hex.AppendFormat("{0:x2}", b); 60 | return hex.ToString(); 61 | } 62 | } 63 | } -------------------------------------------------------------------------------- /source/Octodiff.Tests/Util/SilentProcessRunner.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Diagnostics; 3 | using System.Runtime.InteropServices; 4 | using System.Text; 5 | using System.Threading; 6 | 7 | namespace Octopus.Platform.Util 8 | { 9 | public static class SilentProcessRunner 10 | { 11 | // ReSharper disable once InconsistentNaming 12 | private const int CP_OEMCP = 1; 13 | private static readonly Encoding oemEncoding; 14 | 15 | static SilentProcessRunner() 16 | { 17 | try 18 | { 19 | CPINFOEX info; 20 | if (GetCPInfoEx(CP_OEMCP, 0, out info)) 21 | { 22 | oemEncoding = Encoding.GetEncoding(info.CodePage); 23 | } 24 | else 25 | { 26 | oemEncoding = Encoding.GetEncoding(850); 27 | } 28 | } 29 | catch (Exception) 30 | { 31 | oemEncoding = Encoding.UTF8; 32 | } 33 | } 34 | 35 | public static int ExecuteCommand(string executable, string arguments, string workingDirectory, Action output, Action error) 36 | { 37 | try 38 | { 39 | using (var process = new Process()) 40 | { 41 | process.StartInfo.FileName = executable; 42 | process.StartInfo.Arguments = arguments; 43 | process.StartInfo.WorkingDirectory = workingDirectory; 44 | process.StartInfo.UseShellExecute = false; 45 | process.StartInfo.CreateNoWindow = true; 46 | process.StartInfo.RedirectStandardOutput = true; 47 | process.StartInfo.RedirectStandardError = true; 48 | process.StartInfo.StandardOutputEncoding = oemEncoding; 49 | process.StartInfo.StandardErrorEncoding = oemEncoding; 50 | 51 | using (var outputWaitHandle = new AutoResetEvent(false)) 52 | using (var errorWaitHandle = new AutoResetEvent(false)) 53 | { 54 | process.OutputDataReceived += (sender, e) => 55 | { 56 | if (e.Data == null) 57 | { 58 | outputWaitHandle.Set(); 59 | } 60 | else 61 | { 62 | output(e.Data); 63 | } 64 | }; 65 | 66 | process.ErrorDataReceived += (sender, e) => 67 | { 68 | if (e.Data == null) 69 | { 70 | errorWaitHandle.Set(); 71 | } 72 | else 73 | { 74 | error(e.Data); 75 | } 76 | }; 77 | 78 | process.Start(); 79 | 80 | process.BeginOutputReadLine(); 81 | process.BeginErrorReadLine(); 82 | 83 | process.WaitForExit(); 84 | outputWaitHandle.WaitOne(); 85 | errorWaitHandle.WaitOne(); 86 | 87 | return process.ExitCode; 88 | } 89 | } 90 | } 91 | catch (Exception ex) 92 | { 93 | throw new Exception(string.Format("Error when attempting to execute {0}: {1}", executable, ex.Message), ex); 94 | } 95 | } 96 | 97 | 98 | 99 | [DllImport("kernel32.dll", SetLastError = true)] 100 | private static extern bool GetCPInfoEx([MarshalAs(UnmanagedType.U4)] int CodePage, [MarshalAs(UnmanagedType.U4)] int dwFlags, out CPINFOEX lpCPInfoEx); 101 | 102 | private const int MAX_DEFAULTCHAR = 2; 103 | private const int MAX_LEADBYTES = 12; 104 | private const int MAX_PATH = 260; 105 | 106 | [StructLayout(LayoutKind.Sequential)] 107 | private struct CPINFOEX 108 | { 109 | [MarshalAs(UnmanagedType.U4)] 110 | public int MaxCharSize; 111 | 112 | [MarshalAs(UnmanagedType.ByValArray, SizeConst = MAX_DEFAULTCHAR)] 113 | public byte[] DefaultChar; 114 | 115 | [MarshalAs(UnmanagedType.ByValArray, SizeConst = MAX_LEADBYTES)] 116 | public byte[] LeadBytes; 117 | 118 | public char UnicodeDefaultChar; 119 | 120 | [MarshalAs(UnmanagedType.U4)] 121 | public int CodePage; 122 | 123 | [MarshalAs(UnmanagedType.ByValTStr, SizeConst = MAX_PATH)] 124 | public string CodePageName; 125 | } 126 | } 127 | } -------------------------------------------------------------------------------- /source/Octodiff.sln: -------------------------------------------------------------------------------- 1 | Microsoft Visual Studio Solution File, Format Version 12.00 2 | # Visual Studio 15 3 | VisualStudioVersion = 15.0.26730.12 4 | MinimumVisualStudioVersion = 10.0.40219.1 5 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Octodiff", "Octodiff\Octodiff.csproj", "{BBA7CDC2-DE25-4131-89F2-506712178FC8}" 6 | EndProject 7 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Octodiff.Tests", "Octodiff.Tests\Octodiff.Tests.csproj", "{B7E95BD1-676E-4AC7-9089-3270392776BA}" 8 | EndProject 9 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "_build", "..\build\_build.csproj", "{D7FD8AAD-2386-4BC8-BDBB-B2CFA63DBE57}" 10 | EndProject 11 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Octodiff.Benchmarks", "Octodiff.Benchmarks\Octodiff.Benchmarks.csproj", "{72DD74E4-7B7C-43DF-BF85-92FC8F64682B}" 12 | EndProject 13 | Global 14 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 15 | Debug|Any CPU = Debug|Any CPU 16 | Release|Any CPU = Release|Any CPU 17 | EndGlobalSection 18 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 19 | {D7FD8AAD-2386-4BC8-BDBB-B2CFA63DBE57}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 20 | {D7FD8AAD-2386-4BC8-BDBB-B2CFA63DBE57}.Release|Any CPU.ActiveCfg = Release|Any CPU 21 | {BBA7CDC2-DE25-4131-89F2-506712178FC8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 22 | {BBA7CDC2-DE25-4131-89F2-506712178FC8}.Debug|Any CPU.Build.0 = Debug|Any CPU 23 | {BBA7CDC2-DE25-4131-89F2-506712178FC8}.Release|Any CPU.ActiveCfg = Release|Any CPU 24 | {BBA7CDC2-DE25-4131-89F2-506712178FC8}.Release|Any CPU.Build.0 = Release|Any CPU 25 | {B7E95BD1-676E-4AC7-9089-3270392776BA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 26 | {B7E95BD1-676E-4AC7-9089-3270392776BA}.Debug|Any CPU.Build.0 = Debug|Any CPU 27 | {B7E95BD1-676E-4AC7-9089-3270392776BA}.Release|Any CPU.ActiveCfg = Release|Any CPU 28 | {B7E95BD1-676E-4AC7-9089-3270392776BA}.Release|Any CPU.Build.0 = Release|Any CPU 29 | {72DD74E4-7B7C-43DF-BF85-92FC8F64682B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 30 | {72DD74E4-7B7C-43DF-BF85-92FC8F64682B}.Debug|Any CPU.Build.0 = Debug|Any CPU 31 | {72DD74E4-7B7C-43DF-BF85-92FC8F64682B}.Release|Any CPU.ActiveCfg = Release|Any CPU 32 | {72DD74E4-7B7C-43DF-BF85-92FC8F64682B}.Release|Any CPU.Build.0 = Release|Any CPU 33 | EndGlobalSection 34 | GlobalSection(SolutionProperties) = preSolution 35 | HideSolutionNode = FALSE 36 | EndGlobalSection 37 | GlobalSection(ExtensibilityGlobals) = postSolution 38 | SolutionGuid = {78A05032-DCC5-460F-AEC3-7827471203CF} 39 | EndGlobalSection 40 | EndGlobal 41 | -------------------------------------------------------------------------------- /source/Octodiff.sln.DotSettings: -------------------------------------------------------------------------------- 1 |  2 | <Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /> 3 | <Policy><Descriptor Staticness="Instance" AccessRightKinds="Private" Description="Instance fields (private)"><ElementKinds><Kind Name="FIELD" /><Kind Name="READONLY_FIELD" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /></Policy> 4 | True -------------------------------------------------------------------------------- /source/Octodiff.vssettings: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 1 7 | 1 8 | 1 9 | 0 10 | 1 11 | 0 12 | 1 13 | 1 14 | 0 15 | 1 16 | 1 17 | 1 18 | 1 19 | 0 20 | 0 21 | 1 22 | 1 23 | 1 24 | 1 25 | 1 26 | 1 27 | 1 28 | 1 29 | 1 30 | 1 31 | 1 32 | 1 33 | 1 34 | 1 35 | 0 36 | 1 37 | 0 38 | 1 39 | 0 40 | 0 41 | 1 42 | 1 43 | 1 44 | 0 45 | 0 46 | 1 47 | 0 48 | 0 49 | 0 50 | 0 51 | 0 52 | 1 53 | 0 54 | 0 55 | 0 56 | 0 57 | 0 58 | 0 59 | 0 60 | 0 61 | 0 62 | 1 63 | 1 64 | 65 | 66 | 67 | 68 | -------------------------------------------------------------------------------- /source/Octodiff/CommandLine/DeltaCommand.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using Octodiff.CommandLine.Support; 5 | using Octodiff.Core; 6 | using Octodiff.Diagnostics; 7 | 8 | namespace Octodiff.CommandLine 9 | { 10 | [Command("delta", Description = "Given a signature file and a new file, creates a delta file", Usage = " []")] 11 | class DeltaCommand : ICommand 12 | { 13 | private readonly List> configuration = new List>(); 14 | private readonly OptionSet options; 15 | private string newFilePath; 16 | private string signatureFilePath; 17 | private string deltaFilePath; 18 | 19 | public DeltaCommand() 20 | { 21 | options = new OptionSet(); 22 | options.Positional("signature-file", "The file containing the signature from the basis file.", v => signatureFilePath = v); 23 | options.Positional("new-file", "The file to create the delta from.", v => newFilePath = v); 24 | options.Positional("delta-file", "The file to write the delta to.", v => deltaFilePath = v); 25 | options.Add("progress", "Whether progress should be written to stdout", v => configuration.Add(builder => builder.ProgressReporter = new ConsoleProgressReporter())); 26 | } 27 | 28 | public void GetHelp(TextWriter writer) 29 | { 30 | options.WriteOptionDescriptions(writer); 31 | } 32 | 33 | public int Execute(string[] commandLineArguments) 34 | { 35 | options.Parse(commandLineArguments); 36 | 37 | if (string.IsNullOrWhiteSpace(signatureFilePath)) 38 | throw new OptionException("No signature file was specified", "signature-file"); 39 | if (string.IsNullOrWhiteSpace(newFilePath)) 40 | throw new OptionException("No new file was specified", "new-file"); 41 | 42 | newFilePath = Path.GetFullPath(newFilePath); 43 | signatureFilePath = Path.GetFullPath(signatureFilePath); 44 | 45 | var delta = new DeltaBuilder(); 46 | foreach (var config in configuration) config(delta); 47 | 48 | if (!File.Exists(signatureFilePath)) 49 | { 50 | throw new FileNotFoundException("File not found: " + signatureFilePath, signatureFilePath); 51 | } 52 | 53 | if (!File.Exists(newFilePath)) 54 | { 55 | throw new FileNotFoundException("File not found: " + newFilePath, newFilePath); 56 | } 57 | 58 | if (string.IsNullOrWhiteSpace(deltaFilePath)) 59 | { 60 | deltaFilePath = newFilePath + ".octodelta"; 61 | } 62 | else 63 | { 64 | deltaFilePath = Path.GetFullPath(deltaFilePath); 65 | var directory = Path.GetDirectoryName(deltaFilePath); 66 | if (directory != null && !Directory.Exists(directory)) 67 | { 68 | Directory.CreateDirectory(directory); 69 | } 70 | } 71 | 72 | using (var newFileStream = new FileStream(newFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) 73 | using (var signatureStream = new FileStream(signatureFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) 74 | using (var deltaStream = new FileStream(deltaFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) 75 | { 76 | delta.BuildDelta(newFileStream, new SignatureReader(signatureStream, delta.ProgressReporter), new AggregateCopyOperationsDecorator(new BinaryDeltaWriter(deltaStream))); 77 | } 78 | 79 | return 0; 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /source/Octodiff/CommandLine/ExplainDeltaCommand.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Linq; 4 | using Octodiff.CommandLine.Support; 5 | using Octodiff.Core; 6 | using Octodiff.Diagnostics; 7 | 8 | namespace Octodiff.CommandLine 9 | { 10 | [Command("explain-delta", Description = "Prints instructions from a delta file; useful when debugging.", Usage = "")] 11 | class ExplainDeltaCommand : ICommand 12 | { 13 | private readonly OptionSet options; 14 | private string deltaFilePath; 15 | 16 | public ExplainDeltaCommand() 17 | { 18 | options = new OptionSet(); 19 | options.Positional("delta-file", "The file to read the delta from.", v => deltaFilePath = v); 20 | } 21 | 22 | public void GetHelp(TextWriter writer) 23 | { 24 | options.WriteOptionDescriptions(writer); 25 | } 26 | 27 | public int Execute(string[] commandLineArguments) 28 | { 29 | options.Parse(commandLineArguments); 30 | 31 | if (string.IsNullOrWhiteSpace(deltaFilePath)) 32 | throw new OptionException("No delta file was specified", "delta-file"); 33 | 34 | deltaFilePath = Path.GetFullPath(deltaFilePath); 35 | 36 | if (!File.Exists(deltaFilePath)) 37 | { 38 | throw new FileNotFoundException("File not found: " + deltaFilePath, deltaFilePath); 39 | } 40 | 41 | using (var deltaStream = new FileStream(deltaFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) 42 | { 43 | var reader = new BinaryDeltaReader(deltaStream, NullProgressReporter.Instance); 44 | 45 | reader.Apply((data, offset, count) => 46 | { 47 | if (count > 20) 48 | { 49 | Console.WriteLine("Data: ({0} bytes): {1}...", count, 50 | BitConverter.ToString(data.Skip(offset).Take(20).ToArray())); 51 | } 52 | else 53 | { 54 | Console.WriteLine("Data: ({0} bytes): {1}", count, BitConverter.ToString(data.Skip(offset).Take(count).ToArray())); 55 | } 56 | }, 57 | (start, offset) => Console.WriteLine("Copy: {0:X} to {1:X}", start, offset)); 58 | } 59 | 60 | return 0; 61 | } 62 | } 63 | } -------------------------------------------------------------------------------- /source/Octodiff/CommandLine/HelpCommand.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Linq; 4 | using System.Reflection; 5 | using Octodiff.CommandLine.Support; 6 | 7 | namespace Octodiff.CommandLine 8 | { 9 | [Command("help", "?", "h", Description = "Prints this help text")] 10 | class HelpCommand : ICommand 11 | { 12 | readonly ICommandLocator commands = new CommandLocator(); 13 | 14 | public void GetHelp(TextWriter writer) 15 | { 16 | } 17 | 18 | public int Execute(string[] commandLineArguments) 19 | { 20 | #if NET40 21 | var executable = Path.GetFileNameWithoutExtension(new Uri(typeof(HelpCommand).Assembly.CodeBase).LocalPath); 22 | #else 23 | var executable = Path.GetFileNameWithoutExtension(new Uri(typeof(HelpCommand).GetTypeInfo().Assembly.CodeBase).LocalPath); 24 | #endif 25 | 26 | var commandName = commandLineArguments.FirstOrDefault(); 27 | 28 | if (string.IsNullOrEmpty(commandName)) 29 | { 30 | PrintGeneralHelp(executable); 31 | } 32 | else 33 | { 34 | var commandMeta = commands.Find(commandName); 35 | if (commandMeta == null) 36 | { 37 | Console.ForegroundColor = ConsoleColor.Red; 38 | Console.WriteLine("Command '{0}' is not supported", commandName); 39 | Console.ResetColor(); 40 | PrintGeneralHelp(executable); 41 | } 42 | else 43 | { 44 | var command = commands.Create(commandMeta); 45 | PrintCommandHelp(executable, command, commandMeta, commandName); 46 | } 47 | } 48 | 49 | return 0; 50 | } 51 | 52 | void PrintCommandHelp(string executable, ICommand command, ICommandMetadata commandMetadata, string commandName) 53 | { 54 | Console.ResetColor(); 55 | Console.Write("Usage: "); 56 | Console.ForegroundColor = ConsoleColor.White; 57 | Console.WriteLine(executable + " " + commandName + (!string.IsNullOrWhiteSpace(commandMetadata.Usage) ? " " + commandMetadata.Usage : "") + " []"); 58 | Console.ResetColor(); 59 | Console.WriteLine(); 60 | command.GetHelp(Console.Out); 61 | 62 | Console.WriteLine(); 63 | } 64 | 65 | void PrintGeneralHelp(string executable) 66 | { 67 | Console.ResetColor(); 68 | Console.Write("Usage: "); 69 | Console.ForegroundColor = ConsoleColor.White; 70 | Console.WriteLine(executable + " "); 71 | Console.ResetColor(); 72 | Console.WriteLine(); 73 | Console.WriteLine("Where is one of: "); 74 | Console.WriteLine(); 75 | 76 | foreach (var possible in commands.List().OrderBy(x => x.Name)) 77 | { 78 | Console.ForegroundColor = ConsoleColor.White; 79 | Console.WriteLine(" " + possible.Name.PadRight(15, ' ')); 80 | Console.ResetColor(); 81 | Console.WriteLine(" " + possible.Description); 82 | } 83 | 84 | Console.WriteLine(); 85 | Console.Write("Or use "); 86 | Console.ForegroundColor = ConsoleColor.White; 87 | Console.Write(executable + " help "); 88 | Console.ResetColor(); 89 | Console.WriteLine(" for more details."); 90 | } 91 | } 92 | } -------------------------------------------------------------------------------- /source/Octodiff/CommandLine/PatchCommand.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | using Octodiff.CommandLine.Support; 3 | using Octodiff.Core; 4 | using Octodiff.Diagnostics; 5 | 6 | namespace Octodiff.CommandLine 7 | { 8 | [Command("patch", Description = "Given a basis file, and a delta, produces the new file", Usage = " ")] 9 | class PatchCommand : ICommand 10 | { 11 | private readonly OptionSet options; 12 | private IProgressReporter progressReporter; 13 | private string basisFilePath; 14 | private string deltaFilePath; 15 | private string newFilePath; 16 | private bool skipHashCheck; 17 | 18 | public PatchCommand() 19 | { 20 | options = new OptionSet(); 21 | options.Positional("basis-file", "The file that the delta was created for.", v => basisFilePath = v); 22 | options.Positional("delta-file", "The delta to apply to the basis file", v => deltaFilePath = v); 23 | options.Positional("new-file", "The file to write the result to.", v => newFilePath = v); 24 | options.Add("progress", "Whether progress should be written to stdout", v => progressReporter = new ConsoleProgressReporter()); 25 | options.Add("skip-verification", "Skip checking whether the basis file is the same as the file used to produce the signature that created the delta.", v => skipHashCheck = true); 26 | } 27 | 28 | public void GetHelp(TextWriter writer) 29 | { 30 | options.WriteOptionDescriptions(writer); 31 | } 32 | 33 | public int Execute(string[] commandLineArguments) 34 | { 35 | options.Parse(commandLineArguments); 36 | 37 | if (string.IsNullOrWhiteSpace(basisFilePath)) 38 | throw new OptionException("No basis file was specified", "basis-file"); 39 | if (string.IsNullOrWhiteSpace(deltaFilePath)) 40 | throw new OptionException("No delta file was specified", "delta-file"); 41 | if (string.IsNullOrWhiteSpace(newFilePath)) 42 | throw new OptionException("No new file was specified", "new-file"); 43 | 44 | basisFilePath = Path.GetFullPath(basisFilePath); 45 | deltaFilePath = Path.GetFullPath(deltaFilePath); 46 | newFilePath = Path.GetFullPath(newFilePath); 47 | 48 | if (!File.Exists(basisFilePath)) throw new FileNotFoundException("File not found: " + basisFilePath, basisFilePath); 49 | if (!File.Exists(deltaFilePath)) throw new FileNotFoundException("File not found: " + deltaFilePath, deltaFilePath); 50 | 51 | var directory = Path.GetDirectoryName(newFilePath); 52 | if (directory != null && !Directory.Exists(directory)) 53 | { 54 | Directory.CreateDirectory(directory); 55 | } 56 | 57 | var delta = new DeltaApplier 58 | { 59 | SkipHashCheck = skipHashCheck 60 | }; 61 | 62 | using (var basisStream = new FileStream(basisFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) 63 | using (var deltaStream = new FileStream(deltaFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) 64 | using (var newFileStream = new FileStream(newFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.Read)) 65 | { 66 | delta.Apply(basisStream, new BinaryDeltaReader(deltaStream, progressReporter), newFileStream); 67 | } 68 | 69 | return 0; 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /source/Octodiff/CommandLine/SignatureCommand.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using Octodiff.CommandLine.Support; 5 | using Octodiff.Core; 6 | using Octodiff.Diagnostics; 7 | 8 | namespace Octodiff.CommandLine 9 | { 10 | [Command("signature", "sig", Description = "Given a basis file, creates a signature file", Usage = " []")] 11 | class SignatureCommand : ICommand 12 | { 13 | private readonly List> configuration = new List>(); 14 | private readonly OptionSet options; 15 | private string basisFilePath; 16 | private string signatureFilePath; 17 | 18 | public SignatureCommand() 19 | { 20 | options = new OptionSet(); 21 | options.Positional("basis-file", "The file to read and create a signature from.", v => basisFilePath = v); 22 | options.Positional("signature-file", "The file to write the signature to.", v => signatureFilePath = v); 23 | options.Add("chunk-size=", string.Format("Maximum bytes per chunk. Defaults to {0}. Min of {1}, max of {2}.", SignatureBuilder.DefaultChunkSize, SignatureBuilder.MinimumChunkSize, SignatureBuilder.MaximumChunkSize), v => configuration.Add(builder => builder.ChunkSize = short.Parse(v))); 24 | options.Add("progress", "Whether progress should be written to stdout", v => configuration.Add(builder => builder.ProgressReporter = new ConsoleProgressReporter())); 25 | } 26 | 27 | public void GetHelp(TextWriter writer) 28 | { 29 | options.WriteOptionDescriptions(writer); 30 | } 31 | 32 | public int Execute(string[] commandLineArguments) 33 | { 34 | options.Parse(commandLineArguments); 35 | 36 | if (string.IsNullOrWhiteSpace(basisFilePath)) 37 | throw new OptionException("No basis file was specified", "basis-file"); 38 | 39 | basisFilePath = Path.GetFullPath(basisFilePath); 40 | 41 | var signatureBuilder = new SignatureBuilder(); 42 | foreach (var config in configuration) config(signatureBuilder); 43 | 44 | if (!File.Exists(basisFilePath)) 45 | { 46 | throw new FileNotFoundException("File not found: " + basisFilePath, basisFilePath); 47 | } 48 | 49 | if (string.IsNullOrWhiteSpace(signatureFilePath)) 50 | { 51 | signatureFilePath = basisFilePath + ".octosig"; 52 | } 53 | else 54 | { 55 | signatureFilePath = Path.GetFullPath(signatureFilePath); 56 | var sigDirectory = Path.GetDirectoryName(signatureFilePath); 57 | if (sigDirectory != null && !Directory.Exists(sigDirectory)) 58 | { 59 | Directory.CreateDirectory(sigDirectory); 60 | } 61 | } 62 | 63 | using (var basisStream = new FileStream(basisFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)) 64 | using (var signatureStream = new FileStream(signatureFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) 65 | { 66 | signatureBuilder.Build(basisStream, new SignatureWriter(signatureStream)); 67 | } 68 | 69 | return 0; 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /source/Octodiff/CommandLine/Support/CommandAttribute.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Octodiff.CommandLine.Support 4 | { 5 | [AttributeUsage(AttributeTargets.Class, AllowMultiple = false, Inherited = false)] 6 | class CommandAttribute : Attribute, ICommandMetadata 7 | { 8 | public CommandAttribute(string name, params string[] aliases) 9 | { 10 | Name = name; 11 | Aliases = aliases; 12 | } 13 | 14 | public string Name { get; set; } 15 | public string[] Aliases { get; set; } 16 | public string Description { get; set; } 17 | public string Usage { get; set; } 18 | } 19 | } -------------------------------------------------------------------------------- /source/Octodiff/CommandLine/Support/CommandException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Octodiff.CommandLine.Support 4 | { 5 | class CommandException : Exception 6 | { 7 | public CommandException(string message) 8 | : base(message) 9 | { 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /source/Octodiff/CommandLine/Support/CommandLocator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Reflection; 5 | 6 | namespace Octodiff.CommandLine.Support 7 | { 8 | class CommandLocator : ICommandLocator 9 | { 10 | public ICommandMetadata[] List() 11 | { 12 | return 13 | (from t in assemblyCommands 14 | let attribute = GetCommandAttribute(t) 15 | where attribute != null 16 | select attribute).ToArray(); 17 | } 18 | 19 | public ICommandMetadata Find(string name) 20 | { 21 | name = name.Trim().ToLowerInvariant(); 22 | return (from t in assemblyCommands 23 | let attribute = GetCommandAttribute(t) 24 | where attribute != null 25 | where attribute.Name == name || attribute.Aliases.Any(a => a == name) 26 | select attribute).FirstOrDefault(); 27 | } 28 | 29 | public ICommand Create(ICommandMetadata metadata) 30 | { 31 | var name = metadata.Name; 32 | var found = (from t in assemblyCommands 33 | let attribute = GetCommandAttribute(t) 34 | where attribute != null 35 | where attribute.Name == name || attribute.Aliases.Any(a => a == name) 36 | select t).FirstOrDefault(); 37 | 38 | return found == null ? null : (ICommand) Activator.CreateInstance(found); 39 | } 40 | 41 | IEnumerable assemblyCommands => 42 | #if NET40 43 | typeof(CommandLocator).Assembly.GetTypes().Where(t => typeof(ICommand).IsAssignableFrom(t)); 44 | #else 45 | typeof(CommandLocator).GetTypeInfo().Assembly.GetTypes().Where(t => typeof(ICommand).GetTypeInfo().IsAssignableFrom(t)); 46 | #endif 47 | 48 | ICommandMetadata GetCommandAttribute(Type type) 49 | { 50 | #if NET40 51 | return (ICommandMetadata)type.GetCustomAttributes(typeof(CommandAttribute), true).FirstOrDefault(); 52 | #else 53 | return (ICommandMetadata)type.GetTypeInfo().GetCustomAttributes(typeof(CommandAttribute), true).FirstOrDefault(); 54 | #endif 55 | } 56 | } 57 | } -------------------------------------------------------------------------------- /source/Octodiff/CommandLine/Support/ICommand.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | 3 | namespace Octodiff.CommandLine.Support 4 | { 5 | interface ICommand 6 | { 7 | void GetHelp(TextWriter writer); 8 | int Execute(string[] commandLineArguments); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /source/Octodiff/CommandLine/Support/ICommandLocator.cs: -------------------------------------------------------------------------------- 1 | namespace Octodiff.CommandLine.Support 2 | { 3 | interface ICommandLocator 4 | { 5 | ICommandMetadata[] List(); 6 | ICommandMetadata Find(string name); 7 | ICommand Create(ICommandMetadata metadata); 8 | } 9 | } -------------------------------------------------------------------------------- /source/Octodiff/CommandLine/Support/ICommandMetadata.cs: -------------------------------------------------------------------------------- 1 | namespace Octodiff.CommandLine.Support 2 | { 3 | interface ICommandMetadata 4 | { 5 | string Name { get; } 6 | string[] Aliases { get; } 7 | string Description { get; } 8 | string Usage { get; set; } 9 | } 10 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/Adler32RollingChecksum.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Octodiff.Core 4 | { 5 | [Obsolete("This is non standard implimentation of Adler32, Adler32RollingChecksumV2 should be used instead.", false)] 6 | public class Adler32RollingChecksum : IRollingChecksum 7 | { 8 | public string Name => "Adler32"; 9 | 10 | public UInt32 Calculate(byte[] block, int offset, int count) 11 | { 12 | var a = 1; 13 | var b = 0; 14 | for (var i = offset; i < offset + count; i++) 15 | { 16 | var z = block[i]; 17 | a = (ushort)(z + a); 18 | b = (ushort)(b + a); 19 | } 20 | return (UInt32)((b << 16) | a); 21 | } 22 | 23 | public UInt32 Rotate(UInt32 checksum, byte remove, byte add, int chunkSize) 24 | { 25 | var b = (ushort)(checksum >> 16 & 0xffff); 26 | var a = (ushort)(checksum & 0xffff); 27 | 28 | a = (ushort)((a - remove + add)); 29 | b = (ushort)((b - (chunkSize * remove) + a - 1)); 30 | 31 | return (UInt32)((b << 16) | a); 32 | } 33 | } 34 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/Adler32RollingChecksumV2.cs: -------------------------------------------------------------------------------- 1 | namespace Octodiff.Core 2 | { 3 | public class Adler32RollingChecksumV2 : IRollingChecksum 4 | { 5 | public string Name => "Adler32V2"; 6 | 7 | private const ushort Modulus = 65521; 8 | 9 | public uint Calculate(byte[] block, int offset, int count) 10 | { 11 | var a = 1; 12 | var b = 0; 13 | for (var i = offset; i < offset + count; i++) 14 | { 15 | var z = block[i]; 16 | a = (z + a) % Modulus; 17 | b = (b + a) % Modulus; 18 | } 19 | return (uint)((b << 16) | a); 20 | } 21 | 22 | public uint Rotate(uint checksum, byte remove, byte add, int chunkSize) 23 | { 24 | var b = (ushort)(checksum >> 16 & 0xffff); 25 | var a = (ushort)(checksum & 0xffff); 26 | 27 | a = (ushort)((a - remove + add) % Modulus); 28 | b = (ushort)((b - (chunkSize * remove) + a - 1) % Modulus); 29 | 30 | return (uint)((b << 16) | a); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /source/Octodiff/Core/AggregateCopyOperationsDecorator.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | 3 | namespace Octodiff.Core 4 | { 5 | // This decorator turns any sequential copy operations into a single operation, reducing 6 | // the size of the delta file. 7 | // For example: 8 | // Copy: 0x0000 - 0x0400 9 | // Copy: 0x0401 - 0x0800 10 | // Copy: 0x0801 - 0x0C00 11 | // Gets turned into: 12 | // Copy: 0x0000 - 0x0C00 13 | public class AggregateCopyOperationsDecorator : IDeltaWriter 14 | { 15 | private readonly IDeltaWriter decorated; 16 | private DataRange bufferedCopy; 17 | 18 | public AggregateCopyOperationsDecorator(IDeltaWriter decorated) 19 | { 20 | this.decorated = decorated; 21 | } 22 | 23 | public void WriteDataCommand(Stream source, long offset, long length) 24 | { 25 | FlushCurrentCopyCommand(); 26 | decorated.WriteDataCommand(source, offset, length); 27 | } 28 | 29 | public void WriteMetadata(IHashAlgorithm hashAlgorithm, byte[] expectedNewFileHash) 30 | { 31 | decorated.WriteMetadata(hashAlgorithm, expectedNewFileHash); 32 | } 33 | 34 | public void WriteCopyCommand(DataRange chunk) 35 | { 36 | if (bufferedCopy.Length > 0 && bufferedCopy.StartOffset + bufferedCopy.Length == chunk.StartOffset) 37 | { 38 | bufferedCopy.Length += chunk.Length; 39 | } 40 | else 41 | { 42 | FlushCurrentCopyCommand(); 43 | bufferedCopy = chunk; 44 | } 45 | } 46 | 47 | void FlushCurrentCopyCommand() 48 | { 49 | if (bufferedCopy.Length <= 0) return; 50 | 51 | decorated.WriteCopyCommand(bufferedCopy); 52 | bufferedCopy = new DataRange(); 53 | } 54 | 55 | public void Finish() 56 | { 57 | FlushCurrentCopyCommand(); 58 | decorated.Finish(); 59 | } 60 | } 61 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/BinaryDeltaReader.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.IO; 4 | using Octodiff.Diagnostics; 5 | 6 | namespace Octodiff.Core 7 | { 8 | public class BinaryDeltaReader : IDeltaReader 9 | { 10 | private readonly BinaryReader reader; 11 | private readonly IProgressReporter progressReporter; 12 | private byte[] expectedHash; 13 | private IHashAlgorithm hashAlgorithm; 14 | private bool hasReadMetadata; 15 | private const int DefaultBufferSize = 4 * 1024 * 1024; 16 | 17 | public BinaryDeltaReader(Stream stream, IProgressReporter progressReporter) 18 | { 19 | reader = new BinaryReader(stream); 20 | this.progressReporter = progressReporter ?? NullProgressReporter.Instance; 21 | } 22 | 23 | public byte[] ExpectedHash 24 | { 25 | get 26 | { 27 | EnsureMetadata(); 28 | return expectedHash; 29 | } 30 | } 31 | 32 | public IHashAlgorithm HashAlgorithm 33 | { 34 | get 35 | { 36 | EnsureMetadata(); 37 | return hashAlgorithm; 38 | } 39 | } 40 | 41 | private void EnsureMetadata() 42 | { 43 | if (hasReadMetadata) 44 | return; 45 | 46 | reader.BaseStream.Seek(0, SeekOrigin.Begin); 47 | 48 | var first = reader.ReadBytes(BinaryFormat.DeltaHeader.Length); 49 | if (!StructuralComparisons.StructuralEqualityComparer.Equals(first, BinaryFormat.DeltaHeader)) 50 | throw new CorruptFileFormatException("The delta file appears to be corrupt."); 51 | 52 | var version = reader.ReadByte(); 53 | if (version != BinaryFormat.Version) 54 | throw new CorruptFileFormatException( 55 | "The delta file uses a newer file format than this program can handle."); 56 | 57 | var hashAlgorithmName = reader.ReadString(); 58 | hashAlgorithm = SupportedAlgorithms.Hashing.Create(hashAlgorithmName); 59 | 60 | var hashLength = reader.ReadInt32(); 61 | expectedHash = reader.ReadBytes(hashLength); 62 | var endOfMeta = reader.ReadBytes(BinaryFormat.EndOfMetadata.Length); 63 | if (!StructuralComparisons.StructuralEqualityComparer.Equals(BinaryFormat.EndOfMetadata, endOfMeta)) 64 | throw new CorruptFileFormatException("The signature file appears to be corrupt."); 65 | 66 | hasReadMetadata = true; 67 | } 68 | 69 | public void Apply( 70 | Action writeData, 71 | Action copy) 72 | { 73 | EnsureMetadata(); 74 | 75 | var fileLength = reader.BaseStream.Length; 76 | var buffer = new byte[DefaultBufferSize]; 77 | 78 | while (reader.BaseStream.Position != fileLength) 79 | { 80 | var b = reader.ReadByte(); 81 | 82 | progressReporter.ReportProgress("Applying delta", reader.BaseStream.Position, fileLength); 83 | if (b == BinaryFormat.CopyCommand) 84 | { 85 | var start = reader.ReadInt64(); 86 | var length = reader.ReadInt64(); 87 | copy(start, length); 88 | } 89 | else if (b == BinaryFormat.DataCommand) 90 | { 91 | var length = reader.ReadInt64(); 92 | long soFar = 0; 93 | while (soFar < length) 94 | { 95 | var chunkLength = (int)Math.Min(length - soFar, DefaultBufferSize); 96 | var numberOfBytesRead = reader.Read(buffer, 0, chunkLength); 97 | soFar += numberOfBytesRead; 98 | writeData(buffer, 0, numberOfBytesRead); 99 | } 100 | } 101 | } 102 | } 103 | } 104 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/BinaryDeltaWriter.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | 4 | namespace Octodiff.Core 5 | { 6 | public class BinaryDeltaWriter : IDeltaWriter 7 | { 8 | private readonly BinaryWriter writer; 9 | 10 | public BinaryDeltaWriter(Stream stream) 11 | { 12 | writer = new BinaryWriter(stream); 13 | } 14 | 15 | public void WriteMetadata(IHashAlgorithm hashAlgorithm, byte[] expectedNewFileHash) 16 | { 17 | writer.Write(BinaryFormat.DeltaHeader); 18 | writer.Write(BinaryFormat.Version); 19 | writer.Write(hashAlgorithm.Name); 20 | writer.Write(expectedNewFileHash.Length); 21 | writer.Write(expectedNewFileHash); 22 | writer.Write(BinaryFormat.EndOfMetadata); 23 | } 24 | 25 | public void WriteCopyCommand(DataRange segment) 26 | { 27 | writer.Write(BinaryFormat.CopyCommand); 28 | writer.Write(segment.StartOffset); 29 | writer.Write(segment.Length); 30 | } 31 | 32 | public void WriteDataCommand(Stream source, long offset, long length) 33 | { 34 | const long maxBufferSize = 1024 * 1024; 35 | 36 | writer.Write(BinaryFormat.DataCommand); 37 | writer.Write(length); 38 | 39 | var originalPosition = source.Position; 40 | try 41 | { 42 | source.Seek(offset, SeekOrigin.Begin); 43 | 44 | // ensure we clamp to maxBufferSize before int32 conversion to avoid overflow 45 | var bufferSize = length > maxBufferSize ? maxBufferSize : length; 46 | var buffer = new byte[(int)bufferSize]; 47 | 48 | int read; 49 | long soFar = 0; 50 | while ((read = source.Read(buffer, 0, (int)Math.Min(length - soFar, buffer.Length))) > 0) 51 | { 52 | soFar += read; 53 | 54 | writer.Write(buffer, 0, read); 55 | } 56 | } 57 | finally 58 | { 59 | source.Seek(originalPosition, SeekOrigin.Begin); 60 | } 61 | } 62 | 63 | public void Finish() 64 | { 65 | } 66 | } 67 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/BinaryFormat.cs: -------------------------------------------------------------------------------- 1 | using System.Text; 2 | 3 | namespace Octodiff.Core 4 | { 5 | class BinaryFormat 6 | { 7 | public static readonly byte[] SignatureHeader = Encoding.ASCII.GetBytes("OCTOSIG"); 8 | public static readonly byte[] DeltaHeader = Encoding.ASCII.GetBytes("OCTODELTA"); 9 | public static readonly byte[] EndOfMetadata = Encoding.ASCII.GetBytes(">>>"); 10 | public const byte CopyCommand = 0x60; 11 | public const byte DataCommand = 0x80; 12 | 13 | public const byte Version = 0x01; 14 | } 15 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/ChunkSignature.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Octodiff.Core 4 | { 5 | public class ChunkSignature 6 | { 7 | public long StartOffset; // 8 (but not included in the file on disk) 8 | public short Length; // 2 9 | public byte[] Hash; // 20 10 | public UInt32 RollingChecksum; // 4 11 | // 26 bytes on disk 12 | // 34 bytes in memory 13 | 14 | public override string ToString() 15 | { 16 | return string.Format("{0,6}:{1,6} |{2,20}| {3}", StartOffset, Length, RollingChecksum, BitConverter.ToString(Hash).ToLowerInvariant().Replace("-", "")); 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/ChunkSignatureChecksumComparer.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace Octodiff.Core 4 | { 5 | class ChunkSignatureChecksumComparer : IComparer 6 | { 7 | public int Compare(ChunkSignature x, ChunkSignature y) 8 | { 9 | var comparison = x.RollingChecksum.CompareTo(y.RollingChecksum); 10 | return comparison == 0 ? x.StartOffset.CompareTo(y.StartOffset) : comparison; 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/CompatibilityException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Octodiff.Core 4 | { 5 | public class CompatibilityException : Exception 6 | { 7 | public CompatibilityException(string message) : base(message) 8 | { 9 | 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/CorruptFileFormatException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Octodiff.Core 4 | { 5 | public class CorruptFileFormatException : Exception 6 | { 7 | public CorruptFileFormatException(string message) : base(message) 8 | { 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/DataRange.cs: -------------------------------------------------------------------------------- 1 | namespace Octodiff.Core 2 | { 3 | public struct DataRange 4 | { 5 | public DataRange(long startOffset, long length) 6 | { 7 | StartOffset = startOffset; 8 | Length = length; 9 | } 10 | 11 | public long StartOffset; 12 | public long Length; 13 | } 14 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/DeltaApplier.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.IO; 4 | 5 | namespace Octodiff.Core 6 | { 7 | public class DeltaApplier 8 | { 9 | private const int BufferLength = 4 * 1024 * 1024; 10 | 11 | public DeltaApplier() 12 | { 13 | SkipHashCheck = false; 14 | } 15 | 16 | public bool SkipHashCheck { get; set; } 17 | 18 | public void Apply(Stream basisFileStream, IDeltaReader delta, Stream outputStream) 19 | { 20 | var buffer = new byte[BufferLength]; 21 | 22 | delta.Apply( 23 | writeData: outputStream.Write, 24 | copy: (offset, count) => 25 | { 26 | basisFileStream.Seek(offset, SeekOrigin.Begin); 27 | 28 | int read; 29 | long soFar = 0; 30 | while ((read = basisFileStream.Read(buffer, 0, (int)Math.Min(count - soFar, BufferLength))) > 0) 31 | { 32 | soFar += read; 33 | outputStream.Write(buffer, 0, read); 34 | } 35 | }); 36 | 37 | if (SkipHashCheck) 38 | return; 39 | 40 | outputStream.Seek(0, SeekOrigin.Begin); 41 | 42 | var sourceFileHash = delta.ExpectedHash; 43 | var algorithm = delta.HashAlgorithm; 44 | 45 | var actualHash = algorithm.ComputeHash(outputStream); 46 | 47 | if (!StructuralComparisons.StructuralEqualityComparer.Equals(sourceFileHash, actualHash)) 48 | throw new UsageException( 49 | "Verification of the patched file failed. The SHA1 hash of the patch result file, and the file that was used as input for the delta, do not match. This can happen if the basis file changed since the signatures were calculated."); 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/DeltaBuilder.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.Collections.Generic; 4 | using System.IO; 5 | using System.Linq; 6 | using Octodiff.Diagnostics; 7 | 8 | namespace Octodiff.Core 9 | { 10 | public class DeltaBuilder 11 | { 12 | private const int ReadBufferSize = 4*1024*1024; 13 | 14 | public DeltaBuilder() 15 | { 16 | ProgressReporter = NullProgressReporter.Instance; 17 | } 18 | 19 | public IProgressReporter ProgressReporter { get; set; } 20 | 21 | public void BuildDelta(Stream newFileStream, ISignatureReader signatureReader, IDeltaWriter deltaWriter) 22 | { 23 | var signature = signatureReader.ReadSignature(); 24 | var chunks = signature.Chunks; 25 | 26 | var hash = signature.HashAlgorithm.ComputeHash(newFileStream); 27 | newFileStream.Seek(0, SeekOrigin.Begin); 28 | 29 | deltaWriter.WriteMetadata(signature.HashAlgorithm, hash); 30 | 31 | chunks = OrderChunksByChecksum(chunks); 32 | 33 | int minChunkSize; 34 | int maxChunkSize; 35 | var chunkMap = CreateChunkMap(chunks, out maxChunkSize, out minChunkSize); 36 | 37 | var buffer = new byte[ReadBufferSize]; 38 | long lastMatchPosition = 0; 39 | 40 | var fileSize = newFileStream.Length; 41 | ProgressReporter.ReportProgress("Building delta", 0, fileSize); 42 | 43 | while (true) 44 | { 45 | var startPosition = newFileStream.Position; 46 | var read = newFileStream.Read(buffer, 0, buffer.Length); 47 | if (read < 0) 48 | break; 49 | 50 | var checksumAlgorithm = signature.RollingChecksumAlgorithm; 51 | uint checksum = 0; 52 | 53 | var remainingPossibleChunkSize = maxChunkSize; 54 | 55 | for (var i = 0; i < read - minChunkSize + 1; i++) 56 | { 57 | var readSoFar = startPosition + i; 58 | 59 | var remainingBytes = read - i; 60 | if (remainingBytes < maxChunkSize) 61 | { 62 | remainingPossibleChunkSize = minChunkSize; 63 | } 64 | 65 | if (i == 0 || remainingBytes < maxChunkSize) 66 | { 67 | checksum = checksumAlgorithm.Calculate(buffer, i, remainingPossibleChunkSize); 68 | } 69 | else 70 | { 71 | var remove = buffer[i- 1]; 72 | var add = buffer[i + remainingPossibleChunkSize - 1]; 73 | checksum = checksumAlgorithm.Rotate(checksum, remove, add, remainingPossibleChunkSize); 74 | } 75 | 76 | ProgressReporter.ReportProgress("Building delta", readSoFar, fileSize); 77 | 78 | if (readSoFar - (lastMatchPosition - remainingPossibleChunkSize) < remainingPossibleChunkSize) 79 | continue; 80 | 81 | if (!chunkMap.ContainsKey(checksum)) 82 | continue; 83 | 84 | var startIndex = chunkMap[checksum]; 85 | 86 | for (var j = startIndex; j < chunks.Count && chunks[j].RollingChecksum == checksum; j++) 87 | { 88 | var chunk = chunks[j]; 89 | 90 | var sha = signature.HashAlgorithm.ComputeHash(buffer, i, remainingPossibleChunkSize); 91 | 92 | if (StructuralComparisons.StructuralEqualityComparer.Equals(sha, chunks[j].Hash)) 93 | { 94 | readSoFar = readSoFar + remainingPossibleChunkSize; 95 | 96 | var missing = readSoFar - lastMatchPosition; 97 | if (missing > remainingPossibleChunkSize) 98 | { 99 | deltaWriter.WriteDataCommand(newFileStream, lastMatchPosition, missing - remainingPossibleChunkSize); 100 | } 101 | 102 | deltaWriter.WriteCopyCommand(new DataRange(chunk.StartOffset, chunk.Length)); 103 | lastMatchPosition = readSoFar; 104 | break; 105 | } 106 | } 107 | } 108 | 109 | if (read < buffer.Length) 110 | { 111 | break; 112 | } 113 | 114 | newFileStream.Position = newFileStream.Position - maxChunkSize + 1; 115 | } 116 | 117 | if (newFileStream.Length != lastMatchPosition) 118 | { 119 | deltaWriter.WriteDataCommand(newFileStream, lastMatchPosition, newFileStream.Length - lastMatchPosition); 120 | } 121 | 122 | deltaWriter.Finish(); 123 | } 124 | 125 | private static List OrderChunksByChecksum(List chunks) 126 | { 127 | chunks.Sort(new ChunkSignatureChecksumComparer()); 128 | return chunks; 129 | } 130 | 131 | private Dictionary CreateChunkMap(IList chunks, out int maxChunkSize, out int minChunkSize) 132 | { 133 | ProgressReporter.ReportProgress("Creating chunk map", 0, chunks.Count); 134 | maxChunkSize = 0; 135 | minChunkSize = int.MaxValue; 136 | 137 | var chunkMap = new Dictionary(); 138 | for (var i = 0; i < chunks.Count; i++) 139 | { 140 | var chunk = chunks[i]; 141 | if (chunk.Length > maxChunkSize) maxChunkSize = chunk.Length; 142 | if (chunk.Length < minChunkSize) minChunkSize = chunk.Length; 143 | 144 | if (!chunkMap.ContainsKey(chunk.RollingChecksum)) 145 | { 146 | chunkMap[chunk.RollingChecksum] = i; 147 | } 148 | 149 | ProgressReporter.ReportProgress("Creating chunk map", i, chunks.Count); 150 | } 151 | return chunkMap; 152 | } 153 | } 154 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/DeltaStatistics.cs: -------------------------------------------------------------------------------- 1 | namespace Octodiff.Core 2 | { 3 | class DeltaStatistics 4 | { 5 | public DeltaStatistics() 6 | { 7 | 8 | } 9 | 10 | public int ChunksCopied { get; set; } 11 | public long BytesCopied { get; set; } 12 | } 13 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/HashAlgorithmWrapper.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | using System.Security.Cryptography; 3 | 4 | namespace Octodiff.Core 5 | { 6 | public class HashAlgorithmWrapper : IHashAlgorithm 7 | { 8 | private readonly HashAlgorithm algorithm; 9 | 10 | public HashAlgorithmWrapper(string name, HashAlgorithm algorithm) 11 | { 12 | Name = name; 13 | this.algorithm = algorithm; 14 | } 15 | 16 | public string Name { get; private set; } 17 | public int HashLength { get { return algorithm.HashSize / 8; } } 18 | 19 | public byte[] ComputeHash(Stream stream) 20 | { 21 | return algorithm.ComputeHash(stream); 22 | } 23 | 24 | public byte[] ComputeHash(byte[] buffer, int offset, int length) 25 | { 26 | return algorithm.ComputeHash(buffer, offset, length); 27 | } 28 | } 29 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/IDeltaReader.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Octodiff.Core 4 | { 5 | public interface IDeltaReader 6 | { 7 | byte[] ExpectedHash { get; } 8 | IHashAlgorithm HashAlgorithm { get; } 9 | 10 | /// 11 | /// Reads the delta file. 12 | /// This method will invoke the Action to copy the data from the original file multiple times if needed. 13 | /// This method will also invoke the Action to write data from the delta file to the destination file multiple times if needed. 14 | /// 15 | /// Action to write data from the delta file to the destination file. Parameters: buffer, offset and count. 16 | /// Action to copy data from the original file to the destination file. Parameters: offset and count. 17 | void Apply( 18 | Action writeData, 19 | Action copy 20 | ); 21 | } 22 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/IDeltaWriter.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | 3 | namespace Octodiff.Core 4 | { 5 | public interface IDeltaWriter 6 | { 7 | void WriteMetadata(IHashAlgorithm hashAlgorithm, byte[] expectedNewFileHash); 8 | void WriteCopyCommand(DataRange segment); 9 | void WriteDataCommand(Stream source, long offset, long length); 10 | void Finish(); 11 | } 12 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/IHashAlgorithm.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | 3 | namespace Octodiff.Core 4 | { 5 | public interface IHashAlgorithm 6 | { 7 | string Name { get; } 8 | int HashLength { get; } 9 | byte[] ComputeHash(Stream stream); 10 | byte[] ComputeHash(byte[] buffer, int offset, int length); 11 | } 12 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/IRollingChecksum.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Octodiff.Core 4 | { 5 | public interface IRollingChecksum 6 | { 7 | string Name { get; } 8 | UInt32 Calculate(byte[] block, int offset, int count); 9 | UInt32 Rotate(UInt32 checksum, byte remove, byte add, int chunkSize); 10 | } 11 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/ISignatureReader.cs: -------------------------------------------------------------------------------- 1 | namespace Octodiff.Core 2 | { 3 | public interface ISignatureReader 4 | { 5 | Signature ReadSignature(); 6 | } 7 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/ISignatureWriter.cs: -------------------------------------------------------------------------------- 1 | namespace Octodiff.Core 2 | { 3 | public interface ISignatureWriter 4 | { 5 | void WriteMetadata(IHashAlgorithm hashAlgorithm, IRollingChecksum rollingChecksumAlgorithm); 6 | void WriteChunk(ChunkSignature signature); 7 | } 8 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/Signature.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Security.Cryptography; 3 | 4 | namespace Octodiff.Core 5 | { 6 | public class Signature 7 | { 8 | public Signature(IHashAlgorithm hashAlgorithm, IRollingChecksum rollingChecksumAlgorithm) 9 | { 10 | HashAlgorithm = hashAlgorithm; 11 | RollingChecksumAlgorithm = rollingChecksumAlgorithm; 12 | Chunks = new List(); 13 | } 14 | 15 | public IHashAlgorithm HashAlgorithm { get; private set; } 16 | public IRollingChecksum RollingChecksumAlgorithm { get; private set; } 17 | public List Chunks { get; private set; } 18 | } 19 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/SignatureBuilder.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | using Octodiff.Diagnostics; 3 | 4 | namespace Octodiff.Core 5 | { 6 | public class SignatureBuilder 7 | { 8 | public const short MinimumChunkSize = 128; 9 | public const short DefaultChunkSize = 2048; 10 | public const short MaximumChunkSize = 31 * 1024; 11 | 12 | private short chunkSize; 13 | 14 | public SignatureBuilder() 15 | { 16 | ChunkSize = DefaultChunkSize; 17 | HashAlgorithm = SupportedAlgorithms.Hashing.Default(); 18 | RollingChecksumAlgorithm = SupportedAlgorithms.Checksum.Default(); 19 | ProgressReporter = NullProgressReporter.Instance; 20 | } 21 | 22 | public IProgressReporter ProgressReporter { get; set; } 23 | 24 | public IHashAlgorithm HashAlgorithm { get; set; } 25 | 26 | public IRollingChecksum RollingChecksumAlgorithm { get; set; } 27 | 28 | public short ChunkSize 29 | { 30 | get { return chunkSize; } 31 | set 32 | { 33 | if (value < MinimumChunkSize) 34 | throw new UsageException(string.Format("Chunk size cannot be less than {0}", MinimumChunkSize)); 35 | if (value > MaximumChunkSize) 36 | throw new UsageException(string.Format("Chunk size cannot be exceed {0}", MaximumChunkSize)); 37 | chunkSize = value; 38 | } 39 | } 40 | 41 | public void Build(Stream stream, ISignatureWriter signatureWriter) 42 | { 43 | WriteMetadata(stream, signatureWriter); 44 | WriteChunkSignatures(stream, signatureWriter); 45 | } 46 | 47 | void WriteMetadata(Stream stream, ISignatureWriter signatureWriter) 48 | { 49 | ProgressReporter.ReportProgress("Hashing file", 0, stream.Length); 50 | stream.Seek(0, SeekOrigin.Begin); 51 | 52 | signatureWriter.WriteMetadata(HashAlgorithm, RollingChecksumAlgorithm); 53 | 54 | ProgressReporter.ReportProgress("Hashing file", stream.Length, stream.Length); 55 | } 56 | 57 | void WriteChunkSignatures(Stream stream, ISignatureWriter signatureWriter) 58 | { 59 | var checksumAlgorithm = RollingChecksumAlgorithm; 60 | var hashAlgorithm = HashAlgorithm; 61 | 62 | ProgressReporter.ReportProgress("Building signatures", 0, stream.Length); 63 | stream.Seek(0, SeekOrigin.Begin); 64 | 65 | long start = 0; 66 | int read; 67 | var block = new byte[ChunkSize]; 68 | while ((read = stream.Read(block, 0, block.Length)) > 0) 69 | { 70 | signatureWriter.WriteChunk(new ChunkSignature 71 | { 72 | StartOffset = start, 73 | Length = (short)read, 74 | Hash = hashAlgorithm.ComputeHash(block, 0, read), 75 | RollingChecksum = checksumAlgorithm.Calculate(block, 0, read) 76 | }); 77 | 78 | start += read; 79 | ProgressReporter.ReportProgress("Building signatures", start, stream.Length); 80 | } 81 | } 82 | } 83 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/SignatureReader.cs: -------------------------------------------------------------------------------- 1 | using System.Collections; 2 | using System.Diagnostics; 3 | using System.IO; 4 | using System.Reflection; 5 | using System.Security.Cryptography; 6 | using Octodiff.Diagnostics; 7 | 8 | namespace Octodiff.Core 9 | { 10 | public class SignatureReader : ISignatureReader 11 | { 12 | private readonly IProgressReporter reporter; 13 | private readonly BinaryReader reader; 14 | 15 | public SignatureReader(Stream stream, IProgressReporter reporter) 16 | { 17 | this.reporter = reporter; 18 | this.reader = new BinaryReader(stream); 19 | } 20 | 21 | public Signature ReadSignature() 22 | { 23 | Progress(); 24 | var header = reader.ReadBytes(BinaryFormat.SignatureHeader.Length); 25 | if (!StructuralComparisons.StructuralEqualityComparer.Equals(BinaryFormat.SignatureHeader, header)) 26 | throw new CorruptFileFormatException("The signature file appears to be corrupt."); 27 | 28 | var version = reader.ReadByte(); 29 | if (version != BinaryFormat.Version) 30 | throw new CorruptFileFormatException("The signature file uses a newer file format than this program can handle."); 31 | 32 | var hashAlgorithm = reader.ReadString(); 33 | var rollingChecksumAlgorithm = reader.ReadString(); 34 | 35 | var endOfMeta = reader.ReadBytes(BinaryFormat.EndOfMetadata.Length); 36 | if (!StructuralComparisons.StructuralEqualityComparer.Equals(BinaryFormat.EndOfMetadata, endOfMeta)) 37 | throw new CorruptFileFormatException("The signature file appears to be corrupt."); 38 | 39 | Progress(); 40 | 41 | var hashAlgo = SupportedAlgorithms.Hashing.Create(hashAlgorithm); 42 | var signature = new Signature( 43 | hashAlgo, 44 | SupportedAlgorithms.Checksum.Create(rollingChecksumAlgorithm)); 45 | 46 | var expectedHashLength = hashAlgo.HashLength; 47 | long start = 0; 48 | 49 | var fileLength = reader.BaseStream.Length; 50 | var remainingBytes = fileLength - reader.BaseStream.Position; 51 | var signatureSize = sizeof (ushort) + sizeof (uint) + expectedHashLength; 52 | if (remainingBytes % signatureSize != 0) 53 | throw new CorruptFileFormatException("The signature file appears to be corrupt; at least one chunk has data missing."); 54 | 55 | while (reader.BaseStream.Position < fileLength - 1) 56 | { 57 | var length = reader.ReadInt16(); 58 | var checksum = reader.ReadUInt32(); 59 | var chunkHash = reader.ReadBytes(expectedHashLength); 60 | 61 | signature.Chunks.Add(new ChunkSignature 62 | { 63 | StartOffset = start, 64 | Length = length, 65 | RollingChecksum = checksum, 66 | Hash = chunkHash 67 | }); 68 | 69 | start += length; 70 | 71 | Progress(); 72 | } 73 | 74 | return signature; 75 | } 76 | 77 | void Progress() 78 | { 79 | reporter.ReportProgress("Reading signature", reader.BaseStream.Position, reader.BaseStream.Length); 80 | } 81 | } 82 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/SignatureWriter.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | 3 | namespace Octodiff.Core 4 | { 5 | public class SignatureWriter : ISignatureWriter 6 | { 7 | private readonly BinaryWriter signatureStream; 8 | 9 | public SignatureWriter(Stream signatureStream) 10 | { 11 | this.signatureStream = new BinaryWriter(signatureStream); 12 | } 13 | 14 | public void WriteMetadata(IHashAlgorithm hashAlgorithm, IRollingChecksum rollingChecksumAlgorithm) 15 | { 16 | signatureStream.Write(BinaryFormat.SignatureHeader); 17 | signatureStream.Write(BinaryFormat.Version); 18 | signatureStream.Write(hashAlgorithm.Name); 19 | signatureStream.Write(rollingChecksumAlgorithm.Name); 20 | signatureStream.Write(BinaryFormat.EndOfMetadata); 21 | } 22 | 23 | public void WriteChunk(ChunkSignature signature) 24 | { 25 | signatureStream.Write(signature.Length); 26 | signatureStream.Write(signature.RollingChecksum); 27 | signatureStream.Write(signature.Hash); 28 | } 29 | } 30 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/SupportedAlgorithms.cs: -------------------------------------------------------------------------------- 1 | using System.Security.Cryptography; 2 | 3 | namespace Octodiff.Core 4 | { 5 | public interface IHashingSupportedAlgorithm 6 | { 7 | IHashAlgorithm Default(); 8 | IHashAlgorithm Create(string algorithm); 9 | } 10 | 11 | public class DefaultHashingSupportedAlgorithm 12 | : IHashingSupportedAlgorithm 13 | { 14 | public IHashAlgorithm Sha1() 15 | { 16 | return new HashAlgorithmWrapper("SHA1", SHA1.Create()); 17 | } 18 | 19 | public virtual IHashAlgorithm Default() 20 | { 21 | return Sha1(); 22 | } 23 | 24 | public virtual IHashAlgorithm Create(string algorithm) 25 | { 26 | if (algorithm == "SHA1") 27 | return Sha1(); 28 | 29 | throw new CompatibilityException( 30 | $"The hash algorithm '{algorithm}' is not supported in this version of Octodiff"); 31 | } 32 | } 33 | 34 | public interface IChecksumSupportedAlgorithm 35 | { 36 | IRollingChecksum Default(); 37 | IRollingChecksum Create(string algorithm); 38 | } 39 | 40 | public class DefaultChecksumSupportedAlgorithm 41 | : IChecksumSupportedAlgorithm 42 | { 43 | #pragma warning disable 618 44 | public IRollingChecksum Adler32Rolling(bool useV2 = false) 45 | { 46 | if (useV2) 47 | return new Adler32RollingChecksumV2(); 48 | 49 | return new Adler32RollingChecksum(); 50 | } 51 | #pragma warning restore 618 52 | 53 | public virtual IRollingChecksum Default() 54 | { 55 | return Adler32Rolling(); 56 | } 57 | 58 | public virtual IRollingChecksum Create(string algorithm) 59 | { 60 | switch (algorithm) 61 | { 62 | case "Adler32": 63 | return Adler32Rolling(); 64 | case "Adler32V2": 65 | return Adler32Rolling(true); 66 | } 67 | throw new CompatibilityException( 68 | $"The rolling checksum algorithm '{algorithm}' is not supported in this version of Octodiff"); 69 | } 70 | } 71 | 72 | public static class SupportedAlgorithms 73 | { 74 | public static IHashingSupportedAlgorithm Hashing { get; set; } = new DefaultHashingSupportedAlgorithm(); 75 | 76 | public static IChecksumSupportedAlgorithm Checksum { get; set; } = new DefaultChecksumSupportedAlgorithm(); 77 | } 78 | } -------------------------------------------------------------------------------- /source/Octodiff/Core/UsageException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Octodiff.Core 4 | { 5 | public class UsageException : Exception 6 | { 7 | public UsageException(string message) : base(message) 8 | { 9 | 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /source/Octodiff/Diagnostics/ConsoleProgressReporter.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Octodiff.Diagnostics 4 | { 5 | public class ConsoleProgressReporter : IProgressReporter 6 | { 7 | private string currentOperation; 8 | private int progressPercentage; 9 | 10 | public void ReportProgress(string operation, long currentPosition, long total) 11 | { 12 | var percent = (int)((double)currentPosition/total * 100d + 0.5); 13 | if (currentOperation != operation) 14 | { 15 | progressPercentage = -1; 16 | currentOperation = operation; 17 | } 18 | 19 | if (progressPercentage != percent && percent % 10 == 0) 20 | { 21 | progressPercentage = percent; 22 | Console.WriteLine("{0}: {1}%", currentOperation, percent); 23 | } 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /source/Octodiff/Diagnostics/IProgressReporter.cs: -------------------------------------------------------------------------------- 1 | namespace Octodiff.Diagnostics 2 | { 3 | public interface IProgressReporter 4 | { 5 | void ReportProgress(string operation, long currentPosition, long total); 6 | } 7 | } -------------------------------------------------------------------------------- /source/Octodiff/Diagnostics/NullProgressReporter.cs: -------------------------------------------------------------------------------- 1 | namespace Octodiff.Diagnostics 2 | { 3 | public class NullProgressReporter : IProgressReporter 4 | { 5 | public static NullProgressReporter Instance { get; } = new NullProgressReporter(); 6 | 7 | public void ReportProgress(string operation, long currentPosition, long total) 8 | { 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /source/Octodiff/Octodiff.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0;net462 5 | true 6 | Octodiff 7 | Octopus.Octodiff 8 | false 9 | https://github.com/OctopusDeploy/Octodiff 10 | Apache-2.0 11 | Octopus Deploy 12 | Octopus Deploy Pty Ltd 13 | tool 14 | git 15 | https://github.com/OctopusDeploy/Octodiff/ 16 | true 17 | 100% C# implementation of remote delta compression based on the rsync algorithm 18 | Octodiff 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | Exe 27 | anycpu 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /source/Octodiff/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using Octodiff.CommandLine; 6 | using Octodiff.CommandLine.Support; 7 | using Octodiff.Core; 8 | 9 | namespace Octodiff 10 | { 11 | public class Program 12 | { 13 | public static int Main(string[] args) 14 | { 15 | string[] commandArguments; 16 | var commandName = ExtractCommand(args, out commandArguments); 17 | var locator = new CommandLocator(); 18 | var command = locator.Find(commandName); 19 | 20 | if (command == null) 21 | { 22 | locator.Create(locator.Find("help")).Execute(commandArguments); 23 | return 4; 24 | } 25 | 26 | try 27 | { 28 | var exitCode = locator.Create(command).Execute(commandArguments); 29 | return exitCode; 30 | } 31 | catch (OptionException ex) 32 | { 33 | WriteError(ex); 34 | locator.Create(locator.Find("help")).Execute(new[] {commandName}); 35 | return 4; 36 | } 37 | catch (UsageException ex) 38 | { 39 | WriteError(ex); 40 | return 4; 41 | } 42 | catch (FileNotFoundException ex) 43 | { 44 | WriteError(ex); 45 | return 4; 46 | } 47 | catch (CorruptFileFormatException ex) 48 | { 49 | WriteError(ex); 50 | return 2; 51 | } 52 | catch (IOException ex) 53 | { 54 | WriteError(ex, details: true); 55 | return 1; 56 | } 57 | catch (Exception ex) 58 | { 59 | WriteError(ex, details: true); 60 | return 3; 61 | } 62 | } 63 | 64 | static void WriteError(Exception ex, bool details = false) 65 | { 66 | Console.ForegroundColor = ConsoleColor.Yellow; 67 | Console.WriteLine("Error: " + ex.Message); 68 | Console.ResetColor(); 69 | if (details) 70 | { 71 | Console.WriteLine(ex.ToString()); 72 | } 73 | } 74 | 75 | private static string ExtractCommand(ICollection args, out string[] remaining) 76 | { 77 | remaining = args.Count <= 1 ? new string[0] : args.Skip(1).ToArray(); 78 | return (args.FirstOrDefault() ?? string.Empty).ToLowerInvariant(); 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /source/Octodiff/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | 3 | [assembly: AssemblyTitle("Octodiff")] -------------------------------------------------------------------------------- /source/Tests.ps1: -------------------------------------------------------------------------------- 1 | 2 | function Assert-That ($val, $error) { 3 | if ($val -ne $true) { 4 | Write-Error "Test failed: $error" 5 | } 6 | } 7 | 8 | function Generate-RandomFile ($maxSizeInKb) { 9 | $size = Get-Random -maximum $maxSizeInKb 10 | $name = [System.Guid]::NewGuid().ToString() + ".txt" 11 | $name = (Join-Path (Resolve-Path .) $name) 12 | $data = New-Object System.IO.StreamWriter $name 13 | $rand = new-object System.Random 14 | for ($i = 0; $i -le $size; $i++) { 15 | for ($j = 0; $j -le 1020; $j++){ 16 | $c = [char]$rand.Next(65, 90) 17 | $data.Write($c) 18 | } 19 | $data.WriteLine() 20 | } 21 | $data.Close() 22 | } 23 | 24 | 25 | $here = Split-Path -Parent $MyInvocation.MyCommand.Path 26 | 27 | function Run-OctodiffSimulation($iterations) { 28 | cd $here\..\Octodiff\bin 29 | 30 | mkdir .\Temp -ErrorAction SilentlyContinue 31 | Remove-Item -Recurse -Force .\temp 32 | 33 | mkdir .\Temp -ErrorAction SilentlyContinue | Out-Null 34 | pushd .\Temp 35 | 36 | for ($i = 10; $i -le $iterations; $i++) { 37 | $name = "Package" 38 | mkdir $name -ErrorAction SilentlyContinue | Out-Null 39 | pushd $name | Out-Null 40 | for ($j = 0; $j -lt $i * 10; $j++) { 41 | Generate-RandomFile (1024) 42 | } 43 | popd 44 | 45 | $original = join-path (resolve-path .) ($name + "_orig.nupkg") 46 | dir $name | ToZip $original 47 | 48 | pushd $name | Out-Null 49 | dir | get-random -count (2) | remove-item 50 | for ($j = 0; $j -lt 3; $j++) { 51 | Generate-RandomFile (1024) 52 | } 53 | popd 54 | 55 | $name = $name + "_" + $i 56 | 57 | $newfile = join-path (resolve-path .) ($name + "_new.nupkg") 58 | dir "package" | ToZip $newfile 59 | 60 | $watch = [System.Diagnostics.Stopwatch]::StartNew() 61 | $sigfile = $original + ".octosig" 62 | & ..\Octodiff.exe signature $original $sigfile 63 | Assert-That ($LASTEXITCODE -eq 0) "Error creating signature: exit code $LASTEXITCODE" 64 | 65 | $deltafile = $original + ".octodelta" 66 | & ..\Octodiff.exe delta $sigfile $newfile $deltafile 67 | Assert-That ($LASTEXITCODE -eq 0) "Error creating delta: exit code $LASTEXITCODE" 68 | 69 | $outfile = $newfile + "_2" 70 | & ..\Octodiff.exe patch $original $deltafile $outfile 71 | Assert-That ($LASTEXITCODE -eq 0) "Error applying delta: exit code $LASTEXITCODE" 72 | 73 | 74 | Write-Host "Scenario: $i" 75 | Write-Host " Original size: $((get-item $original).Length / 1024)K" 76 | Write-Host " New size: $((get-item $newfile).Length / 1024)K" 77 | Write-Host " Signature size: $((get-item $sigfile).Length / 1024)K" 78 | Write-Host " Delta size: $((get-item $deltafile).Length / 1024)K" 79 | Write-Host " Time taken: $($watch.ElapsedMilliseconds)ms" 80 | 81 | $oldHash = (get-filehash $newfile).Hash 82 | $newHash = (Get-FileHash ($newfile + "_2")).Hash 83 | write-host " Hashes equal: $($oldHash -eq $newHash)" 84 | } 85 | popd 86 | } 87 | 88 | 89 | Set-StrictMode -V 1.0 90 | [Reflection.Assembly]::LoadWithPartialName("WindowsBase") | Out-Null 91 | function ToZip($fileName, $relativeBaseDirectory=$null, [switch] $appendToZip=$false, $verbose=$true) 92 | { 93 | begin 94 | { 95 | $zipCreated = { (Get-Variable -ErrorAction SilentlyContinue -Name zipFile) -ne $null } 96 | 97 | $mode = [System.IO.FileMode]::Create 98 | if ($appendToZip) 99 | { 100 | $mode = [System.IO.FileMode]::Open 101 | } 102 | $zipFile = [System.IO.Packaging.Package]::Open($fileName, $mode) 103 | } 104 | process 105 | { 106 | if ((&$zipCreated) -and ([System.IO.File]::Exists($_.FullName) -eq $true)) 107 | { 108 | 109 | $zipFileName = $_.FullName 110 | if ($relativeBaseDirectory -ne $null) 111 | { 112 | #$directoryName = [System.IO.Path]::GetDirectoryName($_.FullName) 113 | $zipFileName = $_.FullName.SubString($relativeBaseDirectory.Length, $_.FullName.Length-$relativeBaseDirectory.Length) 114 | } 115 | 116 | 117 | $destFilename = [System.IO.Path]::Combine(".\\", $zipFileName) 118 | #$destFilename = $destFilename.Replace(" ", "_") 119 | $uri = New-Object Uri -ArgumentList ($destFilename, [UriKind]::Relative) 120 | $uri = [System.IO.Packaging.PackUriHelper]::CreatePartUri($uri) 121 | 122 | if ($zipFile.PartExists($uri)) 123 | { 124 | $zipFile.DeletePart($uri); 125 | } 126 | 127 | $part = $zipFile.CreatePart($uri, [string]::Empty, [System.IO.Packaging.CompressionOption]::Normal) 128 | $dest = $part.GetStream() 129 | 130 | $srcStream = New-Object System.IO.FileStream -ArgumentList ($_.FullName, [System.IO.FileMode]::Open, [System.IO.FileAccess]::Read) 131 | try 132 | { 133 | $srcStream.CopyTo($dest) 134 | } 135 | finally 136 | { 137 | $srcStream.Close() 138 | } 139 | } 140 | } 141 | end 142 | { 143 | if (&$zipCreated) 144 | { 145 | $zipFile.Close() 146 | } 147 | } 148 | } 149 | 150 | clear 151 | Run-OctodiffSimulation -iterations 10 --------------------------------------------------------------------------------