├── demo
    ├── mtime_cache_globs.txt
    ├── src
    │   ├── src2.cpp
    │   ├── src1.cpp
    │   ├── header.hpp
    │   └── src3.cpp
    └── CMakeLists.txt
├── .gitignore
├── CHANGELOG.txt
├── .travis.yml
├── LICENSE.txt
├── mtime_cache.gemspec
├── README.md
└── bin
    └── mtime_cache


/demo/mtime_cache_globs.txt:
--------------------------------------------------------------------------------
1 | src/**/*.{%{cpp}}
2 | 
3 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | *.gem
2 | demo/build
3 | demo/.mtime_cache
4 | 


--------------------------------------------------------------------------------
/demo/src/src2.cpp:
--------------------------------------------------------------------------------
1 | #include "header.hpp"
2 | 
3 | void add_at(map& m, const std::string& key, int n)
4 | {
5 |     m[key].push_back(n);
6 | }
7 | 


--------------------------------------------------------------------------------
/demo/src/src1.cpp:
--------------------------------------------------------------------------------
 1 | #include "header.hpp"
 2 | 
 3 | using namespace std;
 4 | 
 5 | int main()
 6 | {
 7 |     map m;
 8 |     add_at(m, "asd", 5);
 9 |     add_at(m, "asd", 15);
10 |     return check_map(m);
11 | }
12 | 


--------------------------------------------------------------------------------
/CHANGELOG.txt:
--------------------------------------------------------------------------------
1 | mtime_cache Changelog
2 | 
3 | 1.0.2 (2016/10/27) - Fixed date of release... ':)
4 | 
5 | 1.0.1 (2016/10/27) - Fixed bug when using globs with no globfile.
6 |                      Fixed gem description.
7 | 
8 | 1.0.0 (2016/10/18) - Initial release
9 | 


--------------------------------------------------------------------------------
/demo/src/header.hpp:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | #include <unordered_map>
 3 | #include <string>
 4 | #include <vector>
 5 | 
 6 | typedef std::unordered_map<std::string, std::vector<int>> map;
 7 | 
 8 | void add_at(map& m, const std::string& key, int n);
 9 | int check_map(const map& m);
10 | 


--------------------------------------------------------------------------------
/demo/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | cmake_minimum_required(VERSION 2.8.7)
 2 | 
 3 | project(mtime_cache_demo)
 4 | 
 5 | if(NOT MSVC)
 6 |     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --std=c++0x")
 7 | endif()
 8 | 
 9 | add_executable(mtime_cache_demo
10 |     src/src1.cpp
11 |     src/src2.cpp
12 |     src/src3.cpp
13 | 	src/header.hpp
14 | )
15 | 


--------------------------------------------------------------------------------
/demo/src/src3.cpp:
--------------------------------------------------------------------------------
 1 | #include "header.hpp"
 2 | 
 3 | #define CHECK(foo) if(!(foo)) return 1;
 4 | 
 5 | int check_map(const map& m)
 6 | {
 7 |     std::string k = "asd";
 8 |     CHECK(m.size() == 1);
 9 |     CHECK(m.find(k) != m.end());
10 |     CHECK(m.at(k).size() == 2);
11 |     CHECK(m.at(k)[0] == 5);
12 |     CHECK(m.at(k)[1] == 15);
13 |     return 0;
14 | }
15 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | language: cpp
 2 | os: linux
 3 | sudo: required
 4 | dist: trusty
 5 | 
 6 | git:
 7 |   depth: 10
 8 | 
 9 | # key part: caching the build directory and the mtime cache
10 | cache:
11 |   directories:
12 |   - demo/build
13 |   - demo/.mtime_cache
14 | 
15 | script:
16 |   - cd demo
17 |   - ruby ../bin/mtime_cache -g mtime_cache_globs.txt -c .mtime_cache/cache.json
18 |   - cd build
19 |   - cmake ..
20 |   - cd ..
21 |   - cmake --build build
22 |   - build/mtime_cache_demo
23 | 


--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
 1 | mtime_cache
 2 | Copyright (c) 2016 Borislav Stanimirov
 3 | 
 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of
 5 | this software and associated documentation files (the "Software"), to deal in
 6 | the Software without restriction, including without limitation the rights to
 7 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 8 | the Software, and to permit persons to whom the Software is furnished to do so,
 9 | subject to the following conditions:
10 | 
11 | The above copyright notice and this permission notice shall be included in all
12 | copies or substantial portions of the Software.
13 | 
14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
16 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
17 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
18 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
19 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.


--------------------------------------------------------------------------------
/mtime_cache.gemspec:
--------------------------------------------------------------------------------
 1 | Gem::Specification.new do |s|
 2 |   s.name        = 'mtime_cache'
 3 |   s.version     = '1.0.2'
 4 |   s.date        = '2016-10-27'
 5 |   s.summary     = 'Cache file mtimes. Helps caching build artifacts on a CIS'
 6 |   s.executables = ['mtime_cache']
 7 |   s.description = <<-DESC
 8 | mtime_cache creates a cache of file modification times, based on a glob pattern.
 9 | If a cache exists it updates unchanged files (unchanged based on MD5 hash) with
10 | the time from the cache.
11 | 
12 | This is useful if you cache your build artifacts for a build process which
13 | detects changes based on source modification time (such as most C or C++ build
14 | systems) on a continuous integration service (such as Travis CI), which clones
15 | the repo for every build.
16 | 
17 | When the repo is cloned, all source files have a modification time equal to the
18 | current time, making the cached build artifacts (for example .o files) obsolete.
19 | mtime_cache allows you to cache the modification times of the files, enabling a
20 | minimal rebuild for each clone.
21 |   DESC
22 |   s.authors     = ['Borislav Stanimirov']
23 |   s.email       = 'b.stanimirov@abv.bg'
24 |   s.files       = ['bin/mtime_cache']
25 |   s.homepage    = 'https://github.com/iboB/mtime_cache'
26 |   s.license     = 'MIT'
27 | end
28 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | mtime_cache
 2 | ===========
 3 | 
 4 | [![Build Status](https://travis-ci.org/iboB/mtime_cache.svg?branch=master)](https://travis-ci.org/iboB/mtime_cache)
 5 | 
 6 | **mtime_cache** helps you make use of build artifacts cache in a CI system (like [Travis CI](https://travis-ci.org/), [AppVeyor](https://www.appveyor.com/) or [Codeship](https://codeship.com/)) for a language whose build typically depends on the source files' modification time rather than their contents (like most C or C++ build systems).
 7 | 
 8 | ## Problem
 9 | 
10 | You have a C/C++ project with many, many files and you're using Travis CI (or AppVeyor, or Codeship) to build and test it. The thing is that Travis CI makes a fresh clone of your project for every build. This leads to builds of 10, or 20, or 40 minutes, or more, even if what's changed is a single source file (or none at all).
11 | 
12 | Frustrated you discover that Travis CI supports caching of build artifacts. And you cache your intermediate (`.o` or `.obj`) file directory. That however seems to do nothing and again a full build is triggered every time. This is because [git doesn't store meta information such as mtime for the files](http://stackoverflow.com/questions/2179722/checking-out-old-file-with-original-create-modified-timestamps/2179825#2179825). So the source files from your fresh clone at Travis CI have modification times newer than the cached `.o` files. So the C/C++ build system "thinks" that they should be rebuilt.
13 | 
14 | ## Solution
15 | 
16 | **mtime_cache**
17 | 
18 | Run the command line tool before you build and it will store the modification time of each source file in a json cache (which you need to cache along with your build artifacts). It will generate a md5 hash for each source file and if it remains the same after the next clone, it will restore the cached mtime for it. So changed files will be re-compiled, but unchanged ones won't be.
19 | 
20 | Making use of the build artifact cache can save you tens of minutes of build time for commits which change little or no source files.
21 | 
22 | Plus it's greener, as it saves CPU time, electricity and the rain forest. 
23 | 
24 | ## Usage
25 | 
26 | ### Installation
27 | 
28 | You can either install mtime_cache as a gem (with `$ gem install mtime_cache`) or make a copy of the script in your repo.
29 | 
30 | ### Running mtime_cache
31 | 
32 | The most basic usage is `$ mtime_cache <globs>`
33 | 
34 | This will generate a file `.mtime_cache.json` in the current directory with cached mtimes for the files matching the globs.
35 | 
36 | You must provide globs to make a cache.
37 | 
38 | #### Globs:
39 | 
40 | Globs are one or more ruby-style glob patterns that must match the source files for which you need an mtime cache.
41 | 
42 | Additionally the tool supports built in extension patterns. You can add an extension pattern in a `%{}` block.
43 | 
44 | Valid globs are:
45 | * `src/*.cpp` - all `.cpp` files in src
46 | * `my/src/**/*.{cpp,hpp}` - all `.cpp` and `.hpp` files in my/src and all its subdirectories
47 | * `some/dir/**/*.{%{cpp}}` - all files typical for C/C++ in some/dir and all its subdirectories
48 | 
49 | The supported extension patterns are:
50 | * `%{cpp}` - common C/C++ extensions: `c,cc,cpp,cxx,h,hpp,hxx,inl,ipp,inc,ixx`
51 | 
52 | #### Additional configuration:
53 | 
54 | * `--cache` or `-c`: Lets you provide a custom cache file instead of using the default `./.mtime_cache.json`. Sample usage: `$ mtime_cache src/*.{c,h} -c .my_cache/mtime_cache.json`
55 | * `--globfile` or `-g`: Lets you provide a text file where each non-empty line is a glob instead of adding all globs as command-line arguments. Sample usage: `mtime_cache -g myglobs.txt`
56 | * `--quiet` or `-q`: Prevents any logging to the standard output whatsoever.
57 | * `--verbose` or `-V`: Shows extra logging. For each file matching a glob a log line will be displayed showing whether its mtime was restored or left unchanged (if it doesn't match the cached md5 hash).
58 | * `--dryrun` or `-d`: Doesn't write to the file system. Only logs and reads matching files, and cache file if it exists.
59 | * `--version` or `-v`: Only displays version and exits.
60 | * `--help` or `-h` or `-?`: Displays help screen and exits.
61 | 
62 | ### Using mtime_cache with Travis CI
63 | 
64 | * Choose a directory name the json cache (for example `.mtime_cache`) and add it to your `.gitignore`. Then add it to your Travis CI cache. Then your cache section in `.travis.yml` might look like this
65 | 
66 | ```yaml
67 | cache:
68 |   directories:
69 |   - my/build/dir
70 |   - .mtime_cache
71 | ```
72 | 
73 | * Run mtime_cache in your script before you trigger the actual build. Then your script section might look like this:
74 | 
75 | ```yaml
76 | script:
77 |   - ./configure
78 |   - ./mtime_cache src/**/*.{%{cpp}} -c .mtime_cache/cache.json
79 |   - make
80 |   - ./test
81 | ```
82 | 
83 | You can also take a look at `.travis.yml` in this repo to see how the demo project is being built.
84 | 
85 | ## Demo
86 | 
87 | There is a demo project in this repo which is build by the Travis CI integration. It has three `.cpp` files and a `.hpp` file. You can see that commits that don't change one of those, don't trigger a build of the C++ code (or that commits that only change one or two `.cpp` files trigger a recompilation of only those).
88 | 
89 | ## License
90 | 
91 | This software is distributed under the MIT Software License.
92 | 
93 | See accompanying file LICENSE.txt or copy [here](https://opensource.org/licenses/MIT).
94 | 
95 | Copyright &copy; 2016 [Borislav Stanimirov](http://github.com/iboB)
96 | 


--------------------------------------------------------------------------------
/bin/mtime_cache:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env ruby
  2 | 
  3 | #
  4 | # mtime_cache
  5 | # Copyright (c) 2016 Borislav Stanimirov
  6 | #
  7 | # Permission is hereby granted, free of charge, to any person obtaining a copy
  8 | # of this software and associated documentation files (the "Software"), to
  9 | # deal in the Software without restriction, including without limitation the
 10 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 11 | # sell copies of the Software, and to permit persons to whom the Software is
 12 | # furnished to do so, subject to the following conditions:
 13 | #
 14 | # The above copyright notice and this permission notice shall be included in
 15 | # all copies or substantial portions of the Software.
 16 | #
 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 22 | # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 23 | # IN THE SOFTWARE.
 24 | #
 25 | 
 26 | require 'digest/md5'
 27 | require 'json'
 28 | require 'fileutils'
 29 | require 'time'
 30 | 
 31 | VERSION = "1.0.2"
 32 | 
 33 | VERSION_TEXT = "mtime_cache v#{VERSION}"
 34 | 
 35 | USAGE = <<ENDUSAGE
 36 | 
 37 | Usage:
 38 |     mtime_cache [<globs>] [-g globfile] [-d] [-q|V] [-c cache]
 39 | ENDUSAGE
 40 | 
 41 | HELP = <<ENDHELP
 42 | 
 43 |     Traverse through globbed files, making a json cache based on their mtime.
 44 |     If a cache exists, changes the mtime of existing unchanged (based on MD5
 45 |     hash) files to the one in the cache.
 46 | 
 47 |     Options:
 48 | 
 49 |     globs           Ruby-compatible glob strings (ex some/path/**/*.java)
 50 |                     A extension pattern is allowd in the form %{pattern}
 51 |                     (ex some/path/*.{%{pattern1},%{pattern2}})
 52 |                     The globs support the following patterns:
 53 |                      %{cpp} - common C++ extensions
 54 | 
 55 |     -g, --globfile  A file with list of globs to perform (one per line)
 56 | 
 57 |     -?, -h, --help  Show this help message.
 58 |     -v, --version   Show the version number (#{VERSION})
 59 |     -q, --quiet     Don't log anything to stdout
 60 |     -V, --verbose   Show extra logging
 61 |     -d, --dryrun    Don't change any files on the filesystem
 62 |     -c, --cache     Specify the cache file for input and output.
 63 |                     [Default is .mtime_cache.json]
 64 | 
 65 | ENDHELP
 66 | 
 67 | param_arg = nil
 68 | ARGS = { :cache => '.mtime_cache.json', :globs => [] }
 69 | 
 70 | ARGV.each do |arg|
 71 |   case arg
 72 |     when '-g', '--globfile'   then param_arg = :globfile
 73 |     when '-h', '-?', '--help' then ARGS[:help] = true
 74 |     when '-v', '--version'    then ARGS[:ver] = true
 75 |     when '-q', '--quiet'      then ARGS[:quiet] = true
 76 |     when '-V', '--verbose'    then ARGS[:verbose] = true
 77 |     when '-d', '--dryrun'     then ARGS[:dry] = true
 78 |     when '-c', '--cache'      then param_arg = :cache
 79 |     else
 80 |       if param_arg
 81 |         ARGS[param_arg] = arg
 82 |         param_arg = nil
 83 |       else
 84 |         ARGS[:globs] << arg
 85 |       end
 86 |   end
 87 | end
 88 | 
 89 | def log(text, level = 0)
 90 |   return if ARGS[:quiet]
 91 |   return if level > 0 && !ARGS[:verbose]
 92 |   puts text
 93 | end
 94 | 
 95 | if ARGS[:ver] || ARGS[:help]
 96 |   log VERSION_TEXT
 97 |   exit if ARGS[:ver]
 98 |   log USAGE
 99 |   log HELP
100 |   exit
101 | end
102 | 
103 | if ARGS[:globs].empty? && !ARGS[:globfile]
104 |   log 'Error: Missing globs'
105 |   log USAGE
106 |   exit 1
107 | end
108 | 
109 | EXTENSION_PATTERNS = {
110 |   :cpp => "c,cc,cpp,cxx,h,hpp,hxx,inl,ipp,inc,ixx"
111 | }
112 | 
113 | cache_file = ARGS[:cache]
114 | 
115 | cache = {}
116 | 
117 | if File.file?(cache_file)
118 |   log "Found #{cache_file}"
119 |   cache = JSON.parse(File.read(cache_file))
120 |   log "Read #{cache.length} entries"
121 | else
122 |   log "#{cache_file} not found. A new one will be created"
123 | end
124 | 
125 | globs = ARGS[:globs].map { |g| g % EXTENSION_PATTERNS }
126 | 
127 | globfile = ARGS[:globfile]
128 | if globfile
129 |   File.open(globfile, 'r').each_line do |line|
130 |     line.strip!
131 |     next if line.empty?
132 |     globs << line % EXTENSION_PATTERNS
133 |   end
134 | end
135 | 
136 | if globs.empty?
137 |   log 'Error: No globs in globfile'
138 |   log USAGE
139 |   exit 1
140 | end
141 | 
142 | files = {}
143 | num_changed = 0
144 | 
145 | def str_from_time(time)
146 |   time.iso8601(9) # nanoseconds precision
147 |   # time.to_i
148 | end
149 | 
150 | globs.each do |glob|
151 |   Dir[glob].each do |file|
152 |     next if !File.file?(file)
153 | 
154 |     mtime = File.mtime(file)
155 |     hash = Digest::MD5.hexdigest(File.read(file))
156 | 
157 |     cached = cache[file]
158 | 
159 |     if cached && cached['hash'] == hash
160 |       cached_mtime_value = cached['mtime']
161 |       cached_mtime = if cached_mtime_value.is_a?(Integer)
162 |                        Time.at(cached_mtime_value)
163 |                      else
164 |                        Time.parse(cached_mtime_value)
165 |                      end
166 |       if cached_mtime < mtime
167 |         mtime = cached_mtime
168 | 
169 |         log "mtime_cache: changing mtime of #{file} to #{str_from_time(mtime)}", 1
170 | 
171 |         File.utime(File.atime(file), mtime, file) if !ARGS[:dry]
172 |         num_changed += 1
173 |       end
174 |     else
175 |       log "mtime_cache: NOT changing mtime of #{file}", 1
176 |     end
177 | 
178 |     files[file] = { 'mtime' => str_from_time(mtime), 'hash' => hash }
179 |   end
180 | end
181 | 
182 | log "Changed mtime of #{num_changed} of #{files.length} files"
183 | log "Writing #{cache_file}"
184 | 
185 | if !ARGS[:dry]
186 |   dirname = File.dirname(cache_file)
187 |   unless File.directory?(dirname)
188 |     FileUtils.mkdir_p(dirname)
189 |   end
190 |   File.open(cache_file, 'w').write(JSON.pretty_generate(files))
191 | end
192 | 


--------------------------------------------------------------------------------