├── .gitignore
├── profiles
    ├── wang_os
    │   ├── dir_distribution.txt
    │   ├── age_distribution.txt
    │   └── size_distribution.txt
    ├── wang_lanl
    │   ├── dir_distribution.txt
    │   ├── age_distribution.txt
    │   └── size_distribution.txt
    ├── agrawal
    │   ├── age_distribution.txt
    │   ├── size_distribution.txt
    │   └── dir_distribution.txt
    ├── grundman
    │   ├── age_distribution.txt
    │   ├── dir_distribution.txt
    │   └── size_distribution.txt
    ├── dabre
    │   ├── age_distribution.txt
    │   ├── dir_distribution.txt
    │   └── size_distribution.txt
    ├── pramod
    │   ├── age_distribution.txt
    │   ├── size_distribution.txt
    │   └── dir_distribution.txt
    ├── douceur
    │   ├── dir_distribution.txt
    │   ├── age_distribution.txt
    │   └── size_distribution.txt
    └── meyer
    │   ├── age_distribution.txt
    │   ├── dir_distribution.txt
    │   └── size_distribution.txt
├── src
    ├── age_list.h
    ├── backend_driver.h
    ├── deltafs_driver.c
    ├── geriatrix.h
    ├── file.h
    ├── ThreadPool.h
    ├── size_bucket.h
    ├── age_bucket.h
    ├── dir_bucket.h
    └── geriatrix.cpp
├── LICENSE.txt
├── utils
    └── rollback.py
├── CMakeLists.txt
└── README.md


/.gitignore:
--------------------------------------------------------------------------------
1 | build/
2 | 


--------------------------------------------------------------------------------
/profiles/wang_os/dir_distribution.txt:
--------------------------------------------------------------------------------
1 | 1
2 | 0 1 0
3 | 
4 | Format:
5 | <number of observations>
6 | <depth> <weight in integers> <subdirs>
7 | 


--------------------------------------------------------------------------------
/profiles/wang_lanl/dir_distribution.txt:
--------------------------------------------------------------------------------
1 | 1
2 | 0 1 0
3 | 
4 | Format:
5 | <number of observations>
6 | <depth> <weight in integers> <subdirs>
7 | 


--------------------------------------------------------------------------------
/profiles/wang_os/age_distribution.txt:
--------------------------------------------------------------------------------
 1 | 10
 2 | 16 1
 3 | 32 1
 4 | 64 2
 5 | 128 3
 6 | 256 19
 7 | 512 30
 8 | 1024 28
 9 | 2048 12
10 | 4096 1
11 | 8192 1
12 | 
13 | Format:
14 | <number of observations>
15 | <age> <weight in integers>
16 | 


--------------------------------------------------------------------------------
/profiles/wang_lanl/age_distribution.txt:
--------------------------------------------------------------------------------
 1 | 12
 2 | 2 1
 3 | 4 1
 4 | 8 1
 5 | 16 1
 6 | 32 3
 7 | 64 35
 8 | 128 51
 9 | 256 42
10 | 512 47
11 | 1024 18
12 | 2048 1
13 | 4096 1
14 | 
15 | Format:
16 | <number of observations>
17 | <age> <weight in integers>
18 | 


--------------------------------------------------------------------------------
/profiles/agrawal/age_distribution.txt:
--------------------------------------------------------------------------------
 1 | 10
 2 | 10.74 1
 3 | 31 2
 4 | 85.98 8
 5 | 240 20
 6 | 687.87 44
 7 | 1945.59 75
 8 | 5502.98 144
 9 | 15768 146
10 | 44023.9 23
11 | 122640 1
12 | 
13 | Format:
14 | <number of observations>
15 | <age> <weight in integers>
16 | 


--------------------------------------------------------------------------------
/profiles/grundman/age_distribution.txt:
--------------------------------------------------------------------------------
 1 | 9
 2 | 15 14143
 3 | 30 67905
 4 | 60 14952
 5 | 120 29470
 6 | 240 9189
 7 | 480 121932
 8 | 960 286283
 9 | 1920 124744
10 | 3840 163830
11 | 
12 | Format:
13 | <number of observations>
14 | <age> <weight in integers>
15 | 


--------------------------------------------------------------------------------
/profiles/agrawal/size_distribution.txt:
--------------------------------------------------------------------------------
 1 | 11
 2 | 8 1
 3 | 32 7
 4 | 128 16
 5 | 512 40
 6 | 2048 50
 7 | 8192 46
 8 | 32768 35
 9 | 131072 20
10 | 524288 10
11 | 2097152 4
12 | 8388608 1
13 | 
14 | Format:
15 | <number of observations>
16 | <size in bytes> <weight in integers>
17 | 


--------------------------------------------------------------------------------
/profiles/wang_lanl/size_distribution.txt:
--------------------------------------------------------------------------------
 1 | 11
 2 | 32 1
 3 | 512 1
 4 | 2048 1
 5 | 4096 1
 6 | 32768 1
 7 | 65536 1
 8 | 131072 47
 9 | 262144 20
10 | 524288 10
11 | 1048576 1
12 | 2097152 1
13 | 
14 | Format:
15 | <number of observations>
16 | <size in bytes> <weight in integers>
17 | 


--------------------------------------------------------------------------------
/profiles/dabre/age_distribution.txt:
--------------------------------------------------------------------------------
 1 | 14
 2 | 1 2
 3 | 2 0
 4 | 4 0
 5 | 8 0
 6 | 16 0
 7 | 32 1907
 8 | 64 3
 9 | 128 4
10 | 256 5541
11 | 512 840
12 | 1024 14392
13 | 2048 18
14 | 4096 23304
15 | 8192 181403
16 | 
17 | Format:
18 | <number of observations>
19 | <age> <weight in integers>
20 | 


--------------------------------------------------------------------------------
/profiles/pramod/age_distribution.txt:
--------------------------------------------------------------------------------
 1 | 16
 2 | 1 32
 3 | 2 0
 4 | 4 0
 5 | 8 0
 6 | 16 0
 7 | 32 0
 8 | 64 0
 9 | 128 0
10 | 256 0
11 | 512 0
12 | 1024 7632
13 | 2048 6780
14 | 4096 2735
15 | 8192 73004
16 | 16384 584199
17 | 32768 211013
18 | 
19 | Format:
20 | <number of observations>
21 | <age> <weight in integers>
22 | 


--------------------------------------------------------------------------------
/profiles/agrawal/dir_distribution.txt:
--------------------------------------------------------------------------------
 1 | 15
 2 | 1 1 1
 3 | 2 2 5
 4 | 3 14 13
 5 | 4 15 19
 6 | 5 23 30
 7 | 6 24 36
 8 | 7 42 32
 9 | 8 22 26
10 | 9 15 22
11 | 10 12 17
12 | 11 7 8
13 | 12 4 5
14 | 13 3 2
15 | 14 2 1
16 | 15 1 1
17 | 
18 | Format:
19 | <number of observations>
20 | <depth> <weight in integers> <subdirs>
21 | 


--------------------------------------------------------------------------------
/profiles/wang_os/size_distribution.txt:
--------------------------------------------------------------------------------
 1 | 15
 2 | 0 12
 3 | 512 8
 4 | 1024 9
 5 | 2048 12
 6 | 4096 18
 7 | 8192 12
 8 | 16384 10
 9 | 32768 9
10 | 65536 5
11 | 131072 1
12 | 262144 1
13 | 524288 1
14 | 1048576 1
15 | 2097152 1
16 | 4194304 1
17 | 
18 | Format:
19 | <number of observations>
20 | <size in bytes> <weight in integers>
21 | 


--------------------------------------------------------------------------------
/profiles/dabre/dir_distribution.txt:
--------------------------------------------------------------------------------
 1 | 14
 2 | 0 1 28
 3 | 1 303 239
 4 | 2 1893 995
 5 | 3 4710 3765
 6 | 4 18597 3389
 7 | 5 30689 6495
 8 | 6 25185 5784
 9 | 7 11252 2843
10 | 8 5540 802
11 | 9 1271 221
12 | 10 306 82
13 | 11 214 20
14 | 12 26 1
15 | 13 6 0
16 | 
17 | Format:
18 | <number of observations>
19 | <depth> <weight in integers> <subdirs>
20 | 


--------------------------------------------------------------------------------
/profiles/douceur/dir_distribution.txt:
--------------------------------------------------------------------------------
 1 | 16
 2 | 0 50 16
 3 | 1 450 32
 4 | 2 1150 64
 5 | 3 1650 32
 6 | 4 1800 16
 7 | 5 1450 8
 8 | 6 1125 4
 9 | 7 625 2
10 | 8 400 1
11 | 9 160 0
12 | 10 100 0
13 | 11 150 0
14 | 12 50 0
15 | 13 20 0
16 | 14 20 0
17 | 15 10 0
18 | 
19 | Format:
20 | <number of observations>
21 | <depth> <weight in integers> <subdirs>
22 | 


--------------------------------------------------------------------------------
/profiles/meyer/age_distribution.txt:
--------------------------------------------------------------------------------
 1 | 24
 2 | 2 3
 3 | 4 90
 4 | 6 70
 5 | 8 30
 6 | 10 20
 7 | 12 10
 8 | 14 14
 9 | 16 10
10 | 18 10
11 | 20 4
12 | 22 4
13 | 24 18
14 | 26 11
15 | 28 8
16 | 30 2
17 | 32 4
18 | 34 5
19 | 36 1
20 | 38 15
21 | 40 8
22 | 42 1
23 | 44 1
24 | 46 2
25 | 48 1
26 | 
27 | Format:
28 | <number of observations>
29 | <age> <weight in integers>
30 | 


--------------------------------------------------------------------------------
/profiles/douceur/age_distribution.txt:
--------------------------------------------------------------------------------
 1 | 20
 2 | 9 25
 3 | 18 30
 4 | 36 50
 5 | 72 60
 6 | 144 80
 7 | 288 90
 8 | 576 120
 9 | 1152 200
10 | 2304 300
11 | 4608 400
12 | 9216 600
13 | 18432 1050
14 | 36864 1250
15 | 73728 1900
16 | 147456 1800
17 | 294912 1500
18 | 589824 700
19 | 1179648 100
20 | 2359296 30
21 | 4718592 25
22 | 
23 | Format:
24 | <number of observations>
25 | <age> <weight in integers>
26 | 


--------------------------------------------------------------------------------
/profiles/meyer/dir_distribution.txt:
--------------------------------------------------------------------------------
 1 | 21
 2 | 0 10 16
 3 | 1 30 32
 4 | 2 100 64
 5 | 3 830 32
 6 | 4 600 16
 7 | 5 800 8
 8 | 6 900 4
 9 | 7 930 2
10 | 8 860 1
11 | 9 780 0
12 | 10 550 0
13 | 11 300 0
14 | 12 190 0
15 | 13 130 0
16 | 14 90 0
17 | 15 50 0
18 | 16 20 0
19 | 17 10 0
20 | 18 5 0
21 | 19 4 0
22 | 20 3 0
23 | 
24 | Format:
25 | <number of observations>
26 | <depth> <weight in integers> <subdirs>
27 | 


--------------------------------------------------------------------------------
/profiles/douceur/size_distribution.txt:
--------------------------------------------------------------------------------
 1 | 25
 2 | 0 180
 3 | 1 0
 4 | 2 10
 5 | 4 15
 6 | 8 90
 7 | 16 25
 8 | 32 165
 9 | 64 380
10 | 128 425
11 | 256 620
12 | 512 880
13 | 1024 1080
14 | 2048 1100
15 | 4096 1000
16 | 8192 1025
17 | 16384 920
18 | 32768 800
19 | 65536 575
20 | 131072 375
21 | 262144 200
22 | 524288 100
23 | 1048576 75
24 | 2097152 20
25 | 4194304 10
26 | 8388608 5
27 | 
28 | Format:
29 | <number of observations>
30 | <size in bytes> <weight in integers>
31 | 


--------------------------------------------------------------------------------
/profiles/dabre/size_distribution.txt:
--------------------------------------------------------------------------------
 1 | 22
 2 | 1024 125594
 3 | 2048 22133
 4 | 4096 22761
 5 | 8192 17448
 6 | 16384 16147
 7 | 32768 10026
 8 | 65536 5874
 9 | 131072 3311
10 | 262144 1833
11 | 524288 1042
12 | 1048576 614
13 | 2097152 289
14 | 4194304 157
15 | 8388608 73
16 | 16777216 45
17 | 33554432 26
18 | 67108864 27
19 | 134217728 5
20 | 268435456 2
21 | 536870912 2
22 | 1073741824 1
23 | 2147483648 4
24 | 
25 | Format:
26 | <number of observations>
27 | <size in bytes> <weight in integers>
28 | 


--------------------------------------------------------------------------------
/profiles/grundman/dir_distribution.txt:
--------------------------------------------------------------------------------
 1 | 26
 2 | 0 1 23
 3 | 1 3644 31
 4 | 2 16136 5
 5 | 3 79382 3
 6 | 4 125928 2
 7 | 5 168632 1
 8 | 6 109214 1
 9 | 7 87633 1
10 | 8 93477 1
11 | 9 96658 1
12 | 10 77672 1
13 | 11 40867 1
14 | 12 31209 1
15 | 13 19282 1
16 | 14 31660 1
17 | 15 14959 1
18 | 16 2935 1
19 | 17 546 1
20 | 18 128 1
21 | 19 14 1
22 | 20 3 1
23 | 21 0 1
24 | 22 0 1
25 | 23 0 1
26 | 24 0 1
27 | 25 9 0
28 | 
29 | Format:
30 | <number of observations>
31 | <depth> <weight in integers> <subdirs>
32 | 


--------------------------------------------------------------------------------
/profiles/grundman/size_distribution.txt:
--------------------------------------------------------------------------------
 1 | 22
 2 | 1024 305720
 3 | 2048 102627
 4 | 4096 102902
 5 | 8192 111260
 6 | 16384 82076
 7 | 32768 53978
 8 | 65536 32438
 9 | 131072 16480
10 | 262144 9636
11 | 524288 6238
12 | 1048576 4010
13 | 2097152 1890
14 | 4194304 1468
15 | 8388608 942
16 | 16777216 366
17 | 33554432 196
18 | 67108864 113
19 | 134217728 42
20 | 268435456 33
21 | 536870912 17
22 | 1073741824 7
23 | 2147483648 4
24 | 4294967296 5
25 | 
26 | Format:
27 | <number of observations>
28 | <size in bytes> <weight in integers>
29 | 


--------------------------------------------------------------------------------
/profiles/meyer/size_distribution.txt:
--------------------------------------------------------------------------------
 1 | 29
 2 | 0 100
 3 | 1 20
 4 | 2 40
 5 | 4 30
 6 | 8 30
 7 | 16 100
 8 | 32 100
 9 | 64 170
10 | 128 220
11 | 256 400
12 | 512 680
13 | 1024 1180
14 | 2048 1230
15 | 4096 1150
16 | 8192 1020
17 | 16384 900
18 | 32768 700
19 | 65536 680
20 | 131072 440
21 | 262144 300
22 | 524288 230
23 | 1048576 175
24 | 2097152 130
25 | 4194304 100
26 | 8388608 60
27 | 16777216 40
28 | 33554432 20
29 | 67108864 10
30 | 134217728 10
31 | 
32 | Format:
33 | <number of observations>
34 | <size in bytes> <weight in integers>
35 | 


--------------------------------------------------------------------------------
/profiles/pramod/size_distribution.txt:
--------------------------------------------------------------------------------
 1 | 26
 2 | 1024 365208
 3 | 2048 131796
 4 | 4096 115274
 5 | 8192 97335
 6 | 16384 65749
 7 | 32768 36649
 8 | 65536 26563
 9 | 131072 18681
10 | 262144 9046
11 | 524288 4537
12 | 1048576 3457
13 | 2097152 2426
14 | 4194304 4395
15 | 8388608 2190
16 | 16777216 1127
17 | 33554432 418
18 | 67108864 245
19 | 134217728 160
20 | 268435456 81
21 | 536870912 12
22 | 1073741824 32
23 | 2147483648 12
24 | 4294967296 1
25 | 8589934592 1
26 | 17179869184 1
27 | 34359738368 1
28 | 
29 | Format:
30 | <number of observations>
31 | <size in bytes> <weight in integers>
32 | 


--------------------------------------------------------------------------------
/profiles/pramod/dir_distribution.txt:
--------------------------------------------------------------------------------
 1 | 37
 2 | 0 0 3
 3 | 1 17 25
 4 | 2 147 7
 5 | 3 604 2
 6 | 4 2498 2
 7 | 5 3994 3
 8 | 6 5525 2
 9 | 7 10947 1
10 | 8 11409 1
11 | 9 12275 1
12 | 10 9786 1
13 | 11 8072 1
14 | 12 9587 1
15 | 13 6856 1
16 | 14 6503 1
17 | 15 3907 1
18 | 16 2720 1
19 | 17 1041 1
20 | 18 781 1
21 | 19 229 1
22 | 20 473 1
23 | 21 117 2
24 | 22 488 1
25 | 23 164 1
26 | 24 456 1
27 | 25 94 1
28 | 26 484 2
29 | 27 85 1
30 | 28 315 1
31 | 29 32 1
32 | 30 226 1
33 | 31 3 1
34 | 32 101 1
35 | 33 2 1
36 | 34 32 1
37 | 35 0 1
38 | 36 10 0
39 | 
40 | Format:
41 | <number of observations>
42 | <depth> <weight in integers> <subdirs>
43 | 


--------------------------------------------------------------------------------
/src/age_list.h:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright (c) 2018 Carnegie Mellon University.
 3 |  *
 4 |  * All rights reserved.
 5 |  *
 6 |  * Use of this source code is governed by a BSD-style license that can be
 7 |  * found in the LICENSE file. See the AUTHORS file for names of contributors.
 8 |  */
 9 | 
10 | #include "file.h"
11 | 
12 | #ifndef AGE_LIST_
13 | #define AGE_LIST_
14 | class AgeList {
15 |   public:
16 |     File *fs;
17 |     size_t size;
18 |     uint64_t count;
19 |     uint64_t total_size;
20 | 
21 |     AgeList(uint64_t size) {
22 |       fs = new File("0", 0, 0, 0);
23 |       fs->prev = fs;
24 |       fs->next = fs;
25 |       this->size = size;
26 |     }
27 | 
28 |     void addFile(File *f) {
29 |       f->next = fs;
30 |       f->prev = fs->prev;
31 |       fs->prev->next = f;
32 |       fs->prev = f;
33 |       count++;
34 |     }
35 | 
36 |     void deleteFile(File *f) {
37 |       if(fs->next == f) {
38 |         fs->next = f->next;
39 |       } else if(fs->prev == f) {
40 |         fs->prev = f->prev;
41 |       }
42 |       f->prev->next = f->next;
43 |       f->next->prev = f->prev;
44 |       f->next = f->prev = NULL;
45 |       count--;
46 |     }
47 | };
48 | #endif  /* AGE_LIST_ */
49 | 


--------------------------------------------------------------------------------
/src/backend_driver.h:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright (c) 2018 Carnegie Mellon University.
 3 |  *
 4 |  * All rights reserved.
 5 |  *
 6 |  * Use of this source code is governed by a BSD-style license that can be
 7 |  * found in the LICENSE file. See the AUTHORS file for names of contributors.
 8 |  */
 9 | 
10 | /*
11 |  * all I/O to the backend is routed through the backend driver.
12 |  * this should allow us to port geriatrix to filesystem that are
13 |  * not accessed through the kernel's POSIX system call API.
14 |  */
15 | 
16 | #ifndef BACKEND_
17 | #define BACKEND_
18 | 
19 | /*
20 |  * the signature on these is setup so that we can directly plug in
21 |  * libc's posix calls for the default posix environment.
22 |  */
23 | struct backend_driver {
24 |     int (*bd_open)(const char *path, int flags, ...);
25 |     int (*bd_close)(int fd);
26 |     ssize_t (*bd_write)(int fd, const void *buf, size_t nbytes);
27 |     int (*bd_access)(const char *path, int mode);
28 |     int (*bd_unlink)(const char *path);
29 |     int (*bd_mkdir)(const char *path, mode_t mode);
30 |     int (*bd_fallocate)(int fd, off_t offset, off_t len);
31 |     int (*bd_stat)(const char *path, struct stat *st);
32 |     int (*bd_chmod)(const char *path, mode_t mode);
33 | };
34 | 
35 | #endif /* BACKEND_ */
36 | 


--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright (c) 2018, Carnegie Mellon University.
 3 |  * All rights reserved.
 4 |  *
 5 |  * Redistribution and use in source and binary forms, with or without
 6 |  * modification, are permitted provided that the following conditions
 7 |  * are met:
 8 |  * 1. Redistributions of source code must retain the above copyright
 9 |  *    notice, this list of conditions and the following disclaimer.
10 |  * 2. Redistributions in binary form must reproduce the above copyright
11 |  *    notice, this list of conditions and the following disclaimer in the
12 |  *    documentation and/or other materials provided with the distribution.
13 |  *
14 |  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
15 |  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
16 |  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
17 |  * A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT
18 |  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
19 |  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
20 |  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
21 |  * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
22 |  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 |  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
24 |  * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
25 |  * POSSIBILITY OF SUCH DAMAGE.
26 |  */
27 | 


--------------------------------------------------------------------------------
/utils/rollback.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python3
 2 | 
 3 | #
 4 | # Copyright (c) 2018 Carnegie Mellon University.
 5 | #
 6 | # All rights reserved.
 7 | #
 8 | # Use of this source code is governed by a BSD-style license that can be
 9 | # found in the LICENSE file. See the AUTHORS file for names of contributors.
10 | #
11 | 
12 | import argparse
13 | from io import BytesIO
14 | 
15 | parser = argparse.ArgumentParser()
16 | parser.add_argument("blkparse", help="blkparse file containing only write operations from blktrace dump, filtered with blkparse -a write ...")
17 | parser.add_argument("offset", help="offset of the partition in blocks", type=int)
18 | parser.add_argument("image", help="original image file path used to dd on partition for running benchmark")
19 | parser.add_argument("partition", help="device file of the partition on which the benchmark was executed")
20 | args = parser.parse_args()
21 | 
22 | def parse_blkparse(blkparse, offset):
23 |     r = set()
24 |     nwrites = 0
25 |     with open(blkparse, "r") as file:
26 |         for line in file:
27 |             fields = line.split()
28 |             if len(fields) < 10: #or fields[5] != 'C':
29 |                 continue
30 |             if fields[6][0] != 'W':
31 |                 continue
32 |             assert fields[6][0] == 'W', fields
33 |             block = (int(fields[7]) - offset) // 8
34 |             nblocks = int(fields[9]) // 8
35 |             nblocks += 1
36 |             for blk in range(block, block + nblocks):
37 |                 r.add(blk)
38 |     return (r, nwrites)
39 | 
40 | def restore_image(written_blocks, nwrites):
41 |     block_size = 4096
42 |     with open(args.image, "rb") as image_file:
43 |         with open(args.partition, "wb") as partition:
44 |             for block in written_blocks:
45 |                 image_file.seek(block * block_size, 0) # seek to byte offset within file
46 |                 data = image_file.read(block_size) # read data block
47 |                 partition.seek(block * block_size, 0) # seek to byte offset in partition
48 |                 bytes_written = partition.write(data)
49 |             partition.close()
50 | 
51 | 
52 | if __name__ == "__main__":
53 |     written_blocks, nwrites = parse_blkparse(args.blkparse, args.offset)
54 |     restore_image(written_blocks, nwrites)
55 |     print("total blocks replaced: ", len(written_blocks))
56 | 


--------------------------------------------------------------------------------
/src/deltafs_driver.c:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright (c) 2018 Carnegie Mellon University.
 3 |  *
 4 |  * All rights reserved.
 5 |  *
 6 |  * Use of this source code is governed by a BSD-style license that can be
 7 |  * found in the LICENSE file. See the AUTHORS file for names of contributors.
 8 |  */
 9 | 
10 | /*
11 |  * this file provides optional DELTAFS support for geriatrix by
12 |  * providing a deltafs backen_driver.  it is only compiled if DELTAFS
13 |  * support is enabled...
14 |  */
15 | 
16 | #include <assert.h>
17 | #include <errno.h>
18 | #include <stdio.h>
19 | #include <stdlib.h>
20 | #include <unistd.h>
21 | #include <sys/stat.h>
22 | 
23 | #include <deltafs/deltafs_api.h>
24 | 
25 | #include "backend_driver.h"
26 | 
27 | /* can't map open directly, since the signatures differ slightly */
28 | static int dback_open(const char *path, int falgs, ...) {
29 |     return(deltafs_open(path, falgs, 0600));
30 | }
31 | 
32 | /* deltafs doesn't have an access function, but it has stat */
33 | static int dback_access(const char *path, int mode) {
34 |     struct stat st;
35 |     assert(mode == F_OK);   /* only support F_OK, check if file is present */
36 |     return(deltafs_stat(path, &st));
37 | }
38 | 
39 | /* 
40 |  * deltafs version of fake posix_fallocate... (XXX could make shared
41 |  * version of this, but too small to bother with?)
42 |  */
43 | static int dback_fallocate(int fd, off_t offset, off_t len) {
44 |     struct stat st;
45 |     off_t newlen, curoff, lastoff, ptr;
46 |     ssize_t rv;
47 | 
48 |     newlen = offset + len;
49 | 
50 |     if (deltafs_fstat(fd, &st) < 0)
51 |         return(errno);
52 | 
53 |     if (st.st_size > newlen)        /* not growing it, assume ok */
54 |         return(0);
55 | 
56 |     if (deltafs_ftruncate(fd, newlen) < 0)   /* grow it */
57 |         return(errno);
58 | 
59 |     curoff = ((st.st_size + (st.st_blksize-1)) / st.st_blksize) * st.st_blksize;
60 |     lastoff = ((newlen + (st.st_blksize-1)) / st.st_blksize) * st.st_blksize;
61 | 
62 |     for (ptr = curoff ; ptr < lastoff ; ptr += st.st_blksize) {
63 |         rv = deltafs_pwrite(fd, "", 1, ptr);
64 |         if (rv < 0)
65 |             return(errno);
66 |         if (rv == 0)
67 |             return(EIO);
68 |     }
69 | 
70 |     return(0);
71 | }
72 | 
73 | /*
74 |  * here is the main driver structure....
75 |  */
76 | struct backend_driver deltafs_backend_driver = {
77 |     dback_open, deltafs_close, deltafs_write, dback_access, deltafs_unlink, 
78 |     deltafs_mkdir, dback_fallocate, deltafs_stat, deltafs_chmod,
79 | };
80 | 


--------------------------------------------------------------------------------
/src/geriatrix.h:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (c) 2018 Carnegie Mellon University.
  3 |  *
  4 |  * All rights reserved.
  5 |  *
  6 |  * Use of this source code is governed by a BSD-style license that can be
  7 |  * found in the LICENSE file. See the AUTHORS file for names of contributors.
  8 |  */
  9 | 
 10 | #include <iostream>
 11 | #include <stdlib.h>
 12 | #include <string>
 13 | #include <boost/container/vector.hpp>
 14 | #include <boost/unordered_map.hpp>
 15 | #include <boost/container/flat_map.hpp>
 16 | #include <boost/tokenizer.hpp>
 17 | #include <boost/random.hpp>
 18 | #include <boost/math/distributions/chi_squared.hpp>
 19 | #include <chrono>
 20 | #include <fstream>
 21 | #include <getopt.h>
 22 | #include <atomic>
 23 | #include <mutex>
 24 | #include <thread>
 25 | #include <fcntl.h>
 26 | #include <inttypes.h>
 27 | #include <unistd.h>
 28 | #include <sys/stat.h>
 29 | #include <signal.h>
 30 | //#include <gperftools/profiler.h>
 31 | #include "ThreadPool.h"
 32 | 
 33 | #include "age_bucket.h"
 34 | #include "age_list.h"
 35 | #include "backend_driver.h"
 36 | 
 37 | using namespace boost::container;
 38 | using namespace boost::unordered;
 39 | using namespace std::chrono;
 40 | 
 41 | std::string mount_point = "";
 42 | int NUM_DIRS = 0;
 43 | int NUM_SIZES = 0;
 44 | int NUM_AGES = 0;
 45 | int fake = 0;
 46 | 
 47 | typedef enum {
 48 |   DIRS,
 49 |   SIZES,
 50 |   AGES
 51 | } distribution_type_t;
 52 | 
 53 | struct size {
 54 |   char *in_file;
 55 |   char *out_file;
 56 |   double *distribution;
 57 |   size_t *arr;
 58 |   double *cutoffs;
 59 |   unordered_map<int, std::string> bucket_keys;
 60 | } s;
 61 | 
 62 | struct dir {
 63 |   char *in_file;
 64 |   char *out_file;
 65 |   double *distribution;
 66 |   int *arr;
 67 |   uint32_t *subdir_arr;
 68 |   unordered_map<int, std::string> bucket_keys;
 69 | } d;
 70 | 
 71 | struct age {
 72 |   char *in_file;
 73 |   char *out_file;
 74 |   double *distribution;
 75 |   double *cutoffs;
 76 |   unordered_map<int, std::string> bucket_keys;
 77 | } a;
 78 | 
 79 | double confidence = 0.0;
 80 | boost::math::chi_squared *dist;
 81 | double goodness_measure = 0.0;
 82 | auto start = std::chrono::high_resolution_clock::now();
 83 | int runtime_max = 0;
 84 | double runtime = 0;
 85 | int runs = 0;
 86 | uint64_t K = 0;
 87 | 
 88 | ThreadPool *pool;
 89 | 
 90 | uint64_t tick = 0;
 91 | uint64_t global_live_file_count = 0;
 92 | uint64_t total_age_weight = 0;
 93 | uint64_t total_size_weight = 0;
 94 | uint64_t total_dir_weight = 0;
 95 | size_t total_disk_capacity = 0;
 96 | size_t live_data_size = 0;
 97 | size_t workload_size = 0;
 98 | 
 99 | enum AGING_TRIGGER {none, convergence, exec_time, workload, accuracy};
100 | 
101 | 


--------------------------------------------------------------------------------
/src/file.h:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright (c) 2018 Carnegie Mellon University.
 3 |  *
 4 |  * All rights reserved.
 5 |  *
 6 |  * Use of this source code is governed by a BSD-style license that can be
 7 |  * found in the LICENSE file. See the AUTHORS file for names of contributors.
 8 |  */
 9 | 
10 | #include <iostream>
11 | 
12 | #ifndef FILE_
13 | #define FILE_
14 | class File {
15 |   public:
16 |     size_t size;
17 |     std::string path;
18 |     uint64_t age;
19 |     int depth; // id of the dir_bucket_keys
20 |     File *prev;
21 |     File *next;
22 |     File *size_next;
23 |     File *size_prev;
24 |     File *dir_next;
25 |     File *dir_prev;
26 |     size_t blk_size;
27 |     long blk_count;
28 | 
29 |     File(const char *name) {
30 |       this->path = name;
31 |       this->size = 0;
32 |       this->age = 0;
33 |       this->depth = 0;
34 |       this->prev = this->next = NULL;
35 |       this->size_next = this->size_prev = NULL;
36 |       this->dir_next = this->dir_prev = NULL;
37 |     }
38 | 
39 |     File(const char *name, size_t size, uint64_t age, int depth) {
40 |       this->path = name;
41 |       this->size = size;
42 |       this->age = age;
43 |       this->prev = this->next = NULL;
44 |       this->size_next = this->size_prev = NULL;
45 |       if(this->size == 0) {
46 |         this->blk_size = 4096;
47 |         this->blk_count = 0;
48 |       } else if(this->size >= 4096) {
49 |         this->blk_size = 4096;
50 |         this->blk_count = this->size / 4096;
51 |       } else if(this->size >= 1024) {
52 |         this->blk_size = 1024;
53 |         this->blk_count = this->size / 1024;
54 |       } else {
55 |         this->blk_size = this->size;
56 |         this->blk_count = 1;
57 |       }
58 |       this->depth = depth;
59 |       this->dir_next = this->dir_prev = NULL;
60 |     }
61 | 
62 |     int createFile();
63 |     int deleteFile();
64 |     int accessFile();
65 | 
66 |     void operator=(const File &f) {
67 |       size = f.size;
68 |       path = f.path;
69 |       age = f.age;
70 |       depth = f.depth;
71 |       blk_size = f.blk_size;
72 |       blk_count = f.blk_count;
73 |       prev = NULL;
74 |       next = NULL;
75 |       size_next = NULL;
76 |       size_prev = NULL;
77 |       dir_next = NULL;
78 |       dir_prev = NULL;
79 |     }
80 | 
81 |     bool operator<(File f) const {
82 |       if(this->age < f.age) {
83 |         return true;
84 |       }
85 |       return false;
86 |     }
87 | 
88 |     friend std::ostream& operator<< (std::ostream &out, const File &f) {
89 |       out << "(path = " << f.path << ", age = " << f.age << ", size = " <<
90 |         f.size << ", depth = " << f.depth << ")";
91 |       return out;
92 |     }
93 | };
94 | 
95 | #endif
96 | 


--------------------------------------------------------------------------------
/src/ThreadPool.h:
--------------------------------------------------------------------------------
  1 | #ifndef THREAD_POOL_H
  2 | #define THREAD_POOL_H
  3 | 
  4 | #include <vector>
  5 | #include <queue>
  6 | #include <memory>
  7 | #include <thread>
  8 | #include <mutex>
  9 | #include <condition_variable>
 10 | #include <future>
 11 | #include <functional>
 12 | #include <stdexcept>
 13 | 
 14 | class ThreadPool {
 15 |   public:
 16 |     // need to keep track of threads so we can join them
 17 |     std::vector< std::thread > workers;
 18 | 
 19 |     ThreadPool(size_t);
 20 |     template<class F, class... Args>
 21 |       auto enqueue(F&& f, Args&&... args) 
 22 |       -> std::future<typename std::result_of<F(Args...)>::type>;
 23 |     ~ThreadPool();
 24 |   private:
 25 |     // the task queue
 26 |     std::queue< std::function<void()> > tasks;
 27 | 
 28 |     // synchronization
 29 |     std::mutex queue_mutex;
 30 |     std::condition_variable condition;
 31 |     bool stop;
 32 | };
 33 | 
 34 | // the constructor just launches some amount of workers
 35 |   inline ThreadPool::ThreadPool(size_t threads)
 36 | :   stop(false)
 37 | {
 38 |   for(size_t i = 0;i<threads;++i)
 39 |     workers.emplace_back(
 40 |         [this]
 41 |         {
 42 |         for(;;)
 43 |         {
 44 |         std::function<void()> task;
 45 | 
 46 |         {
 47 |         std::unique_lock<std::mutex> lock(this->queue_mutex);
 48 |         this->condition.wait(lock,
 49 |           [this]{ return this->stop || !this->tasks.empty(); });
 50 |         if(this->stop && this->tasks.empty())
 51 |         return;
 52 |         task = std::move(this->tasks.front());
 53 |         this->tasks.pop();
 54 | 
 55 |         }
 56 | 
 57 |         task();
 58 |         }
 59 |         }
 60 |         );
 61 | }
 62 | 
 63 | // add new work item to the pool
 64 |   template<class F, class... Args>
 65 | auto ThreadPool::enqueue(F&& f, Args&&... args) 
 66 |   -> std::future<typename std::result_of<F(Args...)>::type>
 67 | {
 68 |   using return_type = typename std::result_of<F(Args...)>::type;
 69 | 
 70 |   auto task = std::make_shared< std::packaged_task<return_type()> >(
 71 |       std::bind(std::forward<F>(f), std::forward<Args>(args)...)
 72 |       );
 73 | 
 74 |   std::future<return_type> res = task->get_future();
 75 |   {
 76 |     std::unique_lock<std::mutex> lock(queue_mutex);
 77 | 
 78 |     // don't allow enqueueing after stopping the pool
 79 |     if(stop)
 80 |       throw std::runtime_error("enqueue on stopped ThreadPool");
 81 | 
 82 |     tasks.emplace([task](){ (*task)(); });
 83 |   }
 84 |   condition.notify_one();
 85 |   return res;
 86 | }
 87 | 
 88 | // the destructor joins all threads
 89 | inline ThreadPool::~ThreadPool()
 90 | {
 91 |   {
 92 |     std::unique_lock<std::mutex> lock(queue_mutex);
 93 |     stop = true;
 94 |   }
 95 |   condition.notify_all();
 96 |   for(std::thread &worker: workers)
 97 |     worker.join();
 98 |   if(!this->tasks.empty()) {
 99 |     std::runtime_error("pending tasks not completed in ThreadPool");
100 |   }
101 | }
102 | 
103 | #endif
104 | 


--------------------------------------------------------------------------------
/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | #
 2 | # CMakeLists.txt  top-level cmake file for geriatrix filesystem aging app
 3 | # 28-Oct-2016  chuck@ece.cmu.edu
 4 | #
 5 | 
 6 | cmake_minimum_required (VERSION 2.8)
 7 | project (geriatrix C CXX)
 8 | 
 9 | # we'll need to check for posix_fallocate
10 | include (CheckFunctionExists)
11 | 
12 | # geriatrix requires c++11
13 | set (CMAKE_CXX_STANDARD 11)
14 | set (CXX_STANDARD_REQUIRED True)
15 | 
16 | # link shared lib with full rpath
17 | set (CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
18 | 
19 | # setup cache variables for ccmake
20 | if (NOT CMAKE_BUILD_TYPE)
21 |     set (CMAKE_BUILD_TYPE Release
22 |          CACHE STRING "Choose the type of build." FORCE)
23 |     set_property (CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS 
24 |                   "Debug" "Release" "RelWithDebInfo" "MinSizeRel")
25 | endif ()
26 | set (DEBUG_SANITIZER Off CACHE STRING "Sanitizer for debug builds")
27 | set_property (CACHE DEBUG_SANITIZER PROPERTY STRINGS
28 |               "Off" "Address" "Thread")
29 | set (CMAKE_PREFIX_PATH "" CACHE STRING "External dependencies path")
30 | 
31 | #
32 | # sanitizer config (XXX: does not probe compiler to see if sanitizer flags
33 | # are supported... )
34 | #
35 | set (as_flags "-fsanitize=address -O1 -fno-omit-frame-pointer")
36 | set (ts_flags "-fsanitize=thread  -O1 -fno-omit-frame-pointer")
37 | if (${CMAKE_BUILD_TYPE} STREQUAL "Debug")
38 |     if (${DEBUG_SANITIZER} STREQUAL "Address")
39 |         set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${as_flags}")
40 |         set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${as_flags}")
41 |     elseif (${DEBUG_SANITIZER} STREQUAL "Thread")
42 |         set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${ts_flags}")
43 |         set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${ts_flags}")
44 |     endif ()
45 | endif ()
46 | 
47 | # geriatrix requires threads
48 | set (CMAKE_THREAD_PREFER_PTHREAD TRUE)
49 | set (THREADS_PREFER_PTHREAD_FLAG TRUE)
50 | find_package (Threads REQUIRED)
51 | list (APPEND geriatrix-depends "${CMAKE_THREAD_LIBS_INIT}")
52 | 
53 | # geriatrix requires boost
54 | set (Boost_NO_BOOST_CMAKE ON)
55 | find_package(Boost)
56 | if (NOT Boost_FOUND AND NOT BOOST_FOUND)
57 |     message ("geriatrix requires boost...  install boost and set")
58 |     message ("CMAKE_PREFIX_PATH to point to boost")
59 |     message (FATAL_ERROR "Aborting...")
60 | endif ()
61 | include_directories (${Boost_INCLUDE_DIRS})
62 | 
63 | # check for posix_fallocate
64 | check_function_exists (posix_fallocate HAS_POSIX_FALLOCATE)
65 | if (NOT HAS_POSIX_FALLOCATE)
66 |     add_definitions (-DNEED_POSIX_FALLOCATE)
67 | endif ()
68 | 
69 | # deltafs is an option
70 | if (DELTAFS)
71 |     find_package (deltafs CONFIG REQUIRED)
72 |     add_definitions (-DDELTAFS)
73 |     list (APPEND geriatrix-depends deltafs)
74 |     list (APPEND geriatrix-drivers src/deltafs_driver.c)
75 | endif ()
76 | 
77 | add_executable (geriatrix ${geriatrix-drivers} src/geriatrix.cpp)
78 | target_link_libraries (geriatrix ${geriatrix-depends})
79 | 
80 | install (TARGETS geriatrix RUNTIME DESTINATION bin)
81 | 


--------------------------------------------------------------------------------
/src/size_bucket.h:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (c) 2018 Carnegie Mellon University.
  3 |  *
  4 |  * All rights reserved.
  5 |  *
  6 |  * Use of this source code is governed by a BSD-style license that can be
  7 |  * found in the LICENSE file. See the AUTHORS file for names of contributors.
  8 |  */
  9 | 
 10 | #include "dir_bucket.h"
 11 | #include <boost/unordered_map.hpp>
 12 | 
 13 | #ifndef SIZE_BUCKET_
 14 | #define SIZE_BUCKET_
 15 | using namespace boost::unordered;
 16 | class SizeBucket {
 17 |   public:
 18 |     uint64_t count; // count of files of particular size
 19 |     File *start; // oldest file of particular size in bucket
 20 |     double ideal_fraction; // ideal fraction of total files in this bucket
 21 |     double actual_fraction; // current fraction of total files
 22 |     uint64_t size; // size of the files in this size bucket
 23 |     int id; // id of size bucket
 24 |     unordered_map<int, DirBucket> *db; // depth to DirBucket map
 25 |     size_t *size_arr;
 26 | 
 27 |     SizeBucket(uint64_t size, int id, size_t *size_arr) {
 28 |       this->size = size;
 29 |       this->size_arr = size_arr;
 30 |       count = 0;
 31 |       start = NULL;
 32 |       ideal_fraction = 0;
 33 |       actual_fraction = 0;
 34 |       this->id = id;
 35 |       db = NULL;
 36 |     }
 37 | 
 38 |     void operator=(const SizeBucket &b) {
 39 |       count = b.count;
 40 |       start = b.start;
 41 |       actual_fraction = b.actual_fraction;
 42 |       ideal_fraction = b.ideal_fraction;
 43 |       size = b.size;
 44 |       id = b.id;
 45 |       db = b.db;
 46 |       size_arr = b.size_arr;
 47 |     }
 48 | 
 49 |     void addFile(File *f, uint64_t live_file_count) {
 50 |       count++;
 51 | 
 52 |       auto d = (db->find(f->depth))->second;
 53 |       db->erase(f->depth);
 54 |       d.count++;
 55 |       if(d.count == 1) {
 56 |         d.start = f;
 57 |       }
 58 |       db->insert(std::pair<size_t, DirBucket>(f->depth, d));
 59 | 
 60 |       actual_fraction = ((double) count / live_file_count);
 61 |       if(start == NULL) {
 62 |         assert(count == 1);
 63 |         start = f;
 64 |         f->size_next = f;
 65 |         f->size_prev = f;
 66 |       } else {
 67 |         f->size_next = start;
 68 |         f->size_prev = start->size_prev;
 69 |         start->size_prev->size_next = f;
 70 |         start->size_prev = f;
 71 |       }
 72 |     }
 73 | 
 74 |     void deleteFile(File *f, uint64_t live_file_count) {
 75 |       /*
 76 |        * Steps in deleting file from SizeBucket
 77 |        * 1. adjust count.
 78 |        * 2. adjust actual fraction.
 79 |        * 3. adjust start if necessary.
 80 |        */
 81 |       count--;
 82 |       actual_fraction = ((double) count / live_file_count);
 83 | 
 84 |       auto d = (db->find(f->depth))->second;
 85 |       db->erase(f->depth);
 86 |       d.count--;
 87 |       if(d.count == 0) {
 88 |         d.start = NULL;
 89 |       } else if(d.start == f) {
 90 |         d.start = f->dir_next;
 91 |       }
 92 |       db->insert(std::pair<size_t, DirBucket>(f->depth, d));
 93 | 
 94 |       if(count == 0) {
 95 |         this->start = NULL;
 96 |       } else if(this->start == f) {
 97 |         this->start = f->size_next;
 98 |       }
 99 |       f->size_prev->size_next = f->size_next;
100 |       f->size_next->size_prev = f->size_prev;
101 |       f->size_next = f->size_prev = NULL;
102 |     }
103 | 
104 |     std::string replace(unordered_map<int, std::string>& size_bucket_keys) {
105 |       auto key = size_bucket_keys[this->id];
106 |       auto new_key = this->getKey();
107 |       size_bucket_keys[this->id] = new_key;
108 |       return key;
109 |     }
110 | 
111 |     std::string getKey() {
112 |       auto difference = this->ideal_fraction - this->actual_fraction;
113 |       return (std::to_string(size_arr[this->id]) + " " + std::to_string(difference));
114 |     }
115 | 
116 |     File *getFileToDelete(int depth) {
117 |       auto d = (db->find(depth))->second;
118 |       return d.getFileToDelete(depth);
119 |     }
120 | 
121 |     void reKey(uint64_t live_file_count, unordered_map<int, std::string>& size_bucket_keys) {
122 |       actual_fraction = ((double) count / live_file_count);
123 |       auto key = size_bucket_keys[this->id];
124 |       auto new_key = this->getKey();
125 |       size_bucket_keys[this->id] = new_key;
126 |     }
127 | 
128 | };
129 | #endif
130 | 


--------------------------------------------------------------------------------
/src/age_bucket.h:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (c) 2018 Carnegie Mellon University.
  3 |  *
  4 |  * All rights reserved.
  5 |  *
  6 |  * Use of this source code is governed by a BSD-style license that can be
  7 |  * found in the LICENSE file. See the AUTHORS file for names of contributors.
  8 |  */
  9 | 
 10 | #include "size_bucket.h"
 11 | 
 12 | #ifndef AGE_BUCKET_
 13 | #define AGE_BUCKET_
 14 | using namespace boost::unordered;
 15 | class AgeBucket {
 16 |   public:
 17 |     File *f; // pointer to start of file as per bucket cutoff
 18 |     unordered_map<size_t, SizeBucket> *sb;
 19 |     uint64_t count; // count of all files in bucket
 20 |     uint64_t cutoff; // cutoff for bucket
 21 |     double ideal_fraction; // ideal fraction of total files in this bucket
 22 |     double actual_fraction; // current fraction of total files
 23 |     bool youngest_bucket; // flag to indicate youngest bucket
 24 |     int id; // bucket id
 25 |     double ratio; // the ratio of what % of files
 26 |     File *last; // youngest file in bucket
 27 | 
 28 |     AgeBucket() {
 29 |       sb = NULL;
 30 |       count = 0;
 31 |       cutoff = 0;
 32 |       actual_fraction = 0;
 33 |       ideal_fraction = 0;
 34 |       youngest_bucket = false;
 35 |       id = 0;
 36 |       ratio = 0;
 37 |       f = NULL;
 38 |       last = NULL;
 39 |     }
 40 | 
 41 |     AgeBucket(int id) {
 42 |       sb = NULL;
 43 |       count = 0;
 44 |       cutoff = 0;
 45 |       actual_fraction = 0;
 46 |       ideal_fraction = 0;
 47 |       youngest_bucket = false;
 48 |       this->id = id;
 49 |       ratio = 0;
 50 |       f = NULL;
 51 |       last = NULL;
 52 |     }
 53 | 
 54 |     /*
 55 |      * This operator overloading is for the comparison of the buckets in the
 56 |      * flat_set.
 57 |      */
 58 |     bool operator<(AgeBucket b) const {
 59 |       auto v1 = ideal_fraction - actual_fraction;
 60 |       auto v2 = b.ideal_fraction - b.actual_fraction;
 61 |       if(v1 < 0) {
 62 |         v1 *= -1;
 63 |       }
 64 |       if(v2 < 0) {
 65 |         v2 *= -1;
 66 |       }
 67 |       if(v1 < v2) {
 68 |         return true;
 69 |       }
 70 |       return false;
 71 |     }
 72 | 
 73 |     void operator=(const AgeBucket &b) {
 74 |       count = b.count;
 75 |       cutoff = b.cutoff;
 76 |       actual_fraction = b.actual_fraction;
 77 |       ideal_fraction = b.ideal_fraction;
 78 |       sb = b.sb;
 79 |       f = b.f;
 80 |       youngest_bucket = b.youngest_bucket;
 81 |       id = b.id;
 82 |       ratio = b.ratio;
 83 |       last = b.last;
 84 |     }
 85 | 
 86 |     void addFile(File *f, uint64_t live_file_count, bool at_front) {
 87 |       count++;
 88 |       auto s = (sb->find(f->size))->second;
 89 |       sb->erase(f->size);
 90 | 
 91 |       auto d = (s.db->find(f->depth))->second;
 92 |       s.db->erase(f->depth);
 93 |       d.addFile(f, live_file_count);
 94 |       s.db->insert(std::pair<int, DirBucket>(f->depth, d));
 95 | 
 96 |       s.count++;
 97 |       if(s.count == 1) {
 98 |         assert(s.start == NULL);
 99 |         s.start = f;
100 |       }
101 |       sb->insert(std::pair<size_t, SizeBucket>(f->size, s));
102 |       actual_fraction = ((double) count / live_file_count);
103 | 
104 |       if(this->f == NULL) {
105 |         this->f = f;
106 |         assert(this->last == NULL);
107 |         this->last = f;
108 |       }
109 | 
110 |       if(at_front) {
111 |         //FIXME: probably assert to see if f->next = this->f
112 |         this->f = f;
113 |       } else {
114 |         this->last = f;
115 |       }
116 |     }
117 | 
118 |     void deleteFile(File *f, uint64_t live_file_count) {
119 |       /*
120 |        * Steps in deleting file from AgeBucket.
121 |        * 1. adjust count.
122 |        * 2. adjust actual_fraction.
123 |        * 3. remove file from size bucket.
124 |        * 4. adjust f if necessary.
125 |        * 5. adjust last if necessary.
126 |        */
127 |       this->count--;
128 |       actual_fraction = ((double) count / live_file_count);
129 |       auto s = (sb->find(f->size))->second;
130 |       sb->erase(f->size);
131 | 
132 |       auto d = (s.db->find(f->depth))->second;
133 |       s.db->erase(f->depth);
134 |       d.deleteFile(f, live_file_count);
135 |       s.db->insert(std::pair<int, DirBucket>(f->depth, d));
136 | 
137 |       s.count--;
138 |       if(s.count == 0) {
139 |         s.start = NULL;
140 |       } else if(s.start == f) {
141 |         s.start = f->size_next;
142 |       }
143 |       sb->insert(std::pair<size_t, SizeBucket>(f->size, s));
144 |       if(count == 0) {
145 |         this->f = NULL;
146 |         this->last = NULL;
147 |       } else if(this->f == f) {
148 |         this->f = f->next;
149 |       }
150 | 
151 |       if((this->last == f) && (count > 0)) {
152 |         this->last = f->prev;
153 |       }
154 |     }
155 | 
156 |     std::string replace(unordered_map<int, std::string>& age_bucket_keys) {
157 |       auto key = age_bucket_keys[this->id];
158 |       auto new_key = this->getKey();
159 |       age_bucket_keys[this->id] = new_key;
160 |       return key;
161 |     }
162 | 
163 |     std::string getKey() {
164 |       auto difference = this->actual_fraction - this->ideal_fraction;
165 |       return (std::to_string(this->id) + " " + std::to_string(difference));
166 |     }
167 | 
168 |     File * getFileToDelete(size_t size, int dir) {
169 |       auto s = (sb->find(size))->second;
170 |       return s.getFileToDelete(dir);
171 |     }
172 | };
173 | #endif
174 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Geriatrix
  2 | 
  3 | Geriatrix is a simple, reproducible file system aging suite that induces
  4 | adequate amount of file and free space fragmentation by performing a series of
  5 | controlled file creates and deletes. Geriatrix is profile-driven and has 8
  6 | built-in aging profiles to facilitate aging.
  7 | 
  8 | Geriatrix has been [published in USENIX ATC 2018](http://cs.cmu.edu/~saukad/geriatrix.pdf).
  9 | 
 10 | ## Building
 11 | 
 12 | Geriatrix requires the following to compile:
 13 | * a c++11 C++ compiler
 14 | * the Boost C++ libraries (http://www.boost.org/)
 15 | 
 16 | It should compile on most POSIX systems that have these prerequisite.
 17 | 
 18 | Geriatrix can be compiled with cmake (http://cmake.org/) or by
 19 | hand editing a basic Makefile.
 20 | 
 21 | To compile Geriatrix with cmake: make a build directory, change
 22 | to that directory, run cmake to generate the build, and then 
 23 | run "make" (and optionally "make install").  To set the 
 24 | installation directory, specify CMAKE_INSTALL_PREFIX on the 
 25 | command line.  If Boost is installed in a non-standard location,
 26 | specify it using CMAKE_PREFIX_PATH.
 27 | 
 28 | Here is an example of building Geriatrix to be installed in
 29 | /opt/bin using a Boost installed in /pkg:
 30 | ```
 31 | cd geriatrix
 32 | mkdir build
 33 | cd build
 34 | cmake -DCMAKE_INSTALL_PREFIX=/opt -DCMAKE_PREFIX_PATH=/pkg ..
 35 | make
 36 | make install
 37 | ```
 38 | 
 39 | ## Parameters
 40 | 
 41 | Geriatrix has a few parameters and setting them correctly is important to
 42 | ensure correct aging. The parameter definitions are:
 43 | 
 44 | - -n: size of your disk image in bytes (total partition size of file system)
 45 | - -u: fullness percentage of file system. Somewhere around 80% is reasonable.
 46 |   Note that this is a fractional value, i.e. 80% full would mean 0.8
 47 | - -r: random seed for a Geriatrix run
 48 | - -m: mount point without trailing / (eg /mnt and not /mnt/)
 49 | - -a: path to the age distribution file from the input aging profile
 50 | - -s: path to the size distribution file from the input aging profile
 51 | - -d: path to the directory depth distribution file from the input aging profile
 52 | - -x: path to write the output of age distribution of file system after aging
 53 | - -y: path to write the output of size distribution of file system after aging
 54 | - -z: path to write the output of dir depth distribution of file system after
 55 |   aging
 56 | - -t: number of threads you want to use for aging. This is a tricky parameter
 57 |   because multithreading might not be able to reproduce exact sequence of file
 58 |   creates / deletes because of the randomness in how the threads are scheduled
 59 | - -i: number of runs over the file system image size. 100 means that Geriatrix
 60 |   will execute until aging workload is equal to (100 * size of file system
 61 |   image)
 62 | - -f: fake mode. This is a mode to just test an aging profile without actually
 63 |   performing file creates or deletes. Essentially just data structure
 64 |   manipulation. Usually you should keep this value as 0 unless you are testing
 65 |   a new aging profile.
 66 | - -p: inject idle time between operations. This is a work-in-progress.
 67 | - -c: confidence interval. If you don't want to wait till perfect aging, you
 68 |   can specify a value between 0 and 1 for a notion of accuracy. 0.9 is aging
 69 |   done with upto 10% error. A value of 0 implies perfect aging.
 70 | - -q: query before quitting. Specifying 1 here will make Geriatrix ask you
 71 |   whether you want to continue aging beyond whatever has been performed.
 72 | - -w: running time limit in mins. Geriatrix will run till either -i parameter
 73 |   is reached or -w parameter is reached, whatever comes first. A -q value of 1
 74 |   will make Geriatrix wait for user input before quitting.
 75 | - -b: backend. Geriatrix supports multiple backends. This should be kept as
 76 |   "posix" (assuming you are benchmarking a posix compliant file system).
 77 | 
 78 | ## Running
 79 | ```
 80 | ./bin/geriatrix -n 21474836480 -u 0.8 -r 42 -m /mnt -a ./profiles/agrawal/age_distribution.txt -s ./profiles/agrawal/size_distribution.txt -d ./profiles/agrawal/dir_distribution.txt -x /tmp/age.out -y /tmp/size.out -z /tmp/dir.out -t 1 -i 1000 -f 0 -p 0 -c 0 -q 1 -w 2880 -b posix
 81 | ```
 82 | The above example shows a 20GB file system image being aged using the built-in
 83 | Agrawal aging profile. The file system is mounted at /mnt. Geriatrix has been
 84 | asked to stop aging after executing a workload of 20TB (1000 * FS image size)
 85 | or 2 days (2880 min), whichever comes first. Since the q parameter is 1,
 86 | Geriatrix will ask if aging should be continued after reaching one of these
 87 | thresholds. The c parameter being 0 specifies that Geriatrix will not quit
 88 | until it reaches perfect relative age distrubtion convergence (refer paper for
 89 | details about what this means).
 90 | 
 91 | ## Output
 92 | 
 93 | After Geriatrix finishes aging (say using the above command), an output similar
 94 | to the following should be seen. Note that this is just an example. Your
 95 | numbers may be different:
 96 | ```
 97 | Aging stopped because perfect convergence was achieved in input distributions.
 98 | ============= OVERALL STATISTICS ===============
 99 |  Total runtime = 300.4352 mins.
100 |  Total number of operations = 1485978
101 |  Number of disk overwrites = 1000
102 |  Total aging workload created = 20971520 MB
103 |  Perfect convergence achieved
104 |  Size distribution dumped in /tmp/size.out
105 |  Dir depth distribution dumped in /tmp/dir.out
106 |  Age distribution dumped in /tmp/age.out
107 | ================================================
108 | ```
109 | 
110 | ## Contact
111 | 
112 | In case of issues or questions, please email saukad@cs.cmu.edu.
113 | 


--------------------------------------------------------------------------------
/src/dir_bucket.h:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright (c) 2018 Carnegie Mellon University.
  3 |  *
  4 |  * All rights reserved.
  5 |  *
  6 |  * Use of this source code is governed by a BSD-style license that can be
  7 |  * found in the LICENSE file. See the AUTHORS file for names of contributors.
  8 |  */
  9 | 
 10 | #include <boost/unordered_map.hpp>
 11 | #include <boost/random.hpp>
 12 | 
 13 | #include "file.h"
 14 | 
 15 | #ifndef DIR_BUCKET_
 16 | #define DIR_BUCKET_
 17 | static int global_live_depth = 0;
 18 | 
 19 | using namespace boost::unordered;
 20 | struct DirBucket {
 21 |   public:
 22 |     uint64_t count; // count of files of particular size
 23 |     double ideal_fraction; // ideal fraction of total files in this bucket
 24 |     double actual_fraction; // current fraction of total files
 25 |     int id; // id of dir bucket
 26 |     int depth; // depth of the dir (root has depth 0)
 27 |     File *start; // oldest file of particular depth
 28 |     std::string prefix; // prefix path for this dir bucket
 29 |     uint64_t sibling_dirs; // count of number of dirs at this level
 30 |     int *dir_arr; // dir arr from input
 31 | 
 32 |     DirBucket(int id) {
 33 |       count = 0;
 34 |       ideal_fraction = 0.0;
 35 |       actual_fraction = 0.0;
 36 |       this->id = id;
 37 |       depth = 0;
 38 |       sibling_dirs = 0;
 39 |     }
 40 | 
 41 |     DirBucket(int depth, uint32_t sibling_dirs, int id, std::string
 42 |               mount_point, int fake, int *dir_arr,
 43 |               int (*mkpath)(const char *, mode_t)) {
 44 |       count = 0;
 45 |       ideal_fraction = 0.0;
 46 |       actual_fraction = 0.0;
 47 |       this->id = id;
 48 |       this->depth = depth;
 49 |       this->sibling_dirs = sibling_dirs;
 50 |       this->dir_arr = dir_arr;
 51 |       prefix = "";
 52 |       auto slash = "/";
 53 |       if(this->depth > 0) {
 54 |         std::string dirs = "";
 55 |         for(auto i=1; i<=this->depth-1; i++) {
 56 |           if(i != 1) {
 57 |             dirs += "/";
 58 |           }
 59 |           dirs += "d" + std::to_string(i);
 60 |         }
 61 |         prefix = dirs;
 62 |         if(sibling_dirs == 0) {
 63 |           if(global_live_depth < this->depth) {
 64 | 
 65 |             std::string full_file_path = mount_point + slash + prefix +
 66 |                 "/d" + std::to_string(this->depth);
 67 |             prefix += "/d" + std::to_string(this->depth);
 68 |             if(!fake) {
 69 |                 int rv = (*mkpath)(full_file_path.c_str(), 0777);
 70 |                 assert(rv == 0);
 71 |             }
 72 | 
 73 |           }
 74 |         } else {
 75 |           for(uint32_t j=1; j<=sibling_dirs; j++) {
 76 |             if(this->depth == 1) {
 77 |               slash = "";
 78 |             }
 79 |             if(global_live_depth < this->depth) {
 80 |               std::string full_file_path = mount_point + slash +
 81 |                 prefix + "/d" + std::to_string(j);
 82 |               if(!fake) {
 83 |                 int rv = (*mkpath)(full_file_path.c_str(), 0777);
 84 |                 assert(rv == 0);
 85 |               }
 86 |             }
 87 |           }
 88 |         }
 89 |         global_live_depth = this->depth;
 90 |       }
 91 |       start = NULL;
 92 |     }
 93 | 
 94 |     void operator=(const DirBucket &b) {
 95 |       count = b.count;
 96 |       actual_fraction = b.actual_fraction;
 97 |       ideal_fraction = b.ideal_fraction;
 98 |       start = b.start;
 99 |       id = b.id;
100 |       depth = b.depth;
101 |       prefix = b.prefix;
102 |       sibling_dirs = b.sibling_dirs;
103 |     }
104 | 
105 |     std::string getKey() {
106 |       auto difference = this->actual_fraction - this->ideal_fraction;
107 |       return (std::to_string(dir_arr[this->id]) + " " +
108 |           std::to_string(difference));
109 |     }
110 | 
111 |     std::string replace(unordered_map<int, std::string>& dir_bucket_keys) {
112 |       auto key = dir_bucket_keys[this->id];
113 |       auto new_key = this->getKey();
114 |       dir_bucket_keys[this->id] = new_key;
115 |       return key;
116 |     }
117 | 
118 |     void addFile(File *f, uint64_t live_file_count) {
119 |       this->count++;
120 |       actual_fraction = ((double) count / live_file_count);
121 |       if(start == NULL) {
122 |         assert(count == 1);
123 |         start = f;
124 |         f->dir_next = f;
125 |         f->dir_prev = f;
126 |       } else {
127 |         assert(count > 1);
128 |         f->dir_next = start;
129 |         f->dir_prev = start->dir_prev;
130 |         start->dir_prev->dir_next = f;
131 |         start->dir_prev = f;
132 |       }
133 |     }
134 | 
135 |     void deleteFile(File *f, uint64_t live_file_count) {
136 |       assert(this->count > 0);
137 |       this->count--;
138 |       actual_fraction = ((double) count / live_file_count);
139 |       if(this->count == 0) {
140 |         this->start = NULL;
141 |       } else if(this->start == f) {
142 |         this->start = f->dir_next;
143 |       }
144 |       f->dir_prev->dir_next = f->dir_next;
145 |       f->dir_next->dir_prev = f->dir_prev;
146 |       f->dir_next = f->dir_prev = NULL;
147 |     }
148 | 
149 |     File *getFileToDelete(int depth) {
150 |       if(this->count == 0) {
151 |         return NULL;
152 |       }
153 |       boost::random::mt19937 gen{static_cast<std::uint32_t>(this->count)};
154 |       boost::random::uniform_int_distribution<> dist{1,
155 |         static_cast<int>(this->count)};
156 |       auto rand = dist(gen);
157 |       auto f = this->start;
158 |       for(auto i=1; i<rand; i++) {
159 |         f = f->dir_next;
160 |       }
161 |       return f;
162 |     }
163 | 
164 |     void reKey(uint64_t live_file_count, unordered_map<int,
165 |         std::string>&dir_bucket_keys) {
166 |       actual_fraction = ((double) count / live_file_count);
167 |       auto key = dir_bucket_keys[this->id];
168 |       auto new_key = this->getKey();
169 |       dir_bucket_keys[this->id] = new_key;
170 |     }
171 | };
172 | #endif
173 | 


--------------------------------------------------------------------------------
/src/geriatrix.cpp:
--------------------------------------------------------------------------------
   1 | /*
   2 |  * Copyright (c) 2018 Carnegie Mellon University.
   3 |  *
   4 |  * All rights reserved.
   5 |  *
   6 |  * Use of this source code is governed by a BSD-style license that can be
   7 |  * found in the LICENSE file. See the AUTHORS file for names of contributors.
   8 |  */
   9 | 
  10 | #include "geriatrix.h"
  11 | 
  12 | #ifdef NEED_POSIX_FALLOCATE
  13 | /*
  14 |  * fake posix_fallocate by ftruncating the file larger and touching
  15 |  * a byte in each block.... returns 0 on success, errno on fail(!!!)
  16 |  * (this is at the top of the file so it can be included in the
  17 |  * posix driver if needed...)
  18 |  */
  19 | static int posix_fallocate(int fd, off_t offset, off_t len) {
  20 |     struct stat st;
  21 |     off_t newlen, curoff, lastoff, ptr;
  22 |     ssize_t rv;
  23 | 
  24 |     newlen = offset + len;
  25 | 
  26 |     if (fstat(fd, &st) < 0)
  27 |         return(errno);
  28 | 
  29 |     if (st.st_size > newlen)        /* not growing it, assume ok */
  30 |         return(0);
  31 | 
  32 |     if (ftruncate(fd, newlen) < 0)   /* grow it */
  33 |         return(errno);
  34 | 
  35 |     curoff = ((st.st_size + (st.st_blksize-1)) / st.st_blksize) * st.st_blksize;
  36 |     lastoff = ((newlen + (st.st_blksize-1)) / st.st_blksize) * st.st_blksize;
  37 | 
  38 |     for (ptr = curoff ; ptr < lastoff ; ptr += st.st_blksize) {
  39 |         if (lseek(fd, ptr, SEEK_SET) < 0)
  40 |             return(errno);
  41 |         rv = write(fd, "", 1);    /* writes a null */
  42 |         if (rv < 0)
  43 |             return(errno);
  44 |         if (rv == 0)
  45 |             return(EIO);
  46 |     }
  47 | 
  48 |     return(0);
  49 | }
  50 | #endif
  51 | 
  52 | /*
  53 |  * backend configuration -- all filesystem aging I/O is routed here!
  54 |  */
  55 | 
  56 | /* posix driver (the default) */
  57 | static struct backend_driver posix_backend_driver = {
  58 |     open, close, write, access, unlink, mkdir, posix_fallocate, stat, chmod,
  59 | };
  60 | 
  61 | #ifdef DELTAFS     /* optional backend for cmu's deltafs */
  62 | extern struct backend_driver deltafs_backend_driver;
  63 | #endif
  64 | 
  65 | /* g_backend is the backend we are using (default=posix) */
  66 | static struct backend_driver *g_backend = &posix_backend_driver;
  67 | 
  68 | /*
  69 |  * mkdir_path(path,mode): make an entire path(ala "mkdir -p").
  70 |  * ret 0 on sucess, -1 on error (errno set by mkdir).
  71 |  */
  72 | static int mkdir_path(const char *path, mode_t mode) {
  73 |   char *pcopy, *slash;
  74 |   mode_t parentmode;
  75 |   int done, olderrno, rv;
  76 |   struct stat st;
  77 | 
  78 |   /* make a copy of it, since we change it (its from a c++ string) */
  79 |   slash = pcopy = strdup(path);
  80 |   if (pcopy == NULL)
  81 |     return(-1);
  82 |   parentmode = mode | S_IWUSR | S_IXUSR;
  83 | 
  84 |   for (done = 0 ; done == 0 ; /*null*/ ) {
  85 |     slash += strspn(slash, "/");   /* first char that is not a "/" */
  86 |     slash += strcspn(slash, "/");  /* next "/" or end of string */
  87 |     if (*slash == '\0') done = 1;  /* hit last directory? */
  88 | 
  89 |     *slash = '\0';
  90 |     rv = g_backend->bd_mkdir(pcopy, done ? mode : parentmode);
  91 |     if (rv < 0) {
  92 |       olderrno = errno;
  93 |       if (g_backend->bd_stat(pcopy, &st) < 0) {   /* not there */
  94 |         errno = olderrno;
  95 |         done = -1;
  96 |         break;
  97 |       }
  98 |       if (!S_ISDIR(st.st_mode)) {
  99 |         errno = ENOTDIR;
 100 |         done = -1;
 101 |         break;
 102 |       }
 103 |       /* could have already be there */
 104 |     } else if (done) {    /* final directory created, apply mode */
 105 |       /* needed if trying to set setuid/setgid/sticky bits */
 106 |       if ((mode & ~(S_IRWXU|S_IRWXG|S_IRWXO)) != 0 &&
 107 |         g_backend->bd_chmod(path, mode) == -1) {
 108 |         done = -1;
 109 |         break;
 110 |       }
 111 |     }
 112 |     if (!done) *slash = '/';
 113 |   }
 114 | 
 115 |   free(pcopy);
 116 |   return( done < 0 ? -1 : 0);
 117 | }
 118 | 
 119 | void issueCreate(const char *path, size_t len) {
 120 |   int fd, rv = 1;
 121 |   fd = g_backend->bd_open(path, O_RDWR|O_CREAT, 0600);
 122 |   assert(fd > -1);
 123 |   if(len > 0) {
 124 |     do {
 125 |       if(rv < 0) {
 126 |         sleep(1);
 127 |       }
 128 |       rv = g_backend->bd_fallocate(fd, 0, len);
 129 |     } while(rv != 0);
 130 |     if(rv != 0) {
 131 |       fprintf(stderr,
 132 |           "issueCreate: fallocate(%s): %s with params fd: %d, len:%lu\n",
 133 |           path, strerror(rv), fd, len);
 134 |       abort();
 135 |     }
 136 |   }
 137 |   rv = g_backend->bd_close(fd);
 138 |   assert(rv == 0);
 139 |   return;
 140 | }
 141 | 
 142 | void issueAccess(const char *path) {
 143 |   int retval = 0;
 144 |   do {
 145 |     retval = g_backend->bd_access(path, F_OK);
 146 |     if(retval == -1) {
 147 |       if(errno == EACCES || errno == ENOENT) {
 148 |         continue;
 149 |       } else {
 150 |         assert(0);
 151 |       }
 152 |     }
 153 |     return;
 154 |   } while(1);
 155 | }
 156 | 
 157 | void issueDelete(const char *path) {
 158 |   int rv;
 159 |   issueAccess(path);
 160 |   rv = g_backend->bd_unlink(path);
 161 |   assert(rv == 0);
 162 |   return;
 163 | }
 164 | 
 165 | /**
 166 |  * Create a file.
 167 |  */
 168 | int File::createFile() {
 169 |   std::string slash = "";
 170 |   if(depth > 1) {
 171 |     slash = "/";
 172 |   }
 173 |   std::string path = mount_point + slash + this->path;
 174 |   size_t size = this->blk_size * this->blk_count;
 175 |   if(!fake)
 176 |     pool->enqueue([path, size] { issueCreate(path.c_str(), size); });
 177 |   return 0;
 178 | }
 179 | 
 180 | /**
 181 |  * Access file
 182 |  */
 183 | int File::accessFile() {
 184 |   std::string slash = "";
 185 |   if(depth != 0) {
 186 |     slash = "/";
 187 |   }
 188 |   std::string full_path = mount_point + slash + this->path;
 189 |   int retval = g_backend->bd_access(full_path.c_str(), F_OK);
 190 |   if(retval == -1) {
 191 |     if(errno == EACCES || errno == ENOENT) {
 192 |       return -1;
 193 |     } else {
 194 |       assert(0);
 195 |     }
 196 |   }
 197 |   return 0;
 198 | }
 199 | 
 200 | /**
 201 |  * Delete a file.
 202 |  */
 203 | int File::deleteFile() {
 204 |   std::string slash = "";
 205 |   if(depth != 0) {
 206 |     slash = "/";
 207 |   }
 208 |   std::string full_path = mount_point + slash + this->path;
 209 |   if(!fake)
 210 |     pool->enqueue([full_path] { issueDelete(full_path.c_str()); });
 211 |   return 0;
 212 | }
 213 | 
 214 | struct BucketCompare {
 215 |   bool operator()(const std::string& lhs, const std::string& rhs) const {
 216 |     double v1 = 0, v2 = 0;
 217 |     long l_id = -1, r_id = -1;
 218 |     int i = 0;
 219 |     boost::char_separator<char> sep(" ");
 220 |     boost::tokenizer<boost::char_separator<char>> tok_l(lhs, sep);
 221 |     for (auto it = tok_l.begin(); it != tok_l.end(); ++it) {
 222 |       if(i == 0) {
 223 |         l_id = std::stol(*it);
 224 |         i++;
 225 |         continue;
 226 |       }
 227 |       v1 = std::stod(*it);
 228 |       break;
 229 |     }
 230 |     boost::tokenizer<boost::char_separator<char>> tok_r(rhs, sep);
 231 |     i = 0;
 232 |     for (auto it = tok_r.begin(); it != tok_r.end(); ++it) {
 233 |       if(i == 0) {
 234 |         r_id = std::stol(*it);
 235 |         i++;
 236 |         continue;
 237 |       }
 238 |       v2 = std::stod(*it);
 239 |       break;
 240 |     }
 241 |     if(v1 < v2) {
 242 |       return true;
 243 |     } else if ((v1 == v2) && (l_id < r_id)) {
 244 |       return true;
 245 |     } else if ((v1 == v2) && (l_id > r_id)) {
 246 |       return false;
 247 |     }
 248 |     return false;
 249 |   }
 250 | };
 251 | 
 252 | AgeList *global_file_list;
 253 | flat_map<std::string, AgeBucket, BucketCompare> age_buckets;
 254 | flat_map<std::string, SizeBucket, BucketCompare> *size_buckets;
 255 | flat_map<std::string, DirBucket, BucketCompare> *dir_buckets;
 256 | 
 257 | void readDistribution(void *input, distribution_type_t type) {
 258 |   int count = 0;
 259 |   struct dir *d = NULL;
 260 |   struct age *a = NULL;
 261 |   struct size *s = NULL;
 262 |   switch(type) {
 263 |     case DIRS: {
 264 |                  d = (struct dir *) input;
 265 |                  std::ifstream infile(d->in_file);
 266 |                  infile >> count;
 267 |                  NUM_DIRS = count;
 268 |                  d->arr = (int *) malloc(sizeof(int) * count);
 269 |                  d->distribution = (double *) malloc(sizeof(double) * count);
 270 |                  d->subdir_arr = (uint32_t *) malloc (sizeof(uint32_t) * count);
 271 |                  for(auto i=0; i<count; i++) {
 272 |                    infile >> d->arr[i];
 273 |                    infile >> d->distribution[i];
 274 |                    infile >> d->subdir_arr[i];
 275 |                  }
 276 |                } break;
 277 | 
 278 |     case SIZES: {
 279 |                   s = (struct size *) input;
 280 |                   std::ifstream infile(s->in_file);
 281 |                   infile >> count;
 282 |                   NUM_SIZES = count;
 283 |                   s->arr = (size_t *) malloc(sizeof(size_t) * count);
 284 |                   s->cutoffs = (double *) malloc(sizeof(double) * count);
 285 |                   s->distribution = (double *) malloc(sizeof(double) * count);
 286 |                   for(auto i=0; i<count; i++) {
 287 |                     infile >> s->arr[i];
 288 |                     infile >> s->distribution[i];
 289 |                     if(i == 0) {
 290 |                       s->cutoffs[i] = s->distribution[i];
 291 |                     } else {
 292 |                       s->cutoffs[i] = s->cutoffs[i-1] + s->distribution[i];
 293 |                     }
 294 |                   }
 295 |                 } break;
 296 | 
 297 |     case AGES: {
 298 |                  a = (struct age *) input;
 299 |                  std::ifstream infile(a->in_file);
 300 |                  infile >> count;
 301 |                  NUM_AGES = count;
 302 |                  a->distribution = (double *) malloc(sizeof(double) * count);
 303 |                  a->cutoffs = (double *) malloc(sizeof(double) * count);
 304 |                  for(auto i=0; i<count; i++) {
 305 |                    infile >> a->cutoffs[i];
 306 |                    infile >> a->distribution[i];
 307 |                  }
 308 |                } break;
 309 |   }
 310 | }
 311 | 
 312 | void init(struct age *a_grp, struct size *s_grp, struct dir *d_grp) {
 313 |   int i, j, k;
 314 |   readDistribution(a_grp, AGES);
 315 |   readDistribution(s_grp, SIZES);
 316 |   readDistribution(d_grp, DIRS);
 317 |   global_file_list = new AgeList(0);
 318 | 
 319 |   for(i=0; i<NUM_AGES; i++) {
 320 |     total_age_weight += a_grp->distribution[i];
 321 |   }
 322 | 
 323 |   for(i=0; i<NUM_SIZES; i++) {
 324 |     total_size_weight += s_grp->distribution[i];
 325 |   }
 326 | 
 327 |   for(i=0; i<NUM_DIRS; i++) {
 328 |     total_dir_weight += d_grp->distribution[i];
 329 |   }
 330 | 
 331 |   size_buckets = new flat_map<std::string, SizeBucket, BucketCompare>;
 332 |   for(i=0; i<NUM_SIZES; i++) {
 333 |     SizeBucket s(s_grp->arr[i], i, s_grp->arr);
 334 |     s.db = new unordered_map<int, DirBucket>();
 335 |     for(j=0; j<NUM_DIRS; j++) {
 336 |       DirBucket d(d_grp->arr[j], d_grp->subdir_arr[j], j,
 337 |                   mount_point, fake, d_grp->arr, mkdir_path);
 338 |       s.db->insert(std::pair<int, DirBucket>(d_grp->arr[j], d));
 339 |     }
 340 |     s.ideal_fraction = s_grp->distribution[i] / total_size_weight;
 341 |     s_grp->bucket_keys[i] = s.getKey();
 342 |     size_buckets->insert(std::pair<std::string,
 343 |         SizeBucket>(s_grp->bucket_keys[i], s));
 344 |   }
 345 | 
 346 |   for(i=0; i<NUM_AGES; i++) {
 347 |     AgeBucket b(i);
 348 |     b.sb = new unordered_map<size_t, SizeBucket>();
 349 |     b.f = NULL;
 350 |     for(j=0; j<NUM_SIZES; j++) {
 351 |       SizeBucket s(s_grp->arr[j], j, s_grp->arr);
 352 |       s.db = new unordered_map<int, DirBucket>();
 353 |       for(k=0; k<NUM_DIRS; k++) {
 354 |         DirBucket d(d_grp->arr[k], d_grp->subdir_arr[k], k, mount_point,
 355 |                     fake, d_grp->arr, mkdir_path);
 356 |         s.db->insert(std::pair<int, DirBucket>(d_grp->arr[k], d));
 357 |       }
 358 |       b.sb->insert(std::pair<size_t, SizeBucket>(s_grp->arr[j], s));
 359 |     }
 360 |     b.ideal_fraction = a_grp->distribution[i] / total_age_weight;
 361 |     if(i == 0) {
 362 |       b.youngest_bucket = true;
 363 |     }
 364 |     b.ratio = 1 - (a_grp->cutoffs[i] / a_grp->cutoffs[NUM_AGES-1]);
 365 |     a_grp->bucket_keys[i] = b.getKey();
 366 |     age_buckets.insert(std::pair<std::string,
 367 |         AgeBucket>(a_grp->bucket_keys[i], b));
 368 |   }
 369 | 
 370 |   dir_buckets = new flat_map<std::string, DirBucket, BucketCompare>;
 371 |   for(i=0; i<NUM_DIRS; i++) {
 372 |     DirBucket d(d_grp->arr[i], d_grp->subdir_arr[i], i, mount_point,
 373 |                 fake, d_grp->arr, mkdir_path);
 374 |     d.ideal_fraction = d_grp->distribution[i] / total_dir_weight;
 375 |     d_grp->bucket_keys[i] = d.getKey();
 376 |     dir_buckets->insert(std::pair<std::string,
 377 |         DirBucket>(d_grp->bucket_keys[i], d));
 378 |   }
 379 | }
 380 | 
 381 | void destroy() {
 382 |   delete dist;
 383 |   delete pool;
 384 |   delete global_file_list;
 385 |   //ProfilerStop();
 386 | }
 387 | 
 388 | void dumpSizeBuckets(char *f = NULL) {
 389 |   FILE *fp = NULL;
 390 |   if(f) {
 391 |     fp = fopen(f, "w");
 392 |     fprintf(fp, "SIZE FRACTION TYPE\n");
 393 |   }
 394 | 
 395 |   if(f) {
 396 |     auto current_id = 0;
 397 |     while(current_id < NUM_SIZES) {
 398 |       auto it = size_buckets->rbegin();
 399 |       auto s = it->second;
 400 |       while(s.id != current_id) {
 401 |         it++;
 402 |         s = it->second;
 403 |       }
 404 |       fprintf(fp, "%" PRIu64 " %f IDEAL\n", s.size, s.ideal_fraction);
 405 |       fprintf(fp, "%" PRIu64 " %f ACTUAL\n", s.size, s.actual_fraction);
 406 |       current_id++;
 407 |     }
 408 |     fclose(fp);
 409 |   }
 410 | }
 411 | 
 412 | void dumpDirBuckets(char *f = NULL) {
 413 |   FILE *fp = NULL;
 414 |   if(f) {
 415 |     fp = fopen(f, "w");
 416 |     fprintf(fp, "DEPTH FRACTION TYPE\n");
 417 |   }
 418 | 
 419 |   if(f) {
 420 |     auto current_id = 0;
 421 |     while(current_id < NUM_DIRS) {
 422 |       auto it = dir_buckets->rbegin();
 423 |       auto d = it->second;
 424 |       while(d.id != current_id) {
 425 |         it++;
 426 |         d = it->second;
 427 |       }
 428 |       fprintf(fp, "%d %f IDEAL\n", d.depth, d.ideal_fraction);
 429 |       fprintf(fp, "%d %f ACTUAL\n", d.depth, d.actual_fraction);
 430 |       current_id++;
 431 |     }
 432 |     fclose(fp);
 433 |   }
 434 | }
 435 | 
 436 | double calculateChiMeanSquared(std::list<double> expected,
 437 |     std::list<double> actual, char *age_dump_file,
 438 |     char *size_dump_file, char *dir_dump_file) {
 439 |   double chi_2 = 0.0;
 440 |   for(auto e_it = expected.begin(), a_it = actual.begin();
 441 |       e_it != expected.end(); e_it++, a_it++) {
 442 |     chi_2 += pow(double((*e_it - *a_it)), 2) / *e_it;
 443 |   }
 444 | 
 445 |   auto goodness_of_my_dist = cdf(*dist, chi_2);
 446 |   if(goodness_of_my_dist <= goodness_measure) {
 447 |     dumpSizeBuckets(size_dump_file);
 448 |     dumpDirBuckets(dir_dump_file);
 449 |     auto current_id = 0;
 450 |     auto fp = fopen(age_dump_file, "w");
 451 |     fprintf(fp, "BUCKET FRACTION TYPE\n");
 452 |     while(current_id < NUM_AGES) {
 453 |       auto it = age_buckets.rbegin();
 454 |       auto a = it->second;
 455 |       while(a.id != current_id) {
 456 |         it++;
 457 |         a = it->second;
 458 |       }
 459 |       fprintf(fp, "%d %f IDEAL\n", a.id, a.ideal_fraction);
 460 |       fprintf(fp, "%d %f ACTUAL\n", a.id, a.actual_fraction);
 461 |       current_id++;
 462 |     }
 463 |     return goodness_of_my_dist;
 464 |   }
 465 |   return 0;
 466 | }
 467 | 
 468 | 
 469 | int dumpAgeBuckets(char *age_dump_file = NULL, char *size_dump_file = NULL,
 470 |     char *dir_dump_file = NULL, int only_calculate_accuracy = 0) {
 471 |   FILE *fp = NULL;
 472 |   if(age_dump_file) {
 473 |     fp = fopen(age_dump_file, "w");
 474 |     fprintf(fp, "BUCKET FRACTION TYPE\n");
 475 |   }
 476 | 
 477 |   std::list <double> expected;
 478 |   std::list <double> actual;
 479 | 
 480 |   if(age_dump_file) {
 481 |     auto current_id = 0;
 482 |     while(current_id < NUM_AGES) {
 483 |       auto it = age_buckets.rbegin();
 484 |       auto a = it->second;
 485 |       while(a.id != current_id) {
 486 |         it++;
 487 |         a = it->second;
 488 |       }
 489 |       fprintf(fp, "%d %f IDEAL\n", a.id, a.ideal_fraction);
 490 |       fprintf(fp, "%d %f ACTUAL\n", a.id, a.actual_fraction);
 491 |       expected.push_back(a.ideal_fraction);
 492 |       actual.push_back(a.actual_fraction);
 493 |       current_id++;
 494 |     }
 495 |     fclose(fp);
 496 |   } else {
 497 |     if(!only_calculate_accuracy) {
 498 |       std::cout << std::endl << 
 499 |         "************ AGE BUCKET DUMP *************" << std::endl;
 500 |     }
 501 |     auto it = age_buckets.rbegin();
 502 |     uint64_t oldest = 0; uint64_t youngest = 0;
 503 |     while(it != age_buckets.rend()) {
 504 |       oldest = 0;
 505 |       youngest = 0;
 506 |       auto a = it->second;
 507 |       if(a.f != NULL) {
 508 |         oldest = a.f->age;
 509 |       }
 510 | 
 511 |       if(a.last != NULL) {
 512 |         youngest = a.last->age;
 513 |       }
 514 |       expected.push_back(a.ideal_fraction);
 515 |       actual.push_back(a.actual_fraction);
 516 | 
 517 |       if(!only_calculate_accuracy) {
 518 |         std::cout << "Bucket = " << a.id << ", Ideal Ratio = " <<
 519 |           a.ideal_fraction << ", Actual Ratio = " << a.actual_fraction <<
 520 |           ", Count = " << a.count << ", Cutoff = " << a.cutoff <<
 521 |           ", Oldest file = " << oldest << ", Youngest file = "
 522 |           << youngest << std::endl;
 523 |       }
 524 |       it++;
 525 |     }
 526 |   }
 527 | 
 528 |   if(confidence > 0.0) {
 529 |     return calculateChiMeanSquared(expected, actual, age_dump_file,
 530 |         size_dump_file, dir_dump_file);
 531 |   }
 532 |   return 0;
 533 | }
 534 | 
 535 | float tossCoin() {
 536 |   return ((float) (rand() % 100)) / 100;
 537 | }
 538 | 
 539 | void reAge(struct age *a_grp, uint64_t future_tick = 0) {
 540 |   int i = 0;
 541 |   AgeBucket a, *ab = new AgeBucket[NUM_AGES];
 542 |   unordered_map<int, AgeBucket> age_bucket_map;
 543 | 
 544 |   for(i=0; i<NUM_AGES; i++) {
 545 |     ab[i] = (age_buckets.find(a_grp->bucket_keys[i]))->second;
 546 |     // revise cutoffs
 547 |     if(future_tick == 0) {
 548 |       ab[i].cutoff = ab[i].ratio * tick;
 549 |     } else {
 550 |       ab[i].cutoff = ab[i].ratio * future_tick;
 551 |     }
 552 |   }
 553 | 
 554 |   /*
 555 |    * Note that the loop below goes to n-1 age buckets.
 556 |    */
 557 |   for(i=0; i<NUM_AGES-1; i++) {
 558 |     if(ab[i].last == NULL) {
 559 |       continue;
 560 |     }
 561 | 
 562 |     while(ab[i].count > 0) {
 563 |       auto f = ab[i].f;
 564 |       if(f->age >= ab[i].cutoff) {
 565 |         break;
 566 |       }
 567 | 
 568 |       ab[i].deleteFile(f, global_live_file_count);
 569 |       ab[i+1].addFile(f, global_live_file_count, false);
 570 |     }
 571 |   }
 572 | 
 573 |   for(i=0; i<NUM_AGES; i++) {
 574 |     auto old_key = ab[i].replace(a_grp->bucket_keys);
 575 |     age_buckets.erase(old_key);
 576 |     age_buckets.insert(std::pair<std::string, AgeBucket>(a_grp->bucket_keys[i],
 577 |           ab[i]));
 578 |   }
 579 | 
 580 |   delete [] ab;
 581 | }
 582 | 
 583 | void dumpStats(struct age *a, struct size *s, struct dir *d) {
 584 |   std::cout << "============= OVERALL STATISTICS ===============" << std::endl;
 585 |   std::cout << " Total runtime = " << runtime << " mins." << std::endl;
 586 |   std::cout << " Total number of operations = " << tick << std::endl;
 587 |   std::cout << " Number of disk overwrites = " << runs << std::endl;
 588 |   std::cout << " Total aging workload created = " <<
 589 |     workload_size / 1048576 << " MB" << std::endl;
 590 |   if (confidence > 0) {
 591 |     std::cout << " Confidence achieved (chi-squared measure) = " <<
 592 |       confidence << std::endl;
 593 |   } else {
 594 |     std::cout << " Perfect convergence achieved" << std::endl;
 595 |   }
 596 |   std::cout << " Size distribution dumped in " << s->out_file << std::endl;
 597 |   std::cout << " Dir depth distribution dumped in " << d->out_file << std::endl;
 598 |   std::cout << " Age distribution dumped in " << a->out_file << std::endl;
 599 |   std::cout << "================================================" << std::endl;
 600 | }
 601 | 
 602 | size_t createFile(int size_arr_position, struct age *a_grp, struct size *s_grp,
 603 |     struct dir *d_grp, int *create_succeeded) {
 604 |   /*
 605 |    * In this function, we need to do the following tasks:
 606 |    *
 607 |    * 1. find what size file we need to create.
 608 |    * 2. find the dir depth we have to create it at
 609 |    * 3. create a file of that size.
 610 |    * 4. increment global_live_file_count
 611 |    * 5. adjust dir_buckets
 612 |    * 6. add file to global_file_list
 613 |    * 7. adjust size_buckets
 614 |    * 8. adjust age_buckets
 615 |    *
 616 |    * IMPORTANT: order of the steps is necessary.
 617 |    */
 618 | 
 619 |   // step 1
 620 |   SizeBucket sb(0, 0, s_grp->arr);
 621 |   if(size_arr_position >= 0) {
 622 |     auto it = size_buckets->find(s_grp->bucket_keys[size_arr_position]);
 623 |     assert(it != size_buckets->end());
 624 |     sb = it->second;
 625 |   } else {
 626 |     // the size bucket farthest away from its ideal fraction
 627 |     auto it = size_buckets->rbegin();
 628 |     while((it->second.size + live_data_size) >= total_disk_capacity) {
 629 |       if (it == size_buckets->rend()) {
 630 |         std::cout << "Cannot create a single file, exhausted all options!"
 631 |           << std::endl;
 632 |         *create_succeeded = -1;
 633 |         return 0;
 634 |       }
 635 |       it++;
 636 |     }
 637 |     sb = it->second;
 638 |   }
 639 | 
 640 |   // step 2
 641 |   auto d_it = dir_buckets->begin();
 642 |   auto d = d_it->second;
 643 | 
 644 |   // step 3
 645 |   char name[PATH_MAX];
 646 |   std::string sibling_dir = "";
 647 |   if((d.depth > 0) && (d.sibling_dirs > 0)) {
 648 |     boost::random::mt19937 gen{static_cast<std::uint64_t>(rand())};
 649 |     boost::random::uniform_int_distribution<std::uint64_t> dist{1,
 650 |       static_cast<std::uint64_t>(d.sibling_dirs)};
 651 |     auto rand_subdir = dist(gen);
 652 |     sibling_dir += "d" + std::to_string(rand_subdir) + "/";
 653 |   }
 654 |   snprintf(name, PATH_MAX, "%s/%s%" PRIu64, d.prefix.c_str(),
 655 |            sibling_dir.c_str(), tick);
 656 |   File *f = new File(name, sb.size, tick, d.depth); // step 2
 657 |   auto retval = f->createFile();
 658 |   assert(retval == 0);
 659 |   auto ret_size = f->size;
 660 | 
 661 |   // step 4
 662 |   global_live_file_count++;
 663 | 
 664 |   // step 5
 665 |   dir_buckets->erase(d_grp->bucket_keys[d.id]);
 666 |   d.count++;
 667 |   d.reKey(global_live_file_count, d_grp->bucket_keys);
 668 |   dir_buckets->insert(std::pair<std::string,
 669 |       DirBucket>(d_grp->bucket_keys[d.id], d));
 670 |   auto old_dir_buckets = dir_buckets;
 671 |   d_it = old_dir_buckets->begin();
 672 |   dir_buckets = new flat_map<std::string, DirBucket, BucketCompare>;
 673 |   while(d_it != old_dir_buckets->end()) {
 674 |     d = d_it->second;
 675 |     d.reKey(global_live_file_count, d_grp->bucket_keys);
 676 |     dir_buckets->insert(std::pair<std::string,
 677 |         DirBucket>(d_grp->bucket_keys[d.id], d));
 678 |     d_it++;
 679 |   }
 680 |   delete old_dir_buckets;
 681 | 
 682 |   // step 6
 683 |   global_file_list->addFile(f);
 684 | 
 685 |   // step 7
 686 |   size_buckets->erase(s_grp->bucket_keys[sb.id]);
 687 |   sb.addFile(f, global_live_file_count);
 688 |   s_grp->bucket_keys[sb.id] = sb.getKey();
 689 |   size_buckets->insert(std::pair<std::string,
 690 |       SizeBucket>(s_grp->bucket_keys[sb.id], sb));
 691 |   auto old_size_buckets = size_buckets;
 692 |   auto s_it = old_size_buckets->begin();
 693 |   size_buckets = new flat_map<std::string, SizeBucket, BucketCompare>;
 694 |   while(s_it != old_size_buckets->end()) {
 695 |     auto s = s_it->second;
 696 |     s.reKey(global_live_file_count, s_grp->bucket_keys);
 697 |     size_buckets->insert(std::pair<std::string,
 698 |         SizeBucket>(s_grp->bucket_keys[s.id], s));
 699 |     s_it++;
 700 |   }
 701 |   delete old_size_buckets;
 702 | 
 703 |   // step 8
 704 |   // youngest bucket
 705 |   AgeBucket ab = ((age_buckets.find(a_grp->bucket_keys[0]))->second);
 706 |   age_buckets.erase(a_grp->bucket_keys[ab.id]);
 707 |   ab.addFile(f, global_live_file_count, false);
 708 |   a_grp->bucket_keys[ab.id] = ab.getKey();
 709 |   age_buckets.insert(std::pair<std::string,
 710 |       AgeBucket>(a_grp->bucket_keys[ab.id], ab));
 711 |   return ret_size;
 712 | }
 713 | 
 714 | size_t deleteFile(struct age *a_grp, struct size *s_grp, struct dir *d_grp) {
 715 |   /*
 716 |    * When deleting a file, we perform the following operations:
 717 |    *
 718 |    * 1. find what size file we should delete.
 719 |    * 2. choose bucket to delete it from.
 720 |    * 3. choose which position in the bucket to delete it from.
 721 |    * 4. perform unlink operation.
 722 |    * 5. decrement global_live_file_count.
 723 |    * 6.  adjust dir buckets
 724 |    * 7. adjust age_buckets
 725 |    * 8. adjust size_buckets
 726 |    * 9. remove file from global_file_list
 727 |    *
 728 |    * NOTE: ORDER IS IMPORTANT
 729 |    */
 730 | 
 731 |   // step 1
 732 |   File *f = NULL;
 733 |   SizeBucket sb(0, 0, s_grp->arr); // dummy SizeBucket
 734 |   AgeBucket ab;
 735 |   DirBucket db(0, 0, 0, mount_point, fake,
 736 |                d_grp->arr, mkdir_path); // dummy DirBucket
 737 | 
 738 |   auto a_it = age_buckets.rbegin();
 739 | 
 740 |   // select age bucket
 741 |   do {
 742 |     ab = a_it->second;
 743 |     auto s_it = size_buckets->begin();
 744 |     // select size bucket
 745 |     do {
 746 |       sb = s_it->second;
 747 |       auto d_it = dir_buckets->rbegin();
 748 |       // select dir bucket
 749 |       do {
 750 |         db = d_it->second;
 751 |         f = ab.getFileToDelete(sb.size, db.depth);
 752 |         d_it++;
 753 |       } while((f == NULL) && (d_it != dir_buckets->rend()));
 754 |       s_it++;
 755 |     } while((f == NULL) && (s_it != size_buckets->end()));
 756 |     a_it++;
 757 |   } while((f == NULL) && (a_it != age_buckets.rend()));
 758 | 
 759 |   if(f == NULL) {
 760 |     std::cout << "Cannot delete a single file of any size!" << std::endl;
 761 |     exit(1);
 762 |   }
 763 | 
 764 |   auto ret_size = f->size;
 765 | 
 766 |   // step 4
 767 |   auto retval = f->deleteFile();
 768 |   assert(retval == 0);
 769 | 
 770 |   // step 5
 771 |   global_live_file_count--;
 772 | 
 773 |   // step 6
 774 |   dir_buckets->erase(d_grp->bucket_keys[db.id]);
 775 |   db.count--;
 776 |   db.reKey(global_live_file_count, d_grp->bucket_keys);
 777 |   dir_buckets->insert(std::pair<std::string,
 778 |       DirBucket>(d_grp->bucket_keys[db.id], db));
 779 |   auto old_dir_buckets = dir_buckets;
 780 |   auto d_it = old_dir_buckets->begin();
 781 |   dir_buckets = new flat_map<std::string, DirBucket, BucketCompare>;
 782 |   while(d_it != old_dir_buckets->end()) {
 783 |     auto d = d_it->second;
 784 |     d.reKey(global_live_file_count, d_grp->bucket_keys);
 785 |     dir_buckets->insert(std::pair<std::string,
 786 |         DirBucket>(d_grp->bucket_keys[d.id], d));
 787 |     d_it++;
 788 |   }
 789 |   delete old_dir_buckets;
 790 | 
 791 |   // step 7
 792 |   age_buckets.erase(a_grp->bucket_keys[ab.id]);
 793 |   ab.deleteFile(f, global_live_file_count);
 794 |   a_grp->bucket_keys[ab.id] = ab.getKey();
 795 |   age_buckets.insert(std::pair<std::string,
 796 |       AgeBucket>(a_grp->bucket_keys[ab.id], ab));
 797 | 
 798 |   // step 8
 799 |   size_buckets->erase(s_grp->bucket_keys[sb.id]);
 800 |   sb.deleteFile(f, global_live_file_count);
 801 |   s_grp->bucket_keys[sb.id] = sb.getKey();
 802 |   size_buckets->insert(std::pair<std::string,
 803 |       SizeBucket>(s_grp->bucket_keys[sb.id], sb));
 804 |   auto old_size_buckets = size_buckets;
 805 |   auto s_it = old_size_buckets->begin();
 806 |   size_buckets = new flat_map<std::string, SizeBucket, BucketCompare>;
 807 |   while(s_it != old_size_buckets->end()) {
 808 |     auto s = s_it->second;
 809 |     s.reKey(global_live_file_count, s_grp->bucket_keys);
 810 |     size_buckets->insert(std::pair<std::string,
 811 |         SizeBucket>(s_grp->bucket_keys[s.id], s));
 812 |     s_it++;
 813 |   }
 814 |   delete old_size_buckets;
 815 | 
 816 |   // step 9
 817 |   global_file_list->deleteFile(f);
 818 | 
 819 |   delete f;
 820 |   return ret_size;
 821 | }
 822 | 
 823 | uint64_t calculateT(struct age *a_grp) {
 824 |   uint64_t T = 0;
 825 |   int64_t t = 0;
 826 |   int i = 0;
 827 |   auto s_i = 0.0;
 828 |   for(i=0; i < NUM_AGES - 1; i++) {
 829 |     auto a = (age_buckets.find(a_grp->bucket_keys[i]))->second;
 830 | 
 831 |     if (i == 0) {
 832 |       s_i = 1 - a.ratio;
 833 |     } else {
 834 |       auto younger_a = (age_buckets.find(a_grp->bucket_keys[i-1]))->second;
 835 |       s_i = younger_a.ratio - a.ratio;
 836 |     }
 837 |     t = 2 * K * ((double)a.ideal_fraction / s_i);
 838 |     if (T < t) {
 839 |       T = t;
 840 |     }
 841 |   }
 842 | 
 843 |   auto a = (age_buckets.find(a_grp->bucket_keys[i]))->second;
 844 |   auto younger_a = (age_buckets.find(a_grp->bucket_keys[i-1]))->second;
 845 |   s_i = younger_a.ratio - a.ratio;
 846 |   t = ((double)(2 * K * (a.ideal_fraction - 1) + K) / s_i);
 847 |   if (t > 0 && T < t) {
 848 |     T = t;
 849 |   }
 850 | 
 851 |   if (s_i * T <= K) {
 852 |     T = (K / s_i);
 853 |   }
 854 |   return T;
 855 | }
 856 | 
 857 | int performOp(bool create, int size_arr_position,
 858 |     int idle_injections, struct age *a, struct size *s, struct dir *d) {
 859 |   tick++;
 860 |   int create_succeeded = 0;
 861 |   if(create) {
 862 |     auto data_added = createFile(size_arr_position, a, s, d, &create_succeeded);
 863 |     if(create_succeeded == 0) {
 864 |       live_data_size += data_added;
 865 |       workload_size += data_added;
 866 |     }
 867 |   }
 868 |   if (!create || create_succeeded == -1) {
 869 |     live_data_size -= deleteFile(a, s, d);
 870 |   }
 871 |   return 0;
 872 | }
 873 | 
 874 | int performRapidAging(size_t till_size, int idle_injections,
 875 |     struct age *a, struct size *s, struct dir *d) {
 876 |   boost::random::mt19937 gen{static_cast<std::uint64_t>(total_size_weight)};
 877 |   boost::random::uniform_int_distribution<std::uint64_t> dist{1,
 878 |     static_cast<std::uint64_t>(total_size_weight)};
 879 |   while(live_data_size < till_size) {
 880 |     auto rand = dist(gen);
 881 |     int j = 0;
 882 |     while(rand > s->cutoffs[j]) {
 883 |       j++;
 884 |       if(j == NUM_SIZES) {
 885 |         j--;
 886 |         break;
 887 |       }
 888 |     }
 889 |     performOp(true, j, idle_injections, a, s, d);
 890 |   }
 891 |   return 0;
 892 | }
 893 | 
 894 | int performStableAging(size_t till_size, int idle_injections,
 895 |     struct age *a, struct size *s, struct dir *d, int runs) {
 896 |   auto future_tick = calculateT(a);
 897 |   reAge(a, future_tick);
 898 |   int confidence_met = 0;
 899 |   AGING_TRIGGER trigger = none;
 900 |   do {
 901 |     if(tossCoin() < 0.5) {
 902 |       performOp(true, -1, idle_injections, a, s, d); // create file
 903 |     } else {
 904 |       performOp(false, -1, idle_injections, a, s, d); // delete file
 905 |     }
 906 |     reAge(a, future_tick);
 907 | 
 908 |     if((tick % 10000) == 0) {
 909 |       auto end = std::chrono::high_resolution_clock::now();
 910 |       auto millis = std::chrono::duration<double,
 911 |            std::milli>(end - start).count();
 912 |       runtime = ((millis / 1000) / 60);
 913 |       std::cout << "Workload = " << workload_size / 1048576 << " MB, Runtime = "
 914 |         << runtime << " mins., Convergence ops = " << future_tick <<
 915 |         ", Operations = " << tick << "..." << std::endl;
 916 |       dumpSizeBuckets();
 917 |       dumpDirBuckets();
 918 |       auto confidence_met = dumpAgeBuckets(a->out_file, s->out_file,
 919 |           d->out_file);
 920 |       if(confidence > 0 && confidence_met == 1) {
 921 |         return 1;
 922 |       }
 923 |     }
 924 |     if(tick >= future_tick) {
 925 |       trigger = convergence;
 926 |       std::cout <<
 927 |         "Aging stopped due to perfect convergence in relative age distribution."
 928 |         << std::endl;
 929 |     } else if(workload_size >= till_size) {
 930 |       std::cout << "Aging stopped because of reaching intended workload size."
 931 |         << std::endl;
 932 |       trigger = workload;
 933 |     } else if(runtime >= runtime_max) {
 934 |       std::cout << "Aging stopped because of reaching runtime limit."
 935 |         << std::endl;
 936 |       trigger = exec_time;
 937 |     } else if(confidence > 0 && confidence_met == 1) {
 938 |       std::cout << "Aging stopped because of meeting intended aging accuracy."
 939 |         << std::endl;
 940 |       trigger = accuracy;
 941 |     }
 942 |   } while(trigger == none);
 943 |   return trigger;
 944 | }
 945 | 
 946 | void usage() {
 947 |   std::cout << std::endl;
 948 |   std::cout << "geriatrix " << std::endl;
 949 |   std::cout << "        -n <disk size in bytes>" << std::endl;
 950 |   std::cout << "        -u <utilization fraction>" << std::endl;
 951 |   std::cout << "        -r <random seed>" << std::endl;
 952 |   std::cout << "        -m <mount point>" << std::endl;
 953 |   std::cout << "        -a <age distribution file>" << std::endl;
 954 |   std::cout << "        -s <size distribution file>" << std::endl;
 955 |   std::cout << "        -d <dir distribution file>" << std::endl;
 956 |   std::cout << "        -x <age distribution out file>" << std::endl;
 957 |   std::cout << "        -y <size distribution out file>" << std::endl;
 958 |   std::cout << "        -z <dir distribution out file>" << std::endl;
 959 |   std::cout << "        -t <t-way concurrency>" << std::endl;
 960 |   std::cout << "        -i <num runs>" << std::endl;
 961 |   std::cout << "        -f <0 / 1 fake>" << std::endl;
 962 |   std::cout << "        -p <0 / 1 idle time>" << std::endl;
 963 |   std::cout << "        -c <confidence fraction between 0 and 1>" << std::endl;
 964 |   std::cout << "        -q <0 / 1 ask before quitting>" << std::endl;
 965 |   std::cout << "        -w <num mins>" << std::endl;
 966 |   std::cout << "        -b <backend (posix, deltafs, etc.)>" << std::endl;
 967 |   std::cout << std::endl;
 968 | }
 969 | 
 970 | int resumeAgingQuery(uint64_t total_disk_capacity, double runtime) {
 971 |   auto decision = 'x';
 972 |   std::cout << "=================== Aging trigger fired  ====================="
 973 |     << std::endl;
 974 |   if(confidence > 0) {
 975 |     std::cout << "Accuracy at this point = " <<
 976 |       dumpAgeBuckets(NULL, NULL, NULL, 1) << std::endl;
 977 |   } else {
 978 |     std::cout << "Perfect convergence mode selected." << std::endl;
 979 |   }
 980 |   std::cout << "Number of disk overwrites = " <<
 981 |     (double)workload_size / total_disk_capacity << std::endl;
 982 |   std::cout << "Runtime till now = " << runtime << " mins." << std::endl;
 983 |   do {
 984 |     std::cout << "Do you want to resume aging (y / n): ";
 985 |     std::cin >> decision;
 986 |     std::cout << std::endl;
 987 |   } while(!((decision == 'y') || (decision == 'n')));
 988 |   if(decision == 'y') {
 989 |     std::cout << "=================================================="
 990 |       << std::endl;
 991 |     if(confidence > 0) {
 992 |       std::cout << "Current confidence level set = " << confidence << "."
 993 |         << std::endl;
 994 |       std::cout << "Enter new confidence level (fraction between 0 and 1): ";
 995 |       std::cin >> confidence;
 996 |       std::cout << std::endl;
 997 |     }
 998 |     std::cout << "Aging currently ran for " << runtime << " mins." << std::endl;
 999 |     std::cout <<
1000 |       "How many more mins do you want to age if confidence is not met: ";
1001 |     std::cin >> runtime_max;
1002 |     start = std::chrono::high_resolution_clock::now();
1003 |     std::cout << std::endl;
1004 |     std::cout << "Number of disk overwrites = " << runs << std::endl;
1005 |     std::cout << "How many more disk overwrites do you want to age for: ";
1006 |     auto more_runs = 0;
1007 |     std::cin >> more_runs;
1008 |     runs += more_runs;
1009 |     std::cout << std::endl;
1010 |     std::cout << "Happy Aging!!!" << std::endl;
1011 |     std::cout << "=================================================="
1012 |       << std::endl;
1013 |     return 1;
1014 |   }
1015 |   return 0;
1016 | }
1017 | 
1018 | void handler(int signo){
1019 |   dumpAgeBuckets(a.out_file);
1020 |   dumpSizeBuckets(s.out_file);
1021 |   dumpDirBuckets(d.out_file);
1022 |   dumpStats(&a, &s, &d);
1023 |   destroy();
1024 |   exit(0);
1025 | }
1026 | 
1027 | int main(int argc, char *argv[]) {
1028 |   if(argc != 37) {
1029 |     usage();
1030 |     exit(1);
1031 |   }
1032 |   char *mybackend = NULL;
1033 |   //uint64_t total_disk_capacity = 0;
1034 |   double utilization = 0.0;
1035 |   int seed = 0;
1036 |   int option = 0;
1037 |   int concurrency = 0;
1038 |   int idle_injections = 0;
1039 |   int query_before_quitting = 0;
1040 |   while((option = getopt(argc, argv,
1041 |                          "n:u:r:m:a:s:d:x:y:z:t:i:f:p:c:q:w:b:")) != EOF) {
1042 |     switch(option) {
1043 |       case 'n': total_disk_capacity = strtoull(optarg, NULL, 10); break;
1044 |       case 'u': utilization = strtod(optarg, NULL); break;
1045 |       case 'r': seed = atoi(optarg); break;
1046 |       case 'm': mount_point = optarg; break;
1047 |       case 'a': a.in_file = optarg; break;
1048 |       case 's': s.in_file = optarg; break;
1049 |       case 'd': d.in_file = optarg; break;
1050 |       case 'x': a.out_file = optarg; break;
1051 |       case 'y': s.out_file = optarg; break;
1052 |       case 'z': d.out_file = optarg; break;
1053 |       case 't': concurrency = atoi(optarg); break;
1054 |       case 'i': runs = atoi(optarg); break;
1055 |       case 'f': fake = atoi(optarg); break;
1056 |       case 'p': idle_injections = atoi(optarg); break;
1057 |       case 'c': confidence = strtod(optarg, NULL); break;
1058 |       case 'q': query_before_quitting = atoi(optarg); break;
1059 |       case 'w': runtime_max = atoi(optarg); break;
1060 |       case 'b': mybackend = optarg; break;
1061 |     }
1062 |   }
1063 |   if (!mybackend || strcmp(mybackend, "posix") == 0) {
1064 |       /* do nothing, posix is the default */
1065 |   } else if (strcmp(mybackend, "deltafs") == 0) {
1066 | #ifdef DELTAFS
1067 |       g_backend = &deltafs_backend_driver;
1068 | #else
1069 |       fprintf(stderr, "error: DELTAFS not enabled in this binary\n");
1070 |       exit(1);
1071 | #endif
1072 |   }
1073 | 
1074 |   assert(total_disk_capacity > 0);
1075 |   srand(seed);
1076 | 
1077 |   init(&a, &s, &d); // initialize the data structures for aging
1078 |   if(confidence > 0.0) {
1079 |     // initialize the chi-squared value for accuracy comparison.
1080 |     dist = new boost::math::chi_squared(NUM_AGES - 1);
1081 |     goodness_measure = cdf(*dist, confidence);
1082 |   }
1083 |   pool = new ThreadPool(concurrency);
1084 |   performRapidAging(total_disk_capacity * utilization, idle_injections,
1085 |       &a, &s, &d);
1086 |   K = tick;
1087 |   struct sigaction sigIntHandler;
1088 | 
1089 |   sigIntHandler.sa_handler = handler;
1090 |   sigemptyset(&sigIntHandler.sa_mask);
1091 |   sigIntHandler.sa_flags = 0;
1092 |   sigaction(SIGINT, &sigIntHandler, NULL);
1093 | 
1094 |   do {
1095 |     performStableAging(total_disk_capacity * runs, idle_injections,
1096 |         &a, &s, &d, runs);
1097 |   } while(query_before_quitting && resumeAgingQuery(total_disk_capacity,
1098 |         runtime));
1099 |   handler(0);
1100 |   return 0;
1101 | }
1102 | 


--------------------------------------------------------------------------------