├── .gitignore ├── MANIFEST.SKIP ├── MANIFEST ├── META.yml ├── t ├── 17.t ├── 16.t ├── 14.t ├── 11.t ├── 20.t ├── 10.t ├── 18.t ├── 15.t ├── 8.t ├── 19.t ├── 12.t ├── 9.t ├── 21.t ├── 5.t ├── 3.t ├── 22.t ├── 4.t ├── 13.t ├── 7.t ├── 2.t ├── 23.t ├── 6.t ├── 1.t └── FastMmapTest.pl ├── .github └── workflows │ └── ci.yml ├── Makefile.PL ├── README ├── mmap_cache_internals.h ├── unix.c ├── win32.c ├── mmap_cache.h ├── mmap_cache_test.c ├── Changes ├── FastMmap.xs ├── ppport.h └── mmap_cache.c /.gitignore: -------------------------------------------------------------------------------- 1 | FastMmap.bs 2 | FastMmap.c 3 | FastMmap.o 4 | MYMETA.json 5 | MYMETA.yml 6 | Makefile 7 | blib/ 8 | mmap_cache.o 9 | pm_to_blib 10 | unix.o 11 | -------------------------------------------------------------------------------- /MANIFEST.SKIP: -------------------------------------------------------------------------------- 1 | ^\.git 2 | ^Cache-FastMmap 3 | pm_to_blib 4 | blib 5 | \.o$ 6 | \.bs$ 7 | ^Makefile$ 8 | ^FastMmap\.c$ 9 | ^MYMETA\. 10 | ^MANIFEST\.bak$ 11 | -------------------------------------------------------------------------------- /MANIFEST: -------------------------------------------------------------------------------- 1 | Changes 2 | FastMmap.xs 3 | lib/Cache/FastMmap.pm 4 | Makefile.PL 5 | MANIFEST This list of files 6 | MANIFEST.SKIP 7 | META.yml 8 | mmap_cache.c 9 | mmap_cache.h 10 | mmap_cache_internals.h 11 | mmap_cache_test.c 12 | ppport.h 13 | README 14 | t/1.t 15 | t/10.t 16 | t/11.t 17 | t/12.t 18 | t/13.t 19 | t/14.t 20 | t/15.t 21 | t/16.t 22 | t/17.t 23 | t/18.t 24 | t/19.t 25 | t/2.t 26 | t/20.t 27 | t/21.t 28 | t/22.t 29 | t/23.t 30 | t/3.t 31 | t/4.t 32 | t/5.t 33 | t/6.t 34 | t/7.t 35 | t/8.t 36 | t/9.t 37 | t/FastMmapTest.pl 38 | unix.c 39 | win32.c 40 | -------------------------------------------------------------------------------- /META.yml: -------------------------------------------------------------------------------- 1 | --- 2 | abstract: "Uses an mmap'ed file to act as a shared memory interprocess cache" 3 | author: 4 | - 'Rob Mueller ' 5 | build_requires: 6 | ExtUtils::MakeMaker: 0 7 | Test::Deep: 0 8 | configure_requires: 9 | ExtUtils::MakeMaker: 0 10 | dynamic_config: 1 11 | generated_by: 'ExtUtils::MakeMaker version 6.66, CPAN::Meta::Converter version 2.133380' 12 | license: unknown 13 | meta-spec: 14 | url: http://module-build.sourceforge.net/META-spec-v1.4.html 15 | version: 1.4 16 | name: Cache-FastMmap 17 | no_index: 18 | directory: 19 | - t 20 | - inc 21 | requires: 22 | Storable: 0 23 | resources: 24 | bugtracker: https://github.com/robmueller/cache-fastmmap/issues 25 | repository: https://github.com/robmueller/cache-fastmmap 26 | version: 1.59 27 | -------------------------------------------------------------------------------- /t/17.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | 4 | use Test::More tests => 186; 5 | BEGIN { use_ok('Cache::FastMmap') }; 6 | use Storable qw(freeze thaw); 7 | use strict; 8 | 9 | ######################### 10 | 11 | # Test that we actually re-use deleted slots in a cache 12 | 13 | my $FC = Cache::FastMmap->new( 14 | page_size => 65536, 15 | num_pages => 1, 16 | init_file => 1, 17 | serializer => '', 18 | start_slots => 89, 19 | ); 20 | ok( defined $FC ); 21 | 22 | ok($FC->set("foo", "a" x 31000), "set foo"); 23 | ok($FC->set("bar", "b" x 31000), "set bar"); 24 | 25 | for (1 .. 90) { 26 | ok($FC->set("a", "$_"), "set $_"); 27 | ok($FC->get("a") eq "$_", "get $_"); 28 | $FC->remove("a"); 29 | } 30 | 31 | ok($FC->get("foo") eq "a" x 31000, "get foo"); 32 | ok($FC->get("bar") eq "b" x 31000, "get bar"); 33 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI Makefile 2 | 3 | on: 4 | push: 5 | branches: '*' 6 | pull_request: 7 | branches: '*' 8 | 9 | jobs: 10 | perl-job: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | perl-version: 16 | - '5.30' 17 | # - '5.32' 18 | # - 'latest' 19 | container: 20 | image: perldocker/perl-tester:${{ matrix.perl-version }} # https://hub.docker.com/r/perldocker/perl-tester 21 | name: Perl ${{ matrix.perl-version }} 22 | steps: 23 | - uses: actions/checkout@v2 24 | - name: Regular tests 25 | run: | 26 | cpanm --installdeps --notest . 27 | cpanm JSON Sereal::Encoder 28 | perl Makefile.PL 29 | make 30 | make test 31 | -------------------------------------------------------------------------------- /Makefile.PL: -------------------------------------------------------------------------------- 1 | use 5.006; 2 | use ExtUtils::MakeMaker; 3 | 4 | WriteMakefile( 5 | 'NAME' => 'Cache::FastMmap', 6 | 'VERSION_FROM' => 'lib/Cache/FastMmap.pm', 7 | 'ABSTRACT_FROM' => 'lib/Cache/FastMmap.pm', 8 | 'AUTHOR' => 'Rob Mueller ', 9 | 'LICENSE' => 'perl', 10 | 'PREREQ_PM' => { 11 | 'Storable' => 0, 12 | 'Test::Deep' => 0, 13 | }, 14 | 'LIBS' => [''], 15 | 'INC' => '-I.', 16 | 'OBJECT' => 'FastMmap.o mmap_cache.o ' . ($^O eq 'MSWin32' ? 'win32.o' : 'unix.o'), 17 | 'META_MERGE' => { 18 | 'resources' => { 19 | 'bugtracker' => 'https://github.com/robmueller/cache-fastmmap/issues', 20 | 'repository' => 'https://github.com/robmueller/cache-fastmmap', 21 | }, 22 | }, 23 | # 'OPTIMIZE' => '-g -DDEBUG -ansi -pedantic', 24 | ); 25 | 26 | -------------------------------------------------------------------------------- /t/16.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | 4 | use Test::More tests => 3; 5 | BEGIN { use_ok('Cache::FastMmap') }; 6 | use strict; 7 | 8 | ######################### 9 | 10 | # Insert your test code below, the Test::More module is use()ed here so read 11 | # its man page ( perldoc Test::More ) for help writing this test script. 12 | 13 | my $FC = Cache::FastMmap->new( 14 | init_file => 1, 15 | num_pages => 1, 16 | page_size => 2 ** 15, 17 | ); 18 | ok( defined $FC ); 19 | 20 | my @d; 21 | 22 | for (1 .. 20) { 23 | 24 | $FC->set($_, $d[$_]=$_) for 1 .. 100; 25 | 26 | for (1 .. 50) { 27 | $FC->remove($_*2); 28 | $d[$_*2] = undef; 29 | 30 | $FC->set($_, $_*2); 31 | $d[$_] = $_*2; 32 | 33 | for my $c (1 .. 100) { 34 | my $v = $FC->get($c); 35 | ($v || 0) == ($d[$c] || 0) 36 | || die "at offset $c, got $v expected $d[$c]"; 37 | } 38 | } 39 | 40 | } 41 | ok(1, "ordering santity tests complete"); 42 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Cache::FastMmap 2 | =========================== 3 | 4 | A shared memory cache through an mmap'ed file. It's core is written 5 | in C for performance. It uses fcntl locking to ensure multiple 6 | processes can safely access the cache at the same time. It uses 7 | a basic LRU algorithm to keep the most used entries in the cache. 8 | 9 | INSTALLATION 10 | 11 | To install this module type the following: 12 | 13 | perl Makefile.PL 14 | make 15 | make test 16 | make install 17 | 18 | DEPENDENCIES 19 | 20 | Storable if you want to store complex structures 21 | 22 | AUTHOR 23 | 24 | Rob Mueller 25 | 26 | DOCUMENTATION 27 | 28 | See the POD documentation. Viewable online at CPAN 29 | 30 | https://metacpan.org/pod/Cache::FastMmap 31 | 32 | COPYRIGHT AND LICENCE 33 | 34 | Copyright (C) 2003-2020 by Fastmail Pty Ltd 35 | 36 | This library is free software; you can redistribute it and/or modify 37 | it under the same terms as Perl itself. 38 | 39 | -------------------------------------------------------------------------------- /t/14.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | 4 | use Test::More tests => 110; 5 | BEGIN { use_ok('Cache::FastMmap') }; 6 | use strict; 7 | 8 | ######################### 9 | 10 | # Insert your test code below, the Test::More module is use()ed here so read 11 | # its man page ( perldoc Test::More ) for help writing this test script. 12 | 13 | my $FC = Cache::FastMmap->new( 14 | enable_stats => 1 15 | ); 16 | 17 | ok( defined $FC ); 18 | 19 | ok( !defined $FC->get("a") ); 20 | $FC->set("a", "b"); 21 | ok( $FC->get("a") eq "b" ); 22 | 23 | # Get 100 times 24 | for (1 .. 100) { 25 | ok( $FC->get("a") eq "b" ); 26 | } 27 | 28 | my ($nreads, $nreadhits) = $FC->get_statistics(); 29 | 30 | cmp_ok( $nreads, '==', 102 ); 31 | cmp_ok( $nreadhits, '==', 101 ); 32 | 33 | ($nreads, $nreadhits) = $FC->get_statistics(1); 34 | 35 | cmp_ok( $nreads, '==', 102 ); 36 | cmp_ok( $nreadhits, '==', 101 ); 37 | 38 | ($nreads, $nreadhits) = $FC->get_statistics(1); 39 | 40 | cmp_ok( $nreads, '==', 0 ); 41 | cmp_ok( $nreadhits, '==', 0 ); 42 | 43 | -------------------------------------------------------------------------------- /t/11.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | 4 | use Test::More tests => 7; 5 | BEGIN { use_ok('Cache::FastMmap') }; 6 | use strict; 7 | 8 | ######################### 9 | 10 | # Test recursive cache use 11 | 12 | sub get_fc { 13 | my $FC; 14 | $FC = Cache::FastMmap->new( 15 | cache_not_found => 1, 16 | serializer => '', 17 | init_file => 1, 18 | num_pages => 89, 19 | page_size => 1024, 20 | read_cb => sub { $FC->get($_[1] . "1"); }, 21 | write_cb => sub { $FC->set($_[1] . "1", $_[2]); }, 22 | delete_cb => sub { $FC->delete($_[1]); }, 23 | write_action => 'write_back', 24 | @_ 25 | ); 26 | return $FC; 27 | } 28 | 29 | my $FC = get_fc(); 30 | ok( defined $FC ); 31 | 32 | $FC->set("foo1", "bar"); 33 | my $V = eval { $FC->get("foo"); }; 34 | ok(!$V, "no return"); 35 | like($@, qr/already locked/, "recurse fail"); 36 | 37 | $FC = undef; 38 | 39 | $FC = get_fc(allow_recursive => 1); 40 | ok( defined $FC ); 41 | 42 | $FC->set("foo1", "bar"); 43 | $V = eval { $FC->get("foo"); }; 44 | ok(!$@, "recurse success 1"); 45 | is($V, "bar", "recurse success 2"); 46 | 47 | -------------------------------------------------------------------------------- /t/20.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | 4 | use Test::More; 5 | use strict; 6 | 7 | ######################### 8 | 9 | # Insert your test code below, the Test::More module is use()ed here so read 10 | # its man page ( perldoc Test::More ) for help writing this test script. 11 | 12 | 13 | if( $^O eq 'MSWin32' ) { 14 | plan skip_all => "permissions parameter is not supported on Windows"; 15 | } 16 | else { 17 | plan tests => 7; 18 | } 19 | 20 | require_ok('Cache::FastMmap'); 21 | 22 | my $old_umask = umask 0000; 23 | note( 'umask returns undef on this system, test results may not be reliable') 24 | unless defined $old_umask; 25 | 26 | my $FC = Cache::FastMmap->new(init_file => 1); 27 | ok( defined $FC ); 28 | my (undef, undef, $Mode) = stat($FC->{share_file}); 29 | $Mode = $Mode & 0777; 30 | is($Mode, 0640, "default persmissions 0640"); 31 | undef $FC; 32 | 33 | my $FC = Cache::FastMmap->new(init_file => 1, permissions => 0600); 34 | ok( defined $FC ); 35 | my (undef, undef, $Mode) = stat($FC->{share_file}); 36 | $Mode = $Mode & 0777; 37 | is($Mode, 0600, "can set to 0600"); 38 | undef $FC; 39 | 40 | my $FC = Cache::FastMmap->new(init_file => 1, permissions => 0666); 41 | ok( defined $FC ); 42 | my (undef, undef, $Mode) = stat($FC->{share_file}); 43 | $Mode = $Mode & 0777; 44 | is($Mode, 0666, "can set to 0666"); 45 | undef $FC; 46 | -------------------------------------------------------------------------------- /t/10.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | 4 | use Test::More tests => 9; 5 | BEGIN { use_ok('Cache::FastMmap') }; 6 | use Data::Dumper; 7 | use strict; 8 | 9 | ######################### 10 | 11 | # Test writeback and cache_not_found option 12 | 13 | # Test a backing store just made of a local hash 14 | my %BackingStore = ( 15 | foo => '123abc', 16 | bar => undef 17 | ); 18 | 19 | my %OrigBackingStore = %BackingStore; 20 | 21 | my $RCBCalled = 0; 22 | 23 | my $FC = Cache::FastMmap->new( 24 | cache_not_found => 1, 25 | serializer => '', 26 | init_file => 1, 27 | num_pages => 89, 28 | page_size => 1024, 29 | context => \%BackingStore, 30 | read_cb => sub { $RCBCalled++; return $_[0]->{$_[1]}; }, 31 | write_cb => sub { $_[0]->{$_[1]} = $_[2]; }, 32 | delete_cb => sub { delete $_[0]->{$_[1]} }, 33 | write_action => 'write_back' 34 | ); 35 | 36 | ok( defined $FC ); 37 | 38 | # Should pull from the backing store 39 | is( $FC->get('foo'), '123abc', "cb get 1"); 40 | is( $FC->get('bar'), undef, "cb get 2"); 41 | is( $RCBCalled, 2, "cb get 2"); 42 | 43 | # Should be in the cache now 44 | is( $FC->get('foo'), '123abc', "cb get 3"); 45 | is( $FC->get('bar'), undef, "cb get 4"); 46 | is( $RCBCalled, 2, "cb get 2"); 47 | 48 | $FC->set('foo', '123abc'); 49 | $FC->set('bar', undef); 50 | 51 | # Should force cache data back to backing store 52 | %BackingStore = (); 53 | $FC->empty(); 54 | 55 | ok( eq_hash(\%BackingStore, \%OrigBackingStore), "items match 1" . Dumper(\%BackingStore, \%OrigBackingStore)); 56 | 57 | -------------------------------------------------------------------------------- /t/18.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | 4 | use Test::More; 5 | 6 | BEGIN { 7 | if ($^O eq "MSWin32") { 8 | plan skip_all => 'No FD_CLOEXEC tests, running on Windows'; 9 | } 10 | if (!-d "/proc/$$") { 11 | plan skip_all => 'No FD_CLOEXEC tests, no /proc filesystem'; 12 | } 13 | } 14 | 15 | 16 | use strict; 17 | use Fcntl; 18 | 19 | ######################### 20 | 21 | # Test fd's are closed on exec 22 | 23 | if (@ARGV) { 24 | my $PipeFd = shift @ARGV; 25 | my $FdCount = scalar(() = glob "/proc/$$/fd/*"); 26 | open(my $PipeFh, ">&=$PipeFd") 27 | || die "Could not reopen fd: $!"; 28 | print($PipeFh "$FdCount\n") 29 | || die "Could not print to pipe: $!"; 30 | exit(0); 31 | } 32 | 33 | require Cache::FastMmap; 34 | my @Caches = map { 35 | Cache::FastMmap->new( 36 | page_size => 4096, 37 | num_pages => 1, 38 | init_file => 1, 39 | serializer => '', 40 | ); 41 | } (1 .. 20); 42 | my $CacheCount = @Caches; 43 | 44 | my $FdCount = scalar(() = glob "/proc/$$/fd/*"); 45 | ok($FdCount > $CacheCount, "More fd's than caches: $FdCount > $CacheCount"); 46 | 47 | pipe(my $ReadPipeFh, my $WritePipeFh) 48 | || die "pipe failed: $!"; 49 | 50 | fcntl($ReadPipeFh, F_SETFD, 0); 51 | fcntl($WritePipeFh, F_SETFD, 0); 52 | 53 | if (!fork) { 54 | exec $^X, $0, fileno($WritePipeFh) 55 | || die "exec failed: $!"; 56 | } 57 | 58 | my $ChildFdCount = <$ReadPipeFh>; 59 | chomp $ChildFdCount; 60 | ok($ChildFdCount < $CacheCount, "Less fd's in child than caches: $ChildFdCount < $CacheCount"); 61 | 62 | done_testing(2); 63 | 64 | -------------------------------------------------------------------------------- /t/15.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | 4 | use Test::More tests => 9; 5 | BEGIN { use_ok('Cache::FastMmap') }; 6 | use Data::Dumper; 7 | use strict; 8 | 9 | ######################### 10 | 11 | # Test writeback and cache_not_found option 12 | 13 | # Test a backing store just made of a local hash 14 | my %BackingStore = ( 15 | foo => { key1 => '123abc' }, 16 | bar => undef 17 | ); 18 | 19 | my %OrigBackingStore = %BackingStore; 20 | 21 | my $RCBCalled = 0; 22 | 23 | my $FC = Cache::FastMmap->new( 24 | cache_not_found => 1, 25 | init_file => 1, 26 | num_pages => 89, 27 | page_size => 1024, 28 | context => \%BackingStore, 29 | read_cb => sub { $RCBCalled++; return $_[0]->{$_[1]}; }, 30 | write_cb => sub { $_[0]->{$_[1]} = $_[2]; }, 31 | delete_cb => sub { delete $_[0]->{$_[1]} }, 32 | write_action => 'write_back' 33 | ); 34 | 35 | ok( defined $FC ); 36 | 37 | # Should pull from the backing store 38 | ok( eq_hash( $FC->get('foo'), { key1 => '123abc' } ), "cb get 1"); 39 | is( $FC->get('bar'), undef, "cb get 2"); 40 | is( $RCBCalled, 2, "cb get 2"); 41 | 42 | # Should be in the cache now 43 | ok( eq_hash( $FC->get('foo'), { key1 => '123abc' } ), "cb get 3"); 44 | is( $FC->get('bar'), undef, "cb get 4"); 45 | is( $RCBCalled, 2, "cb get 2"); 46 | 47 | # Need to make them dirty 48 | $FC->set('foo', { key1 => '123abc' }); 49 | $FC->set('bar', undef); 50 | 51 | # Should force cache data back to backing store 52 | %BackingStore = (); 53 | $FC->empty(); 54 | 55 | ok( eq_hash(\%BackingStore, \%OrigBackingStore), "items match 1" . Dumper(\%BackingStore, \%OrigBackingStore)); 56 | 57 | -------------------------------------------------------------------------------- /t/8.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | 4 | use Test::More tests => 17; 5 | BEGIN { use_ok('Cache::FastMmap') }; 6 | use strict; 7 | 8 | ######################### 9 | 10 | # Insert your test code below, the Test::More module is use()ed here so read 11 | # its man page ( perldoc Test::More ) for help writing this test script. 12 | 13 | my $FC = Cache::FastMmap->new(init_file => 1); 14 | ok( defined $FC ); 15 | 16 | # Test empty cache 17 | ok( !defined $FC->get(''), "empty get('')" ); 18 | 19 | ok( $FC->set('123', 'abc'), "set('123', 'abc')" ); 20 | ok( $FC->get('123') eq 'abc', "get('123') eq 'abc'"); 21 | 22 | ok( $FC->set('123', undef), "set('123', undef)" ); 23 | ok( !defined $FC->get('123'), "!defined get('123')"); 24 | 25 | ok( $FC->set('123', [ 'abc' ]), "set('123', [ 'abc' ])" ); 26 | ok( eq_array($FC->get('123'), [ 'abc' ]), "get('123') eq [ 'abc' ]"); 27 | 28 | # Check UTF8 29 | ok( $FC->set("key\x{263A}", [ "val\x{263A}" ]), "set utf8 key/val" ); 30 | ok( eq_array($FC->get("key\x{263A}"), [ "val\x{263A}" ]), "get utf8 key/val" ); 31 | 32 | is( join(",", sort $FC->get_keys), "123,key\x{263A}", "get_keys 1"); 33 | 34 | my %keys = map { $_->{key} => $_ } $FC->get_keys(2); 35 | is( scalar(keys %keys), 2, "get_keys 2" ); 36 | ok( eq_array($keys{123}->{value}, [ "abc" ]), "get_keys 3"); 37 | ok( eq_array($keys{"key\x{263A}"}->{value}, [ "val\x{263A}" ]), "get_keys 4"); 38 | 39 | # Check clearing actually works 40 | $FC->clear(); 41 | 42 | ok( !defined $FC->get('123'), "post clear 1" ); 43 | ok( !defined $FC->get("key\x{263A}"), "post clear 6" ); 44 | 45 | -------------------------------------------------------------------------------- /t/19.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | 4 | use Test::More; 5 | 6 | BEGIN { 7 | eval "use JSON (); use Sereal ();"; 8 | if ($@) { 9 | plan skip_all => 'No JSON/Sereal, no json/sereal storage tests'; 10 | } else { 11 | plan tests => 5; 12 | } 13 | use_ok('Cache::FastMmap'); 14 | } 15 | 16 | use Time::HiRes qw(time); 17 | use Data::Dumper; 18 | use strict; 19 | 20 | ######################### 21 | 22 | my $FCStorable = Cache::FastMmap->new(serializer => 'storable', init_file => 1); 23 | ok( defined $FCStorable ); 24 | my $FCJson = Cache::FastMmap->new(serializer => 'json', init_file => 1); 25 | ok( defined $FCJson ); 26 | my $FCSereal = Cache::FastMmap->new(serializer => 'sereal', init_file => 1); 27 | ok( defined $FCSereal ); 28 | 29 | eval { $FCJson->set("foo2", { key1 => '123abc', key2 => \"bar" }); }; 30 | ok( $@ =~ /cannot encode reference to scalar/ ); 31 | 32 | my $StorableTime = DoTests($FCStorable); 33 | my $JsonTime = DoTests($FCJson); 34 | my $SerealTime = DoTests($FCSereal); 35 | 36 | # Lets not assume these everywhere as test breakers 37 | # ok ($StorableTime > $SerealTime, "Sereal faster than storable"); 38 | # ok ($StorableTime > $JsonTime, "Json faster than storable"); 39 | 40 | sub DoTests { 41 | my $FC = shift; 42 | 43 | for (1..10000) { 44 | $FC->set("foo$_", { key1 => 'boom', key2 => "woot$_" }); 45 | } 46 | 47 | my $Start = time; 48 | for (1..10000) { 49 | $FC->set("foo$_", { key1 => '123abc', key2 => "bar$_" }); 50 | my $H = $FC->get("foo$_"); 51 | keys %$H == 2 || die; 52 | $H->{key1} eq '123abc' || die; 53 | $H->{key2} eq "bar$_" || die; 54 | } 55 | my $End = time; 56 | return $End-$Start; 57 | } 58 | 59 | -------------------------------------------------------------------------------- /t/12.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | our ($IsWin, $Tests); 4 | 5 | BEGIN { 6 | $IsWin = 0; 7 | $Tests = 7; 8 | 9 | if ($^O eq "MSWin32") { 10 | $IsWin = 1; 11 | $Tests -= 2; 12 | } 13 | } 14 | 15 | use Test::More tests => $Tests; 16 | BEGIN { use_ok('Cache::FastMmap') }; 17 | use strict; 18 | 19 | ######################### 20 | 21 | # Insert your test code below, the Test::More module is use()ed here so read 22 | # its man page ( perldoc Test::More ) for help writing this test script. 23 | 24 | my $FC = Cache::FastMmap->new(init_file => 1, serializer => ''); 25 | ok( defined $FC ); 26 | 27 | # Check get_and_set() 28 | 29 | ok( $FC->set("cnt", 1), "set counter" ); 30 | is( $FC->get_and_set("cnt", sub { return ++$_[1]; }), 2, "get_and_set 1" ); 31 | is( $FC->get_and_set("cnt", sub { return ++$_[1]; }), 3, "get_and_set 2" ); 32 | 33 | # Basic atomicness test 34 | 35 | my $loops = 5000; 36 | if (!$IsWin) { 37 | 38 | $FC->set("cnt", 0); 39 | if (my $pid = fork()) { 40 | for (1 .. $loops) { 41 | $FC->get_and_set("cnt", sub { return ++$_[1]; }); 42 | } 43 | waitpid($pid, 0); 44 | is( $FC->get("cnt"), $loops*2, "get_and_set 1"); 45 | 46 | } else { 47 | for (1 .. $loops) { 48 | $FC->get_and_set("cnt", sub { return ++$_[1]; }); 49 | } 50 | CORE::exit(0); 51 | } 52 | 53 | } 54 | 55 | # Check get_and_remove() 56 | 57 | if (!$IsWin) { 58 | 59 | my $got_but_didnt_remove = 0; 60 | if (my $pid = fork()) { 61 | for (1..$loops) { 62 | $FC->set("cnt", "data"); 63 | my ($got, $did_remove) = $FC->get_and_remove("cnt"); 64 | # With atomicity, we should never get something out, but fail to remove something: 65 | $got_but_didnt_remove++ if $got && !$did_remove; 66 | } 67 | waitpid($pid, 0); 68 | is( $got_but_didnt_remove, 0, "get_and_remove 1" ); 69 | } else { 70 | for (1..$loops) { 71 | $FC->remove("cnt"); 72 | } 73 | CORE::exit(0); 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /t/9.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | 4 | use Test::More tests => 5; 5 | BEGIN { use_ok('Cache::FastMmap') }; 6 | use Storable qw(freeze thaw); 7 | use strict; 8 | 9 | ######################### 10 | 11 | # Insert your test code below, the Test::More module is use()ed here so read 12 | # its man page ( perldoc Test::More ) for help writing this test script. 13 | 14 | my $FC = Cache::FastMmap->new( 15 | page_size => 8192, 16 | num_pages => 3, 17 | init_file => 1, 18 | serializer => '' 19 | ); 20 | ok( defined $FC ); 21 | 22 | sub rand_str { 23 | return join '', map { chr(rand(26) + ord('a')) } 1 .. int($_[0]); 24 | } 25 | 26 | srand(123); 27 | 28 | my $Bad = 0; 29 | 30 | my @LastEntries; 31 | foreach my $i (1 .. 500) { 32 | my $K = rand_str(rand(10) + 10); 33 | my $V = rand_str(rand(20) + 20); 34 | 35 | $FC->set($K, $V); 36 | push @LastEntries, [ $K, $V ]; 37 | shift @LastEntries if @LastEntries > 25; 38 | 39 | foreach my $e (0 .. @LastEntries-1) { 40 | local $_ = $LastEntries[$e]; 41 | if ($_->[1] ne ($FC->get($_->[0]) || '')) { 42 | $Bad = 1; 43 | last; 44 | } 45 | } 46 | last if $Bad; 47 | select(undef, undef, undef, 0.01); 48 | } 49 | 50 | ok( !$Bad ); 51 | 52 | $FC = Cache::FastMmap->new( 53 | page_size => 8192, 54 | num_pages => 3, 55 | init_file => 1 56 | ); 57 | 58 | ok( defined $FC ); 59 | 60 | $Bad = 0; 61 | @LastEntries = (); 62 | 63 | foreach my $i (1 .. 500) { 64 | my $K = rand_str(rand(10) + 10); 65 | my $V = [ rand_str(rand(20) + 20) ]; 66 | 67 | $FC->set($K, $V); 68 | push @LastEntries, [ $K, freeze($V) ]; 69 | shift @LastEntries if @LastEntries > 25; 70 | 71 | foreach my $e (0 .. @LastEntries-1) { 72 | local $_ = $LastEntries[$e]; 73 | if ($_->[1] ne freeze($FC->get($_->[0]) || [])) { 74 | $Bad = 1; 75 | last; 76 | } 77 | } 78 | last if $Bad; 79 | select(undef, undef, undef, 0.01); 80 | } 81 | 82 | ok( !$Bad ); 83 | -------------------------------------------------------------------------------- /t/21.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | 4 | use Test::More; 5 | use strict; 6 | 7 | my $fs; 8 | BEGIN { 9 | my @fs = map { { /(\w+)="([^"]*)"/g } } split /\n/, `findmnt -t tmpfs -P -b -o TARGET,AVAIL | grep /tmp`; 10 | ($fs) = grep { $_->{AVAIL} > 2**33 } @fs; 11 | if (!$fs) { 12 | plan skip_all => 'Large file tests need tmpfs with at least 8G'; 13 | } 14 | } 15 | 16 | BEGIN { 17 | plan tests => 6; 18 | use_ok('Cache::FastMmap') 19 | }; 20 | 21 | 22 | ######################### 23 | 24 | # Insert your test code below, the Test::More module is use()ed here so read 25 | # its man page ( perldoc Test::More ) for help writing this test script. 26 | 27 | my $cache_file = $fs->{TARGET} . "/largecachetest.cache"; 28 | 29 | my %WrittenItems; 30 | my $FC = Cache::FastMmap->new( 31 | share_file => $cache_file, 32 | serializer => '', 33 | init_file => 1, 34 | num_pages => 8, 35 | page_size => 2**30, 36 | ); 37 | 38 | ok( defined $FC ); 39 | 40 | srand(6543); 41 | 42 | # Put 1000 items in the cache - should be big enough :) 43 | for (1 .. 1000) { 44 | my ($Key, $Val) = (RandStr(100), RandStr(1000)); 45 | $FC->set($Key, $Val); 46 | $WrittenItems{$Key} = $Val; 47 | } 48 | 49 | # Get values in cache should be 1000 50 | my %CacheItems = map { $_->{key} => $_->{value} } $FC->get_keys(2); 51 | ok( scalar(keys %CacheItems) == 1000, "1000 items in cache"); 52 | 53 | # Should be able to read all items 54 | my ($Failed, $GetFailed) = (0, 0); 55 | for (keys %WrittenItems) { 56 | $Failed++ if $FC->get($_) ne $WrittenItems{$_}; 57 | $GetFailed++ if $FC->get($_) ne $CacheItems{$_}; 58 | } 59 | 60 | ok( $Failed == 0, "got all written items" ); 61 | ok( $GetFailed == 0, "got all get_keys items" ); 62 | 63 | # Now there should be nothing left 64 | $FC->clear(); 65 | 66 | %CacheItems = map { $_->{key} => $_->{value} } $FC->get_keys(2); 67 | ok( scalar(keys %CacheItems) == 0, "empty cache 2"); 68 | 69 | sub RandStr { 70 | return join '', map { chr(ord('a') + rand(26)) } (1 .. $_[0]); 71 | } 72 | 73 | -------------------------------------------------------------------------------- /t/5.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | 4 | use Test::More tests => 9; 5 | BEGIN { use_ok('Cache::FastMmap') }; 6 | use strict; 7 | 8 | ######################### 9 | 10 | # Insert your test code below, the Test::More module is use()ed here so read 11 | # its man page ( perldoc Test::More ) for help writing this test script. 12 | 13 | # Test a backing store just made of a local hash 14 | my %BackingStore = ( 15 | foo => '123abc', 16 | bar => '456def' 17 | ); 18 | 19 | my %WrittenItems = %BackingStore; 20 | 21 | my $FC = Cache::FastMmap->new( 22 | serializer => '', 23 | init_file => 1, 24 | num_pages => 89, 25 | page_size => 2048, 26 | context => \%BackingStore, 27 | read_cb => sub { return $_[0]->{$_[1]}; }, 28 | write_cb => sub { $_[0]->{$_[1]} = $_[2]; }, 29 | delete_cb => sub { delete $_[0]->{$_[1]} }, 30 | write_action => 'write_through' 31 | ); 32 | 33 | ok( defined $FC ); 34 | 35 | srand(6543); 36 | 37 | # Put 3000 items in the cache 38 | for (1 .. 3000) { 39 | my ($Key, $Val) = (RandStr(10), RandStr(100)); 40 | $FC->set($Key, $Val); 41 | $WrittenItems{$Key} = $Val; 42 | } 43 | 44 | # Get values in cache 45 | my %CacheItems = map { $_->{key} => $_->{value} } $FC->get_keys(2); 46 | 47 | # Reality check approximate number of items in each 48 | ok( scalar(keys %BackingStore) == 3002, "backing store size 1" ); 49 | ok( scalar(keys %CacheItems) > 500, "backing store size 2" ); 50 | 51 | # Should be equal to all items we wrote 52 | ok( eq_hash(\%BackingStore, \%WrittenItems), "items match 1"); 53 | 54 | # Check we can get the items we wrote 55 | is( $FC->get('foo'), '123abc', "cb get 1"); 56 | is( $FC->get('bar'), '456def', "cb get 2"); 57 | 58 | # Read them forward and backward 59 | my $Failed = 0; 60 | for (keys %WrittenItems, reverse keys %WrittenItems) { 61 | $Failed++ if $FC->get($_) ne $WrittenItems{$_}; 62 | } 63 | 64 | ok( $Failed == 0, "got all written items" ); 65 | 66 | ok( eq_hash(\%WrittenItems, \%BackingStore), "items match 2"); 67 | 68 | sub RandStr { 69 | return join '', map { chr(ord('a') + rand(26)) } (1 .. $_[0]); 70 | } 71 | 72 | -------------------------------------------------------------------------------- /t/3.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | 4 | use Test::More tests => 8; 5 | BEGIN { use_ok('Cache::FastMmap') }; 6 | use strict; 7 | 8 | ######################### 9 | 10 | # Insert your test code below, the Test::More module is use()ed here so read 11 | # its man page ( perldoc Test::More ) for help writing this test script. 12 | 13 | my $FC = Cache::FastMmap->new(init_file => 1, serializer => ''); 14 | ok( defined $FC ); 15 | 16 | my (@Keys, $HitRate); 17 | $HitRate = RepeatMixTest($FC, 1000, 0.0, \@Keys); 18 | ok( $HitRate == 0.0, "hit rate 1"); 19 | 20 | $HitRate = RepeatMixTest($FC, 1000, 0.5, \@Keys); 21 | ok( $HitRate == 1.0, "hit rate 2"); 22 | 23 | $HitRate = RepeatMixTest($FC, 1000, 0.8, \@Keys); 24 | ok( $HitRate == 1.0, "hit rate 3"); 25 | 26 | $FC = undef; 27 | @Keys = (); 28 | 29 | # Should be repeatable 30 | srand(123456); 31 | 32 | $FC = Cache::FastMmap->new( 33 | init_file => 1, 34 | page_size => 8192, 35 | serializer => '' 36 | ); 37 | ok( defined $FC ); 38 | 39 | $HitRate = RepeatMixTest($FC, 1000, 0.0, \@Keys); 40 | ok( $HitRate == 0.0, "hit rate 1"); 41 | $HitRate = RepeatMixTest($FC, 10000, 0.5, \@Keys); 42 | ok( $HitRate > 0.8 && $HitRate < 0.95, "hit rate 4 - $HitRate"); 43 | 44 | 45 | sub RepeatMixTest { 46 | my ($FC, $NItems, $Ratio, $WroteKeys) = @_; 47 | 48 | my ($Read, $ReadHit); 49 | 50 | # Lots of random tests 51 | for (1 .. $NItems) { 52 | 53 | # Read/write ratio 54 | if (rand() < $Ratio) { 55 | 56 | # Pick a key from known written ones 57 | my $K = $WroteKeys->[ rand(@$WroteKeys) ]; 58 | my $V = $FC->get($K); 59 | $Read++; 60 | 61 | # Skip if not found in cache 62 | next if !defined $V; 63 | $ReadHit++; 64 | 65 | # Offset of 10 past first chars of value are key 66 | substr($V, 10, length($K)) eq $K 67 | || die "Cache/key not equal: $K, $V"; 68 | 69 | } else { 70 | 71 | my $K = RandStr(16); 72 | my $V = RandStr(10) . $K . RandStr(int(rand(200))); 73 | push @$WroteKeys, $K; 74 | $FC->set($K, $V); 75 | 76 | } 77 | } 78 | 79 | return $Read ? ($ReadHit/$Read) : 0.0; 80 | } 81 | 82 | sub RandStr { 83 | return join '', map { chr(ord('a') + rand(26)) } (1 .. $_[0]); 84 | } 85 | 86 | -------------------------------------------------------------------------------- /t/22.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | 4 | use Test::More tests => 19; 5 | BEGIN { use_ok('Cache::FastMmap') }; 6 | use Data::Dumper; 7 | use strict; 8 | 9 | ######################### 10 | 11 | # Test maintaining write back of expired items after get 12 | 13 | # Test a backing store just made of a local hash 14 | my %BackingStore = (); 15 | 16 | my $FC = Cache::FastMmap->new( 17 | serializer => '', 18 | init_file => 1, 19 | num_pages => 1, 20 | page_size => 8192, 21 | context => \%BackingStore, 22 | write_cb => sub { $_[0]->{$_[1]} = $_[2]; }, 23 | delete_cb => sub { delete $_[0]->{$_[1]} }, 24 | write_action => 'write_back' 25 | ); 26 | 27 | my $epoch = time; 28 | my $now = $epoch; 29 | Cache::FastMmap::_set_time_override($now); 30 | 31 | ok( defined $FC ); 32 | 33 | ok( $FC->set('foo', '123abc', 1), 'store item 1'); 34 | ok( $FC->set('bar', '456def', 2), 'store item 2'); 35 | ok( $FC->set('baz', '789ghi', 3), 'store item 3'); 36 | is( $FC->get('foo'), '123abc', "get item 1"); 37 | is( $FC->get('bar'), '456def', "get item 2"); 38 | is( $FC->get('baz'), '789ghi', "get item 3"); 39 | 40 | $now = $epoch+1; 41 | Cache::FastMmap::_set_time_override($now); 42 | 43 | is( $FC->get('foo'), undef, "get item 1 after sleep 1"); 44 | is( $FC->get('bar'), '456def', "get item 2 after sleep 1"); 45 | is( $FC->get('baz'), '789ghi', "get item 3 after sleep 1"); 46 | 47 | $FC->empty(1); 48 | 49 | ok( eq_hash(\%BackingStore, { foo => '123abc' }), "items match expire 1" ); 50 | 51 | $FC->expire('baz'); 52 | $FC->expire('dummy'); 53 | 54 | ok( eq_hash(\%BackingStore, { foo => '123abc', baz => '789ghi' }), "items match explicit expire" ); 55 | 56 | is( $FC->get('baz'), undef, "get item 3 after explicit expire"); 57 | 58 | $now = $epoch+2; 59 | Cache::FastMmap::_set_time_override($now); 60 | 61 | is( $FC->get('foo'), undef, "get item 1 after sleep 2"); 62 | is( $FC->get('bar'), undef, "get item 2 after sleep 2"); 63 | is( $FC->get('baz'), undef, "get item 3 after sleep 2"); 64 | 65 | $FC->empty(1); 66 | 67 | ok( eq_hash(\%BackingStore, { foo => '123abc', bar => '456def', baz => '789ghi' }), "items match expire 2"); 68 | 69 | $FC->remove('foo'); 70 | 71 | ok( eq_hash(\%BackingStore, { bar => '456def', baz => '789ghi' }), "items match remove 1"); 72 | 73 | 74 | -------------------------------------------------------------------------------- /t/4.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | 4 | use Test::More tests => 9; 5 | BEGIN { use_ok('Cache::FastMmap') }; 6 | use strict; 7 | 8 | ######################### 9 | 10 | # Insert your test code below, the Test::More module is use()ed here so read 11 | # its man page ( perldoc Test::More ) for help writing this test script. 12 | 13 | # Test a backing store just made of a local hash 14 | my %BackingStore = ( 15 | foo => '123abc', 16 | bar => '456def' 17 | ); 18 | 19 | my %WrittenItems = %BackingStore; 20 | 21 | my $FC = Cache::FastMmap->new( 22 | serializer => '', 23 | init_file => 1, 24 | num_pages => 3, 25 | page_size => 32768, 26 | context => \%BackingStore, 27 | read_cb => sub { return $_[0]->{$_[1]}; }, 28 | write_cb => sub { $_[0]->{$_[1]} = $_[2]; }, 29 | delete_cb => sub { delete $_[0]->{$_[1]} }, 30 | write_action => 'write_back' 31 | ); 32 | 33 | ok( defined $FC ); 34 | 35 | srand(6543); 36 | 37 | # Put 100 items in the cache (should be big enough) 38 | for (1 .. 100) { 39 | my ($Key, $Val) = (RandStr(10), RandStr(100)); 40 | $FC->set($Key, $Val); 41 | $WrittenItems{$Key} = $Val; 42 | } 43 | 44 | # Should only be 2 items in the backing store 45 | ok( scalar(keys %BackingStore) == 2, "items match 1"); 46 | 47 | # Should flush back all the items to backing store 48 | $FC->empty(); 49 | 50 | # Get values in cache should be empty 51 | my %CacheItems = map { $_->{key} => $_->{value} } $FC->get_keys(2); 52 | ok( scalar(keys %CacheItems) == 0, "empty cache"); 53 | 54 | # Backing store should be equal to all items we wrote 55 | ok( eq_hash(\%WrittenItems, \%BackingStore), "items match 1"); 56 | 57 | # Should be able to read all items 58 | my $Failed = 0; 59 | for (keys %WrittenItems) { 60 | $Failed++ if $FC->get($_) ne $WrittenItems{$_}; 61 | } 62 | 63 | ok( $Failed == 0, "got all written items 1" ); 64 | 65 | # Empty backing store 66 | %BackingStore = (); 67 | 68 | # Should still be able to read all items 69 | $Failed = 0; 70 | for (keys %WrittenItems) { 71 | $Failed++ if $FC->get($_) ne $WrittenItems{$_}; 72 | } 73 | 74 | ok( $Failed == 0, "got all written items 2" ); 75 | 76 | # Now there should be nothing left 77 | $FC->clear(); 78 | 79 | %CacheItems = map { $_->{key} => $_->{value} } $FC->get_keys(2); 80 | ok( scalar(keys %CacheItems) == 0, "empty cache 2"); 81 | ok( scalar(keys %BackingStore) == 0, "empty backing store 1"); 82 | 83 | sub RandStr { 84 | return join '', map { chr(ord('a') + rand(26)) } (1 .. $_[0]); 85 | } 86 | 87 | -------------------------------------------------------------------------------- /t/13.t: -------------------------------------------------------------------------------- 1 | use strict; 2 | use warnings; 3 | 4 | use Test::More; 5 | use Data::Dumper; 6 | $Data::Dumper::Deparse = 1; 7 | 8 | BEGIN { 9 | note 'Compression testing'; 10 | use_ok('Cache::FastMmap'); 11 | } 12 | 13 | my %compressors = ( 14 | lz4 => 'Compress::LZ4', 15 | snappy => 'Compress::Snappy', 16 | zlib => 'Compress::Zlib', 17 | ); 18 | 19 | for my $compressor ( keys %compressors ) { 20 | note " Testing with $compressors{$compressor}"; 21 | # Avoid prototype mismatch warnings 22 | if ( ! eval "require $compressors{$compressor};" ) { 23 | note " Cannot load $compressors{$compressor}: skipping tests: reason: $@"; 24 | next; 25 | } 26 | 27 | my $FC = Cache::FastMmap->new( 28 | page_size => 8192, 29 | num_pages => 1, 30 | init_file => 1, 31 | serializer => '', 32 | compressor => $compressor 33 | ); 34 | ok( defined $FC, 'create compressing cache' ); 35 | 36 | my $FCNC = Cache::FastMmap->new( 37 | page_size => 8192, 38 | num_pages => 1, 39 | init_file => 1, 40 | serializer => '', 41 | ); 42 | ok( defined $FCNC, 'create non-compressing cache of same size' ); 43 | 44 | my $K1 = rand_str(10); 45 | my $K2 = rand_str(10); 46 | my $V = rand_str(10) x 1000; 47 | 48 | ok( $FC->set($K1, $V), 'set() with large value in compressing cache' ); 49 | ok( $FC->set($K2, $V), 'also set() same value with different key' ); 50 | ok( !$FCNC->set($K1, $V), 'cannot set() same value in non-compressing cache' ); 51 | ok( !$FCNC->set($K2, $V), 'also fail to set() with different key' ); 52 | 53 | my $CV1 = $FC->get($K1); 54 | my $CV2 = $FC->get($K2); 55 | 56 | is( $CV1, $V, 'get() same large value from compressing cache' ); 57 | is( $CV2, $V, 'also get() same value with second key used' ); 58 | 59 | $CV1 = $FCNC->get($K1); 60 | $CV2 = $FCNC->get($K2); 61 | 62 | ok( !defined $CV1, 'cannot get() anything from non-compressing cache' ); 63 | ok( !defined $CV2, 'also fail to get() with second key used' ); 64 | } 65 | 66 | note ' Check support for deprecated `compress` param'; 67 | for ( 1, 'Compress::NonExistent', 'Compress::LZ4' ) { 68 | my $DCNC = Cache::FastMmap->new( 69 | page_size => 8192, 70 | num_pages => 1, 71 | init_file => 1, 72 | serializer => '', 73 | compress => $_, 74 | ); 75 | 76 | ok( defined $DCNC, 'create cache with `compress` param: ' . $_ ); 77 | 78 | my $wanted1 = quotemeta('&$uncompress(my $Tmp = shift())'); 79 | # rt115043: older versions of Data::Dumper would output like below. 80 | my $wanted2 = quotemeta('&$uncompress(my $Tmp = shift @_)'); 81 | my $wanted = qr/$wanted1|$wanted2/; 82 | my $got = Dumper $DCNC->{uncompress}; 83 | like( $got, $wanted, 'using `Compress::Zlib` as compressor' ); 84 | } 85 | 86 | 87 | done_testing; 88 | 89 | sub rand_str { 90 | return join '', map { chr(rand(26) + ord('a')) } 1 .. int($_[0]); 91 | } 92 | 93 | __END__ 94 | -------------------------------------------------------------------------------- /t/7.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | 4 | use Test::More tests => 13; 5 | BEGIN { use_ok('Cache::FastMmap') }; 6 | use strict; 7 | 8 | ######################### 9 | 10 | # Insert your test code below, the Test::More module is use()ed here so read 11 | # its man page ( perldoc Test::More ) for help writing this test script. 12 | 13 | # Test a backing store just made of a local hash 14 | my %BackingStore = ( 15 | foo => '123abc', 16 | bar => '456def' 17 | ); 18 | 19 | my %WrittenItems = %BackingStore; 20 | 21 | my $FC = Cache::FastMmap->new( 22 | serializer => '', 23 | init_file => 1, 24 | num_pages => 89, 25 | page_size => 1024, 26 | context => \%BackingStore, 27 | read_cb => sub { return $_[0]->{$_[1]}; }, 28 | write_cb => sub { $_[0]->{$_[1]} = $_[2]; }, 29 | delete_cb => sub { delete $_[0]->{$_[1]} }, 30 | write_action => 'write_back', 31 | empty_on_exit => 1 32 | ); 33 | 34 | ok( defined $FC ); 35 | 36 | srand(6543); 37 | 38 | # Put 3000 items in the cache 39 | for (1 .. 3000) { 40 | my ($Key, $Val) = (RandStr(10), RandStr(100)); 41 | $FC->set($Key, $Val); 42 | $WrittenItems{$Key} = $Val; 43 | } 44 | 45 | # Get values in cache 46 | my %CacheItems = map { $_->{key} => $_->{value} } $FC->get_keys(2); 47 | 48 | # Reality check approximate number of items in each 49 | ok( scalar(keys %BackingStore) < 2700, "backing store size 1" ); 50 | ok( scalar(keys %CacheItems) > 300, "backing store size 2" ); 51 | 52 | # Merge with backing store items 53 | my %AllItems = (%BackingStore, %CacheItems); 54 | 55 | # Should be equal to all items we wrote 56 | ok( eq_hash(\%AllItems, \%WrittenItems), "items match 1"); 57 | 58 | # Check we can get the items we wrote 59 | is( $FC->get('foo'), '123abc', "cb get 1"); 60 | is( $FC->get('bar'), '456def', "cb get 2"); 61 | 62 | # Read them forward and backward, which should force 63 | # complete flush and read from backing store 64 | my $Failed = 0; 65 | for (keys %WrittenItems, reverse keys %WrittenItems) { 66 | $Failed++ if $FC->get($_) ne $WrittenItems{$_}; 67 | } 68 | 69 | ok( $Failed == 0, "got all written items 1" ); 70 | 71 | # Delete some items (should be random from cache/backing store) 72 | my @DelKeys = (keys %WrittenItems)[0 .. 300]; 73 | for (@DelKeys) { 74 | $FC->remove($_); 75 | delete $WrittenItems{$_}; 76 | } 77 | 78 | # Check it all matches again 79 | %CacheItems = map { $_->{key} => $_->{value} } $FC->get_keys(2); 80 | %AllItems = (%BackingStore, %CacheItems); 81 | ok( eq_hash(\%AllItems, \%WrittenItems), "items match 2"); 82 | 83 | $Failed = 0; 84 | for (keys %WrittenItems) { 85 | $Failed++ if $FC->get($_) ne $WrittenItems{$_}; 86 | } 87 | 88 | ok( $Failed == 0, "got all written items 2" ); 89 | 90 | # Force flushing of cache 91 | $FC->empty(); 92 | 93 | # So all written items should be in backing store 94 | ok( eq_hash(\%WrittenItems, \%BackingStore), "items match 3"); 95 | 96 | my @Keys = $FC->get_keys(0); 97 | ok( scalar(@Keys) == 0, "no items left in cache" ); 98 | 99 | %WrittenItems = %BackingStore = (); 100 | 101 | # Put 3000 items in the cache 102 | for (1 .. 3000) { 103 | my ($Key, $Val) = (RandStr(10), RandStr(100)); 104 | $FC->set($Key, $Val); 105 | $WrittenItems{$Key} = $Val; 106 | } 107 | 108 | # empty_on_exit is set, so this should push to backing store 109 | $FC = undef; 110 | 111 | ok( eq_hash(\%WrittenItems, \%BackingStore), "items match 4"); 112 | 113 | sub RandStr { 114 | return join '', map { chr(ord('a') + rand(26)) } (1 .. $_[0]); 115 | } 116 | 117 | -------------------------------------------------------------------------------- /mmap_cache_internals.h: -------------------------------------------------------------------------------- 1 | #ifndef mmap_cache_internals_h 2 | #define mmap_cache_internals_h 3 | 4 | #ifdef DEBUG 5 | #define ASSERT(x) assert(x) 6 | #include 7 | #else 8 | #define ASSERT(x) 9 | #endif 10 | 11 | #ifdef WIN32 12 | #include 13 | #endif 14 | 15 | /* Cache structure */ 16 | struct mmap_cache { 17 | 18 | /* Current page details */ 19 | void * p_base; 20 | MU32 * p_base_slots; 21 | MU32 p_cur; 22 | MU64 p_offset; 23 | 24 | MU32 p_num_slots; 25 | MU32 p_free_slots; 26 | MU32 p_old_slots; 27 | MU32 p_free_data; 28 | MU32 p_free_bytes; 29 | MU32 p_n_reads; 30 | MU32 p_n_read_hits; 31 | 32 | int p_changed; 33 | 34 | /* General page details */ 35 | MU32 c_num_pages; 36 | MU32 c_page_size; 37 | MU64 c_size; 38 | 39 | /* Pointer to mmapped area */ 40 | void * mm_var; 41 | 42 | /* Cache general details */ 43 | MU32 start_slots; 44 | MU32 expire_time; 45 | int catch_deadlocks; 46 | int enable_stats; 47 | 48 | /* Share mmap file details */ 49 | #ifdef WIN32 50 | HANDLE fh; 51 | #else 52 | int fh; 53 | MU64 inode; 54 | #endif 55 | char * share_file; 56 | int permissions; 57 | int init_file; 58 | int test_file; 59 | int cache_not_found; 60 | 61 | /* Last error string */ 62 | char * last_error; 63 | 64 | }; 65 | 66 | struct mmap_cache_it { 67 | mmap_cache * cache; 68 | MU32 p_cur; 69 | MU32 * slot_ptr; 70 | MU32 * slot_ptr_end; 71 | }; 72 | 73 | /* Macros to access page entries */ 74 | #define PP(p) ((MU32 *)p) 75 | 76 | #define P_Magic(p) (*(PP(p)+0)) 77 | #define P_NumSlots(p) (*(PP(p)+1)) 78 | #define P_FreeSlots(p) (*(PP(p)+2)) 79 | #define P_OldSlots(p) (*(PP(p)+3)) 80 | #define P_FreeData(p) (*(PP(p)+4)) 81 | #define P_FreeBytes(p) (*(PP(p)+5)) 82 | #define P_NReads(p) (*(PP(p)+6)) 83 | #define P_NReadHits(p) (*(PP(p)+7)) 84 | 85 | #define P_HEADERSIZE 32 86 | 87 | /* Macros to access cache slot entries */ 88 | #define SP(s) ((MU32 *)s) 89 | 90 | /* Offset pointer 'p' by 'o' bytes */ 91 | #define PTR_ADD(p,o) ((void *)((char *)p + o)) 92 | 93 | /* Given a data pointer, get key len, value len or combined len */ 94 | #define S_Ptr(b,s) ((MU32 *)PTR_ADD(b, s)) 95 | 96 | #define S_LastAccess(s) (*(s+0)) 97 | #define S_ExpireOn(s) (*(s+1)) 98 | #define S_SlotHash(s) (*(s+2)) 99 | #define S_Flags(s) (*(s+3)) 100 | #define S_KeyLen(s) (*(s+4)) 101 | #define S_ValLen(s) (*(s+5)) 102 | 103 | #define S_KeyPtr(s) ((void *)(s+6)) 104 | #define S_ValPtr(s) (PTR_ADD((void *)(s+6), S_KeyLen(s))) 105 | 106 | /* Length of slot data including key and value data */ 107 | #define S_SlotLen(s) (sizeof(MU32)*6 + S_KeyLen(s) + S_ValLen(s)) 108 | #define KV_SlotLen(k,v) (sizeof(MU32)*6 + k + v) 109 | /* Found key/val len to nearest 4 bytes */ 110 | #define ROUNDLEN(l) ((l) += 3 - (((l)-1) & 3)) 111 | 112 | /* Externs from mmap_cache.c */ 113 | extern char * def_share_file; 114 | extern MU32 def_init_file; 115 | extern MU32 def_test_file; 116 | extern MU32 def_expire_time; 117 | extern MU32 def_c_num_pages; 118 | extern MU32 def_c_page_size; 119 | extern MU32 def_start_slots; 120 | extern char* _mmc_get_def_share_filename(mmap_cache * cache); 121 | 122 | /* Platform specific functions defined in unix.c | win32.c */ 123 | int mmc_open_cache_file(mmap_cache* cache, int * do_init); 124 | int mmc_map_memory(mmap_cache* cache); 125 | int mmc_unmap_memory(mmap_cache* cache); 126 | int mmc_lock_page(mmap_cache* cache, MU64 p_offset); 127 | int mmc_unlock_page(mmap_cache * cache, MU64 p_offset); 128 | int mmc_check_fh(mmap_cache* cache); 129 | int mmc_close_fh(mmap_cache* cache); 130 | int _mmc_set_error(mmap_cache *cache, int err, char * error_string, ...); 131 | char* _mmc_get_def_share_filename(mmap_cache * cache); 132 | 133 | #endif 134 | 135 | -------------------------------------------------------------------------------- /t/2.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | 4 | use Test::More tests => 51; 5 | use Test::Deep; 6 | BEGIN { use_ok('Cache::FastMmap') }; 7 | use strict; 8 | 9 | ######################### 10 | 11 | # Insert your test code below, the Test::More module is use()ed here so read 12 | # its man page ( perldoc Test::More ) for help writing this test script. 13 | 14 | my $FC = Cache::FastMmap->new(init_file => 1, expire_time => 3, serializer => ''); 15 | ok( defined $FC ); 16 | my $FC2 = Cache::FastMmap->new(init_file => 1, expire_time => 5, serializer => ''); 17 | ok( defined $FC2 ); 18 | 19 | my $epoch = time; 20 | my $now = $epoch; 21 | Cache::FastMmap::_set_time_override($now); 22 | 23 | ok( $FC->set('abc', '123'), "expire set 1"); 24 | is( $FC->get('abc'), '123', "expire get 2"); 25 | 26 | ok( $FC2->set('abc', '123'), "expire set 3"); 27 | ok( $FC2->set('def', '123', 3), "expire set 4"); 28 | ok( $FC2->set('ghi', '123', 'now'), "expire set 5"); 29 | ok( $FC2->set('jkl', '123', 'never'), "expire set 6"); 30 | is( $FC2->get('abc'), '123', "expire get 7"); 31 | is( $FC2->get('def'), '123', "expire get 8"); 32 | is( $FC2->get('ghi'), undef, "expire get 9"); 33 | is( $FC2->get('jkl'), '123', "expire get 10"); 34 | 35 | ok( $FC2->set('mno', '123'), "expire get_and_set 1"); 36 | is( scalar $FC2->get_and_set('mno', sub { return ("456", { expire_time => 1 }) }), '456', "expire get_and_set 2"); 37 | is( $FC2->get('mno'), '456', "expire get_and_set 3"); 38 | 39 | my @e = $FC2->get_keys(2); 40 | cmp_deeply( 41 | \@e, 42 | bag( 43 | superhashof({ key => 'abc', value => '123', last_access => num($now, 1), expire_on => num($now+5, 1) }), 44 | superhashof({ key => 'def', value => '123', last_access => num($now, 1), expire_on => num($now+3, 1) }), 45 | superhashof({ key => 'jkl', value => '123', last_access => num($now, 1), expire_on => 0 }), 46 | superhashof({ key => 'mno', value => '456', last_access => num($now, 1), expire_on => num($now+1, 1) }), 47 | ), 48 | "got expected keys" 49 | ) || diag explain [ $now, \@e ]; 50 | 51 | $now = $epoch+2; 52 | Cache::FastMmap::_set_time_override($now); 53 | 54 | ok( $FC->set('def', '456'), "expire set 11"); 55 | is( $FC->get('abc'), '123', "expire get 12"); 56 | is( $FC->get('def'), '456', "expire get 13"); 57 | 58 | is( $FC2->get('abc'), '123', "expire get 14"); 59 | is( $FC2->get('def'), '123', "expire get 15"); 60 | ok( !defined $FC2->get('ghi'), "expire get 16"); 61 | is( $FC2->get('jkl'), '123', "expire get 17"); 62 | 63 | ok( !defined $FC2->get('mno'), "expire get_and_set 4"); 64 | 65 | $now = $epoch+4; 66 | Cache::FastMmap::_set_time_override($now); 67 | 68 | ok( !defined $FC->get('abc'), "expire get 18"); 69 | is( $FC->get('def'), '456', "expire get 19"); 70 | 71 | is( $FC2->get('abc'), '123', "expire get 20"); 72 | ok( !defined $FC2->get('def'), "expire get 21"); 73 | ok( !defined $FC2->get('ghi'), "expire get 22"); 74 | is( $FC2->get('jkl'), '123', "expire get 23"); 75 | 76 | $now = $epoch+6; 77 | Cache::FastMmap::_set_time_override($now); 78 | 79 | ok( !defined $FC->get('abc'), "expire get 24"); 80 | ok( !defined $FC->get('def'), "expire get 25"); 81 | 82 | ok( !defined $FC2->get('abc'), "expire get 26"); 83 | ok( !defined $FC2->get('def'), "expire get 27"); 84 | ok( !defined $FC2->get('ghi'), "expire get 28"); 85 | is( $FC2->get('jkl'), '123', "expire get 29"); 86 | 87 | ok( $FC->set('abc', '123', '1s'), "expire set 31"); 88 | ok( $FC->set('abc', '123', '1m'), "expire set 32"); 89 | ok( $FC->set('abc', '123', '1d'), "expire set 33"); 90 | ok( $FC->set('abc', '123', '1w'), "expire set 34"); 91 | 92 | ok( $FC->set('abc', '123', '1 second'), "expire set 41"); 93 | ok( $FC->set('abc', '123', '1 minute'), "expire set 42"); 94 | ok( $FC->set('abc', '123', '1 day'), "expire set 43"); 95 | ok( $FC->set('abc', '123', '1 week'), "expire set 44"); 96 | 97 | ok( $FC->set('abc', '123', 'now'), "expire set 45"); 98 | ok( $FC->set('abc', '123', 'never'), "expire set 46"); 99 | 100 | ok( $FC->set('abc', '123', 's'), "expire set 47"); 101 | ok( $FC->set('abc', '123', ''), "expire set 48"); 102 | ok( $FC->set('abc', '123', -1), "expire set 49"); 103 | ok( $FC->set('abc', '123', 'garbage'), "expire set 50"); 104 | -------------------------------------------------------------------------------- /t/23.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | 4 | use Test::More tests => 27; 5 | use Test::Deep; 6 | BEGIN { use_ok('Cache::FastMmap') }; 7 | use Data::Dumper; 8 | use strict; 9 | 10 | ######################### 11 | 12 | # Test maintaining expire_on through get_and_set 13 | 14 | # Test a backing store just made of a local hash 15 | my %BackingStore = (); 16 | 17 | my $FC = Cache::FastMmap->new( 18 | serializer => '', 19 | init_file => 1, 20 | num_pages => 1, 21 | page_size => 8192, 22 | context => \%BackingStore, 23 | write_cb => sub { $_[0]->{$_[1]} = $_[2]; }, 24 | write_action => 'write_back', 25 | expire_time => 3, 26 | ); 27 | 28 | my $epoch = time; 29 | my $now = $epoch; 30 | Cache::FastMmap::_set_time_override($now); 31 | 32 | ok( defined $FC ); 33 | 34 | ok( $FC->set('foo', '123abc', 2), 'store item 1'); 35 | ok( $FC->set('bar', '456def', 3), 'store item 2'); 36 | ok( $FC->set('baz', '789ghi'), 'store item 3'); 37 | is( $FC->get('foo'), '123abc', "get item 1"); 38 | is( $FC->get('bar'), '456def', "get item 2"); 39 | is( $FC->get('baz'), '789ghi', "get item 3"); 40 | 41 | $now = $epoch+1; 42 | Cache::FastMmap::_set_time_override($now); 43 | 44 | sub cb { return ( (defined $_[1] ? $_[1] : 'boo') . 'a', { expire_on => $_[2]->{expire_on} }); }; 45 | sub cb2 { return ($_[1] . 'a'); }; 46 | is( $FC->get_and_set('foo', \&cb), '123abca', "get_and_set item 1 after sleep 1"); 47 | is( $FC->get_and_set('bar', \&cb), '456defa', "get_and_set item 2 after sleep 1"); 48 | is( $FC->get_and_set('baz', \&cb2), '789ghia', "get_and_set item 3 after sleep 1"); 49 | is( $FC->get_and_set('gah', \&cb), 'booa', "get_and_set item 4 after sleep 1"); 50 | 51 | my @e = $FC->get_keys(2); 52 | cmp_deeply( 53 | \@e, 54 | bag( 55 | superhashof({ key => 'foo', value => '123abca', last_access => num($now, 1), expire_on => num($now+1, 1) }), 56 | superhashof({ key => 'bar', value => '456defa', last_access => num($now, 1), expire_on => num($now+2, 1) }), 57 | superhashof({ key => 'baz', value => '789ghia', last_access => num($now, 1), expire_on => num($now+3, 1) }), 58 | superhashof({ key => 'gah', value => 'booa', last_access => num($now, 1), expire_on => num($now+3, 1) }), 59 | ), 60 | "got expected keys" 61 | ) || diag explain [ $now, \@e ]; 62 | 63 | $now = $epoch+2; 64 | Cache::FastMmap::_set_time_override($now); 65 | 66 | is( $FC->get('foo'), undef, "get item 1 after sleep 2"); 67 | is( $FC->get('bar'), '456defa', "get item 2 after sleep 2"); 68 | is( $FC->get('baz'), '789ghia', "get item 3 after sleep 2"); 69 | 70 | is( $FC->get_and_set('bar', \&cb), '456defaa', "get_and_set item 2 after sleep 2"); 71 | 72 | @e = $FC->get_keys(2); 73 | cmp_deeply( 74 | \@e, 75 | bag( 76 | superhashof({ key => 'bar', value => '456defaa', last_access => num($now, 1), expire_on => num($now+1, 1) }), 77 | superhashof({ key => 'baz', value => '789ghia', last_access => num($now, 1), expire_on => num($now+2, 1) }), 78 | superhashof({ key => 'gah', value => 'booa', last_access => num($now-1, 1), expire_on => num($now+2, 1) }), 79 | ), 80 | "got expected keys" 81 | ) || diag explain [ $now, \@e ]; 82 | 83 | $now = $epoch+3; 84 | Cache::FastMmap::_set_time_override($now); 85 | 86 | is( $FC->get('foo'), undef, "get item 1 after sleep 3"); 87 | is( $FC->get('bar'), undef, "get item 2 after sleep 3"); 88 | is( $FC->get('baz'), '789ghia', "get item 3 after sleep 3"); 89 | 90 | @e = $FC->get_keys(2); 91 | cmp_deeply( 92 | \@e, 93 | bag( 94 | superhashof({ key => 'baz', value => '789ghia', last_access => num($now, 1), expire_on => num($now+1, 1) }), 95 | superhashof({ key => 'gah', value => 'booa', last_access => num($now-2, 1), expire_on => num($now+1, 1) }), 96 | ), 97 | "got expected keys" 98 | ) || diag explain [ $now, \@e ]; 99 | 100 | $now = $epoch+4; 101 | Cache::FastMmap::_set_time_override($now); 102 | 103 | is( $FC->get('foo'), undef, "get item 1 after sleep 4"); 104 | is( $FC->get('bar'), undef, "get item 2 after sleep 4"); 105 | is( $FC->get('baz'), undef, "get item 3 after sleep 4"); 106 | 107 | @e = $FC->get_keys(2); 108 | cmp_deeply( 109 | \@e, 110 | bag(), 111 | "got expected keys (empty)" 112 | ) || diag explain [ $now, \@e ]; 113 | 114 | $FC->empty(1); 115 | 116 | ok( eq_hash(\%BackingStore, { foo => '123abca', bar => '456defaa', baz => '789ghia', gah => 'booa' }), "items match expire 2"); 117 | 118 | 119 | -------------------------------------------------------------------------------- /t/6.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | 4 | use Test::More; 5 | 6 | my $GetMem; 7 | 8 | BEGIN { 9 | eval "use GTop ();"; 10 | if (!$@) { 11 | my $GTop = GTop->new; 12 | $GetMem = sub { return $GTop->proc_mem($$)->size }; 13 | } elsif (-f "/proc/$$/status") { 14 | $GetMem = sub { open(my $Sh, "/proc/$$/status"); my ($S) = map { /(\d+) kB/ && $1*1024 } grep { /^VmSize:/ } <$Sh>; close($Sh); return $S; } 15 | } 16 | if ($GetMem) { 17 | plan tests => 10; 18 | } else { 19 | plan skip_all => 'No GTop or /proc/, no memory leak tests'; 20 | } 21 | use_ok('Cache::FastMmap'); 22 | } 23 | 24 | use strict; 25 | 26 | ######################### 27 | 28 | # Insert your test code below, the Test::More module is use()ed here so read 29 | # its man page ( perldoc Test::More ) for help writing this test script. 30 | 31 | our ($DidRead, $DidWrite, $DidDelete, $HitCount); 32 | 33 | our $FC; 34 | $FC = Cache::FastMmap->new(init_file => 0, serializer => ''); 35 | $FC = undef; 36 | 37 | TestLeak(\&NewLeak, "new - 1"); 38 | TestLeak(\&NewLeak, "new - 2"); 39 | TestLeak(\&NewLeak2, "new2 - 1"); 40 | TestLeak(\&NewLeak2, "new2 - 2"); 41 | 42 | $FC = Cache::FastMmap->new( 43 | init_file => 1, 44 | serializer => '', 45 | num_pages => 17, 46 | page_size => 65536, 47 | read_cb => sub { $DidRead++; return undef; }, 48 | write_cb => sub { $DidWrite++; }, 49 | delete_cb => sub { $DidDelete++; }, 50 | write_action => 'write_back' 51 | ); 52 | 53 | ok( defined $FC ); 54 | 55 | # Prefill cache to make sure all pages mapped 56 | for (1 .. 10000) { 57 | $FC->set(RandStr(20), RandStr(20)); 58 | } 59 | $FC->get('foo'); 60 | 61 | our $Key = "blah" x 100; 62 | our $Val = "\x{263A}" . RandStr(1000); 63 | 64 | our $IterCount = 100; 65 | 66 | our $StartKey = 1; 67 | SetLeak(); 68 | $StartKey = 1; 69 | GetLeak(); 70 | 71 | our $IterCount = 20000; 72 | 73 | $StartKey = 1; 74 | TestLeak(\&SetLeak, "set"); 75 | 76 | $StartKey = 1; 77 | TestLeak(\&GetLeak, "get"); 78 | 79 | $FC->clear(); 80 | 81 | $StartKey = 1; 82 | TestLeak(\&SetLeak, "set2"); 83 | 84 | our (@a, @b, @c); 85 | @a = $FC->get_keys(0); 86 | @b = $FC->get_keys(1); 87 | @c = $FC->get_keys(2); 88 | @a = @b = @c = (); 89 | 90 | ListLeak(); 91 | TestLeak(\&ListLeak, "list"); 92 | 93 | sub RandStr { 94 | return join '', map { chr(ord('a') + rand(26)) } (1 .. $_[0]); 95 | } 96 | 97 | sub TestLeak { 98 | my $Sub = shift; 99 | my $Test = shift; 100 | 101 | my $Before = $GetMem->(); 102 | eval { 103 | $Sub->(); 104 | }; 105 | if ($@) { 106 | ok(0, "leak test died: $@"); 107 | } 108 | my $After = $GetMem->(); 109 | 110 | my $Extra = ($After - $Before)/1024; 111 | ok( $Extra <= 500, "leak test $Extra > 500k - for $Test"); 112 | } 113 | 114 | sub NewLeak { 115 | 116 | for (1 .. 2000) { 117 | $FC = Cache::FastMmap->new( 118 | init_file => 0, 119 | serializer => '', 120 | num_pages => 17, 121 | page_size => 8192, 122 | read_cb => sub { $DidRead++; return undef; }, 123 | write_cb => sub { $DidWrite++; }, 124 | delete_cb => sub { $DidDelete++; }, 125 | write_action => 'write_back' 126 | ); 127 | } 128 | $FC = undef; 129 | 130 | } 131 | 132 | sub NewLeak2 { 133 | 134 | for (1 .. 2000) { 135 | $FC = Cache::FastMmap->new( 136 | init_file => 1, 137 | serializer => '', 138 | num_pages => 17, 139 | page_size => 8192, 140 | read_cb => sub { $DidRead++; return undef; }, 141 | write_cb => sub { $DidWrite++; }, 142 | delete_cb => sub { $DidDelete++; }, 143 | write_action => 'write_back' 144 | ); 145 | } 146 | $FC = undef; 147 | 148 | } 149 | 150 | sub SetLeak { 151 | for (1 .. $IterCount) { 152 | $Key = "blah" . $StartKey++ . "blah"; 153 | if ($_ % 10 < 6) { $Val = RandStr(int(rand(20))+2); } 154 | elsif ($_ % 10 < 8) { $Val = "\x{263A}" . RandStr(int(rand(20))+2); } 155 | else { $Val = undef; } 156 | 157 | $FC->set($Key, $Val); 158 | } 159 | } 160 | 161 | sub GetLeak { 162 | for (1 .. $IterCount) { 163 | $Key = "blah" . $StartKey++ . "blah"; 164 | $HitCount++ if $FC->get($Key); 165 | } 166 | } 167 | 168 | sub WBLeak { 169 | for (1 .. $IterCount) { 170 | $Key = "blah" . $StartKey++ . "blah"; 171 | if ($_ % 10 < 6) { $Val = RandStr(int(rand(20))+2); } 172 | elsif ($_ % 10 < 8) { $Val = "\x{263A}" . RandStr(int(rand(20))+2); } 173 | else { $Val = undef; } 174 | $FC->set($Key, $Val); 175 | my $PreDidWrite = $DidWrite; 176 | $FC->empty() if $_ % 10 == 0; 177 | $PreDidWrite + 1 == $DidWrite 178 | || die "write count mismatch"; 179 | $FC->get($Key) 180 | && die "get success"; 181 | } 182 | } 183 | 184 | sub ListLeak { 185 | for (1 .. 100) { 186 | @a = $FC->get_keys(0); 187 | @b = $FC->get_keys(1); 188 | @c = $FC->get_keys(2); 189 | @a = @b = @c = (); 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /t/1.t: -------------------------------------------------------------------------------- 1 | 2 | ######################### 3 | 4 | use Test::More tests => 85; 5 | BEGIN { use_ok('Cache::FastMmap') }; 6 | use strict; 7 | 8 | ######################### 9 | 10 | # Insert your test code below, the Test::More module is use()ed here so read 11 | # its man page ( perldoc Test::More ) for help writing this test script. 12 | 13 | my $FC = Cache::FastMmap->new(init_file => 1, serializer => ''); 14 | ok( defined $FC ); 15 | 16 | # Test empty cache 17 | ok( !defined $FC->get(''), "empty get('')" ); 18 | ok( !defined $FC->get(' '), "empty get(' ')" ); 19 | ok( !defined $FC->get(' ' x 1024), "empty get(' ' x 1024)" ); 20 | ok( !defined $FC->get(' ' x 65536), "empty get(' ' x 65536)" ); 21 | 22 | ok( !$FC->exists(''), "false exists('')" ); 23 | ok( !$FC->exists(' '), "false exists(' ')" ); 24 | ok( !$FC->exists(' ' x 1024), "false exists(' ' x 1024)" ); 25 | ok( !$FC->exists(' ' x 65536), "false exists(' ' x 65536)" ); 26 | 27 | # Test basic store/get on key sizes 28 | ok( $FC->set('', 'abc'), "set('', 'abc')" ); 29 | is( $FC->get(''), 'abc', "get('') eq 'abc'"); 30 | ok( $FC->exists(''), "true exists('')"); 31 | my ($R, $DidStore) = $FC->get_and_set('', sub { 'abcd' }); 32 | is ($R, "abcd", "get_and_set('', sub { 'abcd' })" ); 33 | is ($DidStore, 1, "get_and_set did store"); 34 | ok( $FC->exists(''), "true exists('')"); 35 | 36 | ok( $FC->set(' ', 'def'), "set(' ', 'def')" ); 37 | is( $FC->get(' '), 'def', "get(' ') eq 'def'"); 38 | ok( $FC->exists(' '), "true exists(' ')"); 39 | 40 | ok( $FC->set(' ' x 1024, 'ghi'), "set(' ' x 1024, 'ghi')"); 41 | is( $FC->get(' ' x 1024), 'ghi', "get(' ' x 1024) eq 'ghi'"); 42 | ok( $FC->exists(' ' x 1024), "true exists(' ' x 1024)"); 43 | 44 | my ($R, $DidStore) = $FC->get_and_set(' ' x 1024, sub { 'bcde' }); 45 | is($R, "bcde", "get_and_set(' ' x 1024, sub { 'bcde' })" ); 46 | is($DidStore, 1, "get_and_set did store"); 47 | 48 | # Bigger than the page size - should not work 49 | ok( !$FC->set(' ' x 65536, 'jkl'), "set(' ' x 65536, 'jkl')"); 50 | ok( !defined $FC->get(' ' x 65536), "empty get(' ' x 65536)"); 51 | ok( !$FC->exists(' ' x 65536), "false exists(' ' x 65536)"); 52 | 53 | my ($R, $DidStore) = $FC->get_and_set(' ' x 65536, sub { 'cdef' }); 54 | ok( !defined $FC->get(' ' x 65536), "empty get(' ' x 65536)" ); 55 | is($DidStore, 0, "get_and_set did not store"); 56 | 57 | # Test basic store/get on value sizes 58 | ok( $FC->set('abc', ''), "set('abc', '')"); 59 | is( $FC->get('abc'), '', "get('abc') eq ''"); 60 | ok( $FC->exists('abc'), "true exists('abc')"); 61 | 62 | ok( $FC->set('def', 'x'), "set('def', 'x')"); 63 | is( $FC->get('def'), 'x', "get('def') eq 'x'"); 64 | ok( $FC->exists('def'), "true exists('def')"); 65 | 66 | ok( $FC->set('ghi', 'x' . ('y' x 1024) . 'z'), "set('ghi', ...)"); 67 | is( $FC->get('ghi'), 'x' . ('y' x 1024) . 'z', "get('ghi') eq ..."); 68 | ok( $FC->exists('ghi'), "true exists('ghi')"); 69 | 70 | # Bigger than the page size - should not work 71 | ok( !$FC->set('jkl', 'x' . ('y' x 65536) . 'z'), "set('jkl', ...)"); 72 | ok( !defined $FC->get('jkl'), "empty get('jkl')" ); 73 | ok( !$FC->exists('jkl'), "false exists('jkl')"); 74 | 75 | # Ref key should use 'stringy' version 76 | my $Ref = [ ]; 77 | ok( $FC->set($Ref, 'abcd'), "set($Ref)" ); 78 | is( $FC->get($Ref), 'abcd', "get($Ref)" ); 79 | is( $FC->get("$Ref"), 'abcd', "get(\"$Ref\")" ); 80 | ok( $FC->exists("$Ref"), "true exists(\"$Ref\")"); 81 | 82 | # Check UTF8 83 | ok( $FC->set("key\x{263A}", "val"), "set utf8 key" ); 84 | is( $FC->get("key\x{263A}"), "val", "get utf8 key" ); 85 | ok( $FC->exists("key\x{263A}"), "true exists utf8 key"); 86 | 87 | ok( $FC->set("key", "val\x{263A}"), "set utf8 val" ); 88 | is( $FC->get("key"), "val\x{263A}", "get utf8 val" ); 89 | ok( $FC->exists("key"), "true exists utf8 val"); 90 | 91 | ok( $FC->set("key2\x{263A}", "val2\x{263A}"), "set utf8 key/val" ); 92 | is( $FC->get("key2\x{263A}"), "val2\x{263A}", "get utf8 key/val" ); 93 | 94 | # Check clearing actually works 95 | $FC->clear(); 96 | 97 | ok( !defined $FC->get('abc'), "post clear 1" ); 98 | ok( !$FC->exists('abc'), "false exists post clear 1" ); 99 | ok( !defined $FC->get('def'), "post clear 2" ); 100 | ok( !$FC->exists('def'), "false exists post clear 2" ); 101 | ok( !defined $FC->get('ghi'), "post clear 3" ); 102 | ok( !$FC->exists('ghi'), "false exists post clear 3" ); 103 | ok( !defined $FC->get('jkl'), "post clear 4" ); 104 | ok( !$FC->exists('jkl'), "false exists post clear 4" ); 105 | ok( !defined $FC->get("key"), "post clear 5" ); 106 | ok( !$FC->exists('key'), "false exists post clear 5" ); 107 | ok( !defined $FC->get("key\x{263A}"), "post clear 6" ); 108 | ok( !$FC->exists("key"), "false exists post clear 6"); 109 | ok( !defined $FC->get("key2\x{263A}"), "post clear 7" ); 110 | ok( !$FC->exists("key2\x{263A}"), "false exists post clear 7"); 111 | 112 | # Check getting key/value lists 113 | ok( $FC->set("abc", "123"), "get_keys set 1" ); 114 | ok( $FC->set("bcd", "234"), "get_keys set 2" ); 115 | ok( $FC->set("cde", "345"), "get_keys set 3" ); 116 | 117 | is( join(",", sort $FC->get_keys), "abc,bcd,cde", "get_keys 1"); 118 | 119 | my %keys = map { $_->{key} => $_ } $FC->get_keys(2); 120 | is( scalar(keys %keys), 3, "get_keys 2" ); 121 | is($keys{abc}->{value}, "123", "get_keys 3"); 122 | is($keys{bcd}->{value}, "234", "get_keys 4"); 123 | is($keys{cde}->{value}, "345", "get_keys 5"); 124 | 125 | # Test getting key/value lists with UTF8 126 | $FC->set("def\x{263A}", "456\x{263A}"); 127 | 128 | is( join(",", sort $FC->get_keys), "abc,bcd,cde,def\x{263A}", "get_keys 6"); 129 | 130 | %keys = map { $_->{key} => $_ } $FC->get_keys(2); 131 | is( scalar(keys %keys), 4 , "get_keys 7"); 132 | is($keys{abc}->{value}, "123", "get_keys 8"); 133 | is($keys{bcd}->{value}, "234", "get_keys 9"); 134 | is($keys{cde}->{value}, "345", "get_keys 10"); 135 | is($keys{"def\x{263A}"}->{value}, "456\x{263A}", "get_keys 11"); 136 | 137 | # basic multi_* tests 138 | 139 | $FC->multi_set("page1", { k1 => 1, k2 => 2 }); 140 | $FC->multi_set("page2", { k3 => 1, k4 => 2 }); 141 | my $R = $FC->multi_get("page1", [ qw(k1 k2) ]); 142 | is($R->{k1}, 1, "multi_get 1"); 143 | is($R->{k2}, 2, "multi_get 2"); 144 | $R = $FC->multi_get("page2", [ qw(k3 k4) ]); 145 | is($R->{k3}, 1, "multi_get 3"); 146 | is($R->{k4}, 2, "multi_get 4"); 147 | -------------------------------------------------------------------------------- /unix.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * AUTHOR 4 | * 5 | * Rob Mueller 6 | * 7 | * COPYRIGHT AND LICENSE 8 | * 9 | * Copyright (C) 2003 by FastMail IP Partners 10 | * 11 | * This library is free software; you can redistribute it and/or modify 12 | * it under the same terms as Perl itself. 13 | * 14 | */ 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include "mmap_cache.h" 28 | #include "mmap_cache_internals.h" 29 | 30 | char* _mmc_get_def_share_filename(mmap_cache * cache) 31 | { 32 | return def_share_file; 33 | } 34 | 35 | int mmc_open_cache_file(mmap_cache* cache, int * do_init) { 36 | int res, i, fh; 37 | void * tmp; 38 | struct stat statbuf; 39 | 40 | /* Check if file exists */ 41 | res = stat(cache->share_file, &statbuf); 42 | 43 | /* Remove if different size or remove requested */ 44 | if (!res && 45 | (cache->init_file || (statbuf.st_size != cache->c_size))) { 46 | res = remove(cache->share_file); 47 | if (res == -1 && errno != ENOENT) { 48 | return _mmc_set_error(cache, errno, "Unlink of existing share file %s failed", cache->share_file); 49 | } 50 | } 51 | 52 | /* Create file if it doesn't exist */ 53 | res = stat(cache->share_file, &statbuf); 54 | if (res == -1) { 55 | mode_t permissions = (mode_t)cache->permissions; 56 | res = open(cache->share_file, O_WRONLY | O_CREAT | O_EXCL | O_TRUNC | O_APPEND, permissions); 57 | if (res == -1) { 58 | return _mmc_set_error(cache, errno, "Create of share file %s failed", cache->share_file); 59 | } 60 | 61 | /* Fill file with 0's */ 62 | tmp = calloc(1, cache->c_page_size); 63 | if (!tmp) { 64 | return _mmc_set_error(cache, errno, "Calloc of tmp space failed"); 65 | } 66 | 67 | for (i = 0; i < cache->c_num_pages; i++) { 68 | int written = write(res, tmp, cache->c_page_size); 69 | if (written < 0) { 70 | free(tmp); 71 | return _mmc_set_error(cache, errno, "Write to share file %s failed", cache->share_file); 72 | } 73 | if (written < cache->c_page_size) { 74 | free(tmp); 75 | return _mmc_set_error(cache, 0, "Write to share file %s failed; short write (%d of %d bytes written)", cache->share_file, written, cache->c_page_size); 76 | } 77 | } 78 | free(tmp); 79 | 80 | /* Later on initialise page structures */ 81 | *do_init = 1; 82 | 83 | close(res); 84 | } 85 | 86 | /* Open for reading/writing */ 87 | fh = open(cache->share_file, O_RDWR); 88 | if (fh == -1) { 89 | return _mmc_set_error(cache, errno, "Open of share file %s failed", cache->share_file); 90 | } 91 | 92 | /* Automatically close cache fd on exec */ 93 | fcntl(fh, F_SETFD, FD_CLOEXEC); 94 | 95 | fstat(fh, &statbuf); 96 | cache->inode = statbuf.st_ino; 97 | 98 | cache->fh = fh; 99 | 100 | return 0; 101 | 102 | } 103 | 104 | /* 105 | * mmc_map_memory(mmap_cache * cache) 106 | * 107 | * maps the cache file into memory, and sets cache->mm_var as needed. 108 | */ 109 | int mmc_map_memory(mmap_cache* cache) { 110 | /* Map file into memory */ 111 | cache->mm_var = mmap(0, cache->c_size, PROT_READ | PROT_WRITE, MAP_SHARED, cache->fh, 0); 112 | if (cache->mm_var == (void *)MAP_FAILED) { 113 | _mmc_set_error(cache, errno, "Mmap of shared file %s failed", cache->share_file); 114 | mmc_close_fh(cache); 115 | return -1; 116 | } 117 | 118 | return 0; 119 | } 120 | 121 | /* 122 | * mmc_unmap_memory(mmap_cache * cache) 123 | * 124 | * Unmaps cache->mm_var 125 | */ 126 | int mmc_unmap_memory(mmap_cache* cache) { 127 | int res = munmap(cache->mm_var, cache->c_size); 128 | if (res == -1) { 129 | return _mmc_set_error(cache, errno, "Munmap of shared file %s failed", cache->share_file); 130 | } 131 | return res; 132 | } 133 | 134 | int mmc_check_fh(mmap_cache* cache) { 135 | struct stat statbuf; 136 | 137 | fstat(cache->fh, &statbuf); 138 | if (cache->inode != statbuf.st_ino) { 139 | _mmc_set_error(cache, 0, "Underlying cache file fd %d was inode %ld but now %ld, something messed up underlying file descriptors", cache->fh, cache->inode, statbuf.st_ino); 140 | return 0; 141 | } 142 | 143 | return 1; 144 | } 145 | 146 | int mmc_close_fh(mmap_cache* cache) { 147 | int res = close(cache->fh); 148 | return res; 149 | } 150 | 151 | int mmc_lock_page(mmap_cache* cache, MU64 p_offset) { 152 | struct flock lock; 153 | int old_alarm, alarm_left = 10; 154 | int lock_res = -1; 155 | 156 | /* Setup fcntl locking structure */ 157 | lock.l_type = F_WRLCK; 158 | lock.l_whence = SEEK_SET; 159 | lock.l_start = p_offset; 160 | lock.l_len = cache->c_page_size; 161 | 162 | if (cache->catch_deadlocks) 163 | old_alarm = alarm(alarm_left); 164 | 165 | while (lock_res != 0) { 166 | 167 | /* Lock the page (block till done, signal, or timeout) */ 168 | lock_res = fcntl(cache->fh, F_SETLKW, &lock); 169 | 170 | /* Continue immediately if success */ 171 | if (lock_res == 0) { 172 | if (cache->catch_deadlocks) 173 | alarm(old_alarm); 174 | break; 175 | } 176 | 177 | /* Turn off alarm for a moment */ 178 | if (cache->catch_deadlocks) 179 | alarm_left = alarm(0); 180 | 181 | /* Some signal interrupted, and it wasn't the alarm? Rerun lock */ 182 | if (lock_res == -1 && errno == EINTR && alarm_left) { 183 | if (cache->catch_deadlocks) 184 | alarm(alarm_left); 185 | continue; 186 | } 187 | 188 | /* Lock failed? */ 189 | _mmc_set_error(cache, errno, "Lock failed"); 190 | if (cache->catch_deadlocks) 191 | alarm(old_alarm); 192 | return -1; 193 | } 194 | 195 | return 0; 196 | } 197 | 198 | int mmc_unlock_page(mmap_cache * cache, MU64 p_offset) { 199 | struct flock lock; 200 | 201 | /* Setup fcntl locking structure */ 202 | lock.l_type = F_UNLCK; 203 | lock.l_whence = SEEK_SET; 204 | lock.l_start = p_offset; 205 | lock.l_len = cache->c_page_size; 206 | 207 | /* And unlock page */ 208 | fcntl(cache->fh, F_SETLKW, &lock); 209 | 210 | return 0; 211 | } 212 | 213 | 214 | /* 215 | * int _mmc_set_error(mmap_cache *cache, int err, char * error_string, ...) 216 | * 217 | * Set internal error string/state 218 | * 219 | */ 220 | int _mmc_set_error(mmap_cache *cache, int err, char * error_string, ...) { 221 | va_list ap; 222 | static char errbuf[1024]; 223 | 224 | va_start(ap, error_string); 225 | 226 | /* Make sure it's terminated */ 227 | errbuf[1023] = '\0'; 228 | 229 | /* Start with error string passed */ 230 | vsnprintf(errbuf, 1023, error_string, ap); 231 | 232 | /* Add system error code if passed */ 233 | if (err) { 234 | strncat(errbuf, ": ", 1023); 235 | strncat(errbuf, strerror(err), 1023); 236 | } 237 | 238 | /* Save in cache object */ 239 | cache->last_error = errbuf; 240 | 241 | va_end(ap); 242 | 243 | return -1; 244 | } 245 | -------------------------------------------------------------------------------- /t/FastMmapTest.pl: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/perl -w 2 | 3 | use lib '/home/mod_perl/hm/modules'; 4 | use ExtUtils::testlib; 5 | use Cache::FastMmap; 6 | use Data::Dumper; 7 | use POSIX ":sys_wait_h"; 8 | use strict; 9 | 10 | #EdgeTests(); 11 | 12 | my $FC = Cache::FastMmap->new( 13 | init_file => 1, 14 | raw_values => 1 15 | ) || die "Could not create file cache"; 16 | 17 | BasicTests($FC); 18 | $FC->clear(); 19 | 20 | my @Keys; 21 | RepeatMixTest($FC, 0.0, \@Keys); 22 | RepeatMixTest($FC, 0.5, \@Keys); 23 | RepeatMixTest($FC, 0.8, \@Keys); 24 | 25 | ForkTests($FC); 26 | 27 | $FC = Cache::FastMmap->new( 28 | init_file => 1, 29 | page_size => 8192, 30 | raw_values => 1 31 | ) || die "Could not create file cache"; 32 | 33 | BasicTests($FC); 34 | $FC->clear(); 35 | 36 | @Keys = (); 37 | RepeatMixTest($FC, 0.0, \@Keys); 38 | RepeatMixTest($FC, 0.5, \@Keys); 39 | RepeatMixTest($FC, 0.8, \@Keys); 40 | 41 | ForkTests($FC); 42 | 43 | print "All done\n"; 44 | 45 | exit(0); 46 | 47 | sub BasicTests { 48 | my $FC = shift; 49 | 50 | printf "Basic tests\n"; 51 | 52 | # Test empty 53 | !defined $FC->get('') 54 | || die "Not undef on empty get"; 55 | 56 | !defined $FC->get(' ') 57 | || die "Not undef on empty get"; 58 | 59 | !defined $FC->get(' ' x 1024) 60 | || die "Not undef on empty get"; 61 | 62 | !defined $FC->get(' ' x 65536) 63 | || die "Not undef on empty get"; 64 | 65 | # Test basic store/get on key sizes 66 | $FC->set('', 'abc'); 67 | $FC->get('') eq 'abc' 68 | || die "Get mismatch"; 69 | 70 | $FC->set(' ', 'def'); 71 | $FC->get(' ') eq 'def' 72 | || die "Get mismatch"; 73 | 74 | $FC->set(' ' x 1024, 'ghi'); 75 | $FC->get(' ' x 1024) eq 'ghi' 76 | || die "Get mismatch"; 77 | 78 | # Bigger than the page size - shouldn't work 79 | $FC->set(' ' x 65536, 'jkl'); 80 | !defined $FC->get(' ' x 65536) 81 | || die "Get mismatch"; 82 | 83 | # Test basic store/get on value sizes 84 | $FC->set('abc', ''); 85 | $FC->get('abc') eq '' 86 | || die "Get mismatch"; 87 | 88 | $FC->set('def', 'x'); 89 | $FC->get('def') eq 'x' 90 | || die "Get mismatch"; 91 | 92 | $FC->set('ghi', 'x' . ('y' x 1024) . 'z'); 93 | $FC->get('ghi') eq 'x' . ('y' x 1024) . 'z' 94 | || die "Get mismatch"; 95 | 96 | # Bigger than the page size - shouldn't work 97 | $FC->set('jkl', 'x' . ('y' x 65536) . 'z'); 98 | !defined $FC->get('jkl') 99 | || die "Get mismatch"; 100 | 101 | # Ref key should use 'stringy' version 102 | my $Ref = [ ]; 103 | $FC->set($Ref, 'abcd'); 104 | $FC->get($Ref) eq 'abcd' 105 | || die "Get mismatch"; 106 | $FC->get("$Ref") eq 'abcd' 107 | || die "Get mismatch"; 108 | 109 | 110 | # Check utf8 111 | # eval { $FC->set("\x{263A}", "blah\x{263A}"); }; 112 | # $@ || die "Set utf8 succeeded, but should have failed: $@"; 113 | # eval { $FC->set("blah", "\x{263A}"); }; 114 | # $@ || die "Set utf8 succeeded, but should have failed: $@"; 115 | # eval { $FC->get("\x{263A}"); }; 116 | # $@ || die "Set utf8 succeeded, but should have failed: $@"; 117 | 118 | $FC->set("\x{263A}", "blah\x{263A}"); 119 | $FC->get("\x{263A}") eq "blah\x{263A}" 120 | || die "Get mismatch"; 121 | 122 | $FC->clear(); 123 | 124 | $FC->set("abc", "123"); 125 | $FC->set("bcd", "234"); 126 | $FC->set("cde", "345"); 127 | $FC->set("def", "456"); 128 | 129 | join(",", sort $FC->get_keys) eq "abc,bcd,cde,def" 130 | || die "get_keys mismatch"; 131 | 132 | $FC->set("efg\x{263A}", "567\x{263A}"); 133 | 134 | join(",", sort $FC->get_keys) eq "abc,bcd,cde,def,efg\x{263A}" 135 | || die "get_keys mismatch"; 136 | 137 | my %keys = map { $_->{key} => $_ } $FC->get_keys(2); 138 | $keys{abc}->{value} eq "123" 139 | || die "get_keys missing"; 140 | $keys{"efg\x{263A}"}->{value} eq "567\x{263A}" 141 | || die "get_keys missing"; 142 | 143 | } 144 | 145 | sub EdgeTests { 146 | my $FC = Cache::FastMmap->new( 147 | init_file => 1, 148 | num_pages => 1, 149 | raw_values => 1 150 | ) || die "Could not create file cache"; 151 | 152 | printf "Edge tests. Assume implementation\n"; 153 | 154 | $FC->clear(); 155 | 156 | # bytes for kv data 157 | # 65536 - 8*4 - 4*4*89 = 64080 158 | 159 | # adds 4*2 + 1 + 1 = 10 bytes, 64070 rem 160 | $FC->set('a', 'a'); 161 | $FC->get('a') eq 'a' 162 | || die "Get mismatch"; 163 | 164 | # Ensure oldest timestamp 165 | sleep 2; 166 | 167 | # adds 4*2 + 1 + 64051 = 64060, 10 rem 168 | $FC->set('b', 'b' x 64051); 169 | $FC->get('b') eq 'b' x 64051 170 | || die "Get mismatch"; 171 | 172 | sleep 2; 173 | 174 | # adds 4*2 + 1 + 1 = 10 bytes, 0 rem 175 | $FC->set('c', 'c'); 176 | $FC->get('c') eq 'c' 177 | || die "Get mismatch"; 178 | $FC->get('b') eq 'b' x 64051 179 | || die "Get mismatch"; 180 | $FC->get('a') eq 'a' 181 | || die "Get mismatch"; 182 | 183 | # adds 4*2 + 1 + 1 = 10 bytes, force expunge 184 | $FC->set('d', 'd'); 185 | !defined $FC->get('a') 186 | || die "Get mismatch"; 187 | !defined $FC->get('b') 188 | || die "Get mismatch"; 189 | $FC->get('d') eq 'd' 190 | || die "Get mismatch"; 191 | $FC->get('c') eq 'c' 192 | || die "Get mismatch"; 193 | 194 | # Try again 195 | $FC->clear(); 196 | 197 | # adds 4*2 + 1 + 1 = 10 bytes, 64070 rem 198 | $FC->set('a', 'a'); 199 | $FC->get('a') eq 'a' 200 | || die "Get mismatch"; 201 | 202 | # Ensure oldest timestamp 203 | sleep 2; 204 | 205 | # adds 4*2 + 1 + 64052 = 64061, 9 rem 206 | $FC->set('b', 'b' x 64052); 207 | $FC->get('b') eq 'b' x 64052 208 | || die "Get mismatch"; 209 | 210 | sleep 2; 211 | 212 | # adds 4*2 + 1 + 1 = 10 bytes, -1 rem, force expunge 213 | $FC->set('c', 'c'); 214 | $FC->get('c') eq 'c' 215 | || die "Get mismatch"; 216 | 217 | !defined $FC->get('b') 218 | || die "Get mismatch"; 219 | !defined $FC->get('a') 220 | || die "Get mismatch"; 221 | 222 | # adds 4*2 + 1 + 1 = 10 bytes 223 | $FC->set('d', 'd'); 224 | $FC->get('d') eq 'd' 225 | || die "Get mismatch"; 226 | $FC->get('c') eq 'c' 227 | || die "Get mismatch"; 228 | 229 | } 230 | 231 | sub ForkTests { 232 | 233 | # Now fork several children to test cache concurrency 234 | my ($Pid, %Kids); 235 | for (my $j = 0; $j < 8; $j++) { 236 | if (!($Pid = fork())) { 237 | RepeatMixTest($FC, 0.4, \@Keys); 238 | exit; 239 | } 240 | $Kids{$Pid} = 1; 241 | select(undef, undef, undef, 0.001); 242 | } 243 | 244 | # Wait for children to finish 245 | my $Kid; 246 | do { 247 | $Kid = waitpid(-1, WNOHANG); 248 | delete $Kids{$Kid}; 249 | } until $Kid > 0 && !%Kids; 250 | 251 | } 252 | 253 | sub RepeatMixTest { 254 | my ($FC, $Ratio, $WroteKeys) = @_; 255 | 256 | print "Repeat mix tests\n"; 257 | 258 | my ($Read, $ReadHit); 259 | 260 | # Lots of random tests 261 | for (1 .. 10000) { 262 | 263 | # Read/write ratio 264 | if (rand() < $Ratio) { 265 | 266 | # Pick a key from known written ones 267 | my $K = $WroteKeys->[ rand(@$WroteKeys) ]; 268 | my $V = $FC->get($K); 269 | $Read++; 270 | 271 | # Skip if not found in cache 272 | next if !defined $V; 273 | $ReadHit++; 274 | 275 | # Offset of 10 past first chars of value are key 276 | substr($V, 10, length($K)) eq $K 277 | || die "Cache/key not equal: $K, $V"; 278 | 279 | } else { 280 | 281 | my $K = RandStr(16); 282 | my $V = RandStr(10) . $K . RandStr(int(rand(200))); 283 | push @$WroteKeys, $K; 284 | $FC->set($K, $V); 285 | 286 | } 287 | } 288 | 289 | printf "Read hit pct: %5.3f\n", ($ReadHit/$Read) if $Read; 290 | 291 | return; 292 | } 293 | 294 | sub RandStr { 295 | my $Len = shift; 296 | 297 | if (!$::URandom) { 298 | open($::URandom, '/dev/urandom') 299 | || die "Could not open /dev/urandom: $!"; 300 | } 301 | 302 | sysread($::URandom, my $D, $Len); 303 | $D =~ s/(.)/chr(ord($1) % 26 + ord('a'))/ge; 304 | return $D; 305 | } 306 | 307 | -------------------------------------------------------------------------------- /win32.c: -------------------------------------------------------------------------------- 1 | /* 2 | * AUTHOR 3 | * 4 | * Ash Berlin 5 | * 6 | * Based on code by 7 | * Rob Mueller 8 | * 9 | * COPYRIGHT AND LICENSE 10 | * 11 | * Copyright (C) 2007 by Ash Berlin 12 | * 13 | * This library is free software; you can redistribute it and/or modify 14 | * it under the same terms as Perl itself. 15 | * 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | 25 | #include "mmap_cache.h" 26 | #include "mmap_cache_internals.h" 27 | 28 | #ifdef _MSC_VER 29 | #if _MSC_VER <= 1310 30 | #define vsnprintf _vsnprintf 31 | #endif 32 | #endif 33 | 34 | char* _mmc_get_def_share_filename(mmap_cache * cache) 35 | { 36 | int ret; 37 | static char buf[MAX_PATH]; 38 | 39 | ret = GetTempPath(MAX_PATH, buf); 40 | if (ret > MAX_PATH) 41 | { 42 | _mmc_set_error(cache, GetLastError(), "Unable to get temp path"); 43 | return NULL; 44 | } 45 | return strcat(buf, "sharefile"); 46 | } 47 | 48 | int mmc_open_cache_file(mmap_cache* cache, int* do_init) { 49 | int i; 50 | void *tmp; 51 | HANDLE fh, fileMap, findHandle; 52 | WIN32_FIND_DATA statbuf; 53 | 54 | findHandle = FindFirstFile(cache->share_file, &statbuf); 55 | 56 | /* Create file if it doesn't exist */ 57 | if (findHandle == INVALID_HANDLE_VALUE) { 58 | fh = CreateFile(cache->share_file, GENERIC_WRITE, FILE_SHARE_WRITE, NULL, 59 | CREATE_ALWAYS, FILE_ATTRIBUTE_TEMPORARY, NULL); 60 | 61 | if (fh == INVALID_HANDLE_VALUE) { 62 | _mmc_set_error(cache, GetLastError(), "Create of share file %s failed", cache->share_file); 63 | return -1; 64 | } 65 | 66 | /* Fill file with 0's */ 67 | tmp = calloc(1, cache->c_page_size); 68 | if (!tmp) { 69 | _mmc_set_error(cache, GetLastError(), "Calloc of tmp space failed"); 70 | return -1; 71 | } 72 | 73 | for (i = 0; i < cache->c_num_pages; i++) { 74 | DWORD tmpOut; 75 | WriteFile(fh, tmp, cache->c_page_size, &tmpOut, NULL); 76 | } 77 | free(tmp); 78 | 79 | /* Later on initialise page structures */ 80 | *do_init = 1; 81 | 82 | CloseHandle(fh); 83 | 84 | } else { 85 | FindClose(findHandle); 86 | 87 | if (cache->init_file || (statbuf.nFileSizeLow != cache->c_size)) { 88 | *do_init = 1; 89 | 90 | fh = CreateFile(cache->share_file, GENERIC_WRITE, FILE_SHARE_WRITE, NULL, 91 | CREATE_ALWAYS, FILE_ATTRIBUTE_TEMPORARY, NULL); 92 | 93 | if (fh == INVALID_HANDLE_VALUE) { 94 | _mmc_set_error(cache, GetLastError(), "Truncate of existing share file %s failed", cache->share_file); 95 | return -1; 96 | } 97 | CloseHandle(fh); 98 | } 99 | } 100 | 101 | fh = CreateFile(cache->share_file, // File Name 102 | GENERIC_READ|GENERIC_WRITE, // Desired Access 103 | FILE_SHARE_READ|FILE_SHARE_WRITE, // Share mode 104 | NULL, // Security Rights 105 | OPEN_EXISTING, // Creation Mode 106 | FILE_ATTRIBUTE_TEMPORARY, // File Attribs 107 | NULL); // Template File 108 | 109 | if (fh == INVALID_HANDLE_VALUE) { 110 | _mmc_set_error(cache, GetLastError(), "Open of share file \"%s\" failed", cache->share_file); 111 | return -1; 112 | } 113 | 114 | cache->fh = fh; 115 | return 0; 116 | } 117 | 118 | int mmc_map_memory(mmap_cache * cache) { 119 | HANDLE fileMap = CreateFileMapping(cache->fh, NULL, PAGE_READWRITE, 0, cache->c_size, NULL); 120 | if (fileMap == NULL) { 121 | _mmc_set_error(cache, GetLastError(), "CreateFileMapping of %s failed", cache->share_file); 122 | CloseHandle(cache->fh); 123 | return -1; 124 | } 125 | 126 | cache->mm_var = MapViewOfFile(fileMap, FILE_MAP_WRITE|FILE_MAP_READ, 0,0,0); 127 | if (cache->mm_var == NULL) { 128 | _mmc_set_error(cache, GetLastError(), "Mmap of shared file %s failed", cache->share_file); 129 | CloseHandle(fileMap); 130 | CloseHandle(cache->fh); 131 | return -1; 132 | 133 | } 134 | /* If I read the docs right, this will do nothing untill the mm_var is unmapped */ 135 | if (CloseHandle(fileMap) == FALSE) { 136 | _mmc_set_error(cache, GetLastError(), "CloseHandle(fileMap) on shared file %s failed", cache->share_file); 137 | UnmapViewOfFile(cache->mm_var); 138 | CloseHandle(fileMap); 139 | CloseHandle(cache->fh); 140 | return -1; 141 | } 142 | return 0; 143 | } 144 | 145 | int mmc_check_fh(mmap_cache* cache) { 146 | return 1; 147 | } 148 | 149 | int mmc_close_fh(mmap_cache* cache) { 150 | int ret = CloseHandle(cache->fh); 151 | cache->fh = NULL; 152 | return ret; 153 | } 154 | 155 | int mmc_unmap_memory(mmap_cache* cache) { 156 | int res = UnmapViewOfFile(cache->mm_var); 157 | if (res == -1) { 158 | _mmc_set_error(cache, GetLastError(), "Unmmap of shared file %s failed", cache->share_file); 159 | } 160 | return res; 161 | } 162 | 163 | int mmc_lock_page(mmap_cache* cache, MU64 p_offset) { 164 | OVERLAPPED lock; 165 | DWORD lock_res, bytesTransfered; 166 | memset(&lock, 0, sizeof(lock)); 167 | lock.Offset = (DWORD)(p_offset & 0xffffffff); 168 | lock.OffsetHigh = (DWORD)((p_offset >> 32) & 0xffffffff); 169 | lock.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL); 170 | 171 | if (LockFileEx(cache->fh, 0, 0, cache->c_page_size, 0, &lock) == 0) { 172 | _mmc_set_error(cache, GetLastError(), "LockFileEx failed"); 173 | return -1; 174 | } 175 | 176 | lock_res = WaitForSingleObjectEx(lock.hEvent, 10000, FALSE); 177 | 178 | if (lock_res != WAIT_OBJECT_0 || GetOverlappedResult(cache->fh, &lock, &bytesTransfered, FALSE) == FALSE) { 179 | CloseHandle(lock.hEvent); 180 | _mmc_set_error(cache, GetLastError(), "Overlapped Lock failed"); 181 | return -1; 182 | } 183 | return 0; 184 | } 185 | 186 | int mmc_unlock_page(mmap_cache* cache, MU64 p_offset) { 187 | OVERLAPPED lock; 188 | memset(&lock, 0, sizeof(lock)); 189 | lock.Offset = p_offset; 190 | lock.hEvent = 0; 191 | 192 | UnlockFileEx(cache->fh, 0, cache->c_page_size, 0, &lock); 193 | } 194 | 195 | /* 196 | * int _mmc_set_error(mmap_cache *cache, int err, char * error_string, ...) 197 | * 198 | * Set internal error string/state 199 | * 200 | */ 201 | int _mmc_set_error(mmap_cache *cache, int err, char * error_string, ...) { 202 | va_list ap; 203 | static char errbuf[1024]; 204 | char *msgBuff; 205 | 206 | va_start(ap, error_string); 207 | 208 | /* Make sure it's terminated */ 209 | errbuf[1023] = '\0'; 210 | 211 | /* Start with error string passed */ 212 | vsnprintf(errbuf, 1023, error_string, ap); 213 | 214 | /* Add system error code if passed */ 215 | if (err) { 216 | strncat(errbuf, ": ", 1023); 217 | FormatMessage( 218 | FORMAT_MESSAGE_ALLOCATE_BUFFER | 219 | FORMAT_MESSAGE_FROM_SYSTEM, 220 | NULL, 221 | err, 222 | MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), 223 | (LPTSTR) &msgBuff, 224 | 0, NULL ); 225 | strncat(errbuf, msgBuff, 1023); 226 | LocalFree(msgBuff); 227 | } 228 | 229 | /* Save in cache object */ 230 | cache->last_error = errbuf; 231 | 232 | va_end(ap); 233 | 234 | return -1; 235 | } 236 | 237 | -------------------------------------------------------------------------------- /mmap_cache.h: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * AUTHOR 4 | * 5 | * Rob Mueller 6 | * 7 | * COPYRIGHT AND LICENSE 8 | * 9 | * Copyright (C) 2003 by FastMail IP Partners 10 | * 11 | * This library is free software; you can redistribute it and/or modify 12 | * it under the same terms as Perl itself. 13 | * 14 | * mmap_cache 15 | * 16 | * Uses an mmap'ed file to act as a shared memory interprocess cache 17 | * 18 | * The C interface is quite explicit in it's use, in that you have to 19 | * call individual functions to hash a key, lock a page, and find a 20 | * value. This allows a simpler higher level interface to be written 21 | * 22 | * #include 23 | * 24 | * mmap_cache * cache = mmc_new(); 25 | * cache->param = val; 26 | * mmc_init(cache); 27 | * 28 | * // Read a key 29 | * 30 | * // Hash get to find page and slot 31 | * mmc_hash(cache, (void *)key_ptr, (int)key_len, &hash_page, &hash_slot); 32 | * // Lock page 33 | * mmc_lock(cache, hash_page); 34 | * // Get pointer to value data 35 | * mmc_read(cache, hash_slot, (void *)key_ptr, (int)key_len, (void **)&val_ptr, (int *)val_len, &expire_time, &flags); 36 | * // Unlock page 37 | * mmc_unlock(cache); 38 | * 39 | * // Write a key 40 | * 41 | * // Hash get to find page and slot 42 | * mmc_hash(cache, (void *)key_ptr, (int)key_len, &hash_page, &hash_slot); 43 | * // Lock page 44 | * mmc_lock(cache, hash_page); 45 | * // Get pointer to value data 46 | * mmc_write(cache, hash_slot, (void *)key_ptr, (int)key_len, (void *)val_ptr, (int)val_len, expire_time, flags); 47 | * // Unlock page 48 | * mmc_unlock(cache); 49 | * 50 | * DESCRIPTION 51 | * 52 | * This class implements a shared memory cache through an mmap'ed file. It 53 | * uses locking to ensure multiple processes can safely access the cache 54 | * at the same time. It uses a basic LRU algorithm to keep the most used 55 | * entries in the cache. 56 | * 57 | * It tries to be quite efficient through a number of means: 58 | * 59 | * It uses multiple pages within a file, and uses Fcntl to only lock 60 | * a page at a time to reduce contention when multiple processes access 61 | * the cache. 62 | * 63 | * It uses a dual level hashing system (hash to find page, then hash 64 | * within each page to find a slot) to make most I calls O(1) and 65 | * fast 66 | * 67 | * On each I, if there are slots and page space available, only 68 | * the slot has to be updated and the data written at the end of the used 69 | * data space. If either runs out, a re-organisation of the page is 70 | * performed to create new slots/space which is done in an efficient way 71 | * 72 | * The locking is explicitly done in the C interface, so you can create 73 | * a 'read_many' or 'write_many' function that reduces the number of 74 | * locks required 75 | * 76 | * 77 | * IMPLEMENTATION 78 | * 79 | * Each file is made up of a number of 'pages'. The number of 80 | * pages and size of each page is specified when the class is 81 | * constructed. These values are stored in the cache class 82 | * and must be the same for each class connecting to the cache 83 | * file. 84 | * 85 | * NumPages - Number of 'pages' in the cache 86 | * PageSize - Size of each 'page' in the cache 87 | * 88 | * The layout of each page is: 89 | * 90 | * - Magic (4 bytes) - 0x92f7e3b1 magic page start marker 91 | * 92 | * - NumSlots (4 bytes) - Number of hash slots in this page 93 | * 94 | * - FreeSlots (4 bytes) - Number of free slots left in this page. 95 | * This includes all slots with a last access time of 0 96 | * (empty and don't search past) or 1 (empty, but keep searching 97 | * because deleted slot) 98 | * 99 | * - OldSlots (4 bytes) - Of all the free slots, how many were in use 100 | * and are now deleted. This is slots with a last access time of 1 101 | * 102 | * - FreeData (4 bytes) - Offset to free data area to place next item 103 | * 104 | * - FreeBytes (4 bytes) - Bytes left in free data area 105 | * 106 | * - N Reads (4 bytes) - Number of reads performed on this page 107 | * 108 | * - N Read Hits (4 bytes) - Number of reads on this page that have hit 109 | * something in the cache 110 | * 111 | * - Slots (4 bytes * NumSlots) - Hash slots 112 | * 113 | * - Data (to end of page) - Key/value data 114 | * 115 | * Each slot is made of: 116 | * 117 | * - Offset (4 bytes) - offset from start of page to actual data. This 118 | * is 0 if slot is empty, 1 if was used but now empty. This is needed 119 | * so deletes don't require a complete rehash with the linear 120 | * searching method we use 121 | * 122 | * Each data item is made of: 123 | * 124 | * - LastAccess (4 bytes) - Unix time data was last accessed 125 | * 126 | * - ExpireTime (4 bytes) - Unix time data should expire. This is 0 if it 127 | * should never expire 128 | * 129 | * - HashValue (4 bytes) - Value key was hashed to, so we don't have to 130 | * rehash on a re-organisation of the hash table 131 | * 132 | * - Flags (4 bytes) - Various flags 133 | * 134 | * - KeyLen (4 bytes) - Length of key 135 | * 136 | * - ValueLen (4 bytes) - Length of value 137 | * 138 | * - Key (KeyLen bytes) - Key data 139 | * 140 | * - Value (ValueLen bytes) - Value data 141 | * 142 | * Each set/get/delete operation involves: 143 | * 144 | * - Find the page for the key 145 | * - Lock the page 146 | * - Read the page header 147 | * - Find the hash slot for the key 148 | * 149 | * For get's: 150 | * 151 | * - Use linear probing to find correct key, or empty slot 152 | * 153 | * For set's: 154 | * 155 | * - Use linear probing to find empty slot 156 | * - If not enough free slots, do an 'expunge' run 157 | * - Store key/value at FreeData offset, update, and store in slot 158 | * - If not enough space at FreeData offset, do an 'expunge' run 159 | * then store data 160 | * 161 | * For delete's: 162 | * 163 | * - Use linear probing to find correct key, or empty slot 164 | * - Set slot to empty (data cleaned up in expunge run) 165 | * 166 | * An expunge run consists of: 167 | * 168 | * - Scan slots to find used key/value parts. Remove older items 169 | * - If ratio used/free slots too high, increase slot count 170 | * - Compact key/value data into one memory block 171 | * - Restore and update offsets in slots 172 | * 173 | */ 174 | 175 | #include 176 | 177 | /* Main cache structure passed as a pointer to each function */ 178 | typedef struct mmap_cache mmap_cache; 179 | 180 | /* Iterator structure for iterating over items in cache */ 181 | typedef struct mmap_cache_it mmap_cache_it; 182 | 183 | /* Unsigned 32 bit integer */ 184 | typedef uint32_t MU32; 185 | 186 | /* Unsigned 64 bit integer */ 187 | typedef uint64_t MU64; 188 | 189 | /* Magic value for no p_cur */ 190 | #define NOPAGE (~(MU32)0) 191 | 192 | /* Allow overriding "time" for tests */ 193 | void mmc_set_time_override(MU32); 194 | 195 | /* Initialisation/closing/error functions */ 196 | mmap_cache * mmc_new(); 197 | int mmc_init(mmap_cache *); 198 | int mmc_set_param(mmap_cache *, char *, char *); 199 | int mmc_get_param(mmap_cache *, char *); 200 | int mmc_close(mmap_cache *); 201 | char * mmc_error(mmap_cache *); 202 | 203 | /* Functions for find/locking a page */ 204 | int mmc_hash(mmap_cache *, void *, int, MU32 *, MU32 *); 205 | int mmc_lock(mmap_cache *, MU32); 206 | int mmc_unlock(mmap_cache *); 207 | int mmc_is_locked(mmap_cache *); 208 | 209 | /* Functions for getting/setting/deleting values in current page */ 210 | int mmc_read(mmap_cache *, MU32, void *, int, void **, int *, MU32 *, MU32 *); 211 | int mmc_write(mmap_cache *, MU32, void *, int, void *, int, MU32, MU32); 212 | int mmc_delete(mmap_cache *, MU32, void *, int, MU32 *); 213 | 214 | /* Functions of expunging values in current page */ 215 | int mmc_calc_expunge(mmap_cache *, int, int, MU32 *, MU32 ***); 216 | int mmc_do_expunge(mmap_cache *, int, MU32, MU32 **); 217 | 218 | /* Functions for iterating over items in a cache */ 219 | mmap_cache_it * mmc_iterate_new(mmap_cache *); 220 | MU32 * mmc_iterate_next(mmap_cache_it *); 221 | void mmc_iterate_close(mmap_cache_it *); 222 | 223 | /* Retrieve details of a cache page/entry */ 224 | void mmc_get_details(mmap_cache *, MU32 *, void **, int *, void **, int *, MU32 *, MU32 *, MU32 *); 225 | void mmc_get_page_details(mmap_cache * cache, MU32 * nreads, MU32 * nreadhits); 226 | void mmc_reset_page_details(mmap_cache * cache); 227 | 228 | /* Internal functions */ 229 | int _mmc_set_error(mmap_cache *, int, char *, ...); 230 | void _mmc_init_page(mmap_cache *, MU32); 231 | 232 | MU32 * _mmc_find_slot(mmap_cache * , MU32 , void *, int, int ); 233 | void _mmc_delete_slot(mmap_cache * , MU32 *); 234 | 235 | int _mmc_check_expunge(mmap_cache * , int); 236 | 237 | int _mmc_test_page(mmap_cache *); 238 | int _mmc_dump_page(mmap_cache *); 239 | 240 | 241 | -------------------------------------------------------------------------------- /mmap_cache_test.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #if !defined(WIN32) || defined(CYGWIN) 5 | #include 6 | #include 7 | #include 8 | #include 9 | #endif 10 | 11 | #ifdef DEBUG 12 | #define ASSERT(x) assert(x) 13 | #include 14 | #else 15 | #define ASSERT(x) 16 | #endif 17 | 18 | #ifndef WIN32 19 | #include 20 | #else 21 | #include 22 | #include 23 | #include 24 | double last_rand; 25 | double drand48(void) { 26 | last_rand = rand() / (double)(RAND_MAX+1); 27 | 28 | ASSERT(last_rand < 1); 29 | return last_rand; 30 | } 31 | #endif 32 | 33 | #include 34 | #include "mmap_cache.h" 35 | 36 | void * Get(mmap_cache * cache, void * key_ptr, int key_len, int * val_len) { 37 | int found; 38 | void * val_ptr, * val_rtn_ptr = 0; 39 | MU32 hash_page, hash_slot, flags; 40 | 41 | /* Hash key to get page and slot */ 42 | mmc_hash(cache, key_ptr, key_len, &hash_page, &hash_slot); 43 | 44 | /* Get and lock the page */ 45 | mmc_lock(cache, hash_page); 46 | 47 | /* Get value data pointer */ 48 | found = mmc_read(cache, hash_slot, key_ptr, key_len, &val_ptr, val_len, &flags); 49 | 50 | /* If not found, use undef */ 51 | if (found == -1) { 52 | 53 | } else { 54 | 55 | /* Put into our own memory */ 56 | val_rtn_ptr = (void *)malloc(*val_len); 57 | memcpy(val_rtn_ptr, val_ptr, *val_len); 58 | } 59 | 60 | mmc_unlock(cache); 61 | 62 | return val_rtn_ptr; 63 | } 64 | 65 | void Set(mmap_cache * cache, void * key_ptr, int key_len, void * val_ptr, int val_len) { 66 | MU32 hash_page, hash_slot, flags = 0, new_num_slots, ** expunge_items = 0; 67 | int num_expunge; 68 | 69 | /* Hash key to get page and slot */ 70 | mmc_hash(cache, key_ptr, key_len, &hash_page, &hash_slot); 71 | 72 | /* Get and lock the page */ 73 | mmc_lock(cache, hash_page); 74 | 75 | num_expunge = mmc_calc_expunge(cache, 2, key_len + val_len, &new_num_slots, &expunge_items); 76 | if (expunge_items) { 77 | mmc_do_expunge(cache, num_expunge, new_num_slots, expunge_items); 78 | } 79 | 80 | /* Get value data pointer */ 81 | mmc_write(cache, hash_slot, key_ptr, key_len, val_ptr, val_len, 60, flags); 82 | 83 | mmc_unlock(cache); 84 | } 85 | 86 | char * rand_str(int nchar) { 87 | unsigned char * buf = (unsigned char *)malloc(nchar + 1); 88 | int i; 89 | 90 | for (i = 0; i < nchar; i++) { 91 | buf[i] = (char)(rand() % 26) + 'A'; 92 | } 93 | buf[i] = 0; 94 | 95 | return (char *)buf; 96 | } 97 | 98 | char buf[65537]; 99 | 100 | int BasicTests(mmap_cache * cache) { 101 | int val_len, i; 102 | void * val_ptr; 103 | 104 | printf("Basic tests\n"); 105 | 106 | /* Test empty */ 107 | ASSERT(!Get(cache, "", 0, &val_len)); 108 | ASSERT(!Get(cache, " ", 0, &val_len)); 109 | for (i = 0; i < 65536; i++) { buf[i] = ' '; } 110 | ASSERT(!Get(cache, buf, 1024, &val_len)); 111 | ASSERT(!Get(cache, buf, 65536, &val_len)); 112 | 113 | /* Test basic store/get on key sizes */ 114 | Set(cache, "", 0, "abc", 3); 115 | ASSERT(!memcmp(val_ptr = Get(cache, "", 0, &val_len), "abc", 3) && val_len == 3); 116 | free(val_ptr); 117 | Set(cache, " ", 1, "def", 3); 118 | ASSERT(!memcmp(val_ptr = Get(cache, " ", 1, &val_len), "def", 3) && val_len == 3); 119 | free(val_ptr); 120 | Set(cache, buf, 1024, "ghi", 3); 121 | ASSERT(!memcmp(val_ptr = Get(cache, buf, 1024, &val_len), "ghi", 3) && val_len == 3); 122 | free(val_ptr); 123 | 124 | /* Bigger than page size - shouldn't work */ 125 | Set(cache, buf, 65536, "jkl", 3); 126 | ASSERT(!Get(cache, buf, 65536, &val_len)); 127 | 128 | /* Test basic store/get on value sizes */ 129 | Set(cache, "abc", 3, "", 0); 130 | ASSERT((val_ptr = Get(cache, "abc", 3, &val_len)) && val_len == 0); 131 | free(val_ptr); 132 | 133 | Set(cache, "def", 3, "x", 1); 134 | ASSERT(!memcmp(val_ptr = Get(cache, "def", 3, &val_len), "x", 1) && val_len == 1); 135 | free(val_ptr); 136 | 137 | for (i = 0; i < 1024; i++) { buf[i] = 'y'; } 138 | buf[0] = 'z'; buf[1023] = 'w'; 139 | Set(cache, "ghi", 3, buf, 1024); 140 | ASSERT(!memcmp(val_ptr = Get(cache, "ghi", 3, &val_len), buf, 1024) && val_len == 1024); 141 | free(val_ptr); 142 | 143 | /* Bigger than page size - shouldn't work */ 144 | Set(cache, "jkl", 3, buf, 65536); 145 | ASSERT(!Get(cache, "jkl", 3, &val_len)); 146 | 147 | return 0; 148 | } 149 | 150 | int LinearTests(mmap_cache * cache) { 151 | int i, gl; 152 | char * str1, * str2, * str3; 153 | 154 | printf("Linear tests\n"); 155 | 156 | for (i = 0; i < 100000; i++) { 157 | str1 = rand_str(10); 158 | str2 = rand_str(10); 159 | 160 | Set(cache, str1, strlen(str1)+1, str2, strlen(str2)+1); 161 | str3 = Get(cache, str1, strlen(str1)+1, &gl); 162 | ASSERT(strlen(str2)+1 == gl); 163 | ASSERT(!memcmp(str2, str3, strlen(str2)+1)); 164 | 165 | free(str1); 166 | free(str2); 167 | free(str3); 168 | 169 | if (i % 1000 == 0) { 170 | printf("%d\n", i); 171 | } 172 | } 173 | } 174 | 175 | int EdgeTests() { 176 | return 0; 177 | } 178 | 179 | typedef struct key_list { 180 | int n_keys; 181 | int buf_size; 182 | char ** keys; 183 | } key_list; 184 | 185 | key_list * kl_new() { 186 | key_list * kl = (key_list *)malloc(sizeof(key_list)); 187 | kl->buf_size = 8; 188 | kl->keys = (char **)malloc(sizeof(char *) * kl->buf_size); 189 | kl->n_keys = 0; 190 | 191 | return kl; 192 | } 193 | 194 | void kl_push(key_list * kl, char * key) { 195 | if (kl->n_keys < kl->buf_size) { 196 | kl->keys[kl->n_keys++] = key; 197 | return; 198 | } 199 | 200 | kl->buf_size *= 2; 201 | kl->keys = (char **)realloc(kl->keys, sizeof(char *) * kl->buf_size); 202 | kl->keys[kl->n_keys++] = key; 203 | return; 204 | } 205 | 206 | void kl_free(key_list * kl) { 207 | int i; 208 | 209 | for (i = 0; i < kl->n_keys; i++) { 210 | free(kl->keys[i]); 211 | } 212 | } 213 | 214 | int urand_fh = 0; 215 | 216 | void RandSeed() { 217 | #ifdef WIN32 218 | //randomize(); 219 | #else 220 | char buf[8]; 221 | 222 | if (!urand_fh) { 223 | urand_fh = open("/dev/urandom", O_RDONLY); 224 | } 225 | 226 | read(urand_fh, buf, 8); 227 | 228 | srand48(*(long int *)buf); 229 | #endif 230 | } 231 | 232 | int RepeatMixTests(mmap_cache * cache, double ratio, key_list * kl) { 233 | int i, val_len; 234 | int read = 0, read_hit = 0; 235 | char valbuf[256]; 236 | 237 | printf("Repeat mix tests\n"); 238 | 239 | for (i = 0; i < 10000; i++) { 240 | 241 | /* Read/write ratio */ 242 | if (drand48() < ratio) { 243 | /* Pick a key from known written ones */ 244 | char * k = kl->keys[(int)(drand48() * kl->n_keys)]; 245 | void * v = Get(cache, k, strlen(k), &val_len); 246 | read++; 247 | 248 | /* Skip if not found in cache */ 249 | if (!v) { continue; } 250 | read_hit++; 251 | 252 | /* Offset of 10 past first chars of value are key */ 253 | memcpy(valbuf, v+10, strlen(k)); 254 | valbuf[strlen(k)] = '\0'; 255 | ASSERT(!memcmp(valbuf, k, strlen(k))); 256 | 257 | free(v); 258 | 259 | } else { 260 | char * k = rand_str(10 + (int)(drand48() * 10)); 261 | char * v = rand_str(10); 262 | char * ve = rand_str((int)(drand48() * 200)); 263 | strcpy(valbuf, v); 264 | strcat(valbuf, k); 265 | strcat(valbuf, ve); 266 | 267 | kl_push(kl, k); 268 | Set(cache, k, strlen(k), valbuf, strlen(valbuf)); 269 | 270 | free(ve); 271 | free(v); 272 | } 273 | } 274 | 275 | if (read) { 276 | printf("Read hit pct: %5.3f\n", (double)read_hit/read); 277 | } 278 | 279 | return 1; 280 | } 281 | 282 | void IteratorTests(mmap_cache * cache) { 283 | MU32 * entry_ptr; 284 | void * key_ptr, * val_ptr; 285 | int key_len, val_len; 286 | MU32 last_access, expire_time, flags; 287 | mmap_cache_it * it = mmc_iterate_new(cache); 288 | 289 | printf("Iterator tests\n"); 290 | 291 | while ((entry_ptr = mmc_iterate_next(it))) { 292 | mmc_get_details(cache, entry_ptr, 293 | &key_ptr, &key_len, &val_ptr, &val_len, 294 | &last_access, &expire_time, &flags); 295 | 296 | ASSERT(key_len >= 10 && key_len <= 20); 297 | ASSERT(val_len >= 20 && val_len <= 240); 298 | ASSERT(last_access >= 1000000 && last_access <= time(0)); 299 | } 300 | 301 | mmc_iterate_close(it); 302 | } 303 | 304 | int ForkTests(mmap_cache * cache, key_list * kl) { 305 | #ifndef WIN32 306 | int pid, j, k, kid, kids[20], nkids = 0, status; 307 | struct timeval timeout = { 0, 1000 }; 308 | 309 | for (j = 0; j < 8; j++) { 310 | if (!(pid = fork())) { 311 | RandSeed(); 312 | RepeatMixTests(cache, 0.4, kl); 313 | exit(0); 314 | } 315 | kids[nkids++] = pid; 316 | select(0, 0, 0, 0, &timeout); 317 | } 318 | 319 | do { 320 | kid = waitpid(-1, &status, 0); 321 | for (j = 0, k = 0; j < nkids; j++) { 322 | if (kids[j] != kid) { k++; } 323 | kids[j] = kids[k]; 324 | } 325 | nkids--; 326 | } while (kid > 0 && nkids); 327 | 328 | return 0; 329 | #else 330 | #endif 331 | } 332 | 333 | 334 | int main(int argc, char ** argv) { 335 | int res; 336 | key_list * kl; 337 | mmap_cache * cache; 338 | 339 | cache = mmc_new(); 340 | mmc_set_param(cache, "init_file", "1"); 341 | res = mmc_init(cache); 342 | 343 | kl = kl_new(); 344 | 345 | BasicTests(cache); 346 | LinearTests(cache); 347 | 348 | mmc_close(cache); 349 | 350 | cache = mmc_new(); 351 | mmc_set_param(cache, "init_file", "1"); 352 | res = mmc_init(cache); 353 | 354 | RepeatMixTests(cache, 0.0, kl); 355 | RepeatMixTests(cache, 0.5, kl); 356 | RepeatMixTests(cache, 0.8, kl); 357 | 358 | IteratorTests(cache); 359 | 360 | ForkTests(cache, kl); 361 | 362 | kl_free(kl); 363 | mmc_close(cache); 364 | 365 | cache = mmc_new(); 366 | mmc_set_param(cache, "init_file", "1"); 367 | mmc_set_param(cache, "page_size", "8192"); 368 | res = mmc_init(cache); 369 | 370 | kl = kl_new(); 371 | 372 | BasicTests(cache); 373 | RepeatMixTests(cache, 0.0, kl); 374 | RepeatMixTests(cache, 0.5, kl); 375 | RepeatMixTests(cache, 0.8, kl); 376 | 377 | ForkTests(cache, kl); 378 | 379 | kl_free(kl); 380 | mmc_close(cache); 381 | 382 | return 0; 383 | } 384 | 385 | -------------------------------------------------------------------------------- /Changes: -------------------------------------------------------------------------------- 1 | Revision history for Perl extension Cache::FastMmap. 2 | 3 | 1.59 Tue Jun 3 17:25 2025 4 | - Add exists() method to check if key exists in the 5 | cache. This avoid the decompress + deserialize cost 6 | - Initialise pages under lock. This avoids potential 7 | race conditions where multiple processes try and init 8 | a cache file 9 | - Correctly reset p_changed after storing changed details 10 | back to cache. 11 | 12 | 1.58 Mon May 5 16:25 2025 13 | - Catch a rare bug if someone closes our underlying file 14 | handle unexpectedly 15 | 16 | 1.57 Thu Sep 30 16:25 2021 17 | - Add expire($key) method to allow explicitly expiring 18 | a key from the cache. In write-back mode, if the key 19 | is dirty it will be written back, in other modes it's 20 | the same as remove($key) 21 | 22 | 1.56 Mon Dec 14 14:35 2020 23 | - Update MANIFEST to include all tests. It hasn't 24 | been updated in a while 25 | 26 | 1.55 Mon Dec 14 12:10 2020 27 | - Add ability to override internal value of 'time' 28 | everywhere to make tests that check expiry of items 29 | faster and more robust 30 | 31 | 1.54 Sat Dec 12 13:40 2020 32 | - Add Test::Deep as dependency to Makefile.PL 33 | 34 | 1.53 Thu Dec 10 13:40 2020 35 | - Handle expire_on being undef (use cache default) and 36 | return undef if existing value not found in cache. 37 | Makes get_and_set which passes on expire_on just work 38 | in "doesn't exist" case 39 | 40 | 1.52 Thu Dec 10 12:20 2020 41 | - Fix bug where a get() on a key that was expired would 42 | cause that key to be deleted and never written back 43 | even if it was dirty 44 | - Added new expire_on naming so that expire_time 45 | (relative future time) and expire_on (absolute 46 | unix epoch time) are now clear and distinct 47 | - Allow setting expire_on when doing a set() call, 48 | which is useful from get_and_set() to maintain 49 | an items absolute expiry time 50 | 51 | 1.51 Wed Nov 11 17:15 2020 52 | - Skip JSON/Sereal tests if modules not present 53 | - Updated .gitignore 54 | Thanks to https://github.com/szabgab/ 55 | - GitHub Actions config file 56 | Thanks to https://github.com/szabgab/ 57 | 58 | 1.50 Fri Nov 6 20:40 2020 59 | - Allow get_and_set sub to return an options 60 | hash passed to the internal set call 61 | 62 | 1.49 Tue Mar 24 10:15 2020 63 | - Fix windows compilation and test warnings 64 | Thanks to hvn@radiatorsoftware.com 65 | 66 | 1.48 Thu Apr 11 11:30 2019 67 | - Remove use of // so we should work on 5.8 again 68 | - Replace -1 with a NOPAGE constant 69 | - Use 64bit offsets so we support caches > 4G 70 | - Various valgrind code fixes 71 | Thanks to oschwald@gmail.com 72 | 73 | 1.47 Fri Apr 6 11:10 2018 74 | - Allow custom serializer or compressor options 75 | - Update bugtracker link to github 76 | Thanks to toddr@cpan.org 77 | 78 | 1.46 Fri Jul 14 19:40 2017 79 | - Fix tests on older perls (thanks stephanloyd9@gmail.com) 80 | - Use File::Spec for temp dir (thanks fraserbn@gmail.com) 81 | - Fix mmap_cache_internals.h include guard 82 | - Fix get_and_set() always returning 0 for DidStore 83 | - Allow setting permission when creating cache file 84 | - Tweak leak detection tests 85 | - Fix last_access/expire_time checks for impending 86 | 1500000000 seconds since epoch 87 | 88 | 1.45 Thu Mar 14 11:10 2017 89 | - Deprecate raw_values and compress options, add 90 | new compressor and serializer options to allow 91 | different compressors and serializers. 92 | Initial compressor support: zlib, lz4, snappy 93 | Initial serializer support: storable, sereal, json 94 | Thanks to nickt@broadbean.com for initial work 95 | 96 | 1.44 Wed Jun 1 21:45 2016 97 | - Set FD_CLOEXEC flag on cache files when opened. 98 | Particularly useful in Net::Server where 99 | HUPing a process causes it to exec() itself. 100 | Unless you undef the cache references, you'll 101 | leak fd's after each HUP 102 | 103 | 1.43 Fri Oct 23 14:00 2015 104 | - Update copyright + version correctly everywhere 105 | 106 | 1.42 Fri Oct 23 13:30 2015 107 | - Allow get_and_set callback to return an empty list 108 | which means "no change". This allows atomic 109 | "get, and set if not present" type action, but 110 | without resetting the expiry time on each get. 111 | This is basically the same as using the read_cb 112 | 113 | 1.41 Thu Aug 29 15:30 2013 114 | - Actually reuse deleted slots. Meant that if 115 | you used ->remove() a lot, we would re-organise 116 | cache more than needed 117 | - Include typo and meta patches from dsteinbrunner 118 | 119 | 1.40 Mon Dec 5 10:30 2011 120 | - Work around reference holding bug in 121 | Compress::Zlib 122 | 123 | 1.39 Mon Jul 18 09:50 2011 124 | - Remove CImpl and simplify structure into just 125 | Cache::FastMmap module making all XS calls 126 | just function calls in Cache::FastMmap 127 | namespace 128 | 129 | 1.38 Sun Jul 17 18:30 2011 130 | - Fix build process that was completely broken 131 | after moving files around into different 132 | directories 133 | 134 | 1.37 Fri Jul 15 16:30 2011 135 | - Use a lock object with DESTROY method to avoid 136 | an alarm with a die leaving around a locked 137 | paged 138 | 139 | 1.36 Wed Sep 29 13:10 2010 140 | - Disable wrapping fcntl() lock call in alarm, hurts 141 | people that use Time::HiRes::alarm() only to try 142 | and catch buggy deadlock code. Enable with 143 | catch_deadlocks option 144 | 145 | 1.35 Fri Feb 19 12:45 2010 146 | - Fix for returning potential bug that returns old stored 147 | data. Could occur if you mix deletes 148 | (thanks Darrell Bishop) 149 | 150 | 1.34 Fri Jun 19 12:00 2009 151 | - perldoc fix (thanks Jonathan Yu) 152 | 153 | 1.33 Thu Jun 18 12:00 2009 154 | - Update version in META.yml 155 | 156 | 1.32 Thu Jun 18 11:55 2009 157 | - Better LiveCaches tracking via DESTROY 158 | 159 | 1.31 Thu Jun 18 11:40 2009 160 | - when in raw_values => 0 mode, the write_cb is now 161 | correctly called with thawed data, rather than the 162 | raw frozen data 163 | - empty_on_exit correctly called even when a global 164 | cache is left at interpreter exit time (required 165 | Scalar::Util qw(weaken) for object tracking) 166 | 167 | 1.30 Fri May 8 11:10 2009 168 | - Fix for Mandriva compiler (thanks Jean-Christian Hassler) 169 | 170 | 1.29 Fri May 1 17:20 2009 171 | - Support for Windows (thanks to Ash & kmx.at.volny.cz) 172 | (https://rt.cpan.org/Public/Bug/Display.html?id=45210) 173 | (https://rt.cpan.org/Public/Bug/Display.html?id=16501) 174 | 175 | 1.28 Fri Jun 27 11:05 2008 176 | - get_and_set() returns new value + didstore boolean 177 | if called in list context 178 | 179 | 1.27 Wed Jun 18 17:15 2008 180 | - Fix non-ansi C code 181 | - Remove debug flags 182 | 183 | 1.26 Thu May 22 14:50 2008 184 | - Check for write failure when creating file 185 | thanks to Sam Vilain 186 | - Check for $ENV{TMP_DIR} 187 | thanks to Sam Vilain 188 | - Add compress option 189 | - Add basic statistics gathering 190 | 191 | 1.25 Mon Feb 04 13:20 2008 192 | - Fix multi_set bug and add test 193 | (http://rt.cpan.org/Ticket/Display.html?id=32895) 194 | - Test share_file is not a reference 195 | (http://rt.cpan.org/Ticket/Display.html?id=32252) 196 | - Fix C variable declaration error 197 | (http://rt.cpan.org/Ticket/Display.html?id=31223) 198 | - Fix compile warnings in FreeBSD 199 | (http://rt.cpan.org/Ticket/Display.html?id=31900) 200 | - Thanks to all the people that contributed 201 | to the above bugs 202 | 203 | 1.24 Mon Oct 22 13:15 2007 204 | - Add atomic get_and_remove() method thanks to 205 | Darrell Bishop 206 | 207 | 1.23 Wed Oct 17 16:00 2007 208 | - Fix expire time parsing 209 | 210 | 1.22 Wed Oct 17 14:05 2007 211 | - If third parameter to set() is not a references, treat it as 212 | a specify expiry time. Increases compatiability with 213 | Cache::Cache API. Helpful for Catalyst framework 214 | 215 | 1.21 Tue Oct 16 10:40 2007 216 | - if first parameter to new() is a hash ref, use it as 217 | the options hash. Helpful for Catalyst framework 218 | 219 | 1.20 Thu Oct 2 13:40 2007 220 | - add to documentation about page size and cache file locations 221 | - fix t/6.t test failure under new Test::More 222 | 223 | 1.19 Thu Aug 23 09:03 2007 224 | - bad C declaration crept in again, now in svn 225 | 226 | 1.18 Thu Aug 22 17:30 2007 227 | - fix use of $^O not to catch "darwin" 228 | (http://rt.cpan.org/Ticket/Display.html?id=28330) 229 | 230 | 1.17 Thu Aug 22 17:14 2007 231 | - fix declaration in C code that wasn't legal C 232 | 233 | 1.16 Thu May 8 17:12 2007 234 | - fix typo in Changes file (1.15 is 2007, not 2006) 235 | - fix get_keys(2) when undef values in cache 236 | - fix some leak detection tests 237 | 238 | 1.15 Thu May 8 17:12 2007 239 | - catch and rethrow die/exceptions in get_and_set() callbacks 240 | - avoid undef warnings when using cache_not_found mode 241 | - use unique tempfile name rather than the same every time 242 | - add allow_recursive option to allow calls to cache 243 | from within a read/write callback sub 244 | - add checks to die if we try and lock a cache twice, 245 | rather than just causing mysterious errors 246 | - add unlink_on_exit to automatically delete the share_file 247 | when the cache exits. default to true if we created 248 | the share_file, false if we connected to an existing one 249 | - make empty_on_exit only call empty if the pid of the 250 | process we're cleaning up in is the same as the pid we 251 | were created in 252 | - die in CLONE, making it clear threads aren't supported 253 | 254 | 1.14 Thu Oct 20 11:45 2006 255 | - alter calc_expunge to allow more efficient alternate 256 | implementation cases 257 | 258 | 1.13 Thu Oct 20 11:15 2006 259 | - mention UNIX/Win32 compatiability in docs 260 | (http://rt.cpan.org/Ticket/Display.html?id=16501) 261 | - detect page corruption better and croak rather than segfault 262 | (http://rt.cpan.org/Ticket/Display.html?id=17335) 263 | - when running in raw_values => 0 mode, always store 264 | reference to data. Storable doesn't like freeze(SCALAR) 265 | (http://rt.cpan.org/Ticket/Display.html?id=16762) 266 | - Handle edge case of slot count increase when page 267 | already nearly full possibly causing corruption 268 | (can only happen if ->purge() called at just the wrong time) 269 | 270 | 1.12 Thu Oct 19 09:50 2006 271 | - allow writing into a deleted slot 272 | 273 | 1.11 Web Oct 18 15:10 2006 274 | - allow setting default slot count via start_slots argument 275 | 276 | 1.10 Web Oct 18 14:50 2006 277 | - fc_lock() would segfault if no slots were available. 278 | Doesn't happen in normal usage, but can happen if 279 | cache behaviour altered by alternate code 280 | 281 | 1.09 Thu Feb 7 15:50 2005 282 | - add get_and_set() routine to allow atomic reading and 283 | writing of a cache value (thanks to Sreeji Das) 284 | - fix some tests 285 | 286 | 1.08 Thu Aug 26 12:18 2004 287 | - really remove dependency on perl 5.8 288 | 289 | 1.07 Thu Aug 19 22:18 2004 290 | - add extra documentation 291 | - add parameter to empty() method 292 | - add ability to test integrity of cache file 293 | - remove dependency on perl 5.8 294 | 295 | 1.06 Thu May 10 17:18 2004 296 | - add multi_set and multi_get methods 297 | 298 | 1.05 Sat Jan 31 17:24 2004 299 | - fix another edge case where page would get full, but never 300 | expunged when storing references 301 | 302 | 1.04 Sun Jan 25 00:46 2004 303 | - fix test file after new changes 304 | 305 | 1.03 Sun Jan 25 00:21 2004 306 | - fix bad sorting when removing old cache entries 307 | 308 | 1.02 Sat Jan 24 17:05 2004 309 | - fix edge case where page would get full, but never expunged 310 | 311 | 1.01 Sat Dec 13 18:17 2003 312 | - fix leak from improper munmap call 313 | 314 | 1.00 Sat Dec 13 14:19 2003 315 | - initial release 316 | 317 | -------------------------------------------------------------------------------- /FastMmap.xs: -------------------------------------------------------------------------------- 1 | #include "EXTERN.h" 2 | #include "perl.h" 3 | #include "XSUB.h" 4 | 5 | #include "ppport.h" 6 | #include "mmap_cache.h" 7 | 8 | #define FC_UTF8VAL (1<<31) 9 | #define FC_UTF8KEY (1<<30) 10 | #define FC_UNDEF (1<<29) 11 | 12 | #define FC_ENTRY \ 13 | mmap_cache * cache; \ 14 | if (!SvROK(obj)) { \ 15 | croak("Object not reference"); \ 16 | XSRETURN_UNDEF; \ 17 | } \ 18 | obj = SvRV(obj); \ 19 | if (!SvIOKp(obj)) { \ 20 | croak("Object not initialised correctly"); \ 21 | XSRETURN_UNDEF; \ 22 | } \ 23 | cache = INT2PTR(mmap_cache *, SvIV(obj) ); \ 24 | if (!cache) { \ 25 | croak("Object not created correctly"); \ 26 | XSRETURN_UNDEF; \ 27 | } 28 | 29 | 30 | MODULE = Cache::FastMmap PACKAGE = Cache::FastMmap 31 | PROTOTYPES: ENABLE 32 | 33 | SV * 34 | fc_new() 35 | INIT: 36 | mmap_cache * cache; 37 | SV * obj_pnt, * obj; 38 | CODE: 39 | cache = mmc_new(); 40 | 41 | /* Create integer which is pointer to cache object */ 42 | obj_pnt = newSViv(PTR2IV(cache)); 43 | 44 | /* Create reference to integer value. This will be the object */ 45 | obj = newRV_noinc((SV *)obj_pnt); 46 | 47 | RETVAL = obj; 48 | OUTPUT: 49 | RETVAL 50 | 51 | NO_OUTPUT int 52 | fc_set_param(obj, param, val) 53 | SV * obj; 54 | char * param; 55 | char * val; 56 | INIT: 57 | FC_ENTRY 58 | 59 | CODE: 60 | RETVAL = mmc_set_param(cache, param, val); 61 | POSTCALL: 62 | if (RETVAL != 0) { 63 | croak("%s", mmc_error(cache)); 64 | } 65 | 66 | NO_OUTPUT int 67 | fc_init(obj) 68 | SV * obj; 69 | INIT: 70 | FC_ENTRY 71 | 72 | CODE: 73 | RETVAL = mmc_init(cache); 74 | POSTCALL: 75 | if (RETVAL != 0) { 76 | croak("%s", mmc_error(cache)); 77 | } 78 | 79 | 80 | void 81 | fc_close(obj) 82 | SV * obj 83 | INIT: 84 | FC_ENTRY 85 | 86 | CODE: 87 | mmc_close(cache); 88 | sv_setiv(obj, 0); 89 | 90 | 91 | void 92 | fc_hash(obj, key); 93 | SV * obj; 94 | SV * key; 95 | INIT: 96 | int key_len; 97 | void * key_ptr; 98 | MU32 hash_page, hash_slot; 99 | STRLEN pl_key_len; 100 | 101 | FC_ENTRY 102 | 103 | PPCODE: 104 | 105 | /* Get key length, data pointer */ 106 | key_ptr = (void *)SvPV(key, pl_key_len); 107 | key_len = (int)pl_key_len; 108 | 109 | /* Hash key to get page and slot */ 110 | mmc_hash(cache, key_ptr, key_len, &hash_page, &hash_slot); 111 | 112 | XPUSHs(sv_2mortal(newSViv((IV)hash_page))); 113 | XPUSHs(sv_2mortal(newSViv((IV)hash_slot))); 114 | 115 | 116 | NO_OUTPUT int 117 | fc_lock(obj, page); 118 | SV * obj; 119 | UV page; 120 | INIT: 121 | FC_ENTRY 122 | 123 | CODE: 124 | RETVAL = mmc_lock(cache, (MU32)page); 125 | POSTCALL: 126 | if (RETVAL != 0) { 127 | croak("%s", mmc_error(cache)); 128 | } 129 | 130 | 131 | NO_OUTPUT int 132 | fc_unlock(obj); 133 | SV * obj; 134 | INIT: 135 | FC_ENTRY 136 | 137 | CODE: 138 | RETVAL = mmc_unlock(cache); 139 | POSTCALL: 140 | if (RETVAL != 0) { 141 | croak("%s", mmc_error(cache)); 142 | } 143 | 144 | int 145 | fc_is_locked(obj) 146 | SV * obj; 147 | INIT: 148 | FC_ENTRY 149 | 150 | CODE: 151 | /* Write value to cache */ 152 | RETVAL = mmc_is_locked(cache); 153 | 154 | OUTPUT: 155 | RETVAL 156 | 157 | 158 | void 159 | fc_read(obj, hash_slot, key) 160 | SV * obj; 161 | U32 hash_slot; 162 | SV * key; 163 | INIT: 164 | int key_len, val_len, found; 165 | void * key_ptr, * val_ptr; 166 | MU32 expire_on = 0; 167 | MU32 flags = 0; 168 | STRLEN pl_key_len; 169 | SV * val; 170 | 171 | FC_ENTRY 172 | 173 | PPCODE: 174 | 175 | /* Get key length, data pointer */ 176 | key_ptr = (void *)SvPV(key, pl_key_len); 177 | key_len = (int)pl_key_len; 178 | 179 | /* Get value data pointer */ 180 | found = mmc_read(cache, (MU32)hash_slot, key_ptr, key_len, &val_ptr, &val_len, &expire_on, &flags); 181 | 182 | /* If not found, use undef */ 183 | if (found == -1) { 184 | val = &PL_sv_undef; 185 | } else { 186 | 187 | /* Cached an undef value? */ 188 | if (flags & FC_UNDEF) { 189 | val = &PL_sv_undef; 190 | 191 | } else { 192 | 193 | /* Create PERL SV */ 194 | val = sv_2mortal(newSVpvn((const char *)val_ptr, val_len)); 195 | 196 | /* Make UTF8 if stored from UTF8 */ 197 | if (flags & FC_UTF8VAL) { 198 | SvUTF8_on(val); 199 | } 200 | 201 | } 202 | flags = flags & ~(FC_UTF8KEY | FC_UTF8VAL | FC_UNDEF); 203 | } 204 | 205 | XPUSHs(val); 206 | XPUSHs(sv_2mortal(newSViv((IV)flags))); 207 | XPUSHs(sv_2mortal(newSViv((IV)!found))); 208 | XPUSHs(sv_2mortal(newSViv((IV)expire_on))); 209 | 210 | 211 | int 212 | fc_write(obj, hash_slot, key, val, expire_on, in_flags) 213 | SV * obj; 214 | U32 hash_slot; 215 | SV * key; 216 | SV * val; 217 | U32 expire_on; 218 | U32 in_flags; 219 | INIT: 220 | int key_len, val_len; 221 | void * key_ptr, * val_ptr; 222 | STRLEN pl_key_len, pl_val_len; 223 | 224 | FC_ENTRY 225 | 226 | CODE: 227 | 228 | /* Get key length, data pointer */ 229 | key_ptr = (void *)SvPV(key, pl_key_len); 230 | key_len = (int)pl_key_len; 231 | 232 | /* Check for storing undef, and store empty string with undef flag set */ 233 | if (!SvOK(val)) { 234 | in_flags |= FC_UNDEF; 235 | 236 | val_ptr = ""; 237 | val_len = 0; 238 | 239 | } else { 240 | 241 | /* Get key length, data pointer */ 242 | val_ptr = (void *)SvPV(val, pl_val_len); 243 | val_len = (int)pl_val_len; 244 | 245 | /* Set UTF8-ness flag of stored value */ 246 | if (SvUTF8(val)) { 247 | in_flags |= FC_UTF8VAL; 248 | } 249 | if (SvUTF8(key)) { 250 | in_flags |= FC_UTF8KEY; 251 | } 252 | } 253 | 254 | /* Write value to cache */ 255 | RETVAL = mmc_write(cache, (MU32)hash_slot, key_ptr, key_len, val_ptr, val_len, (MU32)expire_on, (MU32)in_flags); 256 | 257 | OUTPUT: 258 | RETVAL 259 | 260 | int 261 | fc_delete(obj, hash_slot, key) 262 | SV * obj; 263 | U32 hash_slot; 264 | SV * key; 265 | INIT: 266 | MU32 out_flags; 267 | int key_len, did_delete; 268 | void * key_ptr; 269 | STRLEN pl_key_len; 270 | 271 | FC_ENTRY 272 | 273 | PPCODE: 274 | 275 | /* Get key length, data pointer */ 276 | key_ptr = (void *)SvPV(key, pl_key_len); 277 | key_len = (int)pl_key_len; 278 | 279 | /* Write value to cache */ 280 | did_delete = mmc_delete(cache, (MU32)hash_slot, key_ptr, key_len, &out_flags); 281 | 282 | XPUSHs(sv_2mortal(newSViv((IV)did_delete))); 283 | XPUSHs(sv_2mortal(newSViv((IV)out_flags))); 284 | 285 | 286 | void 287 | fc_get_page_details(obj) 288 | SV * obj; 289 | INIT: 290 | MU32 nreads = 0, nreadhits = 0; 291 | 292 | FC_ENTRY 293 | 294 | PPCODE: 295 | mmc_get_page_details(cache, &nreads, &nreadhits); 296 | 297 | XPUSHs(sv_2mortal(newSViv((IV)nreads))); 298 | XPUSHs(sv_2mortal(newSViv((IV)nreadhits))); 299 | 300 | 301 | NO_OUTPUT void 302 | fc_reset_page_details(obj) 303 | SV * obj; 304 | INIT: 305 | FC_ENTRY 306 | 307 | CODE: 308 | mmc_reset_page_details(cache); 309 | 310 | 311 | 312 | void 313 | fc_expunge(obj, mode, wb, len) 314 | SV * obj; 315 | int mode; 316 | int wb; 317 | int len; 318 | INIT: 319 | MU32 new_num_slots = 0, ** to_expunge = 0; 320 | int num_expunge, item; 321 | 322 | void * key_ptr, * val_ptr; 323 | int key_len, val_len; 324 | MU32 last_access, expire_on, flags; 325 | 326 | FC_ENTRY 327 | 328 | PPCODE: 329 | 330 | num_expunge = mmc_calc_expunge(cache, mode, len, &new_num_slots, &to_expunge); 331 | if (to_expunge) { 332 | 333 | /* Want list of expunged keys/values? */ 334 | if (wb) { 335 | 336 | for (item = 0; item < num_expunge; item++) { 337 | mmc_get_details(cache, to_expunge[item], 338 | &key_ptr, &key_len, &val_ptr, &val_len, 339 | &last_access, &expire_on, &flags); 340 | 341 | { 342 | HV * ih = (HV *)sv_2mortal((SV *)newHV()); 343 | 344 | SV * key = newSVpvn((const char *)key_ptr, key_len); 345 | SV * val; 346 | 347 | if (flags & FC_UTF8KEY) { 348 | SvUTF8_on(key); 349 | flags ^= FC_UTF8KEY; 350 | } 351 | 352 | if (flags & FC_UNDEF) { 353 | val = newSV(0); 354 | flags ^= FC_UNDEF; 355 | } else { 356 | val = newSVpvn((const char *)val_ptr, val_len); 357 | if (flags & FC_UTF8VAL) { 358 | SvUTF8_on(val); 359 | flags ^= FC_UTF8VAL; 360 | } 361 | } 362 | 363 | /* Store in hash ref */ 364 | hv_store(ih, "key", 3, key, 0); 365 | hv_store(ih, "value", 5, val, 0); 366 | hv_store(ih, "last_access", 11, newSViv((IV)last_access), 0); 367 | hv_store(ih, "expire_on", 9, newSViv((IV)expire_on), 0); 368 | hv_store(ih, "flags", 5, newSViv((IV)flags), 0); 369 | 370 | /* Create reference to hash */ 371 | XPUSHs(sv_2mortal(newRV((SV *)ih))); 372 | } 373 | } 374 | } 375 | 376 | if (!mmc_do_expunge(cache, num_expunge, new_num_slots, to_expunge)) { 377 | croak("%s", mmc_error(cache)); 378 | XSRETURN_UNDEF; 379 | } 380 | } 381 | 382 | 383 | 384 | 385 | void 386 | fc_get_keys(obj, mode) 387 | SV * obj; 388 | int mode; 389 | INIT: 390 | mmap_cache_it * it; 391 | MU32 * entry_ptr; 392 | void * key_ptr, * val_ptr; 393 | int key_len, val_len; 394 | MU32 last_access, expire_on, flags; 395 | 396 | FC_ENTRY 397 | 398 | PPCODE: 399 | 400 | it = mmc_iterate_new(cache); 401 | 402 | /* Iterate over all items */ 403 | while ((entry_ptr = mmc_iterate_next(it))) { 404 | SV * key; 405 | mmc_get_details(cache, entry_ptr, 406 | &key_ptr, &key_len, &val_ptr, &val_len, 407 | &last_access, &expire_on, &flags); 408 | 409 | /* Create key SV, and set UTF8'ness if needed */ 410 | key = newSVpvn((const char *)key_ptr, key_len); 411 | if (flags & FC_UTF8KEY) { 412 | SvUTF8_on(key); 413 | flags ^= FC_UTF8KEY; 414 | } 415 | 416 | /* Mode 0 is just list of keys */ 417 | if (mode == 0) { 418 | XPUSHs(sv_2mortal(key)); 419 | 420 | /* Mode 1/2 is list of hash-refs */ 421 | } else if (mode == 1 || mode == 2) { 422 | HV * ih = (HV *)sv_2mortal((SV *)newHV()); 423 | 424 | /* These things by default */ 425 | hv_store(ih, "key", 3, key, 0); 426 | hv_store(ih, "last_access", 11, newSViv((IV)last_access), 0); 427 | hv_store(ih, "expire_on", 9, newSViv((IV)expire_on), 0); 428 | hv_store(ih, "flags", 5, newSViv((IV)flags), 0); 429 | 430 | /* Add value to hash-ref if mode 2 */ 431 | if (mode == 2) { 432 | SV * val; 433 | if (flags & FC_UNDEF) { 434 | val = newSV(0); 435 | flags ^= FC_UNDEF; 436 | } else { 437 | val = newSVpvn((const char *)val_ptr, val_len); 438 | if (flags & FC_UTF8VAL) { 439 | SvUTF8_on(val); 440 | flags ^= FC_UTF8VAL; 441 | } 442 | } 443 | hv_store(ih, "value", 5, val, 0); 444 | } 445 | 446 | /* Create reference to hash */ 447 | XPUSHs(sv_2mortal(newRV((SV *)ih))); 448 | } 449 | } 450 | 451 | mmc_iterate_close(it); 452 | 453 | 454 | 455 | 456 | 457 | SV * 458 | fc_get(obj, key) 459 | SV * obj; 460 | SV * key; 461 | INIT: 462 | int key_len, val_len, found; 463 | void * key_ptr, * val_ptr; 464 | MU32 hash_page, hash_slot, expire_on, flags; 465 | STRLEN pl_key_len; 466 | SV * val; 467 | 468 | FC_ENTRY 469 | 470 | CODE: 471 | 472 | /* Get key length, data pointer */ 473 | key_ptr = (void *)SvPV(key, pl_key_len); 474 | key_len = (int)pl_key_len; 475 | 476 | /* Hash key to get page and slot */ 477 | mmc_hash(cache, key_ptr, key_len, &hash_page, &hash_slot); 478 | 479 | /* Get and lock the page */ 480 | mmc_lock(cache, hash_page); 481 | 482 | /* Get value data pointer */ 483 | found = mmc_read(cache, hash_slot, key_ptr, key_len, &val_ptr, &val_len, &expire_on, &flags); 484 | 485 | /* If not found, use undef */ 486 | if (found == -1) { 487 | val = &PL_sv_undef; 488 | } else { 489 | 490 | /* Create PERL SV */ 491 | val = newSVpvn((const char *)val_ptr, val_len); 492 | } 493 | 494 | mmc_unlock(cache); 495 | RETVAL = val; 496 | OUTPUT: 497 | RETVAL 498 | 499 | 500 | void 501 | fc_set(obj, key, val) 502 | SV * obj; 503 | SV * key; 504 | SV * val; 505 | INIT: 506 | int key_len, val_len; 507 | void * key_ptr, * val_ptr; 508 | MU32 hash_page, hash_slot, flags = 0; 509 | STRLEN pl_key_len, pl_val_len; 510 | 511 | FC_ENTRY 512 | 513 | CODE: 514 | 515 | /* Get key length, data pointer */ 516 | key_ptr = (void *)SvPV(key, pl_key_len); 517 | key_len = (int)pl_key_len; 518 | 519 | /* Get key length, data pointer */ 520 | val_ptr = (void *)SvPV(val, pl_val_len); 521 | val_len = (int)pl_val_len; 522 | 523 | /* Hash key to get page and slot */ 524 | mmc_hash(cache, key_ptr, key_len, &hash_page, &hash_slot); 525 | 526 | /* Get and lock the page */ 527 | mmc_lock(cache, hash_page); 528 | 529 | /* Get value data pointer */ 530 | mmc_write(cache, hash_slot, key_ptr, key_len, val_ptr, val_len, -1, flags); 531 | 532 | mmc_unlock(cache); 533 | 534 | 535 | NO_OUTPUT void 536 | fc_dump_page(obj); 537 | SV * obj; 538 | INIT: 539 | FC_ENTRY 540 | 541 | CODE: 542 | _mmc_dump_page(cache); 543 | 544 | 545 | NO_OUTPUT void 546 | fc_set_time_override(set_time); 547 | UV set_time; 548 | 549 | CODE: 550 | mmc_set_time_override((MU32)set_time); 551 | 552 | 553 | -------------------------------------------------------------------------------- /ppport.h: -------------------------------------------------------------------------------- 1 | 2 | /* ppport.h -- Perl/Pollution/Portability Version 2.0002 3 | * 4 | * Automatically Created by Devel::PPPort on Wed Nov 19 19:24:25 2003 5 | * 6 | * Do NOT edit this file directly! -- Edit PPPort.pm instead. 7 | * 8 | * Version 2.x, Copyright (C) 2001, Paul Marquess. 9 | * Version 1.x, Copyright (C) 1999, Kenneth Albanowski. 10 | * This code may be used and distributed under the same license as any 11 | * version of Perl. 12 | * 13 | * This version of ppport.h is designed to support operation with Perl 14 | * installations back to 5.004, and has been tested up to 5.8.0. 15 | * 16 | * If this version of ppport.h is failing during the compilation of this 17 | * module, please check if a newer version of Devel::PPPort is available 18 | * on CPAN before sending a bug report. 19 | * 20 | * If you are using the latest version of Devel::PPPort and it is failing 21 | * during compilation of this module, please send a report to perlbug@perl.com 22 | * 23 | * Include all following information: 24 | * 25 | * 1. The complete output from running "perl -V" 26 | * 27 | * 2. This file. 28 | * 29 | * 3. The name & version of the module you were trying to build. 30 | * 31 | * 4. A full log of the build that failed. 32 | * 33 | * 5. Any other information that you think could be relevant. 34 | * 35 | * 36 | * For the latest version of this code, please retreive the Devel::PPPort 37 | * module from CPAN. 38 | * 39 | */ 40 | 41 | /* 42 | * In order for a Perl extension module to be as portable as possible 43 | * across differing versions of Perl itself, certain steps need to be taken. 44 | * Including this header is the first major one, then using dTHR is all the 45 | * appropriate places and using a PL_ prefix to refer to global Perl 46 | * variables is the second. 47 | * 48 | */ 49 | 50 | 51 | /* If you use one of a few functions that were not present in earlier 52 | * versions of Perl, please add a define before the inclusion of ppport.h 53 | * for a static include, or use the GLOBAL request in a single module to 54 | * produce a global definition that can be referenced from the other 55 | * modules. 56 | * 57 | * Function: Static define: Extern define: 58 | * newCONSTSUB() NEED_newCONSTSUB NEED_newCONSTSUB_GLOBAL 59 | * 60 | */ 61 | 62 | 63 | /* To verify whether ppport.h is needed for your module, and whether any 64 | * special defines should be used, ppport.h can be run through Perl to check 65 | * your source code. Simply say: 66 | * 67 | * perl -x ppport.h *.c *.h *.xs foo/bar*.c [etc] 68 | * 69 | * The result will be a list of patches suggesting changes that should at 70 | * least be acceptable, if not necessarily the most efficient solution, or a 71 | * fix for all possible problems. It won't catch where dTHR is needed, and 72 | * doesn't attempt to account for global macro or function definitions, 73 | * nested includes, typemaps, etc. 74 | * 75 | * In order to test for the need of dTHR, please try your module under a 76 | * recent version of Perl that has threading compiled-in. 77 | * 78 | */ 79 | 80 | 81 | /* 82 | #!/usr/bin/perl 83 | @ARGV = ("*.xs") if !@ARGV; 84 | %badmacros = %funcs = %macros = (); $replace = 0; 85 | foreach () { 86 | $funcs{$1} = 1 if /Provide:\s+(\S+)/; 87 | $macros{$1} = 1 if /^#\s*define\s+([a-zA-Z0-9_]+)/; 88 | $replace = $1 if /Replace:\s+(\d+)/; 89 | $badmacros{$2}=$1 if $replace and /^#\s*define\s+([a-zA-Z0-9_]+).*?\s+([a-zA-Z0-9_]+)/; 90 | $badmacros{$1}=$2 if /Replace (\S+) with (\S+)/; 91 | } 92 | foreach $filename (map(glob($_),@ARGV)) { 93 | unless (open(IN, "<$filename")) { 94 | warn "Unable to read from $file: $!\n"; 95 | next; 96 | } 97 | print "Scanning $filename...\n"; 98 | $c = ""; while () { $c .= $_; } close(IN); 99 | $need_include = 0; %add_func = (); $changes = 0; 100 | $has_include = ($c =~ /#.*include.*ppport/m); 101 | 102 | foreach $func (keys %funcs) { 103 | if ($c =~ /#.*define.*\bNEED_$func(_GLOBAL)?\b/m) { 104 | if ($c !~ /\b$func\b/m) { 105 | print "If $func isn't needed, you don't need to request it.\n" if 106 | $changes += ($c =~ s/^.*#.*define.*\bNEED_$func\b.*\n//m); 107 | } else { 108 | print "Uses $func\n"; 109 | $need_include = 1; 110 | } 111 | } else { 112 | if ($c =~ /\b$func\b/m) { 113 | $add_func{$func} =1 ; 114 | print "Uses $func\n"; 115 | $need_include = 1; 116 | } 117 | } 118 | } 119 | 120 | if (not $need_include) { 121 | foreach $macro (keys %macros) { 122 | if ($c =~ /\b$macro\b/m) { 123 | print "Uses $macro\n"; 124 | $need_include = 1; 125 | } 126 | } 127 | } 128 | 129 | foreach $badmacro (keys %badmacros) { 130 | if ($c =~ /\b$badmacro\b/m) { 131 | $changes += ($c =~ s/\b$badmacro\b/$badmacros{$badmacro}/gm); 132 | print "Uses $badmacros{$badmacro} (instead of $badmacro)\n"; 133 | $need_include = 1; 134 | } 135 | } 136 | 137 | if (scalar(keys %add_func) or $need_include != $has_include) { 138 | if (!$has_include) { 139 | $inc = join('',map("#define NEED_$_\n", sort keys %add_func)). 140 | "#include \"ppport.h\"\n"; 141 | $c = "$inc$c" unless $c =~ s/#.*include.*XSUB.*\n/$&$inc/m; 142 | } elsif (keys %add_func) { 143 | $inc = join('',map("#define NEED_$_\n", sort keys %add_func)); 144 | $c = "$inc$c" unless $c =~ s/^.*#.*include.*ppport.*$/$inc$&/m; 145 | } 146 | if (!$need_include) { 147 | print "Doesn't seem to need ppport.h.\n"; 148 | $c =~ s/^.*#.*include.*ppport.*\n//m; 149 | } 150 | $changes++; 151 | } 152 | 153 | if ($changes) { 154 | open(OUT,">/tmp/ppport.h.$$"); 155 | print OUT $c; 156 | close(OUT); 157 | open(DIFF, "diff -u $filename /tmp/ppport.h.$$|"); 158 | while () { s!/tmp/ppport\.h\.$$!$filename.patched!; print STDOUT; } 159 | close(DIFF); 160 | unlink("/tmp/ppport.h.$$"); 161 | } else { 162 | print "Looks OK\n"; 163 | } 164 | } 165 | __DATA__ 166 | */ 167 | 168 | #ifndef _P_P_PORTABILITY_H_ 169 | #define _P_P_PORTABILITY_H_ 170 | 171 | #ifndef PERL_REVISION 172 | # ifndef __PATCHLEVEL_H_INCLUDED__ 173 | # include "patchlevel.h" 174 | # endif 175 | # ifndef PERL_REVISION 176 | # define PERL_REVISION (5) 177 | /* Replace: 1 */ 178 | # define PERL_VERSION PATCHLEVEL 179 | # define PERL_SUBVERSION SUBVERSION 180 | /* Replace PERL_PATCHLEVEL with PERL_VERSION */ 181 | /* Replace: 0 */ 182 | # endif 183 | #endif 184 | 185 | #define PERL_BCDVERSION ((PERL_REVISION * 0x1000000L) + (PERL_VERSION * 0x1000L) + PERL_SUBVERSION) 186 | 187 | /* It is very unlikely that anyone will try to use this with Perl 6 188 | (or greater), but who knows. 189 | */ 190 | #if PERL_REVISION != 5 191 | # error ppport.h only works with Perl version 5 192 | #endif /* PERL_REVISION != 5 */ 193 | 194 | #ifndef ERRSV 195 | # define ERRSV perl_get_sv("@",FALSE) 196 | #endif 197 | 198 | #if (PERL_VERSION < 4) || ((PERL_VERSION == 4) && (PERL_SUBVERSION <= 5)) 199 | /* Replace: 1 */ 200 | # define PL_Sv Sv 201 | # define PL_compiling compiling 202 | # define PL_copline copline 203 | # define PL_curcop curcop 204 | # define PL_curstash curstash 205 | # define PL_defgv defgv 206 | # define PL_dirty dirty 207 | # define PL_dowarn dowarn 208 | # define PL_hints hints 209 | # define PL_na na 210 | # define PL_perldb perldb 211 | # define PL_rsfp_filters rsfp_filters 212 | # define PL_rsfpv rsfp 213 | # define PL_stdingv stdingv 214 | # define PL_sv_no sv_no 215 | # define PL_sv_undef sv_undef 216 | # define PL_sv_yes sv_yes 217 | /* Replace: 0 */ 218 | #endif 219 | 220 | #ifdef HASATTRIBUTE 221 | # if defined(__GNUC__) && defined(__cplusplus) 222 | # define PERL_UNUSED_DECL 223 | # else 224 | # define PERL_UNUSED_DECL __attribute__((unused)) 225 | # endif 226 | #else 227 | # define PERL_UNUSED_DECL 228 | #endif 229 | 230 | #ifndef dNOOP 231 | # define NOOP (void)0 232 | # define dNOOP extern int Perl___notused PERL_UNUSED_DECL 233 | #endif 234 | 235 | #ifndef dTHR 236 | # define dTHR dNOOP 237 | #endif 238 | 239 | #ifndef dTHX 240 | # define dTHX dNOOP 241 | # define dTHXa(x) dNOOP 242 | # define dTHXoa(x) dNOOP 243 | #endif 244 | 245 | #ifndef pTHX 246 | # define pTHX void 247 | # define pTHX_ 248 | # define aTHX 249 | # define aTHX_ 250 | #endif 251 | 252 | #ifndef UVSIZE 253 | # define UVSIZE IVSIZE 254 | #endif 255 | 256 | #ifndef NVTYPE 257 | # if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) 258 | # define NVTYPE long double 259 | # else 260 | # define NVTYPE double 261 | # endif 262 | typedef NVTYPE NV; 263 | #endif 264 | 265 | #ifndef INT2PTR 266 | 267 | #if (IVSIZE == PTRSIZE) && (UVSIZE == PTRSIZE) 268 | # define PTRV UV 269 | # define INT2PTR(any,d) (any)(d) 270 | #else 271 | # if PTRSIZE == LONGSIZE 272 | # define PTRV unsigned long 273 | # else 274 | # define PTRV unsigned 275 | # endif 276 | # define INT2PTR(any,d) (any)(PTRV)(d) 277 | #endif 278 | #define NUM2PTR(any,d) (any)(PTRV)(d) 279 | #define PTR2IV(p) INT2PTR(IV,p) 280 | #define PTR2UV(p) INT2PTR(UV,p) 281 | #define PTR2NV(p) NUM2PTR(NV,p) 282 | #if PTRSIZE == LONGSIZE 283 | # define PTR2ul(p) (unsigned long)(p) 284 | #else 285 | # define PTR2ul(p) INT2PTR(unsigned long,p) 286 | #endif 287 | 288 | #endif /* !INT2PTR */ 289 | 290 | #ifndef boolSV 291 | # define boolSV(b) ((b) ? &PL_sv_yes : &PL_sv_no) 292 | #endif 293 | 294 | #ifndef gv_stashpvn 295 | # define gv_stashpvn(str,len,flags) gv_stashpv(str,flags) 296 | #endif 297 | 298 | #ifndef newSVpvn 299 | # define newSVpvn(data,len) ((len) ? newSVpv ((data), (len)) : newSVpv ("", 0)) 300 | #endif 301 | 302 | #ifndef newRV_inc 303 | /* Replace: 1 */ 304 | # define newRV_inc(sv) newRV(sv) 305 | /* Replace: 0 */ 306 | #endif 307 | 308 | /* DEFSV appears first in 5.004_56 */ 309 | #ifndef DEFSV 310 | # define DEFSV GvSV(PL_defgv) 311 | #endif 312 | 313 | #ifndef SAVE_DEFSV 314 | # define SAVE_DEFSV SAVESPTR(GvSV(PL_defgv)) 315 | #endif 316 | 317 | #ifndef newRV_noinc 318 | # ifdef __GNUC__ 319 | # define newRV_noinc(sv) \ 320 | ({ \ 321 | SV *nsv = (SV*)newRV(sv); \ 322 | SvREFCNT_dec(sv); \ 323 | nsv; \ 324 | }) 325 | # else 326 | # if defined(USE_THREADS) 327 | static SV * newRV_noinc (SV * sv) 328 | { 329 | SV *nsv = (SV*)newRV(sv); 330 | SvREFCNT_dec(sv); 331 | return nsv; 332 | } 333 | # else 334 | # define newRV_noinc(sv) \ 335 | (PL_Sv=(SV*)newRV(sv), SvREFCNT_dec(sv), (SV*)PL_Sv) 336 | # endif 337 | # endif 338 | #endif 339 | 340 | /* Provide: newCONSTSUB */ 341 | 342 | /* newCONSTSUB from IO.xs is in the core starting with 5.004_63 */ 343 | #if (PERL_VERSION < 4) || ((PERL_VERSION == 4) && (PERL_SUBVERSION < 63)) 344 | 345 | #if defined(NEED_newCONSTSUB) 346 | static 347 | #else 348 | extern void newCONSTSUB(HV * stash, char * name, SV *sv); 349 | #endif 350 | 351 | #if defined(NEED_newCONSTSUB) || defined(NEED_newCONSTSUB_GLOBAL) 352 | void 353 | newCONSTSUB(stash,name,sv) 354 | HV *stash; 355 | char *name; 356 | SV *sv; 357 | { 358 | U32 oldhints = PL_hints; 359 | HV *old_cop_stash = PL_curcop->cop_stash; 360 | HV *old_curstash = PL_curstash; 361 | line_t oldline = PL_curcop->cop_line; 362 | PL_curcop->cop_line = PL_copline; 363 | 364 | PL_hints &= ~HINT_BLOCK_SCOPE; 365 | if (stash) 366 | PL_curstash = PL_curcop->cop_stash = stash; 367 | 368 | newSUB( 369 | 370 | #if (PERL_VERSION < 3) || ((PERL_VERSION == 3) && (PERL_SUBVERSION < 22)) 371 | /* before 5.003_22 */ 372 | start_subparse(), 373 | #else 374 | # if (PERL_VERSION == 3) && (PERL_SUBVERSION == 22) 375 | /* 5.003_22 */ 376 | start_subparse(0), 377 | # else 378 | /* 5.003_23 onwards */ 379 | start_subparse(FALSE, 0), 380 | # endif 381 | #endif 382 | 383 | newSVOP(OP_CONST, 0, newSVpv(name,0)), 384 | newSVOP(OP_CONST, 0, &PL_sv_no), /* SvPV(&PL_sv_no) == "" -- GMB */ 385 | newSTATEOP(0, Nullch, newSVOP(OP_CONST, 0, sv)) 386 | ); 387 | 388 | PL_hints = oldhints; 389 | PL_curcop->cop_stash = old_cop_stash; 390 | PL_curstash = old_curstash; 391 | PL_curcop->cop_line = oldline; 392 | } 393 | #endif 394 | 395 | #endif /* newCONSTSUB */ 396 | 397 | #ifndef START_MY_CXT 398 | 399 | /* 400 | * Boilerplate macros for initializing and accessing interpreter-local 401 | * data from C. All statics in extensions should be reworked to use 402 | * this, if you want to make the extension thread-safe. See ext/re/re.xs 403 | * for an example of the use of these macros. 404 | * 405 | * Code that uses these macros is responsible for the following: 406 | * 1. #define MY_CXT_KEY to a unique string, e.g. "DynaLoader_guts" 407 | * 2. Declare a typedef named my_cxt_t that is a structure that contains 408 | * all the data that needs to be interpreter-local. 409 | * 3. Use the START_MY_CXT macro after the declaration of my_cxt_t. 410 | * 4. Use the MY_CXT_INIT macro such that it is called exactly once 411 | * (typically put in the BOOT: section). 412 | * 5. Use the members of the my_cxt_t structure everywhere as 413 | * MY_CXT.member. 414 | * 6. Use the dMY_CXT macro (a declaration) in all the functions that 415 | * access MY_CXT. 416 | */ 417 | 418 | #if defined(MULTIPLICITY) || defined(PERL_OBJECT) || \ 419 | defined(PERL_CAPI) || defined(PERL_IMPLICIT_CONTEXT) 420 | 421 | /* This must appear in all extensions that define a my_cxt_t structure, 422 | * right after the definition (i.e. at file scope). The non-threads 423 | * case below uses it to declare the data as static. */ 424 | #define START_MY_CXT 425 | 426 | #if (PERL_VERSION < 4 || (PERL_VERSION == 4 && PERL_SUBVERSION < 68 )) 427 | /* Fetches the SV that keeps the per-interpreter data. */ 428 | #define dMY_CXT_SV \ 429 | SV *my_cxt_sv = perl_get_sv(MY_CXT_KEY, FALSE) 430 | #else /* >= perl5.004_68 */ 431 | #define dMY_CXT_SV \ 432 | SV *my_cxt_sv = *hv_fetch(PL_modglobal, MY_CXT_KEY, \ 433 | sizeof(MY_CXT_KEY)-1, TRUE) 434 | #endif /* < perl5.004_68 */ 435 | 436 | /* This declaration should be used within all functions that use the 437 | * interpreter-local data. */ 438 | #define dMY_CXT \ 439 | dMY_CXT_SV; \ 440 | my_cxt_t *my_cxtp = INT2PTR(my_cxt_t*,SvUV(my_cxt_sv)) 441 | 442 | /* Creates and zeroes the per-interpreter data. 443 | * (We allocate my_cxtp in a Perl SV so that it will be released when 444 | * the interpreter goes away.) */ 445 | #define MY_CXT_INIT \ 446 | dMY_CXT_SV; \ 447 | /* newSV() allocates one more than needed */ \ 448 | my_cxt_t *my_cxtp = (my_cxt_t*)SvPVX(newSV(sizeof(my_cxt_t)-1));\ 449 | Zero(my_cxtp, 1, my_cxt_t); \ 450 | sv_setuv(my_cxt_sv, PTR2UV(my_cxtp)) 451 | 452 | /* This macro must be used to access members of the my_cxt_t structure. 453 | * e.g. MYCXT.some_data */ 454 | #define MY_CXT (*my_cxtp) 455 | 456 | /* Judicious use of these macros can reduce the number of times dMY_CXT 457 | * is used. Use is similar to pTHX, aTHX etc. */ 458 | #define pMY_CXT my_cxt_t *my_cxtp 459 | #define pMY_CXT_ pMY_CXT, 460 | #define _pMY_CXT ,pMY_CXT 461 | #define aMY_CXT my_cxtp 462 | #define aMY_CXT_ aMY_CXT, 463 | #define _aMY_CXT ,aMY_CXT 464 | 465 | #else /* single interpreter */ 466 | 467 | 468 | #define START_MY_CXT static my_cxt_t my_cxt; 469 | #define dMY_CXT_SV dNOOP 470 | #define dMY_CXT dNOOP 471 | #define MY_CXT_INIT NOOP 472 | #define MY_CXT my_cxt 473 | 474 | #define pMY_CXT void 475 | #define pMY_CXT_ 476 | #define _pMY_CXT 477 | #define aMY_CXT 478 | #define aMY_CXT_ 479 | #define _aMY_CXT 480 | 481 | #endif 482 | 483 | #endif /* START_MY_CXT */ 484 | 485 | #ifndef IVdf 486 | # if IVSIZE == LONGSIZE 487 | # define IVdf "ld" 488 | # define UVuf "lu" 489 | # define UVof "lo" 490 | # define UVxf "lx" 491 | # define UVXf "lX" 492 | # else 493 | # if IVSIZE == INTSIZE 494 | # define IVdf "d" 495 | # define UVuf "u" 496 | # define UVof "o" 497 | # define UVxf "x" 498 | # define UVXf "X" 499 | # endif 500 | # endif 501 | #endif 502 | 503 | #ifndef NVef 504 | # if defined(USE_LONG_DOUBLE) && defined(HAS_LONG_DOUBLE) && \ 505 | defined(PERL_PRIfldbl) /* Not very likely, but let's try anyway. */ 506 | # define NVef PERL_PRIeldbl 507 | # define NVff PERL_PRIfldbl 508 | # define NVgf PERL_PRIgldbl 509 | # else 510 | # define NVef "e" 511 | # define NVff "f" 512 | # define NVgf "g" 513 | # endif 514 | #endif 515 | 516 | #ifndef AvFILLp /* Older perls (<=5.003) lack AvFILLp */ 517 | # define AvFILLp AvFILL 518 | #endif 519 | 520 | #ifdef SvPVbyte 521 | # if PERL_REVISION == 5 && PERL_VERSION < 7 522 | /* SvPVbyte does not work in perl-5.6.1, borrowed version for 5.7.3 */ 523 | # undef SvPVbyte 524 | # define SvPVbyte(sv, lp) \ 525 | ((SvFLAGS(sv) & (SVf_POK|SVf_UTF8)) == (SVf_POK) \ 526 | ? ((lp = SvCUR(sv)), SvPVX(sv)) : my_sv_2pvbyte(aTHX_ sv, &lp)) 527 | static char * 528 | my_sv_2pvbyte(pTHX_ register SV *sv, STRLEN *lp) 529 | { 530 | sv_utf8_downgrade(sv,0); 531 | return SvPV(sv,*lp); 532 | } 533 | # endif 534 | #else 535 | # define SvPVbyte SvPV 536 | #endif 537 | 538 | #endif /* _P_P_PORTABILITY_H_ */ 539 | 540 | /* End of File ppport.h */ 541 | -------------------------------------------------------------------------------- /mmap_cache.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * AUTHOR 4 | * 5 | * Rob Mueller 6 | * 7 | * COPYRIGHT AND LICENSE 8 | * 9 | * Copyright (C) 2003 by FastMail IP Partners 10 | * 11 | * This library is free software; you can redistribute it and/or modify 12 | * it under the same terms as Perl itself. 13 | * 14 | */ 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include "mmap_cache.h" 23 | #include "mmap_cache_internals.h" 24 | 25 | /* Global time_override */ 26 | MU32 time_override = 0; 27 | 28 | void mmc_set_time_override(MU32 set_time) { 29 | time_override = set_time; 30 | } 31 | 32 | /* Default values for a new cache */ 33 | char * def_share_file = "/tmp/sharefile"; 34 | MU32 def_init_file = 0; 35 | MU32 def_test_file = 0; 36 | MU32 def_expire_time = 0; 37 | MU32 def_c_num_pages = 89; 38 | MU32 def_c_page_size = 65536; 39 | MU32 def_start_slots = 89; 40 | 41 | /* 42 | * mmap_cache * mmc_new() 43 | * 44 | * Create a new cache object filled with default values. Values may 45 | * be changed and once ready, you should call mmc_init() to actually 46 | * open the cache file and mmap it. 47 | * 48 | */ 49 | mmap_cache * mmc_new() { 50 | mmap_cache * cache = (mmap_cache *)calloc(1, sizeof(mmap_cache)); 51 | 52 | cache->p_cur = NOPAGE; 53 | 54 | cache->c_num_pages = def_c_num_pages; 55 | cache->c_page_size = def_c_page_size; 56 | 57 | cache->start_slots = def_start_slots; 58 | cache->expire_time = def_expire_time; 59 | 60 | cache->share_file = _mmc_get_def_share_filename(cache); 61 | cache->permissions = 0640; 62 | cache->init_file = def_init_file; 63 | cache->test_file = def_test_file; 64 | 65 | return cache; 66 | } 67 | 68 | int mmc_set_param(mmap_cache * cache, char * param, char * val) { 69 | if (!strcmp(param, "init_file")) { 70 | cache->init_file = atoi(val); 71 | } else if (!strcmp(param, "test_file")) { 72 | cache->test_file = atoi(val); 73 | } else if (!strcmp(param, "page_size")) { 74 | cache->c_page_size = atoi(val); 75 | } else if (!strcmp(param, "num_pages")) { 76 | cache->c_num_pages = atoi(val); 77 | } else if (!strcmp(param, "expire_time")) { 78 | cache->expire_time = atoi(val); 79 | } else if (!strcmp(param, "share_file")) { 80 | cache->share_file = val; 81 | } else if (!strcmp(param, "permissions")) { 82 | cache->permissions = atoi(val); 83 | } else if (!strcmp(param, "start_slots")) { 84 | cache->start_slots = atoi(val); 85 | } else if (!strcmp(param, "catch_deadlocks")) { 86 | cache->catch_deadlocks = atoi(val); 87 | } else if (!strcmp(param, "enable_stats")) { 88 | cache->enable_stats = atoi(val); 89 | } else { 90 | return _mmc_set_error(cache, 0, "Bad set_param parameter: %s", param); 91 | } 92 | 93 | return 0; 94 | } 95 | 96 | int mmc_get_param(mmap_cache * cache, char * param) { 97 | if (!strcmp(param, "page_size")) { 98 | return (int)cache->c_page_size; 99 | } else if (!strcmp(param, "num_pages")) { 100 | return (int)cache->c_num_pages; 101 | } else if (!strcmp(param, "expire_time")) { 102 | return (int)cache->expire_time; 103 | } else { 104 | return _mmc_set_error(cache, 0, "Bad set_param parameter: %s", param); 105 | } 106 | } 107 | 108 | /* 109 | * int mmc_init(mmap_cache * cache) 110 | * 111 | * Initialise the cache object, opening the share file and mmap'ing any 112 | * memory. 113 | * 114 | */ 115 | int mmc_init(mmap_cache * cache) { 116 | int i, do_init = cache->init_file; 117 | MU32 c_num_pages, c_page_size; 118 | MU64 c_size; 119 | 120 | /* Need a share file */ 121 | if (!cache->share_file) { 122 | return _mmc_set_error(cache, 0, "No share file specified"); 123 | } 124 | 125 | /* Basic cache params */ 126 | c_num_pages = cache->c_num_pages; 127 | ASSERT(c_num_pages >= 1 && c_num_pages <= 1000); 128 | 129 | c_page_size = cache->c_page_size; 130 | ASSERT(c_page_size >= 1024 && c_page_size <= 1024*1024*1024); 131 | 132 | ASSERT(cache->start_slots >= 10 && cache->start_slots <= 500); 133 | 134 | cache->c_size = c_size = (MU64)c_num_pages * c_page_size; 135 | 136 | if ( mmc_open_cache_file(cache, &do_init) == -1) return -1; 137 | 138 | /* Map file into memory */ 139 | if ( mmc_map_memory(cache) == -1) return -1; 140 | 141 | /* Initialise pages if new file */ 142 | if (do_init) { 143 | for (i = 0; i < cache->c_num_pages; i++) { 144 | MU64 p_offset = (MU64)i * cache->c_page_size; 145 | mmc_lock_page(cache, p_offset); 146 | _mmc_init_page(cache, i); 147 | mmc_unlock_page(cache, p_offset); 148 | } 149 | 150 | /* Unmap and re-map to stop gtop telling us our memory usage is up */ 151 | if ( mmc_unmap_memory(cache) == -1) return -1; 152 | if ( mmc_map_memory(cache) == -1) return -1; 153 | } 154 | 155 | /* Test pages in file if asked */ 156 | if (cache->test_file) { 157 | for (i = 0; i < cache->c_num_pages; i++) { 158 | int bad_page = 0; 159 | MU64 p_offset = (MU64)i * cache->c_page_size; 160 | 161 | /* Need to lock page, which tests header structure */ 162 | if (mmc_lock(cache, i)) { 163 | /* If that failed, assume bad header, so manually lock */ 164 | mmc_lock_page(cache, p_offset); 165 | bad_page = 1; 166 | 167 | /* If lock succeeded, test page structure */ 168 | } else { 169 | if (!_mmc_test_page(cache)) 170 | bad_page = 1; 171 | } 172 | 173 | /* A bad page, initialise it */ 174 | if (bad_page) { 175 | _mmc_init_page(cache, i); 176 | /* Rerun test on this page, potential infinite 177 | loop if init_page is broken, but then things 178 | are really broken anyway */ 179 | i--; 180 | } 181 | 182 | mmc_unlock_page(cache, p_offset); 183 | cache->p_cur = NOPAGE; 184 | } 185 | } 186 | 187 | return 0; 188 | } 189 | 190 | /* 191 | * int mmc_close(mmap_cache * cache) 192 | * 193 | * Close the given cache, unmmap'ing any memory and closing file 194 | * descriptors. 195 | * 196 | */ 197 | int mmc_close(mmap_cache *cache) { 198 | int res; 199 | 200 | /* Shouldn't call if not init'ed */ 201 | ASSERT(cache->fh); 202 | ASSERT(cache->mm_var); 203 | 204 | /* Shouldn't call if page still locked */ 205 | ASSERT(cache->p_cur == NOPAGE); 206 | 207 | /* Unlock any locked page */ 208 | if (cache->p_cur != NOPAGE) { 209 | mmc_unlock(cache); 210 | } 211 | 212 | /* Close file */ 213 | if (cache->fh) { 214 | mmc_close_fh(cache); 215 | } 216 | 217 | /* Unmap memory */ 218 | if (cache->mm_var) { 219 | res = mmc_unmap_memory(cache); 220 | if (res == -1) { 221 | return _mmc_set_error(cache, errno, "Mmap of shared file %s failed", cache->share_file); 222 | } 223 | } 224 | 225 | free(cache); 226 | 227 | return 0; 228 | } 229 | 230 | char * mmc_error(mmap_cache * cache) { 231 | if (cache->last_error) 232 | return cache->last_error; 233 | return "Unknown error"; 234 | } 235 | 236 | /* 237 | * mmc_lock( 238 | * cache_mmap * cache, MU32 p_cur 239 | * ) 240 | * 241 | * Lock the given page number using fcntl locking. Setup 242 | * cache->p_* fields with correct values for the given page 243 | * 244 | */ 245 | int mmc_lock(mmap_cache * cache, MU32 p_cur) { 246 | MU64 p_offset; 247 | void * p_ptr; 248 | int res = 0; 249 | 250 | /* Argument sanity check */ 251 | if (p_cur == NOPAGE || p_cur > cache->c_num_pages) 252 | return _mmc_set_error(cache, 0, "page %u is NOPAGE or larger than number of pages", p_cur); 253 | 254 | /* Check not already locked */ 255 | if (cache->p_cur != NOPAGE) 256 | return _mmc_set_error(cache, 0, "page %u is already locked, can't lock multiple pages", cache->p_cur); 257 | 258 | /* Setup page details */ 259 | p_offset = (MU64)p_cur * cache->c_page_size; 260 | p_ptr = PTR_ADD(cache->mm_var, p_offset); 261 | 262 | res = mmc_lock_page(cache, p_offset); 263 | if (res) return res; 264 | 265 | if (!(P_Magic(p_ptr) == 0x92f7e3b1)) { 266 | mmc_unlock_page(cache, p_offset); 267 | return _mmc_set_error(cache, 0, "magic page start marker not found. p_cur is %u, offset is %llu", p_cur, p_offset); 268 | } 269 | 270 | /* Copy to cache structure */ 271 | cache->p_num_slots = P_NumSlots(p_ptr); 272 | cache->p_free_slots = P_FreeSlots(p_ptr); 273 | cache->p_old_slots = P_OldSlots(p_ptr); 274 | cache->p_free_data = P_FreeData(p_ptr); 275 | cache->p_free_bytes = P_FreeBytes(p_ptr); 276 | cache->p_n_reads = P_NReads(p_ptr); 277 | cache->p_n_read_hits = P_NReadHits(p_ptr); 278 | 279 | /* Reality check */ 280 | if (cache->p_num_slots < 89 || cache->p_num_slots > cache->c_page_size) 281 | res = _mmc_set_error(cache, 0, "cache num_slots mistmatch"); 282 | else if (cache->p_free_slots < 0 || cache->p_free_slots > cache->p_num_slots) 283 | res = _mmc_set_error(cache, 0, "cache free slots mustmatch"); 284 | else if (cache->p_old_slots > cache->p_free_slots) 285 | res = _mmc_set_error(cache, 0, "cache old slots mistmatch"); 286 | else if (cache->p_free_data + cache->p_free_bytes != cache->c_page_size) 287 | res = _mmc_set_error(cache, 0, "cache free data mistmatch"); 288 | if (res) { 289 | mmc_unlock_page(cache, p_offset); 290 | return res; 291 | } 292 | 293 | /* Check page header */ 294 | ASSERT(P_Magic(p_ptr) == 0x92f7e3b1); 295 | ASSERT(P_NumSlots(p_ptr) >= 89 && P_NumSlots(p_ptr) < cache->c_page_size); 296 | ASSERT(P_FreeSlots(p_ptr) >= 0 && P_FreeSlots(p_ptr) <= P_NumSlots(p_ptr)); 297 | ASSERT(P_OldSlots(p_ptr) <= P_FreeSlots(p_ptr)); 298 | ASSERT(P_FreeData(p_ptr) + P_FreeBytes(p_ptr) == cache->c_page_size); 299 | 300 | /* Setup page pointers */ 301 | cache->p_cur = p_cur; 302 | cache->p_offset = p_offset; 303 | cache->p_base = p_ptr; 304 | cache->p_base_slots = PTR_ADD(p_ptr, P_HEADERSIZE); 305 | 306 | ASSERT(_mmc_test_page(cache)); 307 | 308 | return 0; 309 | } 310 | 311 | /* 312 | * mmc_unlock( 313 | * cache_mmap * cache 314 | * ) 315 | * 316 | * Unlock any currently locked page 317 | * 318 | */ 319 | int mmc_unlock(mmap_cache * cache) { 320 | 321 | ASSERT(cache->p_cur != NOPAGE); 322 | 323 | /* If changed, save page header changes back */ 324 | if (cache->p_changed) { 325 | void * p_ptr = cache->p_base; 326 | 327 | /* Save any changed information back to page */ 328 | P_NumSlots(p_ptr) = cache->p_num_slots; 329 | P_FreeSlots(p_ptr) = cache->p_free_slots; 330 | P_OldSlots(p_ptr) = cache->p_old_slots; 331 | P_FreeData(p_ptr) = cache->p_free_data; 332 | P_FreeBytes(p_ptr) = cache->p_free_bytes; 333 | P_NReads(p_ptr) = cache->p_n_reads; 334 | P_NReadHits(p_ptr) = cache->p_n_read_hits; 335 | 336 | cache->p_changed = 0; 337 | } 338 | 339 | /* Test before unlocking */ 340 | ASSERT(_mmc_test_page(cache)); 341 | 342 | mmc_unlock_page(cache, cache->p_offset); 343 | 344 | cache->p_cur = NOPAGE; 345 | 346 | return 0; 347 | } 348 | 349 | /* 350 | * mmc_is_locked( 351 | * cache_mmap * cache 352 | * ) 353 | * 354 | * Return true if page is locked 355 | * 356 | */ 357 | int mmc_is_locked(mmap_cache * cache) { 358 | 359 | return cache->p_cur != NOPAGE ? 1 : 0; 360 | } 361 | 362 | /* 363 | * int mmc_hash( 364 | * cache_mmap * cache, 365 | * void *key_ptr, int key_len, 366 | * MU32 *hash_page, MU32 *hash_slot 367 | * ) 368 | * 369 | * Hashes the given key, and returns hash value, hash page and hash 370 | * slot part 371 | * 372 | */ 373 | int mmc_hash( 374 | mmap_cache *cache, 375 | void *key_ptr, int key_len, 376 | MU32 *hash_page, MU32 *hash_slot 377 | ) { 378 | MU32 h = 0x92f7e3b1; 379 | unsigned char * uc_key_ptr = (unsigned char *)key_ptr; 380 | unsigned char * uc_key_ptr_end = uc_key_ptr + key_len; 381 | 382 | while (uc_key_ptr != uc_key_ptr_end) { 383 | h = (h << 4) + (h >> 28) + *uc_key_ptr++; 384 | } 385 | 386 | *hash_page = h % cache->c_num_pages; 387 | *hash_slot = h / cache->c_num_pages; 388 | 389 | return 0; 390 | } 391 | 392 | /* 393 | * int mmc_read( 394 | * cache_mmap * cache, MU32 hash_slot, 395 | * void *key_ptr, int key_len, 396 | * void **val_ptr, int *val_len, 397 | * MU32 *expire_on, MU32 *flags 398 | * ) 399 | * 400 | * Read key from current page 401 | * 402 | */ 403 | int mmc_read( 404 | mmap_cache *cache, MU32 hash_slot, 405 | void *key_ptr, int key_len, 406 | void **val_ptr, int *val_len, 407 | MU32 *expire_on_p, MU32 *flags_p 408 | ) { 409 | MU32 * slot_ptr; 410 | 411 | /* Increase read count for page */ 412 | if (cache->enable_stats) { 413 | cache->p_changed = 1; 414 | cache->p_n_reads++; 415 | } 416 | 417 | /* Search slots for key */ 418 | slot_ptr = _mmc_find_slot(cache, hash_slot, key_ptr, key_len, 0); 419 | 420 | /* Did we find a value? */ 421 | if (!slot_ptr || *slot_ptr == 0) { 422 | 423 | /* Return -1 if not */ 424 | return -1; 425 | 426 | /* We found it! Check some other things... */ 427 | } else { 428 | 429 | MU32 * base_det = S_Ptr(cache->p_base, *slot_ptr); 430 | MU32 now = time_override ? time_override : (MU32)time(0); 431 | 432 | MU32 expire_on = S_ExpireOn(base_det); 433 | 434 | /* Sanity check hash matches */ 435 | ASSERT(S_SlotHash(base_det) == hash_slot); 436 | 437 | /* Value expired? */ 438 | if (expire_on && now >= expire_on) { 439 | 440 | /* Return not found, but leave slot. Might need writeback */ 441 | 442 | return -1; 443 | } 444 | 445 | /* Update hit time */ 446 | S_LastAccess(base_det) = now; 447 | 448 | /* Copy values to pointers */ 449 | *flags_p = S_Flags(base_det); 450 | *expire_on_p = expire_on; 451 | *val_len = S_ValLen(base_det); 452 | *val_ptr = S_ValPtr(base_det); 453 | 454 | /* Increase read hit count */ 455 | if (cache->enable_stats) { 456 | cache->p_changed = 1; 457 | cache->p_n_read_hits++; 458 | } 459 | 460 | return 0; 461 | } 462 | } 463 | 464 | /* 465 | * int mmc_write( 466 | * cache_mmap * cache, MU32 hash_slot, 467 | * void *key_ptr, int key_len, 468 | * void *val_ptr, int val_len, 469 | * MU32 expire_on, MU32 flags 470 | * ) 471 | * 472 | * Write key to current page 473 | * 474 | */ 475 | int mmc_write( 476 | mmap_cache *cache, MU32 hash_slot, 477 | void *key_ptr, int key_len, 478 | void *val_ptr, int val_len, 479 | MU32 expire_on, MU32 flags 480 | ) { 481 | int did_store = 0; 482 | MU32 kvlen = KV_SlotLen(key_len, val_len); 483 | 484 | /* Search for slot with given key */ 485 | MU32 * slot_ptr = _mmc_find_slot(cache, hash_slot, key_ptr, key_len, 1); 486 | 487 | /* If all slots full, definitely can't store */ 488 | if (!slot_ptr) 489 | return 0; 490 | 491 | ROUNDLEN(kvlen); 492 | 493 | ASSERT(cache->p_cur != NOPAGE); 494 | 495 | /* If found, delete slot for new value */ 496 | if (*slot_ptr > 1) { 497 | _mmc_delete_slot(cache, slot_ptr); 498 | ASSERT(*slot_ptr == 1); 499 | } 500 | 501 | ASSERT(*slot_ptr <= 1); 502 | 503 | /* If there's space, store the key/value in the data section */ 504 | if (cache->p_free_bytes >= kvlen) { 505 | MU32 * base_det = PTR_ADD(cache->p_base, cache->p_free_data); 506 | MU32 now = time_override ? time_override : (MU32)time(0); 507 | 508 | /* Calculate expiry time */ 509 | if (expire_on == (MU32)-1) 510 | expire_on = cache->expire_time ? now + cache->expire_time : 0; 511 | 512 | /* Store info into slot */ 513 | S_LastAccess(base_det) = now; 514 | S_ExpireOn(base_det) = expire_on; 515 | S_SlotHash(base_det) = hash_slot; 516 | S_Flags(base_det) = flags; 517 | S_KeyLen(base_det) = (MU32)key_len; 518 | S_ValLen(base_det) = (MU32)val_len; 519 | 520 | /* Copy key/value to data section */ 521 | memcpy(S_KeyPtr(base_det), key_ptr, key_len); 522 | memcpy(S_ValPtr(base_det), val_ptr, val_len); 523 | 524 | /* Update used slots/free data info */ 525 | cache->p_free_slots--; 526 | if (*slot_ptr == 1) { cache->p_old_slots--; } 527 | 528 | /* Save new data offset */ 529 | *slot_ptr = cache->p_free_data; 530 | 531 | /* Update free space */ 532 | cache->p_free_bytes -= kvlen; 533 | cache->p_free_data += kvlen; 534 | 535 | /* Ensure changes are saved back */ 536 | cache->p_changed = 1; 537 | 538 | did_store = 1; 539 | } 540 | 541 | return did_store; 542 | } 543 | 544 | /* 545 | * int mmc_delete( 546 | * cache_mmap * cache, MU32 hash_slot, 547 | * void *key_ptr, int key_len 548 | * ) 549 | * 550 | * Delete key from current page 551 | * 552 | */ 553 | int mmc_delete( 554 | mmap_cache *cache, MU32 hash_slot, 555 | void *key_ptr, int key_len, 556 | MU32 * flags 557 | ) { 558 | /* Search slots for key */ 559 | MU32 * slot_ptr = _mmc_find_slot(cache, hash_slot, key_ptr, key_len, 2); 560 | 561 | /* Did we find a value? */ 562 | if (!slot_ptr || *slot_ptr == 0) { 563 | 564 | /* Return 0 if not deleted */ 565 | return 0; 566 | 567 | /* We found it, delete it */ 568 | } else { 569 | 570 | /* Store flags in output pointer */ 571 | MU32 * base_det = S_Ptr(cache->p_base, *slot_ptr); 572 | *flags = S_Flags(base_det); 573 | 574 | _mmc_delete_slot(cache, slot_ptr); 575 | return 1; 576 | } 577 | 578 | } 579 | 580 | int last_access_cmp(const void * a, const void * b) { 581 | MU32 av = S_LastAccess(*(MU32 **)a); 582 | MU32 bv = S_LastAccess(*(MU32 **)b); 583 | if (av < bv) return -1; 584 | if (av > bv) return 1; 585 | return 0; 586 | } 587 | 588 | /* 589 | * int mmc_calc_expunge( 590 | * cache_mmap * cache, int mode, int len, MU32 * new_num_slots, MU32 *** to_expunge 591 | * ) 592 | * 593 | * Calculate entries to expunge from current page. 594 | * 595 | * If len >= 0 596 | * If space available for len bytes & >30% slots free, nothing is expunged 597 | * If len < 0 or not above 598 | * If mode == 0, only expired items are expunged 599 | * If mode == 1, all entries are expunged 600 | * If mode == 2, entries are expunged till 40% free space is created 601 | * 602 | * If expunged is non-null pointer, result is filled with 603 | * a list of slots to expunge 604 | * 605 | * Return value is number of items to expunge 606 | * 607 | */ 608 | int mmc_calc_expunge( 609 | mmap_cache * cache, 610 | int mode, int len, 611 | MU32 * new_num_slots, MU32 *** to_expunge 612 | ) { 613 | double slots_pct; 614 | 615 | ASSERT(cache->p_cur != NOPAGE); 616 | 617 | /* If len >= 0, and space available for len bytes, nothing is expunged */ 618 | if (len >= 0) { 619 | /* Length of key/value data when stored */ 620 | MU32 kvlen = KV_SlotLen(len, 0); 621 | ROUNDLEN(kvlen); 622 | 623 | slots_pct = (double)(cache->p_free_slots - cache->p_old_slots) / cache->p_num_slots; 624 | 625 | /* Nothing to do if hash table more than 30% free slots and enough free space */ 626 | if (slots_pct > 0.3 && cache->p_free_bytes >= kvlen) 627 | return 0; 628 | } 629 | 630 | { 631 | MU32 num_slots = cache->p_num_slots; 632 | 633 | MU32 used_slots = num_slots - cache->p_free_slots; 634 | MU32 * slot_ptr = cache->p_base_slots; 635 | MU32 * slot_end = slot_ptr + num_slots; 636 | 637 | /* Store pointers to used slots */ 638 | MU32 ** copy_base_det = (MU32 **)calloc(used_slots, sizeof(MU32 *)); 639 | MU32 ** copy_base_det_end = copy_base_det + used_slots; 640 | MU32 ** copy_base_det_out = copy_base_det; 641 | MU32 ** copy_base_det_in = copy_base_det + used_slots; 642 | 643 | MU32 page_data_size = cache->c_page_size - num_slots * 4 - P_HEADERSIZE; 644 | MU32 in_slots, data_thresh, used_data = 0; 645 | MU32 now = time_override ? time_override : (MU32)time(0); 646 | 647 | /* Loop for each existing slot, and store in a list */ 648 | for (; slot_ptr != slot_end; slot_ptr++) { 649 | MU32 data_offset = *slot_ptr; 650 | MU32 * base_det = S_Ptr(cache->p_base, data_offset); 651 | MU32 expire_on, kvlen; 652 | 653 | /* Ignore if if free slot */ 654 | if (data_offset <= 1) { 655 | continue; 656 | } 657 | 658 | /* Definitely out if mode == 1 which means expunge all */ 659 | if (mode == 1) { 660 | *copy_base_det_out++ = base_det; 661 | continue; 662 | } 663 | 664 | /* Definitely out if expired, and not dirty */ 665 | expire_on = S_ExpireOn(base_det); 666 | if (expire_on && now >= expire_on) { 667 | *copy_base_det_out++ = base_det; 668 | continue; 669 | } 670 | 671 | /* Track used space */ 672 | kvlen = S_SlotLen(base_det); 673 | ROUNDLEN(kvlen); 674 | ASSERT(kvlen <= page_data_size); 675 | used_data += kvlen; 676 | ASSERT(used_data <= page_data_size); 677 | 678 | /* Potentially in */ 679 | *--copy_base_det_in = base_det; 680 | } 681 | 682 | /* Check that definitely in and out slots add up to used slots */ 683 | ASSERT(copy_base_det_in == copy_base_det_out); 684 | ASSERT(mode != 1 || copy_base_det_out == copy_base_det_end); 685 | 686 | /* Increase slot count if free count is low and there's space to increase */ 687 | slots_pct = (double)(copy_base_det_end - copy_base_det_out) / num_slots; 688 | if (slots_pct > 0.3 && (page_data_size - used_data > (num_slots + 1) * 4 || mode == 2)) { 689 | num_slots = (num_slots * 2) + 1; 690 | } 691 | page_data_size = cache->c_page_size - num_slots * 4 - P_HEADERSIZE; 692 | 693 | /* If mode == 0 or 1, we've just worked out ones to keep and 694 | * which to dispose of, so return results */ 695 | if (mode == 0 || mode == 1) { 696 | *to_expunge = copy_base_det; 697 | *new_num_slots = num_slots; 698 | return (copy_base_det_out - copy_base_det); 699 | } 700 | 701 | /* mode == 2, sort by last access, and remove till enough free space */ 702 | 703 | /* Sort those potentially in by last access */ 704 | in_slots = copy_base_det_end - copy_base_det_in; 705 | qsort((void *)copy_base_det_in, in_slots, sizeof(MU32 *), &last_access_cmp); 706 | 707 | /* Throw out old slots till we have 40% free data space */ 708 | data_thresh = (MU32)(0.6 * page_data_size); 709 | 710 | while (copy_base_det_in != copy_base_det_end && used_data >= data_thresh) { 711 | MU32 * slot_ptr = *copy_base_det_in; 712 | MU32 kvlen = S_SlotLen(slot_ptr); 713 | ROUNDLEN(kvlen); 714 | ASSERT(kvlen <= page_data_size); 715 | used_data -= kvlen; 716 | 717 | ASSERT(used_data >= 0); 718 | 719 | copy_base_det_out = ++copy_base_det_in; 720 | } 721 | ASSERT(used_data < page_data_size); 722 | 723 | *to_expunge = copy_base_det; 724 | *new_num_slots = num_slots; 725 | return (copy_base_det_out - copy_base_det); 726 | } 727 | } 728 | 729 | /* 730 | * int mmc_do_expunge( 731 | * cache_mmap * cache, int num_expunge, MU32 new_num_slots, MU32 ** to_expunge 732 | * ) 733 | * 734 | * Expunge given entries from current page. 735 | * 736 | */ 737 | int mmc_do_expunge( 738 | mmap_cache * cache, 739 | int num_expunge, MU32 new_num_slots, MU32 ** to_expunge 740 | ) { 741 | MU32 * base_slots = cache->p_base_slots; 742 | 743 | MU32 ** to_keep = to_expunge + num_expunge; 744 | MU32 ** to_keep_end = to_expunge + (cache->p_num_slots - cache->p_free_slots); 745 | MU32 new_used_slots = (to_keep_end - to_keep); 746 | 747 | /* Build new slots data and KV data */ 748 | MU32 slot_data_size = new_num_slots * 4; 749 | MU32 * new_slot_data = (MU32 *)calloc(1, slot_data_size); 750 | 751 | MU32 page_data_size = cache->c_page_size - new_num_slots * 4 - P_HEADERSIZE; 752 | 753 | void * new_kv_data = calloc(1, page_data_size); 754 | MU32 new_offset = 0; 755 | 756 | /* Sanity check underlying fd is still the same file */ 757 | if (!mmc_check_fh(cache)) 758 | return 0; 759 | 760 | /* Start all new slots empty */ 761 | memset(new_slot_data, 0, slot_data_size); 762 | 763 | /* Copy entries to keep to new slot entires and data sections */ 764 | for (;to_keep < to_keep_end; to_keep++) { 765 | MU32 * old_base_det = *to_keep; 766 | MU32 * new_slot_ptr; 767 | MU32 kvlen; 768 | 769 | /* Hash key to find starting slot */ 770 | MU32 slot = S_SlotHash(old_base_det) % new_num_slots; 771 | 772 | #ifdef DEBUG 773 | /* Check hash actually matches stored value */ 774 | { 775 | MU32 hash_page_dummy, hash_slot; 776 | mmc_hash(cache, S_KeyPtr(old_base_det), S_KeyLen(old_base_det), &hash_page_dummy, &hash_slot); 777 | 778 | ASSERT(hash_slot == S_SlotHash(old_base_det)); 779 | } 780 | #endif 781 | 782 | /* Find free slot */ 783 | new_slot_ptr = new_slot_data + slot; 784 | while (*new_slot_ptr) { 785 | if (++slot >= new_num_slots) { slot = 0; } 786 | new_slot_ptr = new_slot_data + slot; 787 | } 788 | 789 | /* Copy slot and KV data */ 790 | kvlen = S_SlotLen(old_base_det); 791 | memcpy(PTR_ADD(new_kv_data, new_offset), old_base_det, kvlen); 792 | 793 | /* Store slot data and mark as used */ 794 | *new_slot_ptr = new_offset + new_num_slots * 4 + P_HEADERSIZE; 795 | 796 | ROUNDLEN(kvlen); 797 | new_offset += kvlen; 798 | } 799 | 800 | ASSERT(new_offset <= page_data_size); 801 | 802 | /* printf("page=%d\n", cache->p_cur); 803 | printf("old_slots=%d, new_slots=%d\n", old_num_slots, new_num_slots); 804 | printf("old_used_slots=%d, new_used_slots=%d\n", old_used_slots, new_used_slots);*/ 805 | 806 | /* Store back into mmap'ed file space */ 807 | memcpy(base_slots, new_slot_data, slot_data_size); 808 | memcpy(base_slots + new_num_slots, new_kv_data, new_offset); 809 | 810 | cache->p_num_slots = new_num_slots; 811 | cache->p_free_slots = new_num_slots - new_used_slots; 812 | cache->p_old_slots = 0; 813 | cache->p_free_data = new_offset + new_num_slots * 4 + P_HEADERSIZE; 814 | cache->p_free_bytes = page_data_size - new_offset; 815 | 816 | /* Make sure changes are saved back to mmap'ed file */ 817 | cache->p_changed = 1; 818 | 819 | /* Free allocated memory */ 820 | free(new_kv_data); 821 | free(new_slot_data); 822 | free(to_expunge); 823 | 824 | ASSERT(_mmc_test_page(cache)); 825 | 826 | return 1; 827 | } 828 | 829 | /* 830 | * void mmc_get_page_details(mmap_cache * cache, MU32 * n_reads, MU32 * n_read_hits) 831 | * 832 | * Return details about the current locked page. Currently just 833 | * number of reads and number of reads that hit 834 | * 835 | */ 836 | void mmc_get_page_details(mmap_cache * cache, MU32 * n_reads, MU32 * n_read_hits) { 837 | *n_reads = cache->p_n_reads; 838 | *n_read_hits = cache->p_n_read_hits; 839 | return; 840 | } 841 | 842 | /* 843 | * void mmc_reset_page_details(mmap_cache * cache) 844 | * 845 | * Reset any page details (currently just read hits) 846 | * 847 | */ 848 | void mmc_reset_page_details(mmap_cache * cache) { 849 | cache->p_n_reads = 0; 850 | cache->p_n_read_hits = 0; 851 | cache->p_changed = 1; 852 | return; 853 | } 854 | 855 | /* 856 | * mmap_cache_it * mmc_iterate_new(mmap_cache * cache) 857 | * 858 | * Setup a new iterator to iterate over stored items 859 | * in the cache 860 | * 861 | */ 862 | mmap_cache_it * mmc_iterate_new(mmap_cache * cache) { 863 | mmap_cache_it * it = (mmap_cache_it *)calloc(1, sizeof(mmap_cache_it)); 864 | it->cache = cache; 865 | it->p_cur = NOPAGE; 866 | 867 | return it; 868 | } 869 | 870 | /* 871 | * MU32 * mmc_iterate_next(mmap_cache_it * it) 872 | * 873 | * Move iterator to next item in the cache and return 874 | * pointer to details (0 if there is no next). 875 | * 876 | * You can retrieve details with mmc_get_details(...) 877 | * 878 | */ 879 | MU32 * mmc_iterate_next(mmap_cache_it * it) { 880 | mmap_cache * cache = it->cache; 881 | MU32 * slot_ptr = it->slot_ptr; 882 | MU32 * base_det; 883 | MU32 expire_on; 884 | MU32 now = time_override ? time_override : (MU32)time(0); 885 | 886 | /* Go until we find a slot or exit */ 887 | while (1) { 888 | 889 | /* End of page ... */ 890 | if (slot_ptr == it->slot_ptr_end) { 891 | 892 | if (it->p_cur == NOPAGE) { 893 | it->p_cur = 0; 894 | 895 | /* Unlock current page if any */ 896 | } else { 897 | mmc_unlock(it->cache); 898 | 899 | /* Move to the next page, return 0 if no more pages */ 900 | if (++it->p_cur == cache->c_num_pages) { 901 | it->p_cur = NOPAGE; 902 | it->slot_ptr = 0; 903 | return 0; 904 | } 905 | } 906 | 907 | /* Lock the new page number */ 908 | mmc_lock(it->cache, it->p_cur); 909 | 910 | /* Setup new pointers */ 911 | slot_ptr = cache->p_base_slots; 912 | it->slot_ptr_end = slot_ptr + cache->p_num_slots; 913 | 914 | /* Check again */ 915 | continue; 916 | } 917 | 918 | /* Slot not used */ 919 | if (*slot_ptr <= 1) { 920 | slot_ptr++; 921 | continue; 922 | } 923 | 924 | /* Get pointer to details for this entry */ 925 | base_det = S_Ptr(cache->p_base, *slot_ptr); 926 | 927 | /* Slot expired */ 928 | expire_on = S_ExpireOn(base_det); 929 | if (expire_on && now >= expire_on) { 930 | slot_ptr++; 931 | continue; 932 | } 933 | 934 | break; 935 | } 936 | 937 | /* Move to the next slot for next iteration */ 938 | it->slot_ptr = ++slot_ptr; 939 | 940 | /* Return that we found the next item */ 941 | return base_det; 942 | } 943 | 944 | /* 945 | * void mmc_iterate_close(mmap_cache_it * it) 946 | * 947 | * Finish and dispose of iterator memory 948 | * 949 | */ 950 | void mmc_iterate_close(mmap_cache_it * it) { 951 | /* Unlock page if locked */ 952 | if (it->p_cur != NOPAGE) { 953 | mmc_unlock(it->cache); 954 | } 955 | 956 | /* Free memory */ 957 | free(it); 958 | } 959 | 960 | /* 961 | * void mmc_get_details( 962 | * mmap_cache * cache, 963 | * MU32 * base_det, 964 | * void ** key_ptr, int * key_len, 965 | * void ** val_ptr, int * val_len, 966 | * MU32 * last_access, MU32 * expire_on, MU32 * flags 967 | * ) 968 | * 969 | * Given a base_det pointer to entries details 970 | * (as returned by mmc_iterate_next(...) and 971 | * mmc_calc_expunge(...)) return details of that 972 | * entry in the cache 973 | * 974 | */ 975 | void mmc_get_details( 976 | mmap_cache * cache, 977 | MU32 * base_det, 978 | void ** key_ptr, int * key_len, 979 | void ** val_ptr, int * val_len, 980 | MU32 * last_access, MU32 * expire_on, MU32 * flags 981 | ) { 982 | cache = cache; 983 | 984 | *key_ptr = S_KeyPtr(base_det); 985 | *key_len = S_KeyLen(base_det); 986 | 987 | *val_ptr = S_ValPtr(base_det); 988 | *val_len = S_ValLen(base_det); 989 | 990 | *last_access = S_LastAccess(base_det); 991 | *expire_on = S_ExpireOn(base_det); 992 | *flags = S_Flags(base_det); 993 | } 994 | 995 | 996 | /* 997 | * _mmc_delete_slot( 998 | * mmap_cache * cache, MU32 * slot_ptr 999 | * ) 1000 | * 1001 | * Delete details from the given slot 1002 | * 1003 | */ 1004 | void _mmc_delete_slot( 1005 | mmap_cache * cache, MU32 * slot_ptr 1006 | ) { 1007 | ASSERT(*slot_ptr > 1); 1008 | ASSERT(cache->p_cur != NOPAGE); 1009 | 1010 | /* Set offset to 1 */ 1011 | *slot_ptr = 1; 1012 | 1013 | /* Increase slot free counters */ 1014 | cache->p_free_slots++; 1015 | cache->p_old_slots++; 1016 | 1017 | /* Ensure changes are saved back */ 1018 | cache->p_changed = 1; 1019 | } 1020 | 1021 | /* 1022 | * MU32 * _mmc_find_slot( 1023 | * mmap_cache * cache, MU32 hash_slot, 1024 | * void *key_ptr, int key_len, 1025 | * int mode 1026 | * ) 1027 | * 1028 | * Search current page for a particular 'key'. Use 'hash_slot' to 1029 | * calculate starting slot. Return pointer to slot. 1030 | * 1031 | */ 1032 | MU32 * _mmc_find_slot( 1033 | mmap_cache * cache, MU32 hash_slot, 1034 | void *key_ptr, int key_len, 1035 | int mode 1036 | ) { 1037 | MU32 slots_left, * slots_end; 1038 | /* Modulo hash_slot to find starting slot */ 1039 | MU32 * slot_ptr = cache->p_base_slots + (hash_slot % cache->p_num_slots); 1040 | MU32 * first_deleted = (MU32 *)0; 1041 | 1042 | /* Total slots and pointer to end of slot data to do wrapping */ 1043 | slots_left = cache->p_num_slots; 1044 | slots_end = cache->p_base_slots + slots_left; 1045 | 1046 | ASSERT(cache->p_cur != NOPAGE); 1047 | 1048 | /* Loop with integer probing till we find or don't */ 1049 | while (slots_left--) { 1050 | MU32 data_offset = *slot_ptr; 1051 | ASSERT(data_offset == 0 || data_offset == 1 || 1052 | ((data_offset >= P_HEADERSIZE + cache->p_num_slots*4) && 1053 | (data_offset < cache->c_page_size) && 1054 | ((data_offset & 3) == 0))); 1055 | 1056 | /* data_offset == 0 means empty slot, and no more beyond */ 1057 | /* data_offset == 1 means deleted slot, we can reuse if writing */ 1058 | if (data_offset == 0) { 1059 | /* Return pointer to last checked slot */ 1060 | break; 1061 | } 1062 | if (data_offset == 1 && mode == 1 && 0 == first_deleted) { 1063 | /* Save pointer to first usable slot; if we don't find the key later, 1064 | we'll fall back to returning this. 1065 | */ 1066 | first_deleted = slot_ptr; 1067 | } 1068 | /* deleted slot, keep looking */ 1069 | if (data_offset == 1) { 1070 | 1071 | } else { 1072 | /* Offset is from start of data area */ 1073 | MU32 * base_det = S_Ptr(cache->p_base, data_offset); 1074 | 1075 | /* Two longs are key len and data len */ 1076 | MU32 fkey_len = S_KeyLen(base_det); 1077 | 1078 | /* Key matches? */ 1079 | if (fkey_len == (MU32)key_len && !memcmp(key_ptr, S_KeyPtr(base_det), key_len)) { 1080 | 1081 | /* Yep, found it! */ 1082 | return slot_ptr; 1083 | } 1084 | } 1085 | 1086 | /* Linear probe and wrap at end of slot data... */ 1087 | if (++slot_ptr == slots_end) { slot_ptr = cache->p_base_slots; } 1088 | ASSERT(slot_ptr >= cache->p_base_slots && slot_ptr < slots_end); 1089 | } 1090 | /* No slot found */ 1091 | if (++slots_left == 0) slot_ptr = 0; 1092 | 1093 | if (1 == mode && 0 != first_deleted) 1094 | return first_deleted; 1095 | else 1096 | return slot_ptr; 1097 | } 1098 | 1099 | /* 1100 | * void _mmc_init_page(mmap_cache * cache, int page) 1101 | * 1102 | * Initialise the given page as empty. It's expected 1103 | * that you've already locked the page before doing 1104 | * this 1105 | * 1106 | */ 1107 | void _mmc_init_page(mmap_cache * cache, MU32 p_cur) { 1108 | /* Setup page details */ 1109 | MU64 p_offset = (MU64)p_cur * cache->c_page_size; 1110 | void * p_ptr = PTR_ADD(cache->mm_var, p_offset); 1111 | 1112 | /* Initialise to all 0's */ 1113 | memset(p_ptr, 0, cache->c_page_size); 1114 | 1115 | /* Setup header */ 1116 | P_Magic(p_ptr) = 0x92f7e3b1; 1117 | P_NumSlots(p_ptr) = cache->start_slots; 1118 | P_FreeSlots(p_ptr) = cache->start_slots; 1119 | P_OldSlots(p_ptr) = 0; 1120 | P_FreeData(p_ptr) = P_HEADERSIZE + cache->start_slots * 4; 1121 | P_FreeBytes(p_ptr) = cache->c_page_size - P_FreeData(p_ptr); 1122 | P_NReads(p_ptr) = 0; 1123 | P_NReadHits(p_ptr) = 0; 1124 | 1125 | } 1126 | 1127 | /* 1128 | * int _mmc_test_page(mmap_cache * cache) 1129 | * 1130 | * Test integrity of current page 1131 | * 1132 | */ 1133 | int _mmc_test_page(mmap_cache * cache) { 1134 | MU32 * slot_ptr = cache->p_base_slots; 1135 | MU32 count_free = 0, count_old = 0, max_data_offset = 0; 1136 | MU32 data_size = cache->c_page_size; 1137 | 1138 | ASSERT(cache->p_cur != NOPAGE); 1139 | if (cache->p_cur == NOPAGE) return 0; 1140 | 1141 | for (; slot_ptr < cache->p_base_slots + cache->p_num_slots; slot_ptr++) { 1142 | MU32 data_offset = *slot_ptr; 1143 | 1144 | ASSERT(data_offset == 0 || data_offset == 1 || 1145 | (data_offset >= P_HEADERSIZE + cache->p_num_slots * 4 && 1146 | data_offset < cache->c_page_size)); 1147 | if (!(data_offset == 0 || data_offset == 1 || 1148 | (data_offset >= P_HEADERSIZE + cache->p_num_slots * 4 && 1149 | data_offset < cache->c_page_size))) return 0; 1150 | 1151 | if (data_offset == 1) { 1152 | count_old++; 1153 | } 1154 | if (data_offset <= 1) { 1155 | count_free++; 1156 | continue; 1157 | } 1158 | 1159 | if (data_offset > 1) { 1160 | MU32 * base_det = S_Ptr(cache->p_base, data_offset); 1161 | 1162 | MU32 last_access = S_LastAccess(base_det); 1163 | MU32 expire_on = S_ExpireOn(base_det); 1164 | MU32 key_len = S_KeyLen(base_det); 1165 | MU32 val_len = S_ValLen(base_det); 1166 | MU32 kvlen = S_SlotLen(base_det); 1167 | ROUNDLEN(kvlen); 1168 | 1169 | ASSERT(last_access > 1000000000); 1170 | if (!(last_access > 1000000000)) return 0; 1171 | ASSERT(expire_on == 0 || (expire_on > 1000000000)); 1172 | if (!(expire_on == 0 || (expire_on > 1000000000))) return 0; 1173 | 1174 | ASSERT(key_len >= 0 && key_len < data_size); 1175 | if (!(key_len >= 0 && key_len < data_size)) return 0; 1176 | ASSERT(val_len >= 0 && val_len < data_size); 1177 | if (!(val_len >= 0 && val_len < data_size)) return 0; 1178 | ASSERT(kvlen >= 4*4 && kvlen < data_size); 1179 | if (!(kvlen >= 4*4 && kvlen < data_size)) return 0; 1180 | 1181 | /* Keep track of largest end of data position */ 1182 | if (data_offset + kvlen > max_data_offset) { 1183 | max_data_offset = data_offset + kvlen; 1184 | } 1185 | 1186 | /* Check if key lookup finds same thing */ 1187 | { 1188 | MU32 hash_page, hash_slot, * find_slot_ptr; 1189 | 1190 | /* Hash it */ 1191 | mmc_hash(cache, S_KeyPtr(base_det), (int)key_len, 1192 | &hash_page, &hash_slot); 1193 | ASSERT(hash_slot == S_SlotHash(base_det)); 1194 | if (!(hash_slot == S_SlotHash(base_det))) return 0; 1195 | 1196 | find_slot_ptr = _mmc_find_slot(cache, hash_slot, S_KeyPtr(base_det), key_len, 0); 1197 | 1198 | ASSERT(find_slot_ptr == slot_ptr); 1199 | if (!(find_slot_ptr == slot_ptr)) return 0; 1200 | } 1201 | 1202 | } 1203 | 1204 | } 1205 | 1206 | ASSERT(count_free == cache->p_free_slots); 1207 | if (!(count_free == cache->p_free_slots)) return 0; 1208 | ASSERT(count_old == cache->p_old_slots); 1209 | if (!(count_old == cache->p_old_slots)) return 0; 1210 | ASSERT(cache->p_free_data >= max_data_offset); 1211 | if (!(cache->p_free_data >= max_data_offset)) return 0; 1212 | 1213 | return 1; 1214 | } 1215 | 1216 | /* 1217 | * int _mmc_dump_page(mmap_cache * cache) 1218 | * 1219 | * Dump text version of current page to STDOUT 1220 | * 1221 | */ 1222 | int _mmc_dump_page(mmap_cache * cache) { 1223 | MU32 slot; 1224 | 1225 | ASSERT(cache->p_cur != NOPAGE); 1226 | 1227 | printf("PageNum: %d\n", cache->p_cur); 1228 | printf("\n"); 1229 | printf("PageSize: %d\n", cache->c_page_size); 1230 | printf("BasePage: %p\n", cache->p_base); 1231 | printf("BaseSlots: %p\n", cache->p_base_slots); 1232 | printf("\n"); 1233 | printf("NumSlots: %d\n", cache->p_num_slots); 1234 | printf("FreeSlots: %d\n", cache->p_free_slots); 1235 | printf("OldSlots: %d\n", cache->p_old_slots); 1236 | printf("FreeData: %d\n", cache->p_free_data); 1237 | printf("FreeBytes: %d\n", cache->p_free_bytes); 1238 | 1239 | for (slot = 0; slot < cache->p_num_slots; slot++) { 1240 | MU32 * slot_ptr = cache->p_base_slots + slot; 1241 | 1242 | printf("Slot: %d; OF=%d; ", slot, *slot_ptr); 1243 | 1244 | if (*slot_ptr > 1) { 1245 | MU32 * base_det = S_Ptr(cache->p_base, *slot_ptr); 1246 | MU32 key_len = S_KeyLen(base_det); 1247 | MU32 val_len = S_ValLen(base_det); 1248 | char key[256], val[256]; 1249 | 1250 | printf("LA=%d, ET=%d, HS=%d, FL=%d\n", 1251 | S_LastAccess(base_det), S_ExpireOn(base_det), 1252 | S_SlotHash(base_det), S_Flags(base_det)); 1253 | 1254 | /* Get data */ 1255 | memcpy(key, S_KeyPtr(base_det), key_len > 256 ? 256 : key_len); 1256 | key[key_len] = 0; 1257 | memcpy(val, S_ValPtr(base_det), val_len > 256 ? 256 : val_len); 1258 | val[val_len] = 0; 1259 | 1260 | printf(" K=%s, V=%s\n", key, val); 1261 | 1262 | } 1263 | 1264 | } 1265 | 1266 | return 0; 1267 | } 1268 | 1269 | 1270 | 1271 | 1272 | --------------------------------------------------------------------------------