├── .gitignore ├── LICENSE ├── Makefile ├── README ├── env.sh ├── env ├── BOOST_IOSTREAMS.env ├── BOOST_PROGRAM_OPTIONS.env ├── BOOST_PYTHON.env ├── BOOST_THREAD.env ├── BOOST_UNIT_TEST_FRAMEWORK.env ├── CC.env ├── CXX.env ├── DB_CXX.env ├── HYPERTABLE_HOME.env ├── JDK_DIR.env ├── PERL_INCLUDE.env ├── PERL_LDFLAGS.env ├── PYTHON_INCLUDE.env ├── SLICE_INCLUDE.env └── TCMALLOC.disabled ├── genmodules ├── magic.mk ├── notes ├── README ├── async_table.txt ├── client_mt.txt ├── distributed_table ├── error_handling.txt ├── extended_interface.txt ├── how_to_deploy.txt ├── kdi_perf.txt ├── kdi_transaction.txt ├── lock_server.txt ├── meta_config.txt ├── scan_resume.txt ├── server_arch.txt ├── server_impl.cc ├── server_reorg.txt ├── server_rpc.ice ├── table_to_tablet.txt └── tablet_migration.txt ├── scripts ├── kdi └── kdiWatchdog └── src ├── cc ├── ex │ ├── exception.cc │ ├── exception.h │ ├── module.mk │ └── trace_test.cc ├── flux │ ├── asyncsort.h │ ├── composite.h │ ├── cutoff.h │ ├── cutoff_unittest.cc │ ├── deserialize.h │ ├── filter.h │ ├── flush.h │ ├── heapsort.h │ ├── join.h │ ├── merge.h │ ├── merge_unittest.cc │ ├── modify.h │ ├── module.mk │ ├── observer.h │ ├── select.h │ ├── sequence.h │ ├── serialize.h │ ├── series.h │ ├── sort.h │ ├── stream.h │ ├── tee.h │ ├── threaded_reader.h │ ├── threaded_reader_unittest.cc │ ├── uniq.h │ └── uniqp.h ├── kdi │ ├── CellField.h │ ├── CellField_unittest.cc │ ├── LimitedScanner.h │ ├── RowInterval.cc │ ├── RowInterval.h │ ├── app │ │ ├── cell_test.cc │ │ ├── frag_test.cc │ │ ├── index_compression_test.cc │ │ ├── kdiErase_main.cc │ │ ├── kdiIntervals_main.cc │ │ ├── kdiLoad_main.cc │ │ ├── kdiSample_main.cc │ │ ├── kdiScan_main.cc │ │ ├── kdiTabletGc_main.cc │ │ ├── logscanner_test.cc │ │ ├── module.mk │ │ ├── performance_test.cc │ │ ├── random_load_test.cc │ │ ├── sample_table_test.cc │ │ ├── scan_test.cc │ │ └── scan_visitor.h │ ├── attic │ │ ├── async_table.cc │ │ ├── async_table.h │ │ ├── client │ │ │ ├── cell_packet.h │ │ │ ├── client_connection.cc │ │ │ ├── client_connection.h │ │ │ ├── client_scanner.h │ │ │ ├── client_table.h │ │ │ ├── connection_manager.cc │ │ │ ├── connection_manager.h │ │ │ ├── disabled_module.mk │ │ │ ├── error_buffer.h │ │ │ └── net_test.cc │ │ └── message │ │ │ ├── channel.cc │ │ │ ├── channel.h │ │ │ ├── client_test.cc │ │ │ ├── disabled_module.mk │ │ │ ├── error.h │ │ │ ├── relay.h │ │ │ └── server_test.cc │ ├── cell.h │ ├── cell_filter.cc │ ├── cell_filter.h │ ├── cell_filter_unittest.cc │ ├── cell_merge.h │ ├── cell_ostream.h │ ├── hash │ │ ├── HashedTable.cc │ │ ├── HashedTable.h │ │ └── module.mk │ ├── hyper │ │ ├── Client.cc │ │ ├── Client.h │ │ ├── Table.cc │ │ ├── Table.h │ │ └── module.mk │ ├── local │ │ ├── compactsim.py │ │ ├── disk_table.cc │ │ ├── disk_table.h │ │ ├── disk_table_unittest.cc │ │ ├── disk_table_v0.cc │ │ ├── disk_table_v0_unittest.cc │ │ ├── disk_table_v0_writer.cc │ │ ├── disk_table_writer.cc │ │ ├── disk_table_writer.h │ │ ├── index_cache.cc │ │ ├── index_cache.h │ │ ├── local_table.cc │ │ ├── local_table.h │ │ ├── local_table_unittest.cc │ │ ├── module.mk │ │ └── table_types.h │ ├── logged_memory_table.cc │ ├── logged_memory_table.h │ ├── logged_memory_table_unittest.cc │ ├── marshal │ │ ├── cell_block.h │ │ ├── cell_block_builder.h │ │ ├── cell_block_builder_unittest.cc │ │ └── module.mk │ ├── mem_test.cc │ ├── memory_table.cc │ ├── memory_table.h │ ├── memory_table_unittest.cc │ ├── meta │ │ ├── meta_cache.cc │ │ ├── meta_cache.h │ │ ├── meta_table.cc │ │ ├── meta_table.h │ │ ├── meta_table_unittest.cc │ │ ├── meta_util.cc │ │ ├── meta_util.h │ │ └── module.mk │ ├── module.mk │ ├── mux │ │ ├── MultiTable.cc │ │ ├── MultiTable.h │ │ └── module.mk │ ├── net │ │ ├── ScannerLocator.cc │ │ ├── ScannerLocator.h │ │ ├── StatReporter.ice │ │ ├── StatReporterI.cc │ │ ├── StatReporterI.h │ │ ├── TableLocator.cc │ │ ├── TableLocator.h │ │ ├── TableManager.ice │ │ ├── TableManagerI.cc │ │ ├── TableManagerI.h │ │ ├── kdiNetServer_main.cc │ │ ├── kdiNetStats_main.cc │ │ ├── module.mk │ │ ├── net_table.cc │ │ └── net_table.h │ ├── pred_parse_test.cc │ ├── row_buffer.h │ ├── row_stream.h │ ├── scan_predicate.cc │ ├── scan_predicate.h │ ├── scan_predicate_unittest.cc │ ├── sdstore_cell.cc │ ├── sdstore_cell.h │ ├── sdstore_dynamic_cell.cc │ ├── sdstore_dynamic_cell.h │ ├── sdstore_rowbuffer.cc │ ├── sdstore_rowbuffer.h │ ├── sdstore_rowstream.cc │ ├── sdstore_rowstream.h │ ├── strref.h │ ├── synchronized_table.cc │ ├── synchronized_table.h │ ├── table.cc │ ├── table.h │ ├── table_factory.cc │ ├── table_factory.h │ ├── table_unittest.h │ ├── table_writer.h │ ├── table_writer_unittest.cc │ ├── tablet │ │ ├── CachedFragmentLoader.cc │ │ ├── CachedFragmentLoader.h │ │ ├── CachedLogLoader.cc │ │ ├── CachedLogLoader.h │ │ ├── ConfigManager.h │ │ ├── DiskFragment.cc │ │ ├── DiskFragment.h │ │ ├── DiskFragmentLoader.cc │ │ ├── DiskFragmentLoader.h │ │ ├── DiskFragmentWriter.cc │ │ ├── DiskFragmentWriter.h │ │ ├── FileTracker.cc │ │ ├── FileTracker.h │ │ ├── FragDag.cc │ │ ├── FragDag.h │ │ ├── Fragment.cc │ │ ├── Fragment.h │ │ ├── FragmentLoader.h │ │ ├── FragmentWriter.h │ │ ├── LogEntry.h │ │ ├── LogFragment.cc │ │ ├── LogFragment.h │ │ ├── LogReader.cc │ │ ├── LogReader.h │ │ ├── LogWriter.cc │ │ ├── LogWriter.h │ │ ├── MetaConfigManager.cc │ │ ├── MetaConfigManager.h │ │ ├── Scanner.cc │ │ ├── Scanner.h │ │ ├── SharedCompactor.cc │ │ ├── SharedCompactor.h │ │ ├── SharedLogger.cc │ │ ├── SharedLogger.h │ │ ├── SuperScanner.cc │ │ ├── SuperScanner.h │ │ ├── SuperTablet.cc │ │ ├── SuperTablet.h │ │ ├── SwitchedFragmentLoader.cc │ │ ├── SwitchedFragmentLoader.h │ │ ├── Tablet.cc │ │ ├── Tablet.h │ │ ├── TabletConfig.h │ │ ├── TabletGc.cc │ │ ├── TabletGc.h │ │ ├── TabletName.cc │ │ ├── TabletName.h │ │ ├── WorkQueue.cc │ │ ├── WorkQueue.h │ │ ├── forward.h │ │ └── module.mk │ ├── thread_mem_test.cc │ ├── timestamp.h │ └── tmp │ │ ├── gentable_test.cc │ │ └── module.mk ├── minilzo │ ├── COPYING │ ├── README.LZO │ ├── lzoconf.h │ ├── lzodefs.h │ ├── minilzo.c │ ├── minilzo.h │ └── module.mk ├── oort │ ├── buildstream.h │ ├── checkstream.cc │ ├── exception.h │ ├── fileio.cc │ ├── fileio.h │ ├── headers.cc │ ├── headers.h │ ├── init.cc │ ├── init.h │ ├── module.mk │ ├── ordermgr.h │ ├── partmgr.h │ ├── perfstream.cc │ ├── perfstream.h │ ├── prim.h │ ├── record.cc │ ├── record.h │ ├── recordbuffer.cc │ ├── recordbuffer.h │ ├── recordbuilder.h │ ├── recordstream.cc │ ├── recordstream.h │ ├── serializer.cc │ ├── serializer.h │ ├── splitstream.cc │ ├── typeregistry.h │ └── uniqmgr.h ├── pykdi │ ├── conversion.h │ ├── module.cc │ ├── module.mk │ ├── pyscan.cc │ ├── pyscan.h │ ├── pytable.cc │ └── pytable.h ├── unittest │ ├── main.h │ └── predicates.h └── warp │ ├── EnvironmentVariable.cc │ ├── EnvironmentVariable.h │ ├── EnvironmentVariable_unittest.cc │ ├── PointerValueStore.h │ ├── Runnable.h │ ├── StatTracker.h │ ├── StringAlloc.h │ ├── StringTrie.cc │ ├── StringTrie.h │ ├── WorkerPool.cc │ ├── WorkerPool.h │ ├── adler.cc │ ├── adler.h │ ├── algorithm.h │ ├── algorithm_unittest.cc │ ├── apputil.cc │ ├── apputil.h │ ├── assocvec.h │ ├── assocvec_unittest.cc │ ├── atomic.h │ ├── atomic_unittest.cc │ ├── base64.cc │ ├── base64.h │ ├── base64_test.cc │ ├── base64_unittest.cc │ ├── bit_vector.cc │ ├── bit_vector.h │ ├── bit_vector_unittest.cc │ ├── bloom_filter.cc │ ├── bloom_filter.h │ ├── bloom_filter_unittest.cc │ ├── box2f.h │ ├── btree.h │ ├── btreebuilder.h │ ├── btreereader.h │ ├── buffer.h │ ├── bufferedfile.cc │ ├── bufferedfile.h │ ├── builder.h │ ├── builder_unittest.cc │ ├── call_or_die.cc │ ├── call_or_die.h │ ├── call_queue.cc │ ├── call_queue.h │ ├── charmap.cc │ ├── charmap.h │ ├── circular.h │ ├── circular_unittest.cc │ ├── config.cc │ ├── config.h │ ├── config_unittest.cc │ ├── dir.cc │ ├── dir.h │ ├── file.cc │ ├── file.h │ ├── file_cache.cc │ ├── file_cache.h │ ├── file_cache_unittest.cc │ ├── file_iterator.h │ ├── filestream.h │ ├── filestream_unittest.cc │ ├── framing.h │ ├── fs.cc │ ├── fs.h │ ├── fs_test.cc │ ├── fs_unittest.cc │ ├── fsutil.cc │ ├── fsutil.h │ ├── functional.h │ ├── functor.h │ ├── generator.h │ ├── gzip.cc │ ├── gzip.h │ ├── gzip_unittest.cc │ ├── hash.h │ ├── hash_test.cc │ ├── hashmap.h │ ├── hashtable.h │ ├── hashtable_unittest.cc │ ├── heap.h │ ├── heap_unittest.cc │ ├── hsieh_hash.cc │ ├── hsieh_hash.h │ ├── html_file_tokenizer.cc │ ├── html_file_tokenizer.h │ ├── html_tokenizer.cc │ ├── html_tokenizer.h │ ├── init.cc │ ├── init.h │ ├── interval.h │ ├── interval_test.cc │ ├── interval_unittest.cc │ ├── limited_inserter.h │ ├── limited_inserter_unittest.cc │ ├── line_iterator.h │ ├── line_iterator_unittest.cc │ ├── lockfile.cc │ ├── lockfile.h │ ├── log.cc │ ├── log.h │ ├── lru_cache.h │ ├── lru_cache_unittest.cc │ ├── mapped_region.cc │ ├── mapped_region.h │ ├── md5.cc │ ├── md5.h │ ├── md5_unittest.cc │ ├── memfile.cc │ ├── memfile.h │ ├── memfile_unittest.cc │ ├── memfs.cc │ ├── memfs.h │ ├── memfs_unittest.cc │ ├── mmapfile.cc │ ├── mmapfile.h │ ├── module.mk │ ├── multifile.cc │ ├── multifile.h │ ├── objectpool.h │ ├── objectpool_unittest.cc │ ├── offset.h │ ├── options.cc │ ├── options.h │ ├── options_unittest.cc │ ├── pack.h │ ├── parsing │ ├── module.mk │ ├── numeric.h │ ├── quoted_str.h │ ├── quoted_str_unittest.cc │ └── timestamp.h │ ├── pathmgr.cc │ ├── pathmgr.h │ ├── perffile.cc │ ├── perffile.h │ ├── perfitem.cc │ ├── perfitem.h │ ├── perflog.cc │ ├── perflog.h │ ├── perfmon.h │ ├── perfrusage.cc │ ├── perfvar.h │ ├── pid_file.cc │ ├── pid_file.h │ ├── pid_file_unittest.cc │ ├── plugin.h │ ├── plugin_unittest.cc │ ├── posixdir.cc │ ├── posixdir.h │ ├── posixfs.cc │ ├── posixfs.h │ ├── prime.cc │ ├── prime.h │ ├── prime_test.cc │ ├── rankcollector.h │ ├── rankcollector_test.cc │ ├── rankcollector_unittest.cc │ ├── repr.h │ ├── robots_txt.cc │ ├── robots_txt.h │ ├── robots_txt_unittest.cc │ ├── rollinghash.h │ ├── scorereader.h │ ├── shared.h │ ├── signal_blocker.h │ ├── singleton.h │ ├── stdfile.cc │ ├── stdfile.h │ ├── strhash.h │ ├── string_algorithm.h │ ├── string_data.h │ ├── string_interval.cc │ ├── string_interval.h │ ├── string_pool_builder.h │ ├── string_range.h │ ├── strutil.cc │ ├── strutil.h │ ├── strutil_unittest.cc │ ├── synchronized.h │ ├── synchronized_unittest.cc │ ├── syncqueue.h │ ├── syncqueue_unittest.cc │ ├── sysinfo.cc │ ├── sysinfo.h │ ├── timer.h │ ├── timestamp.cc │ ├── timestamp.h │ ├── timestamp_unittest.cc │ ├── tmpfile.cc │ ├── tmpfile.h │ ├── trie_test.cc │ ├── tuple.h │ ├── tuple_encode.h │ ├── tuple_encode_unittest.cc │ ├── uri.cc │ ├── uri.h │ ├── uri_unittest.cc │ ├── util.h │ ├── varsub.cc │ ├── varsub.h │ ├── varsub_unittest.cc │ ├── vec2f.h │ ├── vint.h │ ├── vintreader.cc │ ├── vintreader.h │ ├── vintwriter.cc │ ├── vintwriter.h │ ├── vstring.h │ ├── xml │ ├── module.mk │ ├── xml_escape.h │ ├── xml_parser.cc │ ├── xml_parser.h │ └── xml_parser_unittest.cc │ ├── zero_escape.cc │ ├── zero_escape.h │ └── zero_escape_unittest.cc └── python ├── alien ├── __init__.py ├── array.py ├── common.py ├── gzdata.py ├── kosmix │ ├── __init__.py │ ├── btree.py │ ├── kdi.py │ ├── record.py │ └── stringdata.py ├── octal.py ├── offset.py ├── prim.py ├── raw.py ├── tarheader.py ├── timestamp.py ├── tools.py ├── vint.py ├── vstruct.py └── vtype.py ├── aview.py ├── binbundle.py ├── kdi ├── __init__.py ├── createTable.py ├── makeMeta.py ├── shrinkMeta.py ├── splits.py └── updateMetaLocation.py └── util ├── __init__.py ├── bag.py ├── column.py ├── config.py ├── docstring.py ├── filemap.py ├── iter.py ├── metafile.py ├── parsetime.py ├── path.py ├── props.py ├── replace_headers.py ├── sample.py ├── shell.py ├── string_interpolate.py ├── term.py ├── uri.py ├── xmlfmt.py └── zero.py /.gitignore: -------------------------------------------------------------------------------- 1 | /bin 2 | /lib 3 | /build 4 | /src/cc/TAGS 5 | -------------------------------------------------------------------------------- /env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ROOT=$(dirname $(readlink -f ${BASH_SOURCE[0]})) 4 | 5 | prepend_path() 6 | { 7 | if ! eval test -z "\"\${$1##*:$2:*}\"" -o -z "\"\${$1%%*:$2}\"" -o -z "\"\${$1##$2:*}\"" -o -z "\"\${$1##$2}\"" ; then 8 | eval "$1=$2:\$$1" 9 | elif eval test -z "\"\$$1\"" ; then 10 | eval "$1=$2" 11 | fi 12 | } 13 | 14 | echo "Setting up environment from: $ROOT" 15 | prepend_path PATH "$ROOT/src/python" 16 | prepend_path PATH "$ROOT/bin" 17 | prepend_path LD_LIBRARY_PATH "$ROOT/lib" 18 | prepend_path PYTHONPATH "$ROOT/src/python" 19 | prepend_path PYTHONPATH "$ROOT/lib/python" 20 | 21 | export PATH LD_LIBRARY_PATH PYTHONPATH 22 | -------------------------------------------------------------------------------- /env/BOOST_IOSTREAMS.env: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | for name in boost_iostreams-mt boost_iostreams ; do 4 | if ld -l$name -o /dev/null 2>/dev/null >/dev/null ; then 5 | echo $name 6 | exit 0 7 | fi 8 | done 9 | 10 | exit 1 11 | -------------------------------------------------------------------------------- /env/BOOST_PROGRAM_OPTIONS.env: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | for name in boost_program_options-mt boost_program_options ; do 4 | if ld -l$name -o /dev/null 2>/dev/null >/dev/null ; then 5 | echo $name 6 | exit 0 7 | fi 8 | done 9 | 10 | exit 1 11 | -------------------------------------------------------------------------------- /env/BOOST_PYTHON.env: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | for name in boost_python-mt boost_python ; do 4 | for python in python2.5 python2.4; do 5 | if ld -l$name -l$python -o/dev/null >/dev/null 2>&1 ; then 6 | echo $name 7 | exit 0 8 | fi 9 | done 10 | done 11 | 12 | exit 1 13 | -------------------------------------------------------------------------------- /env/BOOST_THREAD.env: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Find Boost::Thread, whose .so was renamed between 1.33 to 1.34. 4 | # Wang Lam 08 Apr 2008 5 | # 6 | # 7 | 8 | for name in boost_thread-mt boost_thread ; do 9 | if ld -l$name -o /dev/null 2>/dev/null >/dev/null ; then 10 | echo $name 11 | exit 0 12 | fi 13 | done 14 | 15 | exit 1 16 | -------------------------------------------------------------------------------- /env/BOOST_UNIT_TEST_FRAMEWORK.env: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | for name in boost_unit_test_framework-mt boost_unit_test_framework ; do 4 | if ld -l$name -o /dev/null 2>/dev/null >/dev/null ; then 5 | echo $name 6 | exit 0 7 | fi 8 | done 9 | 10 | exit 1 11 | -------------------------------------------------------------------------------- /env/CC.env: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Prepend ccache and distcc to a command. 4 | # Wang Lam 07 Jun 2007 5 | # 6 | # 7 | 8 | BIN=gcc 9 | 10 | # Only search for distcc if the user has distcc hosts configured. 11 | if [ -n "$DISTCC_HOSTS" -o -r $DISTCC_DIR/hosts -o -r ~/.distcc/hosts ]; then 12 | DISTCC=`which distcc 2>/dev/null` 13 | else 14 | DISTCC= 15 | fi 16 | 17 | CCACHE=`which ccache 2>/dev/null` 18 | 19 | if [ "x$DISTCC" != "x" -a "x$CCACHE" != "x" ]; then 20 | echo CCACHE_PREFIX=$DISTCC $CCACHE $BIN 21 | elif [ "x$DISTCC" != "x" ]; then 22 | echo $DISTCC $BIN 23 | elif [ "x$CCACHE" != "x" ]; then 24 | echo $CCACHE $BIN 25 | else 26 | echo $BIN 27 | fi 28 | 29 | -------------------------------------------------------------------------------- /env/CXX.env: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Prepend ccache and distcc to a command. 4 | # Wang Lam 07 Jun 2007 5 | # 6 | # 7 | 8 | BIN=g++ 9 | 10 | # Only search for distcc if the user has distcc hosts configured. 11 | if [ -n "$DISTCC_HOSTS" -o -r $DISTCC_DIR/hosts -o -r ~/.distcc/hosts ]; then 12 | DISTCC=`which distcc 2>/dev/null` 13 | else 14 | DISTCC= 15 | fi 16 | 17 | CCACHE=`which ccache 2>/dev/null` 18 | 19 | if [ "x$DISTCC" != "x" -a "x$CCACHE" != "x" ]; then 20 | echo CCACHE_PREFIX=$DISTCC $CCACHE $BIN 21 | elif [ "x$DISTCC" != "x" ]; then 22 | echo $DISTCC $BIN 23 | elif [ "x$CCACHE" != "x" ]; then 24 | echo $CCACHE $BIN 25 | else 26 | echo $BIN 27 | fi 28 | 29 | -------------------------------------------------------------------------------- /env/DB_CXX.env: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | for x in db_cxx db_cxx-{4.5,4.3,4.2}; do 4 | if ld -l$x -o /dev/null 2>/dev/null >/dev/null ; then 5 | echo $x 6 | exit 0 7 | fi 8 | done 9 | 10 | exit 1 11 | -------------------------------------------------------------------------------- /env/HYPERTABLE_HOME.env: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | for x in {/usr,/usr/local,$HOME/hypertable{,/build}/0.9.*.*}; do 4 | if [[ -f $x/include/Hypertable/Lib/Table.h && 5 | -f $x/include/Common/Compat.h && 6 | ( -f $x/lib/libHypertable.a || 7 | -f $x/lib/libHypertable.so ) ]]; then 8 | echo $x 9 | exit 10 | fi 11 | done 12 | -------------------------------------------------------------------------------- /env/JDK_DIR.env: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # FC3 image has Sun Java in non-standard place (/usr/local/java) along 4 | # with GCJ in the usual place. Default to /usr/local/java if it 5 | # exists. 6 | for x in /usr/local/java /usr/lib/jvm/java /usr/lib/jvm/java-6-sun; do 7 | if [ -d $x ]; then 8 | echo $x 9 | exit 0 10 | fi 11 | done 12 | 13 | exit 1 14 | -------------------------------------------------------------------------------- /env/PERL_INCLUDE.env: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Find Perl headers, whose location varies by distribution and version. 4 | # Wang Lam 14 Apr 2008 5 | # 6 | # 7 | 8 | eval `perl -V:archlib` 9 | if [ -r $archlib/CORE/perl.h ]; then 10 | echo "$archlib/CORE" 11 | exit 0 12 | else 13 | perl -e 'print grep { -r "$_/CORE/perl.h" } @INC and print "/CORE\n";' 14 | exit $? 15 | fi 16 | 17 | -------------------------------------------------------------------------------- /env/PERL_LDFLAGS.env: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Find the library include path for libperl, whose location varies by 4 | # GNU/Linux distribution. 5 | # Wang Lam 14 Apr 2008 6 | # 7 | # 2008-05-06 : Removed -lperl from output. Use 'perl' as a 8 | # MAGIC_EXTERNAL_DEP instead. This should just print the library 9 | # path. -- Josh 10 | # 11 | 12 | if ld -lperl -o /dev/null 2>/dev/null >/dev/null ; then 13 | exit 0 14 | else 15 | eval `perl -V:archlib` 16 | if ld -L$archlib/CORE -lperl -o /dev/null 2>/dev/null >/dev/null ; then 17 | echo "-L$archlib/CORE" 18 | exit 0 19 | else 20 | # Brute-force search 21 | echo Brute-force: 22 | perl -e 'print "-L" and print grep { -r "$_/CORE/libperl.so" } @INC \ 23 | and print "/CORE\n";' 24 | exit $? 25 | fi 26 | fi 27 | 28 | -------------------------------------------------------------------------------- /env/PYTHON_INCLUDE.env: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Find Python headers, even if python-config is not (yet?) available. 4 | # Wang Lam 14 Apr 2008 5 | # 6 | # 7 | 8 | if python-config --includes 2>/dev/null ; then 9 | exit 0 10 | else 11 | for i in /usr/include/python /usr/include/python{2.5,2.4} ; do 12 | if [ -r ${i}/pyconfig.h ] ; then 13 | echo -I$i 14 | exit 0 15 | fi 16 | done 17 | fi 18 | 19 | exit 1 20 | -------------------------------------------------------------------------------- /env/SLICE_INCLUDE.env: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Find ICE headers, whose location varies by distribution and version. 4 | # Wang Lam 14 Apr 2008 5 | # 6 | # 7 | 8 | for i in /usr/share/slice /usr/share/Ice/slice /usr/share/Ice-?.?.?/slice ; do 9 | if [ -r ${i}/Ice/BuiltinSequences.ice ] ; then 10 | echo $i 11 | exit 0 12 | fi 13 | done 14 | 15 | exit 1 16 | -------------------------------------------------------------------------------- /env/TCMALLOC.disabled: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Find optional tcmalloc library. If it exists, echo it. 4 | for name in tcmalloc; do 5 | if ld -l$name -o/dev/null >/dev/null 2>&1 ; then 6 | echo $name 7 | exit 0 8 | fi 9 | done 10 | 11 | # Didn't find anything.. this is an optional library, so it's ok to 12 | # leave it blank. 13 | -------------------------------------------------------------------------------- /notes/README: -------------------------------------------------------------------------------- 1 | Friendly disclaimer: 2 | 3 | This directory contains a bunch of notes from various times in KDI's 4 | development history. They're more thought experiments than 5 | documentation. Many of the ideas are half-baked, and in some places, 6 | actual KDI development has gone a completely different way. 7 | 8 | -Josh 9 | -------------------------------------------------------------------------------- /notes/error_handling.txt: -------------------------------------------------------------------------------- 1 | #---------------------------------------------------------------------------- 2 | # Error handling strategy 3 | #---------------------------------------------------------------------------- 4 | 5 | Operations on Tables may fail immediately by throwing an exception, or 6 | fail much later somewhere in the system. For performance reasons, we 7 | don't want to wait for each mutation to propagate all the way through 8 | the system before returning from the Table call. Instead, we'll add a 9 | sync() method that will block until all mutations submitted so far 10 | propagate through the system. 11 | 12 | Table: 13 | sync(): 14 | waitForMutations(); 15 | if(errors) 16 | throw errors; 17 | 18 | Additionally, we'll add a special TableError type to contain the error 19 | information. It is not the only exception that may be thrown, but 20 | when it is, it may contain enough information to allow for recovery, 21 | or at least an informative error report. For example, if some cells 22 | mutations failed, the error may contain the specific cells that failed 23 | and why. 24 | 25 | This strategy allows applications to make their own 26 | performance/consistency trade-offs. It is not entirely clear how the 27 | implementation of this strategy will play out yet. 28 | -------------------------------------------------------------------------------- /notes/extended_interface.txt: -------------------------------------------------------------------------------- 1 | #---------------------------------------------------------------------------- 2 | # More interfaces 3 | #---------------------------------------------------------------------------- 4 | 5 | Basic Table: 6 | void set(string row, string column, int64_t time, string value); 7 | void erase(string row, string column, int64_t time); 8 | CellStreamPtr scan(); 9 | void sync(); 10 | 11 | Extended Table: 12 | void put(Cell x); 13 | void erase(ScanPredicate p); 14 | CellStreamPtr scan(ScanPredicate p); 15 | 16 | 17 | 18 | Basic Index: 19 | 20 | 21 | 22 | index.query(userQuery, maxResults, resultOffset, rankByFunction, functionParameters) 23 | ==> result count: (beginIdx, endIdx, maxIdx) 24 | ==> for each result in descending rankScore order: 25 | row, rankScore, optionalData? 26 | -------------------------------------------------------------------------------- /notes/lock_server.txt: -------------------------------------------------------------------------------- 1 | LockServer: 2 | 3 | Keep a map of named locks/values in memory. 4 | 5 | MetaData: 6 | int64_t instanceNumber; 7 | int64_t lockGeneration; 8 | int64_t aclGeneration; 9 | 10 | // For files 11 | int64_t contentGeneration; 12 | int64_t contentChecksum; 13 | 14 | OpenInfo: 15 | string name; 16 | int mode; 17 | int events; 18 | 19 | ClientHandle: 20 | OpenInfo openInfo; 21 | int64_t masterGeneration; 22 | char checkData[CHECK_SZ]; // HMAC of identity, provided by master 23 | // -- master should generate a secret key, then use that to 24 | // supply check data in authenticated handles. Masters should 25 | // verify check data on operations. After fail-over, the new 26 | // master must re-authenticate. Not really sure how the whole 27 | // authentication thing works. Have to check RPC/ICE docs. 28 | 29 | static ClientHandle open(uri, eventMask, mode); 30 | void close(); 31 | 32 | // File-like nodes 33 | void setContents(strref_t data, int64_t casGeneration=-1); 34 | void getContents(vector & data); 35 | void getContentsAndStat(vector & data, MetaData & stat); 36 | 37 | // Dir-like nodes 38 | void readDirectory(vector< pair > & children); 39 | 40 | // All nodes 41 | void getStat(MetaData & stat); 42 | void delete(); 43 | 44 | 45 | // Wrap these in RAII helper 46 | enum LockType { EXCLUSIVE, SHARED }; 47 | void lock(lockType); 48 | bool tryLock(lockType); 49 | void unlock(); 50 | void getSequencer(vector & seq); // must be locked 51 | 52 | // Sequencer helpers -- not sure how useful this stuff is. I was 53 | // thinking that the set sequencer would be from another handle 54 | // somewhere else. 55 | 56 | void setSequencer(strref_t seq); // how useful is this? what if the handle depends on multiple sequencers? 57 | bool checkSequencer(strref_t seq); 58 | -------------------------------------------------------------------------------- /notes/table_to_tablet.txt: -------------------------------------------------------------------------------- 1 | #---------------------------------------------------------------------------- 2 | # Tablet server 3 | #---------------------------------------------------------------------------- 4 | 5 | Current organization: 6 | 7 | kdiNetServer 8 | LocalTable 9 | LoggedMemoryTable 10 | DiskTable 11 | 12 | Future organization: 13 | 14 | kdiTabletServer 15 | TabletManager 16 | SharedTableMgr 17 | SharedLog 18 | Tablet 19 | SharedTable -> SharedTableMgr 20 | DiskTable 21 | 22 | 23 | 24 | MetaTable: 25 | 26 | Row: (tableName, inclusiveUpperBound) 27 | 28 | Cells: 29 | // Equal to inclusiveUpperBound of previous tablet (does not exist for first tablet in table) 30 | ("lowerBound", 0, exclusiveLowerBound) 31 | 32 | ("server", 0, tabletManagerProxy) 33 | 34 | 35 | TabletId: 36 | metaRow (includes table name and upper bound) 37 | lowerBound 38 | 39 | TabletManager: 40 | addTablet(tabletId) 41 | 42 | 43 | 44 | Tablet openTablet(tabletId) 45 | -------------------------------------------------------------------------------- /src/cc/ex/module.mk: -------------------------------------------------------------------------------- 1 | MAGIC_EXTERNAL_DEPS := $(TCMALLOC) 2 | include magic.mk 3 | -------------------------------------------------------------------------------- /src/cc/ex/trace_test.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-07-25 4 | // 5 | // This file is part of the ex library. 6 | // 7 | // The ex library is free software; you can redistribute it and/or modify it 8 | // under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The ex library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | using namespace ex; 24 | 25 | void goBoom() 26 | { 27 | raise("I has an error, kthxbye."); 28 | } 29 | 30 | int main(int ac, char ** av) 31 | { 32 | goBoom(); 33 | return 0; 34 | } 35 | -------------------------------------------------------------------------------- /src/cc/flux/module.mk: -------------------------------------------------------------------------------- 1 | MAGIC_MODULE_DEPS := warp ex 2 | include magic.mk 3 | -------------------------------------------------------------------------------- /src/cc/flux/threaded_reader_unittest.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2009 Josh Taylor (Kosmix Corporation) 3 | // Created 2009-01-10 4 | // 5 | // This file is part of the flux library. 6 | // 7 | // The flux library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The flux library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | using namespace flux; 28 | using namespace std; 29 | 30 | BOOST_AUTO_UNIT_TEST(int_reader) 31 | { 32 | // Sequence of numbers 33 | int const S[] = { 1, 1, 1, 2, 2, 3, 4, 5, 6, 6, 7, 7, 7, 9, 11, 12, 12, 34 | 12, 13, 14, 15 }; 35 | Stream::handle_t s(new Sequence< deque >(deque(S, S+sizeof(S)/sizeof(*S)))); 36 | 37 | // Worker pool -- 3 worker threads 38 | warp::WorkerPool pool(3, "TestPool", false); 39 | 40 | // Threaded reader -- read ahead 4 41 | Stream::handle_t r = makeThreadedReader(pool, 4); 42 | r->pipeFrom(s); 43 | 44 | int x; 45 | for(size_t i = 0; i < sizeof(S)/sizeof(*S); ++i) 46 | { 47 | BOOST_CHECK(r->get(x)); 48 | BOOST_CHECK_EQUAL(x, S[i]); 49 | } 50 | BOOST_CHECK(!r->get(x)); 51 | } 52 | -------------------------------------------------------------------------------- /src/cc/kdi/RowInterval.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-10-28 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_ROWINTERVAL_H 22 | #define KDI_ROWINTERVAL_H 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | namespace kdi { 29 | 30 | /// An interval range of rows 31 | class RowInterval 32 | : public warp::Interval 33 | { 34 | typedef warp::Interval super; 35 | 36 | public: 37 | RowInterval() : super() {} 38 | explicit RowInterval(super const & rows) : super(rows) {} 39 | explicit RowInterval(point_t const & lower, point_t const & upper) : 40 | super(lower, upper) {} 41 | 42 | /// Return the interval as a row predicate string 43 | std::string toRowPredicate() const; 44 | }; 45 | 46 | /// Serialize a RowInterval as a row predicate string 47 | std::ostream & operator<<(std::ostream & out, RowInterval const & x); 48 | 49 | } // namespace kdi 50 | 51 | #endif // KDI_ROWINTERVAL_H 52 | -------------------------------------------------------------------------------- /src/cc/kdi/app/frag_test.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2009 Josh Taylor (Kosmix Corporation) 3 | // Created 2009-01-16 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | using namespace kdi; 31 | using namespace warp; 32 | using namespace std; 33 | 34 | int main(int ac, char ** av) 35 | { 36 | OptionParser op; 37 | 38 | OptionMap opt; 39 | ArgumentList args; 40 | op.parseOrBail(ac, av, opt, args); 41 | 42 | for(ArgumentList::const_iterator i = args.begin(); 43 | i != args.end(); ++i) 44 | { 45 | kdi::local::DiskTablePtr dp = kdi::local::DiskTable::loadTable(*i); 46 | CellStreamPtr scan = dp->scan(""); 47 | 48 | Cell x; 49 | Cell y; 50 | 51 | if(scan->get(x)) 52 | { 53 | y = x; 54 | } 55 | 56 | while(scan->get(x)) { 57 | if(x < y) 58 | { 59 | log("Error: cells out of order - %s", *i); 60 | return 0; 61 | } 62 | y = x; 63 | } 64 | } 65 | 66 | return 0; 67 | } 68 | -------------------------------------------------------------------------------- /src/cc/kdi/app/module.mk: -------------------------------------------------------------------------------- 1 | MAGIC_MODULE_DEPS := kdi_local kdi_net kdi_meta kdi_hash kdi_mux warp_xml 2 | include magic.mk 3 | -------------------------------------------------------------------------------- /src/cc/kdi/attic/async_table.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-11-01 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_ASYNC_TABLE_H 22 | #define KDI_ASYNC_TABLE_H 23 | 24 | namespace kdi { 25 | 26 | 27 | } // namespace kdi 28 | 29 | #endif // KDI_ASYNC_TABLE_H 30 | -------------------------------------------------------------------------------- /src/cc/kdi/attic/client/cell_packet.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-11-07 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_CLIENT_CELL_PACKET_H 22 | #define KDI_CLIENT_CELL_PACKET_H 23 | 24 | namespace kdi { 25 | namespace client { 26 | 27 | struct CellPacket; 28 | class CellPacketBuffer; 29 | typedef boost::shared_ptr CellPacketBufferPtr; 30 | 31 | } // namespace client 32 | } // namespace kdi 33 | 34 | struct CellPacket 35 | { 36 | ArrayOffset 37 | }; 38 | 39 | 40 | #endif // KDI_CLIENT_CELL_PACKET_H 41 | -------------------------------------------------------------------------------- /src/cc/kdi/attic/client/disabled_module.mk: -------------------------------------------------------------------------------- 1 | MAGIC_MODULE_DEPS := kdi boost_system 2 | include magic.mk 3 | -------------------------------------------------------------------------------- /src/cc/kdi/attic/message/disabled_module.mk: -------------------------------------------------------------------------------- 1 | MAGIC_MODULE_DEPS := warp boost_system 2 | include magic.mk 3 | 4 | -------------------------------------------------------------------------------- /src/cc/kdi/cell.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-10-23 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_CELL_H 22 | #define KDI_CELL_H 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | namespace kdi { 30 | 31 | using sdstore::Cell; 32 | 33 | typedef flux::Stream CellStream; 34 | typedef boost::shared_ptr CellStreamPtr; 35 | 36 | using sdstore::makeCell; 37 | using sdstore::makeCellErasure; 38 | 39 | } // namespace kdi 40 | 41 | #endif // KDI_CELL_H 42 | -------------------------------------------------------------------------------- /src/cc/kdi/cell_ostream.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2009 Josh Taylor (Kosmix Corporation) 3 | // Created 2009-01-27 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_CELL_OSTREAM_H 22 | #define KDI_CELL_OSTREAM_H 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | namespace kdi { 29 | 30 | /// Adapter for printing a Cell as fast as possible. 31 | struct WithFastOutput 32 | { 33 | Cell const & cell; 34 | explicit WithFastOutput(Cell const & cell) : cell(cell) {} 35 | }; 36 | 37 | inline std::ostream & operator<<(std::ostream & o, WithFastOutput const & cell) 38 | { 39 | using warp::ReprEscape; 40 | Cell const & c = cell.cell; 41 | return o << "(\"" 42 | << ReprEscape(c.getRow()) 43 | << "\",\"" 44 | << ReprEscape(c.getColumn()) 45 | << "\",@" 46 | << c.getTimestamp() 47 | << ",\"" 48 | << ReprEscape(c.getValue()) 49 | << "\")"; 50 | } 51 | 52 | } // namespace kdi 53 | 54 | #endif // KDI_CELL_OSTREAM_H 55 | -------------------------------------------------------------------------------- /src/cc/kdi/hash/module.mk: -------------------------------------------------------------------------------- 1 | MAGIC_MODULE_DEPS := kdi 2 | include magic.mk 3 | -------------------------------------------------------------------------------- /src/cc/kdi/hyper/Client.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-08-21 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | // Silly Hypertable 22 | #include 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | using namespace kdi; 30 | using namespace kdi::hyper; 31 | 32 | ::Hypertable::ClientPtr const & kdi::hyper::getClient() 33 | { 34 | static boost::mutex mutex; 35 | boost::mutex::scoped_lock lock(mutex); 36 | 37 | static ::Hypertable::ClientPtr client; 38 | if(!client) 39 | { 40 | char const * installDir = getenv("HYPERTABLE_HOME"); 41 | if(!installDir) 42 | { 43 | using namespace ex; 44 | raise("Environment variable HYPERTABLE_HOME not set"); 45 | } 46 | 47 | client = new ::Hypertable::Client(installDir); 48 | } 49 | 50 | return client; 51 | } 52 | -------------------------------------------------------------------------------- /src/cc/kdi/hyper/Client.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-08-21 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_HYPER_CLIENT_H 22 | #define KDI_HYPER_CLIENT_H 23 | 24 | #include 25 | 26 | namespace kdi { 27 | namespace hyper { 28 | 29 | ::Hypertable::ClientPtr const & getClient(); 30 | 31 | } // namespace hyper 32 | } // namespace kdi 33 | 34 | 35 | #endif // KDI_HYPER_CLIENT_H 36 | -------------------------------------------------------------------------------- /src/cc/kdi/hyper/Table.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-08-21 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_HYPER_TABLE_H 22 | #define KDI_HYPER_TABLE_H 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | namespace kdi { 29 | namespace hyper { 30 | 31 | class Table; 32 | 33 | } // namespace hyper 34 | } // namespace kdi 35 | 36 | //---------------------------------------------------------------------------- 37 | // Table 38 | //---------------------------------------------------------------------------- 39 | class kdi::hyper::Table 40 | : public kdi::Table 41 | { 42 | ::Hypertable::TablePtr table; 43 | ::Hypertable::TableMutatorPtr mutator; 44 | 45 | public: 46 | explicit Table(std::string const & tableName); 47 | 48 | virtual void set(strref_t row, strref_t column, int64_t timestamp, 49 | strref_t value); 50 | virtual void erase(strref_t row, strref_t column, int64_t timestamp); 51 | virtual void sync(); 52 | virtual CellStreamPtr scan(ScanPredicate const & pred) const; 53 | }; 54 | 55 | #endif // KDI_HYPER_TABLE_H 56 | -------------------------------------------------------------------------------- /src/cc/kdi/hyper/module.mk: -------------------------------------------------------------------------------- 1 | MAGIC_REQUIRE_VARS := HYPERTABLE_HOME 2 | 3 | MAGIC_FLAGS_CPPFLAGS := -I$(HYPERTABLE_HOME)/include 4 | vpath %.a $(HYPERTABLE_HOME)/lib 5 | vpath %.so $(HYPERTABLE_HOME)/lib 6 | 7 | MAGIC_MODULE_DEPS := kdi 8 | MAGIC_EXTERNAL_DEPS := Hypertable_PREFER_STATIC Hyperspace_PREFER_STATIC HyperCommon_PREFER_STATIC HyperComm_PREFER_STATIC log4cpp $(BOOST_IOSTREAMS) 9 | include magic.mk 10 | -------------------------------------------------------------------------------- /src/cc/kdi/local/local_table_unittest.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-02-05 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | using namespace warp; 27 | using namespace kdi; 28 | using namespace kdi::local; 29 | using namespace kdi::unittest; 30 | using namespace std; 31 | 32 | BOOST_AUTO_UNIT_TEST(table_interface) 33 | { 34 | string tableName = "local+memfs:/test"; 35 | 36 | // Open the table by name 37 | TablePtr table = Table::open(tableName); 38 | BOOST_REQUIRE(table); 39 | 40 | // Test basic interface 41 | testTableInterface(table); 42 | } 43 | -------------------------------------------------------------------------------- /src/cc/kdi/local/module.mk: -------------------------------------------------------------------------------- 1 | MAGIC_MODULE_DEPS := kdi 2 | include magic.mk 3 | -------------------------------------------------------------------------------- /src/cc/kdi/marshal/cell_block.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-11-07 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_MARSHAL_CELL_BLOCK_H 22 | #define KDI_MARSHAL_CELL_BLOCK_H 23 | 24 | #include 25 | #include 26 | 27 | namespace kdi { 28 | namespace marshal { 29 | 30 | /// Row, column, timestamp key for a table Cell. 31 | struct CellKey 32 | { 33 | warp::StringOffset row; 34 | warp::StringOffset column; 35 | int64_t timestamp; 36 | }; 37 | 38 | /// Contents of a table Cell. A null value indicates an erasure 39 | /// for the given key. This structure contains explicit alignment 40 | /// padding. 41 | struct CellData 42 | { 43 | CellKey key; 44 | warp::StringOffset value; // null means erasure 45 | uint32_t __pad; 46 | }; 47 | 48 | /// Ordered collection of Cells. 49 | struct CellBlock 50 | { 51 | enum { 52 | TYPECODE = WARP_PACK4('C','e','l','B'), 53 | VERSION = 0, 54 | FLAGS = 1, 55 | ALIGNMENT = 8, 56 | }; 57 | 58 | warp::ArrayOffset cells; 59 | }; 60 | 61 | } // namespace marshal 62 | } // namespace kdi 63 | 64 | #endif // KDI_MARSHAL_CELL_BLOCK_H 65 | -------------------------------------------------------------------------------- /src/cc/kdi/marshal/module.mk: -------------------------------------------------------------------------------- 1 | MAGIC_MODULE_DEPS := kdi 2 | include magic.mk 3 | -------------------------------------------------------------------------------- /src/cc/kdi/meta/module.mk: -------------------------------------------------------------------------------- 1 | MAGIC_MODULE_DEPS := kdi 2 | include magic.mk 3 | -------------------------------------------------------------------------------- /src/cc/kdi/module.mk: -------------------------------------------------------------------------------- 1 | MAGIC_MODULE_DEPS := oort warp ex 2 | include magic.mk 3 | -------------------------------------------------------------------------------- /src/cc/kdi/mux/module.mk: -------------------------------------------------------------------------------- 1 | MAGIC_MODULE_DEPS := kdi warp 2 | include magic.mk 3 | -------------------------------------------------------------------------------- /src/cc/kdi/net/StatReporter.ice: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2009 Josh Taylor (Kosmix Corporation) 3 | // Created 2009-01-30 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it 8 | // under the terms of the GNU General Public License as published by 9 | // the Free Software Foundation; either version 2 of the License, or 10 | // any later version. 11 | // 12 | // KDI is distributed in the hope that it will be useful, but WITHOUT 13 | // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 14 | // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public 15 | // License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License 18 | // along with this program; if not, write to the Free Software 19 | // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 20 | // 02110-1301, USA. 21 | //---------------------------------------------------------------------------- 22 | 23 | #ifndef KDI_NET_STATREPORTER_ICE 24 | #define KDI_NET_STATREPORTER_ICE 25 | 26 | module kdi { 27 | module net { 28 | module details { 29 | 30 | dictionary StatMap; 31 | 32 | ["ami"] interface StatReporter { 33 | void getStats(out StatMap stats); 34 | }; 35 | 36 | }; // module details 37 | }; // module net 38 | }; // module kdi 39 | 40 | 41 | #endif // KDI_NET_STATREPORTER_ICE 42 | -------------------------------------------------------------------------------- /src/cc/kdi/net/StatReporterI.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2009 Josh Taylor (Kosmix Corporation) 3 | // Created 2009-01-30 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | using namespace kdi::net::details; 26 | using kdi::net::StatReportable; 27 | 28 | namespace { 29 | 30 | class StatReporterI : public StatReporter 31 | { 32 | StatReportable * stats; 33 | 34 | public: 35 | StatReporterI(StatReportable * stats) : stats(stats) {} 36 | 37 | void getStats(StatMap & map, Ice::Current const & cur) 38 | { 39 | stats->getStats(map); 40 | } 41 | }; 42 | } 43 | 44 | Ice::ObjectPtr kdi::net::makeStatReporter(StatReportable * reportable) 45 | { 46 | EX_CHECK_NULL(reportable); 47 | Ice::ObjectPtr p = new StatReporterI(reportable); 48 | return p; 49 | } 50 | -------------------------------------------------------------------------------- /src/cc/kdi/net/StatReporterI.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2009 Josh Taylor (Kosmix Corporation) 3 | // Created 2009-01-30 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_NET_STATREPORTERI_H 22 | #define KDI_NET_STATREPORTERI_H 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | namespace kdi { 30 | namespace net { 31 | 32 | class StatReportable 33 | { 34 | public: 35 | typedef std::map StatMap; 36 | 37 | virtual void getStats(StatMap & stats) const = 0; 38 | 39 | protected: 40 | ~StatReportable() {} 41 | }; 42 | 43 | Ice::ObjectPtr makeStatReporter(StatReportable * reportable); 44 | 45 | } // namespace net 46 | } // namespace kdi 47 | 48 | #endif // KDI_NET_STATREPORTERI_H 49 | -------------------------------------------------------------------------------- /src/cc/kdi/net/TableManager.ice: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-12-10 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it 8 | // under the terms of the GNU General Public License as published by 9 | // the Free Software Foundation; either version 2 of the License, or 10 | // any later version. 11 | // 12 | // KDI is distributed in the hope that it will be useful, but WITHOUT 13 | // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 14 | // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public 15 | // License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License 18 | // along with this program; if not, write to the Free Software 19 | // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 20 | // 02110-1301, USA. 21 | //---------------------------------------------------------------------------- 22 | 23 | #ifndef KDI_NET_TABLEMANAGER_ICE 24 | #define KDI_NET_TABLEMANAGER_ICE 25 | 26 | #include 27 | 28 | module kdi { 29 | module net { 30 | module details { 31 | 32 | ["ami"] interface Scanner { 33 | void getBulk(out Ice::ByteSeq cells, out bool lastBlock); 34 | idempotent void close(); 35 | }; 36 | 37 | ["ami"] interface Table { 38 | idempotent void applyMutations(Ice::ByteSeq cells); 39 | idempotent void sync(); 40 | idempotent Scanner* scan(string predicate); 41 | }; 42 | 43 | interface TableManager { 44 | Table* openTable(string path); 45 | }; 46 | 47 | }; // module details 48 | }; // module net 49 | }; // module kdi 50 | 51 | 52 | #endif // KDI_NET_TABLEMANAGER_ICE 53 | -------------------------------------------------------------------------------- /src/cc/kdi/net/module.mk: -------------------------------------------------------------------------------- 1 | MAGIC_MODULE_DEPS := kdi_tablet kdi_meta 2 | MAGIC_EXTERNAL_DEPS := Ice IceUtil 3 | include magic.mk 4 | -------------------------------------------------------------------------------- /src/cc/kdi/net/net_table.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-12-12 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_NET_NET_TABLE_H 22 | #define KDI_NET_NET_TABLE_H 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | namespace kdi { 30 | namespace net { 31 | 32 | class NetTable; 33 | 34 | } // namespace net 35 | } // namespace kdi 36 | 37 | //---------------------------------------------------------------------------- 38 | // NetTable 39 | //---------------------------------------------------------------------------- 40 | class kdi::net::NetTable 41 | : public kdi::Table, 42 | private boost::noncopyable 43 | { 44 | class Impl; 45 | class Scanner; 46 | 47 | boost::shared_ptr impl; 48 | 49 | public: 50 | NetTable(std::string const & uri); 51 | virtual ~NetTable(); 52 | 53 | // Table interface 54 | virtual void set(strref_t row, strref_t column, int64_t timestamp, 55 | strref_t value); 56 | virtual void erase(strref_t row, strref_t column, int64_t timestamp); 57 | virtual CellStreamPtr scan(ScanPredicate const & pred) const; 58 | virtual void sync(); 59 | virtual RowIntervalStreamPtr scanIntervals() const; 60 | }; 61 | 62 | 63 | #endif // KDI_NET_NET_TABLE_H 64 | -------------------------------------------------------------------------------- /src/cc/kdi/pred_parse_test.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-02-27 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | using namespace kdi; 29 | using namespace warp; 30 | using namespace std; 31 | using namespace ex; 32 | 33 | int main(int ac, char ** av) 34 | { 35 | OptionParser op("%prog "); 36 | 37 | OptionMap opt; 38 | ArgumentList args; 39 | op.parseOrBail(ac, av, opt, args); 40 | 41 | string predStr; 42 | if(!args.empty()) 43 | { 44 | ostringstream oss; 45 | ArgumentList::const_iterator i = args.begin(); 46 | oss << *i; 47 | for(++i; i != args.end(); ++i) 48 | oss << ' ' << *i; 49 | predStr = oss.str(); 50 | } 51 | 52 | cout << "Expression: " << predStr << endl; 53 | try { 54 | ScanPredicate pred(predStr); 55 | cout << "Predicate: " << pred << endl; 56 | } 57 | catch(ValueError const & ex) { 58 | cout << ex << endl; 59 | return 1; 60 | } 61 | 62 | return 0; 63 | } 64 | -------------------------------------------------------------------------------- /src/cc/kdi/row_buffer.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-01-20 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_ROW_BUFFER_H 22 | #define KDI_ROW_BUFFER_H 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | namespace kdi 29 | { 30 | using sdstore::RowBuffer; 31 | 32 | /// Shared pointer to a RowBuffer. 33 | typedef boost::shared_ptr RowBufferPtr; 34 | 35 | /// Stream over RowBuffers. 36 | typedef flux::Stream RowBufferStream; 37 | 38 | /// Shared pointer to a RowBufferStream. 39 | typedef boost::shared_ptr RowBufferStreamPtr; 40 | } 41 | 42 | #endif // KDI_ROW_BUFFER_H 43 | -------------------------------------------------------------------------------- /src/cc/kdi/row_stream.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-01-20 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_ROW_STREAM_H 22 | #define KDI_ROW_STREAM_H 23 | 24 | #include 25 | 26 | namespace kdi 27 | { 28 | using sdstore::RowStream; 29 | } 30 | 31 | #endif // KDI_ROW_STREAM_H 32 | -------------------------------------------------------------------------------- /src/cc/kdi/sdstore_rowstream.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-02-14 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #include 22 | #include 23 | 24 | using namespace sdstore; 25 | using namespace ex; 26 | 27 | RowStream::RowStream(CellStreamHandle const & cellStream) : 28 | input(cellStream) 29 | { 30 | if(!cellStream) 31 | raise("null cellStream"); 32 | } 33 | 34 | bool RowStream::fetch() 35 | { 36 | // Finish current row 37 | Cell x; 38 | while(rowCell) 39 | get(x); 40 | 41 | // Try start another row 42 | if(pending) 43 | { 44 | // Have a cell pending from last get() 45 | rowCell = pending; 46 | pending.release(); 47 | } 48 | else if(!input || !input->get(rowCell)) 49 | { 50 | // No cells left 51 | return false; 52 | } 53 | 54 | // Ready to scan new row 55 | firstInRow = true; 56 | rowName = rowCell.getRow(); 57 | return true; 58 | } 59 | 60 | void RowStream::pipeFrom(CellStreamHandle const & input) 61 | { 62 | rowCell.release(); 63 | pending.release(); 64 | this->input = input; 65 | } 66 | 67 | bool RowStream::get(RowBuffer & buf) 68 | { 69 | buf.clear(); 70 | if(fetch()) 71 | { 72 | Cell x; 73 | while(get(x)) 74 | buf.appendNoCheck(x); 75 | } 76 | return !buf.empty(); 77 | } 78 | -------------------------------------------------------------------------------- /src/cc/kdi/strref.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-10-23 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_STRREF_H 22 | #define KDI_STRREF_H 23 | 24 | #include 25 | 26 | namespace kdi { 27 | 28 | using warp::strref_t; 29 | using warp::binary_data; 30 | 31 | } // namespace kdi 32 | 33 | #endif // KDI_STRREF_H 34 | -------------------------------------------------------------------------------- /src/cc/kdi/tablet/CachedFragmentLoader.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-11-20 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_TABLET_CACHEDFRAGMENTLOADER_H 22 | #define KDI_TABLET_CACHEDFRAGMENTLOADER_H 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | namespace kdi { 31 | namespace tablet { 32 | 33 | /// A cache for loading fragments. If a fragment has already been 34 | /// loaded, it should be reused. 35 | class CachedFragmentLoader 36 | : public FragmentLoader, 37 | private boost::noncopyable 38 | { 39 | typedef std::tr1::unordered_map map_t; 40 | 41 | FragmentLoader * loader; 42 | mutable map_t cache; 43 | mutable size_t nextPurgeSize; 44 | mutable boost::mutex mutex; 45 | 46 | public: 47 | CachedFragmentLoader(FragmentLoader * loader); 48 | FragmentPtr load(std::string const & uri) const; 49 | 50 | private: 51 | void purgeStaleEntries() const; 52 | }; 53 | 54 | } // namespace tablet 55 | } // namespace kdi 56 | 57 | 58 | #endif // KDI_TABLET_CACHEDFRAGMENTLOADER_H 59 | -------------------------------------------------------------------------------- /src/cc/kdi/tablet/DiskFragmentLoader.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2009 Josh Taylor (Kosmix Corporation) 3 | // Created 2009-01-15 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_TABLET_DISKFRAGMENTLOADER_H 22 | #define KDI_TABLET_DISKFRAGMENTLOADER_H 23 | 24 | #include 25 | 26 | namespace warp { class StatTracker; } 27 | 28 | namespace kdi { 29 | namespace tablet { 30 | 31 | class DiskFragmentLoader 32 | : public FragmentLoader 33 | { 34 | warp::StatTracker * tracker; 35 | 36 | public: 37 | explicit DiskFragmentLoader(warp::StatTracker * tracker) : 38 | tracker(tracker) {} 39 | 40 | FragmentPtr load(std::string const & uri) const; 41 | }; 42 | 43 | } // namespace tablet 44 | } // namespace kdi 45 | 46 | #endif // KDI_TABLET_DISKFRAGMENTLOADER_H 47 | -------------------------------------------------------------------------------- /src/cc/kdi/tablet/DiskFragmentWriter.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2009 Josh Taylor (Kosmix Corporation) 3 | // Created 2009-01-15 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #include 22 | #include 23 | 24 | using namespace kdi; 25 | using namespace kdi::tablet; 26 | using namespace warp; 27 | 28 | DiskFragmentWriter::DiskFragmentWriter(ConfigManagerPtr const & configMgr) : 29 | writer(64 << 10), configMgr(configMgr) 30 | { 31 | } 32 | 33 | void DiskFragmentWriter::start(std::string const & table) 34 | { 35 | fn = configMgr->getDataFile(table); 36 | writer.open(fn); 37 | } 38 | 39 | void DiskFragmentWriter::put(Cell const & x) 40 | { 41 | writer.put(x); 42 | } 43 | 44 | std::string DiskFragmentWriter::finish() 45 | { 46 | writer.close(); 47 | return uriPushScheme(fn, "disk"); 48 | } 49 | 50 | size_t DiskFragmentWriter::size() const 51 | { 52 | return writer.size(); 53 | } 54 | -------------------------------------------------------------------------------- /src/cc/kdi/tablet/DiskFragmentWriter.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2009 Josh Taylor (Kosmix Corporation) 3 | // Created 2009-01-15 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_TABLET_DISKFRAGMENTWRITER_H 22 | #define KDI_TABLET_DISKFRAGMENTWRITER_H 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | namespace kdi { 29 | namespace tablet { 30 | 31 | class DiskFragmentWriter 32 | : public FragmentWriter 33 | { 34 | kdi::local::CurDiskTableWriter writer; 35 | std::string fn; 36 | 37 | ConfigManagerPtr configMgr; 38 | 39 | public: 40 | DiskFragmentWriter(ConfigManagerPtr const & configMgr); 41 | 42 | virtual void start(std::string const & table); 43 | virtual void put(Cell const & x); 44 | virtual std::string finish(); 45 | virtual size_t size() const; 46 | }; 47 | 48 | } // namespace tablet 49 | } // namespace kdi 50 | 51 | #endif // KDI_TABLET_DISKFRAGMENTWRITER_H 52 | -------------------------------------------------------------------------------- /src/cc/kdi/tablet/Fragment.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-12-10 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #include 22 | 23 | using namespace kdi::tablet; 24 | using namespace warp; 25 | using namespace std; 26 | 27 | size_t Fragment::getDiskSize(IntervalSet const & rows) const 28 | { 29 | size_t sz = 0; 30 | for(IntervalSet::const_iterator i = rows.begin(); 31 | i != rows.end();) 32 | { 33 | IntervalPoint const & lo = *i; ++i; 34 | IntervalPoint const & hi = *i; ++i; 35 | 36 | sz += getDiskSize(Interval(lo, hi)); 37 | } 38 | 39 | return sz; 40 | } 41 | 42 | size_t Fragment::getDiskSize() const { 43 | Interval unbounded = makeUnboundedInterval(); 44 | return getDiskSize(unbounded); 45 | } 46 | 47 | -------------------------------------------------------------------------------- /src/cc/kdi/tablet/FragmentLoader.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-12-05 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_TABLET_FRAGMENTLOADER_H 22 | #define KDI_TABLET_FRAGMENTLOADER_H 23 | 24 | #include 25 | #include 26 | 27 | namespace kdi { 28 | namespace tablet { 29 | 30 | class FragmentLoader; 31 | 32 | class Fragment; 33 | typedef boost::shared_ptr FragmentPtr; 34 | 35 | } // namespace tablet 36 | } // namespace kdi 37 | 38 | 39 | //---------------------------------------------------------------------------- 40 | // FragmentLoader 41 | //---------------------------------------------------------------------------- 42 | class kdi::tablet::FragmentLoader 43 | { 44 | public: 45 | /// Load a Fragment from a fragment URI. 46 | virtual FragmentPtr load(std::string const & fragmentUri) const = 0; 47 | 48 | protected: 49 | ~FragmentLoader() {} 50 | }; 51 | 52 | #endif // KDI_TABLET_FRAGMENTLOADER_H 53 | -------------------------------------------------------------------------------- /src/cc/kdi/tablet/LogEntry.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-05-20 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_TABLET_LOGENTRY_H 22 | #define KDI_TABLET_LOGENTRY_H 23 | 24 | #include 25 | 26 | namespace kdi { 27 | namespace tablet { 28 | 29 | struct LogEntry; 30 | 31 | } // namespace tablet 32 | } // namespace kdi 33 | 34 | //---------------------------------------------------------------------------- 35 | // LogEntry 36 | //---------------------------------------------------------------------------- 37 | struct kdi::tablet::LogEntry 38 | : public kdi::marshal::CellBlock 39 | { 40 | enum { 41 | TYPECODE = WARP_PACK4('T','b','L','g'), 42 | VERSION = 1 + kdi::marshal::CellBlock::VERSION, 43 | FLAGS = 0, 44 | ALIGNMENT = 8, 45 | }; 46 | 47 | warp::StringOffset tabletName; 48 | }; 49 | 50 | #endif // KDI_TABLET_LOGENTRY_H 51 | -------------------------------------------------------------------------------- /src/cc/kdi/tablet/LogReader.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-05-28 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_TABLET_LOGREADER_H 22 | #define KDI_TABLET_LOGREADER_H 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | namespace kdi { 30 | namespace tablet { 31 | 32 | class LogReader; 33 | 34 | } // namespace tablet 35 | } // namespace kdi 36 | 37 | //---------------------------------------------------------------------------- 38 | // LogReader 39 | //---------------------------------------------------------------------------- 40 | class kdi::tablet::LogReader : public kdi::CellStream 41 | { 42 | oort::RecordStreamHandle input; 43 | std::string tabletName; 44 | 45 | oort::Record logEntry; 46 | marshal::CellData const * nextCell; 47 | marshal::CellData const * endCell; 48 | 49 | public: 50 | LogReader(std::string const & logFn, std::string const & tabletName); 51 | bool get(Cell & x); 52 | }; 53 | 54 | 55 | #endif // KDI_TABLET_LOGREADER_H 56 | -------------------------------------------------------------------------------- /src/cc/kdi/tablet/LogWriter.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-05-20 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_TABLET_LOGWRITER_H 22 | #define KDI_TABLET_LOGWRITER_H 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | namespace kdi { 31 | namespace tablet { 32 | 33 | class LogWriter; 34 | 35 | } // namespace tablet 36 | } // namespace kdi 37 | 38 | //---------------------------------------------------------------------------- 39 | // LogWriter 40 | //---------------------------------------------------------------------------- 41 | class kdi::tablet::LogWriter 42 | : private boost::noncopyable 43 | { 44 | class Impl; 45 | boost::shared_ptr impl; 46 | 47 | public: 48 | explicit LogWriter(std::string const & fileUri); 49 | 50 | /// Sync log to disk. 51 | void sync(); 52 | 53 | /// Write a sequence of cells under a named log entry. The name 54 | /// represents the Tablet to which the cells belong. 55 | void logCells(std::string const & tabletName, 56 | std::vector const & cells); 57 | 58 | /// Get the sharedlog: URI for this log file. 59 | std::string const & getUri() const; 60 | }; 61 | 62 | 63 | #endif // KDI_TABLET_LOGWRITER_H 64 | -------------------------------------------------------------------------------- /src/cc/kdi/tablet/SwitchedFragmentLoader.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2009 Josh Taylor (Kosmix Corporation) 3 | // Created 2009-01-15 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | using namespace kdi::tablet; 26 | using namespace warp; 27 | using namespace ex; 28 | 29 | void SwitchedFragmentLoader::setLoader( 30 | std::string const & scheme, FragmentLoader * loader) 31 | { 32 | EX_CHECK_NULL(loader); 33 | loaders[scheme] = loader; 34 | } 35 | 36 | FragmentPtr SwitchedFragmentLoader::load(std::string const & uri) const 37 | { 38 | map_t::const_iterator i = loaders.find(uriTopScheme(uri)); 39 | if(i == loaders.end()) 40 | raise("unknown fragment scheme: %s", uri); 41 | 42 | return i->second->load(uri); 43 | } 44 | -------------------------------------------------------------------------------- /src/cc/kdi/tablet/SwitchedFragmentLoader.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2009 Josh Taylor (Kosmix Corporation) 3 | // Created 2009-01-15 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_TABLET_SWITCHEDFRAGMENTLOADER_H 22 | #define KDI_TABLET_SWITCHEDFRAGMENTLOADER_H 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | namespace kdi { 29 | namespace tablet { 30 | 31 | class SwitchedFragmentLoader 32 | : public FragmentLoader 33 | { 34 | typedef std::tr1::unordered_map map_t; 35 | map_t loaders; 36 | 37 | public: 38 | void setLoader(std::string const & scheme, FragmentLoader * loader); 39 | FragmentPtr load(std::string const & uri) const; 40 | }; 41 | 42 | } // namespace tablet 43 | } // namespace kdi 44 | 45 | #endif // KDI_TABLET_SWITCHEDFRAGMENTLOADER_H 46 | -------------------------------------------------------------------------------- /src/cc/kdi/tablet/TabletName.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-07-17 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_TABLET_TABLETNAME_H 22 | #define KDI_TABLET_TABLETNAME_H 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | namespace kdi { 29 | namespace tablet { 30 | 31 | class TabletName 32 | { 33 | std::string tableName; 34 | warp::IntervalPoint lastRow; 35 | 36 | public: 37 | /// Reconstruct a tablet name from an encoded string. 38 | explicit TabletName(warp::strref_t encodedName); 39 | 40 | /// Construct a tablet name from a table name and a row upper 41 | /// bound. 42 | TabletName(warp::strref_t tableName, 43 | warp::IntervalPoint lastRow); 44 | 45 | /// Get the tablet name as a single encoded string. 46 | std::string getEncoded() const; 47 | 48 | /// Get the table name for the tablet. 49 | std::string const & getTableName() const 50 | { 51 | return tableName; 52 | } 53 | 54 | /// Get the last row bound for the tablet. 55 | warp::IntervalPoint const & getLastRow() const 56 | { 57 | return lastRow; 58 | } 59 | }; 60 | 61 | } // namespace tablet 62 | } // namespace kdi 63 | 64 | #endif // KDI_TABLET_TABLETNAME_H 65 | -------------------------------------------------------------------------------- /src/cc/kdi/tablet/WorkQueue.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-08-19 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_TABLET_WORKQUEUE_H 22 | #define KDI_TABLET_WORKQUEUE_H 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | namespace kdi { 30 | namespace tablet { 31 | 32 | class WorkQueue; 33 | 34 | } // namespace tablet 35 | } // namespace kdi 36 | 37 | //---------------------------------------------------------------------------- 38 | // WorkQueue 39 | //---------------------------------------------------------------------------- 40 | class kdi::tablet::WorkQueue 41 | : private boost::noncopyable 42 | { 43 | typedef boost::function job_t; 44 | 45 | warp::SyncQueue jobs; 46 | boost::thread_group threads; 47 | bool done; 48 | 49 | public: 50 | explicit WorkQueue(size_t nThreads); 51 | ~WorkQueue(); 52 | 53 | void post(job_t const & job); 54 | void shutdown(); 55 | 56 | private: 57 | void workLoop(); 58 | }; 59 | 60 | #endif // KDI_TABLET_WORKQUEUE_H 61 | -------------------------------------------------------------------------------- /src/cc/kdi/tablet/module.mk: -------------------------------------------------------------------------------- 1 | MAGIC_MODULE_DEPS := kdi kdi_meta kdi_local oort 2 | include magic.mk 3 | -------------------------------------------------------------------------------- /src/cc/kdi/timestamp.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-09-15 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef KDI_TIMESTAMP_H 22 | #define KDI_TIMESTAMP_H 23 | 24 | #include 25 | 26 | namespace kdi { 27 | 28 | using warp::Timestamp; 29 | 30 | } // namespace kdi 31 | 32 | #endif // KDI_TIMESTAMP_H 33 | -------------------------------------------------------------------------------- /src/cc/kdi/tmp/module.mk: -------------------------------------------------------------------------------- 1 | MAGIC_MODULE_DEPS := kdi_local 2 | include magic.mk 3 | -------------------------------------------------------------------------------- /src/cc/minilzo/module.mk: -------------------------------------------------------------------------------- 1 | include magic.mk 2 | -------------------------------------------------------------------------------- /src/cc/oort/exception.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 3 | // Created 2006-01-11 4 | // 5 | // This file is part of the oort library. 6 | // 7 | // The oort library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The oort library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef OORT_EXCEPTION_H 23 | #define OORT_EXCEPTION_H 24 | 25 | #include 26 | 27 | namespace oort 28 | { 29 | EX_DECLARE_EXCEPTION(TypeError, ex::Exception); 30 | EX_DECLARE_EXCEPTION(VersionError, ex::Exception); 31 | } 32 | 33 | #endif // OORT_EXCEPTION_H 34 | -------------------------------------------------------------------------------- /src/cc/oort/headers.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 3 | // Created 2006-02-17 4 | // 5 | // This file is part of the oort library. 6 | // 7 | // The oort library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The oort library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | 24 | namespace 25 | { 26 | oort::vega::RecordHeader::Spec const vegaSpec; 27 | oort::sirius::RecordHeader::Spec const siriusSpec; 28 | } 29 | 30 | oort::vega::RecordHeader::Spec const * const oort::VEGA_SPEC = &vegaSpec; 31 | oort::sirius::RecordHeader::Spec const * const oort::SIRIUS_SPEC = &siriusSpec; 32 | -------------------------------------------------------------------------------- /src/cc/oort/init.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 3 | // Created 2006-08-18 4 | // 5 | // This file is part of the oort library. 6 | // 7 | // The oort library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The oort library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | #include 24 | 25 | void oort::initModule() 26 | { 27 | static bool once = false; 28 | if(!once) 29 | { 30 | once = true; 31 | 32 | warp::initModule(); 33 | 34 | WARP_CALL_INIT(oort_recordstream); 35 | WARP_CALL_INIT(oort_splitstream); 36 | WARP_CALL_INIT(oort_checkstream); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/cc/oort/init.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 3 | // Created 2006-08-18 4 | // 5 | // This file is part of the oort library. 6 | // 7 | // The oort library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The oort library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef OORT_INIT_H 23 | #define OORT_INIT_H 24 | 25 | namespace oort 26 | { 27 | void initModule(); 28 | } 29 | 30 | #endif // OORT_INIT_H 31 | -------------------------------------------------------------------------------- /src/cc/oort/module.mk: -------------------------------------------------------------------------------- 1 | MAGIC_MODULE_DEPS := warp ex minilzo 2 | include magic.mk 3 | -------------------------------------------------------------------------------- /src/cc/oort/recordstream.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 3 | // Created 2006-03-08 4 | // 5 | // This file is part of the oort library. 6 | // 7 | // The oort library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The oort library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef OORT_RECORDSTREAM_H 23 | #define OORT_RECORDSTREAM_H 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | namespace oort 30 | { 31 | typedef flux::Stream RecordStream; 32 | typedef RecordStream::handle_t RecordStreamHandle; 33 | 34 | /// Open a RecordStream by name. The \c flags parameter is the 35 | /// same as for File::open(). 36 | extern RecordStreamHandle openStream(std::string const & uriFn, int flags); 37 | 38 | /// Alias for openStream(uriFn, O_RDONLY). 39 | extern RecordStreamHandle inputStream(std::string const & uriFn); 40 | 41 | /// Alias for openStream(uriFn, O_WRONLY | O_CREAT | O_TRUNC). 42 | extern RecordStreamHandle outputStream(std::string const & uriFn); 43 | 44 | /// Alias for openStream(uriFn, O_WRONLY | O_CREAT | O_APPEND). 45 | extern RecordStreamHandle appendStream(std::string const & uriFn); 46 | 47 | /// Stream open function prototype -- mode is same as used by File::open. 48 | typedef RecordStreamHandle (OpenStreamFunc)(std::string const & uri, int mode); 49 | 50 | /// Register a handler for a particular scheme. 51 | extern void registerScheme(char const * name, OpenStreamFunc * func); 52 | } 53 | 54 | #endif // OORT_RECORDSTREAM_H 55 | -------------------------------------------------------------------------------- /src/cc/oort/serializer.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 3 | // Created 2006-05-24 4 | // 5 | // This file is part of the oort library. 6 | // 7 | // The oort library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The oort library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef OORT_SERIALIZER_H 23 | #define OORT_SERIALIZER_H 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | namespace oort 31 | { 32 | //------------------------------------------------------------------------ 33 | // InputStreamSerializer 34 | //------------------------------------------------------------------------ 35 | /// Convert a Record input stream to a byte stream. 36 | class InputStreamSerializer : public boost::noncopyable 37 | { 38 | typedef boost::scoped_array buf_t; 39 | 40 | RecordStreamHandle input; 41 | HeaderSpec const * spec; 42 | buf_t headerBuf; 43 | 44 | Record rec; 45 | size_t headerLeft; 46 | size_t dataLeft; 47 | 48 | public: 49 | InputStreamSerializer(RecordStreamHandle const & input, 50 | HeaderSpec const * spec); 51 | 52 | size_t read(void * dst, size_t sz); 53 | }; 54 | } 55 | 56 | #endif // OORT_SERIALIZER_H 57 | -------------------------------------------------------------------------------- /src/cc/pykdi/conversion.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-01-11 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef PYKDI_CONVERSION_H 22 | #define PYKDI_CONVERSION_H 23 | 24 | #include 25 | #include 26 | 27 | namespace pykdi { 28 | 29 | /// Convert warp::StringRange to Python string objects. 30 | struct strdata_converter 31 | { 32 | static PyObject * convert(warp::strref_t s) 33 | { 34 | return PyString_FromStringAndSize(s.begin(), s.size()); 35 | } 36 | }; 37 | 38 | } // namespace pykdi 39 | 40 | #endif // PYKDI_CONVERSION_H 41 | -------------------------------------------------------------------------------- /src/cc/pykdi/module.mk: -------------------------------------------------------------------------------- 1 | MAGIC_FLAGS_CPPFLAGS := $(PYTHON_INCLUDE) 2 | 3 | MAGIC_MODULE_DEPS := kdi_meta kdi_net kdi_hash 4 | MAGIC_EXTERNAL_DEPS := $(BOOST_PYTHON) 5 | include magic.mk 6 | 7 | 8 | # Special stuff for Python module 9 | $(build)/pykdi.so: $(_LIB_SO) 10 | @echo "Alias $< as $@" 11 | @ln -nf $< $@ 12 | 13 | TARGETS := $(TARGETS) $(build)/pykdi.so 14 | LIB_INSTALL := 15 | PY_INSTALL := $(build)/pykdi.so 16 | -------------------------------------------------------------------------------- /src/cc/pykdi/pyscan.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-01-11 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #include 22 | 23 | using namespace pykdi; 24 | using namespace kdi; 25 | using namespace boost::python; 26 | using boost::python::objects::stop_iteration_error; 27 | 28 | //---------------------------------------------------------------------------- 29 | // PyScan 30 | //---------------------------------------------------------------------------- 31 | PyScan::PyScan(CellStreamPtr const & scan) : 32 | scan(scan) 33 | { 34 | } 35 | 36 | object PyScan::next() 37 | { 38 | Cell x; 39 | if(!scan->get(x)) 40 | stop_iteration_error(); 41 | 42 | return make_tuple( 43 | str(x.getRow()), 44 | str(x.getColumn()), 45 | x.getTimestamp(), 46 | str(x.getValue())); 47 | } 48 | 49 | void PyScan::defineWrapper() 50 | { 51 | class_("Scan", 52 | "Iterator over a sequence of Table cells.", 53 | no_init) 54 | .def("next", &PyScan::next, 55 | "Get the next (row,column,version,value) cell in the scan.") 56 | .def("__iter__", &PyScan::iter, return_self<>(), 57 | "Return self (an iterable object).") 58 | ; 59 | } 60 | -------------------------------------------------------------------------------- /src/cc/pykdi/pyscan.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-01-11 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef PYKDI_PYSCAN_H 22 | #define PYKDI_PYSCAN_H 23 | 24 | #include 25 | #include 26 | 27 | namespace pykdi { 28 | 29 | class PyScan; 30 | 31 | } // namespace pykdi 32 | 33 | 34 | //---------------------------------------------------------------------------- 35 | // PyScan 36 | //---------------------------------------------------------------------------- 37 | class pykdi::PyScan 38 | { 39 | kdi::CellStreamPtr scan; 40 | 41 | public: 42 | explicit PyScan(kdi::CellStreamPtr const & scan); 43 | boost::python::object next(); 44 | void iter() {} 45 | 46 | public: 47 | static void defineWrapper(); 48 | }; 49 | 50 | #endif // PYKDI_PYSCAN_H 51 | -------------------------------------------------------------------------------- /src/cc/pykdi/pytable.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-01-11 4 | // 5 | // This file is part of KDI. 6 | // 7 | // KDI is free software; you can redistribute it and/or modify it under the 8 | // terms of the GNU General Public License as published by the Free Software 9 | // Foundation; either version 2 of the License, or any later version. 10 | // 11 | // KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | // WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | // FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | // details. 15 | // 16 | // You should have received a copy of the GNU General Public License along 17 | // with this program; if not, write to the Free Software Foundation, Inc., 18 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | //---------------------------------------------------------------------------- 20 | 21 | #ifndef PYKDI_PYTABLE_H 22 | #define PYKDI_PYTABLE_H 23 | 24 | #include 25 | #include 26 | 27 | namespace pykdi { 28 | 29 | class PyTable; 30 | 31 | // Forward declaration 32 | class PyScan; 33 | 34 | } // namespace pykdi 35 | 36 | 37 | //---------------------------------------------------------------------------- 38 | // PyTable 39 | //---------------------------------------------------------------------------- 40 | class pykdi::PyTable 41 | { 42 | std::string uri; 43 | kdi::TablePtr table; 44 | 45 | public: 46 | explicit PyTable(std::string const & uri); 47 | 48 | void set(std::string const & row, std::string const & col, 49 | int64_t rev, std::string const & val); 50 | void erase(std::string const & row, std::string const & col, 51 | int64_t rev); 52 | 53 | void sync(); 54 | 55 | PyScan scan() const; 56 | PyScan scan(kdi::ScanPredicate const & pred) const; 57 | PyScan scan(std::string const & pred) const; 58 | 59 | public: 60 | static void defineWrapper(); 61 | }; 62 | 63 | #endif // PYKDI_PYTABLE_H 64 | -------------------------------------------------------------------------------- /src/cc/unittest/main.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-03-23 4 | // 5 | // This file is part of the unittest library. 6 | // 7 | // The unittest library is free software; you can redistribute it and/or 8 | // modify it under the terms of the GNU General Public License as published 9 | // by the Free Software Foundation; either version 2 of the License, or any 10 | // later version. 11 | // 12 | // The unittest library is distributed in the hope that it will be useful, 13 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef UNITTEST_MAIN_H 23 | #define UNITTEST_MAIN_H 24 | 25 | // This provides main(). Implement test functions like this: 26 | // 27 | // BOOST_AUTO_TEST_CASE(some_test) 28 | // { 29 | // BOOST_CHECK_THROW(raise("Boom!"), RuntimeError); 30 | // BOOST_CHECK_EQUAL(2+2, 4); 31 | // BOOST_CHECK_SMALL(1.0 / 1e50); 32 | // } 33 | 34 | #include 35 | #if ((BOOST_VERSION/100)%1000) < 34 36 | #define BOOST_AUTO_TEST_MAIN 37 | #else 38 | #define BOOST_TEST_MAIN 39 | #define BOOST_TEST_DYN_LINK 40 | #endif 41 | 42 | #include 43 | #include 44 | #include 45 | 46 | #ifndef BOOST_AUTO_UNIT_TEST 47 | // @todo TODO Update use of macro name for Boost 1.33+. 48 | #define BOOST_AUTO_UNIT_TEST BOOST_AUTO_TEST_CASE 49 | #endif 50 | 51 | #endif // UNITTEST_MAIN_H 52 | -------------------------------------------------------------------------------- /src/cc/warp/EnvironmentVariable.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-11-18 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | #include 24 | #include 25 | 26 | using namespace warp; 27 | using namespace ex; 28 | 29 | //---------------------------------------------------------------------------- 30 | // EnvironmentVariable 31 | //---------------------------------------------------------------------------- 32 | bool EnvironmentVariable::get(std::string const & name, std::string & value) 33 | { 34 | if(char const * v = getenv(name.c_str())) 35 | { 36 | value = v; 37 | return true; 38 | } 39 | else 40 | { 41 | return false; 42 | } 43 | } 44 | 45 | void EnvironmentVariable::set(std::string const & name, std::string const & value) 46 | { 47 | if(0 != setenv(name.c_str(), value.c_str(), 1)) 48 | { 49 | raise("setenv (%s=%s) failed: %s", 50 | name, value, getStdError()); 51 | } 52 | } 53 | 54 | void EnvironmentVariable::clear(std::string const & name) 55 | { 56 | if(0 != unsetenv(name.c_str())) 57 | { 58 | raise("clearenv (%s) failed: %s", 59 | name, getStdError()); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/cc/warp/Runnable.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2009 Josh Taylor (Kosmix Corporation) 3 | // Created 2009-01-10 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_RUNNABLE_H 23 | #define WARP_RUNNABLE_H 24 | 25 | namespace warp { 26 | 27 | class Runnable 28 | { 29 | public: 30 | virtual void run() = 0; 31 | 32 | protected: 33 | ~Runnable() {} 34 | }; 35 | 36 | } // namespace warp 37 | 38 | #endif // WARP_RUNNABLE_H 39 | -------------------------------------------------------------------------------- /src/cc/warp/StatTracker.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2009 Josh Taylor (Kosmix Corporation) 3 | // Created 2009-01-15 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_STATTRACKER_H 23 | #define WARP_STATTRACKER_H 24 | 25 | #include 26 | 27 | namespace warp { 28 | 29 | class StatTracker 30 | { 31 | public: 32 | /// Set the named the named stat to the given value 33 | virtual void set(strref_t name, int64_t value) = 0; 34 | 35 | /// Add delta to the named stat 36 | virtual void add(strref_t name, int64_t delta) = 0; 37 | 38 | protected: 39 | ~StatTracker() {} 40 | }; 41 | 42 | class NullStatTracker 43 | : public StatTracker 44 | { 45 | public: 46 | void set(strref_t name, int64_t value) {} 47 | void add(strref_t name, int64_t delta) {} 48 | }; 49 | 50 | } // namespace warp 51 | 52 | #endif // WARP_STATTRACKER_H 53 | -------------------------------------------------------------------------------- /src/cc/warp/StringTrie.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2009 Josh Taylor (Kosmix Corporation) 3 | // Created 2009-04-01 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | 24 | uint16_t const warp::StringTrie_details::BUCKET_SIZES[] = { 0, 1, 3, 7, 17, 37, 67, 127, 257 }; 25 | uint16_t const warp::StringTrie_details::GROW_SIZES[] = { 0, 1, 3, 5, 12, 27, 50, 95, 258 }; 26 | -------------------------------------------------------------------------------- /src/cc/warp/WorkerPool.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2009 Josh Taylor (Kosmix Corporation) 3 | // Created 2009-01-10 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | using namespace warp; 28 | 29 | void WorkerPool::workerLoop() 30 | { 31 | Runnable * task = 0; 32 | while(tasks.pop(task)) 33 | task->run(); 34 | } 35 | 36 | WorkerPool::WorkerPool(size_t nWorkers, std::string const & poolName, 37 | bool verbose) 38 | { 39 | for(size_t i = 0; i < nWorkers; ++i) 40 | { 41 | std::ostringstream oss; 42 | oss << poolName << "-" << i; 43 | workers.create_thread( 44 | warp::callOrDie( 45 | boost::bind( 46 | &WorkerPool::workerLoop, 47 | this), 48 | oss.str(), verbose)); 49 | } 50 | } 51 | 52 | WorkerPool::~WorkerPool() 53 | { 54 | tasks.cancelWaits(); 55 | workers.join_all(); 56 | } 57 | -------------------------------------------------------------------------------- /src/cc/warp/WorkerPool.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2009 Josh Taylor (Kosmix Corporation) 3 | // Created 2009-01-10 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_WORKERPOOL_H 23 | #define WARP_WORKERPOOL_H 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | namespace warp { 32 | 33 | class WorkerPool; 34 | 35 | } // namespace warp 36 | 37 | //---------------------------------------------------------------------------- 38 | // WorkerPool 39 | //---------------------------------------------------------------------------- 40 | class warp::WorkerPool 41 | : private boost::noncopyable 42 | { 43 | warp::SyncQueue tasks; 44 | boost::thread_group workers; 45 | 46 | void workerLoop(); 47 | 48 | public: 49 | WorkerPool(size_t nWorkers, std::string const & poolName, bool verbose); 50 | ~WorkerPool(); 51 | 52 | void submit(Runnable * r) 53 | { 54 | tasks.push(r); 55 | } 56 | }; 57 | 58 | 59 | #endif // WARP_WORKERPOOL_H 60 | -------------------------------------------------------------------------------- /src/cc/warp/adler.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Public domain implementation of Adler-32 checksum from http://en.wikipedia.org/wiki/Adler-32 3 | */ 4 | 5 | #include 6 | 7 | #define MOD_ADLER 65521 8 | 9 | uint32_t warp::adler(const uint8_t *data, size_t len) /* data: Pointer to the data to be summed; len is in bytes */ 10 | { 11 | uint32_t a = 1, b = 0; 12 | 13 | while (len > 0) 14 | { 15 | size_t tlen = len > 5552 ? 5552 : len; 16 | len -= tlen; 17 | do 18 | { 19 | a += *data++; 20 | b += a; 21 | } while (--tlen); 22 | 23 | a %= MOD_ADLER; 24 | b %= MOD_ADLER; 25 | } 26 | 27 | return (b << 16) | a; 28 | } 29 | -------------------------------------------------------------------------------- /src/cc/warp/adler.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Public domain implementation of Adler-32 checksum from http://en.wikipedia.org/wiki/Adler-32 3 | */ 4 | 5 | #include 6 | #include 7 | 8 | namespace warp { 9 | uint32_t adler(const uint8_t *data, size_t len); 10 | } 11 | -------------------------------------------------------------------------------- /src/cc/warp/algorithm_unittest.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-06-14 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | #include 24 | 25 | #include 26 | #include 27 | 28 | using namespace warp; 29 | using namespace ex; 30 | using namespace std; 31 | 32 | BOOST_AUTO_UNIT_TEST(assoc_test) 33 | { 34 | map a; 35 | a["x"] = "y"; 36 | 37 | map b; 38 | b[42] = 3.14159; 39 | 40 | BOOST_CHECK_EQUAL(get_assoc(a, "x"), "y"); 41 | BOOST_CHECK_THROW(get_assoc(a, "a"), KeyError); 42 | BOOST_CHECK_EQUAL(get_assoc(a, "x", "z"), "y"); 43 | BOOST_CHECK_EQUAL(get_assoc(a, "a", "b"), "b"); 44 | 45 | BOOST_CHECK_EQUAL(get_assoc(b, 42), 3.14159); 46 | BOOST_CHECK_THROW(get_assoc(b, 7), KeyError); 47 | BOOST_CHECK_EQUAL(get_assoc(b, 42, 6.0221415e23), 3.14159); 48 | BOOST_CHECK_EQUAL(get_assoc(b, 7, 6.0221415e23), 6.0221415e23); 49 | } 50 | -------------------------------------------------------------------------------- /src/cc/warp/call_or_die.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-12-02 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | using namespace warp; 28 | 29 | namespace { 30 | 31 | void _callOrDie(boost::function const & func, 32 | std::string const & name, bool verbose) 33 | { 34 | if(verbose) 35 | log("%s starting", name); 36 | 37 | try { 38 | func(); 39 | } 40 | catch(ex::Exception const & ex) { 41 | log("%s error: %s", name, ex); 42 | std::terminate(); 43 | } 44 | catch(std::exception const & ex) { 45 | log("%s error: %s", name, ex.what()); 46 | std::terminate(); 47 | } 48 | catch(...) { 49 | log("%s error: unknown exception", name); 50 | std::terminate(); 51 | } 52 | 53 | if(verbose) 54 | log("%s exiting", name); 55 | } 56 | 57 | } 58 | 59 | boost::function warp::callOrDie( 60 | boost::function const & func, 61 | std::string const & name, bool verbose) 62 | { 63 | return boost::bind(&_callOrDie, func, name, verbose); 64 | } 65 | -------------------------------------------------------------------------------- /src/cc/warp/call_or_die.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-12-02 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_CALL_OR_DIE_H 23 | #define WARP_CALL_OR_DIE_H 24 | 25 | #include 26 | #include 27 | 28 | namespace warp { 29 | 30 | /// Wrap a function call so that any uncaught exceptions will be 31 | /// logged and cause the process to terminate. This is useful for 32 | /// wrapping background thread loops. The given name is used as a 33 | /// label for any log messages. If verbose is true, the wrapper 34 | /// function will also log normal start and exit from the 35 | /// function. 36 | boost::function callOrDie( 37 | boost::function const & func, 38 | std::string const & name, bool verbose); 39 | 40 | } // namespace warp 41 | 42 | #endif // WARP_CALL_OR_DIE_H 43 | -------------------------------------------------------------------------------- /src/cc/warp/charmap.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 3 | // Created 2006-08-07 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | 24 | using namespace warp; 25 | 26 | namespace 27 | { 28 | struct LowerCaseMap 29 | { 30 | CharMap m; 31 | LowerCaseMap() 32 | { 33 | char d = 'a' - 'A'; 34 | for(char i = 'A'; i <= 'Z'; ++i) 35 | m[i] += d; 36 | } 37 | }; 38 | 39 | struct UpperCaseMap 40 | { 41 | CharMap m; 42 | UpperCaseMap() 43 | { 44 | char d = 'A' - 'Z'; 45 | for(char i = 'a'; i <= 'z'; ++i) 46 | m[i] += d; 47 | } 48 | }; 49 | } 50 | 51 | CharMap::CharMap() 52 | { 53 | for(int i = 0; i < 256; ++i) 54 | tr[i] = (char)i; 55 | } 56 | 57 | CharMap const & CharMap::toLower() 58 | { 59 | static LowerCaseMap lcMap; 60 | return lcMap.m; 61 | } 62 | 63 | CharMap const & CharMap::toUpper() 64 | { 65 | static UpperCaseMap ucMap; 66 | return ucMap.m; 67 | } 68 | -------------------------------------------------------------------------------- /src/cc/warp/charmap.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 3 | // Created 2006-08-07 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_CHARMAP_H 23 | #define WARP_CHARMAP_H 24 | 25 | #include 26 | 27 | namespace warp 28 | { 29 | class CharMap; 30 | } 31 | 32 | //---------------------------------------------------------------------------- 33 | // CharMap 34 | //---------------------------------------------------------------------------- 35 | class warp::CharMap 36 | { 37 | char tr[256]; 38 | 39 | public: 40 | CharMap(); 41 | 42 | char & operator[](char x) { 43 | return tr[(uint8_t)x]; 44 | } 45 | 46 | char operator[](char x) const { 47 | return tr[(uint8_t)x]; 48 | } 49 | 50 | inline char operator()(char x) const { 51 | return tr[(uint8_t)x]; 52 | } 53 | 54 | static CharMap const & toLower(); 55 | static CharMap const & toUpper(); 56 | }; 57 | 58 | 59 | #endif // WARP_CHARMAP_H 60 | -------------------------------------------------------------------------------- /src/cc/warp/file_cache.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-06-08 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #ifndef WARP_FILE_CACHE_H 29 | #define WARP_FILE_CACHE_H 30 | 31 | namespace warp { 32 | 33 | class FileCache; 34 | typedef boost::shared_ptr FileCachePtr; 35 | 36 | } // namespace warp 37 | 38 | 39 | //---------------------------------------------------------------------------- 40 | // FileCache 41 | //---------------------------------------------------------------------------- 42 | class warp::FileCache 43 | : public boost::enable_shared_from_this, 44 | private boost::noncopyable 45 | { 46 | class FilePool; 47 | class BlockCache; 48 | class Block; 49 | class CachedFile; 50 | 51 | boost::shared_ptr< Synchronized > syncFilePool; 52 | boost::shared_ptr< Synchronized > syncBlockCache; 53 | 54 | private: 55 | FileCache(size_t cacheSize, size_t maxHandles); 56 | 57 | public: 58 | static FileCachePtr make(size_t cacheSize, size_t maxHandles); 59 | FilePtr open(std::string const & uri, int mode=O_RDONLY); 60 | }; 61 | 62 | #endif // WARP_FILE_CACHE_H 63 | -------------------------------------------------------------------------------- /src/cc/warp/filestream_unittest.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-08-16 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | using namespace warp; 29 | 30 | BOOST_AUTO_TEST_CASE(test_ostream) 31 | { 32 | char buf[1024]; 33 | memset(buf, 0, sizeof(buf)); 34 | 35 | FilePtr fp(new MemFile(buf, sizeof(buf), true)); 36 | FileStream s(fp); 37 | 38 | s << "Hi there" << std::endl; 39 | BOOST_CHECK_EQUAL("Hi there\n", buf); 40 | 41 | s.seekg(0); 42 | s << "Eat "; 43 | s.flush(); 44 | 45 | BOOST_CHECK_EQUAL("Eat here\n", buf); 46 | 47 | std::string x; 48 | s >> x; 49 | 50 | BOOST_CHECK_EQUAL(x, "here"); 51 | } 52 | -------------------------------------------------------------------------------- /src/cc/warp/fsutil.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 3 | // Created 2006-06-18 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_FSUTIL_H 23 | #define WARP_FSUTIL_H 24 | 25 | #include 26 | #include 27 | 28 | namespace warp 29 | { 30 | void findFilesNumericSorted(std::string const & dir, 31 | std::string const & ext, 32 | std::vector & out); 33 | 34 | void findDirsNumericSorted(std::string const & dir, 35 | std::vector & out); 36 | 37 | void getExistingDirs(std::string const & base, 38 | std::vector const & children, 39 | std::vector & out, 40 | bool reportMissing); 41 | 42 | void getExistingFiles(std::string const & base, 43 | std::vector const & children, 44 | std::string const & ext, 45 | std::vector & out, 46 | bool reportMissing); 47 | } 48 | 49 | #endif // WARP_FSUTIL_H 50 | -------------------------------------------------------------------------------- /src/cc/warp/generator.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 3 | // Created 2006-01-20 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_GENERATOR_H 23 | #define WARP_GENERATOR_H 24 | 25 | #include 26 | #include 27 | 28 | namespace warp 29 | { 30 | template 31 | class Generator 32 | { 33 | public: 34 | typedef T value_t; 35 | typedef Generator my_t; 36 | typedef boost::shared_ptr handle_t; 37 | 38 | public: 39 | virtual ~Generator() {} 40 | 41 | /// Return the next item in the sequence. May throw 42 | /// StopIteration if sequence is exhausted. 43 | virtual T next() = 0; 44 | 45 | /// Get the next item in sequence. Returns true if item 46 | /// exists, or false on end of sequence (i.e. StopIteration 47 | /// was thrown). 48 | bool getNext(T & x) 49 | { 50 | try { 51 | x = next(); 52 | return true; 53 | } 54 | catch(ex::StopIteration) { 55 | return false; 56 | } 57 | } 58 | }; 59 | } 60 | 61 | #endif // WARP_GENERATOR_H 62 | -------------------------------------------------------------------------------- /src/cc/warp/gzip.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 3 | // Created 2006-01-24 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | 24 | using namespace warp; 25 | using namespace ex; 26 | 27 | //---------------------------------------------------------------------------- 28 | // raiseZlibError 29 | //---------------------------------------------------------------------------- 30 | void warp::raiseZlibError(int zErr) 31 | { 32 | switch(zErr) 33 | { 34 | case Z_NEED_DICT: raise("need dictionary"); 35 | case Z_DATA_ERROR: raise("data error"); 36 | case Z_MEM_ERROR: raise("memory error"); 37 | case Z_VERSION_ERROR: raise("version error: lib=%s, hdr=%s", 38 | ::zlibVersion(), ZLIB_VERSION); 39 | case Z_STREAM_ERROR: raise("stream error"); 40 | default: raise("unknown error: %d", zErr); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/cc/warp/hsieh_hash.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 3 | // Created 2006-08-04 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_HSIEH_HASH_H 23 | #define WARP_HSIEH_HASH_H 24 | 25 | #include 26 | #include 27 | 28 | namespace warp 29 | { 30 | uint32_t hsieh_hash(void const * data, size_t len, uint32_t hash); 31 | 32 | inline uint32_t hsieh_hash(void const * data, size_t len) 33 | { 34 | return hsieh_hash(data, len, uint32_t(len)); 35 | } 36 | 37 | inline uint32_t hsieh_hash(char const * begin, char const * end) 38 | { 39 | return hsieh_hash(begin, end - begin); 40 | } 41 | } 42 | 43 | #endif // WARP_HSIEH_HASH_H 44 | -------------------------------------------------------------------------------- /src/cc/warp/html_file_tokenizer.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-03-08 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_HTML_FILE_TOKENIZER_H 23 | #define WARP_HTML_FILE_TOKENIZER_H 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | namespace warp 31 | { 32 | class HtmlFileTokenizer; 33 | } 34 | 35 | //---------------------------------------------------------------------------- 36 | // HtmlFileTokenizer 37 | //---------------------------------------------------------------------------- 38 | class warp::HtmlFileTokenizer : public boost::noncopyable 39 | { 40 | HtmlTokenizer lastValid; 41 | HtmlTokenizer tok; 42 | 43 | FilePtr input; 44 | Buffer buf; 45 | 46 | size_t bytesIn; 47 | size_t bytesOut; 48 | bool eof; 49 | bool firstTag; 50 | 51 | public: 52 | explicit 53 | HtmlFileTokenizer(FilePtr const & input, size_t bufSize = 64 << 10); 54 | 55 | bool get(HtmlToken & x); 56 | 57 | size_t getBytesIn() const { return bytesIn; } 58 | size_t getBytesOut() const { return bytesOut; } 59 | }; 60 | 61 | #endif // WARP_HTML_FILE_TOKENIZER_H 62 | -------------------------------------------------------------------------------- /src/cc/warp/init.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 3 | // Created 2006-08-13 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | 24 | void warp::initModule() 25 | { 26 | static bool once = false; 27 | if(!once) 28 | { 29 | once = true; 30 | 31 | WARP_CALL_INIT(warp_dir); 32 | WARP_CALL_INIT(warp_file); 33 | WARP_CALL_INIT(warp_file_cache); 34 | WARP_CALL_INIT(warp_fs); 35 | WARP_CALL_INIT(warp_memfs); 36 | WARP_CALL_INIT(warp_mmapfile); 37 | WARP_CALL_INIT(warp_perflog); 38 | WARP_CALL_INIT(warp_perfrusage); 39 | WARP_CALL_INIT(warp_posixdir); 40 | WARP_CALL_INIT(warp_posixfs); 41 | WARP_CALL_INIT(warp_stdfile); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/cc/warp/log.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-03-11 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | using namespace warp; 28 | 29 | void warp::log(strref_t msg) 30 | { 31 | static boost::mutex mutex; 32 | boost::mutex::scoped_lock l(mutex); 33 | 34 | std::cerr << Timestamp::now() << ": " << msg << std::endl; 35 | } 36 | -------------------------------------------------------------------------------- /src/cc/warp/md5_unittest.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-05-29 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | #include 24 | 25 | using namespace warp; 26 | 27 | BOOST_AUTO_UNIT_TEST(hexdigest_test) 28 | { 29 | // Hashes taken from Python md5.new(x).hexdigest() 30 | BOOST_CHECK_EQUAL(md5HexDigest(""), "d41d8cd98f00b204e9800998ecf8427e"); 31 | BOOST_CHECK_EQUAL(md5HexDigest("hello"), "5d41402abc4b2a76b9719d911017c592"); 32 | BOOST_CHECK_EQUAL(md5HexDigest("some more text"), "4281c348eaf83e70ddce0e07221c3d28"); 33 | } 34 | -------------------------------------------------------------------------------- /src/cc/warp/module.mk: -------------------------------------------------------------------------------- 1 | MAGIC_MODULE_DEPS := ex 2 | MAGIC_EXTERNAL_DEPS := $(BOOST_PROGRAM_OPTIONS) $(BOOST_THREAD) z dl $(TCMALLOC) 3 | include magic.mk 4 | -------------------------------------------------------------------------------- /src/cc/warp/options_unittest.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-06-04 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | #include 24 | #include 25 | 26 | using namespace warp; 27 | using namespace std; 28 | 29 | BOOST_AUTO_UNIT_TEST(no_quote_stripping) 30 | { 31 | OptionParser op; 32 | 33 | op.addOption("bool,b", "bool option"); 34 | op.addOption("str,s", value(), "string option"); 35 | 36 | OptionMap opt; 37 | ArgumentList args; 38 | 39 | char const * av[] = { "prog", "-b", "-s", "'blah'", 0 }; 40 | int ac = sizeof(av) / sizeof(*av) - 1; 41 | 42 | op.parse(ac, av, opt, args); 43 | 44 | BOOST_CHECK(hasopt(opt, "bool")); 45 | 46 | string s; 47 | BOOST_CHECK(opt.get("str", s)); 48 | BOOST_CHECK_EQUAL(s, "'blah'"); 49 | } 50 | -------------------------------------------------------------------------------- /src/cc/warp/pack.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-06-07 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_PACK_H 23 | #define WARP_PACK_H 24 | 25 | #include 26 | 27 | /// Pack four bytes into an MSB ordered integer 28 | #define WARP_PACK4(b0, b1, b2, b3) ( ((uint32_t(b3) & 0xff) << 24) | \ 29 | ((uint32_t(b2) & 0xff) << 16) | \ 30 | ((uint32_t(b1) & 0xff) << 8) | \ 31 | ((uint32_t(b0) & 0xff) ) ) 32 | 33 | #endif // WARP_PACK_H 34 | -------------------------------------------------------------------------------- /src/cc/warp/parsing/module.mk: -------------------------------------------------------------------------------- 1 | MAGIC_MODULE_DEPS := warp 2 | include magic.mk 3 | -------------------------------------------------------------------------------- /src/cc/warp/parsing/numeric.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-03-21 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_PARSING_NUMERIC_H 23 | #define WARP_PARSING_NUMERIC_H 24 | 25 | #include 26 | #include 27 | 28 | namespace warp { 29 | namespace parsing { 30 | 31 | using boost::spirit::int_parser; 32 | using boost::spirit::uint_parser; 33 | 34 | int_parser const int64_p = int_parser(); 35 | uint_parser const uint2_p = uint_parser(); 36 | uint_parser const uint4_p = uint_parser(); 37 | 38 | } // namespace parsing 39 | } // namespace warp 40 | 41 | #endif // WARP_PARSING_NUMERIC_H 42 | -------------------------------------------------------------------------------- /src/cc/warp/pathmgr.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 3 | // Created 2006-08-13 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_PATHMGR_H 23 | #define WARP_PATHMGR_H 24 | 25 | #include 26 | #include 27 | 28 | namespace warp 29 | { 30 | class PathManager; 31 | 32 | typedef std::string (PathCompletionFunc)(std::string const & base, 33 | std::string const & path); 34 | 35 | extern std::string defaultPathCompletion(std::string const & base, 36 | std::string const & path); 37 | } 38 | 39 | //---------------------------------------------------------------------------- 40 | // PathManager 41 | //---------------------------------------------------------------------------- 42 | class warp::PathManager : public warp::Singleton 43 | { 44 | friend class Singleton; 45 | 46 | std::string basePath; 47 | PathCompletionFunc * pathComplete; 48 | 49 | PathManager(); 50 | 51 | public: 52 | std::string const & getBase() const { return basePath; } 53 | void setBase(std::string const & base, PathCompletionFunc * comp=0); 54 | std::string resolve(std::string const & userPath) const; 55 | }; 56 | 57 | 58 | #endif // WARP_PATHMGR_H 59 | -------------------------------------------------------------------------------- /src/cc/warp/perfitem.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-01-15 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | #include 24 | 25 | using namespace warp; 26 | 27 | 28 | //---------------------------------------------------------------------------- 29 | // PerformanceItem 30 | //---------------------------------------------------------------------------- 31 | PerformanceItem::PerformanceItem() : 32 | tracker(0), trackerIndex(0), inTeardown(false) 33 | { 34 | } 35 | 36 | PerformanceItem::~PerformanceItem() 37 | { 38 | try { 39 | inTeardown = true; 40 | if(tracker) 41 | tracker->removeItem(this); 42 | } 43 | catch(...) { 44 | // No exceptions... 45 | } 46 | } 47 | 48 | bool PerformanceItem::isTracked() const 49 | { 50 | return tracker != 0; 51 | } 52 | 53 | void PerformanceItem::removeSelf() 54 | { 55 | if(tracker) 56 | tracker->removeItem(this); 57 | } 58 | -------------------------------------------------------------------------------- /src/cc/warp/pid_file.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-08-23 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | using namespace warp; 30 | using namespace ex; 31 | 32 | PidFile::PidFile(std::string const & fn) : 33 | fn(fn) 34 | { 35 | char buf[64]; 36 | int sz = snprintf(buf, sizeof(buf), "%d\n", getpid()); 37 | if(sz < 0) 38 | raise("couldn't write PID into a buffer: %s", 39 | getStdError()); 40 | 41 | FilePtr fp = File::output(fn); 42 | fp->write(buf, sz); 43 | fp->close(); 44 | } 45 | 46 | PidFile::~PidFile() 47 | { 48 | fs::remove(fn); 49 | } 50 | -------------------------------------------------------------------------------- /src/cc/warp/pid_file.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-08-23 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_PID_FILE_H 23 | #define WARP_PID_FILE_H 24 | 25 | #include 26 | #include 27 | 28 | namespace warp 29 | { 30 | /// Write a file containing the process ID (PID) to a specified 31 | /// location. The file is removed when the object is destroyed. 32 | class PidFile; 33 | } 34 | 35 | //---------------------------------------------------------------------------- 36 | // PidFile 37 | //---------------------------------------------------------------------------- 38 | class warp::PidFile : private boost::noncopyable 39 | { 40 | std::string fn; 41 | 42 | public: 43 | explicit PidFile(std::string const & fn); 44 | ~PidFile(); 45 | 46 | std::string const & getName() const { return fn; } 47 | }; 48 | 49 | #endif // WARP_PID_FILE_H 50 | -------------------------------------------------------------------------------- /src/cc/warp/pid_file_unittest.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-08-23 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | using namespace warp; 31 | 32 | BOOST_AUTO_UNIT_TEST(basic_test) 33 | { 34 | std::string const fn("memfs:/pid"); 35 | 36 | // PID file doesn't already exist 37 | BOOST_CHECK(!fs::exists(fn)); 38 | 39 | // Create a PID file 40 | { 41 | PidFile pf(fn); 42 | 43 | // PID file was created 44 | BOOST_CHECK(fs::exists(fn)); 45 | 46 | // Make sure it has our PID 47 | FilePtr fp = File::input(fn); 48 | char buf[256]; 49 | size_t sz = fp->read(buf, sizeof(buf)); 50 | 51 | // Chop newline 52 | while(sz && buf[sz-1] == '\n') 53 | --sz; 54 | 55 | // Parse PID 56 | pid_t p(-1); 57 | BOOST_CHECK(parseInt(p, buf, buf+sz)); 58 | 59 | // Make sure it agrees with what it should be 60 | BOOST_CHECK_EQUAL(getpid(), p); 61 | } 62 | 63 | // Should be gone again 64 | BOOST_CHECK(!fs::exists(fn)); 65 | } 66 | -------------------------------------------------------------------------------- /src/cc/warp/posixdir.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 3 | // Created 2006-09-20 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_POSIXDIR_H 23 | #define WARP_POSIXDIR_H 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | namespace warp 31 | { 32 | class PosixDirectory; 33 | } 34 | 35 | //---------------------------------------------------------------------------- 36 | // PosixDirectory 37 | //---------------------------------------------------------------------------- 38 | class warp::PosixDirectory : public warp::Directory, public boost::noncopyable 39 | { 40 | private: 41 | DIR * dir; 42 | std::string name; 43 | 44 | public: 45 | PosixDirectory(std::string const & uri); 46 | virtual ~PosixDirectory(); 47 | 48 | // Directory interface 49 | virtual void close(); 50 | virtual bool read(std::string & dst); 51 | virtual std::string getPath() const; 52 | }; 53 | 54 | #endif // WARP_POSIXDIR_H 55 | -------------------------------------------------------------------------------- /src/cc/warp/prime.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-02-24 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_PRIME_H 23 | #define WARP_PRIME_H 24 | 25 | #include 26 | 27 | namespace warp 28 | { 29 | /// Probabilistically check \c n for primality using the 30 | /// Miller-Rabin test. Do up to \c k iterations before declaring 31 | /// primality. The probability of a composite number being 32 | /// classified as prime is at most $4^{-k}$. 33 | /// @fix This implementation usually thinks values of \c n over 34 | /// 2^32 are composite due to multiplication overflow problems. 35 | bool isPrime(size_t n, size_t k=20); 36 | 37 | /// Test each odd number greater than or equal to \c n until a 38 | /// probable prime is found. The probability of a composite 39 | /// number being classified as prime is at most $4^{-k}$. 40 | /// @fix This implementation usually thinks values of \c n over 41 | /// 2^32 are composite due to multiplication overflow problems. 42 | size_t nextPrime(size_t n, size_t k=20); 43 | } 44 | 45 | #endif // WARP_PRIME_H 46 | -------------------------------------------------------------------------------- /src/cc/warp/signal_blocker.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-08-23 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_SIGNAL_BLOCKER_H 23 | #define WARP_SIGNAL_BLOCKER_H 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | namespace warp 30 | { 31 | class SignalBlocker : private boost::noncopyable 32 | { 33 | sigset_t old_mask; 34 | 35 | public: 36 | SignalBlocker() 37 | { 38 | sigset_t new_mask; 39 | sigfillset(&new_mask); 40 | if(int err = pthread_sigmask(SIG_BLOCK, &new_mask, &old_mask)) 41 | { 42 | using namespace ex; 43 | raise("failed to block signals: %s", 44 | getStdError(err)); 45 | } 46 | } 47 | 48 | ~SignalBlocker() 49 | { 50 | pthread_sigmask(SIG_SETMASK, &old_mask, 0); 51 | } 52 | }; 53 | } 54 | 55 | #endif // WARP_SIGNAL_BLOCKER_H 56 | -------------------------------------------------------------------------------- /src/cc/warp/singleton.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 3 | // Created 2006-02-21 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_SINGLETON_H 23 | #define WARP_SINGLETON_H 24 | 25 | namespace warp 26 | { 27 | template 28 | class Singleton; 29 | } 30 | 31 | //---------------------------------------------------------------------------- 32 | // Singleton 33 | //---------------------------------------------------------------------------- 34 | template 35 | class warp::Singleton 36 | { 37 | protected: 38 | typedef Singleton friend_t; 39 | Singleton() {} 40 | private: 41 | Singleton(Singleton const & o); 42 | Singleton const & operator=(Singleton const & o); 43 | public: 44 | static DerivedType & get() 45 | { 46 | static DerivedType theOne; 47 | return theOne; 48 | } 49 | }; 50 | 51 | 52 | #endif // WARP_SINGLETON_H 53 | -------------------------------------------------------------------------------- /src/cc/warp/stdfile.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 3 | // Created 2006-01-11 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_STDFILE_H 23 | #define WARP_STDFILE_H 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | namespace warp 30 | { 31 | class StdFile; 32 | } 33 | 34 | 35 | //---------------------------------------------------------------------------- 36 | // StdFile 37 | //---------------------------------------------------------------------------- 38 | /// A File wrapper over the C stdio library. 39 | class warp::StdFile : public File, public boost::noncopyable 40 | { 41 | FILE * fp; 42 | std::string fn; 43 | 44 | public: 45 | StdFile(std::string const & uri, char const * mode); 46 | StdFile(int fd, std::string const & name, char const * mode); 47 | ~StdFile(); 48 | 49 | // File interface 50 | size_t read(void * dst, size_t elemSz, size_t nElem); 51 | size_t readline(char * dst, size_t sz, char delim); 52 | size_t write(void const * src, size_t elemSz, size_t nElem); 53 | void flush(); 54 | void close(); 55 | off_t tell() const; 56 | void seek(off_t offset, int whence); 57 | std::string getName() const; 58 | }; 59 | 60 | 61 | #endif // WARP_STDFILE_H 62 | -------------------------------------------------------------------------------- /src/cc/warp/string_interval.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-12-17 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_STRING_INTERVAL_H 23 | #define WARP_STRING_INTERVAL_H 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | namespace warp { 30 | 31 | /// Get the next string in a lexicographical ordering that is no 32 | /// longer than the given prefix. 33 | /// @returns true if such a string exists 34 | bool getPrefixSuccessor(strref_t prefix, std::string & result); 35 | 36 | /// Return an interval that contains all the strings that start 37 | /// with the given prefix. 38 | Interval makePrefixInterval(strref_t prefix); 39 | 40 | } // namespace warp 41 | 42 | #endif // WARP_STRING_INTERVAL_H 43 | -------------------------------------------------------------------------------- /src/cc/warp/sysinfo.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2009 Josh Taylor (Kosmix Corporation) 3 | // Created 2009-04-01 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_SYSINFO_H 23 | #define WARP_SYSINFO_H 24 | 25 | #include 26 | 27 | namespace warp { 28 | 29 | struct PageCounts 30 | { 31 | size_t total; 32 | size_t resident; 33 | size_t shared; 34 | size_t text; 35 | size_t lib; 36 | size_t data; 37 | size_t dirty; 38 | }; 39 | 40 | size_t getPageSize(); 41 | bool readPageCounts(PageCounts * counts); 42 | 43 | } // namespace warp 44 | 45 | #endif // WARP_SYSINFO_H 46 | -------------------------------------------------------------------------------- /src/cc/warp/tmpfile.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 3 | // Created 2006-04-03 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_TMPFILE_H 23 | #define WARP_TMPFILE_H 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | namespace warp 30 | { 31 | //------------------------------------------------------------------------ 32 | // TmpFileGen 33 | //------------------------------------------------------------------------ 34 | class TmpFileGen : public Generator 35 | { 36 | std::string rootDir; 37 | bool madeDir; 38 | 39 | public: 40 | TmpFileGen(std::string const & rootDir); 41 | FilePtr next(); 42 | }; 43 | 44 | /// Open a unique temporary file in the given directory. The 45 | /// filename will be determined automatically, but the prefix can 46 | /// be specified. The directory must reside on a local 47 | /// filesystem. 48 | FilePtr openTmpFile(std::string const & dir, std::string const & prefix); 49 | } 50 | 51 | #endif // WARP_TMPFILE_H 52 | -------------------------------------------------------------------------------- /src/cc/warp/tuple.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-03-17 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_TUPLE_H 23 | #define WARP_TUPLE_H 24 | 25 | #include 26 | 27 | namespace warp { 28 | 29 | using boost::tuple; 30 | using boost::make_tuple; 31 | using boost::tie; 32 | 33 | typedef boost::tuples::null_type null_type; 34 | 35 | } // namespace warp 36 | 37 | #endif // WARP_TUPLE_H 38 | -------------------------------------------------------------------------------- /src/cc/warp/varsub.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-06-14 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_VARSUB_H 23 | #define WARP_VARSUB_H 24 | 25 | #include 26 | #include 27 | 28 | namespace warp 29 | { 30 | /// Substitute variables of the form "$KEY", "$(KEY)", "${KEY}", 31 | /// or "$[KEY]" with the value associated with "KEY" in the given 32 | /// variable map. If the value is not found, substitute an empty 33 | /// string. Literal dollar signs can be including using "$$". 34 | /// Also, if a single "$" appears with an invalid variable 35 | /// specification, the literal "$" will be retained. The 36 | /// delimited forms of variable naming may contain any characters 37 | /// except for the end delimiter. The non-delimited form must be 38 | /// a valid identifier: an alphabetic character followed by zero 39 | /// or more alpha-numeric characters. 40 | std::string varsub(std::string const & pattern, 41 | std::map const & vars); 42 | } 43 | 44 | #endif // WARP_VARSUB_H 45 | -------------------------------------------------------------------------------- /src/cc/warp/vec2f.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-02-23 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_VEC2F_H 23 | #define WARP_VEC2F_H 24 | 25 | namespace warp 26 | { 27 | /// Two-dimensional vector (in the linear algebra sense) of floats. 28 | class Vec2f; 29 | } 30 | 31 | //---------------------------------------------------------------------------- 32 | // Vec2f 33 | //---------------------------------------------------------------------------- 34 | class warp::Vec2f 35 | { 36 | float d[2]; 37 | 38 | public: 39 | Vec2f() {} 40 | Vec2f(float x, float y) { d[0] = x; d[1] = y; } 41 | 42 | float & operator[](int idx) { return d[idx]; } 43 | float const operator[](int idx) const { return d[idx]; } 44 | }; 45 | 46 | //---------------------------------------------------------------------------- 47 | // Vec2f operators 48 | //---------------------------------------------------------------------------- 49 | namespace warp 50 | { 51 | // Obviously, these operators are far from complete... being lazy. 52 | 53 | inline Vec2f operator-(Vec2f const & a, Vec2f const & b) 54 | { 55 | return Vec2f(b[0]-a[0], b[1]-a[1]); 56 | } 57 | } 58 | 59 | 60 | #endif // WARP_VEC2F_H 61 | -------------------------------------------------------------------------------- /src/cc/warp/vintreader.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 3 | // Created 2006-08-24 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | #include 24 | 25 | using namespace warp; 26 | using namespace ex; 27 | 28 | //---------------------------------------------------------------------------- 29 | // VIntReader 30 | //---------------------------------------------------------------------------- 31 | VIntReader::VIntReader(FilePtr const & fp) : 32 | fp(fp), buf(BUF_SIZE) 33 | { 34 | if(!fp) 35 | raise("null file"); 36 | 37 | buf.flip(); 38 | } 39 | 40 | void VIntReader::seek(off_t pos) 41 | { 42 | // Try to seek within buffered region 43 | off_t posMax = fp->tell(); 44 | off_t posMin = posMax - (buf.limit() - buf.begin()); 45 | if(pos >= posMin && pos <= posMax) 46 | { 47 | // Within buffered region 48 | buf.position(buf.begin() + (pos - posMin)); 49 | } 50 | else 51 | { 52 | // Seek is outside of buffered region 53 | fp->seek(pos); 54 | buf.clear(); 55 | buf.flip(); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/cc/warp/vintreader.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 3 | // Created 2006-08-24 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_VINTREADER_H 23 | #define WARP_VINTREADER_H 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | namespace warp 31 | { 32 | class VIntReader; 33 | } 34 | 35 | //---------------------------------------------------------------------------- 36 | // VIntReader 37 | //---------------------------------------------------------------------------- 38 | class warp::VIntReader : public boost::noncopyable 39 | { 40 | enum { BUF_SIZE = 32 << 10 }; 41 | 42 | FilePtr fp; 43 | Buffer buf; 44 | 45 | public: 46 | VIntReader(FilePtr const & fp); 47 | 48 | template 49 | bool getVInt(Int & x) 50 | { 51 | // Optimistic get 52 | if(warp::getVInt(buf, x)) 53 | return true; 54 | 55 | // That didn't work; refill buffer 56 | buf.compact(); 57 | buf.read(fp); 58 | buf.flip(); 59 | return warp::getVInt(buf, x); 60 | } 61 | 62 | void seek(off_t pos); 63 | 64 | off_t tell() const 65 | { 66 | return fp->tell() - buf.remaining(); 67 | } 68 | }; 69 | 70 | #endif // WARP_VINTREADER_H 71 | -------------------------------------------------------------------------------- /src/cc/warp/vintwriter.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 3 | // Created 2006-08-24 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | #include 24 | 25 | using namespace warp; 26 | using namespace ex; 27 | 28 | //---------------------------------------------------------------------------- 29 | // VIntWriter 30 | //---------------------------------------------------------------------------- 31 | void VIntWriter::spill() 32 | { 33 | buf.flip(); 34 | buf.write(fp); 35 | if(buf.remaining()) 36 | raise("failed to spill buffer to file: %s", 37 | fp->getName()); 38 | buf.clear(); 39 | } 40 | 41 | VIntWriter::VIntWriter(FilePtr const & fp) : 42 | fp(fp), buf(BUF_SIZE) 43 | { 44 | if(!fp) 45 | raise("null file"); 46 | } 47 | 48 | VIntWriter::~VIntWriter() 49 | { 50 | spill(); 51 | } 52 | 53 | void VIntWriter::flush() 54 | { 55 | spill(); 56 | } 57 | -------------------------------------------------------------------------------- /src/cc/warp/xml/module.mk: -------------------------------------------------------------------------------- 1 | MAGIC_MODULE_DEPS := warp ex 2 | MAGIC_EXTERNAL_DEPS := expat 3 | include magic.mk 4 | -------------------------------------------------------------------------------- /src/cc/warp/xml/xml_parser.h: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 3 | // Created 2007-12-20 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #ifndef WARP_XML_XML_PARSER_H 23 | #define WARP_XML_XML_PARSER_H 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | 31 | namespace warp { 32 | namespace xml { 33 | 34 | /// A basic XML parser wrapped over the Expat library. 35 | class XmlParser; 36 | 37 | EX_DECLARE_EXCEPTION(XmlError, ex::RuntimeError); 38 | 39 | } // namespace xml 40 | } // namespace warp 41 | 42 | //---------------------------------------------------------------------------- 43 | // XmlParser 44 | //---------------------------------------------------------------------------- 45 | class warp::xml::XmlParser 46 | { 47 | public: 48 | typedef HashMap AttrMap; 49 | 50 | public: 51 | XmlParser(); 52 | virtual ~XmlParser(); 53 | 54 | void parse(FilePtr const & f, bool clean=false); 55 | 56 | virtual void startElement(strref_t name, AttrMap const & attrs) {} 57 | virtual void endElement(strref_t name) {} 58 | virtual void characterData(strref_t data) {} 59 | }; 60 | 61 | 62 | #endif // WARP_XML_XML_PARSER_H 63 | -------------------------------------------------------------------------------- /src/cc/warp/zero_escape.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-03-17 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | #include 24 | 25 | using namespace warp; 26 | 27 | std::string warp::zeroEscape(strref_t s) 28 | { 29 | std::ostringstream oss; 30 | oss << ZeroEscape(s); 31 | return oss.str(); 32 | } 33 | 34 | std::string warp::zeroUnescape(strref_t s) 35 | { 36 | std::ostringstream oss; 37 | oss << ZeroUnescape(s); 38 | return oss.str(); 39 | } 40 | -------------------------------------------------------------------------------- /src/cc/warp/zero_escape_unittest.cc: -------------------------------------------------------------------------------- 1 | //---------------------------------------------------------- -*- Mode: C++ -*- 2 | // Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 3 | // Created 2008-03-17 4 | // 5 | // This file is part of the warp library. 6 | // 7 | // The warp library is free software; you can redistribute it and/or modify 8 | // it under the terms of the GNU General Public License as published by the 9 | // Free Software Foundation; either version 2 of the License, or any later 10 | // version. 11 | // 12 | // The warp library is distributed in the hope that it will be useful, but 13 | // WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 15 | // Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License along 18 | // with this program; if not, write to the Free Software Foundation, Inc., 19 | // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | //---------------------------------------------------------------------------- 21 | 22 | #include 23 | #include 24 | 25 | using namespace warp; 26 | using namespace ex; 27 | 28 | BOOST_AUTO_UNIT_TEST(basic) 29 | { 30 | // These test strings are strangely formatted because GCC has some 31 | // serious problems with embedded escapes in string literals. 32 | 33 | BOOST_CHECK_EQUAL(zeroEscape(""), ""); 34 | BOOST_CHECK_EQUAL(zeroEscape("foo"), "foo"); 35 | BOOST_CHECK_EQUAL(zeroEscape(binary_data("\0" "foo", 4)), 36 | binary_data("\0" "\x01" "foo", 5)); 37 | BOOST_CHECK_EQUAL(zeroEscape(binary_data("\0" "foo" "\0" "\0" "\0" "\0" "\0", 9)), 38 | binary_data("\0" "\x01" "foo" "\0" "\x05", 7)); 39 | 40 | BOOST_CHECK_EQUAL(zeroUnescape(""), ""); 41 | BOOST_CHECK_EQUAL(zeroUnescape("foo"), "foo"); 42 | BOOST_CHECK_EQUAL(zeroUnescape(binary_data("\0" "\x01" "foo", 5)), 43 | binary_data("\0" "foo", 4)); 44 | BOOST_CHECK_EQUAL(zeroUnescape(binary_data("\0" "\x01" "foo" "\0" "\x05", 7)), 45 | binary_data("\0" "foo" "\0" "\0" "\0" "\0" "\0", 9)); 46 | 47 | BOOST_CHECK_THROW(zeroUnescape(binary_data("foo" "\0", 4)), ValueError); 48 | } 49 | -------------------------------------------------------------------------------- /src/python/alien/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 4 | # Created 2006-03-29 5 | # 6 | # This file is part of Alien. 7 | # 8 | # Alien is free software; you can redistribute it and/or modify it under the 9 | # terms of the GNU General Public License as published by the Free Software 10 | # Foundation; either version 2 of the License, or any later version. 11 | # 12 | # Alien is distributed in the hope that it will be useful, but WITHOUT ANY 13 | # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 15 | # details. 16 | # 17 | # You should have received a copy of the GNU General Public License along 18 | # with this program; if not, write to the Free Software Foundation, Inc., 51 19 | # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | 21 | __all__ = ['array', 'gzdata', 'octal', 'offset', 'prim', 'raw', 'tarheader', 22 | 'timestamp', 'vint', 'vstruct', 'tools', 'kosmix' ] 23 | -------------------------------------------------------------------------------- /src/python/alien/common.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 4 | # Created 2006-03-29 5 | # 6 | # This file is part of Alien. 7 | # 8 | # Alien is free software; you can redistribute it and/or modify it under the 9 | # terms of the GNU General Public License as published by the Free Software 10 | # Foundation; either version 2 of the License, or any later version. 11 | # 12 | # Alien is distributed in the hope that it will be useful, but WITHOUT ANY 13 | # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 15 | # details. 16 | # 17 | # You should have received a copy of the GNU General Public License along 18 | # with this program; if not, write to the Free Software Foundation, Inc., 51 19 | # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | 21 | from alien.prim import * 22 | from alien.raw import RawData, HexData 23 | from alien.vint import VInt, VLong 24 | from alien.vstruct import StructBase 25 | from alien.array import Array, FixedArray 26 | from alien.offset import Offset, ArrayOffset 27 | from alien.tools import sizeof 28 | -------------------------------------------------------------------------------- /src/python/alien/kosmix/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 4 | # Created 2006-03-29 5 | # 6 | # This file is part of Alien. 7 | # 8 | # Alien is free software; you can redistribute it and/or modify it under the 9 | # terms of the GNU General Public License as published by the Free Software 10 | # Foundation; either version 2 of the License, or any later version. 11 | # 12 | # Alien is distributed in the hope that it will be useful, but WITHOUT ANY 13 | # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 15 | # details. 16 | # 17 | # You should have received a copy of the GNU General Public License along 18 | # with this program; if not, write to the Free Software Foundation, Inc., 51 19 | # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | 21 | __all__ = [ 22 | 'btree', 23 | 'kdi', 24 | 'record', 25 | 'stringdata', 26 | ] 27 | -------------------------------------------------------------------------------- /src/python/alien/kosmix/stringdata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright (C) 2007 Josh Taylor (Kosmix Corporation) 4 | # Created 2007-10-20 5 | # 6 | # This file is part of Alien. 7 | # 8 | # Alien is free software; you can redistribute it and/or modify it under the 9 | # terms of the GNU General Public License as published by the Free Software 10 | # Foundation; either version 2 of the License, or any later version. 11 | # 12 | # Alien is distributed in the hope that it will be useful, but WITHOUT ANY 13 | # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 15 | # details. 16 | # 17 | # You should have received a copy of the GNU General Public License along 18 | # with this program; if not, write to the Free Software Foundation, Inc., 51 19 | # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | 21 | from alien.common import * 22 | from alien.gzdata import GZippedData 23 | from util.xmlfmt import xml 24 | 25 | __all__ = dir() 26 | 27 | class StringData(StructBase): 28 | __nobrowse__ = 1 29 | __format__ = { 30 | 0 : [ 31 | ('len', LsbInt), 32 | ('data', RawData(1)), 33 | ], 34 | } 35 | def __get_data(self): 36 | rtype = RawData(self.len) 37 | return rtype(self._buf, self._base + self._offsetof('data')) 38 | data = property(__get_data) 39 | 40 | def _sizeof(cls, ver=None): 41 | return lambda x: LsbInt._sizeof() + x.len 42 | _sizeof = classmethod(_sizeof) 43 | 44 | def __str__(self): 45 | return str(self.data) 46 | 47 | def __xml__(self): 48 | return xml(self.data) 49 | 50 | class GzStringData(GZippedData(RawData(None), None, LsbInt)): 51 | def __str__(self): 52 | return str(self.unzipped) 53 | 54 | class StringOffset(Offset(StringData)): 55 | __nobrowse__ = 1 56 | def __str__(self): 57 | return str(self.target) 58 | 59 | class GzStringOffset(Offset(GzStringData)): 60 | __nobrowse__ = 1 61 | def __str__(self): 62 | return str(self.target) 63 | 64 | 65 | # Export magic... 66 | __all__ = [_x for _x in dir() if _x not in __all__ and not _x.startswith('_')] 67 | -------------------------------------------------------------------------------- /src/python/alien/octal.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 4 | # Created 2006-03-29 5 | # 6 | # This file is part of Alien. 7 | # 8 | # Alien is free software; you can redistribute it and/or modify it under the 9 | # terms of the GNU General Public License as published by the Free Software 10 | # Foundation; either version 2 of the License, or any later version. 11 | # 12 | # Alien is distributed in the hope that it will be useful, but WITHOUT ANY 13 | # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 15 | # details. 16 | # 17 | # You should have received a copy of the GNU General Public License along 18 | # with this program; if not, write to the Free Software Foundation, Inc., 51 19 | # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | 21 | 22 | #---------------------------------------------------------------------------- 23 | # OctalNumber 24 | #---------------------------------------------------------------------------- 25 | def OctalNumber(sz): 26 | class OctalNumber(object): 27 | __size__ = sz 28 | 29 | def __new__(cls, buf, offset=0, ver=None): 30 | x = buf[offset : offset + cls.__size__] 31 | if len(x) > 0 and x[0] == '\x00': 32 | return 0 33 | else: 34 | return int(x, 8) 35 | 36 | def _sizeof(cls, ver=None): 37 | return cls.__size__ 38 | _sizeof = classmethod(_sizeof) 39 | 40 | return OctalNumber 41 | -------------------------------------------------------------------------------- /src/python/alien/tarheader.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 4 | # Created 2006-03-29 5 | # 6 | # This file is part of Alien. 7 | # 8 | # Alien is free software; you can redistribute it and/or modify it under the 9 | # terms of the GNU General Public License as published by the Free Software 10 | # Foundation; either version 2 of the License, or any later version. 11 | # 12 | # Alien is distributed in the hope that it will be useful, but WITHOUT ANY 13 | # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 15 | # details. 16 | # 17 | # You should have received a copy of the GNU General Public License along 18 | # with this program; if not, write to the Free Software Foundation, Inc., 51 19 | # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | 21 | from alien.vstruct import StructBase 22 | from alien.raw import RawData 23 | from alien.octal import OctalNumber 24 | from alien.timestamp import Timestamp 25 | 26 | #---------------------------------------------------------------------------- 27 | # TarHeader 28 | #---------------------------------------------------------------------------- 29 | class TarHeader(StructBase): 30 | __format__ = { 31 | 0 : [ 32 | ('fileName', RawData(100)), 33 | ('fileMode', RawData(8)), 34 | ('ownerId', OctalNumber(8)), 35 | ('groupId', OctalNumber(8)), 36 | ('fileSize', OctalNumber(12)), 37 | ('modTime', Timestamp(OctalNumber(12))), 38 | ('checksum', RawData(8)), 39 | ('typeFlag', RawData(1)), 40 | ('linkTarget', RawData(100)), 41 | ('ustarMagic', RawData(6)), 42 | ('ustarVersion', RawData(2)), 43 | ('ownerName', RawData(32)), 44 | ('groupName', RawData(32)), 45 | ('deviceMajor', OctalNumber(8)), 46 | ('deviceMinor', OctalNumber(8)), 47 | ('filePrefix', RawData(155)), 48 | ('pad', RawData(12)), 49 | ], 50 | } 51 | 52 | 53 | __all__ = ['TarHeader'] 54 | -------------------------------------------------------------------------------- /src/python/alien/timestamp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 4 | # Created 2006-03-29 5 | # 6 | # This file is part of Alien. 7 | # 8 | # Alien is free software; you can redistribute it and/or modify it under the 9 | # terms of the GNU General Public License as published by the Free Software 10 | # Foundation; either version 2 of the License, or any later version. 11 | # 12 | # Alien is distributed in the hope that it will be useful, but WITHOUT ANY 13 | # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 15 | # details. 16 | # 17 | # You should have received a copy of the GNU General Public License along 18 | # with this program; if not, write to the Free Software Foundation, Inc., 51 19 | # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | 21 | import time 22 | 23 | #---------------------------------------------------------------------------- 24 | # Timestamp 25 | #---------------------------------------------------------------------------- 26 | def Timestamp(dtype, dver=None, scaleToSeconds=1.0): 27 | 28 | class Timestamp(object): 29 | def __init__(self, buf, offset=0, ver=None): 30 | self._val = dtype(buf, offset, dver) 31 | 32 | def __str__(self): 33 | return time.ctime(self._val * scaleToSeconds).strip() 34 | 35 | def __repr__(self): 36 | return repr(self._val) 37 | 38 | def _sizeof(cls, ver=None): 39 | return dtype._sizeof(dver) 40 | _sizeof = classmethod(_sizeof) 41 | 42 | def __getRaw(self): 43 | return self._val 44 | raw = property(__getRaw) 45 | 46 | return Timestamp 47 | 48 | #---------------------------------------------------------------------------- 49 | # MsTimestamp 50 | #---------------------------------------------------------------------------- 51 | def MsTimestamp(dtype, dver=None): 52 | return Timestamp(dtype, dver, scaleToSeconds=1e-3) 53 | 54 | #---------------------------------------------------------------------------- 55 | # NanoTimestamp 56 | #---------------------------------------------------------------------------- 57 | def NanoTimestamp(dtype, dver=None): 58 | return Timestamp(dtype, dver, scaleToSeconds=1e-9) 59 | -------------------------------------------------------------------------------- /src/python/kdi/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 4 | # 5 | # This file is part of KDI. 6 | # 7 | # KDI is free software; you can redistribute it and/or modify it under the 8 | # terms of the GNU General Public License as published by the Free Software 9 | # Foundation; either version 2 of the License, or any later version. 10 | # 11 | # KDI is distributed in the hope that it will be useful, but WITHOUT ANY 12 | # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 14 | # details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 51 18 | # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | -------------------------------------------------------------------------------- /src/python/kdi/updateMetaLocation.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 4 | # Created 2008-08-19 5 | # 6 | # This file is part of KDI. 7 | # 8 | # KDI is free software; you can redistribute it and/or modify it under the 9 | # terms of the GNU General Public License as published by the Free Software 10 | # Foundation; either version 2 of the License, or any later version. 11 | # 12 | # KDI is distributed in the hope that it will be useful, but WITHOUT ANY 13 | # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 15 | # details. 16 | # 17 | # You should have received a copy of the GNU General Public License along 18 | # with this program; if not, write to the Free Software Foundation, Inc., 51 19 | # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 20 | 21 | def updateLocations(meta, port): 22 | import pykdi 23 | from util.props import Properties 24 | from cStringIO import StringIO 25 | from util.zero import zeroDecode 26 | 27 | # Open META table 28 | metaTable = pykdi.Table(meta) 29 | 30 | # Go through 'config' cells, extract 'server' variable, and write 31 | # 'location' cells 32 | for row,col,time,val in metaTable.scan('column = "config"'): 33 | table,dummy,last = zeroDecode(row) 34 | cfg = Properties(StringIO(val)) 35 | loc = 'kdi://%s:%s/%s' % (cfg.get('server'), port, table) 36 | metaTable.set(row, 'location', time, loc) 37 | 38 | # Sync changes 39 | metaTable.sync() 40 | 41 | 42 | def main(): 43 | import optparse 44 | op = optparse.OptionParser() 45 | op.add_option('-m','--meta',help='location of META table') 46 | op.add_option('-p','--port',help='port to use for all locations') 47 | opt,args = op.parse_args() 48 | 49 | if not opt.meta: 50 | op.error('need --meta') 51 | if not opt.port: 52 | op.error('need --port') 53 | 54 | updateLocations(opt.meta, opt.port) 55 | 56 | if __name__ == '__main__': 57 | main() 58 | -------------------------------------------------------------------------------- /src/python/util/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 4 | # Created 2006-03-29 5 | # 6 | # This file is part of the util library. 7 | # 8 | # The util library is free software; you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License as published by the 10 | # Free Software Foundation; either version 2 of the License, or any later 11 | # version. 12 | # 13 | # The util library is distributed in the hope that it will be useful, but 14 | # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 15 | # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 16 | # for more details. 17 | # 18 | # You should have received a copy of the GNU General Public License along 19 | # with this program; if not, write to the Free Software Foundation, Inc., 51 20 | # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 21 | 22 | -------------------------------------------------------------------------------- /src/python/util/bag.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright (C) 2005 Josh Taylor (Kosmix Corporation) 4 | # Created 2005-10-05 5 | # 6 | # This file is part of the util library. 7 | # 8 | # The util library is free software; you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License as published by the 10 | # Free Software Foundation; either version 2 of the License, or any later 11 | # version. 12 | # 13 | # The util library is distributed in the hope that it will be useful, but 14 | # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 15 | # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 16 | # for more details. 17 | # 18 | # You should have received a copy of the GNU General Public License along 19 | # with this program; if not, write to the Free Software Foundation, Inc., 51 20 | # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 21 | 22 | class Bag(object): 23 | def __init__(self, **kw): 24 | self.__dict__.update(kw) 25 | def __getitem__(self, k): 26 | return self.__dict__[k] 27 | def __setitem__(self, k, v): 28 | self.__dict__[k] = v 29 | def __iter__(self): 30 | return self.__dict__.iteritems() 31 | def __repr__(self): 32 | return 'Bag('+','.join(['%s=%r'%(k,v) for k,v in self if not k.startswith('_')])+')' 33 | -------------------------------------------------------------------------------- /src/python/util/config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 4 | # Created 2006-03-29 5 | # 6 | # This file is part of the util library. 7 | # 8 | # The util library is free software; you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License as published by the 10 | # Free Software Foundation; either version 2 of the License, or any later 11 | # version. 12 | # 13 | # The util library is distributed in the hope that it will be useful, but 14 | # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 15 | # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 16 | # for more details. 17 | # 18 | # You should have received a copy of the GNU General Public License along 19 | # with this program; if not, write to the Free Software Foundation, Inc., 51 20 | # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 21 | 22 | 23 | #---------------------------------------------------------------------------- 24 | # Config 25 | #---------------------------------------------------------------------------- 26 | class Config(object): 27 | def __init__(self, fn, **kwDef): 28 | self.fn = fn 29 | self.sym = {} 30 | self.sym.update(kwDef) 31 | execfile(fn, self.sym) 32 | 33 | def __getattr__(self, k): 34 | try: 35 | return self.sym[k] 36 | except KeyError, ex: 37 | raise AttributeError('Config %r has no key %r' % (self.fn, k)) 38 | 39 | def __iter__(self): 40 | return self.sym.iteritems() 41 | 42 | def __str__(self): 43 | return '\n'.join(['%s = %r' % (k,v) for k,v in self]) 44 | -------------------------------------------------------------------------------- /src/python/util/sample.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright (C) 2008 Josh Taylor (Kosmix Corporation) 4 | # Created 2008-06-15 5 | # 6 | # This file is part of the util library. 7 | # 8 | # The util library is free software; you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License as published by the 10 | # Free Software Foundation; either version 2 of the License, or any later 11 | # version. 12 | # 13 | # The util library is distributed in the hope that it will be useful, but 14 | # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 15 | # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 16 | # for more details. 17 | # 18 | # You should have received a copy of the GNU General Public License along 19 | # with this program; if not, write to the Free Software Foundation, Inc., 51 20 | # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 21 | 22 | def getLine(fp, pos): 23 | fp.seek(pos) 24 | if pos: 25 | fp.readline() 26 | return fp.readline() 27 | 28 | def sampleLines(fn, num): 29 | from random import randrange 30 | f = file(fn) 31 | try: 32 | f.seek(0, 2) 33 | sz = f.tell() 34 | while num > 0: 35 | pos = randrange(0, sz) 36 | line = getLine(f, pos) 37 | if line: 38 | yield line 39 | num -= 1 40 | except: 41 | f.close() 42 | raise 43 | 44 | 45 | def main(): 46 | import optparse 47 | op = optparse.OptionParser() 48 | op.add_option('-n','--num',type='int',help='samples per file') 49 | op.add_option('-o','--output',help='output file') 50 | opt,args = op.parse_args() 51 | 52 | num = opt.num 53 | if not num: 54 | num = 1 55 | 56 | if opt.output: 57 | out = file(opt.output, 'w') 58 | else: 59 | import sys 60 | out = sys.stdout 61 | 62 | for fn in args: 63 | for line in sampleLines(fn, num): 64 | out.write(line) 65 | if not line.endswith('\n'): 66 | out.write('\n') 67 | 68 | if __name__ == '__main__': 69 | main() 70 | -------------------------------------------------------------------------------- /src/python/util/term.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Copyright (C) 2006 Josh Taylor (Kosmix Corporation) 4 | # Created 2006-08-02 5 | # 6 | # This file is part of the util library. 7 | # 8 | # The util library is free software; you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License as published by the 10 | # Free Software Foundation; either version 2 of the License, or any later 11 | # version. 12 | # 13 | # The util library is distributed in the hope that it will be useful, but 14 | # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 15 | # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 16 | # for more details. 17 | # 18 | # You should have received a copy of the GNU General Public License along 19 | # with this program; if not, write to the Free Software Foundation, Inc., 51 20 | # Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 21 | 22 | import os 23 | 24 | def ioctl_GWINSZ(fd): 25 | try: 26 | ### Discover terminal width 27 | import fcntl, termios, struct, os 28 | cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) 29 | except: 30 | return None 31 | return cr 32 | 33 | def dimensions(): 34 | """dimensions() -> (columns, rows) 35 | 36 | Get dimensions of current terminal. Defaults to (80,25) if it 37 | cannot be determined from the system environment. 38 | """ 39 | ### decide on *some* terminal size 40 | # try open fds 41 | cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) 42 | if not cr: 43 | # ...then ctty 44 | try: 45 | fd = os.open(os.ctermid(), os.O_RDONLY) 46 | cr = ioctl_GWINSZ(fd) 47 | os.close(fd) 48 | except: 49 | pass 50 | if not cr: 51 | # env vars or defaults 52 | cr = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80)) 53 | 54 | # reverse rows, cols 55 | # return cols,rows 56 | return int(cr[1]), int(cr[0]) 57 | 58 | 59 | def main(): 60 | print "Terminal size: cols=%d, rows=%d" % dimensions() 61 | 62 | if __name__ == '__main__': 63 | main() 64 | --------------------------------------------------------------------------------